diff --git a/evaluation_function/evaluation.py b/evaluation_function/evaluation.py index f0b62eb..254b9e2 100755 --- a/evaluation_function/evaluation.py +++ b/evaluation_function/evaluation.py @@ -9,7 +9,7 @@ def validate_fsa(value: str | dict) -> FSA: """Parse a FSA from JSON string or dict.""" if isinstance(value, str): return FSAFrontend.model_validate_json(value).toFSA() - return FSAFrontend.model_validate(value).toFSA() + return FSAFrontend.model_validate(value).toFSA(), FSAFrontend.model_validate(value).config def evaluation_function( response: Any = None, @@ -33,16 +33,9 @@ def evaluation_function( f"Missing FSA data: response or answer is None\n" f"response: {response}\nanswer: {answer}" ) - # Extract and remove config from answer - raw_config = "{}" - if isinstance(answer, dict): - raw_config = answer.pop("config", "{}") - - config = json.loads(raw_config) - # Parse FSAs - student_fsa = validate_fsa(response) - expected_fsa = validate_fsa(answer) + student_fsa, student_config = validate_fsa(response) + expected_fsa, expected_config = validate_fsa(answer) require_minimal = params.get("require_minimal", False) if isinstance(params, dict) else False @@ -53,7 +46,7 @@ def evaluation_function( feedback_items=[( "error", f"Invalid FSA format: {str(e)}\n\n" - f"response: {response}\nanswer: {answer}\nparams: {params}\n\nconfig:{config}" + f"response: {response}\nanswer: {answer}\nparams: {params}\n\nstudent config:{student_config}\n\nexpected config:{expected_config}" )] ) diff --git a/evaluation_function/schemas/fsaFrontend.py b/evaluation_function/schemas/fsaFrontend.py index 1073aad..63aac77 100644 --- a/evaluation_function/schemas/fsaFrontend.py +++ b/evaluation_function/schemas/fsaFrontend.py @@ -53,7 +53,7 @@ class FSAFrontend(BaseModel): default_factory=list, description="F: Set of accepting/final states" ) - + config: str | None = Field(default=None) class Config: schema_extra = { "example": {