Skip to content

fix: add config #43

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Merged
HongleiGu merged 1 commit into from
Jan 31, 2026
Merged

fix: add config #43

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
1 commit
Select commit
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
2 changes: 0 additions & 2 deletions evaluation_function/correction/__init__.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -9,10 +9,8 @@

from .correction import (
analyze_fsa_correction,
check_minimality,
)

__all__ = [
"analyze_fsa_correction",
"check_minimality",
]
281 changes: 166 additions & 115 deletions evaluation_function/correction/correction.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -7,7 +7,9 @@
All detailed "why" feedback comes from are_isomorphic() in validation module.
"""

from typing import List, Optional, Tuple
from typing import List, Optional

from evaluation_function.schemas.params import Params

# Schema imports
from ..schemas import FSA, ValidationError, ErrorCode
Expand All @@ -16,69 +18,60 @@
# Validation imports
from ..validation.validation import (
is_valid_fsa,
is_deterministic,
is_complete,
is_minimal,
fsas_accept_same_language,
get_structured_info_of_fsa,
)

# Algorithm imports for minimality check
from ..algorithms.minimization import hopcroft_minimization


# =============================================================================
# Minimality Check
# =============================================================================

def _check_minimality(fsa: FSA) -> Tuple[bool, Optional[ValidationError]]:
"""Check if FSA is minimal by comparing with its minimized version."""
try:
minimized = hopcroft_minimization(fsa)
if len(minimized.states) < len(fsa.states):
diff = len(fsa.states) - len(minimized.states)
return False, ValidationError(
message=f"Your FSA works correctly, but it's not minimal! You have {len(fsa.states)} states, but only {len(minimized.states)} are needed. You could remove {diff} state(s).",
code=ErrorCode.NOT_MINIMAL,
severity="error",
suggestion="Look for states that behave identically (same transitions and acceptance) - these can be merged into one"
)
return True, None
except Exception:
return True, None


def check_minimality(fsa: FSA) -> bool:
"""Check if FSA is minimal."""
is_min, _ = _check_minimality(fsa)
return is_min


# =============================================================================
# Helper Functions
# Feedback Helpers
# =============================================================================

def _build_feedback(
summary: str,
validation_errors: List[ValidationError],
equivalence_errors: List[ValidationError],
structural_info: Optional[StructuralInfo]
structural_info: Optional[StructuralInfo],
params: Params
) -> FSAFeedback:
"""Build FSAFeedback from errors and analysis."""
all_errors = validation_errors + equivalence_errors

errors = [e for e in all_errors if e.severity == "error"]
warnings = [e for e in all_errors if e.severity in ("warning", "info")]

# Build hints from all error suggestions
hints = [e.suggestion for e in all_errors if e.suggestion]
if structural_info:
if structural_info.unreachable_states:
unreachable = ", ".join(structural_info.unreachable_states)
hints.append(f"Tip: States {{{unreachable}}} can't be reached from your start state - you might want to remove them or add transitions to them")
if structural_info.dead_states:
dead = ", ".join(structural_info.dead_states)
hints.append(f"Tip: States {{{dead}}} can never lead to acceptance - this might be intentional (trap states) or a bug")

# Build language comparison
language = LanguageComparison(are_equivalent=len(equivalence_errors) == 0)


# Remove UI highlights if disabled
if not params.highlight_errors:
for e in all_errors:
e.highlight = None

# Build hints
hints: List[str] = []
if params.feedback_verbosity != "minimal":
hints.extend(e.suggestion for e in all_errors if e.suggestion)

if params.feedback_verbosity == "detailed" and structural_info:
if structural_info.unreachable_states:
unreachable = ", ".join(structural_info.unreachable_states)
hints.append(
f"Tip: States {{{unreachable}}} are unreachable from the start state"
)
if structural_info.dead_states:
dead = ", ".join(structural_info.dead_states)
hints.append(
f"Tip: States {{{dead}}} can never reach an accepting state"
)
else:
structural_info = None
hints = []

language = LanguageComparison(
are_equivalent=len(equivalence_errors) == 0
)

return FSAFeedback(
summary=summary,
errors=errors,
Expand All @@ -90,25 +83,25 @@ def _build_feedback(


def _summarize_errors(errors: List[ValidationError]) -> str:
"""Generate summary from error messages."""
error_types = set()
"""Generate a human-readable summary from error messages."""
categories = set()

for error in errors:
msg = error.message.lower()
if "alphabet" in msg:
error_types.add("alphabet issue")
elif "states" in msg and ("many" in msg or "few" in msg or "needed" in msg):
error_types.add("incorrect number of states")
elif "accepting" in msg or "accept" in msg:
error_types.add("accepting states issue")
elif "transition" in msg or "reading" in msg:
error_types.add("transition issue")

if len(error_types) == 1:
issue = list(error_types)[0]
return f"Almost there! Your FSA has an {issue}. Check the details below."
elif error_types:
return f"Your FSA doesn't quite match the expected language. Issues found: {', '.join(error_types)}"
return f"Your FSA doesn't accept the correct language. Found {len(errors)} issue(s) to fix."
categories.add("alphabet issue")
elif "accept" in msg:
categories.add("accepting states issue")
elif "transition" in msg:
categories.add("transition issue")
elif "state" in msg:
categories.add("state structure issue")

if len(categories) == 1:
return f"Almost there! Your FSA has a {next(iter(categories))}."
elif categories:
return f"Your FSA has multiple issues: {', '.join(categories)}."
return "Your FSA does not match the expected language."


# =============================================================================
Expand All @@ -118,75 +111,133 @@ def _summarize_errors(errors: List[ValidationError]) -> str:
def analyze_fsa_correction(
student_fsa: FSA,
expected_fsa: FSA,
require_minimal: bool = False
params: Params
) -> Result:
"""
Compare student FSA against expected FSA.

Returns Result with:
- is_correct: True if FSAs accept same language
- feedback: Human-readable summary
- fsa_feedback: Structured feedback with ElementHighlight for UI

Args:
student_fsa: The student's FSA
expected_fsa: The reference/expected FSA
require_minimal: Whether to require student FSA to be minimal
Compare student FSA against expected FSA using configurable parameters.
"""

validation_errors: List[ValidationError] = []
equivalence_errors: List[ValidationError] = []
structural_info: Optional[StructuralInfo] = None


# -------------------------------------------------------------------------
# Step 1: Validate student FSA structure
student_errors = is_valid_fsa(student_fsa)
if student_errors:
num_errors = len(student_errors)
if num_errors == 1:
summary = "Your FSA has a structural problem that needs to be fixed first. See the details below."
else:
summary = f"Your FSA has {num_errors} structural problems that need to be fixed first. See the details below."
# -------------------------------------------------------------------------
student_result = is_valid_fsa(student_fsa)
if not student_result.ok():
summary = (
"Your FSA has a structural problem that needs to be fixed first."
if len(student_result.errors) == 1
else f"Your FSA has {len(student_result.errors)} structural problems to fix."
)
return Result(
is_correct=False,
feedback=summary,
fsa_feedback=_build_feedback(summary, student_errors, [], None)
fsa_feedback=_build_feedback(
summary,
student_result.errors,
[],
None,
params
)
)

# Step 2: Validate expected FSA (should not fail)
expected_errors = is_valid_fsa(expected_fsa)
if expected_errors:

# -------------------------------------------------------------------------
# Step 2: Validate expected FSA (should never fail)
# -------------------------------------------------------------------------
expected_result = is_valid_fsa(expected_fsa)
if not expected_result.ok():
return Result(
is_correct=False,
feedback="Oops! There's an issue with the expected answer. Please contact your instructor."
)

# Step 3: Check minimality if required
if require_minimal:
is_min, min_error = _check_minimality(student_fsa)
if not is_min and min_error:
validation_errors.append(min_error)

# Step 4: Structural analysis

# -------------------------------------------------------------------------
# Step 3: Enforce expected automaton type
# -------------------------------------------------------------------------
if params.expected_type == "DFA":
det_result = is_deterministic(student_fsa)
if not det_result.ok():
summary = "Your automaton must be deterministic (a DFA)."
return Result(
is_correct=False,
feedback=summary,
fsa_feedback=_build_feedback(
summary,
det_result.errors,
[],
None,
params
)
)

# -------------------------------------------------------------------------
# Step 4: Optional completeness check
# -------------------------------------------------------------------------
if params.check_completeness:
comp_result = is_complete(student_fsa)
if not comp_result.ok():
validation_errors.extend(comp_result.errors)

# -------------------------------------------------------------------------
# Step 5: Optional minimality check
# -------------------------------------------------------------------------
if params.check_minimality:
min_errors = is_minimal(student_fsa)
validation_errors.extend(min_errors)

# -------------------------------------------------------------------------
# Step 6: Structural analysis (for feedback only)
# -------------------------------------------------------------------------
structural_info = get_structured_info_of_fsa(student_fsa)

# Step 5: Language equivalence (with detailed feedback from are_isomorphic)
equivalence_errors = fsas_accept_same_language(student_fsa, expected_fsa)

if not equivalence_errors and not validation_errors:
# Success message with some stats
state_count = len(student_fsa.states)
feedback = f"Correct! Your FSA with {state_count} state(s) accepts exactly the right language. Well done!"
return Result(
is_correct=True,
feedback=feedback,
fsa_feedback=_build_feedback("Your FSA is correct!", [], [], structural_info)

# -------------------------------------------------------------------------
# Step 7: Language equivalence
# -------------------------------------------------------------------------
equivalence_result = fsas_accept_same_language(
student_fsa, expected_fsa
)
equivalence_errors = equivalence_result.errors

# -------------------------------------------------------------------------
# Step 8: Decide correctness based on evaluation mode
# -------------------------------------------------------------------------
if params.evaluation_mode == "strict":
is_correct = not validation_errors and equivalence_result.ok()
elif params.evaluation_mode == "lenient":
is_correct = equivalence_result.ok()
else: # partial # I dont know what the partial is meant for, always mark as incorrect?
is_correct = False

# -------------------------------------------------------------------------
# Step 9: Build summary
# -------------------------------------------------------------------------
if is_correct:
feedback = (
f"Correct! Your FSA with {len(student_fsa.states)} state(s) "
"accepts exactly the right language. Well done!"
)

# Build result with errors
is_correct = len(equivalence_errors) == 0 and len(validation_errors) == 0
summary = _summarize_errors(equivalence_errors) if equivalence_errors else "Your FSA has some issues to address."

summary = "Your FSA is correct!"
else:
summary = (
_summarize_errors(equivalence_errors)
if equivalence_errors
else "Your FSA has some issues to address."
)
feedback = summary

# -------------------------------------------------------------------------
# Step 10: Return result
# -------------------------------------------------------------------------
return Result(
is_correct=is_correct,
feedback=summary,
fsa_feedback=_build_feedback(summary, validation_errors, equivalence_errors, structural_info)
feedback=feedback,
fsa_feedback=_build_feedback(
summary,
validation_errors,
equivalence_errors,
structural_info,
params
)
)
Loading
Loading