Skip to content
Open
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
9 changes: 8 additions & 1 deletion src/cura_eval/mcq.py
Original file line number Diff line number Diff line change
Expand Up @@ -34,7 +34,14 @@ def extract_letter(text: str, max_letter: str = "J") -> str | None:
if boxed:
return boxed[-1]

marker = re.findall(rf"(?:FINAL\s+)?ANSWER\s*(?:IS\s+)?[:=]?\s*\(?({cls})\)?\b", cleaned)
# Allow a short run of decoration between ANSWER/IS and the letter
# (markdown bold "**Answer:** C", em dashes, fullwidth colons, ...).
# Without this those replies fall through to the standalone rule below,
# where later prose letters silently override the stated answer.
marker = re.findall(
rf"(?:FINAL\s+)?ANSWER\s*(?:IS\s+)?[^A-Za-z0-9]{{0,6}}\(?({cls})\s*\)?\b",
cleaned,
)
if marker:
return marker[-1]

Expand Down
11 changes: 11 additions & 0 deletions tests/test_mcq.py
Original file line number Diff line number Diff line change
Expand Up @@ -10,6 +10,17 @@ def test_answer_marker():
assert extract_letter("answer is (E)", "J") == "E"


def test_answer_marker_survives_markdown_and_unicode_decoration():
# Decorated markers used to miss the Answer rule entirely, so the
# standalone last-occurrence fallback grabbed letters from later prose.
assert extract_letter("**Answer:** C. Option D was considered and rejected.", "J") == "C"
assert extract_letter("**Final Answer:** A\n\nDifferential recap: (B) viral, (C) bacterial.", "J") == "A"
assert extract_letter("Final Answer - C\n\nPlan E rejected due to cost.", "J") == "C"
assert extract_letter("Answer:E", "J") == "E"
assert extract_letter("**The answer is: B**", "J") == "B"
assert extract_letter("ANSWER IS :: G", "J") == "G"


def test_standalone_letter_last_occurrence():
assert extract_letter("Could be A or maybe B", "J") == "B"

Expand Down