From 6def05051e307e7326a2c695e32c7ee80e348485 Mon Sep 17 00:00:00 2001 From: Xeophon <46377542+xeophon@users.noreply.github.com> Date: Sat, 29 Aug 2026 09:47:31 +0200 Subject: [PATCH 1/4] fix(v1): prefer final judge choices over draft boxes --- tests/v1/test_scoring.py | 18 ++++++++++++++---- verifiers/v1/utils/score.py | 31 ++++++++++++++++++++----------- 2 files changed, 34 insertions(+), 15 deletions(-) diff --git a/tests/v1/test_scoring.py b/tests/v1/test_scoring.py index 182cc0873d..5661caedb9 100644 --- a/tests/v1/test_scoring.py +++ b/tests/v1/test_scoring.py @@ -113,7 +113,17 @@ def test_parse_pytest_outcomes_strips_xfail_xpass_reasons() -> None: } -def test_parse_judge_choice_uses_first_choice_after_verdict_marker() -> None: - response = "Final Judgment: B because it is a better answer" - - assert vf.parse_judge_choice(response, choices=("A", "B")) == "B" +def test_parse_judge_choice_prefers_final_marker_then_boxed() -> None: + assert ( + vf.parse_judge_choice( + "Draft: \\boxed{A}\nFinal Judgment: B", choices=("A", "B") + ) + == "B" + ) + assert ( + vf.parse_judge_choice( + "Draft: \\boxed{A}\nFinal Judgment:\nReasoning mentions B", + choices=("A", "B"), + ) + == "A" + ) diff --git a/verifiers/v1/utils/score.py b/verifiers/v1/utils/score.py index f3a9087670..e3aaf7dfd0 100644 --- a/verifiers/v1/utils/score.py +++ b/verifiers/v1/utils/score.py @@ -55,23 +55,32 @@ def parse_judge_choice( return None text = content.rsplit("", 1)[-1].strip() - text = extract_boxed_answer(text, strict=True).strip() or text - text_upper = text.upper() choices_by_upper = {choice.upper(): choice for choice in choices} allowed = "|".join(re.escape(choice) for choice in choices_by_upper) choice_re = rf"(? Date: Sat, 29 Aug 2026 10:00:17 +0200 Subject: [PATCH 2/4] fix(v1): tighten judge choice precedence --- tests/v1/test_scoring.py | 10 ++++++++++ verifiers/v1/utils/score.py | 8 ++++++-- 2 files changed, 16 insertions(+), 2 deletions(-) diff --git a/tests/v1/test_scoring.py b/tests/v1/test_scoring.py index 5661caedb9..676a937dcf 100644 --- a/tests/v1/test_scoring.py +++ b/tests/v1/test_scoring.py @@ -127,3 +127,13 @@ def test_parse_judge_choice_prefers_final_marker_then_boxed() -> None: ) == "A" ) + assert ( + vf.parse_judge_choice( + "Draft: \\boxed{B}\nFinal Judgment: This is a \\boxed{B}", + choices=("A", "B"), + ) + == "B" + ) + assert ( + vf.parse_judge_choice("Draft: \\boxed{A}\nAnswer: B", choices=("A", "B")) == "A" + ) diff --git a/verifiers/v1/utils/score.py b/verifiers/v1/utils/score.py index e3aaf7dfd0..4fe003caf3 100644 --- a/verifiers/v1/utils/score.py +++ b/verifiers/v1/utils/score.py @@ -63,7 +63,7 @@ def parse_judge_choice( horizontal_space = r"[^\S\n]" marker_tail = ( rf"{horizontal_space}*(?:IS{horizontal_space}*)?[:\-]?" - rf"{horizontal_space}*[^\n]*?{choice_re}" + rf"{horizontal_space}*{choice_re}" ) final_choices = re.findall( rf"^{horizontal_space}*FINAL{horizontal_space}+" @@ -78,8 +78,12 @@ def parse_judge_choice( ) boxed = extract_boxed_answer(text, strict=True).strip() + boxed_choices = re.findall(choice_re, boxed.upper()) matches = ( - final_choices or bare_choices or re.findall(choice_re, (boxed or text).upper()) + final_choices + or boxed_choices + or bare_choices + or re.findall(choice_re, text_upper) ) return choices_by_upper.get(matches[-1]) if matches else None From 011b6b71831b34d8f43e2c4f8088eb8e88aa62b3 Mon Sep 17 00:00:00 2001 From: Xeophon <46377542+xeophon@users.noreply.github.com> Date: Sat, 29 Aug 2026 10:41:00 +0200 Subject: [PATCH 3/4] fix(v1): prefer boxed final judge choices --- tests/v1/test_scoring.py | 6 ++++++ verifiers/v1/utils/score.py | 16 +++++++++++++--- 2 files changed, 19 insertions(+), 3 deletions(-) diff --git a/tests/v1/test_scoring.py b/tests/v1/test_scoring.py index 676a937dcf..09bdc8c087 100644 --- a/tests/v1/test_scoring.py +++ b/tests/v1/test_scoring.py @@ -134,6 +134,12 @@ def test_parse_judge_choice_prefers_final_marker_then_boxed() -> None: ) == "B" ) + assert ( + vf.parse_judge_choice( + "Final Judgment: A better choice is \\boxed{B}", choices=("A", "B") + ) + == "B" + ) assert ( vf.parse_judge_choice("Draft: \\boxed{A}\nAnswer: B", choices=("A", "B")) == "A" ) diff --git a/verifiers/v1/utils/score.py b/verifiers/v1/utils/score.py index 4fe003caf3..25e3362d98 100644 --- a/verifiers/v1/utils/score.py +++ b/verifiers/v1/utils/score.py @@ -65,9 +65,18 @@ def parse_judge_choice( rf"{horizontal_space}*(?:IS{horizontal_space}*)?[:\-]?" rf"{horizontal_space}*{choice_re}" ) - final_choices = re.findall( + final_marker = ( rf"^{horizontal_space}*FINAL{horizontal_space}+" - rf"(?:JUDGMENT|ANSWER|VERDICT){marker_tail}", + rf"(?:JUDGMENT|ANSWER|VERDICT)" + ) + final_boxed_choices = re.findall( + rf"{final_marker}[^\n]*{re.escape(BOXED_START.upper())}{horizontal_space}*" + rf"{choice_re}{horizontal_space}*\}}", + text_upper, + re.MULTILINE, + ) + final_choices = re.findall( + rf"{final_marker}{marker_tail}", text_upper, re.MULTILINE, ) @@ -80,7 +89,8 @@ def parse_judge_choice( boxed = extract_boxed_answer(text, strict=True).strip() boxed_choices = re.findall(choice_re, boxed.upper()) matches = ( - final_choices + final_boxed_choices + or final_choices or boxed_choices or bare_choices or re.findall(choice_re, text_upper) From 59239af27bd497669041888e0f263f79df82c10d Mon Sep 17 00:00:00 2001 From: Xeophon <46377542+xeophon@users.noreply.github.com> Date: Sat, 29 Aug 2026 10:49:27 +0200 Subject: [PATCH 4/4] fix(v1): prioritize explicit final judge verdicts --- tests/v1/test_scoring.py | 6 +++++ verifiers/v1/utils/score.py | 46 ++++++++++++++----------------------- 2 files changed, 23 insertions(+), 29 deletions(-) diff --git a/tests/v1/test_scoring.py b/tests/v1/test_scoring.py index 09bdc8c087..c06a747840 100644 --- a/tests/v1/test_scoring.py +++ b/tests/v1/test_scoring.py @@ -138,6 +138,12 @@ def test_parse_judge_choice_prefers_final_marker_then_boxed() -> None: vf.parse_judge_choice( "Final Judgment: A better choice is \\boxed{B}", choices=("A", "B") ) + == "A" + ) + assert ( + vf.parse_judge_choice( + "Final Judgment: \\boxed{A}\nFinal Judgment: B", choices=("A", "B") + ) == "B" ) assert ( diff --git a/verifiers/v1/utils/score.py b/verifiers/v1/utils/score.py index 25e3362d98..d15fd443ed 100644 --- a/verifiers/v1/utils/score.py +++ b/verifiers/v1/utils/score.py @@ -59,42 +59,30 @@ def parse_judge_choice( choices_by_upper = {choice.upper(): choice for choice in choices} allowed = "|".join(re.escape(choice) for choice in choices_by_upper) choice_re = rf"(?