diff --git a/src/cura_eval/judge.py b/src/cura_eval/judge.py index 9860b3d..b0bafdd 100644 --- a/src/cura_eval/judge.py +++ b/src/cura_eval/judge.py @@ -103,10 +103,21 @@ def _format_conversation(conversation: list[dict[str, str]], completion: str) -> def _parse_judge_json(text: str) -> dict | None: cleaned = text.strip() - if cleaned.startswith("```"): - cleaned = cleaned.split("```")[1] + fence = cleaned.find("```") + if fence != -1: + # Judge models often prefix the fenced block with prose + # ("Certainly. ```json ...```"); parse the first fence wherever it is. + cleaned = cleaned[fence + 3 :] if cleaned.startswith("json"): cleaned = cleaned[len("json") :] + cleaned = cleaned.split("```")[0] + else: + # Unfenced reply: take the outermost JSON object so surrounding + # prose does not defeat parsing. + start, end = cleaned.find("{"), cleaned.rfind("}") + if start == -1 or end <= start: + return None + cleaned = cleaned[start : end + 1] try: obj = json.loads(cleaned.strip()) except (ValueError, TypeError): diff --git a/tests/test_judge.py b/tests/test_judge.py index c9ea478..b189c6e 100644 --- a/tests/test_judge.py +++ b/tests/test_judge.py @@ -30,6 +30,16 @@ def test_parse_judge_json_accepts_fenced_and_rejects_malformed(): assert _parse_judge_json("not json") is None +def test_parse_judge_json_handles_prose_around_the_block(): + # Judge replies that open with a sentence before the fenced block used to + # fall through to a raw json.loads and score as JUDGE_PARSE_FAILED. + assert _parse_judge_json('Certainly.\n```json\n{"criteria_met": true}\n```')["criteria_met"] is True + assert _parse_judge_json('Judgment:\n```\n{"criteria_met": false}\n```')["criteria_met"] is False + assert _parse_judge_json('```json\n{"criteria_met": true}\n```\nHope this helps.')["criteria_met"] is True + assert _parse_judge_json('Verdict: {"criteria_met": false, "explanation": "x"} per rubric.')["criteria_met"] is False + assert _parse_judge_json('plain prose, no object at all') is None + + def test_is_reasoning_model_strips_provider_prefix(): assert is_reasoning_model("openai/gpt-5.4") assert is_reasoning_model("o3-mini")