fix: parse judge replies that prefix prose before the fenced block

src/cura_eval/judge.py

@@ -103,10 +103,21 @@ def _format_conversation(conversation: list[dict[str, str]], completion: str) ->
 
 def _parse_judge_json(text: str) -> dict | None:
     cleaned = text.strip()
-    if cleaned.startswith("```"):
-        cleaned = cleaned.split("```")[1]
+    fence = cleaned.find("```")
+    if fence != -1:
+        # Judge models often prefix the fenced block with prose
+        # ("Certainly. ```json ...```"); parse the first fence wherever it is.
+        cleaned = cleaned[fence + 3 :]
         if cleaned.startswith("json"):
             cleaned = cleaned[len("json") :]
+        cleaned = cleaned.split("```")[0]
+    else:
+        # Unfenced reply: take the outermost JSON object so surrounding
+        # prose does not defeat parsing.
+        start, end = cleaned.find("{"), cleaned.rfind("}")
+        if start == -1 or end <= start:
+            return None
+        cleaned = cleaned[start : end + 1]
     try:
         obj = json.loads(cleaned.strip())
     except (ValueError, TypeError):

tests/test_judge.py

@@ -30,6 +30,16 @@ def test_parse_judge_json_accepts_fenced_and_rejects_malformed():
     assert _parse_judge_json("not json") is None
 
 
+def test_parse_judge_json_handles_prose_around_the_block():
+    # Judge replies that open with a sentence before the fenced block used to
+    # fall through to a raw json.loads and score as JUDGE_PARSE_FAILED.
+    assert _parse_judge_json('Certainly.\n```json\n{"criteria_met": true}\n```')["criteria_met"] is True
+    assert _parse_judge_json('Judgment:\n```\n{"criteria_met": false}\n```')["criteria_met"] is False
+    assert _parse_judge_json('```json\n{"criteria_met": true}\n```\nHope this helps.')["criteria_met"] is True
+    assert _parse_judge_json('Verdict: {"criteria_met": false, "explanation": "x"} per rubric.')["criteria_met"] is False
+    assert _parse_judge_json('plain prose, no object at all') is None
+
+
 def test_is_reasoning_model_strips_provider_prefix():
     assert is_reasoning_model("openai/gpt-5.4")
     assert is_reasoning_model("o3-mini")