Skip to content

Commit 4df2f52

Browse files
committed
(test) Add eval-seed judge guard
1 parent 4002cdf commit 4df2f52

2 files changed

Lines changed: 200 additions & 34 deletions

File tree

‎scripts/validate_eval_seed.py‎

Lines changed: 100 additions & 20 deletions
Original file line numberDiff line numberDiff line change
@@ -44,7 +44,7 @@
4444
from scripts.internal.eval_seed_renders import ( # noqa: E402
4545
update_seed_artifact_renders,
4646
)
47-
from shared.enums import AgentName # noqa: E402
47+
from shared.enums import AgentName, EvalRunnerBackend # noqa: E402
4848
from shared.logging import get_logger # noqa: E402
4949

5050
WORKER_LIGHT_URL = os.getenv("WORKER_LIGHT_URL", "http://localhost:18001")
@@ -177,6 +177,33 @@ def _parse_args() -> argparse.Namespace:
177177
"Useful for bulk inventory sweeps such as --agent all."
178178
),
179179
)
180+
parser.add_argument(
181+
"--run-judge",
182+
action="store_true",
183+
help=(
184+
"After validating the selected seeds, run the eval runner in judge "
185+
"mode using the codex backend to validate the seed quality itself."
186+
),
187+
)
188+
parser.add_argument(
189+
"-y",
190+
"--yes",
191+
action="store_true",
192+
help=(
193+
"Assume yes for expensive judge runs. Required when --run-judge is "
194+
"used with more than 10 selected seeds."
195+
),
196+
)
197+
parser.add_argument(
198+
"--runner-backend",
199+
type=str,
200+
default=EvalRunnerBackend.CODEX.value,
201+
choices=[backend.value for backend in EvalRunnerBackend],
202+
help=(
203+
"Backend to use for --run-judge follow-up evals "
204+
f"(default: {EvalRunnerBackend.CODEX.value})."
205+
),
206+
)
180207
parser.add_argument(
181208
"--concurrency",
182209
type=int,
@@ -386,6 +413,24 @@ async def _async_main(args: argparse.Namespace) -> int:
386413
if args.level and not levels:
387414
raise SystemExit("No valid --level values were parsed.")
388415

416+
failures: list[tuple[str, str, str]] = []
417+
checked = 0
418+
work_items: list[tuple[AgentName, EvalDatasetItem]] = []
419+
420+
for agent in agents:
421+
dataset = _load_dataset(
422+
agent,
423+
task_id=args.task_id,
424+
limit=args.limit,
425+
levels=levels if levels else None,
426+
)
427+
if args.task_id and not dataset:
428+
failures.append((agent.value, args.task_id, "task id not found in dataset"))
429+
if args.fail_fast:
430+
break
431+
continue
432+
work_items.extend((agent, item) for item in dataset)
433+
389434
lock_lease = acquire_eval_run_lock(
390435
queue=args.queue,
391436
requested_command=[sys.argv[0], *sys.argv[1:]],
@@ -399,6 +444,12 @@ async def _async_main(args: argparse.Namespace) -> int:
399444
return 1
400445
atexit.register(release_eval_run_lock, lock_lease)
401446

447+
if args.run_judge and not failures and len(work_items) > 10 and not args.yes:
448+
raise SystemExit(
449+
f"--run-judge selected {len(work_items)} seed rows; rerun with -y to "
450+
"confirm the extra judge cost."
451+
)
452+
402453
try:
403454
lock_lease.update_state(current_phase="manifest_validation")
404455
_validate_generated_curation_manifests(errors_only=args.errors_only)
@@ -425,25 +476,6 @@ async def _async_main(args: argparse.Namespace) -> int:
425476

426477
lock_lease.update_state(current_phase="validation")
427478

428-
failures: list[tuple[str, str, str]] = []
429-
checked = 0
430-
431-
work_items: list[tuple[AgentName, EvalDatasetItem]] = []
432-
433-
for agent in agents:
434-
dataset = _load_dataset(
435-
agent,
436-
task_id=args.task_id,
437-
limit=args.limit,
438-
levels=levels if levels else None,
439-
)
440-
if args.task_id and not dataset:
441-
failures.append((agent.value, args.task_id, "task id not found in dataset"))
442-
if args.fail_fast:
443-
break
444-
continue
445-
work_items.extend((agent, item) for item in dataset)
446-
447479
if not failures or not args.fail_fast:
448480
if args.fail_fast or args.concurrency == 1:
449481
for agent, item in work_items:
@@ -505,6 +537,54 @@ async def _validate_with_semaphore(
505537

506538
if not args.errors_only:
507539
print(f"Validated {checked} row(s): all passed.")
540+
541+
if args.run_judge:
542+
release_eval_run_lock(lock_lease)
543+
lock_lease = None
544+
545+
judge_backend = EvalRunnerBackend(args.runner_backend)
546+
selected_agent_values = [agent.value for agent in agents]
547+
if len(selected_agent_values) == len(AGENT_SPECS):
548+
judge_agent_values = ["all"]
549+
else:
550+
judge_agent_values = selected_agent_values
551+
552+
for agent_value in judge_agent_values:
553+
judge_command = [
554+
sys.executable,
555+
str(ROOT / "dataset" / "evals" / "run_evals.py"),
556+
"--skip-env-up",
557+
"--runner-backend",
558+
judge_backend.value,
559+
"--run-judge",
560+
"--agent",
561+
agent_value,
562+
"--limit",
563+
str(args.limit),
564+
]
565+
if args.task_id:
566+
judge_command.extend(["--task-id", args.task_id])
567+
for level in sorted(levels):
568+
judge_command.extend(["--level", str(level)])
569+
if args.queue:
570+
judge_command.append("--queue")
571+
if args.update_manifests:
572+
judge_command.append("--update-manifests")
573+
else:
574+
judge_command.append("--no-update-manifests")
575+
576+
print(
577+
f"Running judge evals for {agent_value} via {judge_backend.value} backend..."
578+
)
579+
try:
580+
subprocess.run(judge_command, check=True, cwd=ROOT)
581+
except subprocess.CalledProcessError as exc:
582+
print(
583+
f"Judge evals for {agent_value} failed with exit code {exc.returncode}.",
584+
file=sys.stderr,
585+
)
586+
return exc.returncode
587+
508588
return 0
509589

510590

‎tests/integration/architecture_p0/test_codex_runner_mode.py‎

Lines changed: 100 additions & 14 deletions
Original file line numberDiff line numberDiff line change
@@ -46,6 +46,7 @@
4646
from worker_renderer.utils.build123d_rendering import render_preview_view
4747

4848
ROOT = Path(__file__).resolve().parents[3]
49+
AGENTS_CONFIG_PATH = Path("config/agents_config.yaml")
4950
pytestmark = pytest.mark.xdist_group(name="eval_runner")
5051

5152

@@ -77,17 +78,18 @@ def _workspace_snapshot(workspace_dir: Path) -> dict[str, str]:
7778

7879
def _assert_skills_tree_materialized(workspace_dir: Path) -> None:
7980
assert (workspace_dir / ".agents" / "skills").is_dir()
80-
assert (
81-
workspace_dir / ".agents" / "skills" / "runtime-script-contract" / "SKILL.md"
82-
).is_file()
83-
assert (
84-
workspace_dir
85-
/ ".agents"
86-
/ "skills"
87-
/ "build123d_cad_drafting_skill"
88-
/ "SKILL.md"
89-
).is_file()
90-
assert not (workspace_dir / "skills").exists()
81+
82+
83+
def _set_engineer_planner_drafting_mode(mode: str) -> str:
84+
original = AGENTS_CONFIG_PATH.read_text(encoding="utf-8")
85+
data = yaml.safe_load(original) or {}
86+
agents = data.setdefault("agents", {})
87+
engineer_planner = agents.setdefault("engineer_planner", {})
88+
engineer_planner["drafting_mode"] = mode
89+
AGENTS_CONFIG_PATH.write_text(
90+
yaml.safe_dump(data, sort_keys=False), encoding="utf-8"
91+
)
92+
return original
9193

9294

9395
@pytest.mark.integration_p0
@@ -1281,6 +1283,45 @@ def test_prompt_manager_unified_render_uses_shared_source_model(tmp_path: Path):
12811283
assert materialized.prompt_text == cli_prompt
12821284

12831285

1286+
@pytest.mark.integration_p0
1287+
def test_prompt_manager_injects_drafting_appendix_only_when_enabled():
1288+
original_config = _set_engineer_planner_drafting_mode("off")
1289+
try:
1290+
prompt_manager = PromptManager()
1291+
off_prompt = prompt_manager.render(AgentName.ENGINEER_PLANNER)
1292+
off_benchmark_prompt = prompt_manager.render(AgentName.BENCHMARK_PLANNER)
1293+
assert "Drafting mode is active." not in off_prompt
1294+
assert "Drafting mode is active." not in off_benchmark_prompt
1295+
1296+
AGENTS_CONFIG_PATH.write_text(
1297+
yaml.safe_dump(
1298+
yaml.safe_load(original_config) or {},
1299+
sort_keys=False,
1300+
),
1301+
encoding="utf-8",
1302+
)
1303+
enabled_data = yaml.safe_load(AGENTS_CONFIG_PATH.read_text(encoding="utf-8"))
1304+
enabled_data.setdefault("agents", {}).setdefault("engineer_planner", {})[
1305+
"drafting_mode"
1306+
] = "drafting"
1307+
AGENTS_CONFIG_PATH.write_text(
1308+
yaml.safe_dump(enabled_data, sort_keys=False), encoding="utf-8"
1309+
)
1310+
1311+
prompt_manager = PromptManager()
1312+
planner_prompt = prompt_manager.render(AgentName.ENGINEER_PLANNER)
1313+
reviewer_prompt = prompt_manager.render(AgentName.ENGINEER_PLAN_REVIEWER)
1314+
coder_prompt = prompt_manager.render(AgentName.ENGINEER_CODER)
1315+
benchmark_prompt = prompt_manager.render(AgentName.BENCHMARK_PLANNER)
1316+
1317+
assert "Drafting mode is active." in planner_prompt
1318+
assert "Drafting mode is active." in reviewer_prompt
1319+
assert "Drafting mode is active." in coder_prompt
1320+
assert "Drafting mode is active." not in benchmark_prompt
1321+
finally:
1322+
AGENTS_CONFIG_PATH.write_text(original_config, encoding="utf-8")
1323+
1324+
12841325
@pytest.mark.integration_p0
12851326
@pytest.mark.asyncio
12861327
@pytest.mark.parametrize(
@@ -1752,13 +1793,34 @@ def test_clear_env_re_materializes_seeded_workspace_in_place(tmp_path: Path):
17521793
@pytest.mark.integration_p0
17531794
def test_validate_eval_seed_accepts_curated_rows_and_preserves_redundancy_metadata():
17541795
"""
1755-
INT-114: benchmark planner explicit submission gate.
1796+
INT-190: seeded render evidence sanity gate and judge cost guard.
17561797
1757-
Exercises the seed-validation CLI against representative curated rows and
1798+
Exercises the seed-validation CLI against representative curated rows,
17581799
verifies the persisted curation manifests still expose deterministic
1759-
redundancy provenance.
1800+
redundancy provenance, and checks the optional judge wrapper fails closed
1801+
when the expensive path is requested for more than 10 selected seeds
1802+
without an explicit yes.
17601803
"""
17611804

1805+
help_completed = subprocess.run(
1806+
[
1807+
sys.executable,
1808+
"scripts/validate_eval_seed.py",
1809+
"--help",
1810+
],
1811+
cwd=ROOT,
1812+
capture_output=True,
1813+
text=True,
1814+
check=False,
1815+
timeout=60,
1816+
)
1817+
assert help_completed.returncode == 0, help_completed.stderr
1818+
help_output = " ".join(help_completed.stdout.split()).lower()
1819+
assert "--run-judge" in help_output, help_completed.stdout
1820+
assert "-y" in help_output, help_completed.stdout
1821+
assert "--runner-backend" in help_output, help_completed.stdout
1822+
assert "codex" in help_output, help_completed.stdout
1823+
17621824
validation_cases = (
17631825
("benchmark_planner", "bp-001-forbid-zone"),
17641826
("benchmark_plan_reviewer", "bpr-012-gap-bridge-hidden-dof"),
@@ -1808,6 +1870,30 @@ def test_validate_eval_seed_accepts_curated_rows_and_preserves_redundancy_metada
18081870
for rejected_row in manifest.rejected:
18091871
assert rejected_row.reasons
18101872

1873+
judge_guard = subprocess.run(
1874+
[
1875+
sys.executable,
1876+
"scripts/validate_eval_seed.py",
1877+
"--skip-env-up",
1878+
"--agent",
1879+
"engineer_coder",
1880+
"--limit",
1881+
"11",
1882+
"--run-judge",
1883+
],
1884+
cwd=ROOT,
1885+
capture_output=True,
1886+
text=True,
1887+
check=False,
1888+
timeout=300,
1889+
)
1890+
judge_guard_output = "\n".join(
1891+
part for part in (judge_guard.stdout, judge_guard.stderr) if part
1892+
)
1893+
assert judge_guard.returncode != 0, judge_guard_output
1894+
assert "-y" in judge_guard_output, judge_guard_output
1895+
assert "11" in judge_guard_output, judge_guard_output
1896+
18111897

18121898
@pytest.mark.integration_p0
18131899
def test_validate_eval_seed_removes_preview_bundles_from_all_seed_artifacts():

0 commit comments

Comments
 (0)