Skip to content

Commit 63fc2d4

Browse files
committed
Strengthen eval seed validation and refresh checks
1 parent 73fc8d9 commit 63fc2d4

5 files changed

Lines changed: 178 additions & 51 deletions

File tree

‎evals/logic/runner.py‎

Lines changed: 2 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -2219,6 +2219,7 @@ async def run_single_eval(
22192219
session_id=session_id,
22202220
agent_name=agent_name,
22212221
spec=spec,
2222+
root=ROOT,
22222223
worker_light_url=WORKER_LIGHT_URL,
22232224
logger=logger,
22242225
)
@@ -2250,6 +2251,7 @@ async def run_single_eval(
22502251
session_id=session_id,
22512252
agent_name=agent_name,
22522253
spec=spec,
2254+
root=ROOT,
22532255
worker_light_url=WORKER_LIGHT_URL,
22542256
logger=logger,
22552257
)

‎evals/logic/seed_maintenance.py‎

Lines changed: 55 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,55 @@
1+
from __future__ import annotations
2+
3+
import hashlib
4+
import json
5+
from pathlib import Path
6+
7+
_PLAN_REVIEW_MANIFEST_NAMES = {
8+
"benchmark_plan_review_manifest.json",
9+
"engineering_plan_review_manifest.json",
10+
}
11+
12+
13+
def refresh_plan_review_manifest_hashes(
14+
artifact_dir: Path, *, fix: bool = False
15+
) -> list[Path]:
16+
"""Refresh deterministic artifact hashes in seeded plan-review manifests."""
17+
18+
updated_manifests: list[Path] = []
19+
20+
for manifest_path in sorted(artifact_dir.rglob("*.json")):
21+
if manifest_path.name not in _PLAN_REVIEW_MANIFEST_NAMES:
22+
continue
23+
24+
try:
25+
manifest = json.loads(manifest_path.read_text())
26+
except Exception:
27+
continue
28+
29+
artifact_hashes = manifest.get("artifact_hashes")
30+
if not isinstance(artifact_hashes, dict) or not artifact_hashes:
31+
continue
32+
33+
updated_hashes = dict(artifact_hashes)
34+
changed = False
35+
for rel_path, expected_hash in artifact_hashes.items():
36+
file_path = artifact_dir / rel_path
37+
if not file_path.exists():
38+
continue
39+
40+
actual_hash = hashlib.sha256(file_path.read_bytes()).hexdigest()
41+
if actual_hash != expected_hash:
42+
updated_hashes[rel_path] = actual_hash
43+
changed = True
44+
45+
if not changed:
46+
continue
47+
48+
updated_manifests.append(manifest_path)
49+
if fix:
50+
manifest["artifact_hashes"] = updated_hashes
51+
manifest_path.write_text(
52+
json.dumps(manifest, indent=2, ensure_ascii=False) + "\n"
53+
)
54+
55+
return updated_manifests

‎evals/logic/workspace.py‎

Lines changed: 49 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -46,6 +46,42 @@ def resolve_seed_artifact_dir(item: EvalDatasetItem, *, root: Path) -> Path | No
4646
return repo_relative
4747

4848

49+
def collect_seed_workspace_artifact_paths(
50+
item: EvalDatasetItem,
51+
*,
52+
root: Path,
53+
) -> list[str]:
54+
"""Return relative paths that should exist for a seeded workspace."""
55+
expected_paths: set[str] = set()
56+
artifact_dir = resolve_seed_artifact_dir(item, root=root)
57+
if (
58+
item.seed_artifact_dir is not None
59+
and artifact_dir is not None
60+
and not artifact_dir.exists()
61+
):
62+
raise FileNotFoundError(f"Seed artifact directory not found: {artifact_dir}")
63+
64+
if artifact_dir is not None and artifact_dir.exists():
65+
for path in sorted(p for p in artifact_dir.rglob("*") if p.is_file()):
66+
expected_paths.add(path.relative_to(artifact_dir).as_posix())
67+
68+
expected_paths.update((item.seed_files or {}).keys())
69+
return sorted(expected_paths)
70+
71+
72+
async def validate_workspace_has_artifacts(
73+
worker: WorkerClient,
74+
*,
75+
artifact_paths: list[str],
76+
) -> list[str]:
77+
"""Return missing relative paths from a workspace-backed filesystem."""
78+
missing: list[str] = []
79+
for rel_path in artifact_paths:
80+
if not await worker.exists(rel_path):
81+
missing.append(rel_path)
82+
return missing
83+
84+
4985
def _refresh_seed_review_manifest_revision(
5086
*,
5187
rel_path: str,
@@ -159,6 +195,7 @@ async def preflight_seeded_entry_contract(
159195
session_id: str,
160196
agent_name: AgentName,
161197
spec: AgentEvalSpec,
198+
root: Path,
162199
worker_light_url: str,
163200
logger: Any,
164201
) -> None:
@@ -231,6 +268,18 @@ async def preflight_seeded_entry_contract(
231268

232269
worker = WorkerClient(base_url=worker_light_url, session_id=session_id)
233270
try:
271+
expected_seed_paths = collect_seed_workspace_artifact_paths(item, root=root)
272+
missing_seed_paths = await validate_workspace_has_artifacts(
273+
worker,
274+
artifact_paths=expected_seed_paths,
275+
)
276+
277+
if missing_seed_paths:
278+
raise ValueError(
279+
"Seeded workspace is missing copied seed artifact(s): "
280+
+ ", ".join(missing_seed_paths)
281+
)
282+
234283
result = await evaluate_node_entry_contract(
235284
contract=contract,
236285
state={

‎scripts/validate_eval_seed.py‎

Lines changed: 30 additions & 51 deletions
Original file line numberDiff line numberDiff line change
@@ -1,6 +1,5 @@
11
import argparse
22
import asyncio
3-
import hashlib
43
import json
54
import os
65
import re
@@ -17,6 +16,9 @@
1716
from controller.clients.worker import WorkerClient # noqa: E402
1817
from evals.logic.curation import load_dataset_curation_manifest # noqa: E402
1918
from evals.logic.models import EvalDatasetItem # noqa: E402
19+
from evals.logic.seed_maintenance import ( # noqa: E402
20+
refresh_plan_review_manifest_hashes as _refresh_plan_review_manifest_hashes,
21+
)
2022
from evals.logic.specs import AGENT_SPECS # noqa: E402
2123
from evals.logic.workspace import ( # noqa: E402
2224
preflight_seeded_entry_contract as _preflight_seeded_entry_contract,
@@ -94,6 +96,14 @@ def _parse_args() -> argparse.Namespace:
9496
action="store_true",
9597
help="Skip running scripts/env_up.sh before validation.",
9698
)
99+
parser.add_argument(
100+
"--fix",
101+
action="store_true",
102+
help=(
103+
"Write deterministic seed repairs back to disk, including plan-review "
104+
"manifest hashes."
105+
),
106+
)
97107
parser.add_argument(
98108
"--fail-fast",
99109
action="store_true",
@@ -193,51 +203,6 @@ def _build_session_id(agent: AgentName, task_id: str) -> str:
193203
return f"seed-check-{agent.value}-{task_id}-{suffix}"[:120]
194204

195205

196-
def _refresh_plan_review_manifest_hashes(
197-
artifact_dir: Path,
198-
) -> list[Path]:
199-
updated_manifests: list[Path] = []
200-
manifest_names = {
201-
"benchmark_plan_review_manifest.json",
202-
"engineering_plan_review_manifest.json",
203-
}
204-
205-
for manifest_path in sorted(artifact_dir.rglob("*.json")):
206-
if manifest_path.name not in manifest_names:
207-
continue
208-
209-
try:
210-
manifest = json.loads(manifest_path.read_text())
211-
except Exception:
212-
continue
213-
214-
artifact_hashes = manifest.get("artifact_hashes")
215-
if not isinstance(artifact_hashes, dict) or not artifact_hashes:
216-
continue
217-
218-
updated_hashes = dict(artifact_hashes)
219-
changed = False
220-
for rel_path, expected_hash in artifact_hashes.items():
221-
file_path = artifact_dir / rel_path
222-
if not file_path.exists():
223-
continue
224-
actual_hash = hashlib.sha256(file_path.read_bytes()).hexdigest()
225-
if actual_hash != expected_hash:
226-
updated_hashes[rel_path] = actual_hash
227-
changed = True
228-
229-
if not changed:
230-
continue
231-
232-
manifest["artifact_hashes"] = updated_hashes
233-
manifest_path.write_text(
234-
json.dumps(manifest, indent=2, ensure_ascii=False) + "\n"
235-
)
236-
updated_manifests.append(manifest_path)
237-
238-
return updated_manifests
239-
240-
241206
def _validate_generated_curation_manifests() -> list[Path]:
242207
manifest_paths = sorted(ROOT.glob("dataset/data/generated/*/v0.0.1/manifest.json"))
243208
validated: list[Path] = []
@@ -255,23 +220,36 @@ def _validate_generated_curation_manifests() -> list[Path]:
255220
return validated
256221

257222

258-
async def _validate_item(agent: AgentName, item: EvalDatasetItem) -> tuple[bool, str]:
223+
async def _validate_item(
224+
agent: AgentName, item: EvalDatasetItem, *, fix: bool
225+
) -> tuple[bool, str]:
259226
session_id = _build_session_id(agent, item.id)
260227
spec = AGENT_SPECS[agent]
261228
try:
262229
resolved_artifact_dir = _resolve_seed_artifact_dir(item, root=ROOT)
263230
if resolved_artifact_dir is not None:
264231
updated_manifests = _refresh_plan_review_manifest_hashes(
265-
resolved_artifact_dir
232+
resolved_artifact_dir, fix=fix
266233
)
267234
for manifest_path in updated_manifests:
235+
event_name = (
236+
"seed_manifest_hashes_refreshed"
237+
if fix
238+
else "seed_manifest_hashes_drift_detected"
239+
)
268240
logger.info(
269-
"seed_manifest_hashes_refreshed",
241+
event_name,
270242
session_id=session_id,
271243
agent_name=agent,
272244
manifest_path=str(manifest_path),
273245
)
274246

247+
if updated_manifests and not fix:
248+
return (
249+
False,
250+
"seed plan-review manifest hashes drifted; rerun with --fix",
251+
)
252+
275253
await _seed_eval_workspace(
276254
item=item,
277255
session_id=session_id,
@@ -285,6 +263,7 @@ async def _validate_item(agent: AgentName, item: EvalDatasetItem) -> tuple[bool,
285263
session_id=session_id,
286264
agent_name=agent,
287265
spec=spec,
266+
root=ROOT,
288267
worker_light_url=WORKER_LIGHT_URL,
289268
logger=logger,
290269
)
@@ -355,7 +334,7 @@ async def _async_main(args: argparse.Namespace) -> int:
355334
if args.fail_fast or args.concurrency == 1:
356335
for agent, item in work_items:
357336
checked += 1
358-
ok, detail = await _validate_item(agent, item)
337+
ok, detail = await _validate_item(agent, item, fix=args.fix)
359338
status = "PASS" if ok else "FAIL"
360339
print(f"{status} {agent.value} {item.id}: {detail}")
361340
if not ok:
@@ -369,7 +348,7 @@ async def _validate_with_semaphore(
369348
agent: AgentName, item: EvalDatasetItem
370349
) -> tuple[AgentName, EvalDatasetItem, bool, str]:
371350
async with semaphore:
372-
ok, detail = await _validate_item(agent, item)
351+
ok, detail = await _validate_item(agent, item, fix=args.fix)
373352
return agent, item, ok, detail
374353

375354
tasks = [

‎tests/integration/architecture_p0/test_codex_runner_mode.py‎

Lines changed: 42 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -15,6 +15,7 @@
1515
verify_planner_workspace,
1616
)
1717
from evals.logic.models import EvalDatasetItem
18+
from evals.logic.seed_maintenance import refresh_plan_review_manifest_hashes
1819
from shared.enums import AgentName
1920
from shared.models.schemas import DatasetCurationManifest, PlannerSubmissionResult
2021

@@ -351,3 +352,44 @@ def test_validate_eval_seed_accepts_curated_rows_and_preserves_redundancy_metada
351352
assert dropped_episode_ids == sorted(set(dropped_episode_ids))
352353
for rejected_row in manifest.rejected:
353354
assert rejected_row.reasons
355+
356+
357+
@pytest.mark.integration_p0
358+
def test_refresh_plan_review_manifest_hashes_can_fix_drift(tmp_path: Path):
359+
artifact_dir = tmp_path / "seed_artifacts"
360+
artifact_dir.mkdir()
361+
362+
payload_path = artifact_dir / "script.py"
363+
payload_path.write_text("print('hello world')\n", encoding="utf-8")
364+
365+
manifest_path = artifact_dir / "benchmark_plan_review_manifest.json"
366+
manifest_path.write_text(
367+
json.dumps(
368+
{
369+
"artifact_hashes": {
370+
"script.py": "deadbeef",
371+
}
372+
},
373+
indent=2,
374+
)
375+
+ "\n",
376+
encoding="utf-8",
377+
)
378+
379+
dry_run_updates = refresh_plan_review_manifest_hashes(artifact_dir, fix=False)
380+
assert dry_run_updates == [manifest_path]
381+
assert (
382+
json.loads(manifest_path.read_text(encoding="utf-8"))["artifact_hashes"][
383+
"script.py"
384+
]
385+
== "deadbeef"
386+
)
387+
388+
fixed_updates = refresh_plan_review_manifest_hashes(artifact_dir, fix=True)
389+
assert fixed_updates == [manifest_path]
390+
391+
updated_manifest = json.loads(manifest_path.read_text(encoding="utf-8"))
392+
assert (
393+
updated_manifest["artifact_hashes"]["script.py"]
394+
== hashlib.sha256(payload_path.read_bytes()).hexdigest()
395+
)

0 commit comments

Comments
 (0)