11import argparse
22import asyncio
3- import hashlib
43import json
54import os
65import re
1716from controller .clients .worker import WorkerClient # noqa: E402
1817from evals .logic .curation import load_dataset_curation_manifest # noqa: E402
1918from evals .logic .models import EvalDatasetItem # noqa: E402
19+ from evals .logic .seed_maintenance import ( # noqa: E402
20+ refresh_plan_review_manifest_hashes as _refresh_plan_review_manifest_hashes ,
21+ )
2022from evals .logic .specs import AGENT_SPECS # noqa: E402
2123from evals .logic .workspace import ( # noqa: E402
2224 preflight_seeded_entry_contract as _preflight_seeded_entry_contract ,
@@ -94,6 +96,14 @@ def _parse_args() -> argparse.Namespace:
9496 action = "store_true" ,
9597 help = "Skip running scripts/env_up.sh before validation." ,
9698 )
99+ parser .add_argument (
100+ "--fix" ,
101+ action = "store_true" ,
102+ help = (
103+ "Write deterministic seed repairs back to disk, including plan-review "
104+ "manifest hashes."
105+ ),
106+ )
97107 parser .add_argument (
98108 "--fail-fast" ,
99109 action = "store_true" ,
@@ -193,51 +203,6 @@ def _build_session_id(agent: AgentName, task_id: str) -> str:
193203 return f"seed-check-{ agent .value } -{ task_id } -{ suffix } " [:120 ]
194204
195205
196- def _refresh_plan_review_manifest_hashes (
197- artifact_dir : Path ,
198- ) -> list [Path ]:
199- updated_manifests : list [Path ] = []
200- manifest_names = {
201- "benchmark_plan_review_manifest.json" ,
202- "engineering_plan_review_manifest.json" ,
203- }
204-
205- for manifest_path in sorted (artifact_dir .rglob ("*.json" )):
206- if manifest_path .name not in manifest_names :
207- continue
208-
209- try :
210- manifest = json .loads (manifest_path .read_text ())
211- except Exception :
212- continue
213-
214- artifact_hashes = manifest .get ("artifact_hashes" )
215- if not isinstance (artifact_hashes , dict ) or not artifact_hashes :
216- continue
217-
218- updated_hashes = dict (artifact_hashes )
219- changed = False
220- for rel_path , expected_hash in artifact_hashes .items ():
221- file_path = artifact_dir / rel_path
222- if not file_path .exists ():
223- continue
224- actual_hash = hashlib .sha256 (file_path .read_bytes ()).hexdigest ()
225- if actual_hash != expected_hash :
226- updated_hashes [rel_path ] = actual_hash
227- changed = True
228-
229- if not changed :
230- continue
231-
232- manifest ["artifact_hashes" ] = updated_hashes
233- manifest_path .write_text (
234- json .dumps (manifest , indent = 2 , ensure_ascii = False ) + "\n "
235- )
236- updated_manifests .append (manifest_path )
237-
238- return updated_manifests
239-
240-
241206def _validate_generated_curation_manifests () -> list [Path ]:
242207 manifest_paths = sorted (ROOT .glob ("dataset/data/generated/*/v0.0.1/manifest.json" ))
243208 validated : list [Path ] = []
@@ -255,23 +220,36 @@ def _validate_generated_curation_manifests() -> list[Path]:
255220 return validated
256221
257222
258- async def _validate_item (agent : AgentName , item : EvalDatasetItem ) -> tuple [bool , str ]:
223+ async def _validate_item (
224+ agent : AgentName , item : EvalDatasetItem , * , fix : bool
225+ ) -> tuple [bool , str ]:
259226 session_id = _build_session_id (agent , item .id )
260227 spec = AGENT_SPECS [agent ]
261228 try :
262229 resolved_artifact_dir = _resolve_seed_artifact_dir (item , root = ROOT )
263230 if resolved_artifact_dir is not None :
264231 updated_manifests = _refresh_plan_review_manifest_hashes (
265- resolved_artifact_dir
232+ resolved_artifact_dir , fix = fix
266233 )
267234 for manifest_path in updated_manifests :
235+ event_name = (
236+ "seed_manifest_hashes_refreshed"
237+ if fix
238+ else "seed_manifest_hashes_drift_detected"
239+ )
268240 logger .info (
269- "seed_manifest_hashes_refreshed" ,
241+ event_name ,
270242 session_id = session_id ,
271243 agent_name = agent ,
272244 manifest_path = str (manifest_path ),
273245 )
274246
247+ if updated_manifests and not fix :
248+ return (
249+ False ,
250+ "seed plan-review manifest hashes drifted; rerun with --fix" ,
251+ )
252+
275253 await _seed_eval_workspace (
276254 item = item ,
277255 session_id = session_id ,
@@ -285,6 +263,7 @@ async def _validate_item(agent: AgentName, item: EvalDatasetItem) -> tuple[bool,
285263 session_id = session_id ,
286264 agent_name = agent ,
287265 spec = spec ,
266+ root = ROOT ,
288267 worker_light_url = WORKER_LIGHT_URL ,
289268 logger = logger ,
290269 )
@@ -355,7 +334,7 @@ async def _async_main(args: argparse.Namespace) -> int:
355334 if args .fail_fast or args .concurrency == 1 :
356335 for agent , item in work_items :
357336 checked += 1
358- ok , detail = await _validate_item (agent , item )
337+ ok , detail = await _validate_item (agent , item , fix = args . fix )
359338 status = "PASS" if ok else "FAIL"
360339 print (f"{ status } { agent .value } { item .id } : { detail } " )
361340 if not ok :
@@ -369,7 +348,7 @@ async def _validate_with_semaphore(
369348 agent : AgentName , item : EvalDatasetItem
370349 ) -> tuple [AgentName , EvalDatasetItem , bool , str ]:
371350 async with semaphore :
372- ok , detail = await _validate_item (agent , item )
351+ ok , detail = await _validate_item (agent , item , fix = args . fix )
373352 return agent , item , ok , detail
374353
375354 tasks = [
0 commit comments