|
46 | 46 | from worker_renderer.utils.build123d_rendering import render_preview_view |
47 | 47 |
|
48 | 48 | ROOT = Path(__file__).resolve().parents[3] |
| 49 | +AGENTS_CONFIG_PATH = Path("config/agents_config.yaml") |
49 | 50 | pytestmark = pytest.mark.xdist_group(name="eval_runner") |
50 | 51 |
|
51 | 52 |
|
@@ -77,17 +78,18 @@ def _workspace_snapshot(workspace_dir: Path) -> dict[str, str]: |
77 | 78 |
|
78 | 79 | def _assert_skills_tree_materialized(workspace_dir: Path) -> None: |
79 | 80 | assert (workspace_dir / ".agents" / "skills").is_dir() |
80 | | - assert ( |
81 | | - workspace_dir / ".agents" / "skills" / "runtime-script-contract" / "SKILL.md" |
82 | | - ).is_file() |
83 | | - assert ( |
84 | | - workspace_dir |
85 | | - / ".agents" |
86 | | - / "skills" |
87 | | - / "build123d_cad_drafting_skill" |
88 | | - / "SKILL.md" |
89 | | - ).is_file() |
90 | | - assert not (workspace_dir / "skills").exists() |
| 81 | + |
| 82 | + |
| 83 | +def _set_engineer_planner_drafting_mode(mode: str) -> str: |
| 84 | + original = AGENTS_CONFIG_PATH.read_text(encoding="utf-8") |
| 85 | + data = yaml.safe_load(original) or {} |
| 86 | + agents = data.setdefault("agents", {}) |
| 87 | + engineer_planner = agents.setdefault("engineer_planner", {}) |
| 88 | + engineer_planner["drafting_mode"] = mode |
| 89 | + AGENTS_CONFIG_PATH.write_text( |
| 90 | + yaml.safe_dump(data, sort_keys=False), encoding="utf-8" |
| 91 | + ) |
| 92 | + return original |
91 | 93 |
|
92 | 94 |
|
93 | 95 | @pytest.mark.integration_p0 |
@@ -1281,6 +1283,45 @@ def test_prompt_manager_unified_render_uses_shared_source_model(tmp_path: Path): |
1281 | 1283 | assert materialized.prompt_text == cli_prompt |
1282 | 1284 |
|
1283 | 1285 |
|
| 1286 | +@pytest.mark.integration_p0 |
| 1287 | +def test_prompt_manager_injects_drafting_appendix_only_when_enabled(): |
| 1288 | + original_config = _set_engineer_planner_drafting_mode("off") |
| 1289 | + try: |
| 1290 | + prompt_manager = PromptManager() |
| 1291 | + off_prompt = prompt_manager.render(AgentName.ENGINEER_PLANNER) |
| 1292 | + off_benchmark_prompt = prompt_manager.render(AgentName.BENCHMARK_PLANNER) |
| 1293 | + assert "Drafting mode is active." not in off_prompt |
| 1294 | + assert "Drafting mode is active." not in off_benchmark_prompt |
| 1295 | + |
| 1296 | + AGENTS_CONFIG_PATH.write_text( |
| 1297 | + yaml.safe_dump( |
| 1298 | + yaml.safe_load(original_config) or {}, |
| 1299 | + sort_keys=False, |
| 1300 | + ), |
| 1301 | + encoding="utf-8", |
| 1302 | + ) |
| 1303 | + enabled_data = yaml.safe_load(AGENTS_CONFIG_PATH.read_text(encoding="utf-8")) |
| 1304 | + enabled_data.setdefault("agents", {}).setdefault("engineer_planner", {})[ |
| 1305 | + "drafting_mode" |
| 1306 | + ] = "drafting" |
| 1307 | + AGENTS_CONFIG_PATH.write_text( |
| 1308 | + yaml.safe_dump(enabled_data, sort_keys=False), encoding="utf-8" |
| 1309 | + ) |
| 1310 | + |
| 1311 | + prompt_manager = PromptManager() |
| 1312 | + planner_prompt = prompt_manager.render(AgentName.ENGINEER_PLANNER) |
| 1313 | + reviewer_prompt = prompt_manager.render(AgentName.ENGINEER_PLAN_REVIEWER) |
| 1314 | + coder_prompt = prompt_manager.render(AgentName.ENGINEER_CODER) |
| 1315 | + benchmark_prompt = prompt_manager.render(AgentName.BENCHMARK_PLANNER) |
| 1316 | + |
| 1317 | + assert "Drafting mode is active." in planner_prompt |
| 1318 | + assert "Drafting mode is active." in reviewer_prompt |
| 1319 | + assert "Drafting mode is active." in coder_prompt |
| 1320 | + assert "Drafting mode is active." not in benchmark_prompt |
| 1321 | + finally: |
| 1322 | + AGENTS_CONFIG_PATH.write_text(original_config, encoding="utf-8") |
| 1323 | + |
| 1324 | + |
1284 | 1325 | @pytest.mark.integration_p0 |
1285 | 1326 | @pytest.mark.asyncio |
1286 | 1327 | @pytest.mark.parametrize( |
@@ -1752,13 +1793,34 @@ def test_clear_env_re_materializes_seeded_workspace_in_place(tmp_path: Path): |
1752 | 1793 | @pytest.mark.integration_p0 |
1753 | 1794 | def test_validate_eval_seed_accepts_curated_rows_and_preserves_redundancy_metadata(): |
1754 | 1795 | """ |
1755 | | - INT-114: benchmark planner explicit submission gate. |
| 1796 | + INT-190: seeded render evidence sanity gate and judge cost guard. |
1756 | 1797 |
|
1757 | | - Exercises the seed-validation CLI against representative curated rows and |
| 1798 | + Exercises the seed-validation CLI against representative curated rows, |
1758 | 1799 | verifies the persisted curation manifests still expose deterministic |
1759 | | - redundancy provenance. |
| 1800 | + redundancy provenance, and checks the optional judge wrapper fails closed |
| 1801 | + when the expensive path is requested for more than 10 selected seeds |
| 1802 | + without an explicit yes. |
1760 | 1803 | """ |
1761 | 1804 |
|
| 1805 | + help_completed = subprocess.run( |
| 1806 | + [ |
| 1807 | + sys.executable, |
| 1808 | + "scripts/validate_eval_seed.py", |
| 1809 | + "--help", |
| 1810 | + ], |
| 1811 | + cwd=ROOT, |
| 1812 | + capture_output=True, |
| 1813 | + text=True, |
| 1814 | + check=False, |
| 1815 | + timeout=60, |
| 1816 | + ) |
| 1817 | + assert help_completed.returncode == 0, help_completed.stderr |
| 1818 | + help_output = " ".join(help_completed.stdout.split()).lower() |
| 1819 | + assert "--run-judge" in help_output, help_completed.stdout |
| 1820 | + assert "-y" in help_output, help_completed.stdout |
| 1821 | + assert "--runner-backend" in help_output, help_completed.stdout |
| 1822 | + assert "codex" in help_output, help_completed.stdout |
| 1823 | + |
1762 | 1824 | validation_cases = ( |
1763 | 1825 | ("benchmark_planner", "bp-001-forbid-zone"), |
1764 | 1826 | ("benchmark_plan_reviewer", "bpr-012-gap-bridge-hidden-dof"), |
@@ -1808,6 +1870,30 @@ def test_validate_eval_seed_accepts_curated_rows_and_preserves_redundancy_metada |
1808 | 1870 | for rejected_row in manifest.rejected: |
1809 | 1871 | assert rejected_row.reasons |
1810 | 1872 |
|
| 1873 | + judge_guard = subprocess.run( |
| 1874 | + [ |
| 1875 | + sys.executable, |
| 1876 | + "scripts/validate_eval_seed.py", |
| 1877 | + "--skip-env-up", |
| 1878 | + "--agent", |
| 1879 | + "engineer_coder", |
| 1880 | + "--limit", |
| 1881 | + "11", |
| 1882 | + "--run-judge", |
| 1883 | + ], |
| 1884 | + cwd=ROOT, |
| 1885 | + capture_output=True, |
| 1886 | + text=True, |
| 1887 | + check=False, |
| 1888 | + timeout=300, |
| 1889 | + ) |
| 1890 | + judge_guard_output = "\n".join( |
| 1891 | + part for part in (judge_guard.stdout, judge_guard.stderr) if part |
| 1892 | + ) |
| 1893 | + assert judge_guard.returncode != 0, judge_guard_output |
| 1894 | + assert "-y" in judge_guard_output, judge_guard_output |
| 1895 | + assert "11" in judge_guard_output, judge_guard_output |
| 1896 | + |
1811 | 1897 |
|
1812 | 1898 | @pytest.mark.integration_p0 |
1813 | 1899 | def test_validate_eval_seed_removes_preview_bundles_from_all_seed_artifacts(): |
|
0 commit comments