From 389f5d741a3a4ab42bdc01d756a7cc435184927f Mon Sep 17 00:00:00 2001 From: chenluo Date: Fri, 28 Aug 2026 15:20:27 +0800 Subject: [PATCH 1/7] feat: add SBD v6 metadata and startup timeline --- .../breakdown/collectors/__init__.py | 8 + .../breakdown/collectors/v6.py | 275 +++++ .../inference_optimizer/breakdown/exporter.py | 46 + .../inference_optimizer/breakdown/schema.py | 109 ++ .../breakdown/session_package.py | 1 + .../inference_optimizer/cli/__init__.py | 120 +- .../inference_optimizer/cli/model_gate.py | 506 +++++++- .../inference_optimizer/cli/preflight.py | 1017 ++++++++++++++--- .../inference_optimizer/session/sbd_v6.py | 260 +++++ .../session/session_paths.py | 30 + .../tests/test_sbd_v6_initial.py | 844 ++++++++++++++ .../tests/test_session_package.py | 4 + 12 files changed, 3056 insertions(+), 164 deletions(-) create mode 100644 src/hyperloom/inference_optimizer/breakdown/collectors/v6.py create mode 100644 src/hyperloom/inference_optimizer/session/sbd_v6.py create mode 100644 src/hyperloom/inference_optimizer/tests/test_sbd_v6_initial.py diff --git a/src/hyperloom/inference_optimizer/breakdown/collectors/__init__.py b/src/hyperloom/inference_optimizer/breakdown/collectors/__init__.py index 51ef8d00b9..68e7b40e6b 100644 --- a/src/hyperloom/inference_optimizer/breakdown/collectors/__init__.py +++ b/src/hyperloom/inference_optimizer/breakdown/collectors/__init__.py @@ -186,6 +186,11 @@ _geak_reconstruct_from_disk as _geak_reconstruct_from_disk, collect_geak as collect_geak, ) +from .v6 import ( + collect_v6_metadata as collect_v6_metadata, + collect_v6_outcome as collect_v6_outcome, + collect_v6_timeline as collect_v6_timeline, +) __all__ = [ "collect_attribution", @@ -209,6 +214,9 @@ "collect_sweep", "collect_telemetry", "collect_token_usage", + "collect_v6_metadata", + "collect_v6_outcome", + "collect_v6_timeline", "collect_workload", "collect_model_info", "collect_recorded_optimizations", diff --git a/src/hyperloom/inference_optimizer/breakdown/collectors/v6.py b/src/hyperloom/inference_optimizer/breakdown/collectors/v6.py new file mode 100644 index 0000000000..e1ceceb20f --- /dev/null +++ b/src/hyperloom/inference_optimizer/breakdown/collectors/v6.py @@ -0,0 +1,275 @@ +"""Additive V6 projections built from the existing V5 evidence.""" + +from __future__ import annotations + +from pathlib import Path +from typing import Any + +from ...session.sbd_v6 import SCHEMA_VERSION_V6, read_timeline_events + + +_SUCCESS_STOP_REASONS = frozenset( + { + "target_reached", + "global_converged", + "time_exhausted", + "max_ticks", + "sweep_done", + "conc_sweep_done", + } +) +_ABORTED_STOP_REASONS = frozenset({"signal", "user_stop_requested"}) +_MODEL_GATE_STOP_REASONS = frozenset( + { + "model_context_window_too_small", + "model_config_incompatible", + "unsupported_model_arch", + } +) + + +def _tool_versions(versions: Any) -> dict[str, str | None]: + if not isinstance(versions, dict): + return {} + tools: dict[str, str | None] = {} + for name, value in versions.items(): + tool = str(name or "").strip() + if not tool: + continue + if isinstance(value, str): + tools[tool] = value or None + continue + if isinstance(value, dict): + label = value.get("version") or value.get("commit") + tools[tool] = str(label) if label not in (None, "") else None + return tools + + +def _architecture(workload: dict[str, Any], model_info: dict[str, Any]) -> dict[str, Any]: + if not workload and not model_info: + return {} + model_class = str(workload.get("model_class") or "").strip() + if not model_class and model_info: + model_class = "moe" if bool(model_info.get("is_moe")) else "dense" + return { + "model_class": model_class, + "model_type": str(model_info.get("model_type") or ""), + "num_hidden_layers": model_info.get("num_hidden_layers"), + "attention_type": str(model_info.get("attention_type") or ""), + "num_experts": model_info.get("num_experts"), + } + + +def _langfuse_projection(langfuse: dict[str, Any]) -> dict[str, Any]: + config = langfuse.get("config") if isinstance(langfuse.get("config"), dict) else {} + trace_url = langfuse.get("trace_url") + if not trace_url: + host = str(config.get("host") or "").rstrip("/") + trace_id = str(langfuse.get("trace_id") or "").strip() + if host and trace_id: + trace_url = f"{host}/trace/{trace_id}" + return { + "enabled": bool(langfuse.get("enabled")), + "trace_url": trace_url or None, + } + + +def collect_v6_metadata( + *, + exported_at_utc: str, + session: dict[str, Any], + workload: dict[str, Any], + model_info: dict[str, Any], + langfuse: dict[str, Any], + versions: dict[str, Any], + state: dict[str, Any], + warnings: list[str], +) -> dict[str, Any]: + """Project V5 session/config sections into the V6 ``metadata`` shape.""" + recovery = session.get("recovery") if isinstance(session.get("recovery"), dict) else {} + task_config = { + "model_name": str(workload.get("model_name") or ""), + "model_path": str(workload.get("model_path") or ""), + "framework_name": str(workload.get("framework_name") or ""), + "framework_version": str(workload.get("framework_version") or ""), + "gpu_type": str(workload.get("gpu_type") or ""), + "tp": workload.get("tp"), + "conc": workload.get("conc"), + "isl": workload.get("isl"), + "osl": workload.get("osl"), + "precision": str(workload.get("precision") or ""), + "max_model_len": workload.get("max_model_len"), + "objective": dict(workload.get("objective") or {}), + "launch_env": dict(state.get("operator_extra_env") or {}), + "launch_server_args": str(state.get("operator_server_args") or state.get("server_args") or ""), + "architecture": _architecture(workload, model_info), + } + return { + "exported_at_utc": exported_at_utc, + "versions": { + "schema_version": SCHEMA_VERSION_V6, + "hyperloom": str(session.get("code_revision") or ""), + "framework": str(workload.get("framework_name") or "") or None, + "framework_version": str(workload.get("framework_version") or "") or None, + "tools": _tool_versions(versions), + }, + "session": { + "session_id": str(session.get("session_id") or ""), + "claw_session_id": session.get("claw_session_id"), + "sandbox_user_id": session.get("sandbox_user_id"), + "created_at_utc": str(session.get("created_at_utc") or ""), + "start_ts": str(session.get("start_ts") or ""), + "ended_at_utc": str(session.get("ended_at_utc") or ""), + "host": str(session.get("host") or ""), + "session_dir": str(session.get("session_dir") or ""), + "user_data_path": str(session.get("user_data_path") or ""), + "code_revision": str(session.get("code_revision") or ""), + "pid": int(session.get("pid") or 0), + "max_minutes": int(session.get("max_minutes") or 0), + "elapsed_minutes": float(session.get("elapsed_minutes") or 0.0), + "tick_count": int(session.get("tick_count") or 0), + "recovery": { + "recovered": bool(recovery.get("recovered")), + "crash_count": int(recovery.get("crash_count") or 0), + "degraded_mode": bool(recovery.get("degraded_mode")), + }, + }, + "task_config": task_config, + "langfuse": _langfuse_projection(langfuse), + "warnings": list(warnings), + } + + +def collect_v6_timeline( + session_dir: Path, + warnings: list[str], + *, + state: dict[str, Any] | None = None, + recorded_operations: list[dict[str, Any]] | None = None, +) -> list[dict[str, Any]]: + """Load durable install and model-gate events without mutating V5 state.""" + del state, recorded_operations + return read_timeline_events(session_dir, warnings=warnings) + + +def _outcome_status(stop_reason: str) -> str: + if stop_reason in _SUCCESS_STOP_REASONS: + return "completed" + if stop_reason in _ABORTED_STOP_REASONS or not stop_reason: + return "aborted" + return "failed" + + +def _stage_reached( + state: dict[str, Any], + stop_reason: str, + timeline: list[dict[str, Any]], +) -> str: + if stop_reason in _MODEL_GATE_STOP_REASONS: + return "model_gate" + phase = str(state.get("phase") or "").strip().upper() + history = state.get("phase_history") + if isinstance(history, list): + for row in reversed(history): + if isinstance(row, dict) and str(row.get("to_phase") or "").strip(): + phase = str(row.get("to_phase") or "").strip().upper() + break + if phase == "PRELUDE": + if state.get("roofline_snapshots") or state.get("last_roofline") or state.get("roofline_attempts"): + return "roofline" + if state.get("last_profile_trace") or state.get("last_profile") or state.get("profile_attempts"): + return "profile" + if state.get("warm_replay_attempted") or state.get("warm_replay_outcome") or state.get("warm_replay_pending"): + return "warm_replay" + enablement = state.get("enablement") + if isinstance(enablement, dict) and any( + ( + int(enablement.get("attempts") or 0) > 0, + bool(enablement.get("pending")), + bool(enablement.get("validation_pending")), + bool(enablement.get("succeeded")), + bool(enablement.get("launch_log")), + bool(enablement.get("inflight_task_id")), + ) + ): + return "enablement" + baseline_tput = state.get("baseline_tput") + if ( + isinstance(baseline_tput, (int, float)) + and baseline_tput > 0 + or state.get("last_baseline") + or state.get("baseline_attempts") + or int(state.get("baseline_failure_streak") or 0) > 0 + ): + return "baseline" + if ( + state.get("warm_start_ts") + or state.get("warm_start_recipe") + or state.get("warm_start_pitfalls") + or state.get("warm_start_lessons") + or state.get("warm_start_context") + ): + return "warm_start" + phase_map = { + "FRAMEWORK_AGENT": "framework_agent", + "EXPLORE": "framework_agent", + "KERNEL_AGENT": ( + "kernel" + if any(state.get(key) for key in ("last_kernel_opt", "last_fusion", "last_gemm_tuning", "last_collective")) + else "kernel_agent" + ), + "SWEEP": ("conc_sweep" if state.get("last_conc_sweep") or state.get("last_conc_sweep_watermark") else "sweep"), + "CLOSE": "close", + } + if phase in phase_map: + return phase_map[phase] + if timeline: + return str(timeline[-1].get("type") or "") + return "install" + + +def collect_v6_outcome( + *, + session: dict[str, Any], + baseline: dict[str, Any], + final: dict[str, Any], + optimizations: dict[str, Any], + state: dict[str, Any], + timeline: list[dict[str, Any]], +) -> dict[str, Any]: + """Project V5 result sections into the V6 ``outcome`` shape.""" + stop_reason = str(session.get("stop_reason") or "").strip() + validation = optimizations.get("validation") + if not isinstance(validation, dict): + validation = {} + return { + "stop_reason": stop_reason, + "status": _outcome_status(stop_reason), + "stage_reached": _stage_reached(state, stop_reason, timeline), + "baseline": { + "throughput_tok_s_per_gpu": baseline.get("throughput_tok_s_per_gpu"), + "accuracy": baseline.get("accuracy"), + "ttft_mean_ms": baseline.get("ttft_mean_ms"), + "e2el_mean_ms": baseline.get("e2el_mean_ms"), + }, + "final": { + "throughput_tok_s_per_gpu": final.get("throughput_tok_s_per_gpu"), + "gain_pct": final.get("cumulative_gain_pct_validated", 0.0), + "action_path": list(final.get("action_path") or []), + "extra_envs": dict(final.get("extra_envs") or {}), + "extra_server_args": str(final.get("extra_server_args") or ""), + }, + "validation": { + "attributed_gain_pct": validation.get("attributed_total_gain_pct", 0.0), + "unattributed_gain_pct": validation.get("unattributed_gain_pct", 0.0), + "reconciliation_gap_pct": validation.get("reconciliation_gap_pct"), + "notes": list(validation.get("notes") or []), + }, + } + + +__all__ = [ + "collect_v6_metadata", + "collect_v6_outcome", + "collect_v6_timeline", +] diff --git a/src/hyperloom/inference_optimizer/breakdown/exporter.py b/src/hyperloom/inference_optimizer/breakdown/exporter.py index f3ec196c7d..233c9375e0 100644 --- a/src/hyperloom/inference_optimizer/breakdown/exporter.py +++ b/src/hyperloom/inference_optimizer/breakdown/exporter.py @@ -560,6 +560,48 @@ def _pick(section: str, collector_value: Any) -> Any: warnings, default={}, ) + v6_warnings = list(warnings) + timeline = _safe_collect( + "timeline", + lambda: collectors.collect_v6_timeline( + sd, + v6_warnings, + state=state, + recorded_operations=recorded_operations, + ), + v6_warnings, + default=[], + ) + outcome = _safe_collect( + "outcome", + lambda: collectors.collect_v6_outcome( + session=session_section, + baseline=baseline, + final=final, + optimizations=optimizations, + state=state, + timeline=timeline, + ), + v6_warnings, + default={}, + ) + metadata = _safe_collect( + "metadata", + lambda: collectors.collect_v6_metadata( + exported_at_utc=exported_at, + session=session_section, + workload=workload, + model_info=model_info, + langfuse=langfuse, + versions=versions, + state=state, + warnings=v6_warnings, + ), + v6_warnings, + default={}, + ) + if isinstance(metadata, dict): + metadata["warnings"] = list(v6_warnings) breakdown = { "schema_version": schema_version, @@ -619,6 +661,10 @@ def _pick(section: str, collector_value: Any) -> Any: "versions": versions, # Enablement attempt-runtime observability; {} → hidden. "enablement": enablement, + "metadata": metadata, + "outcome": outcome, + "timeline": timeline, + "close": {}, "warnings": warnings, "source_files": source_files, } diff --git a/src/hyperloom/inference_optimizer/breakdown/schema.py b/src/hyperloom/inference_optimizer/breakdown/schema.py index dbc8a1a2b8..de0a99adbc 100644 --- a/src/hyperloom/inference_optimizer/breakdown/schema.py +++ b/src/hyperloom/inference_optimizer/breakdown/schema.py @@ -13,6 +13,8 @@ from typing import Any, Literal, TypedDict +from ..session.sbd_v6 import SCHEMA_VERSION_V6 + #: Historical collector-only schema retained for archived-reader identification. SCHEMA_VERSION_V2 = "hyperloom.session_breakdown.v2" @@ -2923,6 +2925,101 @@ class Integrity(TypedDict, total=False): conflicts: list[dict[str, Any]] +class V6MetadataVersions(TypedDict, total=False): + """Version identifiers projected into V6 metadata.""" + + schema_version: str + hyperloom: str + framework: str | None + framework_version: str | None + tools: dict[str, str | None] + + +class V6MetadataSession(TypedDict, total=False): + """Session identity and lifecycle fields exposed by V6 metadata.""" + + session_id: str + claw_session_id: str | None + sandbox_user_id: str | None + created_at_utc: str + start_ts: str + ended_at_utc: str + host: str + session_dir: str + user_data_path: str + code_revision: str + pid: int + max_minutes: int + elapsed_minutes: float + tick_count: int + recovery: dict[str, Any] + + +class V6TaskConfig(TypedDict, total=False): + """Launch-time workload and model architecture projected into V6.""" + + model_name: str + model_path: str + framework_name: str + framework_version: str + gpu_type: str + tp: int | None + conc: int | None + isl: int | None + osl: int | None + precision: str + max_model_len: int | None + objective: dict[str, Any] + launch_env: dict[str, str] + launch_server_args: str + architecture: dict[str, Any] + + +class V6Metadata(TypedDict, total=False): + """V6 task identity, configuration, versions, and trace entrypoint.""" + + exported_at_utc: str + versions: V6MetadataVersions + session: V6MetadataSession + task_config: V6TaskConfig + langfuse: dict[str, Any] + warnings: list[str] + + +class V6Outcome(TypedDict, total=False): + """V6 session result projection for downstream consumers.""" + + stop_reason: str + status: Literal["completed", "failed", "aborted"] + stage_reached: str + baseline: dict[str, Any] + final: dict[str, Any] + validation: dict[str, Any] + + +class V6TimelineEvent(TypedDict, total=False): + """One ordered V6 business-stage event; CLOSE is intentionally excluded.""" + + type: str + kind: str + status: str + start_time: str + end_time: str + ext: dict[str, Any] + + +class V6Close(TypedDict, total=False): + """V6 session finalization result exposed outside the business timeline.""" + + status: Literal["succeeded", "failed", "degraded"] + start_time: str + end_time: str + close_sequence_done: bool + steps: list[dict[str, Any]] + robustness: dict[str, Any] + artifacts: dict[str, Any] + + class SessionBreakdown(TypedDict, total=False): """Top-level wire shape of ``session_breakdown.json``. @@ -3018,6 +3115,10 @@ class SessionBreakdown(TypedDict, total=False): versions: dict[str, KernelToolMetadata] # Enablement attempt-runtime observability; {} → dashboard hides the block. enablement: EnablementBreakdown + metadata: V6Metadata + outcome: V6Outcome + timeline: list[V6TimelineEvent] + close: V6Close warnings: list[str] source_files: SourceFiles @@ -3028,6 +3129,7 @@ class SessionBreakdown(TypedDict, total=False): "SCHEMA_VERSION_V2", "SCHEMA_VERSION_V3", "SCHEMA_VERSION_V5", + "SCHEMA_VERSION_V6", "Adoption", "AdoptedKernel", "ArtifactRef", @@ -3122,6 +3224,13 @@ class SessionBreakdown(TypedDict, total=False): "TokenUsageAttribution", "TokenUsageBucket", "TokenUsageTimelineEntry", + "V6Metadata", + "V6MetadataSession", + "V6MetadataVersions", + "V6Close", + "V6Outcome", + "V6TaskConfig", + "V6TimelineEvent", "Workload", "WorkloadObjective", ] diff --git a/src/hyperloom/inference_optimizer/breakdown/session_package.py b/src/hyperloom/inference_optimizer/breakdown/session_package.py index 00bea4fc3a..5e029527e7 100644 --- a/src/hyperloom/inference_optimizer/breakdown/session_package.py +++ b/src/hyperloom/inference_optimizer/breakdown/session_package.py @@ -76,6 +76,7 @@ "reports/kernel_optimization_summary.json", "reports/kernel_roofline.json", "reports/conc_sweep_summary.json", + "reports/sbd_v6/*.json", "reports/trace/*.jsonl", # ── target analysis ─────────────────────────────────────────────── "target_analysis/target_baseline.json", diff --git a/src/hyperloom/inference_optimizer/cli/__init__.py b/src/hyperloom/inference_optimizer/cli/__init__.py index 5fbbb1cc69..7ae423e325 100644 --- a/src/hyperloom/inference_optimizer/cli/__init__.py +++ b/src/hyperloom/inference_optimizer/cli/__init__.py @@ -43,11 +43,14 @@ _autodetect_gpu_type, _gpu_runner_type, _load_model_max_position_embeddings, + _finish_model_gate, _preflight_context_window, _preflight_model_config_compat, _preflight_unsupported_model_arch, + _record_resumed_model_gate, _resolve_gpu_type, _resolve_max_model_len, + _start_model_gate, ) from ..model_config_utils import ( summarize_model_config, @@ -108,6 +111,7 @@ ENV_USER_DATA_PATH, asset_system_prompts_dir, make_session_dir, + workspace_root, ) @@ -126,6 +130,8 @@ ) from .preflight import ( _check_gfx_arch_resolvable, + _mark_pending_install_event_failed, + _persist_install_event, _preflight as _preflight, ) @@ -1831,6 +1837,96 @@ def _exit_code_for_stop_reason(stop_reason: str | None) -> int: return 0 if (stop_reason or "") in _SUCCESS_STOP_REASONS else 1 +def _preflight_failure_session_dir(args: argparse.Namespace) -> Path: + """Return a safe session directory for a pre-session install failure.""" + resume_from = str(getattr(args, "resume_from", "") or "").strip() + if resume_from: + candidate = Path(resume_from).expanduser().resolve() + try: + candidate.relative_to(workspace_root().resolve()) + except (OSError, ValueError): + pass + else: + if candidate.is_dir(): + return candidate + + return _new_preflight_failure_session_dir(args) + + +def _new_preflight_failure_session_dir( + args: argparse.Namespace, + *, + failed_attempt: bool = False, +) -> Path: + """Create a standalone session for a failed preflight attempt.""" + model_name = resolve_model_display_name(args) + if not model_name: + model_name = Path(os.environ.get("MODEL_PATH", "")).name + if not model_name: + resume_from = str(getattr(args, "resume_from", "") or "").strip() + if resume_from: + model_name = Path(resume_from).expanduser().parent.name + if failed_attempt: + model_name = f"{model_name or 'preflight'}-failed-attempt" + return make_session_dir(model_name=model_name or "preflight-failure") + + +def _persist_preflight_failure_artifacts( + args: argparse.Namespace, + exc: BaseException, +) -> Path | None: + """Best-effort materialize the failed install event and final SBD.""" + _mark_pending_install_event_failed(args, exc) + try: + session_dir = _preflight_failure_session_dir(args) + except Exception: # noqa: BLE001 — never replace the original preflight failure + log.warning("failed to create a session for SBD V6 preflight failure", exc_info=True) + return None + + session_lock = SessionLock(session_dir) + try: + session_lock.acquire() + except Exception as lock_exc: # noqa: BLE001 — a busy resume must not mutate the active session + session_lock.release() + if not str(getattr(args, "resume_from", "") or "").strip(): + log.warning("failed to lock SBD V6 preflight failure session", exc_info=True) + return None + log.warning( + "resume session unavailable for preflight failure artifacts (%s); using an isolated failed-attempt session", + lock_exc, + ) + try: + session_dir = _new_preflight_failure_session_dir(args, failed_attempt=True) + session_lock = SessionLock(session_dir) + session_lock.acquire() + except Exception: # noqa: BLE001 — never replace the original preflight failure + session_lock.release() + log.warning("failed to create an isolated SBD V6 preflight failure session", exc_info=True) + return None + + try: + if not (session_dir / "manifest.json").is_file(): + try: + manifest_args = argparse.Namespace(**vars(args)) + if not getattr(manifest_args, "model", None): + manifest_args.model = os.environ.get("MODEL_PATH", "") + write_manifest(session_dir, args=manifest_args) + except Exception: # noqa: BLE001 — the install event can still stand alone + log.warning("failed to write manifest for SBD V6 preflight failure", exc_info=True) + + _persist_install_event(args, session_dir) + try: + from ..breakdown import write_breakdown_json + + write_breakdown_json(session_dir) + except Exception: # noqa: BLE001 — never replace the original preflight failure + log.warning("failed to write SBD V6 preflight failure breakdown", exc_info=True) + finally: + session_lock.release() + print(f"Preflight failure artifacts: {session_dir}", file=sys.stderr) + return session_dir + + async def _run_optimize(args: argparse.Namespace) -> int: """Run the ``optimize`` subcommand end to end. @@ -1971,7 +2067,14 @@ async def _run_optimize(args: argparse.Namespace) -> int: # Capture provider intent before _preflight() fills missing endpoints # (preflight may populate OPENAI_BASE_URL from ANTHROPIC_BASE_URL). codex_follows_claude = _codex_model_should_follow_claude() - resolved_urls = _preflight(args) + try: + resolved_urls = _preflight(args) + except BaseException as exc: + try: + _persist_preflight_failure_artifacts(args, exc) + except BaseException: # noqa: BLE001 — preserve the original failure exactly + log.warning("failed to preserve SBD V6 preflight failure", exc_info=True) + raise _resolve_models_for_run( args, @@ -1979,7 +2082,6 @@ async def _run_optimize(args: argparse.Namespace) -> int: claude_follows_codex=claude_follows_codex, codex_follows_claude=codex_follows_claude, ) - # Before either session branch: these are read by the fresh-launch seeding # AND by the resume path, so this is the one place that covers both. _preflight_agentx_backend(args) @@ -2022,6 +2124,7 @@ async def _run_optimize(args: argparse.Namespace) -> int: # Single-optimizer guard: take the session lock before any state.json / # lease access. Held for the whole run. session_lock = _acquire_session_lock_or_exit(session_dir) + _persist_install_event(args, session_dir) try: manifest = load_manifest(session_dir) @@ -2259,6 +2362,16 @@ async def _run_optimize(args: argparse.Namespace) -> int: reanchor_budget = bool(prior_stop or prior_crash >= 3) _begin_resume_leg(state, reanchor_budget=reanchor_budget) state.save(session_dir) + _record_resumed_model_gate( + args, + session_dir, + workload_overrides={ + "model_path": str(state.model_path or manifest.get("model_path") or ""), + "model_name": str(state.model_name or manifest.get("model_name") or ""), + "framework": str(state.framework or manifest.get("framework") or ""), + "gpu_type": str(state.gpu_type or manifest.get("gpu_type") or ""), + }, + ) if reanchor_budget: override_note = " (--force-resume override)" if force_resume and prior_stop in gated_terminal else "" print(f" → cleared stop_reason and reset crash_count (was {prior_crash}) for fresh resume{override_note}") @@ -2427,6 +2540,7 @@ async def _run_optimize(args: argparse.Namespace) -> int: # and the owner pid is published for the robustness monitor. session_lock = _acquire_session_lock_or_exit(session_dir) manifest = write_manifest(session_dir, args=args) + _persist_install_event(args, session_dir) # One-shot Langfuse startup marker so a run killed before a breakdown # still leaves a correlatable trace. Best-effort, never fatal. try: @@ -2472,6 +2586,7 @@ async def _run_optimize(args: argparse.Namespace) -> int: session_id=manifest["session_id"], compute_partition=compute_partition, ) + _start_model_gate(args, session_dir) # Unsupported-model preflight: reject multimodal/vision configs (runs after seed, before heavy bring-up). if _preflight_unsupported_model_arch(args, session_dir): sys.exit(2) @@ -2482,6 +2597,7 @@ async def _run_optimize(args: argparse.Namespace) -> int: # Context-window preflight: reject when ISL+OSL+headroom exceeds max_position_embeddings (no stretch by policy). if _preflight_context_window(args, session_dir): sys.exit(2) + _finish_model_gate(args, session_dir) # Recipe KB T0 anchor (after seed for recipe_canonical_id, before Coordinator); skipped when --degraded-kb. recipe_kb_client = _bootstrap_recipe_kb( args, diff --git a/src/hyperloom/inference_optimizer/cli/model_gate.py b/src/hyperloom/inference_optimizer/cli/model_gate.py index 42df18a65e..88e601b520 100644 --- a/src/hyperloom/inference_optimizer/cli/model_gate.py +++ b/src/hyperloom/inference_optimizer/cli/model_gate.py @@ -14,12 +14,13 @@ import os import struct import sys -from collections.abc import Callable +from collections.abc import Callable, Mapping from dataclasses import dataclass from pathlib import Path from typing import Any from .. import gpu_types as _gpu_types +from ...common.timeutil import now_iso from ..model_config_utils import ( # noqa: F401 - re-exported for callers/tests GEMMA2_ARCHITECTURES as _GEMMA2_ARCHITECTURES, _config_architectures, @@ -1806,6 +1807,255 @@ def _detect_incompatible_model_config( _MAX_MODEL_LEN_HEADROOM = 4096 +_MODEL_GATE_ORDER = ( + "unsupported_model_arch", + "model_config_compat", + "context_window", +) +_MODEL_GATE_EVENT_ATTR = "_sbd_v6_model_gate_event" + + +def _model_gate_workload(args: argparse.Namespace) -> dict[str, Any]: + model_path = str(getattr(args, "model", "") or "") + return { + "model_path": model_path, + "model_name": str(getattr(args, "model_display_name", "") or "") + or (Path(model_path).name if model_path else ""), + "framework": str(getattr(args, "framework", "") or os.environ.get("FRAMEWORK", "")), + "gpu_type": str(getattr(args, "gpu_type", "") or os.environ.get("TARGET_GPU_TYPE", "")), + "isl": int(getattr(args, "isl", 0) or 0), + "osl": int(getattr(args, "osl", 0) or 0), + "allow_mm_text_fallback": bool(getattr(args, "allow_mm_text_fallback", True)), + "headroom_tokens": _context_headroom_tokens(), + "headroom_env": _CONTEXT_HEADROOM_ENV, + } + + +def _new_model_gate_event(args: argparse.Namespace) -> dict[str, Any]: + return { + "type": "model_gate", + "kind": "model_gate", + "status": "succeeded", + "start_time": now_iso(timespec="seconds"), + "end_time": "", + "ext": { + "run_kind": "fresh", + "skip_reason": None, + "failed_gate_id": None, + "workload": _model_gate_workload(args), + "checks": [], + "degraded": {"active": False, "warnings": []}, + }, + } + + +def _load_model_gate_event(args: argparse.Namespace, session_dir: Path) -> dict[str, Any]: + from ..session.sbd_v6 import read_timeline_event_for_update + + event = getattr(args, _MODEL_GATE_EVENT_ATTR, None) + if not isinstance(event, dict): + event = read_timeline_event_for_update(session_dir, "model_gate") + if event is None or str(event.get("type") or "") != "model_gate": + event = _new_model_gate_event(args) + setattr(args, _MODEL_GATE_EVENT_ATTR, event) + event.setdefault("kind", "model_gate") + event.setdefault("status", "succeeded") + event.setdefault("start_time", now_iso(timespec="seconds")) + event.setdefault("end_time", "") + ext = event.get("ext") + if not isinstance(ext, dict): + ext = {} + event["ext"] = ext + ext.setdefault("run_kind", "fresh") + ext.setdefault("skip_reason", None) + ext.setdefault("failed_gate_id", None) + ext["workload"] = _model_gate_workload(args) + checks = ext.get("checks") + ext["checks"] = [row for row in checks if isinstance(row, dict)] if isinstance(checks, list) else [] + degraded = ext.get("degraded") + if not isinstance(degraded, dict): + degraded = {} + ext["degraded"] = degraded + degraded["active"] = bool(degraded.get("active")) + warnings = degraded.get("warnings") + degraded["warnings"] = [row for row in warnings if isinstance(row, dict)] if isinstance(warnings, list) else [] + return event + + +def _write_model_gate_event(session_dir: Path, event: dict[str, Any]) -> bool: + from ..session.sbd_v6 import write_timeline_event + + try: + write_timeline_event(session_dir, event) + except Exception: # noqa: BLE001 — observability must never change gate behavior + log.warning("failed to persist SBD V6 model-gate event", exc_info=True) + return False + return True + + +def _model_gate_status( + checks: list[dict[str, Any]], + *, + skip_reason: str | None = None, +) -> str: + statuses = {str(check.get("status") or "") for check in checks} + if "failed" in statuses: + return "failed" + if "warned" in statuses or "unknown" in statuses: + return "degraded" + if skip_reason: + return "skipped" + return "succeeded" + + +def _model_gate_check_order(check: dict[str, Any]) -> int: + try: + return int(check.get("order") or 0) + except (TypeError, ValueError): + return len(_MODEL_GATE_ORDER) + 1 + + +def _record_model_gate_check( + args: argparse.Namespace, + session_dir: Path, + check: dict[str, Any], + *, + failure: dict[str, Any] | None = None, + degraded_warning: dict[str, Any] | None = None, +) -> None: + try: + event = _load_model_gate_event(args, session_dir) + ext = event["ext"] + checks = [ + row for row in ext.get("checks", []) if isinstance(row, dict) and row.get("gate_id") != check.get("gate_id") + ] + checks.append(check) + checks.sort(key=_model_gate_check_order) + if failure is not None: + failed_order = int(check.get("order") or 0) + present = {str(row.get("gate_id") or "") for row in checks} + for order, gate_id in enumerate(_MODEL_GATE_ORDER, start=1): + if order > failed_order and gate_id not in present: + checks.append( + { + "gate_id": gate_id, + "order": order, + "status": "skipped", + "skip_reason": "prior_gate_failed", + "detail": {}, + } + ) + checks.sort(key=_model_gate_check_order) + ext["failed_gate_id"] = str(check.get("gate_id") or "") + ext["failure"] = failure + event["end_time"] = now_iso(timespec="seconds") + if degraded_warning is not None: + degraded = ext.setdefault("degraded", {"active": False, "warnings": []}) + degraded["active"] = True + degraded.setdefault("warnings", []).append(degraded_warning) + ext["checks"] = checks + event["status"] = _model_gate_status( + checks, + skip_reason=str(ext.get("skip_reason") or "") or None, + ) + _write_model_gate_event(session_dir, event) + except Exception: # noqa: BLE001 — V6 observability must never change gate behavior + log.warning("failed to record SBD V6 model-gate check", exc_info=True) + + +def _start_model_gate(args: argparse.Namespace, session_dir: Path) -> None: + """Create the model-gate event before the first check executes.""" + try: + event = _new_model_gate_event(args) + setattr(args, _MODEL_GATE_EVENT_ATTR, event) + _write_model_gate_event(session_dir, event) + except Exception: # noqa: BLE001 — V6 observability must never change launch behavior + log.warning("failed to initialize SBD V6 model-gate event", exc_info=True) + + +def _finish_model_gate(args: argparse.Namespace, session_dir: Path) -> None: + """Finalize a successfully completed three-check model-gate chain.""" + try: + event = _load_model_gate_event(args, session_dir) + ext = event["ext"] + event["status"] = _model_gate_status( + ext["checks"], + skip_reason=str(ext.get("skip_reason") or "") or None, + ) + event["end_time"] = now_iso(timespec="seconds") + _write_model_gate_event(session_dir, event) + except Exception: # noqa: BLE001 — V6 observability must never change launch behavior + log.warning("failed to finalize SBD V6 model-gate event", exc_info=True) + + +def _record_resumed_model_gate( + args: argparse.Namespace, + session_dir: Path, + *, + workload_overrides: Mapping[str, Any] | None = None, +) -> None: + """Persist the explicit V6 skip required by the resume path.""" + try: + timestamp = now_iso(timespec="seconds") + event = _new_model_gate_event(args) + event["status"] = "skipped" + event["start_time"] = timestamp + event["end_time"] = timestamp + event["ext"]["run_kind"] = "resume" + event["ext"]["skip_reason"] = "resume" + if workload_overrides: + event["ext"]["workload"].update(workload_overrides) + event["ext"]["checks"] = [ + { + "gate_id": gate_id, + "order": order, + "status": "skipped", + "skip_reason": "resume", + "detail": {}, + } + for order, gate_id in enumerate(_MODEL_GATE_ORDER, start=1) + ] + setattr(args, _MODEL_GATE_EVENT_ATTR, event) + _write_model_gate_event(session_dir, event) + except Exception: # noqa: BLE001 — V6 observability must never change resume behavior + log.warning("failed to record resumed SBD V6 model-gate event", exc_info=True) + + +def _write_model_gate_breakdown( + args: argparse.Namespace, + session_dir: Path, + *, + failure_label: str, +) -> None: + """Write the fail-fast SBD and then persist its truthful artifact status.""" + try: + from ..breakdown import write_breakdown_json + + write_breakdown_json(session_dir) + except Exception as exc: # noqa: BLE001 — never mask the gate failure + print( + f"WARNING: failed to write session_breakdown.json on {failure_label} fail-fast: {exc!r}", + file=sys.stderr, + ) + return + + try: + event = _load_model_gate_event(args, session_dir) + ext = event.get("ext") + failure = ext.get("failure") if isinstance(ext, dict) else None + artifacts = failure.get("artifacts") if isinstance(failure, dict) else None + if not isinstance(artifacts, dict): + return + artifacts["breakdown_written"] = True + if not _write_model_gate_event(session_dir, event): + return + write_breakdown_json(session_dir) + except Exception: # noqa: BLE001 — V6 refresh must not mask the gate failure + log.warning( + "failed to refresh session_breakdown.json with model-gate artifact status", + exc_info=True, + ) + def _context_headroom_tokens() -> int: """Resolve the context headroom (tokens); env override, else default. @@ -1892,13 +2142,71 @@ def _preflight_context_window(args: argparse.Namespace, session_dir: Path) -> bo isl = int(getattr(args, "isl", 0) or 0) osl = int(getattr(args, "osl", 0) or 0) if isl <= 0 or osl <= 0: + _record_model_gate_check( + args, + session_dir, + { + "gate_id": "context_window", + "order": 3, + "status": "skipped", + "skip_reason": "isl_osl_unset", + "detail": { + "isl": isl, + "osl": osl, + "headroom": _context_headroom_tokens(), + "required": None, + "max_position_embeddings": None, + "fits": None, + "policy": "no_context_length_override", + }, + }, + ) return False maxpos = _load_model_max_position_embeddings(str(getattr(args, "model", "") or "")) if not maxpos: + headroom = _context_headroom_tokens() + _record_model_gate_check( + args, + session_dir, + { + "gate_id": "context_window", + "order": 3, + "status": "skipped", + "skip_reason": "max_position_unknown", + "detail": { + "isl": isl, + "osl": osl, + "headroom": headroom, + "required": isl + osl + headroom, + "max_position_embeddings": None, + "fits": None, + "policy": "no_context_length_override", + }, + }, + ) return False headroom = _context_headroom_tokens() required = isl + osl + headroom if maxpos >= required: + _record_model_gate_check( + args, + session_dir, + { + "gate_id": "context_window", + "order": 3, + "status": "passed", + "skip_reason": None, + "detail": { + "isl": isl, + "osl": osl, + "headroom": headroom, + "required": required, + "max_position_embeddings": maxpos, + "fits": True, + "policy": "no_context_length_override", + }, + }, + ) return False reason = ( @@ -1938,17 +2246,38 @@ def _preflight_context_window(args: argparse.Namespace, session_dir: Path) -> bo f"WARNING: failed to persist context-window stop report: {exc!r}", file=sys.stderr, ) + _record_model_gate_check( + args, + session_dir, + { + "gate_id": "context_window", + "order": 3, + "status": "failed", + "skip_reason": None, + "detail": { + "isl": isl, + "osl": osl, + "headroom": headroom, + "required": required, + "max_position_embeddings": maxpos, + "fits": False, + "policy": "no_context_length_override", + }, + }, + failure={ + "gate_id": "context_window", + "stop_reason": "model_context_window_too_small", + "exit_code": 2, + "message": reason, + "artifacts": { + "final_json": "reports/final.json" if (session_dir / "reports" / "final.json").is_file() else None, + "breakdown_written": False, + }, + }, + ) # Delivery-artifact parity: emit session_breakdown.json here too since # fail-fast exits before coordinator.run()'s finally. - try: - from ..breakdown import write_breakdown_json - - write_breakdown_json(session_dir) - except Exception as exc: # noqa: BLE001 — best-effort; never mask the reason - print( - f"WARNING: failed to write session_breakdown.json on context fail-fast: {exc!r}", - file=sys.stderr, - ) + _write_model_gate_breakdown(args, session_dir, failure_label="context") # Langfuse parity: this gate exits before coordinator.run()'s finally, so # push the breakdown to Langfuse here too. _emit_breakdown_to_langfuse(session_dir) @@ -1985,6 +2314,25 @@ def _preflight_model_config_compat( framework=framework, ) if detail is None: + model_dir = resolve_local_model_dir(model) or Path(model) + config_path = model_dir / "config.json" + absent = not config_path.is_file() + _record_model_gate_check( + args, + session_dir, + { + "gate_id": "model_config_compat", + "order": 2, + "status": "skipped" if absent else "passed", + "skip_reason": "config_absent_soft_pass" if absent else None, + "detail": { + "config_path": str(config_path) if config_path.is_file() else None, + "incompatible": False, + "reason": None, + "detector": None, + }, + }, + ) return False name = Path(model).name or model reason = ( @@ -2018,15 +2366,35 @@ def _preflight_model_config_compat( f"WARNING: failed to persist model-config stop report: {exc!r}", file=sys.stderr, ) - try: - from ..breakdown import write_breakdown_json - - write_breakdown_json(session_dir) - except Exception as exc: # noqa: BLE001 — best-effort; never mask the reason - print( - f"WARNING: failed to write session_breakdown.json on config fail-fast: {exc!r}", - file=sys.stderr, - ) + model_dir = resolve_local_model_dir(model) or Path(model) + config_path = model_dir / "config.json" + _record_model_gate_check( + args, + session_dir, + { + "gate_id": "model_config_compat", + "order": 2, + "status": "failed", + "skip_reason": None, + "detail": { + "config_path": str(config_path) if config_path.is_file() else None, + "incompatible": True, + "reason": detail, + "detector": None, + }, + }, + failure={ + "gate_id": "model_config_compat", + "stop_reason": "model_config_incompatible", + "exit_code": 2, + "message": reason, + "artifacts": { + "final_json": "reports/final.json" if (session_dir / "reports" / "final.json").is_file() else None, + "breakdown_written": False, + }, + }, + ) + _write_model_gate_breakdown(args, session_dir, failure_label="config") # Langfuse parity: this gate exits before coordinator.run()'s finally, so # push the breakdown to Langfuse here too. _emit_breakdown_to_langfuse(session_dir) @@ -2076,11 +2444,49 @@ def _preflight_unsupported_model_arch( except Exception: # noqa: BLE001 — registry import must never block the gate is_scriptable = str(framework).strip().lower() == "xdit" if is_scriptable: + _record_model_gate_check( + args, + session_dir, + { + "gate_id": "unsupported_model_arch", + "order": 1, + "status": "skipped", + "skip_reason": "scriptable_framework", + "verdict": None, + "detail": { + "architecture": None, + "model_type": None, + "signal": None, + "allow_mm_text_fallback": bool(getattr(args, "allow_mm_text_fallback", True)), + "action": "proceed", + }, + }, + ) return False model = str(getattr(args, "model", "") or "") hit = _detect_unsupported_model(model) if hit is None: + config = _load_model_config_dict(model) + architectures = _config_architectures(config) if isinstance(config, dict) else [] + _record_model_gate_check( + args, + session_dir, + { + "gate_id": "unsupported_model_arch", + "order": 1, + "status": "passed" if isinstance(config, dict) else "unknown", + "skip_reason": None, + "verdict": "plain_text" if isinstance(config, dict) else None, + "detail": { + "architecture": architectures[0] if architectures else None, + "model_type": str(config.get("model_type") or "") if isinstance(config, dict) else None, + "signal": None, + "allow_mm_text_fallback": bool(getattr(args, "allow_mm_text_fallback", True)), + "action": "proceed", + }, + }, + ) return False name = Path(model).name or model @@ -2121,6 +2527,30 @@ def _preflight_unsupported_model_arch( f"WARNING: failed to persist degraded-mode marker: {exc!r}", file=sys.stderr, ) + _record_model_gate_check( + args, + session_dir, + { + "gate_id": "unsupported_model_arch", + "order": 1, + "status": "warned", + "skip_reason": None, + "verdict": verdict, + "detail": { + "architecture": arch, + "model_type": mt, + "signal": str(hit.get("signal") or ""), + "allow_mm_text_fallback": allow_fallback, + "action": "proceed", + }, + }, + degraded_warning={ + "kind": "multimodal_text_fallback", + "architecture": arch, + "model_type": mt, + "signal": str(hit.get("signal") or ""), + }, + ) return False reason = ( @@ -2159,17 +2589,37 @@ def _preflight_unsupported_model_arch( f"WARNING: failed to persist unsupported-model stop report: {exc!r}", file=sys.stderr, ) + _record_model_gate_check( + args, + session_dir, + { + "gate_id": "unsupported_model_arch", + "order": 1, + "status": "failed", + "skip_reason": None, + "verdict": verdict, + "detail": { + "architecture": arch, + "model_type": mt, + "signal": str(hit.get("signal") or ""), + "allow_mm_text_fallback": allow_fallback, + "action": "fail_fast", + }, + }, + failure={ + "gate_id": "unsupported_model_arch", + "stop_reason": "unsupported_model_arch", + "exit_code": 2, + "message": reason, + "artifacts": { + "final_json": "reports/final.json" if (session_dir / "reports" / "final.json").is_file() else None, + "breakdown_written": False, + }, + }, + ) # Delivery-artifact parity: emit session_breakdown.json here too since # fail-fast exits before coordinator.run()'s finally. - try: - from ..breakdown import write_breakdown_json - - write_breakdown_json(session_dir) - except Exception as exc: # noqa: BLE001 — best-effort; never mask the reason - print( - f"WARNING: failed to write session_breakdown.json on unsupported-model fail-fast: {exc!r}", - file=sys.stderr, - ) + _write_model_gate_breakdown(args, session_dir, failure_label="unsupported-model") # Langfuse parity: this gate exits before coordinator.run()'s finally, so # push the breakdown to Langfuse here too. _emit_breakdown_to_langfuse(session_dir) diff --git a/src/hyperloom/inference_optimizer/cli/preflight.py b/src/hyperloom/inference_optimizer/cli/preflight.py index cfe6260c80..7237b70784 100644 --- a/src/hyperloom/inference_optimizer/cli/preflight.py +++ b/src/hyperloom/inference_optimizer/cli/preflight.py @@ -15,6 +15,7 @@ import subprocess import sys import tempfile +from collections.abc import Callable from pathlib import Path from typing import Any, NamedTuple @@ -39,6 +40,7 @@ RESOLVED_FRAMEWORK_PYTHON_ENV, detect_gfx_arch, ) +from hyperloom.common.timeutil import now_iso from .credentials import ( _is_stale_proxy_url, @@ -126,7 +128,7 @@ def _provider_only_mode() -> str: return "" -def _normalize_legacy_deepseek_env() -> None: +def _normalize_legacy_deepseek_env() -> dict[str, Any]: """Rewrite a retired ``DEEPSEEK_*`` configuration into the standard variables. DeepSeek serves the Anthropic protocol on ``/anthropic`` and the OpenAI @@ -135,6 +137,8 @@ def _normalize_legacy_deepseek_env() -> None: the only place in the runtime that reads the retired variables; everything downstream sees just the two protocol sides. """ + before = dict(os.environ) + had_legacy_config = any(os.environ.get(key) for key in LEGACY_DEEPSEEK_ENV_KEYS) updates = deepseek_compat_env() if updates: for key, value in updates.items(): @@ -146,9 +150,25 @@ def _normalize_legacy_deepseek_env() -> None: # A gateway that serves only its own models supplies the model ids too. # Exported (not just resolved) so subprocesses, GEAKv4 and the kernel-agent # installer inherit them instead of falling back to an AMD Claude id. - for key, value in provider_model_defaults().items(): + model_defaults = provider_model_defaults() + for key, value in model_defaults.items(): os.environ[key] = value print(f"Preflight: {key} -> {value} (implied by the configured gateway)") + changed = sorted(key for key in {*updates, *model_defaults} if before.get(key) != os.environ.get(key)) + if changed: + status = "applied" + skip_reason = None + elif had_legacy_config or updates or model_defaults: + status = "already_present" + skip_reason = None + else: + status = "skipped" + skip_reason = "legacy_env_absent" + return { + "status": status, + "skip_reason": skip_reason, + "detail": {"keys_set": changed}, + } def _restore_provider_only_mode(provider_mode: str, snapshot: dict[str, str | None]) -> None: @@ -200,7 +220,7 @@ def _is_placeholder_tracelens_path(value: str) -> bool: return False -def _load_dotenv_fallback() -> None: +def _load_dotenv_fallback() -> dict[str, Any]: """Source missing vars from ``$REPO_ROOT/.env``; env always wins (no-clobber). Always parses ``.env`` and loads any key not already present in the @@ -209,7 +229,11 @@ def _load_dotenv_fallback() -> None: """ env_file = _resolve_dotenv_file() if env_file is None: - return + return { + "status": "skipped", + "skip_reason": "dotenv_missing", + "detail": {"vars_loaded": 0, "source": None}, + } parsed: dict[str, str] = {} loaded = 0 for raw in env_file.read_text(encoding="utf-8", errors="replace").splitlines(): @@ -242,6 +266,11 @@ def _load_dotenv_fallback() -> None: loaded += 1 if loaded: print(f"Preflight: loaded {loaded} missing var(s) from {env_file} (env wins)") + return { + "status": "applied" if loaded else "already_present", + "skip_reason": None, + "detail": {"vars_loaded": loaded, "source": str(env_file)}, + } def _prepend_path(var: str, entry: str) -> None: @@ -308,12 +337,13 @@ def _parse_env_assignments(text: str) -> dict[str, str]: return out -def _correct_kernel_agent_path_vars(file_vars: dict[str, str], env_path: Path) -> None: +def _correct_kernel_agent_path_vars(file_vars: dict[str, str], env_path: Path) -> list[str]: """Overwrite invalid inherited path-class vars with the env file's value. Only fires when the inherited value is unset/non-existent AND the file value points at an existing dir; a valid inherited value keeps env-wins semantics. """ + corrected: list[str] = [] for key in _KERNEL_AGENT_PATH_VARS: file_val = file_vars.get(key) if not file_val or not Path(file_val).is_dir(): @@ -328,9 +358,11 @@ def _correct_kernel_agent_path_vars(file_vars: dict[str, str], env_path: Path) - file=sys.stderr, ) os.environ[key] = file_val + corrected.append(key) + return corrected -def _load_kernel_agent_env_fallback() -> None: +def _load_kernel_agent_env_fallback() -> dict[str, Any]: """Auto-source the installer-written kernel-agent env file (``$KERNEL_AGENT_ENV`` or ``$USER_DATA_PATH/runtime/kernel-agent.env.sh``). @@ -350,14 +382,26 @@ def _load_kernel_agent_env_fallback() -> None: # Root is set: no bootstrap, but still correct invalid path vars from the # env file when resolvable. if not candidate: - return + return { + "status": "already_present", + "skip_reason": None, + "detail": {"vars_loaded": 0, "env_file": None}, + } env_path = Path(candidate) if not env_path.is_file(): - return + return { + "status": "already_present", + "skip_reason": None, + "detail": {"vars_loaded": 0, "env_file": str(env_path)}, + } try: text = env_path.read_text(encoding="utf-8", errors="replace") except OSError: - return + return { + "status": "already_present", + "skip_reason": None, + "detail": {"vars_loaded": 0, "env_file": str(env_path)}, + } file_vars, dropped_file_vars = filter_untrusted_env_mapping( _parse_env_assignments(text), allow_predicate=is_allowed_kernel_agent_env_key, @@ -367,8 +411,16 @@ def _load_kernel_agent_env_fallback() -> None: f"Preflight: WARNING — ignoring unsupported kernel-agent env key {key} from {env_path}", file=sys.stderr, ) - _correct_kernel_agent_path_vars(file_vars, env_path) - return + corrected = _correct_kernel_agent_path_vars(file_vars, env_path) + return { + "status": "applied" if corrected else "already_present", + "skip_reason": None, + "detail": { + "vars_loaded": 0, + "env_file": str(env_path), + "corrected_keys": corrected, + }, + } if not candidate: print( @@ -419,7 +471,7 @@ def _load_kernel_agent_env_fallback() -> None: if key not in os.environ: os.environ[key] = value loaded += 1 - _correct_kernel_agent_path_vars(file_vars, env_path) + corrected = _correct_kernel_agent_path_vars(file_vars, env_path) if "HYPERLOOM_KERNEL_AGENT_ROOT" not in os.environ: print( f"Preflight: ERROR — sourced {env_path} ({loaded} vars) but " @@ -434,9 +486,18 @@ def _load_kernel_agent_env_fallback() -> None: f"{env_path} (env wins, HYPERLOOM_KERNEL_AGENT_ROOT=" f"{os.environ['HYPERLOOM_KERNEL_AGENT_ROOT']})" ) - - -def _ensure_python_sdks(python_exe: str, pip_extra: list[str]) -> None: + return { + "status": "applied" if loaded or corrected else "already_present", + "skip_reason": None, + "detail": { + "vars_loaded": loaded, + "env_file": str(env_path), + "corrected_keys": corrected, + }, + } + + +def _ensure_python_sdks(python_exe: str, pip_extra: list[str]) -> dict[str, Any]: """Probe-then-install runtime-imported Python SDKs using the same interpreter that imports them. Avoids first-tick BackendError after baseline burns wall time; same-interpreter install avoids @@ -458,6 +519,8 @@ def _ensure_python_sdks(python_exe: str, pip_extra: list[str]) -> None: ("openai", "openai>=1.50"), ("httpx", "httpx>=0.27"), ) + installed: list[str] = [] + already_present: list[str] = [] for module_name, pip_spec in candidates: check = subprocess.run( [python_exe, "-c", f"import {module_name}"], @@ -465,6 +528,7 @@ def _ensure_python_sdks(python_exe: str, pip_extra: list[str]) -> None: ) if check.returncode == 0: print(f"Preflight: {module_name} OK") + already_present.append(pip_spec) continue print(f"Preflight: {module_name} not importable, installing {pip_spec} ...") subprocess.run( @@ -472,6 +536,17 @@ def _ensure_python_sdks(python_exe: str, pip_extra: list[str]) -> None: check=True, ) print(f"Preflight: installed {pip_spec}") + installed.append(pip_spec) + return { + "status": "applied" if installed else "already_present", + "skip_reason": None, + "target": ",".join(spec for _, spec in candidates), + "interpreter": python_exe, + "detail": { + "installed": installed, + "already_present": already_present, + }, + } _RAY_VERSION = "2.44.1" @@ -548,7 +623,7 @@ def _ray_smoke(python_exe: str) -> subprocess.CompletedProcess: ) -def _ensure_ray(python_exe: str, pip_extra: list[str]) -> None: +def _ensure_ray(python_exe: str, pip_extra: list[str]) -> dict[str, Any]: """Probe-then-install Ray using the interpreter that will import it. Ray is used broadly (multi-node scheduling, kernel/profile/recover @@ -567,7 +642,15 @@ def _ensure_ray(python_exe: str, pip_extra: list[str]) -> None: check = _ray_smoke(python_exe) if check.returncode == 0: print("Preflight: ray OK") - return + return { + "status": "already_present", + "skip_reason": None, + "target": "ray", + "interpreter": python_exe, + "spec": _RAY_INSTALL_SPEC, + "version_after": _RAY_VERSION, + "message": None, + } reason = (check.stderr or check.stdout or "unknown Ray smoke failure").strip().splitlines()[-1] print(f"Preflight: ray/click invalid ({reason}), installing {_RAY_INSTALL_SPEC} + {_CLICK_INSTALL_SPEC} ...") subprocess.run( @@ -579,6 +662,15 @@ def _ensure_ray(python_exe: str, pip_extra: list[str]) -> None: reason = (check.stderr or check.stdout or "unknown Ray smoke failure").strip() raise RuntimeError(f"Ray install completed but smoke test still failed: {reason}") print("Preflight: ray installed OK") + return { + "status": "applied", + "skip_reason": None, + "target": "ray", + "interpreter": python_exe, + "spec": _RAY_INSTALL_SPEC, + "version_after": _RAY_VERSION, + "message": reason, + } # InferenceX benchmark_serving client-side deps. Mirrors the ``_BENCH_SERVING_DEPS`` @@ -597,7 +689,7 @@ def _ensure_ray(python_exe: str, pip_extra: list[str]) -> None: ) -def _ensure_bench_serving_deps(python_exe: str, pip_extra: list[str]) -> None: +def _ensure_bench_serving_deps(python_exe: str, pip_extra: list[str]) -> dict[str, Any]: """Probe-then-install the InferenceX benchmark_serving client deps in python_exe. ``assets/install.sh:ensure_bench_serving_deps`` installs these into the @@ -627,16 +719,32 @@ def _ensure_bench_serving_deps(python_exe: str, pip_extra: list[str]) -> None: missing = [line.strip() for line in (result.stdout or "").splitlines() if line.strip()] if not missing: print("Preflight: benchmark_serving client deps OK") - return + return { + "status": "already_present", + "skip_reason": None, + "target": "benchmark_serving client deps", + "interpreter": python_exe, + "detail": {"installed": [], "already_present": mods}, + } print(f"Preflight: installing benchmark_serving client deps: {' '.join(missing)} ...") subprocess.run( [python_exe, "-m", "pip", "install", "--quiet", "--no-cache-dir", *pip_extra, *missing], check=True, ) print("Preflight: benchmark_serving client deps installed OK") - - -def _ensure_framework_deps(args, python_exe: str, pip_extra: list[str]) -> None: + return { + "status": "applied", + "skip_reason": None, + "target": "benchmark_serving client deps", + "interpreter": python_exe, + "detail": { + "installed": missing, + "already_present": [module for module in mods if module not in missing], + }, + } + + +def _ensure_framework_deps(args, python_exe: str, pip_extra: list[str]) -> dict[str, Any]: """Install the selected framework's declared runtime deps into python_exe. Resolution mirrors the CLI's own order (``--framework`` > ``$FRAMEWORK`` > @@ -661,8 +769,29 @@ def _ensure_framework_deps(args, python_exe: str, pip_extra: list[str]) -> None: # Frameworks that ship no manifest are the common case; stay quiet unless # the manifest actually asked for something or was partly rejected. if outcome.skipped_reason and not (outcome.refused or outcome.invalid): - return - framework_deps.report(outcome, prefix="Preflight: framework deps") + status = "skipped" + else: + framework_deps.report(outcome, prefix="Preflight: framework deps") + if outcome.failed or outcome.refused or outcome.invalid: + status = "warned" + elif outcome.installed: + status = "applied" + else: + status = "already_present" + return { + "status": status, + "skip_reason": outcome.skipped_reason or None, + "target": framework, + "interpreter": python_exe, + "detail": { + "manifest": str(outcome.manifest) if outcome.manifest is not None else None, + "installed": list(outcome.installed), + "already_present": list(outcome.already_present), + "refused": list(outcome.refused), + "invalid": list(outcome.invalid), + "failed": list(outcome.failed), + }, + } # Escape hatch for the serving-framework gate below, mirroring @@ -913,7 +1042,7 @@ def _resolve_framework_build(framework: str, interpreters: list[str]) -> tuple[s return inconclusive or refuted or (None, missing) -def _check_serving_framework(args, benchmark_python: str) -> None: +def _check_serving_framework(args, benchmark_python: str) -> dict[str, Any]: """Fail fast when the selected serving framework is not importable here. The optimizer patches and rebuilds framework code in place, so the package @@ -937,21 +1066,37 @@ def _check_serving_framework(args, benchmark_python: str) -> None: ).strip().lower() or framework_registry.DEFAULT_FRAMEWORK if framework_registry.is_scriptable(framework): - return + return { + "status": "skipped", + "skip_reason": "scriptable_framework", + "target": framework, + } if os.environ.get(SKIP_FRAMEWORK_CHECK_ENV, "").strip(): print(f"Preflight: {SKIP_FRAMEWORK_CHECK_ENV} set; skipping the {framework} importability check") - return + return { + "status": "skipped", + "skip_reason": "skip_framework_check_env", + "target": framework, + } remote_base_url = os.environ.get("BENCHMARK_BASE_URL", "").strip() if remote_base_url: print(f"Preflight: BENCHMARK_BASE_URL={remote_base_url}; skipping the local {framework} check (remote server)") - return + return { + "status": "skipped", + "skip_reason": "remote_benchmark_url", + "target": framework, + } from hyperloom.inference_optimizer.multi_node._internal.external_state import external_service_url from hyperloom.orchestrator.actions.executors._multi_node_env import is_multi_node if is_multi_node() and external_service_url(): print(f"Preflight: external multi-node mode; skipping the local {framework} check (serving is on remote pods)") - return + return { + "status": "skipped", + "skip_reason": "external_multi_node", + "target": framework, + } interpreters = _framework_probe_interpreters(framework, benchmark_python) found, probe = _resolve_framework_build(framework, interpreters) @@ -978,20 +1123,37 @@ def _check_serving_framework(args, benchmark_python: str) -> None: evidence = _rocm_evidence(framework) if found and probe.verdict is True: print(f"Preflight: {framework} importable ({found}); {evidence} confirms a ROCm build") - return + return { + "status": "applied", + "skip_reason": None, + "target": framework, + "detail": {"probe_interpreter": found, "rocm_verified": True}, + } if found and probe.verdict is None: print( f"Preflight: WARNING — {framework} is importable ({found}) but could not verify a ROCm build " f"via {evidence}{_probe_detail_block(probe.detail)}" ) - return + return { + "status": "warned", + "skip_reason": None, + "target": framework, + "message": probe.detail or "ROCm build could not be verified", + "detail": {"probe_interpreter": found, "rocm_verified": None}, + } if not found and probe.timed_out: # A timeout proves nothing, so blocking here would fail a merely slow host. print( f"Preflight: WARNING — the {framework} probe timed out; proceeding without verifying it" f"{_probe_detail_block(probe.detail)}" ) - return + return { + "status": "warned", + "skip_reason": None, + "target": framework, + "message": probe.detail or "framework probe timed out", + "detail": {"probe_interpreter": None, "rocm_verified": None}, + } # Every path below stops the run, so it needs a remedy that works. Both of # them name --install-framework, which setup rejects outside its own set. @@ -1008,7 +1170,13 @@ def _check_serving_framework(args, benchmark_python: str) -> None: "existing checkout on this host. Continuing; the benchmark will fail if it\n" f"genuinely needs {framework} here." ) - return + return { + "status": "warned", + "skip_reason": None, + "target": framework, + "message": probe.detail or "framework availability could not be verified", + "detail": {"probe_interpreter": found, "rocm_verified": probe.verdict}, + } if found: print( @@ -1020,7 +1188,7 @@ def _check_serving_framework(args, benchmark_python: str) -> None: f"To proceed anyway, set {SKIP_FRAMEWORK_CHECK_ENV}=1.", file=sys.stderr, ) - sys.exit(2) + raise SystemExit(2) if _in_container(): remedy = ( @@ -1051,7 +1219,7 @@ def _check_serving_framework(args, benchmark_python: str) -> None: "for a local run that is expected to serve from somewhere unprobed.", file=sys.stderr, ) - sys.exit(2) + raise SystemExit(2) # RUN_EVAL values that disable the accuracy gate (mirrors _workload_envs). @@ -1207,7 +1375,12 @@ def _resolved_eval_disabled(args: argparse.Namespace) -> bool: return bool(state.get("eval_disabled")) -def _ensure_lm_eval_dep(python_exe: str, pip_extra: list[str], *, eval_disabled: bool = False) -> None: +def _ensure_lm_eval_dep( + python_exe: str, + pip_extra: list[str], + *, + eval_disabled: bool = False, +) -> dict[str, Any]: """Probe-then-install ``lm_eval`` in python_exe when the accuracy gate is on. ``install.sh`` defers ``lm_eval`` to InferenceX's ``benchmark_lib.sh`` runtime @@ -1245,12 +1418,30 @@ def _ensure_lm_eval_dep(python_exe: str, pip_extra: list[str], *, eval_disabled: from hyperloom.orchestrator.actions.executors._multi_node_env import is_multi_node if not is_multi_node(): - return # single-node: InferenceX installs lm_eval itself, see above + return { + "status": "skipped", + "skip_reason": "single_node_runtime_install", + "target": "lm_eval[api]", + "interpreter": python_exe, + "message": "single-node InferenceX installs lm_eval on first use", + } if eval_disabled: - return # --no-eval: no eval anywhere, so the harness is never loaded + return { + "status": "skipped", + "skip_reason": "eval_disabled", + "target": "lm_eval[api]", + "interpreter": python_exe, + "message": "accuracy evaluation is disabled", + } run_eval = os.environ.get("RUN_EVAL") if run_eval is not None and run_eval.strip().lower() in _RUN_EVAL_FALSE_VALUES: - return # accuracy gate disabled; lm_eval not required + return { + "status": "skipped", + "skip_reason": "eval_disabled", + "target": "lm_eval[api]", + "interpreter": python_exe, + "message": f"RUN_EVAL={run_eval}", + } missing = _probe_missing_lm_eval_deps(python_exe) if missing is None: # Absence is unproven, so installing would be a guess that could replace @@ -1258,15 +1449,32 @@ def _ensure_lm_eval_dep(python_exe: str, pip_extra: list[str], *, eval_disabled: # already breaks the benchmark far more loudly than a missing accuracy # gate would, so this warns and changes nothing. print("Preflight: WARNING — cannot run the lm_eval probe; leaving the interpreter untouched") - return + return { + "status": "warned", + "skip_reason": None, + "target": "lm_eval[api]", + "interpreter": python_exe, + "message": "lm_eval dependency probe could not run", + } if not missing: print("Preflight: lm_eval[api] OK") - return + return { + "status": "already_present", + "skip_reason": None, + "target": "lm_eval[api]", + "interpreter": python_exe, + } if "lm_eval" in missing: print(f"Preflight: installing lm_eval[api]@{_LM_EVAL_PINNED_REF[:12]} (accuracy gate) ...") _install_pinned_lm_eval(python_exe, pip_extra) print("Preflight: lm_eval[api] installed OK") - return + return { + "status": "applied", + "skip_reason": None, + "target": "lm_eval[api]", + "interpreter": python_exe, + "detail": {"installed": list(missing)}, + } # The image already ships lm_eval. Install only the absent extra so pip # cannot resolve a different lm_eval build over the one baked in, which # would silently swap out a version the image pinned on purpose -- the same @@ -1288,6 +1496,13 @@ def _ensure_lm_eval_dep(python_exe: str, pip_extra: list[str], *, eval_disabled: check=True, ) print(f"Preflight: {' '.join(targets)} installed OK") + return { + "status": "applied", + "skip_reason": None, + "target": "lm_eval[api]", + "interpreter": python_exe, + "detail": {"installed": list(targets)}, + } def _unset_hip_visible_devices() -> None: @@ -1307,7 +1522,7 @@ def _unset_hip_visible_devices() -> None: ) -def _check_gpu_visibility() -> None: +def _check_gpu_visibility() -> dict[str, Any]: """Best-effort informational check of visible GPU count vs ``$TP`` (silent when rocm-smi is absent). Skipped in external multi-node mode: there the server runs on remote GPU @@ -1326,7 +1541,11 @@ def _check_gpu_visibility() -> None: if is_multi_node() and external_service_url(): print("Preflight: external multi-node mode; skipping local GPU visibility check (GPUs are on remote pods)") - return + return { + "status": "skipped", + "skip_reason": "external_multi_node", + "detail": {"visible": None, "tp_requested": None, "warn": None}, + } try: proc = subprocess.run( ["rocm-smi", "--showid"], @@ -1335,9 +1554,17 @@ def _check_gpu_visibility() -> None: timeout=5, ) except (FileNotFoundError, subprocess.TimeoutExpired, PermissionError, OSError): - return + return { + "status": "skipped", + "skip_reason": "rocm_smi_unavailable", + "detail": {"visible": None, "tp_requested": None, "warn": None}, + } if proc.returncode != 0: - return + return { + "status": "skipped", + "skip_reason": "rocm_smi_failed", + "detail": {"visible": None, "tp_requested": None, "warn": None}, + } # rocm-smi --showid emits multiple GPU[ lines per GPU; deduplicate by GPU index. visible_indices: set[str] = set() for line in (proc.stdout or "").splitlines(): @@ -1352,24 +1579,40 @@ def _check_gpu_visibility() -> None: except ValueError: wanted = 1 if visible == 0: - print("Preflight: WARNING — rocm-smi sees 0 GPUs; benchmark will fail") - return + warning = "rocm-smi sees 0 GPUs; benchmark will fail" + print(f"Preflight: WARNING — {warning}") + return { + "status": "warned", + "skip_reason": None, + "detail": {"visible": visible, "tp_requested": wanted, "warn": warning}, + } if wanted > visible: - print( - f"Preflight: WARNING — TP={wanted} but rocm-smi sees {visible} " - f"GPU(s); sglang/vllm may fail to load weights. Lower TP or " - f"adjust ROCR_VISIBLE_DEVICES." - ) + warning = f"TP={wanted} but rocm-smi sees {visible} GPU(s); sglang/vllm may fail to load weights" + print(f"Preflight: WARNING — {warning}. Lower TP or adjust ROCR_VISIBLE_DEVICES.") + return { + "status": "warned", + "skip_reason": None, + "detail": {"visible": visible, "tp_requested": wanted, "warn": warning}, + } + return { + "status": "applied", + "skip_reason": None, + "detail": {"visible": visible, "tp_requested": wanted, "warn": None}, + } -def _check_shm_disk() -> None: +def _check_shm_disk() -> dict[str, Any]: """Warn (not fail-fast) on tight ``/dev/shm`` (vLLM/NCCL IPC needs headroom).""" try: usage = shutil.disk_usage("/dev/shm") # nosec B108 - mountpoint probe, not temp file creation. except (FileNotFoundError, OSError): - return + return { + "status": "skipped", + "skip_reason": "shm_unavailable", + "detail": {"shm_free_gib": None, "min_gib": 16}, + } + free_gb = usage.free / (1024**3) if usage.free < _DEV_SHM_MIN_FREE_BYTES: - free_gb = usage.free / (1024**3) total_gb = usage.total / (1024**3) print( f"Preflight: WARNING — /dev/shm has {free_gb:.1f} GiB free of " @@ -1377,6 +1620,11 @@ def _check_shm_disk() -> None: f"NCCL shm segments may collide with stale entries; if the " f"first server launch hangs >5min, clear /dev/shm/{{vllm,nccl,cuda}}*" ) + return { + "status": "warned" if usage.free < _DEV_SHM_MIN_FREE_BYTES else "applied", + "skip_reason": None, + "detail": {"shm_free_gib": round(free_gb, 1), "min_gib": 16}, + } def _check_gfx_arch_resolvable(gpu_type: str | None = None) -> None: @@ -1407,7 +1655,7 @@ def _check_gfx_arch_resolvable(gpu_type: str | None = None) -> None: ) -def _check_platform_tuning() -> None: +def _check_platform_tuning() -> dict[str, Any]: """Record host CPU tuning state and warn on settings that skew results. Within one session every trial runs on this same node, so host tuning @@ -1435,7 +1683,11 @@ def _check_platform_tuning() -> None: """ plat = probe_cpu_platform() if plat is None: - return # Not Linux sysfs, or a container without it; stay silent. + return { + "status": "skipped", + "skip_reason": "platform_probe_unavailable", + "detail": {"smt": None, "governor": "unknown", "cpb": None}, + } print( f"Preflight: platform [{socket.gethostname()}] — SMT {plat.smt or '?'}, " @@ -1453,6 +1705,16 @@ def _check_platform_tuning() -> None: "Preflight: WARNING — Core Performance Boost is disabled; CPU-side " "work (sampling, scheduling, tokenization) will run below rated clocks" ) + warned = plat.governor not in ("performance", "unknown") or plat.boost == "off" + return { + "status": "warned" if warned else "applied", + "skip_reason": None, + "detail": { + "smt": plat.smt, + "governor": plat.governor, + "cpb": {"on": True, "off": False}.get(plat.boost), + }, + } _TRACELENS_REQUIRED_CLIS: tuple[str, ...] = ("TraceLens_generate_perf_report_pytorch_inference",) @@ -1479,7 +1741,7 @@ def _tracelens_required_at_preflight(no_kernel: bool, enable_roofline: bool) -> return not (no_kernel and not enable_roofline) -def _check_tracelens_cli() -> None: +def _check_tracelens_cli() -> dict[str, Any]: """Hard-gate TraceLens CLI presence — abort before Coordinator starts (SKILL IR-2). Pod-local /opt/venv/bin/TraceLens_* console_scripts don't persist across pod restarts, so install.sh @@ -1488,7 +1750,12 @@ def _check_tracelens_cli() -> None: """ missing = [name for name in _TRACELENS_REQUIRED_CLIS if shutil.which(name) is None] if not missing: - return + return { + "status": "applied", + "skip_reason": None, + "target": "TraceLens", + "message": None, + } session_dir = str(_workspace_root_resolve()) print( f"ERROR: TraceLens CLI(s) not on PATH: {missing}. The pod-local " @@ -1503,10 +1770,10 @@ def _check_tracelens_cli() -> None: f"then retry `python -m hyperloom.inference_optimizer.cli optimize`. Refusing to start.", file=sys.stderr, ) - sys.exit(2) + raise SystemExit(2) -def _check_tracelens_root_exists() -> None: +def _check_tracelens_root_exists() -> dict[str, Any]: """Hard-gate an explicitly set ``TRACELENS_ROOT`` at preflight. An operator-supplied TRACELENS_ROOT that points at a missing checkout (stale @@ -1515,7 +1782,11 @@ def _check_tracelens_root_exists() -> None: """ override = os.environ.get("TRACELENS_ROOT") if not override or Path(override).is_dir(): - return + return { + "status": "applied", + "skip_reason": None, + "target": "TRACELENS_ROOT", + } print( f"ERROR: TRACELENS_ROOT={override} does not point at an existing " f"TraceLens checkout. It was likely inherited from a stale shell or an " @@ -1546,7 +1817,7 @@ def _emit_preflight_diagnostics( magpie_python: str, anthropic_base_url: str | None, args: argparse.Namespace | None = None, -) -> None: +) -> dict[str, Any]: """One canonical, grep-friendly diagnostics block at the end of preflight. Args: @@ -1604,13 +1875,40 @@ def _emit_preflight_diagnostics( print(f" pr_degraded_reason = {pr_reason}") # Surface Recipe KB offline-queue state; dead-letter pile-up signals a cold start. + queue_status: dict[str, Any] + diagnostics_status = "applied" + diagnostics_message: str | None = None try: - _print_recipe_kb_queue_status() + queue_status = _print_recipe_kb_queue_status() except Exception as exc: # noqa: BLE001 — defensive print(f" recipe_kb_queue = ") - - -def _print_recipe_kb_queue_status() -> None: + queue_status = { + "pending": None, + "dead_letter": None, + "flushed": None, + "root": None, + } + diagnostics_status = "warned" + diagnostics_message = f"recipe KB queue probe failed: {exc!r}" + return { + "status": diagnostics_status, + "skip_reason": None, + "message": diagnostics_message, + "detail": { + "asset_root": str(asset_root()), + "session_dir": str(_session_dir_resolve()), + "magpie_python": magpie_python, + "inferencex_path": os.environ.get("INFERENCEX_PATH") or None, + "aiter_jit_cache": dict(probe), + "recipe_kb_queue": queue_status, + "cold_start_timeout_sec": int(cold_cap) if str(cold_cap).isdigit() else cold_cap, + "warm_timeout_sec": BASELINE_DEFAULT_TIMEOUT_SEC, + "anthropic_base_url": anthropic_base_url, + }, + } + + +def _print_recipe_kb_queue_status() -> dict[str, Any]: """Emit a one-line summary of the Recipe KB offline NDJSON queue (dead-letter = permanent-reject signal). Note: @@ -1655,6 +1953,12 @@ def _count(p: Path) -> int: f"Specialists for affected anchors will start cold " f"(no priors). See {dead}." ) + return { + "pending": p_n, + "dead_letter": d_n, + "flushed": f_n, + "root": str(pending.parent), + } _INFERENCEX_REPO_DEFAULT = "https://github.com/SemiAnalysisAI/InferenceX.git" @@ -1931,6 +2235,192 @@ def _clone_inferencex(dest: Path) -> str | None: return None +def _begin_install_event(args: argparse.Namespace | None) -> dict[str, Any]: + event: dict[str, Any] = { + "type": "install", + "kind": "install", + "status": "succeeded", + "start_time": now_iso(timespec="seconds"), + "end_time": "", + "ext": { + "run_kind": "resume" if bool(getattr(args, "resume_from", None)) else "fresh", + "hard_fail_step_id": None, + "runtime_snapshot": {}, + "steps": [], + }, + } + try: + from ..session.sbd_v6 import set_pending_install_event + + set_pending_install_event(args, event) + except Exception: # noqa: BLE001 — V6 observability must never change preflight behavior + log.warning("failed to initialize SBD V6 install event", exc_info=True) + return event + + +def _record_install_step( + event: dict[str, Any], + *, + step_id: str, + category: str, + status: str, + skip_reason: str | None = None, + message: str | None = None, + **fields: Any, +) -> None: + step: dict[str, Any] = { + "step_id": step_id, + "category": category, + "status": status, + "skip_reason": skip_reason, + } + if message is not None: + step["message"] = message + step.update(fields) + event["ext"]["steps"].append(step) + + +def _fail_install_step( + event: dict[str, Any], + *, + step_id: str, + category: str, + exc: BaseException, +) -> None: + failure_fields: dict[str, Any] = {} + if isinstance(exc, SystemExit): + failure_fields["detail"] = {"exit_code": exc.code} + _record_install_step( + event, + step_id=step_id, + category=category, + status="failed", + message=str(exc) or type(exc).__name__, + error_class=type(exc).__name__, + **failure_fields, + ) + event["status"] = "failed" + event["end_time"] = now_iso(timespec="seconds") + event["ext"]["hard_fail_step_id"] = step_id + + +def _mark_pending_install_event_failed( + args: argparse.Namespace | None, + exc: BaseException, +) -> dict[str, Any] | None: + """Mark an unwrapped preflight exception on the pending install event.""" + try: + from ..session.sbd_v6 import pending_install_event + + event = pending_install_event(args) + if event is None: + event = _begin_install_event(args) + if str(event.get("status") or "") != "failed": + _fail_install_step( + event, + step_id="unhandled_preflight", + category="check", + exc=exc, + ) + return event + except Exception: # noqa: BLE001 — never replace the original preflight failure + log.warning("failed to finalize SBD V6 install failure", exc_info=True) + return None + + +def _run_install_step( + event: dict[str, Any], + *, + step_id: str, + category: str, + action: Callable[[], Any], + success_status: str = "applied", + **success_fields: Any, +) -> Any: + try: + result = action() + except BaseException as exc: + try: + _fail_install_step(event, step_id=step_id, category=category, exc=exc) + except Exception: # noqa: BLE001 — preserve the original preflight exception + log.warning("failed to record SBD V6 install-step failure", exc_info=True) + raise + try: + outcome = dict(result) if isinstance(result, dict) else {} + status = str(outcome.pop("status", success_status) or success_status) + skip_reason = outcome.pop("skip_reason", None) + message = outcome.pop("message", None) + fields = {**success_fields, **outcome} + _record_install_step( + event, + step_id=step_id, + category=category, + status=status, + skip_reason=skip_reason, + message=message, + **fields, + ) + except Exception: # noqa: BLE001 — V6 observability must never change preflight behavior + log.warning("failed to record SBD V6 install step", exc_info=True) + return result + + +def _resolved_provider_mode(resolved_urls: tuple[str, str] | None) -> str | None: + anthropic_url, openai_url = resolved_urls or ("", "") + if anthropic_url and openai_url: + return "mixed" + if anthropic_url: + return "anthropic" + if openai_url: + return "openai" + return None + + +def _finish_install_event( + event: dict[str, Any], + *, + args: argparse.Namespace | None, + benchmark_backend: str, + benchmark_python: str, + magpie_python: str, + inferencex_path: str, + resolved_urls: tuple[str, str] | None, +) -> None: + no_kernel = bool(getattr(args, "no_kernel", False)) if args is not None else False + enable_roofline = bool(getattr(args, "enable_roofline", True)) if args is not None else True + tracelens_required = _tracelens_required_at_preflight(no_kernel, enable_roofline) + event["ext"]["runtime_snapshot"] = { + "benchmark_backend": benchmark_backend, + "benchmark_interpreter": benchmark_python, + "magpie_python": magpie_python, + "inferencex_path": inferencex_path, + "magpie_path": os.environ.get("MAGPIE_PATH") or None, + "tracelens_required": tracelens_required, + "tracelens_route_hint": "agent" if not no_kernel else ("bypass" if enable_roofline else None), + "provider_mode": _resolved_provider_mode(resolved_urls), + } + statuses = {str(step.get("status") or "") for step in event["ext"]["steps"]} + if "failed" in statuses: + event["status"] = "failed" + elif "warned" in statuses: + event["status"] = "degraded" + elif statuses and statuses == {"skipped"}: + event["status"] = "skipped" + else: + event["status"] = "succeeded" + event["end_time"] = now_iso(timespec="seconds") + + +def _persist_install_event(args: argparse.Namespace | None, session_dir: Path) -> None: + """Persist the pre-session install trace without changing launch behavior.""" + from ..session.sbd_v6 import persist_pending_install_event + + try: + persist_pending_install_event(args, session_dir) + except Exception as exc: # noqa: BLE001 + log.warning("failed to persist SBD V6 install event: %s", exc) + + def _preflight( args: argparse.Namespace | None = None, ) -> tuple[str, str] | None: @@ -1948,25 +2438,69 @@ def _preflight( tuple[str, str] | None: ``(anthropic_base_url, openai_base_url)``, or ``None`` when neither base URL is configured. """ - _load_dotenv_fallback() + install_event = _begin_install_event(args) + _run_install_step( + install_event, + step_id="load_dotenv", + category="normalize", + action=_load_dotenv_fallback, + ) # ``.env`` is operator configuration, so both the single-provider intent and # the restore baseline are taken after it loads. Only what the installer env # file injects on top is undone below. provider_mode = _provider_only_mode() provider_snapshot = {key: os.environ.get(key) for key in (*_PROVIDER_FALLBACK_KEYS, *_ANTHROPIC_FALLBACK_KEYS)} - _load_kernel_agent_env_fallback() + _run_install_step( + install_event, + step_id="load_kernel_agent_env", + category="normalize", + action=_load_kernel_agent_env_fallback, + ) _derive_runtime_paths() _restore_provider_only_mode(provider_mode, provider_snapshot) - _normalize_legacy_deepseek_env() + _run_install_step( + install_event, + step_id="normalize_legacy_deepseek_env", + category="normalize", + action=_normalize_legacy_deepseek_env, + ) # Fail fast on missing credentials after the fallback loaders. - _validate_credentials() + _run_install_step( + install_event, + step_id="validate_credentials", + category="check", + action=_validate_credentials, + detail={"exit_code": 0}, + ) # Same timing, same reason: run after the loaders so a withdrawn KB # override set in ``.env`` is caught, and before any KB read happens. from hyperloom.agents.framework.kb import prepare_kb_environment - prepare_kb_environment() + kb_withdrawn_override = bool(os.environ.get("FRAMEWORK_AGENT_KB_DIR", "").strip()) + kb_enabled = not bool(getattr(args, "degraded_kb", False)) if args is not None else True + + def _prepare_kb_install_step() -> dict[str, Any]: + prepare_kb_environment() + return { + "status": ("skipped" if not kb_enabled else "warned" if kb_withdrawn_override else "applied"), + "skip_reason": "explicit_flag" if not kb_enabled else None, + "detail": { + "recipe_kb": { + "enabled": kb_enabled, + "reason": "explicit_flag" if not kb_enabled else None, + }, + "kb_withdrawn_override": kb_withdrawn_override, + }, + } + + _run_install_step( + install_event, + step_id="prepare_kb_environment", + category="degrade", + action=_prepare_kb_install_step, + ) # --- Auth alias export (internal LLM aliases only) --- # These aliases feed OpenAI-protocol consumers, so they are filled from the @@ -1994,7 +2528,8 @@ def _preflight( resolve_benchmark_interpreter as _resolve_benchmark_interpreter, ) - _magpie_backend_active = _resolve_active_backend_name() == "magpie" + benchmark_backend = _resolve_active_backend_name() + _magpie_backend_active = benchmark_backend == "magpie" # Interpreter used for benchmark-runtime installs (Ray). For bypass this is # sys.executable; for Magpie it's the Magpie-importable venv. benchmark_python = _resolve_benchmark_interpreter() @@ -2006,7 +2541,12 @@ def _preflight( # --- Python SDK auto-install (claude-agent-sdk / openai / httpx) --- # Must precede Coordinator import (ClaudeBackend lazy-imports the SDK). - _ensure_python_sdks(sys.executable, pip_extra) + _run_install_step( + install_event, + step_id="ensure_python_sdks", + category="install", + action=lambda: _ensure_python_sdks(sys.executable, pip_extra), + ) # --- Resolve Anthropic + OpenAI base URLs (split entrypoints) --- # Explicit operator values on each side are preserved; a missing side falls @@ -2083,40 +2623,84 @@ def _preflight( # --- ROCm env hygiene + GPU/shm sanity (defensive WARN-only) --- _unset_hip_visible_devices() - _check_gpu_visibility() - _check_shm_disk() - _check_platform_tuning() + _run_install_step( + install_event, + step_id="check_gpu_visibility", + category="check", + action=_check_gpu_visibility, + ) + _run_install_step( + install_event, + step_id="check_shm_disk", + category="check", + action=_check_shm_disk, + ) + _run_install_step( + install_event, + step_id="check_platform_tuning", + category="check", + action=_check_platform_tuning, + ) # --- Runtime dep install --- # 1. Ray — used broadly (multi-node scheduling, kernel/profile/recover # executors), not only by Magpie, so it is installed regardless of backend. # Install it with the active backend's interpreter so a bypass-only box # gets Ray in its own venv instead of Magpie's. - _ensure_ray(benchmark_python, pip_extra) + _run_install_step( + install_event, + step_id="ensure_ray", + category="install", + action=lambda: _ensure_ray(benchmark_python, pip_extra), + ) # 1b. InferenceX benchmark_serving client deps — required by every serving # benchmark client launch. install.sh installs these into the install-time # $PYTHON, but the bypass runner launches the client with the active # benchmark interpreter; ensure them there too so a bypass-only box whose # sys.executable differs from /opt/venv can still import the client. - _ensure_bench_serving_deps(benchmark_python, pip_extra) + _run_install_step( + install_event, + step_id="ensure_bench_serving_deps", + category="install", + action=lambda: _ensure_bench_serving_deps(benchmark_python, pip_extra), + ) # 1c. lm_eval — GSM8K accuracy gate, multi-node only (the helper gates # itself). The multi-node magpie remote-compat client path runs # ``python -m lm_eval`` directly, with no InferenceX runtime shim to install # the harness, so every RUN_EVAL=true baseline there would otherwise abort # with baseline_accuracy_failed. - _ensure_lm_eval_dep(benchmark_python, pip_extra, eval_disabled=_resolved_eval_disabled(args)) + _run_install_step( + install_event, + step_id="ensure_lm_eval", + category="install", + action=lambda: _ensure_lm_eval_dep( + benchmark_python, + pip_extra, + eval_disabled=_resolved_eval_disabled(args), + ), + ) # 1d. Per-framework runtime deps declared in assets/framework_deps/. This is # the pass that covers the documented flow: install.sh runs before # --framework is known, so its own attempt usually no-ops and a scriptable # framework would otherwise reach baseline with nothing installed. - _ensure_framework_deps(args, benchmark_python, pip_extra) + _run_install_step( + install_event, + step_id="framework_deps", + category="install", + action=lambda: _ensure_framework_deps(args, benchmark_python, pip_extra), + ) # 1e. The serving framework itself, checked after 1d so everything that could # have supplied it has run. Without this the failure surfaces far from its cause. - _check_serving_framework(args, benchmark_python) + _run_install_step( + install_event, + step_id="check_serving_framework", + category="check", + action=lambda: _check_serving_framework(args, benchmark_python), + ) # 2. Magpie — the benchmark engine the Magpie backend shells out to. # Skipped entirely when the @@ -2125,41 +2709,70 @@ def _preflight( # Magpie-importable venv (via resolve_benchmark_interpreter), so a # bypass-only environment never resolves the Magpie venv / /opt/venv. magpie_python = benchmark_python - if not _magpie_backend_active: - print(f"Preflight: benchmark backend is {_resolve_active_backend_name()!r}; skipping Magpie install/import") - check = None - else: - check = subprocess.run([magpie_python, "-c", "import Magpie"], capture_output=True) - if _magpie_backend_active and check is not None and check.returncode != 0: - magpie_repo = os.environ.get("MAGPIE_REPO", "https://github.com/AMD-AGI/Magpie.git") - magpie_ref = os.environ.get("MAGPIE_REF", "e6833b8183c6c41adf6038252337550876ca0433") - magpie_spec = os.environ.get( - "MAGPIE_PACKAGE_SPEC", - f"magpie-eval @ git+{magpie_repo}@{magpie_ref}", - ) - print(f"Preflight: Magpie not importable; installing {magpie_spec} ...") - subprocess.run( - [magpie_python, "-m", "pip", "install", "--quiet", *pip_extra, magpie_spec], - check=True, + magpie_installed = False + magpie_spec: str | None = None + try: + if not _magpie_backend_active: + print(f"Preflight: benchmark backend is {benchmark_backend!r}; skipping Magpie install/import") + check = None + else: + check = subprocess.run([magpie_python, "-c", "import Magpie"], capture_output=True) + if _magpie_backend_active and check is not None and check.returncode != 0: + magpie_repo = os.environ.get("MAGPIE_REPO", "https://github.com/AMD-AGI/Magpie.git") + magpie_ref = os.environ.get("MAGPIE_REF", "e6833b8183c6c41adf6038252337550876ca0433") + magpie_spec = os.environ.get( + "MAGPIE_PACKAGE_SPEC", + f"magpie-eval @ git+{magpie_repo}@{magpie_ref}", + ) + print(f"Preflight: Magpie not importable; installing {magpie_spec} ...") + subprocess.run( + [magpie_python, "-m", "pip", "install", "--quiet", *pip_extra, magpie_spec], + check=True, + ) + magpie_installed = True + print("Preflight: Magpie installed OK") + if _magpie_backend_active and not os.environ.get("MAGPIE_PATH", "").strip(): + magpie_root = subprocess.run( + [ + magpie_python, + "-c", + "from pathlib import Path; import Magpie; print(Path(Magpie.__file__).resolve().parent.parent)", + ], + capture_output=True, + text=True, + check=True, + ).stdout.strip() + if magpie_root: + os.environ["MAGPIE_PATH"] = magpie_root + print(f"Preflight: MAGPIE_PATH resolved from installed package: {magpie_root}") + except BaseException as exc: + _fail_install_step( + install_event, + step_id="ensure_magpie", + category="install", + exc=exc, ) - print("Preflight: Magpie installed OK") - if _magpie_backend_active and not os.environ.get("MAGPIE_PATH", "").strip(): - magpie_root = subprocess.run( - [ - magpie_python, - "-c", - "from pathlib import Path; import Magpie; print(Path(Magpie.__file__).resolve().parent.parent)", - ], - capture_output=True, - text=True, - check=True, - ).stdout.strip() - if magpie_root: - os.environ["MAGPIE_PATH"] = magpie_root - print(f"Preflight: MAGPIE_PATH resolved from installed package: {magpie_root}") + raise + _record_install_step( + install_event, + step_id="ensure_magpie", + category="install", + status=("skipped" if not _magpie_backend_active else "applied" if magpie_installed else "already_present"), + skip_reason=None if _magpie_backend_active else "benchmark_backend_not_magpie", + message=( + f"installed {magpie_spec}" + if magpie_installed + else f"benchmark backend is {benchmark_backend!r}" + if not _magpie_backend_active + else None + ), + target="magpie-eval", + interpreter=magpie_python, + ) # 3. InferenceX — required for GSM8K accuracy eval; lm-eval deps auto-install at runtime via benchmark_lib.sh. inferencex_path = os.environ.get("INFERENCEX_PATH", "").strip() + inferencex_cloned = False if not inferencex_path: from ..session.paths import ( magpie_dir as _magpie_default, @@ -2209,6 +2822,7 @@ def _preflight( dest = _open_source_default() / _inferencex_dest_name(_ref) print(f"Preflight: no InferenceX checkout at {_ref[:12]}; cloning into {dest} ...") inferencex_path = _clone_inferencex(dest) + inferencex_cloned = bool(inferencex_path) if not (inferencex_path and _inferencex_checkout_ok(inferencex_path)): print( "Preflight: ERROR — InferenceX checkout missing and clone " @@ -2217,7 +2831,14 @@ def _preflight( "src/hyperloom/inference_optimizer/assets/install.sh.", file=sys.stderr, ) - sys.exit(2) + exc = SystemExit(2) + _fail_install_step( + install_event, + step_id="clone_inferencex", + category="install", + exc=exc, + ) + raise exc # Guard against a read-only INFERENCEX_PATH: Magpie stages benchmark scripts # there, so a non-writable tree fails the run before server boot. if not os.access(inferencex_path, os.W_OK): @@ -2229,10 +2850,32 @@ def _preflight( f"Hyperloom clone a fresh one).", file=sys.stderr, ) - sys.exit(2) + exc = SystemExit(2) + _fail_install_step( + install_event, + step_id="clone_inferencex", + category="install", + exc=exc, + ) + raise exc # Always overwrite (not setdefault): a stale/broken INFERENCEX_PATH must not # survive into the child env. The validated value wins. os.environ["INFERENCEX_PATH"] = inferencex_path + _record_install_step( + install_event, + step_id="clone_inferencex", + category="install", + status="applied" if inferencex_cloned else "already_present", + skip_reason=None, + target="InferenceX", + version_after=_inferencex_head_sha(inferencex_path) or None, + detail={ + "ref": os.environ.get("INFERENCEX_REF") or _INFERENCEX_REF_DEFAULT, + "dest": inferencex_path, + "writable": os.access(inferencex_path, os.W_OK), + "exit_code": 0, + }, + ) # --- Magpie/InferenceX eval-concurrency compatibility ------------------- # Preflight installs Magpie and clones InferenceX itself (above), entirely @@ -2244,15 +2887,42 @@ def _preflight( # flag ("Unknown parameter: --concurrent-requests"), aborting every # RUN_EVAL=true baseline before any results*.json exists. Patch the trees we # just materialized, now that both paths are known. - if _magpie_backend_active: - # Trust patch first, mirroring install.sh: the eval-concurrency strip - # removes the very `run_eval ... --concurrent-requests` line the legacy - # MI300X trust patcher matches on, so the reverse order would leave a - # tree permanently unpatchable by that path. - _ensure_client_trust_compat(os.environ.get("MAGPIE_PATH", "")) - _ensure_eval_concurrency_compat(os.environ.get("MAGPIE_PATH", ""), inferencex_path) - - _report_inferencex_patch_anchors(inferencex_path) + try: + if _magpie_backend_active: + # Trust patch first, mirroring install.sh: the eval-concurrency strip + # removes the very `run_eval ... --concurrent-requests` line the legacy + # MI300X trust patcher matches on, so the reverse order would leave a + # tree permanently unpatchable by that path. + trust_ok = _ensure_client_trust_compat(os.environ.get("MAGPIE_PATH", "")) + concurrency_ok = _ensure_eval_concurrency_compat( + os.environ.get("MAGPIE_PATH", ""), + inferencex_path, + ) + else: + trust_ok = True + concurrency_ok = True + anchors_ok = _report_inferencex_patch_anchors(inferencex_path) + except BaseException as exc: + _fail_install_step( + install_event, + step_id="patch_magpie_eval_concurrency", + category="patch", + exc=exc, + ) + raise + patch_ok = trust_ok and concurrency_ok and anchors_ok + _record_install_step( + install_event, + step_id="patch_magpie_eval_concurrency", + category="patch", + status=("skipped" if not _magpie_backend_active else "applied" if patch_ok else "warned"), + skip_reason=None if _magpie_backend_active else "benchmark_backend_not_magpie", + detail={ + "client_trust_compatible": trust_ok, + "eval_concurrency_compatible": concurrency_ok, + "inferencex_patch_anchors_ok": anchors_ok, + }, + ) # --- node / claude / codex CLI presence (WARN-only) --- _check_node_claude_cli() @@ -2262,10 +2932,20 @@ def _preflight( no_kernel = getattr(args, "no_kernel", False) if args else False enable_roofline = getattr(args, "enable_roofline", True) if args else True if _tracelens_required_at_preflight(no_kernel, enable_roofline): - _check_tracelens_cli() + _run_install_step( + install_event, + step_id="check_tracelens_cli", + category="check", + action=_check_tracelens_cli, + ) # Fail fast on a stale/placeholder TRACELENS_ROOT before the Coordinator # starts, rather than ~10h later in trace_analyze. - _check_tracelens_root_exists() + _run_install_step( + install_event, + step_id="check_tracelens_root", + category="check", + action=_check_tracelens_root_exists, + ) else: _missing_tl = [n for n in _TRACELENS_REQUIRED_CLIS if shutil.which(n) is None] if _missing_tl: @@ -2273,22 +2953,70 @@ def _preflight( f"Preflight: WARNING — TraceLens CLI(s) not on PATH: {_missing_tl} " f"(skipped; --no-kernel + roofline disabled)" ) + _record_install_step( + install_event, + step_id="check_tracelens_cli", + category="check", + status="skipped", + skip_reason="no_kernel_and_roofline_disabled", + target="TraceLens", + message=f"missing CLIs: {_missing_tl}" if _missing_tl else None, + ) + _record_install_step( + install_event, + step_id="check_tracelens_root", + category="check", + status="skipped", + skip_reason="tracelens_not_required", + target="TRACELENS_ROOT", + ) # --- IR-3: PR Monitor reachability probe (soft degrade) --- if args is not None: - _run_ir3_preflight(args) + _run_install_step( + install_event, + step_id="ir3_pr_monitor_probe", + category="degrade", + action=lambda: _run_ir3_preflight(args), + ) + else: + _record_install_step( + install_event, + step_id="ir3_pr_monitor_probe", + category="degrade", + status="skipped", + skip_reason="args_unavailable", + ) # --- Single canonical diagnostics block --- - _emit_preflight_diagnostics( - magpie_python=magpie_python, - anthropic_base_url=(resolved_urls[0] if resolved_urls is not None else None), - args=args, + _run_install_step( + install_event, + step_id="diagnostics_snapshot", + category="diagnostic", + action=lambda: _emit_preflight_diagnostics( + magpie_python=magpie_python, + anthropic_base_url=(resolved_urls[0] if resolved_urls is not None else None), + args=args, + ), ) + try: + _finish_install_event( + install_event, + args=args, + benchmark_backend=benchmark_backend, + benchmark_python=benchmark_python, + magpie_python=magpie_python, + inferencex_path=inferencex_path, + resolved_urls=resolved_urls, + ) + except Exception: # noqa: BLE001 — V6 observability must never change preflight behavior + log.warning("failed to finalize SBD V6 install event", exc_info=True) + return resolved_urls -def _run_ir3_preflight(args: argparse.Namespace) -> None: +def _run_ir3_preflight(args: argparse.Namespace) -> dict[str, Any]: """IR-3 — PR Monitor reachability probe (soft degrade); never raises/exits. Recipe KB enablement is controlled by ``--degraded-kb`` (``recipe_kb_enabled``); @@ -2310,7 +3038,14 @@ def _run_ir3_preflight(args: argparse.Namespace) -> None: args.pr_degraded_reason = "explicit_flag" if explicit_pr else None if explicit_kb and explicit_pr: - return + return { + "status": "skipped", + "skip_reason": "explicit_flag", + "detail": { + "pr_monitor": {"enabled": False, "reason": "explicit_flag"}, + "marker": None, + }, + } user_data = _workspace_root_resolve() marker_path = user_data / "runtime" / "recipe_kb" / ".kb_preflight.json" @@ -2354,3 +3089,17 @@ def _run_ir3_preflight(args: argparse.Namespace) -> None: if not explicit_pr and not marker.get("pr_reachable", False) and not marker.get("pr_skipped", False): args.pr_monitor_enabled = False args.pr_degraded_reason = "ir3_auto" + enabled = bool(args.pr_monitor_enabled) + status = "skipped" if explicit_pr else "applied" if enabled else "warned" + event_reason = "explicit_flag" if explicit_pr else "ir3_unreachable" if not enabled else None + return { + "status": status, + "skip_reason": event_reason, + "detail": { + "pr_monitor": { + "enabled": enabled, + "reason": event_reason, + }, + "marker": str(marker_path), + }, + } diff --git a/src/hyperloom/inference_optimizer/session/sbd_v6.py b/src/hyperloom/inference_optimizer/session/sbd_v6.py new file mode 100644 index 0000000000..5e05651a0b --- /dev/null +++ b/src/hyperloom/inference_optimizer/session/sbd_v6.py @@ -0,0 +1,260 @@ +"""Small write/read surface for additive SBD V6 timeline events.""" + +from __future__ import annotations + +import re +from argparse import Namespace +from pathlib import Path +from typing import Any + +from hyperloom.common.io import atomic_write_json +from hyperloom.common.jsonio import read_json + +from .session_paths import ( + sbd_v6_install_path, + sbd_v6_model_gate_path, + sbd_v6_timeline_dir, + sbd_v6_timeline_event_path, +) + + +SCHEMA_VERSION_V6 = "hyperloom.session_breakdown.v6.0" +_PENDING_INSTALL_ATTR = "_sbd_v6_install_event" +_STORAGE_SEQUENCE_KEY = "__sbd_v6_timeline_sequence" +_EVENT_TYPES = ("install", "model_gate") +_EVENT_FILE_RE = re.compile(r"^(?P\d+)-(?P[a-z0-9_]+)\.json$") + + +def _event_path(session_dir: Path | str, event_type: str) -> Path: + root = Path(session_dir) + if event_type == "install": + return sbd_v6_install_path(root) + if event_type == "model_gate": + return sbd_v6_model_gate_path(root) + raise ValueError(f"unsupported SBD V6 timeline event type: {event_type!r}") + + +def _public_event(event: dict[str, Any]) -> dict[str, Any]: + value = dict(event) + value.pop(_STORAGE_SEQUENCE_KEY, None) + return value + + +def _write_event(path: Path, event: dict[str, Any]) -> None: + atomic_write_json( + path, + _public_event(event), + indent=2, + sort_keys=True, + ensure_ascii=False, + trailing_newline=True, + make_parents=True, + ) + + +def _history_files(session_dir: Path | str) -> list[tuple[int, str, Path]]: + root = sbd_v6_timeline_dir(Path(session_dir)) + if not root.is_dir(): + return [] + files: list[tuple[int, str, Path]] = [] + for path in root.glob("*.json"): + match = _EVENT_FILE_RE.fullmatch(path.name) + if match is None: + continue + event_type = match.group("event_type") + if event_type not in _EVENT_TYPES: + continue + files.append((int(match.group("sequence")), event_type, path)) + files.sort(key=lambda row: row[0]) + return files + + +def _read_event_file( + path: Path, + event_type: str, + warnings: list[str] | None = None, +) -> dict[str, Any] | None: + try: + event = read_json(path, require_dict=True, strict=True) + except Exception as exc: + if warnings is not None: + warnings.append(f"timeline.{event_type}: failed to parse {path}: {exc!r}") + return None + if str(event.get("type") or "") != event_type: + if warnings is not None: + warnings.append(f"timeline.{event_type}: ignored event with type={event.get('type')!r}") + return None + return _public_event(event) + + +def _event_identity(event_type: str, event: dict[str, Any]) -> tuple[str, str, str] | None: + ext = event.get("ext") if isinstance(event.get("ext"), dict) else {} + start_time = str(event.get("start_time") or "") + run_kind = str(ext.get("run_kind") or "") + if not start_time and not run_kind: + return None + return event_type, start_time, run_kind + + +def _ensure_timeline_history(session_dir: Path | str) -> list[tuple[int, str, Path]]: + history = _history_files(session_dir) + root = Path(session_dir) + stored_events: list[tuple[int, str, Path, dict[str, Any]]] = [] + for sequence, event_type, path in history: + event = _read_event_file(path, event_type) + if event is not None: + stored_events.append((sequence, event_type, path, event)) + + next_sequence = max((sequence for sequence, _, _ in history), default=0) + for event_type in _EVENT_TYPES: + legacy_path = _event_path(root, event_type) + if not legacy_path.is_file(): + continue + event = _read_event_file(legacy_path, event_type) + if event is None: + continue + if any(stored_event == event for _, _, _, stored_event in stored_events): + continue + identity = _event_identity(event_type, event) + matching = next( + ( + row + for row in reversed(stored_events) + if identity is not None and _event_identity(row[1], row[3]) == identity + ), + None, + ) + if matching is not None: + _write_event(matching[2], event) + stored_events[stored_events.index(matching)] = (*matching[:3], event) + continue + next_sequence += 1 + path = sbd_v6_timeline_event_path(root, next_sequence, event_type) + _write_event(path, event) + stored_events.append((next_sequence, event_type, path, event)) + return _history_files(root) + + +def write_timeline_event(session_dir: Path | str, event: dict[str, Any]) -> Path: + """Persist one event without replacing an earlier run of the same stage.""" + event_type = str(event.get("type") or "").strip() + latest_path = _event_path(session_dir, event_type) + history = _ensure_timeline_history(session_dir) + + raw_sequence = event.get(_STORAGE_SEQUENCE_KEY) + try: + sequence = int(raw_sequence) if raw_sequence is not None else 0 + except (TypeError, ValueError): + sequence = 0 + if sequence > 0: + conflicting_type = next( + (stored_type for stored_sequence, stored_type, _ in history if stored_sequence == sequence), + None, + ) + if conflicting_type is not None and conflicting_type != event_type: + raise ValueError(f"SBD V6 timeline sequence {sequence} belongs to {conflicting_type!r}, not {event_type!r}") + else: + sequence = max((stored_sequence for stored_sequence, _, _ in history), default=0) + 1 + event[_STORAGE_SEQUENCE_KEY] = sequence + + _write_event( + sbd_v6_timeline_event_path(Path(session_dir), sequence, event_type), + event, + ) + _write_event(latest_path, event) + return latest_path + + +def read_timeline_event( + session_dir: Path | str, + event_type: str, +) -> dict[str, Any] | None: + """Read the latest persisted event of one type.""" + _event_path(session_dir, event_type) + for _, stored_type, path in reversed(_ensure_timeline_history(session_dir)): + if stored_type != event_type: + continue + event = _read_event_file(path, event_type) + if event is not None: + return event + + path = _event_path(session_dir, event_type) + if not path.is_file(): + return None + return _read_event_file(path, event_type) + + +def read_timeline_event_for_update( + session_dir: Path | str, + event_type: str, +) -> dict[str, Any] | None: + """Read the latest event with its private storage sequence attached.""" + _event_path(session_dir, event_type) + for sequence, stored_type, path in reversed(_ensure_timeline_history(session_dir)): + if stored_type != event_type: + continue + event = _read_event_file(path, event_type) + if event is not None: + event[_STORAGE_SEQUENCE_KEY] = sequence + return event + return None + + +def read_timeline_events( + session_dir: Path | str, + *, + warnings: list[str] | None = None, +) -> list[dict[str, Any]]: + """Read all persisted V6 events in execution order.""" + history = _ensure_timeline_history(session_dir) + if history: + events: list[dict[str, Any]] = [] + for _, event_type, path in history: + event = _read_event_file(path, event_type, warnings) + if event is not None: + events.append(event) + return events + + events = [] + for event_type in _EVENT_TYPES: + path = _event_path(session_dir, event_type) + if not path.is_file(): + continue + event = _read_event_file(path, event_type, warnings) + if event is not None: + events.append(event) + return events + + +def set_pending_install_event(args: Namespace | None, event: dict[str, Any]) -> None: + """Attach the pre-session install event to the parsed CLI namespace.""" + if args is not None: + setattr(args, _PENDING_INSTALL_ATTR, event) + + +def pending_install_event(args: Namespace | None) -> dict[str, Any] | None: + """Return the in-memory install event captured before session creation.""" + if args is None: + return None + event = getattr(args, _PENDING_INSTALL_ATTR, None) + return event if isinstance(event, dict) else None + + +def persist_pending_install_event(args: Namespace | None, session_dir: Path | str) -> Path | None: + """Persist the install event once the session directory exists.""" + event = pending_install_event(args) + if event is None: + return None + return write_timeline_event(session_dir, event) + + +__all__ = [ + "SCHEMA_VERSION_V6", + "pending_install_event", + "persist_pending_install_event", + "read_timeline_event", + "read_timeline_event_for_update", + "read_timeline_events", + "set_pending_install_event", + "write_timeline_event", +] diff --git a/src/hyperloom/inference_optimizer/session/session_paths.py b/src/hyperloom/inference_optimizer/session/session_paths.py index 5b259ac647..99a8fdf339 100644 --- a/src/hyperloom/inference_optimizer/session/session_paths.py +++ b/src/hyperloom/inference_optimizer/session/session_paths.py @@ -272,6 +272,31 @@ def reports_dir(session_dir: Path) -> Path: return Path(session_dir) / "reports" +def sbd_v6_dir(session_dir: Path) -> Path: + """Compute ``/reports/sbd_v6/`` for V6 timeline source events.""" + return reports_dir(session_dir) / "sbd_v6" + + +def sbd_v6_timeline_dir(session_dir: Path) -> Path: + """Compute the append-only V6 timeline event directory.""" + return sbd_v6_dir(session_dir) / "timeline" + + +def sbd_v6_timeline_event_path(session_dir: Path, sequence: int, event_type: str) -> Path: + """Compute one ordered V6 timeline event path.""" + return sbd_v6_timeline_dir(session_dir) / f"{int(sequence):06d}-{event_type}.json" + + +def sbd_v6_install_path(session_dir: Path) -> Path: + """Compute the persisted V6 ``install`` timeline event path.""" + return sbd_v6_dir(session_dir) / "install.json" + + +def sbd_v6_model_gate_path(session_dir: Path) -> Path: + """Compute the persisted V6 ``model_gate`` timeline event path.""" + return sbd_v6_dir(session_dir) / "model_gate.json" + + def enablement_dir(session_dir: Path) -> Path: """``/reports/enablement/`` — enablement round artifacts. @@ -880,6 +905,11 @@ def failure_evidence_path(session_dir: Path, failure_id: str) -> Path: "research_hints_md", "runs_dir", "runs_root", + "sbd_v6_dir", + "sbd_v6_install_path", + "sbd_v6_model_gate_path", + "sbd_v6_timeline_dir", + "sbd_v6_timeline_event_path", "state_path", "target_analysis_dir", "target_analysis_report_md", diff --git a/src/hyperloom/inference_optimizer/tests/test_sbd_v6_initial.py b/src/hyperloom/inference_optimizer/tests/test_sbd_v6_initial.py new file mode 100644 index 0000000000..ba2941f2e7 --- /dev/null +++ b/src/hyperloom/inference_optimizer/tests/test_sbd_v6_initial.py @@ -0,0 +1,844 @@ +"""Focused coverage for the additive SBD V6 bootstrap fields and stages.""" + +from __future__ import annotations + +import argparse +import asyncio +import json +import os +from pathlib import Path + +import pytest + +from hyperloom.inference_optimizer.breakdown import exporter +from hyperloom.inference_optimizer.breakdown.collectors.v6 import collect_v6_timeline +from hyperloom.inference_optimizer.breakdown.schema import SCHEMA_VERSION_V5 +from hyperloom.inference_optimizer.session.sbd_v6 import ( + SCHEMA_VERSION_V6, + read_timeline_event, + read_timeline_events, + write_timeline_event, +) + + +def _write_json(path: Path, payload: dict) -> None: + path.parent.mkdir(parents=True, exist_ok=True) + path.write_text(json.dumps(payload), encoding="utf-8") + + +def _gate_args(model: Path, **overrides) -> argparse.Namespace: + values = { + "model": str(model), + "model_display_name": model.name, + "framework": "sglang", + "gpu_type": "MI300X", + "isl": 1024, + "osl": 1024, + "allow_mm_text_fallback": True, + } + values.update(overrides) + return argparse.Namespace(**values) + + +def _seed_state(session_dir: Path, monkeypatch, model: Path) -> None: + monkeypatch.setenv("INFERENCE_OPTIMIZER_CURRENT_SESSION_DIR", str(session_dir)) + from hyperloom.orchestrator.state.shared_state import SharedState + + session_dir.mkdir(parents=True, exist_ok=True) + (session_dir / "reports").mkdir(parents=True, exist_ok=True) + SharedState(session_id="sbd-v6-test", model_name=model.name, model_path=str(model)).save(session_dir) + + +def _write_model_config(model: Path, payload: dict) -> None: + _write_json(model / "config.json", payload) + (model / "tokenizer_config.json").write_text("{}", encoding="utf-8") + + +def _model_gate_from_breakdown(session_dir: Path) -> dict: + breakdown = json.loads((session_dir / "session_breakdown.json").read_text(encoding="utf-8")) + assert breakdown["schema_version"] == SCHEMA_VERSION_V5 + assert breakdown["metadata"]["versions"]["schema_version"] == SCHEMA_VERSION_V6 + assert breakdown["outcome"]["status"] == "failed" + assert breakdown["outcome"]["stage_reached"] == "model_gate" + return next(event for event in breakdown["timeline"] if event["type"] == "model_gate") + + +def test_v6_projection_is_additive_to_v5_breakdown(tmp_path): + state = { + "session_id": "session-v6", + "model_name": "Qwen-Test", + "model_path": "/models/qwen-test", + "framework": "sglang", + "gpu_type": "MI300X", + "phase": "CLOSE", + "start_ts": "2026-08-27T01:00:00+00:00", + "stop_ts": "2026-08-27T02:00:00+00:00", + "stop_reason": "target_reached", + "tick": 9, + "baseline_tput": 100.0, + "baseline_accuracy": 0.75, + "current_best": { + "tput": 125.0, + "extra_envs": {"SGLANG_USE_AITER": "1"}, + "extra_server_args": "--watchdog-timeout 1800", + }, + "cumulative_gain_validated": 25.0, + "operator_extra_env": {"TP": "8"}, + "operator_server_args": "--context-length 11264", + "model_info": { + "model_type": "qwen3", + "num_hidden_layers": 36, + "attention_type": "GQA", + "num_experts": None, + }, + } + manifest = { + "session_id": "session-v6", + "created_at_utc": "2026-08-27T01:00:00+00:00", + "host": "test-host", + "code_revision": "abc1234", + "pid": 42, + "max_minutes": 180, + "model_name": "Qwen-Test", + "model_path": "/models/qwen-test", + "framework": "sglang", + "framework_version": "0.5.17", + "gpu_type": "MI300X", + "tp": 8, + "workload": { + "conc": 64, + "isl": 1024, + "osl": 1024, + "precision": "bf16", + "max_model_len": 11264, + }, + "objective": {"kind": "throughput", "value": 120.0}, + } + _write_json(tmp_path / "state.json", state) + _write_json(tmp_path / "manifest.json", manifest) + + before = exporter.build(tmp_path) + write_timeline_event( + tmp_path, + { + "type": "install", + "kind": "install", + "status": "succeeded", + "start_time": "2026-08-27T00:58:00+00:00", + "end_time": "2026-08-27T00:59:00+00:00", + "ext": {"run_kind": "fresh", "hard_fail_step_id": None, "runtime_snapshot": {}, "steps": []}, + }, + ) + write_timeline_event( + tmp_path, + { + "type": "model_gate", + "kind": "model_gate", + "status": "succeeded", + "start_time": "2026-08-27T00:59:00+00:00", + "end_time": "2026-08-27T01:00:00+00:00", + "ext": {"run_kind": "fresh", "checks": []}, + }, + ) + + after = exporter.build(tmp_path) + + assert after["schema_version"] == SCHEMA_VERSION_V5 + v6_keys = {"metadata", "outcome", "timeline", "close"} + assert {key: value for key, value in after.items() if key not in v6_keys} == { + key: value for key, value in before.items() if key not in v6_keys + } + assert after["metadata"]["versions"]["schema_version"] == SCHEMA_VERSION_V6 + assert after["metadata"]["versions"]["hyperloom"] == "abc1234" + assert after["metadata"]["task_config"]["launch_env"] == {"TP": "8"} + assert after["outcome"]["status"] == "completed" + assert after["outcome"]["stage_reached"] == "close" + assert "token_usage" not in after["outcome"] + assert [event["type"] for event in after["timeline"]] == ["install", "model_gate"] + assert after["close"] == {} + assert all(event["type"] != "close" for event in after["timeline"]) + + +def test_invalid_v6_event_does_not_change_v5_warnings(tmp_path): + before = exporter.build(tmp_path) + path = tmp_path / "reports" / "sbd_v6" / "install.json" + path.parent.mkdir(parents=True) + path.write_text("{invalid", encoding="utf-8") + + after = exporter.build(tmp_path) + + assert after["warnings"] == before["warnings"] + assert any("timeline.install" in warning for warning in after["metadata"]["warnings"]) + assert after["timeline"] == [] + + +def test_install_event_stays_pending_until_session_creation(tmp_path): + from hyperloom.inference_optimizer.cli import preflight + + args = argparse.Namespace(resume_from=None, no_kernel=True, enable_roofline=False) + event = preflight._begin_install_event(args) + preflight._run_install_step( + event, + step_id="load_dotenv", + category="normalize", + action=lambda: { + "status": "already_present", + "skip_reason": None, + "detail": {"vars_loaded": 0, "source": "/repo/.env"}, + }, + ) + preflight._run_install_step( + event, + step_id="check_shm_disk", + category="check", + action=lambda: { + "status": "warned", + "skip_reason": None, + "detail": {"shm_free_gib": 8.0, "min_gib": 16}, + }, + ) + preflight._finish_install_event( + event, + args=args, + benchmark_backend="bypass", + benchmark_python="python", + magpie_python="python", + inferencex_path="/opt/InferenceX", + resolved_urls=("", "https://api.openai.com/v1"), + ) + + assert not (tmp_path / "reports" / "sbd_v6" / "install.json").exists() + preflight._persist_install_event(args, tmp_path) + + persisted = read_timeline_event(tmp_path, "install") + assert persisted is not None + assert persisted["status"] == "degraded" + assert [step["step_id"] for step in persisted["ext"]["steps"]] == [ + "load_dotenv", + "check_shm_disk", + ] + assert persisted["ext"]["runtime_snapshot"]["provider_mode"] == "openai" + + +def test_preflight_hard_failure_creates_session_and_final_sbd(tmp_path, monkeypatch): + import hyperloom.inference_optimizer.cli as optimizer_cli + from hyperloom.inference_optimizer.cli import preflight + from hyperloom.inference_optimizer.session.paths import ENV_CURRENT_SESSION_DIR + + workspace = tmp_path / "sessions" + model = tmp_path / "Qwen-Test" + monkeypatch.setenv("USER_DATA_PATH", str(workspace)) + monkeypatch.delenv(ENV_CURRENT_SESSION_DIR, raising=False) + monkeypatch.setattr( + optimizer_cli, + "clean_stale_aiter_locks", + lambda: {"dir": "", "deleted": 0, "skipped_fresh": 0, "errors": 0}, + ) + + def fail_preflight(args): + event = preflight._begin_install_event(args) + + def reject_credentials(): + raise SystemExit(2) + + preflight._run_install_step( + event, + step_id="validate_credentials", + category="check", + action=reject_credentials, + ) + + monkeypatch.setattr(optimizer_cli, "_preflight", fail_preflight) + args = optimizer_cli._build_parser().parse_args(["optimize", "--model", str(model)]) + + with pytest.raises(SystemExit) as exc: + asyncio.run(optimizer_cli._run_optimize(args)) + + assert exc.value.code == 2 + session_dir = Path(os.environ[ENV_CURRENT_SESSION_DIR]) + assert session_dir.is_relative_to(workspace) + install = read_timeline_event(session_dir, "install") + assert install is not None + assert install["status"] == "failed" + assert install["ext"]["hard_fail_step_id"] == "validate_credentials" + assert (session_dir / "manifest.json").is_file() + breakdown = json.loads((session_dir / "session_breakdown.json").read_text(encoding="utf-8")) + assert [(event["type"], event["status"]) for event in breakdown["timeline"]] == [("install", "failed")] + + +def test_unwrapped_preflight_failure_is_persisted_as_failed(tmp_path, monkeypatch): + import hyperloom.inference_optimizer.cli as optimizer_cli + from hyperloom.inference_optimizer.cli import preflight + from hyperloom.inference_optimizer.session.paths import ENV_CURRENT_SESSION_DIR + + workspace = tmp_path / "sessions" + model = tmp_path / "Qwen-Test" + monkeypatch.setenv("USER_DATA_PATH", str(workspace)) + monkeypatch.delenv(ENV_CURRENT_SESSION_DIR, raising=False) + monkeypatch.setattr( + optimizer_cli, + "clean_stale_aiter_locks", + lambda: {"dir": "", "deleted": 0, "skipped_fresh": 0, "errors": 0}, + ) + monkeypatch.setattr(preflight, "_load_dotenv_fallback", lambda: None) + monkeypatch.setattr(preflight, "_provider_only_mode", lambda: "") + monkeypatch.setattr(preflight, "_load_kernel_agent_env_fallback", lambda: None) + + def fail_runtime_paths(): + raise RuntimeError("runtime path resolution failed") + + monkeypatch.setattr(preflight, "_derive_runtime_paths", fail_runtime_paths) + args = optimizer_cli._build_parser().parse_args(["optimize", "--model", str(model)]) + + with pytest.raises(RuntimeError, match="runtime path resolution failed"): + asyncio.run(optimizer_cli._run_optimize(args)) + + session_dir = Path(os.environ[ENV_CURRENT_SESSION_DIR]) + install = read_timeline_event(session_dir, "install") + assert install is not None + assert install["status"] == "failed" + assert install["end_time"] + assert install["ext"]["hard_fail_step_id"] == "unhandled_preflight" + failure = install["ext"]["steps"][-1] + assert failure["step_id"] == "unhandled_preflight" + assert failure["error_class"] == "RuntimeError" + assert failure["message"] == "runtime path resolution failed" + + +def test_busy_resume_preflight_failure_uses_isolated_session(tmp_path, monkeypatch): + import hyperloom.inference_optimizer.cli as optimizer_cli + from hyperloom.inference_optimizer.cli import preflight + from hyperloom.inference_optimizer.session.paths import ENV_CURRENT_SESSION_DIR + + workspace = tmp_path / "sessions" + resume_dir = workspace / "Qwen-Test" / "active-session" + original_install = { + "type": "install", + "kind": "install", + "status": "succeeded", + "start_time": "2026-08-27T01:00:00+00:00", + "end_time": "2026-08-27T01:01:00+00:00", + "ext": {"run_kind": "fresh", "steps": []}, + } + _write_json(resume_dir / "reports" / "sbd_v6" / "install.json", original_install) + _write_json(resume_dir / "session_breakdown.json", {"sentinel": "active"}) + monkeypatch.setenv("USER_DATA_PATH", str(workspace)) + monkeypatch.delenv("MODEL_PATH", raising=False) + monkeypatch.delenv(ENV_CURRENT_SESSION_DIR, raising=False) + monkeypatch.setattr( + optimizer_cli, + "clean_stale_aiter_locks", + lambda: {"dir": "", "deleted": 0, "skipped_fresh": 0, "errors": 0}, + ) + + class FakeSessionLock: + def __init__(self, session_dir): + self.session_dir = Path(session_dir) + + def acquire(self): + if self.session_dir == resume_dir: + raise optimizer_cli.SessionAlreadyRunning(resume_dir, {"pid": 123}) + return self + + def release(self): + return None + + def fail_preflight(args): + preflight._begin_install_event(args) + raise RuntimeError("resume preflight failed") + + monkeypatch.setattr(optimizer_cli, "SessionLock", FakeSessionLock) + monkeypatch.setattr(optimizer_cli, "_preflight", fail_preflight) + args = optimizer_cli._build_parser().parse_args(["optimize", "--resume-from", str(resume_dir)]) + + with pytest.raises(RuntimeError, match="resume preflight failed"): + asyncio.run(optimizer_cli._run_optimize(args)) + + assert json.loads((resume_dir / "reports" / "sbd_v6" / "install.json").read_text(encoding="utf-8")) == ( + original_install + ) + assert json.loads((resume_dir / "session_breakdown.json").read_text(encoding="utf-8")) == {"sentinel": "active"} + failed_session = Path(os.environ[ENV_CURRENT_SESSION_DIR]) + assert failed_session != resume_dir + install = read_timeline_event(failed_session, "install") + assert install is not None + assert install["status"] == "failed" + assert install["ext"]["run_kind"] == "resume" + + +def test_timeline_history_retains_fresh_and_resume_events(tmp_path, monkeypatch): + from hyperloom.inference_optimizer.cli import model_gate + + write_timeline_event( + tmp_path, + { + "type": "install", + "kind": "install", + "status": "succeeded", + "start_time": "2026-08-27T01:00:00+00:00", + "end_time": "2026-08-27T01:01:00+00:00", + "ext": {"run_kind": "fresh", "steps": []}, + }, + ) + timestamps = iter( + ( + "2026-08-27T01:02:00+00:00", + "2026-08-27T01:03:00+00:00", + "2026-08-27T02:02:00+00:00", + "2026-08-27T02:03:00+00:00", + "2026-08-27T02:04:00+00:00", + ) + ) + monkeypatch.setattr(model_gate, "now_iso", lambda **_kwargs: next(timestamps)) + fresh_args = _gate_args(tmp_path / "model") + model_gate._start_model_gate(fresh_args, tmp_path) + model_gate._finish_model_gate(fresh_args, tmp_path) + write_timeline_event( + tmp_path, + { + "type": "install", + "kind": "install", + "status": "succeeded", + "start_time": "2026-08-27T02:00:00+00:00", + "end_time": "2026-08-27T02:01:00+00:00", + "ext": {"run_kind": "resume", "steps": []}, + }, + ) + resume_args = _gate_args(tmp_path / "model", resume_from=str(tmp_path)) + model_gate._record_resumed_model_gate(resume_args, tmp_path) + + expected = [ + ("install", "succeeded", "fresh"), + ("model_gate", "succeeded", "fresh"), + ("install", "succeeded", "resume"), + ("model_gate", "skipped", "resume"), + ] + assert [ + (event["type"], event["status"], event["ext"]["run_kind"]) for event in read_timeline_events(tmp_path) + ] == expected + assert [ + (event["type"], event["status"], event["ext"]["run_kind"]) + for event in collect_v6_timeline(tmp_path, [], state={}, recorded_operations=[]) + ] == expected + latest_gate = read_timeline_event(tmp_path, "model_gate") + assert latest_gate is not None + assert latest_gate["ext"]["run_kind"] == "resume" + + +def test_timeline_history_bootstraps_legacy_fixed_events(tmp_path): + _write_json( + tmp_path / "reports" / "sbd_v6" / "install.json", + { + "type": "install", + "kind": "install", + "status": "succeeded", + "start_time": "2026-08-27T01:00:00+00:00", + "end_time": "2026-08-27T01:01:00+00:00", + "ext": {"run_kind": "fresh", "steps": []}, + }, + ) + _write_json( + tmp_path / "reports" / "sbd_v6" / "model_gate.json", + { + "type": "model_gate", + "kind": "model_gate", + "status": "succeeded", + "start_time": "2026-08-27T01:02:00+00:00", + "end_time": "2026-08-27T01:03:00+00:00", + "ext": {"run_kind": "fresh", "checks": []}, + }, + ) + + write_timeline_event( + tmp_path, + { + "type": "install", + "kind": "install", + "status": "succeeded", + "start_time": "2026-08-27T02:00:00+00:00", + "end_time": "2026-08-27T02:01:00+00:00", + "ext": {"run_kind": "resume", "steps": []}, + }, + ) + + assert [(event["type"], event["ext"]["run_kind"]) for event in read_timeline_events(tmp_path)] == [ + ("install", "fresh"), + ("model_gate", "fresh"), + ("install", "resume"), + ] + + +def test_timeline_history_recovers_after_partial_legacy_migration(tmp_path): + fresh_install = { + "type": "install", + "kind": "install", + "status": "succeeded", + "start_time": "2026-08-27T01:00:00+00:00", + "end_time": "2026-08-27T01:01:00+00:00", + "ext": {"run_kind": "fresh", "steps": []}, + } + _write_json(tmp_path / "reports" / "sbd_v6" / "install.json", fresh_install) + _write_json(tmp_path / "reports" / "sbd_v6" / "timeline" / "000001-install.json", fresh_install) + _write_json( + tmp_path / "reports" / "sbd_v6" / "model_gate.json", + { + "type": "model_gate", + "kind": "model_gate", + "status": "succeeded", + "start_time": "2026-08-27T01:02:00+00:00", + "end_time": "2026-08-27T01:03:00+00:00", + "ext": {"run_kind": "fresh", "checks": []}, + }, + ) + + write_timeline_event( + tmp_path, + { + "type": "install", + "kind": "install", + "status": "succeeded", + "start_time": "2026-08-27T02:00:00+00:00", + "end_time": "2026-08-27T02:01:00+00:00", + "ext": {"run_kind": "resume", "steps": []}, + }, + ) + + assert [(event["type"], event["ext"]["run_kind"]) for event in read_timeline_events(tmp_path)] == [ + ("install", "fresh"), + ("model_gate", "fresh"), + ("install", "resume"), + ] + + +def test_preflight_records_install_steps_in_execution_order(tmp_path, monkeypatch): + from hyperloom.agents.framework import kb as framework_kb + from hyperloom.inference_optimizer.cli import preflight + from hyperloom.inference_optimizer.session.sbd_v6 import pending_install_event + from hyperloom.orchestrator.actions.executors import benchmark_backend + + inferencex = tmp_path / "InferenceX" + inferencex.mkdir() + monkeypatch.setenv("INFERENCEX_PATH", str(inferencex)) + monkeypatch.setattr(benchmark_backend, "resolve_backend_name", lambda: "bypass") + monkeypatch.setattr(benchmark_backend, "resolve_benchmark_interpreter", lambda: "python") + monkeypatch.setattr(preflight, "_provider_only_mode", lambda: "") + monkeypatch.setattr(preflight, "_load_dotenv_fallback", lambda: None) + monkeypatch.setattr(preflight, "_load_kernel_agent_env_fallback", lambda: None) + monkeypatch.setattr(preflight, "_derive_runtime_paths", lambda: None) + monkeypatch.setattr(preflight, "_restore_provider_only_mode", lambda *_args: None) + monkeypatch.setattr(preflight, "_normalize_legacy_deepseek_env", lambda: None) + monkeypatch.setattr(preflight, "_validate_credentials", lambda: None) + monkeypatch.setattr(framework_kb, "prepare_kb_environment", lambda: None) + monkeypatch.setattr(preflight, "_ensure_python_sdks", lambda *_args: None) + monkeypatch.setattr(preflight, "_resolve_llm_endpoints", lambda: ("", "")) + monkeypatch.setattr(preflight, "_unset_hip_visible_devices", lambda: None) + monkeypatch.setattr(preflight, "_check_gpu_visibility", lambda: None) + monkeypatch.setattr(preflight, "_check_shm_disk", lambda: None) + monkeypatch.setattr(preflight, "_check_platform_tuning", lambda: None) + monkeypatch.setattr(preflight, "_ensure_ray", lambda *_args: None) + monkeypatch.setattr(preflight, "_ensure_bench_serving_deps", lambda *_args: None) + monkeypatch.setattr(preflight, "_ensure_lm_eval_dep", lambda *_args, **_kwargs: None) + monkeypatch.setattr(preflight, "_ensure_framework_deps", lambda *_args: None) + monkeypatch.setattr(preflight, "_check_serving_framework", lambda *_args: None) + monkeypatch.setattr(preflight, "_inferencex_checkout_ok", lambda *_args, **_kwargs: True) + monkeypatch.setattr(preflight, "_inferencex_head_sha", lambda *_args: "abc123") + monkeypatch.setattr(preflight, "_report_inferencex_patch_anchors", lambda *_args: True) + monkeypatch.setattr(preflight, "_check_node_claude_cli", lambda: None) + monkeypatch.setattr(preflight.shutil, "which", lambda name: f"/usr/bin/{name}") + monkeypatch.setattr( + preflight, + "_run_ir3_preflight", + lambda _args: {"status": "applied", "skip_reason": None}, + ) + monkeypatch.setattr( + preflight, + "_emit_preflight_diagnostics", + lambda **_kwargs: {"status": "applied", "skip_reason": None}, + ) + args = argparse.Namespace( + resume_from=None, + degraded_kb=True, + framework="sglang", + no_eval=True, + no_kernel=True, + enable_roofline=False, + ) + + preflight._preflight(args) + + event = pending_install_event(args) + assert event is not None + assert [step["step_id"] for step in event["ext"]["steps"]] == [ + "load_dotenv", + "load_kernel_agent_env", + "normalize_legacy_deepseek_env", + "validate_credentials", + "prepare_kb_environment", + "ensure_python_sdks", + "check_gpu_visibility", + "check_shm_disk", + "check_platform_tuning", + "ensure_ray", + "ensure_bench_serving_deps", + "ensure_lm_eval", + "framework_deps", + "check_serving_framework", + "ensure_magpie", + "clone_inferencex", + "patch_magpie_eval_concurrency", + "check_tracelens_cli", + "check_tracelens_root", + "ir3_pr_monitor_probe", + "diagnostics_snapshot", + ] + steps = {step["step_id"]: step for step in event["ext"]["steps"]} + assert steps["prepare_kb_environment"]["status"] == "skipped" + assert steps["prepare_kb_environment"]["skip_reason"] == "explicit_flag" + assert steps["ensure_magpie"]["message"] == "benchmark backend is 'bypass'" + + +def test_ir3_unreachable_uses_v6_reason_without_changing_v5_state_reason(tmp_path, monkeypatch): + from hyperloom.inference_optimizer.cli import preflight + + monkeypatch.setattr(preflight, "_workspace_root_resolve", lambda: tmp_path) + monkeypatch.setattr(preflight.subprocess, "run", lambda *_args, **_kwargs: None) + args = argparse.Namespace(degraded_kb=False, degraded_pr=False, pr_monitor_url="") + + outcome = preflight._run_ir3_preflight(args) + + assert args.pr_degraded_reason == "ir3_auto" + assert outcome["status"] == "warned" + assert outcome["skip_reason"] == "ir3_unreachable" + assert outcome["detail"]["pr_monitor"] == { + "enabled": False, + "reason": "ir3_unreachable", + } + + +@pytest.mark.parametrize( + ("scenario", "failed_gate_id", "expected_statuses"), + [ + ("unsupported", "unsupported_model_arch", ["failed", "skipped", "skipped"]), + ("config", "model_config_compat", ["passed", "failed", "skipped"]), + ("context", "context_window", ["passed", "passed", "failed"]), + ], +) +def test_each_model_gate_failure_is_written_to_final_sbd( + tmp_path, + monkeypatch, + scenario, + failed_gate_id, + expected_statuses, +): + from hyperloom.inference_optimizer.cli import model_gate + + monkeypatch.setattr(model_gate, "_emit_breakdown_to_langfuse", lambda _session_dir: None) + model = tmp_path / scenario + if scenario == "unsupported": + _write_model_config( + model, + { + "architectures": ["Gemma3ForConditionalGeneration"], + "model_type": "gemma3", + }, + ) + elif scenario == "config": + _write_model_config( + model, + { + "architectures": ["LlamaForCausalLM"], + "model_type": "llama", + "rope_scaling": {"factor": 2.0}, + }, + ) + else: + _write_model_config( + model, + { + "architectures": ["LlamaForCausalLM"], + "model_type": "llama", + "max_position_embeddings": 2048, + }, + ) + session_dir = tmp_path / f"session-{scenario}" + _seed_state(session_dir, monkeypatch, model) + args = _gate_args(model) + model_gate._start_model_gate(args, session_dir) + + if scenario == "unsupported": + assert model_gate._preflight_unsupported_model_arch(args, session_dir) is True + else: + assert model_gate._preflight_unsupported_model_arch(args, session_dir) is False + if scenario == "config": + assert model_gate._preflight_model_config_compat(args, session_dir) is True + else: + assert model_gate._preflight_model_config_compat(args, session_dir) is False + assert model_gate._preflight_context_window(args, session_dir) is True + + event = _model_gate_from_breakdown(session_dir) + assert event["status"] == "failed" + assert event["ext"]["failed_gate_id"] == failed_gate_id + assert [check["gate_id"] for check in event["ext"]["checks"]] == [ + "unsupported_model_arch", + "model_config_compat", + "context_window", + ] + assert [check["status"] for check in event["ext"]["checks"]] == expected_statuses + assert event["ext"]["failure"]["artifacts"]["breakdown_written"] is True + + +def test_resume_model_gate_records_three_explicit_skips(tmp_path): + from hyperloom.inference_optimizer.cli import model_gate + + args = _gate_args(tmp_path / "model") + model_gate._record_resumed_model_gate( + args, + tmp_path, + workload_overrides={ + "model_path": "/models/resumed", + "model_name": "resumed-model", + "framework": "vllm", + "gpu_type": "MI355X", + }, + ) + + event = read_timeline_event(tmp_path, "model_gate") + assert event is not None + assert event["status"] == "skipped" + assert event["ext"]["run_kind"] == "resume" + assert event["ext"]["skip_reason"] == "resume" + assert event["ext"]["workload"]["model_path"] == "/models/resumed" + assert event["ext"]["workload"]["model_name"] == "resumed-model" + assert event["ext"]["workload"]["framework"] == "vllm" + assert event["ext"]["workload"]["gpu_type"] == "MI355X" + assert [check["skip_reason"] for check in event["ext"]["checks"]] == [ + "resume", + "resume", + "resume", + ] + + +def test_model_gate_event_write_failure_does_not_change_gate_result(tmp_path, monkeypatch): + from hyperloom.inference_optimizer.cli import model_gate + from hyperloom.inference_optimizer.session import sbd_v6 + + model = tmp_path / "healthy" + _write_model_config( + model, + { + "architectures": ["LlamaForCausalLM"], + "model_type": "llama", + "max_position_embeddings": 8192, + }, + ) + monkeypatch.setattr( + sbd_v6, + "write_timeline_event", + lambda *_args, **_kwargs: (_ for _ in ()).throw(OSError("disk unavailable")), + ) + + args = _gate_args(model) + model_gate._start_model_gate(args, tmp_path) + + assert model_gate._preflight_unsupported_model_arch(args, tmp_path) is False + + +def test_model_gate_projection_failure_does_not_change_gate_result(tmp_path, monkeypatch): + from hyperloom.inference_optimizer.cli import model_gate + + model = tmp_path / "healthy" + _write_model_config( + model, + { + "architectures": ["LlamaForCausalLM"], + "model_type": "llama", + "max_position_embeddings": 8192, + }, + ) + monkeypatch.setattr( + model_gate, + "_load_model_gate_event", + lambda *_args, **_kwargs: (_ for _ in ()).throw(ValueError("bad V6 projection")), + ) + + assert model_gate._preflight_unsupported_model_arch(_gate_args(model), tmp_path) is False + + +def test_install_projection_failure_does_not_change_step_result(monkeypatch): + from hyperloom.inference_optimizer.cli import preflight + + monkeypatch.setattr( + preflight, + "_record_install_step", + lambda *_args, **_kwargs: (_ for _ in ()).throw(ValueError("bad V6 projection")), + ) + + assert ( + preflight._run_install_step( + {"ext": {"steps": []}}, + step_id="unchanged", + category="check", + action=lambda: "original-result", + ) + == "original-result" + ) + + +def test_corrupt_model_gate_event_is_safely_normalized(tmp_path): + from hyperloom.inference_optimizer.cli import model_gate + + path = tmp_path / "reports" / "sbd_v6" / "model_gate.json" + _write_json( + path, + { + "type": "model_gate", + "ext": { + "checks": [{"gate_id": "legacy", "order": "invalid", "status": "unknown"}], + "degraded": [], + }, + }, + ) + args = _gate_args(tmp_path / "model") + + model_gate._record_model_gate_check( + args, + tmp_path, + { + "gate_id": "unsupported_model_arch", + "order": 1, + "status": "passed", + "skip_reason": None, + "detail": {}, + }, + ) + + event = read_timeline_event(tmp_path, "model_gate") + assert event is not None + assert event["status"] == "degraded" + assert event["ext"]["checks"][0]["gate_id"] == "unsupported_model_arch" + assert event["ext"]["degraded"] == {"active": False, "warnings": []} + + +def test_fresh_model_gate_with_only_soft_skips_succeeds(tmp_path): + from hyperloom.inference_optimizer.cli import model_gate + + args = _gate_args(tmp_path / "model") + model_gate._start_model_gate(args, tmp_path) + for order, gate_id in enumerate(model_gate._MODEL_GATE_ORDER, start=1): + model_gate._record_model_gate_check( + args, + tmp_path, + { + "gate_id": gate_id, + "order": order, + "status": "skipped", + "skip_reason": "soft_pass", + "detail": {}, + }, + ) + model_gate._finish_model_gate(args, tmp_path) + + event = read_timeline_event(tmp_path, "model_gate") + assert event is not None + assert event["status"] == "succeeded" + assert event["ext"]["skip_reason"] is None diff --git a/src/hyperloom/inference_optimizer/tests/test_session_package.py b/src/hyperloom/inference_optimizer/tests/test_session_package.py index 47dbf18c62..38ce374b0b 100644 --- a/src/hyperloom/inference_optimizer/tests/test_session_package.py +++ b/src/hyperloom/inference_optimizer/tests/test_session_package.py @@ -40,6 +40,8 @@ def _build_session(sd: Path) -> None: _write(sd / "reports" / "optimization_journal.json", "[]") _write(sd / "reports" / "kernel_optimization_summary.json", "{}") _write(sd / "reports" / "kernel_roofline.json", "{}") + _write(sd / "reports" / "sbd_v6" / "install.json", "{}") + _write(sd / "reports" / "sbd_v6" / "model_gate.json", "{}") _write(sd / "reports" / "trace" / "decision_trace.jsonl", "{}\n") _write(sd / "reports" / "trace" / "llm_calls.jsonl", "{}\n") _write(sd / "target_analysis" / "target_baseline.json", "{}") @@ -102,6 +104,8 @@ def test_package_includes_curated_excludes_noise(tmp_path: Path) -> None: "reports/optimization_journal.json", "reports/kernel_optimization_summary.json", "reports/kernel_roofline.json", + "reports/sbd_v6/install.json", + "reports/sbd_v6/model_gate.json", "reports/trace/decision_trace.jsonl", "reports/trace/llm_calls.jsonl", "target_analysis/target_baseline.json", From b6408de25fe0613d6a7613cd0841f1ac577c7125 Mon Sep 17 00:00:00 2001 From: chenluo Date: Fri, 28 Aug 2026 15:23:55 +0800 Subject: [PATCH 2/7] feat: add merged Framework Agent SBD timeline --- .../breakdown/collectors/v6.py | 1911 ++++++++++++++++- .../breakdown/recorder/instrument.py | 3 +- ...test_enablement_coordinator_wiring_unit.py | 1 + .../tests/test_framework_agent_authoring.py | 1 + .../tests/test_research_scout.py | 1 + .../tests/test_sbd_v6_initial.py | 965 +++++++++ .../tests/test_specialist_lifecycle.py | 4 +- .../orchestrator/enablement/params.py | 1 + src/hyperloom/orchestrator/phases/explore.py | 15 +- .../orchestrator/phases/framework.py | 1 + src/hyperloom/orchestrator/phases/internal.py | 3 + 11 files changed, 2900 insertions(+), 6 deletions(-) diff --git a/src/hyperloom/inference_optimizer/breakdown/collectors/v6.py b/src/hyperloom/inference_optimizer/breakdown/collectors/v6.py index e1ceceb20f..38f786d7c1 100644 --- a/src/hyperloom/inference_optimizer/breakdown/collectors/v6.py +++ b/src/hyperloom/inference_optimizer/breakdown/collectors/v6.py @@ -2,9 +2,13 @@ from __future__ import annotations +import re +from datetime import datetime from pathlib import Path from typing import Any +from hyperloom.common.jsonio import read_json, read_jsonl + from ...session.sbd_v6 import SCHEMA_VERSION_V6, read_timeline_events @@ -26,6 +30,22 @@ "unsupported_model_arch", } ) +_FRAMEWORK_PHASES = frozenset({"FRAMEWORK_AGENT", "EXPLORE"}) +_FRAMEWORK_EXIT_REASON_MAP = { + "explore_no_more_leverage": "optimize_no_more_leverage", + "plateau_explore": "optimize_no_more_leverage", + "explore_phase_budget_exhausted": "optimize_phase_budget_exhausted", + "explore_budget_cap": "optimize_budget_cap", + "explore_force_exit_low_budget": "optimize_force_exit_low_budget", +} +_AUTHORING_TASK_KINDS = frozenset( + { + "explore_apply_retry", + "framework_authoring", + "framework_local_explore", + } +) +_MACRO_CYCLE_RE = re.compile(r"(?:^|\s)macro_cycle\s*=\s*(-?\d+)(?=\s|$)") def _tool_versions(versions: Any) -> dict[str, str | None]: @@ -140,6 +160,1869 @@ def collect_v6_metadata( } +def _mapping(value: Any) -> dict[str, Any]: + return value if isinstance(value, dict) else {} + + +def _dict_rows(value: Any) -> list[dict[str, Any]]: + return [row for row in value if isinstance(row, dict)] if isinstance(value, list) else [] + + +def _dict_value_rows(value: Any) -> list[dict[str, Any]]: + return [row for row in value.values() if isinstance(row, dict)] if isinstance(value, dict) else [] + + +def _first(*values: Any) -> Any: + for value in values: + if value is not None and value != "": + return value + return None + + +def _optional_int(value: Any) -> int | None: + if value is None or value == "" or isinstance(value, bool): + return None + try: + return int(value) + except (TypeError, ValueError): + return None + + +def _optional_float(value: Any) -> float | None: + if value is None or value == "" or isinstance(value, bool): + return None + try: + return float(value) + except (TypeError, ValueError): + return None + + +def _optional_bool(value: Any) -> bool | None: + if isinstance(value, bool): + return value + if isinstance(value, int) and value in (0, 1): + return bool(value) + if isinstance(value, str): + normalized = value.strip().lower() + if normalized in {"1", "true", "yes", "on", "passed", "succeeded"}: + return True + if normalized in {"0", "false", "no", "off", "failed"}: + return False + return None + + +def _string_list(value: Any) -> list[str]: + if not isinstance(value, (list, tuple, set)): + return [] + return [str(item) for item in value if item not in (None, "")] + + +def _nested(mapping: dict[str, Any], *path: str) -> Any: + value: Any = mapping + for key in path: + if not isinstance(value, dict): + return None + value = value.get(key) + return value + + +def _row_cycle(row: dict[str, Any]) -> int | None: + for value in ( + row.get("macro_cycle"), + row.get("cycle"), + _nested(row, "outputs", "macro_cycle"), + _nested(row, "outputs", "cycle"), + _nested(row, "inputs", "macro_cycle"), + _nested(row, "inputs", "cycle"), + _nested(row, "metadata", "extras", "macro_cycle"), + _nested(row, "metadata", "extras", "cycle"), + ): + parsed = _optional_int(value) + if parsed is not None: + return parsed + return None + + +def _row_timestamp(row: dict[str, Any]) -> str: + return str( + _first( + row.get("ts"), + row.get("ended_at"), + row.get("completed_at"), + row.get("started_at"), + ) + or "" + ) + + +def _prompt_macro_cycle(value: Any) -> int | None: + match = _MACRO_CYCLE_RE.search(str(value or "")) + return _optional_int(match.group(1)) if match else None + + +def _timestamp_number(value: Any) -> float | None: + text = str(value or "").strip() + if not text: + return None + try: + return datetime.fromisoformat(text.replace("Z", "+00:00")).timestamp() + except (TypeError, ValueError): + return None + + +def _operation_name(operation: dict[str, Any]) -> str: + return str(operation.get("name") or operation.get("kind") or "").strip().lower() + + +def _declared_source_phase(row: dict[str, Any]) -> str: + return ( + str( + _first( + row.get("source_phase"), + _nested(row, "outputs", "source_phase"), + _nested(row, "inputs", "source_phase"), + _nested(row, "metadata", "extras", "source_phase"), + ) + or "" + ) + .strip() + .upper() + ) + + +def _source_phase(row: dict[str, Any]) -> str: + return _declared_source_phase(row) or str(row.get("phase") or "").strip().upper() + + +def _specialist_payloads(row: dict[str, Any]) -> tuple[dict[str, Any], ...]: + return ( + row, + _mapping(row.get("inputs")), + _mapping(row.get("outputs")), + _mapping(row.get("extensions")), + ) + + +def _is_framework_specialist(row: dict[str, Any], *, allow_legacy: bool) -> bool: + declared_phase = _declared_source_phase(row) + phase = declared_phase or str(row.get("phase") or "").strip().upper() + if not declared_phase and str(row.get("source") or "").strip().lower() == "specialist_recorder_hook": + phase = "" + if phase: + return phase in _FRAMEWORK_PHASES + + agent = str(row.get("agent") or "").strip().lower() + if agent in {"framework_agent", "explore"}: + return True + if agent in {"enablement", "kernel", "kernel_agent", "prelude", "internal"}: + return False + + payloads = _specialist_payloads(row) + if any(_optional_bool(payload.get("enablement")) is True for payload in payloads): + return False + if any( + _optional_bool(payload.get("framework_agent_authoring")) is True + or bool(payload.get("framework_agent_candidate_id")) + or bool(payload.get("framework_batch_id")) + or str(payload.get("task_kind") or "").strip().lower() in _AUTHORING_TASK_KINDS + for payload in payloads + ): + return True + return allow_legacy + + +def _operation_task_id(operation: dict[str, Any]) -> str: + return str( + _first( + _nested(operation, "extensions", "task_id"), + _nested(operation, "outputs", "task_id"), + _nested(operation, "metadata", "extras", "task_id"), + ) + or "" + ) + + +def _candidate_id(value: Any) -> str: + candidate = _mapping(value) + return str( + _first( + candidate.get("candidate_id"), + candidate.get("pr_url"), + candidate.get("url"), + candidate.get("ref"), + candidate.get("head_sha"), + ) + or "" + ) + + +def _is_framework_operation(operation: dict[str, Any]) -> bool: + name = _operation_name(operation) + phase = _source_phase(operation) + agent = str(operation.get("agent") or "").strip().lower() + outputs = _mapping(operation.get("outputs")) + if name == "framework_agent": + return True + if name == "explore": + if phase: + return phase in _FRAMEWORK_PHASES + return agent in {"framework_agent", "explore"} + if name in {"integrate", "integrate_patch"}: + if outputs.get("enablement") or outputs.get("enablement_landing"): + return False + if phase: + return phase in _FRAMEWORK_PHASES + return bool( + agent in {"framework_agent", "explore"} + or outputs.get("framework_agent_authoring") + or outputs.get("framework_agent_candidate_id") + ) + return name.startswith("specialist") and _is_framework_specialist(operation, allow_legacy=True) + + +def _new_framework_window(cycle: int, start_time: str = "") -> dict[str, Any]: + return { + "cycle": cycle, + "start_time": start_time, + "end_time": "", + "rows": [], + "exit_row": {}, + } + + +def _framework_windows( + state: dict[str, Any], + recorded_operations: list[dict[str, Any]], +) -> list[dict[str, Any]]: + windows: list[dict[str, Any]] = [] + active: dict[str, Any] | None = None + history = _dict_rows(state.get("phase_history")) + for row in history: + from_phase = str(row.get("from_phase") or "").strip().upper() + to_phase = str(row.get("to_phase") or "").strip().upper() + if not from_phase and not to_phase: + continue + cycle = _row_cycle(row) + if cycle is None: + cycle = int(state.get("macro_cycle") or 0) + from_framework = from_phase in _FRAMEWORK_PHASES + to_framework = to_phase in _FRAMEWORK_PHASES + if to_framework and not from_framework: + active = _new_framework_window(cycle, str(row.get("ts") or "")) + active["rows"].append(row) + windows.append(active) + continue + if from_framework and to_framework: + if active is None or int(active["cycle"]) != cycle: + active = _new_framework_window(cycle) + windows.append(active) + active["rows"].append(row) + continue + if from_framework and not to_framework: + if active is None or int(active["cycle"]) != cycle: + active = next( + ( + window + for window in reversed(windows) + if int(window["cycle"]) == cycle and not window["end_time"] + ), + None, + ) + if active is None: + active = _new_framework_window(cycle) + windows.append(active) + active["rows"].append(row) + active["end_time"] = str(row.get("ts") or "") + active["exit_row"] = row + active = None + continue + current_phase = str(state.get("phase") or "").strip().upper() + current_cycle = int(state.get("macro_cycle") or 0) + if current_phase in _FRAMEWORK_PHASES and not any( + int(window["cycle"]) == current_cycle and not window["end_time"] for window in windows + ): + windows.append(_new_framework_window(current_cycle, str(state.get("phase_started_ts") or ""))) + + evidence_rows: list[dict[str, Any]] = [] + for operation in recorded_operations: + if not _is_framework_operation(operation): + continue + if _operation_name(operation).startswith("specialist") and not _is_framework_specialist( + operation, + allow_legacy=False, + ): + continue + evidence_rows.append(operation) + evidence_rows.extend(_dict_rows(state.get("framework_agent_batches"))) + evidence_rows.extend(_dict_rows(state.get("framework_agent_phase_progress"))) + evidence_rows.extend( + row for row in _dict_rows(state.get("specialist_rounds")) if _is_framework_specialist(row, allow_legacy=False) + ) + evidence_rows.extend(_dict_rows(state.get("framework_config_exploration_results"))) + explore_search = _mapping(state.get("explore_search")) + tested = explore_search.get("tested") + if isinstance(tested, dict): + evidence_rows.extend(row for row in tested.values() if isinstance(row, dict)) + evidence_rows.extend(_dict_rows(explore_search.get("winners_history"))) + known_cycles = {int(window["cycle"]) for window in windows} + for row in evidence_rows: + cycle = _row_cycle(row) + if cycle is None or cycle in known_cycles: + continue + windows.append(_new_framework_window(cycle)) + known_cycles.add(cycle) + + windows.sort( + key=lambda window: ( + int(window["cycle"]), + _timestamp_number(window.get("start_time")) or float("inf"), + ) + ) + return windows + + +def _row_in_window(row: dict[str, Any], window: dict[str, Any], window_count: int) -> bool: + cycle = _row_cycle(row) + if cycle is not None and cycle != int(window["cycle"]): + return False + row_ts = _timestamp_number(_row_timestamp(row)) + start_ts = _timestamp_number(window.get("start_time")) + end_ts = _timestamp_number(window.get("end_time")) + if row_ts is not None and (start_ts is not None or end_ts is not None): + return (start_ts is None or row_ts >= start_ts) and (end_ts is None or row_ts <= end_ts) + return cycle is not None or window_count == 1 + + +def _window_operations( + recorded_operations: list[dict[str, Any]], + window: dict[str, Any], + window_count: int, +) -> list[dict[str, Any]]: + return [ + operation + for operation in recorded_operations + if _is_framework_operation(operation) and _row_in_window(operation, window, window_count) + ] + + +def _window_state_rows( + state: dict[str, Any], + field: str, + window: dict[str, Any], + window_count: int, +) -> list[dict[str, Any]]: + return [row for row in _dict_rows(state.get(field)) if _row_in_window(row, window, window_count)] + + +def _window_evidence(window: dict[str, Any]) -> dict[str, Any]: + evidence: dict[str, Any] = {} + for row in _dict_rows(window.get("rows")): + evidence.update(_mapping(row.get("evidence"))) + return evidence + + +def _operation_value(operations: list[dict[str, Any]], *paths: tuple[str, ...]) -> Any: + for operation in reversed(operations): + for path in paths: + value = _nested(operation, *path) + if value is not None and value != "": + return value + return None + + +def _framework_policy( + state: dict[str, Any], + operations: list[dict[str, Any]], + evidence: dict[str, Any], +) -> dict[str, Any]: + overrides = _mapping(state.get("plateau_overrides")) + stack_rebench_enabled = _optional_bool( + _operation_value( + operations, + ("outputs", "stack_rebench_enabled"), + ("outputs", "enable_stack_rebench"), + ("inputs", "stack_rebench_enabled"), + ("inputs", "enable_stack_rebench"), + ) + ) + return { + "keep_threshold_pct": _optional_float( + _first( + _operation_value( + operations, + ("outputs", "keep_threshold_pct"), + ("inputs", "keep_threshold_pct"), + ("decisions", "evidence", "keep_threshold_pct"), + ), + evidence.get("keep_threshold_pct"), + ) + ), + "stack_stable_threshold_pct": _optional_float( + _operation_value( + operations, + ("outputs", "stack_stable_threshold_pct"), + ("inputs", "stack_stable_threshold_pct"), + ) + ), + "stack_rebench_enabled": stack_rebench_enabled, + "variant_timeout_sec": _optional_int( + _first( + _operation_value( + operations, + ("outputs", "variant_timeout_sec"), + ("inputs", "variant_timeout_sec"), + ), + state.get("explore_variant_timeout_sec_override"), + ) + ), + "overtime_kill_ratio": _optional_float( + _first( + _operation_value( + operations, + ("outputs", "explore_overtime_kill_ratio"), + ("outputs", "overtime_kill_ratio"), + ("inputs", "explore_overtime_kill_ratio"), + ), + state.get("explore_overtime_kill_ratio"), + ) + ), + "force_exit_budget_pct": _optional_float( + _first(evidence.get("force_exit_budget_pct"), overrides.get("force_exit_budget_pct")) + ), + "config_arm": { + "keep_gain_threshold_pct": _optional_float( + _first(evidence.get("keep_gain_threshold_pct"), overrides.get("explore_keep_gain_pct")) + ), + "empty_streak_threshold": _optional_int( + _first(evidence.get("empty_streak_threshold"), overrides.get("explore_empty_streak")) + ), + "lookback": _optional_int(_first(evidence.get("lookback"), overrides.get("explore_lookback"))), + }, + "source_arm": { + "no_keep_streak_threshold": _optional_int( + _first(evidence.get("source_threshold"), evidence.get("no_keep_streak_threshold")) + ), + "discovery_retry_limit": _optional_int( + _first(evidence.get("retry_limit"), evidence.get("discovery_retry_limit")) + ), + "authoring_enabled": _optional_bool(state.get("framework_agent_authoring_enabled")), + }, + } + + +def _specialist_rows( + state: dict[str, Any], + operations: list[dict[str, Any]], + window: dict[str, Any], + window_count: int, +) -> list[dict[str, Any]]: + ordered: list[str] = [] + by_key: dict[str, dict[str, Any]] = {} + + def _upsert(row: dict[str, Any]) -> None: + key = str(_first(row.get("task_id"), row.get("round_id")) or "") + if not key: + key = f"row:{len(ordered)}" + if key not in by_key: + ordered.append(key) + by_key[key] = dict(row) + return + merged = dict(row) + merged.update(by_key[key]) + by_key[key] = merged + + for row in _window_state_rows(state, "specialist_rounds", window, window_count): + if not _is_framework_specialist(row, allow_legacy=True): + continue + _upsert(row) + for operation in operations: + if not _operation_name(operation).startswith("specialist"): + continue + row = dict(_mapping(operation.get("inputs"))) + row.update(_mapping(operation.get("outputs"))) + row.setdefault("task_id", _operation_task_id(operation)) + row.setdefault("cycle", _row_cycle(operation)) + row.setdefault("status", operation.get("status")) + row.setdefault("completed_at", operation.get("ended_at")) + row.setdefault("source_phase", operation.get("phase")) + _upsert(row) + return [by_key[key] for key in ordered] + + +def _specialist_role( + row: dict[str, Any], + candidate_map: dict[str, str], + source_task_ids: set[str], +) -> str: + task_id = str(row.get("task_id") or "") + domain = str(row.get("domain") or "").strip().lower() + task_kind = str(row.get("task_kind") or "").strip().lower() + if ( + row.get("candidate_discovery") + or task_kind == "candidate_discovery" + or domain == "candidate_discovery_specialist" + ): + return "discovery" + authoring_marker = _optional_bool(row.get("framework_agent_authoring")) is True + candidate_id = str(_first(row.get("framework_agent_candidate_id"), row.get("candidate_id")) or "") + if ( + task_id in candidate_map + or task_id in source_task_ids + or authoring_marker + or candidate_id + or task_kind in _AUTHORING_TASK_KINDS + or bool(_patch_refs(row)) + ): + return "authoring" + return "config" + + +def _specialist_status(row: dict[str, Any]) -> str: + raw = str(row.get("status") or "").strip().lower() + if raw in {"failed", "error", "timed_out", "timeout"} or row.get("error"): + return "failed" + proposals = row.get("proposal_set") + if bool(row.get("empty")) or isinstance(proposals, list) and not proposals: + return "empty" + if raw in {"empty", "skipped"}: + return "empty" + return "succeeded" + + +def _proposal_names(row: dict[str, Any]) -> list[str]: + names: list[str] = [] + for proposal in _dict_rows(row.get("proposal_set")): + name = str(_first(proposal.get("name"), proposal.get("variant_name"), proposal.get("proposal_id")) or "") + if name and name not in names: + names.append(name) + return names + + +def _config_specialist_run(row: dict[str, Any]) -> dict[str, Any]: + return { + "round_id": str(_first(row.get("round_id"), row.get("task_id")) or ""), + "task_id": str(row.get("task_id") or ""), + "status": _specialist_status(row), + "domain": str(row.get("domain") or ""), + "scope": _first(row.get("scope"), None), + "tags": _string_list(row.get("tags")), + "gap_canonical_id": _first(row.get("gap_canonical_id"), None), + "proposal_msg_id": _first(row.get("proposal_msg_id"), None), + "proposal_names": _proposal_names(row), + "reason": _first(row.get("reason"), row.get("error"), None), + } + + +def _config_source_index(rows: list[dict[str, Any]]) -> dict[str, dict[str, Any]]: + index: dict[str, dict[str, Any]] = {} + for row in rows: + info = { + "specialist_round_id": _first(row.get("round_id"), row.get("task_id"), None), + "proposal_msg_id": _first(row.get("proposal_msg_id"), None), + } + for proposal in _dict_rows(row.get("proposal_set")): + for key in ( + proposal.get("fingerprint"), + proposal.get("name"), + proposal.get("variant_name"), + proposal.get("proposal_id"), + ): + text = str(key or "").strip() + if text: + index.setdefault(text, info) + return index + + +def _config_variant(raw: dict[str, Any], source_index: dict[str, dict[str, Any]]) -> dict[str, Any]: + metrics = _mapping(raw.get("metrics")) + variant = _mapping(raw.get("variant")) + combined = dict(metrics) + combined.update(variant) + combined.update(raw) + name = str(_first(combined.get("name"), combined.get("variant_name")) or "") + fingerprint = str(combined.get("fingerprint") or "") + source = source_index.get(fingerprint) or source_index.get(name) or {} + outcome = str(combined.get("outcome") or "").strip().upper() + stack_tput = _optional_float(combined.get("stack_rebench_tput")) + stack_ran = _optional_bool(combined.get("stack_rebench_ran")) + if stack_ran is None and (stack_tput is not None or combined.get("stack_rebench_workspace")): + stack_ran = True + stack_stable = _optional_bool(combined.get("stack_rebench_stable")) + if stack_stable is None and stack_ran: + if outcome == "KEEP_UNSTABLE": + stack_stable = False + elif outcome == "KEEP": + stack_stable = True + return { + "name": name, + "fingerprint": fingerprint, + "source": { + "provenance": str(combined.get("provenance") or ""), + "scope": _first(combined.get("scope"), None), + "specialist_round_id": _first(combined.get("specialist_round_id"), source.get("specialist_round_id")), + "proposal_msg_id": _first(combined.get("proposal_msg_id"), source.get("proposal_msg_id")), + "critic_iteration": _optional_int(combined.get("critic_iteration")), + }, + "config_delta": { + "extra_server_args": str(combined.get("extra_server_args") or ""), + "extra_envs": dict(_mapping(combined.get("extra_envs"))), + "remove_args": _string_list(combined.get("remove_args")), + "unset_envs": _string_list(combined.get("unset_envs")), + "args_mode": _first(combined.get("args_mode"), None), + }, + "accepted_kernels": _string_list(combined.get("accepted_kernels")), + "measurement": { + "base_tput": _optional_float(combined.get("base_tput")), + "decision_tput": _optional_float(_first(combined.get("decision_tput"), combined.get("tput"))), + "gain_pct": _optional_float(combined.get("gain_pct")), + "runtime_sec": _optional_float(combined.get("runtime_sec")), + "estimated_output_throughput": _optional_float(combined.get("estimated_output_throughput")), + }, + "accuracy": { + "required": _optional_bool(_first(combined.get("accuracy_required"), combined.get("require_accuracy"))), + "reference": _optional_float(_first(combined.get("accuracy_reference"), combined.get("accuracy_baseline"))), + "value": _optional_float(combined.get("accuracy")), + "passed": _optional_bool(_first(combined.get("accuracy_pass"), combined.get("accuracy_passed"))), + }, + "stack_rebench": { + "ran": stack_ran, + "tput": stack_tput, + "stable": stack_stable, + }, + "outcome": outcome, + "reason": _first(combined.get("reason"), None), + "stage": _first(combined.get("stage"), None), + "failure": { + "error_class": _first(combined.get("error_class"), None), + "error_excerpt": _first(combined.get("error_excerpt"), combined.get("error"), None), + }, + "artifacts": { + "workspace": _first(combined.get("workspace"), combined.get("single_workspace"), None), + "stack_rebench_workspace": _first(combined.get("stack_rebench_workspace"), None), + "server_log_path": _first(combined.get("server_log_path"), None), + "raw_result_path": _first(combined.get("raw_result_path"), None), + }, + } + + +def _round_variant_rows(outputs: dict[str, Any]) -> list[dict[str, Any]]: + tested = _nested(outputs, "explore_search_update", "tested") + tested_by_fingerprint = tested if isinstance(tested, dict) else {} + round_id = str(outputs.get("round_id") or "") + ordered: list[dict[str, Any]] = [] + seen: set[str] = set() + for outcome in _dict_rows(outputs.get("per_variant_outcomes")): + fingerprint = str(outcome.get("fingerprint") or "") + merged = dict(_mapping(tested_by_fingerprint.get(fingerprint))) + merged.update(outcome) + key = fingerprint or str(outcome.get("variant_name") or len(ordered)) + seen.add(key) + ordered.append(merged) + for fingerprint, tested_row in tested_by_fingerprint.items(): + if not isinstance(tested_row, dict): + continue + if round_id and str(tested_row.get("round_id") or "") not in {"", round_id}: + continue + key = str(fingerprint or tested_row.get("name") or len(ordered)) + if key in seen: + continue + merged = dict(tested_row) + merged.setdefault("fingerprint", str(fingerprint)) + ordered.append(merged) + return ordered + + +def _config_round_from_operation( + operation: dict[str, Any], + source_index: dict[str, dict[str, Any]], +) -> dict[str, Any]: + outputs = _mapping(operation.get("outputs")) + inputs = _mapping(operation.get("inputs")) + last_round = _mapping(_nested(outputs, "explore_search_update", "last_round")) + input_stack = _mapping(_first(outputs.get("input_stack"), inputs.get("input_stack"))) + variants = [_config_variant(row, source_index) for row in _round_variant_rows(outputs)] + decision_mode = _first(outputs.get("decision_mode"), inputs.get("decision_mode")) + if decision_mode is None: + modes = { + str(_nested(row, "metrics", "overtime_anchor_kind") or row.get("overtime_anchor_kind") or "") + for row in _dict_rows(outputs.get("per_variant_outcomes")) + } + modes.discard("") + decision_mode = next(iter(modes)) if len(modes) == 1 else None + return { + "round_id": str( + _first( + outputs.get("round_id"), + _nested(operation, "metadata", "extras", "round_id"), + operation.get("operation_id"), + ) + or "" + ), + "task_id": _operation_task_id(operation), + "status": "succeeded" + if str(_first(outputs.get("status"), operation.get("status")) or "").lower() + in {"succeeded", "success", "completed", "kept", "keep"} + else "failed", + "framework": str( + _first(outputs.get("framework"), _nested(outputs, "workload", "framework"), inputs.get("framework")) or "" + ), + "workload_signature": str( + _first( + outputs.get("workload_signature"), + next( + ( + row.get("workload_signature") + for row in _mapping(_nested(outputs, "explore_search_update", "tested")).values() + if isinstance(row, dict) and row.get("workload_signature") + ), + None, + ), + ) + or "" + ), + "throughput_unit": _first(outputs.get("throughput_unit"), inputs.get("throughput_unit"), None), + "decision_mode": decision_mode, + "input_stack": { + "throughput": _optional_float( + _first(input_stack.get("throughput"), outputs.get("base_tput"), last_round.get("base_tput")) + ), + "accuracy": _optional_float( + _first(input_stack.get("accuracy"), outputs.get("accuracy_baseline"), inputs.get("accuracy_baseline")) + ), + "extra_server_args": str( + _first( + input_stack.get("extra_server_args"), + outputs.get("base_extra_args"), + last_round.get("base_extra_args"), + "", + ) + or "" + ), + "extra_envs": dict(_mapping(_first(input_stack.get("extra_envs"), outputs.get("base_extra_envs")))), + "remove_args": _string_list(_first(input_stack.get("remove_args"), outputs.get("base_remove_args"), [])), + "unset_envs": _string_list(_first(input_stack.get("unset_envs"), outputs.get("base_unset_envs"), [])), + "args_mode": _first(input_stack.get("args_mode"), outputs.get("base_args_mode"), None), + }, + "variants": variants, + } + + +def _fallback_config_rounds( + state: dict[str, Any], + window: dict[str, Any], + window_count: int, + source_index: dict[str, dict[str, Any]], + existing_ids: set[str], +) -> list[dict[str, Any]]: + explore_search = _mapping(state.get("explore_search")) + tested = explore_search.get("tested") + grouped: dict[str, list[dict[str, Any]]] = {} + if isinstance(tested, dict): + for fingerprint, row in tested.items(): + if not isinstance(row, dict) or not _row_in_window(row, window, window_count): + continue + round_id = str(row.get("round_id") or "") + if not round_id or round_id in existing_ids: + continue + item = dict(row) + item.setdefault("fingerprint", str(fingerprint)) + grouped.setdefault(round_id, []).append(item) + rounds: list[dict[str, Any]] = [] + for round_id, rows in grouped.items(): + measured_outcomes = {"KEEP", "REVERT", "KEEP_UNSTABLE", "KILLED_OVERTIME"} + rounds.append( + { + "round_id": round_id, + "task_id": "", + "status": ( + "succeeded" + if any(str(row.get("outcome") or "").upper() in measured_outcomes for row in rows) + else "failed" + ), + "framework": str(next((row.get("framework") for row in rows if row.get("framework")), "")), + "workload_signature": str( + next((row.get("workload_signature") for row in rows if row.get("workload_signature")), "") + ), + "throughput_unit": None, + "decision_mode": None, + "input_stack": { + "throughput": _optional_float(next((row.get("base_tput") for row in rows), None)), + "accuracy": None, + "extra_server_args": "", + "extra_envs": {}, + "remove_args": [], + "unset_envs": [], + "args_mode": None, + }, + "variants": [_config_variant(row, source_index) for row in rows], + } + ) + return rounds + + +def _config_arm( + state: dict[str, Any], + operations: list[dict[str, Any]], + specialist_rows: list[dict[str, Any]], + specialist_roles: dict[str, str], + window: dict[str, Any], + window_count: int, + evidence: dict[str, Any], + policy: dict[str, Any], +) -> dict[str, Any]: + config_rows = [ + row + for row in specialist_rows + if specialist_roles.get(str(_first(row.get("task_id"), row.get("round_id")) or "")) == "config" + ] + specialist_runs = [_config_specialist_run(row) for row in config_rows] + source_index = _config_source_index(config_rows) + rounds = [ + _config_round_from_operation(operation, source_index) + for operation in operations + if _operation_name(operation) == "explore" + ] + existing_ids = {str(row.get("round_id") or "") for row in rounds} + rounds.extend(_fallback_config_rounds(state, window, window_count, source_index, existing_ids)) + recorded_task_ids = {str(row.get("task_id") or "") for row in rounds} + for row in _window_state_rows(state, "framework_config_exploration_results", window, window_count): + task_id = str(row.get("task_id") or "") + round_id = str(_first(row.get("round_id"), task_id) or "") + if not round_id or task_id in recorded_task_ids or round_id in existing_ids: + continue + rounds.append( + { + "round_id": round_id, + "task_id": task_id, + "status": "succeeded" if str(row.get("status") or "succeeded").lower() != "failed" else "failed", + "framework": str(row.get("framework") or ""), + "workload_signature": str(row.get("workload_signature") or ""), + "throughput_unit": _first(row.get("throughput_unit"), None), + "decision_mode": _first(row.get("decision_mode"), None), + "input_stack": { + "throughput": None, + "accuracy": None, + "extra_server_args": "", + "extra_envs": {}, + "remove_args": [], + "unset_envs": [], + "args_mode": None, + }, + "variants": [], + } + ) + policy_config = _mapping(policy.get("config_arm")) + explore_search = _mapping(state.get("explore_search")) + tested_rows = [ + row for row in _dict_value_rows(explore_search.get("tested")) if _row_in_window(row, window, window_count) + ] + winner_rows = [ + row for row in _dict_rows(explore_search.get("winners_history")) if _row_in_window(row, window, window_count) + ] + recent_keep_gain = _optional_float(evidence.get("recent_keep_gain_pct")) + lookback = _optional_int(policy_config.get("lookback")) + if recent_keep_gain is None and lookback is not None and lookback > 0: + recent_keep_gain = round( + sum(_optional_float(row.get("gain_pct")) or 0.0 for row in winner_rows[-lookback:]), + 4, + ) + empty_streak = _optional_int(evidence.get("empty_streak")) + if empty_streak is None: + empty_streak = 0 + for row in reversed(specialist_runs): + if row["status"] != "empty": + break + empty_streak += 1 + tested_this_cycle = _optional_int(evidence.get("tested_this_cycle")) + if tested_this_cycle is None: + tested_this_cycle = len(tested_rows) + triggered_value = _optional_bool(evidence.get("config_arm_plateaued")) + if triggered_value is None: + keep_gain_threshold = _optional_float(policy_config.get("keep_gain_threshold_pct")) + empty_streak_threshold = _optional_int(policy_config.get("empty_streak_threshold")) + if recent_keep_gain is not None and keep_gain_threshold is not None and empty_streak_threshold is not None: + exhausted = empty_streak >= empty_streak_threshold or ( + not specialist_runs and tested_this_cycle >= empty_streak_threshold + ) + triggered_value = recent_keep_gain < keep_gain_threshold and exhausted + return { + "plateau": { + "triggered": triggered_value, + "recent_keep_gain_pct": recent_keep_gain, + "empty_streak": empty_streak, + "tested_this_cycle": tested_this_cycle, + }, + "specialist_runs": specialist_runs, + "rounds": rounds, + } + + +def _candidate_row(candidate: dict[str, Any]) -> dict[str, Any]: + audit = _mapping(candidate.get("audit")) + raw_verdict = str(_first(candidate.get("verdict"), audit.get("verdict"), candidate.get("applicability")) or "") + verdict = raw_verdict.strip().lower() + if verdict not in {"worth_a_bench", "already_present", "not_applicable"}: + verdict = "" + route = str(_first(candidate.get("route"), audit.get("recommended_next_step")) or "").strip() + if route in {"direct_apply", "direct"}: + route = "direct_framework" + if route not in {"direct_framework", "author_via_specialist"}: + route = "" + return { + "candidate_id": _candidate_id(candidate), + "source_ref": str( + _first(candidate.get("pr_url"), candidate.get("ref"), candidate.get("head_sha"), candidate.get("diff_url")) + or "" + ), + "repo": str( + _first(candidate.get("repo"), candidate.get("repo_url"), candidate.get("discovered_repo_url")) or "" + ), + "title": str(candidate.get("title") or ""), + "changed_files": _string_list(candidate.get("changed_files")), + "verdict": verdict, + "route": route, + "gap_canonical_id": _first(candidate.get("gap_canonical_id"), None), + "reason": _first(candidate.get("reason"), audit.get("reason"), candidate.get("rationale"), None), + } + + +def _candidate_index(state: dict[str, Any]) -> dict[str, dict[str, Any]]: + index: dict[str, dict[str, Any]] = {} + for batch in _dict_rows(state.get("framework_agent_batches")): + for candidate in _dict_rows(batch.get("candidates")): + candidate_id = _candidate_id(candidate) + if candidate_id: + index[candidate_id] = candidate + return index + + +def _candidate_discovery_runs( + state: dict[str, Any], + specialist_rows: list[dict[str, Any]], + specialist_roles: dict[str, str], + window: dict[str, Any], + window_count: int, +) -> list[dict[str, Any]]: + discovery_rows = [ + row + for row in specialist_rows + if specialist_roles.get(str(_first(row.get("task_id"), row.get("round_id")) or "")) == "discovery" + ] + used_tasks: set[str] = set() + timestamped_runs: list[tuple[str, dict[str, Any]]] = [] + + def _append_run(run: dict[str, Any], source: dict[str, Any]) -> None: + timestamped_runs.append((_row_timestamp(source), run)) + + for batch in _dict_rows(state.get("framework_agent_batches")): + batch_candidates = _dict_rows(batch.get("candidates")) + batch_ids = {_candidate_id(candidate) for candidate in batch_candidates if _candidate_id(candidate)} + task_id = str(batch.get("task_id") or "") + matched: dict[str, Any] | None = None + for row in discovery_rows: + row_task_id = str(row.get("task_id") or "") + proposal_ids = {_candidate_id(proposal) for proposal in _dict_rows(row.get("proposal_set"))} + if task_id and row_task_id == task_id: + matched = row + break + if row_task_id and str(batch.get("batch_id") or "").endswith(row_task_id[:8]): + matched = row + break + if batch_ids and proposal_ids.intersection(batch_ids): + matched = row + break + if matched is not None: + task_id = task_id or str(matched.get("task_id") or "") + used_tasks.add(str(matched.get("task_id") or "")) + elif not _row_in_window(batch, window, window_count): + continue + discovered_candidates = _dict_rows((matched or {}).get("proposal_set")) or batch_candidates + _append_run( + { + "task_id": task_id, + "status": _specialist_status(matched or batch) + if matched is not None + else ("succeeded" if batch_candidates else "empty"), + "batch_id": _first(batch.get("batch_id"), None), + "gap_canonical_id": _first( + batch.get("gap_canonical_id"), + (matched or {}).get("gap_canonical_id"), + None, + ), + "reason": _first(batch.get("reason"), (matched or {}).get("reason"), None), + "candidates": [_candidate_row(candidate) for candidate in discovered_candidates], + }, + matched or batch, + ) + for row in discovery_rows: + task_id = str(row.get("task_id") or "") + if task_id in used_tasks: + continue + _append_run( + { + "task_id": task_id, + "status": _specialist_status(row), + "batch_id": _first(row.get("batch_id"), None), + "gap_canonical_id": _first(row.get("gap_canonical_id"), None), + "reason": _first(row.get("reason"), row.get("error"), None), + "candidates": [_candidate_row(candidate) for candidate in _dict_rows(row.get("proposal_set"))], + }, + row, + ) + + failed_markers = 0 + terminal_retry_rows: list[dict[str, Any]] = [] + for row in _dict_rows(window.get("rows")): + evidence = _mapping(row.get("evidence")) + event = str(evidence.get("event") or "").strip() + reason = str(row.get("reason") or "").strip() + if event == "framework_agent_discover_failed": + failed_markers += 1 + _append_run( + { + "task_id": str(_first(evidence.get("task_id"), evidence.get("failed_task_id")) or ""), + "status": "failed", + "batch_id": _first(evidence.get("batch_id"), None), + "gap_canonical_id": _first(evidence.get("gap_canonical_id"), None), + "reason": _first(evidence.get("error"), reason, None), + "candidates": [], + }, + row, + ) + elif event == "framework_agent_phase_done" and reason == "discover_empty_payload": + _append_run( + { + "task_id": str(evidence.get("task_id") or ""), + "status": "empty", + "batch_id": _first(evidence.get("batch_id"), None), + "gap_canonical_id": _first(evidence.get("gap_canonical_id"), None), + "reason": reason, + "candidates": [], + }, + row, + ) + elif event == "framework_agent_phase_done" and reason == "discover_retries_exhausted": + terminal_retry_rows.append(row) + + if failed_markers == 0: + for row in terminal_retry_rows: + evidence = _mapping(row.get("evidence")) + _append_run( + { + "task_id": str(_first(evidence.get("task_id"), evidence.get("failed_task_id")) or ""), + "status": "failed", + "batch_id": _first(evidence.get("batch_id"), None), + "gap_canonical_id": _first(evidence.get("gap_canonical_id"), None), + "reason": str(row.get("reason") or "discover_retries_exhausted"), + "candidates": [], + }, + row, + ) + + if timestamped_runs and all(_timestamp_number(timestamp) is not None for timestamp, _ in timestamped_runs): + timestamped_runs.sort(key=lambda item: _timestamp_number(item[0]) or 0.0) + return [run for _, run in timestamped_runs] + + +def _patch_refs(row: dict[str, Any]) -> list[str]: + refs: list[str] = [] + for key in ("patch_refs", "patches_written", "patches", "artifacts_written"): + values = row.get(key) + if not isinstance(values, list): + continue + for value in values: + if isinstance(value, dict): + value = _first(value.get("path"), value.get("patch_path"), value.get("target"), value.get("rel_target")) + text = str(value or "").strip() + if text and text not in refs: + refs.append(text) + for proposal in _dict_rows(row.get("proposal_set")): + for value in _patch_refs(proposal): + if value not in refs: + refs.append(value) + return refs + + +def _authoring_runs( + state: dict[str, Any], + specialist_rows: list[dict[str, Any]], + specialist_roles: dict[str, str], + progress_rows: list[dict[str, Any]], +) -> list[dict[str, Any]]: + candidate_map = { + str(key): str(value) + for key, value in _mapping(state.get("framework_agent_specialist_candidate_map")).items() + if key and value + } + progress_by_task = { + str(row.get("specialist_task_id") or ""): row for row in progress_rows if row.get("specialist_task_id") + } + runs: list[dict[str, Any]] = [] + used_tasks: set[str] = set() + for row in specialist_rows: + key = str(_first(row.get("task_id"), row.get("round_id")) or "") + if specialist_roles.get(key) != "authoring": + continue + task_id = str(row.get("task_id") or "") + progress = progress_by_task.get(task_id, {}) + candidate_id = str( + _first( + row.get("candidate_id"), + row.get("framework_agent_candidate_id"), + candidate_map.get(task_id), + progress.get("candidate_id"), + ) + or "" + ) + reauthor_attempt = _optional_int( + _first( + row.get("reauthor_attempt"), + row.get("apply_retry_attempt"), + progress.get("reauthor_attempt"), + ) + ) + kind = ( + "reauthor" + if reauthor_attempt is not None and reauthor_attempt > 0 + else "local_authoring" + if candidate_id.startswith("local_explore:") or bool(row.get("framework_local_explore")) + else "candidate_authoring" + if candidate_id + else "" + ) + runs.append( + { + "task_id": task_id, + "candidate_id": candidate_id, + "kind": kind, + "status": _specialist_status(row), + "reauthor_attempt": reauthor_attempt, + "specialist_domain": str(row.get("domain") or ""), + "gap_canonical_id": _first(row.get("gap_canonical_id"), None), + "patch_refs": _patch_refs(row), + "reason": _first(row.get("reason"), row.get("error"), progress.get("rationale"), None), + } + ) + if task_id: + used_tasks.add(task_id) + for progress in progress_rows: + task_id = str(progress.get("specialist_task_id") or "") + if not task_id or task_id in used_tasks: + continue + candidate_id = str(progress.get("candidate_id") or candidate_map.get(task_id) or "") + runs.append( + { + "task_id": task_id, + "candidate_id": candidate_id, + "kind": "local_authoring" if candidate_id.startswith("local_explore:") else "candidate_authoring", + "status": "failed" + if str(progress.get("status") or "").lower() in {"dispatch_failed", "author_failed", "recovery_failed"} + else "empty" + if str(progress.get("provenance") or "").lower() == "authored_empty" + else "succeeded", + "reauthor_attempt": _optional_int(progress.get("reauthor_attempt")), + "specialist_domain": str(progress.get("domain") or ""), + "gap_canonical_id": _first(progress.get("gap_canonical_id"), None), + "patch_refs": _patch_refs(progress), + "reason": _first(progress.get("rationale"), progress.get("error"), None), + } + ) + return runs + + +def _source_attempt_status(value: Any, *, kept: Any = None) -> str: + status = str(value or "").strip().lower() + if status in {"keep", "kept", "promoted", "adopted"} or kept is True: + return "KEEP" + if status in {"kept_inert", "keep_inert"}: + return "KEEP_INERT" + if status in {"revert", "reverted", "rejected", "accuracy_unavailable_reject"}: + return "REVERT" + if status in {"critic_denied", "rejected_by_critic", "needs_review_no_evidence", "reauthor_cap"}: + return "CRITIC_DENIED" + if status in {"skipped", "already_present", "not_applicable", "author_empty", "no_patch", "no_patches"}: + return "SKIPPED" + if status in { + "failed", + "error", + "apply_failed", + "bench_reverted", + "enqueue_failed", + "dispatch_failed", + "materialize_failed", + "no_result_failed", + "apply_fail_cap", + "recovery_failed", + "repeated_review_abort", + }: + return "FAILED" + return "" + + +def _source_attempt( + operation: dict[str, Any] | None, + progress: dict[str, Any], + candidate_index: dict[str, dict[str, Any]], +) -> dict[str, Any]: + operation = operation or {} + outputs = _mapping(operation.get("outputs")) + inputs = _mapping(operation.get("inputs")) + candidate = _mapping(_first(outputs.get("candidate"), inputs.get("candidate"))) + candidate_id = str( + _first( + _candidate_id(candidate), + outputs.get("framework_agent_candidate_id"), + inputs.get("framework_agent_candidate_id"), + _nested(operation, "metadata", "extras", "candidate_id"), + progress.get("candidate_id"), + ) + or "" + ) + known_candidate = candidate_index.get(candidate_id, {}) + if not candidate: + candidate = known_candidate + task_id = str(_first(_operation_task_id(operation), progress.get("integrate_task_id")) or "") + source_task_id = str( + _first(outputs.get("specialist_task_id"), inputs.get("specialist_task_id"), progress.get("specialist_task_id")) + or "" + ) + reauthor_attempt = _optional_int( + _first(outputs.get("reauthor_attempt"), inputs.get("reauthor_attempt"), progress.get("reauthor_attempt")) + ) + route = str( + _first( + outputs.get("route"), + inputs.get("route"), + candidate.get("route"), + known_candidate.get("route"), + ) + or "" + ).strip() + if route in {"direct", "direct_apply"}: + route = "direct_framework" + patch_source = str(_first(outputs.get("patch_source"), inputs.get("patch_source")) or "").strip().lower() + if patch_source not in {"specialist_authored", "upstream_pr"}: + patch_source = "" + provenance = str(_first(outputs.get("provenance"), inputs.get("provenance")) or "").strip().lower() + if not patch_source and provenance == "upstream_pr": + patch_source = "upstream_pr" + authoring_marker = _optional_bool( + _first(outputs.get("framework_agent_authoring"), inputs.get("framework_agent_authoring")) + ) + if not patch_source and (route == "direct_framework" or source_task_id and task_id and source_task_id == task_id): + patch_source = "upstream_pr" + if not patch_source and ( + route in {"author_via_specialist", "local_authoring", "reauthor"} + or source_task_id + and source_task_id != task_id + ): + patch_source = "specialist_authored" + if not patch_source and _operation_name(operation) == "framework_agent": + patch_source = "upstream_pr" + if not patch_source and authoring_marker is True and not source_task_id: + patch_source = "specialist_authored" + lever_kind = str(_first(outputs.get("lever_kind"), inputs.get("lever_kind")) or "").strip().lower() + if not lever_kind: + lever_kind = "upstream_pr" if patch_source == "upstream_pr" else "source_patch" if patch_source else "" + if lever_kind not in {"config", "source_patch", "upstream_pr", "enablement"}: + lever_kind = "" + if not route: + if reauthor_attempt and reauthor_attempt > 0: + route = "reauthor" + elif candidate_id.startswith("local_explore:"): + route = "local_authoring" + elif patch_source == "upstream_pr": + route = "direct_framework" + elif patch_source == "specialist_authored": + route = "author_via_specialist" + parity = _mapping(outputs.get("switch_off_parity")) + stack_rebench = _mapping(outputs.get("stack_rebench")) + files = _string_list(_first(outputs.get("target_files"), candidate.get("changed_files"), [])) + applied_artifacts = _dict_rows(outputs.get("artifacts_applied")) + if not files: + files = [ + str(_first(row.get("rel_target"), row.get("target")) or "") + for row in applied_artifacts + if _first(row.get("rel_target"), row.get("target")) + ] + raw_status = _first(outputs.get("status"), progress.get("status"), operation.get("status")) + return { + "attempt_id": str(_first(operation.get("operation_id"), task_id, candidate_id) or ""), + "task_id": task_id or None, + "candidate_id": candidate_id or None, + "source_task_id": source_task_id or None, + "patch_source": patch_source or None, + "lever_kind": lever_kind or None, + "route": route or None, + "status": _source_attempt_status(raw_status, kept=progress.get("kept")), + "before_tput": _optional_float(_first(outputs.get("base_tput"), progress.get("pre_tput"))), + "after_tput": _optional_float(_first(outputs.get("output_throughput"), progress.get("post_tput"))), + "local_gain_pct": _optional_float( + _first(outputs.get("delta_pct"), outputs.get("gain_pct"), progress.get("gain_pct")) + ), + "ts": str(_first(operation.get("ended_at"), progress.get("ts")) or ""), + "source_ref": _first( + candidate.get("pr_url"), + candidate.get("ref"), + candidate.get("head_sha"), + candidate.get("diff_url"), + None, + ), + "files": files, + "reason": _first(outputs.get("reason"), progress.get("rationale"), outputs.get("error"), None), + "gates": { + "accuracy_passed": _optional_bool(outputs.get("accuracy_pass")), + "keep_threshold_pct": _optional_float(outputs.get("keep_threshold_pct")), + "switch_off_parity_passed": _optional_bool(_first(parity.get("ok"), parity.get("passed"))), + "stack_rebench_passed": _optional_bool(_first(stack_rebench.get("stable"), stack_rebench.get("ok"))), + }, + "framework_levers": _dict_rows(outputs.get("framework_levers")), + "config_delta": { + "extra_server_args": str(outputs.get("extra_server_args_applied") or ""), + "extra_envs": dict( + _mapping(_first(outputs.get("extra_envs_applied"), outputs.get("config_changes_applied"))) + ), + }, + "artifacts": { + "patches_applied": _string_list(outputs.get("patches_applied")), + "source_snapshot": _first(outputs.get("source_snapshot"), None), + "source_manifest": _first(outputs.get("source_manifest"), None), + "workspace": _first(outputs.get("workspace"), None), + }, + "failure": { + "error_class": _first(outputs.get("error_class"), progress.get("error_class"), None), + "error": _first(outputs.get("error"), progress.get("error"), None), + }, + } + + +def _source_attempts( + state: dict[str, Any], + operations: list[dict[str, Any]], + progress_rows: list[dict[str, Any]], +) -> list[dict[str, Any]]: + candidates = _candidate_index(state) + progress_by_integrate = { + str(row.get("integrate_task_id") or ""): row for row in progress_rows if row.get("integrate_task_id") + } + progress_by_candidate = { + str(row.get("candidate_id") or ""): row for row in progress_rows if row.get("candidate_id") + } + progress_by_source = { + str(row.get("specialist_task_id") or ""): row for row in progress_rows if row.get("specialist_task_id") + } + used_progress: set[int] = set() + attempts: list[dict[str, Any]] = [] + for operation in operations: + if _operation_name(operation) not in {"framework_agent", "integrate", "integrate_patch"}: + continue + outputs = _mapping(operation.get("outputs")) + candidate_id = str( + _first( + _candidate_id(outputs.get("candidate")), + outputs.get("framework_agent_candidate_id"), + _nested(operation, "metadata", "extras", "candidate_id"), + ) + or "" + ) + task_id = _operation_task_id(operation) + source_task_id = str(outputs.get("specialist_task_id") or "") + progress = ( + progress_by_integrate.get(task_id) + or progress_by_candidate.get(candidate_id) + or progress_by_source.get(source_task_id) + or {} + ) + if progress: + used_progress.add(id(progress)) + attempts.append(_source_attempt(operation, progress, candidates)) + for progress in progress_rows: + if id(progress) in used_progress or str(progress.get("status") or "").lower() == "cycle_boundary": + continue + attempts.append(_source_attempt(None, progress, candidates)) + attempts.sort(key=lambda row: row.get("ts") or "") + return attempts + + +def _critic_cycle_indexes( + session_dir: Path, + warnings: list[str], + state: dict[str, Any], + operations: list[dict[str, Any]], +) -> tuple[dict[str, int], dict[str, int]]: + task_cycles: dict[str, int] = {} + proposal_cycles: dict[str, int] = {} + candidate_cycles: dict[str, int] = {} + + def _index_row(row: dict[str, Any]) -> None: + cycle = _row_cycle(row) + if cycle is None: + return + task_id = str( + _first( + _operation_task_id(row), + row.get("task_id"), + row.get("integrate_task_id"), + row.get("specialist_task_id"), + ) + or "" + ) + if task_id: + task_cycles.setdefault(task_id, cycle) + proposal_id = str( + _first( + row.get("proposal_msg_id"), + _nested(row, "inputs", "proposal_msg_id"), + _nested(row, "outputs", "proposal_msg_id"), + _nested(row, "metadata", "extras", "proposal_msg_id"), + ) + or "" + ) + if proposal_id: + proposal_cycles.setdefault(proposal_id, cycle) + candidate_id = str( + _first( + row.get("candidate_id"), + row.get("framework_agent_candidate_id"), + _nested(row, "inputs", "framework_agent_candidate_id"), + _nested(row, "outputs", "framework_agent_candidate_id"), + _candidate_id(_nested(row, "inputs", "candidate")), + _candidate_id(_nested(row, "outputs", "candidate")), + ) + or "" + ) + if candidate_id: + candidate_cycles.setdefault(candidate_id, cycle) + + for operation in operations: + _index_row(operation) + for field in ( + "framework_agent_phase_progress", + "framework_agent_batches", + "specialist_rounds", + ): + for row in _dict_rows(state.get(field)): + _index_row(row) + cycle = _row_cycle(row) + if cycle is None: + continue + for candidate in _dict_rows(row.get("candidates")): + candidate_id = _candidate_id(candidate) + if candidate_id: + candidate_cycles.setdefault(candidate_id, cycle) + + map_path = session_dir / "reports" / "trace" / "proposal_task_map.jsonl" + if map_path.is_file(): + map_rows = read_jsonl( + map_path, + require_dict=True, + skip_malformed=True, + on_error=lambda exc: warnings.append( + f"timeline.framework_agent.critic: failed to parse {map_path}: {exc!r}" + ), + ) + for row in map_rows: + proposal_id = str(row.get("proposal_msg_id") or "") + task_id = str(row.get("task_id") or "") + cycle = task_cycles.get(task_id) + if proposal_id and cycle is not None: + proposal_cycles.setdefault(proposal_id, cycle) + return proposal_cycles, candidate_cycles + + +def _critic_review_rows( + session_dir: Path, + warnings: list[str], + state: dict[str, Any], + operations: list[dict[str, Any]], + window: dict[str, Any], + window_count: int, +) -> list[dict[str, Any]]: + root = session_dir / "critic-workdir" + if not root.is_dir(): + return [] + proposal_cycles, candidate_cycles = _critic_cycle_indexes(session_dir, warnings, state, operations) + rows: list[dict[str, Any]] = [] + for workdir in sorted(root.iterdir(), key=lambda path: path.name): + if not workdir.is_dir(): + continue + loaded: dict[str, dict[str, Any]] = {} + failed = False + for name in ("request", "judge_bundle", "review", "emit"): + path = workdir / f"{name}.json" + if not path.is_file(): + loaded[name] = {} + continue + try: + loaded[name] = read_json(path, require_dict=True, strict=True) + except Exception as exc: + warnings.append(f"timeline.framework_agent.critic: failed to parse {path}: {exc!r}") + failed = True + break + if failed: + continue + request = loaded["request"] + judge = loaded["judge_bundle"] + review = loaded["review"] + emit = loaded["emit"] + review_phase = ( + str( + _first( + judge.get("phase"), + _nested(request, "context", "phase"), + _nested(judge, "merged_context", "phase"), + ) + or "" + ) + .strip() + .upper() + ) + if review_phase and review_phase not in _FRAMEWORK_PHASES: + continue + proposals = { + str(proposal.get("msg_id") or ""): proposal + for proposal in _dict_rows(judge.get("proposals")) + if proposal.get("msg_id") + } + effective = {} + envelope = _mapping(emit.get("intent_envelope")) + for intent in _dict_rows(envelope.get("intents")): + if str(intent.get("intent_type") or "") != "review_verdict": + continue + payload = _mapping(intent.get("payload")) + target = str(payload.get("target_proposal_msg_id") or "") + if target: + effective[target] = payload + for verdict_row in _dict_rows(review.get("review_verdicts")): + proposal_id = str(verdict_row.get("target_proposal_msg_id") or "") + proposal = proposals.get(proposal_id, {}) + payload = _mapping(proposal.get("payload")) + params = _mapping(payload.get("params")) + candidate = _mapping(_first(payload.get("candidate"), params.get("candidate"))) + action = str( + _first(proposal.get("action_name"), payload.get("action_name"), payload.get("kind")) or "" + ).lower() + candidate_id = str( + _first( + payload.get("framework_agent_candidate_id"), + params.get("framework_agent_candidate_id"), + _candidate_id(candidate), + ) + or "" + ) + if action in {"params", "backends", "explore"}: + arm = "config" + target_action = "explore" + elif action in {"framework_agent", "integrate", "integrate_patch"}: + arm = "source" + target_action = "integrate_patch" + elif action == "specialist": + task_kind = str(params.get("task_kind") or "").strip().lower() + source_marker = bool( + candidate_id + or _optional_bool(params.get("framework_agent_authoring")) is True + or _optional_bool(params.get("candidate_discovery")) is True + or task_kind in _AUTHORING_TASK_KINDS + or bool(_patch_refs(params)) + ) + arm = "source" if source_marker else "config" + target_action = "specialist" + else: + continue + cycle_row = { + "cycle": _first( + payload.get("cycle"), + payload.get("macro_cycle"), + params.get("cycle"), + params.get("macro_cycle"), + proposal.get("cycle"), + proposal.get("macro_cycle"), + proposal_cycles.get(proposal_id), + candidate_cycles.get(candidate_id), + _nested(request, "context", "macro_cycle"), + _nested(request, "context", "cycle"), + _prompt_macro_cycle(request.get("raw_prompt")), + _nested(judge, "merged_context", "macro_cycle"), + _nested(judge, "merged_context", "cycle"), + ), + "ts": _first(verdict_row.get("ts"), emit.get("ts"), review.get("ts")), + } + if not _row_in_window(cycle_row, window, window_count): + continue + effective_row = _mapping(effective.get(proposal_id)) + risks = [ + { + "severity": str(risk.get("severity") or ""), + "risk": str(_first(risk.get("risk"), risk.get("summary"), risk.get("reason")) or ""), + } + for risk in _dict_rows(verdict_row.get("risks")) + ] + followup_task_ids = _string_list( + _first(effective_row.get("followup_task_ids"), verdict_row.get("followup_task_ids"), []) + ) + rows.append( + { + "proposal_msg_id": proposal_id, + "candidate_id": candidate_id or None, + "variant_name": _first(payload.get("variant_name"), params.get("variant_name"), None), + "arm": arm, + "target_action": target_action, + "source": "critic" + if str(verdict_row.get("source") or "critic") == "critic" + else "critic_unavailable", + "verdict": str(verdict_row.get("verdict") or ""), + "effective_verdict": str( + _first( + effective_row.get("verdict"), + verdict_row.get("effective_verdict"), + verdict_row.get("verdict"), + ) + or "" + ), + "reasoning": str(verdict_row.get("reasoning") or ""), + "confidence": _first(verdict_row.get("confidence"), None), + "failure_reason_code": _first(verdict_row.get("failure_reason_code"), None), + "required_evidence": _string_list(verdict_row.get("required_evidence")), + "risks": risks, + "advice_text": _first(verdict_row.get("advice_text"), effective_row.get("advice_text"), None), + "alternative_action": _first(verdict_row.get("alternative_action"), None), + "followup_task_ids": followup_task_ids, + "ts": str(_first(verdict_row.get("ts"), emit.get("ts"), review.get("ts")) or ""), + "review_path": (workdir / "review.json").relative_to(session_dir).as_posix(), + } + ) + rows.sort(key=lambda row: row.get("ts") or "") + return rows + + +def _source_arm( + state: dict[str, Any], + operations: list[dict[str, Any]], + specialist_rows: list[dict[str, Any]], + specialist_roles: dict[str, str], + progress_rows: list[dict[str, Any]], + window: dict[str, Any], + window_count: int, + evidence: dict[str, Any], +) -> dict[str, Any]: + consecutive_no_keep = _optional_int( + _first(evidence.get("source_consecutive_no_keep"), evidence.get("consecutive_no_keep")) + ) + if consecutive_no_keep is None: + consecutive_no_keep = 0 + for row in reversed(progress_rows): + if str(row.get("status") or "").lower() == "cycle_boundary": + break + if bool(row.get("kept")) or str(row.get("status") or "").lower() == "kept": + break + if str(row.get("status") or "").lower() == "dispatch_failed": + continue + consecutive_no_keep += 1 + candidates_exhausted = _optional_bool( + _first(evidence.get("source_candidates_exhausted"), evidence.get("candidates_exhausted")) + ) + if candidates_exhausted is None: + state_cycle = _optional_int(state.get("macro_cycle")) + if window_count == 1 or state_cycle == int(window["cycle"]): + candidates_exhausted = _optional_bool(state.get("framework_agent_phase_done")) + triggered = _optional_bool(evidence.get("source_arm_plateaued")) + if triggered is None: + threshold = _optional_int(_first(evidence.get("source_threshold"), evidence.get("threshold"))) + if candidates_exhausted is True: + triggered = True + elif threshold is not None: + triggered = consecutive_no_keep >= threshold + return { + "plateau": { + "triggered": triggered, + "consecutive_no_keep": consecutive_no_keep, + "candidates_exhausted": candidates_exhausted, + }, + "candidate_discovery_runs": _candidate_discovery_runs( + state, + specialist_rows, + specialist_roles, + window, + window_count, + ), + "authoring_runs": _authoring_runs(state, specialist_rows, specialist_roles, progress_rows), + "attempts": _source_attempts(state, operations, progress_rows), + } + + +def _framework_exit( + window: dict[str, Any], + config_arm: dict[str, Any], + source_arm: dict[str, Any], +) -> dict[str, Any]: + exit_row = _mapping(window.get("exit_row")) + evidence = _mapping(exit_row.get("evidence")) + raw_reason = str(_first(evidence.get("passed_through_reason"), exit_row.get("reason")) or "") + reason = _FRAMEWORK_EXIT_REASON_MAP.get(raw_reason, raw_reason) or None + trigger = _first(evidence.get("trigger"), evidence.get("evidence"), None) + if trigger == "phase_budget_cap": + trigger = "budget_cap" + if trigger is None: + trigger = { + "optimize_no_more_leverage": "both_arms_plateaued", + "optimize_phase_budget_exhausted": "phase_budget_exhausted", + "optimize_budget_cap": "budget_cap", + "optimize_force_exit_low_budget": "force_exit", + }.get(str(reason or "")) + switch_bottleneck = _optional_bool(evidence.get("switch_bottleneck")) + if switch_bottleneck is None: + plateau_values = ( + _nested(config_arm, "plateau", "triggered"), + _nested(source_arm, "plateau", "triggered"), + ) + if any(value is True for value in plateau_values): + switch_bottleneck = True + elif all(value is False for value in plateau_values): + switch_bottleneck = False + return { + "reason": reason, + "trigger": trigger, + "hint": _first(evidence.get("hint"), None), + "switch_bottleneck": switch_bottleneck, + } + + +def _framework_failure(window: dict[str, Any]) -> dict[str, Any]: + exit_row = _mapping(window.get("exit_row")) + evidence = _mapping(exit_row.get("evidence")) + reason = str(exit_row.get("reason") or "").lower() + is_failure = bool(evidence.get("error") or evidence.get("error_class") or reason.endswith("_failed")) + if is_failure: + return { + "failed_task_id": _first(evidence.get("failed_task_id"), evidence.get("task_id"), None), + "error_class": _first(evidence.get("error_class"), None), + "error": _first(evidence.get("error"), None), + } + + rows = _dict_rows(window.get("rows")) + terminal_discovery_failure = next( + ( + row + for row in reversed(rows) + if _nested(row, "evidence", "event") == "framework_agent_phase_done" + and str(row.get("reason") or "") == "discover_retries_exhausted" + ), + None, + ) + if terminal_discovery_failure is None: + return {"failed_task_id": None, "error_class": None, "error": None} + + failed_discovery = next( + (row for row in reversed(rows) if _nested(row, "evidence", "event") == "framework_agent_discover_failed"), + terminal_discovery_failure, + ) + failure_evidence = _mapping(failed_discovery.get("evidence")) + return { + "failed_task_id": _first( + failure_evidence.get("failed_task_id"), + failure_evidence.get("task_id"), + None, + ), + "error_class": _first(failure_evidence.get("error_class"), None), + "error": _first( + failure_evidence.get("error"), + terminal_discovery_failure.get("reason"), + None, + ), + } + + +def _framework_event( + session_dir: Path, + state: dict[str, Any], + recorded_operations: list[dict[str, Any]], + warnings: list[str], + window: dict[str, Any], + window_count: int, +) -> dict[str, Any]: + operations = _window_operations(recorded_operations, window, window_count) + progress_rows = _window_state_rows(state, "framework_agent_phase_progress", window, window_count) + candidate_map = { + str(key): str(value) + for key, value in _mapping(state.get("framework_agent_specialist_candidate_map")).items() + if key and value + } + source_task_ids = { + str(row.get("specialist_task_id") or "") for row in progress_rows if row.get("specialist_task_id") + } + specialist_rows = _specialist_rows(state, operations, window, window_count) + specialist_roles = { + str(_first(row.get("task_id"), row.get("round_id")) or ""): _specialist_role( + row, + candidate_map, + source_task_ids, + ) + for row in specialist_rows + } + evidence = _window_evidence(window) + policy = _framework_policy(state, operations, evidence) + config_arm = _config_arm( + state, + operations, + specialist_rows, + specialist_roles, + window, + window_count, + evidence, + policy, + ) + source_arm = _source_arm( + state, + operations, + specialist_rows, + specialist_roles, + progress_rows, + window, + window_count, + evidence, + ) + exit_details = _framework_exit(window, config_arm, source_arm) + failure = _framework_failure(window) + has_work = bool( + config_arm["specialist_runs"] + or config_arm["rounds"] + or any(run["candidates"] for run in source_arm["candidate_discovery_runs"]) + or source_arm["authoring_runs"] + or source_arm["attempts"] + ) + if any(value is not None for value in failure.values()): + status = "failed" + elif not window.get("end_time"): + status = "degraded" + elif has_work: + status = "succeeded" + else: + status = "skipped" + return { + "type": "framework_agent", + "kind": "agent", + "status": status, + "start_time": str(window.get("start_time") or ""), + "end_time": str(window.get("end_time") or ""), + "ext": { + "macro_cycle": int(window["cycle"]), + "policy": policy, + "critic_reviews": _critic_review_rows( + session_dir, + warnings, + state, + recorded_operations, + window, + window_count, + ), + "config_arm": config_arm, + "source_arm": source_arm, + "exit": exit_details, + "failure": failure, + }, + } + + def collect_v6_timeline( session_dir: Path, warnings: list[str], @@ -147,9 +2030,31 @@ def collect_v6_timeline( state: dict[str, Any] | None = None, recorded_operations: list[dict[str, Any]] | None = None, ) -> list[dict[str, Any]]: - """Load durable install and model-gate events without mutating V5 state.""" - del state, recorded_operations - return read_timeline_events(session_dir, warnings=warnings) + """Load durable events and project framework work without mutating V5 state.""" + timeline = read_timeline_events(session_dir, warnings=warnings) + state = state if isinstance(state, dict) else {} + operations = [row for row in recorded_operations or [] if isinstance(row, dict)] + windows = _framework_windows(state, operations) + for window in windows: + timeline.append( + _framework_event( + session_dir, + state, + operations, + warnings, + window, + len(windows), + ) + ) + indexed = list(enumerate(timeline)) + indexed.sort( + key=lambda row: ( + _timestamp_number(_first(row[1].get("start_time"), row[1].get("end_time"))) is None, + _timestamp_number(_first(row[1].get("start_time"), row[1].get("end_time"))) or 0.0, + row[0], + ) + ) + return [event for _, event in indexed] def _outcome_status(stop_reason: str) -> str: diff --git a/src/hyperloom/inference_optimizer/breakdown/recorder/instrument.py b/src/hyperloom/inference_optimizer/breakdown/recorder/instrument.py index 4db9a25f06..14c4ac0820 100644 --- a/src/hyperloom/inference_optimizer/breakdown/recorder/instrument.py +++ b/src/hyperloom/inference_optimizer/breakdown/recorder/instrument.py @@ -3662,6 +3662,7 @@ def record_specialist_round( domains.append(str(entry.get("domain"))) domains.extend(str(tag) for tag in (entry.get("tags") or []) if str(tag)) domains = list(dict.fromkeys(domain for domain in domains if domain)) + source_phase = str(entry.get("source_phase") or "EXPLORE").strip().upper() record_subject( session_dir, subject_id=round_subject_id, @@ -3713,7 +3714,7 @@ def record_specialist_round( root_operation_id=operation_id, kind="specialist", name=f"specialist round {round_id}", - phase=phase or str(entry.get("phase") or ""), + phase=source_phase, status="succeeded" if entry.get("completed_at") else "partial", source="specialist_recorder_hook", executor_class="llm_agent", diff --git a/src/hyperloom/inference_optimizer/tests/test_enablement_coordinator_wiring_unit.py b/src/hyperloom/inference_optimizer/tests/test_enablement_coordinator_wiring_unit.py index 68c880f76b..baac53b5f9 100644 --- a/src/hyperloom/inference_optimizer/tests/test_enablement_coordinator_wiring_unit.py +++ b/src/hyperloom/inference_optimizer/tests/test_enablement_coordinator_wiring_unit.py @@ -92,6 +92,7 @@ def test_build_params_actionable_failure_tags_enablement(monkeypatch): params = Coordinator._build_enablement_specialist_params(fake, _MISSING_ARCH_LOG) assert params is not None assert params["domain"] == "enablement_specialist" + assert params["source_phase"] == "ENABLEMENT" # Reuses FRAMEWORK authoring machinery + tags the objective. assert params["framework_agent_authoring"] is True assert params["enablement"] is True diff --git a/src/hyperloom/inference_optimizer/tests/test_framework_agent_authoring.py b/src/hyperloom/inference_optimizer/tests/test_framework_agent_authoring.py index 34a26dc381..c08732bf9d 100644 --- a/src/hyperloom/inference_optimizer/tests/test_framework_agent_authoring.py +++ b/src/hyperloom/inference_optimizer/tests/test_framework_agent_authoring.py @@ -226,6 +226,7 @@ def test_materialize_unknown_route_dispatches_both_tracks( params = spec["params"] assert params["framework_agent_authoring"] is True assert params["domain"] == "serving_specialist" + assert params["source_phase"] == "FRAMEWORK_AGENT" assert params["framework_agent_candidate_id"] == _CANDIDATE["pr_url"] assert params.get("task_kind") == "framework_authoring" pr_lead = params.get("pr_lead") or {} diff --git a/src/hyperloom/inference_optimizer/tests/test_research_scout.py b/src/hyperloom/inference_optimizer/tests/test_research_scout.py index d90bfd8fc4..e05342813c 100644 --- a/src/hyperloom/inference_optimizer/tests/test_research_scout.py +++ b/src/hyperloom/inference_optimizer/tests/test_research_scout.py @@ -161,6 +161,7 @@ async def test_internal_research_scout_task_is_readonly(tmp_path: Path): assert task is not None assert task.params["mode"] == "research" + assert task.params["source_phase"] == "PRELUDE" assert task.side_effects == ["writes_results"] assert task.params["seen_pr_ids"] == ["https://pr/seen"] assert "Does this vLLM version support the backend?" in task.params["notes"] diff --git a/src/hyperloom/inference_optimizer/tests/test_sbd_v6_initial.py b/src/hyperloom/inference_optimizer/tests/test_sbd_v6_initial.py index ba2941f2e7..a262d8376e 100644 --- a/src/hyperloom/inference_optimizer/tests/test_sbd_v6_initial.py +++ b/src/hyperloom/inference_optimizer/tests/test_sbd_v6_initial.py @@ -842,3 +842,968 @@ def test_fresh_model_gate_with_only_soft_skips_succeeds(tmp_path): assert event is not None assert event["status"] == "succeeded" assert event["ext"]["skip_reason"] is None + + +def test_framework_timeline_merges_legacy_framework_and_explore(tmp_path): + state = { + "phase": "KERNEL_AGENT", + "macro_cycle": 0, + "framework_agent_phase_done": True, + "phase_history": [ + { + "from_phase": "PRELUDE", + "to_phase": "FRAMEWORK_AGENT", + "reason": "prelude_complete", + "ts": "2026-08-27T01:00:00+00:00", + "cycle": 0, + }, + { + "from_phase": "FRAMEWORK_AGENT", + "to_phase": "EXPLORE", + "reason": "framework_agent_phase_done", + "ts": "2026-08-27T01:10:00+00:00", + "cycle": 0, + }, + { + "from_phase": "EXPLORE", + "to_phase": "KERNEL_AGENT", + "reason": "explore_no_more_leverage", + "ts": "2026-08-27T01:20:00+00:00", + "cycle": 0, + "evidence": { + "recent_keep_gain_pct": 5.0, + "keep_gain_threshold_pct": 6.0, + "empty_streak": 2, + "empty_streak_threshold": 2, + "lookback": 6, + "tested_this_cycle": 1, + "config_arm_plateaued": True, + "source_consecutive_no_keep": 1, + "source_threshold": 3, + "source_candidates_exhausted": True, + "source_arm_plateaued": True, + "switch_bottleneck": True, + "evidence": "both_arms_plateaued", + }, + }, + ], + "specialist_rounds": [ + { + "round_id": "spec-config-1", + "task_id": "spec-config-task", + "domain": "serving_specialist", + "cycle": 0, + "completed_at": "2026-08-27T01:04:00+00:00", + "proposal_set": [ + { + "name": "chunked-prefill", + "fingerprint": "fp-config-1", + } + ], + } + ], + "explore_search": { + "tested": { + "fp-config-1": { + "fingerprint": "fp-config-1", + "name": "chunked-prefill", + "outcome": "KEEP", + "tput": 105.0, + "base_tput": 100.0, + "gain_pct": 5.0, + "round_id": "config-round-1", + "cycle": 0, + "workload_signature": "qwen-tp8-c64", + "framework": "sglang", + "stack_rebench_tput": 104.0, + "stack_rebench_workspace": "runs/config-round-1/rebench", + } + }, + "winners_history": [{"gain_pct": 5.0, "cycle": 0}], + }, + "framework_agent_batches": [ + { + "batch_id": "legacy-batch", + "candidates": [ + { + "pr_url": "https://example.test/pr/7", + "route": "direct_framework", + "audit": {"verdict": "worth_a_bench"}, + } + ], + } + ], + "framework_agent_phase_progress": [ + { + "candidate_id": "https://example.test/pr/7", + "status": "kept", + "kept": True, + "pre_tput": 105.0, + "post_tput": 108.0, + "gain_pct": 2.857, + "cycle": 0, + "ts": "2026-08-27T01:08:00+00:00", + } + ], + } + operations = [ + { + "operation_id": "op-source-1", + "name": "framework_agent", + "phase": "FRAMEWORK_AGENT", + "macro_cycle": 0, + "status": "succeeded", + "ended_at": "2026-08-27T01:08:00+00:00", + "outputs": { + "status": "kept", + "candidate": { + "pr_url": "https://example.test/pr/7", + "route": "direct_framework", + "changed_files": ["python/server.py"], + }, + "base_tput": 105.0, + "output_throughput": 108.0, + "delta_pct": 2.857, + "accuracy_pass": True, + "keep_threshold_pct": 1.0, + "patches_applied": ["patches/pr-7.patch"], + "target_files": ["python/server.py"], + "workspace": "runs/framework/pr-7", + }, + "extensions": {"task_id": "source-task-1"}, + }, + { + "operation_id": "op-config-1", + "name": "explore", + "phase": "EXPLORE", + "macro_cycle": 0, + "status": "succeeded", + "ended_at": "2026-08-27T01:16:00+00:00", + "outputs": { + "status": "succeeded", + "round_id": "config-round-1", + "framework": "sglang", + "base_tput": 100.0, + "per_variant_outcomes": [ + { + "variant_name": "chunked-prefill", + "outcome": "KEEP", + "fingerprint": "fp-config-1", + "provenance": "specialist:serving_specialist", + "scope": "domain", + "metrics": {"tput": 105.0, "gain_pct": 5.0}, + "variant": { + "extra_server_args": "--enable-chunked-prefill", + "extra_envs": {"SGLANG_CHUNKED_PREFILL": "1"}, + }, + } + ], + "explore_search_update": { + "tested": state["explore_search"]["tested"], + "last_round": { + "round_id": "config-round-1", + "base_tput": 100.0, + "base_extra_args": "--base-flag", + }, + }, + }, + "extensions": {"task_id": "config-task-1"}, + }, + ] + + timeline = collect_v6_timeline(tmp_path, [], state=state, recorded_operations=operations) + + assert [event["type"] for event in timeline] == ["framework_agent"] + event = timeline[0] + assert event["start_time"] == "2026-08-27T01:00:00+00:00" + assert event["end_time"] == "2026-08-27T01:20:00+00:00" + assert "summary" not in event + assert event["ext"]["policy"]["stack_rebench_enabled"] is None + assert event["ext"]["config_arm"]["rounds"][0]["workload_signature"] == "qwen-tp8-c64" + assert event["ext"]["config_arm"]["rounds"][0]["input_stack"]["extra_server_args"] == "--base-flag" + variant = event["ext"]["config_arm"]["rounds"][0]["variants"][0] + assert variant["stack_rebench"] == {"ran": True, "tput": 104.0, "stable": True} + attempt = event["ext"]["source_arm"]["attempts"][0] + assert attempt["patch_source"] == "upstream_pr" + assert attempt["lever_kind"] == "upstream_pr" + assert attempt["route"] == "direct_framework" + assert attempt["status"] == "KEEP" + assert event["ext"]["exit"] == { + "reason": "optimize_no_more_leverage", + "trigger": "both_arms_plateaued", + "hint": None, + "switch_bottleneck": True, + } + + +def test_framework_timeline_projects_pr1301_source_and_critic_data(tmp_path): + state = { + "phase": "KERNEL_AGENT", + "macro_cycle": 2, + "framework_agent_authoring_enabled": True, + "framework_agent_phase_done": False, + "phase_history": [ + { + "from_phase": "SWEEP", + "to_phase": "FRAMEWORK_AGENT", + "reason": "cycle_reloop", + "ts": "2026-08-27T02:00:00+00:00", + "cycle": 2, + }, + { + "from_phase": "FRAMEWORK_AGENT", + "to_phase": "KERNEL_AGENT", + "reason": "optimize_phase_budget_exhausted", + "ts": "2026-08-27T02:20:00+00:00", + "cycle": 2, + "evidence": { + "source_consecutive_no_keep": 0, + "source_threshold": 3, + "source_candidates_exhausted": False, + "source_arm_plateaued": False, + "recent_keep_gain_pct": 0.0, + "keep_gain_threshold_pct": 1.0, + "empty_streak": 0, + "empty_streak_threshold": 3, + "lookback": 6, + "tested_this_cycle": 0, + "config_arm_plateaued": False, + "switch_bottleneck": False, + }, + }, + ], + "specialist_rounds": [ + { + "round_id": "discover-round", + "task_id": "discover-task-1234", + "domain": "candidate_discovery_specialist", + "cycle": 2, + "completed_at": "2026-08-27T02:04:00+00:00", + "proposal_set": [ + { + "pr_url": "https://example.test/pr/9", + "title": "Fuse host copies", + "verdict": "worth_a_bench", + "route": "author_via_specialist", + }, + { + "pr_url": "https://example.test/pr/10", + "title": "Unrelated backend", + "verdict": "not_applicable", + "reason": "wrong framework", + }, + ], + }, + { + "round_id": "author-round", + "task_id": "author-task-1", + "domain": "serving_specialist", + "cycle": 2, + "completed_at": "2026-08-27T02:08:00+00:00", + "proposal_set": [{"patches_written": ["patches/pr-9.patch"]}], + }, + ], + "framework_agent_batches": [ + { + "batch_id": "discovery-0-discover", + "candidates": [ + { + "pr_url": "https://example.test/pr/9", + "route": "author_via_specialist", + "audit": {"verdict": "worth_a_bench"}, + } + ], + } + ], + "framework_agent_specialist_candidate_map": { + "author-task-1": "https://example.test/pr/9", + }, + "framework_agent_phase_progress": [ + { + "candidate_id": "https://example.test/pr/9", + "batch_id": "discovery-0-discover", + "status": "kept", + "kept": True, + "gain_pct": 4.0, + "pre_tput": 100.0, + "post_tput": 104.0, + "specialist_task_id": "author-task-1", + "integrate_task_id": "integrate-task-1", + "cycle": 2, + "ts": "2026-08-27T02:15:00+00:00", + } + ], + } + operations = [ + { + "operation_id": "op-integrate-1", + "name": "integrate_patch", + "phase": "FRAMEWORK_AGENT", + "agent": "framework_agent", + "macro_cycle": 2, + "status": "succeeded", + "ended_at": "2026-08-27T02:15:00+00:00", + "outputs": { + "status": "kept", + "framework_agent_authoring": True, + "specialist_task_id": "author-task-1", + "base_tput": 100.0, + "output_throughput": 104.0, + "delta_pct": 4.0, + "accuracy_pass": True, + "keep_threshold_pct": 1.0, + "patches_applied": ["patches/pr-9.patch"], + "target_files": ["python/worker.py"], + "source_snapshot": "optimization_stack/src/author-task-1", + "source_manifest": "optimization_stack/src/author-task-1/manifest.json", + "workspace": "runs/integrate-task-1", + "switch_off_parity": {"ran": True, "ok": True}, + "stack_rebench": {"stable": True}, + "framework_levers": [{"switch": "SGLANG_FAST_COPY", "default_on": True}], + }, + "extensions": {"task_id": "integrate-task-1"}, + } + ] + critic_dir = tmp_path / "critic-workdir" / "000000" + _write_json( + critic_dir / "judge_bundle.json", + { + "merged_context": {"macro_cycle": 2}, + "proposals": [ + { + "msg_id": "proposal-1", + "action_name": "integrate_patch", + "payload": { + "params": { + "framework_agent_candidate_id": "https://example.test/pr/9", + } + }, + } + ], + }, + ) + _write_json( + critic_dir / "review.json", + { + "review_verdicts": [ + { + "target_proposal_msg_id": "proposal-1", + "verdict": "needs_review", + "source": "critic", + "reasoning": "needs parity evidence", + "confidence": "high", + "required_evidence": ["switch-off parity"], + "risks": [{"severity": "major", "risk": "default behavior may change"}], + } + ] + }, + ) + _write_json( + critic_dir / "emit.json", + { + "intent_envelope": { + "intents": [ + { + "intent_type": "review_verdict", + "payload": { + "target_proposal_msg_id": "proposal-1", + "verdict": "approve", + "advice_text": "retain the switch-off check", + }, + } + ] + } + }, + ) + + timeline = collect_v6_timeline(tmp_path, [], state=state, recorded_operations=operations) + + event = timeline[0] + discovery = event["ext"]["source_arm"]["candidate_discovery_runs"][0] + assert discovery["task_id"] == "discover-task-1234" + assert [candidate["verdict"] for candidate in discovery["candidates"]] == [ + "worth_a_bench", + "not_applicable", + ] + authoring = event["ext"]["source_arm"]["authoring_runs"][0] + assert authoring["candidate_id"] == "https://example.test/pr/9" + assert authoring["patch_refs"] == ["patches/pr-9.patch"] + attempt = event["ext"]["source_arm"]["attempts"][0] + assert attempt["patch_source"] == "specialist_authored" + assert attempt["lever_kind"] == "source_patch" + assert attempt["route"] == "author_via_specialist" + assert attempt["status"] == "KEEP" + assert attempt["gates"] == { + "accuracy_passed": True, + "keep_threshold_pct": 1.0, + "switch_off_parity_passed": True, + "stack_rebench_passed": True, + } + review = event["ext"]["critic_reviews"][0] + assert review["arm"] == "source" + assert review["target_action"] == "integrate_patch" + assert review["verdict"] == "needs_review" + assert review["effective_verdict"] == "approve" + assert "token" not in json.dumps(event).lower() + + +def test_framework_timeline_keeps_config_serving_specialist_out_of_source_arm(tmp_path): + state = { + "phase": "KERNEL_AGENT", + "macro_cycle": 0, + "phase_history": [ + { + "from_phase": "PRELUDE", + "to_phase": "FRAMEWORK_AGENT", + "ts": "2026-08-27T03:00:00+00:00", + "cycle": 0, + }, + { + "from_phase": "FRAMEWORK_AGENT", + "to_phase": "KERNEL_AGENT", + "ts": "2026-08-27T03:10:00+00:00", + "cycle": 0, + }, + ], + "specialist_rounds": [ + { + "round_id": "config-round", + "task_id": "config-task", + "domain": "serving_specialist", + "source_phase": "FRAMEWORK_AGENT", + "cycle": 0, + "proposal_set": [ + { + "name": "larger-page-size", + "extra_server_args": "--page-size 32", + } + ], + } + ], + } + critic_dir = tmp_path / "critic-workdir" / "000000" + _write_json( + critic_dir / "request.json", + { + "context": {"phase": "FRAMEWORK_AGENT"}, + "raw_prompt": "=== Shared session state ===\nmacro_cycle=0\n", + }, + ) + _write_json( + critic_dir / "judge_bundle.json", + { + "phase": "FRAMEWORK_AGENT", + "proposals": [ + { + "msg_id": "config-proposal", + "action_name": "specialist", + "payload": { + "params": { + "domain": "serving_specialist", + "source_phase": "FRAMEWORK_AGENT", + } + }, + } + ], + }, + ) + _write_json( + critic_dir / "review.json", + { + "review_verdicts": [ + { + "target_proposal_msg_id": "config-proposal", + "verdict": "approve", + } + ] + }, + ) + _write_json( + critic_dir / "emit.json", + { + "intent_envelope": { + "intents": [ + { + "intent_type": "review_verdict", + "payload": { + "target_proposal_msg_id": "config-proposal", + "verdict": "approve", + }, + } + ] + } + }, + ) + + event = collect_v6_timeline(tmp_path, [], state=state, recorded_operations=[])[0] + + assert [row["task_id"] for row in event["ext"]["config_arm"]["specialist_runs"]] == ["config-task"] + assert event["ext"]["source_arm"]["authoring_runs"] == [] + assert event["ext"]["critic_reviews"][0]["arm"] == "config" + + +def test_framework_timeline_ignores_kernel_specialist_without_framework_evidence(tmp_path): + state = { + "phase": "KERNEL_AGENT", + "macro_cycle": 4, + "specialist_rounds": [ + { + "round_id": "kernel-specialist", + "task_id": "kernel-specialist", + "domain": "kernel_specialist", + "cycle": 4, + "completed_at": "2026-08-27T04:00:00+00:00", + "proposal_set": [{"name": "kernel-rewrite"}], + } + ], + } + + operations = [ + { + "operation_id": "op-kernel-specialist", + "kind": "specialist", + "name": "specialist round kernel-specialist", + "phase": "EXPLORE", + "source": "specialist_recorder_hook", + "macro_cycle": 4, + "status": "succeeded", + "outputs": state["specialist_rounds"][0], + } + ] + + assert collect_v6_timeline(tmp_path, [], state=state, recorded_operations=operations) == [] + + +def test_framework_timeline_recovers_direct_upstream_patch_source(tmp_path): + candidate_id = "https://example.test/pr/11" + state = { + "phase": "KERNEL_AGENT", + "macro_cycle": 0, + "phase_history": [ + { + "from_phase": "PRELUDE", + "to_phase": "FRAMEWORK_AGENT", + "ts": "2026-08-27T03:00:00+00:00", + "cycle": 0, + }, + { + "from_phase": "FRAMEWORK_AGENT", + "to_phase": "KERNEL_AGENT", + "ts": "2026-08-27T03:10:00+00:00", + "cycle": 0, + }, + ], + "framework_agent_batches": [ + { + "batch_id": "discovery-0", + "candidates": [ + { + "pr_url": candidate_id, + "route": "direct_framework", + } + ], + } + ], + "framework_agent_phase_progress": [ + { + "candidate_id": candidate_id, + "integrate_task_id": "integrate-direct-1", + "status": "kept", + "kept": True, + "cycle": 0, + } + ], + } + operations = [ + { + "operation_id": "op-integrate-direct", + "name": "integrate_patch", + "phase": "FRAMEWORK_AGENT", + "macro_cycle": 0, + "status": "succeeded", + "outputs": { + "status": "kept", + "framework_agent_authoring": True, + "specialist_task_id": "integrate-direct-1", + }, + "extensions": {"task_id": "integrate-direct-1"}, + } + ] + + event = collect_v6_timeline(tmp_path, [], state=state, recorded_operations=operations)[0] + attempt = event["ext"]["source_arm"]["attempts"][0] + + assert attempt["candidate_id"] == candidate_id + assert attempt["patch_source"] == "upstream_pr" + assert attempt["lever_kind"] == "upstream_pr" + assert attempt["route"] == "direct_framework" + + +def test_framework_timeline_keeps_macro_cycles_isolated(tmp_path): + state = { + "phase": "KERNEL_AGENT", + "macro_cycle": 1, + "phase_history": [ + { + "from_phase": "PRELUDE", + "to_phase": "FRAMEWORK_AGENT", + "reason": "prelude_complete", + "ts": "2026-08-27T03:00:00+00:00", + "cycle": 0, + }, + { + "from_phase": "FRAMEWORK_AGENT", + "to_phase": "KERNEL_AGENT", + "reason": "optimize_no_more_leverage", + "ts": "2026-08-27T03:10:00+00:00", + "cycle": 0, + }, + { + "from_phase": "SWEEP", + "to_phase": "FRAMEWORK_AGENT", + "reason": "cycle_reloop", + "ts": "2026-08-27T04:00:00+00:00", + "cycle": 1, + }, + { + "from_phase": "FRAMEWORK_AGENT", + "to_phase": "KERNEL_AGENT", + "reason": "optimize_no_more_leverage", + "ts": "2026-08-27T04:10:00+00:00", + "cycle": 1, + }, + ], + } + operations = [ + { + "operation_id": "cycle-0", + "name": "explore", + "phase": "FRAMEWORK_AGENT", + "macro_cycle": 0, + "status": "succeeded", + "outputs": {"status": "succeeded", "round_id": "round-0"}, + }, + { + "operation_id": "cycle-1", + "name": "explore", + "phase": "FRAMEWORK_AGENT", + "macro_cycle": 1, + "status": "succeeded", + "outputs": {"status": "succeeded", "round_id": "round-1"}, + }, + ] + + timeline = collect_v6_timeline(tmp_path, [], state=state, recorded_operations=operations) + + assert [event["ext"]["macro_cycle"] for event in timeline] == [0, 1] + assert [event["ext"]["config_arm"]["rounds"][0]["round_id"] for event in timeline] == [ + "round-0", + "round-1", + ] + + +def test_framework_timeline_excludes_kernel_phase_explore_rebench(tmp_path): + state = { + "phase": "KERNEL_AGENT", + "macro_cycle": 0, + "phase_history": [ + { + "from_phase": "PRELUDE", + "to_phase": "FRAMEWORK_AGENT", + "ts": "2026-08-27T03:00:00+00:00", + "cycle": 0, + }, + { + "from_phase": "FRAMEWORK_AGENT", + "to_phase": "KERNEL_AGENT", + "ts": "2026-08-27T03:10:00+00:00", + "cycle": 0, + }, + ], + } + operations = [ + { + "operation_id": "framework-round", + "name": "explore", + "phase": "FRAMEWORK_AGENT", + "agent": "explore", + "macro_cycle": 0, + "status": "succeeded", + "ended_at": "2026-08-27T03:05:00+00:00", + "outputs": {"status": "succeeded", "round_id": "framework-round"}, + }, + { + "operation_id": "kernel-rebench", + "name": "explore", + "phase": "KERNEL_AGENT", + "agent": "explore", + "macro_cycle": 0, + "status": "succeeded", + "ended_at": "2026-08-27T03:06:00+00:00", + "outputs": {"status": "succeeded", "round_id": "kernel-rebench"}, + }, + ] + + event = collect_v6_timeline(tmp_path, [], state=state, recorded_operations=operations)[0] + + assert [row["round_id"] for row in event["ext"]["config_arm"]["rounds"]] == ["framework-round"] + + +def test_framework_timeline_projects_discovery_history_outcomes_in_order(tmp_path): + state = { + "phase": "KERNEL_AGENT", + "macro_cycle": 0, + "phase_history": [ + { + "from_phase": "PRELUDE", + "to_phase": "FRAMEWORK_AGENT", + "ts": "2026-08-27T03:00:00+00:00", + "cycle": 0, + }, + { + "from_phase": "FRAMEWORK_AGENT", + "to_phase": "FRAMEWORK_AGENT", + "reason": "framework_agent_discover_failed", + "evidence": { + "event": "framework_agent_discover_failed", + "attempt": 1, + "limit": 3, + "error": "TimeoutError('upstream unavailable')", + }, + "ts": "2026-08-27T03:01:00+00:00", + "cycle": 0, + }, + { + "from_phase": "FRAMEWORK_AGENT", + "to_phase": "FRAMEWORK_AGENT", + "reason": "discover_empty_payload", + "evidence": { + "event": "framework_agent_phase_done", + "failure_count": 0, + "retry_limit": 3, + }, + "ts": "2026-08-27T03:03:00+00:00", + "cycle": 0, + }, + { + "from_phase": "FRAMEWORK_AGENT", + "to_phase": "KERNEL_AGENT", + "reason": "framework_agent_phase_done", + "ts": "2026-08-27T03:04:00+00:00", + "cycle": 0, + }, + ], + "framework_agent_batches": [ + { + "batch_id": "discovery-0", + "ts": "2026-08-27T03:02:00+00:00", + "cycle": 0, + "candidates": [ + { + "pr_url": "https://example.test/pr/12", + "route": "direct_framework", + } + ], + } + ], + } + + event = collect_v6_timeline(tmp_path, [], state=state, recorded_operations=[])[0] + runs = event["ext"]["source_arm"]["candidate_discovery_runs"] + + assert [run["status"] for run in runs] == ["failed", "succeeded", "empty"] + assert runs[0]["reason"] == "TimeoutError('upstream unavailable')" + assert runs[1]["batch_id"] == "discovery-0" + assert runs[1]["candidates"][0]["candidate_id"] == "https://example.test/pr/12" + assert runs[2]["reason"] == "discover_empty_payload" + assert event["status"] == "succeeded" + assert event["ext"]["failure"] == { + "failed_task_id": None, + "error_class": None, + "error": None, + } + + +def test_framework_timeline_marks_exhausted_discovery_retries_failed(tmp_path): + state = { + "phase": "KERNEL_AGENT", + "macro_cycle": 0, + "phase_history": [ + { + "from_phase": "PRELUDE", + "to_phase": "FRAMEWORK_AGENT", + "ts": "2026-08-27T03:00:00+00:00", + "cycle": 0, + }, + { + "from_phase": "FRAMEWORK_AGENT", + "to_phase": "FRAMEWORK_AGENT", + "reason": "framework_agent_discover_failed", + "evidence": { + "event": "framework_agent_discover_failed", + "attempt": 1, + "limit": 3, + "error": "TimeoutError('first')", + }, + "ts": "2026-08-27T03:01:00+00:00", + "cycle": 0, + }, + { + "from_phase": "FRAMEWORK_AGENT", + "to_phase": "FRAMEWORK_AGENT", + "reason": "framework_agent_discover_failed", + "evidence": { + "event": "framework_agent_discover_failed", + "attempt": 3, + "limit": 3, + "error": "TimeoutError('last')", + }, + "ts": "2026-08-27T03:02:00+00:00", + "cycle": 0, + }, + { + "from_phase": "FRAMEWORK_AGENT", + "to_phase": "FRAMEWORK_AGENT", + "reason": "discover_retries_exhausted", + "evidence": { + "event": "framework_agent_phase_done", + "failure_count": 3, + "retry_limit": 3, + }, + "ts": "2026-08-27T03:03:00+00:00", + "cycle": 0, + }, + { + "from_phase": "FRAMEWORK_AGENT", + "to_phase": "KERNEL_AGENT", + "reason": "framework_agent_phase_done", + "ts": "2026-08-27T03:04:00+00:00", + "cycle": 0, + }, + ], + } + + event = collect_v6_timeline(tmp_path, [], state=state, recorded_operations=[])[0] + runs = event["ext"]["source_arm"]["candidate_discovery_runs"] + + assert [run["status"] for run in runs] == ["failed", "failed"] + assert event["status"] == "failed" + assert event["ext"]["failure"] == { + "failed_task_id": None, + "error_class": None, + "error": "TimeoutError('last')", + } + + +def test_framework_timeline_assigns_critic_reviews_from_request_cycle(tmp_path): + state = { + "phase": "KERNEL_AGENT", + "macro_cycle": 1, + "phase_history": [ + { + "from_phase": "PRELUDE", + "to_phase": "FRAMEWORK_AGENT", + "ts": "2026-08-27T03:00:00+00:00", + "cycle": 0, + }, + { + "from_phase": "FRAMEWORK_AGENT", + "to_phase": "KERNEL_AGENT", + "ts": "2026-08-27T03:10:00+00:00", + "cycle": 0, + }, + { + "from_phase": "SWEEP", + "to_phase": "FRAMEWORK_AGENT", + "ts": "2026-08-27T04:00:00+00:00", + "cycle": 1, + }, + { + "from_phase": "FRAMEWORK_AGENT", + "to_phase": "KERNEL_AGENT", + "ts": "2026-08-27T04:10:00+00:00", + "cycle": 1, + }, + ], + } + for cycle in (0, 1): + proposal_id = f"proposal-cycle-{cycle}" + critic_dir = tmp_path / "critic-workdir" / f"{cycle:06d}" + _write_json( + critic_dir / "request.json", + { + "context": {"phase": "FRAMEWORK_AGENT"}, + "raw_prompt": ( + f"=== Shared session state ===\nmacro_cycle={cycle}\n" + if cycle == 0 + else "=== Shared session state ===\n" + ), + }, + ) + _write_json( + critic_dir / "judge_bundle.json", + { + "phase": "FRAMEWORK_AGENT", + "proposals": [ + { + "msg_id": proposal_id, + "action_name": "integrate_patch", + "payload": { + "framework_agent_candidate_id": f"candidate-{cycle}", + }, + } + ], + }, + ) + _write_json( + critic_dir / "review.json", + { + "review_verdicts": [ + { + "target_proposal_msg_id": proposal_id, + "verdict": "approve", + } + ] + }, + ) + _write_json( + critic_dir / "emit.json", + { + "intent_envelope": { + "intents": [ + { + "intent_type": "review_verdict", + "payload": { + "target_proposal_msg_id": proposal_id, + "verdict": "approve", + }, + } + ] + } + }, + ) + + _write_json( + tmp_path / "reports" / "trace" / "proposal_task_map.jsonl", + { + "proposal_msg_id": "proposal-cycle-1", + "task_id": "integrate-cycle-1", + }, + ) + operations = [ + { + "operation_id": "op-cycle-1", + "name": "integrate_patch", + "phase": "FRAMEWORK_AGENT", + "macro_cycle": 1, + "extensions": {"task_id": "integrate-cycle-1"}, + "outputs": {"status": "reverted"}, + } + ] + + timeline = collect_v6_timeline(tmp_path, [], state=state, recorded_operations=operations) + + assert [[review["proposal_msg_id"] for review in event["ext"]["critic_reviews"]] for event in timeline] == [ + ["proposal-cycle-0"], + ["proposal-cycle-1"], + ] diff --git a/src/hyperloom/inference_optimizer/tests/test_specialist_lifecycle.py b/src/hyperloom/inference_optimizer/tests/test_specialist_lifecycle.py index dce7b51eaa..247421d2e6 100644 --- a/src/hyperloom/inference_optimizer/tests/test_specialist_lifecycle.py +++ b/src/hyperloom/inference_optimizer/tests/test_specialist_lifecycle.py @@ -259,7 +259,7 @@ async def test_build_specialist_round_entry_carries_full_payload(coord): coord_obj = Coordinator.__new__(Coordinator) task = _StubTask( task_id="t-build", - params={"round_id": "round-9"}, + params={"round_id": "round-9", "source_phase": "KERNEL_AGENT"}, ) payload = _done_payload( domain="serving_specialist", @@ -290,6 +290,7 @@ async def test_build_specialist_round_entry_carries_full_payload(coord): "confidence", "new_findings", "residual_questions", + "source_phase", } assert expected_keys.issubset(entry.keys()) assert entry["round_id"] == "round-9" @@ -297,6 +298,7 @@ async def test_build_specialist_round_entry_carries_full_payload(coord): assert entry["proposals_total"] == 2 assert entry["empty"] is False assert entry["confidence"] == 0.62 + assert entry["source_phase"] == "KERNEL_AGENT" @pytest.mark.asyncio diff --git a/src/hyperloom/orchestrator/enablement/params.py b/src/hyperloom/orchestrator/enablement/params.py index 0f393c5d9b..e6e0f3ddc2 100644 --- a/src/hyperloom/orchestrator/enablement/params.py +++ b/src/hyperloom/orchestrator/enablement/params.py @@ -265,6 +265,7 @@ def _build_enablement_specialist_params(self, launch_log: str, *, attempt: int = params_out: dict[str, Any] = { "domain": "enablement_specialist", + "source_phase": "ENABLEMENT", "gap_canonical_id": gap_cid, "gap_symptom": (f"{framework or '?'} cannot launch {model or 'the target model'}: {signature.kind}"), "gap_layer": "framework", diff --git a/src/hyperloom/orchestrator/phases/explore.py b/src/hyperloom/orchestrator/phases/explore.py index 1c03b192ae..727cd267b4 100644 --- a/src/hyperloom/orchestrator/phases/explore.py +++ b/src/hyperloom/orchestrator/phases/explore.py @@ -1868,7 +1868,18 @@ def _build_specialist_round_entry( proposals = done_payload.get("proposal_set") or [] if not isinstance(proposals, list): proposals = [] - round_id = str((task.params or {}).get("round_id") or task.task_id) + task_params = task.params or {} + round_id = str(task_params.get("round_id") or task.task_id) + source_phase = ( + str( + task_params.get("source_phase") + or done_payload.get("source_phase") + or getattr(getattr(self, "shared_state", None), "phase", "") + or "" + ) + .strip() + .upper() + ) from ..specialists.domains import normalize_dispatch_tags # Knowledge-domain tags; reported tags win over dispatch params. @@ -1892,6 +1903,8 @@ def _build_specialist_round_entry( "new_findings": list(done_payload.get("new_findings") or []), "residual_questions": list(done_payload.get("residual_questions") or []), } + if source_phase: + entry["source_phase"] = source_phase gpu_ids = done_payload.get("allocated_gpu_ids") or [] if isinstance(gpu_ids, list) and gpu_ids: entry["allocated_gpu_ids"] = [ diff --git a/src/hyperloom/orchestrator/phases/framework.py b/src/hyperloom/orchestrator/phases/framework.py index 86da3aafbb..66e3f19d83 100644 --- a/src/hyperloom/orchestrator/phases/framework.py +++ b/src/hyperloom/orchestrator/phases/framework.py @@ -359,6 +359,7 @@ async def _enqueue_framework_agent_authoring_specialist( "gap_layer": "framework", "framework": str(candidate.get("framework") or getattr(state, "framework", "") or "").strip().lower(), "task_kind": "framework_authoring", + "source_phase": "FRAMEWORK_AGENT", "pr_lead": {"title": title, "url": pr_url, "diff_url": diff_url}, "lever_kind": LEVER_UPSTREAM_PR, # Provenance markers for the dispatcher-side authored-patch bridge. diff --git a/src/hyperloom/orchestrator/phases/internal.py b/src/hyperloom/orchestrator/phases/internal.py index d5fc6c787a..b17d4d510b 100644 --- a/src/hyperloom/orchestrator/phases/internal.py +++ b/src/hyperloom/orchestrator/phases/internal.py @@ -47,6 +47,7 @@ async def _enqueue_internal_research_scout_task( ) params: dict[str, Any] = { "domain": "research_scout_specialist", + "source_phase": str(getattr(self.shared_state, "phase", "") or "PRELUDE").strip().upper(), "gap_canonical_id": f"gap.research_scout.round{int(round_id)}", "gap_symptom": ( "Collect proven priors (reference launch scripts, model " @@ -180,6 +181,7 @@ async def _enqueue_internal_static_recon_task( idempotency_key = "internal-static-recon-prelude" params: dict[str, Any] = { "domain": "static_recon_specialist", + "source_phase": str(getattr(state, "phase", "") or "PRELUDE").strip().upper(), "gap_canonical_id": "gap.static_recon.prelude", "gap_symptom": ( "Grep the framework source for un-bridged capability switches " @@ -287,6 +289,7 @@ async def _maybe_enqueue_trajectory_reviewer(self) -> None: domain = hint[0] if hint else "serving_specialist" params: dict[str, Any] = { "domain": domain, + "source_phase": str(getattr(state, "phase", "") or "INTERNAL").strip().upper(), "gap_canonical_id": f"gap.trajectory_review.cycle{cycle}", "gap_symptom": ( "The search has plateaued. Review the optimization trajectory " From 5479dd7b8a96603fd468137f9e5c047dee8317f2 Mon Sep 17 00:00:00 2001 From: chenluo Date: Fri, 28 Aug 2026 16:48:50 +0800 Subject: [PATCH 3/7] fix: harden Framework Agent timeline attribution --- .../breakdown/collectors/v6.py | 65 +++++--- .../breakdown/recorder/instrument.py | 38 +++-- .../tests/test_sbd_v6_initial.py | 157 ++++++++++++++++++ 3 files changed, 222 insertions(+), 38 deletions(-) diff --git a/src/hyperloom/inference_optimizer/breakdown/collectors/v6.py b/src/hyperloom/inference_optimizer/breakdown/collectors/v6.py index 38f786d7c1..f6878f7ceb 100644 --- a/src/hyperloom/inference_optimizer/breakdown/collectors/v6.py +++ b/src/hyperloom/inference_optimizer/breakdown/collectors/v6.py @@ -1502,6 +1502,30 @@ def _source_attempts( progress_rows: list[dict[str, Any]], ) -> list[dict[str, Any]]: candidates = _candidate_index(state) + source_operations = [ + operation + for operation in operations + if _operation_name(operation) in {"framework_agent", "integrate", "integrate_patch"} + ] + candidate_operation_counts: dict[str, int] = {} + source_operation_counts: dict[str, int] = {} + for operation in source_operations: + outputs = _mapping(operation.get("outputs")) + inputs = _mapping(operation.get("inputs")) + candidate_id = str( + _first( + _candidate_id(_first(outputs.get("candidate"), inputs.get("candidate"))), + outputs.get("framework_agent_candidate_id"), + inputs.get("framework_agent_candidate_id"), + _nested(operation, "metadata", "extras", "candidate_id"), + ) + or "" + ) + if candidate_id: + candidate_operation_counts[candidate_id] = candidate_operation_counts.get(candidate_id, 0) + 1 + source_task_id = str(_first(outputs.get("specialist_task_id"), inputs.get("specialist_task_id")) or "") + if source_task_id: + source_operation_counts[source_task_id] = source_operation_counts.get(source_task_id, 0) + 1 progress_by_integrate = { str(row.get("integrate_task_id") or ""): row for row in progress_rows if row.get("integrate_task_id") } @@ -1513,26 +1537,25 @@ def _source_attempts( } used_progress: set[int] = set() attempts: list[dict[str, Any]] = [] - for operation in operations: - if _operation_name(operation) not in {"framework_agent", "integrate", "integrate_patch"}: - continue + for operation in source_operations: outputs = _mapping(operation.get("outputs")) + inputs = _mapping(operation.get("inputs")) candidate_id = str( _first( - _candidate_id(outputs.get("candidate")), + _candidate_id(_first(outputs.get("candidate"), inputs.get("candidate"))), outputs.get("framework_agent_candidate_id"), + inputs.get("framework_agent_candidate_id"), _nested(operation, "metadata", "extras", "candidate_id"), ) or "" ) task_id = _operation_task_id(operation) - source_task_id = str(outputs.get("specialist_task_id") or "") - progress = ( - progress_by_integrate.get(task_id) - or progress_by_candidate.get(candidate_id) - or progress_by_source.get(source_task_id) - or {} - ) + source_task_id = str(_first(outputs.get("specialist_task_id"), inputs.get("specialist_task_id")) or "") + progress = progress_by_integrate.get(task_id) or {} + if not progress and source_operation_counts.get(source_task_id) == 1: + progress = progress_by_source.get(source_task_id) or {} + if not progress and candidate_operation_counts.get(candidate_id) == 1: + progress = progress_by_candidate.get(candidate_id) or {} if progress: used_progress.add(id(progress)) attempts.append(_source_attempt(operation, progress, candidates)) @@ -1981,14 +2004,23 @@ def _framework_event( window_count, evidence, ) + critic_reviews = _critic_review_rows( + session_dir, + warnings, + state, + recorded_operations, + window, + window_count, + ) exit_details = _framework_exit(window, config_arm, source_arm) failure = _framework_failure(window) has_work = bool( config_arm["specialist_runs"] or config_arm["rounds"] - or any(run["candidates"] for run in source_arm["candidate_discovery_runs"]) + or source_arm["candidate_discovery_runs"] or source_arm["authoring_runs"] or source_arm["attempts"] + or critic_reviews ) if any(value is not None for value in failure.values()): status = "failed" @@ -2007,14 +2039,7 @@ def _framework_event( "ext": { "macro_cycle": int(window["cycle"]), "policy": policy, - "critic_reviews": _critic_review_rows( - session_dir, - warnings, - state, - recorded_operations, - window, - window_count, - ), + "critic_reviews": critic_reviews, "config_arm": config_arm, "source_arm": source_arm, "exit": exit_details, diff --git a/src/hyperloom/inference_optimizer/breakdown/recorder/instrument.py b/src/hyperloom/inference_optimizer/breakdown/recorder/instrument.py index 14c4ac0820..45a5eeab0f 100644 --- a/src/hyperloom/inference_optimizer/breakdown/recorder/instrument.py +++ b/src/hyperloom/inference_optimizer/breakdown/recorder/instrument.py @@ -3637,10 +3637,9 @@ def record_specialist_round( a no-op. entry (dict[str, Any]): the specialist round entry (keyed by ``round_id``); an empty/non-dict value is a no-op. - phase (str): the phase the round ran in; falls back to - ``entry["phase"]``, then to the reader's timestamp backfill. A - specialist runs in more than one phase, so this cannot be a - constant. + phase (str): the runtime phase used when the entry does not already + declare ``source_phase``. A specialist runs in more than one phase, + so this cannot be a constant. producer (str): the breakdown producer label (defaults to the Coordinator). """ @@ -3648,21 +3647,24 @@ def record_specialist_round( trace_skip(reason="no session_dir" if not session_dir else "empty entry", section="specialist_rounds") return try: - key = str(entry.get("round_id") or "") or None + source_phase = str(entry.get("source_phase") or phase or entry.get("phase") or "").strip().upper() + recorded_entry = dict(entry) + if source_phase: + recorded_entry.setdefault("source_phase", source_phase) + key = str(recorded_entry.get("round_id") or "") or None _recorder(session_dir, producer).record_item( "specialist_runs", - dict(entry), + recorded_entry, key=key, ) - round_id = str(entry.get("round_id") or key or entry.get("task_id") or "unknown") + round_id = str(recorded_entry.get("round_id") or key or recorded_entry.get("task_id") or "unknown") operation_id = _stable_id("op", "specialist", round_id) round_subject_id = _stable_id("subject", "specialist-round", round_id) - domains = list(entry.get("domains") or []) - if entry.get("domain"): - domains.append(str(entry.get("domain"))) - domains.extend(str(tag) for tag in (entry.get("tags") or []) if str(tag)) + domains = list(recorded_entry.get("domains") or []) + if recorded_entry.get("domain"): + domains.append(str(recorded_entry.get("domain"))) + domains.extend(str(tag) for tag in (recorded_entry.get("tags") or []) if str(tag)) domains = list(dict.fromkeys(domain for domain in domains if domain)) - source_phase = str(entry.get("source_phase") or "EXPLORE").strip().upper() record_subject( session_dir, subject_id=round_subject_id, @@ -3671,7 +3673,7 @@ def record_specialist_round( name=round_id, attributes={ "domains": domains, - "proposals_total": entry.get("proposals_total"), + "proposals_total": recorded_entry.get("proposals_total"), }, producer=producer, ) @@ -3689,7 +3691,7 @@ def record_specialist_round( ) domain_subjects.append({"subject_id": domain_id, "subject_type": "specialist_domain"}) proposal_subjects: list[dict[str, Any]] = [] - proposals = entry.get("proposal_set") + proposals = recorded_entry.get("proposal_set") if isinstance(proposals, list): for index, proposal in enumerate(proposals): if not isinstance(proposal, Mapping): @@ -3715,18 +3717,18 @@ def record_specialist_round( kind="specialist", name=f"specialist round {round_id}", phase=source_phase, - status="succeeded" if entry.get("completed_at") else "partial", + status="succeeded" if recorded_entry.get("completed_at") else "partial", source="specialist_recorder_hook", executor_class="llm_agent", purpose="proposal", - scope=str(entry.get("scope") or ""), + scope=str(recorded_entry.get("scope") or ""), strategy_group="specialist", strategy="multi_domain", producer=producer, - ended_at=str(entry.get("completed_at") or entry.get("dispatched_at") or ""), + ended_at=str(recorded_entry.get("completed_at") or recorded_entry.get("dispatched_at") or ""), subject={"subject_id": round_subject_id, "subject_type": "specialist_round"}, subjects=domain_subjects + proposal_subjects, - outputs=dict(entry), + outputs=recorded_entry, adoption_refs=[], extensions={"downstream_relation": "proposal_only"}, ) diff --git a/src/hyperloom/inference_optimizer/tests/test_sbd_v6_initial.py b/src/hyperloom/inference_optimizer/tests/test_sbd_v6_initial.py index a262d8376e..ca90fd08f6 100644 --- a/src/hyperloom/inference_optimizer/tests/test_sbd_v6_initial.py +++ b/src/hyperloom/inference_optimizer/tests/test_sbd_v6_initial.py @@ -1807,3 +1807,160 @@ def test_framework_timeline_assigns_critic_reviews_from_request_cycle(tmp_path): ["proposal-cycle-0"], ["proposal-cycle-1"], ] + + +def test_specialist_recorder_preserves_runtime_phase_when_entry_has_no_source_phase(tmp_path, monkeypatch): + from hyperloom.inference_optimizer.breakdown.recorder import instrument + + captured: dict[str, dict] = {} + + class Recorder: + def record_item(self, stream, item, *, key=None): + captured["ledger"] = {"stream": stream, "item": item, "key": key} + + monkeypatch.setattr(instrument, "_recorder", lambda *_args, **_kwargs: Recorder()) + monkeypatch.setattr(instrument, "record_subject", lambda *_args, **_kwargs: None) + monkeypatch.setattr( + instrument, + "record_operation", + lambda *_args, **kwargs: captured.setdefault("operation", kwargs), + ) + monkeypatch.setattr(instrument, "record_trace_event", lambda *_args, **_kwargs: None) + + instrument.record_specialist_round( + tmp_path, + { + "round_id": "kernel-specialist", + "task_id": "kernel-task", + "completed_at": "2026-08-28T01:00:00+00:00", + }, + phase="KERNEL_AGENT", + ) + + assert captured["ledger"]["item"]["source_phase"] == "KERNEL_AGENT" + assert captured["operation"]["phase"] == "KERNEL_AGENT" + assert captured["operation"]["outputs"]["source_phase"] == "KERNEL_AGENT" + + +def test_framework_timeline_treats_empty_discovery_as_executed_work(tmp_path): + state = { + "phase": "KERNEL_AGENT", + "macro_cycle": 1, + "phase_history": [ + { + "from_phase": "PRELUDE", + "to_phase": "FRAMEWORK_AGENT", + "cycle": 1, + "ts": "2026-08-28T01:00:00+00:00", + }, + { + "from_phase": "FRAMEWORK_AGENT", + "to_phase": "KERNEL_AGENT", + "cycle": 1, + "ts": "2026-08-28T01:05:00+00:00", + "reason": "optimize_no_more_leverage", + }, + ], + "specialist_rounds": [ + { + "round_id": "discovery-empty", + "task_id": "discovery-task", + "source_phase": "FRAMEWORK_AGENT", + "cycle": 1, + "task_kind": "candidate_discovery", + "domain": "candidate_discovery_specialist", + "proposal_set": [], + "empty": True, + "completed_at": "2026-08-28T01:03:00+00:00", + } + ], + } + + events = collect_v6_timeline(tmp_path, [], state=state, recorded_operations=[]) + + assert len(events) == 1 + assert events[0]["status"] == "succeeded" + assert events[0]["ext"]["source_arm"]["candidate_discovery_runs"] == [ + { + "task_id": "discovery-task", + "status": "empty", + "batch_id": None, + "gap_canonical_id": None, + "reason": None, + "candidates": [], + } + ] + + +def test_framework_timeline_does_not_copy_final_progress_into_earlier_retry(tmp_path): + state = { + "phase": "KERNEL_AGENT", + "macro_cycle": 1, + "phase_history": [ + { + "from_phase": "PRELUDE", + "to_phase": "FRAMEWORK_AGENT", + "cycle": 1, + "ts": "2026-08-28T01:00:00+00:00", + }, + { + "from_phase": "FRAMEWORK_AGENT", + "to_phase": "KERNEL_AGENT", + "cycle": 1, + "ts": "2026-08-28T01:10:00+00:00", + }, + ], + "framework_agent_phase_progress": [ + { + "candidate_id": "candidate-1", + "integrate_task_id": "integrate-2", + "status": "kept", + "kept": True, + "pre_tput": 100.0, + "post_tput": 120.0, + "gain_pct": 20.0, + "cycle": 1, + "ts": "2026-08-28T01:08:00+00:00", + } + ], + } + operations = [ + { + "operation_id": "operation-1", + "name": "integrate_patch", + "phase": "FRAMEWORK_AGENT", + "macro_cycle": 1, + "extensions": {"task_id": "integrate-1"}, + "outputs": { + "framework_agent_candidate_id": "candidate-1", + "specialist_task_id": "specialist-1", + "status": "apply_failed", + }, + "ended_at": "2026-08-28T01:04:00+00:00", + }, + { + "operation_id": "operation-2", + "name": "integrate_patch", + "phase": "FRAMEWORK_AGENT", + "macro_cycle": 1, + "extensions": {"task_id": "integrate-2"}, + "outputs": { + "framework_agent_candidate_id": "candidate-1", + "specialist_task_id": "specialist-1", + "status": "kept", + }, + "ended_at": "2026-08-28T01:08:00+00:00", + }, + ] + + event = collect_v6_timeline(tmp_path, [], state=state, recorded_operations=operations)[0] + attempts = event["ext"]["source_arm"]["attempts"] + + assert attempts[0]["status"] == "FAILED" + assert attempts[0]["before_tput"] is None + assert attempts[0]["after_tput"] is None + assert attempts[0]["local_gain_pct"] is None + assert attempts[1]["status"] == "KEEP" + assert attempts[1]["before_tput"] == 100.0 + assert attempts[1]["after_tput"] == 120.0 + assert attempts[1]["local_gain_pct"] == 20.0 From 1900f4ef638122c680536740b0e93bbfe0f90028 Mon Sep 17 00:00:00 2001 From: chenluo Date: Mon, 31 Aug 2026 14:27:44 +0800 Subject: [PATCH 4/7] fix: preserve Framework Agent SBD evidence --- .../breakdown/collectors/v6.py | 339 +++++++----------- .../breakdown/critic_reviews.py | 244 +++++++++++++ .../inference_optimizer/breakdown/exporter.py | 3 + .../breakdown/recorder/instrument.py | 49 ++- .../inference_optimizer/breakdown/schema.py | 6 + .../tests/test_framework_agent_authoring.py | 32 +- .../tests/test_sbd_v6_initial.py | 196 +++++++++- src/hyperloom/orchestrator/loop/dispatcher.py | 5 + src/hyperloom/orchestrator/loop/writeback.py | 17 +- src/hyperloom/orchestrator/phases/explore.py | 33 +- .../orchestrator/phases/framework.py | 15 +- .../orchestrator/roles/critic_agent.py | 2 + 12 files changed, 712 insertions(+), 229 deletions(-) create mode 100644 src/hyperloom/inference_optimizer/breakdown/critic_reviews.py diff --git a/src/hyperloom/inference_optimizer/breakdown/collectors/v6.py b/src/hyperloom/inference_optimizer/breakdown/collectors/v6.py index f6878f7ceb..033549bde8 100644 --- a/src/hyperloom/inference_optimizer/breakdown/collectors/v6.py +++ b/src/hyperloom/inference_optimizer/breakdown/collectors/v6.py @@ -2,13 +2,13 @@ from __future__ import annotations -import re from datetime import datetime from pathlib import Path from typing import Any from hyperloom.common.jsonio import read_json, read_jsonl +from ..critic_reviews import FRAMEWORK_REVIEW_FIELDS, normalize_framework_reviews from ...session.sbd_v6 import SCHEMA_VERSION_V6, read_timeline_events @@ -45,7 +45,6 @@ "framework_local_explore", } ) -_MACRO_CYCLE_RE = re.compile(r"(?:^|\s)macro_cycle\s*=\s*(-?\d+)(?=\s|$)") def _tool_versions(versions: Any) -> dict[str, str | None]: @@ -255,11 +254,6 @@ def _row_timestamp(row: dict[str, Any]) -> str: ) -def _prompt_macro_cycle(value: Any) -> int | None: - match = _MACRO_CYCLE_RE.search(str(value or "")) - return _optional_int(match.group(1)) if match else None - - def _timestamp_number(value: Any) -> float | None: text = str(value or "").strip() if not text: @@ -306,28 +300,35 @@ def _specialist_payloads(row: dict[str, Any]) -> tuple[dict[str, Any], ...]: def _is_framework_specialist(row: dict[str, Any], *, allow_legacy: bool) -> bool: declared_phase = _declared_source_phase(row) phase = declared_phase or str(row.get("phase") or "").strip().upper() - if not declared_phase and str(row.get("source") or "").strip().lower() == "specialist_recorder_hook": + legacy_recorder_hook = ( + not declared_phase and str(row.get("source") or "").strip().lower() == "specialist_recorder_hook" + ) + if legacy_recorder_hook: phase = "" if phase: return phase in _FRAMEWORK_PHASES agent = str(row.get("agent") or "").strip().lower() - if agent in {"framework_agent", "explore"}: - return True - if agent in {"enablement", "kernel", "kernel_agent", "prelude", "internal"}: - return False + if not legacy_recorder_hook: + if agent in {"framework_agent", "explore"}: + return True + if agent in {"enablement", "kernel", "kernel_agent", "prelude", "internal"}: + return False payloads = _specialist_payloads(row) if any(_optional_bool(payload.get("enablement")) is True for payload in payloads): return False if any( _optional_bool(payload.get("framework_agent_authoring")) is True + or _optional_bool(payload.get("candidate_discovery")) is True or bool(payload.get("framework_agent_candidate_id")) or bool(payload.get("framework_batch_id")) - or str(payload.get("task_kind") or "").strip().lower() in _AUTHORING_TASK_KINDS + or str(payload.get("task_kind") or "").strip().lower() in {*_AUTHORING_TASK_KINDS, "candidate_discovery"} for payload in payloads ): return True + if legacy_recorder_hook: + return False return allow_legacy @@ -536,15 +537,6 @@ def _framework_policy( evidence: dict[str, Any], ) -> dict[str, Any]: overrides = _mapping(state.get("plateau_overrides")) - stack_rebench_enabled = _optional_bool( - _operation_value( - operations, - ("outputs", "stack_rebench_enabled"), - ("outputs", "enable_stack_rebench"), - ("inputs", "stack_rebench_enabled"), - ("inputs", "enable_stack_rebench"), - ) - ) return { "keep_threshold_pct": _optional_float( _first( @@ -557,14 +549,6 @@ def _framework_policy( evidence.get("keep_threshold_pct"), ) ), - "stack_stable_threshold_pct": _optional_float( - _operation_value( - operations, - ("outputs", "stack_stable_threshold_pct"), - ("inputs", "stack_stable_threshold_pct"), - ) - ), - "stack_rebench_enabled": stack_rebench_enabled, "variant_timeout_sec": _optional_int( _first( _operation_value( @@ -742,17 +726,11 @@ def _config_variant(raw: dict[str, Any], source_index: dict[str, dict[str, Any]] name = str(_first(combined.get("name"), combined.get("variant_name")) or "") fingerprint = str(combined.get("fingerprint") or "") source = source_index.get(fingerprint) or source_index.get(name) or {} - outcome = str(combined.get("outcome") or "").strip().upper() - stack_tput = _optional_float(combined.get("stack_rebench_tput")) - stack_ran = _optional_bool(combined.get("stack_rebench_ran")) - if stack_ran is None and (stack_tput is not None or combined.get("stack_rebench_workspace")): - stack_ran = True - stack_stable = _optional_bool(combined.get("stack_rebench_stable")) - if stack_stable is None and stack_ran: - if outcome == "KEEP_UNSTABLE": - stack_stable = False - elif outcome == "KEEP": - stack_stable = True + raw_outcome = str(combined.get("outcome") or "").strip().upper() + outcome = "REVERT" if raw_outcome == "KEEP_UNSTABLE" else raw_outcome + stage = _first(combined.get("stage"), None) + if str(stage or "").strip().lower() == "stack_rebench": + stage = None return { "name": name, "fingerprint": fingerprint, @@ -784,21 +762,15 @@ def _config_variant(raw: dict[str, Any], source_index: dict[str, dict[str, Any]] "value": _optional_float(combined.get("accuracy")), "passed": _optional_bool(_first(combined.get("accuracy_pass"), combined.get("accuracy_passed"))), }, - "stack_rebench": { - "ran": stack_ran, - "tput": stack_tput, - "stable": stack_stable, - }, "outcome": outcome, "reason": _first(combined.get("reason"), None), - "stage": _first(combined.get("stage"), None), + "stage": stage, "failure": { "error_class": _first(combined.get("error_class"), None), "error_excerpt": _first(combined.get("error_excerpt"), combined.get("error"), None), }, "artifacts": { "workspace": _first(combined.get("workspace"), combined.get("single_workspace"), None), - "stack_rebench_workspace": _first(combined.get("stack_rebench_workspace"), None), "server_log_path": _first(combined.get("server_log_path"), None), "raw_result_path": _first(combined.get("raw_result_path"), None), }, @@ -1170,7 +1142,7 @@ def _append_run(run: dict[str, Any], source: dict[str, Any]) -> None: row, ) - failed_markers = 0 + failed_markers = sum(_specialist_status(row) == "failed" for row in discovery_rows) terminal_retry_rows: list[dict[str, Any]] = [] for row in _dict_rows(window.get("rows")): evidence = _mapping(row.get("evidence")) @@ -1201,7 +1173,12 @@ def _append_run(run: dict[str, Any], source: dict[str, Any]) -> None: }, row, ) - elif event == "framework_agent_phase_done" and reason == "discover_retries_exhausted": + elif event == "framework_agent_phase_done" and reason in { + "discover_retries_exhausted", + "no_candidates_and_discovery_exhausted", + }: + if _optional_int(evidence.get("failure_count")) in {None, 0}: + continue terminal_retry_rows.append(row) if failed_markers == 0: @@ -1310,17 +1287,24 @@ def _authoring_runs( if not task_id or task_id in used_tasks: continue candidate_id = str(progress.get("candidate_id") or candidate_map.get(task_id) or "") + reauthor_attempt = _optional_int(progress.get("reauthor_attempt")) runs.append( { "task_id": task_id, "candidate_id": candidate_id, - "kind": "local_authoring" if candidate_id.startswith("local_explore:") else "candidate_authoring", + "kind": ( + "reauthor" + if reauthor_attempt is not None and reauthor_attempt > 0 + else "local_authoring" + if candidate_id.startswith("local_explore:") + else "candidate_authoring" + ), "status": "failed" if str(progress.get("status") or "").lower() in {"dispatch_failed", "author_failed", "recovery_failed"} else "empty" if str(progress.get("provenance") or "").lower() == "authored_empty" else "succeeded", - "reauthor_attempt": _optional_int(progress.get("reauthor_attempt")), + "reauthor_attempt": reauthor_attempt, "specialist_domain": str(progress.get("domain") or ""), "gap_canonical_id": _first(progress.get("gap_canonical_id"), None), "patch_refs": _patch_refs(progress), @@ -1436,7 +1420,6 @@ def _source_attempt( elif patch_source == "specialist_authored": route = "author_via_specialist" parity = _mapping(outputs.get("switch_off_parity")) - stack_rebench = _mapping(outputs.get("stack_rebench")) files = _string_list(_first(outputs.get("target_files"), candidate.get("changed_files"), [])) applied_artifacts = _dict_rows(outputs.get("artifacts_applied")) if not files: @@ -1474,7 +1457,6 @@ def _source_attempt( "accuracy_passed": _optional_bool(outputs.get("accuracy_pass")), "keep_threshold_pct": _optional_float(outputs.get("keep_threshold_pct")), "switch_off_parity_passed": _optional_bool(_first(parity.get("ok"), parity.get("passed"))), - "stack_rebench_passed": _optional_bool(_first(stack_rebench.get("stable"), stack_rebench.get("ok"))), }, "framework_levers": _dict_rows(outputs.get("framework_levers")), "config_delta": { @@ -1660,160 +1642,83 @@ def _critic_review_rows( operations: list[dict[str, Any]], window: dict[str, Any], window_count: int, + critic_iterations: list[dict[str, Any]], ) -> list[dict[str, Any]]: - root = session_dir / "critic-workdir" - if not root.is_dir(): - return [] proposal_cycles, candidate_cycles = _critic_cycle_indexes(session_dir, warnings, state, operations) rows: list[dict[str, Any]] = [] - for workdir in sorted(root.iterdir(), key=lambda path: path.name): - if not workdir.is_dir(): - continue - loaded: dict[str, dict[str, Any]] = {} - failed = False - for name in ("request", "judge_bundle", "review", "emit"): - path = workdir / f"{name}.json" - if not path.is_file(): - loaded[name] = {} - continue - try: - loaded[name] = read_json(path, require_dict=True, strict=True) - except Exception as exc: - warnings.append(f"timeline.framework_agent.critic: failed to parse {path}: {exc!r}") - failed = True - break - if failed: - continue - request = loaded["request"] - judge = loaded["judge_bundle"] - review = loaded["review"] - emit = loaded["emit"] - review_phase = ( - str( - _first( - judge.get("phase"), - _nested(request, "context", "phase"), - _nested(judge, "merged_context", "phase"), - ) - or "" - ) - .strip() - .upper() - ) - if review_phase and review_phase not in _FRAMEWORK_PHASES: - continue - proposals = { - str(proposal.get("msg_id") or ""): proposal - for proposal in _dict_rows(judge.get("proposals")) - if proposal.get("msg_id") + seen: set[tuple[str, ...]] = set() + + def _append_review(row: dict[str, Any]) -> None: + proposal_id = str(row.get("proposal_msg_id") or "") + candidate_id = str(row.get("candidate_id") or "") + cycle_row = { + "cycle": _first( + row.get("macro_cycle"), + row.get("cycle"), + proposal_cycles.get(proposal_id), + candidate_cycles.get(candidate_id), + ), + "ts": row.get("ts"), } - effective = {} - envelope = _mapping(emit.get("intent_envelope")) - for intent in _dict_rows(envelope.get("intents")): - if str(intent.get("intent_type") or "") != "review_verdict": - continue - payload = _mapping(intent.get("payload")) - target = str(payload.get("target_proposal_msg_id") or "") - if target: - effective[target] = payload - for verdict_row in _dict_rows(review.get("review_verdicts")): - proposal_id = str(verdict_row.get("target_proposal_msg_id") or "") - proposal = proposals.get(proposal_id, {}) - payload = _mapping(proposal.get("payload")) - params = _mapping(payload.get("params")) - candidate = _mapping(_first(payload.get("candidate"), params.get("candidate"))) - action = str( - _first(proposal.get("action_name"), payload.get("action_name"), payload.get("kind")) or "" - ).lower() - candidate_id = str( - _first( - payload.get("framework_agent_candidate_id"), - params.get("framework_agent_candidate_id"), - _candidate_id(candidate), - ) - or "" + if not _row_in_window(cycle_row, window, window_count): + return + projected = {field: row.get(field) for field in FRAMEWORK_REVIEW_FIELDS} + if projected.get("review_path"): + projected["review_path"] = str(projected["review_path"]).replace("\\", "/") + identity = tuple( + str(projected.get(field) or "") + for field in ( + "proposal_msg_id", + "candidate_id", + "variant_name", + "arm", + "target_action", + "verdict", + "effective_verdict", + "ts", ) - if action in {"params", "backends", "explore"}: - arm = "config" - target_action = "explore" - elif action in {"framework_agent", "integrate", "integrate_patch"}: - arm = "source" - target_action = "integrate_patch" - elif action == "specialist": - task_kind = str(params.get("task_kind") or "").strip().lower() - source_marker = bool( - candidate_id - or _optional_bool(params.get("framework_agent_authoring")) is True - or _optional_bool(params.get("candidate_discovery")) is True - or task_kind in _AUTHORING_TASK_KINDS - or bool(_patch_refs(params)) - ) - arm = "source" if source_marker else "config" - target_action = "specialist" - else: + ) + if identity in seen: + return + seen.add(identity) + rows.append(projected) + + for iteration in critic_iterations: + for durable_row in _dict_rows(iteration.get("framework_reviews")): + durable_row.setdefault("ts", iteration.get("ts")) + durable_row.setdefault("review_path", iteration.get("review_path")) + durable_row.setdefault("phase", iteration.get("phase")) + durable_row.setdefault("macro_cycle", _first(iteration.get("macro_cycle"), iteration.get("cycle"))) + _append_review(durable_row) + + root = session_dir / "critic-workdir" + if root.is_dir(): + for workdir in sorted(root.iterdir(), key=lambda path: path.name): + if not workdir.is_dir(): continue - cycle_row = { - "cycle": _first( - payload.get("cycle"), - payload.get("macro_cycle"), - params.get("cycle"), - params.get("macro_cycle"), - proposal.get("cycle"), - proposal.get("macro_cycle"), - proposal_cycles.get(proposal_id), - candidate_cycles.get(candidate_id), - _nested(request, "context", "macro_cycle"), - _nested(request, "context", "cycle"), - _prompt_macro_cycle(request.get("raw_prompt")), - _nested(judge, "merged_context", "macro_cycle"), - _nested(judge, "merged_context", "cycle"), - ), - "ts": _first(verdict_row.get("ts"), emit.get("ts"), review.get("ts")), - } - if not _row_in_window(cycle_row, window, window_count): + loaded: dict[str, dict[str, Any]] = {} + failed = False + for name in ("request", "judge_bundle", "review", "emit"): + path = workdir / f"{name}.json" + if not path.is_file(): + loaded[name] = {} + continue + try: + loaded[name] = read_json(path, require_dict=True, strict=True) + except Exception as exc: + warnings.append(f"timeline.framework_agent.critic: failed to parse {path}: {exc!r}") + failed = True + break + if failed: continue - effective_row = _mapping(effective.get(proposal_id)) - risks = [ - { - "severity": str(risk.get("severity") or ""), - "risk": str(_first(risk.get("risk"), risk.get("summary"), risk.get("reason")) or ""), - } - for risk in _dict_rows(verdict_row.get("risks")) - ] - followup_task_ids = _string_list( - _first(effective_row.get("followup_task_ids"), verdict_row.get("followup_task_ids"), []) - ) - rows.append( - { - "proposal_msg_id": proposal_id, - "candidate_id": candidate_id or None, - "variant_name": _first(payload.get("variant_name"), params.get("variant_name"), None), - "arm": arm, - "target_action": target_action, - "source": "critic" - if str(verdict_row.get("source") or "critic") == "critic" - else "critic_unavailable", - "verdict": str(verdict_row.get("verdict") or ""), - "effective_verdict": str( - _first( - effective_row.get("verdict"), - verdict_row.get("effective_verdict"), - verdict_row.get("verdict"), - ) - or "" - ), - "reasoning": str(verdict_row.get("reasoning") or ""), - "confidence": _first(verdict_row.get("confidence"), None), - "failure_reason_code": _first(verdict_row.get("failure_reason_code"), None), - "required_evidence": _string_list(verdict_row.get("required_evidence")), - "risks": risks, - "advice_text": _first(verdict_row.get("advice_text"), effective_row.get("advice_text"), None), - "alternative_action": _first(verdict_row.get("alternative_action"), None), - "followup_task_ids": followup_task_ids, - "ts": str(_first(verdict_row.get("ts"), emit.get("ts"), review.get("ts")) or ""), - "review_path": (workdir / "review.json").relative_to(session_dir).as_posix(), - } - ) + for review_row in normalize_framework_reviews( + request=loaded["request"], + judge_bundle=loaded["judge_bundle"], + review=loaded["review"], + emit=loaded["emit"], + review_path=(workdir / "review.json").relative_to(session_dir).as_posix(), + ): + _append_review(review_row) rows.sort(key=lambda row: row.get("ts") or "") return rows @@ -1910,7 +1815,10 @@ def _framework_exit( } -def _framework_failure(window: dict[str, Any]) -> dict[str, Any]: +def _framework_failure( + window: dict[str, Any], + specialist_rows: list[dict[str, Any]], +) -> dict[str, Any]: exit_row = _mapping(window.get("exit_row")) evidence = _mapping(exit_row.get("evidence")) reason = str(exit_row.get("reason") or "").lower() @@ -1928,7 +1836,8 @@ def _framework_failure(window: dict[str, Any]) -> dict[str, Any]: row for row in reversed(rows) if _nested(row, "evidence", "event") == "framework_agent_phase_done" - and str(row.get("reason") or "") == "discover_retries_exhausted" + and str(row.get("reason") or "") in {"discover_retries_exhausted", "no_candidates_and_discovery_exhausted"} + and _optional_int(_nested(row, "evidence", "failure_count")) not in {None, 0} ), None, ) @@ -1936,19 +1845,34 @@ def _framework_failure(window: dict[str, Any]) -> dict[str, Any]: return {"failed_task_id": None, "error_class": None, "error": None} failed_discovery = next( + ( + row + for row in reversed(specialist_rows) + if _specialist_role(row, {}, set()) == "discovery" and _specialist_status(row) == "failed" + ), + None, + ) or next( (row for row in reversed(rows) if _nested(row, "evidence", "event") == "framework_agent_discover_failed"), terminal_discovery_failure, ) failure_evidence = _mapping(failed_discovery.get("evidence")) return { "failed_task_id": _first( + failed_discovery.get("task_id"), failure_evidence.get("failed_task_id"), failure_evidence.get("task_id"), None, ), - "error_class": _first(failure_evidence.get("error_class"), None), + "error_class": _first( + failed_discovery.get("error_class"), + failure_evidence.get("error_class"), + "candidate_discovery_failed", + ), "error": _first( + failed_discovery.get("error"), + failed_discovery.get("run_error"), failure_evidence.get("error"), + failed_discovery.get("reason"), terminal_discovery_failure.get("reason"), None, ), @@ -1959,6 +1883,7 @@ def _framework_event( session_dir: Path, state: dict[str, Any], recorded_operations: list[dict[str, Any]], + critic_iterations: list[dict[str, Any]], warnings: list[str], window: dict[str, Any], window_count: int, @@ -2011,9 +1936,10 @@ def _framework_event( recorded_operations, window, window_count, + critic_iterations, ) exit_details = _framework_exit(window, config_arm, source_arm) - failure = _framework_failure(window) + failure = _framework_failure(window, specialist_rows) has_work = bool( config_arm["specialist_runs"] or config_arm["rounds"] @@ -2054,11 +1980,13 @@ def collect_v6_timeline( *, state: dict[str, Any] | None = None, recorded_operations: list[dict[str, Any]] | None = None, + critic_iterations: list[dict[str, Any]] | None = None, ) -> list[dict[str, Any]]: """Load durable events and project framework work without mutating V5 state.""" timeline = read_timeline_events(session_dir, warnings=warnings) state = state if isinstance(state, dict) else {} operations = [row for row in recorded_operations or [] if isinstance(row, dict)] + critic_iterations = [row for row in critic_iterations or [] if isinstance(row, dict)] windows = _framework_windows(state, operations) for window in windows: timeline.append( @@ -2066,6 +1994,7 @@ def collect_v6_timeline( session_dir, state, operations, + critic_iterations, warnings, window, len(windows), diff --git a/src/hyperloom/inference_optimizer/breakdown/critic_reviews.py b/src/hyperloom/inference_optimizer/breakdown/critic_reviews.py new file mode 100644 index 0000000000..3e28544e02 --- /dev/null +++ b/src/hyperloom/inference_optimizer/breakdown/critic_reviews.py @@ -0,0 +1,244 @@ +"""Normalize durable Framework Agent critic-review evidence.""" + +from __future__ import annotations + +import re +from typing import Any + + +FRAMEWORK_REVIEW_FIELDS = ( + "proposal_msg_id", + "candidate_id", + "variant_name", + "arm", + "target_action", + "source", + "verdict", + "effective_verdict", + "reasoning", + "confidence", + "failure_reason_code", + "required_evidence", + "risks", + "advice_text", + "alternative_action", + "followup_task_ids", + "ts", + "review_path", +) + +_FRAMEWORK_PHASES = frozenset({"FRAMEWORK_AGENT", "EXPLORE"}) +_AUTHORING_TASK_KINDS = frozenset( + { + "explore_apply_retry", + "framework_authoring", + "framework_local_explore", + } +) +_MACRO_CYCLE_RE = re.compile(r"(?:^|\s)macro_cycle\s*=\s*(-?\d+)(?=\s|$)") + + +def _mapping(value: Any) -> dict[str, Any]: + return dict(value) if isinstance(value, dict) else {} + + +def _dict_rows(value: Any) -> list[dict[str, Any]]: + return [dict(row) for row in value or [] if isinstance(row, dict)] if isinstance(value, list) else [] + + +def _first(*values: Any) -> Any: + return next((value for value in values if value is not None and value != ""), None) + + +def _nested(value: Any, *keys: str) -> Any: + current = value + for key in keys: + if not isinstance(current, dict): + return None + current = current.get(key) + return current + + +def _optional_bool(value: Any) -> bool | None: + if isinstance(value, bool): + return value + if isinstance(value, str): + normalized = value.strip().lower() + if normalized in {"1", "true", "yes", "on"}: + return True + if normalized in {"0", "false", "no", "off"}: + return False + if isinstance(value, (int, float)): + return bool(value) + return None + + +def _string_list(value: Any) -> list[str]: + if not isinstance(value, (list, tuple, set)): + return [] + return [str(item) for item in value if str(item)] + + +def _candidate_id(value: Any) -> str: + candidate = _mapping(value) + return str( + _first( + candidate.get("candidate_id"), + candidate.get("pr_url"), + candidate.get("url"), + candidate.get("ref"), + candidate.get("head_sha"), + ) + or "" + ) + + +def _prompt_macro_cycle(value: Any) -> int | None: + match = _MACRO_CYCLE_RE.search(str(value or "")) + if match is None: + return None + try: + return int(match.group(1)) + except (TypeError, ValueError): + return None + + +def _has_patch_refs(params: dict[str, Any]) -> bool: + return any(isinstance(params.get(key), list) and bool(params.get(key)) for key in ("patch_refs", "patches_written")) + + +def normalize_framework_reviews( + *, + request: dict[str, Any] | None, + judge_bundle: dict[str, Any] | None, + review: dict[str, Any] | None, + emit: dict[str, Any] | None, + review_path: str | None, +) -> list[dict[str, Any]]: + """Return compact V6 Framework review rows from one Critic iteration.""" + request = _mapping(request) + judge = _mapping(judge_bundle) + review = _mapping(review) + emit = _mapping(emit) + normalized_review_path = str(review_path or "").replace("\\", "/") or None + review_phase = ( + str( + _first( + judge.get("phase"), + _nested(request, "context", "phase"), + _nested(judge, "merged_context", "phase"), + ) + or "" + ) + .strip() + .upper() + ) + if review_phase and review_phase not in _FRAMEWORK_PHASES: + return [] + + proposals = { + str(proposal.get("msg_id") or ""): proposal + for proposal in _dict_rows(judge.get("proposals")) + if proposal.get("msg_id") + } + effective: dict[str, dict[str, Any]] = {} + for intent in _dict_rows(_nested(emit, "intent_envelope", "intents")): + if str(intent.get("intent_type") or "") != "review_verdict": + continue + payload = _mapping(intent.get("payload")) + target = str(payload.get("target_proposal_msg_id") or "") + if target: + effective[target] = payload + + rows: list[dict[str, Any]] = [] + for verdict_row in _dict_rows(review.get("review_verdicts")): + proposal_id = str(verdict_row.get("target_proposal_msg_id") or "") + proposal = proposals.get(proposal_id, {}) + payload = _mapping(proposal.get("payload")) + params = _mapping(payload.get("params")) + candidate = _mapping(_first(payload.get("candidate"), params.get("candidate"))) + action = str(_first(proposal.get("action_name"), payload.get("action_name"), payload.get("kind")) or "").lower() + candidate_id = str( + _first( + payload.get("framework_agent_candidate_id"), + params.get("framework_agent_candidate_id"), + _candidate_id(candidate), + ) + or "" + ) + if action in {"params", "backends", "explore"}: + arm = "config" + target_action = "explore" + elif action in {"framework_agent", "integrate", "integrate_patch"}: + arm = "source" + target_action = "integrate_patch" + elif action == "specialist": + task_kind = str(params.get("task_kind") or "").strip().lower() + source_marker = bool( + candidate_id + or _optional_bool(params.get("framework_agent_authoring")) is True + or _optional_bool(params.get("candidate_discovery")) is True + or task_kind in _AUTHORING_TASK_KINDS + or _has_patch_refs(params) + ) + arm = "source" if source_marker else "config" + target_action = "specialist" + else: + continue + + effective_row = effective.get(proposal_id, {}) + rows.append( + { + "proposal_msg_id": proposal_id, + "candidate_id": candidate_id or None, + "variant_name": _first(payload.get("variant_name"), params.get("variant_name"), None), + "arm": arm, + "target_action": target_action, + "source": "critic" if str(verdict_row.get("source") or "critic") == "critic" else "critic_unavailable", + "verdict": str(verdict_row.get("verdict") or ""), + "effective_verdict": str( + _first( + effective_row.get("verdict"), + verdict_row.get("effective_verdict"), + verdict_row.get("verdict"), + ) + or "" + ), + "reasoning": str(verdict_row.get("reasoning") or ""), + "confidence": _first(verdict_row.get("confidence"), None), + "failure_reason_code": _first(verdict_row.get("failure_reason_code"), None), + "required_evidence": _string_list(verdict_row.get("required_evidence")), + "risks": [ + { + "severity": str(risk.get("severity") or ""), + "risk": str(_first(risk.get("risk"), risk.get("summary"), risk.get("reason")) or ""), + } + for risk in _dict_rows(verdict_row.get("risks")) + ], + "advice_text": _first(verdict_row.get("advice_text"), effective_row.get("advice_text"), None), + "alternative_action": _first(verdict_row.get("alternative_action"), None), + "followup_task_ids": _string_list( + _first(effective_row.get("followup_task_ids"), verdict_row.get("followup_task_ids"), []) + ), + "ts": str(_first(verdict_row.get("ts"), emit.get("ts"), review.get("ts")) or ""), + "review_path": normalized_review_path, + "phase": review_phase, + "macro_cycle": _first( + payload.get("cycle"), + payload.get("macro_cycle"), + params.get("cycle"), + params.get("macro_cycle"), + proposal.get("cycle"), + proposal.get("macro_cycle"), + _nested(request, "context", "macro_cycle"), + _nested(request, "context", "cycle"), + _prompt_macro_cycle(request.get("raw_prompt")), + _nested(judge, "merged_context", "macro_cycle"), + _nested(judge, "merged_context", "cycle"), + ), + } + ) + return rows + + +__all__ = ["FRAMEWORK_REVIEW_FIELDS", "normalize_framework_reviews"] diff --git a/src/hyperloom/inference_optimizer/breakdown/exporter.py b/src/hyperloom/inference_optimizer/breakdown/exporter.py index 233c9375e0..f9ed1078fd 100644 --- a/src/hyperloom/inference_optimizer/breakdown/exporter.py +++ b/src/hyperloom/inference_optimizer/breakdown/exporter.py @@ -568,6 +568,9 @@ def _pick(section: str, collector_value: Any) -> Any: v6_warnings, state=state, recorded_operations=recorded_operations, + critic_iterations=( + critic_robustness.get("critic_iterations", []) if isinstance(critic_robustness, dict) else [] + ), ), v6_warnings, default=[], diff --git a/src/hyperloom/inference_optimizer/breakdown/recorder/instrument.py b/src/hyperloom/inference_optimizer/breakdown/recorder/instrument.py index 45a5eeab0f..03196ece15 100644 --- a/src/hyperloom/inference_optimizer/breakdown/recorder/instrument.py +++ b/src/hyperloom/inference_optimizer/breakdown/recorder/instrument.py @@ -42,6 +42,7 @@ from hyperloom.common.timeutil import iso_z, now_iso from ..agent_ownership import UNATTRIBUTED, agent_from_phase, patch_author +from ..critic_reviews import normalize_framework_reviews from .trace import trace_skip log = logging.getLogger(__name__) @@ -3750,6 +3751,8 @@ def record_critic_iteration( session_dir: Path | str | None, *, iter_n: int, + request: dict[str, Any] | None = None, + judge_bundle: dict[str, Any] | None = None, review: dict[str, Any] | None, emit: dict[str, Any] | None, workdir: Path | str | None, @@ -3758,9 +3761,10 @@ def record_critic_iteration( ) -> None: """Record one ``critic_robustness.critic_iterations`` item. - Recorded per-iteration (idempotent on ``iter_n``) so the critic backend's - workdir pruning never erases history; payload mirrors - ``collectors.collect_critic_robustness``. + Recorded per-iteration under a session-unique identity so workdir pruning + and resume-time turn-index reuse never erase history; payload mirrors + ``collectors.collect_critic_robustness`` and retains normalized Framework + review rows for the V6 timeline. ``kb_priors`` (when provided) carries the per-iteration KB integration trace: whether the historical priors were used, the request, the response, @@ -3770,7 +3774,9 @@ def record_critic_iteration( Args: session_dir (Path | str | None): the session directory; a falsy value is a no-op. - iter_n (int): the critic iteration number (idempotency key). + iter_n (int): the process-local critic iteration number. + request (dict[str, Any] | None): the critic request payload. + judge_bundle (dict[str, Any] | None): the proposal bundle reviewed. review (dict[str, Any] | None): the critic review payload. emit (dict[str, Any] | None): the critic emit payload. workdir (Path | str | None): the critic backend workdir holding the @@ -3786,6 +3792,14 @@ def record_critic_iteration( review = review if isinstance(review, dict) else {} emit = emit if isinstance(emit, dict) else {} wd = Path(workdir) if workdir else None + request = request if isinstance(request, dict) else read_json(wd / "request.json", default={}) if wd else {} + judge_bundle = ( + judge_bundle + if isinstance(judge_bundle, dict) + else read_json(wd / "judge_bundle.json", default={}) + if wd + else {} + ) payload = { "iter": int(iter_n), "ts": str(emit.get("ts") or review.get("ts") or ""), @@ -3798,14 +3812,35 @@ def record_critic_iteration( "review_path": _rel(wd / "review.json", session_dir) if wd else None, "kb_writes": list(emit.get("kb_writes") or []) if isinstance(emit.get("kb_writes"), list) else [], } + framework_reviews = normalize_framework_reviews( + request=request, + judge_bundle=judge_bundle, + review=review, + emit=emit, + review_path=_rel(wd / "review.json", session_dir).replace("\\", "/") if wd else None, + ) + if framework_reviews: + payload["framework_reviews"] = framework_reviews if isinstance(kb_priors, dict) and kb_priors: payload["kb_priors"] = kb_priors + iteration_id = _stable_id( + "critic-iteration", + iter_n, + payload.get("ts"), + [row.get("proposal_msg_id") for row in framework_reviews], + payload.get("topic"), + request, + judge_bundle, + review, + emit, + ) + payload["iteration_id"] = iteration_id _recorder(session_dir, producer).record_item( "critic_iterations", payload, - key=str(iter_n), + key=iteration_id, ) - operation_id = _stable_id("op", "critic", iter_n) + operation_id = _stable_id("op", iteration_id) artifact_refs: list[str] = [] for name in ("request_path", "judge_bundle_path", "emit_path", "review_path"): path = payload.get(name) @@ -3864,7 +3899,7 @@ def record_critic_iteration( write_key = ( write.get("write_id") or write.get("point_id") or write.get("edge_id") or write.get("kind") or index ) - write_operation_id = _stable_id("op", "kb-write", iter_n, write_key) + write_operation_id = _stable_id("op", "kb-write", iteration_id, write_key) result_payload = write.get("result") if isinstance(write.get("result"), Mapping) else {} write_status = _operation_status(result_payload.get("status") or write.get("status") or "succeeded") record_operation( diff --git a/src/hyperloom/inference_optimizer/breakdown/schema.py b/src/hyperloom/inference_optimizer/breakdown/schema.py index de0a99adbc..b6c9a8a092 100644 --- a/src/hyperloom/inference_optimizer/breakdown/schema.py +++ b/src/hyperloom/inference_optimizer/breakdown/schema.py @@ -1064,6 +1064,8 @@ class CriticIteration(TypedDict, total=False): """One critic-agent review pass over a proposed change. Attributes: + iteration_id (str): Stable session-unique identity for this persisted + review pass, including resume-time reuse of ``iter``. iter (int): Iteration index. ts (str): ISO UTC timestamp of the review. topic (str): What was reviewed (e.g. ``kernel_opt:k001`` / ``backends:flag_X``). @@ -1074,8 +1076,11 @@ class CriticIteration(TypedDict, total=False): judge_bundle_path (str): Path to the judge bundle. emit_path (str): Path to the emitted review output. review_path (str): Path to the review record. + framework_reviews (list[dict[str, Any]]): Durable normalized V6 + Framework review rows. """ + iteration_id: str iter: int ts: str topic: str # what was reviewed (kernel_opt:k001, backends:flag_X, ...) @@ -1085,6 +1090,7 @@ class CriticIteration(TypedDict, total=False): judge_bundle_path: str emit_path: str review_path: str + framework_reviews: list[dict[str, Any]] class RobustnessSignal(TypedDict, total=False): diff --git a/src/hyperloom/inference_optimizer/tests/test_framework_agent_authoring.py b/src/hyperloom/inference_optimizer/tests/test_framework_agent_authoring.py index c08732bf9d..1a9daa4c39 100644 --- a/src/hyperloom/inference_optimizer/tests/test_framework_agent_authoring.py +++ b/src/hyperloom/inference_optimizer/tests/test_framework_agent_authoring.py @@ -246,6 +246,33 @@ def test_materialize_authoring_disabled_runs_diff_track_only( assert kinds == ["integrate_patch"] +def test_reauthor_attempt_propagates_into_specialist_and_integrate_params(tmp_path: Path): + from hyperloom.orchestrator.phases.explore import _forward_integrate_source + + stub = _Stub(tmp_path, authoring=True) + + task_id = asyncio.run( + stub._enqueue_framework_agent_authoring_specialist( + dict(_CANDIDATE), + {}, + reauthor_attempt=1, + ) + ) + + specialist_task = stub.tasks._queued[-1] + assert task_id == specialist_task.task_id + assert specialist_task.params["reauthor_attempt"] == 1 + round_entry = stub._build_specialist_round_entry( + task=specialist_task, + done_payload={"proposal_set": [], "empty": True}, + source=f"specialist:{task_id}", + ) + assert round_entry["reauthor_attempt"] == 1 + integrate_params: dict[str, Any] = {} + _forward_integrate_source(specialist_task.params, integrate_params) + assert integrate_params["reauthor_attempt"] == 1 + + @pytest.mark.parametrize( ("route", "authoring", "kinds"), [ @@ -378,6 +405,7 @@ def test_record_authored_outcome_writes_progress_and_rolls_max_gain( "framework_agent_candidate_id": "pr-42", "framework_batch_id": "b1", "specialist_task_id": "s-1", + "reauthor_attempt": 1, }, ) result = SimpleNamespace( @@ -398,6 +426,7 @@ def test_record_authored_outcome_writes_progress_and_rolls_max_gain( assert row["provenance"] == "authored" assert row["candidate_id"] == "pr-42" assert row["gain_pct"] == pytest.approx(6.5) + assert row["reauthor_attempt"] == 1 assert stub.shared_state.framework_agent_batches[0]["max_gain_pct_observed_in_batch"] == pytest.approx(6.5) @@ -621,7 +650,7 @@ async def _noop_async(*_args: Any, **_kwargs: Any) -> None: task = SimpleNamespace( task_id="integrate-cross-phase", kind="integrate_patch", - params={"framework_agent_authoring": True}, + params={"framework_agent_authoring": True, "reauthor_attempt": 1}, ) result = SubAgentResult( task_id=task.task_id, @@ -632,6 +661,7 @@ async def _noop_async(*_args: Any, **_kwargs: Any) -> None: await DispatcherCollaborator(stub)._reap_dispatched_task(task, result, None) assert recorded == ["reverted"] + assert result.result["reauthor_attempt"] == 1 def test_empty_outcome_fires_when_patch_dropped_by_vetting(tmp_path: Path): diff --git a/src/hyperloom/inference_optimizer/tests/test_sbd_v6_initial.py b/src/hyperloom/inference_optimizer/tests/test_sbd_v6_initial.py index ca90fd08f6..9b301b2d0f 100644 --- a/src/hyperloom/inference_optimizer/tests/test_sbd_v6_initial.py +++ b/src/hyperloom/inference_optimizer/tests/test_sbd_v6_initial.py @@ -6,7 +6,9 @@ import asyncio import json import os +import shutil from pathlib import Path +from types import SimpleNamespace import pytest @@ -1018,11 +1020,11 @@ def test_framework_timeline_merges_legacy_framework_and_explore(tmp_path): assert event["start_time"] == "2026-08-27T01:00:00+00:00" assert event["end_time"] == "2026-08-27T01:20:00+00:00" assert "summary" not in event - assert event["ext"]["policy"]["stack_rebench_enabled"] is None + assert "stack_rebench_enabled" not in event["ext"]["policy"] assert event["ext"]["config_arm"]["rounds"][0]["workload_signature"] == "qwen-tp8-c64" assert event["ext"]["config_arm"]["rounds"][0]["input_stack"]["extra_server_args"] == "--base-flag" variant = event["ext"]["config_arm"]["rounds"][0]["variants"][0] - assert variant["stack_rebench"] == {"ran": True, "tput": 104.0, "stable": True} + assert "stack_rebench" not in variant attempt = event["ext"]["source_arm"]["attempts"][0] assert attempt["patch_source"] == "upstream_pr" assert attempt["lever_kind"] == "upstream_pr" @@ -1101,6 +1103,10 @@ def test_framework_timeline_projects_pr1301_source_and_critic_data(tmp_path): "cycle": 2, "completed_at": "2026-08-27T02:08:00+00:00", "proposal_set": [{"patches_written": ["patches/pr-9.patch"]}], + "task_kind": "framework_authoring", + "framework_agent_authoring": True, + "framework_agent_candidate_id": "https://example.test/pr/9", + "reauthor_attempt": 1, }, ], "framework_agent_batches": [ @@ -1129,6 +1135,7 @@ def test_framework_timeline_projects_pr1301_source_and_critic_data(tmp_path): "post_tput": 104.0, "specialist_task_id": "author-task-1", "integrate_task_id": "integrate-task-1", + "reauthor_attempt": 1, "cycle": 2, "ts": "2026-08-27T02:15:00+00:00", } @@ -1147,6 +1154,7 @@ def test_framework_timeline_projects_pr1301_source_and_critic_data(tmp_path): "status": "kept", "framework_agent_authoring": True, "specialist_task_id": "author-task-1", + "reauthor_attempt": 1, "base_tput": 100.0, "output_throughput": 104.0, "delta_pct": 4.0, @@ -1227,6 +1235,8 @@ def test_framework_timeline_projects_pr1301_source_and_critic_data(tmp_path): ] authoring = event["ext"]["source_arm"]["authoring_runs"][0] assert authoring["candidate_id"] == "https://example.test/pr/9" + assert authoring["kind"] == "reauthor" + assert authoring["reauthor_attempt"] == 1 assert authoring["patch_refs"] == ["patches/pr-9.patch"] attempt = event["ext"]["source_arm"]["attempts"][0] assert attempt["patch_source"] == "specialist_authored" @@ -1237,7 +1247,6 @@ def test_framework_timeline_projects_pr1301_source_and_critic_data(tmp_path): "accuracy_passed": True, "keep_threshold_pct": 1.0, "switch_off_parity_passed": True, - "stack_rebench_passed": True, } review = event["ext"]["critic_reviews"][0] assert review["arm"] == "source" @@ -1364,6 +1373,7 @@ def test_framework_timeline_ignores_kernel_specialist_without_framework_evidence "kind": "specialist", "name": "specialist round kernel-specialist", "phase": "EXPLORE", + "agent": "explore", "source": "specialist_recorder_hook", "macro_cycle": 4, "status": "succeeded", @@ -1690,7 +1700,7 @@ def test_framework_timeline_marks_exhausted_discovery_retries_failed(tmp_path): assert event["status"] == "failed" assert event["ext"]["failure"] == { "failed_task_id": None, - "error_class": None, + "error_class": "candidate_discovery_failed", "error": "TimeoutError('last')", } @@ -1964,3 +1974,181 @@ def test_framework_timeline_does_not_copy_final_progress_into_earlier_retry(tmp_ assert attempts[1]["before_tput"] == 100.0 assert attempts[1]["after_tput"] == 120.0 assert attempts[1]["local_gain_pct"] == 20.0 + + +def test_failed_discovery_uses_task_params_and_actual_terminal_reason(tmp_path): + from hyperloom.orchestrator.phases.explore import ExplorePhase + + phase = object.__new__(ExplorePhase) + phase.shared_state = SimpleNamespace(phase="KERNEL_AGENT") + task = SimpleNamespace( + task_id="discovery-task", + params={ + "source_phase": "FRAMEWORK_AGENT", + "domain": "candidate_discovery_specialist", + "task_kind": "candidate_discovery", + "candidate_discovery": True, + "gap_canonical_id": "gap.framework.candidate_discovery.sglang", + }, + ) + entry = phase._build_specialist_round_entry( + task=task, + done_payload={}, + source="specialist:discovery-task", + run_error="TimeoutError('upstream unavailable')", + ) + state = { + "phase": "FRAMEWORK_AGENT", + "macro_cycle": 0, + "phase_history": [ + { + "from_phase": "PRELUDE", + "to_phase": "FRAMEWORK_AGENT", + "cycle": 0, + "ts": "2026-08-28T00:00:00+00:00", + }, + { + "from_phase": "FRAMEWORK_AGENT", + "to_phase": "FRAMEWORK_AGENT", + "cycle": 0, + "reason": "no_candidates_and_discovery_exhausted", + "evidence": { + "event": "framework_agent_phase_done", + "failure_count": 1, + "retry_limit": 3, + }, + "ts": "2026-08-28T00:05:00+00:00", + }, + ], + "specialist_rounds": [entry], + } + + event = collect_v6_timeline(tmp_path, [], state=state, recorded_operations=[])[0] + + assert entry["domain"] == "candidate_discovery_specialist" + assert entry["task_kind"] == "candidate_discovery" + assert entry["candidate_discovery"] is True + assert entry["status"] == "failed" + assert entry["run_error"] == "TimeoutError('upstream unavailable')" + assert event["ext"]["source_arm"]["candidate_discovery_runs"] == [ + { + "task_id": "discovery-task", + "status": "failed", + "batch_id": None, + "gap_canonical_id": "gap.framework.candidate_discovery.sglang", + "reason": "TimeoutError('upstream unavailable')", + "candidates": [], + } + ] + assert event["status"] == "failed" + assert event["ext"]["failure"] == { + "failed_task_id": "discovery-task", + "error_class": "candidate_discovery_failed", + "error": "TimeoutError('upstream unavailable')", + } + + +def test_framework_critic_reviews_survive_pruning_and_reused_iteration_number(tmp_path): + from hyperloom.inference_optimizer.breakdown.recorder import instrument + from hyperloom.inference_optimizer.breakdown.recorder.assembler import assemble_parts + + state = { + "session_id": "durable-critic", + "phase": "KERNEL_AGENT", + "macro_cycle": 0, + "phase_history": [ + { + "from_phase": "PRELUDE", + "to_phase": "FRAMEWORK_AGENT", + "cycle": 0, + "ts": "2026-08-28T01:00:00+00:00", + }, + { + "from_phase": "FRAMEWORK_AGENT", + "to_phase": "KERNEL_AGENT", + "cycle": 0, + "ts": "2026-08-28T01:10:00+00:00", + }, + ], + } + _write_json(tmp_path / "state.json", state) + _write_json(tmp_path / "manifest.json", {"session_id": "durable-critic"}) + workdir = tmp_path / "critic-workdir" / "000000" + + for index in (1, 2): + proposal_id = f"proposal-{index}" + timestamp = f"2026-08-28T01:0{index}:00+00:00" + request = {"context": {"phase": "FRAMEWORK_AGENT", "macro_cycle": 0}} + judge_bundle = { + "phase": "FRAMEWORK_AGENT", + "proposals": [ + { + "msg_id": proposal_id, + "action_name": "integrate_patch", + "payload": {"framework_agent_candidate_id": f"candidate-{index}"}, + } + ], + } + review = { + "ts": timestamp, + "review_verdicts": [ + { + "target_proposal_msg_id": proposal_id, + "verdict": "approve", + "reasoning": f"review {index}", + } + ], + } + emit = { + "ts": timestamp, + "intent_envelope": { + "intents": [ + { + "intent_type": "review_verdict", + "payload": { + "target_proposal_msg_id": proposal_id, + "verdict": "approve", + }, + } + ] + }, + } + for name, payload in ( + ("request", request), + ("judge_bundle", judge_bundle), + ("review", review), + ("emit", emit), + ): + _write_json(workdir / f"{name}.json", payload) + instrument.record_critic_iteration( + tmp_path, + iter_n=0, + request=request, + judge_bundle=judge_bundle, + review=review, + emit=emit, + workdir=workdir, + ) + + assembled = assemble_parts(tmp_path) + critic_iterations = assembled["critic_robustness"]["critic_iterations"] + assert len(critic_iterations) == 2 + assert len({row["iteration_id"] for row in critic_iterations}) == 2 + + timeline = collect_v6_timeline( + tmp_path, + [], + state=state, + recorded_operations=assembled.get("operations", []), + critic_iterations=critic_iterations, + ) + assert [row["proposal_msg_id"] for row in timeline[0]["ext"]["critic_reviews"]] == [ + "proposal-1", + "proposal-2", + ] + + shutil.rmtree(tmp_path / "critic-workdir") + breakdown = exporter.build(tmp_path) + reviews = breakdown["timeline"][0]["ext"]["critic_reviews"] + assert [row["proposal_msg_id"] for row in reviews] == ["proposal-1", "proposal-2"] + assert all("\\" not in row["review_path"] for row in reviews) diff --git a/src/hyperloom/orchestrator/loop/dispatcher.py b/src/hyperloom/orchestrator/loop/dispatcher.py index f9cf8e54ac..6369258374 100644 --- a/src/hyperloom/orchestrator/loop/dispatcher.py +++ b/src/hyperloom/orchestrator/loop/dispatcher.py @@ -1186,6 +1186,10 @@ async def _reap_dispatched_task( "specialist auto-retry hook failed for task=%s", task.task_id, ) + if isinstance(result.result, dict): + reauthor_attempt = (getattr(task, "params", None) or {}).get("reauthor_attempt") + if reauthor_attempt not in (None, ""): + result.result.setdefault("reauthor_attempt", reauthor_attempt) try: await self.bus.append_and_seq( Message.new( @@ -1221,6 +1225,7 @@ async def _reap_dispatched_task( task=task, done_payload=done_payload, source=(f"{SPECIALIST_FROM_AGENT_PREFIX}{task.task_id}"), + run_error=str(result.error or ""), ) except Exception: # noqa: BLE001 — defensive log.exception( diff --git a/src/hyperloom/orchestrator/loop/writeback.py b/src/hyperloom/orchestrator/loop/writeback.py index 0be3986fe0..40bb077083 100644 --- a/src/hyperloom/orchestrator/loop/writeback.py +++ b/src/hyperloom/orchestrator/loop/writeback.py @@ -2244,6 +2244,7 @@ async def _record_specialist_result( task: Task, done_payload: dict[str, Any], source: str, + run_error: str = "", ) -> None: """Common bookkeeping for any specialist task termination (dispatcher loop + intent routing); idempotent on round_id, failures logged not raised. @@ -2252,8 +2253,11 @@ async def _record_specialist_result( done_payload: The specialist's done payload (proposal_set, domain, summary, etc.). source: The emitting agent string (``specialist:``). + run_error: Dispatch failure text when the specialist produced no + usable payload. """ - domain = str(done_payload.get("domain") or "").strip() + task_params = task.params or {} + domain = str(done_payload.get("domain") or task_params.get("domain") or "").strip() proposals = done_payload.get("proposal_set") or [] if not isinstance(proposals, list): proposals = [] @@ -2263,6 +2267,7 @@ async def _record_specialist_result( task=task, done_payload=done_payload, source=source, + run_error=run_error, ) # Advisory multi-model scoring of the proposal_set; informational only, gates nothing. Defensive. _scorer = getattr(self, "_proposal_scorer", None) @@ -2272,8 +2277,8 @@ async def _record_specialist_result( gap={ "domain": domain, "gap_canonical_id": done_payload.get("gap_canonical_id", ""), - "gap_symptom": (task.params or {}).get("gap_symptom"), - "gap_evidence": (task.params or {}).get("gap_evidence"), + "gap_symptom": task_params.get("gap_symptom"), + "gap_evidence": task_params.get("gap_evidence"), "summary": done_payload.get("summary", ""), }, proposals=proposals, @@ -2320,12 +2325,14 @@ async def _record_specialist_result( { "task_id": task.task_id, "domain": domain, - "gap_canonical_id": str(done_payload.get("gap_canonical_id") or ""), + "gap_canonical_id": str( + done_payload.get("gap_canonical_id") or task_params.get("gap_canonical_id") or "" + ), "empty": is_empty, "proposals_total": len(proposals), "confidence": done_payload.get("confidence"), "summary": str(done_payload.get("summary") or "")[:480], - "reason": str(done_payload.get("reason") or "")[:480], + "reason": str(run_error or done_payload.get("reason") or "")[:480], "ts": datetime.now(timezone.utc).isoformat(), } ) diff --git a/src/hyperloom/orchestrator/phases/explore.py b/src/hyperloom/orchestrator/phases/explore.py index 727cd267b4..4fb9e75ef0 100644 --- a/src/hyperloom/orchestrator/phases/explore.py +++ b/src/hyperloom/orchestrator/phases/explore.py @@ -85,7 +85,7 @@ def _forward_integrate_source( # ``lever_kind`` travels with the proposal: the patch that lands moved the # same lever the specialist was dispatched against, and re-deriving it at # writeback time is how attribution drifts. - for key in ("gap_canonical_id", "gap_layer", "lever_kind"): + for key in ("gap_canonical_id", "gap_layer", "lever_kind", "reauthor_attempt", "apply_retry_attempt"): value = src.get(key) if value not in (None, "", [], {}): dst[key] = value @@ -1852,6 +1852,7 @@ def _build_specialist_round_entry( task: Task, done_payload: dict[str, Any], source: str, + run_error: str = "", ) -> dict[str, Any]: """Translate a specialist done payload into a SharedState.specialist_rounds[] row; round_id defaults to task_id for idempotent overwrite. @@ -1860,6 +1861,7 @@ def _build_specialist_round_entry( done_payload: The specialist done payload (proposal_set, domain, tags, summary, etc.). source: The emitting agent string, recorded on the row. + run_error: Dispatch failure text when no valid payload was produced. Returns: A specialist-round row dict suitable for @@ -1891,18 +1893,41 @@ def _build_specialist_round_entry( "task_id": task.task_id, "source": source or "coordinator", "completed_at": datetime.now(timezone.utc).isoformat(), - "domain": str(done_payload.get("domain") or ""), + "domain": str(done_payload.get("domain") or task_params.get("domain") or ""), "tags": list(tags), - "gap_canonical_id": str(done_payload.get("gap_canonical_id") or ""), + "gap_canonical_id": str(done_payload.get("gap_canonical_id") or task_params.get("gap_canonical_id") or ""), "empty": bool(done_payload.get("empty")) or len(proposals) == 0, "proposals_total": len(proposals), "proposal_set": list(proposals), "summary": str(done_payload.get("summary") or "")[:480], - "reason": str(done_payload.get("reason") or "")[:480], + "reason": str(run_error or done_payload.get("reason") or "")[:480], "confidence": done_payload.get("confidence"), "new_findings": list(done_payload.get("new_findings") or []), "residual_questions": list(done_payload.get("residual_questions") or []), } + for key in ( + "task_kind", + "scope", + "proposal_msg_id", + "framework_agent_candidate_id", + "framework_batch_id", + "reauthor_attempt", + "apply_retry_attempt", + ): + value = done_payload.get(key) + if value in (None, "", [], {}): + value = task_params.get(key) + if value not in (None, "", [], {}): + entry[key] = value + for key in ("candidate_discovery", "framework_agent_authoring"): + if bool(done_payload.get(key) or task_params.get(key)): + entry[key] = True + if run_error: + entry["status"] = "failed" + entry["error"] = str(run_error)[:1000] + entry["run_error"] = str(run_error)[:1000] + elif done_payload.get("status") not in (None, ""): + entry["status"] = str(done_payload.get("status")) if source_phase: entry["source_phase"] = source_phase gpu_ids = done_payload.get("allocated_gpu_ids") or [] diff --git a/src/hyperloom/orchestrator/phases/framework.py b/src/hyperloom/orchestrator/phases/framework.py index 66e3f19d83..62685715e2 100644 --- a/src/hyperloom/orchestrator/phases/framework.py +++ b/src/hyperloom/orchestrator/phases/framework.py @@ -149,7 +149,7 @@ async def _pump_framework_agent_phase(self) -> None: return self._record_framework_agent_phase_done( reason="no_candidates_and_discovery_exhausted", - failure_count=int(getattr(state, "framework_agent_empty_discoveries", 0) or 0), + failure_count=int(getattr(state, "framework_agent_discover_failures", 0) or 0), ) state.framework_agent_phase_done = True state.save(self.session_dir) @@ -366,6 +366,7 @@ async def _enqueue_framework_agent_authoring_specialist( "framework_agent_authoring": True, "framework_agent_candidate_id": cand_id, "framework_batch_id": batch_id, + "reauthor_attempt": int(reauthor_attempt), "framework_audit": (audit if isinstance(audit, dict) else {}), "source": "coordinator_internal", "notes": notes, @@ -1439,6 +1440,7 @@ def _record_framework_agent_phase_done( evidence={ "event": "framework_agent_phase_done", "failure_count": int(failure_count), + "empty_count": int(getattr(state, "framework_agent_empty_discoveries", 0) or 0), "retry_limit": int(_fa_client.DISCOVER_FAILURE_RETRY_LIMIT), "batches_discovered": len(getattr(state, "framework_agent_batches", None) or []), "outcome_class": outcome_class, @@ -2151,6 +2153,7 @@ def _record_framework_agent_authored_outcome( "accuracy_pass": res.get("accuracy_pass"), "specialist_task_id": spec_tid, "integrate_task_id": str(getattr(task, "task_id", "") or ""), + "reauthor_attempt": res.get("reauthor_attempt", params.get("reauthor_attempt")), }, ) if not recorded: @@ -2250,7 +2253,10 @@ def _record_framework_agent_dispatch_failure( rationale=run_error[:500], provenance="dispatch_failed", gain_pct=0.0, - extra={"specialist_task_id": str(getattr(task, "task_id", "") or "")}, + extra={ + "specialist_task_id": str(getattr(task, "task_id", "") or ""), + "reauthor_attempt": params.get("reauthor_attempt"), + }, ) if not recorded: return @@ -2365,7 +2371,10 @@ def _record_framework_agent_authoring_empty_outcome( rationale=reason, provenance="authored_empty", gain_pct=0.0, - extra={"specialist_task_id": str(getattr(task, "task_id", "") or "")}, + extra={ + "specialist_task_id": str(getattr(task, "task_id", "") or ""), + "reauthor_attempt": params.get("reauthor_attempt"), + }, ) if not recorded: return diff --git a/src/hyperloom/orchestrator/roles/critic_agent.py b/src/hyperloom/orchestrator/roles/critic_agent.py index 0660c27617..ee9cfd86f2 100644 --- a/src/hyperloom/orchestrator/roles/critic_agent.py +++ b/src/hyperloom/orchestrator/roles/critic_agent.py @@ -867,6 +867,8 @@ async def run( instrument.record_critic_iteration( self.session_dir, iter_n=turn_idx, + request=request, + judge_bundle=judge_bundle, review=review, emit=emit, workdir=workdir, From ddb80294da41f1b351652e02d7214bcabd34f659 Mon Sep 17 00:00:00 2001 From: chenluo Date: Mon, 31 Aug 2026 15:23:38 +0800 Subject: [PATCH 5/7] Fix SBD V6 startup failure reporting --- .../breakdown/collectors/v6.py | 9 +- .../breakdown/session_package.py | 1 + .../inference_optimizer/cli/__init__.py | 20 ++- .../tests/test_sbd_v6_initial.py | 117 ++++++++++++++++++ .../tests/test_session_package.py | 6 + 5 files changed, 150 insertions(+), 3 deletions(-) diff --git a/src/hyperloom/inference_optimizer/breakdown/collectors/v6.py b/src/hyperloom/inference_optimizer/breakdown/collectors/v6.py index 033549bde8..c71bf450b2 100644 --- a/src/hyperloom/inference_optimizer/breakdown/collectors/v6.py +++ b/src/hyperloom/inference_optimizer/breakdown/collectors/v6.py @@ -2098,12 +2098,19 @@ def collect_v6_outcome( ) -> dict[str, Any]: """Project V5 result sections into the V6 ``outcome`` shape.""" stop_reason = str(session.get("stop_reason") or "").strip() + outcome_status = _outcome_status(stop_reason) + for event in reversed(timeline): + if not isinstance(event, dict) or str(event.get("type") or "") not in {"install", "model_gate"}: + continue + if str(event.get("status") or "").strip().lower() == "failed": + outcome_status = "failed" + break validation = optimizations.get("validation") if not isinstance(validation, dict): validation = {} return { "stop_reason": stop_reason, - "status": _outcome_status(stop_reason), + "status": outcome_status, "stage_reached": _stage_reached(state, stop_reason, timeline), "baseline": { "throughput_tok_s_per_gpu": baseline.get("throughput_tok_s_per_gpu"), diff --git a/src/hyperloom/inference_optimizer/breakdown/session_package.py b/src/hyperloom/inference_optimizer/breakdown/session_package.py index 5e029527e7..4231bc6e8b 100644 --- a/src/hyperloom/inference_optimizer/breakdown/session_package.py +++ b/src/hyperloom/inference_optimizer/breakdown/session_package.py @@ -77,6 +77,7 @@ "reports/kernel_roofline.json", "reports/conc_sweep_summary.json", "reports/sbd_v6/*.json", + "reports/sbd_v6/timeline/*.json", "reports/trace/*.jsonl", # ── target analysis ─────────────────────────────────────────────── "target_analysis/target_baseline.json", diff --git a/src/hyperloom/inference_optimizer/cli/__init__.py b/src/hyperloom/inference_optimizer/cli/__init__.py index 7ae423e325..d7fd702179 100644 --- a/src/hyperloom/inference_optimizer/cli/__init__.py +++ b/src/hyperloom/inference_optimizer/cli/__init__.py @@ -1837,6 +1837,22 @@ def _exit_code_for_stop_reason(stop_reason: str | None) -> int: return 0 if (stop_reason or "") in _SUCCESS_STOP_REASONS else 1 +def _is_valid_resume_session_dir(candidate: Path) -> bool: + """Return whether ``candidate`` is an initialized, readable session.""" + if not (candidate / "manifest.json").is_file() or not (candidate / "state.json").is_file(): + return False + try: + manifest = load_manifest(candidate) + state = SharedState.load_or_init(candidate) + except Exception: # noqa: BLE001 — invalid resume input must fall back without masking preflight + return False + if not isinstance(manifest, Mapping): + return False + manifest_session_id = str(manifest.get("session_id") or "").strip() + state_session_id = str(state.session_id or "").strip() + return bool(manifest_session_id and state_session_id and manifest_session_id == state_session_id) + + def _preflight_failure_session_dir(args: argparse.Namespace) -> Path: """Return a safe session directory for a pre-session install failure.""" resume_from = str(getattr(args, "resume_from", "") or "").strip() @@ -1847,10 +1863,10 @@ def _preflight_failure_session_dir(args: argparse.Namespace) -> Path: except (OSError, ValueError): pass else: - if candidate.is_dir(): + if candidate.is_dir() and _is_valid_resume_session_dir(candidate): return candidate - return _new_preflight_failure_session_dir(args) + return _new_preflight_failure_session_dir(args, failed_attempt=bool(resume_from)) def _new_preflight_failure_session_dir( diff --git a/src/hyperloom/inference_optimizer/tests/test_sbd_v6_initial.py b/src/hyperloom/inference_optimizer/tests/test_sbd_v6_initial.py index 9b301b2d0f..a2cd0357bf 100644 --- a/src/hyperloom/inference_optimizer/tests/test_sbd_v6_initial.py +++ b/src/hyperloom/inference_optimizer/tests/test_sbd_v6_initial.py @@ -266,6 +266,7 @@ def reject_credentials(): assert (session_dir / "manifest.json").is_file() breakdown = json.loads((session_dir / "session_breakdown.json").read_text(encoding="utf-8")) assert [(event["type"], event["status"]) for event in breakdown["timeline"]] == [("install", "failed")] + assert breakdown["outcome"]["status"] == "failed" def test_unwrapped_preflight_failure_is_persisted_as_failed(tmp_path, monkeypatch): @@ -305,6 +306,8 @@ def fail_runtime_paths(): assert failure["step_id"] == "unhandled_preflight" assert failure["error_class"] == "RuntimeError" assert failure["message"] == "runtime path resolution failed" + breakdown = json.loads((session_dir / "session_breakdown.json").read_text(encoding="utf-8")) + assert breakdown["outcome"]["status"] == "failed" def test_busy_resume_preflight_failure_uses_isolated_session(tmp_path, monkeypatch): @@ -314,6 +317,9 @@ def test_busy_resume_preflight_failure_uses_isolated_session(tmp_path, monkeypat workspace = tmp_path / "sessions" resume_dir = workspace / "Qwen-Test" / "active-session" + model = tmp_path / "Qwen-Test" + _seed_state(resume_dir, monkeypatch, model) + _write_json(resume_dir / "manifest.json", {"schema_version": 4, "session_id": "sbd-v6-test"}) original_install = { "type": "install", "kind": "install", @@ -368,6 +374,117 @@ def fail_preflight(args): assert install["ext"]["run_kind"] == "resume" +def test_resume_preflight_failure_overrides_completed_outcome(tmp_path, monkeypatch): + import hyperloom.inference_optimizer.cli as optimizer_cli + from hyperloom.inference_optimizer.cli import preflight + from hyperloom.inference_optimizer.session.paths import ENV_CURRENT_SESSION_DIR + from hyperloom.orchestrator.state.shared_state import SharedState + + workspace = tmp_path / "sessions" + resume_dir = workspace / "Qwen-Test" / "completed-session" + model = tmp_path / "Qwen-Test" + _seed_state(resume_dir, monkeypatch, model) + state = SharedState.load_or_init(resume_dir) + state.phase = "CLOSE" + state.stop_reason = "target_reached" + state.save(resume_dir) + _write_json(resume_dir / "manifest.json", {"schema_version": 4, "session_id": "sbd-v6-test"}) + write_timeline_event( + resume_dir, + { + "type": "install", + "kind": "install", + "status": "succeeded", + "start_time": "2026-08-27T01:00:00+00:00", + "end_time": "2026-08-27T01:01:00+00:00", + "ext": {"run_kind": "fresh", "steps": []}, + }, + ) + monkeypatch.setenv("USER_DATA_PATH", str(workspace)) + monkeypatch.delenv(ENV_CURRENT_SESSION_DIR, raising=False) + monkeypatch.setattr( + optimizer_cli, + "clean_stale_aiter_locks", + lambda: {"dir": "", "deleted": 0, "skipped_fresh": 0, "errors": 0}, + ) + + def fail_preflight(args): + preflight._begin_install_event(args) + raise RuntimeError("resume preflight failed") + + monkeypatch.setattr(optimizer_cli, "_preflight", fail_preflight) + args = optimizer_cli._build_parser().parse_args(["optimize", "--resume-from", str(resume_dir)]) + + with pytest.raises(RuntimeError, match="resume preflight failed"): + asyncio.run(optimizer_cli._run_optimize(args)) + + breakdown = json.loads((resume_dir / "session_breakdown.json").read_text(encoding="utf-8")) + assert breakdown["outcome"]["stop_reason"] == "target_reached" + assert breakdown["outcome"]["status"] == "failed" + assert breakdown["timeline"][-1]["type"] == "install" + assert breakdown["timeline"][-1]["status"] == "failed" + + +@pytest.mark.parametrize("invalid_artifact", ["missing", "manifest", "state"]) +def test_invalid_resume_preflight_failure_does_not_mutate_requested_directory( + tmp_path, + monkeypatch, + invalid_artifact, +): + import hyperloom.inference_optimizer.cli as optimizer_cli + from hyperloom.inference_optimizer.cli import preflight + from hyperloom.inference_optimizer.session.paths import ENV_CURRENT_SESSION_DIR + + workspace = tmp_path / "sessions" + resume_dir = workspace / "not-a-session" + _write_json(resume_dir / "session_breakdown.json", {"sentinel": "unchanged"}) + if invalid_artifact != "missing": + manifest = resume_dir / "manifest.json" + state = resume_dir / "state.json" + manifest.write_text( + "{invalid" if invalid_artifact == "manifest" else json.dumps({"session_id": "not-a-session"}), + encoding="utf-8", + ) + state.write_text( + "{invalid" if invalid_artifact == "state" else json.dumps({"session_id": "not-a-session"}), + encoding="utf-8", + ) + before = { + path.relative_to(resume_dir).as_posix(): path.read_bytes() for path in resume_dir.rglob("*") if path.is_file() + } + monkeypatch.setenv("USER_DATA_PATH", str(workspace)) + monkeypatch.delenv("MODEL_PATH", raising=False) + monkeypatch.delenv(ENV_CURRENT_SESSION_DIR, raising=False) + monkeypatch.setattr( + optimizer_cli, + "clean_stale_aiter_locks", + lambda: {"dir": "", "deleted": 0, "skipped_fresh": 0, "errors": 0}, + ) + + def fail_preflight(args): + preflight._begin_install_event(args) + raise RuntimeError("invalid resume preflight failed") + + monkeypatch.setattr(optimizer_cli, "_preflight", fail_preflight) + args = optimizer_cli._build_parser().parse_args(["optimize", "--resume-from", str(resume_dir)]) + + with pytest.raises(RuntimeError, match="invalid resume preflight failed"): + asyncio.run(optimizer_cli._run_optimize(args)) + + after = { + path.relative_to(resume_dir).as_posix(): path.read_bytes() for path in resume_dir.rglob("*") if path.is_file() + } + assert after == before + assert not (resume_dir / "runtime").exists() + assert not (resume_dir / "reports").exists() + failed_session = Path(os.environ[ENV_CURRENT_SESSION_DIR]) + assert failed_session != resume_dir + assert "failed-attempt" in failed_session.parent.name + install = read_timeline_event(failed_session, "install") + assert install is not None + assert install["status"] == "failed" + + def test_timeline_history_retains_fresh_and_resume_events(tmp_path, monkeypatch): from hyperloom.inference_optimizer.cli import model_gate diff --git a/src/hyperloom/inference_optimizer/tests/test_session_package.py b/src/hyperloom/inference_optimizer/tests/test_session_package.py index 38ce374b0b..0c9ef97b0e 100644 --- a/src/hyperloom/inference_optimizer/tests/test_session_package.py +++ b/src/hyperloom/inference_optimizer/tests/test_session_package.py @@ -42,6 +42,8 @@ def _build_session(sd: Path) -> None: _write(sd / "reports" / "kernel_roofline.json", "{}") _write(sd / "reports" / "sbd_v6" / "install.json", "{}") _write(sd / "reports" / "sbd_v6" / "model_gate.json", "{}") + _write(sd / "reports" / "sbd_v6" / "timeline" / "000001-install.json", "{}") + _write(sd / "reports" / "sbd_v6" / "timeline" / "000002-model_gate.json", "{}") _write(sd / "reports" / "trace" / "decision_trace.jsonl", "{}\n") _write(sd / "reports" / "trace" / "llm_calls.jsonl", "{}\n") _write(sd / "target_analysis" / "target_baseline.json", "{}") @@ -106,6 +108,8 @@ def test_package_includes_curated_excludes_noise(tmp_path: Path) -> None: "reports/kernel_roofline.json", "reports/sbd_v6/install.json", "reports/sbd_v6/model_gate.json", + "reports/sbd_v6/timeline/000001-install.json", + "reports/sbd_v6/timeline/000002-model_gate.json", "reports/trace/decision_trace.jsonl", "reports/trace/llm_calls.jsonl", "target_analysis/target_baseline.json", @@ -217,6 +221,8 @@ def test_loose_files_dropped_at_dest_root(tmp_path: Path) -> None: "session_breakdown.json", "state.json", "reports/final.json", + "reports/sbd_v6/timeline/000001-install.json", + "reports/sbd_v6/timeline/000002-model_gate.json", "reports/trace/decision_trace.jsonl", "kernel-agent/runs/20260609T010022Z/20260609T012416Z_tl-abc/tracelens/analysis.md", "runs/baseline/abc/measure_round/benchmark_sglang_x/benchmark_report.json", From 0539eb709d4f55f88f954f32a5de0a36762fd467 Mon Sep 17 00:00:00 2001 From: chenluo Date: Mon, 31 Aug 2026 16:00:25 +0800 Subject: [PATCH 6/7] fix: address SBD V6 review findings --- CHANGELOG.md | 6 + docs/reference/session-breakdown.md | 11 + .../inference_optimizer/breakdown/SKILL.md | 4 + .../breakdown/collectors/_common.py | 47 ++++ .../breakdown/collectors/v6.py | 94 +------ .../breakdown/critic_reviews.py | 100 ++------ .../breakdown/recorder/instrument.py | 10 + .../inference_optimizer/breakdown/schema.py | 4 + .../breakdown/session_package.py | 2 +- .../inference_optimizer/cli/__init__.py | 73 ++---- .../inference_optimizer/cli/model_gate.py | 57 ++--- .../inference_optimizer/cli/preflight.py | 12 +- .../inference_optimizer/session/sbd_v6.py | 170 ++++++------- .../session/session_paths.py | 14 +- .../tests/test_coordinator_runtime.py | 12 +- .../tests/test_critic_agent_backend.py | 3 + .../tests/test_sbd_v6_initial.py | 232 +++++++++++++----- .../tests/test_session_package.py | 6 +- .../orchestrator/loop/coordinator.py | 1 + .../orchestrator/roles/critic_agent.py | 17 +- 20 files changed, 451 insertions(+), 424 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 03bdea6433..fb697c8851 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -7,6 +7,12 @@ The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.1.0/), ### Added +- **Session breakdown exports now include the additive V6 startup contract.** + The existing V5 payload remains intact while `metadata`, `outcome`, + `timeline`, and `close` provide the V6 read model. Install and model-gate + source events use one ordered timeline ledger that preserves fresh and resume + attempts, and write failures are surfaced through `metadata.warnings`. + - **KernelForge now ships inside Hyperloom as the built-in kernel-opt agent.** Its source was snapshotted from `AMD-BRAIN-Internal/KernelForge` at `85b49f2f` (upstream `main`, PR #53 included) into `src/kernelforge/`; diff --git a/docs/reference/session-breakdown.md b/docs/reference/session-breakdown.md index 54eee0557a..9c55a56a1f 100644 --- a/docs/reference/session-breakdown.md +++ b/docs/reference/session-breakdown.md @@ -87,6 +87,10 @@ The following JSON structure shows all top-level fields in `session_breakdown.js "critic_robustness": { /* §14 Critic iterations + Robustness signals */ }, "telemetry": { /* §15 Telemetry artefact paths */ }, "optimizations": { /* canonical adopted-optimization API */ }, + "metadata": { /* additive V6 metadata and launch configuration */ }, + "outcome": { /* additive V6 terminal result */ }, + "timeline": [ /* additive V6 ordered stage events */ ], + "close": { /* additive V6 close-stage result */ }, "warnings": [ /* string[] — non-fatal collector warnings */ ], "source_files": { /* §17 SourceFiles — raw artefact paths */ }, @@ -117,6 +121,13 @@ The following JSON structure shows all top-level fields in `session_breakdown.js The `session` (SessionMeta) section also carries `user_data_path` and a `recovery` sub-object in addition to the fields documented in §3. +The additive V6 surface is identified by +`metadata.versions.schema_version = "hyperloom.session_breakdown.v6.0"` while +the existing top-level V5 contract remains unchanged. Startup source events are +stored in execution order under `reports/sbd_v6/timeline/`; writer failures are +reported through `metadata.warnings` rather than being indistinguishable from a +stage that never ran. + All sections use the `total=False` TypedDict convention — every field is optional. Consumers should expect partial documents when a session ended early (`baseline_failed`, `time_exhausted` before kernel-opt diff --git a/src/hyperloom/inference_optimizer/breakdown/SKILL.md b/src/hyperloom/inference_optimizer/breakdown/SKILL.md index ac50d84c72..fc9e836b3c 100644 --- a/src/hyperloom/inference_optimizer/breakdown/SKILL.md +++ b/src/hyperloom/inference_optimizer/breakdown/SKILL.md @@ -42,6 +42,10 @@ authoritative. | `sweep` | Grid size, best_overall, pareto_front, every variant's benchmark numbers. | | `critic_robustness` | Per-iter critic verdicts + robustness signals. | | `telemetry` | Paths to `benchmark_report.json` / `torch_trace` / `system_profile` / server logs + aggregated GPU monitor. `telemetry.orchestration_context` carries the compaction-loop health: `seed_prompts`, `delta_prompts`, `compactions`, `degenerate_compactions`, `tick_count`, `compactions_per_tick`, `delta_ratio`, `context_tokens_at_compaction`. See `docs/reference/session-breakdown.md §telemetry.orchestration_context`. | +| `metadata` | Additive V6 schema/version, session, launch configuration, Langfuse, and warning metadata. | +| `outcome` | Additive V6 terminal status, reached stage, stop reason, and final measured result. | +| `timeline` | Additive V6 ordered stage events; startup source events live under `reports/sbd_v6/timeline/`. | +| `close` | Additive V6 close-stage payload; currently empty until the close-stage collector is implemented. | | `warnings` | Best-effort caveats (missing files, partial sections, reconstructed fields). | | `source_files` | Mapping from logical section to relative path under `session_dir`. | diff --git a/src/hyperloom/inference_optimizer/breakdown/collectors/_common.py b/src/hyperloom/inference_optimizer/breakdown/collectors/_common.py index 829ed1edb8..851a334756 100644 --- a/src/hyperloom/inference_optimizer/breakdown/collectors/_common.py +++ b/src/hyperloom/inference_optimizer/breakdown/collectors/_common.py @@ -21,7 +21,54 @@ from ...session.paths import is_path_within +_FRAMEWORK_PHASES = frozenset({"FRAMEWORK_AGENT", "EXPLORE"}) +_AUTHORING_TASK_KINDS = frozenset( + { + "explore_apply_retry", + "framework_authoring", + "framework_local_explore", + } +) + + # Shared helpers +def _mapping(value: Any) -> dict[str, Any]: + """Return ``value`` when it is a dict, otherwise an empty mapping.""" + return value if isinstance(value, dict) else {} + + +def _dict_rows(value: Any) -> list[dict[str, Any]]: + """Keep only dictionary rows from a list-shaped value.""" + return [row for row in value if isinstance(row, dict)] if isinstance(value, list) else [] + + +def _first(*values: Any) -> Any: + """Return the first value that is neither ``None`` nor an empty string.""" + return next((value for value in values if value is not None and value != ""), None) + + +def _optional_bool(value: Any) -> bool | None: + """Coerce conventional boolean spellings without accepting arbitrary numbers.""" + if isinstance(value, bool): + return value + if isinstance(value, int) and value in (0, 1): + return bool(value) + if isinstance(value, str): + normalized = value.strip().lower() + if normalized in {"1", "true", "yes", "on", "passed", "succeeded"}: + return True + if normalized in {"0", "false", "no", "off", "failed"}: + return False + return None + + +def _string_list(value: Any) -> list[str]: + """Normalize a list-like value to non-empty strings.""" + if not isinstance(value, (list, tuple, set)): + return [] + return [str(item) for item in value if item not in (None, "")] + + def _load_json_safe( path: Path | None, warnings: list[str], diff --git a/src/hyperloom/inference_optimizer/breakdown/collectors/v6.py b/src/hyperloom/inference_optimizer/breakdown/collectors/v6.py index c71bf450b2..6b930729d7 100644 --- a/src/hyperloom/inference_optimizer/breakdown/collectors/v6.py +++ b/src/hyperloom/inference_optimizer/breakdown/collectors/v6.py @@ -2,7 +2,6 @@ from __future__ import annotations -from datetime import datetime from pathlib import Path from typing import Any @@ -10,6 +9,19 @@ from ..critic_reviews import FRAMEWORK_REVIEW_FIELDS, normalize_framework_reviews from ...session.sbd_v6 import SCHEMA_VERSION_V6, read_timeline_events +from ._common import ( + _AUTHORING_TASK_KINDS, + _FRAMEWORK_PHASES, + _dict_rows, + _first, + _mapping, + _optional_bool, + _parse_iso_unix as _timestamp_number, + _safe_get as _nested, + _string_list, + _to_float as _optional_float, + _to_int as _optional_int, +) _SUCCESS_STOP_REASONS = frozenset( @@ -30,7 +42,6 @@ "unsupported_model_arch", } ) -_FRAMEWORK_PHASES = frozenset({"FRAMEWORK_AGENT", "EXPLORE"}) _FRAMEWORK_EXIT_REASON_MAP = { "explore_no_more_leverage": "optimize_no_more_leverage", "plateau_explore": "optimize_no_more_leverage", @@ -38,13 +49,6 @@ "explore_budget_cap": "optimize_budget_cap", "explore_force_exit_low_budget": "optimize_force_exit_low_budget", } -_AUTHORING_TASK_KINDS = frozenset( - { - "explore_apply_retry", - "framework_authoring", - "framework_local_explore", - } -) def _tool_versions(versions: Any) -> dict[str, str | None]: @@ -159,72 +163,10 @@ def collect_v6_metadata( } -def _mapping(value: Any) -> dict[str, Any]: - return value if isinstance(value, dict) else {} - - -def _dict_rows(value: Any) -> list[dict[str, Any]]: - return [row for row in value if isinstance(row, dict)] if isinstance(value, list) else [] - - def _dict_value_rows(value: Any) -> list[dict[str, Any]]: return [row for row in value.values() if isinstance(row, dict)] if isinstance(value, dict) else [] -def _first(*values: Any) -> Any: - for value in values: - if value is not None and value != "": - return value - return None - - -def _optional_int(value: Any) -> int | None: - if value is None or value == "" or isinstance(value, bool): - return None - try: - return int(value) - except (TypeError, ValueError): - return None - - -def _optional_float(value: Any) -> float | None: - if value is None or value == "" or isinstance(value, bool): - return None - try: - return float(value) - except (TypeError, ValueError): - return None - - -def _optional_bool(value: Any) -> bool | None: - if isinstance(value, bool): - return value - if isinstance(value, int) and value in (0, 1): - return bool(value) - if isinstance(value, str): - normalized = value.strip().lower() - if normalized in {"1", "true", "yes", "on", "passed", "succeeded"}: - return True - if normalized in {"0", "false", "no", "off", "failed"}: - return False - return None - - -def _string_list(value: Any) -> list[str]: - if not isinstance(value, (list, tuple, set)): - return [] - return [str(item) for item in value if item not in (None, "")] - - -def _nested(mapping: dict[str, Any], *path: str) -> Any: - value: Any = mapping - for key in path: - if not isinstance(value, dict): - return None - value = value.get(key) - return value - - def _row_cycle(row: dict[str, Any]) -> int | None: for value in ( row.get("macro_cycle"), @@ -254,16 +196,6 @@ def _row_timestamp(row: dict[str, Any]) -> str: ) -def _timestamp_number(value: Any) -> float | None: - text = str(value or "").strip() - if not text: - return None - try: - return datetime.fromisoformat(text.replace("Z", "+00:00")).timestamp() - except (TypeError, ValueError): - return None - - def _operation_name(operation: dict[str, Any]) -> str: return str(operation.get("name") or operation.get("kind") or "").strip().lower() diff --git a/src/hyperloom/inference_optimizer/breakdown/critic_reviews.py b/src/hyperloom/inference_optimizer/breakdown/critic_reviews.py index 3e28544e02..a923705dea 100644 --- a/src/hyperloom/inference_optimizer/breakdown/critic_reviews.py +++ b/src/hyperloom/inference_optimizer/breakdown/critic_reviews.py @@ -2,9 +2,20 @@ from __future__ import annotations -import re from typing import Any +from .collectors._common import ( + _AUTHORING_TASK_KINDS, + _FRAMEWORK_PHASES, + _dict_rows, + _first, + _mapping, + _optional_bool, + _safe_get as _nested, + _string_list, + _to_int, +) + FRAMEWORK_REVIEW_FIELDS = ( "proposal_msg_id", @@ -27,58 +38,6 @@ "review_path", ) -_FRAMEWORK_PHASES = frozenset({"FRAMEWORK_AGENT", "EXPLORE"}) -_AUTHORING_TASK_KINDS = frozenset( - { - "explore_apply_retry", - "framework_authoring", - "framework_local_explore", - } -) -_MACRO_CYCLE_RE = re.compile(r"(?:^|\s)macro_cycle\s*=\s*(-?\d+)(?=\s|$)") - - -def _mapping(value: Any) -> dict[str, Any]: - return dict(value) if isinstance(value, dict) else {} - - -def _dict_rows(value: Any) -> list[dict[str, Any]]: - return [dict(row) for row in value or [] if isinstance(row, dict)] if isinstance(value, list) else [] - - -def _first(*values: Any) -> Any: - return next((value for value in values if value is not None and value != ""), None) - - -def _nested(value: Any, *keys: str) -> Any: - current = value - for key in keys: - if not isinstance(current, dict): - return None - current = current.get(key) - return current - - -def _optional_bool(value: Any) -> bool | None: - if isinstance(value, bool): - return value - if isinstance(value, str): - normalized = value.strip().lower() - if normalized in {"1", "true", "yes", "on"}: - return True - if normalized in {"0", "false", "no", "off"}: - return False - if isinstance(value, (int, float)): - return bool(value) - return None - - -def _string_list(value: Any) -> list[str]: - if not isinstance(value, (list, tuple, set)): - return [] - return [str(item) for item in value if str(item)] - - def _candidate_id(value: Any) -> str: candidate = _mapping(value) return str( @@ -93,16 +52,6 @@ def _candidate_id(value: Any) -> str: ) -def _prompt_macro_cycle(value: Any) -> int | None: - match = _MACRO_CYCLE_RE.search(str(value or "")) - if match is None: - return None - try: - return int(match.group(1)) - except (TypeError, ValueError): - return None - - def _has_patch_refs(params: dict[str, Any]) -> bool: return any(isinstance(params.get(key), list) and bool(params.get(key)) for key in ("patch_refs", "patches_written")) @@ -223,18 +172,19 @@ def normalize_framework_reviews( "ts": str(_first(verdict_row.get("ts"), emit.get("ts"), review.get("ts")) or ""), "review_path": normalized_review_path, "phase": review_phase, - "macro_cycle": _first( - payload.get("cycle"), - payload.get("macro_cycle"), - params.get("cycle"), - params.get("macro_cycle"), - proposal.get("cycle"), - proposal.get("macro_cycle"), - _nested(request, "context", "macro_cycle"), - _nested(request, "context", "cycle"), - _prompt_macro_cycle(request.get("raw_prompt")), - _nested(judge, "merged_context", "macro_cycle"), - _nested(judge, "merged_context", "cycle"), + "macro_cycle": _to_int( + _first( + payload.get("cycle"), + payload.get("macro_cycle"), + params.get("cycle"), + params.get("macro_cycle"), + proposal.get("cycle"), + proposal.get("macro_cycle"), + _nested(request, "context", "macro_cycle"), + _nested(request, "context", "cycle"), + _nested(judge, "merged_context", "macro_cycle"), + _nested(judge, "merged_context", "cycle"), + ) ), } ) diff --git a/src/hyperloom/inference_optimizer/breakdown/recorder/instrument.py b/src/hyperloom/inference_optimizer/breakdown/recorder/instrument.py index 03196ece15..eb9499432a 100644 --- a/src/hyperloom/inference_optimizer/breakdown/recorder/instrument.py +++ b/src/hyperloom/inference_optimizer/breakdown/recorder/instrument.py @@ -3812,6 +3812,16 @@ def record_critic_iteration( "review_path": _rel(wd / "review.json", session_dir) if wd else None, "kb_writes": list(emit.get("kb_writes") or []) if isinstance(emit.get("kb_writes"), list) else [], } + request_context = request.get("context") if isinstance(request.get("context"), dict) else {} + phase = str(request_context.get("phase") or "").strip().upper() + if phase: + payload["phase"] = phase + try: + macro_cycle = int(request_context["macro_cycle"]) + except (KeyError, TypeError, ValueError): + macro_cycle = None + if macro_cycle is not None: + payload["macro_cycle"] = macro_cycle framework_reviews = normalize_framework_reviews( request=request, judge_bundle=judge_bundle, diff --git a/src/hyperloom/inference_optimizer/breakdown/schema.py b/src/hyperloom/inference_optimizer/breakdown/schema.py index b6c9a8a092..35fdd9d5b8 100644 --- a/src/hyperloom/inference_optimizer/breakdown/schema.py +++ b/src/hyperloom/inference_optimizer/breakdown/schema.py @@ -1076,6 +1076,8 @@ class CriticIteration(TypedDict, total=False): judge_bundle_path (str): Path to the judge bundle. emit_path (str): Path to the emitted review output. review_path (str): Path to the review record. + phase (str): Coordinator phase captured with the critic request. + macro_cycle (int): Coordinator macro cycle captured with the request. framework_reviews (list[dict[str, Any]]): Durable normalized V6 Framework review rows. """ @@ -1090,6 +1092,8 @@ class CriticIteration(TypedDict, total=False): judge_bundle_path: str emit_path: str review_path: str + phase: str + macro_cycle: int framework_reviews: list[dict[str, Any]] diff --git a/src/hyperloom/inference_optimizer/breakdown/session_package.py b/src/hyperloom/inference_optimizer/breakdown/session_package.py index 4231bc6e8b..fac5397070 100644 --- a/src/hyperloom/inference_optimizer/breakdown/session_package.py +++ b/src/hyperloom/inference_optimizer/breakdown/session_package.py @@ -76,8 +76,8 @@ "reports/kernel_optimization_summary.json", "reports/kernel_roofline.json", "reports/conc_sweep_summary.json", - "reports/sbd_v6/*.json", "reports/sbd_v6/timeline/*.json", + "reports/sbd_v6/write_warnings.jsonl", "reports/trace/*.jsonl", # ── target analysis ─────────────────────────────────────────────── "target_analysis/target_baseline.json", diff --git a/src/hyperloom/inference_optimizer/cli/__init__.py b/src/hyperloom/inference_optimizer/cli/__init__.py index d7fd702179..7aec431a41 100644 --- a/src/hyperloom/inference_optimizer/cli/__init__.py +++ b/src/hyperloom/inference_optimizer/cli/__init__.py @@ -111,7 +111,6 @@ ENV_USER_DATA_PATH, asset_system_prompts_dir, make_session_dir, - workspace_root, ) @@ -1837,38 +1836,6 @@ def _exit_code_for_stop_reason(stop_reason: str | None) -> int: return 0 if (stop_reason or "") in _SUCCESS_STOP_REASONS else 1 -def _is_valid_resume_session_dir(candidate: Path) -> bool: - """Return whether ``candidate`` is an initialized, readable session.""" - if not (candidate / "manifest.json").is_file() or not (candidate / "state.json").is_file(): - return False - try: - manifest = load_manifest(candidate) - state = SharedState.load_or_init(candidate) - except Exception: # noqa: BLE001 — invalid resume input must fall back without masking preflight - return False - if not isinstance(manifest, Mapping): - return False - manifest_session_id = str(manifest.get("session_id") or "").strip() - state_session_id = str(state.session_id or "").strip() - return bool(manifest_session_id and state_session_id and manifest_session_id == state_session_id) - - -def _preflight_failure_session_dir(args: argparse.Namespace) -> Path: - """Return a safe session directory for a pre-session install failure.""" - resume_from = str(getattr(args, "resume_from", "") or "").strip() - if resume_from: - candidate = Path(resume_from).expanduser().resolve() - try: - candidate.relative_to(workspace_root().resolve()) - except (OSError, ValueError): - pass - else: - if candidate.is_dir() and _is_valid_resume_session_dir(candidate): - return candidate - - return _new_preflight_failure_session_dir(args, failed_attempt=bool(resume_from)) - - def _new_preflight_failure_session_dir( args: argparse.Namespace, *, @@ -1889,12 +1856,15 @@ def _new_preflight_failure_session_dir( def _persist_preflight_failure_artifacts( args: argparse.Namespace, - exc: BaseException, + exc: Exception, ) -> Path | None: """Best-effort materialize the failed install event and final SBD.""" _mark_pending_install_event_failed(args, exc) try: - session_dir = _preflight_failure_session_dir(args) + session_dir = _new_preflight_failure_session_dir( + args, + failed_attempt=bool(str(getattr(args, "resume_from", "") or "").strip()), + ) except Exception: # noqa: BLE001 — never replace the original preflight failure log.warning("failed to create a session for SBD V6 preflight failure", exc_info=True) return None @@ -1902,23 +1872,10 @@ def _persist_preflight_failure_artifacts( session_lock = SessionLock(session_dir) try: session_lock.acquire() - except Exception as lock_exc: # noqa: BLE001 — a busy resume must not mutate the active session + except Exception: # noqa: BLE001 — never replace the original preflight failure session_lock.release() - if not str(getattr(args, "resume_from", "") or "").strip(): - log.warning("failed to lock SBD V6 preflight failure session", exc_info=True) - return None - log.warning( - "resume session unavailable for preflight failure artifacts (%s); using an isolated failed-attempt session", - lock_exc, - ) - try: - session_dir = _new_preflight_failure_session_dir(args, failed_attempt=True) - session_lock = SessionLock(session_dir) - session_lock.acquire() - except Exception: # noqa: BLE001 — never replace the original preflight failure - session_lock.release() - log.warning("failed to create an isolated SBD V6 preflight failure session", exc_info=True) - return None + log.warning("failed to lock SBD V6 preflight failure session", exc_info=True) + return None try: if not (session_dir / "manifest.json").is_file(): @@ -1927,16 +1884,22 @@ def _persist_preflight_failure_artifacts( if not getattr(manifest_args, "model", None): manifest_args.model = os.environ.get("MODEL_PATH", "") write_manifest(session_dir, args=manifest_args) - except Exception: # noqa: BLE001 — the install event can still stand alone + except Exception as write_exc: # noqa: BLE001 — the install event can still stand alone log.warning("failed to write manifest for SBD V6 preflight failure", exc_info=True) + from ..session.sbd_v6 import record_write_warning + + record_write_warning(session_dir, component="preflight_failure.manifest", exc=write_exc) _persist_install_event(args, session_dir) try: from ..breakdown import write_breakdown_json write_breakdown_json(session_dir) - except Exception: # noqa: BLE001 — never replace the original preflight failure + except Exception as write_exc: # noqa: BLE001 — never replace the original preflight failure log.warning("failed to write SBD V6 preflight failure breakdown", exc_info=True) + from ..session.sbd_v6 import record_write_warning + + record_write_warning(session_dir, component="preflight_failure.breakdown", exc=write_exc) finally: session_lock.release() print(f"Preflight failure artifacts: {session_dir}", file=sys.stderr) @@ -2085,10 +2048,10 @@ async def _run_optimize(args: argparse.Namespace) -> int: codex_follows_claude = _codex_model_should_follow_claude() try: resolved_urls = _preflight(args) - except BaseException as exc: + except Exception as exc: # noqa: BLE001 — only unexpected defects create diagnostic sessions try: _persist_preflight_failure_artifacts(args, exc) - except BaseException: # noqa: BLE001 — preserve the original failure exactly + except Exception: # noqa: BLE001 — preserve the original failure exactly log.warning("failed to preserve SBD V6 preflight failure", exc_info=True) raise diff --git a/src/hyperloom/inference_optimizer/cli/model_gate.py b/src/hyperloom/inference_optimizer/cli/model_gate.py index 88e601b520..656965ff90 100644 --- a/src/hyperloom/inference_optimizer/cli/model_gate.py +++ b/src/hyperloom/inference_optimizer/cli/model_gate.py @@ -1883,16 +1883,26 @@ def _load_model_gate_event(args: argparse.Namespace, session_dir: Path) -> dict[ def _write_model_gate_event(session_dir: Path, event: dict[str, Any]) -> bool: - from ..session.sbd_v6 import write_timeline_event + from ..session.sbd_v6 import record_write_warning, write_timeline_event try: write_timeline_event(session_dir, event) - except Exception: # noqa: BLE001 — observability must never change gate behavior + except Exception as exc: # noqa: BLE001 — observability must never change gate behavior log.warning("failed to persist SBD V6 model-gate event", exc_info=True) + if not record_write_warning(session_dir, component="model_gate.event", exc=exc): + log.debug("failed to persist SBD V6 model-gate write warning", exc_info=True) return False return True +def _record_model_gate_warning(session_dir: Path, *, component: str, exc: BaseException) -> None: + """Best-effort retain a model-gate observability failure for export.""" + from ..session.sbd_v6 import record_write_warning + + if not record_write_warning(session_dir, component=component, exc=exc): + log.debug("failed to persist SBD V6 model-gate warning", exc_info=True) + + def _model_gate_status( checks: list[dict[str, Any]], *, @@ -1959,8 +1969,9 @@ def _record_model_gate_check( skip_reason=str(ext.get("skip_reason") or "") or None, ) _write_model_gate_event(session_dir, event) - except Exception: # noqa: BLE001 — V6 observability must never change gate behavior + except Exception as exc: # noqa: BLE001 — V6 observability must never change gate behavior log.warning("failed to record SBD V6 model-gate check", exc_info=True) + _record_model_gate_warning(session_dir, component="model_gate.check", exc=exc) def _start_model_gate(args: argparse.Namespace, session_dir: Path) -> None: @@ -1969,8 +1980,9 @@ def _start_model_gate(args: argparse.Namespace, session_dir: Path) -> None: event = _new_model_gate_event(args) setattr(args, _MODEL_GATE_EVENT_ATTR, event) _write_model_gate_event(session_dir, event) - except Exception: # noqa: BLE001 — V6 observability must never change launch behavior + except Exception as exc: # noqa: BLE001 — V6 observability must never change launch behavior log.warning("failed to initialize SBD V6 model-gate event", exc_info=True) + _record_model_gate_warning(session_dir, component="model_gate.start", exc=exc) def _finish_model_gate(args: argparse.Namespace, session_dir: Path) -> None: @@ -1984,8 +1996,9 @@ def _finish_model_gate(args: argparse.Namespace, session_dir: Path) -> None: ) event["end_time"] = now_iso(timespec="seconds") _write_model_gate_event(session_dir, event) - except Exception: # noqa: BLE001 — V6 observability must never change launch behavior + except Exception as exc: # noqa: BLE001 — V6 observability must never change launch behavior log.warning("failed to finalize SBD V6 model-gate event", exc_info=True) + _record_model_gate_warning(session_dir, component="model_gate.finish", exc=exc) def _record_resumed_model_gate( @@ -2017,17 +2030,17 @@ def _record_resumed_model_gate( ] setattr(args, _MODEL_GATE_EVENT_ATTR, event) _write_model_gate_event(session_dir, event) - except Exception: # noqa: BLE001 — V6 observability must never change resume behavior + except Exception as exc: # noqa: BLE001 — V6 observability must never change resume behavior log.warning("failed to record resumed SBD V6 model-gate event", exc_info=True) + _record_model_gate_warning(session_dir, component="model_gate.resume", exc=exc) def _write_model_gate_breakdown( - args: argparse.Namespace, session_dir: Path, *, failure_label: str, ) -> None: - """Write the fail-fast SBD and then persist its truthful artifact status.""" + """Write the fail-fast SBD once without masking the gate failure.""" try: from ..breakdown import write_breakdown_json @@ -2037,24 +2050,7 @@ def _write_model_gate_breakdown( f"WARNING: failed to write session_breakdown.json on {failure_label} fail-fast: {exc!r}", file=sys.stderr, ) - return - - try: - event = _load_model_gate_event(args, session_dir) - ext = event.get("ext") - failure = ext.get("failure") if isinstance(ext, dict) else None - artifacts = failure.get("artifacts") if isinstance(failure, dict) else None - if not isinstance(artifacts, dict): - return - artifacts["breakdown_written"] = True - if not _write_model_gate_event(session_dir, event): - return - write_breakdown_json(session_dir) - except Exception: # noqa: BLE001 — V6 refresh must not mask the gate failure - log.warning( - "failed to refresh session_breakdown.json with model-gate artifact status", - exc_info=True, - ) + _record_model_gate_warning(session_dir, component=f"model_gate.{failure_label}.breakdown", exc=exc) def _context_headroom_tokens() -> int: @@ -2271,13 +2267,12 @@ def _preflight_context_window(args: argparse.Namespace, session_dir: Path) -> bo "message": reason, "artifacts": { "final_json": "reports/final.json" if (session_dir / "reports" / "final.json").is_file() else None, - "breakdown_written": False, }, }, ) # Delivery-artifact parity: emit session_breakdown.json here too since # fail-fast exits before coordinator.run()'s finally. - _write_model_gate_breakdown(args, session_dir, failure_label="context") + _write_model_gate_breakdown(session_dir, failure_label="context") # Langfuse parity: this gate exits before coordinator.run()'s finally, so # push the breakdown to Langfuse here too. _emit_breakdown_to_langfuse(session_dir) @@ -2390,11 +2385,10 @@ def _preflight_model_config_compat( "message": reason, "artifacts": { "final_json": "reports/final.json" if (session_dir / "reports" / "final.json").is_file() else None, - "breakdown_written": False, }, }, ) - _write_model_gate_breakdown(args, session_dir, failure_label="config") + _write_model_gate_breakdown(session_dir, failure_label="config") # Langfuse parity: this gate exits before coordinator.run()'s finally, so # push the breakdown to Langfuse here too. _emit_breakdown_to_langfuse(session_dir) @@ -2613,13 +2607,12 @@ def _preflight_unsupported_model_arch( "message": reason, "artifacts": { "final_json": "reports/final.json" if (session_dir / "reports" / "final.json").is_file() else None, - "breakdown_written": False, }, }, ) # Delivery-artifact parity: emit session_breakdown.json here too since # fail-fast exits before coordinator.run()'s finally. - _write_model_gate_breakdown(args, session_dir, failure_label="unsupported-model") + _write_model_gate_breakdown(session_dir, failure_label="unsupported-model") # Langfuse parity: this gate exits before coordinator.run()'s finally, so # push the breakdown to Langfuse here too. _emit_breakdown_to_langfuse(session_dir) diff --git a/src/hyperloom/inference_optimizer/cli/preflight.py b/src/hyperloom/inference_optimizer/cli/preflight.py index 7237b70784..fefd769967 100644 --- a/src/hyperloom/inference_optimizer/cli/preflight.py +++ b/src/hyperloom/inference_optimizer/cli/preflight.py @@ -2413,12 +2413,20 @@ def _finish_install_event( def _persist_install_event(args: argparse.Namespace | None, session_dir: Path) -> None: """Persist the pre-session install trace without changing launch behavior.""" - from ..session.sbd_v6 import persist_pending_install_event + from ..session.sbd_v6 import persist_pending_install_event, record_write_warning try: - persist_pending_install_event(args, session_dir) + path = persist_pending_install_event(args, session_dir) except Exception as exc: # noqa: BLE001 + log.warning("failed to persist SBD V6 install event", exc_info=True) + if not record_write_warning(session_dir, component="install.event", exc=exc): + log.debug("failed to persist SBD V6 install-event write warning", exc_info=True) + return + if path is None: + exc = RuntimeError("pending install event is unavailable") log.warning("failed to persist SBD V6 install event: %s", exc) + if not record_write_warning(session_dir, component="install.event", exc=exc): + log.debug("failed to persist SBD V6 install-event write warning", exc_info=True) def _preflight( diff --git a/src/hyperloom/inference_optimizer/session/sbd_v6.py b/src/hyperloom/inference_optimizer/session/sbd_v6.py index 5e05651a0b..51a3c036b7 100644 --- a/src/hyperloom/inference_optimizer/session/sbd_v6.py +++ b/src/hyperloom/inference_optimizer/session/sbd_v6.py @@ -7,14 +7,14 @@ from pathlib import Path from typing import Any -from hyperloom.common.io import atomic_write_json -from hyperloom.common.jsonio import read_json +from hyperloom.common.io import append_jsonl, atomic_write_json +from hyperloom.common.jsonio import read_json, read_jsonl +from hyperloom.common.timeutil import now_iso from .session_paths import ( - sbd_v6_install_path, - sbd_v6_model_gate_path, sbd_v6_timeline_dir, sbd_v6_timeline_event_path, + sbd_v6_write_warnings_path, ) @@ -25,13 +25,10 @@ _EVENT_FILE_RE = re.compile(r"^(?P\d+)-(?P[a-z0-9_]+)\.json$") -def _event_path(session_dir: Path | str, event_type: str) -> Path: - root = Path(session_dir) - if event_type == "install": - return sbd_v6_install_path(root) - if event_type == "model_gate": - return sbd_v6_model_gate_path(root) - raise ValueError(f"unsupported SBD V6 timeline event type: {event_type!r}") +def _validate_event_type(event_type: str) -> str: + if event_type not in _EVENT_TYPES: + raise ValueError(f"unsupported SBD V6 timeline event type: {event_type!r}") + return event_type def _public_event(event: dict[str, Any]) -> dict[str, Any]: @@ -87,59 +84,10 @@ def _read_event_file( return _public_event(event) -def _event_identity(event_type: str, event: dict[str, Any]) -> tuple[str, str, str] | None: - ext = event.get("ext") if isinstance(event.get("ext"), dict) else {} - start_time = str(event.get("start_time") or "") - run_kind = str(ext.get("run_kind") or "") - if not start_time and not run_kind: - return None - return event_type, start_time, run_kind - - -def _ensure_timeline_history(session_dir: Path | str) -> list[tuple[int, str, Path]]: - history = _history_files(session_dir) - root = Path(session_dir) - stored_events: list[tuple[int, str, Path, dict[str, Any]]] = [] - for sequence, event_type, path in history: - event = _read_event_file(path, event_type) - if event is not None: - stored_events.append((sequence, event_type, path, event)) - - next_sequence = max((sequence for sequence, _, _ in history), default=0) - for event_type in _EVENT_TYPES: - legacy_path = _event_path(root, event_type) - if not legacy_path.is_file(): - continue - event = _read_event_file(legacy_path, event_type) - if event is None: - continue - if any(stored_event == event for _, _, _, stored_event in stored_events): - continue - identity = _event_identity(event_type, event) - matching = next( - ( - row - for row in reversed(stored_events) - if identity is not None and _event_identity(row[1], row[3]) == identity - ), - None, - ) - if matching is not None: - _write_event(matching[2], event) - stored_events[stored_events.index(matching)] = (*matching[:3], event) - continue - next_sequence += 1 - path = sbd_v6_timeline_event_path(root, next_sequence, event_type) - _write_event(path, event) - stored_events.append((next_sequence, event_type, path, event)) - return _history_files(root) - - def write_timeline_event(session_dir: Path | str, event: dict[str, Any]) -> Path: """Persist one event without replacing an earlier run of the same stage.""" - event_type = str(event.get("type") or "").strip() - latest_path = _event_path(session_dir, event_type) - history = _ensure_timeline_history(session_dir) + event_type = _validate_event_type(str(event.get("type") or "").strip()) + history = _history_files(session_dir) raw_sequence = event.get(_STORAGE_SEQUENCE_KEY) try: @@ -157,12 +105,9 @@ def write_timeline_event(session_dir: Path | str, event: dict[str, Any]) -> Path sequence = max((stored_sequence for stored_sequence, _, _ in history), default=0) + 1 event[_STORAGE_SEQUENCE_KEY] = sequence - _write_event( - sbd_v6_timeline_event_path(Path(session_dir), sequence, event_type), - event, - ) - _write_event(latest_path, event) - return latest_path + path = sbd_v6_timeline_event_path(Path(session_dir), sequence, event_type) + _write_event(path, event) + return path def read_timeline_event( @@ -170,18 +115,14 @@ def read_timeline_event( event_type: str, ) -> dict[str, Any] | None: """Read the latest persisted event of one type.""" - _event_path(session_dir, event_type) - for _, stored_type, path in reversed(_ensure_timeline_history(session_dir)): + _validate_event_type(event_type) + for _, stored_type, path in reversed(_history_files(session_dir)): if stored_type != event_type: continue event = _read_event_file(path, event_type) if event is not None: return event - - path = _event_path(session_dir, event_type) - if not path.is_file(): - return None - return _read_event_file(path, event_type) + return None def read_timeline_event_for_update( @@ -189,8 +130,8 @@ def read_timeline_event_for_update( event_type: str, ) -> dict[str, Any] | None: """Read the latest event with its private storage sequence attached.""" - _event_path(session_dir, event_type) - for sequence, stored_type, path in reversed(_ensure_timeline_history(session_dir)): + _validate_event_type(event_type) + for sequence, stored_type, path in reversed(_history_files(session_dir)): if stored_type != event_type: continue event = _read_event_file(path, event_type) @@ -206,26 +147,71 @@ def read_timeline_events( warnings: list[str] | None = None, ) -> list[dict[str, Any]]: """Read all persisted V6 events in execution order.""" - history = _ensure_timeline_history(session_dir) - if history: - events: list[dict[str, Any]] = [] - for _, event_type, path in history: - event = _read_event_file(path, event_type, warnings) - if event is not None: - events.append(event) - return events - - events = [] - for event_type in _EVENT_TYPES: - path = _event_path(session_dir, event_type) - if not path.is_file(): - continue + if warnings is not None: + for warning in read_write_warnings(session_dir, warnings=warnings): + if warning not in warnings: + warnings.append(warning) + events: list[dict[str, Any]] = [] + for _, event_type, path in _history_files(session_dir): event = _read_event_file(path, event_type, warnings) if event is not None: events.append(event) return events +def record_write_warning( + session_dir: Path | str, + *, + component: str, + exc: BaseException, +) -> bool: + """Best-effort persist a V6 writer failure for the next export.""" + try: + append_jsonl( + sbd_v6_write_warnings_path(Path(session_dir)), + { + "ts": now_iso(timespec="seconds"), + "component": str(component or "unknown"), + "error_class": type(exc).__name__, + "message": str(exc) or repr(exc), + }, + make_parents=True, + ensure_ascii=False, + sort_keys=True, + ) + except Exception: # noqa: BLE001 — warning persistence cannot change runtime behavior + return False + return True + + +def read_write_warnings( + session_dir: Path | str, + *, + warnings: list[str] | None = None, +) -> list[str]: + """Read durable V6 writer failures without mutating the session.""" + path = sbd_v6_write_warnings_path(Path(session_dir)) + if not path.is_file(): + return [] + rows = read_jsonl( + path, + require_dict=True, + skip_malformed=True, + on_error=( + (lambda exc: warnings.append(f"timeline.write_warnings: failed to parse {path}: {exc!r}")) + if warnings is not None + else None + ), + ) + result: list[str] = [] + for row in rows: + component = str(row.get("component") or "unknown") + error_class = str(row.get("error_class") or "Error") + message = str(row.get("message") or "write failed") + result.append(f"sbd_v6.write.{component}: {error_class}: {message}") + return result + + def set_pending_install_event(args: Namespace | None, event: dict[str, Any]) -> None: """Attach the pre-session install event to the parsed CLI namespace.""" if args is not None: @@ -252,9 +238,11 @@ def persist_pending_install_event(args: Namespace | None, session_dir: Path | st "SCHEMA_VERSION_V6", "pending_install_event", "persist_pending_install_event", + "read_write_warnings", "read_timeline_event", "read_timeline_event_for_update", "read_timeline_events", + "record_write_warning", "set_pending_install_event", "write_timeline_event", ] diff --git a/src/hyperloom/inference_optimizer/session/session_paths.py b/src/hyperloom/inference_optimizer/session/session_paths.py index 99a8fdf339..1c00ee06a6 100644 --- a/src/hyperloom/inference_optimizer/session/session_paths.py +++ b/src/hyperloom/inference_optimizer/session/session_paths.py @@ -287,14 +287,9 @@ def sbd_v6_timeline_event_path(session_dir: Path, sequence: int, event_type: str return sbd_v6_timeline_dir(session_dir) / f"{int(sequence):06d}-{event_type}.json" -def sbd_v6_install_path(session_dir: Path) -> Path: - """Compute the persisted V6 ``install`` timeline event path.""" - return sbd_v6_dir(session_dir) / "install.json" - - -def sbd_v6_model_gate_path(session_dir: Path) -> Path: - """Compute the persisted V6 ``model_gate`` timeline event path.""" - return sbd_v6_dir(session_dir) / "model_gate.json" +def sbd_v6_write_warnings_path(session_dir: Path) -> Path: + """Compute the durable V6 write-warning ledger path.""" + return sbd_v6_dir(session_dir) / "write_warnings.jsonl" def enablement_dir(session_dir: Path) -> Path: @@ -906,10 +901,9 @@ def failure_evidence_path(session_dir: Path, failure_id: str) -> Path: "runs_dir", "runs_root", "sbd_v6_dir", - "sbd_v6_install_path", - "sbd_v6_model_gate_path", "sbd_v6_timeline_dir", "sbd_v6_timeline_event_path", + "sbd_v6_write_warnings_path", "state_path", "target_analysis_dir", "target_analysis_report_md", diff --git a/src/hyperloom/inference_optimizer/tests/test_coordinator_runtime.py b/src/hyperloom/inference_optimizer/tests/test_coordinator_runtime.py index 33eec6e336..17118b634f 100644 --- a/src/hyperloom/inference_optimizer/tests/test_coordinator_runtime.py +++ b/src/hyperloom/inference_optimizer/tests/test_coordinator_runtime.py @@ -342,9 +342,17 @@ def __init__(self, name: str, session_dir: Path) -> None: super().__init__(name) self._session_dir = session_dir self.trace_ctx_calls = 0 + self.trace_contexts: list[dict[str, object]] = [] - def set_trace_context(self, *, tick: int | None = None, phase: str | None = None) -> None: + def set_trace_context( + self, + *, + tick: int | None = None, + phase: str | None = None, + macro_cycle: int | None = None, + ) -> None: self.trace_ctx_calls += 1 + self.trace_contexts.append({"tick": tick, "phase": phase, "macro_cycle": macro_cycle}) async def run( self, @@ -390,6 +398,8 @@ async def test_self_tracing_backend_failure_is_recorded_exactly_once(session_dir # The surviving row is the backend's richer one (real review model). assert {r["model"] for r in rows} == {"claude-opus-4-7"} assert {r["component"] for r in rows} == {"critic"} + assert backends["critic"].trace_contexts + assert all("macro_cycle" in context for context in backends["critic"].trace_contexts) finally: await c.stop() diff --git a/src/hyperloom/inference_optimizer/tests/test_critic_agent_backend.py b/src/hyperloom/inference_optimizer/tests/test_critic_agent_backend.py index 468012281f..98acec85c5 100644 --- a/src/hyperloom/inference_optimizer/tests/test_critic_agent_backend.py +++ b/src/hyperloom/inference_optimizer/tests/test_critic_agent_backend.py @@ -1209,12 +1209,15 @@ async def test_static_context_override_wins_over_manifest( runtime_caller_factory=lambda: fake_caller, static_context={"model": "explicit-m", "framework": "vllm", "gpu_type": "mi355x"}, ) + backend.set_trace_context(tick=8, phase="FRAMEWORK_AGENT", macro_cycle=3) await backend.run("prompt") request = json.loads((fake_session_dir / "critic-workdir" / "000000" / "request.json").read_text(encoding="utf-8")) assert request["context"] == { "model": "explicit-m", "framework": "vllm", "gpu_type": "mi355x", + "phase": "FRAMEWORK_AGENT", + "macro_cycle": 3, } diff --git a/src/hyperloom/inference_optimizer/tests/test_sbd_v6_initial.py b/src/hyperloom/inference_optimizer/tests/test_sbd_v6_initial.py index a2cd0357bf..c2a81a82cb 100644 --- a/src/hyperloom/inference_optimizer/tests/test_sbd_v6_initial.py +++ b/src/hyperloom/inference_optimizer/tests/test_sbd_v6_initial.py @@ -14,6 +14,7 @@ from hyperloom.inference_optimizer.breakdown import exporter from hyperloom.inference_optimizer.breakdown.collectors.v6 import collect_v6_timeline +from hyperloom.inference_optimizer.breakdown.critic_reviews import normalize_framework_reviews from hyperloom.inference_optimizer.breakdown.schema import SCHEMA_VERSION_V5 from hyperloom.inference_optimizer.session.sbd_v6 import ( SCHEMA_VERSION_V6, @@ -146,7 +147,7 @@ def test_v6_projection_is_additive_to_v5_breakdown(tmp_path): after = exporter.build(tmp_path) assert after["schema_version"] == SCHEMA_VERSION_V5 - v6_keys = {"metadata", "outcome", "timeline", "close"} + v6_keys = {"exported_at_utc", "metadata", "outcome", "timeline", "close"} assert {key: value for key, value in after.items() if key not in v6_keys} == { key: value for key, value in before.items() if key not in v6_keys } @@ -163,7 +164,7 @@ def test_v6_projection_is_additive_to_v5_breakdown(tmp_path): def test_invalid_v6_event_does_not_change_v5_warnings(tmp_path): before = exporter.build(tmp_path) - path = tmp_path / "reports" / "sbd_v6" / "install.json" + path = tmp_path / "reports" / "sbd_v6" / "timeline" / "000001-install.json" path.parent.mkdir(parents=True) path.write_text("{invalid", encoding="utf-8") @@ -209,7 +210,7 @@ def test_install_event_stays_pending_until_session_creation(tmp_path): resolved_urls=("", "https://api.openai.com/v1"), ) - assert not (tmp_path / "reports" / "sbd_v6" / "install.json").exists() + assert not (tmp_path / "reports" / "sbd_v6" / "timeline").exists() preflight._persist_install_event(args, tmp_path) persisted = read_timeline_event(tmp_path, "install") @@ -222,7 +223,8 @@ def test_install_event_stays_pending_until_session_creation(tmp_path): assert persisted["ext"]["runtime_snapshot"]["provider_mode"] == "openai" -def test_preflight_hard_failure_creates_session_and_final_sbd(tmp_path, monkeypatch): +@pytest.mark.parametrize("failure", [SystemExit(2), KeyboardInterrupt()]) +def test_expected_preflight_exit_does_not_create_session(tmp_path, monkeypatch, failure): import hyperloom.inference_optimizer.cli as optimizer_cli from hyperloom.inference_optimizer.cli import preflight from hyperloom.inference_optimizer.session.paths import ENV_CURRENT_SESSION_DIR @@ -241,7 +243,7 @@ def fail_preflight(args): event = preflight._begin_install_event(args) def reject_credentials(): - raise SystemExit(2) + raise failure preflight._run_install_step( event, @@ -253,20 +255,60 @@ def reject_credentials(): monkeypatch.setattr(optimizer_cli, "_preflight", fail_preflight) args = optimizer_cli._build_parser().parse_args(["optimize", "--model", str(model)]) - with pytest.raises(SystemExit) as exc: + with pytest.raises(type(failure)) as exc: asyncio.run(optimizer_cli._run_optimize(args)) - assert exc.value.code == 2 - session_dir = Path(os.environ[ENV_CURRENT_SESSION_DIR]) - assert session_dir.is_relative_to(workspace) - install = read_timeline_event(session_dir, "install") - assert install is not None - assert install["status"] == "failed" - assert install["ext"]["hard_fail_step_id"] == "validate_credentials" - assert (session_dir / "manifest.json").is_file() - breakdown = json.loads((session_dir / "session_breakdown.json").read_text(encoding="utf-8")) - assert [(event["type"], event["status"]) for event in breakdown["timeline"]] == [("install", "failed")] - assert breakdown["outcome"]["status"] == "failed" + if isinstance(failure, SystemExit): + assert exc.value.code == 2 + assert ENV_CURRENT_SESSION_DIR not in os.environ + assert not workspace.exists() or not list(workspace.rglob("session_breakdown.json")) + + +@pytest.mark.parametrize("failure", [SystemExit(2), KeyboardInterrupt()]) +def test_expected_resume_preflight_exit_does_not_mutate_session(tmp_path, monkeypatch, failure): + import hyperloom.inference_optimizer.cli as optimizer_cli + from hyperloom.inference_optimizer.cli import preflight + from hyperloom.inference_optimizer.session.paths import ENV_CURRENT_SESSION_DIR + + workspace = tmp_path / "sessions" + resume_dir = workspace / "Qwen-Test" / "existing-session" + _write_json(resume_dir / "session_breakdown.json", {"sentinel": "unchanged"}) + before = { + path.relative_to(resume_dir).as_posix(): path.read_bytes() for path in resume_dir.rglob("*") if path.is_file() + } + monkeypatch.setenv("USER_DATA_PATH", str(workspace)) + monkeypatch.delenv(ENV_CURRENT_SESSION_DIR, raising=False) + monkeypatch.setattr( + optimizer_cli, + "clean_stale_aiter_locks", + lambda: {"dir": "", "deleted": 0, "skipped_fresh": 0, "errors": 0}, + ) + + def fail_preflight(args): + event = preflight._begin_install_event(args) + + def reject_environment(): + raise failure + + preflight._run_install_step( + event, + step_id="validate_environment", + category="check", + action=reject_environment, + ) + + monkeypatch.setattr(optimizer_cli, "_preflight", fail_preflight) + args = optimizer_cli._build_parser().parse_args(["optimize", "--resume-from", str(resume_dir)]) + + with pytest.raises(type(failure)): + asyncio.run(optimizer_cli._run_optimize(args)) + + after = { + path.relative_to(resume_dir).as_posix(): path.read_bytes() for path in resume_dir.rglob("*") if path.is_file() + } + assert after == before + assert ENV_CURRENT_SESSION_DIR not in os.environ + assert not list(workspace.rglob("*failed-attempt*")) def test_unwrapped_preflight_failure_is_persisted_as_failed(tmp_path, monkeypatch): @@ -310,7 +352,7 @@ def fail_runtime_paths(): assert breakdown["outcome"]["status"] == "failed" -def test_busy_resume_preflight_failure_uses_isolated_session(tmp_path, monkeypatch): +def test_resume_preflight_failure_uses_isolated_session(tmp_path, monkeypatch): import hyperloom.inference_optimizer.cli as optimizer_cli from hyperloom.inference_optimizer.cli import preflight from hyperloom.inference_optimizer.session.paths import ENV_CURRENT_SESSION_DIR @@ -328,7 +370,8 @@ def test_busy_resume_preflight_failure_uses_isolated_session(tmp_path, monkeypat "end_time": "2026-08-27T01:01:00+00:00", "ext": {"run_kind": "fresh", "steps": []}, } - _write_json(resume_dir / "reports" / "sbd_v6" / "install.json", original_install) + original_install_public = json.loads(json.dumps(original_install)) + write_timeline_event(resume_dir, original_install) _write_json(resume_dir / "session_breakdown.json", {"sentinel": "active"}) monkeypatch.setenv("USER_DATA_PATH", str(workspace)) monkeypatch.delenv("MODEL_PATH", raising=False) @@ -339,32 +382,17 @@ def test_busy_resume_preflight_failure_uses_isolated_session(tmp_path, monkeypat lambda: {"dir": "", "deleted": 0, "skipped_fresh": 0, "errors": 0}, ) - class FakeSessionLock: - def __init__(self, session_dir): - self.session_dir = Path(session_dir) - - def acquire(self): - if self.session_dir == resume_dir: - raise optimizer_cli.SessionAlreadyRunning(resume_dir, {"pid": 123}) - return self - - def release(self): - return None - def fail_preflight(args): preflight._begin_install_event(args) raise RuntimeError("resume preflight failed") - monkeypatch.setattr(optimizer_cli, "SessionLock", FakeSessionLock) monkeypatch.setattr(optimizer_cli, "_preflight", fail_preflight) args = optimizer_cli._build_parser().parse_args(["optimize", "--resume-from", str(resume_dir)]) with pytest.raises(RuntimeError, match="resume preflight failed"): asyncio.run(optimizer_cli._run_optimize(args)) - assert json.loads((resume_dir / "reports" / "sbd_v6" / "install.json").read_text(encoding="utf-8")) == ( - original_install - ) + assert read_timeline_events(resume_dir) == [original_install_public] assert json.loads((resume_dir / "session_breakdown.json").read_text(encoding="utf-8")) == {"sentinel": "active"} failed_session = Path(os.environ[ENV_CURRENT_SESSION_DIR]) assert failed_session != resume_dir @@ -374,7 +402,7 @@ def fail_preflight(args): assert install["ext"]["run_kind"] == "resume" -def test_resume_preflight_failure_overrides_completed_outcome(tmp_path, monkeypatch): +def test_resume_preflight_failure_does_not_overwrite_completed_outcome(tmp_path, monkeypatch): import hyperloom.inference_optimizer.cli as optimizer_cli from hyperloom.inference_optimizer.cli import preflight from hyperloom.inference_optimizer.session.paths import ENV_CURRENT_SESSION_DIR @@ -400,6 +428,8 @@ def test_resume_preflight_failure_overrides_completed_outcome(tmp_path, monkeypa "ext": {"run_kind": "fresh", "steps": []}, }, ) + exporter.write_breakdown_json(resume_dir) + original_breakdown = (resume_dir / "session_breakdown.json").read_bytes() monkeypatch.setenv("USER_DATA_PATH", str(workspace)) monkeypatch.delenv(ENV_CURRENT_SESSION_DIR, raising=False) monkeypatch.setattr( @@ -418,8 +448,13 @@ def fail_preflight(args): with pytest.raises(RuntimeError, match="resume preflight failed"): asyncio.run(optimizer_cli._run_optimize(args)) - breakdown = json.loads((resume_dir / "session_breakdown.json").read_text(encoding="utf-8")) - assert breakdown["outcome"]["stop_reason"] == "target_reached" + assert (resume_dir / "session_breakdown.json").read_bytes() == original_breakdown + assert [(event["type"], event["status"]) for event in read_timeline_events(resume_dir)] == [ + ("install", "succeeded") + ] + failed_session = Path(os.environ[ENV_CURRENT_SESSION_DIR]) + assert failed_session != resume_dir + breakdown = json.loads((failed_session / "session_breakdown.json").read_text(encoding="utf-8")) assert breakdown["outcome"]["status"] == "failed" assert breakdown["timeline"][-1]["type"] == "install" assert breakdown["timeline"][-1]["status"] == "failed" @@ -544,7 +579,7 @@ def test_timeline_history_retains_fresh_and_resume_events(tmp_path, monkeypatch) assert latest_gate["ext"]["run_kind"] == "resume" -def test_timeline_history_bootstraps_legacy_fixed_events(tmp_path): +def test_timeline_reads_ignore_flat_files_without_mutating_session(tmp_path): _write_json( tmp_path / "reports" / "sbd_v6" / "install.json", { @@ -568,26 +603,16 @@ def test_timeline_history_bootstraps_legacy_fixed_events(tmp_path): }, ) - write_timeline_event( - tmp_path, - { - "type": "install", - "kind": "install", - "status": "succeeded", - "start_time": "2026-08-27T02:00:00+00:00", - "end_time": "2026-08-27T02:01:00+00:00", - "ext": {"run_kind": "resume", "steps": []}, - }, - ) + before = {path.name: path.read_bytes() for path in (tmp_path / "reports" / "sbd_v6").iterdir()} - assert [(event["type"], event["ext"]["run_kind"]) for event in read_timeline_events(tmp_path)] == [ - ("install", "fresh"), - ("model_gate", "fresh"), - ("install", "resume"), - ] + assert read_timeline_events(tmp_path) == [] + assert read_timeline_event(tmp_path, "install") is None + assert not (tmp_path / "reports" / "sbd_v6" / "timeline").exists() + after = {path.name: path.read_bytes() for path in (tmp_path / "reports" / "sbd_v6").iterdir()} + assert after == before -def test_timeline_history_recovers_after_partial_legacy_migration(tmp_path): +def test_timeline_writer_does_not_migrate_flat_files(tmp_path): fresh_install = { "type": "install", "kind": "install", @@ -597,7 +622,6 @@ def test_timeline_history_recovers_after_partial_legacy_migration(tmp_path): "ext": {"run_kind": "fresh", "steps": []}, } _write_json(tmp_path / "reports" / "sbd_v6" / "install.json", fresh_install) - _write_json(tmp_path / "reports" / "sbd_v6" / "timeline" / "000001-install.json", fresh_install) _write_json( tmp_path / "reports" / "sbd_v6" / "model_gate.json", { @@ -623,10 +647,12 @@ def test_timeline_history_recovers_after_partial_legacy_migration(tmp_path): ) assert [(event["type"], event["ext"]["run_kind"]) for event in read_timeline_events(tmp_path)] == [ - ("install", "fresh"), - ("model_gate", "fresh"), - ("install", "resume"), + ("install", "resume") ] + assert (tmp_path / "reports" / "sbd_v6" / "timeline" / "000001-install.json").is_file() + assert json.loads((tmp_path / "reports" / "sbd_v6" / "install.json").read_text(encoding="utf-8")) == ( + fresh_install + ) def test_preflight_records_install_steps_in_execution_order(tmp_path, monkeypatch): @@ -803,7 +829,7 @@ def test_each_model_gate_failure_is_written_to_final_sbd( "context_window", ] assert [check["status"] for check in event["ext"]["checks"]] == expected_statuses - assert event["ext"]["failure"]["artifacts"]["breakdown_written"] is True + assert "breakdown_written" not in event["ext"]["failure"]["artifacts"] def test_resume_model_gate_records_three_explicit_skips(tmp_path): @@ -860,6 +886,20 @@ def test_model_gate_event_write_failure_does_not_change_gate_result(tmp_path, mo model_gate._start_model_gate(args, tmp_path) assert model_gate._preflight_unsupported_model_arch(args, tmp_path) is False + breakdown = exporter.build(tmp_path) + assert any("sbd_v6.write.model_gate.event" in warning for warning in breakdown["metadata"]["warnings"]) + + +def test_model_gate_fail_fast_writes_breakdown_once(tmp_path, monkeypatch): + from hyperloom.inference_optimizer import breakdown + from hyperloom.inference_optimizer.cli import model_gate + + writes: list[Path] = [] + monkeypatch.setattr(breakdown, "write_breakdown_json", lambda session_dir: writes.append(Path(session_dir))) + + model_gate._write_model_gate_breakdown(tmp_path, failure_label="test") + + assert writes == [tmp_path] def test_model_gate_projection_failure_does_not_change_gate_result(tmp_path, monkeypatch): @@ -903,10 +943,37 @@ def test_install_projection_failure_does_not_change_step_result(monkeypatch): ) +def test_install_event_write_failure_is_exported_as_v6_warning(tmp_path, monkeypatch): + from hyperloom.inference_optimizer.cli import preflight + from hyperloom.inference_optimizer.session import sbd_v6 + + args = argparse.Namespace(resume_from=None, no_kernel=True, enable_roofline=False) + preflight._begin_install_event(args) + monkeypatch.setattr( + sbd_v6, + "write_timeline_event", + lambda *_args, **_kwargs: (_ for _ in ()).throw(OSError("install disk unavailable")), + ) + + preflight._persist_install_event(args, tmp_path) + + breakdown = exporter.build(tmp_path) + assert any("sbd_v6.write.install.event" in warning for warning in breakdown["metadata"]["warnings"]) + + +def test_missing_pending_install_event_is_exported_as_v6_warning(tmp_path): + from hyperloom.inference_optimizer.cli import preflight + + preflight._persist_install_event(argparse.Namespace(), tmp_path) + + breakdown = exporter.build(tmp_path) + assert any("pending install event is unavailable" in warning for warning in breakdown["metadata"]["warnings"]) + + def test_corrupt_model_gate_event_is_safely_normalized(tmp_path): from hyperloom.inference_optimizer.cli import model_gate - path = tmp_path / "reports" / "sbd_v6" / "model_gate.json" + path = tmp_path / "reports" / "sbd_v6" / "timeline" / "000001-model_gate.json" _write_json( path, { @@ -1859,12 +1926,8 @@ def test_framework_timeline_assigns_critic_reviews_from_request_cycle(tmp_path): _write_json( critic_dir / "request.json", { - "context": {"phase": "FRAMEWORK_AGENT"}, - "raw_prompt": ( - f"=== Shared session state ===\nmacro_cycle={cycle}\n" - if cycle == 0 - else "=== Shared session state ===\n" - ), + "context": {"phase": "FRAMEWORK_AGENT", "macro_cycle": cycle}, + "raw_prompt": "=== Shared session state ===\nmacro_cycle=99\n", }, ) _write_json( @@ -1936,6 +1999,37 @@ def test_framework_timeline_assigns_critic_reviews_from_request_cycle(tmp_path): ] +def test_framework_review_does_not_parse_macro_cycle_from_prompt(): + reviews = normalize_framework_reviews( + request={ + "context": {"phase": "FRAMEWORK_AGENT"}, + "raw_prompt": "=== Shared session state ===\nmacro_cycle=7\n", + }, + judge_bundle={ + "phase": "FRAMEWORK_AGENT", + "proposals": [ + { + "msg_id": "proposal-1", + "action_name": "integrate_patch", + "payload": {"framework_agent_candidate_id": "candidate-1"}, + } + ], + }, + review={ + "review_verdicts": [ + { + "target_proposal_msg_id": "proposal-1", + "verdict": "approve", + } + ] + }, + emit={"intent_envelope": {"intents": []}}, + review_path=None, + ) + + assert reviews[0]["macro_cycle"] is None + + def test_specialist_recorder_preserves_runtime_phase_when_entry_has_no_source_phase(tmp_path, monkeypatch): from hyperloom.inference_optimizer.breakdown.recorder import instrument @@ -2251,6 +2345,8 @@ def test_framework_critic_reviews_survive_pruning_and_reused_iteration_number(tm critic_iterations = assembled["critic_robustness"]["critic_iterations"] assert len(critic_iterations) == 2 assert len({row["iteration_id"] for row in critic_iterations}) == 2 + assert all(row["phase"] == "FRAMEWORK_AGENT" for row in critic_iterations) + assert all(row["macro_cycle"] == 0 for row in critic_iterations) timeline = collect_v6_timeline( tmp_path, diff --git a/src/hyperloom/inference_optimizer/tests/test_session_package.py b/src/hyperloom/inference_optimizer/tests/test_session_package.py index 0c9ef97b0e..42b5a53f32 100644 --- a/src/hyperloom/inference_optimizer/tests/test_session_package.py +++ b/src/hyperloom/inference_optimizer/tests/test_session_package.py @@ -40,10 +40,9 @@ def _build_session(sd: Path) -> None: _write(sd / "reports" / "optimization_journal.json", "[]") _write(sd / "reports" / "kernel_optimization_summary.json", "{}") _write(sd / "reports" / "kernel_roofline.json", "{}") - _write(sd / "reports" / "sbd_v6" / "install.json", "{}") - _write(sd / "reports" / "sbd_v6" / "model_gate.json", "{}") _write(sd / "reports" / "sbd_v6" / "timeline" / "000001-install.json", "{}") _write(sd / "reports" / "sbd_v6" / "timeline" / "000002-model_gate.json", "{}") + _write(sd / "reports" / "sbd_v6" / "write_warnings.jsonl", "{}\n") _write(sd / "reports" / "trace" / "decision_trace.jsonl", "{}\n") _write(sd / "reports" / "trace" / "llm_calls.jsonl", "{}\n") _write(sd / "target_analysis" / "target_baseline.json", "{}") @@ -106,10 +105,9 @@ def test_package_includes_curated_excludes_noise(tmp_path: Path) -> None: "reports/optimization_journal.json", "reports/kernel_optimization_summary.json", "reports/kernel_roofline.json", - "reports/sbd_v6/install.json", - "reports/sbd_v6/model_gate.json", "reports/sbd_v6/timeline/000001-install.json", "reports/sbd_v6/timeline/000002-model_gate.json", + "reports/sbd_v6/write_warnings.jsonl", "reports/trace/decision_trace.jsonl", "reports/trace/llm_calls.jsonl", "target_analysis/target_baseline.json", diff --git a/src/hyperloom/orchestrator/loop/coordinator.py b/src/hyperloom/orchestrator/loop/coordinator.py index 7b7999ac72..3794c9032e 100644 --- a/src/hyperloom/orchestrator/loop/coordinator.py +++ b/src/hyperloom/orchestrator/loop/coordinator.py @@ -2024,6 +2024,7 @@ async def _reactor_pass(self, agent_name: str) -> None: _set_trace_ctx( tick=int(self.shared_state.tick or 0), phase=(self.shared_state.phase or "") or None, + macro_cycle=int(self.shared_state.macro_cycle or 0), ) except Exception: # noqa: BLE001 pass diff --git a/src/hyperloom/orchestrator/roles/critic_agent.py b/src/hyperloom/orchestrator/roles/critic_agent.py index ee9cfd86f2..de98b15b8e 100644 --- a/src/hyperloom/orchestrator/roles/critic_agent.py +++ b/src/hyperloom/orchestrator/roles/critic_agent.py @@ -515,11 +515,12 @@ class CriticAgentBackend: # dataclass field) to avoid descriptor binding as a method. _client: Any = field(default=None, init=False, repr=False) _turn_idx: int = field(default=0, init=False, repr=False) - # Trace context (tick / phase) the Coordinator stamps before each reactor + # Trace context the Coordinator stamps before each reactor # ``run()`` so the critic's self-written llm_calls row carries the timeline # keys. _trace_tick: int | None = field(default=None, init=False, repr=False) _trace_phase: str | None = field(default=None, init=False, repr=False) + _trace_macro_cycle: int | None = field(default=None, init=False, repr=False) # Proposal msg_ids reviewed by the current turn, snapshotted for llm_calls # attribution. _trace_reviewed_msg_ids: list[str] | None = field( @@ -731,6 +732,8 @@ async def run( context = dict(self._static_context) if self._trace_phase: context["phase"] = str(self._trace_phase).strip().upper() + if self._trace_macro_cycle is not None: + context["macro_cycle"] = self._trace_macro_cycle request: dict[str, Any] = { "kind": "coordinator_inbox", "session_id": session_id, @@ -1405,12 +1408,14 @@ def set_trace_context( *, tick: int | None = None, phase: str | None = None, + macro_cycle: int | None = None, ) -> None: - """Stamp the timeline keys for the next reactor turn's trace row. + """Stamp the timeline keys for the next reactor turn and request. The Coordinator calls this before ``run()`` (it owns ``shared_state``) - so the critic's self-written ``llm_calls`` row carries the same - tick/phase the in-process reactor trace would have. Best-effort: a + so the critic request carries the current phase/macro-cycle and its + self-written ``llm_calls`` row carries the same tick/phase as the + in-process reactor trace. Best-effort: a bad value degrades to ``None`` rather than raising. """ try: @@ -1418,6 +1423,10 @@ def set_trace_context( except (TypeError, ValueError): self._trace_tick = None self._trace_phase = (str(phase) or None) if phase else None + try: + self._trace_macro_cycle = int(macro_cycle) if macro_cycle is not None else None + except (TypeError, ValueError): + self._trace_macro_cycle = None def _trace_critic_llm_call( self, From 4f5f0a7d1aa6b3046da00d91ae2f8dc0e7065d6a Mon Sep 17 00:00:00 2001 From: chenluo Date: Mon, 31 Aug 2026 16:05:49 +0800 Subject: [PATCH 7/7] style: apply Ruff formatting --- src/hyperloom/inference_optimizer/breakdown/critic_reviews.py | 1 + .../inference_optimizer/tests/test_sbd_v6_initial.py | 4 +--- 2 files changed, 2 insertions(+), 3 deletions(-) diff --git a/src/hyperloom/inference_optimizer/breakdown/critic_reviews.py b/src/hyperloom/inference_optimizer/breakdown/critic_reviews.py index a923705dea..2192ac9b68 100644 --- a/src/hyperloom/inference_optimizer/breakdown/critic_reviews.py +++ b/src/hyperloom/inference_optimizer/breakdown/critic_reviews.py @@ -38,6 +38,7 @@ "review_path", ) + def _candidate_id(value: Any) -> str: candidate = _mapping(value) return str( diff --git a/src/hyperloom/inference_optimizer/tests/test_sbd_v6_initial.py b/src/hyperloom/inference_optimizer/tests/test_sbd_v6_initial.py index c2a81a82cb..4509370af3 100644 --- a/src/hyperloom/inference_optimizer/tests/test_sbd_v6_initial.py +++ b/src/hyperloom/inference_optimizer/tests/test_sbd_v6_initial.py @@ -650,9 +650,7 @@ def test_timeline_writer_does_not_migrate_flat_files(tmp_path): ("install", "resume") ] assert (tmp_path / "reports" / "sbd_v6" / "timeline" / "000001-install.json").is_file() - assert json.loads((tmp_path / "reports" / "sbd_v6" / "install.json").read_text(encoding="utf-8")) == ( - fresh_install - ) + assert json.loads((tmp_path / "reports" / "sbd_v6" / "install.json").read_text(encoding="utf-8")) == (fresh_install) def test_preflight_records_install_steps_in_execution_order(tmp_path, monkeypatch):