diff --git a/src/hyperloom/inference_optimizer/breakdown/collectors/__init__.py b/src/hyperloom/inference_optimizer/breakdown/collectors/__init__.py index 51ef8d00b9..68e7b40e6b 100644 --- a/src/hyperloom/inference_optimizer/breakdown/collectors/__init__.py +++ b/src/hyperloom/inference_optimizer/breakdown/collectors/__init__.py @@ -186,6 +186,11 @@ _geak_reconstruct_from_disk as _geak_reconstruct_from_disk, collect_geak as collect_geak, ) +from .v6 import ( + collect_v6_metadata as collect_v6_metadata, + collect_v6_outcome as collect_v6_outcome, + collect_v6_timeline as collect_v6_timeline, +) __all__ = [ "collect_attribution", @@ -209,6 +214,9 @@ "collect_sweep", "collect_telemetry", "collect_token_usage", + "collect_v6_metadata", + "collect_v6_outcome", + "collect_v6_timeline", "collect_workload", "collect_model_info", "collect_recorded_optimizations", diff --git a/src/hyperloom/inference_optimizer/breakdown/collectors/v6.py b/src/hyperloom/inference_optimizer/breakdown/collectors/v6.py new file mode 100644 index 0000000000..38f786d7c1 --- /dev/null +++ b/src/hyperloom/inference_optimizer/breakdown/collectors/v6.py @@ -0,0 +1,2180 @@ +"""Additive V6 projections built from the existing V5 evidence.""" + +from __future__ import annotations + +import re +from datetime import datetime +from pathlib import Path +from typing import Any + +from hyperloom.common.jsonio import read_json, read_jsonl + +from ...session.sbd_v6 import SCHEMA_VERSION_V6, read_timeline_events + + +_SUCCESS_STOP_REASONS = frozenset( + { + "target_reached", + "global_converged", + "time_exhausted", + "max_ticks", + "sweep_done", + "conc_sweep_done", + } +) +_ABORTED_STOP_REASONS = frozenset({"signal", "user_stop_requested"}) +_MODEL_GATE_STOP_REASONS = frozenset( + { + "model_context_window_too_small", + "model_config_incompatible", + "unsupported_model_arch", + } +) +_FRAMEWORK_PHASES = frozenset({"FRAMEWORK_AGENT", "EXPLORE"}) +_FRAMEWORK_EXIT_REASON_MAP = { + "explore_no_more_leverage": "optimize_no_more_leverage", + "plateau_explore": "optimize_no_more_leverage", + "explore_phase_budget_exhausted": "optimize_phase_budget_exhausted", + "explore_budget_cap": "optimize_budget_cap", + "explore_force_exit_low_budget": "optimize_force_exit_low_budget", +} +_AUTHORING_TASK_KINDS = frozenset( + { + "explore_apply_retry", + "framework_authoring", + "framework_local_explore", + } +) +_MACRO_CYCLE_RE = re.compile(r"(?:^|\s)macro_cycle\s*=\s*(-?\d+)(?=\s|$)") + + +def _tool_versions(versions: Any) -> dict[str, str | None]: + if not isinstance(versions, dict): + return {} + tools: dict[str, str | None] = {} + for name, value in versions.items(): + tool = str(name or "").strip() + if not tool: + continue + if isinstance(value, str): + tools[tool] = value or None + continue + if isinstance(value, dict): + label = value.get("version") or value.get("commit") + tools[tool] = str(label) if label not in (None, "") else None + return tools + + +def _architecture(workload: dict[str, Any], model_info: dict[str, Any]) -> dict[str, Any]: + if not workload and not model_info: + return {} + model_class = str(workload.get("model_class") or "").strip() + if not model_class and model_info: + model_class = "moe" if bool(model_info.get("is_moe")) else "dense" + return { + "model_class": model_class, + "model_type": str(model_info.get("model_type") or ""), + "num_hidden_layers": model_info.get("num_hidden_layers"), + "attention_type": str(model_info.get("attention_type") or ""), + "num_experts": model_info.get("num_experts"), + } + + +def _langfuse_projection(langfuse: dict[str, Any]) -> dict[str, Any]: + config = langfuse.get("config") if isinstance(langfuse.get("config"), dict) else {} + trace_url = langfuse.get("trace_url") + if not trace_url: + host = str(config.get("host") or "").rstrip("/") + trace_id = str(langfuse.get("trace_id") or "").strip() + if host and trace_id: + trace_url = f"{host}/trace/{trace_id}" + return { + "enabled": bool(langfuse.get("enabled")), + "trace_url": trace_url or None, + } + + +def collect_v6_metadata( + *, + exported_at_utc: str, + session: dict[str, Any], + workload: dict[str, Any], + model_info: dict[str, Any], + langfuse: dict[str, Any], + versions: dict[str, Any], + state: dict[str, Any], + warnings: list[str], +) -> dict[str, Any]: + """Project V5 session/config sections into the V6 ``metadata`` shape.""" + recovery = session.get("recovery") if isinstance(session.get("recovery"), dict) else {} + task_config = { + "model_name": str(workload.get("model_name") or ""), + "model_path": str(workload.get("model_path") or ""), + "framework_name": str(workload.get("framework_name") or ""), + "framework_version": str(workload.get("framework_version") or ""), + "gpu_type": str(workload.get("gpu_type") or ""), + "tp": workload.get("tp"), + "conc": workload.get("conc"), + "isl": workload.get("isl"), + "osl": workload.get("osl"), + "precision": str(workload.get("precision") or ""), + "max_model_len": workload.get("max_model_len"), + "objective": dict(workload.get("objective") or {}), + "launch_env": dict(state.get("operator_extra_env") or {}), + "launch_server_args": str(state.get("operator_server_args") or state.get("server_args") or ""), + "architecture": _architecture(workload, model_info), + } + return { + "exported_at_utc": exported_at_utc, + "versions": { + "schema_version": SCHEMA_VERSION_V6, + "hyperloom": str(session.get("code_revision") or ""), + "framework": str(workload.get("framework_name") or "") or None, + "framework_version": str(workload.get("framework_version") or "") or None, + "tools": _tool_versions(versions), + }, + "session": { + "session_id": str(session.get("session_id") or ""), + "claw_session_id": session.get("claw_session_id"), + "sandbox_user_id": session.get("sandbox_user_id"), + "created_at_utc": str(session.get("created_at_utc") or ""), + "start_ts": str(session.get("start_ts") or ""), + "ended_at_utc": str(session.get("ended_at_utc") or ""), + "host": str(session.get("host") or ""), + "session_dir": str(session.get("session_dir") or ""), + "user_data_path": str(session.get("user_data_path") or ""), + "code_revision": str(session.get("code_revision") or ""), + "pid": int(session.get("pid") or 0), + "max_minutes": int(session.get("max_minutes") or 0), + "elapsed_minutes": float(session.get("elapsed_minutes") or 0.0), + "tick_count": int(session.get("tick_count") or 0), + "recovery": { + "recovered": bool(recovery.get("recovered")), + "crash_count": int(recovery.get("crash_count") or 0), + "degraded_mode": bool(recovery.get("degraded_mode")), + }, + }, + "task_config": task_config, + "langfuse": _langfuse_projection(langfuse), + "warnings": list(warnings), + } + + +def _mapping(value: Any) -> dict[str, Any]: + return value if isinstance(value, dict) else {} + + +def _dict_rows(value: Any) -> list[dict[str, Any]]: + return [row for row in value if isinstance(row, dict)] if isinstance(value, list) else [] + + +def _dict_value_rows(value: Any) -> list[dict[str, Any]]: + return [row for row in value.values() if isinstance(row, dict)] if isinstance(value, dict) else [] + + +def _first(*values: Any) -> Any: + for value in values: + if value is not None and value != "": + return value + return None + + +def _optional_int(value: Any) -> int | None: + if value is None or value == "" or isinstance(value, bool): + return None + try: + return int(value) + except (TypeError, ValueError): + return None + + +def _optional_float(value: Any) -> float | None: + if value is None or value == "" or isinstance(value, bool): + return None + try: + return float(value) + except (TypeError, ValueError): + return None + + +def _optional_bool(value: Any) -> bool | None: + if isinstance(value, bool): + return value + if isinstance(value, int) and value in (0, 1): + return bool(value) + if isinstance(value, str): + normalized = value.strip().lower() + if normalized in {"1", "true", "yes", "on", "passed", "succeeded"}: + return True + if normalized in {"0", "false", "no", "off", "failed"}: + return False + return None + + +def _string_list(value: Any) -> list[str]: + if not isinstance(value, (list, tuple, set)): + return [] + return [str(item) for item in value if item not in (None, "")] + + +def _nested(mapping: dict[str, Any], *path: str) -> Any: + value: Any = mapping + for key in path: + if not isinstance(value, dict): + return None + value = value.get(key) + return value + + +def _row_cycle(row: dict[str, Any]) -> int | None: + for value in ( + row.get("macro_cycle"), + row.get("cycle"), + _nested(row, "outputs", "macro_cycle"), + _nested(row, "outputs", "cycle"), + _nested(row, "inputs", "macro_cycle"), + _nested(row, "inputs", "cycle"), + _nested(row, "metadata", "extras", "macro_cycle"), + _nested(row, "metadata", "extras", "cycle"), + ): + parsed = _optional_int(value) + if parsed is not None: + return parsed + return None + + +def _row_timestamp(row: dict[str, Any]) -> str: + return str( + _first( + row.get("ts"), + row.get("ended_at"), + row.get("completed_at"), + row.get("started_at"), + ) + or "" + ) + + +def _prompt_macro_cycle(value: Any) -> int | None: + match = _MACRO_CYCLE_RE.search(str(value or "")) + return _optional_int(match.group(1)) if match else None + + +def _timestamp_number(value: Any) -> float | None: + text = str(value or "").strip() + if not text: + return None + try: + return datetime.fromisoformat(text.replace("Z", "+00:00")).timestamp() + except (TypeError, ValueError): + return None + + +def _operation_name(operation: dict[str, Any]) -> str: + return str(operation.get("name") or operation.get("kind") or "").strip().lower() + + +def _declared_source_phase(row: dict[str, Any]) -> str: + return ( + str( + _first( + row.get("source_phase"), + _nested(row, "outputs", "source_phase"), + _nested(row, "inputs", "source_phase"), + _nested(row, "metadata", "extras", "source_phase"), + ) + or "" + ) + .strip() + .upper() + ) + + +def _source_phase(row: dict[str, Any]) -> str: + return _declared_source_phase(row) or str(row.get("phase") or "").strip().upper() + + +def _specialist_payloads(row: dict[str, Any]) -> tuple[dict[str, Any], ...]: + return ( + row, + _mapping(row.get("inputs")), + _mapping(row.get("outputs")), + _mapping(row.get("extensions")), + ) + + +def _is_framework_specialist(row: dict[str, Any], *, allow_legacy: bool) -> bool: + declared_phase = _declared_source_phase(row) + phase = declared_phase or str(row.get("phase") or "").strip().upper() + if not declared_phase and str(row.get("source") or "").strip().lower() == "specialist_recorder_hook": + phase = "" + if phase: + return phase in _FRAMEWORK_PHASES + + agent = str(row.get("agent") or "").strip().lower() + if agent in {"framework_agent", "explore"}: + return True + if agent in {"enablement", "kernel", "kernel_agent", "prelude", "internal"}: + return False + + payloads = _specialist_payloads(row) + if any(_optional_bool(payload.get("enablement")) is True for payload in payloads): + return False + if any( + _optional_bool(payload.get("framework_agent_authoring")) is True + or bool(payload.get("framework_agent_candidate_id")) + or bool(payload.get("framework_batch_id")) + or str(payload.get("task_kind") or "").strip().lower() in _AUTHORING_TASK_KINDS + for payload in payloads + ): + return True + return allow_legacy + + +def _operation_task_id(operation: dict[str, Any]) -> str: + return str( + _first( + _nested(operation, "extensions", "task_id"), + _nested(operation, "outputs", "task_id"), + _nested(operation, "metadata", "extras", "task_id"), + ) + or "" + ) + + +def _candidate_id(value: Any) -> str: + candidate = _mapping(value) + return str( + _first( + candidate.get("candidate_id"), + candidate.get("pr_url"), + candidate.get("url"), + candidate.get("ref"), + candidate.get("head_sha"), + ) + or "" + ) + + +def _is_framework_operation(operation: dict[str, Any]) -> bool: + name = _operation_name(operation) + phase = _source_phase(operation) + agent = str(operation.get("agent") or "").strip().lower() + outputs = _mapping(operation.get("outputs")) + if name == "framework_agent": + return True + if name == "explore": + if phase: + return phase in _FRAMEWORK_PHASES + return agent in {"framework_agent", "explore"} + if name in {"integrate", "integrate_patch"}: + if outputs.get("enablement") or outputs.get("enablement_landing"): + return False + if phase: + return phase in _FRAMEWORK_PHASES + return bool( + agent in {"framework_agent", "explore"} + or outputs.get("framework_agent_authoring") + or outputs.get("framework_agent_candidate_id") + ) + return name.startswith("specialist") and _is_framework_specialist(operation, allow_legacy=True) + + +def _new_framework_window(cycle: int, start_time: str = "") -> dict[str, Any]: + return { + "cycle": cycle, + "start_time": start_time, + "end_time": "", + "rows": [], + "exit_row": {}, + } + + +def _framework_windows( + state: dict[str, Any], + recorded_operations: list[dict[str, Any]], +) -> list[dict[str, Any]]: + windows: list[dict[str, Any]] = [] + active: dict[str, Any] | None = None + history = _dict_rows(state.get("phase_history")) + for row in history: + from_phase = str(row.get("from_phase") or "").strip().upper() + to_phase = str(row.get("to_phase") or "").strip().upper() + if not from_phase and not to_phase: + continue + cycle = _row_cycle(row) + if cycle is None: + cycle = int(state.get("macro_cycle") or 0) + from_framework = from_phase in _FRAMEWORK_PHASES + to_framework = to_phase in _FRAMEWORK_PHASES + if to_framework and not from_framework: + active = _new_framework_window(cycle, str(row.get("ts") or "")) + active["rows"].append(row) + windows.append(active) + continue + if from_framework and to_framework: + if active is None or int(active["cycle"]) != cycle: + active = _new_framework_window(cycle) + windows.append(active) + active["rows"].append(row) + continue + if from_framework and not to_framework: + if active is None or int(active["cycle"]) != cycle: + active = next( + ( + window + for window in reversed(windows) + if int(window["cycle"]) == cycle and not window["end_time"] + ), + None, + ) + if active is None: + active = _new_framework_window(cycle) + windows.append(active) + active["rows"].append(row) + active["end_time"] = str(row.get("ts") or "") + active["exit_row"] = row + active = None + continue + current_phase = str(state.get("phase") or "").strip().upper() + current_cycle = int(state.get("macro_cycle") or 0) + if current_phase in _FRAMEWORK_PHASES and not any( + int(window["cycle"]) == current_cycle and not window["end_time"] for window in windows + ): + windows.append(_new_framework_window(current_cycle, str(state.get("phase_started_ts") or ""))) + + evidence_rows: list[dict[str, Any]] = [] + for operation in recorded_operations: + if not _is_framework_operation(operation): + continue + if _operation_name(operation).startswith("specialist") and not _is_framework_specialist( + operation, + allow_legacy=False, + ): + continue + evidence_rows.append(operation) + evidence_rows.extend(_dict_rows(state.get("framework_agent_batches"))) + evidence_rows.extend(_dict_rows(state.get("framework_agent_phase_progress"))) + evidence_rows.extend( + row for row in _dict_rows(state.get("specialist_rounds")) if _is_framework_specialist(row, allow_legacy=False) + ) + evidence_rows.extend(_dict_rows(state.get("framework_config_exploration_results"))) + explore_search = _mapping(state.get("explore_search")) + tested = explore_search.get("tested") + if isinstance(tested, dict): + evidence_rows.extend(row for row in tested.values() if isinstance(row, dict)) + evidence_rows.extend(_dict_rows(explore_search.get("winners_history"))) + known_cycles = {int(window["cycle"]) for window in windows} + for row in evidence_rows: + cycle = _row_cycle(row) + if cycle is None or cycle in known_cycles: + continue + windows.append(_new_framework_window(cycle)) + known_cycles.add(cycle) + + windows.sort( + key=lambda window: ( + int(window["cycle"]), + _timestamp_number(window.get("start_time")) or float("inf"), + ) + ) + return windows + + +def _row_in_window(row: dict[str, Any], window: dict[str, Any], window_count: int) -> bool: + cycle = _row_cycle(row) + if cycle is not None and cycle != int(window["cycle"]): + return False + row_ts = _timestamp_number(_row_timestamp(row)) + start_ts = _timestamp_number(window.get("start_time")) + end_ts = _timestamp_number(window.get("end_time")) + if row_ts is not None and (start_ts is not None or end_ts is not None): + return (start_ts is None or row_ts >= start_ts) and (end_ts is None or row_ts <= end_ts) + return cycle is not None or window_count == 1 + + +def _window_operations( + recorded_operations: list[dict[str, Any]], + window: dict[str, Any], + window_count: int, +) -> list[dict[str, Any]]: + return [ + operation + for operation in recorded_operations + if _is_framework_operation(operation) and _row_in_window(operation, window, window_count) + ] + + +def _window_state_rows( + state: dict[str, Any], + field: str, + window: dict[str, Any], + window_count: int, +) -> list[dict[str, Any]]: + return [row for row in _dict_rows(state.get(field)) if _row_in_window(row, window, window_count)] + + +def _window_evidence(window: dict[str, Any]) -> dict[str, Any]: + evidence: dict[str, Any] = {} + for row in _dict_rows(window.get("rows")): + evidence.update(_mapping(row.get("evidence"))) + return evidence + + +def _operation_value(operations: list[dict[str, Any]], *paths: tuple[str, ...]) -> Any: + for operation in reversed(operations): + for path in paths: + value = _nested(operation, *path) + if value is not None and value != "": + return value + return None + + +def _framework_policy( + state: dict[str, Any], + operations: list[dict[str, Any]], + evidence: dict[str, Any], +) -> dict[str, Any]: + overrides = _mapping(state.get("plateau_overrides")) + stack_rebench_enabled = _optional_bool( + _operation_value( + operations, + ("outputs", "stack_rebench_enabled"), + ("outputs", "enable_stack_rebench"), + ("inputs", "stack_rebench_enabled"), + ("inputs", "enable_stack_rebench"), + ) + ) + return { + "keep_threshold_pct": _optional_float( + _first( + _operation_value( + operations, + ("outputs", "keep_threshold_pct"), + ("inputs", "keep_threshold_pct"), + ("decisions", "evidence", "keep_threshold_pct"), + ), + evidence.get("keep_threshold_pct"), + ) + ), + "stack_stable_threshold_pct": _optional_float( + _operation_value( + operations, + ("outputs", "stack_stable_threshold_pct"), + ("inputs", "stack_stable_threshold_pct"), + ) + ), + "stack_rebench_enabled": stack_rebench_enabled, + "variant_timeout_sec": _optional_int( + _first( + _operation_value( + operations, + ("outputs", "variant_timeout_sec"), + ("inputs", "variant_timeout_sec"), + ), + state.get("explore_variant_timeout_sec_override"), + ) + ), + "overtime_kill_ratio": _optional_float( + _first( + _operation_value( + operations, + ("outputs", "explore_overtime_kill_ratio"), + ("outputs", "overtime_kill_ratio"), + ("inputs", "explore_overtime_kill_ratio"), + ), + state.get("explore_overtime_kill_ratio"), + ) + ), + "force_exit_budget_pct": _optional_float( + _first(evidence.get("force_exit_budget_pct"), overrides.get("force_exit_budget_pct")) + ), + "config_arm": { + "keep_gain_threshold_pct": _optional_float( + _first(evidence.get("keep_gain_threshold_pct"), overrides.get("explore_keep_gain_pct")) + ), + "empty_streak_threshold": _optional_int( + _first(evidence.get("empty_streak_threshold"), overrides.get("explore_empty_streak")) + ), + "lookback": _optional_int(_first(evidence.get("lookback"), overrides.get("explore_lookback"))), + }, + "source_arm": { + "no_keep_streak_threshold": _optional_int( + _first(evidence.get("source_threshold"), evidence.get("no_keep_streak_threshold")) + ), + "discovery_retry_limit": _optional_int( + _first(evidence.get("retry_limit"), evidence.get("discovery_retry_limit")) + ), + "authoring_enabled": _optional_bool(state.get("framework_agent_authoring_enabled")), + }, + } + + +def _specialist_rows( + state: dict[str, Any], + operations: list[dict[str, Any]], + window: dict[str, Any], + window_count: int, +) -> list[dict[str, Any]]: + ordered: list[str] = [] + by_key: dict[str, dict[str, Any]] = {} + + def _upsert(row: dict[str, Any]) -> None: + key = str(_first(row.get("task_id"), row.get("round_id")) or "") + if not key: + key = f"row:{len(ordered)}" + if key not in by_key: + ordered.append(key) + by_key[key] = dict(row) + return + merged = dict(row) + merged.update(by_key[key]) + by_key[key] = merged + + for row in _window_state_rows(state, "specialist_rounds", window, window_count): + if not _is_framework_specialist(row, allow_legacy=True): + continue + _upsert(row) + for operation in operations: + if not _operation_name(operation).startswith("specialist"): + continue + row = dict(_mapping(operation.get("inputs"))) + row.update(_mapping(operation.get("outputs"))) + row.setdefault("task_id", _operation_task_id(operation)) + row.setdefault("cycle", _row_cycle(operation)) + row.setdefault("status", operation.get("status")) + row.setdefault("completed_at", operation.get("ended_at")) + row.setdefault("source_phase", operation.get("phase")) + _upsert(row) + return [by_key[key] for key in ordered] + + +def _specialist_role( + row: dict[str, Any], + candidate_map: dict[str, str], + source_task_ids: set[str], +) -> str: + task_id = str(row.get("task_id") or "") + domain = str(row.get("domain") or "").strip().lower() + task_kind = str(row.get("task_kind") or "").strip().lower() + if ( + row.get("candidate_discovery") + or task_kind == "candidate_discovery" + or domain == "candidate_discovery_specialist" + ): + return "discovery" + authoring_marker = _optional_bool(row.get("framework_agent_authoring")) is True + candidate_id = str(_first(row.get("framework_agent_candidate_id"), row.get("candidate_id")) or "") + if ( + task_id in candidate_map + or task_id in source_task_ids + or authoring_marker + or candidate_id + or task_kind in _AUTHORING_TASK_KINDS + or bool(_patch_refs(row)) + ): + return "authoring" + return "config" + + +def _specialist_status(row: dict[str, Any]) -> str: + raw = str(row.get("status") or "").strip().lower() + if raw in {"failed", "error", "timed_out", "timeout"} or row.get("error"): + return "failed" + proposals = row.get("proposal_set") + if bool(row.get("empty")) or isinstance(proposals, list) and not proposals: + return "empty" + if raw in {"empty", "skipped"}: + return "empty" + return "succeeded" + + +def _proposal_names(row: dict[str, Any]) -> list[str]: + names: list[str] = [] + for proposal in _dict_rows(row.get("proposal_set")): + name = str(_first(proposal.get("name"), proposal.get("variant_name"), proposal.get("proposal_id")) or "") + if name and name not in names: + names.append(name) + return names + + +def _config_specialist_run(row: dict[str, Any]) -> dict[str, Any]: + return { + "round_id": str(_first(row.get("round_id"), row.get("task_id")) or ""), + "task_id": str(row.get("task_id") or ""), + "status": _specialist_status(row), + "domain": str(row.get("domain") or ""), + "scope": _first(row.get("scope"), None), + "tags": _string_list(row.get("tags")), + "gap_canonical_id": _first(row.get("gap_canonical_id"), None), + "proposal_msg_id": _first(row.get("proposal_msg_id"), None), + "proposal_names": _proposal_names(row), + "reason": _first(row.get("reason"), row.get("error"), None), + } + + +def _config_source_index(rows: list[dict[str, Any]]) -> dict[str, dict[str, Any]]: + index: dict[str, dict[str, Any]] = {} + for row in rows: + info = { + "specialist_round_id": _first(row.get("round_id"), row.get("task_id"), None), + "proposal_msg_id": _first(row.get("proposal_msg_id"), None), + } + for proposal in _dict_rows(row.get("proposal_set")): + for key in ( + proposal.get("fingerprint"), + proposal.get("name"), + proposal.get("variant_name"), + proposal.get("proposal_id"), + ): + text = str(key or "").strip() + if text: + index.setdefault(text, info) + return index + + +def _config_variant(raw: dict[str, Any], source_index: dict[str, dict[str, Any]]) -> dict[str, Any]: + metrics = _mapping(raw.get("metrics")) + variant = _mapping(raw.get("variant")) + combined = dict(metrics) + combined.update(variant) + combined.update(raw) + name = str(_first(combined.get("name"), combined.get("variant_name")) or "") + fingerprint = str(combined.get("fingerprint") or "") + source = source_index.get(fingerprint) or source_index.get(name) or {} + outcome = str(combined.get("outcome") or "").strip().upper() + stack_tput = _optional_float(combined.get("stack_rebench_tput")) + stack_ran = _optional_bool(combined.get("stack_rebench_ran")) + if stack_ran is None and (stack_tput is not None or combined.get("stack_rebench_workspace")): + stack_ran = True + stack_stable = _optional_bool(combined.get("stack_rebench_stable")) + if stack_stable is None and stack_ran: + if outcome == "KEEP_UNSTABLE": + stack_stable = False + elif outcome == "KEEP": + stack_stable = True + return { + "name": name, + "fingerprint": fingerprint, + "source": { + "provenance": str(combined.get("provenance") or ""), + "scope": _first(combined.get("scope"), None), + "specialist_round_id": _first(combined.get("specialist_round_id"), source.get("specialist_round_id")), + "proposal_msg_id": _first(combined.get("proposal_msg_id"), source.get("proposal_msg_id")), + "critic_iteration": _optional_int(combined.get("critic_iteration")), + }, + "config_delta": { + "extra_server_args": str(combined.get("extra_server_args") or ""), + "extra_envs": dict(_mapping(combined.get("extra_envs"))), + "remove_args": _string_list(combined.get("remove_args")), + "unset_envs": _string_list(combined.get("unset_envs")), + "args_mode": _first(combined.get("args_mode"), None), + }, + "accepted_kernels": _string_list(combined.get("accepted_kernels")), + "measurement": { + "base_tput": _optional_float(combined.get("base_tput")), + "decision_tput": _optional_float(_first(combined.get("decision_tput"), combined.get("tput"))), + "gain_pct": _optional_float(combined.get("gain_pct")), + "runtime_sec": _optional_float(combined.get("runtime_sec")), + "estimated_output_throughput": _optional_float(combined.get("estimated_output_throughput")), + }, + "accuracy": { + "required": _optional_bool(_first(combined.get("accuracy_required"), combined.get("require_accuracy"))), + "reference": _optional_float(_first(combined.get("accuracy_reference"), combined.get("accuracy_baseline"))), + "value": _optional_float(combined.get("accuracy")), + "passed": _optional_bool(_first(combined.get("accuracy_pass"), combined.get("accuracy_passed"))), + }, + "stack_rebench": { + "ran": stack_ran, + "tput": stack_tput, + "stable": stack_stable, + }, + "outcome": outcome, + "reason": _first(combined.get("reason"), None), + "stage": _first(combined.get("stage"), None), + "failure": { + "error_class": _first(combined.get("error_class"), None), + "error_excerpt": _first(combined.get("error_excerpt"), combined.get("error"), None), + }, + "artifacts": { + "workspace": _first(combined.get("workspace"), combined.get("single_workspace"), None), + "stack_rebench_workspace": _first(combined.get("stack_rebench_workspace"), None), + "server_log_path": _first(combined.get("server_log_path"), None), + "raw_result_path": _first(combined.get("raw_result_path"), None), + }, + } + + +def _round_variant_rows(outputs: dict[str, Any]) -> list[dict[str, Any]]: + tested = _nested(outputs, "explore_search_update", "tested") + tested_by_fingerprint = tested if isinstance(tested, dict) else {} + round_id = str(outputs.get("round_id") or "") + ordered: list[dict[str, Any]] = [] + seen: set[str] = set() + for outcome in _dict_rows(outputs.get("per_variant_outcomes")): + fingerprint = str(outcome.get("fingerprint") or "") + merged = dict(_mapping(tested_by_fingerprint.get(fingerprint))) + merged.update(outcome) + key = fingerprint or str(outcome.get("variant_name") or len(ordered)) + seen.add(key) + ordered.append(merged) + for fingerprint, tested_row in tested_by_fingerprint.items(): + if not isinstance(tested_row, dict): + continue + if round_id and str(tested_row.get("round_id") or "") not in {"", round_id}: + continue + key = str(fingerprint or tested_row.get("name") or len(ordered)) + if key in seen: + continue + merged = dict(tested_row) + merged.setdefault("fingerprint", str(fingerprint)) + ordered.append(merged) + return ordered + + +def _config_round_from_operation( + operation: dict[str, Any], + source_index: dict[str, dict[str, Any]], +) -> dict[str, Any]: + outputs = _mapping(operation.get("outputs")) + inputs = _mapping(operation.get("inputs")) + last_round = _mapping(_nested(outputs, "explore_search_update", "last_round")) + input_stack = _mapping(_first(outputs.get("input_stack"), inputs.get("input_stack"))) + variants = [_config_variant(row, source_index) for row in _round_variant_rows(outputs)] + decision_mode = _first(outputs.get("decision_mode"), inputs.get("decision_mode")) + if decision_mode is None: + modes = { + str(_nested(row, "metrics", "overtime_anchor_kind") or row.get("overtime_anchor_kind") or "") + for row in _dict_rows(outputs.get("per_variant_outcomes")) + } + modes.discard("") + decision_mode = next(iter(modes)) if len(modes) == 1 else None + return { + "round_id": str( + _first( + outputs.get("round_id"), + _nested(operation, "metadata", "extras", "round_id"), + operation.get("operation_id"), + ) + or "" + ), + "task_id": _operation_task_id(operation), + "status": "succeeded" + if str(_first(outputs.get("status"), operation.get("status")) or "").lower() + in {"succeeded", "success", "completed", "kept", "keep"} + else "failed", + "framework": str( + _first(outputs.get("framework"), _nested(outputs, "workload", "framework"), inputs.get("framework")) or "" + ), + "workload_signature": str( + _first( + outputs.get("workload_signature"), + next( + ( + row.get("workload_signature") + for row in _mapping(_nested(outputs, "explore_search_update", "tested")).values() + if isinstance(row, dict) and row.get("workload_signature") + ), + None, + ), + ) + or "" + ), + "throughput_unit": _first(outputs.get("throughput_unit"), inputs.get("throughput_unit"), None), + "decision_mode": decision_mode, + "input_stack": { + "throughput": _optional_float( + _first(input_stack.get("throughput"), outputs.get("base_tput"), last_round.get("base_tput")) + ), + "accuracy": _optional_float( + _first(input_stack.get("accuracy"), outputs.get("accuracy_baseline"), inputs.get("accuracy_baseline")) + ), + "extra_server_args": str( + _first( + input_stack.get("extra_server_args"), + outputs.get("base_extra_args"), + last_round.get("base_extra_args"), + "", + ) + or "" + ), + "extra_envs": dict(_mapping(_first(input_stack.get("extra_envs"), outputs.get("base_extra_envs")))), + "remove_args": _string_list(_first(input_stack.get("remove_args"), outputs.get("base_remove_args"), [])), + "unset_envs": _string_list(_first(input_stack.get("unset_envs"), outputs.get("base_unset_envs"), [])), + "args_mode": _first(input_stack.get("args_mode"), outputs.get("base_args_mode"), None), + }, + "variants": variants, + } + + +def _fallback_config_rounds( + state: dict[str, Any], + window: dict[str, Any], + window_count: int, + source_index: dict[str, dict[str, Any]], + existing_ids: set[str], +) -> list[dict[str, Any]]: + explore_search = _mapping(state.get("explore_search")) + tested = explore_search.get("tested") + grouped: dict[str, list[dict[str, Any]]] = {} + if isinstance(tested, dict): + for fingerprint, row in tested.items(): + if not isinstance(row, dict) or not _row_in_window(row, window, window_count): + continue + round_id = str(row.get("round_id") or "") + if not round_id or round_id in existing_ids: + continue + item = dict(row) + item.setdefault("fingerprint", str(fingerprint)) + grouped.setdefault(round_id, []).append(item) + rounds: list[dict[str, Any]] = [] + for round_id, rows in grouped.items(): + measured_outcomes = {"KEEP", "REVERT", "KEEP_UNSTABLE", "KILLED_OVERTIME"} + rounds.append( + { + "round_id": round_id, + "task_id": "", + "status": ( + "succeeded" + if any(str(row.get("outcome") or "").upper() in measured_outcomes for row in rows) + else "failed" + ), + "framework": str(next((row.get("framework") for row in rows if row.get("framework")), "")), + "workload_signature": str( + next((row.get("workload_signature") for row in rows if row.get("workload_signature")), "") + ), + "throughput_unit": None, + "decision_mode": None, + "input_stack": { + "throughput": _optional_float(next((row.get("base_tput") for row in rows), None)), + "accuracy": None, + "extra_server_args": "", + "extra_envs": {}, + "remove_args": [], + "unset_envs": [], + "args_mode": None, + }, + "variants": [_config_variant(row, source_index) for row in rows], + } + ) + return rounds + + +def _config_arm( + state: dict[str, Any], + operations: list[dict[str, Any]], + specialist_rows: list[dict[str, Any]], + specialist_roles: dict[str, str], + window: dict[str, Any], + window_count: int, + evidence: dict[str, Any], + policy: dict[str, Any], +) -> dict[str, Any]: + config_rows = [ + row + for row in specialist_rows + if specialist_roles.get(str(_first(row.get("task_id"), row.get("round_id")) or "")) == "config" + ] + specialist_runs = [_config_specialist_run(row) for row in config_rows] + source_index = _config_source_index(config_rows) + rounds = [ + _config_round_from_operation(operation, source_index) + for operation in operations + if _operation_name(operation) == "explore" + ] + existing_ids = {str(row.get("round_id") or "") for row in rounds} + rounds.extend(_fallback_config_rounds(state, window, window_count, source_index, existing_ids)) + recorded_task_ids = {str(row.get("task_id") or "") for row in rounds} + for row in _window_state_rows(state, "framework_config_exploration_results", window, window_count): + task_id = str(row.get("task_id") or "") + round_id = str(_first(row.get("round_id"), task_id) or "") + if not round_id or task_id in recorded_task_ids or round_id in existing_ids: + continue + rounds.append( + { + "round_id": round_id, + "task_id": task_id, + "status": "succeeded" if str(row.get("status") or "succeeded").lower() != "failed" else "failed", + "framework": str(row.get("framework") or ""), + "workload_signature": str(row.get("workload_signature") or ""), + "throughput_unit": _first(row.get("throughput_unit"), None), + "decision_mode": _first(row.get("decision_mode"), None), + "input_stack": { + "throughput": None, + "accuracy": None, + "extra_server_args": "", + "extra_envs": {}, + "remove_args": [], + "unset_envs": [], + "args_mode": None, + }, + "variants": [], + } + ) + policy_config = _mapping(policy.get("config_arm")) + explore_search = _mapping(state.get("explore_search")) + tested_rows = [ + row for row in _dict_value_rows(explore_search.get("tested")) if _row_in_window(row, window, window_count) + ] + winner_rows = [ + row for row in _dict_rows(explore_search.get("winners_history")) if _row_in_window(row, window, window_count) + ] + recent_keep_gain = _optional_float(evidence.get("recent_keep_gain_pct")) + lookback = _optional_int(policy_config.get("lookback")) + if recent_keep_gain is None and lookback is not None and lookback > 0: + recent_keep_gain = round( + sum(_optional_float(row.get("gain_pct")) or 0.0 for row in winner_rows[-lookback:]), + 4, + ) + empty_streak = _optional_int(evidence.get("empty_streak")) + if empty_streak is None: + empty_streak = 0 + for row in reversed(specialist_runs): + if row["status"] != "empty": + break + empty_streak += 1 + tested_this_cycle = _optional_int(evidence.get("tested_this_cycle")) + if tested_this_cycle is None: + tested_this_cycle = len(tested_rows) + triggered_value = _optional_bool(evidence.get("config_arm_plateaued")) + if triggered_value is None: + keep_gain_threshold = _optional_float(policy_config.get("keep_gain_threshold_pct")) + empty_streak_threshold = _optional_int(policy_config.get("empty_streak_threshold")) + if recent_keep_gain is not None and keep_gain_threshold is not None and empty_streak_threshold is not None: + exhausted = empty_streak >= empty_streak_threshold or ( + not specialist_runs and tested_this_cycle >= empty_streak_threshold + ) + triggered_value = recent_keep_gain < keep_gain_threshold and exhausted + return { + "plateau": { + "triggered": triggered_value, + "recent_keep_gain_pct": recent_keep_gain, + "empty_streak": empty_streak, + "tested_this_cycle": tested_this_cycle, + }, + "specialist_runs": specialist_runs, + "rounds": rounds, + } + + +def _candidate_row(candidate: dict[str, Any]) -> dict[str, Any]: + audit = _mapping(candidate.get("audit")) + raw_verdict = str(_first(candidate.get("verdict"), audit.get("verdict"), candidate.get("applicability")) or "") + verdict = raw_verdict.strip().lower() + if verdict not in {"worth_a_bench", "already_present", "not_applicable"}: + verdict = "" + route = str(_first(candidate.get("route"), audit.get("recommended_next_step")) or "").strip() + if route in {"direct_apply", "direct"}: + route = "direct_framework" + if route not in {"direct_framework", "author_via_specialist"}: + route = "" + return { + "candidate_id": _candidate_id(candidate), + "source_ref": str( + _first(candidate.get("pr_url"), candidate.get("ref"), candidate.get("head_sha"), candidate.get("diff_url")) + or "" + ), + "repo": str( + _first(candidate.get("repo"), candidate.get("repo_url"), candidate.get("discovered_repo_url")) or "" + ), + "title": str(candidate.get("title") or ""), + "changed_files": _string_list(candidate.get("changed_files")), + "verdict": verdict, + "route": route, + "gap_canonical_id": _first(candidate.get("gap_canonical_id"), None), + "reason": _first(candidate.get("reason"), audit.get("reason"), candidate.get("rationale"), None), + } + + +def _candidate_index(state: dict[str, Any]) -> dict[str, dict[str, Any]]: + index: dict[str, dict[str, Any]] = {} + for batch in _dict_rows(state.get("framework_agent_batches")): + for candidate in _dict_rows(batch.get("candidates")): + candidate_id = _candidate_id(candidate) + if candidate_id: + index[candidate_id] = candidate + return index + + +def _candidate_discovery_runs( + state: dict[str, Any], + specialist_rows: list[dict[str, Any]], + specialist_roles: dict[str, str], + window: dict[str, Any], + window_count: int, +) -> list[dict[str, Any]]: + discovery_rows = [ + row + for row in specialist_rows + if specialist_roles.get(str(_first(row.get("task_id"), row.get("round_id")) or "")) == "discovery" + ] + used_tasks: set[str] = set() + timestamped_runs: list[tuple[str, dict[str, Any]]] = [] + + def _append_run(run: dict[str, Any], source: dict[str, Any]) -> None: + timestamped_runs.append((_row_timestamp(source), run)) + + for batch in _dict_rows(state.get("framework_agent_batches")): + batch_candidates = _dict_rows(batch.get("candidates")) + batch_ids = {_candidate_id(candidate) for candidate in batch_candidates if _candidate_id(candidate)} + task_id = str(batch.get("task_id") or "") + matched: dict[str, Any] | None = None + for row in discovery_rows: + row_task_id = str(row.get("task_id") or "") + proposal_ids = {_candidate_id(proposal) for proposal in _dict_rows(row.get("proposal_set"))} + if task_id and row_task_id == task_id: + matched = row + break + if row_task_id and str(batch.get("batch_id") or "").endswith(row_task_id[:8]): + matched = row + break + if batch_ids and proposal_ids.intersection(batch_ids): + matched = row + break + if matched is not None: + task_id = task_id or str(matched.get("task_id") or "") + used_tasks.add(str(matched.get("task_id") or "")) + elif not _row_in_window(batch, window, window_count): + continue + discovered_candidates = _dict_rows((matched or {}).get("proposal_set")) or batch_candidates + _append_run( + { + "task_id": task_id, + "status": _specialist_status(matched or batch) + if matched is not None + else ("succeeded" if batch_candidates else "empty"), + "batch_id": _first(batch.get("batch_id"), None), + "gap_canonical_id": _first( + batch.get("gap_canonical_id"), + (matched or {}).get("gap_canonical_id"), + None, + ), + "reason": _first(batch.get("reason"), (matched or {}).get("reason"), None), + "candidates": [_candidate_row(candidate) for candidate in discovered_candidates], + }, + matched or batch, + ) + for row in discovery_rows: + task_id = str(row.get("task_id") or "") + if task_id in used_tasks: + continue + _append_run( + { + "task_id": task_id, + "status": _specialist_status(row), + "batch_id": _first(row.get("batch_id"), None), + "gap_canonical_id": _first(row.get("gap_canonical_id"), None), + "reason": _first(row.get("reason"), row.get("error"), None), + "candidates": [_candidate_row(candidate) for candidate in _dict_rows(row.get("proposal_set"))], + }, + row, + ) + + failed_markers = 0 + terminal_retry_rows: list[dict[str, Any]] = [] + for row in _dict_rows(window.get("rows")): + evidence = _mapping(row.get("evidence")) + event = str(evidence.get("event") or "").strip() + reason = str(row.get("reason") or "").strip() + if event == "framework_agent_discover_failed": + failed_markers += 1 + _append_run( + { + "task_id": str(_first(evidence.get("task_id"), evidence.get("failed_task_id")) or ""), + "status": "failed", + "batch_id": _first(evidence.get("batch_id"), None), + "gap_canonical_id": _first(evidence.get("gap_canonical_id"), None), + "reason": _first(evidence.get("error"), reason, None), + "candidates": [], + }, + row, + ) + elif event == "framework_agent_phase_done" and reason == "discover_empty_payload": + _append_run( + { + "task_id": str(evidence.get("task_id") or ""), + "status": "empty", + "batch_id": _first(evidence.get("batch_id"), None), + "gap_canonical_id": _first(evidence.get("gap_canonical_id"), None), + "reason": reason, + "candidates": [], + }, + row, + ) + elif event == "framework_agent_phase_done" and reason == "discover_retries_exhausted": + terminal_retry_rows.append(row) + + if failed_markers == 0: + for row in terminal_retry_rows: + evidence = _mapping(row.get("evidence")) + _append_run( + { + "task_id": str(_first(evidence.get("task_id"), evidence.get("failed_task_id")) or ""), + "status": "failed", + "batch_id": _first(evidence.get("batch_id"), None), + "gap_canonical_id": _first(evidence.get("gap_canonical_id"), None), + "reason": str(row.get("reason") or "discover_retries_exhausted"), + "candidates": [], + }, + row, + ) + + if timestamped_runs and all(_timestamp_number(timestamp) is not None for timestamp, _ in timestamped_runs): + timestamped_runs.sort(key=lambda item: _timestamp_number(item[0]) or 0.0) + return [run for _, run in timestamped_runs] + + +def _patch_refs(row: dict[str, Any]) -> list[str]: + refs: list[str] = [] + for key in ("patch_refs", "patches_written", "patches", "artifacts_written"): + values = row.get(key) + if not isinstance(values, list): + continue + for value in values: + if isinstance(value, dict): + value = _first(value.get("path"), value.get("patch_path"), value.get("target"), value.get("rel_target")) + text = str(value or "").strip() + if text and text not in refs: + refs.append(text) + for proposal in _dict_rows(row.get("proposal_set")): + for value in _patch_refs(proposal): + if value not in refs: + refs.append(value) + return refs + + +def _authoring_runs( + state: dict[str, Any], + specialist_rows: list[dict[str, Any]], + specialist_roles: dict[str, str], + progress_rows: list[dict[str, Any]], +) -> list[dict[str, Any]]: + candidate_map = { + str(key): str(value) + for key, value in _mapping(state.get("framework_agent_specialist_candidate_map")).items() + if key and value + } + progress_by_task = { + str(row.get("specialist_task_id") or ""): row for row in progress_rows if row.get("specialist_task_id") + } + runs: list[dict[str, Any]] = [] + used_tasks: set[str] = set() + for row in specialist_rows: + key = str(_first(row.get("task_id"), row.get("round_id")) or "") + if specialist_roles.get(key) != "authoring": + continue + task_id = str(row.get("task_id") or "") + progress = progress_by_task.get(task_id, {}) + candidate_id = str( + _first( + row.get("candidate_id"), + row.get("framework_agent_candidate_id"), + candidate_map.get(task_id), + progress.get("candidate_id"), + ) + or "" + ) + reauthor_attempt = _optional_int( + _first( + row.get("reauthor_attempt"), + row.get("apply_retry_attempt"), + progress.get("reauthor_attempt"), + ) + ) + kind = ( + "reauthor" + if reauthor_attempt is not None and reauthor_attempt > 0 + else "local_authoring" + if candidate_id.startswith("local_explore:") or bool(row.get("framework_local_explore")) + else "candidate_authoring" + if candidate_id + else "" + ) + runs.append( + { + "task_id": task_id, + "candidate_id": candidate_id, + "kind": kind, + "status": _specialist_status(row), + "reauthor_attempt": reauthor_attempt, + "specialist_domain": str(row.get("domain") or ""), + "gap_canonical_id": _first(row.get("gap_canonical_id"), None), + "patch_refs": _patch_refs(row), + "reason": _first(row.get("reason"), row.get("error"), progress.get("rationale"), None), + } + ) + if task_id: + used_tasks.add(task_id) + for progress in progress_rows: + task_id = str(progress.get("specialist_task_id") or "") + if not task_id or task_id in used_tasks: + continue + candidate_id = str(progress.get("candidate_id") or candidate_map.get(task_id) or "") + runs.append( + { + "task_id": task_id, + "candidate_id": candidate_id, + "kind": "local_authoring" if candidate_id.startswith("local_explore:") else "candidate_authoring", + "status": "failed" + if str(progress.get("status") or "").lower() in {"dispatch_failed", "author_failed", "recovery_failed"} + else "empty" + if str(progress.get("provenance") or "").lower() == "authored_empty" + else "succeeded", + "reauthor_attempt": _optional_int(progress.get("reauthor_attempt")), + "specialist_domain": str(progress.get("domain") or ""), + "gap_canonical_id": _first(progress.get("gap_canonical_id"), None), + "patch_refs": _patch_refs(progress), + "reason": _first(progress.get("rationale"), progress.get("error"), None), + } + ) + return runs + + +def _source_attempt_status(value: Any, *, kept: Any = None) -> str: + status = str(value or "").strip().lower() + if status in {"keep", "kept", "promoted", "adopted"} or kept is True: + return "KEEP" + if status in {"kept_inert", "keep_inert"}: + return "KEEP_INERT" + if status in {"revert", "reverted", "rejected", "accuracy_unavailable_reject"}: + return "REVERT" + if status in {"critic_denied", "rejected_by_critic", "needs_review_no_evidence", "reauthor_cap"}: + return "CRITIC_DENIED" + if status in {"skipped", "already_present", "not_applicable", "author_empty", "no_patch", "no_patches"}: + return "SKIPPED" + if status in { + "failed", + "error", + "apply_failed", + "bench_reverted", + "enqueue_failed", + "dispatch_failed", + "materialize_failed", + "no_result_failed", + "apply_fail_cap", + "recovery_failed", + "repeated_review_abort", + }: + return "FAILED" + return "" + + +def _source_attempt( + operation: dict[str, Any] | None, + progress: dict[str, Any], + candidate_index: dict[str, dict[str, Any]], +) -> dict[str, Any]: + operation = operation or {} + outputs = _mapping(operation.get("outputs")) + inputs = _mapping(operation.get("inputs")) + candidate = _mapping(_first(outputs.get("candidate"), inputs.get("candidate"))) + candidate_id = str( + _first( + _candidate_id(candidate), + outputs.get("framework_agent_candidate_id"), + inputs.get("framework_agent_candidate_id"), + _nested(operation, "metadata", "extras", "candidate_id"), + progress.get("candidate_id"), + ) + or "" + ) + known_candidate = candidate_index.get(candidate_id, {}) + if not candidate: + candidate = known_candidate + task_id = str(_first(_operation_task_id(operation), progress.get("integrate_task_id")) or "") + source_task_id = str( + _first(outputs.get("specialist_task_id"), inputs.get("specialist_task_id"), progress.get("specialist_task_id")) + or "" + ) + reauthor_attempt = _optional_int( + _first(outputs.get("reauthor_attempt"), inputs.get("reauthor_attempt"), progress.get("reauthor_attempt")) + ) + route = str( + _first( + outputs.get("route"), + inputs.get("route"), + candidate.get("route"), + known_candidate.get("route"), + ) + or "" + ).strip() + if route in {"direct", "direct_apply"}: + route = "direct_framework" + patch_source = str(_first(outputs.get("patch_source"), inputs.get("patch_source")) or "").strip().lower() + if patch_source not in {"specialist_authored", "upstream_pr"}: + patch_source = "" + provenance = str(_first(outputs.get("provenance"), inputs.get("provenance")) or "").strip().lower() + if not patch_source and provenance == "upstream_pr": + patch_source = "upstream_pr" + authoring_marker = _optional_bool( + _first(outputs.get("framework_agent_authoring"), inputs.get("framework_agent_authoring")) + ) + if not patch_source and (route == "direct_framework" or source_task_id and task_id and source_task_id == task_id): + patch_source = "upstream_pr" + if not patch_source and ( + route in {"author_via_specialist", "local_authoring", "reauthor"} + or source_task_id + and source_task_id != task_id + ): + patch_source = "specialist_authored" + if not patch_source and _operation_name(operation) == "framework_agent": + patch_source = "upstream_pr" + if not patch_source and authoring_marker is True and not source_task_id: + patch_source = "specialist_authored" + lever_kind = str(_first(outputs.get("lever_kind"), inputs.get("lever_kind")) or "").strip().lower() + if not lever_kind: + lever_kind = "upstream_pr" if patch_source == "upstream_pr" else "source_patch" if patch_source else "" + if lever_kind not in {"config", "source_patch", "upstream_pr", "enablement"}: + lever_kind = "" + if not route: + if reauthor_attempt and reauthor_attempt > 0: + route = "reauthor" + elif candidate_id.startswith("local_explore:"): + route = "local_authoring" + elif patch_source == "upstream_pr": + route = "direct_framework" + elif patch_source == "specialist_authored": + route = "author_via_specialist" + parity = _mapping(outputs.get("switch_off_parity")) + stack_rebench = _mapping(outputs.get("stack_rebench")) + files = _string_list(_first(outputs.get("target_files"), candidate.get("changed_files"), [])) + applied_artifacts = _dict_rows(outputs.get("artifacts_applied")) + if not files: + files = [ + str(_first(row.get("rel_target"), row.get("target")) or "") + for row in applied_artifacts + if _first(row.get("rel_target"), row.get("target")) + ] + raw_status = _first(outputs.get("status"), progress.get("status"), operation.get("status")) + return { + "attempt_id": str(_first(operation.get("operation_id"), task_id, candidate_id) or ""), + "task_id": task_id or None, + "candidate_id": candidate_id or None, + "source_task_id": source_task_id or None, + "patch_source": patch_source or None, + "lever_kind": lever_kind or None, + "route": route or None, + "status": _source_attempt_status(raw_status, kept=progress.get("kept")), + "before_tput": _optional_float(_first(outputs.get("base_tput"), progress.get("pre_tput"))), + "after_tput": _optional_float(_first(outputs.get("output_throughput"), progress.get("post_tput"))), + "local_gain_pct": _optional_float( + _first(outputs.get("delta_pct"), outputs.get("gain_pct"), progress.get("gain_pct")) + ), + "ts": str(_first(operation.get("ended_at"), progress.get("ts")) or ""), + "source_ref": _first( + candidate.get("pr_url"), + candidate.get("ref"), + candidate.get("head_sha"), + candidate.get("diff_url"), + None, + ), + "files": files, + "reason": _first(outputs.get("reason"), progress.get("rationale"), outputs.get("error"), None), + "gates": { + "accuracy_passed": _optional_bool(outputs.get("accuracy_pass")), + "keep_threshold_pct": _optional_float(outputs.get("keep_threshold_pct")), + "switch_off_parity_passed": _optional_bool(_first(parity.get("ok"), parity.get("passed"))), + "stack_rebench_passed": _optional_bool(_first(stack_rebench.get("stable"), stack_rebench.get("ok"))), + }, + "framework_levers": _dict_rows(outputs.get("framework_levers")), + "config_delta": { + "extra_server_args": str(outputs.get("extra_server_args_applied") or ""), + "extra_envs": dict( + _mapping(_first(outputs.get("extra_envs_applied"), outputs.get("config_changes_applied"))) + ), + }, + "artifacts": { + "patches_applied": _string_list(outputs.get("patches_applied")), + "source_snapshot": _first(outputs.get("source_snapshot"), None), + "source_manifest": _first(outputs.get("source_manifest"), None), + "workspace": _first(outputs.get("workspace"), None), + }, + "failure": { + "error_class": _first(outputs.get("error_class"), progress.get("error_class"), None), + "error": _first(outputs.get("error"), progress.get("error"), None), + }, + } + + +def _source_attempts( + state: dict[str, Any], + operations: list[dict[str, Any]], + progress_rows: list[dict[str, Any]], +) -> list[dict[str, Any]]: + candidates = _candidate_index(state) + progress_by_integrate = { + str(row.get("integrate_task_id") or ""): row for row in progress_rows if row.get("integrate_task_id") + } + progress_by_candidate = { + str(row.get("candidate_id") or ""): row for row in progress_rows if row.get("candidate_id") + } + progress_by_source = { + str(row.get("specialist_task_id") or ""): row for row in progress_rows if row.get("specialist_task_id") + } + used_progress: set[int] = set() + attempts: list[dict[str, Any]] = [] + for operation in operations: + if _operation_name(operation) not in {"framework_agent", "integrate", "integrate_patch"}: + continue + outputs = _mapping(operation.get("outputs")) + candidate_id = str( + _first( + _candidate_id(outputs.get("candidate")), + outputs.get("framework_agent_candidate_id"), + _nested(operation, "metadata", "extras", "candidate_id"), + ) + or "" + ) + task_id = _operation_task_id(operation) + source_task_id = str(outputs.get("specialist_task_id") or "") + progress = ( + progress_by_integrate.get(task_id) + or progress_by_candidate.get(candidate_id) + or progress_by_source.get(source_task_id) + or {} + ) + if progress: + used_progress.add(id(progress)) + attempts.append(_source_attempt(operation, progress, candidates)) + for progress in progress_rows: + if id(progress) in used_progress or str(progress.get("status") or "").lower() == "cycle_boundary": + continue + attempts.append(_source_attempt(None, progress, candidates)) + attempts.sort(key=lambda row: row.get("ts") or "") + return attempts + + +def _critic_cycle_indexes( + session_dir: Path, + warnings: list[str], + state: dict[str, Any], + operations: list[dict[str, Any]], +) -> tuple[dict[str, int], dict[str, int]]: + task_cycles: dict[str, int] = {} + proposal_cycles: dict[str, int] = {} + candidate_cycles: dict[str, int] = {} + + def _index_row(row: dict[str, Any]) -> None: + cycle = _row_cycle(row) + if cycle is None: + return + task_id = str( + _first( + _operation_task_id(row), + row.get("task_id"), + row.get("integrate_task_id"), + row.get("specialist_task_id"), + ) + or "" + ) + if task_id: + task_cycles.setdefault(task_id, cycle) + proposal_id = str( + _first( + row.get("proposal_msg_id"), + _nested(row, "inputs", "proposal_msg_id"), + _nested(row, "outputs", "proposal_msg_id"), + _nested(row, "metadata", "extras", "proposal_msg_id"), + ) + or "" + ) + if proposal_id: + proposal_cycles.setdefault(proposal_id, cycle) + candidate_id = str( + _first( + row.get("candidate_id"), + row.get("framework_agent_candidate_id"), + _nested(row, "inputs", "framework_agent_candidate_id"), + _nested(row, "outputs", "framework_agent_candidate_id"), + _candidate_id(_nested(row, "inputs", "candidate")), + _candidate_id(_nested(row, "outputs", "candidate")), + ) + or "" + ) + if candidate_id: + candidate_cycles.setdefault(candidate_id, cycle) + + for operation in operations: + _index_row(operation) + for field in ( + "framework_agent_phase_progress", + "framework_agent_batches", + "specialist_rounds", + ): + for row in _dict_rows(state.get(field)): + _index_row(row) + cycle = _row_cycle(row) + if cycle is None: + continue + for candidate in _dict_rows(row.get("candidates")): + candidate_id = _candidate_id(candidate) + if candidate_id: + candidate_cycles.setdefault(candidate_id, cycle) + + map_path = session_dir / "reports" / "trace" / "proposal_task_map.jsonl" + if map_path.is_file(): + map_rows = read_jsonl( + map_path, + require_dict=True, + skip_malformed=True, + on_error=lambda exc: warnings.append( + f"timeline.framework_agent.critic: failed to parse {map_path}: {exc!r}" + ), + ) + for row in map_rows: + proposal_id = str(row.get("proposal_msg_id") or "") + task_id = str(row.get("task_id") or "") + cycle = task_cycles.get(task_id) + if proposal_id and cycle is not None: + proposal_cycles.setdefault(proposal_id, cycle) + return proposal_cycles, candidate_cycles + + +def _critic_review_rows( + session_dir: Path, + warnings: list[str], + state: dict[str, Any], + operations: list[dict[str, Any]], + window: dict[str, Any], + window_count: int, +) -> list[dict[str, Any]]: + root = session_dir / "critic-workdir" + if not root.is_dir(): + return [] + proposal_cycles, candidate_cycles = _critic_cycle_indexes(session_dir, warnings, state, operations) + rows: list[dict[str, Any]] = [] + for workdir in sorted(root.iterdir(), key=lambda path: path.name): + if not workdir.is_dir(): + continue + loaded: dict[str, dict[str, Any]] = {} + failed = False + for name in ("request", "judge_bundle", "review", "emit"): + path = workdir / f"{name}.json" + if not path.is_file(): + loaded[name] = {} + continue + try: + loaded[name] = read_json(path, require_dict=True, strict=True) + except Exception as exc: + warnings.append(f"timeline.framework_agent.critic: failed to parse {path}: {exc!r}") + failed = True + break + if failed: + continue + request = loaded["request"] + judge = loaded["judge_bundle"] + review = loaded["review"] + emit = loaded["emit"] + review_phase = ( + str( + _first( + judge.get("phase"), + _nested(request, "context", "phase"), + _nested(judge, "merged_context", "phase"), + ) + or "" + ) + .strip() + .upper() + ) + if review_phase and review_phase not in _FRAMEWORK_PHASES: + continue + proposals = { + str(proposal.get("msg_id") or ""): proposal + for proposal in _dict_rows(judge.get("proposals")) + if proposal.get("msg_id") + } + effective = {} + envelope = _mapping(emit.get("intent_envelope")) + for intent in _dict_rows(envelope.get("intents")): + if str(intent.get("intent_type") or "") != "review_verdict": + continue + payload = _mapping(intent.get("payload")) + target = str(payload.get("target_proposal_msg_id") or "") + if target: + effective[target] = payload + for verdict_row in _dict_rows(review.get("review_verdicts")): + proposal_id = str(verdict_row.get("target_proposal_msg_id") or "") + proposal = proposals.get(proposal_id, {}) + payload = _mapping(proposal.get("payload")) + params = _mapping(payload.get("params")) + candidate = _mapping(_first(payload.get("candidate"), params.get("candidate"))) + action = str( + _first(proposal.get("action_name"), payload.get("action_name"), payload.get("kind")) or "" + ).lower() + candidate_id = str( + _first( + payload.get("framework_agent_candidate_id"), + params.get("framework_agent_candidate_id"), + _candidate_id(candidate), + ) + or "" + ) + if action in {"params", "backends", "explore"}: + arm = "config" + target_action = "explore" + elif action in {"framework_agent", "integrate", "integrate_patch"}: + arm = "source" + target_action = "integrate_patch" + elif action == "specialist": + task_kind = str(params.get("task_kind") or "").strip().lower() + source_marker = bool( + candidate_id + or _optional_bool(params.get("framework_agent_authoring")) is True + or _optional_bool(params.get("candidate_discovery")) is True + or task_kind in _AUTHORING_TASK_KINDS + or bool(_patch_refs(params)) + ) + arm = "source" if source_marker else "config" + target_action = "specialist" + else: + continue + cycle_row = { + "cycle": _first( + payload.get("cycle"), + payload.get("macro_cycle"), + params.get("cycle"), + params.get("macro_cycle"), + proposal.get("cycle"), + proposal.get("macro_cycle"), + proposal_cycles.get(proposal_id), + candidate_cycles.get(candidate_id), + _nested(request, "context", "macro_cycle"), + _nested(request, "context", "cycle"), + _prompt_macro_cycle(request.get("raw_prompt")), + _nested(judge, "merged_context", "macro_cycle"), + _nested(judge, "merged_context", "cycle"), + ), + "ts": _first(verdict_row.get("ts"), emit.get("ts"), review.get("ts")), + } + if not _row_in_window(cycle_row, window, window_count): + continue + effective_row = _mapping(effective.get(proposal_id)) + risks = [ + { + "severity": str(risk.get("severity") or ""), + "risk": str(_first(risk.get("risk"), risk.get("summary"), risk.get("reason")) or ""), + } + for risk in _dict_rows(verdict_row.get("risks")) + ] + followup_task_ids = _string_list( + _first(effective_row.get("followup_task_ids"), verdict_row.get("followup_task_ids"), []) + ) + rows.append( + { + "proposal_msg_id": proposal_id, + "candidate_id": candidate_id or None, + "variant_name": _first(payload.get("variant_name"), params.get("variant_name"), None), + "arm": arm, + "target_action": target_action, + "source": "critic" + if str(verdict_row.get("source") or "critic") == "critic" + else "critic_unavailable", + "verdict": str(verdict_row.get("verdict") or ""), + "effective_verdict": str( + _first( + effective_row.get("verdict"), + verdict_row.get("effective_verdict"), + verdict_row.get("verdict"), + ) + or "" + ), + "reasoning": str(verdict_row.get("reasoning") or ""), + "confidence": _first(verdict_row.get("confidence"), None), + "failure_reason_code": _first(verdict_row.get("failure_reason_code"), None), + "required_evidence": _string_list(verdict_row.get("required_evidence")), + "risks": risks, + "advice_text": _first(verdict_row.get("advice_text"), effective_row.get("advice_text"), None), + "alternative_action": _first(verdict_row.get("alternative_action"), None), + "followup_task_ids": followup_task_ids, + "ts": str(_first(verdict_row.get("ts"), emit.get("ts"), review.get("ts")) or ""), + "review_path": (workdir / "review.json").relative_to(session_dir).as_posix(), + } + ) + rows.sort(key=lambda row: row.get("ts") or "") + return rows + + +def _source_arm( + state: dict[str, Any], + operations: list[dict[str, Any]], + specialist_rows: list[dict[str, Any]], + specialist_roles: dict[str, str], + progress_rows: list[dict[str, Any]], + window: dict[str, Any], + window_count: int, + evidence: dict[str, Any], +) -> dict[str, Any]: + consecutive_no_keep = _optional_int( + _first(evidence.get("source_consecutive_no_keep"), evidence.get("consecutive_no_keep")) + ) + if consecutive_no_keep is None: + consecutive_no_keep = 0 + for row in reversed(progress_rows): + if str(row.get("status") or "").lower() == "cycle_boundary": + break + if bool(row.get("kept")) or str(row.get("status") or "").lower() == "kept": + break + if str(row.get("status") or "").lower() == "dispatch_failed": + continue + consecutive_no_keep += 1 + candidates_exhausted = _optional_bool( + _first(evidence.get("source_candidates_exhausted"), evidence.get("candidates_exhausted")) + ) + if candidates_exhausted is None: + state_cycle = _optional_int(state.get("macro_cycle")) + if window_count == 1 or state_cycle == int(window["cycle"]): + candidates_exhausted = _optional_bool(state.get("framework_agent_phase_done")) + triggered = _optional_bool(evidence.get("source_arm_plateaued")) + if triggered is None: + threshold = _optional_int(_first(evidence.get("source_threshold"), evidence.get("threshold"))) + if candidates_exhausted is True: + triggered = True + elif threshold is not None: + triggered = consecutive_no_keep >= threshold + return { + "plateau": { + "triggered": triggered, + "consecutive_no_keep": consecutive_no_keep, + "candidates_exhausted": candidates_exhausted, + }, + "candidate_discovery_runs": _candidate_discovery_runs( + state, + specialist_rows, + specialist_roles, + window, + window_count, + ), + "authoring_runs": _authoring_runs(state, specialist_rows, specialist_roles, progress_rows), + "attempts": _source_attempts(state, operations, progress_rows), + } + + +def _framework_exit( + window: dict[str, Any], + config_arm: dict[str, Any], + source_arm: dict[str, Any], +) -> dict[str, Any]: + exit_row = _mapping(window.get("exit_row")) + evidence = _mapping(exit_row.get("evidence")) + raw_reason = str(_first(evidence.get("passed_through_reason"), exit_row.get("reason")) or "") + reason = _FRAMEWORK_EXIT_REASON_MAP.get(raw_reason, raw_reason) or None + trigger = _first(evidence.get("trigger"), evidence.get("evidence"), None) + if trigger == "phase_budget_cap": + trigger = "budget_cap" + if trigger is None: + trigger = { + "optimize_no_more_leverage": "both_arms_plateaued", + "optimize_phase_budget_exhausted": "phase_budget_exhausted", + "optimize_budget_cap": "budget_cap", + "optimize_force_exit_low_budget": "force_exit", + }.get(str(reason or "")) + switch_bottleneck = _optional_bool(evidence.get("switch_bottleneck")) + if switch_bottleneck is None: + plateau_values = ( + _nested(config_arm, "plateau", "triggered"), + _nested(source_arm, "plateau", "triggered"), + ) + if any(value is True for value in plateau_values): + switch_bottleneck = True + elif all(value is False for value in plateau_values): + switch_bottleneck = False + return { + "reason": reason, + "trigger": trigger, + "hint": _first(evidence.get("hint"), None), + "switch_bottleneck": switch_bottleneck, + } + + +def _framework_failure(window: dict[str, Any]) -> dict[str, Any]: + exit_row = _mapping(window.get("exit_row")) + evidence = _mapping(exit_row.get("evidence")) + reason = str(exit_row.get("reason") or "").lower() + is_failure = bool(evidence.get("error") or evidence.get("error_class") or reason.endswith("_failed")) + if is_failure: + return { + "failed_task_id": _first(evidence.get("failed_task_id"), evidence.get("task_id"), None), + "error_class": _first(evidence.get("error_class"), None), + "error": _first(evidence.get("error"), None), + } + + rows = _dict_rows(window.get("rows")) + terminal_discovery_failure = next( + ( + row + for row in reversed(rows) + if _nested(row, "evidence", "event") == "framework_agent_phase_done" + and str(row.get("reason") or "") == "discover_retries_exhausted" + ), + None, + ) + if terminal_discovery_failure is None: + return {"failed_task_id": None, "error_class": None, "error": None} + + failed_discovery = next( + (row for row in reversed(rows) if _nested(row, "evidence", "event") == "framework_agent_discover_failed"), + terminal_discovery_failure, + ) + failure_evidence = _mapping(failed_discovery.get("evidence")) + return { + "failed_task_id": _first( + failure_evidence.get("failed_task_id"), + failure_evidence.get("task_id"), + None, + ), + "error_class": _first(failure_evidence.get("error_class"), None), + "error": _first( + failure_evidence.get("error"), + terminal_discovery_failure.get("reason"), + None, + ), + } + + +def _framework_event( + session_dir: Path, + state: dict[str, Any], + recorded_operations: list[dict[str, Any]], + warnings: list[str], + window: dict[str, Any], + window_count: int, +) -> dict[str, Any]: + operations = _window_operations(recorded_operations, window, window_count) + progress_rows = _window_state_rows(state, "framework_agent_phase_progress", window, window_count) + candidate_map = { + str(key): str(value) + for key, value in _mapping(state.get("framework_agent_specialist_candidate_map")).items() + if key and value + } + source_task_ids = { + str(row.get("specialist_task_id") or "") for row in progress_rows if row.get("specialist_task_id") + } + specialist_rows = _specialist_rows(state, operations, window, window_count) + specialist_roles = { + str(_first(row.get("task_id"), row.get("round_id")) or ""): _specialist_role( + row, + candidate_map, + source_task_ids, + ) + for row in specialist_rows + } + evidence = _window_evidence(window) + policy = _framework_policy(state, operations, evidence) + config_arm = _config_arm( + state, + operations, + specialist_rows, + specialist_roles, + window, + window_count, + evidence, + policy, + ) + source_arm = _source_arm( + state, + operations, + specialist_rows, + specialist_roles, + progress_rows, + window, + window_count, + evidence, + ) + exit_details = _framework_exit(window, config_arm, source_arm) + failure = _framework_failure(window) + has_work = bool( + config_arm["specialist_runs"] + or config_arm["rounds"] + or any(run["candidates"] for run in source_arm["candidate_discovery_runs"]) + or source_arm["authoring_runs"] + or source_arm["attempts"] + ) + if any(value is not None for value in failure.values()): + status = "failed" + elif not window.get("end_time"): + status = "degraded" + elif has_work: + status = "succeeded" + else: + status = "skipped" + return { + "type": "framework_agent", + "kind": "agent", + "status": status, + "start_time": str(window.get("start_time") or ""), + "end_time": str(window.get("end_time") or ""), + "ext": { + "macro_cycle": int(window["cycle"]), + "policy": policy, + "critic_reviews": _critic_review_rows( + session_dir, + warnings, + state, + recorded_operations, + window, + window_count, + ), + "config_arm": config_arm, + "source_arm": source_arm, + "exit": exit_details, + "failure": failure, + }, + } + + +def collect_v6_timeline( + session_dir: Path, + warnings: list[str], + *, + state: dict[str, Any] | None = None, + recorded_operations: list[dict[str, Any]] | None = None, +) -> list[dict[str, Any]]: + """Load durable events and project framework work without mutating V5 state.""" + timeline = read_timeline_events(session_dir, warnings=warnings) + state = state if isinstance(state, dict) else {} + operations = [row for row in recorded_operations or [] if isinstance(row, dict)] + windows = _framework_windows(state, operations) + for window in windows: + timeline.append( + _framework_event( + session_dir, + state, + operations, + warnings, + window, + len(windows), + ) + ) + indexed = list(enumerate(timeline)) + indexed.sort( + key=lambda row: ( + _timestamp_number(_first(row[1].get("start_time"), row[1].get("end_time"))) is None, + _timestamp_number(_first(row[1].get("start_time"), row[1].get("end_time"))) or 0.0, + row[0], + ) + ) + return [event for _, event in indexed] + + +def _outcome_status(stop_reason: str) -> str: + if stop_reason in _SUCCESS_STOP_REASONS: + return "completed" + if stop_reason in _ABORTED_STOP_REASONS or not stop_reason: + return "aborted" + return "failed" + + +def _stage_reached( + state: dict[str, Any], + stop_reason: str, + timeline: list[dict[str, Any]], +) -> str: + if stop_reason in _MODEL_GATE_STOP_REASONS: + return "model_gate" + phase = str(state.get("phase") or "").strip().upper() + history = state.get("phase_history") + if isinstance(history, list): + for row in reversed(history): + if isinstance(row, dict) and str(row.get("to_phase") or "").strip(): + phase = str(row.get("to_phase") or "").strip().upper() + break + if phase == "PRELUDE": + if state.get("roofline_snapshots") or state.get("last_roofline") or state.get("roofline_attempts"): + return "roofline" + if state.get("last_profile_trace") or state.get("last_profile") or state.get("profile_attempts"): + return "profile" + if state.get("warm_replay_attempted") or state.get("warm_replay_outcome") or state.get("warm_replay_pending"): + return "warm_replay" + enablement = state.get("enablement") + if isinstance(enablement, dict) and any( + ( + int(enablement.get("attempts") or 0) > 0, + bool(enablement.get("pending")), + bool(enablement.get("validation_pending")), + bool(enablement.get("succeeded")), + bool(enablement.get("launch_log")), + bool(enablement.get("inflight_task_id")), + ) + ): + return "enablement" + baseline_tput = state.get("baseline_tput") + if ( + isinstance(baseline_tput, (int, float)) + and baseline_tput > 0 + or state.get("last_baseline") + or state.get("baseline_attempts") + or int(state.get("baseline_failure_streak") or 0) > 0 + ): + return "baseline" + if ( + state.get("warm_start_ts") + or state.get("warm_start_recipe") + or state.get("warm_start_pitfalls") + or state.get("warm_start_lessons") + or state.get("warm_start_context") + ): + return "warm_start" + phase_map = { + "FRAMEWORK_AGENT": "framework_agent", + "EXPLORE": "framework_agent", + "KERNEL_AGENT": ( + "kernel" + if any(state.get(key) for key in ("last_kernel_opt", "last_fusion", "last_gemm_tuning", "last_collective")) + else "kernel_agent" + ), + "SWEEP": ("conc_sweep" if state.get("last_conc_sweep") or state.get("last_conc_sweep_watermark") else "sweep"), + "CLOSE": "close", + } + if phase in phase_map: + return phase_map[phase] + if timeline: + return str(timeline[-1].get("type") or "") + return "install" + + +def collect_v6_outcome( + *, + session: dict[str, Any], + baseline: dict[str, Any], + final: dict[str, Any], + optimizations: dict[str, Any], + state: dict[str, Any], + timeline: list[dict[str, Any]], +) -> dict[str, Any]: + """Project V5 result sections into the V6 ``outcome`` shape.""" + stop_reason = str(session.get("stop_reason") or "").strip() + validation = optimizations.get("validation") + if not isinstance(validation, dict): + validation = {} + return { + "stop_reason": stop_reason, + "status": _outcome_status(stop_reason), + "stage_reached": _stage_reached(state, stop_reason, timeline), + "baseline": { + "throughput_tok_s_per_gpu": baseline.get("throughput_tok_s_per_gpu"), + "accuracy": baseline.get("accuracy"), + "ttft_mean_ms": baseline.get("ttft_mean_ms"), + "e2el_mean_ms": baseline.get("e2el_mean_ms"), + }, + "final": { + "throughput_tok_s_per_gpu": final.get("throughput_tok_s_per_gpu"), + "gain_pct": final.get("cumulative_gain_pct_validated", 0.0), + "action_path": list(final.get("action_path") or []), + "extra_envs": dict(final.get("extra_envs") or {}), + "extra_server_args": str(final.get("extra_server_args") or ""), + }, + "validation": { + "attributed_gain_pct": validation.get("attributed_total_gain_pct", 0.0), + "unattributed_gain_pct": validation.get("unattributed_gain_pct", 0.0), + "reconciliation_gap_pct": validation.get("reconciliation_gap_pct"), + "notes": list(validation.get("notes") or []), + }, + } + + +__all__ = [ + "collect_v6_metadata", + "collect_v6_outcome", + "collect_v6_timeline", +] diff --git a/src/hyperloom/inference_optimizer/breakdown/exporter.py b/src/hyperloom/inference_optimizer/breakdown/exporter.py index f3ec196c7d..233c9375e0 100644 --- a/src/hyperloom/inference_optimizer/breakdown/exporter.py +++ b/src/hyperloom/inference_optimizer/breakdown/exporter.py @@ -560,6 +560,48 @@ def _pick(section: str, collector_value: Any) -> Any: warnings, default={}, ) + v6_warnings = list(warnings) + timeline = _safe_collect( + "timeline", + lambda: collectors.collect_v6_timeline( + sd, + v6_warnings, + state=state, + recorded_operations=recorded_operations, + ), + v6_warnings, + default=[], + ) + outcome = _safe_collect( + "outcome", + lambda: collectors.collect_v6_outcome( + session=session_section, + baseline=baseline, + final=final, + optimizations=optimizations, + state=state, + timeline=timeline, + ), + v6_warnings, + default={}, + ) + metadata = _safe_collect( + "metadata", + lambda: collectors.collect_v6_metadata( + exported_at_utc=exported_at, + session=session_section, + workload=workload, + model_info=model_info, + langfuse=langfuse, + versions=versions, + state=state, + warnings=v6_warnings, + ), + v6_warnings, + default={}, + ) + if isinstance(metadata, dict): + metadata["warnings"] = list(v6_warnings) breakdown = { "schema_version": schema_version, @@ -619,6 +661,10 @@ def _pick(section: str, collector_value: Any) -> Any: "versions": versions, # Enablement attempt-runtime observability; {} → hidden. "enablement": enablement, + "metadata": metadata, + "outcome": outcome, + "timeline": timeline, + "close": {}, "warnings": warnings, "source_files": source_files, } diff --git a/src/hyperloom/inference_optimizer/breakdown/recorder/instrument.py b/src/hyperloom/inference_optimizer/breakdown/recorder/instrument.py index da41747f0a..766781ade9 100644 --- a/src/hyperloom/inference_optimizer/breakdown/recorder/instrument.py +++ b/src/hyperloom/inference_optimizer/breakdown/recorder/instrument.py @@ -3655,6 +3655,7 @@ def record_specialist_round( domains.append(str(entry.get("domain"))) domains.extend(str(tag) for tag in (entry.get("tags") or []) if str(tag)) domains = list(dict.fromkeys(domain for domain in domains if domain)) + source_phase = str(entry.get("source_phase") or "EXPLORE").strip().upper() record_subject( session_dir, subject_id=round_subject_id, @@ -3706,7 +3707,7 @@ def record_specialist_round( root_operation_id=operation_id, kind="specialist", name=f"specialist round {round_id}", - phase="EXPLORE", + phase=source_phase, status="succeeded" if entry.get("completed_at") else "partial", source="specialist_recorder_hook", executor_class="llm_agent", diff --git a/src/hyperloom/inference_optimizer/breakdown/schema.py b/src/hyperloom/inference_optimizer/breakdown/schema.py index 2e01e2114d..e91e1510e5 100644 --- a/src/hyperloom/inference_optimizer/breakdown/schema.py +++ b/src/hyperloom/inference_optimizer/breakdown/schema.py @@ -13,6 +13,8 @@ from typing import Any, Literal, TypedDict +from ..session.sbd_v6 import SCHEMA_VERSION_V6 + #: Historical collector-only schema retained for archived-reader identification. SCHEMA_VERSION_V2 = "hyperloom.session_breakdown.v2" @@ -2920,6 +2922,101 @@ class Integrity(TypedDict, total=False): conflicts: list[dict[str, Any]] +class V6MetadataVersions(TypedDict, total=False): + """Version identifiers projected into V6 metadata.""" + + schema_version: str + hyperloom: str + framework: str | None + framework_version: str | None + tools: dict[str, str | None] + + +class V6MetadataSession(TypedDict, total=False): + """Session identity and lifecycle fields exposed by V6 metadata.""" + + session_id: str + claw_session_id: str | None + sandbox_user_id: str | None + created_at_utc: str + start_ts: str + ended_at_utc: str + host: str + session_dir: str + user_data_path: str + code_revision: str + pid: int + max_minutes: int + elapsed_minutes: float + tick_count: int + recovery: dict[str, Any] + + +class V6TaskConfig(TypedDict, total=False): + """Launch-time workload and model architecture projected into V6.""" + + model_name: str + model_path: str + framework_name: str + framework_version: str + gpu_type: str + tp: int | None + conc: int | None + isl: int | None + osl: int | None + precision: str + max_model_len: int | None + objective: dict[str, Any] + launch_env: dict[str, str] + launch_server_args: str + architecture: dict[str, Any] + + +class V6Metadata(TypedDict, total=False): + """V6 task identity, configuration, versions, and trace entrypoint.""" + + exported_at_utc: str + versions: V6MetadataVersions + session: V6MetadataSession + task_config: V6TaskConfig + langfuse: dict[str, Any] + warnings: list[str] + + +class V6Outcome(TypedDict, total=False): + """V6 session result projection for downstream consumers.""" + + stop_reason: str + status: Literal["completed", "failed", "aborted"] + stage_reached: str + baseline: dict[str, Any] + final: dict[str, Any] + validation: dict[str, Any] + + +class V6TimelineEvent(TypedDict, total=False): + """One ordered V6 business-stage event; CLOSE is intentionally excluded.""" + + type: str + kind: str + status: str + start_time: str + end_time: str + ext: dict[str, Any] + + +class V6Close(TypedDict, total=False): + """V6 session finalization result exposed outside the business timeline.""" + + status: Literal["succeeded", "failed", "degraded"] + start_time: str + end_time: str + close_sequence_done: bool + steps: list[dict[str, Any]] + robustness: dict[str, Any] + artifacts: dict[str, Any] + + class SessionBreakdown(TypedDict, total=False): """Top-level wire shape of ``session_breakdown.json``. @@ -3015,6 +3112,10 @@ class SessionBreakdown(TypedDict, total=False): versions: dict[str, KernelToolMetadata] # Enablement attempt-runtime observability; {} → dashboard hides the block. enablement: EnablementBreakdown + metadata: V6Metadata + outcome: V6Outcome + timeline: list[V6TimelineEvent] + close: V6Close warnings: list[str] source_files: SourceFiles @@ -3025,6 +3126,7 @@ class SessionBreakdown(TypedDict, total=False): "SCHEMA_VERSION_V2", "SCHEMA_VERSION_V3", "SCHEMA_VERSION_V5", + "SCHEMA_VERSION_V6", "Adoption", "AdoptedKernel", "ArtifactRef", @@ -3119,6 +3221,13 @@ class SessionBreakdown(TypedDict, total=False): "TokenUsageAttribution", "TokenUsageBucket", "TokenUsageTimelineEntry", + "V6Metadata", + "V6MetadataSession", + "V6MetadataVersions", + "V6Close", + "V6Outcome", + "V6TaskConfig", + "V6TimelineEvent", "Workload", "WorkloadObjective", ] diff --git a/src/hyperloom/inference_optimizer/breakdown/session_package.py b/src/hyperloom/inference_optimizer/breakdown/session_package.py index 00bea4fc3a..5e029527e7 100644 --- a/src/hyperloom/inference_optimizer/breakdown/session_package.py +++ b/src/hyperloom/inference_optimizer/breakdown/session_package.py @@ -76,6 +76,7 @@ "reports/kernel_optimization_summary.json", "reports/kernel_roofline.json", "reports/conc_sweep_summary.json", + "reports/sbd_v6/*.json", "reports/trace/*.jsonl", # ── target analysis ─────────────────────────────────────────────── "target_analysis/target_baseline.json", diff --git a/src/hyperloom/inference_optimizer/cli/__init__.py b/src/hyperloom/inference_optimizer/cli/__init__.py index 7e56ae9277..fa4c631efb 100644 --- a/src/hyperloom/inference_optimizer/cli/__init__.py +++ b/src/hyperloom/inference_optimizer/cli/__init__.py @@ -43,11 +43,14 @@ _autodetect_gpu_type, _gpu_runner_type, _load_model_max_position_embeddings, + _finish_model_gate, _preflight_context_window, _preflight_model_config_compat, _preflight_unsupported_model_arch, + _record_resumed_model_gate, _resolve_gpu_type, _resolve_max_model_len, + _start_model_gate, ) from ..model_config_utils import ( summarize_model_config, @@ -108,6 +111,7 @@ ENV_USER_DATA_PATH, asset_system_prompts_dir, make_session_dir, + workspace_root, ) @@ -126,6 +130,8 @@ ) from .preflight import ( _check_gfx_arch_resolvable, + _mark_pending_install_event_failed, + _persist_install_event, _preflight as _preflight, ) @@ -1637,6 +1643,96 @@ def _exit_code_for_stop_reason(stop_reason: str | None) -> int: return 0 if (stop_reason or "") in _SUCCESS_STOP_REASONS else 1 +def _preflight_failure_session_dir(args: argparse.Namespace) -> Path: + """Return a safe session directory for a pre-session install failure.""" + resume_from = str(getattr(args, "resume_from", "") or "").strip() + if resume_from: + candidate = Path(resume_from).expanduser().resolve() + try: + candidate.relative_to(workspace_root().resolve()) + except (OSError, ValueError): + pass + else: + if candidate.is_dir(): + return candidate + + return _new_preflight_failure_session_dir(args) + + +def _new_preflight_failure_session_dir( + args: argparse.Namespace, + *, + failed_attempt: bool = False, +) -> Path: + """Create a standalone session for a failed preflight attempt.""" + model_name = resolve_model_display_name(args) + if not model_name: + model_name = Path(os.environ.get("MODEL_PATH", "")).name + if not model_name: + resume_from = str(getattr(args, "resume_from", "") or "").strip() + if resume_from: + model_name = Path(resume_from).expanduser().parent.name + if failed_attempt: + model_name = f"{model_name or 'preflight'}-failed-attempt" + return make_session_dir(model_name=model_name or "preflight-failure") + + +def _persist_preflight_failure_artifacts( + args: argparse.Namespace, + exc: BaseException, +) -> Path | None: + """Best-effort materialize the failed install event and final SBD.""" + _mark_pending_install_event_failed(args, exc) + try: + session_dir = _preflight_failure_session_dir(args) + except Exception: # noqa: BLE001 — never replace the original preflight failure + log.warning("failed to create a session for SBD V6 preflight failure", exc_info=True) + return None + + session_lock = SessionLock(session_dir) + try: + session_lock.acquire() + except Exception as lock_exc: # noqa: BLE001 — a busy resume must not mutate the active session + session_lock.release() + if not str(getattr(args, "resume_from", "") or "").strip(): + log.warning("failed to lock SBD V6 preflight failure session", exc_info=True) + return None + log.warning( + "resume session unavailable for preflight failure artifacts (%s); using an isolated failed-attempt session", + lock_exc, + ) + try: + session_dir = _new_preflight_failure_session_dir(args, failed_attempt=True) + session_lock = SessionLock(session_dir) + session_lock.acquire() + except Exception: # noqa: BLE001 — never replace the original preflight failure + session_lock.release() + log.warning("failed to create an isolated SBD V6 preflight failure session", exc_info=True) + return None + + try: + if not (session_dir / "manifest.json").is_file(): + try: + manifest_args = argparse.Namespace(**vars(args)) + if not getattr(manifest_args, "model", None): + manifest_args.model = os.environ.get("MODEL_PATH", "") + write_manifest(session_dir, args=manifest_args) + except Exception: # noqa: BLE001 — the install event can still stand alone + log.warning("failed to write manifest for SBD V6 preflight failure", exc_info=True) + + _persist_install_event(args, session_dir) + try: + from ..breakdown import write_breakdown_json + + write_breakdown_json(session_dir) + except Exception: # noqa: BLE001 — never replace the original preflight failure + log.warning("failed to write SBD V6 preflight failure breakdown", exc_info=True) + finally: + session_lock.release() + print(f"Preflight failure artifacts: {session_dir}", file=sys.stderr) + return session_dir + + async def _run_optimize(args: argparse.Namespace) -> int: """Run the ``optimize`` subcommand end to end. @@ -1770,7 +1866,14 @@ async def _run_optimize(args: argparse.Namespace) -> int: # Capture provider intent before _preflight() fills missing endpoints # (preflight may populate OPENAI_BASE_URL from ANTHROPIC_BASE_URL). codex_follows_claude = _codex_model_should_follow_claude() - resolved_urls = _preflight(args) + try: + resolved_urls = _preflight(args) + except BaseException as exc: + try: + _persist_preflight_failure_artifacts(args, exc) + except BaseException: # noqa: BLE001 — preserve the original failure exactly + log.warning("failed to preserve SBD V6 preflight failure", exc_info=True) + raise _resolve_models_for_run( args, @@ -1778,7 +1881,6 @@ async def _run_optimize(args: argparse.Namespace) -> int: claude_follows_codex=claude_follows_codex, codex_follows_claude=codex_follows_claude, ) - # Before either session branch: these are read by the fresh-launch seeding # AND by the resume path, so this is the one place that covers both. _preflight_agentx_backend(args) @@ -1821,6 +1923,7 @@ async def _run_optimize(args: argparse.Namespace) -> int: # Single-optimizer guard: take the session lock before any state.json / # lease access. Held for the whole run. session_lock = _acquire_session_lock_or_exit(session_dir) + _persist_install_event(args, session_dir) try: manifest = load_manifest(session_dir) @@ -2034,6 +2137,16 @@ async def _run_optimize(args: argparse.Namespace) -> int: reanchor_budget = bool(prior_stop or prior_crash >= 3) _begin_resume_leg(state, reanchor_budget=reanchor_budget) state.save(session_dir) + _record_resumed_model_gate( + args, + session_dir, + workload_overrides={ + "model_path": str(state.model_path or manifest.get("model_path") or ""), + "model_name": str(state.model_name or manifest.get("model_name") or ""), + "framework": str(state.framework or manifest.get("framework") or ""), + "gpu_type": str(state.gpu_type or manifest.get("gpu_type") or ""), + }, + ) if reanchor_budget: override_note = " (--force-resume override)" if force_resume and prior_stop in gated_terminal else "" print(f" → cleared stop_reason and reset crash_count (was {prior_crash}) for fresh resume{override_note}") @@ -2202,6 +2315,7 @@ async def _run_optimize(args: argparse.Namespace) -> int: # and the owner pid is published for the robustness monitor. session_lock = _acquire_session_lock_or_exit(session_dir) manifest = write_manifest(session_dir, args=args) + _persist_install_event(args, session_dir) # One-shot Langfuse startup marker so a run killed before a breakdown # still leaves a correlatable trace. Best-effort, never fatal. try: @@ -2230,6 +2344,7 @@ async def _run_optimize(args: argparse.Namespace) -> int: args, session_id=manifest["session_id"], ) + _start_model_gate(args, session_dir) # Unsupported-model preflight: reject multimodal/vision configs (runs after seed, before heavy bring-up). if _preflight_unsupported_model_arch(args, session_dir): sys.exit(2) @@ -2240,6 +2355,7 @@ async def _run_optimize(args: argparse.Namespace) -> int: # Context-window preflight: reject when ISL+OSL+headroom exceeds max_position_embeddings (no stretch by policy). if _preflight_context_window(args, session_dir): sys.exit(2) + _finish_model_gate(args, session_dir) # Recipe KB T0 anchor (after seed for recipe_canonical_id, before Coordinator); skipped when --degraded-kb. recipe_kb_client = _bootstrap_recipe_kb( args, diff --git a/src/hyperloom/inference_optimizer/cli/model_gate.py b/src/hyperloom/inference_optimizer/cli/model_gate.py index 42df18a65e..88e601b520 100644 --- a/src/hyperloom/inference_optimizer/cli/model_gate.py +++ b/src/hyperloom/inference_optimizer/cli/model_gate.py @@ -14,12 +14,13 @@ import os import struct import sys -from collections.abc import Callable +from collections.abc import Callable, Mapping from dataclasses import dataclass from pathlib import Path from typing import Any from .. import gpu_types as _gpu_types +from ...common.timeutil import now_iso from ..model_config_utils import ( # noqa: F401 - re-exported for callers/tests GEMMA2_ARCHITECTURES as _GEMMA2_ARCHITECTURES, _config_architectures, @@ -1806,6 +1807,255 @@ def _detect_incompatible_model_config( _MAX_MODEL_LEN_HEADROOM = 4096 +_MODEL_GATE_ORDER = ( + "unsupported_model_arch", + "model_config_compat", + "context_window", +) +_MODEL_GATE_EVENT_ATTR = "_sbd_v6_model_gate_event" + + +def _model_gate_workload(args: argparse.Namespace) -> dict[str, Any]: + model_path = str(getattr(args, "model", "") or "") + return { + "model_path": model_path, + "model_name": str(getattr(args, "model_display_name", "") or "") + or (Path(model_path).name if model_path else ""), + "framework": str(getattr(args, "framework", "") or os.environ.get("FRAMEWORK", "")), + "gpu_type": str(getattr(args, "gpu_type", "") or os.environ.get("TARGET_GPU_TYPE", "")), + "isl": int(getattr(args, "isl", 0) or 0), + "osl": int(getattr(args, "osl", 0) or 0), + "allow_mm_text_fallback": bool(getattr(args, "allow_mm_text_fallback", True)), + "headroom_tokens": _context_headroom_tokens(), + "headroom_env": _CONTEXT_HEADROOM_ENV, + } + + +def _new_model_gate_event(args: argparse.Namespace) -> dict[str, Any]: + return { + "type": "model_gate", + "kind": "model_gate", + "status": "succeeded", + "start_time": now_iso(timespec="seconds"), + "end_time": "", + "ext": { + "run_kind": "fresh", + "skip_reason": None, + "failed_gate_id": None, + "workload": _model_gate_workload(args), + "checks": [], + "degraded": {"active": False, "warnings": []}, + }, + } + + +def _load_model_gate_event(args: argparse.Namespace, session_dir: Path) -> dict[str, Any]: + from ..session.sbd_v6 import read_timeline_event_for_update + + event = getattr(args, _MODEL_GATE_EVENT_ATTR, None) + if not isinstance(event, dict): + event = read_timeline_event_for_update(session_dir, "model_gate") + if event is None or str(event.get("type") or "") != "model_gate": + event = _new_model_gate_event(args) + setattr(args, _MODEL_GATE_EVENT_ATTR, event) + event.setdefault("kind", "model_gate") + event.setdefault("status", "succeeded") + event.setdefault("start_time", now_iso(timespec="seconds")) + event.setdefault("end_time", "") + ext = event.get("ext") + if not isinstance(ext, dict): + ext = {} + event["ext"] = ext + ext.setdefault("run_kind", "fresh") + ext.setdefault("skip_reason", None) + ext.setdefault("failed_gate_id", None) + ext["workload"] = _model_gate_workload(args) + checks = ext.get("checks") + ext["checks"] = [row for row in checks if isinstance(row, dict)] if isinstance(checks, list) else [] + degraded = ext.get("degraded") + if not isinstance(degraded, dict): + degraded = {} + ext["degraded"] = degraded + degraded["active"] = bool(degraded.get("active")) + warnings = degraded.get("warnings") + degraded["warnings"] = [row for row in warnings if isinstance(row, dict)] if isinstance(warnings, list) else [] + return event + + +def _write_model_gate_event(session_dir: Path, event: dict[str, Any]) -> bool: + from ..session.sbd_v6 import write_timeline_event + + try: + write_timeline_event(session_dir, event) + except Exception: # noqa: BLE001 — observability must never change gate behavior + log.warning("failed to persist SBD V6 model-gate event", exc_info=True) + return False + return True + + +def _model_gate_status( + checks: list[dict[str, Any]], + *, + skip_reason: str | None = None, +) -> str: + statuses = {str(check.get("status") or "") for check in checks} + if "failed" in statuses: + return "failed" + if "warned" in statuses or "unknown" in statuses: + return "degraded" + if skip_reason: + return "skipped" + return "succeeded" + + +def _model_gate_check_order(check: dict[str, Any]) -> int: + try: + return int(check.get("order") or 0) + except (TypeError, ValueError): + return len(_MODEL_GATE_ORDER) + 1 + + +def _record_model_gate_check( + args: argparse.Namespace, + session_dir: Path, + check: dict[str, Any], + *, + failure: dict[str, Any] | None = None, + degraded_warning: dict[str, Any] | None = None, +) -> None: + try: + event = _load_model_gate_event(args, session_dir) + ext = event["ext"] + checks = [ + row for row in ext.get("checks", []) if isinstance(row, dict) and row.get("gate_id") != check.get("gate_id") + ] + checks.append(check) + checks.sort(key=_model_gate_check_order) + if failure is not None: + failed_order = int(check.get("order") or 0) + present = {str(row.get("gate_id") or "") for row in checks} + for order, gate_id in enumerate(_MODEL_GATE_ORDER, start=1): + if order > failed_order and gate_id not in present: + checks.append( + { + "gate_id": gate_id, + "order": order, + "status": "skipped", + "skip_reason": "prior_gate_failed", + "detail": {}, + } + ) + checks.sort(key=_model_gate_check_order) + ext["failed_gate_id"] = str(check.get("gate_id") or "") + ext["failure"] = failure + event["end_time"] = now_iso(timespec="seconds") + if degraded_warning is not None: + degraded = ext.setdefault("degraded", {"active": False, "warnings": []}) + degraded["active"] = True + degraded.setdefault("warnings", []).append(degraded_warning) + ext["checks"] = checks + event["status"] = _model_gate_status( + checks, + skip_reason=str(ext.get("skip_reason") or "") or None, + ) + _write_model_gate_event(session_dir, event) + except Exception: # noqa: BLE001 — V6 observability must never change gate behavior + log.warning("failed to record SBD V6 model-gate check", exc_info=True) + + +def _start_model_gate(args: argparse.Namespace, session_dir: Path) -> None: + """Create the model-gate event before the first check executes.""" + try: + event = _new_model_gate_event(args) + setattr(args, _MODEL_GATE_EVENT_ATTR, event) + _write_model_gate_event(session_dir, event) + except Exception: # noqa: BLE001 — V6 observability must never change launch behavior + log.warning("failed to initialize SBD V6 model-gate event", exc_info=True) + + +def _finish_model_gate(args: argparse.Namespace, session_dir: Path) -> None: + """Finalize a successfully completed three-check model-gate chain.""" + try: + event = _load_model_gate_event(args, session_dir) + ext = event["ext"] + event["status"] = _model_gate_status( + ext["checks"], + skip_reason=str(ext.get("skip_reason") or "") or None, + ) + event["end_time"] = now_iso(timespec="seconds") + _write_model_gate_event(session_dir, event) + except Exception: # noqa: BLE001 — V6 observability must never change launch behavior + log.warning("failed to finalize SBD V6 model-gate event", exc_info=True) + + +def _record_resumed_model_gate( + args: argparse.Namespace, + session_dir: Path, + *, + workload_overrides: Mapping[str, Any] | None = None, +) -> None: + """Persist the explicit V6 skip required by the resume path.""" + try: + timestamp = now_iso(timespec="seconds") + event = _new_model_gate_event(args) + event["status"] = "skipped" + event["start_time"] = timestamp + event["end_time"] = timestamp + event["ext"]["run_kind"] = "resume" + event["ext"]["skip_reason"] = "resume" + if workload_overrides: + event["ext"]["workload"].update(workload_overrides) + event["ext"]["checks"] = [ + { + "gate_id": gate_id, + "order": order, + "status": "skipped", + "skip_reason": "resume", + "detail": {}, + } + for order, gate_id in enumerate(_MODEL_GATE_ORDER, start=1) + ] + setattr(args, _MODEL_GATE_EVENT_ATTR, event) + _write_model_gate_event(session_dir, event) + except Exception: # noqa: BLE001 — V6 observability must never change resume behavior + log.warning("failed to record resumed SBD V6 model-gate event", exc_info=True) + + +def _write_model_gate_breakdown( + args: argparse.Namespace, + session_dir: Path, + *, + failure_label: str, +) -> None: + """Write the fail-fast SBD and then persist its truthful artifact status.""" + try: + from ..breakdown import write_breakdown_json + + write_breakdown_json(session_dir) + except Exception as exc: # noqa: BLE001 — never mask the gate failure + print( + f"WARNING: failed to write session_breakdown.json on {failure_label} fail-fast: {exc!r}", + file=sys.stderr, + ) + return + + try: + event = _load_model_gate_event(args, session_dir) + ext = event.get("ext") + failure = ext.get("failure") if isinstance(ext, dict) else None + artifacts = failure.get("artifacts") if isinstance(failure, dict) else None + if not isinstance(artifacts, dict): + return + artifacts["breakdown_written"] = True + if not _write_model_gate_event(session_dir, event): + return + write_breakdown_json(session_dir) + except Exception: # noqa: BLE001 — V6 refresh must not mask the gate failure + log.warning( + "failed to refresh session_breakdown.json with model-gate artifact status", + exc_info=True, + ) + def _context_headroom_tokens() -> int: """Resolve the context headroom (tokens); env override, else default. @@ -1892,13 +2142,71 @@ def _preflight_context_window(args: argparse.Namespace, session_dir: Path) -> bo isl = int(getattr(args, "isl", 0) or 0) osl = int(getattr(args, "osl", 0) or 0) if isl <= 0 or osl <= 0: + _record_model_gate_check( + args, + session_dir, + { + "gate_id": "context_window", + "order": 3, + "status": "skipped", + "skip_reason": "isl_osl_unset", + "detail": { + "isl": isl, + "osl": osl, + "headroom": _context_headroom_tokens(), + "required": None, + "max_position_embeddings": None, + "fits": None, + "policy": "no_context_length_override", + }, + }, + ) return False maxpos = _load_model_max_position_embeddings(str(getattr(args, "model", "") or "")) if not maxpos: + headroom = _context_headroom_tokens() + _record_model_gate_check( + args, + session_dir, + { + "gate_id": "context_window", + "order": 3, + "status": "skipped", + "skip_reason": "max_position_unknown", + "detail": { + "isl": isl, + "osl": osl, + "headroom": headroom, + "required": isl + osl + headroom, + "max_position_embeddings": None, + "fits": None, + "policy": "no_context_length_override", + }, + }, + ) return False headroom = _context_headroom_tokens() required = isl + osl + headroom if maxpos >= required: + _record_model_gate_check( + args, + session_dir, + { + "gate_id": "context_window", + "order": 3, + "status": "passed", + "skip_reason": None, + "detail": { + "isl": isl, + "osl": osl, + "headroom": headroom, + "required": required, + "max_position_embeddings": maxpos, + "fits": True, + "policy": "no_context_length_override", + }, + }, + ) return False reason = ( @@ -1938,17 +2246,38 @@ def _preflight_context_window(args: argparse.Namespace, session_dir: Path) -> bo f"WARNING: failed to persist context-window stop report: {exc!r}", file=sys.stderr, ) + _record_model_gate_check( + args, + session_dir, + { + "gate_id": "context_window", + "order": 3, + "status": "failed", + "skip_reason": None, + "detail": { + "isl": isl, + "osl": osl, + "headroom": headroom, + "required": required, + "max_position_embeddings": maxpos, + "fits": False, + "policy": "no_context_length_override", + }, + }, + failure={ + "gate_id": "context_window", + "stop_reason": "model_context_window_too_small", + "exit_code": 2, + "message": reason, + "artifacts": { + "final_json": "reports/final.json" if (session_dir / "reports" / "final.json").is_file() else None, + "breakdown_written": False, + }, + }, + ) # Delivery-artifact parity: emit session_breakdown.json here too since # fail-fast exits before coordinator.run()'s finally. - try: - from ..breakdown import write_breakdown_json - - write_breakdown_json(session_dir) - except Exception as exc: # noqa: BLE001 — best-effort; never mask the reason - print( - f"WARNING: failed to write session_breakdown.json on context fail-fast: {exc!r}", - file=sys.stderr, - ) + _write_model_gate_breakdown(args, session_dir, failure_label="context") # Langfuse parity: this gate exits before coordinator.run()'s finally, so # push the breakdown to Langfuse here too. _emit_breakdown_to_langfuse(session_dir) @@ -1985,6 +2314,25 @@ def _preflight_model_config_compat( framework=framework, ) if detail is None: + model_dir = resolve_local_model_dir(model) or Path(model) + config_path = model_dir / "config.json" + absent = not config_path.is_file() + _record_model_gate_check( + args, + session_dir, + { + "gate_id": "model_config_compat", + "order": 2, + "status": "skipped" if absent else "passed", + "skip_reason": "config_absent_soft_pass" if absent else None, + "detail": { + "config_path": str(config_path) if config_path.is_file() else None, + "incompatible": False, + "reason": None, + "detector": None, + }, + }, + ) return False name = Path(model).name or model reason = ( @@ -2018,15 +2366,35 @@ def _preflight_model_config_compat( f"WARNING: failed to persist model-config stop report: {exc!r}", file=sys.stderr, ) - try: - from ..breakdown import write_breakdown_json - - write_breakdown_json(session_dir) - except Exception as exc: # noqa: BLE001 — best-effort; never mask the reason - print( - f"WARNING: failed to write session_breakdown.json on config fail-fast: {exc!r}", - file=sys.stderr, - ) + model_dir = resolve_local_model_dir(model) or Path(model) + config_path = model_dir / "config.json" + _record_model_gate_check( + args, + session_dir, + { + "gate_id": "model_config_compat", + "order": 2, + "status": "failed", + "skip_reason": None, + "detail": { + "config_path": str(config_path) if config_path.is_file() else None, + "incompatible": True, + "reason": detail, + "detector": None, + }, + }, + failure={ + "gate_id": "model_config_compat", + "stop_reason": "model_config_incompatible", + "exit_code": 2, + "message": reason, + "artifacts": { + "final_json": "reports/final.json" if (session_dir / "reports" / "final.json").is_file() else None, + "breakdown_written": False, + }, + }, + ) + _write_model_gate_breakdown(args, session_dir, failure_label="config") # Langfuse parity: this gate exits before coordinator.run()'s finally, so # push the breakdown to Langfuse here too. _emit_breakdown_to_langfuse(session_dir) @@ -2076,11 +2444,49 @@ def _preflight_unsupported_model_arch( except Exception: # noqa: BLE001 — registry import must never block the gate is_scriptable = str(framework).strip().lower() == "xdit" if is_scriptable: + _record_model_gate_check( + args, + session_dir, + { + "gate_id": "unsupported_model_arch", + "order": 1, + "status": "skipped", + "skip_reason": "scriptable_framework", + "verdict": None, + "detail": { + "architecture": None, + "model_type": None, + "signal": None, + "allow_mm_text_fallback": bool(getattr(args, "allow_mm_text_fallback", True)), + "action": "proceed", + }, + }, + ) return False model = str(getattr(args, "model", "") or "") hit = _detect_unsupported_model(model) if hit is None: + config = _load_model_config_dict(model) + architectures = _config_architectures(config) if isinstance(config, dict) else [] + _record_model_gate_check( + args, + session_dir, + { + "gate_id": "unsupported_model_arch", + "order": 1, + "status": "passed" if isinstance(config, dict) else "unknown", + "skip_reason": None, + "verdict": "plain_text" if isinstance(config, dict) else None, + "detail": { + "architecture": architectures[0] if architectures else None, + "model_type": str(config.get("model_type") or "") if isinstance(config, dict) else None, + "signal": None, + "allow_mm_text_fallback": bool(getattr(args, "allow_mm_text_fallback", True)), + "action": "proceed", + }, + }, + ) return False name = Path(model).name or model @@ -2121,6 +2527,30 @@ def _preflight_unsupported_model_arch( f"WARNING: failed to persist degraded-mode marker: {exc!r}", file=sys.stderr, ) + _record_model_gate_check( + args, + session_dir, + { + "gate_id": "unsupported_model_arch", + "order": 1, + "status": "warned", + "skip_reason": None, + "verdict": verdict, + "detail": { + "architecture": arch, + "model_type": mt, + "signal": str(hit.get("signal") or ""), + "allow_mm_text_fallback": allow_fallback, + "action": "proceed", + }, + }, + degraded_warning={ + "kind": "multimodal_text_fallback", + "architecture": arch, + "model_type": mt, + "signal": str(hit.get("signal") or ""), + }, + ) return False reason = ( @@ -2159,17 +2589,37 @@ def _preflight_unsupported_model_arch( f"WARNING: failed to persist unsupported-model stop report: {exc!r}", file=sys.stderr, ) + _record_model_gate_check( + args, + session_dir, + { + "gate_id": "unsupported_model_arch", + "order": 1, + "status": "failed", + "skip_reason": None, + "verdict": verdict, + "detail": { + "architecture": arch, + "model_type": mt, + "signal": str(hit.get("signal") or ""), + "allow_mm_text_fallback": allow_fallback, + "action": "fail_fast", + }, + }, + failure={ + "gate_id": "unsupported_model_arch", + "stop_reason": "unsupported_model_arch", + "exit_code": 2, + "message": reason, + "artifacts": { + "final_json": "reports/final.json" if (session_dir / "reports" / "final.json").is_file() else None, + "breakdown_written": False, + }, + }, + ) # Delivery-artifact parity: emit session_breakdown.json here too since # fail-fast exits before coordinator.run()'s finally. - try: - from ..breakdown import write_breakdown_json - - write_breakdown_json(session_dir) - except Exception as exc: # noqa: BLE001 — best-effort; never mask the reason - print( - f"WARNING: failed to write session_breakdown.json on unsupported-model fail-fast: {exc!r}", - file=sys.stderr, - ) + _write_model_gate_breakdown(args, session_dir, failure_label="unsupported-model") # Langfuse parity: this gate exits before coordinator.run()'s finally, so # push the breakdown to Langfuse here too. _emit_breakdown_to_langfuse(session_dir) diff --git a/src/hyperloom/inference_optimizer/cli/preflight.py b/src/hyperloom/inference_optimizer/cli/preflight.py index 0d7a3e703b..9e7f6e5447 100644 --- a/src/hyperloom/inference_optimizer/cli/preflight.py +++ b/src/hyperloom/inference_optimizer/cli/preflight.py @@ -15,6 +15,7 @@ import subprocess import sys import tempfile +from collections.abc import Callable from pathlib import Path from typing import Any, NamedTuple @@ -35,6 +36,7 @@ from hyperloom.common.gpu_identity import AMD_GPU_DISPATCH_IDENTITIES from hyperloom.common.platform_probe import probe_cpu_platform from hyperloom.common.provenance import detect_gfx_arch +from hyperloom.common.timeutil import now_iso from .credentials import ( _is_stale_proxy_url, @@ -122,7 +124,7 @@ def _provider_only_mode() -> str: return "" -def _normalize_legacy_deepseek_env() -> None: +def _normalize_legacy_deepseek_env() -> dict[str, Any]: """Rewrite a retired ``DEEPSEEK_*`` configuration into the standard variables. DeepSeek serves the Anthropic protocol on ``/anthropic`` and the OpenAI @@ -131,6 +133,8 @@ def _normalize_legacy_deepseek_env() -> None: the only place in the runtime that reads the retired variables; everything downstream sees just the two protocol sides. """ + before = dict(os.environ) + had_legacy_config = any(os.environ.get(key) for key in LEGACY_DEEPSEEK_ENV_KEYS) updates = deepseek_compat_env() if updates: for key, value in updates.items(): @@ -142,9 +146,25 @@ def _normalize_legacy_deepseek_env() -> None: # A gateway that serves only its own models supplies the model ids too. # Exported (not just resolved) so subprocesses, GEAKv4 and the kernel-agent # installer inherit them instead of falling back to an AMD Claude id. - for key, value in provider_model_defaults().items(): + model_defaults = provider_model_defaults() + for key, value in model_defaults.items(): os.environ[key] = value print(f"Preflight: {key} -> {value} (implied by the configured gateway)") + changed = sorted(key for key in {*updates, *model_defaults} if before.get(key) != os.environ.get(key)) + if changed: + status = "applied" + skip_reason = None + elif had_legacy_config or updates or model_defaults: + status = "already_present" + skip_reason = None + else: + status = "skipped" + skip_reason = "legacy_env_absent" + return { + "status": status, + "skip_reason": skip_reason, + "detail": {"keys_set": changed}, + } def _restore_provider_only_mode(provider_mode: str, snapshot: dict[str, str | None]) -> None: @@ -196,7 +216,7 @@ def _is_placeholder_tracelens_path(value: str) -> bool: return False -def _load_dotenv_fallback() -> None: +def _load_dotenv_fallback() -> dict[str, Any]: """Source missing vars from ``$REPO_ROOT/.env``; env always wins (no-clobber). Always parses ``.env`` and loads any key not already present in the @@ -205,7 +225,11 @@ def _load_dotenv_fallback() -> None: """ env_file = _resolve_dotenv_file() if env_file is None: - return + return { + "status": "skipped", + "skip_reason": "dotenv_missing", + "detail": {"vars_loaded": 0, "source": None}, + } parsed: dict[str, str] = {} loaded = 0 for raw in env_file.read_text(encoding="utf-8", errors="replace").splitlines(): @@ -238,6 +262,11 @@ def _load_dotenv_fallback() -> None: loaded += 1 if loaded: print(f"Preflight: loaded {loaded} missing var(s) from {env_file} (env wins)") + return { + "status": "applied" if loaded else "already_present", + "skip_reason": None, + "detail": {"vars_loaded": loaded, "source": str(env_file)}, + } def _prepend_path(var: str, entry: str) -> None: @@ -304,12 +333,13 @@ def _parse_env_assignments(text: str) -> dict[str, str]: return out -def _correct_kernel_agent_path_vars(file_vars: dict[str, str], env_path: Path) -> None: +def _correct_kernel_agent_path_vars(file_vars: dict[str, str], env_path: Path) -> list[str]: """Overwrite invalid inherited path-class vars with the env file's value. Only fires when the inherited value is unset/non-existent AND the file value points at an existing dir; a valid inherited value keeps env-wins semantics. """ + corrected: list[str] = [] for key in _KERNEL_AGENT_PATH_VARS: file_val = file_vars.get(key) if not file_val or not Path(file_val).is_dir(): @@ -324,9 +354,11 @@ def _correct_kernel_agent_path_vars(file_vars: dict[str, str], env_path: Path) - file=sys.stderr, ) os.environ[key] = file_val + corrected.append(key) + return corrected -def _load_kernel_agent_env_fallback() -> None: +def _load_kernel_agent_env_fallback() -> dict[str, Any]: """Auto-source the installer-written kernel-agent env file (``$KERNEL_AGENT_ENV`` or ``$USER_DATA_PATH/runtime/kernel-agent.env.sh``). @@ -346,14 +378,26 @@ def _load_kernel_agent_env_fallback() -> None: # Root is set: no bootstrap, but still correct invalid path vars from the # env file when resolvable. if not candidate: - return + return { + "status": "already_present", + "skip_reason": None, + "detail": {"vars_loaded": 0, "env_file": None}, + } env_path = Path(candidate) if not env_path.is_file(): - return + return { + "status": "already_present", + "skip_reason": None, + "detail": {"vars_loaded": 0, "env_file": str(env_path)}, + } try: text = env_path.read_text(encoding="utf-8", errors="replace") except OSError: - return + return { + "status": "already_present", + "skip_reason": None, + "detail": {"vars_loaded": 0, "env_file": str(env_path)}, + } file_vars, dropped_file_vars = filter_untrusted_env_mapping( _parse_env_assignments(text), allow_predicate=is_allowed_kernel_agent_env_key, @@ -363,8 +407,16 @@ def _load_kernel_agent_env_fallback() -> None: f"Preflight: WARNING — ignoring unsupported kernel-agent env key {key} from {env_path}", file=sys.stderr, ) - _correct_kernel_agent_path_vars(file_vars, env_path) - return + corrected = _correct_kernel_agent_path_vars(file_vars, env_path) + return { + "status": "applied" if corrected else "already_present", + "skip_reason": None, + "detail": { + "vars_loaded": 0, + "env_file": str(env_path), + "corrected_keys": corrected, + }, + } if not candidate: print( @@ -415,7 +467,7 @@ def _load_kernel_agent_env_fallback() -> None: if key not in os.environ: os.environ[key] = value loaded += 1 - _correct_kernel_agent_path_vars(file_vars, env_path) + corrected = _correct_kernel_agent_path_vars(file_vars, env_path) if "HYPERLOOM_KERNEL_AGENT_ROOT" not in os.environ: print( f"Preflight: ERROR — sourced {env_path} ({loaded} vars) but " @@ -430,9 +482,18 @@ def _load_kernel_agent_env_fallback() -> None: f"{env_path} (env wins, HYPERLOOM_KERNEL_AGENT_ROOT=" f"{os.environ['HYPERLOOM_KERNEL_AGENT_ROOT']})" ) - - -def _ensure_python_sdks(python_exe: str, pip_extra: list[str]) -> None: + return { + "status": "applied" if loaded or corrected else "already_present", + "skip_reason": None, + "detail": { + "vars_loaded": loaded, + "env_file": str(env_path), + "corrected_keys": corrected, + }, + } + + +def _ensure_python_sdks(python_exe: str, pip_extra: list[str]) -> dict[str, Any]: """Probe-then-install runtime-imported Python SDKs using the same interpreter that imports them. Avoids first-tick BackendError after baseline burns wall time; same-interpreter install avoids @@ -454,6 +515,8 @@ def _ensure_python_sdks(python_exe: str, pip_extra: list[str]) -> None: ("openai", "openai>=1.50"), ("httpx", "httpx>=0.27"), ) + installed: list[str] = [] + already_present: list[str] = [] for module_name, pip_spec in candidates: check = subprocess.run( [python_exe, "-c", f"import {module_name}"], @@ -461,6 +524,7 @@ def _ensure_python_sdks(python_exe: str, pip_extra: list[str]) -> None: ) if check.returncode == 0: print(f"Preflight: {module_name} OK") + already_present.append(pip_spec) continue print(f"Preflight: {module_name} not importable, installing {pip_spec} ...") subprocess.run( @@ -468,6 +532,17 @@ def _ensure_python_sdks(python_exe: str, pip_extra: list[str]) -> None: check=True, ) print(f"Preflight: installed {pip_spec}") + installed.append(pip_spec) + return { + "status": "applied" if installed else "already_present", + "skip_reason": None, + "target": ",".join(spec for _, spec in candidates), + "interpreter": python_exe, + "detail": { + "installed": installed, + "already_present": already_present, + }, + } _RAY_VERSION = "2.44.1" @@ -544,7 +619,7 @@ def _ray_smoke(python_exe: str) -> subprocess.CompletedProcess: ) -def _ensure_ray(python_exe: str, pip_extra: list[str]) -> None: +def _ensure_ray(python_exe: str, pip_extra: list[str]) -> dict[str, Any]: """Probe-then-install Ray using the interpreter that will import it. Ray is used broadly (multi-node scheduling, kernel/profile/recover @@ -563,7 +638,15 @@ def _ensure_ray(python_exe: str, pip_extra: list[str]) -> None: check = _ray_smoke(python_exe) if check.returncode == 0: print("Preflight: ray OK") - return + return { + "status": "already_present", + "skip_reason": None, + "target": "ray", + "interpreter": python_exe, + "spec": _RAY_INSTALL_SPEC, + "version_after": _RAY_VERSION, + "message": None, + } reason = (check.stderr or check.stdout or "unknown Ray smoke failure").strip().splitlines()[-1] print(f"Preflight: ray/click invalid ({reason}), installing {_RAY_INSTALL_SPEC} + {_CLICK_INSTALL_SPEC} ...") subprocess.run( @@ -575,6 +658,15 @@ def _ensure_ray(python_exe: str, pip_extra: list[str]) -> None: reason = (check.stderr or check.stdout or "unknown Ray smoke failure").strip() raise RuntimeError(f"Ray install completed but smoke test still failed: {reason}") print("Preflight: ray installed OK") + return { + "status": "applied", + "skip_reason": None, + "target": "ray", + "interpreter": python_exe, + "spec": _RAY_INSTALL_SPEC, + "version_after": _RAY_VERSION, + "message": reason, + } # InferenceX benchmark_serving client-side deps. Mirrors the ``_BENCH_SERVING_DEPS`` @@ -593,7 +685,7 @@ def _ensure_ray(python_exe: str, pip_extra: list[str]) -> None: ) -def _ensure_bench_serving_deps(python_exe: str, pip_extra: list[str]) -> None: +def _ensure_bench_serving_deps(python_exe: str, pip_extra: list[str]) -> dict[str, Any]: """Probe-then-install the InferenceX benchmark_serving client deps in python_exe. ``assets/install.sh:ensure_bench_serving_deps`` installs these into the @@ -623,16 +715,32 @@ def _ensure_bench_serving_deps(python_exe: str, pip_extra: list[str]) -> None: missing = [line.strip() for line in (result.stdout or "").splitlines() if line.strip()] if not missing: print("Preflight: benchmark_serving client deps OK") - return + return { + "status": "already_present", + "skip_reason": None, + "target": "benchmark_serving client deps", + "interpreter": python_exe, + "detail": {"installed": [], "already_present": mods}, + } print(f"Preflight: installing benchmark_serving client deps: {' '.join(missing)} ...") subprocess.run( [python_exe, "-m", "pip", "install", "--quiet", "--no-cache-dir", *pip_extra, *missing], check=True, ) print("Preflight: benchmark_serving client deps installed OK") - - -def _ensure_framework_deps(args, python_exe: str, pip_extra: list[str]) -> None: + return { + "status": "applied", + "skip_reason": None, + "target": "benchmark_serving client deps", + "interpreter": python_exe, + "detail": { + "installed": missing, + "already_present": [module for module in mods if module not in missing], + }, + } + + +def _ensure_framework_deps(args, python_exe: str, pip_extra: list[str]) -> dict[str, Any]: """Install the selected framework's declared runtime deps into python_exe. Resolution mirrors the CLI's own order (``--framework`` > ``$FRAMEWORK`` > @@ -657,8 +765,29 @@ def _ensure_framework_deps(args, python_exe: str, pip_extra: list[str]) -> None: # Frameworks that ship no manifest are the common case; stay quiet unless # the manifest actually asked for something or was partly rejected. if outcome.skipped_reason and not (outcome.refused or outcome.invalid): - return - framework_deps.report(outcome, prefix="Preflight: framework deps") + status = "skipped" + else: + framework_deps.report(outcome, prefix="Preflight: framework deps") + if outcome.failed or outcome.refused or outcome.invalid: + status = "warned" + elif outcome.installed: + status = "applied" + else: + status = "already_present" + return { + "status": status, + "skip_reason": outcome.skipped_reason or None, + "target": framework, + "interpreter": python_exe, + "detail": { + "manifest": str(outcome.manifest) if outcome.manifest is not None else None, + "installed": list(outcome.installed), + "already_present": list(outcome.already_present), + "refused": list(outcome.refused), + "invalid": list(outcome.invalid), + "failed": list(outcome.failed), + }, + } # Escape hatch for the serving-framework gate below, mirroring @@ -909,7 +1038,7 @@ def _resolve_framework_build(framework: str, interpreters: list[str]) -> tuple[s return inconclusive or refuted or (None, missing) -def _check_serving_framework(args, benchmark_python: str) -> None: +def _check_serving_framework(args, benchmark_python: str) -> dict[str, Any]: """Fail fast when the selected serving framework is not importable here. The optimizer patches and rebuilds framework code in place, so the package @@ -933,41 +1062,74 @@ def _check_serving_framework(args, benchmark_python: str) -> None: ).strip().lower() or framework_registry.DEFAULT_FRAMEWORK if framework_registry.is_scriptable(framework): - return + return { + "status": "skipped", + "skip_reason": "scriptable_framework", + "target": framework, + } if os.environ.get(SKIP_FRAMEWORK_CHECK_ENV, "").strip(): print(f"Preflight: {SKIP_FRAMEWORK_CHECK_ENV} set; skipping the {framework} importability check") - return + return { + "status": "skipped", + "skip_reason": "skip_framework_check_env", + "target": framework, + } remote_base_url = os.environ.get("BENCHMARK_BASE_URL", "").strip() if remote_base_url: print(f"Preflight: BENCHMARK_BASE_URL={remote_base_url}; skipping the local {framework} check (remote server)") - return + return { + "status": "skipped", + "skip_reason": "remote_benchmark_url", + "target": framework, + } from hyperloom.inference_optimizer.multi_node._internal.external_state import external_service_url from hyperloom.orchestrator.actions.executors._multi_node_env import is_multi_node if is_multi_node() and external_service_url(): print(f"Preflight: external multi-node mode; skipping the local {framework} check (serving is on remote pods)") - return + return { + "status": "skipped", + "skip_reason": "external_multi_node", + "target": framework, + } interpreters = _framework_probe_interpreters(framework, benchmark_python) found, probe = _resolve_framework_build(framework, interpreters) evidence = _rocm_evidence(framework) if found and probe.verdict is True: print(f"Preflight: {framework} importable ({found}); {evidence} confirms a ROCm build") - return + return { + "status": "applied", + "skip_reason": None, + "target": framework, + "detail": {"probe_interpreter": found, "rocm_verified": True}, + } if found and probe.verdict is None: print( f"Preflight: WARNING — {framework} is importable ({found}) but could not verify a ROCm build " f"via {evidence}{_probe_detail_block(probe.detail)}" ) - return + return { + "status": "warned", + "skip_reason": None, + "target": framework, + "message": probe.detail or "ROCm build could not be verified", + "detail": {"probe_interpreter": found, "rocm_verified": None}, + } if not found and probe.timed_out: # A timeout proves nothing, so blocking here would fail a merely slow host. print( f"Preflight: WARNING — the {framework} probe timed out; proceeding without verifying it" f"{_probe_detail_block(probe.detail)}" ) - return + return { + "status": "warned", + "skip_reason": None, + "target": framework, + "message": probe.detail or "framework probe timed out", + "detail": {"probe_interpreter": None, "rocm_verified": None}, + } # Every path below stops the run, so it needs a remedy that works. Both of # them name --install-framework, which setup rejects outside its own set. @@ -984,7 +1146,13 @@ def _check_serving_framework(args, benchmark_python: str) -> None: "existing checkout on this host. Continuing; the benchmark will fail if it\n" f"genuinely needs {framework} here." ) - return + return { + "status": "warned", + "skip_reason": None, + "target": framework, + "message": probe.detail or "framework availability could not be verified", + "detail": {"probe_interpreter": found, "rocm_verified": probe.verdict}, + } if found: print( @@ -996,7 +1164,7 @@ def _check_serving_framework(args, benchmark_python: str) -> None: f"To proceed anyway, set {SKIP_FRAMEWORK_CHECK_ENV}=1.", file=sys.stderr, ) - sys.exit(2) + raise SystemExit(2) if _in_container(): remedy = ( @@ -1027,7 +1195,7 @@ def _check_serving_framework(args, benchmark_python: str) -> None: "for a local run that is expected to serve from somewhere unprobed.", file=sys.stderr, ) - sys.exit(2) + raise SystemExit(2) # RUN_EVAL values that disable the accuracy gate (mirrors _workload_envs). @@ -1183,7 +1351,12 @@ def _resolved_eval_disabled(args: argparse.Namespace) -> bool: return bool(state.get("eval_disabled")) -def _ensure_lm_eval_dep(python_exe: str, pip_extra: list[str], *, eval_disabled: bool = False) -> None: +def _ensure_lm_eval_dep( + python_exe: str, + pip_extra: list[str], + *, + eval_disabled: bool = False, +) -> dict[str, Any]: """Probe-then-install ``lm_eval`` in python_exe when the accuracy gate is on. ``install.sh`` defers ``lm_eval`` to InferenceX's ``benchmark_lib.sh`` runtime @@ -1221,12 +1394,30 @@ def _ensure_lm_eval_dep(python_exe: str, pip_extra: list[str], *, eval_disabled: from hyperloom.orchestrator.actions.executors._multi_node_env import is_multi_node if not is_multi_node(): - return # single-node: InferenceX installs lm_eval itself, see above + return { + "status": "skipped", + "skip_reason": "single_node_runtime_install", + "target": "lm_eval[api]", + "interpreter": python_exe, + "message": "single-node InferenceX installs lm_eval on first use", + } if eval_disabled: - return # --no-eval: no eval anywhere, so the harness is never loaded + return { + "status": "skipped", + "skip_reason": "eval_disabled", + "target": "lm_eval[api]", + "interpreter": python_exe, + "message": "accuracy evaluation is disabled", + } run_eval = os.environ.get("RUN_EVAL") if run_eval is not None and run_eval.strip().lower() in _RUN_EVAL_FALSE_VALUES: - return # accuracy gate disabled; lm_eval not required + return { + "status": "skipped", + "skip_reason": "eval_disabled", + "target": "lm_eval[api]", + "interpreter": python_exe, + "message": f"RUN_EVAL={run_eval}", + } missing = _probe_missing_lm_eval_deps(python_exe) if missing is None: # Absence is unproven, so installing would be a guess that could replace @@ -1234,15 +1425,32 @@ def _ensure_lm_eval_dep(python_exe: str, pip_extra: list[str], *, eval_disabled: # already breaks the benchmark far more loudly than a missing accuracy # gate would, so this warns and changes nothing. print("Preflight: WARNING — cannot run the lm_eval probe; leaving the interpreter untouched") - return + return { + "status": "warned", + "skip_reason": None, + "target": "lm_eval[api]", + "interpreter": python_exe, + "message": "lm_eval dependency probe could not run", + } if not missing: print("Preflight: lm_eval[api] OK") - return + return { + "status": "already_present", + "skip_reason": None, + "target": "lm_eval[api]", + "interpreter": python_exe, + } if "lm_eval" in missing: print(f"Preflight: installing lm_eval[api]@{_LM_EVAL_PINNED_REF[:12]} (accuracy gate) ...") _install_pinned_lm_eval(python_exe, pip_extra) print("Preflight: lm_eval[api] installed OK") - return + return { + "status": "applied", + "skip_reason": None, + "target": "lm_eval[api]", + "interpreter": python_exe, + "detail": {"installed": list(missing)}, + } # The image already ships lm_eval. Install only the absent extra so pip # cannot resolve a different lm_eval build over the one baked in, which # would silently swap out a version the image pinned on purpose -- the same @@ -1264,6 +1472,13 @@ def _ensure_lm_eval_dep(python_exe: str, pip_extra: list[str], *, eval_disabled: check=True, ) print(f"Preflight: {' '.join(targets)} installed OK") + return { + "status": "applied", + "skip_reason": None, + "target": "lm_eval[api]", + "interpreter": python_exe, + "detail": {"installed": list(targets)}, + } def _unset_hip_visible_devices() -> None: @@ -1283,7 +1498,7 @@ def _unset_hip_visible_devices() -> None: ) -def _check_gpu_visibility() -> None: +def _check_gpu_visibility() -> dict[str, Any]: """Best-effort informational check of visible GPU count vs ``$TP`` (silent when rocm-smi is absent). Skipped in external multi-node mode: there the server runs on remote GPU @@ -1302,7 +1517,11 @@ def _check_gpu_visibility() -> None: if is_multi_node() and external_service_url(): print("Preflight: external multi-node mode; skipping local GPU visibility check (GPUs are on remote pods)") - return + return { + "status": "skipped", + "skip_reason": "external_multi_node", + "detail": {"visible": None, "tp_requested": None, "warn": None}, + } try: proc = subprocess.run( ["rocm-smi", "--showid"], @@ -1311,9 +1530,17 @@ def _check_gpu_visibility() -> None: timeout=5, ) except (FileNotFoundError, subprocess.TimeoutExpired, PermissionError, OSError): - return + return { + "status": "skipped", + "skip_reason": "rocm_smi_unavailable", + "detail": {"visible": None, "tp_requested": None, "warn": None}, + } if proc.returncode != 0: - return + return { + "status": "skipped", + "skip_reason": "rocm_smi_failed", + "detail": {"visible": None, "tp_requested": None, "warn": None}, + } # rocm-smi --showid emits multiple GPU[ lines per GPU; deduplicate by GPU index. visible_indices: set[str] = set() for line in (proc.stdout or "").splitlines(): @@ -1328,24 +1555,40 @@ def _check_gpu_visibility() -> None: except ValueError: wanted = 1 if visible == 0: - print("Preflight: WARNING — rocm-smi sees 0 GPUs; benchmark will fail") - return + warning = "rocm-smi sees 0 GPUs; benchmark will fail" + print(f"Preflight: WARNING — {warning}") + return { + "status": "warned", + "skip_reason": None, + "detail": {"visible": visible, "tp_requested": wanted, "warn": warning}, + } if wanted > visible: - print( - f"Preflight: WARNING — TP={wanted} but rocm-smi sees {visible} " - f"GPU(s); sglang/vllm may fail to load weights. Lower TP or " - f"adjust ROCR_VISIBLE_DEVICES." - ) + warning = f"TP={wanted} but rocm-smi sees {visible} GPU(s); sglang/vllm may fail to load weights" + print(f"Preflight: WARNING — {warning}. Lower TP or adjust ROCR_VISIBLE_DEVICES.") + return { + "status": "warned", + "skip_reason": None, + "detail": {"visible": visible, "tp_requested": wanted, "warn": warning}, + } + return { + "status": "applied", + "skip_reason": None, + "detail": {"visible": visible, "tp_requested": wanted, "warn": None}, + } -def _check_shm_disk() -> None: +def _check_shm_disk() -> dict[str, Any]: """Warn (not fail-fast) on tight ``/dev/shm`` (vLLM/NCCL IPC needs headroom).""" try: usage = shutil.disk_usage("/dev/shm") # nosec B108 - mountpoint probe, not temp file creation. except (FileNotFoundError, OSError): - return + return { + "status": "skipped", + "skip_reason": "shm_unavailable", + "detail": {"shm_free_gib": None, "min_gib": 16}, + } + free_gb = usage.free / (1024**3) if usage.free < _DEV_SHM_MIN_FREE_BYTES: - free_gb = usage.free / (1024**3) total_gb = usage.total / (1024**3) print( f"Preflight: WARNING — /dev/shm has {free_gb:.1f} GiB free of " @@ -1353,6 +1596,11 @@ def _check_shm_disk() -> None: f"NCCL shm segments may collide with stale entries; if the " f"first server launch hangs >5min, clear /dev/shm/{{vllm,nccl,cuda}}*" ) + return { + "status": "warned" if usage.free < _DEV_SHM_MIN_FREE_BYTES else "applied", + "skip_reason": None, + "detail": {"shm_free_gib": round(free_gb, 1), "min_gib": 16}, + } def _check_gfx_arch_resolvable(gpu_type: str | None = None) -> None: @@ -1383,7 +1631,7 @@ def _check_gfx_arch_resolvable(gpu_type: str | None = None) -> None: ) -def _check_platform_tuning() -> None: +def _check_platform_tuning() -> dict[str, Any]: """Record host CPU tuning state and warn on settings that skew results. Within one session every trial runs on this same node, so host tuning @@ -1411,7 +1659,11 @@ def _check_platform_tuning() -> None: """ plat = probe_cpu_platform() if plat is None: - return # Not Linux sysfs, or a container without it; stay silent. + return { + "status": "skipped", + "skip_reason": "platform_probe_unavailable", + "detail": {"smt": None, "governor": "unknown", "cpb": None}, + } print( f"Preflight: platform [{socket.gethostname()}] — SMT {plat.smt or '?'}, " @@ -1429,6 +1681,16 @@ def _check_platform_tuning() -> None: "Preflight: WARNING — Core Performance Boost is disabled; CPU-side " "work (sampling, scheduling, tokenization) will run below rated clocks" ) + warned = plat.governor not in ("performance", "unknown") or plat.boost == "off" + return { + "status": "warned" if warned else "applied", + "skip_reason": None, + "detail": { + "smt": plat.smt, + "governor": plat.governor, + "cpb": {"on": True, "off": False}.get(plat.boost), + }, + } _TRACELENS_REQUIRED_CLIS: tuple[str, ...] = ("TraceLens_generate_perf_report_pytorch_inference",) @@ -1455,7 +1717,7 @@ def _tracelens_required_at_preflight(no_kernel: bool, enable_roofline: bool) -> return not (no_kernel and not enable_roofline) -def _check_tracelens_cli() -> None: +def _check_tracelens_cli() -> dict[str, Any]: """Hard-gate TraceLens CLI presence — abort before Coordinator starts (SKILL IR-2). Pod-local /opt/venv/bin/TraceLens_* console_scripts don't persist across pod restarts, so install.sh @@ -1464,7 +1726,12 @@ def _check_tracelens_cli() -> None: """ missing = [name for name in _TRACELENS_REQUIRED_CLIS if shutil.which(name) is None] if not missing: - return + return { + "status": "applied", + "skip_reason": None, + "target": "TraceLens", + "message": None, + } session_dir = str(_workspace_root_resolve()) print( f"ERROR: TraceLens CLI(s) not on PATH: {missing}. The pod-local " @@ -1479,10 +1746,10 @@ def _check_tracelens_cli() -> None: f"then retry `python -m hyperloom.inference_optimizer.cli optimize`. Refusing to start.", file=sys.stderr, ) - sys.exit(2) + raise SystemExit(2) -def _check_tracelens_root_exists() -> None: +def _check_tracelens_root_exists() -> dict[str, Any]: """Hard-gate an explicitly set ``TRACELENS_ROOT`` at preflight. An operator-supplied TRACELENS_ROOT that points at a missing checkout (stale @@ -1491,7 +1758,11 @@ def _check_tracelens_root_exists() -> None: """ override = os.environ.get("TRACELENS_ROOT") if not override or Path(override).is_dir(): - return + return { + "status": "applied", + "skip_reason": None, + "target": "TRACELENS_ROOT", + } print( f"ERROR: TRACELENS_ROOT={override} does not point at an existing " f"TraceLens checkout. It was likely inherited from a stale shell or an " @@ -1522,7 +1793,7 @@ def _emit_preflight_diagnostics( magpie_python: str, anthropic_base_url: str | None, args: argparse.Namespace | None = None, -) -> None: +) -> dict[str, Any]: """One canonical, grep-friendly diagnostics block at the end of preflight. Args: @@ -1580,13 +1851,40 @@ def _emit_preflight_diagnostics( print(f" pr_degraded_reason = {pr_reason}") # Surface Recipe KB offline-queue state; dead-letter pile-up signals a cold start. + queue_status: dict[str, Any] + diagnostics_status = "applied" + diagnostics_message: str | None = None try: - _print_recipe_kb_queue_status() + queue_status = _print_recipe_kb_queue_status() except Exception as exc: # noqa: BLE001 — defensive print(f" recipe_kb_queue = ") - - -def _print_recipe_kb_queue_status() -> None: + queue_status = { + "pending": None, + "dead_letter": None, + "flushed": None, + "root": None, + } + diagnostics_status = "warned" + diagnostics_message = f"recipe KB queue probe failed: {exc!r}" + return { + "status": diagnostics_status, + "skip_reason": None, + "message": diagnostics_message, + "detail": { + "asset_root": str(asset_root()), + "session_dir": str(_session_dir_resolve()), + "magpie_python": magpie_python, + "inferencex_path": os.environ.get("INFERENCEX_PATH") or None, + "aiter_jit_cache": dict(probe), + "recipe_kb_queue": queue_status, + "cold_start_timeout_sec": int(cold_cap) if str(cold_cap).isdigit() else cold_cap, + "warm_timeout_sec": BASELINE_DEFAULT_TIMEOUT_SEC, + "anthropic_base_url": anthropic_base_url, + }, + } + + +def _print_recipe_kb_queue_status() -> dict[str, Any]: """Emit a one-line summary of the Recipe KB offline NDJSON queue (dead-letter = permanent-reject signal). Note: @@ -1631,6 +1929,12 @@ def _count(p: Path) -> int: f"Specialists for affected anchors will start cold " f"(no priors). See {dead}." ) + return { + "pending": p_n, + "dead_letter": d_n, + "flushed": f_n, + "root": str(pending.parent), + } _INFERENCEX_REPO_DEFAULT = "https://github.com/SemiAnalysisAI/InferenceX.git" @@ -1907,6 +2211,192 @@ def _clone_inferencex(dest: Path) -> str | None: return None +def _begin_install_event(args: argparse.Namespace | None) -> dict[str, Any]: + event: dict[str, Any] = { + "type": "install", + "kind": "install", + "status": "succeeded", + "start_time": now_iso(timespec="seconds"), + "end_time": "", + "ext": { + "run_kind": "resume" if bool(getattr(args, "resume_from", None)) else "fresh", + "hard_fail_step_id": None, + "runtime_snapshot": {}, + "steps": [], + }, + } + try: + from ..session.sbd_v6 import set_pending_install_event + + set_pending_install_event(args, event) + except Exception: # noqa: BLE001 — V6 observability must never change preflight behavior + log.warning("failed to initialize SBD V6 install event", exc_info=True) + return event + + +def _record_install_step( + event: dict[str, Any], + *, + step_id: str, + category: str, + status: str, + skip_reason: str | None = None, + message: str | None = None, + **fields: Any, +) -> None: + step: dict[str, Any] = { + "step_id": step_id, + "category": category, + "status": status, + "skip_reason": skip_reason, + } + if message is not None: + step["message"] = message + step.update(fields) + event["ext"]["steps"].append(step) + + +def _fail_install_step( + event: dict[str, Any], + *, + step_id: str, + category: str, + exc: BaseException, +) -> None: + failure_fields: dict[str, Any] = {} + if isinstance(exc, SystemExit): + failure_fields["detail"] = {"exit_code": exc.code} + _record_install_step( + event, + step_id=step_id, + category=category, + status="failed", + message=str(exc) or type(exc).__name__, + error_class=type(exc).__name__, + **failure_fields, + ) + event["status"] = "failed" + event["end_time"] = now_iso(timespec="seconds") + event["ext"]["hard_fail_step_id"] = step_id + + +def _mark_pending_install_event_failed( + args: argparse.Namespace | None, + exc: BaseException, +) -> dict[str, Any] | None: + """Mark an unwrapped preflight exception on the pending install event.""" + try: + from ..session.sbd_v6 import pending_install_event + + event = pending_install_event(args) + if event is None: + event = _begin_install_event(args) + if str(event.get("status") or "") != "failed": + _fail_install_step( + event, + step_id="unhandled_preflight", + category="check", + exc=exc, + ) + return event + except Exception: # noqa: BLE001 — never replace the original preflight failure + log.warning("failed to finalize SBD V6 install failure", exc_info=True) + return None + + +def _run_install_step( + event: dict[str, Any], + *, + step_id: str, + category: str, + action: Callable[[], Any], + success_status: str = "applied", + **success_fields: Any, +) -> Any: + try: + result = action() + except BaseException as exc: + try: + _fail_install_step(event, step_id=step_id, category=category, exc=exc) + except Exception: # noqa: BLE001 — preserve the original preflight exception + log.warning("failed to record SBD V6 install-step failure", exc_info=True) + raise + try: + outcome = dict(result) if isinstance(result, dict) else {} + status = str(outcome.pop("status", success_status) or success_status) + skip_reason = outcome.pop("skip_reason", None) + message = outcome.pop("message", None) + fields = {**success_fields, **outcome} + _record_install_step( + event, + step_id=step_id, + category=category, + status=status, + skip_reason=skip_reason, + message=message, + **fields, + ) + except Exception: # noqa: BLE001 — V6 observability must never change preflight behavior + log.warning("failed to record SBD V6 install step", exc_info=True) + return result + + +def _resolved_provider_mode(resolved_urls: tuple[str, str] | None) -> str | None: + anthropic_url, openai_url = resolved_urls or ("", "") + if anthropic_url and openai_url: + return "mixed" + if anthropic_url: + return "anthropic" + if openai_url: + return "openai" + return None + + +def _finish_install_event( + event: dict[str, Any], + *, + args: argparse.Namespace | None, + benchmark_backend: str, + benchmark_python: str, + magpie_python: str, + inferencex_path: str, + resolved_urls: tuple[str, str] | None, +) -> None: + no_kernel = bool(getattr(args, "no_kernel", False)) if args is not None else False + enable_roofline = bool(getattr(args, "enable_roofline", True)) if args is not None else True + tracelens_required = _tracelens_required_at_preflight(no_kernel, enable_roofline) + event["ext"]["runtime_snapshot"] = { + "benchmark_backend": benchmark_backend, + "benchmark_interpreter": benchmark_python, + "magpie_python": magpie_python, + "inferencex_path": inferencex_path, + "magpie_path": os.environ.get("MAGPIE_PATH") or None, + "tracelens_required": tracelens_required, + "tracelens_route_hint": "agent" if not no_kernel else ("bypass" if enable_roofline else None), + "provider_mode": _resolved_provider_mode(resolved_urls), + } + statuses = {str(step.get("status") or "") for step in event["ext"]["steps"]} + if "failed" in statuses: + event["status"] = "failed" + elif "warned" in statuses: + event["status"] = "degraded" + elif statuses and statuses == {"skipped"}: + event["status"] = "skipped" + else: + event["status"] = "succeeded" + event["end_time"] = now_iso(timespec="seconds") + + +def _persist_install_event(args: argparse.Namespace | None, session_dir: Path) -> None: + """Persist the pre-session install trace without changing launch behavior.""" + from ..session.sbd_v6 import persist_pending_install_event + + try: + persist_pending_install_event(args, session_dir) + except Exception as exc: # noqa: BLE001 + log.warning("failed to persist SBD V6 install event: %s", exc) + + def _preflight( args: argparse.Namespace | None = None, ) -> tuple[str, str] | None: @@ -1924,25 +2414,69 @@ def _preflight( tuple[str, str] | None: ``(anthropic_base_url, openai_base_url)``, or ``None`` when neither base URL is configured. """ - _load_dotenv_fallback() + install_event = _begin_install_event(args) + _run_install_step( + install_event, + step_id="load_dotenv", + category="normalize", + action=_load_dotenv_fallback, + ) # ``.env`` is operator configuration, so both the single-provider intent and # the restore baseline are taken after it loads. Only what the installer env # file injects on top is undone below. provider_mode = _provider_only_mode() provider_snapshot = {key: os.environ.get(key) for key in (*_PROVIDER_FALLBACK_KEYS, *_ANTHROPIC_FALLBACK_KEYS)} - _load_kernel_agent_env_fallback() + _run_install_step( + install_event, + step_id="load_kernel_agent_env", + category="normalize", + action=_load_kernel_agent_env_fallback, + ) _derive_runtime_paths() _restore_provider_only_mode(provider_mode, provider_snapshot) - _normalize_legacy_deepseek_env() + _run_install_step( + install_event, + step_id="normalize_legacy_deepseek_env", + category="normalize", + action=_normalize_legacy_deepseek_env, + ) # Fail fast on missing credentials after the fallback loaders. - _validate_credentials() + _run_install_step( + install_event, + step_id="validate_credentials", + category="check", + action=_validate_credentials, + detail={"exit_code": 0}, + ) # Same timing, same reason: run after the loaders so a withdrawn KB # override set in ``.env`` is caught, and before any KB read happens. from hyperloom.agents.framework.kb import prepare_kb_environment - prepare_kb_environment() + kb_withdrawn_override = bool(os.environ.get("FRAMEWORK_AGENT_KB_DIR", "").strip()) + kb_enabled = not bool(getattr(args, "degraded_kb", False)) if args is not None else True + + def _prepare_kb_install_step() -> dict[str, Any]: + prepare_kb_environment() + return { + "status": ("skipped" if not kb_enabled else "warned" if kb_withdrawn_override else "applied"), + "skip_reason": "explicit_flag" if not kb_enabled else None, + "detail": { + "recipe_kb": { + "enabled": kb_enabled, + "reason": "explicit_flag" if not kb_enabled else None, + }, + "kb_withdrawn_override": kb_withdrawn_override, + }, + } + + _run_install_step( + install_event, + step_id="prepare_kb_environment", + category="degrade", + action=_prepare_kb_install_step, + ) # --- Auth alias export (internal LLM aliases only) --- # These aliases feed OpenAI-protocol consumers, so they are filled from the @@ -1970,7 +2504,8 @@ def _preflight( resolve_benchmark_interpreter as _resolve_benchmark_interpreter, ) - _magpie_backend_active = _resolve_active_backend_name() == "magpie" + benchmark_backend = _resolve_active_backend_name() + _magpie_backend_active = benchmark_backend == "magpie" # Interpreter used for benchmark-runtime installs (Ray). For bypass this is # sys.executable; for Magpie it's the Magpie-importable venv. benchmark_python = _resolve_benchmark_interpreter() @@ -1982,7 +2517,12 @@ def _preflight( # --- Python SDK auto-install (claude-agent-sdk / openai / httpx) --- # Must precede Coordinator import (ClaudeBackend lazy-imports the SDK). - _ensure_python_sdks(sys.executable, pip_extra) + _run_install_step( + install_event, + step_id="ensure_python_sdks", + category="install", + action=lambda: _ensure_python_sdks(sys.executable, pip_extra), + ) # --- Resolve Anthropic + OpenAI base URLs (split entrypoints) --- # Explicit operator values on each side are preserved; a missing side falls @@ -2059,40 +2599,84 @@ def _preflight( # --- ROCm env hygiene + GPU/shm sanity (defensive WARN-only) --- _unset_hip_visible_devices() - _check_gpu_visibility() - _check_shm_disk() - _check_platform_tuning() + _run_install_step( + install_event, + step_id="check_gpu_visibility", + category="check", + action=_check_gpu_visibility, + ) + _run_install_step( + install_event, + step_id="check_shm_disk", + category="check", + action=_check_shm_disk, + ) + _run_install_step( + install_event, + step_id="check_platform_tuning", + category="check", + action=_check_platform_tuning, + ) # --- Runtime dep install --- # 1. Ray — used broadly (multi-node scheduling, kernel/profile/recover # executors), not only by Magpie, so it is installed regardless of backend. # Install it with the active backend's interpreter so a bypass-only box # gets Ray in its own venv instead of Magpie's. - _ensure_ray(benchmark_python, pip_extra) + _run_install_step( + install_event, + step_id="ensure_ray", + category="install", + action=lambda: _ensure_ray(benchmark_python, pip_extra), + ) # 1b. InferenceX benchmark_serving client deps — required by every serving # benchmark client launch. install.sh installs these into the install-time # $PYTHON, but the bypass runner launches the client with the active # benchmark interpreter; ensure them there too so a bypass-only box whose # sys.executable differs from /opt/venv can still import the client. - _ensure_bench_serving_deps(benchmark_python, pip_extra) + _run_install_step( + install_event, + step_id="ensure_bench_serving_deps", + category="install", + action=lambda: _ensure_bench_serving_deps(benchmark_python, pip_extra), + ) # 1c. lm_eval — GSM8K accuracy gate, multi-node only (the helper gates # itself). The multi-node magpie remote-compat client path runs # ``python -m lm_eval`` directly, with no InferenceX runtime shim to install # the harness, so every RUN_EVAL=true baseline there would otherwise abort # with baseline_accuracy_failed. - _ensure_lm_eval_dep(benchmark_python, pip_extra, eval_disabled=_resolved_eval_disabled(args)) + _run_install_step( + install_event, + step_id="ensure_lm_eval", + category="install", + action=lambda: _ensure_lm_eval_dep( + benchmark_python, + pip_extra, + eval_disabled=_resolved_eval_disabled(args), + ), + ) # 1d. Per-framework runtime deps declared in assets/framework_deps/. This is # the pass that covers the documented flow: install.sh runs before # --framework is known, so its own attempt usually no-ops and a scriptable # framework would otherwise reach baseline with nothing installed. - _ensure_framework_deps(args, benchmark_python, pip_extra) + _run_install_step( + install_event, + step_id="framework_deps", + category="install", + action=lambda: _ensure_framework_deps(args, benchmark_python, pip_extra), + ) # 1e. The serving framework itself, checked after 1d so everything that could # have supplied it has run. Without this the failure surfaces far from its cause. - _check_serving_framework(args, benchmark_python) + _run_install_step( + install_event, + step_id="check_serving_framework", + category="check", + action=lambda: _check_serving_framework(args, benchmark_python), + ) # 2. Magpie — the benchmark engine the Magpie backend shells out to. # Skipped entirely when the @@ -2101,41 +2685,70 @@ def _preflight( # Magpie-importable venv (via resolve_benchmark_interpreter), so a # bypass-only environment never resolves the Magpie venv / /opt/venv. magpie_python = benchmark_python - if not _magpie_backend_active: - print(f"Preflight: benchmark backend is {_resolve_active_backend_name()!r}; skipping Magpie install/import") - check = None - else: - check = subprocess.run([magpie_python, "-c", "import Magpie"], capture_output=True) - if _magpie_backend_active and check is not None and check.returncode != 0: - magpie_repo = os.environ.get("MAGPIE_REPO", "https://github.com/AMD-AGI/Magpie.git") - magpie_ref = os.environ.get("MAGPIE_REF", "e6833b8183c6c41adf6038252337550876ca0433") - magpie_spec = os.environ.get( - "MAGPIE_PACKAGE_SPEC", - f"magpie-eval @ git+{magpie_repo}@{magpie_ref}", - ) - print(f"Preflight: Magpie not importable; installing {magpie_spec} ...") - subprocess.run( - [magpie_python, "-m", "pip", "install", "--quiet", *pip_extra, magpie_spec], - check=True, + magpie_installed = False + magpie_spec: str | None = None + try: + if not _magpie_backend_active: + print(f"Preflight: benchmark backend is {benchmark_backend!r}; skipping Magpie install/import") + check = None + else: + check = subprocess.run([magpie_python, "-c", "import Magpie"], capture_output=True) + if _magpie_backend_active and check is not None and check.returncode != 0: + magpie_repo = os.environ.get("MAGPIE_REPO", "https://github.com/AMD-AGI/Magpie.git") + magpie_ref = os.environ.get("MAGPIE_REF", "e6833b8183c6c41adf6038252337550876ca0433") + magpie_spec = os.environ.get( + "MAGPIE_PACKAGE_SPEC", + f"magpie-eval @ git+{magpie_repo}@{magpie_ref}", + ) + print(f"Preflight: Magpie not importable; installing {magpie_spec} ...") + subprocess.run( + [magpie_python, "-m", "pip", "install", "--quiet", *pip_extra, magpie_spec], + check=True, + ) + magpie_installed = True + print("Preflight: Magpie installed OK") + if _magpie_backend_active and not os.environ.get("MAGPIE_PATH", "").strip(): + magpie_root = subprocess.run( + [ + magpie_python, + "-c", + "from pathlib import Path; import Magpie; print(Path(Magpie.__file__).resolve().parent.parent)", + ], + capture_output=True, + text=True, + check=True, + ).stdout.strip() + if magpie_root: + os.environ["MAGPIE_PATH"] = magpie_root + print(f"Preflight: MAGPIE_PATH resolved from installed package: {magpie_root}") + except BaseException as exc: + _fail_install_step( + install_event, + step_id="ensure_magpie", + category="install", + exc=exc, ) - print("Preflight: Magpie installed OK") - if _magpie_backend_active and not os.environ.get("MAGPIE_PATH", "").strip(): - magpie_root = subprocess.run( - [ - magpie_python, - "-c", - "from pathlib import Path; import Magpie; print(Path(Magpie.__file__).resolve().parent.parent)", - ], - capture_output=True, - text=True, - check=True, - ).stdout.strip() - if magpie_root: - os.environ["MAGPIE_PATH"] = magpie_root - print(f"Preflight: MAGPIE_PATH resolved from installed package: {magpie_root}") + raise + _record_install_step( + install_event, + step_id="ensure_magpie", + category="install", + status=("skipped" if not _magpie_backend_active else "applied" if magpie_installed else "already_present"), + skip_reason=None if _magpie_backend_active else "benchmark_backend_not_magpie", + message=( + f"installed {magpie_spec}" + if magpie_installed + else f"benchmark backend is {benchmark_backend!r}" + if not _magpie_backend_active + else None + ), + target="magpie-eval", + interpreter=magpie_python, + ) # 3. InferenceX — required for GSM8K accuracy eval; lm-eval deps auto-install at runtime via benchmark_lib.sh. inferencex_path = os.environ.get("INFERENCEX_PATH", "").strip() + inferencex_cloned = False if not inferencex_path: from ..session.paths import ( magpie_dir as _magpie_default, @@ -2185,6 +2798,7 @@ def _preflight( dest = _open_source_default() / _inferencex_dest_name(_ref) print(f"Preflight: no InferenceX checkout at {_ref[:12]}; cloning into {dest} ...") inferencex_path = _clone_inferencex(dest) + inferencex_cloned = bool(inferencex_path) if not (inferencex_path and _inferencex_checkout_ok(inferencex_path)): print( "Preflight: ERROR — InferenceX checkout missing and clone " @@ -2193,7 +2807,14 @@ def _preflight( "src/hyperloom/inference_optimizer/assets/install.sh.", file=sys.stderr, ) - sys.exit(2) + exc = SystemExit(2) + _fail_install_step( + install_event, + step_id="clone_inferencex", + category="install", + exc=exc, + ) + raise exc # Guard against a read-only INFERENCEX_PATH: Magpie stages benchmark scripts # there, so a non-writable tree fails the run before server boot. if not os.access(inferencex_path, os.W_OK): @@ -2205,10 +2826,32 @@ def _preflight( f"Hyperloom clone a fresh one).", file=sys.stderr, ) - sys.exit(2) + exc = SystemExit(2) + _fail_install_step( + install_event, + step_id="clone_inferencex", + category="install", + exc=exc, + ) + raise exc # Always overwrite (not setdefault): a stale/broken INFERENCEX_PATH must not # survive into the child env. The validated value wins. os.environ["INFERENCEX_PATH"] = inferencex_path + _record_install_step( + install_event, + step_id="clone_inferencex", + category="install", + status="applied" if inferencex_cloned else "already_present", + skip_reason=None, + target="InferenceX", + version_after=_inferencex_head_sha(inferencex_path) or None, + detail={ + "ref": os.environ.get("INFERENCEX_REF") or _INFERENCEX_REF_DEFAULT, + "dest": inferencex_path, + "writable": os.access(inferencex_path, os.W_OK), + "exit_code": 0, + }, + ) # --- Magpie/InferenceX eval-concurrency compatibility ------------------- # Preflight installs Magpie and clones InferenceX itself (above), entirely @@ -2220,15 +2863,42 @@ def _preflight( # flag ("Unknown parameter: --concurrent-requests"), aborting every # RUN_EVAL=true baseline before any results*.json exists. Patch the trees we # just materialized, now that both paths are known. - if _magpie_backend_active: - # Trust patch first, mirroring install.sh: the eval-concurrency strip - # removes the very `run_eval ... --concurrent-requests` line the legacy - # MI300X trust patcher matches on, so the reverse order would leave a - # tree permanently unpatchable by that path. - _ensure_client_trust_compat(os.environ.get("MAGPIE_PATH", "")) - _ensure_eval_concurrency_compat(os.environ.get("MAGPIE_PATH", ""), inferencex_path) - - _report_inferencex_patch_anchors(inferencex_path) + try: + if _magpie_backend_active: + # Trust patch first, mirroring install.sh: the eval-concurrency strip + # removes the very `run_eval ... --concurrent-requests` line the legacy + # MI300X trust patcher matches on, so the reverse order would leave a + # tree permanently unpatchable by that path. + trust_ok = _ensure_client_trust_compat(os.environ.get("MAGPIE_PATH", "")) + concurrency_ok = _ensure_eval_concurrency_compat( + os.environ.get("MAGPIE_PATH", ""), + inferencex_path, + ) + else: + trust_ok = True + concurrency_ok = True + anchors_ok = _report_inferencex_patch_anchors(inferencex_path) + except BaseException as exc: + _fail_install_step( + install_event, + step_id="patch_magpie_eval_concurrency", + category="patch", + exc=exc, + ) + raise + patch_ok = trust_ok and concurrency_ok and anchors_ok + _record_install_step( + install_event, + step_id="patch_magpie_eval_concurrency", + category="patch", + status=("skipped" if not _magpie_backend_active else "applied" if patch_ok else "warned"), + skip_reason=None if _magpie_backend_active else "benchmark_backend_not_magpie", + detail={ + "client_trust_compatible": trust_ok, + "eval_concurrency_compatible": concurrency_ok, + "inferencex_patch_anchors_ok": anchors_ok, + }, + ) # --- node / claude / codex CLI presence (WARN-only) --- _check_node_claude_cli() @@ -2238,10 +2908,20 @@ def _preflight( no_kernel = getattr(args, "no_kernel", False) if args else False enable_roofline = getattr(args, "enable_roofline", True) if args else True if _tracelens_required_at_preflight(no_kernel, enable_roofline): - _check_tracelens_cli() + _run_install_step( + install_event, + step_id="check_tracelens_cli", + category="check", + action=_check_tracelens_cli, + ) # Fail fast on a stale/placeholder TRACELENS_ROOT before the Coordinator # starts, rather than ~10h later in trace_analyze. - _check_tracelens_root_exists() + _run_install_step( + install_event, + step_id="check_tracelens_root", + category="check", + action=_check_tracelens_root_exists, + ) else: _missing_tl = [n for n in _TRACELENS_REQUIRED_CLIS if shutil.which(n) is None] if _missing_tl: @@ -2249,22 +2929,70 @@ def _preflight( f"Preflight: WARNING — TraceLens CLI(s) not on PATH: {_missing_tl} " f"(skipped; --no-kernel + roofline disabled)" ) + _record_install_step( + install_event, + step_id="check_tracelens_cli", + category="check", + status="skipped", + skip_reason="no_kernel_and_roofline_disabled", + target="TraceLens", + message=f"missing CLIs: {_missing_tl}" if _missing_tl else None, + ) + _record_install_step( + install_event, + step_id="check_tracelens_root", + category="check", + status="skipped", + skip_reason="tracelens_not_required", + target="TRACELENS_ROOT", + ) # --- IR-3: PR Monitor reachability probe (soft degrade) --- if args is not None: - _run_ir3_preflight(args) + _run_install_step( + install_event, + step_id="ir3_pr_monitor_probe", + category="degrade", + action=lambda: _run_ir3_preflight(args), + ) + else: + _record_install_step( + install_event, + step_id="ir3_pr_monitor_probe", + category="degrade", + status="skipped", + skip_reason="args_unavailable", + ) # --- Single canonical diagnostics block --- - _emit_preflight_diagnostics( - magpie_python=magpie_python, - anthropic_base_url=(resolved_urls[0] if resolved_urls is not None else None), - args=args, + _run_install_step( + install_event, + step_id="diagnostics_snapshot", + category="diagnostic", + action=lambda: _emit_preflight_diagnostics( + magpie_python=magpie_python, + anthropic_base_url=(resolved_urls[0] if resolved_urls is not None else None), + args=args, + ), ) + try: + _finish_install_event( + install_event, + args=args, + benchmark_backend=benchmark_backend, + benchmark_python=benchmark_python, + magpie_python=magpie_python, + inferencex_path=inferencex_path, + resolved_urls=resolved_urls, + ) + except Exception: # noqa: BLE001 — V6 observability must never change preflight behavior + log.warning("failed to finalize SBD V6 install event", exc_info=True) + return resolved_urls -def _run_ir3_preflight(args: argparse.Namespace) -> None: +def _run_ir3_preflight(args: argparse.Namespace) -> dict[str, Any]: """IR-3 — PR Monitor reachability probe (soft degrade); never raises/exits. Recipe KB enablement is controlled by ``--degraded-kb`` (``recipe_kb_enabled``); @@ -2286,7 +3014,14 @@ def _run_ir3_preflight(args: argparse.Namespace) -> None: args.pr_degraded_reason = "explicit_flag" if explicit_pr else None if explicit_kb and explicit_pr: - return + return { + "status": "skipped", + "skip_reason": "explicit_flag", + "detail": { + "pr_monitor": {"enabled": False, "reason": "explicit_flag"}, + "marker": None, + }, + } user_data = _workspace_root_resolve() marker_path = user_data / "runtime" / "recipe_kb" / ".kb_preflight.json" @@ -2330,3 +3065,17 @@ def _run_ir3_preflight(args: argparse.Namespace) -> None: if not explicit_pr and not marker.get("pr_reachable", False) and not marker.get("pr_skipped", False): args.pr_monitor_enabled = False args.pr_degraded_reason = "ir3_auto" + enabled = bool(args.pr_monitor_enabled) + status = "skipped" if explicit_pr else "applied" if enabled else "warned" + event_reason = "explicit_flag" if explicit_pr else "ir3_unreachable" if not enabled else None + return { + "status": status, + "skip_reason": event_reason, + "detail": { + "pr_monitor": { + "enabled": enabled, + "reason": event_reason, + }, + "marker": str(marker_path), + }, + } diff --git a/src/hyperloom/inference_optimizer/session/sbd_v6.py b/src/hyperloom/inference_optimizer/session/sbd_v6.py new file mode 100644 index 0000000000..5e05651a0b --- /dev/null +++ b/src/hyperloom/inference_optimizer/session/sbd_v6.py @@ -0,0 +1,260 @@ +"""Small write/read surface for additive SBD V6 timeline events.""" + +from __future__ import annotations + +import re +from argparse import Namespace +from pathlib import Path +from typing import Any + +from hyperloom.common.io import atomic_write_json +from hyperloom.common.jsonio import read_json + +from .session_paths import ( + sbd_v6_install_path, + sbd_v6_model_gate_path, + sbd_v6_timeline_dir, + sbd_v6_timeline_event_path, +) + + +SCHEMA_VERSION_V6 = "hyperloom.session_breakdown.v6.0" +_PENDING_INSTALL_ATTR = "_sbd_v6_install_event" +_STORAGE_SEQUENCE_KEY = "__sbd_v6_timeline_sequence" +_EVENT_TYPES = ("install", "model_gate") +_EVENT_FILE_RE = re.compile(r"^(?P\d+)-(?P[a-z0-9_]+)\.json$") + + +def _event_path(session_dir: Path | str, event_type: str) -> Path: + root = Path(session_dir) + if event_type == "install": + return sbd_v6_install_path(root) + if event_type == "model_gate": + return sbd_v6_model_gate_path(root) + raise ValueError(f"unsupported SBD V6 timeline event type: {event_type!r}") + + +def _public_event(event: dict[str, Any]) -> dict[str, Any]: + value = dict(event) + value.pop(_STORAGE_SEQUENCE_KEY, None) + return value + + +def _write_event(path: Path, event: dict[str, Any]) -> None: + atomic_write_json( + path, + _public_event(event), + indent=2, + sort_keys=True, + ensure_ascii=False, + trailing_newline=True, + make_parents=True, + ) + + +def _history_files(session_dir: Path | str) -> list[tuple[int, str, Path]]: + root = sbd_v6_timeline_dir(Path(session_dir)) + if not root.is_dir(): + return [] + files: list[tuple[int, str, Path]] = [] + for path in root.glob("*.json"): + match = _EVENT_FILE_RE.fullmatch(path.name) + if match is None: + continue + event_type = match.group("event_type") + if event_type not in _EVENT_TYPES: + continue + files.append((int(match.group("sequence")), event_type, path)) + files.sort(key=lambda row: row[0]) + return files + + +def _read_event_file( + path: Path, + event_type: str, + warnings: list[str] | None = None, +) -> dict[str, Any] | None: + try: + event = read_json(path, require_dict=True, strict=True) + except Exception as exc: + if warnings is not None: + warnings.append(f"timeline.{event_type}: failed to parse {path}: {exc!r}") + return None + if str(event.get("type") or "") != event_type: + if warnings is not None: + warnings.append(f"timeline.{event_type}: ignored event with type={event.get('type')!r}") + return None + return _public_event(event) + + +def _event_identity(event_type: str, event: dict[str, Any]) -> tuple[str, str, str] | None: + ext = event.get("ext") if isinstance(event.get("ext"), dict) else {} + start_time = str(event.get("start_time") or "") + run_kind = str(ext.get("run_kind") or "") + if not start_time and not run_kind: + return None + return event_type, start_time, run_kind + + +def _ensure_timeline_history(session_dir: Path | str) -> list[tuple[int, str, Path]]: + history = _history_files(session_dir) + root = Path(session_dir) + stored_events: list[tuple[int, str, Path, dict[str, Any]]] = [] + for sequence, event_type, path in history: + event = _read_event_file(path, event_type) + if event is not None: + stored_events.append((sequence, event_type, path, event)) + + next_sequence = max((sequence for sequence, _, _ in history), default=0) + for event_type in _EVENT_TYPES: + legacy_path = _event_path(root, event_type) + if not legacy_path.is_file(): + continue + event = _read_event_file(legacy_path, event_type) + if event is None: + continue + if any(stored_event == event for _, _, _, stored_event in stored_events): + continue + identity = _event_identity(event_type, event) + matching = next( + ( + row + for row in reversed(stored_events) + if identity is not None and _event_identity(row[1], row[3]) == identity + ), + None, + ) + if matching is not None: + _write_event(matching[2], event) + stored_events[stored_events.index(matching)] = (*matching[:3], event) + continue + next_sequence += 1 + path = sbd_v6_timeline_event_path(root, next_sequence, event_type) + _write_event(path, event) + stored_events.append((next_sequence, event_type, path, event)) + return _history_files(root) + + +def write_timeline_event(session_dir: Path | str, event: dict[str, Any]) -> Path: + """Persist one event without replacing an earlier run of the same stage.""" + event_type = str(event.get("type") or "").strip() + latest_path = _event_path(session_dir, event_type) + history = _ensure_timeline_history(session_dir) + + raw_sequence = event.get(_STORAGE_SEQUENCE_KEY) + try: + sequence = int(raw_sequence) if raw_sequence is not None else 0 + except (TypeError, ValueError): + sequence = 0 + if sequence > 0: + conflicting_type = next( + (stored_type for stored_sequence, stored_type, _ in history if stored_sequence == sequence), + None, + ) + if conflicting_type is not None and conflicting_type != event_type: + raise ValueError(f"SBD V6 timeline sequence {sequence} belongs to {conflicting_type!r}, not {event_type!r}") + else: + sequence = max((stored_sequence for stored_sequence, _, _ in history), default=0) + 1 + event[_STORAGE_SEQUENCE_KEY] = sequence + + _write_event( + sbd_v6_timeline_event_path(Path(session_dir), sequence, event_type), + event, + ) + _write_event(latest_path, event) + return latest_path + + +def read_timeline_event( + session_dir: Path | str, + event_type: str, +) -> dict[str, Any] | None: + """Read the latest persisted event of one type.""" + _event_path(session_dir, event_type) + for _, stored_type, path in reversed(_ensure_timeline_history(session_dir)): + if stored_type != event_type: + continue + event = _read_event_file(path, event_type) + if event is not None: + return event + + path = _event_path(session_dir, event_type) + if not path.is_file(): + return None + return _read_event_file(path, event_type) + + +def read_timeline_event_for_update( + session_dir: Path | str, + event_type: str, +) -> dict[str, Any] | None: + """Read the latest event with its private storage sequence attached.""" + _event_path(session_dir, event_type) + for sequence, stored_type, path in reversed(_ensure_timeline_history(session_dir)): + if stored_type != event_type: + continue + event = _read_event_file(path, event_type) + if event is not None: + event[_STORAGE_SEQUENCE_KEY] = sequence + return event + return None + + +def read_timeline_events( + session_dir: Path | str, + *, + warnings: list[str] | None = None, +) -> list[dict[str, Any]]: + """Read all persisted V6 events in execution order.""" + history = _ensure_timeline_history(session_dir) + if history: + events: list[dict[str, Any]] = [] + for _, event_type, path in history: + event = _read_event_file(path, event_type, warnings) + if event is not None: + events.append(event) + return events + + events = [] + for event_type in _EVENT_TYPES: + path = _event_path(session_dir, event_type) + if not path.is_file(): + continue + event = _read_event_file(path, event_type, warnings) + if event is not None: + events.append(event) + return events + + +def set_pending_install_event(args: Namespace | None, event: dict[str, Any]) -> None: + """Attach the pre-session install event to the parsed CLI namespace.""" + if args is not None: + setattr(args, _PENDING_INSTALL_ATTR, event) + + +def pending_install_event(args: Namespace | None) -> dict[str, Any] | None: + """Return the in-memory install event captured before session creation.""" + if args is None: + return None + event = getattr(args, _PENDING_INSTALL_ATTR, None) + return event if isinstance(event, dict) else None + + +def persist_pending_install_event(args: Namespace | None, session_dir: Path | str) -> Path | None: + """Persist the install event once the session directory exists.""" + event = pending_install_event(args) + if event is None: + return None + return write_timeline_event(session_dir, event) + + +__all__ = [ + "SCHEMA_VERSION_V6", + "pending_install_event", + "persist_pending_install_event", + "read_timeline_event", + "read_timeline_event_for_update", + "read_timeline_events", + "set_pending_install_event", + "write_timeline_event", +] diff --git a/src/hyperloom/inference_optimizer/session/session_paths.py b/src/hyperloom/inference_optimizer/session/session_paths.py index 5b259ac647..99a8fdf339 100644 --- a/src/hyperloom/inference_optimizer/session/session_paths.py +++ b/src/hyperloom/inference_optimizer/session/session_paths.py @@ -272,6 +272,31 @@ def reports_dir(session_dir: Path) -> Path: return Path(session_dir) / "reports" +def sbd_v6_dir(session_dir: Path) -> Path: + """Compute ``/reports/sbd_v6/`` for V6 timeline source events.""" + return reports_dir(session_dir) / "sbd_v6" + + +def sbd_v6_timeline_dir(session_dir: Path) -> Path: + """Compute the append-only V6 timeline event directory.""" + return sbd_v6_dir(session_dir) / "timeline" + + +def sbd_v6_timeline_event_path(session_dir: Path, sequence: int, event_type: str) -> Path: + """Compute one ordered V6 timeline event path.""" + return sbd_v6_timeline_dir(session_dir) / f"{int(sequence):06d}-{event_type}.json" + + +def sbd_v6_install_path(session_dir: Path) -> Path: + """Compute the persisted V6 ``install`` timeline event path.""" + return sbd_v6_dir(session_dir) / "install.json" + + +def sbd_v6_model_gate_path(session_dir: Path) -> Path: + """Compute the persisted V6 ``model_gate`` timeline event path.""" + return sbd_v6_dir(session_dir) / "model_gate.json" + + def enablement_dir(session_dir: Path) -> Path: """``/reports/enablement/`` — enablement round artifacts. @@ -880,6 +905,11 @@ def failure_evidence_path(session_dir: Path, failure_id: str) -> Path: "research_hints_md", "runs_dir", "runs_root", + "sbd_v6_dir", + "sbd_v6_install_path", + "sbd_v6_model_gate_path", + "sbd_v6_timeline_dir", + "sbd_v6_timeline_event_path", "state_path", "target_analysis_dir", "target_analysis_report_md", diff --git a/src/hyperloom/inference_optimizer/tests/test_enablement_coordinator_wiring_unit.py b/src/hyperloom/inference_optimizer/tests/test_enablement_coordinator_wiring_unit.py index 68c880f76b..baac53b5f9 100644 --- a/src/hyperloom/inference_optimizer/tests/test_enablement_coordinator_wiring_unit.py +++ b/src/hyperloom/inference_optimizer/tests/test_enablement_coordinator_wiring_unit.py @@ -92,6 +92,7 @@ def test_build_params_actionable_failure_tags_enablement(monkeypatch): params = Coordinator._build_enablement_specialist_params(fake, _MISSING_ARCH_LOG) assert params is not None assert params["domain"] == "enablement_specialist" + assert params["source_phase"] == "ENABLEMENT" # Reuses FRAMEWORK authoring machinery + tags the objective. assert params["framework_agent_authoring"] is True assert params["enablement"] is True diff --git a/src/hyperloom/inference_optimizer/tests/test_framework_agent_authoring.py b/src/hyperloom/inference_optimizer/tests/test_framework_agent_authoring.py index 383618bfef..0e2a897f00 100644 --- a/src/hyperloom/inference_optimizer/tests/test_framework_agent_authoring.py +++ b/src/hyperloom/inference_optimizer/tests/test_framework_agent_authoring.py @@ -226,6 +226,7 @@ def test_materialize_unknown_route_dispatches_both_tracks( params = spec["params"] assert params["framework_agent_authoring"] is True assert params["domain"] == "serving_specialist" + assert params["source_phase"] == "FRAMEWORK_AGENT" assert params["framework_agent_candidate_id"] == _CANDIDATE["pr_url"] assert params.get("task_kind") == "framework_authoring" pr_lead = params.get("pr_lead") or {} diff --git a/src/hyperloom/inference_optimizer/tests/test_research_scout.py b/src/hyperloom/inference_optimizer/tests/test_research_scout.py index d90bfd8fc4..e05342813c 100644 --- a/src/hyperloom/inference_optimizer/tests/test_research_scout.py +++ b/src/hyperloom/inference_optimizer/tests/test_research_scout.py @@ -161,6 +161,7 @@ async def test_internal_research_scout_task_is_readonly(tmp_path: Path): assert task is not None assert task.params["mode"] == "research" + assert task.params["source_phase"] == "PRELUDE" assert task.side_effects == ["writes_results"] assert task.params["seen_pr_ids"] == ["https://pr/seen"] assert "Does this vLLM version support the backend?" in task.params["notes"] diff --git a/src/hyperloom/inference_optimizer/tests/test_sbd_v6_initial.py b/src/hyperloom/inference_optimizer/tests/test_sbd_v6_initial.py new file mode 100644 index 0000000000..a262d8376e --- /dev/null +++ b/src/hyperloom/inference_optimizer/tests/test_sbd_v6_initial.py @@ -0,0 +1,1809 @@ +"""Focused coverage for the additive SBD V6 bootstrap fields and stages.""" + +from __future__ import annotations + +import argparse +import asyncio +import json +import os +from pathlib import Path + +import pytest + +from hyperloom.inference_optimizer.breakdown import exporter +from hyperloom.inference_optimizer.breakdown.collectors.v6 import collect_v6_timeline +from hyperloom.inference_optimizer.breakdown.schema import SCHEMA_VERSION_V5 +from hyperloom.inference_optimizer.session.sbd_v6 import ( + SCHEMA_VERSION_V6, + read_timeline_event, + read_timeline_events, + write_timeline_event, +) + + +def _write_json(path: Path, payload: dict) -> None: + path.parent.mkdir(parents=True, exist_ok=True) + path.write_text(json.dumps(payload), encoding="utf-8") + + +def _gate_args(model: Path, **overrides) -> argparse.Namespace: + values = { + "model": str(model), + "model_display_name": model.name, + "framework": "sglang", + "gpu_type": "MI300X", + "isl": 1024, + "osl": 1024, + "allow_mm_text_fallback": True, + } + values.update(overrides) + return argparse.Namespace(**values) + + +def _seed_state(session_dir: Path, monkeypatch, model: Path) -> None: + monkeypatch.setenv("INFERENCE_OPTIMIZER_CURRENT_SESSION_DIR", str(session_dir)) + from hyperloom.orchestrator.state.shared_state import SharedState + + session_dir.mkdir(parents=True, exist_ok=True) + (session_dir / "reports").mkdir(parents=True, exist_ok=True) + SharedState(session_id="sbd-v6-test", model_name=model.name, model_path=str(model)).save(session_dir) + + +def _write_model_config(model: Path, payload: dict) -> None: + _write_json(model / "config.json", payload) + (model / "tokenizer_config.json").write_text("{}", encoding="utf-8") + + +def _model_gate_from_breakdown(session_dir: Path) -> dict: + breakdown = json.loads((session_dir / "session_breakdown.json").read_text(encoding="utf-8")) + assert breakdown["schema_version"] == SCHEMA_VERSION_V5 + assert breakdown["metadata"]["versions"]["schema_version"] == SCHEMA_VERSION_V6 + assert breakdown["outcome"]["status"] == "failed" + assert breakdown["outcome"]["stage_reached"] == "model_gate" + return next(event for event in breakdown["timeline"] if event["type"] == "model_gate") + + +def test_v6_projection_is_additive_to_v5_breakdown(tmp_path): + state = { + "session_id": "session-v6", + "model_name": "Qwen-Test", + "model_path": "/models/qwen-test", + "framework": "sglang", + "gpu_type": "MI300X", + "phase": "CLOSE", + "start_ts": "2026-08-27T01:00:00+00:00", + "stop_ts": "2026-08-27T02:00:00+00:00", + "stop_reason": "target_reached", + "tick": 9, + "baseline_tput": 100.0, + "baseline_accuracy": 0.75, + "current_best": { + "tput": 125.0, + "extra_envs": {"SGLANG_USE_AITER": "1"}, + "extra_server_args": "--watchdog-timeout 1800", + }, + "cumulative_gain_validated": 25.0, + "operator_extra_env": {"TP": "8"}, + "operator_server_args": "--context-length 11264", + "model_info": { + "model_type": "qwen3", + "num_hidden_layers": 36, + "attention_type": "GQA", + "num_experts": None, + }, + } + manifest = { + "session_id": "session-v6", + "created_at_utc": "2026-08-27T01:00:00+00:00", + "host": "test-host", + "code_revision": "abc1234", + "pid": 42, + "max_minutes": 180, + "model_name": "Qwen-Test", + "model_path": "/models/qwen-test", + "framework": "sglang", + "framework_version": "0.5.17", + "gpu_type": "MI300X", + "tp": 8, + "workload": { + "conc": 64, + "isl": 1024, + "osl": 1024, + "precision": "bf16", + "max_model_len": 11264, + }, + "objective": {"kind": "throughput", "value": 120.0}, + } + _write_json(tmp_path / "state.json", state) + _write_json(tmp_path / "manifest.json", manifest) + + before = exporter.build(tmp_path) + write_timeline_event( + tmp_path, + { + "type": "install", + "kind": "install", + "status": "succeeded", + "start_time": "2026-08-27T00:58:00+00:00", + "end_time": "2026-08-27T00:59:00+00:00", + "ext": {"run_kind": "fresh", "hard_fail_step_id": None, "runtime_snapshot": {}, "steps": []}, + }, + ) + write_timeline_event( + tmp_path, + { + "type": "model_gate", + "kind": "model_gate", + "status": "succeeded", + "start_time": "2026-08-27T00:59:00+00:00", + "end_time": "2026-08-27T01:00:00+00:00", + "ext": {"run_kind": "fresh", "checks": []}, + }, + ) + + after = exporter.build(tmp_path) + + assert after["schema_version"] == SCHEMA_VERSION_V5 + v6_keys = {"metadata", "outcome", "timeline", "close"} + assert {key: value for key, value in after.items() if key not in v6_keys} == { + key: value for key, value in before.items() if key not in v6_keys + } + assert after["metadata"]["versions"]["schema_version"] == SCHEMA_VERSION_V6 + assert after["metadata"]["versions"]["hyperloom"] == "abc1234" + assert after["metadata"]["task_config"]["launch_env"] == {"TP": "8"} + assert after["outcome"]["status"] == "completed" + assert after["outcome"]["stage_reached"] == "close" + assert "token_usage" not in after["outcome"] + assert [event["type"] for event in after["timeline"]] == ["install", "model_gate"] + assert after["close"] == {} + assert all(event["type"] != "close" for event in after["timeline"]) + + +def test_invalid_v6_event_does_not_change_v5_warnings(tmp_path): + before = exporter.build(tmp_path) + path = tmp_path / "reports" / "sbd_v6" / "install.json" + path.parent.mkdir(parents=True) + path.write_text("{invalid", encoding="utf-8") + + after = exporter.build(tmp_path) + + assert after["warnings"] == before["warnings"] + assert any("timeline.install" in warning for warning in after["metadata"]["warnings"]) + assert after["timeline"] == [] + + +def test_install_event_stays_pending_until_session_creation(tmp_path): + from hyperloom.inference_optimizer.cli import preflight + + args = argparse.Namespace(resume_from=None, no_kernel=True, enable_roofline=False) + event = preflight._begin_install_event(args) + preflight._run_install_step( + event, + step_id="load_dotenv", + category="normalize", + action=lambda: { + "status": "already_present", + "skip_reason": None, + "detail": {"vars_loaded": 0, "source": "/repo/.env"}, + }, + ) + preflight._run_install_step( + event, + step_id="check_shm_disk", + category="check", + action=lambda: { + "status": "warned", + "skip_reason": None, + "detail": {"shm_free_gib": 8.0, "min_gib": 16}, + }, + ) + preflight._finish_install_event( + event, + args=args, + benchmark_backend="bypass", + benchmark_python="python", + magpie_python="python", + inferencex_path="/opt/InferenceX", + resolved_urls=("", "https://api.openai.com/v1"), + ) + + assert not (tmp_path / "reports" / "sbd_v6" / "install.json").exists() + preflight._persist_install_event(args, tmp_path) + + persisted = read_timeline_event(tmp_path, "install") + assert persisted is not None + assert persisted["status"] == "degraded" + assert [step["step_id"] for step in persisted["ext"]["steps"]] == [ + "load_dotenv", + "check_shm_disk", + ] + assert persisted["ext"]["runtime_snapshot"]["provider_mode"] == "openai" + + +def test_preflight_hard_failure_creates_session_and_final_sbd(tmp_path, monkeypatch): + import hyperloom.inference_optimizer.cli as optimizer_cli + from hyperloom.inference_optimizer.cli import preflight + from hyperloom.inference_optimizer.session.paths import ENV_CURRENT_SESSION_DIR + + workspace = tmp_path / "sessions" + model = tmp_path / "Qwen-Test" + monkeypatch.setenv("USER_DATA_PATH", str(workspace)) + monkeypatch.delenv(ENV_CURRENT_SESSION_DIR, raising=False) + monkeypatch.setattr( + optimizer_cli, + "clean_stale_aiter_locks", + lambda: {"dir": "", "deleted": 0, "skipped_fresh": 0, "errors": 0}, + ) + + def fail_preflight(args): + event = preflight._begin_install_event(args) + + def reject_credentials(): + raise SystemExit(2) + + preflight._run_install_step( + event, + step_id="validate_credentials", + category="check", + action=reject_credentials, + ) + + monkeypatch.setattr(optimizer_cli, "_preflight", fail_preflight) + args = optimizer_cli._build_parser().parse_args(["optimize", "--model", str(model)]) + + with pytest.raises(SystemExit) as exc: + asyncio.run(optimizer_cli._run_optimize(args)) + + assert exc.value.code == 2 + session_dir = Path(os.environ[ENV_CURRENT_SESSION_DIR]) + assert session_dir.is_relative_to(workspace) + install = read_timeline_event(session_dir, "install") + assert install is not None + assert install["status"] == "failed" + assert install["ext"]["hard_fail_step_id"] == "validate_credentials" + assert (session_dir / "manifest.json").is_file() + breakdown = json.loads((session_dir / "session_breakdown.json").read_text(encoding="utf-8")) + assert [(event["type"], event["status"]) for event in breakdown["timeline"]] == [("install", "failed")] + + +def test_unwrapped_preflight_failure_is_persisted_as_failed(tmp_path, monkeypatch): + import hyperloom.inference_optimizer.cli as optimizer_cli + from hyperloom.inference_optimizer.cli import preflight + from hyperloom.inference_optimizer.session.paths import ENV_CURRENT_SESSION_DIR + + workspace = tmp_path / "sessions" + model = tmp_path / "Qwen-Test" + monkeypatch.setenv("USER_DATA_PATH", str(workspace)) + monkeypatch.delenv(ENV_CURRENT_SESSION_DIR, raising=False) + monkeypatch.setattr( + optimizer_cli, + "clean_stale_aiter_locks", + lambda: {"dir": "", "deleted": 0, "skipped_fresh": 0, "errors": 0}, + ) + monkeypatch.setattr(preflight, "_load_dotenv_fallback", lambda: None) + monkeypatch.setattr(preflight, "_provider_only_mode", lambda: "") + monkeypatch.setattr(preflight, "_load_kernel_agent_env_fallback", lambda: None) + + def fail_runtime_paths(): + raise RuntimeError("runtime path resolution failed") + + monkeypatch.setattr(preflight, "_derive_runtime_paths", fail_runtime_paths) + args = optimizer_cli._build_parser().parse_args(["optimize", "--model", str(model)]) + + with pytest.raises(RuntimeError, match="runtime path resolution failed"): + asyncio.run(optimizer_cli._run_optimize(args)) + + session_dir = Path(os.environ[ENV_CURRENT_SESSION_DIR]) + install = read_timeline_event(session_dir, "install") + assert install is not None + assert install["status"] == "failed" + assert install["end_time"] + assert install["ext"]["hard_fail_step_id"] == "unhandled_preflight" + failure = install["ext"]["steps"][-1] + assert failure["step_id"] == "unhandled_preflight" + assert failure["error_class"] == "RuntimeError" + assert failure["message"] == "runtime path resolution failed" + + +def test_busy_resume_preflight_failure_uses_isolated_session(tmp_path, monkeypatch): + import hyperloom.inference_optimizer.cli as optimizer_cli + from hyperloom.inference_optimizer.cli import preflight + from hyperloom.inference_optimizer.session.paths import ENV_CURRENT_SESSION_DIR + + workspace = tmp_path / "sessions" + resume_dir = workspace / "Qwen-Test" / "active-session" + original_install = { + "type": "install", + "kind": "install", + "status": "succeeded", + "start_time": "2026-08-27T01:00:00+00:00", + "end_time": "2026-08-27T01:01:00+00:00", + "ext": {"run_kind": "fresh", "steps": []}, + } + _write_json(resume_dir / "reports" / "sbd_v6" / "install.json", original_install) + _write_json(resume_dir / "session_breakdown.json", {"sentinel": "active"}) + monkeypatch.setenv("USER_DATA_PATH", str(workspace)) + monkeypatch.delenv("MODEL_PATH", raising=False) + monkeypatch.delenv(ENV_CURRENT_SESSION_DIR, raising=False) + monkeypatch.setattr( + optimizer_cli, + "clean_stale_aiter_locks", + lambda: {"dir": "", "deleted": 0, "skipped_fresh": 0, "errors": 0}, + ) + + class FakeSessionLock: + def __init__(self, session_dir): + self.session_dir = Path(session_dir) + + def acquire(self): + if self.session_dir == resume_dir: + raise optimizer_cli.SessionAlreadyRunning(resume_dir, {"pid": 123}) + return self + + def release(self): + return None + + def fail_preflight(args): + preflight._begin_install_event(args) + raise RuntimeError("resume preflight failed") + + monkeypatch.setattr(optimizer_cli, "SessionLock", FakeSessionLock) + monkeypatch.setattr(optimizer_cli, "_preflight", fail_preflight) + args = optimizer_cli._build_parser().parse_args(["optimize", "--resume-from", str(resume_dir)]) + + with pytest.raises(RuntimeError, match="resume preflight failed"): + asyncio.run(optimizer_cli._run_optimize(args)) + + assert json.loads((resume_dir / "reports" / "sbd_v6" / "install.json").read_text(encoding="utf-8")) == ( + original_install + ) + assert json.loads((resume_dir / "session_breakdown.json").read_text(encoding="utf-8")) == {"sentinel": "active"} + failed_session = Path(os.environ[ENV_CURRENT_SESSION_DIR]) + assert failed_session != resume_dir + install = read_timeline_event(failed_session, "install") + assert install is not None + assert install["status"] == "failed" + assert install["ext"]["run_kind"] == "resume" + + +def test_timeline_history_retains_fresh_and_resume_events(tmp_path, monkeypatch): + from hyperloom.inference_optimizer.cli import model_gate + + write_timeline_event( + tmp_path, + { + "type": "install", + "kind": "install", + "status": "succeeded", + "start_time": "2026-08-27T01:00:00+00:00", + "end_time": "2026-08-27T01:01:00+00:00", + "ext": {"run_kind": "fresh", "steps": []}, + }, + ) + timestamps = iter( + ( + "2026-08-27T01:02:00+00:00", + "2026-08-27T01:03:00+00:00", + "2026-08-27T02:02:00+00:00", + "2026-08-27T02:03:00+00:00", + "2026-08-27T02:04:00+00:00", + ) + ) + monkeypatch.setattr(model_gate, "now_iso", lambda **_kwargs: next(timestamps)) + fresh_args = _gate_args(tmp_path / "model") + model_gate._start_model_gate(fresh_args, tmp_path) + model_gate._finish_model_gate(fresh_args, tmp_path) + write_timeline_event( + tmp_path, + { + "type": "install", + "kind": "install", + "status": "succeeded", + "start_time": "2026-08-27T02:00:00+00:00", + "end_time": "2026-08-27T02:01:00+00:00", + "ext": {"run_kind": "resume", "steps": []}, + }, + ) + resume_args = _gate_args(tmp_path / "model", resume_from=str(tmp_path)) + model_gate._record_resumed_model_gate(resume_args, tmp_path) + + expected = [ + ("install", "succeeded", "fresh"), + ("model_gate", "succeeded", "fresh"), + ("install", "succeeded", "resume"), + ("model_gate", "skipped", "resume"), + ] + assert [ + (event["type"], event["status"], event["ext"]["run_kind"]) for event in read_timeline_events(tmp_path) + ] == expected + assert [ + (event["type"], event["status"], event["ext"]["run_kind"]) + for event in collect_v6_timeline(tmp_path, [], state={}, recorded_operations=[]) + ] == expected + latest_gate = read_timeline_event(tmp_path, "model_gate") + assert latest_gate is not None + assert latest_gate["ext"]["run_kind"] == "resume" + + +def test_timeline_history_bootstraps_legacy_fixed_events(tmp_path): + _write_json( + tmp_path / "reports" / "sbd_v6" / "install.json", + { + "type": "install", + "kind": "install", + "status": "succeeded", + "start_time": "2026-08-27T01:00:00+00:00", + "end_time": "2026-08-27T01:01:00+00:00", + "ext": {"run_kind": "fresh", "steps": []}, + }, + ) + _write_json( + tmp_path / "reports" / "sbd_v6" / "model_gate.json", + { + "type": "model_gate", + "kind": "model_gate", + "status": "succeeded", + "start_time": "2026-08-27T01:02:00+00:00", + "end_time": "2026-08-27T01:03:00+00:00", + "ext": {"run_kind": "fresh", "checks": []}, + }, + ) + + write_timeline_event( + tmp_path, + { + "type": "install", + "kind": "install", + "status": "succeeded", + "start_time": "2026-08-27T02:00:00+00:00", + "end_time": "2026-08-27T02:01:00+00:00", + "ext": {"run_kind": "resume", "steps": []}, + }, + ) + + assert [(event["type"], event["ext"]["run_kind"]) for event in read_timeline_events(tmp_path)] == [ + ("install", "fresh"), + ("model_gate", "fresh"), + ("install", "resume"), + ] + + +def test_timeline_history_recovers_after_partial_legacy_migration(tmp_path): + fresh_install = { + "type": "install", + "kind": "install", + "status": "succeeded", + "start_time": "2026-08-27T01:00:00+00:00", + "end_time": "2026-08-27T01:01:00+00:00", + "ext": {"run_kind": "fresh", "steps": []}, + } + _write_json(tmp_path / "reports" / "sbd_v6" / "install.json", fresh_install) + _write_json(tmp_path / "reports" / "sbd_v6" / "timeline" / "000001-install.json", fresh_install) + _write_json( + tmp_path / "reports" / "sbd_v6" / "model_gate.json", + { + "type": "model_gate", + "kind": "model_gate", + "status": "succeeded", + "start_time": "2026-08-27T01:02:00+00:00", + "end_time": "2026-08-27T01:03:00+00:00", + "ext": {"run_kind": "fresh", "checks": []}, + }, + ) + + write_timeline_event( + tmp_path, + { + "type": "install", + "kind": "install", + "status": "succeeded", + "start_time": "2026-08-27T02:00:00+00:00", + "end_time": "2026-08-27T02:01:00+00:00", + "ext": {"run_kind": "resume", "steps": []}, + }, + ) + + assert [(event["type"], event["ext"]["run_kind"]) for event in read_timeline_events(tmp_path)] == [ + ("install", "fresh"), + ("model_gate", "fresh"), + ("install", "resume"), + ] + + +def test_preflight_records_install_steps_in_execution_order(tmp_path, monkeypatch): + from hyperloom.agents.framework import kb as framework_kb + from hyperloom.inference_optimizer.cli import preflight + from hyperloom.inference_optimizer.session.sbd_v6 import pending_install_event + from hyperloom.orchestrator.actions.executors import benchmark_backend + + inferencex = tmp_path / "InferenceX" + inferencex.mkdir() + monkeypatch.setenv("INFERENCEX_PATH", str(inferencex)) + monkeypatch.setattr(benchmark_backend, "resolve_backend_name", lambda: "bypass") + monkeypatch.setattr(benchmark_backend, "resolve_benchmark_interpreter", lambda: "python") + monkeypatch.setattr(preflight, "_provider_only_mode", lambda: "") + monkeypatch.setattr(preflight, "_load_dotenv_fallback", lambda: None) + monkeypatch.setattr(preflight, "_load_kernel_agent_env_fallback", lambda: None) + monkeypatch.setattr(preflight, "_derive_runtime_paths", lambda: None) + monkeypatch.setattr(preflight, "_restore_provider_only_mode", lambda *_args: None) + monkeypatch.setattr(preflight, "_normalize_legacy_deepseek_env", lambda: None) + monkeypatch.setattr(preflight, "_validate_credentials", lambda: None) + monkeypatch.setattr(framework_kb, "prepare_kb_environment", lambda: None) + monkeypatch.setattr(preflight, "_ensure_python_sdks", lambda *_args: None) + monkeypatch.setattr(preflight, "_resolve_llm_endpoints", lambda: ("", "")) + monkeypatch.setattr(preflight, "_unset_hip_visible_devices", lambda: None) + monkeypatch.setattr(preflight, "_check_gpu_visibility", lambda: None) + monkeypatch.setattr(preflight, "_check_shm_disk", lambda: None) + monkeypatch.setattr(preflight, "_check_platform_tuning", lambda: None) + monkeypatch.setattr(preflight, "_ensure_ray", lambda *_args: None) + monkeypatch.setattr(preflight, "_ensure_bench_serving_deps", lambda *_args: None) + monkeypatch.setattr(preflight, "_ensure_lm_eval_dep", lambda *_args, **_kwargs: None) + monkeypatch.setattr(preflight, "_ensure_framework_deps", lambda *_args: None) + monkeypatch.setattr(preflight, "_check_serving_framework", lambda *_args: None) + monkeypatch.setattr(preflight, "_inferencex_checkout_ok", lambda *_args, **_kwargs: True) + monkeypatch.setattr(preflight, "_inferencex_head_sha", lambda *_args: "abc123") + monkeypatch.setattr(preflight, "_report_inferencex_patch_anchors", lambda *_args: True) + monkeypatch.setattr(preflight, "_check_node_claude_cli", lambda: None) + monkeypatch.setattr(preflight.shutil, "which", lambda name: f"/usr/bin/{name}") + monkeypatch.setattr( + preflight, + "_run_ir3_preflight", + lambda _args: {"status": "applied", "skip_reason": None}, + ) + monkeypatch.setattr( + preflight, + "_emit_preflight_diagnostics", + lambda **_kwargs: {"status": "applied", "skip_reason": None}, + ) + args = argparse.Namespace( + resume_from=None, + degraded_kb=True, + framework="sglang", + no_eval=True, + no_kernel=True, + enable_roofline=False, + ) + + preflight._preflight(args) + + event = pending_install_event(args) + assert event is not None + assert [step["step_id"] for step in event["ext"]["steps"]] == [ + "load_dotenv", + "load_kernel_agent_env", + "normalize_legacy_deepseek_env", + "validate_credentials", + "prepare_kb_environment", + "ensure_python_sdks", + "check_gpu_visibility", + "check_shm_disk", + "check_platform_tuning", + "ensure_ray", + "ensure_bench_serving_deps", + "ensure_lm_eval", + "framework_deps", + "check_serving_framework", + "ensure_magpie", + "clone_inferencex", + "patch_magpie_eval_concurrency", + "check_tracelens_cli", + "check_tracelens_root", + "ir3_pr_monitor_probe", + "diagnostics_snapshot", + ] + steps = {step["step_id"]: step for step in event["ext"]["steps"]} + assert steps["prepare_kb_environment"]["status"] == "skipped" + assert steps["prepare_kb_environment"]["skip_reason"] == "explicit_flag" + assert steps["ensure_magpie"]["message"] == "benchmark backend is 'bypass'" + + +def test_ir3_unreachable_uses_v6_reason_without_changing_v5_state_reason(tmp_path, monkeypatch): + from hyperloom.inference_optimizer.cli import preflight + + monkeypatch.setattr(preflight, "_workspace_root_resolve", lambda: tmp_path) + monkeypatch.setattr(preflight.subprocess, "run", lambda *_args, **_kwargs: None) + args = argparse.Namespace(degraded_kb=False, degraded_pr=False, pr_monitor_url="") + + outcome = preflight._run_ir3_preflight(args) + + assert args.pr_degraded_reason == "ir3_auto" + assert outcome["status"] == "warned" + assert outcome["skip_reason"] == "ir3_unreachable" + assert outcome["detail"]["pr_monitor"] == { + "enabled": False, + "reason": "ir3_unreachable", + } + + +@pytest.mark.parametrize( + ("scenario", "failed_gate_id", "expected_statuses"), + [ + ("unsupported", "unsupported_model_arch", ["failed", "skipped", "skipped"]), + ("config", "model_config_compat", ["passed", "failed", "skipped"]), + ("context", "context_window", ["passed", "passed", "failed"]), + ], +) +def test_each_model_gate_failure_is_written_to_final_sbd( + tmp_path, + monkeypatch, + scenario, + failed_gate_id, + expected_statuses, +): + from hyperloom.inference_optimizer.cli import model_gate + + monkeypatch.setattr(model_gate, "_emit_breakdown_to_langfuse", lambda _session_dir: None) + model = tmp_path / scenario + if scenario == "unsupported": + _write_model_config( + model, + { + "architectures": ["Gemma3ForConditionalGeneration"], + "model_type": "gemma3", + }, + ) + elif scenario == "config": + _write_model_config( + model, + { + "architectures": ["LlamaForCausalLM"], + "model_type": "llama", + "rope_scaling": {"factor": 2.0}, + }, + ) + else: + _write_model_config( + model, + { + "architectures": ["LlamaForCausalLM"], + "model_type": "llama", + "max_position_embeddings": 2048, + }, + ) + session_dir = tmp_path / f"session-{scenario}" + _seed_state(session_dir, monkeypatch, model) + args = _gate_args(model) + model_gate._start_model_gate(args, session_dir) + + if scenario == "unsupported": + assert model_gate._preflight_unsupported_model_arch(args, session_dir) is True + else: + assert model_gate._preflight_unsupported_model_arch(args, session_dir) is False + if scenario == "config": + assert model_gate._preflight_model_config_compat(args, session_dir) is True + else: + assert model_gate._preflight_model_config_compat(args, session_dir) is False + assert model_gate._preflight_context_window(args, session_dir) is True + + event = _model_gate_from_breakdown(session_dir) + assert event["status"] == "failed" + assert event["ext"]["failed_gate_id"] == failed_gate_id + assert [check["gate_id"] for check in event["ext"]["checks"]] == [ + "unsupported_model_arch", + "model_config_compat", + "context_window", + ] + assert [check["status"] for check in event["ext"]["checks"]] == expected_statuses + assert event["ext"]["failure"]["artifacts"]["breakdown_written"] is True + + +def test_resume_model_gate_records_three_explicit_skips(tmp_path): + from hyperloom.inference_optimizer.cli import model_gate + + args = _gate_args(tmp_path / "model") + model_gate._record_resumed_model_gate( + args, + tmp_path, + workload_overrides={ + "model_path": "/models/resumed", + "model_name": "resumed-model", + "framework": "vllm", + "gpu_type": "MI355X", + }, + ) + + event = read_timeline_event(tmp_path, "model_gate") + assert event is not None + assert event["status"] == "skipped" + assert event["ext"]["run_kind"] == "resume" + assert event["ext"]["skip_reason"] == "resume" + assert event["ext"]["workload"]["model_path"] == "/models/resumed" + assert event["ext"]["workload"]["model_name"] == "resumed-model" + assert event["ext"]["workload"]["framework"] == "vllm" + assert event["ext"]["workload"]["gpu_type"] == "MI355X" + assert [check["skip_reason"] for check in event["ext"]["checks"]] == [ + "resume", + "resume", + "resume", + ] + + +def test_model_gate_event_write_failure_does_not_change_gate_result(tmp_path, monkeypatch): + from hyperloom.inference_optimizer.cli import model_gate + from hyperloom.inference_optimizer.session import sbd_v6 + + model = tmp_path / "healthy" + _write_model_config( + model, + { + "architectures": ["LlamaForCausalLM"], + "model_type": "llama", + "max_position_embeddings": 8192, + }, + ) + monkeypatch.setattr( + sbd_v6, + "write_timeline_event", + lambda *_args, **_kwargs: (_ for _ in ()).throw(OSError("disk unavailable")), + ) + + args = _gate_args(model) + model_gate._start_model_gate(args, tmp_path) + + assert model_gate._preflight_unsupported_model_arch(args, tmp_path) is False + + +def test_model_gate_projection_failure_does_not_change_gate_result(tmp_path, monkeypatch): + from hyperloom.inference_optimizer.cli import model_gate + + model = tmp_path / "healthy" + _write_model_config( + model, + { + "architectures": ["LlamaForCausalLM"], + "model_type": "llama", + "max_position_embeddings": 8192, + }, + ) + monkeypatch.setattr( + model_gate, + "_load_model_gate_event", + lambda *_args, **_kwargs: (_ for _ in ()).throw(ValueError("bad V6 projection")), + ) + + assert model_gate._preflight_unsupported_model_arch(_gate_args(model), tmp_path) is False + + +def test_install_projection_failure_does_not_change_step_result(monkeypatch): + from hyperloom.inference_optimizer.cli import preflight + + monkeypatch.setattr( + preflight, + "_record_install_step", + lambda *_args, **_kwargs: (_ for _ in ()).throw(ValueError("bad V6 projection")), + ) + + assert ( + preflight._run_install_step( + {"ext": {"steps": []}}, + step_id="unchanged", + category="check", + action=lambda: "original-result", + ) + == "original-result" + ) + + +def test_corrupt_model_gate_event_is_safely_normalized(tmp_path): + from hyperloom.inference_optimizer.cli import model_gate + + path = tmp_path / "reports" / "sbd_v6" / "model_gate.json" + _write_json( + path, + { + "type": "model_gate", + "ext": { + "checks": [{"gate_id": "legacy", "order": "invalid", "status": "unknown"}], + "degraded": [], + }, + }, + ) + args = _gate_args(tmp_path / "model") + + model_gate._record_model_gate_check( + args, + tmp_path, + { + "gate_id": "unsupported_model_arch", + "order": 1, + "status": "passed", + "skip_reason": None, + "detail": {}, + }, + ) + + event = read_timeline_event(tmp_path, "model_gate") + assert event is not None + assert event["status"] == "degraded" + assert event["ext"]["checks"][0]["gate_id"] == "unsupported_model_arch" + assert event["ext"]["degraded"] == {"active": False, "warnings": []} + + +def test_fresh_model_gate_with_only_soft_skips_succeeds(tmp_path): + from hyperloom.inference_optimizer.cli import model_gate + + args = _gate_args(tmp_path / "model") + model_gate._start_model_gate(args, tmp_path) + for order, gate_id in enumerate(model_gate._MODEL_GATE_ORDER, start=1): + model_gate._record_model_gate_check( + args, + tmp_path, + { + "gate_id": gate_id, + "order": order, + "status": "skipped", + "skip_reason": "soft_pass", + "detail": {}, + }, + ) + model_gate._finish_model_gate(args, tmp_path) + + event = read_timeline_event(tmp_path, "model_gate") + assert event is not None + assert event["status"] == "succeeded" + assert event["ext"]["skip_reason"] is None + + +def test_framework_timeline_merges_legacy_framework_and_explore(tmp_path): + state = { + "phase": "KERNEL_AGENT", + "macro_cycle": 0, + "framework_agent_phase_done": True, + "phase_history": [ + { + "from_phase": "PRELUDE", + "to_phase": "FRAMEWORK_AGENT", + "reason": "prelude_complete", + "ts": "2026-08-27T01:00:00+00:00", + "cycle": 0, + }, + { + "from_phase": "FRAMEWORK_AGENT", + "to_phase": "EXPLORE", + "reason": "framework_agent_phase_done", + "ts": "2026-08-27T01:10:00+00:00", + "cycle": 0, + }, + { + "from_phase": "EXPLORE", + "to_phase": "KERNEL_AGENT", + "reason": "explore_no_more_leverage", + "ts": "2026-08-27T01:20:00+00:00", + "cycle": 0, + "evidence": { + "recent_keep_gain_pct": 5.0, + "keep_gain_threshold_pct": 6.0, + "empty_streak": 2, + "empty_streak_threshold": 2, + "lookback": 6, + "tested_this_cycle": 1, + "config_arm_plateaued": True, + "source_consecutive_no_keep": 1, + "source_threshold": 3, + "source_candidates_exhausted": True, + "source_arm_plateaued": True, + "switch_bottleneck": True, + "evidence": "both_arms_plateaued", + }, + }, + ], + "specialist_rounds": [ + { + "round_id": "spec-config-1", + "task_id": "spec-config-task", + "domain": "serving_specialist", + "cycle": 0, + "completed_at": "2026-08-27T01:04:00+00:00", + "proposal_set": [ + { + "name": "chunked-prefill", + "fingerprint": "fp-config-1", + } + ], + } + ], + "explore_search": { + "tested": { + "fp-config-1": { + "fingerprint": "fp-config-1", + "name": "chunked-prefill", + "outcome": "KEEP", + "tput": 105.0, + "base_tput": 100.0, + "gain_pct": 5.0, + "round_id": "config-round-1", + "cycle": 0, + "workload_signature": "qwen-tp8-c64", + "framework": "sglang", + "stack_rebench_tput": 104.0, + "stack_rebench_workspace": "runs/config-round-1/rebench", + } + }, + "winners_history": [{"gain_pct": 5.0, "cycle": 0}], + }, + "framework_agent_batches": [ + { + "batch_id": "legacy-batch", + "candidates": [ + { + "pr_url": "https://example.test/pr/7", + "route": "direct_framework", + "audit": {"verdict": "worth_a_bench"}, + } + ], + } + ], + "framework_agent_phase_progress": [ + { + "candidate_id": "https://example.test/pr/7", + "status": "kept", + "kept": True, + "pre_tput": 105.0, + "post_tput": 108.0, + "gain_pct": 2.857, + "cycle": 0, + "ts": "2026-08-27T01:08:00+00:00", + } + ], + } + operations = [ + { + "operation_id": "op-source-1", + "name": "framework_agent", + "phase": "FRAMEWORK_AGENT", + "macro_cycle": 0, + "status": "succeeded", + "ended_at": "2026-08-27T01:08:00+00:00", + "outputs": { + "status": "kept", + "candidate": { + "pr_url": "https://example.test/pr/7", + "route": "direct_framework", + "changed_files": ["python/server.py"], + }, + "base_tput": 105.0, + "output_throughput": 108.0, + "delta_pct": 2.857, + "accuracy_pass": True, + "keep_threshold_pct": 1.0, + "patches_applied": ["patches/pr-7.patch"], + "target_files": ["python/server.py"], + "workspace": "runs/framework/pr-7", + }, + "extensions": {"task_id": "source-task-1"}, + }, + { + "operation_id": "op-config-1", + "name": "explore", + "phase": "EXPLORE", + "macro_cycle": 0, + "status": "succeeded", + "ended_at": "2026-08-27T01:16:00+00:00", + "outputs": { + "status": "succeeded", + "round_id": "config-round-1", + "framework": "sglang", + "base_tput": 100.0, + "per_variant_outcomes": [ + { + "variant_name": "chunked-prefill", + "outcome": "KEEP", + "fingerprint": "fp-config-1", + "provenance": "specialist:serving_specialist", + "scope": "domain", + "metrics": {"tput": 105.0, "gain_pct": 5.0}, + "variant": { + "extra_server_args": "--enable-chunked-prefill", + "extra_envs": {"SGLANG_CHUNKED_PREFILL": "1"}, + }, + } + ], + "explore_search_update": { + "tested": state["explore_search"]["tested"], + "last_round": { + "round_id": "config-round-1", + "base_tput": 100.0, + "base_extra_args": "--base-flag", + }, + }, + }, + "extensions": {"task_id": "config-task-1"}, + }, + ] + + timeline = collect_v6_timeline(tmp_path, [], state=state, recorded_operations=operations) + + assert [event["type"] for event in timeline] == ["framework_agent"] + event = timeline[0] + assert event["start_time"] == "2026-08-27T01:00:00+00:00" + assert event["end_time"] == "2026-08-27T01:20:00+00:00" + assert "summary" not in event + assert event["ext"]["policy"]["stack_rebench_enabled"] is None + assert event["ext"]["config_arm"]["rounds"][0]["workload_signature"] == "qwen-tp8-c64" + assert event["ext"]["config_arm"]["rounds"][0]["input_stack"]["extra_server_args"] == "--base-flag" + variant = event["ext"]["config_arm"]["rounds"][0]["variants"][0] + assert variant["stack_rebench"] == {"ran": True, "tput": 104.0, "stable": True} + attempt = event["ext"]["source_arm"]["attempts"][0] + assert attempt["patch_source"] == "upstream_pr" + assert attempt["lever_kind"] == "upstream_pr" + assert attempt["route"] == "direct_framework" + assert attempt["status"] == "KEEP" + assert event["ext"]["exit"] == { + "reason": "optimize_no_more_leverage", + "trigger": "both_arms_plateaued", + "hint": None, + "switch_bottleneck": True, + } + + +def test_framework_timeline_projects_pr1301_source_and_critic_data(tmp_path): + state = { + "phase": "KERNEL_AGENT", + "macro_cycle": 2, + "framework_agent_authoring_enabled": True, + "framework_agent_phase_done": False, + "phase_history": [ + { + "from_phase": "SWEEP", + "to_phase": "FRAMEWORK_AGENT", + "reason": "cycle_reloop", + "ts": "2026-08-27T02:00:00+00:00", + "cycle": 2, + }, + { + "from_phase": "FRAMEWORK_AGENT", + "to_phase": "KERNEL_AGENT", + "reason": "optimize_phase_budget_exhausted", + "ts": "2026-08-27T02:20:00+00:00", + "cycle": 2, + "evidence": { + "source_consecutive_no_keep": 0, + "source_threshold": 3, + "source_candidates_exhausted": False, + "source_arm_plateaued": False, + "recent_keep_gain_pct": 0.0, + "keep_gain_threshold_pct": 1.0, + "empty_streak": 0, + "empty_streak_threshold": 3, + "lookback": 6, + "tested_this_cycle": 0, + "config_arm_plateaued": False, + "switch_bottleneck": False, + }, + }, + ], + "specialist_rounds": [ + { + "round_id": "discover-round", + "task_id": "discover-task-1234", + "domain": "candidate_discovery_specialist", + "cycle": 2, + "completed_at": "2026-08-27T02:04:00+00:00", + "proposal_set": [ + { + "pr_url": "https://example.test/pr/9", + "title": "Fuse host copies", + "verdict": "worth_a_bench", + "route": "author_via_specialist", + }, + { + "pr_url": "https://example.test/pr/10", + "title": "Unrelated backend", + "verdict": "not_applicable", + "reason": "wrong framework", + }, + ], + }, + { + "round_id": "author-round", + "task_id": "author-task-1", + "domain": "serving_specialist", + "cycle": 2, + "completed_at": "2026-08-27T02:08:00+00:00", + "proposal_set": [{"patches_written": ["patches/pr-9.patch"]}], + }, + ], + "framework_agent_batches": [ + { + "batch_id": "discovery-0-discover", + "candidates": [ + { + "pr_url": "https://example.test/pr/9", + "route": "author_via_specialist", + "audit": {"verdict": "worth_a_bench"}, + } + ], + } + ], + "framework_agent_specialist_candidate_map": { + "author-task-1": "https://example.test/pr/9", + }, + "framework_agent_phase_progress": [ + { + "candidate_id": "https://example.test/pr/9", + "batch_id": "discovery-0-discover", + "status": "kept", + "kept": True, + "gain_pct": 4.0, + "pre_tput": 100.0, + "post_tput": 104.0, + "specialist_task_id": "author-task-1", + "integrate_task_id": "integrate-task-1", + "cycle": 2, + "ts": "2026-08-27T02:15:00+00:00", + } + ], + } + operations = [ + { + "operation_id": "op-integrate-1", + "name": "integrate_patch", + "phase": "FRAMEWORK_AGENT", + "agent": "framework_agent", + "macro_cycle": 2, + "status": "succeeded", + "ended_at": "2026-08-27T02:15:00+00:00", + "outputs": { + "status": "kept", + "framework_agent_authoring": True, + "specialist_task_id": "author-task-1", + "base_tput": 100.0, + "output_throughput": 104.0, + "delta_pct": 4.0, + "accuracy_pass": True, + "keep_threshold_pct": 1.0, + "patches_applied": ["patches/pr-9.patch"], + "target_files": ["python/worker.py"], + "source_snapshot": "optimization_stack/src/author-task-1", + "source_manifest": "optimization_stack/src/author-task-1/manifest.json", + "workspace": "runs/integrate-task-1", + "switch_off_parity": {"ran": True, "ok": True}, + "stack_rebench": {"stable": True}, + "framework_levers": [{"switch": "SGLANG_FAST_COPY", "default_on": True}], + }, + "extensions": {"task_id": "integrate-task-1"}, + } + ] + critic_dir = tmp_path / "critic-workdir" / "000000" + _write_json( + critic_dir / "judge_bundle.json", + { + "merged_context": {"macro_cycle": 2}, + "proposals": [ + { + "msg_id": "proposal-1", + "action_name": "integrate_patch", + "payload": { + "params": { + "framework_agent_candidate_id": "https://example.test/pr/9", + } + }, + } + ], + }, + ) + _write_json( + critic_dir / "review.json", + { + "review_verdicts": [ + { + "target_proposal_msg_id": "proposal-1", + "verdict": "needs_review", + "source": "critic", + "reasoning": "needs parity evidence", + "confidence": "high", + "required_evidence": ["switch-off parity"], + "risks": [{"severity": "major", "risk": "default behavior may change"}], + } + ] + }, + ) + _write_json( + critic_dir / "emit.json", + { + "intent_envelope": { + "intents": [ + { + "intent_type": "review_verdict", + "payload": { + "target_proposal_msg_id": "proposal-1", + "verdict": "approve", + "advice_text": "retain the switch-off check", + }, + } + ] + } + }, + ) + + timeline = collect_v6_timeline(tmp_path, [], state=state, recorded_operations=operations) + + event = timeline[0] + discovery = event["ext"]["source_arm"]["candidate_discovery_runs"][0] + assert discovery["task_id"] == "discover-task-1234" + assert [candidate["verdict"] for candidate in discovery["candidates"]] == [ + "worth_a_bench", + "not_applicable", + ] + authoring = event["ext"]["source_arm"]["authoring_runs"][0] + assert authoring["candidate_id"] == "https://example.test/pr/9" + assert authoring["patch_refs"] == ["patches/pr-9.patch"] + attempt = event["ext"]["source_arm"]["attempts"][0] + assert attempt["patch_source"] == "specialist_authored" + assert attempt["lever_kind"] == "source_patch" + assert attempt["route"] == "author_via_specialist" + assert attempt["status"] == "KEEP" + assert attempt["gates"] == { + "accuracy_passed": True, + "keep_threshold_pct": 1.0, + "switch_off_parity_passed": True, + "stack_rebench_passed": True, + } + review = event["ext"]["critic_reviews"][0] + assert review["arm"] == "source" + assert review["target_action"] == "integrate_patch" + assert review["verdict"] == "needs_review" + assert review["effective_verdict"] == "approve" + assert "token" not in json.dumps(event).lower() + + +def test_framework_timeline_keeps_config_serving_specialist_out_of_source_arm(tmp_path): + state = { + "phase": "KERNEL_AGENT", + "macro_cycle": 0, + "phase_history": [ + { + "from_phase": "PRELUDE", + "to_phase": "FRAMEWORK_AGENT", + "ts": "2026-08-27T03:00:00+00:00", + "cycle": 0, + }, + { + "from_phase": "FRAMEWORK_AGENT", + "to_phase": "KERNEL_AGENT", + "ts": "2026-08-27T03:10:00+00:00", + "cycle": 0, + }, + ], + "specialist_rounds": [ + { + "round_id": "config-round", + "task_id": "config-task", + "domain": "serving_specialist", + "source_phase": "FRAMEWORK_AGENT", + "cycle": 0, + "proposal_set": [ + { + "name": "larger-page-size", + "extra_server_args": "--page-size 32", + } + ], + } + ], + } + critic_dir = tmp_path / "critic-workdir" / "000000" + _write_json( + critic_dir / "request.json", + { + "context": {"phase": "FRAMEWORK_AGENT"}, + "raw_prompt": "=== Shared session state ===\nmacro_cycle=0\n", + }, + ) + _write_json( + critic_dir / "judge_bundle.json", + { + "phase": "FRAMEWORK_AGENT", + "proposals": [ + { + "msg_id": "config-proposal", + "action_name": "specialist", + "payload": { + "params": { + "domain": "serving_specialist", + "source_phase": "FRAMEWORK_AGENT", + } + }, + } + ], + }, + ) + _write_json( + critic_dir / "review.json", + { + "review_verdicts": [ + { + "target_proposal_msg_id": "config-proposal", + "verdict": "approve", + } + ] + }, + ) + _write_json( + critic_dir / "emit.json", + { + "intent_envelope": { + "intents": [ + { + "intent_type": "review_verdict", + "payload": { + "target_proposal_msg_id": "config-proposal", + "verdict": "approve", + }, + } + ] + } + }, + ) + + event = collect_v6_timeline(tmp_path, [], state=state, recorded_operations=[])[0] + + assert [row["task_id"] for row in event["ext"]["config_arm"]["specialist_runs"]] == ["config-task"] + assert event["ext"]["source_arm"]["authoring_runs"] == [] + assert event["ext"]["critic_reviews"][0]["arm"] == "config" + + +def test_framework_timeline_ignores_kernel_specialist_without_framework_evidence(tmp_path): + state = { + "phase": "KERNEL_AGENT", + "macro_cycle": 4, + "specialist_rounds": [ + { + "round_id": "kernel-specialist", + "task_id": "kernel-specialist", + "domain": "kernel_specialist", + "cycle": 4, + "completed_at": "2026-08-27T04:00:00+00:00", + "proposal_set": [{"name": "kernel-rewrite"}], + } + ], + } + + operations = [ + { + "operation_id": "op-kernel-specialist", + "kind": "specialist", + "name": "specialist round kernel-specialist", + "phase": "EXPLORE", + "source": "specialist_recorder_hook", + "macro_cycle": 4, + "status": "succeeded", + "outputs": state["specialist_rounds"][0], + } + ] + + assert collect_v6_timeline(tmp_path, [], state=state, recorded_operations=operations) == [] + + +def test_framework_timeline_recovers_direct_upstream_patch_source(tmp_path): + candidate_id = "https://example.test/pr/11" + state = { + "phase": "KERNEL_AGENT", + "macro_cycle": 0, + "phase_history": [ + { + "from_phase": "PRELUDE", + "to_phase": "FRAMEWORK_AGENT", + "ts": "2026-08-27T03:00:00+00:00", + "cycle": 0, + }, + { + "from_phase": "FRAMEWORK_AGENT", + "to_phase": "KERNEL_AGENT", + "ts": "2026-08-27T03:10:00+00:00", + "cycle": 0, + }, + ], + "framework_agent_batches": [ + { + "batch_id": "discovery-0", + "candidates": [ + { + "pr_url": candidate_id, + "route": "direct_framework", + } + ], + } + ], + "framework_agent_phase_progress": [ + { + "candidate_id": candidate_id, + "integrate_task_id": "integrate-direct-1", + "status": "kept", + "kept": True, + "cycle": 0, + } + ], + } + operations = [ + { + "operation_id": "op-integrate-direct", + "name": "integrate_patch", + "phase": "FRAMEWORK_AGENT", + "macro_cycle": 0, + "status": "succeeded", + "outputs": { + "status": "kept", + "framework_agent_authoring": True, + "specialist_task_id": "integrate-direct-1", + }, + "extensions": {"task_id": "integrate-direct-1"}, + } + ] + + event = collect_v6_timeline(tmp_path, [], state=state, recorded_operations=operations)[0] + attempt = event["ext"]["source_arm"]["attempts"][0] + + assert attempt["candidate_id"] == candidate_id + assert attempt["patch_source"] == "upstream_pr" + assert attempt["lever_kind"] == "upstream_pr" + assert attempt["route"] == "direct_framework" + + +def test_framework_timeline_keeps_macro_cycles_isolated(tmp_path): + state = { + "phase": "KERNEL_AGENT", + "macro_cycle": 1, + "phase_history": [ + { + "from_phase": "PRELUDE", + "to_phase": "FRAMEWORK_AGENT", + "reason": "prelude_complete", + "ts": "2026-08-27T03:00:00+00:00", + "cycle": 0, + }, + { + "from_phase": "FRAMEWORK_AGENT", + "to_phase": "KERNEL_AGENT", + "reason": "optimize_no_more_leverage", + "ts": "2026-08-27T03:10:00+00:00", + "cycle": 0, + }, + { + "from_phase": "SWEEP", + "to_phase": "FRAMEWORK_AGENT", + "reason": "cycle_reloop", + "ts": "2026-08-27T04:00:00+00:00", + "cycle": 1, + }, + { + "from_phase": "FRAMEWORK_AGENT", + "to_phase": "KERNEL_AGENT", + "reason": "optimize_no_more_leverage", + "ts": "2026-08-27T04:10:00+00:00", + "cycle": 1, + }, + ], + } + operations = [ + { + "operation_id": "cycle-0", + "name": "explore", + "phase": "FRAMEWORK_AGENT", + "macro_cycle": 0, + "status": "succeeded", + "outputs": {"status": "succeeded", "round_id": "round-0"}, + }, + { + "operation_id": "cycle-1", + "name": "explore", + "phase": "FRAMEWORK_AGENT", + "macro_cycle": 1, + "status": "succeeded", + "outputs": {"status": "succeeded", "round_id": "round-1"}, + }, + ] + + timeline = collect_v6_timeline(tmp_path, [], state=state, recorded_operations=operations) + + assert [event["ext"]["macro_cycle"] for event in timeline] == [0, 1] + assert [event["ext"]["config_arm"]["rounds"][0]["round_id"] for event in timeline] == [ + "round-0", + "round-1", + ] + + +def test_framework_timeline_excludes_kernel_phase_explore_rebench(tmp_path): + state = { + "phase": "KERNEL_AGENT", + "macro_cycle": 0, + "phase_history": [ + { + "from_phase": "PRELUDE", + "to_phase": "FRAMEWORK_AGENT", + "ts": "2026-08-27T03:00:00+00:00", + "cycle": 0, + }, + { + "from_phase": "FRAMEWORK_AGENT", + "to_phase": "KERNEL_AGENT", + "ts": "2026-08-27T03:10:00+00:00", + "cycle": 0, + }, + ], + } + operations = [ + { + "operation_id": "framework-round", + "name": "explore", + "phase": "FRAMEWORK_AGENT", + "agent": "explore", + "macro_cycle": 0, + "status": "succeeded", + "ended_at": "2026-08-27T03:05:00+00:00", + "outputs": {"status": "succeeded", "round_id": "framework-round"}, + }, + { + "operation_id": "kernel-rebench", + "name": "explore", + "phase": "KERNEL_AGENT", + "agent": "explore", + "macro_cycle": 0, + "status": "succeeded", + "ended_at": "2026-08-27T03:06:00+00:00", + "outputs": {"status": "succeeded", "round_id": "kernel-rebench"}, + }, + ] + + event = collect_v6_timeline(tmp_path, [], state=state, recorded_operations=operations)[0] + + assert [row["round_id"] for row in event["ext"]["config_arm"]["rounds"]] == ["framework-round"] + + +def test_framework_timeline_projects_discovery_history_outcomes_in_order(tmp_path): + state = { + "phase": "KERNEL_AGENT", + "macro_cycle": 0, + "phase_history": [ + { + "from_phase": "PRELUDE", + "to_phase": "FRAMEWORK_AGENT", + "ts": "2026-08-27T03:00:00+00:00", + "cycle": 0, + }, + { + "from_phase": "FRAMEWORK_AGENT", + "to_phase": "FRAMEWORK_AGENT", + "reason": "framework_agent_discover_failed", + "evidence": { + "event": "framework_agent_discover_failed", + "attempt": 1, + "limit": 3, + "error": "TimeoutError('upstream unavailable')", + }, + "ts": "2026-08-27T03:01:00+00:00", + "cycle": 0, + }, + { + "from_phase": "FRAMEWORK_AGENT", + "to_phase": "FRAMEWORK_AGENT", + "reason": "discover_empty_payload", + "evidence": { + "event": "framework_agent_phase_done", + "failure_count": 0, + "retry_limit": 3, + }, + "ts": "2026-08-27T03:03:00+00:00", + "cycle": 0, + }, + { + "from_phase": "FRAMEWORK_AGENT", + "to_phase": "KERNEL_AGENT", + "reason": "framework_agent_phase_done", + "ts": "2026-08-27T03:04:00+00:00", + "cycle": 0, + }, + ], + "framework_agent_batches": [ + { + "batch_id": "discovery-0", + "ts": "2026-08-27T03:02:00+00:00", + "cycle": 0, + "candidates": [ + { + "pr_url": "https://example.test/pr/12", + "route": "direct_framework", + } + ], + } + ], + } + + event = collect_v6_timeline(tmp_path, [], state=state, recorded_operations=[])[0] + runs = event["ext"]["source_arm"]["candidate_discovery_runs"] + + assert [run["status"] for run in runs] == ["failed", "succeeded", "empty"] + assert runs[0]["reason"] == "TimeoutError('upstream unavailable')" + assert runs[1]["batch_id"] == "discovery-0" + assert runs[1]["candidates"][0]["candidate_id"] == "https://example.test/pr/12" + assert runs[2]["reason"] == "discover_empty_payload" + assert event["status"] == "succeeded" + assert event["ext"]["failure"] == { + "failed_task_id": None, + "error_class": None, + "error": None, + } + + +def test_framework_timeline_marks_exhausted_discovery_retries_failed(tmp_path): + state = { + "phase": "KERNEL_AGENT", + "macro_cycle": 0, + "phase_history": [ + { + "from_phase": "PRELUDE", + "to_phase": "FRAMEWORK_AGENT", + "ts": "2026-08-27T03:00:00+00:00", + "cycle": 0, + }, + { + "from_phase": "FRAMEWORK_AGENT", + "to_phase": "FRAMEWORK_AGENT", + "reason": "framework_agent_discover_failed", + "evidence": { + "event": "framework_agent_discover_failed", + "attempt": 1, + "limit": 3, + "error": "TimeoutError('first')", + }, + "ts": "2026-08-27T03:01:00+00:00", + "cycle": 0, + }, + { + "from_phase": "FRAMEWORK_AGENT", + "to_phase": "FRAMEWORK_AGENT", + "reason": "framework_agent_discover_failed", + "evidence": { + "event": "framework_agent_discover_failed", + "attempt": 3, + "limit": 3, + "error": "TimeoutError('last')", + }, + "ts": "2026-08-27T03:02:00+00:00", + "cycle": 0, + }, + { + "from_phase": "FRAMEWORK_AGENT", + "to_phase": "FRAMEWORK_AGENT", + "reason": "discover_retries_exhausted", + "evidence": { + "event": "framework_agent_phase_done", + "failure_count": 3, + "retry_limit": 3, + }, + "ts": "2026-08-27T03:03:00+00:00", + "cycle": 0, + }, + { + "from_phase": "FRAMEWORK_AGENT", + "to_phase": "KERNEL_AGENT", + "reason": "framework_agent_phase_done", + "ts": "2026-08-27T03:04:00+00:00", + "cycle": 0, + }, + ], + } + + event = collect_v6_timeline(tmp_path, [], state=state, recorded_operations=[])[0] + runs = event["ext"]["source_arm"]["candidate_discovery_runs"] + + assert [run["status"] for run in runs] == ["failed", "failed"] + assert event["status"] == "failed" + assert event["ext"]["failure"] == { + "failed_task_id": None, + "error_class": None, + "error": "TimeoutError('last')", + } + + +def test_framework_timeline_assigns_critic_reviews_from_request_cycle(tmp_path): + state = { + "phase": "KERNEL_AGENT", + "macro_cycle": 1, + "phase_history": [ + { + "from_phase": "PRELUDE", + "to_phase": "FRAMEWORK_AGENT", + "ts": "2026-08-27T03:00:00+00:00", + "cycle": 0, + }, + { + "from_phase": "FRAMEWORK_AGENT", + "to_phase": "KERNEL_AGENT", + "ts": "2026-08-27T03:10:00+00:00", + "cycle": 0, + }, + { + "from_phase": "SWEEP", + "to_phase": "FRAMEWORK_AGENT", + "ts": "2026-08-27T04:00:00+00:00", + "cycle": 1, + }, + { + "from_phase": "FRAMEWORK_AGENT", + "to_phase": "KERNEL_AGENT", + "ts": "2026-08-27T04:10:00+00:00", + "cycle": 1, + }, + ], + } + for cycle in (0, 1): + proposal_id = f"proposal-cycle-{cycle}" + critic_dir = tmp_path / "critic-workdir" / f"{cycle:06d}" + _write_json( + critic_dir / "request.json", + { + "context": {"phase": "FRAMEWORK_AGENT"}, + "raw_prompt": ( + f"=== Shared session state ===\nmacro_cycle={cycle}\n" + if cycle == 0 + else "=== Shared session state ===\n" + ), + }, + ) + _write_json( + critic_dir / "judge_bundle.json", + { + "phase": "FRAMEWORK_AGENT", + "proposals": [ + { + "msg_id": proposal_id, + "action_name": "integrate_patch", + "payload": { + "framework_agent_candidate_id": f"candidate-{cycle}", + }, + } + ], + }, + ) + _write_json( + critic_dir / "review.json", + { + "review_verdicts": [ + { + "target_proposal_msg_id": proposal_id, + "verdict": "approve", + } + ] + }, + ) + _write_json( + critic_dir / "emit.json", + { + "intent_envelope": { + "intents": [ + { + "intent_type": "review_verdict", + "payload": { + "target_proposal_msg_id": proposal_id, + "verdict": "approve", + }, + } + ] + } + }, + ) + + _write_json( + tmp_path / "reports" / "trace" / "proposal_task_map.jsonl", + { + "proposal_msg_id": "proposal-cycle-1", + "task_id": "integrate-cycle-1", + }, + ) + operations = [ + { + "operation_id": "op-cycle-1", + "name": "integrate_patch", + "phase": "FRAMEWORK_AGENT", + "macro_cycle": 1, + "extensions": {"task_id": "integrate-cycle-1"}, + "outputs": {"status": "reverted"}, + } + ] + + timeline = collect_v6_timeline(tmp_path, [], state=state, recorded_operations=operations) + + assert [[review["proposal_msg_id"] for review in event["ext"]["critic_reviews"]] for event in timeline] == [ + ["proposal-cycle-0"], + ["proposal-cycle-1"], + ] diff --git a/src/hyperloom/inference_optimizer/tests/test_session_package.py b/src/hyperloom/inference_optimizer/tests/test_session_package.py index 47dbf18c62..38ce374b0b 100644 --- a/src/hyperloom/inference_optimizer/tests/test_session_package.py +++ b/src/hyperloom/inference_optimizer/tests/test_session_package.py @@ -40,6 +40,8 @@ def _build_session(sd: Path) -> None: _write(sd / "reports" / "optimization_journal.json", "[]") _write(sd / "reports" / "kernel_optimization_summary.json", "{}") _write(sd / "reports" / "kernel_roofline.json", "{}") + _write(sd / "reports" / "sbd_v6" / "install.json", "{}") + _write(sd / "reports" / "sbd_v6" / "model_gate.json", "{}") _write(sd / "reports" / "trace" / "decision_trace.jsonl", "{}\n") _write(sd / "reports" / "trace" / "llm_calls.jsonl", "{}\n") _write(sd / "target_analysis" / "target_baseline.json", "{}") @@ -102,6 +104,8 @@ def test_package_includes_curated_excludes_noise(tmp_path: Path) -> None: "reports/optimization_journal.json", "reports/kernel_optimization_summary.json", "reports/kernel_roofline.json", + "reports/sbd_v6/install.json", + "reports/sbd_v6/model_gate.json", "reports/trace/decision_trace.jsonl", "reports/trace/llm_calls.jsonl", "target_analysis/target_baseline.json", diff --git a/src/hyperloom/inference_optimizer/tests/test_specialist_lifecycle.py b/src/hyperloom/inference_optimizer/tests/test_specialist_lifecycle.py index dce7b51eaa..247421d2e6 100644 --- a/src/hyperloom/inference_optimizer/tests/test_specialist_lifecycle.py +++ b/src/hyperloom/inference_optimizer/tests/test_specialist_lifecycle.py @@ -259,7 +259,7 @@ async def test_build_specialist_round_entry_carries_full_payload(coord): coord_obj = Coordinator.__new__(Coordinator) task = _StubTask( task_id="t-build", - params={"round_id": "round-9"}, + params={"round_id": "round-9", "source_phase": "KERNEL_AGENT"}, ) payload = _done_payload( domain="serving_specialist", @@ -290,6 +290,7 @@ async def test_build_specialist_round_entry_carries_full_payload(coord): "confidence", "new_findings", "residual_questions", + "source_phase", } assert expected_keys.issubset(entry.keys()) assert entry["round_id"] == "round-9" @@ -297,6 +298,7 @@ async def test_build_specialist_round_entry_carries_full_payload(coord): assert entry["proposals_total"] == 2 assert entry["empty"] is False assert entry["confidence"] == 0.62 + assert entry["source_phase"] == "KERNEL_AGENT" @pytest.mark.asyncio diff --git a/src/hyperloom/orchestrator/enablement/params.py b/src/hyperloom/orchestrator/enablement/params.py index 0f393c5d9b..e6e0f3ddc2 100644 --- a/src/hyperloom/orchestrator/enablement/params.py +++ b/src/hyperloom/orchestrator/enablement/params.py @@ -265,6 +265,7 @@ def _build_enablement_specialist_params(self, launch_log: str, *, attempt: int = params_out: dict[str, Any] = { "domain": "enablement_specialist", + "source_phase": "ENABLEMENT", "gap_canonical_id": gap_cid, "gap_symptom": (f"{framework or '?'} cannot launch {model or 'the target model'}: {signature.kind}"), "gap_layer": "framework", diff --git a/src/hyperloom/orchestrator/phases/explore.py b/src/hyperloom/orchestrator/phases/explore.py index 1c03b192ae..727cd267b4 100644 --- a/src/hyperloom/orchestrator/phases/explore.py +++ b/src/hyperloom/orchestrator/phases/explore.py @@ -1868,7 +1868,18 @@ def _build_specialist_round_entry( proposals = done_payload.get("proposal_set") or [] if not isinstance(proposals, list): proposals = [] - round_id = str((task.params or {}).get("round_id") or task.task_id) + task_params = task.params or {} + round_id = str(task_params.get("round_id") or task.task_id) + source_phase = ( + str( + task_params.get("source_phase") + or done_payload.get("source_phase") + or getattr(getattr(self, "shared_state", None), "phase", "") + or "" + ) + .strip() + .upper() + ) from ..specialists.domains import normalize_dispatch_tags # Knowledge-domain tags; reported tags win over dispatch params. @@ -1892,6 +1903,8 @@ def _build_specialist_round_entry( "new_findings": list(done_payload.get("new_findings") or []), "residual_questions": list(done_payload.get("residual_questions") or []), } + if source_phase: + entry["source_phase"] = source_phase gpu_ids = done_payload.get("allocated_gpu_ids") or [] if isinstance(gpu_ids, list) and gpu_ids: entry["allocated_gpu_ids"] = [ diff --git a/src/hyperloom/orchestrator/phases/framework.py b/src/hyperloom/orchestrator/phases/framework.py index e975ab0cfe..adccd7bf51 100644 --- a/src/hyperloom/orchestrator/phases/framework.py +++ b/src/hyperloom/orchestrator/phases/framework.py @@ -359,6 +359,7 @@ async def _enqueue_framework_agent_authoring_specialist( "gap_layer": "framework", "framework": str(candidate.get("framework") or getattr(state, "framework", "") or "").strip().lower(), "task_kind": "framework_authoring", + "source_phase": "FRAMEWORK_AGENT", "pr_lead": {"title": title, "url": pr_url, "diff_url": diff_url}, "lever_kind": LEVER_UPSTREAM_PR, # Provenance markers for the dispatcher-side authored-patch bridge. diff --git a/src/hyperloom/orchestrator/phases/internal.py b/src/hyperloom/orchestrator/phases/internal.py index d5fc6c787a..b17d4d510b 100644 --- a/src/hyperloom/orchestrator/phases/internal.py +++ b/src/hyperloom/orchestrator/phases/internal.py @@ -47,6 +47,7 @@ async def _enqueue_internal_research_scout_task( ) params: dict[str, Any] = { "domain": "research_scout_specialist", + "source_phase": str(getattr(self.shared_state, "phase", "") or "PRELUDE").strip().upper(), "gap_canonical_id": f"gap.research_scout.round{int(round_id)}", "gap_symptom": ( "Collect proven priors (reference launch scripts, model " @@ -180,6 +181,7 @@ async def _enqueue_internal_static_recon_task( idempotency_key = "internal-static-recon-prelude" params: dict[str, Any] = { "domain": "static_recon_specialist", + "source_phase": str(getattr(state, "phase", "") or "PRELUDE").strip().upper(), "gap_canonical_id": "gap.static_recon.prelude", "gap_symptom": ( "Grep the framework source for un-bridged capability switches " @@ -287,6 +289,7 @@ async def _maybe_enqueue_trajectory_reviewer(self) -> None: domain = hint[0] if hint else "serving_specialist" params: dict[str, Any] = { "domain": domain, + "source_phase": str(getattr(state, "phase", "") or "INTERNAL").strip().upper(), "gap_canonical_id": f"gap.trajectory_review.cycle{cycle}", "gap_symptom": ( "The search has plateaued. Review the optimization trajectory "