diff --git a/src/hyperloom/common/gain_math.py b/src/hyperloom/common/gain_math.py index a3b7c391d..46bec20f5 100644 --- a/src/hyperloom/common/gain_math.py +++ b/src/hyperloom/common/gain_math.py @@ -1,7 +1,12 @@ # SPDX-FileCopyrightText: 2026 Advanced Micro Devices, Inc. # SPDX-License-Identifier: MIT -"""Throughput percentage-gain helpers (``gain_math``). Stdlib-only.""" +"""Throughput percentage-gain helpers (``gain_math``). + +Default paths stay stdlib-only so the report-time collector does not import +Magpie/torch. The optional ``use_composite`` branch of +:func:`conc_pair_comparison` lazily imports :mod:`hyperloom.common.perf_metric`. +""" from __future__ import annotations @@ -35,16 +40,22 @@ def incremental_gain_pct(new: float, ref: float) -> float | None: def conc_pair_comparison( baseline_points: list[dict[str, Any]], optimized_points: list[dict[str, Any]], + *, + use_composite: bool = False, ) -> tuple[list[dict[str, Any]], dict[str, Any]]: """Pair curve points by CONC (outer join), compute per-conc speedup, and aggregate. Shared by the conc-sweep post-hook and the breakdown collector, which must - produce byte-identical rows/summary from the same curves. Stdlib-only so - the collector never drags in Magpie/torch at report time. + produce byte-identical rows/summary from the same curves. Default ranking + is output-tput ratio and stays stdlib-only. When *use_composite* is true + and both arms have a full perf triple, speedup is ``1+S`` (``delta_pct`` + is ``S*100``) with per-pair output-tput fallback. Args: baseline_points: Curve rows for the baseline arm. optimized_points: Curve rows for the optimized arm. + use_composite: Rank on composite score *S* when both points have a + full triple. Default ``False`` (output-tput ratio). Returns: A tuple of ``(per_conc_rows, summary_dict)``. @@ -56,6 +67,14 @@ def _norm_conc(p: dict[str, Any]) -> int | float | str: return int(raw) return raw # type: ignore[return-value] + score_fn = None + snap_fn = None + if use_composite: + from hyperloom.common.perf_metric import composite_score, perf_snapshot_from_mapping + + score_fn = composite_score + snap_fn = perf_snapshot_from_mapping + by_conc_b = {_norm_conc(p): p for p in baseline_points} by_conc_o = {_norm_conc(p): p for p in optimized_points} rows: list[dict[str, Any]] = [] @@ -71,13 +90,25 @@ def _norm_conc(p: dict[str, Any]) -> int | float | str: ot = to_float(o.get("output_throughput")) speedup: float | None = None delta_pct: float | None = None - if bt is not None and bt > 0 and ot is not None and ot > 0: - speedup = ot / bt - delta_pct = (speedup - 1.0) * 100.0 - speedups.append(speedup) - successful_pairs += 1 - else: - failed_pairs += 1 + used_composite = False + if score_fn is not None and snap_fn is not None: + b_snap = snap_fn(b) + o_snap = snap_fn(o) + if b_snap is not None and o_snap is not None: + score = score_fn(o_snap, b_snap) + speedup = 1.0 + score + delta_pct = score * 100.0 + used_composite = True + speedups.append(speedup) + successful_pairs += 1 + if speedup is None: + if bt is not None and bt > 0 and ot is not None and ot > 0: + speedup = ot / bt + delta_pct = (speedup - 1.0) * 100.0 + speedups.append(speedup) + successful_pairs += 1 + else: + failed_pairs += 1 rows.append( { "conc": c, @@ -87,6 +118,7 @@ def _norm_conc(p: dict[str, Any]) -> int | float | str: "delta_pct": delta_pct, "baseline_status": b.get("status"), "optimized_status": o.get("status"), + "used_composite": used_composite, } ) summary: dict[str, Any] = { @@ -96,6 +128,7 @@ def _norm_conc(p: dict[str, Any]) -> int | float | str: "best_speedup": None, "median_speedup": None, "mean_speedup": None, + "metric": "composite_v1" if use_composite else "output_throughput", } if speedups: best_idx, best_val = max( diff --git a/src/hyperloom/common/perf_metric.py b/src/hyperloom/common/perf_metric.py new file mode 100644 index 000000000..2d92e7f39 --- /dev/null +++ b/src/hyperloom/common/perf_metric.py @@ -0,0 +1,295 @@ +# SPDX-FileCopyrightText: 2026 Advanced Micro Devices, Inc. +# SPDX-License-Identifier: MIT + +"""Composite performance metric (input tput + intvty p90 + output tput). + +Weighted improvement vs the baseline triple, gated behind +``HYPERLOOM_PERF_METRIC=composite_v1``. Noise floors default to 0 (raw +``Δ``); set ``HYPERLOOM_PERF_NOISE_PCT`` to subtract a band. Serving / +AgentX workloads only; scriptable frameworks keep output-tput grading. +""" + +from __future__ import annotations + +from typing import Any, Mapping + +from hyperloom.common.env import env_str + +COMPOSITE_V1 = "composite_v1" +_DEFAULT_WEIGHTS = (0.55, 0.30, 0.15) +_DEFAULT_NOISE_PCT = (0.0, 0.0, 0.0) + + +def composite_metric_enabled() -> bool: + """True when the composite grading flag is on.""" + return env_str("HYPERLOOM_PERF_METRIC").lower() == COMPOSITE_V1 + + +def composite_grading_enabled(framework: str | None = None) -> bool: + """Composite grading is opt-in and limited to non-scriptable serving runs.""" + if not composite_metric_enabled(): + return False + if not framework: + return True + from hyperloom.inference_optimizer import framework_registry + + return not framework_registry.is_scriptable(framework) + + +def _parse_triple(name: str, default: tuple[float, float, float]) -> tuple[float, float, float]: + raw = env_str(name) + if not raw: + return default + parts = [p.strip() for p in raw.split(",") if p.strip()] + if len(parts) != 3: + return default + out: list[float] = [] + for part in parts: + try: + out.append(float(part)) + except ValueError: + return default + return out[0], out[1], out[2] + + +def parse_weights() -> tuple[float, float, float]: + """Return ``(w_in, w_intv, w_out)`` from ``HYPERLOOM_PERF_WEIGHTS``.""" + return _parse_triple("HYPERLOOM_PERF_WEIGHTS", _DEFAULT_WEIGHTS) + + +def parse_noise_pct() -> tuple[float, float, float]: + """Return noise floors ``(in, intv, out)`` in percent from env (default 0).""" + return _parse_triple("HYPERLOOM_PERF_NOISE_PCT", _DEFAULT_NOISE_PCT) + + +def perf_snapshot_from_mapping(source: Mapping[str, Any] | None) -> dict[str, float] | None: + """Extract the perf triple when all three axes are positive.""" + if not isinstance(source, Mapping): + return None + out = source.get("output_throughput", source.get("tput")) + inp = source.get("input_throughput") + intv = source.get("intvty_p90") + if not all(isinstance(v, (int, float)) and float(v) > 0 for v in (out, inp, intv)): + return None + snap: dict[str, float] = { + "output_throughput": float(out), + "input_throughput": float(inp), + "intvty_p90": float(intv), + } + tpot = source.get("tpot_p90_ms") + if isinstance(tpot, (int, float)) and float(tpot) > 0: + snap["tpot_p90_ms"] = float(tpot) + return snap + + +def resolve_baseline_perf(state: Any) -> dict[str, float] | None: + """Read the session baseline perf triple from shared state.""" + raw = getattr(state, "baseline_perf", None) + if isinstance(raw, dict): + return perf_snapshot_from_mapping(raw) + return None + + +def delta_improvement(new: float, base: float) -> float: + """Fractional improvement (0 when not strictly better).""" + if base <= 0 or new <= 0: + return 0.0 + return max(0.0, (float(new) - float(base)) / float(base)) + + +def noise_adjusted_delta(delta: float, noise_pct: float) -> float: + """Subtract a noise floor (percent points) from a fractional delta.""" + return max(0.0, float(delta) - float(noise_pct) / 100.0) + + +def composite_score( + candidate: Mapping[str, float], + baseline: Mapping[str, float], + *, + weights: tuple[float, float, float] | None = None, + noise_pct: tuple[float, float, float] | None = None, +) -> float: + """Weighted noise-adjusted improvement vs baseline on all three axes.""" + w_in, w_intv, w_out = weights or parse_weights() + n_in, n_intv, n_out = noise_pct or parse_noise_pct() + d_in = noise_adjusted_delta( + delta_improvement(float(candidate["input_throughput"]), float(baseline["input_throughput"])), + n_in, + ) + d_intv = noise_adjusted_delta( + delta_improvement(float(candidate["intvty_p90"]), float(baseline["intvty_p90"])), + n_intv, + ) + d_out = noise_adjusted_delta( + delta_improvement(float(candidate["output_throughput"]), float(baseline["output_throughput"])), + n_out, + ) + return w_in * d_in + w_intv * d_intv + w_out * d_out + + +def score_gain_pct( + candidate: Mapping[str, float], + anchor: Mapping[str, float], + baseline: Mapping[str, float], +) -> float | None: + """Incremental composite-score gain of *candidate* over *anchor* (both vs *baseline*).""" + anchor_score = composite_score(anchor, baseline) + cand_score = composite_score(candidate, baseline) + if cand_score <= 0: + return None + if anchor_score <= 0: + return cand_score * 100.0 + return (cand_score - anchor_score) / anchor_score * 100.0 + + +def keep_gain_pct( + candidate: Mapping[str, Any] | None, + *, + state: Any = None, + framework: str | None = None, + base_tput: float | None = None, +) -> tuple[float | None, bool]: + """KEEP gain percent, using the composite score when the flag and triples are present. + + Returns: + ``(gain_pct, used_composite)``. Composite ``gain_pct`` is ``None`` when + *S* did not improve (same meaning as :func:`score_gain_pct`). Output-tput + fallback uses :func:`hyperloom.common.gain_math.gain_pct`. + """ + from hyperloom.common.gain_math import gain_pct as tput_gain_pct + + fw = framework or (getattr(state, "framework", None) if state is not None else None) + cand_snap = perf_snapshot_from_mapping(candidate) + baseline = resolve_baseline_perf(state) + if composite_grading_enabled(fw) and cand_snap and baseline: + anchor = perf_snapshot_from_mapping(getattr(state, "current_best", None) if state is not None else None) + return score_gain_pct(cand_snap, anchor or baseline, baseline), True + new_tput: float | None = None + if isinstance(candidate, Mapping): + raw = candidate.get("output_throughput", candidate.get("tput", candidate.get("new_tput"))) + if isinstance(raw, (int, float)): + new_tput = float(raw) + return tput_gain_pct(new_tput, float(base_tput or 0.0)), False + + +def session_gain_pct( + candidate: Mapping[str, Any] | None, + *, + state: Any = None, + framework: str | None = None, + base_tput: float | None = None, +) -> tuple[float | None, bool]: + """Session-total gain percent vs the session baseline (not vs ``current_best``). + + Composite ``gain_pct`` is ``S * 100`` (including ``0.0`` when no axis + improved). KEEP incremental grading is :func:`keep_gain_pct`. + """ + from hyperloom.common.gain_math import gain_pct as tput_gain_pct + + fw = framework or (getattr(state, "framework", None) if state is not None else None) + cand_snap = perf_snapshot_from_mapping(candidate) + baseline = resolve_baseline_perf(state) + if composite_grading_enabled(fw) and cand_snap and baseline: + return composite_score(cand_snap, baseline) * 100.0, True + new_tput: float | None = None + if isinstance(candidate, Mapping): + raw = candidate.get("output_throughput", candidate.get("tput", candidate.get("new_tput"))) + if isinstance(raw, (int, float)) and not isinstance(raw, bool): + new_tput = float(raw) + bt = base_tput + if bt is None and state is not None: + bt = getattr(state, "baseline_tput", None) + return tput_gain_pct(new_tput, float(bt or 0.0)), False + + +def session_gain_from_measurement( + new_tput: float, + *, + state: Any = None, + candidate: Mapping[str, Any] | None = None, + base_tput: float | None = None, +) -> tuple[float | None, bool]: + """Session gain for a measured output tput, filling axes from *candidate* then ``current_best``.""" + mapping: dict[str, Any] = {} + cb = getattr(state, "current_best", None) if state is not None else None + if isinstance(cb, Mapping): + mapping.update(cb) + if isinstance(candidate, Mapping): + mapping.update(candidate) + mapping["tput"] = float(new_tput) + mapping["output_throughput"] = float(new_tput) + return session_gain_pct(mapping, state=state, base_tput=base_tput) + + +def perf_axes_from_mapping(source: Mapping[str, Any] | None) -> dict[str, float]: + """Positive input / intvty / output / tpot fields for stack-lift payloads.""" + if not isinstance(source, Mapping): + return {} + out: dict[str, float] = {} + for key in ("input_throughput", "intvty_p90", "tpot_p90_ms", "output_throughput"): + val = source.get(key) + if isinstance(val, (int, float)) and float(val) > 0: + out[key] = float(val) + if "output_throughput" not in out: + tput = source.get("tput", source.get("new_tput")) + if isinstance(tput, (int, float)) and float(tput) > 0: + out["output_throughput"] = float(tput) + if "output_throughput" in out: + out["tput"] = out["output_throughput"] + return out + + +def resolve_grading_anchor_score(state: Any) -> float: + """Composite score of the config candidates are composed on (0 before any lift).""" + cb = getattr(state, "current_best", None) + baseline = resolve_baseline_perf(state) + if baseline and isinstance(cb, dict): + anchor = perf_snapshot_from_mapping(cb) + if anchor: + return composite_score(anchor, baseline) + return 0.0 + + +def session_composite_score(state: Any) -> float | None: + """Composite score *S* of ``current_best`` vs the session baseline, or None.""" + fw = getattr(state, "framework", None) if state is not None else None + if not composite_grading_enabled(fw): + return None + baseline = resolve_baseline_perf(state) + snap = perf_snapshot_from_mapping(getattr(state, "current_best", None) if state is not None else None) + if baseline is None or snap is None: + return None + return composite_score(snap, baseline) + + +def composite_watermark_levels(state: Any) -> tuple[float, float] | None: + """``(1+S_now, 1+S_last_snapshot)`` for the 10% roofline watermark, or None.""" + score = session_composite_score(state) + if score is None: + return None + last_s = getattr(state, "last_roofline_score", None) if state is not None else None + if not isinstance(last_s, (int, float)) or isinstance(last_s, bool): + last_s = 0.0 + return 1.0 + float(score), 1.0 + float(last_s) + + +__all__ = [ + "COMPOSITE_V1", + "composite_grading_enabled", + "composite_metric_enabled", + "composite_score", + "composite_watermark_levels", + "delta_improvement", + "keep_gain_pct", + "noise_adjusted_delta", + "parse_noise_pct", + "parse_weights", + "perf_axes_from_mapping", + "perf_snapshot_from_mapping", + "resolve_baseline_perf", + "resolve_grading_anchor_score", + "score_gain_pct", + "session_composite_score", + "session_gain_from_measurement", + "session_gain_pct", +] diff --git a/src/hyperloom/common/tests/test_perf_metric.py b/src/hyperloom/common/tests/test_perf_metric.py new file mode 100644 index 000000000..b243c9206 --- /dev/null +++ b/src/hyperloom/common/tests/test_perf_metric.py @@ -0,0 +1,173 @@ +# SPDX-FileCopyrightText: 2026 Advanced Micro Devices, Inc. +# SPDX-License-Identifier: MIT + +from __future__ import annotations + +import pytest + +from hyperloom.common.perf_metric import ( + composite_score, + noise_adjusted_delta, + perf_snapshot_from_mapping, + score_gain_pct, +) + +_BASELINE = { + "input_throughput": 24252.0, + "output_throughput": 170.0, + "intvty_p90": 745.19, +} + + +def test_default_noise_is_zero_uses_raw_delta(): + candidate = dict(_BASELINE) + candidate["output_throughput"] = _BASELINE["output_throughput"] * 1.02 + score = composite_score(candidate, _BASELINE) + assert abs(score - 0.15 * 0.02) < 1e-9 + + +def test_noise_floor_zeros_small_output_gain(): + candidate = dict(_BASELINE) + candidate["output_throughput"] = _BASELINE["output_throughput"] * 1.02 + score = composite_score(candidate, _BASELINE, noise_pct=(2.0, 2.0, 2.0)) + assert score < 1e-9 + + +def test_input_gain_dominates_output_noise(): + candidate = dict(_BASELINE) + candidate["input_throughput"] = _BASELINE["input_throughput"] * 1.10 + candidate["output_throughput"] = _BASELINE["output_throughput"] * 1.05 + score = composite_score( + candidate, + _BASELINE, + weights=(0.55, 0.30, 0.15), + noise_pct=(2.0, 2.0, 2.0), + ) + assert score > 0.04 + + +def test_score_gain_pct_incremental(): + anchor = dict(_BASELINE) + candidate = dict(_BASELINE) + candidate["input_throughput"] *= 1.10 + gain = score_gain_pct(candidate, anchor, _BASELINE) + assert gain is not None + assert gain > 0.0 + + +def test_keep_gain_pct_falls_back_to_output_tput(): + from hyperloom.common.perf_metric import keep_gain_pct + + gain, used = keep_gain_pct( + {"output_throughput": 110.0}, + base_tput=100.0, + ) + assert used is False + assert gain == pytest.approx(10.0) + + +def test_keep_gain_pct_uses_composite_when_flag_on(monkeypatch): + from types import SimpleNamespace + + from hyperloom.common.perf_metric import keep_gain_pct + + monkeypatch.setenv("HYPERLOOM_PERF_METRIC", "composite_v1") + state = SimpleNamespace( + framework="sglang", + baseline_perf=dict(_BASELINE), + current_best=dict(_BASELINE), + ) + candidate = dict(_BASELINE) + candidate["input_throughput"] = _BASELINE["input_throughput"] * 1.20 + gain, used = keep_gain_pct(candidate, state=state, base_tput=_BASELINE["output_throughput"]) + assert used is True + assert gain is not None + assert gain > 1.0 + + +def test_perf_snapshot_requires_all_axes(): + assert perf_snapshot_from_mapping({"output_throughput": 1.0}) is None + assert perf_snapshot_from_mapping(_BASELINE) == _BASELINE + + +def test_noise_adjusted_delta(): + assert abs(noise_adjusted_delta(0.05, 2.0) - 0.03) < 1e-9 + + +def test_session_gain_pct_falls_back_to_output_tput(): + from hyperloom.common.perf_metric import session_gain_pct + + gain, used = session_gain_pct({"output_throughput": 110.0}, base_tput=100.0) + assert used is False + assert gain == pytest.approx(10.0) + + +def test_session_gain_pct_uses_score_vs_baseline(monkeypatch): + from types import SimpleNamespace + + from hyperloom.common.perf_metric import session_gain_pct + + monkeypatch.setenv("HYPERLOOM_PERF_METRIC", "composite_v1") + state = SimpleNamespace(framework="sglang", baseline_perf=dict(_BASELINE), current_best=dict(_BASELINE)) + candidate = dict(_BASELINE) + candidate["input_throughput"] = _BASELINE["input_throughput"] * 1.20 + gain, used = session_gain_pct(candidate, state=state, base_tput=_BASELINE["output_throughput"]) + assert used is True + assert gain == pytest.approx(11.0) + + +def test_session_gain_from_measurement_reads_current_best_axes(monkeypatch): + from types import SimpleNamespace + + from hyperloom.common.perf_metric import session_gain_from_measurement + + monkeypatch.setenv("HYPERLOOM_PERF_METRIC", "composite_v1") + state = SimpleNamespace( + framework="sglang", + baseline_tput=_BASELINE["output_throughput"], + baseline_perf=dict(_BASELINE), + current_best={ + "tput": _BASELINE["output_throughput"], + "input_throughput": _BASELINE["input_throughput"] * 1.20, + "intvty_p90": _BASELINE["intvty_p90"], + }, + ) + gain, used = session_gain_from_measurement( + _BASELINE["output_throughput"], + state=state, + base_tput=_BASELINE["output_throughput"], + ) + assert used is True + assert gain == pytest.approx(11.0) + + +def test_session_composite_score_none_when_flag_off(): + from types import SimpleNamespace + + from hyperloom.common.perf_metric import session_composite_score + + state = SimpleNamespace(framework="sglang", baseline_perf=dict(_BASELINE), current_best=dict(_BASELINE)) + assert session_composite_score(state) is None + + +def test_composite_watermark_levels_treat_missing_last_score_as_zero(monkeypatch): + from types import SimpleNamespace + + from hyperloom.common.perf_metric import composite_watermark_levels, session_composite_score + + monkeypatch.setenv("HYPERLOOM_PERF_METRIC", "composite_v1") + candidate = dict(_BASELINE) + candidate["input_throughput"] = _BASELINE["input_throughput"] * 1.20 + state = SimpleNamespace( + framework="sglang", + baseline_perf=dict(_BASELINE), + current_best=candidate, + last_roofline_score=None, + ) + assert session_composite_score(state) == pytest.approx(0.11) + cur, last = composite_watermark_levels(state) + assert cur == pytest.approx(1.11) + assert last == pytest.approx(1.0) + state.last_roofline_score = 0.11 + cur, last = composite_watermark_levels(state) + assert cur / last == pytest.approx(1.0) diff --git a/src/hyperloom/inference_optimizer/agentx/mapping.py b/src/hyperloom/inference_optimizer/agentx/mapping.py index b7bba8162..ad46c1295 100644 --- a/src/hyperloom/inference_optimizer/agentx/mapping.py +++ b/src/hyperloom/inference_optimizer/agentx/mapping.py @@ -90,9 +90,12 @@ def map_aiperf( rc = int(stat(m, "request_count") or 0) isl = stat(m, "input_sequence_length") + intvty_p90 = stat(m, "output_token_throughput_per_user", "p90", default=0.0) + return { "request_throughput": stat(m, "request_throughput"), "output_throughput": out_tput, + "input_throughput": in_tput, "total_token_throughput": total_tput, "completed": rc, "total_input_tokens": int(stat(m, "total_isl") or (isl * max(1, rc)) or 0), @@ -104,8 +107,10 @@ def map_aiperf( "std_ttft_ms": stat(m, "time_to_first_token", "std"), "mean_tpot_ms": stat(m, "inter_token_latency", "avg"), "median_tpot_ms": stat(m, "inter_token_latency", "p50"), + "p90_tpot_ms": stat(m, "inter_token_latency", "p90"), "p99_tpot_ms": stat(m, "inter_token_latency", "p99"), "std_tpot_ms": stat(m, "inter_token_latency", "std"), + "intvty_p90_tok_s_user": intvty_p90, "mean_itl_ms": stat(m, "inter_token_latency", "avg"), "median_itl_ms": stat(m, "inter_token_latency", "p50"), "p99_itl_ms": stat(m, "inter_token_latency", "p99"), diff --git a/src/hyperloom/inference_optimizer/assets/agentx/map_aiperf.py b/src/hyperloom/inference_optimizer/assets/agentx/map_aiperf.py index 180b0136e..d3ca51c59 100755 --- a/src/hyperloom/inference_optimizer/assets/agentx/map_aiperf.py +++ b/src/hyperloom/inference_optimizer/assets/agentx/map_aiperf.py @@ -59,9 +59,11 @@ def map_aiperf(export, *, noncanonical_reasons=None): total_tput = _stat(m, "total_token_throughput") or ((in_tput or 0) + (out_tput or 0)) rc = int(_stat(m, "request_count") or 0) isl = _stat(m, "input_sequence_length") + intvty_p90 = _stat(m, "output_token_throughput_per_user", "p90", default=0.0) return { "request_throughput": _stat(m, "request_throughput"), "output_throughput": out_tput, + "input_throughput": in_tput, "total_token_throughput": total_tput, "completed": rc, "total_input_tokens": int(_stat(m, "total_isl") or (isl * max(1, rc)) or 0), @@ -73,8 +75,10 @@ def map_aiperf(export, *, noncanonical_reasons=None): "std_ttft_ms": _stat(m, "time_to_first_token", "std"), "mean_tpot_ms": _stat(m, "inter_token_latency", "avg"), "median_tpot_ms": _stat(m, "inter_token_latency", "p50"), + "p90_tpot_ms": _stat(m, "inter_token_latency", "p90"), "p99_tpot_ms": _stat(m, "inter_token_latency", "p99"), "std_tpot_ms": _stat(m, "inter_token_latency", "std"), + "intvty_p90_tok_s_user": intvty_p90, "mean_itl_ms": _stat(m, "inter_token_latency", "avg"), "median_itl_ms": _stat(m, "inter_token_latency", "p50"), "p99_itl_ms": _stat(m, "inter_token_latency", "p99"), diff --git a/src/hyperloom/inference_optimizer/breakdown/collectors/kernels.py b/src/hyperloom/inference_optimizer/breakdown/collectors/kernels.py index 8fff06fb9..0621eacab 100644 --- a/src/hyperloom/inference_optimizer/breakdown/collectors/kernels.py +++ b/src/hyperloom/inference_optimizer/breakdown/collectors/kernels.py @@ -1024,6 +1024,10 @@ def _load_conc_variant_point(variant_dir: Path, *, arm: str, conc: int) -> dict[ "output_throughput": tput_f, "request_throughput": data.get("request_throughput"), "total_token_throughput": data.get("total_token_throughput"), + "input_throughput": _to_float(data.get("input_throughput")), + "intvty_p90": _to_float(data.get("intvty_p90")) + or _to_float(data.get("intvty_p90_tok_s_user")), + "tpot_p90_ms": _to_float(data.get("tpot_p90_ms")), "raw_result_path": _rel( result_path, variant_dir.parents[2] if len(variant_dir.parents) >= 3 else variant_dir, @@ -1042,6 +1046,9 @@ def _recover_conc_sweep_summary_from_runs( warnings: list[str], ) -> dict[str, Any]: """Recover a conc_sweep summary from raw run workspaces when the report is stale.""" + from hyperloom.common.perf_metric import composite_grading_enabled + + use_composite = composite_grading_enabled() runs_dir = session_dir / "runs" / "conc_sweep" if not runs_dir.exists(): return {} @@ -1072,7 +1079,9 @@ def _recover_conc_sweep_summary_from_runs( continue baseline_points.sort(key=lambda p: p["conc"]) optimized_points.sort(key=lambda p: p["conc"]) - comparison, summary = conc_pair_comparison(baseline_points, optimized_points) + comparison, summary = conc_pair_comparison( + baseline_points, optimized_points, use_composite=use_composite + ) pairs = int(summary.get("successful_pairs") or 0) payload = { "schema_version": "recovered-v1", diff --git a/src/hyperloom/inference_optimizer/tests/test_conc_sweep.py b/src/hyperloom/inference_optimizer/tests/test_conc_sweep.py index 831ced6ef..49c7e10a6 100644 --- a/src/hyperloom/inference_optimizer/tests/test_conc_sweep.py +++ b/src/hyperloom/inference_optimizer/tests/test_conc_sweep.py @@ -176,6 +176,72 @@ def test_build_comparison_mismatched_concs_outer_join(): assert summary["successful_pairs"] == 1 +def test_build_comparison_flag_off_ignores_input_lift(): + """Default ranking stays output-tput even when triples are present.""" + baseline = [ + { + "conc": 1, + "output_throughput": 100.0, + "input_throughput": 1000.0, + "intvty_p90": 50.0, + "status": "succeeded", + } + ] + optimized = [ + { + "conc": 1, + "output_throughput": 100.0, + "input_throughput": 1200.0, + "intvty_p90": 50.0, + "status": "succeeded", + } + ] + rows, summary = _build_comparison(baseline, optimized) + assert rows[0]["speedup"] == pytest.approx(1.0) + assert rows[0]["used_composite"] is False + assert summary["metric"] == "output_throughput" + + +def test_build_comparison_composite_uses_score(): + """Flag-on pairing: flat output + 20% input lift is S=0.11 (speedup 1.11).""" + baseline = [ + { + "conc": 1, + "output_throughput": 100.0, + "input_throughput": 1000.0, + "intvty_p90": 50.0, + "status": "succeeded", + } + ] + optimized = [ + { + "conc": 1, + "output_throughput": 100.0, + "input_throughput": 1200.0, + "intvty_p90": 50.0, + "status": "succeeded", + } + ] + rows, summary = _build_comparison(baseline, optimized, use_composite=True) + assert rows[0]["speedup"] == pytest.approx(1.11) + assert rows[0]["delta_pct"] == pytest.approx(11.0) + assert rows[0]["used_composite"] is True + assert summary["metric"] == "composite_v1" + assert summary["best_speedup"] == pytest.approx(1.11) + + +def test_build_comparison_composite_falls_back_without_triple(): + """Missing triple on either arm falls back to output-tput ratio.""" + rows, summary = _build_comparison( + [{"conc": 1, "output_throughput": 100.0, "status": "succeeded"}], + [{"conc": 1, "output_throughput": 130.0, "status": "succeeded"}], + use_composite=True, + ) + assert rows[0]["speedup"] == pytest.approx(1.30) + assert rows[0]["used_composite"] is False + assert summary["metric"] == "composite_v1" + + # Skip paths @pytest.mark.parametrize( "override, reason", @@ -360,6 +426,58 @@ async def _fake_run_grid(*, grid: list[GridVariant], **_kw): assert not final_json_path.exists() +def test_run_conc_sweep_composite_uses_score( + session_dir: Path, + baseline_yaml: Path, + monkeypatch, +): + """Flag on + full triples: per-conc ranking uses *S*, not output-tput ratio.""" + monkeypatch.setenv("HYPERLOOM_PERF_METRIC", "composite_v1") + state = _make_state(baseline_config_path=str(baseline_yaml)) + state.framework = "sglang" + + async def _fake_run_grid(*, grid: list[GridVariant], **_kw): + out: list[VariantResult] = [] + for v in grid: + vr = _fake_variant(v.name, throughput=100.0, envs=v.extra_envs) + if v.name.startswith("baseline_"): + vr.input_throughput = 1000.0 + vr.intvty_p90 = 50.0 + else: + vr.input_throughput = 1200.0 + vr.intvty_p90 = 50.0 + out.append(vr) + return out + + with ( + patch( + "hyperloom.orchestrator.kernel.conc_sweep.run_grid", + side_effect=_fake_run_grid, + ), + patch( + "hyperloom.orchestrator.kernel.conc_sweep.materialize_config_with_envs", + side_effect=_fake_materialize, + ), + ): + payload = asyncio.run( + run_conc_sweep( + state, + session_dir, + concs=[1, 4], + ) + ) + + assert payload["status"] == "succeeded" + assert payload["summary"]["metric"] == "composite_v1" + assert payload["summary"]["best_speedup"] == pytest.approx(1.11) + for row in payload["comparison"]: + assert row["used_composite"] is True + assert row["speedup"] == pytest.approx(1.11) + assert row["delta_pct"] == pytest.approx(11.0) + assert row["baseline_tput"] == pytest.approx(100.0) + assert row["optimized_tput"] == pytest.approx(100.0) + + def test_run_conc_sweep_canonicalizes_gpu_type_to_runner( session_dir: Path, baseline_yaml: Path, diff --git a/src/hyperloom/inference_optimizer/tests/test_coordinator_sync_helpers_coverage_unit.py b/src/hyperloom/inference_optimizer/tests/test_coordinator_sync_helpers_coverage_unit.py index 4de81960e..da22e9b50 100644 --- a/src/hyperloom/inference_optimizer/tests/test_coordinator_sync_helpers_coverage_unit.py +++ b/src/hyperloom/inference_optimizer/tests/test_coordinator_sync_helpers_coverage_unit.py @@ -281,6 +281,71 @@ def test_current_tput_from_validated_gain(coord: Coordinator) -> None: assert coord._current_tput_from_validated_gain() == pytest.approx(110.0) +def test_current_tput_from_validated_gain_prefers_current_best(coord: Coordinator) -> None: + coord.shared_state.baseline_tput = 100.0 + coord.shared_state.cumulative_gain_validated = 50.0 + coord.shared_state.current_best = {"tput": 105.0} + assert coord._current_tput_from_validated_gain() == pytest.approx(105.0) + + +def test_current_tput_from_validated_gain_does_not_invert_composite( + coord: Coordinator, monkeypatch +) -> None: + monkeypatch.setenv("HYPERLOOM_PERF_METRIC", "composite_v1") + coord.shared_state.framework = "sglang" + coord.shared_state.baseline_tput = 100.0 + coord.shared_state.cumulative_gain_validated = 11.0 + coord.shared_state.current_best = {} + assert coord._current_tput_from_validated_gain() == 0.0 + + +def test_update_cumulative_gain_validated_uses_composite(coord: Coordinator, monkeypatch) -> None: + monkeypatch.setenv("HYPERLOOM_PERF_METRIC", "composite_v1") + ss = coord.shared_state + ss.framework = "sglang" + ss.baseline_tput = 100.0 + ss.baseline_perf = { + "output_throughput": 100.0, + "input_throughput": 1000.0, + "intvty_p90": 50.0, + } + ss.current_best = { + "tput": 100.0, + "output_throughput": 100.0, + "input_throughput": 1200.0, + "intvty_p90": 50.0, + } + ss.optimization_stack = [{}] + coord._update_cumulative_gain_validated(100.0) + assert ss.cumulative_gain_validated == pytest.approx(11.0) + + +def test_append_stack_gain_entry_uses_composite(monkeypatch) -> None: + from hyperloom.orchestrator.state.shared_state import SharedState + + monkeypatch.setenv("HYPERLOOM_PERF_METRIC", "composite_v1") + s = SharedState() + s.framework = "sglang" + s.baseline_tput = 100.0 + s.baseline_perf = { + "output_throughput": 100.0, + "input_throughput": 1000.0, + "intvty_p90": 50.0, + } + gain = s.append_stack_gain_entry( + action="explore", + variant_name="v1", + new_tput=100.0, + candidate={ + "output_throughput": 100.0, + "input_throughput": 1200.0, + "intvty_p90": 50.0, + }, + ) + assert gain == pytest.approx(11.0) + assert s.gain_per_stack_entry == [pytest.approx(11.0)] + + def test_needs_roofline_for_watermark_guards(coord: Coordinator) -> None: ss = coord.shared_state # pending roofline -> never re-arm @@ -298,6 +363,88 @@ def test_needs_roofline_for_watermark_guards(coord: Coordinator) -> None: assert coord._needs_roofline_for_watermark() is True +def _composite_watermark_state(ss, *, input_scale: float = 1.0, output_scale: float = 1.0): + ss.framework = "sglang" + ss.last_roofline_tput = 100.0 + ss.last_roofline_score = None + ss.baseline_tput = 100.0 + ss.baseline_perf = { + "output_throughput": 100.0, + "input_throughput": 1000.0, + "intvty_p90": 50.0, + } + ss.current_best = { + "tput": 100.0 * output_scale, + "output_throughput": 100.0 * output_scale, + "input_throughput": 1000.0 * input_scale, + "intvty_p90": 50.0, + } + + +def test_needs_roofline_for_watermark_composite_input_step( + coord: Coordinator, monkeypatch +) -> None: + monkeypatch.setenv("HYPERLOOM_PERF_METRIC", "composite_v1") + ss = coord.shared_state + _composite_watermark_state(ss, input_scale=1.20) + assert coord._needs_roofline_for_watermark() is True + ss.last_roofline_score = 0.11 + assert coord._needs_roofline_for_watermark() is False + + +def test_needs_roofline_for_watermark_composite_does_not_invert_s( + coord: Coordinator, monkeypatch +) -> None: + monkeypatch.setenv("HYPERLOOM_PERF_METRIC", "composite_v1") + ss = coord.shared_state + _composite_watermark_state(ss) + ss.cumulative_gain_validated = 50.0 + assert coord._needs_roofline_for_watermark() is False + + +def test_needs_roofline_for_watermark_composite_output_only_does_not_fire( + coord: Coordinator, monkeypatch +) -> None: + monkeypatch.setenv("HYPERLOOM_PERF_METRIC", "composite_v1") + ss = coord.shared_state + _composite_watermark_state(ss, output_scale=1.20) + assert coord._needs_roofline_for_watermark() is False + + +def test_needs_roofline_for_watermark_composite_missing_triple_uses_tput( + coord: Coordinator, monkeypatch +) -> None: + monkeypatch.setenv("HYPERLOOM_PERF_METRIC", "composite_v1") + ss = coord.shared_state + ss.framework = "sglang" + ss.last_roofline_tput = 100.0 + ss.baseline_tput = 100.0 + ss.current_best = {"tput": 120.0} + assert coord._needs_roofline_for_watermark() is True + + +def test_stamp_roofline_watermark_records_composite_score(monkeypatch) -> None: + from hyperloom.orchestrator.state.shared_state import SharedState + + monkeypatch.setenv("HYPERLOOM_PERF_METRIC", "composite_v1") + s = SharedState() + s.framework = "sglang" + s.baseline_perf = { + "output_throughput": 100.0, + "input_throughput": 1000.0, + "intvty_p90": 50.0, + } + s.current_best = { + "tput": 100.0, + "output_throughput": 100.0, + "input_throughput": 1200.0, + "intvty_p90": 50.0, + } + s.stamp_roofline_watermark(100.0) + assert s.last_roofline_tput == 100.0 + assert s.last_roofline_score == pytest.approx(0.11) + + # -- gap extraction -------------------------------------------------------- def test_extract_gaps_from_baseline_empty(coord: Coordinator) -> None: coord.shared_state.baseline_tput = 0.0 diff --git a/src/hyperloom/inference_optimizer/tests/test_explore_executor.py b/src/hyperloom/inference_optimizer/tests/test_explore_executor.py index 0ce4ae3fc..85376fbb2 100644 --- a/src/hyperloom/inference_optimizer/tests/test_explore_executor.py +++ b/src/hyperloom/inference_optimizer/tests/test_explore_executor.py @@ -97,7 +97,13 @@ def _write_baseline_yaml(path: Path) -> None: yaml.safe_dump(cfg, f) -def _fake_workspace(slot: Path, *, tput: float = 800.0) -> Path: +def _fake_workspace( + slot: Path, + *, + tput: float = 800.0, + input_throughput: float | None = None, + intvty_p90: float | None = None, +) -> Path: workspace = slot / "benchmark_sglang_20260519_001122" workspace.mkdir(parents=True) (workspace / "benchmark_report.json").write_text( @@ -120,6 +126,16 @@ def _fake_workspace(slot: Path, *, tput: float = 800.0) -> Path: } ) ) + if input_throughput is not None or intvty_p90 is not None: + (workspace / "inferencex_result.json").write_text( + json.dumps( + { + "output_throughput": tput, + "input_throughput": input_throughput, + "intvty_p90_tok_s_user": intvty_p90, + } + ) + ) return workspace @@ -1437,6 +1453,68 @@ def _fake_run(cmd, *args, **kwargs): assert "stack_unstable" in rejected_reasons +@pytest.mark.asyncio +async def test_explore_stack_rebench_confirms_composite_keep( + sub_agent_runner, + tmp_path, + monkeypatch, +): + """Flag on: round-2 0.5% floor is *S*, so a flat-output input lift stays KEEP.""" + monkeypatch.setenv("HYPERLOOM_PERF_METRIC", "composite_v1") + sub, tr, _ = sub_agent_runner + baseline_perf = { + "output_throughput": 800.0, + "input_throughput": 10000.0, + "intvty_p90": 700.0, + } + state = SharedState() + state.framework = "sglang" + state.baseline_tput = 800.0 + state.baseline_perf = dict(baseline_perf) + state.current_best = {"action": "baseline", "tput": 800.0, **baseline_perf} + sub.shared_state = state + base = tmp_path / "base.yaml" + _write_baseline_yaml(base) + + def _fake_run(cmd, *args, **kwargs): + out_idx = cmd.index("--output-dir") + slot = Path(cmd[out_idx + 1]) + _fake_workspace(slot, tput=800.0, input_throughput=12000.0, intvty_p90=700.0) + return subprocess.CompletedProcess(args=cmd, returncode=0, stdout="ok", stderr="") + + task = await tr.create( + kind="explore", + params={ + "config_path": str(base), + "output_dir": str(tmp_path / "explore-composite-rebench"), + "base_tput": 800.0, + "grid": [ + { + "name": "input_lift", + "extra_args": "--input-lift", + "extra_envs": {}, + "provenance": "llm_direct", + } + ], + "variant_timeout_sec": 10, + "stack_stable_threshold_pct": 0.5, + }, + idempotency_key="ex-composite-rebench", + ) + sub.register_executor("explore", ExploreExecutor(session_dir=tmp_path)) + with patch( + "hyperloom.orchestrator.actions.executors._grid_runner.run_with_session_kill", + side_effect=_fake_run, + ): + res = await sub.run_task(task) + + out = res.result + assert {w["name"] for w in out["winners"]} == {"input_lift"} + assert out["keep_unstable_in_stack"] == [] + assert out["winners"][0]["gain_pct"] > 1.0 + assert out["winners"][0]["tput"] == 800.0 + + def test_default_keep_and_stack_stable_thresholds(): """Pin the KEEP gate (1.0%) and the lower stack-rebench stability floor (0.5%).""" from hyperloom.orchestrator.actions.executors.explore import ( diff --git a/src/hyperloom/inference_optimizer/tests/test_framework_agent_executor.py b/src/hyperloom/inference_optimizer/tests/test_framework_agent_executor.py index 6fc6b2e14..a8436352f 100644 --- a/src/hyperloom/inference_optimizer/tests/test_framework_agent_executor.py +++ b/src/hyperloom/inference_optimizer/tests/test_framework_agent_executor.py @@ -360,6 +360,67 @@ async def fake_bench(self, *, params, output_root, slug, **_kwargs): # noqa: AR assert (repo / "src.py").read_text().endswith("return 2\n") +@pytest.mark.asyncio +async def test_executor_keep_uses_composite_metric(tmp_path: Path, monkeypatch): + """Flag on: input-only lift KEEPs even when output tput is flat.""" + from types import SimpleNamespace + + monkeypatch.setenv("HYPERLOOM_PERF_METRIC", "composite_v1") + session_dir = tmp_path / "session" + session_dir.mkdir() + repo = tmp_path / "framework" + init_git_repo(repo) + patch_path = tmp_path / "p.patch" + patch_path.write_text(_VALID_PATCH, encoding="utf-8") + + baseline_perf = { + "output_throughput": 1000.0, + "input_throughput": 10000.0, + "intvty_p90": 700.0, + } + state = SimpleNamespace( + framework="sglang", + baseline_tput=1000.0, + baseline_perf=dict(baseline_perf), + current_best={"action": "baseline", "tput": 1000.0, **baseline_perf}, + baseline_accuracy=0.0, + ) + + executor = FrameworkAgentExecutor(session_dir=session_dir) + + async def fake_bench(self, *, params, output_root, slug, **_kwargs): # noqa: ARG001 + return ( + { + "status": "succeeded", + "output_throughput": 1000.0, + "input_throughput": 12000.0, + "intvty_p90": 700.0, + }, + {"accuracy_pass": None}, + ) + + ctx = _make_ctx( + "t-fp-composite", + { + "candidate": _make_candidate(), + "patches": [str(patch_path)], + "framework_source_root": str(repo), + "base_tput": 1000.0, + "keep_threshold_pct": 1.0, + "framework": "sglang", + }, + extra={"shared_state": state}, + ) + with patch.object(FrameworkAgentExecutor, "_bench_candidate", new=fake_bench): + result = await executor(ctx) + + assert result["status"] == "kept" + assert result["output_throughput"] == 1000.0 + assert result["delta_pct"] > 1.0 + assert result["input_throughput"] == pytest.approx(12000.0) + assert "composite gain" in result["reason"] + + @pytest.mark.asyncio async def test_bench_is_bounded_by_the_session_budget(tmp_path: Path): """The candidate bench is handed the session budget, as the other arms are. diff --git a/src/hyperloom/inference_optimizer/tests/test_integrate_patch_coverage_unit.py b/src/hyperloom/inference_optimizer/tests/test_integrate_patch_coverage_unit.py index 9ceb7ab26..4ddabbfee 100644 --- a/src/hyperloom/inference_optimizer/tests/test_integrate_patch_coverage_unit.py +++ b/src/hyperloom/inference_optimizer/tests/test_integrate_patch_coverage_unit.py @@ -421,6 +421,70 @@ async def test_keep_path(tmp_path, monkeypatch): assert (repo / "src.py").read_text().endswith("return 2\n") +@pytest.mark.asyncio +async def test_keep_path_uses_composite_metric(tmp_path, monkeypatch): + """Flag on: input-only lift KEEPs even when output tput is flat.""" + monkeypatch.setenv("HYPERLOOM_PERF_METRIC", "composite_v1") + session = tmp_path / "s" + session.mkdir() + repo = tmp_path / "fw" + _init_git_repo(repo) + _write_workspace(session, "spec") + + class _SS: + framework = "sglang" + baseline_tput = 100.0 + baseline_perf = { + "output_throughput": 100.0, + "input_throughput": 10000.0, + "intvty_p90": 700.0, + } + current_best = { + "action": "baseline", + "tput": 100.0, + "output_throughput": 100.0, + "input_throughput": 10000.0, + "intvty_p90": 700.0, + } + baseline_accuracy = 0.0 + + def get_specialist_patch_verdict(self, tid): + return "approve" + + ex = IntegratePatchExecutor(session_dir=session) + monkeypatch.setattr( + IntegratePatchExecutor, + "_bench_patch", + _stub_bench( + { + "output_throughput": 100.0, + "input_throughput": 12000.0, + "intvty_p90": 700.0, + "status": "succeeded", + }, + {"accuracy_pass": None}, + ), + ) + res = await ex( + _make_ctx( + "t", + { + "specialist_task_id": "spec", + "framework_source_root": str(repo), + "base_tput": 100.0, + "enable_stack_rebench": False, + "framework": "sglang", + }, + extra={"shared_state": _SS()}, + ) + ) + assert res["status"] == "kept" + assert res["output_throughput"] == 100.0 + assert res["delta_pct"] > 1.0 + assert res["input_throughput"] == pytest.approx(12000.0) + assert "composite gain" in res["reason"] + + def _stub_confirm(result: dict): async def _c(self, **kwargs): return result @@ -528,6 +592,54 @@ async def _fake_run_grid(**_kwargs): assert not any("stack_rebench_failed" in w for w in result.warnings) +@pytest.mark.asyncio +async def test_measure_stack_rebench_composite_floor_ignores_flat_tput(tmp_path, monkeypatch): + """Flag on: +20% input / flat output clears the 0.5% *S* floor.""" + monkeypatch.setenv("HYPERLOOM_PERF_METRIC", "composite_v1") + from unittest.mock import patch + + from hyperloom.orchestrator.actions.executors._grid_runner import GridVariant, VariantResult + from hyperloom.orchestrator.actions.executors import _stack_rebench as sr + + baseline = { + "output_throughput": 800.0, + "input_throughput": 10000.0, + "intvty_p90": 700.0, + } + measured = VariantResult( + name="v", + extra_server_args="", + extra_envs={}, + status="succeeded", + output_throughput=800.0, + input_throughput=12000.0, + intvty_p90=700.0, + workspace=str(tmp_path / "ws"), + ) + + async def _fake_run_grid(**_kwargs): + return [measured] + + with patch.object(sr, "run_grid", new=_fake_run_grid): + result = await sr.measure_stack_rebench( + config_path=tmp_path / "base.yaml", + base_extra_args="", + variant=GridVariant("v"), + base_tput=800.0, + stable_threshold_pct=0.5, + output_slot=tmp_path / "slot", + variant_timeout_sec=600, + framework="sglang", + anchor_perf=baseline, + baseline_perf=baseline, + ) + + assert result.used_composite is True + assert result.stable is True + assert result.tput == 800.0 + assert result.stable_gain_pct is not None and result.stable_gain_pct > 0.5 + + @pytest.mark.asyncio async def test_keep_confirmed_by_rebench(tmp_path, monkeypatch): session = tmp_path / "s" diff --git a/src/hyperloom/inference_optimizer/tests/test_kernel_integrate_and_report.py b/src/hyperloom/inference_optimizer/tests/test_kernel_integrate_and_report.py index 9168f10f1..3d6b81472 100644 --- a/src/hyperloom/inference_optimizer/tests/test_kernel_integrate_and_report.py +++ b/src/hyperloom/inference_optimizer/tests/test_kernel_integrate_and_report.py @@ -86,7 +86,14 @@ def _write_baseline_yaml(path: Path) -> None: yaml.safe_dump(cfg, f) -def _fake_workspace(slot: Path, *, tput: float = 800.0, accuracy: float | None = None) -> Path: +def _fake_workspace( + slot: Path, + *, + tput: float = 800.0, + accuracy: float | None = None, + input_throughput: float | None = None, + intvty_p90: float | None = None, +) -> Path: workspace = slot / "benchmark_sglang_smoke" workspace.mkdir(parents=True, exist_ok=True) if accuracy is not None: @@ -115,6 +122,17 @@ def _fake_workspace(slot: Path, *, tput: float = 800.0, accuracy: float | None = } ) ) + if input_throughput is not None or intvty_p90 is not None: + (workspace / "inferencex_result.json").write_text( + json.dumps( + { + "output_throughput": tput, + "input_throughput": input_throughput, + "intvty_p90_tok_s_user": intvty_p90, + } + ), + encoding="utf-8", + ) return workspace @@ -288,6 +306,70 @@ def _fake_run(cmd, *args, **kwargs): assert "workspace" in res +@pytest.mark.asyncio +async def test_integrate_handler_keep_uses_composite_metric( + session_dir, + tmp_path, + monkeypatch, +): + """Flag on: input-only lift KEEPs even when output tput is flat.""" + monkeypatch.setenv("HYPERLOOM_PERF_METRIC", "composite_v1") + base_yaml = tmp_path / "base.yaml" + _write_baseline_yaml(base_yaml) + baseline_perf = { + "output_throughput": 800.0, + "input_throughput": 10000.0, + "intvty_p90": 700.0, + } + state = SharedState.load_or_init(session_dir) + state.framework = "sglang" + state.baseline_tput = 800.0 + state.baseline_perf = dict(baseline_perf) + state.current_best = { + "action": "baseline", + "tput": 800.0, + **baseline_perf, + } + state.save(session_dir) + + def _fake_run(cmd, *args, **kwargs): + out_idx = cmd.index("--output-dir") + slot = Path(cmd[out_idx + 1]) + _fake_workspace( + slot, + tput=800.0, + input_throughput=12000.0, + intvty_p90=700.0, + ) + return subprocess.CompletedProcess( + args=cmd, + returncode=0, + stdout="ok", + stderr="", + ) + + target, patch_file = _write_patch_pair(tmp_path) + payload = { + "base_tput": 800.0, + "config_path": str(base_yaml), + "kernel_id": "k_composite", + "patch_path": str(patch_file), + "target_file": str(target), + "allow_unknown_target": True, + "skip_rebuild": True, + "framework": "sglang", + } + with patch("hyperloom.orchestrator.actions.executors.baseline.run_with_session_kill", side_effect=_fake_run): + res = await krh.integrate_handler(payload, session_dir=session_dir) + + assert res["status"] == "ok" + assert res["decision"] == "KEEP" + assert res["new_tput"] == 800.0 + assert res["gain_pct"] > 1.0 + assert res["input_throughput"] == pytest.approx(12000.0) + assert res["intvty_p90"] == pytest.approx(700.0) + + @pytest.mark.asyncio async def test_integrate_handler_keeps_positive_stack_increment( session_dir, diff --git a/src/hyperloom/inference_optimizer/tests/test_map_aiperf.py b/src/hyperloom/inference_optimizer/tests/test_map_aiperf.py index cb8f0c226..9ee332746 100644 --- a/src/hyperloom/inference_optimizer/tests/test_map_aiperf.py +++ b/src/hyperloom/inference_optimizer/tests/test_map_aiperf.py @@ -32,7 +32,8 @@ def _sample(): "total_output_tokens": {"unit": "tok", "avg": 2100.0}, "benchmark_duration": {"unit": "s", "avg": 14.0}, "time_to_first_token": _metric(120.0, p50=110.0, p99=200.0, std=15.0), - "inter_token_latency": _metric(20.0, p50=18.0, p99=40.0, std=5.0), + "inter_token_latency": _metric(20.0, p50=18.0, p90=34.3, p99=40.0, std=5.0), + "output_token_throughput_per_user": _metric(745.19, p50=700.0, p99=800.0, std=10.0), "request_latency": _metric(900.0, p50=850.0, p99=1500.0, std=120.0), "theoretical_prefix_cache_hit": {"unit": "%", "avg": 0.73}, } @@ -50,6 +51,7 @@ def test_map_core_throughput_and_counts(): r = map_aiperf(_sample()) assert r["request_throughput"] == 3.0 assert r["output_throughput"] == 500.0 + assert r["input_throughput"] == 1500.0 assert r["total_token_throughput"] == 2000.0 assert r["completed"] == 42 assert r["total_input_tokens"] == 4200 @@ -67,6 +69,8 @@ def test_map_latency_fields(): assert r["p99_itl_ms"] == 40.0 # tpot mirrors inter_token_latency in the aiperf schema assert r["mean_tpot_ms"] == 20.0 + assert r["p90_tpot_ms"] == 34.3 + assert r["intvty_p90_tok_s_user"] == 745.19 assert r["mean_e2el_ms"] == 900.0 assert r["p99_e2el_ms"] == 1500.0 diff --git a/src/hyperloom/inference_optimizer/tests/test_sweep_executor_unit.py b/src/hyperloom/inference_optimizer/tests/test_sweep_executor_unit.py index ff728d79c..74ec20b8d 100644 --- a/src/hyperloom/inference_optimizer/tests/test_sweep_executor_unit.py +++ b/src/hyperloom/inference_optimizer/tests/test_sweep_executor_unit.py @@ -150,6 +150,104 @@ def test_pareto_front_ignores_non_numeric(): assert pareto_front(entries) == [] +def test_pareto_front_max_total_tput_vs_max_intvty(): + """Composite Pareto: maximize total tput AND maximize p90 intvty.""" + entries = [ + {"status": "succeeded", "total_token_throughput": 1000, "intvty_p90": 500, "name": "a"}, + {"status": "succeeded", "total_token_throughput": 900, "intvty_p90": 800, "name": "b"}, + {"status": "succeeded", "total_token_throughput": 800, "intvty_p90": 400, "name": "c"}, + {"status": "succeeded", "total_token_throughput": 1100, "intvty_p90": 500, "name": "d"}, + {"status": "failed", "total_token_throughput": 9999, "intvty_p90": 9999, "name": "fail"}, + ] + front = pareto_front( + entries, + x_key="total_token_throughput", + y_key="intvty_p90", + y_higher_is_better=True, + ) + names = {e["name"] for e in front} + assert names == {"b", "d"} + + +def test_best_entry_for_each_conc_composite_prefers_score(monkeypatch): + """Flag on: a flat-output input lift beats a higher-output cell at the same conc.""" + monkeypatch.setenv("HYPERLOOM_PERF_METRIC", "composite_v1") + from hyperloom.orchestrator.actions.executors._grid_base import best_entry_for_each_conc + + baseline = { + "output_throughput": 800.0, + "input_throughput": 10000.0, + "intvty_p90": 700.0, + } + high_out = { + "status": "succeeded", + "conc": 4, + "name": "high_out", + "output_throughput": 900.0, + "input_throughput": 10000.0, + "intvty_p90": 700.0, + } + input_lift = { + "status": "succeeded", + "conc": 4, + "name": "input_lift", + "output_throughput": 800.0, + "input_throughput": 12000.0, + "intvty_p90": 700.0, + } + best = best_entry_for_each_conc( + [high_out, input_lift], + framework="sglang", + baseline_perf=baseline, + ) + assert best["4"]["name"] == "input_lift" + + +def test_best_entry_for_each_conc_falls_back_without_triple(monkeypatch): + monkeypatch.setenv("HYPERLOOM_PERF_METRIC", "composite_v1") + from hyperloom.orchestrator.actions.executors._grid_base import best_entry_for_each_conc + + entries = [ + {"status": "succeeded", "conc": 4, "name": "low", "output_throughput": 100.0}, + {"status": "succeeded", "conc": 4, "name": "high", "output_throughput": 200.0}, + ] + best = best_entry_for_each_conc(entries, framework="sglang", baseline_perf=None) + assert best["4"]["name"] == "high" + + +def test_select_sweep_pareto_composite_then_fallback(monkeypatch): + monkeypatch.setenv("HYPERLOOM_PERF_METRIC", "composite_v1") + from hyperloom.orchestrator.actions.executors._grid_base import select_sweep_pareto + + composite_cells = [ + { + "status": "succeeded", + "total_token_throughput": 1000, + "intvty_p90": 500, + "output_throughput": 10, + "e2el_mean_ms": 1, + "name": "a", + }, + { + "status": "succeeded", + "total_token_throughput": 900, + "intvty_p90": 800, + "output_throughput": 999, + "e2el_mean_ms": 1, + "name": "b", + }, + ] + front = select_sweep_pareto(composite_cells, framework="sglang") + assert {e["name"] for e in front} == {"a", "b"} + + tput_only = [ + {"status": "succeeded", "output_throughput": 100, "e2el_mean_ms": 10, "name": "fast"}, + {"status": "succeeded", "output_throughput": 90, "e2el_mean_ms": 20, "name": "slow"}, + ] + fallback = select_sweep_pareto(tput_only, framework="sglang") + assert [e["name"] for e in fallback] == ["fast"] + + # ---- SweepExecutor.__call__ ---- @@ -210,5 +308,60 @@ async def fake_run_grid(**kwargs): out = await ex(ctx) assert out["status"] == "succeeded" assert out["grid_size"] == 1 - assert out["best_for_each_conc"]["4"]["output_throughput"] == 120.0 - assert len(out["pareto_front"]) == 1 +async def test_call_success_composite_ranks_on_score(tmp_path, monkeypatch): + """Flag on: best-per-conc is *S*; Pareto is total tput vs p90 intvty.""" + monkeypatch.setenv("HYPERLOOM_PERF_METRIC", "composite_v1") + cfg = tmp_path / "c.yaml" + cfg.write_text("k: v\n", encoding="utf-8") + monkeypatch.setattr(sw, "materialize_config_with_envs", lambda *a, **k: cfg) + + async def fake_run_grid(**kwargs): + return [ + VariantResult( + name="high_out", + extra_server_args="", + extra_envs={"CONC": "4", "ISL": "1024", "OSL": "1024"}, + status="succeeded", + output_throughput=900.0, + total_token_throughput=1000.0, + input_throughput=10000.0, + intvty_p90=500.0, + e2el_mean_ms=40.0, + ), + VariantResult( + name="input_lift", + extra_server_args="", + extra_envs={"CONC": "4", "ISL": "8192", "OSL": "1024"}, + status="succeeded", + output_throughput=800.0, + total_token_throughput=900.0, + input_throughput=12000.0, + intvty_p90=800.0, + e2el_mean_ms=40.0, + ), + ] + + monkeypatch.setattr(sw, "run_grid", fake_run_grid) + baseline_perf = { + "output_throughput": 800.0, + "input_throughput": 10000.0, + "intvty_p90": 700.0, + } + ex = sw.SweepExecutor(session_dir=tmp_path) + ctx = _ctx( + tmp_path, + { + "config_path": str(cfg), + "conc_values": [4], + "isl_osl_configs": ["1024:1024", "8192:1024"], + }, + ) + ctx.extra["shared_state"] = SimpleNamespace( + framework="sglang", + baseline_perf=baseline_perf, + model_path="", + ) + out = await ex(ctx) + assert out["status"] == "succeeded" + assert out["best_for_each_conc"]["4"]["name"] == "input_lift" + assert {e["name"] for e in out["pareto_front"]} == {"high_out", "input_lift"} diff --git a/src/hyperloom/inference_optimizer/tests/test_sweep_phase_auto.py b/src/hyperloom/inference_optimizer/tests/test_sweep_phase_auto.py index a1d3d5b8f..53f1d2233 100644 --- a/src/hyperloom/inference_optimizer/tests/test_sweep_phase_auto.py +++ b/src/hyperloom/inference_optimizer/tests/test_sweep_phase_auto.py @@ -224,7 +224,9 @@ def test_pending_keep_kernel_ids_do_not_retry_needs_review(): assert state.next_pending_keep_kernel_id() == "k001" -def _patch_stack_validation_internals(monkeypatch, *, new_tput: float, revert_status: str = "ok"): +def _patch_stack_validation_internals( + monkeypatch, *, new_tput: float, revert_status: str = "ok", bench: dict | None = None +): """Stub apply/revert/bench so the real stack-validation decision path runs.""" import hyperloom.orchestrator.kernel.request_handlers as krh import hyperloom.orchestrator.actions.executors.baseline as baseline_mod @@ -243,11 +245,14 @@ def __init__(self, *, session_dir): self.session_dir = session_dir async def __call__(self, ctx): - return { + payload = { "output_throughput": new_tput, "report_path": "/tmp/report", "workspace": "/tmp/workspace", } + if bench: + payload.update(bench) + return payload monkeypatch.setattr(krh, "_maybe_apply_kernel_patch", _fake_apply) monkeypatch.setattr(krh, "_maybe_revert_kernel_patch", _fake_revert) @@ -350,6 +355,42 @@ async def test_stack_validation_keeps_on_positive_increment_over_current_best( assert result["revert_result"]["status"] == "skipped" +@pytest.mark.asyncio +async def test_stack_validation_keep_uses_composite_metric(tmp_path: Path, monkeypatch): + """Flag on: input-only lift KEEPs the leftover stack even when output tput is flat.""" + monkeypatch.setenv("HYPERLOOM_PERF_METRIC", "composite_v1") + baseline_perf = { + "output_throughput": 100.0, + "input_throughput": 10000.0, + "intvty_p90": 700.0, + } + c = _stack_validation_coordinator(tmp_path) + c.shared_state.framework = "sglang" + c.shared_state.baseline_perf = dict(baseline_perf) + c.shared_state.current_best = { + "action": "integrate", + "tput": 110.0, + "kernel_id": "k_prev", + "output_throughput": 110.0, + "input_throughput": 10000.0, + "intvty_p90": 700.0, + } + stack = c._stack_entries_for_validation(["k001", "k004"]) + _patch_stack_validation_internals( + monkeypatch, + new_tput=110.0, + bench={"input_throughput": 12000.0, "intvty_p90": 700.0}, + ) + + result = await c._run_kernel_stack_validation_e2e(stack) + + assert result["decision"] == "KEEP" + assert result["new_tput"] == 110.0 + assert result["stack_incremental_gain_pct"] > 1.0 + assert result["input_throughput"] == pytest.approx(12000.0) + assert result["revert_result"]["status"] == "skipped" + + @pytest.mark.asyncio async def test_positive_needs_review_stack_validation_promotes_combo(tmp_path: Path): """Two positive sub-threshold kernel patches should get one combined E2E validation.""" diff --git a/src/hyperloom/inference_optimizer/tests/test_warm_replay.py b/src/hyperloom/inference_optimizer/tests/test_warm_replay.py index 2c953faad..c3813db66 100644 --- a/src/hyperloom/inference_optimizer/tests/test_warm_replay.py +++ b/src/hyperloom/inference_optimizer/tests/test_warm_replay.py @@ -49,6 +49,7 @@ class _StubSharedState: cumulative_gain_validated_ts: str = "" cumulative_gain_validated_stack_len: int = 0 current_best: dict = field(default_factory=dict) + baseline_perf: dict | None = None tick: int = 0 phase: str = "PRELUDE" conc: int = 64 @@ -59,10 +60,17 @@ class _StubSharedState: def save(self, *args, **kwargs): # noqa: D401 — stub pass - def append_stack_gain_entry(self, *, action, variant_name, new_tput, extra_server_args="", ts=None): - from hyperloom.common.gain_math import gain_pct + def append_stack_gain_entry( + self, *, action, variant_name, new_tput, extra_server_args="", ts=None, candidate=None + ): + from hyperloom.common.perf_metric import session_gain_from_measurement - entry_gain_pct = gain_pct(float(new_tput or 0.0), float(self.baseline_tput or 0.0)) + entry_gain_pct, _used = session_gain_from_measurement( + float(new_tput or 0.0), + state=self, + candidate=candidate, + base_tput=float(self.baseline_tput or 0.0), + ) self.gain_per_stack_entry.append(entry_gain_pct) return entry_gain_pct @@ -1102,6 +1110,115 @@ def test_promote_warm_replay_adopts_on_any_positive_gain(tmp_path): assert coord.shared_state.current_best["action"] == "replay_warm_recipe" +def test_promote_warm_replay_composite_adopts_flat_output_input_lift(tmp_path, monkeypatch): + """Flag on: +20% input / flat output reproduces; the old tput bar would have drifted.""" + monkeypatch.setenv("HYPERLOOM_PERF_METRIC", "composite_v1") + baseline_perf = { + "output_throughput": 600.0, + "input_throughput": 10000.0, + "intvty_p90": 700.0, + } + coord = _make_coord(tmp_path, warm_start_recipe=_warm_recipe_t1()) + coord.shared_state.baseline_perf = dict(baseline_perf) + coord.shared_state.current_best = {"action": "baseline", "tput": 600.0, **baseline_perf} + coord.shared_state.warm_replay_outcome = { + "status": "in_flight", + "expected_gain_pct": 25.0, + "warm_recipe_tier": "exact", + } + result = { + "status": "succeeded", + "output_throughput": 600.0, + "input_throughput": 12000.0, + "intvty_p90": 700.0, + } + coord._promote_warm_replay( + result, + task=_StubTask( + params={ + "extra_server_args": "--attention-backend AITER", + "baseline_tput_anchor": 600.0, + } + ), + ) + + outcome = coord.shared_state.warm_replay_outcome + assert outcome["status"] == "reproduced" + assert outcome["used_composite"] is True + # S = 0.55 * 20% input = 11%; output did not move. + assert outcome["actual_gain_pct"] == pytest.approx(11.0) + assert outcome["throughput_after"] == 600.0 + assert outcome.get("below_historical_reproduce_pct") is not True + assert coord.shared_state.current_best["action"] == "replay_warm_recipe" + assert coord.shared_state.current_best["tput"] == 600.0 + assert coord.shared_state.current_best["input_throughput"] == 12000.0 + assert len(coord.shared_state.optimization_stack) == 1 + assert coord.shared_state.gain_per_stack_entry == [pytest.approx(11.0)] + assert coord.shared_state.cumulative_gain_validated == pytest.approx(11.0) + + +def test_promote_warm_replay_composite_combined_contract_uses_score_bar(tmp_path, monkeypatch): + """Flag on + combined 1% bar: the same flat-output input lift still clears KEEP.""" + monkeypatch.setenv("HYPERLOOM_PERF_METRIC", "composite_v1") + baseline_perf = { + "output_throughput": 600.0, + "input_throughput": 10000.0, + "intvty_p90": 700.0, + } + coord = _make_coord(tmp_path, warm_start_recipe=_warm_recipe_t1()) + coord.shared_state.baseline_perf = dict(baseline_perf) + coord.shared_state.current_best = {"action": "baseline", "tput": 600.0, **baseline_perf} + coord.shared_state.warm_replay_outcome = {"status": "in_flight", "expected_gain_pct": 0.0} + coord._promote_warm_replay( + { + "status": "succeeded", + "output_throughput": 600.0, + "input_throughput": 12000.0, + "intvty_p90": 700.0, + }, + task=_StubTask( + params={ + "extra_server_args": "--current", + "baseline_tput_anchor": 600.0, + "combined_current_contract": True, + "combined_keep_threshold_pct": 1.0, + } + ), + ) + + outcome = coord.shared_state.warm_replay_outcome + assert outcome["status"] == "reproduced" + assert outcome["used_composite"] is True + assert outcome["actual_gain_pct"] == pytest.approx(11.0) + + +def test_promote_warm_replay_composite_falls_back_without_triple(tmp_path, monkeypatch): + """Flag on but no intvty: still the output-tput path, so flat output is drift.""" + monkeypatch.setenv("HYPERLOOM_PERF_METRIC", "composite_v1") + coord = _make_coord(tmp_path, warm_start_recipe=_warm_recipe_t1()) + coord.shared_state.baseline_perf = { + "output_throughput": 600.0, + "input_throughput": 10000.0, + "intvty_p90": 700.0, + } + coord.shared_state.warm_replay_outcome = {"status": "in_flight", "expected_gain_pct": 25.0} + coord._promote_warm_replay( + {"status": "succeeded", "output_throughput": 600.0, "input_throughput": 12000.0}, + task=_StubTask( + params={ + "extra_server_args": "--attention-backend AITER", + "baseline_tput_anchor": 600.0, + } + ), + ) + + outcome = coord.shared_state.warm_replay_outcome + assert outcome["status"] == "drift" + assert outcome["used_composite"] is False + assert outcome["actual_gain_pct"] == 0.0 + assert coord.shared_state.optimization_stack == [] + + def test_promote_warm_replay_no_gain_is_drift(tmp_path): """Zero or negative measured gain → ``drift``, no stack push.""" coord = _make_coord(tmp_path, warm_start_recipe=_warm_recipe_t1()) @@ -1763,7 +1880,7 @@ async def test_warm_replay_falls_back_to_flat_gain_pct_for_arbor_seed(tmp_path): def test_promote_warm_replay_cumulative_gain_uses_tput_ratio(tmp_path): - """Cumulative gain after warm-replay = (tput / baseline_tput - 1) × 100, the authoritative formula.""" + """Flag off: cumulative gain after warm-replay is (tput / baseline_tput - 1) × 100.""" coord = _make_coord(tmp_path, warm_start_recipe=_warm_recipe_t1()) coord.shared_state.warm_replay_outcome = { "status": "in_flight", diff --git a/src/hyperloom/orchestrator/actions/executors/_geak_sweep.py b/src/hyperloom/orchestrator/actions/executors/_geak_sweep.py index 638a7b716..85c3a32f3 100644 --- a/src/hyperloom/orchestrator/actions/executors/_geak_sweep.py +++ b/src/hyperloom/orchestrator/actions/executors/_geak_sweep.py @@ -23,7 +23,7 @@ from hyperloom.common.env_safety import build_benchmark_env from hyperloom.common.jsonio import read_json -from ._grid_base import pareto_front +from ._grid_base import best_entry_for_each_conc, select_sweep_pareto log = logging.getLogger(__name__) @@ -90,6 +90,8 @@ async def sweep_via_geak( variant_timeout_sec: int, repeats: int = 3, pin_num_prompts: bool = False, + framework: str | None = None, + state: Any = None, ) -> dict[str, Any]: """Run a CONC × (ISL, OSL) sweep on the GEAK-optimized server. @@ -214,6 +216,15 @@ def _run() -> subprocess.CompletedProcess: ttft = summ.get("ttft_ms_median") tpot = summ.get("tpot_ms_median") e2el = summ.get("e2el_ms_median") + total_tput = summ.get("total_token_throughput") + if total_tput is None: + total_tput = summ.get("total_throughput_tok_s_median") + intvty = summ.get("intvty_p90_tok_s_user") + if intvty is None: + intvty = summ.get("intvty_p90") + input_tput = summ.get("input_throughput") + if input_tput is None: + input_tput = summ.get("input_throughput_tok_s_median") if proc.returncode == 0 and isinstance(tput, (int, float)) and tput > 0: succeeded = True entry.update( @@ -224,6 +235,12 @@ def _run() -> subprocess.CompletedProcess: "tpot_mean_ms": tpot, } ) + if isinstance(total_tput, (int, float)) and total_tput > 0: + entry["total_token_throughput"] = float(total_tput) + if isinstance(intvty, (int, float)) and intvty > 0: + entry["intvty_p90"] = float(intvty) + if isinstance(input_tput, (int, float)) and input_tput > 0: + entry["input_throughput"] = float(input_tput) else: err = (proc.stderr or "")[-500:] or "no throughput" entry.update({"status": "failed", "error": err}) @@ -248,17 +265,16 @@ def _run() -> subprocess.CompletedProcess: ) entries.append(entry) - front = pareto_front(entries, latency_key="ttft_mean_ms") - best_for_each_conc: dict[str, dict[str, Any]] = {} - for e in entries: - if e["status"] != "succeeded": - continue - cur = best_for_each_conc.get(str(e["conc"])) - if cur is None or ( - isinstance(e.get("output_throughput"), (int, float)) - and e["output_throughput"] > cur.get("output_throughput", 0) - ): - best_for_each_conc[str(e["conc"])] = e + front = select_sweep_pareto( + entries, + framework=framework, + fallback_latency_key="ttft_mean_ms", + ) + best_for_each_conc = best_entry_for_each_conc( + entries, + framework=framework, + state=state, + ) succeeded = [e for e in entries if e["status"] == "succeeded"] return { diff --git a/src/hyperloom/orchestrator/actions/executors/_grid_base.py b/src/hyperloom/orchestrator/actions/executors/_grid_base.py index 20ab25325..c43fc87d7 100644 --- a/src/hyperloom/orchestrator/actions/executors/_grid_base.py +++ b/src/hyperloom/orchestrator/actions/executors/_grid_base.py @@ -234,6 +234,9 @@ class VariantResult: ttft_mean_ms (float | None): Mean time-to-first-token (ms). e2el_mean_ms (float | None): Mean end-to-end latency (ms). tpot_mean_ms (float | None): Mean time-per-output-token (ms). + input_throughput (float | None): Input tokens/sec (prefill), if measured. + tpot_p90_ms (float | None): p90 inter-token latency (ms), if measured. + intvty_p90 (float | None): p90 output tok/s/user (interactivity), if measured. workspace (str | None): Path to the located ``benchmark_*`` workspace. report_path (str | None): Path to ``benchmark_report.json`` if present. raw_result_path (str | None): Path to the raw result JSON, if salvaged. @@ -269,6 +272,9 @@ class VariantResult: ttft_mean_ms: float | None = None e2el_mean_ms: float | None = None tpot_mean_ms: float | None = None + input_throughput: float | None = None + tpot_p90_ms: float | None = None + intvty_p90: float | None = None workspace: str | None = None report_path: str | None = None raw_result_path: str | None = None @@ -319,6 +325,9 @@ def to_dict(self) -> dict[str, Any]: "ttft_mean_ms": self.ttft_mean_ms, "e2el_mean_ms": self.e2el_mean_ms, "tpot_mean_ms": self.tpot_mean_ms, + "input_throughput": self.input_throughput, + "tpot_p90_ms": self.tpot_p90_ms, + "intvty_p90": self.intvty_p90, "workspace": self.workspace, "report_path": self.report_path, "raw_result_path": self.raw_result_path, @@ -334,42 +343,150 @@ def to_dict(self) -> dict[str, Any]: } +def _numeric(value: Any) -> float | None: + """Return a finite number, excluding bool (a subclass of int).""" + if isinstance(value, bool) or not isinstance(value, (int, float)): + return None + return float(value) + + def pareto_front( entries: list[dict[str, Any]], *, latency_key: str = "e2el_mean_ms", + x_key: str = "output_throughput", + y_key: str | None = None, + y_higher_is_better: bool = False, ) -> list[dict[str, Any]]: - """Naive O(N²) Pareto for (max ``output_throughput``, min *latency_key*). + """Naive O(N²) Pareto on two axes. + + Default (flag off): maximize ``output_throughput``, minimize *latency_key* + (native sweep: ``e2el_mean_ms``; GEAK sweep: ``ttft_mean_ms``). + + Composite Pareto: maximize ``total_token_throughput`` and maximize + ``intvty_p90`` (pass ``y_higher_is_better=True``). Args: - entries (list[dict[str, Any]]): Sweep result entries to filter. - latency_key (str): Which latency metric to minimize. The native sweep - reads ``e2el_mean_ms``; the GEAK sweep reads ``ttft_mean_ms`` - because ``bench_summary.json`` carries no e2el. + entries: Sweep result entries to filter. + latency_key: Minimize axis when *y_key* is omitted (backward compatible). + x_key: Maximize axis. + y_key: Second axis; defaults to *latency_key*. + y_higher_is_better: When True, maximize *y_key*; when False, minimize it. Returns: - list[dict[str, Any]]: The non-dominated subset of succeeded entries. + The non-dominated subset of succeeded entries that have both axes. """ + y = latency_key if y_key is None else y_key + + def _y_at_least(other: float, cand: float) -> bool: + return other >= cand if y_higher_is_better else other <= cand + + def _y_strictly_better(other: float, cand: float) -> bool: + return other > cand if y_higher_is_better else other < cand + succ = [ e for e in entries - if e["status"] == "succeeded" - and isinstance(e.get("output_throughput"), (int, float)) - and isinstance(e.get(latency_key), (int, float)) + if e.get("status") == "succeeded" + and _numeric(e.get(x_key)) is not None + and _numeric(e.get(y)) is not None ] front: list[dict[str, Any]] = [] for cand in succ: + cand_x = _numeric(cand.get(x_key)) + cand_y = _numeric(cand.get(y)) + if cand_x is None or cand_y is None: + continue dominated = False for other in succ: if other is cand: continue + other_x = _numeric(other.get(x_key)) + other_y = _numeric(other.get(y)) + if other_x is None or other_y is None: + continue if ( - other["output_throughput"] >= cand["output_throughput"] - and other[latency_key] <= cand[latency_key] - and (other["output_throughput"] > cand["output_throughput"] or other[latency_key] < cand[latency_key]) + other_x >= cand_x + and _y_at_least(other_y, cand_y) + and (other_x > cand_x or _y_strictly_better(other_y, cand_y)) ): dominated = True break if not dominated: front.append(cand) return front + + +def select_sweep_pareto( + entries: list[dict[str, Any]], + *, + framework: str | None = None, + fallback_latency_key: str = "e2el_mean_ms", +) -> list[dict[str, Any]]: + """Pareto for a sweep grid: composite axes when the flag is on, else tput/latency. + + Flag on: max total token throughput vs max p90 intvty. If no cell has both + axes, fall back to output-tput vs *fallback_latency_key*. + """ + from hyperloom.common.perf_metric import composite_grading_enabled + + if composite_grading_enabled(framework): + front = pareto_front( + entries, + x_key="total_token_throughput", + y_key="intvty_p90", + y_higher_is_better=True, + ) + if front: + return front + return pareto_front(entries, latency_key=fallback_latency_key) + + +def best_entry_for_each_conc( + entries: list[dict[str, Any]], + *, + framework: str | None = None, + state: Any = None, + baseline_perf: Any = None, +) -> dict[str, dict[str, Any]]: + """Best succeeded cell per concurrency. + + Flag on with a baseline triple: highest composite *S*. Cells missing a + triple do not compete on *S*; if a conc has no scored cell, fall back to + highest output throughput. Flag off: highest output throughput. + """ + from hyperloom.common.perf_metric import ( + composite_grading_enabled, + composite_score, + perf_snapshot_from_mapping, + resolve_baseline_perf, + ) + + baseline = baseline_perf or resolve_baseline_perf(state) + use_composite = composite_grading_enabled(framework) and bool(baseline) + scored: dict[str, tuple[float, dict[str, Any]]] = {} + tput_best: dict[str, dict[str, Any]] = {} + for e in entries: + if e.get("status") != "succeeded": + continue + conc = str(e.get("conc")) + tput = _numeric(e.get("output_throughput")) + if tput is not None: + cur = tput_best.get(conc) + cur_tput = _numeric(cur.get("output_throughput")) if cur is not None else None + if cur is None or cur_tput is None or tput > cur_tput: + tput_best[conc] = e + if use_composite and baseline is not None: + snap = perf_snapshot_from_mapping(e) + if snap is None: + continue + score = composite_score(snap, baseline) + prev = scored.get(conc) + if prev is None or score > prev[0]: + scored[conc] = (score, e) + if use_composite: + out: dict[str, dict[str, Any]] = {} + for conc in set(scored) | set(tput_best): + out[conc] = scored[conc][1] if conc in scored else tput_best[conc] + return out + return tput_best diff --git a/src/hyperloom/orchestrator/actions/executors/_grid_runner.py b/src/hyperloom/orchestrator/actions/executors/_grid_runner.py index 4c9a51a46..cff022774 100644 --- a/src/hyperloom/orchestrator/actions/executors/_grid_runner.py +++ b/src/hyperloom/orchestrator/actions/executors/_grid_runner.py @@ -2710,6 +2710,9 @@ def _skip_rest_for_budget(idx: int, *, spent_on: str, rounds_left: int = variant ttft_mean_ms=measurement.get("ttft_mean_ms"), e2el_mean_ms=measurement.get("e2el_mean_ms"), tpot_mean_ms=measurement.get("tpot_mean_ms"), + input_throughput=measurement.get("input_throughput"), + tpot_p90_ms=measurement.get("tpot_p90_ms"), + intvty_p90=measurement.get("intvty_p90"), workspace=str(workspace), report_path=str(report_path) if report_path.exists() else None, raw_result_path=measurement.get("raw_result_path"), diff --git a/src/hyperloom/orchestrator/actions/executors/_stack_rebench.py b/src/hyperloom/orchestrator/actions/executors/_stack_rebench.py index 922203e82..73d74b045 100644 --- a/src/hyperloom/orchestrator/actions/executors/_stack_rebench.py +++ b/src/hyperloom/orchestrator/actions/executors/_stack_rebench.py @@ -4,15 +4,17 @@ """Shared full-stack rebench step. Re-benches a variant layered on the current stack and compares it against a -stability floor (``base_tput * (1 + threshold%)``). Used by the explore ledger -(post-KEEP confirmation) and by integrate_patch (KEEP gate for patches). +stability floor. Throughput mode uses ``base_tput * (1 + threshold%)``. +Composite mode (flag on, serving, full triples) uses the same 0.5% threshold +on incremental *S*. Used by the explore ledger (post-KEEP confirmation) and +by integrate_patch (KEEP gate for patches). """ from __future__ import annotations from dataclasses import dataclass, field from pathlib import Path -from typing import Any +from typing import Any, Mapping from ..stop_attribution import stopped_by_the_run_class from ._grid_runner import GridVariant, run_grid @@ -43,10 +45,18 @@ class StackRebenchResult: # did not happen" apart from "the confirmation failed": :attr:`stable` is # ``False`` for both, and only one of them is evidence about the variant. error_class: str = "" + input_throughput: float | None = None + intvty_p90: float | None = None + tpot_p90_ms: float | None = None + stable_gain_pct: float | None = None + used_composite: bool = False + _stable: bool | None = field(default=None, repr=False) @property def stable(self) -> bool: - """True when the measured throughput cleared the stability floor.""" + """True when the confirmation cleared the stability floor.""" + if self._stable is not None: + return self._stable return self.tput is not None and self.tput >= self.stable_floor @@ -72,6 +82,9 @@ async def measure_stack_rebench( serving_lease: Any = None, session_deadline_sec: float | None = None, variant_expected_sec: float | None = None, + framework: str | None = None, + anchor_perf: Mapping[str, Any] | None = None, + baseline_perf: Mapping[str, Any] | None = None, ) -> StackRebenchResult: """Run ``variant`` once on the stack and grade it against the floor. @@ -117,10 +130,22 @@ async def measure_stack_rebench( workspace: str | None = None warnings: list[str] = [] error_class = "" + input_throughput: float | None = None + intvty_p90: float | None = None + tpot_p90_ms: float | None = None if rb is not None and rb.status == "succeeded": tput = rb.output_throughput workspace = rb.workspace warnings = list(rb.nonfatal_warnings) + raw_in = getattr(rb, "input_throughput", None) + raw_intv = getattr(rb, "intvty_p90", None) + raw_tpot = getattr(rb, "tpot_p90_ms", None) + if isinstance(raw_in, (int, float)) and float(raw_in) > 0: + input_throughput = float(raw_in) + if isinstance(raw_intv, (int, float)) and float(raw_intv) > 0: + intvty_p90 = float(raw_intv) + if isinstance(raw_tpot, (int, float)) and float(raw_tpot) > 0: + tpot_p90_ms = float(raw_tpot) elif rb is not None and stopped_by_the_run_class(getattr(rb, "error_class", "")) is not None: error_class = rb.error_class warnings.append(f"stack_rebench_skipped:{error_class}") @@ -128,13 +153,45 @@ async def measure_stack_rebench( warnings.append(f"stack_rebench_failed:{(rb.error or '')[-120:]}") else: warnings.append("stack_rebench_no_result") - stable_floor = base_tput * (1.0 + stable_threshold_pct / 100.0) + stable_floor = base_tput * (1.0 + stable_threshold_pct / 100.0) if base_tput > 0 else 0.0 + used_composite = False + stable_gain_pct: float | None = None + if tput is None: + stable_flag = False + else: + from hyperloom.common.perf_metric import ( + composite_grading_enabled, + perf_snapshot_from_mapping, + score_gain_pct, + ) + + cand = perf_snapshot_from_mapping( + { + "output_throughput": tput, + "input_throughput": input_throughput, + "intvty_p90": intvty_p90, + "tpot_p90_ms": tpot_p90_ms, + } + ) + if composite_grading_enabled(framework) and cand and baseline_perf and anchor_perf: + used_composite = True + graded = score_gain_pct(cand, anchor_perf, baseline_perf) + stable_gain_pct = 0.0 if graded is None else float(graded) + stable_flag = stable_gain_pct >= float(stable_threshold_pct) + else: + stable_flag = tput >= stable_floor return StackRebenchResult( tput=tput, workspace=workspace, warnings=warnings, stable_floor=stable_floor, error_class=error_class, + input_throughput=input_throughput, + intvty_p90=intvty_p90, + tpot_p90_ms=tpot_p90_ms, + stable_gain_pct=stable_gain_pct, + used_composite=used_composite, + _stable=stable_flag, ) diff --git a/src/hyperloom/orchestrator/actions/executors/benchmark_result.py b/src/hyperloom/orchestrator/actions/executors/benchmark_result.py index 800dfa370..ea13ecba9 100644 --- a/src/hyperloom/orchestrator/actions/executors/benchmark_result.py +++ b/src/hyperloom/orchestrator/actions/executors/benchmark_result.py @@ -712,6 +712,12 @@ def _merge_raw_result( measurement["ttft_p99_ms"] = to_float(raw.get("p99_ttft_ms")) if measurement.get("tpot_mean_ms") is None: measurement["tpot_mean_ms"] = to_float(raw.get("mean_tpot_ms")) + if measurement.get("input_throughput") is None: + measurement["input_throughput"] = to_float(raw.get("input_throughput")) + if measurement.get("tpot_p90_ms") is None: + measurement["tpot_p90_ms"] = to_float(raw.get("p90_tpot_ms")) + if measurement.get("intvty_p90") is None: + measurement["intvty_p90"] = to_float(raw.get("intvty_p90_tok_s_user")) if measurement.get("e2el_mean_ms") is None: measurement["e2el_mean_ms"] = first_float( raw.get("mean_e2el_ms"), diff --git a/src/hyperloom/orchestrator/actions/executors/explore.py b/src/hyperloom/orchestrator/actions/executors/explore.py index 5643dd31c..953f57b26 100644 --- a/src/hyperloom/orchestrator/actions/executors/explore.py +++ b/src/hyperloom/orchestrator/actions/executors/explore.py @@ -38,6 +38,12 @@ from hyperloom.common.coerce import to_str_list from hyperloom.common.gain_math import gain_pct +from hyperloom.common.perf_metric import ( + composite_grading_enabled, + perf_snapshot_from_mapping, + resolve_baseline_perf, + score_gain_pct, +) from hyperloom.common.model_paths import resolve_session_model_path from hyperloom.common.timeutil import now_iso from hyperloom.inference_optimizer.session.session_paths import runs_dir @@ -1088,6 +1094,13 @@ async def __call__(self, ctx) -> dict[str, Any]: stack_unset_envs = list(dict.fromkeys(base_unset_envs)) stack_base_args_mode = base_args_mode running_base_tput = base_tput + use_composite = composite_grading_enabled(framework) + baseline_perf = resolve_baseline_perf(ss) if use_composite else None + running_base_perf = ( + perf_snapshot_from_mapping(getattr(ss, "current_best", None)) or baseline_perf + if use_composite + else None + ) # In-batch KEEP'd entries (for full vs incremental stack recompose). in_batch_keeps: list[dict[str, Any]] = [] @@ -1513,15 +1526,34 @@ def _stopped_by_the_run(result: Any, *, variant: GridVariant, idx: int, round_la # Decision-round gain is the cost gate: only variants that # clear keep_threshold (and the accuracy gate) earn a warm # stack-rebench round. - gain = gain_pct(r.output_throughput, running_base_tput) - outcome = "FAILED" - reason: str = "" - if r.status != "succeeded" or gain is None: - reason = (r.error or "")[-1200:] or "no_measurement" - elif gain < keep_threshold_pct: - outcome = "REVERT" - reason = "gain_below_threshold" + cand_snap = perf_snapshot_from_mapping( + { + "output_throughput": r.output_throughput, + "input_throughput": r.input_throughput, + "intvty_p90": r.intvty_p90, + "tpot_p90_ms": r.tpot_p90_ms, + } + ) + gain: float | None + if use_composite and baseline_perf and running_base_perf and cand_snap: + gain = score_gain_pct(cand_snap, running_base_perf, baseline_perf) + outcome = "FAILED" + reason = "" + if r.status != "succeeded" or gain is None: + reason = (r.error or "")[-1200:] or "no_measurement" + elif gain < keep_threshold_pct: + outcome = "REVERT" + reason = "gain_below_threshold" else: + gain = gain_pct(r.output_throughput, running_base_tput) + outcome = "FAILED" + reason = "" + if r.status != "succeeded" or gain is None: + reason = (r.error or "")[-1200:] or "no_measurement" + elif gain < keep_threshold_pct: + outcome = "REVERT" + reason = "gain_below_threshold" + if outcome == "FAILED" and not reason: # Accuracy gate. For serving it runs only for high-risk # variants. For scriptable frameworks the image-quality # gate is the sole correctness signal, so every variant is @@ -1579,6 +1611,9 @@ def _stopped_by_the_run(result: Any, *, variant: GridVariant, idx: int, round_la "status": r.status, "tput": decision_tput, "decision_tput": decision_tput, + "input_throughput": r.input_throughput, + "intvty_p90": r.intvty_p90, + "tpot_p90_ms": r.tpot_p90_ms, "gain_pct": gain, "base_tput": running_base_tput, "round_id": round_id, @@ -1663,6 +1698,9 @@ def _stopped_by_the_run(result: Any, *, variant: GridVariant, idx: int, round_la "accuracy": accuracy_value, "tput": decision_tput, "decision_tput": decision_tput, + "input_throughput": r.input_throughput, + "intvty_p90": r.intvty_p90, + "tpot_p90_ms": r.tpot_p90_ms, "single_workspace": r.workspace, "round_id": round_id, "accepted_at_round": round_id, @@ -1727,6 +1765,9 @@ def _stopped_by_the_run(result: Any, *, variant: GridVariant, idx: int, round_la serving_lease=variant_lease, session_deadline_sec=session_deadline_sec, variant_expected_sec=decision_expected_sec, + framework=framework, + anchor_perf=running_base_perf, + baseline_perf=baseline_perf, ) # A confirmation the run stopped is not a failed # confirmation: grading it would evict a variant as @@ -1746,16 +1787,27 @@ def _stopped_by_the_run(result: Any, *, variant: GridVariant, idx: int, round_la stable_floor = rebench.stable_floor # Rebench missed the stability floor: evict as REVERT. if not rebench.stable: - log.warning( - "explore: variant %s KEEP -> KEEP_UNSTABLE " - "(stack_rebench_tput=%s vs stable_floor=%.2f " - "with running_base_tput=%.2f * (1+%.2f%%))", - gv.name, - stack_rebench_tput, - stable_floor, - running_base_tput, - stack_stable_threshold_pct, - ) + if rebench.used_composite: + log.warning( + "explore: variant %s KEEP -> KEEP_UNSTABLE " + "(composite gain=%s vs floor=%.2f%%)", + gv.name, + f"{rebench.stable_gain_pct:.2f}%" + if rebench.stable_gain_pct is not None + else "n/a", + stack_stable_threshold_pct, + ) + else: + log.warning( + "explore: variant %s KEEP -> KEEP_UNSTABLE " + "(stack_rebench_tput=%s vs stable_floor=%.2f " + "with running_base_tput=%.2f * (1+%.2f%%))", + gv.name, + stack_rebench_tput, + stable_floor, + running_base_tput, + stack_stable_threshold_pct, + ) round_tested[fp]["outcome"] = "KEEP_UNSTABLE" round_tested[fp]["stack_rebench_tput"] = stack_rebench_tput round_tested[fp]["stack_rebench_workspace"] = stack_rebench_workspace @@ -1789,16 +1841,35 @@ def _stopped_by_the_run(result: Any, *, variant: GridVariant, idx: int, round_la in_batch_keeps.pop() continue else: - # Stable — the warm round-2 tput is the headline; - # recompute gain from it and fold the variant onto - # the stack. - gain = gain_pct(stack_rebench_tput, running_base_tput) + # Stable — the warm round-2 measurement is the + # headline. Recompute gain with the same metric + # round 1 used (composite *S* or output tput). + if rebench.used_composite and rebench.stable_gain_pct is not None: + gain = float(rebench.stable_gain_pct) + else: + gain = gain_pct(stack_rebench_tput, running_base_tput) stack_extra_args = next_effective_args if persist_effective_args else next_stack_args stack_extra_envs = next_envs stack_remove_args = list(run_remove_args) stack_unset_envs = list(run_unset_envs) stack_base_args_mode = "replace" if persist_effective_args else "append" running_base_tput = stack_rebench_tput + rb_snap = perf_snapshot_from_mapping( + { + "output_throughput": stack_rebench_tput, + "input_throughput": rebench.input_throughput, + "intvty_p90": rebench.intvty_p90, + "tpot_p90_ms": rebench.tpot_p90_ms, + } + ) + if use_composite and rb_snap: + running_base_perf = rb_snap + keep_entry["input_throughput"] = rb_snap["input_throughput"] + keep_entry["intvty_p90"] = rb_snap["intvty_p90"] + if "tpot_p90_ms" in rb_snap: + keep_entry["tpot_p90_ms"] = rb_snap["tpot_p90_ms"] + elif use_composite and cand_snap: + running_base_perf = cand_snap keep_entry["gain_pct"] = gain keep_entry["tput"] = stack_rebench_tput keep_entry["stack_rebench_tput"] = stack_rebench_tput @@ -1818,6 +1889,8 @@ def _stopped_by_the_run(result: Any, *, variant: GridVariant, idx: int, round_la stack_unset_envs = list(run_unset_envs) stack_base_args_mode = "replace" if persist_effective_args else "append" running_base_tput = decision_tput or running_base_tput + if use_composite and cand_snap: + running_base_perf = cand_snap winners.append(keep_entry) winners_history_update.append( diff --git a/src/hyperloom/orchestrator/actions/executors/framework_agent.py b/src/hyperloom/orchestrator/actions/executors/framework_agent.py index 23dda7f0e..35608d5a5 100644 --- a/src/hyperloom/orchestrator/actions/executors/framework_agent.py +++ b/src/hyperloom/orchestrator/actions/executors/framework_agent.py @@ -14,6 +14,7 @@ from hyperloom.common.env import is_truthy from hyperloom.common.model_paths import resolve_session_model_path +from hyperloom.common.perf_metric import keep_gain_pct, perf_axes_from_mapping from hyperloom.common.url_safety import require_http_url from hyperloom.inference_optimizer.session.session_paths import runs_dir from ._accuracy_gate import ( @@ -826,9 +827,22 @@ def _undo_candidate() -> None: params.get("keep_threshold_pct", self.keep_threshold_pct), ) new_tput = bench_result.get("output_throughput") - delta_pct: float | None = None + tput_delta_pct: float | None = None if isinstance(new_tput, (int, float)) and new_tput > 0 and base_tput > 0: - delta_pct = (float(new_tput) - base_tput) / base_tput * 100.0 + tput_delta_pct = (float(new_tput) - base_tput) / base_tput * 100.0 + shared_state = extra.get("shared_state") or extra.get("state") + framework = str(params.get("framework") or getattr(shared_state, "framework", "") or "") + graded, used_composite = keep_gain_pct( + bench_result, + state=shared_state, + framework=framework, + base_tput=base_tput, + ) + if used_composite: + delta_pct = 0.0 if graded is None else float(graded) + else: + delta_pct = tput_delta_pct + perf_axes = perf_axes_from_mapping(bench_result) accuracy_pass = gate_evidence.get("accuracy_pass") # Source patches require the accuracy gate for a KEEP: a measured @@ -869,7 +883,7 @@ async def _record_outcome(outcome: str) -> None: await self._write_kb_record( candidate=candidate, outcome=outcome, - tps_delta_pct=float(delta_pct or 0.0), + tps_delta_pct=float(tput_delta_pct or 0.0), patch_path=str(applied[0]) if applied else "", extra=extra, accuracy_delta_pct=acc_delta_pct, @@ -884,7 +898,8 @@ async def _record_outcome(outcome: str) -> None: if delta_pct is None: reasons.append("no measurable throughput") elif delta_pct < keep_threshold_pct: - reasons.append(f"throughput delta {delta_pct:+.2f}% < keep_threshold {keep_threshold_pct:.2f}%") + metric = "composite gain" if used_composite else "throughput delta" + reasons.append(f"{metric} {delta_pct:+.2f}% < keep_threshold {keep_threshold_pct:.2f}%") if acc_block and acc_reason: reasons.append(acc_reason) # Distinguish "accuracy required but unevaluated" (None, not a @@ -898,12 +913,14 @@ async def _record_outcome(outcome: str) -> None: stash_state, stash_note, { + **perf_axes, "status": revert_status, "candidate": candidate, "batch_id": batch_id, "patches_applied": [], "patches_reverted": [str(p) for p in reverted], "output_throughput": new_tput, + "tput": new_tput, "delta_pct": delta_pct, "accuracy_pass": accuracy_pass, "base_tput": base_tput, @@ -931,6 +948,7 @@ async def _record_outcome(outcome: str) -> None: stash_state, stash_note, { + **perf_axes, "status": "apply_failed", "error_class": "keep_commit_failed", "error": commit_err or "git commit failed", @@ -939,6 +957,7 @@ async def _record_outcome(outcome: str) -> None: "patches_applied": [], "patches_reverted": [str(p) for p in reverted], "output_throughput": new_tput, + "tput": new_tput, "delta_pct": delta_pct, "accuracy_pass": accuracy_pass, "base_tput": base_tput, @@ -955,19 +974,24 @@ async def _record_outcome(outcome: str) -> None: stash_state, stash_note, { + **perf_axes, "status": "kept", "candidate": candidate, "batch_id": batch_id, "patches_applied": [str(p) for p in applied], "patches_reverted": [], "output_throughput": new_tput, + "tput": new_tput, "delta_pct": delta_pct, "accuracy_pass": accuracy_pass, "base_tput": base_tput, "keep_threshold_pct": keep_threshold_pct, "keep_commit_sha": keep_sha, "patch_source_mode": patch_source_mode, - "reason": (f"throughput delta {delta_pct:+.2f}% >= {keep_threshold_pct:.2f}%"), + "reason": ( + f"{'composite gain' if used_composite else 'throughput delta'} " + f"{delta_pct:+.2f}% >= {keep_threshold_pct:.2f}%" + ), "bench_result": bench_result, "workspace": str(output_root), }, @@ -1205,6 +1229,9 @@ async def _bench_candidate( "name": r.name, "status": r.status, "output_throughput": getattr(r, "output_throughput", None), + "input_throughput": getattr(r, "input_throughput", None), + "intvty_p90": getattr(r, "intvty_p90", None), + "tpot_p90_ms": getattr(r, "tpot_p90_ms", None), "ttft_ms": getattr(r, "ttft_ms", None), "itl_ms": getattr(r, "itl_ms", None), "result_dir": str(getattr(r, "result_dir", "")), diff --git a/src/hyperloom/orchestrator/actions/executors/integrate_patch.py b/src/hyperloom/orchestrator/actions/executors/integrate_patch.py index 2eb221e2d..551ca5e98 100644 --- a/src/hyperloom/orchestrator/actions/executors/integrate_patch.py +++ b/src/hyperloom/orchestrator/actions/executors/integrate_patch.py @@ -21,6 +21,7 @@ from hyperloom.common.coerce import to_str_list from hyperloom.common.env_safety import filter_untrusted_env_mapping, is_allowed_variant_env_key from hyperloom.common.model_paths import resolve_session_model_path +from hyperloom.common.perf_metric import keep_gain_pct, perf_axes_from_mapping from hyperloom.common.timeutil import now_iso from hyperloom.inference_optimizer.gpu_types import amd_gpu_dispatch_identity from hyperloom.inference_optimizer.session.session_paths import runs_dir @@ -3053,9 +3054,21 @@ async def _gate_perf( ) new_tput = bench_result.get("output_throughput") - delta_pct = None + tput_delta_pct = None if isinstance(new_tput, (int, float)) and new_tput > 0 and base_tput > 0: - delta_pct = (float(new_tput) - base_tput) / base_tput * 100.0 + tput_delta_pct = (float(new_tput) - base_tput) / base_tput * 100.0 + framework = str(params.get("framework") or getattr(shared_state, "framework", "") or "") + graded, used_composite = keep_gain_pct( + bench_result, + state=shared_state, + framework=framework, + base_tput=base_tput, + ) + if used_composite: + delta_pct = 0.0 if graded is None else float(graded) + else: + delta_pct = tput_delta_pct + perf_axes = perf_axes_from_mapping(bench_result) accuracy_pass: bool | None = gate_evidence.get("accuracy_pass") fw_authored = bool(params.get("framework_agent_authoring") or params.get("framework_agent_candidate_id")) @@ -3143,7 +3156,7 @@ async def _gate_perf( await self._maybe_write_framework_kb_record( done_payload=done_payload, outcome=kb_outcome, - tps_delta_pct=float(delta_pct or 0.0), + tps_delta_pct=float(tput_delta_pct or 0.0), extra=extra, accuracy_delta_pct=acc_delta_pct, config_fingerprint=cfg_fingerprint, @@ -3221,7 +3234,8 @@ async def _gate_perf( if delta_pct is None: reasons.append("no measurable throughput") elif delta_pct < keep_threshold_pct: - reasons.append(f"throughput delta {delta_pct:+.2f}% < keep_threshold {keep_threshold_pct:.2f}%") + metric = "composite gain" if used_composite else "throughput delta" + reasons.append(f"{metric} {delta_pct:+.2f}% < keep_threshold {keep_threshold_pct:.2f}%") if acc_block and acc_reason: reasons.append(acc_reason) _probe_reason = eval_probe_summary(gate_evidence.get("eval_probe")) @@ -3234,7 +3248,7 @@ async def _gate_perf( await self._maybe_write_framework_kb_record( done_payload=done_payload, outcome="reverted_smoke_fail", - tps_delta_pct=float(delta_pct or 0.0), + tps_delta_pct=float(tput_delta_pct or 0.0), extra=extra, accuracy_delta_pct=acc_delta_pct, config_fingerprint=cfg_fingerprint, @@ -3271,6 +3285,7 @@ async def _gate_perf( base_tput=base_tput, session_deadline_sec=session_deadline_sec, variant_expected_sec=variant_expected_sec, + shared_state=shared_state, ) rb_acc_block, rb_acc_reason, _rb_degraded = accuracy_keep_block( confirm["accuracy_pass"], @@ -3282,9 +3297,17 @@ async def _gate_perf( reverted = self._revert_patches(framework_root, applied) reasons = [] if not confirm["stable"]: - reasons.append( - f"stack rebench {confirm['tput']} below stability floor {confirm['stable_floor']:.2f}" - ) + if confirm.get("used_composite"): + gain = confirm.get("stable_gain_pct") + reasons.append( + f"stack rebench composite gain " + f"{0.0 if gain is None else float(gain):+.2f}% " + f"below stability floor {self._rebench_stable_threshold_pct(params):.2f}%" + ) + else: + reasons.append( + f"stack rebench {confirm['tput']} below stability floor {confirm['stable_floor']:.2f}" + ) if confirm["accuracy_pass"] is False: reasons.append("accuracy regression on rebench") elif rb_acc_block and rb_acc_reason: @@ -3297,7 +3320,7 @@ async def _gate_perf( await self._maybe_write_framework_kb_record( done_payload=done_payload, outcome="reverted_smoke_fail", - tps_delta_pct=float(delta_pct or 0.0), + tps_delta_pct=float(tput_delta_pct or 0.0), extra=extra, accuracy_delta_pct=acc_delta_pct, config_fingerprint=cfg_fingerprint, @@ -3326,14 +3349,16 @@ async def _gate_perf( ) if isinstance(confirm["tput"], (int, float)) and confirm["tput"] > 0: new_tput = confirm["tput"] - delta_pct = (float(new_tput) - base_tput) / base_tput * 100.0 + tput_delta_pct = (float(new_tput) - base_tput) / base_tput * 100.0 + if not used_composite: + delta_pct = tput_delta_pct if confirm["accuracy_pass"] is not None: accuracy_pass = confirm["accuracy_pass"] await self._maybe_write_framework_kb_record( done_payload=done_payload, outcome="integrated", - tps_delta_pct=float(delta_pct or 0.0), + tps_delta_pct=float(tput_delta_pct or 0.0), extra=extra, accuracy_delta_pct=acc_delta_pct, config_fingerprint=cfg_fingerprint, @@ -3431,6 +3456,7 @@ async def _gate_perf( stash_state, stash_note, { + **perf_axes, "status": "kept", "specialist_task_id": specialist_task_id, # Proposal ownership must survive delegated-result persistence @@ -3448,11 +3474,15 @@ async def _gate_perf( "extra_server_args_applied": extra_server_args_applied, "extra_envs_applied": extra_envs_applied, "output_throughput": new_tput, + "tput": new_tput, "delta_pct": delta_pct, "accuracy_pass": accuracy_pass, "base_tput": base_tput, "keep_threshold_pct": keep_threshold_pct, - "reason": (f"throughput delta {delta_pct:+.2f}% >= {keep_threshold_pct:.2f}%"), + "reason": ( + f"{'composite gain' if used_composite else 'throughput delta'} " + f"{delta_pct:+.2f}% >= {keep_threshold_pct:.2f}%" + ), "bench_result": bench_result, "workspace": str(output_root), "source_snapshot": source_snapshot_dir, @@ -4251,6 +4281,9 @@ async def _bench_patch( "name": r.name, "status": r.status, "output_throughput": getattr(r, "output_throughput", None), + "input_throughput": getattr(r, "input_throughput", None), + "intvty_p90": getattr(r, "intvty_p90", None), + "tpot_p90_ms": getattr(r, "tpot_p90_ms", None), "ttft_ms": getattr(r, "ttft_ms", None), "itl_ms": getattr(r, "itl_ms", None), # Benchmark dir; ``_grade_accuracy`` locates accuracy artifacts here. @@ -4414,6 +4447,7 @@ async def _confirm_stack_rebench( base_tput: float, session_deadline_sec: float | None = None, variant_expected_sec: float | None = None, + shared_state: Any = None, ) -> dict[str, Any]: """Re-bench the patched stack once more and re-grade throughput + accuracy. @@ -4457,6 +4491,15 @@ async def _confirm_stack_rebench( _rt_rb = params.get("runtime_override") if isinstance(_rt_rb, dict) and _rt_rb: variant.runtime_override = dict(_rt_rb) + from hyperloom.common.perf_metric import perf_snapshot_from_mapping, resolve_baseline_perf + + framework = str(params.get("framework") or getattr(shared_state, "framework", "") or "") + baseline_perf = resolve_baseline_perf(shared_state) if shared_state is not None else None + anchor_perf = ( + perf_snapshot_from_mapping(getattr(shared_state, "current_best", None)) or baseline_perf + if shared_state is not None + else None + ) rebench = await measure_stack_rebench( config_path=config_path, base_extra_args=base_extra_args, @@ -4473,6 +4516,9 @@ async def _confirm_stack_rebench( base_args_mode=str(params.get("base_args_mode") or "append"), session_deadline_sec=session_deadline_sec, variant_expected_sec=variant_expected_sec, + framework=framework, + anchor_perf=anchor_perf, + baseline_perf=baseline_perf, ) return self._graded_rebench(rebench, params=params, override_result_dir=override_result_dir) @@ -4533,6 +4579,11 @@ def _graded_rebench( "warnings": rebench.warnings, "stable_floor": rebench.stable_floor, "accuracy_pass": accuracy_pass, + "input_throughput": rebench.input_throughput, + "intvty_p90": rebench.intvty_p90, + "tpot_p90_ms": rebench.tpot_p90_ms, + "stable_gain_pct": rebench.stable_gain_pct, + "used_composite": rebench.used_composite, } diff --git a/src/hyperloom/orchestrator/actions/executors/report.py b/src/hyperloom/orchestrator/actions/executors/report.py index b27ed9696..f0b8e0c00 100644 --- a/src/hyperloom/orchestrator/actions/executors/report.py +++ b/src/hyperloom/orchestrator/actions/executors/report.py @@ -501,6 +501,44 @@ def _platform_fingerprint(gpu_type: str | None = None) -> dict[str, Any]: return platform_fingerprint(gpu_type, multi_node=multi_node) +def _append_composite_perf_section(lines: list[str], summary: dict[str, Any]) -> None: + """Render composite perf axes when baseline perf data is available.""" + from hyperloom.common.perf_metric import composite_metric_enabled, composite_score, perf_snapshot_from_mapping + + baseline = perf_snapshot_from_mapping(summary.get("baseline_perf")) + if not baseline: + return + cb = summary.get("current_best") or {} + cb_snap = perf_snapshot_from_mapping(cb) if isinstance(cb, dict) else None + lines.append("## Composite perf (input / intvty p90 / output)") + lines.append("") + lines.append( + f"- baseline input tput : `{baseline['input_throughput']:.1f}` tok/s" + ) + lines.append( + f"- baseline output tput: `{baseline['output_throughput']:.1f}` tok/s" + ) + lines.append( + f"- baseline intvty p90 : `{baseline['intvty_p90']:.1f}` tok/s/user" + ) + if cb_snap: + lines.append( + f"- current_best input : `{cb_snap['input_throughput']:.1f}` tok/s" + ) + lines.append( + f"- current_best output : `{cb_snap['output_throughput']:.1f}` tok/s" + ) + lines.append( + f"- current_best intvty : `{cb_snap['intvty_p90']:.1f}` tok/s/user" + ) + score = composite_score(cb_snap, baseline) + lines.append(f"- current_best score : `{score:.4f}`") + if composite_metric_enabled(): + lines.append("- grading mode : `composite_v1` (flag on)") + else: + lines.append("- grading mode : `output_throughput` (composite flag off)") + + def _build_summary_dict( state: SharedState, ev_counts: dict[str, int], @@ -539,6 +577,7 @@ def _build_summary_dict( "stop_reason": stop_reason, "stop_reason_explanation": _explain_stop_reason(stop_reason, state), "baseline_tput": state.baseline_tput, + "baseline_perf": dict(getattr(state, "baseline_perf", None) or {}), "baseline_accuracy": state.baseline_accuracy, "current_best": state.current_best, # Validated gain (what the run actually delivered). @@ -652,6 +691,7 @@ def _format_md(summary: dict[str, Any]) -> str: lines.append(f"- ttft_mean : `{cb.get('ttft_mean_ms'):.1f}` ms") if cb.get("e2el_mean_ms") is not None: lines.append(f"- e2el_mean : `{cb.get('e2el_mean_ms'):.1f}` ms") + _append_composite_perf_section(lines, summary) lines.append("") lines.extend(_format_completeness_annotations(summary)) lines.append("## Run summary") diff --git a/src/hyperloom/orchestrator/actions/executors/sweep.py b/src/hyperloom/orchestrator/actions/executors/sweep.py index 47b82ee1c..40171e1dc 100644 --- a/src/hyperloom/orchestrator/actions/executors/sweep.py +++ b/src/hyperloom/orchestrator/actions/executors/sweep.py @@ -21,7 +21,11 @@ sweep_grid: [{conc, isl, osl, output_throughput, ttft_mean_ms, e2el_mean_ms, status, workspace, error}] pareto_front: subset of sweep_grid that's not dominated - best_for_each_conc: dict[conc → entry with highest tput] + (output tput vs e2el, or total tput vs p90 intvty when + HYPERLOOM_PERF_METRIC=composite_v1) + best_for_each_conc: dict[conc → best cell] + (highest output tput, or highest composite *S* when + the flag is on) """ from __future__ import annotations @@ -34,7 +38,7 @@ from hyperloom.common.coerce import to_int from hyperloom.common.model_paths import resolve_session_model_path from hyperloom.inference_optimizer.session.session_paths import runs_dir -from ._grid_base import pareto_front +from ._grid_base import best_entry_for_each_conc, select_sweep_pareto from ._grid_runner import ( GridVariant, VariantResult, @@ -218,11 +222,13 @@ async def __call__(self, ctx) -> dict[str, Any]: ``workspace``. """ params = ctx.task.params or {} + extra = getattr(ctx, "extra", None) or {} + shared_state = extra.get("shared_state") or extra.get("state") + framework = str(getattr(shared_state, "framework", "") or "") if shared_state is not None else "" # GEAK reuse path: sweep the optimized server via GEAK's own bench_e2e.sh # + the already-built overlay. ps_result = params.get("geak_result") or {} if ps_result.get("bench_script") and ps_result.get("status") == "ok": - extra = getattr(ctx, "extra", None) or {} output_root = Path( params.get("output_dir") or extra.get("workspace") @@ -236,13 +242,13 @@ async def __call__(self, ctx) -> dict[str, Any]: isl_osl_configs=list(params.get("isl_osl_configs") or self.default_isl_osl_configs), output_root=output_root, variant_timeout_sec=int(params.get("variant_timeout_sec", self.variant_timeout_sec)), + framework=framework, + state=shared_state, ) config_path = Path(params.get("config_path") or self.default_config_path or default_baseline_config()) if not config_path.exists(): return {"status": "failed", "error_class": "missing_config", "error": f"config not found: {config_path}"} - extra = getattr(ctx, "extra", None) or {} - shared_state = extra.get("shared_state") or extra.get("state") output_root = Path( params.get("output_dir") or extra.get("workspace") or runs_dir(self.session_dir, "sweep", ctx.task.task_id) ) @@ -364,20 +370,12 @@ async def __call__(self, ctx) -> dict[str, Any]: # Surface skipped combos so the grid stays complete; they never enter # Pareto / best selections. entries.extend(skipped_variants) - front = pareto_front(entries) - - # Best per CONC. - best_for_each_conc: dict[int, dict[str, Any]] = {} - for e in entries: - if e["status"] != "succeeded": - continue - cur = best_for_each_conc.get(e["conc"]) - if cur is None or ( - isinstance(e.get("output_throughput"), (int, float)) - and isinstance(cur.get("output_throughput"), (int, float)) - and e["output_throughput"] > cur["output_throughput"] - ): - best_for_each_conc[e["conc"]] = e + front = select_sweep_pareto(entries, framework=framework) + best_for_each_conc = best_entry_for_each_conc( + entries, + framework=framework, + state=shared_state, + ) successful_entries = [e for e in entries if e.get("status") == "succeeded"] @@ -386,7 +384,7 @@ async def __call__(self, ctx) -> dict[str, Any]: "grid_size": len(entries), "sweep_grid": entries, "pareto_front": front, - "best_for_each_conc": {str(k): v for k, v in best_for_each_conc.items()}, + "best_for_each_conc": best_for_each_conc, "workspace": output_root.as_posix(), } diff --git a/src/hyperloom/orchestrator/kernel/conc_sweep.py b/src/hyperloom/orchestrator/kernel/conc_sweep.py index 457fdd943..c070251bb 100644 --- a/src/hyperloom/orchestrator/kernel/conc_sweep.py +++ b/src/hyperloom/orchestrator/kernel/conc_sweep.py @@ -123,6 +123,9 @@ def _point_from_variant(v: VariantResult, *, arm: str) -> dict[str, Any]: "output_throughput": v.output_throughput, "request_throughput": v.request_throughput, "total_token_throughput": v.total_token_throughput, + "input_throughput": v.input_throughput, + "intvty_p90": v.intvty_p90, + "tpot_p90_ms": v.tpot_p90_ms, "ttft_mean_ms": v.ttft_mean_ms, "e2el_mean_ms": v.e2el_mean_ms, "duration_seconds": v.duration_seconds, @@ -136,6 +139,13 @@ def _point_from_variant(v: VariantResult, *, arm: str) -> dict[str, Any]: } +def _conc_pair_use_composite(state: SharedState) -> bool: + """True when conc-sweep pairing should rank on composite score *S*.""" + from hyperloom.common.perf_metric import composite_grading_enabled + + return composite_grading_enabled(str(getattr(state, "framework", "") or "") or None) + + def _budget_limited_without_valid_pair( *, budget_exhausted: bool, @@ -175,6 +185,9 @@ def _write_csv(csv_path: Path, points: list[dict[str, Any]]) -> None: "output_throughput", "request_throughput", "total_token_throughput", + "input_throughput", + "intvty_p90", + "tpot_p90_ms", "ttft_mean_ms", "e2el_mean_ms", "duration_seconds", @@ -1092,7 +1105,9 @@ def _flush_partial_conc_sweep_report( # noqa: PLR0913 b_pts.sort(key=lambda p: p["conc"]) o_pts.sort(key=lambda p: p["conc"]) - comparison, summary = conc_pair_comparison(b_pts, o_pts) + comparison, summary = conc_pair_comparison( + b_pts, o_pts, use_composite=_conc_pair_use_composite(state) + ) p: dict[str, Any] = { "schema_version": SCHEMA_VERSION, "status": "in_progress" if partial else "unknown", @@ -1379,7 +1394,11 @@ async def run_conc_sweep( baseline_points.sort(key=lambda p: p["conc"]) optimized_points.sort(key=lambda p: p["conc"]) - comparison, summary = conc_pair_comparison(baseline_points, optimized_points) + comparison, summary = conc_pair_comparison( + baseline_points, + optimized_points, + use_composite=_conc_pair_use_composite(state), + ) budget_limited_no_pair = _budget_limited_without_valid_pair( budget_exhausted=budget_exhausted, summary=summary, diff --git a/src/hyperloom/orchestrator/kernel/request_handlers.py b/src/hyperloom/orchestrator/kernel/request_handlers.py index 0f01831a8..542a15339 100644 --- a/src/hyperloom/orchestrator/kernel/request_handlers.py +++ b/src/hyperloom/orchestrator/kernel/request_handlers.py @@ -8049,8 +8049,11 @@ def _restore_aiter_rebuild_env() -> None: new_tput = float(bench_result.get("output_throughput") or 0.0) from hyperloom.common.gain_math import gain_pct_or_zero, incremental_gain_pct + from hyperloom.common.perf_metric import keep_gain_pct - gain_pct = gain_pct_or_zero(new_tput, base_tput) + tput_gain_pct = gain_pct_or_zero(new_tput, base_tput) + gain_pct = tput_gain_pct + used_composite = False stack_positive_keep = False stack_incremental_gain_pct: float | None = None try: @@ -8059,21 +8062,44 @@ def _restore_aiter_rebuild_env() -> None: state = SharedState.load_or_init(session_dir) current_best = state.current_best or {} current_best_tput = float(current_best.get("tput") or 0.0) - if current_best_tput > 0: - stack_incremental_gain_pct = incremental_gain_pct(new_tput, current_best_tput) + framework = str(payload.get("framework") or getattr(state, "framework", "") or "") + graded, used_composite = keep_gain_pct( + bench_result, + state=state, + framework=framework, + base_tput=base_tput, + ) + if used_composite: + gain_pct = 0.0 if graded is None else float(graded) + stack_incremental_gain_pct = gain_pct + else: + gain_pct = tput_gain_pct + if current_best_tput > 0: + stack_incremental_gain_pct = incremental_gain_pct(new_tput, current_best_tput) stack_positive_keep = ( bool(state.optimization_stack) and str(current_best.get("action") or "") == "integrate" - and current_best_tput > 0 + and stack_incremental_gain_pct is not None and stack_incremental_gain_pct >= STACK_INCREMENTAL_KEEP_THRESHOLD_PCT ) + if not used_composite: + stack_positive_keep = stack_positive_keep and current_best_tput > 0 except Exception: # noqa: BLE001 - fall back to the original threshold stack_positive_keep = False - decision = ( - "KEEP" - if (gain_pct > keep_threshold_pct or stack_positive_keep) - else ("REVERT" if gain_pct < -keep_threshold_pct else "NEEDS_REVIEW") - ) + used_composite = False + gain_pct = tput_gain_pct + if used_composite: + decision = ( + "KEEP" + if (gain_pct > keep_threshold_pct or stack_positive_keep) + else ("REVERT" if gain_pct <= 0.0 else "NEEDS_REVIEW") + ) + else: + decision = ( + "KEEP" + if (gain_pct > keep_threshold_pct or stack_positive_keep) + else ("REVERT" if gain_pct < -keep_threshold_pct else "NEEDS_REVIEW") + ) # Accuracy gate: a kernel patch only KEEPs if it also holds accuracy. Graded # ONLY for a candidate that already cleared the throughput bar, so a @@ -8182,6 +8208,9 @@ def _restore_aiter_rebuild_env() -> None: "base_tput": base_tput, "new_tput": new_tput, "gain_pct": gain_pct, + "input_throughput": bench_result.get("input_throughput"), + "intvty_p90": bench_result.get("intvty_p90"), + "tpot_p90_ms": bench_result.get("tpot_p90_ms"), "report_path": bench_result.get("report_path"), "workspace": bench_result.get("workspace"), "extra_server_args": extra_args, diff --git a/src/hyperloom/orchestrator/loop/coordinator.py b/src/hyperloom/orchestrator/loop/coordinator.py index 6be09d17f..57b3c9b7d 100644 --- a/src/hyperloom/orchestrator/loop/coordinator.py +++ b/src/hyperloom/orchestrator/loop/coordinator.py @@ -1011,6 +1011,7 @@ def router(self) -> IntentRouter: "_run_kernel_opt_entry_batch": "phase_kernel", "_current_tput_from_validated_gain": "phase_kernel", "_last_measured_roofline_tput": "phase_kernel", + "_roofline_watermark_levels": "phase_kernel", "_needs_roofline_for_watermark": "phase_kernel", "_maybe_enqueue_watermark_roofline": "phase_kernel", "_cached_kernel_request": "phase_kernel", diff --git a/src/hyperloom/orchestrator/loop/writeback.py b/src/hyperloom/orchestrator/loop/writeback.py index da12df623..8573b4c0a 100644 --- a/src/hyperloom/orchestrator/loop/writeback.py +++ b/src/hyperloom/orchestrator/loop/writeback.py @@ -12,6 +12,7 @@ from types import SimpleNamespace from typing import Any, Mapping from hyperloom.common.coerce import to_float, to_str_list +from hyperloom.common.perf_metric import perf_axes_from_mapping from hyperloom.common.io import append_jsonl from hyperloom.inference_optimizer.breakdown.agent_ownership import ( patch_owner_phase, @@ -505,7 +506,10 @@ def _update_cumulative_gain_validated( Call only when ``baseline_tput > 0`` and ``new_tput`` is a positive measured throughput. The caller remains responsible for any surrounding - guard (e.g. ``if self.shared_state.baseline_tput > 0``). + guard (e.g. ``if self.shared_state.baseline_tput > 0``). Flag-on + serving runs with a full triple store ``S * 100`` vs the session + baseline; otherwise this is still + ``(new_tput - baseline_tput) / baseline_tput * 100``. Args: new_tput: The newly measured throughput to promote as the validated @@ -518,7 +522,14 @@ def _update_cumulative_gain_validated( ts: Author-time stamp the caller already minted for this promotion; defaults to now. """ - validated_gain = (float(new_tput) - self.shared_state.baseline_tput) / self.shared_state.baseline_tput * 100.0 + from hyperloom.common.perf_metric import session_gain_from_measurement + + gain, _used = session_gain_from_measurement( + float(new_tput), + state=self.shared_state, + base_tput=self.shared_state.baseline_tput, + ) + validated_gain = float(gain if gain is not None else 0.0) ts = str(ts or datetime.now(timezone.utc).isoformat()) self.shared_state.cumulative_gain_validated = float(validated_gain) self.shared_state.cumulative_gain_validated_ts = ts @@ -576,6 +587,7 @@ async def _record_integrate_keep(self, result: dict[str, Any]) -> None: "e2el_mean_ms": result.get("e2el_mean_ms"), "tpot_mean_ms": result.get("tpot_mean_ms"), "workspace": result.get("workspace"), + **perf_axes_from_mapping(result), }, gap_canonical_id=str(result.get("gap_canonical_id") or "").strip(), entry_extra={ @@ -2572,7 +2584,31 @@ def _lift_to_current_best( for not beating the current anchor. """ anchor = resolve_grading_anchor_tput(self.shared_state) - if anchor > 0 and float(best_tput) <= anchor: + framework = str(getattr(self.shared_state, "framework", "") or "") + from hyperloom.common.perf_metric import ( + composite_grading_enabled, + composite_score, + perf_snapshot_from_mapping, + resolve_baseline_perf, + ) + + baseline_perf = resolve_baseline_perf(self.shared_state) + use_composite = composite_grading_enabled(framework) and bool(baseline_perf) + previous = self.shared_state.current_best or {} + cand_snap = perf_snapshot_from_mapping(bv) if isinstance(bv, dict) else None + anchor_snap = perf_snapshot_from_mapping(previous) or baseline_perf + if use_composite and cand_snap and anchor_snap: + cand_score = composite_score(cand_snap, baseline_perf) + anchor_score = composite_score(anchor_snap, baseline_perf) + if cand_score <= anchor_score: + log.info( + "current_best held at score %.4f: %s winner scored %.4f (no lift)", + anchor_score, + task_kind, + cand_score, + ) + return False + elif anchor > 0 and float(best_tput) <= anchor: log.info( "current_best held at %.1f: %s winner measured %.1f (no lift)", anchor, @@ -2580,7 +2616,8 @@ def _lift_to_current_best( float(best_tput), ) return False - previous = self.shared_state.current_best or {} + if not isinstance(previous, dict): + previous = {} base_args = "" if isinstance(previous, dict): base_args = strip_benchmark_harness_flags(previous.get("extra_server_args")) @@ -2764,6 +2801,7 @@ def _lift_to_current_best( variant_name=variant_name, new_tput=best_tput, extra_server_args=full_args, + candidate=bv if isinstance(bv, dict) else None, ) # Merge envs: start from previous stack top envs so source-layer KEEPs @@ -2785,8 +2823,13 @@ def _lift_to_current_best( "ttft_mean_ms": bv.get("ttft_mean_ms") if isinstance(bv, dict) else None, "e2el_mean_ms": bv.get("e2el_mean_ms") if isinstance(bv, dict) else None, "tpot_mean_ms": bv.get("tpot_mean_ms") if isinstance(bv, dict) else None, + "input_throughput": (bv.get("input_throughput") if isinstance(bv, dict) else None), + "tpot_p90_ms": (bv.get("tpot_p90_ms") if isinstance(bv, dict) else None), + "intvty_p90": (bv.get("intvty_p90") if isinstance(bv, dict) else None), "workspace": bv.get("workspace") if isinstance(bv, dict) else None, } + if use_composite and cand_snap and baseline_perf: + current_best["perf_score"] = composite_score(cand_snap, baseline_perf) if isinstance(bv, dict): for _ctrl_key in ("remove_args", "unset_envs", "args_mode"): if bv.get(_ctrl_key): @@ -3089,9 +3132,11 @@ async def _promote_baseline( # must not reset it back to the bare reference config. if anchor_accepted and not (getattr(self.shared_state, "optimization_stack", None) or []): anchor_tput = float(self.shared_state.baseline_tput or 0.0) - self.shared_state.current_best = { + current_best = { "action": "baseline", - "tput": (anchor_tput if anchor_tput > 0 else (float(tput) if isinstance(tput, (int, float)) else None)), + "tput": ( + anchor_tput if anchor_tput > 0 else (float(tput) if isinstance(tput, (int, float)) else None) + ), "hot_tput": (float(tput) if isinstance(tput, (int, float)) else None), "cold_tput": ( float(warmup_anchor) if isinstance(warmup_anchor, (int, float)) and warmup_anchor > 0 else None @@ -3099,8 +3144,22 @@ async def _promote_baseline( "ttft_mean_ms": result.get("ttft_mean_ms"), "e2el_mean_ms": result.get("e2el_mean_ms"), "tpot_mean_ms": result.get("tpot_mean_ms"), + "input_throughput": result.get("input_throughput"), + "tpot_p90_ms": result.get("tpot_p90_ms"), + "intvty_p90": result.get("intvty_p90"), "workspace": result.get("workspace"), } + from hyperloom.common.perf_metric import composite_score, perf_snapshot_from_mapping + + snap = perf_snapshot_from_mapping(result) + if snap: + self.shared_state.baseline_perf = dict(snap) + current_best["input_throughput"] = snap["input_throughput"] + current_best["intvty_p90"] = snap["intvty_p90"] + if snap.get("tpot_p90_ms") is not None: + current_best["tpot_p90_ms"] = snap["tpot_p90_ms"] + current_best["perf_score"] = composite_score(snap, snap) + self.shared_state.current_best = current_best changed = True if anchor_accepted: audit_decision = "promoted" @@ -3426,11 +3485,12 @@ async def _promote_profile( audit_extras["framework_rewrite_evidence"] = evidence_path audit_extras["framework_rewrite_candidate_count"] = result.get("framework_rewrite_candidate_count") changed = True - # On a successful profile, re-anchor last_roofline_tput and clear the pending field. + # On a successful profile, re-anchor last_roofline_tput (and *S*) + # and clear the pending field. if profile_status == "succeeded": anchor_tput = self._current_tput_from_validated_gain() if anchor_tput > 0: - self.shared_state.last_roofline_tput = float(anchor_tput) + self.shared_state.stamp_roofline_watermark(anchor_tput) changed = True if task is not None and self.shared_state.auto_roofline_pending_task_id == task.task_id: self.shared_state.auto_roofline_pending_task_id = "" @@ -3481,9 +3541,10 @@ async def _promote_roofline( # Reset the roofline failure streak on a successful snapshot. if hasattr(self.shared_state, "roofline_failure_streak"): self.shared_state.roofline_failure_streak = 0 - # Re-anchor the 10% watermark step on the projected current tput -- - # but only for a roofline that actually produced an analysis. The - # anchor is what stops the watermark firing again until throughput + # Re-anchor the 10% watermark step on the projected current tput + # (and *S* when the composite flag is on) -- but only for a + # roofline that actually produced an analysis. The anchor is what + # stops the watermark firing again until the comparable quantity # climbs another 10%, so anchoring on an empty one buys a whole # cycle of silence for a snapshot that says nothing: the specialist # keeps reading "(none)" while the anchor insists a roofline was @@ -3492,7 +3553,7 @@ async def _promote_roofline( if str((self.shared_state.last_trace_analyze or {}).get("analysis_md_text") or ""): anchor_tput = self._current_tput_from_validated_gain() if anchor_tput > 0: - self.shared_state.last_roofline_tput = float(anchor_tput) + self.shared_state.stamp_roofline_watermark(anchor_tput) else: log.warning( "roofline %s produced no analysis; leaving the watermark " @@ -4039,6 +4100,7 @@ async def _promote_integrate_patch( # Durable source-layer handles so current_best stays relaunchable # and reproducible in the GEAK baseline. **_source_layer_handles(result), + **perf_axes_from_mapping(result), } if source_phase: lift["source_phase"] = source_phase @@ -4220,6 +4282,7 @@ async def _promote_framework_agent( # if writeback runs after the state machine has advanced. "source_phase": "FRAMEWORK_AGENT", "provenance": "framework_agent", + **perf_axes_from_mapping(result), } lifted = self._lift_to_current_best(_FRAMEWORK_STACK_ACTION, float(new_tput), lift) if lifted and self.shared_state.baseline_tput > 0: @@ -5398,6 +5461,8 @@ async def _validate_geak_via_geak_harness(self, *, reason: str) -> dict[str, Any # Single-point validated replay pins the headline protocol (num_prompts # etc.) so it is protocol-identical to the reported result. pin_num_prompts=True, + framework=str(getattr(self.shared_state, "framework", "") or ""), + state=self.shared_state, ) if str(res.get("status") or "") == "succeeded" and geak_sp > 1.0: # Rebench-first: write the headline from the GEAK-harness MEASURED diff --git a/src/hyperloom/orchestrator/phases/kernel.py b/src/hyperloom/orchestrator/phases/kernel.py index 9b455a716..727ce7e1f 100644 --- a/src/hyperloom/orchestrator/phases/kernel.py +++ b/src/hyperloom/orchestrator/phases/kernel.py @@ -30,6 +30,13 @@ ) from ..state.task_registry import TERMINAL_STATES from ..bus.message_bus import Message +from hyperloom.common.perf_metric import ( + composite_grading_enabled, + composite_score, + perf_axes_from_mapping, + perf_snapshot_from_mapping, + resolve_baseline_perf, +) from ..loop.coordinator_helpers import ( _GEAK_MEASUREMENT_DIVERGENCE_WARN_PCT, _MAX_ROOFLINE_FAILURE_RETRIES, @@ -378,7 +385,7 @@ async def _maybe_reprofile_for_kernel(self) -> None: after != before or snapshots_after != snapshots_before or snapshot_id_after != snapshot_id_before ) if after > 0 and snapshot_landed: - self.shared_state.last_roofline_tput = after + self.shared_state.stamp_roofline_watermark(after) self.shared_state.last_profile_status = "succeeded" # Record the workload (incl. serving_config) that this trace reflects; # _profile_config_changed derives the config signature from it, so @@ -1499,22 +1506,44 @@ def _promote_geak_from_candidate( return if measured <= 0: return - # KEEP guard (aligns GEAK with forge / integrate_patch): a measured + # KEEP guard (aligns GEAK with forge / integrate_handler): a measured # rebench that does not beat the current best must NOT overwrite the # headline / stack / gain. Backstops every promote entry point (2a, 2b, # crash-recovery) so a low-but-valid measurement can never lower best. cb_now = self.shared_state.current_best if isinstance(self.shared_state.current_best, dict) else {} cb_tput = cb_now.get("tput") - if isinstance(cb_tput, (int, float)) and cb_tput > 0 and measured <= float(cb_tput): + framework = str(getattr(self.shared_state, "framework", "") or "") + baseline_perf = resolve_baseline_perf(self.shared_state) + cand_snap = perf_snapshot_from_mapping( + { + "output_throughput": measured, + "input_throughput": result.get("input_throughput"), + "intvty_p90": result.get("intvty_p90"), + "tpot_p90_ms": result.get("tpot_p90_ms"), + } + ) + anchor_snap = perf_snapshot_from_mapping(cb_now) or baseline_perf + composite_keep = bool( + composite_grading_enabled(framework) + and cand_snap is not None + and anchor_snap is not None + and baseline_perf is not None + ) + if composite_keep: + beats_best = composite_score(cand_snap, baseline_perf) > composite_score(anchor_snap, baseline_perf) + else: + beats_best = not (isinstance(cb_tput, (int, float)) and cb_tput > 0 and measured <= float(cb_tput)) + if not beats_best: + cb_tput_f = float(cb_tput) if isinstance(cb_tput, (int, float)) else 0.0 log.info( "geak promote skipped: measured %.3f did not beat current_best %.3f", measured, - float(cb_tput), + cb_tput_f, ) self._reject_geak_kernel_journey( result, measured_tput=measured, - current_best_tput=float(cb_tput), + current_best_tput=cb_tput_f, provenance="geak_promote_rejected", ) try: @@ -1525,7 +1554,7 @@ def _promote_geak_from_candidate( "decision": "REJECTED", "reason": "rebench_did_not_beat_current_best", "measured_tput": measured, - "current_best_tput": float(cb_tput), + "current_best_tput": cb_tput_f, } instrument.record_geak_operation( self.session_dir, @@ -1559,6 +1588,7 @@ def _promote_geak_from_candidate( "ttft_mean_ms": result.get("ttft_ms"), "tpot_mean_ms": result.get("tpot_ms"), "workspace": result.get("eval_dir"), + **perf_axes_from_mapping({**result, "output_throughput": measured, "tput": measured}), }, entry_extra=self._geak_stack_entry_extra(result, overlay_loaded=overlay_loaded), ) @@ -3233,7 +3263,7 @@ async def _validate_gemm_tuning_e2e(self, result: dict[str, Any]) -> None: applied.get("detail"), ) - if decision == "KEEP" and new_tput > running_tput and not apply_blockers: + if decision == "KEEP" and not apply_blockers: stacked_envs.update(env) running_tput = new_tput kept.append( @@ -3259,6 +3289,7 @@ async def _validate_gemm_tuning_e2e(self, result: dict[str, Any]) -> None: "extra_envs": dict(env), "source_phase": "KERNEL_AGENT", "workspace": result.get("workspace"), + **perf_axes_from_mapping(integrate_verdict), }, entry_extra={ "tuned_file": adopted_tuned_file, @@ -4019,6 +4050,7 @@ def _promote_collective_integrate_keep( "source_phase": "KERNEL_AGENT", "provenance": "forge_collective", "workspace": integrate_result.get("workspace"), + **perf_axes_from_mapping(integrate_result), }, entry_extra={ "backend": "forge", @@ -4267,6 +4299,7 @@ def _promote_fusion_integrate_keep( "source_phase": "KERNEL_AGENT", "provenance": "forge_fusion", "workspace": integrate_result.get("workspace"), + **perf_axes_from_mapping(integrate_result), }, entry_extra={ "backend": "forge", @@ -4285,19 +4318,32 @@ def _promote_fusion_integrate_keep( ) def _current_tput_from_validated_gain(self) -> float: - """Project current tput from ``baseline_tput * (1 + cumulative_gain_validated/100)``; 0.0 when baseline unknown (watermark not-yet-armed). + """Return the live measured tput used for watermark math. + + Prefers ``current_best`` output tok/s. Flag-off fallback reconstructs + from ``baseline_tput * (1 + cumulative_gain_validated/100)``. Composite + ``cumulative_gain_validated`` is *S* and must not be inverted. Returns: - The projected current throughput, or ``0.0`` when the baseline is - unknown. + The measured/projected current throughput, or ``0.0`` when unknown. """ state = self.shared_state + cb = getattr(state, "current_best", None) + if isinstance(cb, dict): + for key in ("output_throughput", "tput"): + raw = cb.get(key) + if isinstance(raw, (int, float)) and not isinstance(raw, bool) and float(raw) > 0: + return float(raw) try: base = float(state.baseline_tput or 0.0) except (TypeError, ValueError): base = 0.0 if base <= 0: return 0.0 + from hyperloom.common.perf_metric import composite_grading_enabled + + if composite_grading_enabled(str(getattr(state, "framework", "") or "") or None): + return 0.0 try: gain = float(state.cumulative_gain_validated or 0.0) except (TypeError, ValueError): @@ -4318,14 +4364,32 @@ def _last_measured_roofline_tput(self) -> float: return tput return 0.0 + def _roofline_watermark_levels(self, last_rl: float) -> tuple[float, float]: + """Comparable ``(current, last)`` pair for the 10% roofline watermark. + + Flag on with a full triple: ``(1+S_now, 1+S_at_last_snapshot)``. + Otherwise output tok/s vs ``last_rl``. + """ + from hyperloom.common.perf_metric import composite_watermark_levels + + levels = composite_watermark_levels(self.shared_state) + if levels is not None: + return levels + return self._current_tput_from_validated_gain(), last_rl + def _needs_roofline_for_watermark(self) -> bool: - """True iff projected tput crossed the watermark over ``last_roofline_tput`` (False until PRELUDE roofline ran, or while auto_roofline_pending_task_id is in-flight). + """True iff the comparable watermark crossed over the last snapshot. + + Flag off: projected output tput vs ``last_roofline_tput``. Flag on + with a full triple: ``(1+S)`` vs ``(1+last_roofline_score)``. False + until PRELUDE roofline ran, or while auto_roofline_pending_task_id is + in-flight. Returns: - ``True`` when a fresh roofline is warranted because projected tput - crossed the watermark ratio; ``False`` otherwise (including the - bootstrap and in-flight re-arm guards, and once the failure streak - has exhausted ``_MAX_ROOFLINE_FAILURE_RETRIES``). + ``True`` when a fresh roofline is warranted because the comparable + quantity crossed the watermark ratio; ``False`` otherwise + (including the bootstrap and in-flight re-arm guards, and once the + failure streak has exhausted ``_MAX_ROOFLINE_FAILURE_RETRIES``). """ state = self.shared_state try: @@ -4349,10 +4413,10 @@ def _needs_roofline_for_watermark(self) -> bool: last_rl = 0.0 if last_rl <= 0: return False - cur = self._current_tput_from_validated_gain() - if cur <= 0: + cur, last = self._roofline_watermark_levels(last_rl) + if cur <= 0 or last <= 0: return False - return cur / last_rl >= _resolve_roofline_watermark_ratio() + return cur / last >= _resolve_roofline_watermark_ratio() async def _release_finished_roofline_gate(self) -> None: """Drop an in-flight marker that names a roofline which already finished. @@ -4405,12 +4469,17 @@ async def _maybe_enqueue_watermark_roofline( ) return False self.shared_state.auto_roofline_pending_task_id = task.task_id + try: + last_rl = float(self.shared_state.last_roofline_tput or 0.0) + except (TypeError, ValueError): + last_rl = 0.0 + cur, last = self._roofline_watermark_levels(last_rl) log.info( "watermark-roofline (%s): enqueued task=%s (cur=%.2f, last_roofline=%.2f, ratio>=%.2f)", reason, task.task_id, - self._current_tput_from_validated_gain(), - float(self.shared_state.last_roofline_tput or 0.0), + cur, + last, self._ROOFLINE_WATERMARK_RATIO, ) return True diff --git a/src/hyperloom/orchestrator/phases/kernel_stack.py b/src/hyperloom/orchestrator/phases/kernel_stack.py index fa3d6a0c2..c5c8a5527 100644 --- a/src/hyperloom/orchestrator/phases/kernel_stack.py +++ b/src/hyperloom/orchestrator/phases/kernel_stack.py @@ -483,16 +483,41 @@ async def _run_kernel_stack_validation_e2e( new_tput = 0.0 gain_pct = -100.0 incremental_gain_pct = -100.0 + perf_axes: dict[str, float] = {} else: + from hyperloom.common.perf_metric import keep_gain_pct, perf_axes_from_mapping + base_tput = float(self.shared_state.baseline_tput or 0.0) - # The stack is applied on top of current_best, so the KEEP - # decision uses the incremental gain over current_best, not the - # total gain over the original baseline. + # The stack is applied on top of current_best. KEEP is that + # incremental gain: composite *S* when the flag and triples are + # present, otherwise output-tput % vs the live tput anchor. decision_base = resolve_grading_anchor_tput(self.shared_state) new_tput = float(bench_result.get("output_throughput") or 0.0) - gain_pct = (new_tput - base_tput) / base_tput * 100.0 if base_tput > 0 else 0.0 - incremental_gain_pct = (new_tput - decision_base) / decision_base * 100.0 if decision_base > 0 else 0.0 - decision = "KEEP" if incremental_gain_pct > KERNEL_STACK_VALIDATION_KEEP_THRESHOLD_PCT else "REVERT" + tput_gain_pct = (new_tput - base_tput) / base_tput * 100.0 if base_tput > 0 else 0.0 + tput_incremental = ( + (new_tput - decision_base) / decision_base * 100.0 if decision_base > 0 else 0.0 + ) + framework = str(getattr(self.shared_state, "framework", "") or "") + graded, used_composite = keep_gain_pct( + bench_result if isinstance(bench_result, dict) else None, + state=self.shared_state, + framework=framework, + base_tput=decision_base, + ) + if used_composite: + incremental_gain_pct = 0.0 if graded is None else float(graded) + gain_pct = incremental_gain_pct + else: + incremental_gain_pct = tput_incremental + gain_pct = tput_gain_pct + decision = ( + "KEEP" + if incremental_gain_pct > KERNEL_STACK_VALIDATION_KEEP_THRESHOLD_PCT + else "REVERT" + ) + perf_axes = perf_axes_from_mapping( + bench_result if isinstance(bench_result, dict) else None + ) # bench_result already carries accuracy (RUN_EVAL defaults true here). if decision == "KEEP" and isinstance(bench_result, dict): @@ -541,6 +566,7 @@ async def _run_kernel_stack_validation_e2e( } result = { + **perf_axes, "status": top_status, "decision": decision, "patch_cleanup_status": cs, @@ -550,6 +576,8 @@ async def _run_kernel_stack_validation_e2e( "target_file": "+".join(str(e.get("target_file") or "") for e in entries), "base_tput": float(self.shared_state.baseline_tput or 0.0), "new_tput": new_tput, + "output_throughput": new_tput, + "tput": new_tput, "gain_pct": gain_pct, "stack_incremental_gain_pct": incremental_gain_pct, "stack_incremental_keep_threshold_pct": (KERNEL_STACK_VALIDATION_KEEP_THRESHOLD_PCT), diff --git a/src/hyperloom/orchestrator/phases/prelude.py b/src/hyperloom/orchestrator/phases/prelude.py index e43f94c1a..71412402c 100644 --- a/src/hyperloom/orchestrator/phases/prelude.py +++ b/src/hyperloom/orchestrator/phases/prelude.py @@ -2028,7 +2028,19 @@ def _promote_warm_replay( # nothing about whether it still computes correctly here. if not self._warm_replay_accuracy_ok(result, task, outcome): return - measured_gain = (single_round_tput / baseline_tput - 1.0) * 100.0 + from hyperloom.common.perf_metric import keep_gain_pct, perf_axes_from_mapping + + tput_gain = (single_round_tput / baseline_tput - 1.0) * 100.0 + graded, used_composite = keep_gain_pct( + result, + state=state, + framework=str(getattr(state, "framework", "") or ""), + base_tput=baseline_tput, + ) + # Flag on + full triples: grade the same *S* KEEP other serving + # surfaces use. Otherwise the historical output-tput % vs this + # session's baseline. ``None`` means *S* did not improve. + measured_gain = (0.0 if graded is None else float(graded)) if used_composite else tput_gain result["combined_gain_pct"] = round(measured_gain, 3) decision_params = (task.params if task is not None else {}) or {} combined_current_contract = bool(decision_params.get("combined_current_contract")) @@ -2054,9 +2066,12 @@ def _promote_warm_replay( outcome["actual_gain_pct"] = round(measured_gain, 3) outcome["throughput_after"] = tput outcome["keep_threshold_pct"] = keep_threshold + outcome["used_composite"] = used_composite if expected_gain > 0: historical_bar = expected_gain * min_reproduce - if measured_gain > 0 and measured_gain < historical_bar: + # Donor ``expected_gain_pct`` is an output-tput claim, so this + # advisory flag stays on tput even when KEEP graded *S*. + if tput_gain > 0 and tput_gain < historical_bar: outcome["below_historical_reproduce_pct"] = True outcome["historical_reproduce_bar_pct"] = round( historical_bar, @@ -2189,6 +2204,7 @@ def _promote_warm_replay( "replay_warm_recipe", float(single_round_tput), { + **perf_axes_from_mapping(result), "name": "warm_replay", "candidate_extra_server_args": warm_args, "candidate_extra_envs": warm_envs, @@ -2217,8 +2233,9 @@ def _promote_warm_replay( if baseline_tput > 0: self._update_cumulative_gain_validated(single_round_tput) log.info( - "warm-replay REPRODUCED: measured=+%.2f%% (expected=+%.2f%%, " + "warm-replay REPRODUCED: %s=+%.2f%% (expected=+%.2f%%, " "min_required=+%.2f%%); pushed warm_replay onto stack", + "composite gain" if used_composite else "throughput delta", measured_gain, expected_gain, expected_gain * min_reproduce if expected_gain > 0 else 0.0, @@ -2256,9 +2273,11 @@ def _promote_warm_replay( } outcome["kernel"] = dict(kernel_outcome) outcome["status"] = "drift" - outcome["reason"] = f"measured {measured_gain:+.2f}% below keep threshold {keep_threshold:+.2f}%" + metric = "composite gain" if used_composite else "throughput delta" + outcome["reason"] = f"{metric} {measured_gain:+.2f}% below keep threshold {keep_threshold:+.2f}%" log.info( - "warm-replay DRIFT: measured=%+.2f%% threshold=%+.2f%%", + "warm-replay DRIFT: %s=%+.2f%% threshold=%+.2f%%", + metric, measured_gain, keep_threshold, ) diff --git a/src/hyperloom/orchestrator/state/objective.py b/src/hyperloom/orchestrator/state/objective.py index f45edc0e0..696a99c0e 100644 --- a/src/hyperloom/orchestrator/state/objective.py +++ b/src/hyperloom/orchestrator/state/objective.py @@ -118,7 +118,12 @@ def gap_pct(self, state: "SharedState") -> float: @dataclass class TargetGainObjective(_RatioObjective): - """Reach ``target_gain_pct`` % over baseline_tput (progress = cumulative_gain_validated / target, capped at 1.0).""" + """Reach ``target_gain_pct`` % over the session baseline. + + Progress is ``cumulative_gain_validated / target``, capped at 1.0. + That validated figure is output-tput % by default, or composite *S* × 100 + when ``HYPERLOOM_PERF_METRIC=composite_v1``. + """ target_gain_pct: float diff --git a/src/hyperloom/orchestrator/state/shared_state.py b/src/hyperloom/orchestrator/state/shared_state.py index 22db84cc2..febaaba40 100644 --- a/src/hyperloom/orchestrator/state/shared_state.py +++ b/src/hyperloom/orchestrator/state/shared_state.py @@ -26,6 +26,7 @@ per-writer detail (variant_name, extra_server_args, extra_envs, workspace, latency means) cumulative_gain_validated float — % over baseline at the last full-stack rebench + (output-tput % , or composite *S* × 100 when the flag is on) stop_reason str — set when graceful stop fires stop_ts str — ISO timestamp of the first stop_reason write resumed_ts str — ISO timestamp of the most recent --resume @@ -579,6 +580,8 @@ class SharedState(_RenderMixin, _ExploreStateMixin): conc_sweep_variant_timeout_sec: int = 1800 target_summary: str = "" baseline_tput: float = 0.0 + # Baseline perf triple (input/output tput + intvty p90) for composite grading. + baseline_perf: dict[str, Any] = field(default_factory=dict) # Internal-only baseline cold+hot double-run switch; default-on keeps EXPLORE # warm-decision apples-to-apples with the baseline measurement basis. baseline_double_run: bool = True @@ -693,8 +696,10 @@ class SharedState(_RenderMixin, _ExploreStateMixin): optimization_stack: list[dict[str, Any]] = field(default_factory=list) # Index-aligned with ``optimization_stack``: per-entry incremental gain pct; missing => None. gain_per_stack_entry: list[float | None] = field(default_factory=list) - # Total gain over ``baseline_tput``, stamped only from a measurement taken + # Total gain over the session baseline, stamped only from a measurement taken # with the whole stack applied; standalone validate_stack denied by PolicyGate. + # Flag off: ``(tput - baseline_tput) / baseline_tput * 100``. Flag on + + # full triple: composite score *S* × 100. cumulative_gain_validated: float = 0.0 cumulative_gain_validated_ts: str = "" # ``optimization_stack`` length at last successful inline rebench; longer => new KEEPs need validation. @@ -710,8 +715,12 @@ class SharedState(_RenderMixin, _ExploreStateMixin): # main-flow rebench; kept OUT of current_best / optimization_stack / the # headline gain until validated. Cleared once promoted from a measured rebench. geak_pending: dict[str, Any] = field(default_factory=dict) - # Tput watermark for gain-driven roofline refresh; Coordinator re-enqueues at a compound 10% step. + # Tput (or 1+S when the composite flag is on) watermark for gain-driven + # roofline refresh; Coordinator re-enqueues at a compound 10% step. last_roofline_tput: float = 0.0 + # *S* at the last successful roofline; None until a composite-capable + # snapshot has been stamped (PRELUDE / resume treat None as 0). + last_roofline_score: float | None = None stop_reason: str = "" # When the session first stopped, and therefore its end time for # consumers. Stamped by the first ``set_stop_reason`` write and left alone @@ -3674,8 +3683,13 @@ def append_stack_gain_entry( new_tput: float, extra_server_args: str = "", ts: str | None = None, + candidate: Mapping[str, Any] | None = None, ) -> float | None: - """Mirror an optimization_stack append into gain_per_stack_entry; computes ``(new_tput-baseline_tput)/baseline_tput*100`` and appends. Returns gain_pct (None when baseline_tput is 0 or new_tput non-positive). + """Mirror an optimization_stack append into gain_per_stack_entry. + + Flag off: ``(new_tput-baseline_tput)/baseline_tput*100``. Flag on with + a full triple: ``S * 100`` vs the session baseline. Returns gain_pct + (None when baseline_tput is 0 or new_tput non-positive on the tput path). Args: action (str): The action that produced the stack entry. @@ -3683,10 +3697,13 @@ def append_stack_gain_entry( new_tput (float): The measured throughput for the entry. extra_server_args (str): The extra server args for the entry. ts (str | None): Optional ISO timestamp for the entry. + candidate (Mapping | None): Perf axes for the entry when known + (used for composite session accounting). Returns: - float | None: The computed incremental gain pct, or ``None`` when - ``baseline_tput`` is 0 or ``new_tput`` is non-positive. + float | None: The computed session-total gain pct, or ``None`` when + ``baseline_tput`` is 0 or ``new_tput`` is non-positive on the + output-tput path. """ try: base = float(self.baseline_tput or 0.0) @@ -3696,12 +3713,32 @@ def append_stack_gain_entry( tput = float(new_tput or 0.0) except (TypeError, ValueError): tput = 0.0 - from hyperloom.common.gain_math import gain_pct + from hyperloom.common.perf_metric import session_gain_from_measurement - entry_gain_pct = gain_pct(tput, base) + entry_gain_pct, _used = session_gain_from_measurement( + tput, + state=self, + candidate=candidate, + base_tput=base, + ) self.gain_per_stack_entry.append(entry_gain_pct) return entry_gain_pct + def stamp_roofline_watermark(self, tput: float | None = None) -> None: + """Re-anchor the 10% roofline watermark after a successful snapshot. + + Writes ``last_roofline_tput`` when ``tput`` is positive. When the + composite flag is on and the session has a full triple, also stamps + ``last_roofline_score`` to the current *S*. + """ + if isinstance(tput, (int, float)) and not isinstance(tput, bool) and float(tput) > 0: + self.last_roofline_tput = float(tput) + from hyperloom.common.perf_metric import session_composite_score + + score = session_composite_score(self) + if score is not None: + self.last_roofline_score = float(score) + # Time-budget helpers (consumed by Coordinator._compose_prompt) def elapsed_minutes(self, *, now: datetime | None = None) -> float: """Wall-clock minutes since ``start_ts`` (0.0 when empty/unparseable).