fix(agentx): close AgentX round timeout gaps + overhead-cap visibility

src/hyperloom/inference_optimizer/assets/agentx/aiperf_client.sh

@@ -255,6 +255,16 @@ FRT="${AGENTX_FAILED_REQUEST_THRESHOLD:-0.10}"
 # leaderboard measurement -- by construction rather than by promise.
 CANON_ENTRIES=393
 CANON_DURATION=3600
+# Warmup is measurement-defining and was missing from this list until a measured
+# run exposed the gap: the agentic warmup is what puts the KV/radix cache under
+# realistic pressure before the window opens, so replaying at 1 request/lane
+# instead of 10 measures a materially emptier cache. It carries no scenario
+# marker either -- aiperf has no concept of "how much warmup is enough" -- so a
+# reduced-warmup round came back submission_valid=true and looked publishable.
+# On a 743B model the canonical 10/lane is a ~2h warmup, which is exactly when
+# an operator reaches for this knob, so the hole was reachable in practice.
+CANON_WARMUP_PER_LANE=10
+CANON_WARMUP_GRACE=1800
 # The corpus this model family canonically replays, before any operator pin.
 # CANON_DS is resolved with the corpus above. The family whitelist behind it is
 # a derivation, not a registry -- a model upstream runs on the full corpus but
@@ -274,6 +284,10 @@ NONCANON=()
 [ "$DURATION" != "$CANON_DURATION" ] && NONCANON+=("duration=${DURATION}s(canonical ${CANON_DURATION}s)")
 [ -n "${AGENTX_MAX_CTX:-}" ] && NONCANON+=("client_context_cap=${AGENTX_MAX_CTX}")
 [ "${AGENTX_UNSAFE_OVERRIDE:-false}" = "true" ] && NONCANON+=("unsafe_override_forced")
+[ "$WARMLANE" != "$CANON_WARMUP_PER_LANE" ] && \
+  NONCANON+=("warmup_per_lane=${WARMLANE}(canonical ${CANON_WARMUP_PER_LANE})")
+[ "$WARMGRACE" != "$CANON_WARMUP_GRACE" ] && \
+  NONCANON+=("warmup_grace=${WARMGRACE}s(canonical ${CANON_WARMUP_GRACE}s)")
 
 SMOKE_ARGS=()
 if [ "$DURATION" -lt "$CANON_DURATION" ] || [ "${AGENTX_UNSAFE_OVERRIDE:-false}" = "true" ]; then
@@ -348,6 +362,24 @@ if [ "${PROFILE:-0}" = "1" ]; then
   # the upstream profile it lands squarely inside setup and captures nothing.
   PWARM="${AGENTX_PROFILE_WARMUP_S:-2700}"
   PWIN="${AGENTX_PROFILE_WINDOW_S:-20}"
+  # The delay is a blind wall clock: it does not know which phase aiperf is in,
+  # and the two ways to get it wrong are NOT symmetric. Opening late is fatal --
+  # aiperf exits, the branch below only logs a warning, and the round produces no
+  # trace at all. Opening early merely captures a still-loaded system slightly
+  # before steady state, which TraceLens can still use. Measured: a 743B model
+  # spends ~2.5h in the agentic warmup, so a delay tuned on a 35B round lands
+  # either mid-warmup or past the end depending on which way the estimate erred.
+  #
+  # So clamp toward "early". The round cannot outlast the measurement window plus
+  # the warmup that precedes it, and the only number known here is the window, so
+  # cap the delay at DURATION - PWIN - margin and say when the cap bites. A
+  # capture inside warmup is a usable trace; a capture that never happens is not.
+  _pmax=$(( DURATION - PWIN - 60 ))
+  [ "$_pmax" -lt 0 ] && _pmax=0
+  if [ "$PWARM" -gt "$_pmax" ]; then
+    log "WARN profile delay ${PWARM}s exceeds the safe bound for a ${DURATION}s window; clamping to ${_pmax}s so the capture cannot land after the round ends"
+    PWARM="$_pmax"
+  fi
   log "PROFILE=1: self-bracketing profile window (delay=${PWARM}s window=${PWIN}s)"
   run_aiperf & APID=$!
   sleep "$PWARM"

src/hyperloom/inference_optimizer/tests/test_agentx_baseline_timeout.py

@@ -83,6 +83,27 @@ def test_overhead_budget_is_tunable(monkeypatch):
     assert agentx_baseline_timeout_sec() == AGENTX_DEFAULT_DURATION_SEC + 3600
 
 
+def test_default_overhead_warns_it_may_not_fit_every_model(monkeypatch, caplog):
+    """A raw aiperf run against Kimi-K3 (conc=64) measured warmup alone taking
+    ~12075s -- longer than this whole default cap. Nothing here can tell a
+    long-context/slow-prefill model apart from GLM-5.2/Qwen3.8, the models this
+    constant was measured on, so the gap must be surfaced instead of silently
+    assumed to fit every model.
+    """
+    _clear(monkeypatch)
+    with caplog.at_level("WARNING"):
+        agentx_baseline_timeout_sec()
+    assert any("AGENTX_BASELINE_OVERHEAD_SEC" in r.message for r in caplog.records)
+
+
+def test_explicit_overhead_override_suppresses_the_warning(monkeypatch, caplog):
+    _clear(monkeypatch)
+    monkeypatch.setenv("AGENTX_BASELINE_OVERHEAD_SEC", "20000")
+    with caplog.at_level("WARNING"):
+        agentx_baseline_timeout_sec()
+    assert not any("AGENTX_BASELINE_OVERHEAD_SEC" in r.message for r in caplog.records)
+
+
 def test_explicit_cap_wins_outright(monkeypatch):
     _clear(monkeypatch)
     monkeypatch.setenv("AGENTX_DURATION", "7200")

src/hyperloom/inference_optimizer/tests/test_agentx_rebaseline_timeout.py

@@ -0,0 +1,72 @@
+###############################################################################
+# SPDX-FileCopyrightText: 2026 Advanced Micro Devices, Inc.
+# SPDX-License-Identifier: MIT
+#
+# See LICENSE for license information.
+###############################################################################
+
+"""The producer side of the AgentX timeout defect.
+
+Raising the variant cap was necessary and not sufficient. Integrate passes an
+explicit ``timeout_sec`` into the re-baseline task, and ``_resolve_timeout``
+deliberately lets an explicit param outrank the AgentX derivation -- a contract
+with its own test. So the raise belongs where the param is produced, next to
+the existing cold-start raise, which exists for exactly the same reason: an
+explicit param suppresses the executor's own sizing branch.
+
+Measured on Qwen3.8: a round whose server answered all 685 chat/completions
+with 200 was cut at exactly its 7200s param, mid-warmup, after which the client
+could no longer connect. aiperf reports the cancelled warmup credit as
+``warmup_failure``, so nothing in the abort reason names the timeout.
+"""
+
+from hyperloom.orchestrator.kernel.request_handlers import (
+    _agentx_rebaseline_timeout,
+)
+
+# the two values observed killing real rounds
+OBSERVED = (7200, 9000)
+
+
+def test_default_path_is_untouched(monkeypatch):
+    """AgentX off: the resolved value passes through, exactly as before."""
+    monkeypatch.delenv("HYPERLOOM_AGENTX", raising=False)
+    for value in (*OBSERVED, 60, 50000):
+        assert _agentx_rebaseline_timeout(value) == value
+
+
+def test_default_path_untouched_with_stale_agentx_vars(monkeypatch):
+    """A leftover AGENTX_* var must not switch the raise on by itself."""
+    monkeypatch.delenv("HYPERLOOM_AGENTX", raising=False)
+    monkeypatch.setenv("AGENTX_DURATION", "3600")
+    monkeypatch.setenv("AGENTX_BASELINE_OVERHEAD_SEC", "28800")
+    assert _agentx_rebaseline_timeout(7200) == 7200
+
+
+def test_raises_the_observed_killers(monkeypatch):
+    monkeypatch.setenv("HYPERLOOM_AGENTX", "1")
+    monkeypatch.setenv("AGENTX_DURATION", "3600")
+    monkeypatch.setenv("AGENTX_BASELINE_OVERHEAD_SEC", "28800")
+    monkeypatch.delenv("AGENTX_BASELINE_TIMEOUT_SEC", raising=False)
+    for value in OBSERVED:
+        assert _agentx_rebaseline_timeout(value) == 32400
+
+
+def test_never_lowers_a_larger_value(monkeypatch):
+    monkeypatch.setenv("HYPERLOOM_AGENTX", "1")
+    monkeypatch.setenv("AGENTX_DURATION", "3600")
+    monkeypatch.setenv("AGENTX_BASELINE_OVERHEAD_SEC", "7200")
+    monkeypatch.delenv("AGENTX_BASELINE_TIMEOUT_SEC", raising=False)
+    assert _agentx_rebaseline_timeout(50000) == 50000
+
+
+def test_tracks_the_baseline_derivation(monkeypatch):
+    """One number, not two: it follows baseline's own resolver."""
+    monkeypatch.setenv("HYPERLOOM_AGENTX", "1")
+    monkeypatch.setenv("AGENTX_DURATION", "3600")
+    monkeypatch.setenv("AGENTX_BASELINE_OVERHEAD_SEC", "7200")
+    monkeypatch.delenv("AGENTX_BASELINE_TIMEOUT_SEC", raising=False)
+    assert _agentx_rebaseline_timeout(7200) == 10800
+
+    monkeypatch.setenv("AGENTX_BASELINE_TIMEOUT_SEC", "44000")
+    assert _agentx_rebaseline_timeout(7200) == 44000

src/hyperloom/inference_optimizer/tests/test_agentx_variant_timeout.py

@@ -0,0 +1,76 @@
+###############################################################################
+# SPDX-FileCopyrightText: 2026 Advanced Micro Devices, Inc.
+# SPDX-License-Identifier: MIT
+#
+# See LICENSE for license information.
+###############################################################################
+
+"""The variant hard cap has to survive a canonical AgentX warmup.
+
+Found by running the AgentX path end to end with nothing disabled: a GLM-5.2
+variant launched 09:47:41 was killed at 11:47:41.575 -- the synthetic cap minus
+its reserve -- with twenty-plus connections dropping in the same millisecond
+while the server was still prefilling with 55 requests running. aiperf reports
+that as ``warmup_failure`` because a cancelled root warmup credit is terminal,
+so the subprocess kill never appears in the abort reason and the round looks
+like a workload problem.
+
+The caps involved are all sized for the synthetic 1024/1024 shape: 7800s for
+integrate, 2400s for explore, 1800s for the conc sweep. ``baseline`` already
+derives an AgentX-aware cap; only that path got it.
+"""
+
+from hyperloom.orchestrator.actions.executors._grid_runner import (
+    agentx_variant_timeout_sec,
+)
+
+# the three synthetic caps that killed real rounds
+SYNTHETIC_CAPS = (1800, 2400, 7800)
+
+
+def test_default_path_is_untouched(monkeypatch):
+    """AgentX off must behave exactly as before -- it is an opt-in branch.
+
+    This is the property that matters most: AgentX is a new benchmark branch,
+    not the default, so with it disabled every cap has to come back unchanged.
+    """
+    monkeypatch.delenv("HYPERLOOM_AGENTX", raising=False)
+    for cap in (*SYNTHETIC_CAPS, 99, 36000):
+        assert agentx_variant_timeout_sec(cap) == cap
+
+
+def test_default_path_untouched_even_with_agentx_vars_present(monkeypatch):
+    """Leftover AGENTX_* vars must not switch the branch on by themselves."""
+    monkeypatch.delenv("HYPERLOOM_AGENTX", raising=False)
+    monkeypatch.setenv("AGENTX_DURATION", "3600")
+    monkeypatch.setenv("AGENTX_BASELINE_OVERHEAD_SEC", "28800")
+    assert agentx_variant_timeout_sec(7800) == 7800
+
+
+def test_agentx_raises_the_synthetic_defaults(monkeypatch):
+    monkeypatch.setenv("HYPERLOOM_AGENTX", "1")
+    monkeypatch.setenv("AGENTX_DURATION", "3600")
+    monkeypatch.setenv("AGENTX_BASELINE_OVERHEAD_SEC", "7200")
+    monkeypatch.delenv("AGENTX_BASELINE_TIMEOUT_SEC", raising=False)
+    for cap in SYNTHETIC_CAPS:
+        assert agentx_variant_timeout_sec(cap) == 10800
+
+
+def test_never_lowers_an_operator_choice(monkeypatch):
+    monkeypatch.setenv("HYPERLOOM_AGENTX", "1")
+    monkeypatch.setenv("AGENTX_DURATION", "3600")
+    monkeypatch.setenv("AGENTX_BASELINE_OVERHEAD_SEC", "7200")
+    monkeypatch.delenv("AGENTX_BASELINE_TIMEOUT_SEC", raising=False)
+    assert agentx_variant_timeout_sec(36000) == 36000
+
+
+def test_tracks_the_baseline_derivation(monkeypatch):
+    """One number, not two: the cap follows baseline's own resolver."""
+    monkeypatch.setenv("HYPERLOOM_AGENTX", "1")
+    monkeypatch.setenv("AGENTX_DURATION", "3600")
+    monkeypatch.setenv("AGENTX_BASELINE_OVERHEAD_SEC", "28800")
+    monkeypatch.delenv("AGENTX_BASELINE_TIMEOUT_SEC", raising=False)
+    assert agentx_variant_timeout_sec(7800) == 32400
+
+    monkeypatch.setenv("AGENTX_BASELINE_TIMEOUT_SEC", "50000")
+    assert agentx_variant_timeout_sec(7800) == 50000

src/hyperloom/inference_optimizer/tests/test_aiperf_client_sh.py

@@ -586,3 +586,51 @@ def test_inherited_noncanonical_marker_does_not_leak_in(tmp_path):
     out = _result(res)
     assert not out["submission_invalid_reasons"]
     assert out["submission_valid"] is not False
+
+
+def test_reduced_warmup_is_flagged_non_canonical(tmp_path):
+    """Warmup is measurement-defining, so trimming it must void submittability.
+
+    Measured on a 743B model: the canonical 10 requests/lane is a ~2h warmup, so
+    an operator reaches for this knob under real time pressure. aiperf has no
+    concept of "enough warmup", so the scenario stamps submission_valid=true and
+    the round looks publishable while having measured a materially emptier cache.
+    Only the client knows the canonical value, so only the client can object.
+    """
+    bench, bind, res = _sandbox(tmp_path)
+    r = _run(bench, bind, res, tmp_path, AGENTX_WARMUP_REQUESTS_PER_LANE="1")
+    assert r.returncode == 0, r.stderr
+    out = _result(res)
+    assert out["submission_valid"] is False
+    assert any(
+        "warmup_per_lane=1" in x for x in out["submission_invalid_reasons"]
+    ), out["submission_invalid_reasons"]
+
+
+def test_reduced_warmup_grace_is_flagged_non_canonical(tmp_path):
+    """Same for the drain window: a shorter grace truncates the warmup it gates."""
+    bench, bind, res = _sandbox(tmp_path)
+    r = _run(bench, bind, res, tmp_path, AGENTX_WARMUP_GRACE_PERIOD="60")
+    assert r.returncode == 0, r.stderr
+    out = _result(res)
+    assert out["submission_valid"] is False
+    assert any(
+        "warmup_grace=60s" in x for x in out["submission_invalid_reasons"]
+    ), out["submission_invalid_reasons"]
+
+
+def test_canonical_warmup_is_not_flagged(tmp_path):
+    """The canonical values must not trip the new check (no false positive)."""
+    bench, bind, res = _sandbox(tmp_path)
+    r = _run(
+        bench,
+        bind,
+        res,
+        tmp_path,
+        AGENTX_WARMUP_REQUESTS_PER_LANE="10",
+        AGENTX_WARMUP_GRACE_PERIOD="1800",
+    )
+    assert r.returncode == 0, r.stderr
+    out = _result(res)
+    assert not out["submission_invalid_reasons"]
+    assert out["submission_valid"] is not False

src/hyperloom/orchestrator/actions/executors/_grid_runner.py

@@ -1293,6 +1293,46 @@
     return _STOPPED_BY_THE_RUN.get(int(returncode))
 
 
+def agentx_variant_timeout_sec(cap: int) -> int:
+    """Raise a variant's hard cap to what an AgentX round actually needs.
+
+    Every variant cap in the tree is sized for the synthetic 1024/1024 shape --
+    7800s for integrate, 2400s for explore, 1800s for the conc sweep. A
+    canonical AgentX warmup is 10 requests per lane against real agentic
+    traces, which on a 700B-class model runs well past two hours before the
+    measured round even begins, so those caps kill the round mid-warmup.
+    Measured on GLM-5.2: a variant launched 09:47:41 was killed at
+    11:47:41.575, twenty-plus connections dropping in the same millisecond
+    while the server was still prefilling with 55 requests running. Downstream
+    that reads as a warmup failure, because aiperf treats a cancelled root
+    warmup credit as terminal -- so the real cause (a subprocess kill) is
+    invisible in the abort reason.
+
+    ``baseline`` already derives an AgentX-aware cap; only that one path got
+    it. This reuses the same derivation rather than introducing a second number
+    to keep in sync, and never lowers a cap, so an operator who asked for
+    longer keeps it.
+
+    AgentX is an opt-in benchmark branch: with it disabled this returns ``cap``
+    untouched and the default path is unaffected.
+
+    Args:
+        cap: The declared hard timeout for the round, in seconds.
+
+    Returns:
+        int: ``cap``, or the AgentX-derived cap when that is larger.
+    """
+    # Local import: baseline imports from this module, and the rest of the file
+    # already resolves _workload_envs this way.
+    from ._workload_envs import agentx_enabled
+
+    if not agentx_enabled():
+        return cap
+    from .baseline import agentx_baseline_timeout_sec
+
+    return max(cap, agentx_baseline_timeout_sec())
+
+
 def session_clamped_timeout_sec(
     cap: int,
     session_deadline_sec: float | None,
@@ -1575,7 +1615,20 @@
         Returns:
             int: The hard timeout to grant this round, in seconds.
         """
-        cap = int(variant_timeout_sec)
+        declared = int(variant_timeout_sec)
+        cap = agentx_variant_timeout_sec(declared)
+        if cap != declared:
+            log.info(
+                "grid_runner: variant %d/%d name=%s %s cap raised %ds -> %ds "
+                "(AgentX: AGENTX_DURATION + overhead; the synthetic default "
+                "cannot cover a canonical agentic warmup)",
+                idx + 1,
+                len(grid),
+                name,
+                round_label,
+                declared,
+                cap,
+            )
         clamped = session_clamped_timeout_sec(cap, session_deadline_sec, reserve_sec=reserve_sec)
         if clamped == cap:
             return cap

src/hyperloom/orchestrator/actions/executors/_workload_envs.py

@@ -81,6 +81,14 @@
 # serialization-safe on a large TP=8 MoE. Tunable via
 # HYPERLOOM_PROFILE_MAX_STEPS_CAP.
 _DEFAULT_PROFILE_MAX_STEPS = 128
+# AgentX counterpart of the cap above. The 128 is calibrated in decode steps
+# against the synthetic 1024/1024 shape; an agentic step carries a measured ISL
+# p50 of 56k-96k tokens, so the same step count buffers orders of magnitude more
+# profiler events in HOST RAM. Measured on DeepSeek-V4-Pro: eight vLLM workers at
+# 113-127 GB each, Ray reported 1012/1024 GB and killed the capture three times.
+# The AgentX client bounds the window by wall clock anyway (~20s of steady
+# state), so the extra steps buy nothing. HYPERLOOM_PROFILE_MAX_ITERS overrides.
+_AGENTX_PROFILE_MAX_ITERS = 8
 # Default profile OSL ceiling when --profile-osl / PROFILE_OSL is unset: the
 # profile reuses min(served OSL, this) so its trace stays light.
 _PROFILE_DEFAULT_OSL = 1024
@@ -1066,6 +1074,29 @@ def materialize_config_with_envs(
         # host RAM until the OOM killer arrives.
         if agentx_enabled():
             delay_iters = 0
+            # ...and the bound itself has to come down, because the cap above is
+            # sized in DECODE STEPS against the synthetic OSL. Under AgentX the
+            # captured work per step is agentic: measured ISL p50 was 56k-96k
+            # tokens, two orders of magnitude past the 1024/1024 shape the cap
+            # was calibrated on. At the stock cap a DeepSeek-V4 profile round put
+            # each of the eight vLLM workers at 113-127 GB of HOST RAM -- Ray
+            # reported 1012/1024 GB and killed them mid-capture, three attempts
+            # in a row, so the round produced no trace at all.
+            #
+            # A shorter capture is not a worse trace here: the client already
+            # bounds the window by wall clock (~20s of steady state), so the
+            # extra steps buy nothing and only inflate the in-memory event
+            # buffer. HYPERLOOM_PROFILE_MAX_ITERS still overrides this below.
+            if max_iters > _AGENTX_PROFILE_MAX_ITERS:
+                log.info(
+                    "AgentX: lowering captured profile steps %d -> %d. The cap is "
+                    "calibrated on the synthetic ISL/OSL shape; an agentic step "
+                    "carries orders of magnitude more, and the torch profiler "
+                    "buffers events in host RAM until the OOM killer arrives.",
+                    max_iters,
+                    _AGENTX_PROFILE_MAX_ITERS,
+                )
+                max_iters = _AGENTX_PROFILE_MAX_ITERS
         # Operator hard-override of captured steps (e.g. a small eager FlyDSL
         # profile). Honored verbatim; warn when outside the safe band rather
         # than silently clamping.