From f2c0d209e0b31210ac43d737879f11e5ef2bdcc5 Mon Sep 17 00:00:00 2001 From: Zeng Date: Fri, 21 Aug 2026 12:05:12 +0800 Subject: [PATCH 01/12] fix(agentx): close three gaps a real 743B/862B run walked straight into All three were found by running the AgentX path end to end on GLM-5.2-MXFP4 and DeepSeek-V4-Pro (8x MI355X, TP=8, full 062126 corpus). Each one silently cost a multi-hour round. 1. Warmup was missing from the non-canonical list. The agentic warmup is what puts the cache under realistic pressure before the window opens, so 1 request/lane instead of 10 measures a materially emptier cache. aiperf has no notion of "enough warmup", so such a round came back submission_valid=true and looked publishable. On a 743B model the canonical 10/lane is a ~2h warmup, which is exactly when an operator reaches for the knob -- the hole was reachable in practice, not theoretical. Both AGENTX_WARMUP_REQUESTS_PER_LANE and AGENTX_WARMUP_GRACE_PERIOD now register as deviations. 2. The self-bracketed profile delay could open after the round had ended. AGENTX_PROFILE_WARMUP_S is a blind wall clock with no idea which phase aiperf is in, and the two failure directions are not symmetric: opening late yields NO trace (aiperf exits, the branch only warns), opening early yields a capture of a still-loaded system that TraceLens can use. Observed both ways on the same model in one day. Clamp toward early at DURATION - window - 60s and say so when the clamp bites. 3. The profile capture bound is calibrated on the synthetic shape. 128 decode steps is serialization-safe at 1024/1024; an agentic step carries a measured ISL p50 of 56k-96k tokens. On DeepSeek-V4-Pro that put each of the eight vLLM workers at 113-127 GB of HOST RAM -- Ray reported 1012/1024 GB and killed the capture, three attempts running, so the roofline arm produced no trace at all. AgentX now caps at 8; the client bounds the window by wall clock anyway, so the extra steps only inflate the in-memory event buffer. HYPERLOOM_PROFILE_MAX_ITERS still overrides. Tests: three new cases in test_aiperf_client_sh.py, negative-verified by reverting the client fix (the two "must flag" cases go red, the no-false-positive case stays green). Full file: 37 passed on a POSIX box. Co-Authored-By: Claude Opus 5 --- .../assets/agentx/aiperf_client.sh | 32 +++++++++++++ .../tests/test_aiperf_client_sh.py | 48 +++++++++++++++++++ .../actions/executors/_workload_envs.py | 31 ++++++++++++ 3 files changed, 111 insertions(+) diff --git a/src/hyperloom/inference_optimizer/assets/agentx/aiperf_client.sh b/src/hyperloom/inference_optimizer/assets/agentx/aiperf_client.sh index 730637494b..0f36a46001 100755 --- a/src/hyperloom/inference_optimizer/assets/agentx/aiperf_client.sh +++ b/src/hyperloom/inference_optimizer/assets/agentx/aiperf_client.sh @@ -255,6 +255,16 @@ FRT="${AGENTX_FAILED_REQUEST_THRESHOLD:-0.10}" # leaderboard measurement -- by construction rather than by promise. CANON_ENTRIES=393 CANON_DURATION=3600 +# Warmup is measurement-defining and was missing from this list until a measured +# run exposed the gap: the agentic warmup is what puts the KV/radix cache under +# realistic pressure before the window opens, so replaying at 1 request/lane +# instead of 10 measures a materially emptier cache. It carries no scenario +# marker either -- aiperf has no concept of "how much warmup is enough" -- so a +# reduced-warmup round came back submission_valid=true and looked publishable. +# On a 743B model the canonical 10/lane is a ~2h warmup, which is exactly when +# an operator reaches for this knob, so the hole was reachable in practice. +CANON_WARMUP_PER_LANE=10 +CANON_WARMUP_GRACE=1800 # The corpus this model family canonically replays, before any operator pin. # CANON_DS is resolved with the corpus above. The family whitelist behind it is # a derivation, not a registry -- a model upstream runs on the full corpus but @@ -274,6 +284,10 @@ NONCANON=() [ "$DURATION" != "$CANON_DURATION" ] && NONCANON+=("duration=${DURATION}s(canonical ${CANON_DURATION}s)") [ -n "${AGENTX_MAX_CTX:-}" ] && NONCANON+=("client_context_cap=${AGENTX_MAX_CTX}") [ "${AGENTX_UNSAFE_OVERRIDE:-false}" = "true" ] && NONCANON+=("unsafe_override_forced") +[ "$WARMLANE" != "$CANON_WARMUP_PER_LANE" ] && \ + NONCANON+=("warmup_per_lane=${WARMLANE}(canonical ${CANON_WARMUP_PER_LANE})") +[ "$WARMGRACE" != "$CANON_WARMUP_GRACE" ] && \ + NONCANON+=("warmup_grace=${WARMGRACE}s(canonical ${CANON_WARMUP_GRACE}s)") SMOKE_ARGS=() if [ "$DURATION" -lt "$CANON_DURATION" ] || [ "${AGENTX_UNSAFE_OVERRIDE:-false}" = "true" ]; then @@ -348,6 +362,24 @@ if [ "${PROFILE:-0}" = "1" ]; then # the upstream profile it lands squarely inside setup and captures nothing. PWARM="${AGENTX_PROFILE_WARMUP_S:-2700}" PWIN="${AGENTX_PROFILE_WINDOW_S:-20}" + # The delay is a blind wall clock: it does not know which phase aiperf is in, + # and the two ways to get it wrong are NOT symmetric. Opening late is fatal -- + # aiperf exits, the branch below only logs a warning, and the round produces no + # trace at all. Opening early merely captures a still-loaded system slightly + # before steady state, which TraceLens can still use. Measured: a 743B model + # spends ~2.5h in the agentic warmup, so a delay tuned on a 35B round lands + # either mid-warmup or past the end depending on which way the estimate erred. + # + # So clamp toward "early". The round cannot outlast the measurement window plus + # the warmup that precedes it, and the only number known here is the window, so + # cap the delay at DURATION - PWIN - margin and say when the cap bites. A + # capture inside warmup is a usable trace; a capture that never happens is not. + _pmax=$(( DURATION - PWIN - 60 )) + [ "$_pmax" -lt 0 ] && _pmax=0 + if [ "$PWARM" -gt "$_pmax" ]; then + log "WARN profile delay ${PWARM}s exceeds the safe bound for a ${DURATION}s window; clamping to ${_pmax}s so the capture cannot land after the round ends" + PWARM="$_pmax" + fi log "PROFILE=1: self-bracketing profile window (delay=${PWARM}s window=${PWIN}s)" run_aiperf & APID=$! sleep "$PWARM" diff --git a/src/hyperloom/inference_optimizer/tests/test_aiperf_client_sh.py b/src/hyperloom/inference_optimizer/tests/test_aiperf_client_sh.py index c5ace05798..fb877ddc68 100644 --- a/src/hyperloom/inference_optimizer/tests/test_aiperf_client_sh.py +++ b/src/hyperloom/inference_optimizer/tests/test_aiperf_client_sh.py @@ -586,3 +586,51 @@ def test_inherited_noncanonical_marker_does_not_leak_in(tmp_path): out = _result(res) assert not out["submission_invalid_reasons"] assert out["submission_valid"] is not False + + +def test_reduced_warmup_is_flagged_non_canonical(tmp_path): + """Warmup is measurement-defining, so trimming it must void submittability. + + Measured on a 743B model: the canonical 10 requests/lane is a ~2h warmup, so + an operator reaches for this knob under real time pressure. aiperf has no + concept of "enough warmup", so the scenario stamps submission_valid=true and + the round looks publishable while having measured a materially emptier cache. + Only the client knows the canonical value, so only the client can object. + """ + bench, bind, res = _sandbox(tmp_path) + r = _run(bench, bind, res, tmp_path, AGENTX_WARMUP_REQUESTS_PER_LANE="1") + assert r.returncode == 0, r.stderr + out = _result(res) + assert out["submission_valid"] is False + assert any( + "warmup_per_lane=1" in x for x in out["submission_invalid_reasons"] + ), out["submission_invalid_reasons"] + + +def test_reduced_warmup_grace_is_flagged_non_canonical(tmp_path): + """Same for the drain window: a shorter grace truncates the warmup it gates.""" + bench, bind, res = _sandbox(tmp_path) + r = _run(bench, bind, res, tmp_path, AGENTX_WARMUP_GRACE_PERIOD="60") + assert r.returncode == 0, r.stderr + out = _result(res) + assert out["submission_valid"] is False + assert any( + "warmup_grace=60s" in x for x in out["submission_invalid_reasons"] + ), out["submission_invalid_reasons"] + + +def test_canonical_warmup_is_not_flagged(tmp_path): + """The canonical values must not trip the new check (no false positive).""" + bench, bind, res = _sandbox(tmp_path) + r = _run( + bench, + bind, + res, + tmp_path, + AGENTX_WARMUP_REQUESTS_PER_LANE="10", + AGENTX_WARMUP_GRACE_PERIOD="1800", + ) + assert r.returncode == 0, r.stderr + out = _result(res) + assert not out["submission_invalid_reasons"] + assert out["submission_valid"] is not False diff --git a/src/hyperloom/orchestrator/actions/executors/_workload_envs.py b/src/hyperloom/orchestrator/actions/executors/_workload_envs.py index 6cc6ff8cb6..060dbb93b5 100644 --- a/src/hyperloom/orchestrator/actions/executors/_workload_envs.py +++ b/src/hyperloom/orchestrator/actions/executors/_workload_envs.py @@ -81,6 +81,14 @@ # serialization-safe on a large TP=8 MoE. Tunable via # HYPERLOOM_PROFILE_MAX_STEPS_CAP. _DEFAULT_PROFILE_MAX_STEPS = 128 +# AgentX counterpart of the cap above. The 128 is calibrated in decode steps +# against the synthetic 1024/1024 shape; an agentic step carries a measured ISL +# p50 of 56k-96k tokens, so the same step count buffers orders of magnitude more +# profiler events in HOST RAM. Measured on DeepSeek-V4-Pro: eight vLLM workers at +# 113-127 GB each, Ray reported 1012/1024 GB and killed the capture three times. +# The AgentX client bounds the window by wall clock anyway (~20s of steady +# state), so the extra steps buy nothing. HYPERLOOM_PROFILE_MAX_ITERS overrides. +_AGENTX_PROFILE_MAX_ITERS = 8 # Default profile OSL ceiling when --profile-osl / PROFILE_OSL is unset: the # profile reuses min(served OSL, this) so its trace stays light. _PROFILE_DEFAULT_OSL = 1024 @@ -1066,6 +1074,29 @@ def materialize_config_with_envs( # host RAM until the OOM killer arrives. if agentx_enabled(): delay_iters = 0 + # ...and the bound itself has to come down, because the cap above is + # sized in DECODE STEPS against the synthetic OSL. Under AgentX the + # captured work per step is agentic: measured ISL p50 was 56k-96k + # tokens, two orders of magnitude past the 1024/1024 shape the cap + # was calibrated on. At the stock cap a DeepSeek-V4 profile round put + # each of the eight vLLM workers at 113-127 GB of HOST RAM -- Ray + # reported 1012/1024 GB and killed them mid-capture, three attempts + # in a row, so the round produced no trace at all. + # + # A shorter capture is not a worse trace here: the client already + # bounds the window by wall clock (~20s of steady state), so the + # extra steps buy nothing and only inflate the in-memory event + # buffer. HYPERLOOM_PROFILE_MAX_ITERS still overrides this below. + if max_iters > _AGENTX_PROFILE_MAX_ITERS: + log.info( + "AgentX: lowering captured profile steps %d -> %d. The cap is " + "calibrated on the synthetic ISL/OSL shape; an agentic step " + "carries orders of magnitude more, and the torch profiler " + "buffers events in host RAM until the OOM killer arrives.", + max_iters, + _AGENTX_PROFILE_MAX_ITERS, + ) + max_iters = _AGENTX_PROFILE_MAX_ITERS # Operator hard-override of captured steps (e.g. a small eager FlyDSL # profile). Honored verbatim; warn when outside the safe band rather # than silently clamping. From f98db4811efd0f7bb930790ecab3407798f0b83d Mon Sep 17 00:00:00 2001 From: Zeng Date: Sun, 23 Aug 2026 21:12:51 +0800 Subject: [PATCH 02/12] fix(agentx): variant rounds are killed mid-warmup by a synthetic-sized cap Found by running the AgentX path end to end with no phase disabled -- the first time that has been done, because every earlier run in this campaign passed --no-kernel and friends. A GLM-5.2 variant launched 09:47:41 died at 11:47:41.575: twenty-plus connections dropped in the SAME millisecond while the server was healthy and still prefilling with 55 requests running. A simultaneous mass disconnect against a live server is a subprocess kill, not a workload problem -- but aiperf treats a cancelled root warmup credit as terminal, so it surfaces as `warmup_failure` and the real cause never appears in the abort reason. Every variant cap is sized for the synthetic 1024/1024 shape: 7800s for integrate, 2400s for explore, 1800s for the conc sweep. A canonical AgentX warmup is 10 requests per lane over real agentic traces and runs past two hours on a 700B-class model before the measured round begins, so those caps kill the round by arithmetic rather than by chance. baseline already derives an AgentX-aware cap; only that path got it. Raised at the single choke point every variant round resolves through, reusing baseline's own resolver so there is not a second number to keep in sync, and never lowering a cap an operator asked for. AgentX is an opt-in benchmark branch: the gate means that with it off this is a no-op and the default path is untouched -- asserted directly, including with stale AGENTX_* vars present. This also retires three hypotheses that looked right and were not: concurrency contention (canonical conc=64 clears baseline with zero errors), a mis-passed --warmup-grace-period (removing it still failed at 706/707), and poisoned corpus entries (the recurring trace ids are an artefact of seed 42 fixing dispatch order; their sizes are the 9th and 88th percentile). Co-Authored-By: Claude Opus 5 --- .../tests/test_agentx_variant_timeout.py | 76 +++++++++++++++++++ .../actions/executors/_grid_runner.py | 55 +++++++++++++- 2 files changed, 130 insertions(+), 1 deletion(-) create mode 100644 src/hyperloom/inference_optimizer/tests/test_agentx_variant_timeout.py diff --git a/src/hyperloom/inference_optimizer/tests/test_agentx_variant_timeout.py b/src/hyperloom/inference_optimizer/tests/test_agentx_variant_timeout.py new file mode 100644 index 0000000000..e833ecc776 --- /dev/null +++ b/src/hyperloom/inference_optimizer/tests/test_agentx_variant_timeout.py @@ -0,0 +1,76 @@ +############################################################################### +# SPDX-FileCopyrightText: 2026 Advanced Micro Devices, Inc. +# SPDX-License-Identifier: MIT +# +# See LICENSE for license information. +############################################################################### + +"""The variant hard cap has to survive a canonical AgentX warmup. + +Found by running the AgentX path end to end with nothing disabled: a GLM-5.2 +variant launched 09:47:41 was killed at 11:47:41.575 -- the synthetic cap minus +its reserve -- with twenty-plus connections dropping in the same millisecond +while the server was still prefilling with 55 requests running. aiperf reports +that as ``warmup_failure`` because a cancelled root warmup credit is terminal, +so the subprocess kill never appears in the abort reason and the round looks +like a workload problem. + +The caps involved are all sized for the synthetic 1024/1024 shape: 7800s for +integrate, 2400s for explore, 1800s for the conc sweep. ``baseline`` already +derives an AgentX-aware cap; only that path got it. +""" + +from hyperloom.orchestrator.actions.executors._grid_runner import ( + agentx_variant_timeout_sec, +) + +# the three synthetic caps that killed real rounds +SYNTHETIC_CAPS = (1800, 2400, 7800) + + +def test_default_path_is_untouched(monkeypatch): + """AgentX off must behave exactly as before -- it is an opt-in branch. + + This is the property that matters most: AgentX is a new benchmark branch, + not the default, so with it disabled every cap has to come back unchanged. + """ + monkeypatch.delenv("HYPERLOOM_AGENTX", raising=False) + for cap in (*SYNTHETIC_CAPS, 99, 36000): + assert agentx_variant_timeout_sec(cap) == cap + + +def test_default_path_untouched_even_with_agentx_vars_present(monkeypatch): + """Leftover AGENTX_* vars must not switch the branch on by themselves.""" + monkeypatch.delenv("HYPERLOOM_AGENTX", raising=False) + monkeypatch.setenv("AGENTX_DURATION", "3600") + monkeypatch.setenv("AGENTX_BASELINE_OVERHEAD_SEC", "28800") + assert agentx_variant_timeout_sec(7800) == 7800 + + +def test_agentx_raises_the_synthetic_defaults(monkeypatch): + monkeypatch.setenv("HYPERLOOM_AGENTX", "1") + monkeypatch.setenv("AGENTX_DURATION", "3600") + monkeypatch.setenv("AGENTX_BASELINE_OVERHEAD_SEC", "7200") + monkeypatch.delenv("AGENTX_BASELINE_TIMEOUT_SEC", raising=False) + for cap in SYNTHETIC_CAPS: + assert agentx_variant_timeout_sec(cap) == 10800 + + +def test_never_lowers_an_operator_choice(monkeypatch): + monkeypatch.setenv("HYPERLOOM_AGENTX", "1") + monkeypatch.setenv("AGENTX_DURATION", "3600") + monkeypatch.setenv("AGENTX_BASELINE_OVERHEAD_SEC", "7200") + monkeypatch.delenv("AGENTX_BASELINE_TIMEOUT_SEC", raising=False) + assert agentx_variant_timeout_sec(36000) == 36000 + + +def test_tracks_the_baseline_derivation(monkeypatch): + """One number, not two: the cap follows baseline's own resolver.""" + monkeypatch.setenv("HYPERLOOM_AGENTX", "1") + monkeypatch.setenv("AGENTX_DURATION", "3600") + monkeypatch.setenv("AGENTX_BASELINE_OVERHEAD_SEC", "28800") + monkeypatch.delenv("AGENTX_BASELINE_TIMEOUT_SEC", raising=False) + assert agentx_variant_timeout_sec(7800) == 32400 + + monkeypatch.setenv("AGENTX_BASELINE_TIMEOUT_SEC", "50000") + assert agentx_variant_timeout_sec(7800) == 50000 diff --git a/src/hyperloom/orchestrator/actions/executors/_grid_runner.py b/src/hyperloom/orchestrator/actions/executors/_grid_runner.py index 4c9a51a46b..d36a760a2a 100644 --- a/src/hyperloom/orchestrator/actions/executors/_grid_runner.py +++ b/src/hyperloom/orchestrator/actions/executors/_grid_runner.py @@ -1293,6 +1293,46 @@ def stopped_by_the_run(returncode: int | None) -> StoppedByTheRun | None: return _STOPPED_BY_THE_RUN.get(int(returncode)) +def agentx_variant_timeout_sec(cap: int) -> int: + """Raise a variant's hard cap to what an AgentX round actually needs. + + Every variant cap in the tree is sized for the synthetic 1024/1024 shape -- + 7800s for integrate, 2400s for explore, 1800s for the conc sweep. A + canonical AgentX warmup is 10 requests per lane against real agentic + traces, which on a 700B-class model runs well past two hours before the + measured round even begins, so those caps kill the round mid-warmup. + Measured on GLM-5.2: a variant launched 09:47:41 was killed at + 11:47:41.575, twenty-plus connections dropping in the same millisecond + while the server was still prefilling with 55 requests running. Downstream + that reads as a warmup failure, because aiperf treats a cancelled root + warmup credit as terminal -- so the real cause (a subprocess kill) is + invisible in the abort reason. + + ``baseline`` already derives an AgentX-aware cap; only that one path got + it. This reuses the same derivation rather than introducing a second number + to keep in sync, and never lowers a cap, so an operator who asked for + longer keeps it. + + AgentX is an opt-in benchmark branch: with it disabled this returns ``cap`` + untouched and the default path is unaffected. + + Args: + cap: The declared hard timeout for the round, in seconds. + + Returns: + int: ``cap``, or the AgentX-derived cap when that is larger. + """ + # Local import: baseline imports from this module, and the rest of the file + # already resolves _workload_envs this way. + from ._workload_envs import agentx_enabled + + if not agentx_enabled(): + return cap + from .baseline import agentx_baseline_timeout_sec + + return max(cap, agentx_baseline_timeout_sec()) + + def session_clamped_timeout_sec( cap: int, session_deadline_sec: float | None, @@ -1575,7 +1615,20 @@ def _round_timeout_sec(idx: int, name: str, *, round_label: str, reserve_sec: fl Returns: int: The hard timeout to grant this round, in seconds. """ - cap = int(variant_timeout_sec) + declared = int(variant_timeout_sec) + cap = agentx_variant_timeout_sec(declared) + if cap != declared: + log.info( + "grid_runner: variant %d/%d name=%s %s cap raised %ds -> %ds " + "(AgentX: AGENTX_DURATION + overhead; the synthetic default " + "cannot cover a canonical agentic warmup)", + idx + 1, + len(grid), + name, + round_label, + declared, + cap, + ) clamped = session_clamped_timeout_sec(cap, session_deadline_sec, reserve_sec=reserve_sec) if clamped == cap: return cap From c972a80b5d207e10735be52b9eb2d44aa03f8784 Mon Sep 17 00:00:00 2001 From: Zeng Date: Mon, 24 Aug 2026 08:48:35 +0800 Subject: [PATCH 03/12] fix(agentx): raise the re-baseline timeout where the param is produced Second half of the variant-cap defect, found by running the fixed path further. Raising the variant cap carried GLM-5.2 to the roofline round for the first time, and then rounds kept dying with `warmup_failure` anyway. The reason is not in the round: a Qwen3.8 baseline whose server answered all 685 chat/completions with 200 was cut at exactly its 7200s timeout, mid-warmup, after which the client could no longer connect. The log names it plainly -- `baseline_executor: timeout=7200s (explicit task param)`, and 9000s elsewhere. Integrate passes an explicit timeout_sec, sized for the synthetic shape, and a canonical AgentX warmup (10/lane over real agentic traces) does not fit either value. aiperf reports the cancelled warmup credit as `warmup_failure`, so the timeout never appears in the abort reason and it reads as a workload problem. Raised at the producer, not the consumer. `_resolve_timeout` deliberately lets an explicit param outrank the AgentX derivation and has a test pinning that contract; the first attempt here overrode it and broke test_explicit_task_param_still_outranks_agentx, which is exactly the kind of deliberate decision a test exists to defend. So this sits next to _cold_start_rebaseline_timeout, which was written for the very same reason: an explicit param suppresses the executor's own sizing branch. Gated on agentx_enabled(), never lowers a larger value, and follows baseline's own resolver so there is no second number to keep in sync. AgentX is an opt-in benchmark branch and must not move the default path -- asserted directly, including with stale AGENTX_* vars in the environment. Co-Authored-By: Claude Opus 5 --- .../tests/test_agentx_rebaseline_timeout.py | 72 +++++++++++++++++++ .../orchestrator/kernel/request_handlers.py | 57 +++++++++++++-- 2 files changed, 125 insertions(+), 4 deletions(-) create mode 100644 src/hyperloom/inference_optimizer/tests/test_agentx_rebaseline_timeout.py diff --git a/src/hyperloom/inference_optimizer/tests/test_agentx_rebaseline_timeout.py b/src/hyperloom/inference_optimizer/tests/test_agentx_rebaseline_timeout.py new file mode 100644 index 0000000000..1e907ed876 --- /dev/null +++ b/src/hyperloom/inference_optimizer/tests/test_agentx_rebaseline_timeout.py @@ -0,0 +1,72 @@ +############################################################################### +# SPDX-FileCopyrightText: 2026 Advanced Micro Devices, Inc. +# SPDX-License-Identifier: MIT +# +# See LICENSE for license information. +############################################################################### + +"""The producer side of the AgentX timeout defect. + +Raising the variant cap was necessary and not sufficient. Integrate passes an +explicit ``timeout_sec`` into the re-baseline task, and ``_resolve_timeout`` +deliberately lets an explicit param outrank the AgentX derivation -- a contract +with its own test. So the raise belongs where the param is produced, next to +the existing cold-start raise, which exists for exactly the same reason: an +explicit param suppresses the executor's own sizing branch. + +Measured on Qwen3.8: a round whose server answered all 685 chat/completions +with 200 was cut at exactly its 7200s param, mid-warmup, after which the client +could no longer connect. aiperf reports the cancelled warmup credit as +``warmup_failure``, so nothing in the abort reason names the timeout. +""" + +from hyperloom.orchestrator.kernel.request_handlers import ( + _agentx_rebaseline_timeout, +) + +# the two values observed killing real rounds +OBSERVED = (7200, 9000) + + +def test_default_path_is_untouched(monkeypatch): + """AgentX off: the resolved value passes through, exactly as before.""" + monkeypatch.delenv("HYPERLOOM_AGENTX", raising=False) + for value in (*OBSERVED, 60, 50000): + assert _agentx_rebaseline_timeout(value) == value + + +def test_default_path_untouched_with_stale_agentx_vars(monkeypatch): + """A leftover AGENTX_* var must not switch the raise on by itself.""" + monkeypatch.delenv("HYPERLOOM_AGENTX", raising=False) + monkeypatch.setenv("AGENTX_DURATION", "3600") + monkeypatch.setenv("AGENTX_BASELINE_OVERHEAD_SEC", "28800") + assert _agentx_rebaseline_timeout(7200) == 7200 + + +def test_raises_the_observed_killers(monkeypatch): + monkeypatch.setenv("HYPERLOOM_AGENTX", "1") + monkeypatch.setenv("AGENTX_DURATION", "3600") + monkeypatch.setenv("AGENTX_BASELINE_OVERHEAD_SEC", "28800") + monkeypatch.delenv("AGENTX_BASELINE_TIMEOUT_SEC", raising=False) + for value in OBSERVED: + assert _agentx_rebaseline_timeout(value) == 32400 + + +def test_never_lowers_a_larger_value(monkeypatch): + monkeypatch.setenv("HYPERLOOM_AGENTX", "1") + monkeypatch.setenv("AGENTX_DURATION", "3600") + monkeypatch.setenv("AGENTX_BASELINE_OVERHEAD_SEC", "7200") + monkeypatch.delenv("AGENTX_BASELINE_TIMEOUT_SEC", raising=False) + assert _agentx_rebaseline_timeout(50000) == 50000 + + +def test_tracks_the_baseline_derivation(monkeypatch): + """One number, not two: it follows baseline's own resolver.""" + monkeypatch.setenv("HYPERLOOM_AGENTX", "1") + monkeypatch.setenv("AGENTX_DURATION", "3600") + monkeypatch.setenv("AGENTX_BASELINE_OVERHEAD_SEC", "7200") + monkeypatch.delenv("AGENTX_BASELINE_TIMEOUT_SEC", raising=False) + assert _agentx_rebaseline_timeout(7200) == 10800 + + monkeypatch.setenv("AGENTX_BASELINE_TIMEOUT_SEC", "44000") + assert _agentx_rebaseline_timeout(7200) == 44000 diff --git a/src/hyperloom/orchestrator/kernel/request_handlers.py b/src/hyperloom/orchestrator/kernel/request_handlers.py index 23bd2b82ad..79aa376fe9 100644 --- a/src/hyperloom/orchestrator/kernel/request_handlers.py +++ b/src/hyperloom/orchestrator/kernel/request_handlers.py @@ -7532,6 +7532,53 @@ def _grade_integrate_accuracy( } +def _agentx_rebaseline_timeout(resolved_sec: int) -> int: + """Raise a re-baseline timeout to what an AgentX round needs. + + Same shape, and the same root cause, as + :func:`_cold_start_rebaseline_timeout`: the explicit ``timeout_sec`` that + integrate passes suppresses the baseline executor's own AgentX branch, so a + value sized for the synthetic shape becomes the only budget the round gets. + Observed values are 7200s and 9000s; a canonical AgentX warmup is 10 + requests per lane over real agentic traces and does not fit either. + + Measured on Qwen3.8: a round whose server answered all 685 + chat/completions with 200 was cut at exactly its 7200s param, mid-warmup, + after which the client could no longer connect. Nothing in the abort reason + names the timeout -- aiperf reports the cancelled warmup credit as + ``warmup_failure``, so it reads as a workload problem. + + Raised here, where the param is produced, rather than in the executor that + consumes it: ``_resolve_timeout`` deliberately lets an explicit param + outrank the AgentX derivation, and that contract has a test on it. AgentX + is an opt-in branch, so with it disabled this returns ``resolved_sec`` + untouched and the default path is unaffected. + + Args: + resolved_sec: The timeout the payload/contract resolved to. + + Returns: + int: ``resolved_sec``, or the AgentX-derived cap when that is larger. + """ + from ..actions.executors._workload_envs import agentx_enabled + + if not agentx_enabled(): + return resolved_sec + from ..actions.executors.baseline import agentx_baseline_timeout_sec + + agentx_sec = agentx_baseline_timeout_sec() + if agentx_sec <= resolved_sec: + return resolved_sec + log.warning( + "integrate_handler: raising re-baseline timeout %ds -> %ds " + "(AgentX: AGENTX_DURATION + overhead; a synthetic-sized param cannot " + "cover a canonical agentic warmup and kills the round mid-warmup)", + resolved_sec, + agentx_sec, + ) + return agentx_sec + + def _cold_start_rebaseline_timeout(resolved_sec: int) -> int: """Raise a re-baseline timeout to the cold-start cap when the JIT cache is empty. @@ -7777,10 +7824,12 @@ async def integrate_handler( fake_task_id = f"integrate-{kernel_id or 'anon'}" workspace = unique_runs_dir(session_dir, "integrate", fake_task_id) baseline_executor = BaselineExecutor(session_dir=session_dir) - rebaseline_timeout_sec = _cold_start_rebaseline_timeout( - _integrate_rebaseline_timeout_sec( - payload, - default_timeout_sec=baseline_executor.default_timeout_sec, + rebaseline_timeout_sec = _agentx_rebaseline_timeout( + _cold_start_rebaseline_timeout( + _integrate_rebaseline_timeout_sec( + payload, + default_timeout_sec=baseline_executor.default_timeout_sec, + ) ) ) fake_task = Task( From b01ab13cbb827b0cb9fd1b108dfdb10d885b0db7 Mon Sep 17 00:00:00 2001 From: Zeng Date: Thu, 27 Aug 2026 17:08:38 +0800 Subject: [PATCH 04/12] fix(agentx): surface when the baseline overhead cap is unvalidated for a model AGENTX_BASELINE_OVERHEAD_SEC's default (7200s) is calibrated on GLM-5.2/Qwen3.8 measurements. A raw aiperf run against Kimi-K3 at concurrency=64 measured warmup alone draining in ~12075s, already past the whole default cap before profiling even starts. Warn when the default is used unmodified so an operator sees this before the round is silently killed hours in, instead of only after. Co-Authored-By: Claude Sonnet 5 --- .../tests/test_agentx_baseline_timeout.py | 21 ++++++++++++++++ .../actions/executors/baseline.py | 24 ++++++++++++++++--- 2 files changed, 42 insertions(+), 3 deletions(-) diff --git a/src/hyperloom/inference_optimizer/tests/test_agentx_baseline_timeout.py b/src/hyperloom/inference_optimizer/tests/test_agentx_baseline_timeout.py index a307327619..c37a40857f 100644 --- a/src/hyperloom/inference_optimizer/tests/test_agentx_baseline_timeout.py +++ b/src/hyperloom/inference_optimizer/tests/test_agentx_baseline_timeout.py @@ -83,6 +83,27 @@ def test_overhead_budget_is_tunable(monkeypatch): assert agentx_baseline_timeout_sec() == AGENTX_DEFAULT_DURATION_SEC + 3600 +def test_default_overhead_warns_it_may_not_fit_every_model(monkeypatch, caplog): + """A raw aiperf run against Kimi-K3 (conc=64) measured warmup alone taking + ~12075s -- longer than this whole default cap. Nothing here can tell a + long-context/slow-prefill model apart from GLM-5.2/Qwen3.8, the models this + constant was measured on, so the gap must be surfaced instead of silently + assumed to fit every model. + """ + _clear(monkeypatch) + with caplog.at_level("WARNING"): + agentx_baseline_timeout_sec() + assert any("AGENTX_BASELINE_OVERHEAD_SEC" in r.message for r in caplog.records) + + +def test_explicit_overhead_override_suppresses_the_warning(monkeypatch, caplog): + _clear(monkeypatch) + monkeypatch.setenv("AGENTX_BASELINE_OVERHEAD_SEC", "20000") + with caplog.at_level("WARNING"): + agentx_baseline_timeout_sec() + assert not any("AGENTX_BASELINE_OVERHEAD_SEC" in r.message for r in caplog.records) + + def test_explicit_cap_wins_outright(monkeypatch): _clear(monkeypatch) monkeypatch.setenv("AGENTX_DURATION", "7200") diff --git a/src/hyperloom/orchestrator/actions/executors/baseline.py b/src/hyperloom/orchestrator/actions/executors/baseline.py index 091ceb700e..90c2fa99a5 100644 --- a/src/hyperloom/orchestrator/actions/executors/baseline.py +++ b/src/hyperloom/orchestrator/actions/executors/baseline.py @@ -645,9 +645,27 @@ def _int(name: str, default: int) -> int: explicit = _int("AGENTX_BASELINE_TIMEOUT_SEC", 0) if explicit: return explicit - return _int("AGENTX_DURATION", AGENTX_DEFAULT_DURATION_SEC) + _int( - "AGENTX_BASELINE_OVERHEAD_SEC", AGENTX_BASELINE_OVERHEAD_SEC - ) + + overhead_explicit = (src.get("AGENTX_BASELINE_OVERHEAD_SEC") or "").strip() + overhead = _int("AGENTX_BASELINE_OVERHEAD_SEC", AGENTX_BASELINE_OVERHEAD_SEC) + if not overhead_explicit: + # AGENTX_BASELINE_OVERHEAD_SEC's default is calibrated on GLM-5.2/Qwen3.8 + # (measured 4774s/6676s warmup+compile). A raw aiperf run against + # Kimi-K3 (conc=64, ISL ~115k avg) measured warmup alone draining in + # ~12075s -- already past this whole cap before profiling even starts. + # Nothing here can tell long-context/slow-prefill models apart from the + # ones this constant was measured on, so surface it instead of letting + # the round run untimed-out until the flat cap kills it mid-warmup. + log.warning( + "agentx_baseline_timeout_sec: using default AGENTX_BASELINE_OVERHEAD_SEC=%ds " + "(no explicit override set). This default is calibrated on GLM-5.2/Qwen3.8 " + "and may be far too small for long-context or slow-prefill models -- a raw " + "aiperf run against Kimi-K3 at concurrency=64 measured warmup alone taking " + "~12075s. If this round is for such a model, set AGENTX_BASELINE_OVERHEAD_SEC " + "explicitly.", + overhead, + ) + return _int("AGENTX_DURATION", AGENTX_DEFAULT_DURATION_SEC) + overhead # Cold-start settings and probes live in ``_aiter_jit`` and are re-exported From fa8f5758c79358281f0888cdd49b48fd7938adfd Mon Sep 17 00:00:00 2001 From: Zeng Date: Thu, 27 Aug 2026 18:09:57 +0800 Subject: [PATCH 05/12] fix(agentx): close four review findings in the closed-loop timeout path - baseline.py: treat an invalid AGENTX_BASELINE_OVERHEAD_SEC override (non-integer, zero, or negative) the same as unset, so the "using the default overhead" warning still fires instead of being silently suppressed by a broken value. - _grid_runner.py: _skip_rest_for_budget now gates admission on the AgentX-raised variant cap (agentx_variant_timeout_sec), not the declared one, when no per-variant estimate is available -- gating on the declared cap could admit a variant that the raised cap then gets clamped back down for, reproducing the mid-warmup kill this cap-raise exists to prevent. - _grid_runner.py: capture the measure round's actual granted timeout in a local variable and log that value on TimeoutExpired, instead of the outer variant_timeout_sec, so the timeout reported during diagnosis matches what was actually enforced. - _workload_envs.py: warn when AgentX's max-iters clamp lands below the steady-state floor, mirroring the existing warning on the manual HYPERLOOM_PROFILE_MAX_ITERS override path. Co-Authored-By: Claude Sonnet 5 --- .../tests/test_grid_runner.py | 36 +++++++++++++++++++ .../tests/test_profile_and_kernel_handlers.py | 25 +++++++++++++ .../actions/executors/_grid_runner.py | 15 +++++--- .../actions/executors/_workload_envs.py | 8 +++++ .../actions/executors/baseline.py | 14 ++++++-- 5 files changed, 92 insertions(+), 6 deletions(-) diff --git a/src/hyperloom/inference_optimizer/tests/test_grid_runner.py b/src/hyperloom/inference_optimizer/tests/test_grid_runner.py index 2cec7bf2d1..d4d2d06da4 100644 --- a/src/hyperloom/inference_optimizer/tests/test_grid_runner.py +++ b/src/hyperloom/inference_optimizer/tests/test_grid_runner.py @@ -1723,6 +1723,42 @@ async def test_without_an_estimate_the_stricter_backstop_check_is_kept(self, tmp assert recorded == [] assert [r.status for r in results] == ["skipped"] + @pytest.mark.asyncio + async def test_without_an_estimate_agentx_gates_on_its_raised_cap(self, tmp_path, monkeypatch): + """The AgentX-raised cap, not the declared one, must gate admission. + + Gating on the declared ``variant_timeout_sec`` (600) would admit this + variant with 700s left on the clock; the round is then handed the + AgentX-raised cap (10800s) by ``_round_timeout_sec``, which + ``session_clamped_timeout_sec`` immediately clamps back down to the + ~700s actually remaining -- reproducing the mid-warmup kill this + AgentX cap-raise exists to prevent. + """ + monkeypatch.setenv("HYPERLOOM_AGENTX", "1") + monkeypatch.setenv("AGENTX_DURATION", "3600") + monkeypatch.setenv("AGENTX_BASELINE_OVERHEAD_SEC", "7200") + monkeypatch.delenv("AGENTX_BASELINE_TIMEOUT_SEC", raising=False) + base = tmp_path / "base.yaml" + _write_baseline_yaml_overrides(base) + recorded: list[dict] = [] + + with patch( + "hyperloom.orchestrator.actions.executors._grid_runner.run_with_session_kill", + side_effect=_capture_launches(recorded), + ): + results = await run_grid( + base_yaml_path=base, + base_extra_args="", + grid=[GridVariant("v0")], + output_root=tmp_path / "out", + variant_timeout_sec=600, + session_deadline_sec=time.monotonic() + 700.0, + variant_expected_sec=None, + ) + + assert recorded == [] + assert [r.status for r in results] == ["skipped"] + class TestSessionBudgetTimeoutClamp: """A granted cap never exceeds what the session can still pay for. diff --git a/src/hyperloom/inference_optimizer/tests/test_profile_and_kernel_handlers.py b/src/hyperloom/inference_optimizer/tests/test_profile_and_kernel_handlers.py index 2f28432140..2dc4ec9f77 100644 --- a/src/hyperloom/inference_optimizer/tests/test_profile_and_kernel_handlers.py +++ b/src/hyperloom/inference_optimizer/tests/test_profile_and_kernel_handlers.py @@ -838,6 +838,31 @@ def test_materialize_profile_window_sglang_skill_formula( assert body["num_steps"] == 128 +def test_materialize_profile_agentx_clamp_warns_below_steady_floor( + tmp_path, + monkeypatch, + caplog, +): + """AgentX's tighter capture cap (8) must warn when it undercuts steady_floor. + + CONC=32/OSL=1024/R=1.0 -> steady_floor=ceil(1024*2/64)=32, far above the + AgentX cap of 8. The manual HYPERLOOM_PROFILE_MAX_ITERS override already + warns in this situation; the AgentX auto-clamp must match it instead of + silently capturing a trace with no steady-state window. + """ + import yaml + + _clear_workload_env(monkeypatch) + monkeypatch.setenv("HYPERLOOM_AGENTX", "1") + src = _profile_yaml(tmp_path, "vllm", {"CONC": 32, "ISL": 256, "OSL": 1024}) + with caplog.at_level("WARNING"): + out = _materialize_config_with_envs(src, tmp_path) + rendered = yaml.safe_load(out.read_text()) + extra = rendered["benchmark"]["envs"]["EXTRA_VLLM_ARGS"] + assert "--profiler-config.max_iterations 8" in extra, extra + assert any("steady-state floor" in r.message for r in caplog.records) + + def test_materialize_persists_inferencex_path_for_magpie( tmp_path, monkeypatch, diff --git a/src/hyperloom/orchestrator/actions/executors/_grid_runner.py b/src/hyperloom/orchestrator/actions/executors/_grid_runner.py index d36a760a2a..f2e8af7e0a 100644 --- a/src/hyperloom/orchestrator/actions/executors/_grid_runner.py +++ b/src/hyperloom/orchestrator/actions/executors/_grid_runner.py @@ -1764,11 +1764,17 @@ def _skip_rest_for_budget(idx: int, *, spent_on: str, rounds_left: int = variant return False remaining_sec = session_deadline_sec - time.monotonic() # Falls back to a single ``variant_timeout_sec`` when no estimate was - # given, which is what callers that cannot estimate already got. + # given, which is what callers that cannot estimate already got. Must + # be the AgentX-raised cap, not the declared one: the declared cap + # understates what the round will actually be granted, so this check + # would admit a variant it cannot fit, which then gets its timeout + # clamped down to the (too-small) remaining budget by + # ``session_clamped_timeout_sec`` -- reproducing the mid-warmup kill + # this module's AgentX cap-raise exists to prevent. required_sec = ( float(variant_expected_sec) * rounds_left if variant_expected_sec is not None - else float(variant_timeout_sec) + else float(agentx_variant_timeout_sec(variant_timeout_sec)) ) if remaining_sec >= required_sec: return False @@ -2291,6 +2297,7 @@ def _skip_rest_for_budget(idx: int, *, spent_on: str, rounds_left: int = variant # leak destinations per-variant. slot_workspaces_before = snapshot_workspaces(slot) variant_started_unix = time.time() + measure_cap_sec = _round_timeout_sec(i, variant.name, round_label="measure") try: rc, stdout, stderr = await _reported_magpie( i, @@ -2298,7 +2305,7 @@ def _skip_rest_for_budget(idx: int, *, spent_on: str, rounds_left: int = variant magpie_python=magpie_python, config_path=cfg_path, output_dir=slot, - timeout_sec=_round_timeout_sec(i, variant.name, round_label="measure"), + timeout_sec=measure_cap_sec, cwd=cwd, result_dir=result_dir, soft_deadline_sec=soft_deadline_sec, @@ -2319,7 +2326,7 @@ def _skip_rest_for_budget(idx: int, *, spent_on: str, rounds_left: int = variant i + 1, len(grid), variant.name, - variant_timeout_sec, + measure_cap_sec, exc, ) _write_variant_abort_marker( diff --git a/src/hyperloom/orchestrator/actions/executors/_workload_envs.py b/src/hyperloom/orchestrator/actions/executors/_workload_envs.py index 060dbb93b5..f0933f4cdd 100644 --- a/src/hyperloom/orchestrator/actions/executors/_workload_envs.py +++ b/src/hyperloom/orchestrator/actions/executors/_workload_envs.py @@ -1097,6 +1097,14 @@ def materialize_config_with_envs( _AGENTX_PROFILE_MAX_ITERS, ) max_iters = _AGENTX_PROFILE_MAX_ITERS + if max_iters < steady_floor: + log.warning( + "AgentX: capped profile steps %d is below the steady-state " + "floor of %d; the trace may lack a steady-state window " + "(trace_split_no_steady_state).", + max_iters, + steady_floor, + ) # Operator hard-override of captured steps (e.g. a small eager FlyDSL # profile). Honored verbatim; warn when outside the safe band rather # than silently clamping. diff --git a/src/hyperloom/orchestrator/actions/executors/baseline.py b/src/hyperloom/orchestrator/actions/executors/baseline.py index 90c2fa99a5..55022731fb 100644 --- a/src/hyperloom/orchestrator/actions/executors/baseline.py +++ b/src/hyperloom/orchestrator/actions/executors/baseline.py @@ -642,13 +642,23 @@ def _int(name: str, default: int) -> int: return default return value if value > 0 else default + def _is_valid_override(name: str) -> bool: + raw = (src.get(name) or "").strip() + try: + return int(raw) > 0 + except ValueError: + return False + explicit = _int("AGENTX_BASELINE_TIMEOUT_SEC", 0) if explicit: return explicit - overhead_explicit = (src.get("AGENTX_BASELINE_OVERHEAD_SEC") or "").strip() overhead = _int("AGENTX_BASELINE_OVERHEAD_SEC", AGENTX_BASELINE_OVERHEAD_SEC) - if not overhead_explicit: + # Same validity bar as `_int` itself (parses to a positive int) rather than + # "non-empty string" -- otherwise an invalid override (e.g. "abc" or "-1") + # both silently falls back to the default AND suppresses the warning meant + # to flag exactly that case. + if not _is_valid_override("AGENTX_BASELINE_OVERHEAD_SEC"): # AGENTX_BASELINE_OVERHEAD_SEC's default is calibrated on GLM-5.2/Qwen3.8 # (measured 4774s/6676s warmup+compile). A raw aiperf run against # Kimi-K3 (conc=64, ISL ~115k avg) measured warmup alone draining in From 0c155946ef1942391e010c6848ff387cae5f82f1 Mon Sep 17 00:00:00 2001 From: Zeng Date: Thu, 27 Aug 2026 18:27:38 +0800 Subject: [PATCH 06/12] fix(agentx): correct the profile-delay clamp and warmup non-canon checks - aiperf_client.sh: the _pmax safe bound for the self-bracketing profile delay only accounted for DURATION (the measurement window), not the warmup drain (bounded by WARMGRACE) that precedes it. On a long-warmup model this clamped an operator-tuned PWARM down to a fraction of what it needed to be, forcing the capture to fire mid-warmup -- the exact failure this self-bracketing exists to prevent. - aiperf_client.sh: WARMLANE/WARMGRACE non-canon checks used `!=`, so raising either above canonical (e.g. a longer grace period to let a large model's warmup fully drain) was flagged as a deviation even though it doesn't change what gets replayed. Changed to `-lt` so only a reduction is flagged. - aiperf_client.sh: CANON_WARMUP_PER_LANE/CANON_WARMUP_GRACE were declared twice with duplicated literals; now declared once, next to the WARMLANE/WARMGRACE defaults that derive from them. Co-Authored-By: Claude Sonnet 5 --- .../assets/agentx/aiperf_client.sh | 34 ++++++--- .../tests/test_aiperf_client_sh.py | 74 +++++++++++++++++++ 2 files changed, 97 insertions(+), 11 deletions(-) diff --git a/src/hyperloom/inference_optimizer/assets/agentx/aiperf_client.sh b/src/hyperloom/inference_optimizer/assets/agentx/aiperf_client.sh index 0f36a46001..0ca52c8db0 100755 --- a/src/hyperloom/inference_optimizer/assets/agentx/aiperf_client.sh +++ b/src/hyperloom/inference_optimizer/assets/agentx/aiperf_client.sh @@ -185,8 +185,10 @@ DURATION="${AGENTX_DURATION:-3600}" # period for them to drain before profiling starts. This replaces the old # --warmup-duration / --num-warmup-sessions pair, which the scenario does not # use and which measured a different thing entirely. -WARMLANE="${AGENTX_WARMUP_REQUESTS_PER_LANE:-10}" -WARMGRACE="${AGENTX_WARMUP_GRACE_PERIOD:-1800}" +CANON_WARMUP_PER_LANE=10 +CANON_WARMUP_GRACE=1800 +WARMLANE="${AGENTX_WARMUP_REQUESTS_PER_LANE:-$CANON_WARMUP_PER_LANE}" +WARMGRACE="${AGENTX_WARMUP_GRACE_PERIOD:-$CANON_WARMUP_GRACE}" # Per-trajectory-tree idle cap. NOT the same thing as the scenario's 10s # whole-system cap, and NOT scenario-locked -- upstream passes it explicitly @@ -263,8 +265,8 @@ CANON_DURATION=3600 # reduced-warmup round came back submission_valid=true and looked publishable. # On a 743B model the canonical 10/lane is a ~2h warmup, which is exactly when # an operator reaches for this knob, so the hole was reachable in practice. -CANON_WARMUP_PER_LANE=10 -CANON_WARMUP_GRACE=1800 +# (CANON_WARMUP_PER_LANE/CANON_WARMUP_GRACE are declared above, alongside +# WARMLANE/WARMGRACE, so the canonical value and the default can't drift apart.) # The corpus this model family canonically replays, before any operator pin. # CANON_DS is resolved with the corpus above. The family whitelist behind it is # a derivation, not a registry -- a model upstream runs on the full corpus but @@ -284,9 +286,14 @@ NONCANON=() [ "$DURATION" != "$CANON_DURATION" ] && NONCANON+=("duration=${DURATION}s(canonical ${CANON_DURATION}s)") [ -n "${AGENTX_MAX_CTX:-}" ] && NONCANON+=("client_context_cap=${AGENTX_MAX_CTX}") [ "${AGENTX_UNSAFE_OVERRIDE:-false}" = "true" ] && NONCANON+=("unsafe_override_forced") -[ "$WARMLANE" != "$CANON_WARMUP_PER_LANE" ] && \ +# `-lt`, not `!=`: only a *smaller* value under-pressures the cache or risks +# truncating the drain before it finishes. A larger value is strictly more +# warmup than canonical -- e.g. an operator raising the grace period so a +# large model's warmup has room to drain -- and does not change what gets +# replayed, so it must not be flagged as a deviation. +[ "$WARMLANE" -lt "$CANON_WARMUP_PER_LANE" ] && \ NONCANON+=("warmup_per_lane=${WARMLANE}(canonical ${CANON_WARMUP_PER_LANE})") -[ "$WARMGRACE" != "$CANON_WARMUP_GRACE" ] && \ +[ "$WARMGRACE" -lt "$CANON_WARMUP_GRACE" ] && \ NONCANON+=("warmup_grace=${WARMGRACE}s(canonical ${CANON_WARMUP_GRACE}s)") SMOKE_ARGS=() @@ -370,11 +377,16 @@ if [ "${PROFILE:-0}" = "1" ]; then # spends ~2.5h in the agentic warmup, so a delay tuned on a 35B round lands # either mid-warmup or past the end depending on which way the estimate erred. # - # So clamp toward "early". The round cannot outlast the measurement window plus - # the warmup that precedes it, and the only number known here is the window, so - # cap the delay at DURATION - PWIN - margin and say when the cap bites. A - # capture inside warmup is a usable trace; a capture that never happens is not. - _pmax=$(( DURATION - PWIN - 60 )) + # So clamp toward "early". The round cannot outlast the warmup drain plus the + # measurement window that follows it -- WARMGRACE bounds the former, DURATION + # the latter -- so cap the delay at WARMGRACE + DURATION - PWIN - margin and + # say when the cap bites. Omitting WARMGRACE would treat DURATION as if it + # were the whole round's clock instead of just the measurement phase, and + # clamp an operator-tuned PWARM (e.g. ~2.5h for a 743B model's warmup) down to + # a fraction of that -- forcing the capture to fire mid-warmup, the exact + # failure this self-bracketing exists to avoid. A capture inside warmup is a + # usable trace; a capture that never happens is not. + _pmax=$(( WARMGRACE + DURATION - PWIN - 60 )) [ "$_pmax" -lt 0 ] && _pmax=0 if [ "$PWARM" -gt "$_pmax" ]; then log "WARN profile delay ${PWARM}s exceeds the safe bound for a ${DURATION}s window; clamping to ${_pmax}s so the capture cannot land after the round ends" diff --git a/src/hyperloom/inference_optimizer/tests/test_aiperf_client_sh.py b/src/hyperloom/inference_optimizer/tests/test_aiperf_client_sh.py index fb877ddc68..59acc6e741 100644 --- a/src/hyperloom/inference_optimizer/tests/test_aiperf_client_sh.py +++ b/src/hyperloom/inference_optimizer/tests/test_aiperf_client_sh.py @@ -485,6 +485,54 @@ def test_profile_posts_bare_when_there_are_no_bounds(tmp_path, env): assert "-d" not in argv +def test_profile_delay_safe_bound_accounts_for_warmup_grace(tmp_path): + """The `_pmax` clamp must not treat DURATION as the whole round's clock. + + A large model's warmup drain is bounded by WARMGRACE, not DURATION, and + happens *before* the measurement window opens. With WARMGRACE=65, + DURATION=5, PWIN=0: the old DURATION-only bound (5 - 0 - 60 = -55, clamped + to 0) would force *any* positive PWARM to clamp to 0 and fire immediately + -- squarely inside warmup. The WARMGRACE-inclusive bound (65 + 5 - 0 - 60 + = 10) correctly leaves room for a delay that clears the drain first. + """ + bench, bind, res = _sandbox(tmp_path) + r = _run( + bench, + bind, + res, + tmp_path, + PROFILE="1", + AGENTX_PROFILE_WARMUP_S="8", + AGENTX_PROFILE_WINDOW_S="0", + AGENTX_DURATION="5", + AGENTX_WARMUP_GRACE_PERIOD="65", + FAKE_AIPERF_SLEEP="0.2", + AGENTX_CURL_MARKER=str(tmp_path / "curl.txt"), + ) + assert r.returncode == 0, r.stderr + assert "clamping to" not in (r.stdout + r.stderr) + + +def test_profile_delay_is_still_clamped_when_it_exceeds_the_full_bound(tmp_path): + """A PWARM that exceeds even WARMGRACE + DURATION is still clamped early.""" + bench, bind, res = _sandbox(tmp_path) + r = _run( + bench, + bind, + res, + tmp_path, + PROFILE="1", + AGENTX_PROFILE_WARMUP_S="30", + AGENTX_PROFILE_WINDOW_S="0", + AGENTX_DURATION="5", + AGENTX_WARMUP_GRACE_PERIOD="65", + FAKE_AIPERF_SLEEP="0.2", + AGENTX_CURL_MARKER=str(tmp_path / "curl.txt"), + ) + assert r.returncode == 0, r.stderr + assert "clamping to 10s" in (r.stdout + r.stderr) + + def test_agentx_server_script_override_without_framework(tmp_path): """An explicit AGENTX_SERVER_SCRIPT still resolves when FRAMEWORK is unset.""" bench, bind, res = _sandbox(tmp_path, make_builtin=False) @@ -634,3 +682,29 @@ def test_canonical_warmup_is_not_flagged(tmp_path): out = _result(res) assert not out["submission_invalid_reasons"] assert out["submission_valid"] is not False + + +def test_raised_warmup_grace_is_not_flagged_non_canonical(tmp_path): + """A *longer* grace period is more warmup, not less, and must not be flagged. + + An operator raising this so a large model's warmup has room to fully drain + (e.g. 4h for Kimi-K3-scale warmup) does not change what gets replayed -- + only how long the client is willing to wait for it. Flagging it the same + as a truncated drain would make the correct run non-submittable. + """ + bench, bind, res = _sandbox(tmp_path) + r = _run(bench, bind, res, tmp_path, AGENTX_WARMUP_GRACE_PERIOD="14400") + assert r.returncode == 0, r.stderr + out = _result(res) + assert not out["submission_invalid_reasons"] + assert out["submission_valid"] is not False + + +def test_raised_warmup_per_lane_is_not_flagged_non_canonical(tmp_path): + """Symmetric with the grace period: more warmup requests is not a deviation.""" + bench, bind, res = _sandbox(tmp_path) + r = _run(bench, bind, res, tmp_path, AGENTX_WARMUP_REQUESTS_PER_LANE="20") + assert r.returncode == 0, r.stderr + out = _result(res) + assert not out["submission_invalid_reasons"] + assert out["submission_valid"] is not False From 666c5c68466d34c5add99f564c05cdbc0e420c5c Mon Sep 17 00:00:00 2001 From: Zeng Date: Thu, 27 Aug 2026 19:30:46 +0800 Subject: [PATCH 07/12] fix(agentx): make the rebaseline and profile-clamp paths persisted-state aware - request_handlers.py: _agentx_rebaseline_timeout only checked the ambient HYPERLOOM_AGENTX env var, so an integrate call driven from a subprocess that did not inherit it silently fell back to the non-agentic timeout. Now checks agentx_active(), which also honors the persisted benchmark_mode on SharedState. - _workload_envs.py: extracted agentx_active() as the shared persisted-state check (agentx_kb_write_blocked already had this logic inline) so both call sites agree on what "AgentX is on" means. - _workload_envs.py: when HYPERLOOM_PROFILE_MAX_STEPS_CAP is set explicitly and AgentX's clamp overrides it anyway, warn instead of silently overriding -- otherwise a deliberate operator setting appears to have no effect with no trace of why. Co-Authored-By: Claude Sonnet 5 --- .../tests/test_agentx_rebaseline_timeout.py | 22 +++++++ .../tests/test_profile_and_kernel_handlers.py | 25 +++++++ .../tests/test_workload_envs.py | 34 ++++++++++ .../actions/executors/_workload_envs.py | 66 ++++++++++++++----- .../orchestrator/kernel/request_handlers.py | 14 ++-- 5 files changed, 139 insertions(+), 22 deletions(-) diff --git a/src/hyperloom/inference_optimizer/tests/test_agentx_rebaseline_timeout.py b/src/hyperloom/inference_optimizer/tests/test_agentx_rebaseline_timeout.py index 1e907ed876..e4f37b94e3 100644 --- a/src/hyperloom/inference_optimizer/tests/test_agentx_rebaseline_timeout.py +++ b/src/hyperloom/inference_optimizer/tests/test_agentx_rebaseline_timeout.py @@ -20,6 +20,8 @@ ``warmup_failure``, so nothing in the abort reason names the timeout. """ +from types import SimpleNamespace + from hyperloom.orchestrator.kernel.request_handlers import ( _agentx_rebaseline_timeout, ) @@ -70,3 +72,23 @@ def test_tracks_the_baseline_derivation(monkeypatch): monkeypatch.setenv("AGENTX_BASELINE_TIMEOUT_SEC", "44000") assert _agentx_rebaseline_timeout(7200) == 44000 + + +def test_persisted_benchmark_mode_raises_without_the_env_var(monkeypatch): + """A re-baseline driven from a subprocess that never inherited + ``HYPERLOOM_AGENTX`` must still get the raise from the session's + persisted ``benchmark_mode`` -- otherwise it reproduces the exact + mid-warmup kill this function exists to prevent. + """ + monkeypatch.delenv("HYPERLOOM_AGENTX", raising=False) + monkeypatch.setenv("AGENTX_DURATION", "3600") + monkeypatch.setenv("AGENTX_BASELINE_OVERHEAD_SEC", "28800") + monkeypatch.delenv("AGENTX_BASELINE_TIMEOUT_SEC", raising=False) + shared_state = SimpleNamespace(benchmark_mode="agentx") + assert _agentx_rebaseline_timeout(7200, shared_state=shared_state) == 32400 + + +def test_unrelated_benchmark_mode_does_not_raise(monkeypatch): + monkeypatch.delenv("HYPERLOOM_AGENTX", raising=False) + shared_state = SimpleNamespace(benchmark_mode="synthetic") + assert _agentx_rebaseline_timeout(7200, shared_state=shared_state) == 7200 diff --git a/src/hyperloom/inference_optimizer/tests/test_profile_and_kernel_handlers.py b/src/hyperloom/inference_optimizer/tests/test_profile_and_kernel_handlers.py index 2dc4ec9f77..970d8a7dce 100644 --- a/src/hyperloom/inference_optimizer/tests/test_profile_and_kernel_handlers.py +++ b/src/hyperloom/inference_optimizer/tests/test_profile_and_kernel_handlers.py @@ -863,6 +863,31 @@ def test_materialize_profile_agentx_clamp_warns_below_steady_floor( assert any("steady-state floor" in r.message for r in caplog.records) +def test_materialize_profile_agentx_clamp_warns_on_explicit_override( + tmp_path, + monkeypatch, + caplog, +): + """An explicit HYPERLOOM_PROFILE_MAX_STEPS_CAP must not be silently overridden. + + Without this, an operator who explicitly raised the cap (e.g. to widen the + profiler's steady-state window) would see it clamped back to 8 by the + AgentX branch with no indication their override had no effect. + """ + import yaml + + _clear_workload_env(monkeypatch) + monkeypatch.setenv("HYPERLOOM_AGENTX", "1") + monkeypatch.setenv("HYPERLOOM_PROFILE_MAX_STEPS_CAP", "64") + src = _profile_yaml(tmp_path, "vllm", {"CONC": 32, "ISL": 256, "OSL": 1024}) + with caplog.at_level("WARNING"): + out = _materialize_config_with_envs(src, tmp_path) + rendered = yaml.safe_load(out.read_text()) + extra = rendered["benchmark"]["envs"]["EXTRA_VLLM_ARGS"] + assert "--profiler-config.max_iterations 8" in extra, extra + assert any("explicit HYPERLOOM_PROFILE_MAX_STEPS_CAP=64" in r.message for r in caplog.records) + + def test_materialize_persists_inferencex_path_for_magpie( tmp_path, monkeypatch, diff --git a/src/hyperloom/inference_optimizer/tests/test_workload_envs.py b/src/hyperloom/inference_optimizer/tests/test_workload_envs.py index ebf227e1c0..c0bd063bad 100644 --- a/src/hyperloom/inference_optimizer/tests/test_workload_envs.py +++ b/src/hyperloom/inference_optimizer/tests/test_workload_envs.py @@ -678,6 +678,40 @@ def test_quality_ref_zero_config_baseline_writes_session_ref(monkeypatch, tmp_pa assert bench["envs"]["XDIT_QUALITY_REF_WRITE"] == expected +# --------------------------------------------------------------------------- +# agentx_active: persisted benchmark_mode as a fallback for a missing env var +# --------------------------------------------------------------------------- + + +def test_agentx_active_true_from_env_var(monkeypatch): + _clear_env(monkeypatch) + monkeypatch.setenv("HYPERLOOM_AGENTX", "1") + assert we.agentx_active() is True + + +def test_agentx_active_false_with_neither_signal(monkeypatch): + _clear_env(monkeypatch) + assert we.agentx_active() is False + assert we.agentx_active(SimpleNamespace(benchmark_mode="")) is False + + +def test_agentx_active_true_from_persisted_state_without_env_var(monkeypatch): + # A subprocess/SDK caller that never inherited HYPERLOOM_AGENTX must still + # be recognized as AgentX-active from the session's persisted mode. + _clear_env(monkeypatch) + assert we.agentx_active(SimpleNamespace(benchmark_mode="agentx")) is True + + +def test_agentx_kb_write_blocked_matches_agentx_active(monkeypatch): + # agentx_kb_write_blocked delegates to agentx_active; both signals still work. + _clear_env(monkeypatch) + assert we.agentx_kb_write_blocked() is False + monkeypatch.setenv("HYPERLOOM_AGENTX", "1") + assert we.agentx_kb_write_blocked() is True + _clear_env(monkeypatch) + assert we.agentx_kb_write_blocked(SimpleNamespace(benchmark_mode="agentx")) is True + + # --------------------------------------------------------------------------- # Scriptable baseline sampling cost (measurement contract values) # --------------------------------------------------------------------------- diff --git a/src/hyperloom/orchestrator/actions/executors/_workload_envs.py b/src/hyperloom/orchestrator/actions/executors/_workload_envs.py index f0933f4cdd..38e7580e3f 100644 --- a/src/hyperloom/orchestrator/actions/executors/_workload_envs.py +++ b/src/hyperloom/orchestrator/actions/executors/_workload_envs.py @@ -135,6 +135,27 @@ def agentx_enabled(env: dict[str, str] | None = None) -> bool: return str(raw).strip().lower() in _AGENTX_TRUE_VALUES +def agentx_active(shared_state: Any = None) -> bool: + """Whether AgentX is enabled, preferring persisted state over the ambient env var. + + ``benchmark_mode`` is stamped at seed precisely so it survives a restart, + while ``HYPERLOOM_AGENTX`` only describes the shell that happens to be + running -- an SDK caller, or a re-baseline/variant round driven from a + subprocess that did not inherit it, would otherwise miss the AgentX-sized + timeout or publish the agentic number under a synthetic tag. Either saying + "agentx" is enough. + + Args: + shared_state: Session state, when the caller has one. + + Returns: + True when AgentX is enabled for this session, by either signal. + """ + if agentx_enabled(): + return True + return str(getattr(shared_state, "benchmark_mode", "") or "").strip().lower() == "agentx" + + def agentx_kb_write_blocked(shared_state: Any = None) -> bool: """Whether an agentic measurement must stay out of the cross-session KB. @@ -151,21 +172,13 @@ def agentx_kb_write_blocked(shared_state: Any = None) -> bool: finalize, the runtime amend, and the T0 anchor), they were not all found at once, and a fourth should have something obvious to call. - Prefers the persisted mode over the ambient env var. ``benchmark_mode`` is - stamped at seed precisely so it survives a restart, while ``HYPERLOOM_AGENTX`` - only describes the shell that happens to be running -- an SDK caller, or a - CLOSE re-driven in a subprocess that did not inherit it, would otherwise - publish the agentic number. Either saying "agentx" is enough. - Args: shared_state: Session state, when the caller has one. Returns: True when the caller must skip its Recipe KB write. """ - if agentx_enabled(): - return True - return str(getattr(shared_state, "benchmark_mode", "") or "").strip().lower() == "agentx" + return agentx_active(shared_state) def apply_agentx_switch(bench: dict[str, Any], model_path: str | None = None) -> None: @@ -1005,8 +1018,10 @@ def materialize_config_with_envs( safe_conc = max(conc_val, 1) # Cap captured decode steps at a serialization-safe default so the # torch-profiler trace can be written without starving the engine RPC. + _cap_raw = os.environ.get("HYPERLOOM_PROFILE_MAX_STEPS_CAP", "").strip() + cap_explicit = _cap_raw.isdigit() and int(_cap_raw) >= 1 try: - cap = int(os.environ.get("HYPERLOOM_PROFILE_MAX_STEPS_CAP", "").strip() or _DEFAULT_PROFILE_MAX_STEPS) + cap = int(_cap_raw or _DEFAULT_PROFILE_MAX_STEPS) except (TypeError, ValueError): cap = _DEFAULT_PROFILE_MAX_STEPS if cap < 1: @@ -1088,14 +1103,29 @@ def materialize_config_with_envs( # extra steps buy nothing and only inflate the in-memory event # buffer. HYPERLOOM_PROFILE_MAX_ITERS still overrides this below. if max_iters > _AGENTX_PROFILE_MAX_ITERS: - log.info( - "AgentX: lowering captured profile steps %d -> %d. The cap is " - "calibrated on the synthetic ISL/OSL shape; an agentic step " - "carries orders of magnitude more, and the torch profiler " - "buffers events in host RAM until the OOM killer arrives.", - max_iters, - _AGENTX_PROFILE_MAX_ITERS, - ) + if cap_explicit: + # The operator asked for this cap explicitly (e.g. to widen + # the steady-state window); silently overriding it with no + # trace of the original value would hide why a deliberate + # HYPERLOOM_PROFILE_MAX_STEPS_CAP setting had no effect. + log.warning( + "AgentX: explicit HYPERLOOM_PROFILE_MAX_STEPS_CAP=%d is " + "being overridden to %d. The cap is calibrated on the " + "synthetic ISL/OSL shape; an agentic step carries orders " + "of magnitude more, and the torch profiler buffers " + "events in host RAM until the OOM killer arrives.", + max_iters, + _AGENTX_PROFILE_MAX_ITERS, + ) + else: + log.info( + "AgentX: lowering captured profile steps %d -> %d. The cap is " + "calibrated on the synthetic ISL/OSL shape; an agentic step " + "carries orders of magnitude more, and the torch profiler " + "buffers events in host RAM until the OOM killer arrives.", + max_iters, + _AGENTX_PROFILE_MAX_ITERS, + ) max_iters = _AGENTX_PROFILE_MAX_ITERS if max_iters < steady_floor: log.warning( diff --git a/src/hyperloom/orchestrator/kernel/request_handlers.py b/src/hyperloom/orchestrator/kernel/request_handlers.py index 79aa376fe9..b91e7d4203 100644 --- a/src/hyperloom/orchestrator/kernel/request_handlers.py +++ b/src/hyperloom/orchestrator/kernel/request_handlers.py @@ -7532,7 +7532,7 @@ def _grade_integrate_accuracy( } -def _agentx_rebaseline_timeout(resolved_sec: int) -> int: +def _agentx_rebaseline_timeout(resolved_sec: int, *, shared_state: Any = None) -> int: """Raise a re-baseline timeout to what an AgentX round needs. Same shape, and the same root cause, as @@ -7556,13 +7556,16 @@ def _agentx_rebaseline_timeout(resolved_sec: int) -> int: Args: resolved_sec: The timeout the payload/contract resolved to. + shared_state: Session state, so a persisted ``benchmark_mode`` still + triggers the raise when this integrate call runs in a subprocess + that did not inherit ``HYPERLOOM_AGENTX``. Returns: int: ``resolved_sec``, or the AgentX-derived cap when that is larger. """ - from ..actions.executors._workload_envs import agentx_enabled + from ..actions.executors._workload_envs import agentx_active - if not agentx_enabled(): + if not agentx_active(shared_state): return resolved_sec from ..actions.executors.baseline import agentx_baseline_timeout_sec @@ -7824,13 +7827,16 @@ async def integrate_handler( fake_task_id = f"integrate-{kernel_id or 'anon'}" workspace = unique_runs_dir(session_dir, "integrate", fake_task_id) baseline_executor = BaselineExecutor(session_dir=session_dir) + from ..state.shared_state import SharedState + rebaseline_timeout_sec = _agentx_rebaseline_timeout( _cold_start_rebaseline_timeout( _integrate_rebaseline_timeout_sec( payload, default_timeout_sec=baseline_executor.default_timeout_sec, ) - ) + ), + shared_state=SharedState.load_or_init(session_dir), ) fake_task = Task( task_id=fake_task_id, From c3a1442baa51dfd157fe2765ae559a967e116a32 Mon Sep 17 00:00:00 2001 From: Zeng Date: Thu, 27 Aug 2026 19:54:50 +0800 Subject: [PATCH 08/12] fix(agentx): derive the baseline overhead, and close two silent-deviation gaps Addresses the M1/M2/M3 findings of the PR 1309 architecture review. - baseline.py: AGENTX_BASELINE_OVERHEAD_SEC was a flat 7200s covering setup, corpus load, warmup and first-compile, calibrated on GLM-5.2/Qwen3.8. Warmup is the share that varies by model, and it already has an operator-visible bound in the client -- AGENTX_WARMUP_GRACE_PERIOD. A model whose warmup runs long is a model whose operator has already had to raise that knob, so the overhead now derives from it (5400s non-warmup + grace) instead of asking for a second, independent number meaning the same thing. At canonical settings the sum is unchanged, preserving the measured calibration. Every input is logged, and the "nothing has been tuned for this model" warning now fires only when neither knob is set. - _workload_envs.py: HYPERLOOM_PROFILE_MAX_ITERS is applied after the AgentX capture clamp and wins, which is intended -- but it lifts a host-RAM bound, not a serialization one, and neither existing warning could report it. `cap` defaults to _DEFAULT_PROFILE_MAX_STEPS (128), so the obvious override of 128 was neither below the steady-state floor nor above the cap and restored the full OOM exposure in silence. Still honoured verbatim; now warns. - aiperf_client.sh: AGENTX_FAILED_REQUEST_THRESHOLD was missing from the non-canonical list. Raising it keeps alive a run upstream's 0.10 would have aborted, and aiperf stamps no scenario marker for it, so the round came back submission_valid=true. Only a larger ratio is flagged; tightening it measures a strictly cleaner run. CANON_FRT is declared once and feeds the default, so the canonical value and the default cannot drift apart. - Cross-referenced the three synthetic-sized variant-timeout defaults with agentx_variant_timeout_sec, and added a CHANGELOG entry with operator notes for the two behaviour changes (derived cap, submission_valid on a raised failure threshold). Co-Authored-By: Claude Opus 5 (1M context) --- CHANGELOG.md | 45 +++++++++++ .../assets/agentx/aiperf_client.sh | 16 +++- .../tests/test_agentx_baseline_timeout.py | 54 +++++++++++++ .../tests/test_aiperf_client_sh.py | 42 ++++++++++ .../tests/test_profile_and_kernel_handlers.py | 48 ++++++++++++ .../actions/executors/_grid_base.py | 6 +- .../actions/executors/_workload_envs.py | 18 +++++ .../actions/executors/baseline.py | 77 ++++++++++++++----- .../actions/executors/integrate_patch.py | 4 + .../orchestrator/kernel/conc_sweep.py | 3 + 10 files changed, 292 insertions(+), 21 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index e0333b5c29..5a1677d12f 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -5,6 +5,51 @@ The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.1.0/), ## [Unreleased] +### Fixed + +- **The AgentX baseline overhead is derived from the warmup bound instead of a + flat constant.** `AGENTX_BASELINE_OVERHEAD_SEC` was a single measured number + (7200s, calibrated on GLM-5.2/Qwen3.8) covering setup, corpus load, warmup and + first-compile. Warmup is the share that actually varies by model, and it + already has an operator-visible bound in the client: + `AGENTX_WARMUP_GRACE_PERIOD`. A model whose warmup runs long is therefore a + model whose operator has already raised that knob — a raw aiperf run against + Kimi-K3 at concurrency 64 measured warmup alone at ~12075s, past the entire + flat cap. The overhead is now `5400s non-warmup + AGENTX_WARMUP_GRACE_PERIOD`, + and every input is logged at INFO so a field timeout can be read back to the + values that produced it.
+ **Operator note**: at canonical settings the cap is unchanged + (5400 + 1800 = 7200), so nothing moves for existing synthetic or GLM-5.2-class + runs. Raising `AGENTX_WARMUP_GRACE_PERIOD` now also raises the baseline + timeout by the same amount — which is the point, but it means the round's + worst-case wall clock grows with that knob. `AGENTX_BASELINE_OVERHEAD_SEC` + still overrides the derivation outright, and the "nothing has been tuned for + this model" warning now fires only when *neither* knob is set. + +- **Overriding `HYPERLOOM_PROFILE_MAX_ITERS` under AgentX no longer lifts the + host-RAM capture bound silently.** The AgentX branch clamps captured profile + steps to 8 because an agentic step carries orders of magnitude more profiler + events than the synthetic shape the normal cap is sized against — at the stock + cap a DeepSeek-V4 round was OOM-killed mid-capture three times in a row. The + operator override is applied afterwards and wins, which is intended, but the + two existing warnings could not report it: `cap` defaults to 128, so the + obvious `HYPERLOOM_PROFILE_MAX_ITERS=128` was neither below the steady-state + floor nor above the cap and restored the full exposure without printing + anything. The override is still honoured verbatim; it now warns. + +- **A loosened `AGENTX_FAILED_REQUEST_THRESHOLD` is flagged as a non-canonical + workload.** Raising the abort ratio keeps alive a run that upstream's 0.10 + would have aborted, and the surviving requests are then mapped as an ordinary + measurement. aiperf stamps no scenario marker for it — the threshold is the + client's own safety net, not part of the scenario — so the round came back + `submission_valid=true`. Only a *larger* ratio is flagged; tightening it + measures a strictly cleaner run.
+ **Operator note**: a run that raises this knob is now stamped + `submission_valid=false` with `failed_request_threshold=(canonical 0.10)` + in `submission_invalid_reasons`, and `benchmark_result.py` will refuse the + measurement. Rounds that previously passed on a raised threshold will now be + rejected — which is the intended correction, not a regression. + ## [v1.0.0] - 2026-08-26 Current packaged version (`pyproject.toml`). See [release notes](docs/release-notes.md) and the diff --git a/src/hyperloom/inference_optimizer/assets/agentx/aiperf_client.sh b/src/hyperloom/inference_optimizer/assets/agentx/aiperf_client.sh index 0ca52c8db0..9d94fc1e7d 100755 --- a/src/hyperloom/inference_optimizer/assets/agentx/aiperf_client.sh +++ b/src/hyperloom/inference_optimizer/assets/agentx/aiperf_client.sh @@ -236,7 +236,10 @@ AIPERF="${AIPERF_BIN:-aiperf}" # map_aiperf.py carries no error counters. This is the safety net that turns a # server/client context mismatch into an honest failure instead of a fabricated # win on the surviving short sessions. Matches upstream's 0.10. -FRT="${AGENTX_FAILED_REQUEST_THRESHOLD:-0.10}" +# Declared as one value, like CANON_WARMUP_*/WARMLANE below, so the canonical +# ratio and the default cannot drift apart. +CANON_FRT=0.10 +FRT="${AGENTX_FAILED_REQUEST_THRESHOLD:-$CANON_FRT}" # ── Non-canonical workloads may run, but may never be submittable ──────────── # The scenario enforces a 900s duration floor, so a shortened AGENTX_DURATION is @@ -295,6 +298,17 @@ NONCANON=() NONCANON+=("warmup_per_lane=${WARMLANE}(canonical ${CANON_WARMUP_PER_LANE})") [ "$WARMGRACE" -lt "$CANON_WARMUP_GRACE" ] && \ NONCANON+=("warmup_grace=${WARMGRACE}s(canonical ${CANON_WARMUP_GRACE}s)") +# The abort threshold is measurement-defining for the same reason warmup is: +# raising it keeps a run alive that upstream's 0.10 would have aborted, and the +# surviving requests are then mapped as a normal measurement. aiperf stamps no +# scenario marker for it -- the threshold is the client's own safety net, not +# something the scenario knows about -- so a loosened round comes back +# submission_valid=true. Only a *larger* ratio deviates; tightening it below +# canonical measures a strictly cleaner run. Compared with awk because the +# ratio is a decimal, which `-lt` cannot handle. +if awk "BEGIN{exit !(($FRT) > ($CANON_FRT))}" 2>/dev/null; then + NONCANON+=("failed_request_threshold=${FRT}(canonical ${CANON_FRT})") +fi SMOKE_ARGS=() if [ "$DURATION" -lt "$CANON_DURATION" ] || [ "${AGENTX_UNSAFE_OVERRIDE:-false}" = "true" ]; then diff --git a/src/hyperloom/inference_optimizer/tests/test_agentx_baseline_timeout.py b/src/hyperloom/inference_optimizer/tests/test_agentx_baseline_timeout.py index c37a40857f..5e9da193dd 100644 --- a/src/hyperloom/inference_optimizer/tests/test_agentx_baseline_timeout.py +++ b/src/hyperloom/inference_optimizer/tests/test_agentx_baseline_timeout.py @@ -26,6 +26,7 @@ from hyperloom.orchestrator.actions.executors.baseline import ( AGENTX_BASELINE_OVERHEAD_SEC, + AGENTX_CANON_WARMUP_GRACE_SEC, AGENTX_DEFAULT_DURATION_SEC, BASELINE_DEFAULT_TIMEOUT_SEC, BaselineExecutor, @@ -43,6 +44,7 @@ def _clear(monkeypatch): "AGENTX_DURATION", "AGENTX_BASELINE_TIMEOUT_SEC", "AGENTX_BASELINE_OVERHEAD_SEC", + "AGENTX_WARMUP_GRACE_PERIOD", ): monkeypatch.delenv(k, raising=False) @@ -104,6 +106,58 @@ def test_explicit_overhead_override_suppresses_the_warning(monkeypatch, caplog): assert not any("AGENTX_BASELINE_OVERHEAD_SEC" in r.message for r in caplog.records) +def test_overhead_tracks_the_warmup_grace_the_operator_set(monkeypatch): + """The knob that bounds the warmup must also size the cap that has to cover it. + + A model whose warmup runs long is a model whose operator has already had to + raise AGENTX_WARMUP_GRACE_PERIOD for the round to finish -- the Kimi-K3 + case, where the flat overhead was smaller than the warmup itself. Deriving + from that same knob is what stops the two numbers disagreeing. + """ + _clear(monkeypatch) + monkeypatch.setenv("AGENTX_WARMUP_GRACE_PERIOD", "14400") + grown = 14400 - AGENTX_CANON_WARMUP_GRACE_SEC + assert agentx_baseline_timeout_sec() == ( + AGENTX_DEFAULT_DURATION_SEC + AGENTX_BASELINE_OVERHEAD_SEC + grown + ) + + +def test_canonical_grace_reproduces_the_measured_constant(monkeypatch): + """Splitting the constant must not move it: same inputs, same number.""" + _clear(monkeypatch) + monkeypatch.setenv("AGENTX_WARMUP_GRACE_PERIOD", str(AGENTX_CANON_WARMUP_GRACE_SEC)) + assert agentx_baseline_timeout_sec() == ( + AGENTX_DEFAULT_DURATION_SEC + AGENTX_BASELINE_OVERHEAD_SEC + ) + + +def test_explicit_overhead_outranks_the_derivation(monkeypatch): + """A pinned overhead is an answer, not an input: the grace must not add to it.""" + _clear(monkeypatch) + monkeypatch.setenv("AGENTX_WARMUP_GRACE_PERIOD", "14400") + monkeypatch.setenv("AGENTX_BASELINE_OVERHEAD_SEC", "3600") + assert agentx_baseline_timeout_sec() == AGENTX_DEFAULT_DURATION_SEC + 3600 + + +def test_a_tuned_grace_suppresses_the_uncalibrated_warning(monkeypatch, caplog): + """The warning is about nobody having sized this model, not about the default.""" + _clear(monkeypatch) + monkeypatch.setenv("AGENTX_WARMUP_GRACE_PERIOD", "14400") + with caplog.at_level("WARNING"): + agentx_baseline_timeout_sec() + assert not any("AGENTX_BASELINE_OVERHEAD_SEC" in r.message for r in caplog.records) + + +@pytest.mark.parametrize("bad", ["", " ", "abc", "0", "-1"]) +def test_unparseable_grace_falls_back_to_canonical(monkeypatch, bad): + """A typo in the grace must not shrink the cap below the measured default.""" + _clear(monkeypatch) + monkeypatch.setenv("AGENTX_WARMUP_GRACE_PERIOD", bad) + assert agentx_baseline_timeout_sec() == ( + AGENTX_DEFAULT_DURATION_SEC + AGENTX_BASELINE_OVERHEAD_SEC + ) + + def test_explicit_cap_wins_outright(monkeypatch): _clear(monkeypatch) monkeypatch.setenv("AGENTX_DURATION", "7200") diff --git a/src/hyperloom/inference_optimizer/tests/test_aiperf_client_sh.py b/src/hyperloom/inference_optimizer/tests/test_aiperf_client_sh.py index 59acc6e741..4bab9027ed 100644 --- a/src/hyperloom/inference_optimizer/tests/test_aiperf_client_sh.py +++ b/src/hyperloom/inference_optimizer/tests/test_aiperf_client_sh.py @@ -708,3 +708,45 @@ def test_raised_warmup_per_lane_is_not_flagged_non_canonical(tmp_path): out = _result(res) assert not out["submission_invalid_reasons"] assert out["submission_valid"] is not False + + +def test_raised_failed_request_threshold_is_flagged_non_canonical(tmp_path): + """Loosening the abort threshold is measurement-defining and carries no marker. + + Raising it keeps alive a run that upstream's 0.10 would have aborted, and + the requests that did survive are then mapped as an ordinary measurement. + aiperf stamps nothing for this -- the threshold is the client's own safety + net, not part of the scenario -- so without the client objecting the round + comes back submission_valid=true. + """ + bench, bind, res = _sandbox(tmp_path) + r = _run(bench, bind, res, tmp_path, AGENTX_FAILED_REQUEST_THRESHOLD="0.5") + assert r.returncode == 0, r.stderr + out = _result(res) + assert out["submission_valid"] is False + assert any( + "failed_request_threshold=0.5" in x for x in out["submission_invalid_reasons"] + ), out["submission_invalid_reasons"] + + +def test_tightened_failed_request_threshold_is_not_flagged(tmp_path): + """A stricter threshold measures a cleaner run, so it is not a deviation.""" + bench, bind, res = _sandbox(tmp_path) + r = _run(bench, bind, res, tmp_path, AGENTX_FAILED_REQUEST_THRESHOLD="0.01") + assert r.returncode == 0, r.stderr + out = _result(res) + assert not out["submission_invalid_reasons"] + assert out["submission_valid"] is not False + + +def test_canonical_failed_request_threshold_is_not_flagged(tmp_path): + """Restating the canonical ratio must not trip the check, in either spelling.""" + for spelling in ("0.10", "0.1"): + base = tmp_path / spelling.replace(".", "_") + base.mkdir() + bench, bind, res = _sandbox(base) + r = _run(bench, bind, res, tmp_path, AGENTX_FAILED_REQUEST_THRESHOLD=spelling) + assert r.returncode == 0, r.stderr + out = _result(res) + assert not out["submission_invalid_reasons"], spelling + assert out["submission_valid"] is not False, spelling diff --git a/src/hyperloom/inference_optimizer/tests/test_profile_and_kernel_handlers.py b/src/hyperloom/inference_optimizer/tests/test_profile_and_kernel_handlers.py index 970d8a7dce..e8d79c04ba 100644 --- a/src/hyperloom/inference_optimizer/tests/test_profile_and_kernel_handlers.py +++ b/src/hyperloom/inference_optimizer/tests/test_profile_and_kernel_handlers.py @@ -888,6 +888,54 @@ def test_materialize_profile_agentx_clamp_warns_on_explicit_override( assert any("explicit HYPERLOOM_PROFILE_MAX_STEPS_CAP=64" in r.message for r in caplog.records) +def test_materialize_profile_max_iters_override_warns_it_undoes_the_agentx_bound( + tmp_path, + monkeypatch, + caplog, +): + """Overriding the AgentX capture bound must say so -- 128 warns about nothing else. + + HYPERLOOM_PROFILE_MAX_ITERS is applied after the AgentX clamp and wins, so + it restores exactly the host-RAM exposure the clamp exists to remove. The + two pre-existing warnings cannot cover this: ``cap`` defaults to + _DEFAULT_PROFILE_MAX_STEPS (128), so an override of 128 is neither below + steady_floor's band nor above the cap, and the bound would be lifted in + silence. + """ + import yaml + + _clear_workload_env(monkeypatch) + monkeypatch.setenv("HYPERLOOM_AGENTX", "1") + monkeypatch.setenv("HYPERLOOM_PROFILE_MAX_ITERS", "128") + src = _profile_yaml(tmp_path, "vllm", {"CONC": 32, "ISL": 256, "OSL": 1024}) + with caplog.at_level("WARNING"): + out = _materialize_config_with_envs(src, tmp_path) + rendered = yaml.safe_load(out.read_text()) + extra = rendered["benchmark"]["envs"]["EXTRA_VLLM_ARGS"] + # The override is still honored verbatim; this is a visibility fix only. + assert "--profiler-config.max_iterations 128" in extra, extra + assert any( + "HYPERLOOM_PROFILE_MAX_ITERS=128 overrides the AgentX capture bound of 8" in r.message + for r in caplog.records + ), [r.message for r in caplog.records] + + +def test_materialize_profile_max_iters_override_is_quiet_without_agentx( + tmp_path, + monkeypatch, + caplog, +): + """The new warning is AgentX-only; the synthetic path has no host-RAM bound.""" + import yaml + + _clear_workload_env(monkeypatch) + monkeypatch.setenv("HYPERLOOM_PROFILE_MAX_ITERS", "128") + src = _profile_yaml(tmp_path, "vllm", {"CONC": 32, "ISL": 256, "OSL": 1024}) + with caplog.at_level("WARNING"): + _materialize_config_with_envs(src, tmp_path) + assert not any("AgentX capture bound" in r.message for r in caplog.records) + + def test_materialize_persists_inferencex_path_for_magpie( tmp_path, monkeypatch, diff --git a/src/hyperloom/orchestrator/actions/executors/_grid_base.py b/src/hyperloom/orchestrator/actions/executors/_grid_base.py index 20ab253255..a4a5ade271 100644 --- a/src/hyperloom/orchestrator/actions/executors/_grid_base.py +++ b/src/hyperloom/orchestrator/actions/executors/_grid_base.py @@ -63,7 +63,11 @@ def variant_fingerprint( ) -_VARIANT_TIMEOUT_SEC_DEFAULT = 7800 # 130 min; matches BASELINE_DEFAULT_TIMEOUT_SEC +# 130 min; matches BASELINE_DEFAULT_TIMEOUT_SEC. Sized for the synthetic +# ISL/OSL shape: an AgentX round does not fit it, and is not meant to -- see +# ``agentx_variant_timeout_sec`` in ``_grid_runner``, which raises whatever cap +# reaches it rather than expecting this default to cover both workloads. +_VARIANT_TIMEOUT_SEC_DEFAULT = 7800 @dataclass diff --git a/src/hyperloom/orchestrator/actions/executors/_workload_envs.py b/src/hyperloom/orchestrator/actions/executors/_workload_envs.py index 38e7580e3f..a1da421558 100644 --- a/src/hyperloom/orchestrator/actions/executors/_workload_envs.py +++ b/src/hyperloom/orchestrator/actions/executors/_workload_envs.py @@ -1147,6 +1147,24 @@ def materialize_config_with_envs( delay_iters = 8 if delay_iters < 0: delay_iters = 0 + # The AgentX clamp above is a HOST RAM bound, and this override + # silently undoes it. Neither check below stands in for saying so: + # ``cap`` defaults to _DEFAULT_PROFILE_MAX_STEPS, so the obvious + # HYPERLOOM_PROFILE_MAX_ITERS=128 lands exactly on it, trips + # neither branch, and restores the very bound that kept the + # profiler from being OOM-killed -- without printing anything. + if agentx_enabled() and max_iters > _AGENTX_PROFILE_MAX_ITERS: + log.warning( + "HYPERLOOM_PROFILE_MAX_ITERS=%d overrides the AgentX capture " + "bound of %d. That bound is a host-RAM limit, not a " + "serialization one: an agentic step carries orders of " + "magnitude more events than the synthetic shape ``cap`` is " + "sized against, and at the stock cap a DeepSeek-V4 profile " + "round was OOM-killed mid-capture three times in a row. " + "Unset it to restore the bound.", + max_iters, + _AGENTX_PROFILE_MAX_ITERS, + ) if max_iters < steady_floor: log.warning( "HYPERLOOM_PROFILE_MAX_ITERS=%d is below the steady-state " diff --git a/src/hyperloom/orchestrator/actions/executors/baseline.py b/src/hyperloom/orchestrator/actions/executors/baseline.py index 55022731fb..c129e19051 100644 --- a/src/hyperloom/orchestrator/actions/executors/baseline.py +++ b/src/hyperloom/orchestrator/actions/executors/baseline.py @@ -622,6 +622,19 @@ def _classify_subprocess_error( AGENTX_BASELINE_OVERHEAD_SEC = 7200 # setup + corpus + warmup + first-compile AGENTX_DEFAULT_DURATION_SEC = 3600 # mirrors aiperf_client.sh's default +# The warmup share of that overhead is not a constant either, and it is the +# share that actually varies by model: aiperf_client.sh bounds the warmup drain +# with AGENTX_WARMUP_GRACE_PERIOD, so a model whose warmup runs long is a model +# whose operator has already had to raise that knob for the round to complete at +# all. Splitting the flat 7200 at its canonical grace lets the cap follow that +# same knob instead of asking for a second, independent number that means the +# same thing -- which is how the constant came to be wrong for Kimi-K3 (a raw +# aiperf run measured warmup alone at ~12075s, past this entire cap). At +# canonical settings the sum is unchanged, so the measured GLM-5.2/Qwen3.8 +# calibration this number carries is preserved exactly. +AGENTX_CANON_WARMUP_GRACE_SEC = 1800 # aiperf_client.sh's CANON_WARMUP_GRACE +_AGENTX_NON_WARMUP_OVERHEAD_SEC = AGENTX_BASELINE_OVERHEAD_SEC - AGENTX_CANON_WARMUP_GRACE_SEC + def agentx_baseline_timeout_sec(env: "Mapping[str, str] | None" = None) -> int: """Resolve the AgentX baseline cap: explicit, else duration + overhead. @@ -653,29 +666,55 @@ def _is_valid_override(name: str) -> bool: if explicit: return explicit - overhead = _int("AGENTX_BASELINE_OVERHEAD_SEC", AGENTX_BASELINE_OVERHEAD_SEC) # Same validity bar as `_int` itself (parses to a positive int) rather than # "non-empty string" -- otherwise an invalid override (e.g. "abc" or "-1") # both silently falls back to the default AND suppresses the warning meant # to flag exactly that case. - if not _is_valid_override("AGENTX_BASELINE_OVERHEAD_SEC"): - # AGENTX_BASELINE_OVERHEAD_SEC's default is calibrated on GLM-5.2/Qwen3.8 - # (measured 4774s/6676s warmup+compile). A raw aiperf run against - # Kimi-K3 (conc=64, ISL ~115k avg) measured warmup alone draining in - # ~12075s -- already past this whole cap before profiling even starts. - # Nothing here can tell long-context/slow-prefill models apart from the - # ones this constant was measured on, so surface it instead of letting - # the round run untimed-out until the flat cap kills it mid-warmup. - log.warning( - "agentx_baseline_timeout_sec: using default AGENTX_BASELINE_OVERHEAD_SEC=%ds " - "(no explicit override set). This default is calibrated on GLM-5.2/Qwen3.8 " - "and may be far too small for long-context or slow-prefill models -- a raw " - "aiperf run against Kimi-K3 at concurrency=64 measured warmup alone taking " - "~12075s. If this round is for such a model, set AGENTX_BASELINE_OVERHEAD_SEC " - "explicitly.", - overhead, - ) - return _int("AGENTX_DURATION", AGENTX_DEFAULT_DURATION_SEC) + overhead + if _is_valid_override("AGENTX_BASELINE_OVERHEAD_SEC"): + overhead = _int("AGENTX_BASELINE_OVERHEAD_SEC", AGENTX_BASELINE_OVERHEAD_SEC) + grace = None + else: + # Derive the warmup share from the same knob that bounds it in the + # client, so the cap tracks the round the operator actually configured + # rather than the one this constant was measured on. + grace = _int("AGENTX_WARMUP_GRACE_PERIOD", AGENTX_CANON_WARMUP_GRACE_SEC) + overhead = _AGENTX_NON_WARMUP_OVERHEAD_SEC + grace + if not _is_valid_override("AGENTX_WARMUP_GRACE_PERIOD"): + # Nothing has been tuned for this model at all. The derivation + # above is only as good as its warmup bound, and at the canonical + # grace that bound is the GLM-5.2/Qwen3.8 measurement (4774s/6676s + # warmup+compile). A raw aiperf run against Kimi-K3 (conc=64, ISL + # ~115k avg) measured warmup alone draining in ~12075s -- past this + # whole cap before profiling starts. Nothing here can tell such a + # model apart, so say so rather than let the round be killed + # mid-warmup by a cap nobody chose. + log.warning( + "agentx_baseline_timeout_sec: neither AGENTX_BASELINE_OVERHEAD_SEC nor " + "AGENTX_WARMUP_GRACE_PERIOD is set, so the overhead falls back to the " + "canonical %ds (= %ds non-warmup + %ds canonical warmup grace). That " + "grace is calibrated on GLM-5.2/Qwen3.8 and may be far too small for a " + "long-context or slow-prefill model -- a raw aiperf run against Kimi-K3 " + "at concurrency=64 measured warmup alone taking ~12075s. Raise " + "AGENTX_WARMUP_GRACE_PERIOD (the client honours it too, so the warmup " + "and this cap stay consistent) or pin AGENTX_BASELINE_OVERHEAD_SEC.", + overhead, + _AGENTX_NON_WARMUP_OVERHEAD_SEC, + AGENTX_CANON_WARMUP_GRACE_SEC, + ) + duration = _int("AGENTX_DURATION", AGENTX_DEFAULT_DURATION_SEC) + total = duration + overhead + # Log every input, so a timeout in the field can be read back to the value + # that produced it instead of guessing which knob was in play. + log.info( + "agentx_baseline_timeout_sec: %ds = duration %ds + overhead %ds (%s)", + total, + duration, + overhead, + "explicit AGENTX_BASELINE_OVERHEAD_SEC" + if grace is None + else f"{_AGENTX_NON_WARMUP_OVERHEAD_SEC}s non-warmup + {grace}s warmup grace", + ) + return total # Cold-start settings and probes live in ``_aiter_jit`` and are re-exported diff --git a/src/hyperloom/orchestrator/actions/executors/integrate_patch.py b/src/hyperloom/orchestrator/actions/executors/integrate_patch.py index 16c0e6bf45..3f22c00529 100644 --- a/src/hyperloom/orchestrator/actions/executors/integrate_patch.py +++ b/src/hyperloom/orchestrator/actions/executors/integrate_patch.py @@ -84,6 +84,10 @@ DEFAULT_KEEP_THRESHOLD_PCT = 1.0 # grid noise floor; KEEP is re-confirmed by a stack rebench +# Synthetic-sized, mirroring _grid_base._VARIANT_TIMEOUT_SEC_DEFAULT. AgentX +# rounds are raised past it by ``agentx_variant_timeout_sec`` where the cap is +# consumed, so this stays the synthetic default rather than growing to cover a +# workload it was never measured against. DEFAULT_VARIANT_TIMEOUT_SEC = 7800 _HYPERLOOM_AUTO_STASH_MSG = "hyperloom-auto-stash: preserving user changes before candidate run" # Deliberately shares no substring with the auto-stash tag: _find_hyperloom_auto_stash diff --git a/src/hyperloom/orchestrator/kernel/conc_sweep.py b/src/hyperloom/orchestrator/kernel/conc_sweep.py index 457fdd9433..fde7f71292 100644 --- a/src/hyperloom/orchestrator/kernel/conc_sweep.py +++ b/src/hyperloom/orchestrator/kernel/conc_sweep.py @@ -55,6 +55,9 @@ DEFAULT_NUM_PROMPTS_FACTOR = 5 # Per-variant timeout (seconds); override via ``--conc-sweep-timeout-sec``. +# Synthetic-sized, like every other variant-timeout default here; under AgentX +# ``agentx_variant_timeout_sec`` raises it at the point of use, so this number +# is a floor for the synthetic sweep rather than a bound on an agentic round. DEFAULT_VARIANT_TIMEOUT_SEC = 1800 # Total wall-clock budget (seconds); override via ``--conc-sweep-total-budget-sec``. From 3f730cd25409133f3bb9285c8f17d90b29d3ac8e Mon Sep 17 00:00:00 2001 From: Zeng Date: Thu, 27 Aug 2026 19:59:46 +0800 Subject: [PATCH 09/12] fix(tests): drop an unused yaml import (ruff F401) The AgentX-off counterpart of the MAX_ITERS override test asserts only on log records, so it never renders the materialized config and does not need yaml. Co-Authored-By: Claude Opus 5 (1M context) --- .../tests/test_profile_and_kernel_handlers.py | 2 -- 1 file changed, 2 deletions(-) diff --git a/src/hyperloom/inference_optimizer/tests/test_profile_and_kernel_handlers.py b/src/hyperloom/inference_optimizer/tests/test_profile_and_kernel_handlers.py index e8d79c04ba..23947d9ad8 100644 --- a/src/hyperloom/inference_optimizer/tests/test_profile_and_kernel_handlers.py +++ b/src/hyperloom/inference_optimizer/tests/test_profile_and_kernel_handlers.py @@ -926,8 +926,6 @@ def test_materialize_profile_max_iters_override_is_quiet_without_agentx( caplog, ): """The new warning is AgentX-only; the synthetic path has no host-RAM bound.""" - import yaml - _clear_workload_env(monkeypatch) monkeypatch.setenv("HYPERLOOM_PROFILE_MAX_ITERS", "128") src = _profile_yaml(tmp_path, "vllm", {"CONC": 32, "ISL": 256, "OSL": 1024}) From 67d5ae0bec5051fc8c410e1a2d45892fb4ba81ca Mon Sep 17 00:00:00 2001 From: Zeng Date: Thu, 27 Aug 2026 20:06:58 +0800 Subject: [PATCH 10/12] style: apply ruff format to this branch's test files CI runs `ruff format --check .` (lint.yml), which this branch has been failing since 0c155946e -- the warmup non-canon assertions added there, and the failed-request-threshold ones added on top of them, were hand-wrapped rather than formatted. Formatting only, no behaviour change; the three files pass on a clean checkout (69 passed). Co-Authored-By: Claude Opus 5 (1M context) --- .../tests/test_agentx_baseline_timeout.py | 12 +++--------- .../tests/test_aiperf_client_sh.py | 14 +++++--------- .../tests/test_profile_and_kernel_handlers.py | 3 +-- 3 files changed, 9 insertions(+), 20 deletions(-) diff --git a/src/hyperloom/inference_optimizer/tests/test_agentx_baseline_timeout.py b/src/hyperloom/inference_optimizer/tests/test_agentx_baseline_timeout.py index 5e9da193dd..0c49abd596 100644 --- a/src/hyperloom/inference_optimizer/tests/test_agentx_baseline_timeout.py +++ b/src/hyperloom/inference_optimizer/tests/test_agentx_baseline_timeout.py @@ -117,18 +117,14 @@ def test_overhead_tracks_the_warmup_grace_the_operator_set(monkeypatch): _clear(monkeypatch) monkeypatch.setenv("AGENTX_WARMUP_GRACE_PERIOD", "14400") grown = 14400 - AGENTX_CANON_WARMUP_GRACE_SEC - assert agentx_baseline_timeout_sec() == ( - AGENTX_DEFAULT_DURATION_SEC + AGENTX_BASELINE_OVERHEAD_SEC + grown - ) + assert agentx_baseline_timeout_sec() == (AGENTX_DEFAULT_DURATION_SEC + AGENTX_BASELINE_OVERHEAD_SEC + grown) def test_canonical_grace_reproduces_the_measured_constant(monkeypatch): """Splitting the constant must not move it: same inputs, same number.""" _clear(monkeypatch) monkeypatch.setenv("AGENTX_WARMUP_GRACE_PERIOD", str(AGENTX_CANON_WARMUP_GRACE_SEC)) - assert agentx_baseline_timeout_sec() == ( - AGENTX_DEFAULT_DURATION_SEC + AGENTX_BASELINE_OVERHEAD_SEC - ) + assert agentx_baseline_timeout_sec() == (AGENTX_DEFAULT_DURATION_SEC + AGENTX_BASELINE_OVERHEAD_SEC) def test_explicit_overhead_outranks_the_derivation(monkeypatch): @@ -153,9 +149,7 @@ def test_unparseable_grace_falls_back_to_canonical(monkeypatch, bad): """A typo in the grace must not shrink the cap below the measured default.""" _clear(monkeypatch) monkeypatch.setenv("AGENTX_WARMUP_GRACE_PERIOD", bad) - assert agentx_baseline_timeout_sec() == ( - AGENTX_DEFAULT_DURATION_SEC + AGENTX_BASELINE_OVERHEAD_SEC - ) + assert agentx_baseline_timeout_sec() == (AGENTX_DEFAULT_DURATION_SEC + AGENTX_BASELINE_OVERHEAD_SEC) def test_explicit_cap_wins_outright(monkeypatch): diff --git a/src/hyperloom/inference_optimizer/tests/test_aiperf_client_sh.py b/src/hyperloom/inference_optimizer/tests/test_aiperf_client_sh.py index 4bab9027ed..f23fe86938 100644 --- a/src/hyperloom/inference_optimizer/tests/test_aiperf_client_sh.py +++ b/src/hyperloom/inference_optimizer/tests/test_aiperf_client_sh.py @@ -650,9 +650,7 @@ def test_reduced_warmup_is_flagged_non_canonical(tmp_path): assert r.returncode == 0, r.stderr out = _result(res) assert out["submission_valid"] is False - assert any( - "warmup_per_lane=1" in x for x in out["submission_invalid_reasons"] - ), out["submission_invalid_reasons"] + assert any("warmup_per_lane=1" in x for x in out["submission_invalid_reasons"]), out["submission_invalid_reasons"] def test_reduced_warmup_grace_is_flagged_non_canonical(tmp_path): @@ -662,9 +660,7 @@ def test_reduced_warmup_grace_is_flagged_non_canonical(tmp_path): assert r.returncode == 0, r.stderr out = _result(res) assert out["submission_valid"] is False - assert any( - "warmup_grace=60s" in x for x in out["submission_invalid_reasons"] - ), out["submission_invalid_reasons"] + assert any("warmup_grace=60s" in x for x in out["submission_invalid_reasons"]), out["submission_invalid_reasons"] def test_canonical_warmup_is_not_flagged(tmp_path): @@ -724,9 +720,9 @@ def test_raised_failed_request_threshold_is_flagged_non_canonical(tmp_path): assert r.returncode == 0, r.stderr out = _result(res) assert out["submission_valid"] is False - assert any( - "failed_request_threshold=0.5" in x for x in out["submission_invalid_reasons"] - ), out["submission_invalid_reasons"] + assert any("failed_request_threshold=0.5" in x for x in out["submission_invalid_reasons"]), out[ + "submission_invalid_reasons" + ] def test_tightened_failed_request_threshold_is_not_flagged(tmp_path): diff --git a/src/hyperloom/inference_optimizer/tests/test_profile_and_kernel_handlers.py b/src/hyperloom/inference_optimizer/tests/test_profile_and_kernel_handlers.py index 23947d9ad8..68ad3117d6 100644 --- a/src/hyperloom/inference_optimizer/tests/test_profile_and_kernel_handlers.py +++ b/src/hyperloom/inference_optimizer/tests/test_profile_and_kernel_handlers.py @@ -915,8 +915,7 @@ def test_materialize_profile_max_iters_override_warns_it_undoes_the_agentx_bound # The override is still honored verbatim; this is a visibility fix only. assert "--profiler-config.max_iterations 128" in extra, extra assert any( - "HYPERLOOM_PROFILE_MAX_ITERS=128 overrides the AgentX capture bound of 8" in r.message - for r in caplog.records + "HYPERLOOM_PROFILE_MAX_ITERS=128 overrides the AgentX capture bound of 8" in r.message for r in caplog.records ), [r.message for r in caplog.records] From bf8da7b25e787aaaee93081185380aeea74edaac Mon Sep 17 00:00:00 2001 From: Zeng Date: Thu, 27 Aug 2026 22:09:31 +0800 Subject: [PATCH 11/12] fix(agentx): re-state AIPERF_HTTP_TCP_USER_TIMEOUT after the AIPERF_* scrub TCP_USER_TIMEOUT bounds how long Linux tolerates an established connection making no progress, and an agentic turn against a long-context model makes none for as long as the server is prefill-bound. aiperf's stock 30s therefore aborts otherwise-live connections mid-prefill, which surfaces as a warmup failure with no server-side error to match it. Upstream's Kimi-K3 arms (benchmarks/single_node/agentic/kimik3_fp4_mi355x_mtp.sh and the atom/b300 variants) and the DSv4 SGLang arms all export 900000 for exactly this reason -- the DSv4 comment names the 30s default it is overriding. This file scrubs every inherited AIPERF_* except AIPERF_BIN, so an operator who exported it upstream had no effect and the client kept the stock bound. Exported after the scrub like every other AIPERF_ setting here, tunable through AGENTX_HTTP_TCP_USER_TIMEOUT. Co-Authored-By: Claude Opus 5 (1M context) --- CHANGELOG.md | 11 +++++++ .../assets/agentx/aiperf_client.sh | 12 +++++++ .../tests/test_aiperf_client_sh.py | 33 +++++++++++++++++++ 3 files changed, 56 insertions(+) diff --git a/CHANGELOG.md b/CHANGELOG.md index 5a1677d12f..b2de64e0d5 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -37,6 +37,17 @@ The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.1.0/), floor nor above the cap and restored the full exposure without printing anything. The override is still honoured verbatim; it now warns. +- **`AIPERF_HTTP_TCP_USER_TIMEOUT` is re-stated after the `AIPERF_*` scrub.** + `TCP_USER_TIMEOUT` bounds how long Linux tolerates an established connection + making no progress, and an agentic turn against a long-context model makes + none for as long as the server is prefill-bound. aiperf's stock 30s therefore + aborts otherwise-live connections mid-prefill, surfacing as a warmup failure + with no server-side error to match it. Upstream's Kimi-K3 and DSv4 recipes all + export `900000` (15 min); Hyperloom scrubs every inherited `AIPERF_*` except + `AIPERF_BIN`, so an operator setting it had no effect and the client ran on + the stock bound. Now exported after the scrub, tunable via + `AGENTX_HTTP_TCP_USER_TIMEOUT`. + - **A loosened `AGENTX_FAILED_REQUEST_THRESHOLD` is flagged as a non-canonical workload.** Raising the abort ratio keeps alive a run that upstream's 0.10 would have aborted, and the surviving requests are then mapped as an ordinary diff --git a/src/hyperloom/inference_optimizer/assets/agentx/aiperf_client.sh b/src/hyperloom/inference_optimizer/assets/agentx/aiperf_client.sh index 9d94fc1e7d..81d5d548cf 100755 --- a/src/hyperloom/inference_optimizer/assets/agentx/aiperf_client.sh +++ b/src/hyperloom/inference_optimizer/assets/agentx/aiperf_client.sh @@ -29,6 +29,9 @@ # non-submittable -- see the smoke note below), # AGENTX_REALTIME_METRICS (rolling stats block; default true), # AGENTX_DATASET_CONFIG_TIMEOUT (default 1800), AGENTX_LIVE_ASSISTANT, +# AGENTX_HTTP_TCP_USER_TIMEOUT (no-TCP-progress bound in ms; default 900000, +# matching upstream's long-context recipes -- aiperf's stock 30s aborts +# live connections while the server is prefill-bound), # AGENTX_MMAP_CACHE_DIR (dataset mmap cache; defaults under $HF_HUB_CACHE), # AGENTX_MAX_CTX (explicit opt-in client-side context cap; NEVER inferred # from $MAX_MODEL_LEN -- see the replay-context note below), @@ -214,6 +217,15 @@ done < <(env) # aiperf validates SERVICE_PROFILE_CONFIGURE_TIMEOUT >= DATASET_CONFIGURATION_TIMEOUT. export AIPERF_DATASET_CONFIGURATION_TIMEOUT="${AGENTX_DATASET_CONFIG_TIMEOUT:-1800}" export AIPERF_SERVICE_PROFILE_CONFIGURE_TIMEOUT="${AGENTX_DATASET_CONFIG_TIMEOUT:-1800}" +# TCP_USER_TIMEOUT bounds how long Linux tolerates an established connection +# making no progress -- and an agentic turn against a long-context model makes +# no TCP progress for as long as the server is prefill-bound. aiperf's stock +# 30s therefore aborts otherwise-live connections mid-prefill, which surfaces +# as a warmup failure with no server-side error to match it. Upstream's +# Kimi-K3 and DSv4 recipes all export 900000 (15 min) for exactly this, and the +# scrub above would drop an inherited copy, so it has to be re-stated here or +# the request timeout is left to a bound two orders of magnitude too small. +export AIPERF_HTTP_TCP_USER_TIMEOUT="${AGENTX_HTTP_TCP_USER_TIMEOUT:-900000}" # Pre-canned assistant replay (recorded responses drive later turns). export AIPERF_DATASET_WEKA_LIVE_ASSISTANT_RESPONSES="${AGENTX_LIVE_ASSISTANT:-0}" # Headless realtime metrics are opt-in on current aiperf, and the scrub above diff --git a/src/hyperloom/inference_optimizer/tests/test_aiperf_client_sh.py b/src/hyperloom/inference_optimizer/tests/test_aiperf_client_sh.py index f23fe86938..f61eea7693 100644 --- a/src/hyperloom/inference_optimizer/tests/test_aiperf_client_sh.py +++ b/src/hyperloom/inference_optimizer/tests/test_aiperf_client_sh.py @@ -56,6 +56,7 @@ def _fake_builtin(write_pid: bool) -> str: echo "AIPERF_DATASET_CONFIGURATION_TIMEOUT=${AIPERF_DATASET_CONFIGURATION_TIMEOUT:-UNSET}" echo "AIPERF_SERVICE_PROFILE_CONFIGURE_TIMEOUT=${AIPERF_SERVICE_PROFILE_CONFIGURE_TIMEOUT:-UNSET}" echo "AIPERF_DATASET_MMAP_CACHE_DIR=${AIPERF_DATASET_MMAP_CACHE_DIR:-UNSET}" + echo "AIPERF_HTTP_TCP_USER_TIMEOUT=${AIPERF_HTTP_TCP_USER_TIMEOUT:-UNSET}" echo "AIPERF_UI_REALTIME_METRICS_ENABLED=${AIPERF_UI_REALTIME_METRICS_ENABLED:-UNSET}" } > "${AGENTX_TEST_MARKER}" exit "${FAKE_RC:-0}" @@ -160,6 +161,38 @@ def test_scrub_keeps_aiperf_bin_drops_others(tmp_path): assert "AIPERF_FOO=UNSET" in marker # stray AIPERF_* scrubbed +def test_tcp_user_timeout_survives_the_scrub(tmp_path): + """The scrub must not leave aiperf on its 30s stock TCP_USER_TIMEOUT. + + That bound is how long Linux tolerates an established connection making no + progress, and an agentic turn against a long-context model makes none for + as long as the server is prefill-bound. Upstream's Kimi-K3 and DSv4 recipes + export 900000; because the scrub above drops any inherited copy, this file + has to re-state it or the connection dies mid-prefill and the round fails + as a warmup error with no matching server-side fault. + """ + bench, bind, res = _sandbox(tmp_path) + r = _run(bench, bind, res, tmp_path) + assert r.returncode == 0, r.stderr + assert "AIPERF_HTTP_TCP_USER_TIMEOUT=900000" in (tmp_path / "marker.txt").read_text() + + +def test_tcp_user_timeout_is_tunable_through_the_agentx_name(tmp_path): + """Operators tune it through AGENTX_, like every other knob in this file.""" + bench, bind, res = _sandbox(tmp_path) + r = _run(bench, bind, res, tmp_path, AGENTX_HTTP_TCP_USER_TIMEOUT="1200000") + assert r.returncode == 0, r.stderr + assert "AIPERF_HTTP_TCP_USER_TIMEOUT=1200000" in (tmp_path / "marker.txt").read_text() + + +def test_inherited_tcp_user_timeout_does_not_win(tmp_path): + """An inherited AIPERF_ copy is scrubbed; ours is authoritative.""" + bench, bind, res = _sandbox(tmp_path) + r = _run(bench, bind, res, tmp_path, AIPERF_HTTP_TCP_USER_TIMEOUT="30000") + assert r.returncode == 0, r.stderr + assert "AIPERF_HTTP_TCP_USER_TIMEOUT=900000" in (tmp_path / "marker.txt").read_text() + + def test_gpu_type_uppercase_resolves_builtin(tmp_path): bench, bind, res = _sandbox(tmp_path) r = _run(bench, bind, res, tmp_path, GPU_TYPE="MI300X") From 7ca67ec0c30e2632a7ca841e0639b198317337fe Mon Sep 17 00:00:00 2001 From: Zeng Date: Sun, 30 Aug 2026 18:21:52 +0800 Subject: [PATCH 12/12] fix(agentx): raise the inner Magpie timeout to the AgentX baseline cap The Magpie benchmark config's flat timeout_seconds (7200s default from baseline_vllm.yaml) is one wall-clock deadline over server boot + warmup + the measurement window + result export. At the model's native context an AgentX baseline's boot+warmup alone overruns it, so Magpie SIGKILLs the benchmark before aiperf writes inferencex_result.json -- a 0-tput baseline that fails the session, while the raised outer subprocess cap goes unused (the two layers were left inconsistent). Set bench["timeout_seconds"] in apply_agentx_switch to the same agentx_baseline_timeout_sec() the outer cap already uses, so the inner and outer caps stay consistent. AgentX-only: the switch returns early when AgentX is off, so the default (synthetic) cap is untouched. Measured on Kimi-K3 (vLLM, TP8, conc8, 1M ctx): boot+warmup ~46 min + a 3600s window overran the 7200s cap and the baseline landed at 0 tok/s despite a clean profiling run. Co-Authored-By: Claude Opus 5 (1M context) --- .../tests/test_agentx_budget_and_guards.py | 55 +++++++++++++++++++ .../actions/executors/_workload_envs.py | 16 ++++++ 2 files changed, 71 insertions(+) diff --git a/src/hyperloom/inference_optimizer/tests/test_agentx_budget_and_guards.py b/src/hyperloom/inference_optimizer/tests/test_agentx_budget_and_guards.py index 7600d50161..98a3606f2a 100644 --- a/src/hyperloom/inference_optimizer/tests/test_agentx_budget_and_guards.py +++ b/src/hyperloom/inference_optimizer/tests/test_agentx_budget_and_guards.py @@ -303,3 +303,58 @@ def test_verdict_gate_spares_scriptable_runs_under_agentx(monkeypatch): """ _on(monkeypatch) assert _valid(_measurement()) is True + + +# --- inner Magpie timeout follows the AgentX cap ------------------------------ + + +def test_agentx_switch_raises_the_inner_magpie_timeout(monkeypatch): + """The flat Magpie ``timeout_seconds`` must follow the raised AgentX cap. + + The flat cap covers server boot + warmup + the measurement window + export + as one deadline. At the model's native context AgentX's boot+warmup alone + overruns the synthetic 7200s default, so the benchmark is SIGKILLed before + aiperf writes its result -- a 0-tput baseline that fails the session. The + switch lifts the inner cap to the same budget the outer subprocess timeout + uses, so the two layers stay consistent. + """ + _on(monkeypatch) + monkeypatch.setenv("AGENTX_BASELINE_TIMEOUT_SEC", "25200") + from hyperloom.orchestrator.actions.executors._workload_envs import ( + apply_agentx_switch, + ) + from hyperloom.orchestrator.actions.executors.baseline import ( + agentx_baseline_timeout_sec, + ) + + bench = {"framework": "vllm", "model": "/models/x", "timeout_seconds": 7200} + apply_agentx_switch(bench) + assert bench["timeout_seconds"] == agentx_baseline_timeout_sec() + assert bench["timeout_seconds"] > 7200 + assert bench["benchmark_script"] == "aiperf_client.sh" + + +def test_agentx_switch_leaves_the_inner_timeout_alone_without_agentx(monkeypatch): + """The default (synthetic) cap must be untouched when AgentX is off.""" + _off(monkeypatch) + from hyperloom.orchestrator.actions.executors._workload_envs import ( + apply_agentx_switch, + ) + + bench = {"framework": "vllm", "model": "/models/x", "timeout_seconds": 7200} + apply_agentx_switch(bench) + assert bench["timeout_seconds"] == 7200 + assert "benchmark_script" not in bench + + +def test_agentx_switch_skips_scriptable_inner_timeout(monkeypatch): + """A scriptable framework returns early, so its cap is never rewritten.""" + _on(monkeypatch) + monkeypatch.setenv("AGENTX_BASELINE_TIMEOUT_SEC", "25200") + from hyperloom.orchestrator.actions.executors._workload_envs import ( + apply_agentx_switch, + ) + + bench = {"framework": "xdit", "model": "/models/x", "timeout_seconds": 7200} + apply_agentx_switch(bench) + assert bench["timeout_seconds"] == 7200 diff --git a/src/hyperloom/orchestrator/actions/executors/_workload_envs.py b/src/hyperloom/orchestrator/actions/executors/_workload_envs.py index 67d09281f9..069d1e6e61 100644 --- a/src/hyperloom/orchestrator/actions/executors/_workload_envs.py +++ b/src/hyperloom/orchestrator/actions/executors/_workload_envs.py @@ -192,6 +192,22 @@ def apply_agentx_switch(bench: dict[str, Any], model_path: str | None = None) -> return envs = bench.setdefault("envs", {}) bench["benchmark_script"] = "aiperf_client.sh" + # The Magpie benchmark config's flat wall-clock cap (``benchmark.timeout_seconds``, + # e.g. 7200s from baseline_vllm.yaml) is one deadline over server boot + warmup + + # the measurement window + result export. AgentX runs at the model's native + # context (``max_model_len`` lifted from the synthetic 6144 to e.g. 1M), so boot + + # warmup alone can consume ~45 min before the window even opens; the flat cap then + # SIGKILLs the benchmark before aiperf writes ``inferencex_result.json`` -- a 0-tput + # baseline that fails the session. Raise the inner cap to the same AgentX budget the + # outer subprocess timeout already uses (``agentx_baseline_timeout_sec``) so the two + # layers stay consistent. AgentX-only: this function returned early above when AgentX + # is off, so the default (synthetic) cap is untouched. The import is function-local + # to avoid a circular dependency (``baseline`` imports this module at load time). + from hyperloom.orchestrator.actions.executors.baseline import ( + agentx_baseline_timeout_sec, + ) + + bench["timeout_seconds"] = agentx_baseline_timeout_sec() envs["RUN_EVAL"] = "false" envs["MODEL"] = str(model_path or bench.get("model") or os.environ.get("MODEL_PATH", "")).strip() envs["FRAMEWORK"] = framework