From 156e3e2c4592c613f19d1cd8cd2be5b9cd3d21e6 Mon Sep 17 00:00:00 2001 From: YaoCheng Date: Tue, 25 Aug 2026 06:43:37 +0000 Subject: [PATCH 1/4] add glm5.2 profile script --- deploy/docker/Dockerfile.sglang | 6 +- deploy/docker/patch.upstream.status.md | 3 +- deploy/docker/patches/sglang_dsa/README.md | 43 +++- .../sglang_dsa/dsa_indexer_idle_metadata.diff | 54 +++++ .../scripts/apply_sglang_dsa_patches.sh | 12 +- examples/sglang_1p1d_glm5.2/README.md | 99 +++++++++ .../cluster/cluster.peermem.sh | 9 + examples/sglang_1p1d_glm5.2/common.sh | 34 +++- examples/sglang_1p1d_glm5.2/engine/capture.sh | 192 ++++++++++++++++++ examples/sglang_1p1d_glm5.2/engine/leg.sh | 6 +- examples/sglang_1p1d_glm5.2/engine/up.sh | 38 ++++ 11 files changed, 482 insertions(+), 14 deletions(-) create mode 100644 deploy/docker/patches/sglang_dsa/dsa_indexer_idle_metadata.diff create mode 100755 examples/sglang_1p1d_glm5.2/engine/capture.sh diff --git a/deploy/docker/Dockerfile.sglang b/deploy/docker/Dockerfile.sglang index 0fce3544f..86253a900 100644 --- a/deploy/docker/Dockerfile.sglang +++ b/deploy/docker/Dockerfile.sglang @@ -82,12 +82,12 @@ COPY infera ./infera RUN pip install --no-cache-dir ".[sglang]" "setuptools>=83.0.0" # ---- GLM-5.2 DSA patch set (PD + DP-attention on gfx950) -------------------- -# WHAT: four patches that together make PD + DP-attention work for GLM-5.2, with +# WHAT: five patches that together make PD + DP-attention work for GLM-5.2, with # EAGLE MTP on top; without them it crashes on the first batch or deadlocks the DP # group. patches/sglang_dsa/README.md carries the per-patch rationale. # -# Three of the four are context diffs at --fuzz=0 against the pinned base, so a -# base bump FAILS here instead of mis-applying. The fourth is an anchor script +# Four of the five are context diffs at --fuzz=0 against the pinned base, so a +# base bump FAILS here instead of mis-applying. The fifth is an anchor script # instead: Dockerfile.sglang.gfx942 needs the same fix on a v0.5.16 base where # those diffs cannot apply, and one source of truth beats two. # diff --git a/deploy/docker/patch.upstream.status.md b/deploy/docker/patch.upstream.status.md index 4ae904116..0a163219c 100644 --- a/deploy/docker/patch.upstream.status.md +++ b/deploy/docker/patch.upstream.status.md @@ -22,7 +22,7 @@ Column meanings: ## sglang — `patches/sglang_dsa/` (baked by `Dockerfile.sglang` and `Dockerfile.sglang.gfx942`, `APPLY_SGLANG_DSA_PATCHES=1`) -Patch 01 is an anchor script and is baked by both images. The other three are +Patch 01 is an anchor script and is baked by both images. The other four are `--fuzz=0` diffs cut against the mi35x base (v0.5.17) and are not applied by the gfx942 image, which substitutes `dsa_page_table_rows` and `draft_cuda_graph_dp_vote` at runtime with `--json-model-override-args '{"index_share_for_mtp_iteration":false}'` @@ -36,6 +36,7 @@ at runtime with `--json-model-override-args '{"index_share_for_mtp_iteration":fa | `sglang_dsa/dsa_dp_sync.diff` | `seq_lens.max().item()` is a host sync on a branch only *some* DP ranks take → DP collectives desync → deadlock | none found | [sglang#33973](https://github.com/sgl-project/sglang/pull/33973) — this file **is** that PR's diff, so it drops by deletion when it merges | **yes** (`dorado269`) | OPEN | | `sglang_dsa/dsa_page_table_rows.diff` | page table has one row per **request**, top-k one per **token** under MTP → `assert page_table.shape[0] == topk_indices.shape[0]` | none found | [sglang#32209](https://github.com/sgl-project/sglang/pull/32209) solves the same row mismatch by **trimming q/top-k**; porting that half here fails at conc=32 and is unresolved | no (`HZY-Wade`) | OPEN | | `sglang_dsa/draft_cuda_graph_dp_vote.diff` | the draft graph/eager choice is per-rank, so under PD + DP-attention + MTP a DP group splits across the two paths and deadlocks on the first routed request | [sglang#32527](https://github.com/sgl-project/sglang/issues/32527) | [sglang#32209](https://github.com/sgl-project/sglang/pull/32209) carries the same vote at the same site; we take only that half | no (`HZY-Wade`) | OPEN | +| `sglang_dsa/dsa_indexer_idle_metadata.diff` | EAGLE eager IDLE selects a per-step DSA backend without `forward_metadata`; the indexer reads it before the existing empty-batch guard → scheduler exits during warmup | none found for this exact traceback | [sglang#32209](https://github.com/sgl-project/sglang/pull/32209) carries an early CUDA IDLE short-circuit; [#31683](https://github.com/sgl-project/sglang/pull/31683) widens a later guard but is too late for the v0.5.17 ordering | no | OPEN at last recorded checks; re-check before release | Background, already present in the base and **not** patched by us: [sglang#30378](https://github.com/sgl-project/sglang/pull/30378) / diff --git a/deploy/docker/patches/sglang_dsa/README.md b/deploy/docker/patches/sglang_dsa/README.md index e7aeaeaff..03f3b7818 100644 --- a/deploy/docker/patches/sglang_dsa/README.md +++ b/deploy/docker/patches/sglang_dsa/README.md @@ -1,7 +1,7 @@ # sglang DSA patches -Patches that make **PD disaggregation + DP-attention** work for GLM-5.2, and, -with the fourth, **EAGLE MTP** on top. Without them the combination crashes on +Patches that make **PD disaggregation + DP-attention** work for GLM-5.2, with +**EAGLE MTP** on top. Without them the combination crashes on the first batch or deadlocks the whole DP group under concurrency. They apply to the sglang tree bundled in the ROCm engine images (an editable @@ -10,7 +10,7 @@ same arm of the set — see [Applying](#applying): | image | base | arm | |---|---|---| -| `Dockerfile.sglang` (gfx950 / MI355X) | `lmsysorg/sglang:v0.5.17-rocm720-mi35x` | 01 + dp_sync + page_table_rows + draft_dp_vote | +| `Dockerfile.sglang` (gfx950 / MI355X) | `lmsysorg/sglang:v0.5.17-rocm720-mi35x` | 01 + idle_metadata + dp_sync + page_table_rows + draft_dp_vote | | `Dockerfile.sglang.gfx942` (gfx942 / MI325X) | `lmsysorg/sglang:v0.5.16-rocm720-mi30x` | **01 only**, plus a mandatory runtime flag | | # | patch | fixes | @@ -19,6 +19,7 @@ same arm of the set — see [Applying](#applying): | 02a | `dsa_dp_sync.diff` | a host sync on a branch only *some* DP ranks take → collectives desync → deadlock. This file is upstream PR sglang#33973 verbatim | | 02b | `dsa_page_table_rows.diff` | page table has one row per **request**, top-k one per **token** under MTP → `assert page_table.shape[0] == topk_indices.shape[0]` | | 04 | `draft_cuda_graph_dp_vote.diff` | the draft graph/eager choice is made **per rank** from rank-dependent inputs and diverges on the PD decode leg → deadlock | +| 05 | `dsa_indexer_idle_metadata.diff` | EAGLE eager IDLE reaches the per-step DSA indexer before that backend has `forward_metadata` → scheduler exits during warmup | Patch 01 is a **script** and the rest are **context diffs**, and that is the whole reason the two images can differ: the diffs are `--fuzz=0` against one @@ -50,7 +51,7 @@ which takes `DSA_PATCH_SET`: | arm | used by | applies | verification | |---|---|---|---| -| `full` (default) | `Dockerfile.sglang` | 01 + 02a + 02b + 04 | 7 bytecode markers | +| `full` (default) | `Dockerfile.sglang` | 01 + 02a + 02b + 04 + 05 | 8 bytecode markers | | `indexer` | `Dockerfile.sglang.gfx942` | 01 | the two patch-01 markers | Set `APPLY_SGLANG_DSA_PATCHES=0` for a stock engine to A/B against. @@ -65,7 +66,8 @@ By hand, against the pinned base: ```bash cd /sgl-workspace/sglang python3 patch_dsa_indexer_hip_dp_padded_rows.py -for d in dsa_dp_sync.diff dsa_page_table_rows.diff draft_cuda_graph_dp_vote.diff; do +for d in dsa_indexer_idle_metadata.diff dsa_dp_sync.diff \ + dsa_page_table_rows.diff draft_cuda_graph_dp_vote.diff; do patch -p1 --fuzz=0 < "$d" done ``` @@ -219,6 +221,37 @@ extracts the paged-MQA backend, restructuring the `is_aiter()` dispatch this patch hangs off. Either landing in a future base drifts the anchors, which fails the build rather than mis-applying — but re-cut patch 01 when bumping past them. +### Patch 05: EAGLE eager IDLE has no per-step DSA metadata + +On the EAGLE eager path, an IDLE DP rank skips +`draft_attn_backend.init_forward_metadata()`, then `draft_forward()` selects a +per-step backend through `ForwardContext`. The generic eager IDLE runner updates +`model_runner.attn_backend`; that is not the per-step backend the DSA indexer +reads. If IDLE is that backend's first use, even the attribute is absent: + +```text +AttributeError: 'DeepseekSparseAttnBackend' object has no attribute + 'forward_metadata' +``` + +The existing empty-`seq_lens` guard returns an all-invalid top-k tensor, but on +v0.5.17 it sits hundreds of lines after `get_indexer_metadata()` and therefore +cannot protect this access. Patch 05 adds the equivalent IDLE return immediately +after `x_meta` is available and before metadata lookup. It retains +`_broadcast_indexer_topk_from_rank0`: IDLE ranks still need the same DP +synchronization primitive as active ranks. + +This is the CUDA IDLE short-circuit strategy carried in upstream +[#32209](https://github.com/sgl-project/sglang/pull/32209), adapted to the +v0.5.17 ordering. [#31683](https://github.com/sgl-project/sglang/pull/31683) +widens the later empty-batch guard, which is insufficient by itself on this +base because metadata has already been read. + +**Validation status.** The failure was reproduced on MI355X TP8/DP8 with PD, +DP-attention, EAGLE(3,1,4), and decode CUDA graphs disabled. The patch has +build-time apply/bytecode coverage; repeated two-node cold-start and concurrency +validation is still pending. + ### Prerequisite There is no longer one. GLM-5.2 MTP used to need a backport of sglang #30265 to diff --git a/deploy/docker/patches/sglang_dsa/dsa_indexer_idle_metadata.diff b/deploy/docker/patches/sglang_dsa/dsa_indexer_idle_metadata.diff new file mode 100644 index 000000000..a4eb1b5b8 --- /dev/null +++ b/deploy/docker/patches/sglang_dsa/dsa_indexer_idle_metadata.diff @@ -0,0 +1,54 @@ +PATCH 05 -- DSA indexer: short-circuit EAGLE IDLE before metadata lookup +================================================================================ +WHAT An EAGLE eager IDLE forward selects a per-step DSA backend through + ForwardContext, but EagleDraftWorker skips draft_attn_backend metadata + initialization for IDLE. DSAIndexer.forward_cuda() asks that backend for + indexer metadata before reaching its existing empty-seq-lens guard, so a + backend whose first use is IDLE raises: + + AttributeError: 'DeepseekSparseAttnBackend' object has no attribute + 'forward_metadata' + + Return the same all-invalid top-k tensor as the existing empty-seq-lens + branch before any metadata access. Keep the rank-0 broadcast: IDLE ranks + still have to issue the same DP synchronization primitive as active ranks. + +SCOPE v0.5.17 only. This is a --fuzz=0 context diff for the pinned MI355X base. + The gfx942 v0.5.16 arm is unchanged and has not reproduced this failure. + +UPSTREAM + #31683 widens a later empty-seq-lens guard to cover IDLE. + #32209 carries an early CUDA IDLE short-circuit alongside its EAGLE/DP + fixes. This cut is placed before get_indexer_metadata(), which is required + on v0.5.17 because that lookup precedes the existing guard. + +EVIDENCE + Reproduced on 2026-08-24 with GLM-5.2-MXFP4, MI355X, TP8/DP8, + DP-attention, EAGLE(3,1,4), PD decode, and decode CUDA graphs disabled. + Multiple scheduler ranks failed during disaggregation warmup at + dsa_indexer.py:get_indexer_metadata before the server reported ready. + +diff --git a/python/sglang/srt/layers/attention/dsa/dsa_indexer.py b/python/sglang/srt/layers/attention/dsa/dsa_indexer.py +--- a/python/sglang/srt/layers/attention/dsa/dsa_indexer.py ++++ b/python/sglang/srt/layers/attention/dsa/dsa_indexer.py +@@ -1558,6 +1558,20 @@ + # a tuple like (x_fp8, x_scale[, y]). Use `x_meta` for shape/device queries. + x_meta = x[0] if isinstance(x, tuple) else x + ++ # EAGLE IDLE forwards skip per-step draft-backend metadata initialization. ++ # Return the same sentinel top-k as the empty-seq-lens branch below before ++ # get_indexer_metadata dereferences a forward_metadata that does not exist. ++ _glm52_idle_indexer = forward_batch.forward_mode.is_idle() ++ if _glm52_idle_indexer: ++ topk_result = torch.full( ++ (x_meta.shape[0], self.index_topk), ++ -1, ++ dtype=torch.int, ++ device=x_meta.device, ++ ) ++ topk_result = _broadcast_indexer_topk_from_rank0(topk_result) ++ return maybe_capture_indexer_topk(layer_id, topk_result) ++ + in_piecewise_or_breakable_cuda_graph = ( + _is_in_piecewise_or_breakable_cuda_graph() + ) diff --git a/deploy/docker/scripts/apply_sglang_dsa_patches.sh b/deploy/docker/scripts/apply_sglang_dsa_patches.sh index 75fae8334..aa94b7af9 100755 --- a/deploy/docker/scripts/apply_sglang_dsa_patches.sh +++ b/deploy/docker/scripts/apply_sglang_dsa_patches.sh @@ -8,8 +8,9 @@ # TWO ARMS, because the set is not uniformly portable across our engine bases: # # DSA_PATCH_SET=full (default; Dockerfile.sglang, mi35x / v0.5.17) -# patch 01 + dsa_dp_sync + dsa_page_table_rows + draft_cuda_graph_dp_vote. -# The three diffs are `--fuzz=0` against that one release, so this arm +# patch 01 + dsa_indexer_idle_metadata + dsa_dp_sync + +# dsa_page_table_rows + draft_cuda_graph_dp_vote. +# The four diffs are `--fuzz=0` against that one release, so this arm # only works there. # DSA_PATCH_SET=indexer (Dockerfile.sglang.gfx942, mi30x / v0.5.16) # patch 01 only. 02b is substituted at RUNTIME by @@ -32,6 +33,10 @@ # with a convincing stack. Compare the host amdgpu version against the # container's /opt/rocm/.info/version before attributing one to sglang -- see # the driver precondition in patches/sglang_dsa/README.md. +# dsa_dp_sync has no substitute and is not carried there: it has not been +# re-cut or measured on v0.5.16. +# dsa_indexer_idle_metadata is likewise not carried: the failure has not +# been reproduced or evaluated on that base. # # WHY BYTECODE VERIFICATION. Python caches compiled modules in __pycache__ keyed # on the source mtime. A patch script that restores a backup with shutil.copy2 @@ -64,7 +69,7 @@ PATCH01=patch_dsa_indexer_hip_dp_padded_rows.py # from each other and from patch 01. dsa_dp_sync.diff is upstream PR sglang#33973 # verbatim, so it drops by deleting the file the day that merges. case "$DSA_PATCH_SET" in - full) PATCHES=(dsa_dp_sync.diff dsa_page_table_rows.diff draft_cuda_graph_dp_vote.diff) ;; + full) PATCHES=(dsa_indexer_idle_metadata.diff dsa_dp_sync.diff dsa_page_table_rows.diff draft_cuda_graph_dp_vote.diff) ;; indexer) PATCHES=() ;; esac @@ -93,6 +98,7 @@ if [ "$DSA_PATCH_SET" = "full" ]; then # missing makes the patch inert rather than absent, which is the failure mode # that matters: an inert 04 looks exactly like a working one until load. MARKERS+=( + "dsa_indexer.py:_glm52_idle_indexer" "decode.py:force_disable_draft_cuda_graph" "dp_attn.py:can_run_draft_cuda_graph" "eagle_draft_cuda_graph_runner.py:can_run_dp_draft_cuda_graph" diff --git a/examples/sglang_1p1d_glm5.2/README.md b/examples/sglang_1p1d_glm5.2/README.md index ceff243f4..e4680de15 100644 --- a/examples/sglang_1p1d_glm5.2/README.md +++ b/examples/sglang_1p1d_glm5.2/README.md @@ -27,6 +27,7 @@ different fabrics; those numbers are not published with this kit. | `engine/up.sh` | bring up both nodes: containers → etcd + kvd → both legs → router | | `engine/smoke.sh` | service check **plus** positive evidence for each of the five features | | `engine/bench.sh` | reference throughput sweep using SGLang's own `bench_serving` | +| `engine/capture.sh` | take one torch trace per PD role out of a running load (opt-in) | | `engine/down.sh` | tear down and wait for VRAM to actually free | | `preflight_rdma.sh` | RDMA preflight: registration-mode probe + cross-node fabric measurement | @@ -223,6 +224,104 @@ enough requests to reach steady state. **This kit ships no agentic benchmark client**, by design. Point the customer's own harness at the router endpoint. +### Profiling + +Off unless asked for. Two switches, `PROFILE_PREFILL` and `PROFILE_DECODE`, and the **same +values must be given to `up` and to `capture`** — `up` decides whether the control plane +exists at all, `capture` reads them to pick which roles to sample. + +**Decode.** The common case, and the expensive one: + +```bash +PROFILE_DECODE=1 bash cluster/cluster.peermem.sh up +nohup bash cluster/cluster.peermem.sh bench 64 & # load, in the background +PROFILE_DECODE=1 bash cluster/cluster.peermem.sh capture # 20s window out of it +``` + +**Prefill.** Same three commands, one switch changed: + +```bash +PROFILE_PREFILL=1 bash cluster/cluster.peermem.sh up +nohup bash cluster/cluster.peermem.sh bench 64 & +PROFILE_PREFILL=1 bash cluster/cluster.peermem.sh capture +``` + +Cheaper than the decode case in a way worth knowing: the prefill leg runs its **normal** +configuration. There is no graph to turn off, because the extend path does not use one, so +the only distortion is the python router — the leg itself is the same one you benchmarked. +Measured on this stack, a prefill capture came back with 31 steps and 31 of them carrying +GPU operators (coverage 1.00) with nothing disabled. + +What you do have to think about is whether there is enough prefill work in the window. At +steady state most of the machine is decoding; prefill only runs when a new request arrives, +so a long `OSL` starves it. If the step count comes back low, shorten the output +(`OSL=128`) or raise the concurrency, and re-check the histogram. + +**Both at once.** Supported, and the only way to get a P and a D trace from the *same* +window — the two starts are issued together from one shell, so the windows line up: + +```bash +PROFILE_PREFILL=1 PROFILE_DECODE=1 bash cluster/cluster.peermem.sh up +nohup bash cluster/cluster.peermem.sh bench 64 & +PROFILE_PREFILL=1 PROFILE_DECODE=1 bash cluster/cluster.peermem.sh capture +``` + +Note the asymmetry this creates. `PROFILE_DECODE=1` turns the decode graphs off, and +`bench_serving` is closed-loop: a request holds its concurrency slot until its last token, so +a decode leg running ~4× slower completes requests ~4× slower and **prefill is fed ~4× less +often**. What that does and does not damage: + +- **Survives.** The prefill kernels themselves. The two legs are different GPUs on different + nodes, and what an extend step costs is set by its batch shape and the model, not by why it + got scheduled. "Which operators dominate prefill" — the usual reason to profile it — holds. +- **Thins out.** The step count in the window. A 20 s window may come back with single digits. +- **Skews.** The batch-shape mix. Sparser arrivals mean fewer requests queued when the + scheduler forms an extend batch, so it biases toward `bs=1`, and `bs` drives the MoE expert + distribution and the GEMM shapes. Shape-dependent conclusions are taken from a thinner + batch distribution than real steady state. +- **Breaks.** Anything with wall-clock in the denominator — occupancy, inter-step gaps, "how + busy is prefill", throughput — and all queueing and TTFT figures. + +So: **profile prefill on its own if you want its operators** — it needs no graph disabled, so +the extra round is nearly free. Use both roles together for the one thing only it can give +you, a P and D pair from the same window, where the decode distortion is already priced in. + +(The mechanism above is reasoned from the closed-loop setup, not measured; the 35.6 → 137.2 ms +decode figure is.) + +`bench` is deliberately not given a detach flag — `nohup … &` already does it, and the one +thing a flag would buy (surviving an ssh drop mid-run) is already caught by `capture`, which +refuses to start, and warns again after warm-up, if no `bench_serving` is running. Give the +load enough requests to outlast `WARMUP_S + WINDOW_S`: `N=` overrides the `10 × C` +default. + +`capture` waits `WARMUP_S` (default 60) for the load to reach steady state, opens a +`WINDOW_S` (default 20) window, starts both roles from **one** shell inside the container +so their windows line up, stops with a single call, waits for the per-rank files to stop +growing in the shared `TRACE_OUT` mount, and returns the output directory. Trace inspection +is a separate operation; no tar, `docker cp`, or SSH transfer is involved. + +Three things about this are easy to get wrong and are handled for you: + +- **The rust router cannot do it.** `--enable-profiling` makes `launch_rust.py` raise + `SystemExit` rather than degrade, so the router simply never comes up. `start_router` + switches the backend to `python` when it sees the flag. The python router forwards more + slowly, so **throughput from a profiling run is not comparable to a normal one** — say so + in whatever you write up. +- **Decode CUDA graphs hide the kernels you are profiling for.** A graph replays as one + opaque launch: the trace keeps the per-step annotation and loses everything inside it. + Measured here: 134 decode steps in a window, **1** of them with GPU operators, and the + resulting kernel ranking was not merely imprecise but *reordered*. So `PROFILE_DECODE=1` + also sets `CUDA_GRAPH=0` on the decode leg — which costs real throughput (decode TPOT + measured 35.6 → 137.2 ms), which is why it is not a default. Prefill needs no equivalent; + its extend path does not run through a graph. `--enable-profile-cuda-graph` is **not** a + substitute: it instruments graph *capture*, which happens once at start-up. + Set `DECODE_CUDA_GRAPH=1` to profile with graphs on anyway — that is the run you compare + against to show how much the graph was hiding. + +The traces are ordinary torch/chrome traces — open one in Perfetto, or point an analyzer at +the output directory. This kit does not ship one. + ## 6. Tear down ```bash diff --git a/examples/sglang_1p1d_glm5.2/cluster/cluster.peermem.sh b/examples/sglang_1p1d_glm5.2/cluster/cluster.peermem.sh index 3c37236aa..f25912b55 100755 --- a/examples/sglang_1p1d_glm5.2/cluster/cluster.peermem.sh +++ b/examples/sglang_1p1d_glm5.2/cluster/cluster.peermem.sh @@ -45,6 +45,15 @@ export KIT_DIR="$KIT" # Requires an infera-sglang build NEWER than 0.2.0. The default below is a placeholder. export INFERA_IMAGE="${INFERA_IMAGE:-}" +# Development source overlay. /apps is shared by both nodes, and the engine image's +# WORKDIR is /opt/infera, so mounting this checkout there makes every `python -m infera...` +# invocation import the current working tree. Set INFERA_SRC= to disable the overlay. +# export INFERA_SRC="${INFERA_SRC-$(cd "$KIT/../.." && pwd)}" + +# Shared writable trace directory. common.sh bind-mounts this exact absolute path into both +# engine containers, so capture.sh can write there without a tar/docker-cp/SSH fetch stage. +export TRACE_OUT="${TRACE_OUT:-$KIT/profiles}" + # MODEL_MOUNT is bind-mounted into the container; MODEL must live under it. # Prefer LOCAL storage on both nodes — a slow mount blows the ready timeout # (see cluster/README.md section 2). diff --git a/examples/sglang_1p1d_glm5.2/common.sh b/examples/sglang_1p1d_glm5.2/common.sh index 3dcb5e6e3..75657e6ad 100755 --- a/examples/sglang_1p1d_glm5.2/common.sh +++ b/examples/sglang_1p1d_glm5.2/common.sh @@ -34,8 +34,20 @@ start_container(){ require_env INFERA_IMAGE "the engine image" require_env MODEL_MOUNT "host dir holding the weights, bind-mounted into the container" local ep=(--entrypoint '') + local loaded_infera if [ "${ENTRYPOINT_KEEP:-0}" = "1" ]; then ep=(); fi local mounts=(-v "$MODEL_MOUNT:$MODEL_MOUNT") + + if [ -n "${INFERA_SRC:-}" ]; then + [ -f "$INFERA_SRC/pyproject.toml" ] \ + || die "INFERA_SRC is not an Infera checkout on $(hostname -s): $INFERA_SRC" + mounts+=(-v "$INFERA_SRC:/opt/infera:ro") + fi + + if [ -n "${TRACE_OUT:-}" ]; then + mkdir -p "$TRACE_OUT" || die "could not create TRACE_OUT on $(hostname -s): $TRACE_OUT" + mounts+=(-v "$TRACE_OUT:$TRACE_OUT") + fi # Only mount a host RDMA provider library if the site says it needs one. # HOST_RDMA_MOUNT must be the in-container path YOUR image's entrypoint reads — mount it # elsewhere and the injection silently no-ops: zero devices, and the leg still serves. @@ -50,6 +62,15 @@ start_container(){ "${mounts[@]}" "${ep[@]}" "$INFERA_IMAGE" sleep infinity >/dev/null \ || die "docker run failed for $CTR" log "container '$CTR' up ($INFERA_IMAGE)" + if [ -n "${INFERA_SRC:-}" ]; then + loaded_infera="$(docker exec "$CTR" python3 -c 'import infera; print(infera.__file__)')" \ + || die "could not import the mounted Infera source in $CTR" + case "$loaded_infera" in + /opt/infera/infera/*) log "development source active: $INFERA_SRC -> $loaded_infera" ;; + *) die "mounted Infera source is not active; imported $loaded_infera" ;; + esac + fi + [ -n "${TRACE_OUT:-}" ] && log "trace output mounted rw: $TRACE_OUT" docker exec "$CTR" python3 -c 'import torch;print(" gpu gate:", torch.cuda.is_available(), torch.cuda.device_count())' || true # Zero devices is not an error — mooncake falls back to a 5-20x slower transport and the # leg serves fine. Hence a warning, not a number to skim past. @@ -83,10 +104,21 @@ start_etcd(){ # what: the infera router, inside $CTR on the prefill node. # why : the module is `infera.server`, NOT `infera.router` — the latter has no __main__. # how : POLICY=kv-aware|round-robin, coupled to GMU_PREFILL; kv-aware needs the tokenizer. +# ROUTER_EXTRA_ARGS appends flags verbatim; engine/up.sh uses it for --enable-profiling. start_router(){ local ip="${1:?router bind ip}" require_env TOKENIZER "tokenizer path the router loads for kv-aware routing" local policy="${ROUTER_POLICY:-kv-aware}" backend="${ROUTER_BACKEND:-rust}" + local extra="${ROUTER_EXTRA_ARGS:-}" + # The profile control plane exists only in the python router. launch_rust.py does not + # degrade when it sees --enable-profiling — it raises SystemExit, so the router never + # starts and the failure surfaces as "router did not come up" 60s later. Flip the backend + # here, in the one function that owns it, rather than making every caller remember. + if [ "$backend" = "rust" ] && [[ " $extra " == *" --enable-profiling "* ]]; then + warn "profiling requested -> router backend rust->python (rust has no profiling control plane)" + warn " expect lower absolute throughput than a rust-router run; do not compare the two." + backend=python + fi local kv_args=() if [ "$policy" = "kv-aware" ]; then # Cost = w * (request_blocks - hits) + active_blocks. Prefill is compute-bound and a hit @@ -104,7 +136,7 @@ start_router(){ --host 0.0.0.0 --port $ROUTER_PORT --router-backend $backend \ --discovery-backend etcd --etcd-endpoint $ip:$ETCD_PORT \ --request-transport http --kv-event-transport zmq \ - --router-policy $policy ${kv_args[*]} > /tmp/router.log 2>&1" + --router-policy $policy ${kv_args[*]} $extra > /tmp/router.log 2>&1" wait_health "http://$ip:$ROUTER_PORT/health" 12 5 \ || { docker exec "$CTR" tail -30 /tmp/router.log; die "router did not come up"; } log "router up on :$ROUTER_PORT (backend=$backend policy=$policy)" diff --git a/examples/sglang_1p1d_glm5.2/engine/capture.sh b/examples/sglang_1p1d_glm5.2/engine/capture.sh new file mode 100755 index 000000000..850828374 --- /dev/null +++ b/examples/sglang_1p1d_glm5.2/engine/capture.sh @@ -0,0 +1,192 @@ +#!/usr/bin/env bash +# Copyright (c) 2026, Advanced Micro Devices, Inc. All rights reserved. +# SPDX-License-Identifier: MIT +# what: cut a window out of a running load and capture one torch trace per PD role. +# why : prefill and decode overlap in wall-clock time, so a single trace cannot be split by +# role after the fact. The router's role selector is what guarantees the prefill +# directory holds prefill operators and nothing else. +# how : DO NOT run this directly — run it through cluster/.sh, e.g. +# PROFILE_DECODE=1 bash cluster/.sh up +# nohup bash cluster/.sh bench 64 & # load, in the background +# PROFILE_DECODE=1 bash cluster/.sh capture +# The same PROFILE_* values must be given to `up` and to `capture`: `up` decides +# whether the control plane exists at all, `capture` reads them to pick roles. +# +# Knobs (all optional): +# WARMUP_S=60 seconds to let the load settle before opening the window +# WINDOW_S=20 length of the sampled window +# TRACE_OUT=DIR shared host path mounted rw into both engine containers (default ../profiles) +# REQUIRE_LOAD=0 skip the "is a bench running" check (you are driving load some other way) +set -uo pipefail +DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"; source "$DIR/../common.sh" + +require_env PREFILL_NODE; require_env DECODE_NODE; require_env PREFILL_IP +SSH_CMD="${SSH_CMD:-ssh -o StrictHostKeyChecking=no}" +on(){ local h="$1"; shift; $SSH_CMD "$h" "$*" $SSH_CMD -> docker +# exec -> bash -c). base64 is [A-Za-z0-9+/=] only, so nothing in the payload can be +# re-interpreted on the way. Same reason up.sh stages /run_kvd.sh as a file. +put_script(){ + local h="$1" path="$2" b64 + b64="$(base64 -w0)" || die "base64 -w0 failed on this host" + on "$h" "docker exec $CTR bash -c 'echo $b64 | base64 -d > $path && chmod +x $path'" \ + || die "could not stage $path inside $CTR on $h" +} + +bench_running(){ on "$PREFILL_NODE" "docker exec $CTR pgrep -f 'sglang.bench_serving' >/dev/null 2>&1"; } + +# ---- 1/6 preflight ------------------------------------------------------------------------- +log "=== 1/6 preflight (roles: ${ROLES[*]}) ===" + +# Direct output is safe only when the exact host path is a writable bind mount. Without this +# check Docker would create the directory in the container layer and capture would appear to +# succeed while the host sees nothing. +for r in "${ROLES[@]}"; do + h="$(node_of "$r")" + mounted="$(on "$h" "docker inspect -f '{{range .Mounts}}{{if eq .Destination \"$TRACE_ROOT\"}}{{.RW}}{{end}}{{end}}' $CTR" | tr -d '\r\n')" + [ "$mounted" = "true" ] \ + || die "TRACE_OUT is not mounted rw in $CTR on $h: $TRACE_ROOT +Re-run 'bash cluster/.sh down', then bring up the stack with the same TRACE_OUT." +done + +# Probe the control plane with a role that cannot exist. In infera/server/app.py the handler +# checks the 403 gate FIRST and rejects the role BEFORE selecting or contacting any worker, +# so this tells us whether profiling is enabled without touching a running profile: +# 400 -> enabled (it got as far as validating the role) 403 -> not enabled +code="$(on "$PREFILL_NODE" "docker exec $CTR curl -s -o /dev/null -w '%{http_code}' -m 10 \ + -X POST '$URL/v1/admin/profile/start?role=__probe__'" | tr -d '\r\n')" +case "$code" in + 400) log " control plane ON (probe -> 400 invalid role, as expected)" ;; + 403) die "the router has profiling disabled. Re-run bring-up with the switch set: + PROFILE_PREFILL=1 PROFILE_DECODE=1 bash cluster/.sh up" ;; + 000|"") die "no answer from the router at $URL — is the stack up?" ;; + *) warn " unexpected probe status '$code'; continuing, but read /tmp/router.log if start fails" ;; +esac + +# A role with no ACTIVE worker makes /v1/admin/profile/start 404. Catch it here rather than +# after the warm-up, which is a minute of waiting for nothing. +workers="$(on "$PREFILL_NODE" "docker exec $CTR curl -s -m10 $URL/v1/workers" | tr -d ' \r')" +for r in "${ROLES[@]}"; do + n="$(printf '%s' "$workers" | grep -c "\"disagg_mode\":\"$r\"")" + [ "${n:-0}" -ge 1 ] || die "no $r worker registered with the router — start?role=$r would 404" + log " $r workers registered: $n" +done + +if [ "${REQUIRE_LOAD:-1}" = "1" ] && ! bench_running; then + die "no load in flight. An idle window profiles an empty scheduler loop, not your model. +Start one first, in another shell or in the background: + nohup bash cluster/.sh bench 64 & +Give it enough requests to outlast WARMUP_S + WINDOW_S (${WARMUP_S}s + ${WINDOW_S}s here) — N= raises it. +Set REQUIRE_LOAD=0 if you are driving load some other way." +fi + +# ---- 2/6 warm-up --------------------------------------------------------------------------- +log "=== 2/6 warm-up ${WARMUP_S}s ===" +# Profile steady state, not ramp-up: the first requests hit a cold radix cache and a decode +# leg with nothing queued, and their step times are not the ones worth optimising. +sleep "$WARMUP_S" +if [ "${REQUIRE_LOAD:-1}" = "1" ] && ! bench_running; then + warn " the load finished during warm-up — the window will catch an idle engine." + warn " Send more requests (N=) or shorten WARMUP_S." +fi + +# ---- 3/6 output directories ---------------------------------------------------------------- +log "=== 3/6 output directories ===" +# SGLang does not create output_dir. When it is missing the export fails inside the profiler +# callback long after /start_profile has already answered 200 — you find out at the end, with +# an empty directory and no error anywhere. +for r in "${ROLES[@]}"; do + on "$(node_of "$r")" "docker exec $CTR mkdir -p $CTR_TRACE/$r" \ + || die "could not create $CTR_TRACE/$r in $CTR on $(node_of "$r")" +done + +# ---- 4/6 start ----------------------------------------------------------------------------- +log "=== 4/6 start (window ${WINDOW_S}s) ===" +# The request body. Three keys are not obvious: +# with_stack=false MUST be explicit. SGLang's default is True (`with_stack if +# with_stack is not None else True`), which adds millions of +# python_function events — measured 122 MB vs 14 MB per rank, ~80% of +# the file — that no downstream analysis reads. +# record_shapes=true operator input shapes; without them no FLOPs/bytes roofline is possible. +# activities spelled out rather than left null, so the engine does not choose. +body(){ printf '{"output_dir":"%s/%s","record_shapes":true,"with_stack":false,"activities":["CPU","GPU"]}' "$CTR_TRACE" "$1"; } + +# Both starts are issued from inside ONE container, backgrounded, and waited on together. +# Issuing them from the host in sequence puts a full round-trip between the two roles' +# windows; from one shell the skew is sub-millisecond. Both go to the router regardless of +# role — it is the router that fans out to the worker on the other node. +{ + echo '#!/bin/bash' + for r in "${ROLES[@]}"; do + printf "curl -sS -m 60 -X POST -H 'Content-Type: application/json' -d '%s' '%s/v1/admin/profile/start?role=%s' > /tmp/capture_start_%s.out 2>&1 &\n" \ + "$(body "$r")" "$URL" "$r" "$r" + done + echo 'wait' + for r in "${ROLES[@]}"; do + printf 'printf " %%-8s " %s; cat /tmp/capture_start_%s.out; echo\n' "$r" "$r" + done +} | put_script "$PREFILL_NODE" /run_capture_start.sh +on "$PREFILL_NODE" "docker exec $CTR /run_capture_start.sh" \ + || die "profile start failed — see the output above and /tmp/router.log in $CTR" + +sleep "$WINDOW_S" + +# ---- 5/6 stop ------------------------------------------------------------------------------ +log "=== 5/6 stop ===" +# One call, not one per role. Two sequential stops would leave the first role recording for +# however long the first round-trip took, which is exactly the skew step 4 avoided. With both +# roles selected that means a broadcast (no selector); with one, a selector, so the other leg +# is not sent a stop it never started. +sel=""; [ "${#ROLES[@]}" -eq 1 ] && sel="?role=${ROLES[0]}" +# The router's own HTTP client has a hardcoded 30s read timeout (infera/server/app.py). A stop +# that has to flush eight ranks can exceed it, and the engine keeps writing regardless — so a +# failure here is not evidence that the stop failed. The flush check below is what decides. +on "$PREFILL_NODE" "docker exec $CTR curl -sS -m 180 -X POST \ + -H 'Content-Type: application/json' -d '{}' '$URL/v1/admin/profile/stop$sel'" \ + || warn " stop returned an error — this is often just the router's 30s read timeout; checking the files" +echo + +# ---- 6/6 flush ------------------------------------------------------------------------------- +log "=== 6/6 flush ===" +# A trace file appearing is not a trace file being finished: torch writes it from the profiler +# callback after stop has already returned. Wait for the byte count to stop moving rather than +# for a fixed sleep, which is either too short or wastes minutes. +wait_flush(){ + local h="$1" role="$2" path="$CTR_TRACE/$2" prev=-1 cur + for _ in $(seq 1 "${FLUSH_TRIES:-30}"); do + cur="$(on "$h" "docker exec $CTR bash -c 'du -sb $path 2>/dev/null | cut -f1'" | tr -d '\r\n')" + cur="${cur:-0}" + if [ "$cur" -gt 0 ] && [ "$cur" = "$prev" ]; then + log " $h/$role: $path stable at $cur bytes"; return 0 + fi + prev="$cur"; sleep "${FLUSH_GAP:-10}" + done + warn " $h/$role: $path never stopped growing (last: $prev bytes)" + return 1 +} + +for r in "${ROLES[@]}"; do + wait_flush "$(node_of "$r")" "$r" +done +log "traces available -> $OUT" + +log "capture done -> $OUT" +exit 0 diff --git a/examples/sglang_1p1d_glm5.2/engine/leg.sh b/examples/sglang_1p1d_glm5.2/engine/leg.sh index b7fc1eb4f..8dfdb8fa4 100755 --- a/examples/sglang_1p1d_glm5.2/engine/leg.sh +++ b/examples/sglang_1p1d_glm5.2/engine/leg.sh @@ -13,6 +13,8 @@ # MTP=0|1 EAGLE speculative decoding (decode leg only unless PREFILL_MTP=1) # KVAWARE=0|1 publish KV events so the router can route by cache locality # KVD=0|1 wire the infera-kvd HiCacheStorage backend (L2 host RAM + L3) +# CUDA_GRAPH=0|1 0 makes the leg profileable at a large throughput cost. up.sh sets it +# from PROFILE_DECODE; do not set it by hand for a serving run. set -uo pipefail DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"; source "$DIR/../common.sh" @@ -145,7 +147,9 @@ CAR_ARGS=() # client-side cache-hit metric reads 0 and a prefix-reuse target cannot be checked at all. EXTRA_ARGS=(--enable-cache-report) -log "$ROLE on $MY_IP:$PORT — tp=$TP dpa=$DPA mtp=$MTP kvaware=$KVAWARE kvd=$KVD gmu=$GMU chunk=$CHUNK ctx=$CTX nic=$NIC ib=$RDMA_IB_DEVICES" +[ "${CUDA_GRAPH:-1}" = "0" ] && EXTRA_ARGS+=("${CUDA_GRAPH_OFF_ARG:-"--disable-cuda-graph"}") + +log "$ROLE on $MY_IP:$PORT — tp=$TP dpa=$DPA mtp=$MTP kvaware=$KVAWARE kvd=$KVD gmu=$GMU chunk=$CHUNK ctx=$CTX nic=$NIC ib=$RDMA_IB_DEVICES cuda_graph=${CUDA_GRAPH:-1}" # Pass the transport/recipe env explicitly rather than relying on `docker exec`'s environment: # the variables above are exported in THIS shell, on the host, not inside the container. diff --git a/examples/sglang_1p1d_glm5.2/engine/up.sh b/examples/sglang_1p1d_glm5.2/engine/up.sh index a4c4edae0..e13015a0a 100755 --- a/examples/sglang_1p1d_glm5.2/engine/up.sh +++ b/examples/sglang_1p1d_glm5.2/engine/up.sh @@ -20,6 +20,37 @@ require_env RDMA_IB_DEVICES; require_env MC_GID_INDEX SSH_CMD="${SSH_CMD:-ssh -o StrictHostKeyChecking=no}" on(){ local h="$1"; shift; $SSH_CMD "$h" "$*"; } +# ---- profiling (off by default; nothing below changes unless a PROFILE_* is set) ------------ +# Two switches, not one, because the two legs are profiled for different reasons and only one +# of them has to be slowed down to be readable. Usage: +# PROFILE_DECODE=1 bash cluster/.sh up +# PROFILE_DECODE=1 bash cluster/.sh capture +# Both commands need the same values — capture.sh reads them to pick which roles to sample. +PROFILE_PREFILL="${PROFILE_PREFILL:-0}" +PROFILE_DECODE="${PROFILE_DECODE:-0}" +ROUTER_EXTRA_ARGS="${ROUTER_EXTRA_ARGS:-}" +if [ "$PROFILE_PREFILL" = "1" ] || [ "$PROFILE_DECODE" = "1" ]; then + # The control plane is on the ROUTER, not the legs: POST /v1/admin/profile/{start,stop} + # answers 403 unless the server carries this flag. start_router() also swaps the backend + # to python, because the rust one hard-exits on it. + ROUTER_EXTRA_ARGS="$ROUTER_EXTRA_ARGS --enable-profiling" + log "profiling ON (prefill=$PROFILE_PREFILL decode=$PROFILE_DECODE)" +fi + +# Decode CUDA graphs hide exactly what a decode profile is for. A graph replays as one opaque +# launch, so the trace keeps the per-step annotations and loses the kernels inside them — +# measured on this stack: 134 decode steps in the window, 1 of them with GPU operators. +# Prefill needs no equivalent; its extend path does not run through a graph. +# Not free, so it is tied to PROFILE_DECODE rather than left on: decode TPOT measured +# 35.6 -> 137.2 ms with graphs off. Set DECODE_CUDA_GRAPH=1 explicitly to profile WITH graphs +# on anyway — that is the run you compare against to show how much the graph was hiding. +if [ "$PROFILE_DECODE" = "1" ]; then + DECODE_CUDA_GRAPH="${DECODE_CUDA_GRAPH:-0}" +else + DECODE_CUDA_GRAPH="${DECODE_CUDA_GRAPH:-1}" +fi +[ "$DECODE_CUDA_GRAPH" = "0" ] && warn "decode CUDA graphs OFF (profiling) — throughput numbers from this run are NOT comparable to a normal one" + # The env every remote invocation needs. Kept in one string so the two legs cannot drift. # PREFILL_MTP is forwarded as well as consumed below: leg.sh reads it directly to decide # whether a prefill leg may emit MTP args, so passing only MTP= leaves that gate always shut. @@ -34,7 +65,10 @@ ${MC_MS_FILTERS:+MC_MS_FILTERS=$MC_MS_FILTERS} \ ${MC_MS_AUTO_DISC:+MC_MS_AUTO_DISC=$MC_MS_AUTO_DISC} \ ${RDMAV_FORK_SAFE:+RDMAV_FORK_SAFE=$RDMAV_FORK_SAFE} \ ${HOST_RDMA_LIB:+HOST_RDMA_LIB=$HOST_RDMA_LIB} \ +${HOST_RDMA_MOUNT:+HOST_RDMA_MOUNT=$HOST_RDMA_MOUNT} \ ${ENTRYPOINT_KEEP:+ENTRYPOINT_KEEP=$ENTRYPOINT_KEEP} \ +${INFERA_SRC:+INFERA_SRC=$INFERA_SRC} \ +${TRACE_OUT:+TRACE_OUT=$TRACE_OUT} \ ${GMU_PREFILL:+GMU_PREFILL=$GMU_PREFILL} ${GMU_DECODE:+GMU_DECODE=$GMU_DECODE}" log "=== 1/4 containers ===" @@ -69,6 +103,7 @@ on "$PREFILL_NODE" "$COMMON_ENV ROLE=prefill MY_IP=$PREFILL_IP PORT=$PREFILL_POR bash $KIT_DIR/engine/leg.sh" on "$DECODE_NODE" "$COMMON_ENV ROLE=decode MY_IP=$DECODE_IP PORT=$DECODE_PORT \ DPA=${DECODE_DPA:-1} MTP=${DECODE_MTP:-1} KVD=${DECODE_KVD:-0} \ + CUDA_GRAPH=$DECODE_CUDA_GRAPH \ bash $KIT_DIR/engine/leg.sh" # Poll /health from INSIDE each node's container. Never curl a PD leg's port from another @@ -93,8 +128,11 @@ log "=== 4/4 router (prefill node) ===" # are serving means its first health check already reflects a paired deployment. on "$PREFILL_NODE" "$COMMON_ENV ROUTER_POLICY=${ROUTER_POLICY:-kv-aware} \ ROUTER_BACKEND=${ROUTER_BACKEND:-rust} \ + ${ROUTER_EXTRA_ARGS:+ROUTER_EXTRA_ARGS='$ROUTER_EXTRA_ARGS'} \ ${KV_PREFILL_W:+KV_PREFILL_W=$KV_PREFILL_W} ${KV_DECODE_W:+KV_DECODE_W=$KV_DECODE_W} \ bash -c 'source $KIT_DIR/common.sh; start_router $PREFILL_IP'" log "up. endpoint: http://$PREFILL_IP:$ROUTER_PORT" log "verify with: bash cluster/.sh smoke" +[ -n "$ROUTER_EXTRA_ARGS" ] && log "profile with: bash cluster/.sh capture (needs load in flight)" +exit 0 From 07063e14c12216b919d0fa809d7b84f27b417b83 Mon Sep 17 00:00:00 2001 From: YaoCheng Date: Tue, 25 Aug 2026 10:12:32 +0000 Subject: [PATCH 2/4] add mooncake trace replay scripts --- examples/sglang_1p1d_glm5.2/README.md | 84 ++++++- examples/sglang_1p1d_glm5.2/cluster/README.md | 26 ++ .../cluster/cluster.peermem.sh | 34 ++- examples/sglang_1p1d_glm5.2/engine/capture.sh | 34 ++- .../sglang_1p1d_glm5.2/engine/trace_replay.sh | 223 ++++++++++++++++++ 5 files changed, 389 insertions(+), 12 deletions(-) create mode 100755 examples/sglang_1p1d_glm5.2/engine/trace_replay.sh diff --git a/examples/sglang_1p1d_glm5.2/README.md b/examples/sglang_1p1d_glm5.2/README.md index e4680de15..3a42a4930 100644 --- a/examples/sglang_1p1d_glm5.2/README.md +++ b/examples/sglang_1p1d_glm5.2/README.md @@ -27,6 +27,7 @@ different fabrics; those numbers are not published with this kit. | `engine/up.sh` | bring up both nodes: containers → etcd + kvd → both legs → router | | `engine/smoke.sh` | service check **plus** positive evidence for each of the five features | | `engine/bench.sh` | reference throughput sweep using SGLang's own `bench_serving` | +| `engine/trace_replay.sh` | replay a Mooncake production trace with AIPerf — the only load here with a real shared prefix (opt-in) | | `engine/capture.sh` | take one torch trace per PD role out of a running load (opt-in) | | `engine/down.sh` | tear down and wait for VRAM to actually free | | `preflight_rdma.sh` | RDMA preflight: registration-mode probe + cross-node fabric measurement | @@ -221,8 +222,64 @@ flags are load-bearing in ways that are not obvious from the flag name: `--num-prompts` is recomputed per concurrency (`10 × C`), so each arm of a sweep gets enough requests to reach steady state. -**This kit ships no agentic benchmark client**, by design. Point the customer's own -harness at the router endpoint. +### Trace replay + +The sweep above cannot measure prefix reuse, and prefix reuse is the property that makes +an agentic workload behave the way it does. `trace_replay.sh` replays a **Mooncake-format +production trace** at the timestamps the trace recorded: + +```bash +bash cluster/cluster.peermem.sh trace_replay prepare # offline checks + shape report +bash cluster/cluster.peermem.sh trace_replay run # send the load +``` + +Set `AIPERF_TRACE` in the wrapper first; §5 there has the download line for the FAST25 +arxiv trace. `prepare` slices the file, pulls the client image, checks that the trace, +tokenizer, output directory and router are all reachable **from the node that will +generate the load**, and prints the slice's ISL/OSL distribution and theoretical cache +hit rate. None of it touches the deployment, so it is safe to run against a warming stack. + +Why this is a different measurement rather than a second opinion on the same one: a +Mooncake entry carries `hash_ids`, and [AIPerf](https://github.com/ai-dynamo/aiperf) +expands each hash id into a real block of tokens. Equal hash ids therefore produce an +equal token *prefix*, so the radix cache, kvd and `kv-aware` routing are all exercised. +In the FAST25 arxiv trace one hash id appears in **every** request — a single shared +system prefix across 12,031 requests, median input 6,909 tokens. + +**Do not compare these numbers to `bench` numbers.** Two independent reasons: + +- `bench` builds every prompt independently, so it has no shared prefix and `trace_replay` + does. That is the point of having both. +- `trace_replay` sends `ignore_eos:true` and `bench` does not. AIPerf's `mooncake_trace` + loader sets `max_tokens` from the recorded `output_length` but injects no `min_tokens` + (its `--force-min-tokens` default is honored only by the `baseten_trace` loader), so + without `ignore_eos` a reasoning model that emits EOS early replays *shorter* than the + recording. `bench` uses the `sglang-oai-chat` backend, which sends neither `ignore_eos` + nor `include_usage` ([sgl-project/sglang#10746](https://github.com/sgl-project/sglang/issues/10746)), + so its output lengths are EOS-bound and its TPOT is computed against a fallback length. + +Both scripts pass `--temperature 1.0 --top-p 0.95` for the same reason, and it matters +*more* here: trace prompts are longer than the sweep's, and this reasoning model falls +into repetition on a long prompt at temperature 0 — after which MTP predicts the loop +perfectly, acceptance length pins at 4.00, and the run reads like KV corruption. + +Three knobs decide whether the run means anything: + +| knob | default | what goes wrong at the extremes | +|---|---|---| +| `REQUESTS` | 2000 | `all` is 12,031 requests over 58.9 min of trace. The first run over any slice synthesizes and tokenizes every prompt **before** sending anything (~14M tokens of text for the default slice), so a cold cache looks like a hang | +| `MAX_CONC` | 256 | A **ceiling**, not a target — fixed-schedule sends at the recorded timestamps regardless. Too low and arrivals queue behind the ceiling, which is the closed-loop behaviour `bench` already measures. Too high and a deployment that cannot keep up grows unbounded queue depth, after which every percentile describes the queue | +| `SPEEDUP` | unset | Scales every timestamp. The way to ask "what if this traffic arrived 2× faster" without changing its shape. Unset keeps the trace's own pace and keeps the synthesis pipeline out of the path | + +The client runs from `nvcr.io/nvidia/ai-dynamo/aiperf` (255 MB) and **not** inside `$CTR`: +the engine image ships Python 3.10 and AIPerf requires ≥ 3.11. `AIPERF_NODE` defaults to +the prefill node because that needs no extra configuration, but it is not the neutral +choice — prompt synthesis is CPU-bound and competes with the engine's own scheduler and +tokenizer processes. Any node that can route to the router works. + +Read the **OSL-mismatch block** in AIPerf's summary before quoting a decode-side number. +That is where a thinking model that stopped short of the requested output length shows up, +and it reports a percentage rather than a failure. ### Profiling @@ -291,9 +348,25 @@ decode figure is.) `bench` is deliberately not given a detach flag — `nohup … &` already does it, and the one thing a flag would buy (surviving an ssh drop mid-run) is already caught by `capture`, which -refuses to start, and warns again after warm-up, if no `bench_serving` is running. Give the -load enough requests to outlast `WARMUP_S + WINDOW_S`: `N=` overrides the `10 × C` -default. +refuses to start, and warns again after warm-up, if no load is running. Give the load enough +requests to outlast `WARMUP_S + WINDOW_S`: `N=` overrides the `10 × C` default. + +**Profiling under trace replay** is the only way to get prefill operators from a load that +has a real shared prefix, since `bench`'s random dataset has none by construction. `capture` +has to be told which load generator to look for, because the two live on opposite sides of +the container boundary — `bench_serving` runs inside `$CTR` on the prefill node, AIPerf runs +in its own container on `$AIPERF_NODE`: + +```bash +PROFILE_PREFILL=1 bash cluster/cluster.peermem.sh up +nohup bash cluster/cluster.peermem.sh trace_replay run & +LOAD_KIND=trace_replay PROFILE_PREFILL=1 bash cluster/cluster.peermem.sh capture +``` + +Size the slice for the window: a cold cache spends minutes synthesizing prompts before its +first request, so `WARMUP_S` starts counting against an idle engine unless the replay is +already sending. Either run `trace_replay prepare` and one throwaway `run` first to warm the +cache, or raise `WARMUP_S`. `capture` waits `WARMUP_S` (default 60) for the load to reach steady state, opens a `WINDOW_S` (default 20) window, starts both roles from **one** shell inside the container @@ -431,6 +504,7 @@ Stated plainly rather than implied. | **these scripts as written** | **validated** — `preflight_rdma.sh mode` → `up` → `smoke` → `bench` → `down` on a 2-node MI355X mode-B cluster, with no edits outside `cluster/cluster.dmabuf.sh`. Long context checked separately (needle, to 238K tokens) and under a real agentic workload at concurrency 8 | | `preflight_rdma.sh` | `mode` validated on both nodes and its verdict followed. `fabric` not exercised | | `cluster.peermem.sh`, `round-robin` routing | **not validated** — no peer-mem cluster was available, and the shipped `kv-aware` default is what ran | +| `engine/trace_replay.sh` | **not validated** — written against AIPerf's source and the published NGC image, never executed against a live deployment. The Python-3.10-in-the-engine-image constraint that shapes it was measured on `lmsysorg/sglang-rocm` rather than on `rocm/infera` itself. The AIPerf flags it passes are taken from a 0.13.0 checkout while the pinned image is 0.12.0, so a rejected flag is the expected first failure | If you run this kit and it does not come up, that is worth reporting. diff --git a/examples/sglang_1p1d_glm5.2/cluster/README.md b/examples/sglang_1p1d_glm5.2/cluster/README.md index da9c218e4..0fc2cb2eb 100644 --- a/examples/sglang_1p1d_glm5.2/cluster/README.md +++ b/examples/sglang_1p1d_glm5.2/cluster/README.md @@ -107,6 +107,32 @@ Run it on **each** node — the two can legitimately differ. the [main README](../README.md#recommended-configuration) explains what each one buys and which pairs are coupled. +### 5. Trace replay (optional) + +Only read by `engine/trace_replay.sh`, and only when `AIPERF_TRACE` is set. Leave it +unset and this block does nothing. + +| field | what it must be | +|---|---| +| `AIPERF_TRACE` | absolute path to a Mooncake-format trace JSONL, readable on **both** this host and `$AIPERF_NODE` | +| `AIPERF_OUT` | artifacts, the per-run command file and the mmap dataset cache. Must resolve to the **same** path on both hosts, for the same reason `KIT_DIR` must | +| `AIPERF_NODE` | which node generates the load. Defaults to `$PREFILL_NODE` | +| `AIPERF_IMAGE` | pinned to a published NGC tag. Override only to run a locally built AIPerf | + +Two of these are worth understanding before you set them. + +**`AIPERF_NODE` defaults to the prefill node because that needs no extra configuration, +not because it is the neutral choice.** AIPerf synthesizes and tokenizes every prompt in +the slice before it sends anything, which is CPU-bound work competing with the engine's +own scheduler and tokenizer processes on that node. Any host that can route to +`$PREFILL_IP:$ROUTER_PORT` works, and a host that is not serving removes that +interference entirely. + +**The client cannot run inside the engine container.** That image ships Python 3.10 and +AIPerf requires ≥ 3.11, so `trace_replay.sh` uses its own container. At 255 MB it is not +the pull an engine image is. If your site cannot reach `nvcr.io`, build AIPerf's own +Dockerfile and point `AIPERF_IMAGE` at the result — nothing else changes. + ## Schedulers where `ssh ` does not work `engine/up.sh` reaches each node through `$SSH_CMD`, defaulting to diff --git a/examples/sglang_1p1d_glm5.2/cluster/cluster.peermem.sh b/examples/sglang_1p1d_glm5.2/cluster/cluster.peermem.sh index f25912b55..00323a10b 100755 --- a/examples/sglang_1p1d_glm5.2/cluster/cluster.peermem.sh +++ b/examples/sglang_1p1d_glm5.2/cluster/cluster.peermem.sh @@ -21,7 +21,7 @@ # If preflight instead reports "no peer-mem module loaded", use # cluster.dmabuf.sh. See cluster/README.md for the full mapping. # -# Usage: bash cluster/cluster.peermem.sh up | smoke | bench [conc...] | down +# Usage: bash cluster/cluster.peermem.sh up | smoke | bench [conc...] | trace_replay | down set -euo pipefail KIT="$(cd "$(dirname "${BASH_SOURCE[0]}")/.." && pwd)" @@ -107,4 +107,36 @@ export DECODE_KVD=0 # off on decode by design export GMU_PREFILL=0.70 export GMU_DECODE=0.85 +# --------------------------------------------------------------------------- +# 5. Trace replay — optional, only read by engine/trace_replay.sh +# --------------------------------------------------------------------------- +# AIPerf replays a Mooncake-format production trace at the timestamps it recorded, which is +# the one load in this kit that carries a real shared prefix. Leave AIPERF_TRACE unset and +# nothing here does anything. +# +# The client CANNOT live in the engine container: that image ships Python 3.10 and AIPerf +# requires >= 3.11. It runs from the published NGC image instead — 255 MB, so pulling it is +# not the concern that pulling an engine image is. +export AIPERF_IMAGE="${AIPERF_IMAGE:-nvcr.io/nvidia/ai-dynamo/aiperf:0.12.0}" + +# The trace, on a path BOTH this host and $AIPERF_NODE can read. A few MB each, so there is no +# need to clone the repo: +# B=https://raw.githubusercontent.com/kvcache-ai/Mooncake/main/FAST25-release/traces +# curl -LO $B/conversation_trace.jsonl # 12031 reqs, avg ISL 12035, avg OSL 343 +# curl -LO $B/toolagent_trace.jsonl # 23608 reqs, avg ISL 8596, avg OSL 182 +# conversation is the default choice; toolagent is closer to an agentic workload. Upstream's +# third file, synthetic_trace.jsonl, has GENERATED (Poisson) arrival times rather than recorded +# ones, which makes it the wrong input for a fixed-schedule replay of real traffic. +# export AIPERF_TRACE="/conversation_trace.jsonl" + +# Which node generates the load. Defaults to the prefill node, which is the simple choice but +# not the neutral one: AIPerf synthesizes and tokenizes every prompt before sending anything, +# and that competes for CPU with the engine's own scheduler and tokenizer processes. Point it +# at any node that can route to $PREFILL_IP:$ROUTER_PORT to remove that interference. +# export AIPERF_NODE="" + +# Artifacts, the generated per-run command file, and the mmap dataset cache. Must be the same +# path on both hosts — trace_replay.sh checks this rather than letting it fail obscurely later. +export AIPERF_OUT="${AIPERF_OUT:-$KIT/aiperf}" + exec bash "$KIT/engine/${1:-up}.sh" "${@:2}" diff --git a/examples/sglang_1p1d_glm5.2/engine/capture.sh b/examples/sglang_1p1d_glm5.2/engine/capture.sh index 850828374..e90c66f64 100755 --- a/examples/sglang_1p1d_glm5.2/engine/capture.sh +++ b/examples/sglang_1p1d_glm5.2/engine/capture.sh @@ -16,7 +16,8 @@ # WARMUP_S=60 seconds to let the load settle before opening the window # WINDOW_S=20 length of the sampled window # TRACE_OUT=DIR shared host path mounted rw into both engine containers (default ../profiles) -# REQUIRE_LOAD=0 skip the "is a bench running" check (you are driving load some other way) +# LOAD_KIND=bench which load generator to look for: bench | trace_replay +# REQUIRE_LOAD=0 skip the "is a load running" check (you are driving load some other way) set -uo pipefail DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"; source "$DIR/../common.sh" @@ -51,7 +52,24 @@ put_script(){ || die "could not stage $path inside $CTR on $h" } -bench_running(){ on "$PREFILL_NODE" "docker exec $CTR pgrep -f 'sglang.bench_serving' >/dev/null 2>&1"; } +# what: is there load in flight? why: an idle window profiles an empty scheduler loop. +# The two load generators this kit ships live on opposite sides of the container boundary, so +# this cannot be one pgrep with a wider pattern: +# bench.sh runs sglang.bench_serving INSIDE $CTR on the prefill node +# trace_replay.sh runs AIPerf in its own container on $AIPERF_NODE, which need not be a +# serving node at all — so the probe is a host-side pgrep there +# LOAD_KIND selects which one to look for; anything else means you are driving load some other +# way, which is what REQUIRE_LOAD=0 is for. +bench_running(){ + case "${LOAD_KIND:-bench}" in + bench) + on "$PREFILL_NODE" "docker exec $CTR pgrep -f 'sglang.bench_serving' >/dev/null 2>&1" ;; + trace_replay) + on "${AIPERF_NODE:-$PREFILL_NODE}" "pgrep -f 'aiperf profile' >/dev/null 2>&1" ;; + *) + die "unknown LOAD_KIND '${LOAD_KIND}'. Use bench | trace_replay, or set REQUIRE_LOAD=0." ;; + esac +} # ---- 1/6 preflight ------------------------------------------------------------------------- log "=== 1/6 preflight (roles: ${ROLES[*]}) ===" @@ -91,10 +109,14 @@ for r in "${ROLES[@]}"; do done if [ "${REQUIRE_LOAD:-1}" = "1" ] && ! bench_running; then - die "no load in flight. An idle window profiles an empty scheduler loop, not your model. + die "no ${LOAD_KIND:-bench} load in flight. An idle window profiles an empty scheduler loop, not your model. Start one first, in another shell or in the background: - nohup bash cluster/.sh bench 64 & -Give it enough requests to outlast WARMUP_S + WINDOW_S (${WARMUP_S}s + ${WINDOW_S}s here) — N= raises it. + nohup bash cluster/.sh bench 64 & # LOAD_KIND=bench (default) + nohup bash cluster/.sh trace_replay run & # LOAD_KIND=trace_replay +Give it enough requests to outlast WARMUP_S + WINDOW_S (${WARMUP_S}s + ${WINDOW_S}s here): N= +raises it for bench, and for trace_replay the run lasts as long as the replayed window does +(START_MS/END_MS, or the whole trace). Note that a cold trace_replay cache spends minutes +synthesizing prompts before its first request, so start it well before you expect load. Set REQUIRE_LOAD=0 if you are driving load some other way." fi @@ -105,7 +127,7 @@ log "=== 2/6 warm-up ${WARMUP_S}s ===" sleep "$WARMUP_S" if [ "${REQUIRE_LOAD:-1}" = "1" ] && ! bench_running; then warn " the load finished during warm-up — the window will catch an idle engine." - warn " Send more requests (N=) or shorten WARMUP_S." + warn " Send more requests (N= for bench, a wider START_MS/END_MS window for trace_replay) or shorten WARMUP_S." fi # ---- 3/6 output directories ---------------------------------------------------------------- diff --git a/examples/sglang_1p1d_glm5.2/engine/trace_replay.sh b/examples/sglang_1p1d_glm5.2/engine/trace_replay.sh new file mode 100755 index 000000000..ca3ada5d6 --- /dev/null +++ b/examples/sglang_1p1d_glm5.2/engine/trace_replay.sh @@ -0,0 +1,223 @@ +#!/usr/bin/env bash +# Copyright (c) 2026, Advanced Micro Devices, Inc. All rights reserved. +# SPDX-License-Identifier: MIT +# what: replay a Mooncake-format production trace against the router with AIPerf, at the +# timestamps the trace recorded. +# why : bench.sh sweeps `--dataset-name random`, which builds every prompt independently — no +# shared prefix by construction, so it cannot measure prefix reuse at all. A Mooncake +# trace carries hash_ids and AIPerf expands each one into a real token block, so the +# radix cache, kvd and kv-aware routing are all exercised. Read "Trace replay" in the +# README before quoting any number from here against a bench.sh number. +# how : DO NOT run this directly — run it through cluster/.sh: +# bash cluster/.sh trace_replay prepare # read-only checks + shape report +# bash cluster/.sh trace_replay run # send the load +# bash cluster/.sh trace_replay # both, in order +# +# AIPerf runs in its OWN container, never in $CTR: the engine image ships Python 3.10 and AIPerf +# needs >= 3.11. There is deliberately no row-count knob either — the loader applies the +# timestamp window before prompt synthesis, so START_MS/END_MS saves everything a pre-sliced +# file would, without cutting a session in half on a trace that carries session_id. +# +# Knobs (all optional): +# START_MS/END_MS replay only a window of the trace, in trace milliseconds +# MAX_CONC=256 in-flight ceiling; 0 omits the flag. Not neutral either way — see README +# WORKERS=16 AIPerf worker processes. Matters when $AIPERF_NODE is a serving node +# BLOCK_SIZE=512 tokens per hash id, i.e. the block size the TRACE was recorded at +# REQ_TIMEOUT=900 per-request timeout in seconds +# SPEEDUP= scale every timestamp (2.0 = twice as fast). Also reshapes hash_ids +# IGNORE_EOS=1 send ignore_eos so output length equals the trace's output_length +set -uo pipefail +DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"; source "$DIR/../common.sh" + +require_env PREFILL_NODE; require_env PREFILL_IP +require_env MODEL "model directory — AIPerf loads its tokenizer from here" +require_env MODEL_MOUNT "host dir holding the weights, bind-mounted into the AIPerf container" +require_env AIPERF_TRACE "absolute path to the Mooncake-format trace JSONL, readable on \$AIPERF_NODE" + +SSH_CMD="${SSH_CMD:-ssh -o StrictHostKeyChecking=no}" +on(){ local h="$1"; shift; $SSH_CMD "$h" "$*" /dev/null 2>&1 || docker pull $AIPERF_IMAGE" \ + || die "could not obtain $AIPERF_IMAGE on $AIPERF_NODE" + log " $AIPERF_IMAGE present" + + log "=== prepare 2/4 paths visible on $AIPERF_NODE ===" + mkdir -p "$AIPERF_OUT" || die "could not create $AIPERF_OUT" + on "$AIPERF_NODE" "test -d '$AIPERF_OUT'" \ + || die "AIPERF_OUT is not visible on $AIPERF_NODE: $AIPERF_OUT +It must resolve to the same path on both hosts, like \$KIT_DIR already does." + on "$AIPERF_NODE" "test -r '$AIPERF_TRACE'" \ + || die "the trace is not readable on $AIPERF_NODE: $AIPERF_TRACE" + # AIPerf treats --tokenizer as a local path only when it exists, and as a HuggingFace repo id + # otherwise — which HF_HUB_OFFLINE then turns into a hard failure at startup. + on "$AIPERF_NODE" "test -r '$MODEL/tokenizer_config.json' || test -r '$MODEL/tokenizer.json'" \ + || die "no tokenizer files under $MODEL on $AIPERF_NODE" + log " trace, tokenizer and output dir all readable on $AIPERF_NODE" + # Auto-promotion to fixed-schedule hinges on the first record carrying a timestamp. Without + # one the replay runs but silently ignores the trace's pacing, which is the point of a replay. + head -1 "$AIPERF_TRACE" | grep -q '"timestamp"' \ + || warn " first record has no \"timestamp\" — AIPerf will NOT auto-promote to fixed-schedule" + + log "=== prepare 3/4 router reachable from $AIPERF_NODE ===" + # From the HOST: the AIPerf container is --network=host, so the host's view is the container's. + on "$AIPERF_NODE" "curl -sf -m10 '$URL/health' >/dev/null" \ + || die "no answer from the router at $URL from $AIPERF_NODE — is the stack up, and is that IP routable from there?" + workers="$(on "$AIPERF_NODE" "curl -s -m10 '$URL/v1/workers'" | tr -d ' \r')" + for role in prefill decode; do + n="$(printf '%s' "$workers" | grep -c "\"disagg_mode\":\"$role\"")" + [ "${n:-0}" -ge 1 ] || die "no $role worker registered with the router — replaying now would measure half a deployment" + log " $role workers registered: $n" + done + + log "=== prepare 4/4 trace shape ===" + # ISL/OSL distributions, prefix groups and the theoretical hit rate at this block size. A hit + # rate near zero means this trace exercises no prefix reuse, which is cheaper to learn here + # than from results. Reads the WHOLE file — it does not know about START_MS/END_MS. + aiperf_in_image "aiperf analyze-trace --input-file $AIPERF_TRACE --block-size $BLOCK_SIZE --output-file $SHAPE" \ + || warn " analyze-trace failed — continuing, the replay does not depend on it" + log " shape report -> $SHAPE" +} + +# ---- run ------------------------------------------------------------------------------------- +run_replay(){ + local ts run_dir args extra + ts="$(date +%Y%m%d_%H%M%S)" + run_dir="$AIPERF_OUT/$ts" + + [ -r "$AIPERF_TRACE" ] || die "trace not readable: $AIPERF_TRACE" + mkdir -p "$run_dir" || die "could not create $run_dir" + + # Optional flags accumulate into $args; --extra-inputs takes a LIST, so $extra is separate and + # goes LAST or the flags after it are swallowed as more of its values. + # + # auto-offset shifts the first timestamp to 0 so the replay starts immediately, and is mutually + # exclusive with an explicit start offset. Both offsets are applied by the loader. + args="--fixed-schedule" + if [ -n "${START_MS:-}" ]; then + args="$args --fixed-schedule-start-offset $START_MS" + else + args="$args --fixed-schedule-auto-offset" + fi + [ -n "${END_MS:-}" ] && args="$args --fixed-schedule-end-offset $END_MS" + + # Not only a time-axis knob: any value routes the trace through AIPerf's Synthesizer, which + # rewrites hash_ids — shared prefix blocks keep their ids, each request is truncated at its + # first non-shared block and the tail gets fresh unique ids. Near a no-op on a prefix-chained + # trace, but prepare's hit rate then describes the raw file, not what replays. + [ -n "${SPEEDUP:-}" ] && args="$args --synthesis-speedup-ratio $SPEEDUP" + + # A CEILING, not a target — fixed-schedule sends at the recorded timestamps regardless. Too + # low degenerates into the closed-loop behaviour bench.sh already measures; too high lets a + # deployment that cannot keep up grow unbounded queue depth, after which every percentile + # describes the queue. 0 omits the flag if a future AIPerf rejects the pairing. + [ "$MAX_CONC" != "0" ] && args="$args --concurrency $MAX_CONC" + + # temperature/top_p are the checkpoint's own generation_config defaults and deliberately NOT + # greedy, for the reason bench.sh passes them — and it matters more here, where prompts are + # longer. ignore_eos is the one thing bench.sh does not send: the mooncake_trace loader sets + # max_tokens from output_length but injects no min_tokens (--force-min-tokens is honored only + # by baseten_trace), so without it a thinking model that stops early replays short. That also + # makes decode-side numbers here incomparable to bench.sh — see the README. + extra="temperature:1.0 top_p:0.95" + [ "${IGNORE_EOS:-1}" = "1" ] && extra="$extra ignore_eos:true" + + log "=== replay ===" + log " trace : $AIPERF_TRACE" + [ -n "${START_MS:-}${END_MS:-}" ] && log " window : ${START_MS:-start}..${END_MS:-end} ms" + log " endpoint : $URL (model $SERVED)" + log " client : $AIPERF_NODE ($AIPERF_IMAGE)" + log " artifacts : $run_dir" + log " options : $args --extra-inputs $extra" + # A cold cache spends minutes synthesizing prompts with nothing on the wire; say so, or it + # reads as a hang. + log " a cold cache synthesizes and tokenizes every prompt BEFORE the first request goes out" + + aiperf_in_image "aiperf profile \ + --model $SERVED --tokenizer $MODEL \ + --endpoint-type chat --streaming \ + --url $URL \ + --input-file $AIPERF_TRACE --custom-dataset-type mooncake_trace \ + --isl-block-size $BLOCK_SIZE \ + --workers-max $WORKERS \ + --request-timeout-seconds $REQ_TIMEOUT \ + --no-gpu-telemetry --ui none \ + --artifact-dir $run_dir \ + $args --extra-inputs $extra" \ + || die "replay failed — read the output above, then $run_dir for whatever AIPerf wrote" + + log "replay done -> $run_dir" + # Where a thinking model that stopped short of the requested output length shows up, as a + # percentage rather than a failure. + log "read the OSL-mismatch block in the summary before quoting any decode-side number" +} + +case "${1:-all}" in + prepare) prepare ;; + run) run_replay ;; + all) prepare && run_replay ;; + *) die "unknown subcommand '$1'. Use: prepare | run | (nothing, for both)" ;; +esac +exit 0 From e43ae9560d27b91347ec1106ceb4660b26fb998d Mon Sep 17 00:00:00 2001 From: YaoCheng Date: Wed, 26 Aug 2026 05:13:34 +0000 Subject: [PATCH 3/4] fix trace replay local tokenlizer bugs --- examples/sglang_1p1d_glm5.2/cluster/README.md | 6 +++++- .../engine/aiperf_compat/sitecustomize.py | 19 +++++++++++++++++ .../sglang_1p1d_glm5.2/engine/trace_replay.sh | 21 ++++++++++++++----- 3 files changed, 40 insertions(+), 6 deletions(-) create mode 100644 examples/sglang_1p1d_glm5.2/engine/aiperf_compat/sitecustomize.py diff --git a/examples/sglang_1p1d_glm5.2/cluster/README.md b/examples/sglang_1p1d_glm5.2/cluster/README.md index 0fc2cb2eb..8a321a123 100644 --- a/examples/sglang_1p1d_glm5.2/cluster/README.md +++ b/examples/sglang_1p1d_glm5.2/cluster/README.md @@ -110,7 +110,11 @@ and which pairs are coupled. ### 5. Trace replay (optional) Only read by `engine/trace_replay.sh`, and only when `AIPERF_TRACE` is set. Leave it -unset and this block does nothing. +unset and nothing here does anything. + +```bash +AIPERF_TRACE=<...> bash cluster/cluster.dmabuf.sh trace_replay prepare +``` | field | what it must be | |---|---| diff --git a/examples/sglang_1p1d_glm5.2/engine/aiperf_compat/sitecustomize.py b/examples/sglang_1p1d_glm5.2/engine/aiperf_compat/sitecustomize.py new file mode 100644 index 000000000..5901840e4 --- /dev/null +++ b/examples/sglang_1p1d_glm5.2/engine/aiperf_compat/sitecustomize.py @@ -0,0 +1,19 @@ +"""Compatibility for AIPerf 0.12 offline workers using local tokenizers.""" + +from pathlib import Path + +from aiperf.common.tokenizer import Tokenizer # type: ignore[import-not-found] + + +_original_resolve_local_snapshot = Tokenizer._resolve_local_snapshot.__func__ + + +@classmethod +def _resolve_local_snapshot(cls, name: str, revision: str) -> str: + local_path = Path(name) + if local_path.is_dir(): + return str(local_path.resolve()) + return _original_resolve_local_snapshot(cls, name, revision) + + +Tokenizer._resolve_local_snapshot = _resolve_local_snapshot diff --git a/examples/sglang_1p1d_glm5.2/engine/trace_replay.sh b/examples/sglang_1p1d_glm5.2/engine/trace_replay.sh index ca3ada5d6..828eb60dd 100755 --- a/examples/sglang_1p1d_glm5.2/engine/trace_replay.sh +++ b/examples/sglang_1p1d_glm5.2/engine/trace_replay.sh @@ -40,6 +40,7 @@ on(){ local h="$1"; shift; $SSH_CMD "$h" "$*" "$console_log" || die "could not create $console_log" + # Preserve the complete replay stdout/stderr (including Rich's final tables) while still + # streaming it to the invoking terminal. From here onward both streams share their ordering. + exec > >(tee -a "$console_log") 2>&1 # Optional flags accumulate into $args; --extra-inputs takes a LIST, so $extra is separate and # goes LAST or the flags after it are swallowed as more of its values. @@ -190,6 +200,7 @@ run_replay(){ log " endpoint : $URL (model $SERVED)" log " client : $AIPERF_NODE ($AIPERF_IMAGE)" log " artifacts : $run_dir" + log " console : $console_log" log " options : $args --extra-inputs $extra" # A cold cache spends minutes synthesizing prompts with nothing on the wire; say so, or it # reads as a hang. From a7728e3f77b19bfb7aa5c5f2250972ae121648d9 Mon Sep 17 00:00:00 2001 From: YaoCheng Date: Wed, 26 Aug 2026 11:29:45 +0000 Subject: [PATCH 4/4] add glm5.2 1p1d example lm_eval --- examples/sglang_1p1d_glm5.2/README.md | 159 +++++++- examples/sglang_1p1d_glm5.2/cluster/README.md | 12 +- .../cluster/cluster.dmabuf.sh | 101 +++++- .../cluster/cluster.peermem.sh | 59 ++- .../engine/aiperf_compat/sitecustomize.py | 1 - examples/sglang_1p1d_glm5.2/engine/lm_eval.sh | 176 +++++++++ .../engine/tools/probe_accuracy.py | 342 ++++++++++++++++++ .../engine/tools/summarise_eval.py | 257 +++++++++++++ 8 files changed, 1089 insertions(+), 18 deletions(-) create mode 100755 examples/sglang_1p1d_glm5.2/engine/lm_eval.sh create mode 100755 examples/sglang_1p1d_glm5.2/engine/tools/probe_accuracy.py create mode 100755 examples/sglang_1p1d_glm5.2/engine/tools/summarise_eval.py diff --git a/examples/sglang_1p1d_glm5.2/README.md b/examples/sglang_1p1d_glm5.2/README.md index 3a42a4930..8e2ab9d01 100644 --- a/examples/sglang_1p1d_glm5.2/README.md +++ b/examples/sglang_1p1d_glm5.2/README.md @@ -6,12 +6,13 @@ by the **infera router**, with **DP-attention**, **MTP** (EAGLE speculative deco and the **kvd** cache tiers all on. Two files in [`cluster/`](cluster/) hold everything site-specific. Fill in one of -them and the deployment is three commands. +them and the deployment is four commands. ```bash bash preflight_rdma.sh mode # which wrapper do I need? bash cluster/cluster.peermem.sh up # (or cluster.dmabuf.sh) — bring it up -bash cluster/cluster.peermem.sh smoke # prove it works +bash cluster/cluster.peermem.sh smoke # prove it is serving +bash cluster/cluster.peermem.sh lm_eval # prove it is serving CORRECTLY ``` This exact shape has been measured under two independent agentic benchmarks on two @@ -26,6 +27,7 @@ different fabrics; those numbers are not published with this kit. | `engine/leg.sh` | the real launcher for one PD leg. The tuned recipe lives here; no site values do | | `engine/up.sh` | bring up both nodes: containers → etcd + kvd → both legs → router | | `engine/smoke.sh` | service check **plus** positive evidence for each of the five features | +| `engine/lm_eval.sh` | accuracy: does it serve the model **correctly**, not just quickly. Nothing to install | | `engine/bench.sh` | reference throughput sweep using SGLang's own `bench_serving` | | `engine/trace_replay.sh` | replay a Mooncake production trace with AIPerf — the only load here with a real shared prefix (opt-in) | | `engine/capture.sh` | take one torch trace per PD role out of a running load (opt-in) | @@ -197,6 +199,131 @@ so the chain of thought lands in `reasoning_content` — but it is billed agains comes back empty with `finish_reason: "length"`, and the check reads as a failure on a deployment that is serving correctly. +### Accuracy + +`smoke` proves the features are on and `bench` proves the stack is fast. Neither reads +a single answer. Between MXFP4 weights, an `fp8_e4m3` KV cache, sparse attention on +ROCm, KV moved across a wire by mooncake, EAGLE speculation and a radix cache shared +between requests, there are six independent ways for this stack to serve fluent, fast, +**wrong** tokens — and every one of them looks green in `smoke` and `bench`. + +```bash +bash cluster/cluster.peermem.sh lm_eval probe # ~40 s, gates the rest +bash cluster/cluster.peermem.sh lm_eval quick # ~2 min, GSM8K 200 +bash cluster/cluster.peermem.sh lm_eval full # ~13 min, the full GSM8K set twice +bash cluster/cluster.peermem.sh lm_eval # probe, then quick +``` + +The evaluator is `sglang.test.run_eval`, which ships **inside the engine image** and +talks OpenAI `/v1/chat/completions`. Nothing is installed. That matters beyond +convenience: pip-installing an eval harness into `$CTR` would drag a `torch` dependency +across the ROCm build the engine is running on. + +**`probe` runs first and gates the rest**, because the failures it catches all produce a +*number* rather than an error, and that number is indistinguishable from a real +regression. It sends ~30 requests and checks four things: both legs are registered; +`content` actually arrives; the same question always gets the same **answer**; and a +shared prefix does not change the answer to the question behind it. Its exit status is 0 +only when every hard check passes, so it can gate a script. Soft findings are printed +but never fail. + +That last check is the reason this script exists in the form it does. Prefix reuse +changing an answer is the one failure mode nothing else in the kit can see: `bench` +builds every prompt independently, so a radix-cache or kvd lookup returning the wrong +block is never exercised there at all. + +**Reading the results.** Measured on this deployment, for calibration: + +| what | healthy reading | +|---|---| +| `probe` | `PROBE PASSED`, no hard failures | +| GSM8K 200 | **0.940–0.965** (six runs, mean 0.952) | +| GSM8K full (1314 scored) | **0.945**, 95% CI [0.932, 0.956], 376 s | +| `mixed_prefix_gsm8k` full (1299 scored) | **0.945**, 95% CI [0.932, 0.956], 380 s | +| prefix-reuse delta | **+0.000** — shared prefixes changed nothing | +| MTP acceptance length | mean **3.01–3.10**, min 2.42, max 3.54 | + +`full` runs `gsm8k` and `mixed_prefix_gsm8k` as a **pair**, and the pairing is the +point. They ask the same questions; only the second one puts them behind partially +shared few-shot prefixes. A single absolute score has no baseline to be compared +against — nobody knows what GLM-5.2-MXFP4 with an fp8 KV cache "should" score — but the +*gap between these two* is a measurement of prefix reuse that needs no external +baseline. The summary reports that gap with its own interval and says whether it +contains zero. + +Four things about this are counter-intuitive: + +- **`temperature=0` is not byte-reproducible here, and that is expected.** Ten serial + repeats of one GSM8K question all returned 72 while their completion lengths ranged + over 214–289 tokens. MTP verification, dp8 attention and batch-dependent kernel + selection each perturb the numerics without changing the argmax that matters. This is + why `probe` compares the **answer** and not the bytes: a byte-for-byte check would + fail on every healthy run, which is worse than no check at all. +- **Two runs of `quick` that differ are usually the same measurement.** 200 questions at + a true rate of 0.95 gives a 95% interval of about [0.913, 0.974]. Six runs on an + unchanged deployment here spanned 0.940 to 0.965 — a 2.5-point spread with nothing + changing between them, and every value inside that one interval. The interval is + printed next to every score for exactly this reason. On the full set it narrows to + roughly ±1.2 points, enough to see a ~3-point regression and no smaller. The + intervals are Wilson rather than the textbook + `p ± z·√(p(1−p)/n)`, which collapses to zero width at a perfect score — a 16-question + run that happens to get everything right would otherwise report `[1.000, 1.000]`. +- **The full set scores 1314 questions, not the 1319 GSM8K ships.** `GSM8KEval` slices + the few-shot examples off the front of the evaluation set so they cannot leak into it, + and `mixed_prefix_gsm8k` reserves a further 15 for its secondary prefix pool, leaving + 1299. The `scored` column is counted from the report rather than assumed, because the + dataset size is the wrong denominator for the interval — and a run that failed requests + would not show up as a smaller count anyway, since a failed request is scored 0 and + stays in the set. +- **A low score on `gpqa` or `aime25` is a `MAX_TOKENS` artefact, not a finding.** Both + need long chains of thought, GLM-5.2 bills reasoning against the same budget, and at + the 2048 default they truncate and score near chance — `gpqa` measured 0.25, which is + exactly random for a four-way choice. + +Of the eleven evals `run_eval` registers, three do **not** work here: `humaneval` (no +`human_eval` package in the image), `math` (grades equality with a second model, so it +needs an OpenAI key) and `mmmu` (vision, and this is a text checkpoint). Besides the two +`full` runs by default, `mmlu`, `gpqa`, `aime25` and `mgsm_en` were each executed against +this image and work; `mgsm` and `longbench_v2` were not tried. Any of them can be +requested by name: `lm_eval full mmlu`. + +### Comparing two runs + +Since the absolute number cannot be checked against a published one, the useful practice +is to record a run on a stack you trust and compare against **that**. The question that +actually comes up is never "is 0.945 good" but "did this config change, leg restart or +new image move anything", and results are timestamped per run so both sides are still on +disk: + +```bash +python3 engine/tools/summarise_eval.py lm_eval/ lm_eval/ +python3 engine/tools/summarise_eval.py --gate lm_eval/ lm_eval/ +``` + +``` + eval A B delta 95% CI of delta verdict + gsm8k 0.945 0.850 -0.095 [-0.118, -0.072] REGRESSED + mixed_prefix_gsm8k 0.945 0.945 -0.000 [-0.018, +0.017] same +``` + +The same script prints the single-run table, so any past result directory can be +re-summarised without re-running the eval. `--gate` exits non-zero when some eval dropped +by more than measurement noise; without it the comparison only reports, which matches how +the rest of this kit treats scores. + +**The delta gets its own interval, and that is not the same as checking whether the two +scores' intervals overlap.** Overlap is the intuitive test and it is too conservative: +uncertainties combine in quadrature, not by addition, so two runs can have visibly +overlapping intervals and still differ significantly. Reading overlap hides real +regressions, which is the one thing this comparison exists to catch. The verdict column +asks instead whether the difference's own interval — Newcombe's composition of the two +Wilson intervals — contains zero. + +`same` therefore means this pair of runs cannot tell the two configurations apart — not +that they are identical. With 200 questions per side that interval is about ±5 points +wide, so a genuine 3-point regression reads as `same`. Use `full` on both sides when the +answer has to be trusted. + ### Reference sweep ```bash @@ -229,21 +356,26 @@ an agentic workload behave the way it does. `trace_replay.sh` replays a **Moonca production trace** at the timestamps the trace recorded: ```bash -bash cluster/cluster.peermem.sh trace_replay prepare # offline checks + shape report +bash cluster/cluster.peermem.sh trace_replay prepare # read-only checks + shape report bash cluster/cluster.peermem.sh trace_replay run # send the load ``` -Set `AIPERF_TRACE` in the wrapper first; §5 there has the download line for the FAST25 -arxiv trace. `prepare` slices the file, pulls the client image, checks that the trace, +Set `AIPERF_TRACE` in the wrapper first; §5 there has the download lines for the two +Mooncake FAST25 traces. `prepare` pulls the client image, checks that the trace, tokenizer, output directory and router are all reachable **from the node that will -generate the load**, and prints the slice's ISL/OSL distribution and theoretical cache -hit rate. None of it touches the deployment, so it is safe to run against a warming stack. +generate the load**, and prints the trace's ISL/OSL distribution and theoretical cache +hit rate. It is read-only end to end, so it is safe to run against a warming stack. + +That shape report reads the **whole file** and knows nothing about `START_MS`/`END_MS`, +so it describes the recording rather than the window you are about to replay. Nothing in +`prepare` writes a sliced copy either — the window is applied by AIPerf's loader at +`run` time. Why this is a different measurement rather than a second opinion on the same one: a Mooncake entry carries `hash_ids`, and [AIPerf](https://github.com/ai-dynamo/aiperf) expands each hash id into a real block of tokens. Equal hash ids therefore produce an equal token *prefix*, so the radix cache, kvd and `kv-aware` routing are all exercised. -In the FAST25 arxiv trace one hash id appears in **every** request — a single shared +In `conversation_trace.jsonl` one hash id appears in **every** request — a single shared system prefix across 12,031 requests, median input 6,909 tokens. **Do not compare these numbers to `bench` numbers.** Two independent reasons: @@ -267,10 +399,15 @@ Three knobs decide whether the run means anything: | knob | default | what goes wrong at the extremes | |---|---|---| -| `REQUESTS` | 2000 | `all` is 12,031 requests over 58.9 min of trace. The first run over any slice synthesizes and tokenizes every prompt **before** sending anything (~14M tokens of text for the default slice), so a cold cache looks like a hang | +| `START_MS` / `END_MS` | unset | The replay window, in trace milliseconds. Unset replays the whole recording: `conversation_trace.jsonl` is 12,031 requests over 58.9 min, and its 288,500 hash-id blocks expand to ~148M tokens of prompt at the default `BLOCK_SIZE=512`. The first run over any (trace, window) pair synthesizes and tokenizes **all** of it before a single request goes out, so a cold cache looks like a hang | | `MAX_CONC` | 256 | A **ceiling**, not a target — fixed-schedule sends at the recorded timestamps regardless. Too low and arrivals queue behind the ceiling, which is the closed-loop behaviour `bench` already measures. Too high and a deployment that cannot keep up grows unbounded queue depth, after which every percentile describes the queue | | `SPEEDUP` | unset | Scales every timestamp. The way to ask "what if this traffic arrived 2× faster" without changing its shape. Unset keeps the trace's own pace and keeps the synthesis pipeline out of the path | +There is deliberately **no request-count knob**. The loader applies the timestamp window +before prompt synthesis, so `START_MS`/`END_MS` saves everything a pre-sliced file would +save while cutting on a time boundary rather than mid-session — which matters on any +trace that carries `session_id`. + The client runs from `nvcr.io/nvidia/ai-dynamo/aiperf` (255 MB) and **not** inside `$CTR`: the engine image ships Python 3.10 and AIPerf requires ≥ 3.11. `AIPERF_NODE` defaults to the prefill node because that needs no extra configuration, but it is not the neutral @@ -504,7 +641,9 @@ Stated plainly rather than implied. | **these scripts as written** | **validated** — `preflight_rdma.sh mode` → `up` → `smoke` → `bench` → `down` on a 2-node MI355X mode-B cluster, with no edits outside `cluster/cluster.dmabuf.sh`. Long context checked separately (needle, to 238K tokens) and under a real agentic workload at concurrency 8 | | `preflight_rdma.sh` | `mode` validated on both nodes and its verdict followed. `fabric` not exercised | | `cluster.peermem.sh`, `round-robin` routing | **not validated** — no peer-mem cluster was available, and the shipped `kv-aware` default is what ran | -| `engine/trace_replay.sh` | **not validated** — written against AIPerf's source and the published NGC image, never executed against a live deployment. The Python-3.10-in-the-engine-image constraint that shapes it was measured on `lmsysorg/sglang-rocm` rather than on `rocm/infera` itself. The AIPerf flags it passes are taken from a 0.13.0 checkout while the pinned image is 0.12.0, so a rejected flag is the expected first failure | +| `engine/lm_eval.sh` | **validated** — `probe`, `quick`, `full` and the failure path all executed against a live 1P1D deployment on MI355X (`rocm/infera:sglang-v0.2.10`, sglang 0.5.17), and every number quoted under "Accuracy" comes from those runs. The eval availability list in its header is measured rather than read off upstream, with the two exceptions it names (`mgsm`, `longbench_v2`) marked as untried | +| `engine/tools/summarise_eval.py` | **validated** — single-run and compare modes both exercised on real result directories. The `REGRESSED` verdict and `--gate`'s non-zero exit were checked against a fabricated lower-score directory, since no real regression was available to trigger them | +| `engine/trace_replay.sh` | **validated** — `prepare` and `run` both executed against a live 1P1D deployment, replaying `conversation_trace.jsonl` through the pinned AIPerf 0.12.0 image to completion, with and without a `START_MS`/`END_MS` window. Two concerns this row used to carry did not materialise: 0.12.0 accepted every flag the script passes (they were read off a 0.13.0 checkout), and the Python-3.10 constraint that forces the client out of `$CTR` was confirmed on `rocm/infera` itself, not just on the base image. Not exercised: the other traces, `SPEEDUP`, `IGNORE_EOS=0`, and pointing `AIPERF_NODE` at a non-serving host | If you run this kit and it does not come up, that is worth reporting. diff --git a/examples/sglang_1p1d_glm5.2/cluster/README.md b/examples/sglang_1p1d_glm5.2/cluster/README.md index 8a321a123..eb42f34f4 100644 --- a/examples/sglang_1p1d_glm5.2/cluster/README.md +++ b/examples/sglang_1p1d_glm5.2/cluster/README.md @@ -28,7 +28,8 @@ a puzzle to solve. ## The fields you must fill in -Both wrappers have the same four blocks. Placeholders read ``. +Both wrappers have the same five blocks. Placeholders read ``. Only the first +four are needed to serve; §5 is read by `engine/trace_replay.sh` alone. ### 1. Nodes @@ -52,11 +53,20 @@ config preflight, but a *wrong-but-routable* one is not — it just hangs. | `MODEL` | the checkpoint, which must live **under** `MODEL_MOUNT` | | `TOKENIZER` | usually the same path; the router loads it for kv-aware routing | | `HOST_RDMA_LIB` / `HOST_RDMA_MOUNT` / `ENTRYPOINT_KEEP` | only if your image injects a host RDMA provider library at entrypoint — see below | +| `TRACE_OUT` | where `engine/capture.sh` writes torch traces. Bind-mounted at this **exact absolute path** into both engine containers, so it must be writable on both nodes. Shipped as `$KIT_DIR/profiles` | +| `INFERA_SRC` | optional development overlay: an Infera checkout mounted read-only over the image's `/opt/infera`. Shipped commented out | The checkpoint is ~400 GB and both legs read it during bring-up. Prefer local storage on both nodes: a slow mount can take an order of magnitude longer and blow the ready timeout — which presents as a crash loop, not as slow storage. +`TRACE_OUT` is the one field here that a serving deployment never touches and a +profiling run cannot work without. `capture.sh` verifies the mount with `docker inspect` +before it starts a profile, because without that check Docker would create the directory +inside the container layer and the capture would report success while the host stayed +empty. `INFERA_SRC` is checked the same way, by importing `infera` in the started +container and refusing to continue if the mounted tree is not what got imported. + **If your image injects a host RDMA provider library**, all three variables are required together. `HOST_RDMA_LIB` is the host path (point it at the *symlink*, so nodes carrying different provider builds both resolve); `HOST_RDMA_MOUNT` is the in-container diff --git a/examples/sglang_1p1d_glm5.2/cluster/cluster.dmabuf.sh b/examples/sglang_1p1d_glm5.2/cluster/cluster.dmabuf.sh index a20ac3052..bdac3d7a1 100755 --- a/examples/sglang_1p1d_glm5.2/cluster/cluster.dmabuf.sh +++ b/examples/sglang_1p1d_glm5.2/cluster/cluster.dmabuf.sh @@ -31,16 +31,40 @@ # If preflight instead reports "peer-mem present", use cluster.peermem.sh. # See cluster/README.md for the full mapping. # -# Usage: bash cluster/cluster.dmabuf.sh up | smoke | bench [conc...] | down +# Usage: bash cluster/cluster.dmabuf.sh up | smoke | lm_eval | bench [conc...] | +# trace_replay [prepare|run] | capture | down set -euo pipefail KIT="$(cd "$(dirname "${BASH_SOURCE[0]}")/.." && pwd)" +# --------------------------------------------------------------------------- +# 0. Before the first bring-up — READ THIS +# --------------------------------------------------------------------------- +# Every value in §2 and §3 is a MEASUREMENT, not a preference, and each one fails +# SILENTLY when it is wrong: a NIC without ODP pins and duplicates the KV pool, and a +# provider mismatch drops everything to TCP at 5-20x the latency. The stack keeps serving +# tokens either way. Run this ON BOTH NODES first and reconcile §2/§3 with what it says: +# +# IMAGE= bash preflight_rdma.sh mode +# -> "peermem: absent" + "mode B: VIABLE" confirms this wrapper is the right one. +# If it reports peer-mem present, use cluster/cluster.peermem.sh instead. +# -> it names the ODP NIC for RDMA_IB_DEVICES and the RoCEv2 index for MC_GID_INDEX, +# the latter being per-NODE rather than cluster-wide. +# +# IMAGE= DUMP_PATH= srun -N2 --ntasks-per-node=1 \ +# bash preflight_rdma.sh fabric +# -> cross-node bandwidth over rdma AND tcp. If the two are close, RDMA is not +# actually carrying the traffic and everything below measures the fallback path. + # --------------------------------------------------------------------------- # 1. Nodes # --------------------------------------------------------------------------- # Each node's SSH-reachable name and its DATA-PLANE IP. Do NOT use the # management/public NIC — the legs advertise these to each other. If ssh to # compute nodes is blocked, set SSH_CMD (see cluster/README.md). +# +# These are CONTROL-plane addresses and must be routable BETWEEN the nodes; a +# point-to-point /31 link address is not. The KV payload does not ride them — it goes +# over the RDMA device named in §3. export PREFILL_NODE="" export DECODE_NODE="" export PREFILL_IP="" @@ -54,14 +78,53 @@ export KIT_DIR="$KIT" # 2. Image and weights # --------------------------------------------------------------------------- # Requires an infera-sglang build NEWER than 0.2.0. The default below is a placeholder. +# The main Infera README names `rocm/infera:sglang-v0.1.1` — that one is too old for this +# kit, so take the newest `sglang-v0.2.x` tag published on Docker Hub instead. export INFERA_IMAGE="${INFERA_IMAGE:-}" +# Development source overlay. When both nodes share a filesystem, and because the engine +# image's WORKDIR is /opt/infera, mounting this checkout there makes every +# `python -m infera...` invocation import the current working tree. common.sh verifies the +# mount actually took and refuses to continue if the container imported anything else. +# Set INFERA_SRC= to disable the overlay. +# export INFERA_SRC="${INFERA_SRC-$(cd "$KIT/../.." && pwd)}" + +# Shared writable trace directory, required by `capture`. common.sh bind-mounts this exact +# absolute path into both engine containers, so the torch traces land on shared storage +# with no tar/docker-cp/SSH fetch stage. capture.sh checks the mount before it profiles. +export TRACE_OUT="${TRACE_OUT:-$KIT/profiles}" + # MODEL_MOUNT is bind-mounted into the container; MODEL must live under it. -# Shared storage works but loads slowly — ~400 GB read by both legs at once. +# Shared storage works but loads slowly — ~400 GB over ~280 shards, read by both legs at +# once, so the first bring-up is minutes of silence rather than a hang. That is what +# leg.sh's INFERA_SGLANG_READY_TIMEOUT (3600s) is absorbing; if it still trips, stage the +# weights onto local NVMe per node and repoint this. export MODEL_MOUNT="" export MODEL="$MODEL_MOUNT/GLM-5.2-MXFP4" export TOKENIZER="$MODEL" +# If your image injects a host RDMA provider library at entrypoint, set all three: the +# host library (the SYMLINK, so differing per-node builds resolve), the in-container path +# THAT image reads, and the entrypoint. Any one alone silently does nothing. +# +# Why this exists: the image's SGLang layer ships one build of the userspace provider, +# and a host whose kernel module is newer speaks a later ABI. A mismatch does NOT error — +# ibv_get_device_list simply returns ZERO devices, mooncake finds no RDMA and falls back +# to TCP at 5-20x the latency, and the deployment looks healthy the whole time. In this +# mode that also costs you dma-buf itself, which is the only reason to be on wrapper B. +# +# What each of the three does, and how it fails alone: +# HOST_RDMA_LIB the host-side SYMLINK, not the resolved .so.1.x.y.z — per-node builds +# differ, and the symlink is what makes one line work on both nodes +# HOST_RDMA_MOUNT the exact in-container path THAT image's entrypoint reads. Point it +# elsewhere and the bind-mount lands where nothing looks: a silent no-op +# ENTRYPOINT_KEEP common.sh otherwise starts the container with --entrypoint '', which +# skips the injection and leaves the stale provider in place +# The entrypoint is a pass-through when the host path is absent, so leaving this on is safe. +# `docker exec $CTR ibv_devinfo` inside a running container is the check that it worked. +# export HOST_RDMA_LIB=/usr/lib/x86_64-linux-gnu/lib.so +# export HOST_RDMA_MOUNT=/host-/lib.so ENTRYPOINT_KEEP=1 + # --------------------------------------------------------------------------- # 3. Transport — mode B (dma-buf on the ODP NIC) # --------------------------------------------------------------------------- @@ -107,4 +170,38 @@ export DECODE_KVD=0 # off on decode by design export GMU_PREFILL=0.70 export GMU_DECODE=0.85 +# --------------------------------------------------------------------------- +# 5. Trace replay — optional, only read by engine/trace_replay.sh +# --------------------------------------------------------------------------- +# AIPerf replays a Mooncake-format production trace at the timestamps it recorded, which is +# the one load in this kit that carries a real shared prefix. Leave AIPERF_TRACE unset and +# nothing here does anything. +# +# The client CANNOT live in the engine container: that image ships Python 3.10 and AIPerf +# requires >= 3.11. It runs from the published NGC image instead — 255 MB, so pulling it is +# not the concern that pulling an engine image is. +export AIPERF_IMAGE="${AIPERF_IMAGE:-nvcr.io/nvidia/ai-dynamo/aiperf:0.12.0}" + +# The trace, on a path BOTH this host and $AIPERF_NODE can read. A few MB each, so there is no +# need to clone the repo: +# B=https://raw.githubusercontent.com/kvcache-ai/Mooncake/main/FAST25-release/traces +# curl -LO $B/conversation_trace.jsonl # 12031 reqs, avg ISL 12035, avg OSL 343 +# curl -LO $B/toolagent_trace.jsonl # 23608 reqs, avg ISL 8596, avg OSL 182 +# conversation is the default choice; toolagent is closer to an agentic workload. Upstream's +# third file, synthetic_trace.jsonl, has GENERATED (Poisson) arrival times rather than recorded +# ones, which makes it the wrong input for a fixed-schedule replay of real traffic. +# export AIPERF_TRACE="/conversation_trace.jsonl" + +# Which node generates the load. Defaults to the prefill node, which is the simple choice but +# not the neutral one: AIPerf synthesizes and tokenizes every prompt before sending anything, +# and that competes for CPU with the engine's own scheduler and tokenizer processes. Point it +# at any node that can route to $PREFILL_IP:$ROUTER_PORT to remove that interference. +# export AIPERF_NODE="" + +# Artifacts, the generated per-run command file, and the mmap dataset cache. Must be the same +# path on both hosts — trace_replay.sh checks this rather than letting it fail obscurely later. +# Alongside profiles/, never inside it: that one is the torch-trace mount common.sh binds +# into both engine containers, and mixing the two makes neither directory mean one thing. +export AIPERF_OUT="${AIPERF_OUT:-$KIT/aiperf}" + exec bash "$KIT/engine/${1:-up}.sh" "${@:2}" diff --git a/examples/sglang_1p1d_glm5.2/cluster/cluster.peermem.sh b/examples/sglang_1p1d_glm5.2/cluster/cluster.peermem.sh index 00323a10b..f01ffcd00 100755 --- a/examples/sglang_1p1d_glm5.2/cluster/cluster.peermem.sh +++ b/examples/sglang_1p1d_glm5.2/cluster/cluster.peermem.sh @@ -21,16 +21,44 @@ # If preflight instead reports "no peer-mem module loaded", use # cluster.dmabuf.sh. See cluster/README.md for the full mapping. # -# Usage: bash cluster/cluster.peermem.sh up | smoke | bench [conc...] | trace_replay | down +# Usage: bash cluster/cluster.peermem.sh up | smoke | lm_eval | bench [conc...] | +# trace_replay [prepare|run] | capture | down set -euo pipefail KIT="$(cd "$(dirname "${BASH_SOURCE[0]}")/.." && pwd)" +# --------------------------------------------------------------------------- +# 0. Before the first bring-up — READ THIS +# --------------------------------------------------------------------------- +# Every value in §2 and §3 is a MEASUREMENT, not a preference, and each one fails +# SILENTLY when it is wrong: the wrong registration mode duplicates the KV pool or drops +# to TCP at 5-20x the latency, and the stack keeps serving tokens either way. Picking +# this wrapper because mode A sounds better, rather than because preflight said so, is +# the most expensive way to fill this file in. +# +# Run both of these ON BOTH NODES first, and reconcile §2/§3 with what they print: +# +# IMAGE= bash preflight_rdma.sh mode +# -> "peermem: present" + "mode A: VIABLE" confirms this wrapper is the right one. +# If it says "no peer-mem module loaded", switch to cluster/cluster.dmabuf.sh. +# -> the device list is the ground truth for RDMA_IB_DEVICES. +# -> the RoCEv2 GID index is the ground truth for MC_GID_INDEX, which is per-NODE. +# +# IMAGE= DUMP_PATH= srun -N2 --ntasks-per-node=1 \ +# bash preflight_rdma.sh fabric +# -> cross-node bandwidth over rdma AND tcp. If the two are close, RDMA is not +# actually carrying the traffic and everything below measures the fallback path. + # --------------------------------------------------------------------------- # 1. Nodes # --------------------------------------------------------------------------- # Each node's SSH-reachable name and its DATA-PLANE IP. Do NOT use the # management/public NIC — the legs advertise these to each other for the KV # handoff. See cluster/README.md section 1. +# +# These are the CONTROL-plane addresses: the legs advertise them to each other and the +# router listens on one of them, so they must be routable BETWEEN the nodes. A +# point-to-point /31 link address is not. The KV payload does not ride these at all — +# it goes over the RDMA rails named in §3. export PREFILL_NODE="" export DECODE_NODE="" export PREFILL_IP="" @@ -43,6 +71,8 @@ export KIT_DIR="$KIT" # 2. Image and weights # --------------------------------------------------------------------------- # Requires an infera-sglang build NEWER than 0.2.0. The default below is a placeholder. +# The main Infera README names `rocm/infera:sglang-v0.1.1` — that one is too old for this +# kit, so take the newest `sglang-v0.2.x` tag published on Docker Hub instead. export INFERA_IMAGE="${INFERA_IMAGE:-}" # Development source overlay. /apps is shared by both nodes, and the engine image's @@ -56,15 +86,34 @@ export TRACE_OUT="${TRACE_OUT:-$KIT/profiles}" # MODEL_MOUNT is bind-mounted into the container; MODEL must live under it. # Prefer LOCAL storage on both nodes — a slow mount blows the ready timeout -# (see cluster/README.md section 2). +# (see cluster/README.md section 2). GLM-5.2-MXFP4 is ~400 GB over ~280 shards and BOTH +# legs read all of it at once, so on shared storage the first load is minutes of silence; +# that is what leg.sh's INFERA_SGLANG_READY_TIMEOUT (3600s) is absorbing. If it still +# trips, stage the weights onto local NVMe per node and repoint this. export MODEL_MOUNT="" export MODEL="$MODEL_MOUNT/GLM-5.2-MXFP4" export TOKENIZER="$MODEL" # If your image injects a host RDMA provider library at entrypoint, set all three: the # host library (the SYMLINK, so differing per-node builds resolve), the in-container path -# THAT image reads, and the entrypoint. Any one alone silently does nothing — see §2. -# export HOST_RDMA_LIB=/usr/lib/x86_64-linux-gnu/lib.so.1 +# THAT image reads, and the entrypoint. Any one alone silently does nothing. +# +# Why this exists: the image's SGLang layer ships one build of the userspace provider, +# and a host whose kernel module is newer speaks a later ABI. A mismatch does NOT error — +# ibv_get_device_list simply returns ZERO devices, mooncake finds no RDMA and falls back +# to TCP at 5-20x the latency, and the deployment looks healthy the whole time. The +# injection copies the host build over the container's at start. +# +# What each of the three does, and how it fails alone: +# HOST_RDMA_LIB the host-side SYMLINK, not the resolved .so.1.x.y.z — per-node builds +# differ, and the symlink is what makes one line work on both nodes +# HOST_RDMA_MOUNT the exact in-container path THAT image's entrypoint reads. Point it +# elsewhere and the bind-mount lands where nothing looks: a silent no-op +# ENTRYPOINT_KEEP common.sh otherwise starts the container with --entrypoint '', which +# skips the injection and leaves the stale provider in place +# The entrypoint is a pass-through when the host path is absent, so leaving this on is safe. +# `docker exec $CTR ibv_devinfo` inside a running container is the check that it worked. +# export HOST_RDMA_LIB=/usr/lib/x86_64-linux-gnu/lib.so # export HOST_RDMA_MOUNT=/host-/lib.so ENTRYPOINT_KEEP=1 # --------------------------------------------------------------------------- @@ -137,6 +186,8 @@ export AIPERF_IMAGE="${AIPERF_IMAGE:-nvcr.io/nvidia/ai-dynamo/aiperf:0.12.0}" # Artifacts, the generated per-run command file, and the mmap dataset cache. Must be the same # path on both hosts — trace_replay.sh checks this rather than letting it fail obscurely later. +# Alongside profiles/, never inside it: that one is the torch-trace mount common.sh binds +# into both engine containers, and mixing the two makes neither directory mean one thing. export AIPERF_OUT="${AIPERF_OUT:-$KIT/aiperf}" exec bash "$KIT/engine/${1:-up}.sh" "${@:2}" diff --git a/examples/sglang_1p1d_glm5.2/engine/aiperf_compat/sitecustomize.py b/examples/sglang_1p1d_glm5.2/engine/aiperf_compat/sitecustomize.py index 5901840e4..65634fd51 100644 --- a/examples/sglang_1p1d_glm5.2/engine/aiperf_compat/sitecustomize.py +++ b/examples/sglang_1p1d_glm5.2/engine/aiperf_compat/sitecustomize.py @@ -4,7 +4,6 @@ from aiperf.common.tokenizer import Tokenizer # type: ignore[import-not-found] - _original_resolve_local_snapshot = Tokenizer._resolve_local_snapshot.__func__ diff --git a/examples/sglang_1p1d_glm5.2/engine/lm_eval.sh b/examples/sglang_1p1d_glm5.2/engine/lm_eval.sh new file mode 100755 index 000000000..8ff6203e8 --- /dev/null +++ b/examples/sglang_1p1d_glm5.2/engine/lm_eval.sh @@ -0,0 +1,176 @@ +#!/usr/bin/env bash +# Copyright (c) 2026, Advanced Micro Devices, Inc. All rights reserved. +# SPDX-License-Identifier: MIT +# what: check that this deployment serves the model CORRECTLY, not just quickly. +# why : smoke.sh proves the features are on and bench.sh proves the stack is fast; neither +# reads a single answer. MXFP4 weights, an fp8_e4m3 KV cache, sparse attention on ROCm, +# KV moved by mooncake, EAGLE speculation and a radix cache shared between requests are +# six independent ways to serve fluent, fast, WRONG tokens — and all six look green in +# smoke and bench. +# how : DO NOT run this directly — run it through cluster/.sh: +# bash cluster/.sh lm_eval probe # ~40 s, gates the rest +# bash cluster/.sh lm_eval quick # ~2 min, GSM8K 200 +# bash cluster/.sh lm_eval full # ~13 min, the full set twice +# bash cluster/.sh lm_eval # probe, then quick +# +# The evaluator is sglang.test.run_eval, which ships inside the engine image (sglang 0.5.17) +# and talks OpenAI /v1/chat/completions, so nothing is installed into $CTR. Read "Accuracy" +# in the README before quoting a number from here: it says what a score can and cannot show. +# +# Knobs (all optional): +# NUM_EXAMPLES questions per eval. quick defaults to 200, full to everything +# THREADS=32 concurrency, and an accuracy variable too — a score that holds at 1 and +# drops at 256 indicts batching rather than the model +# MAX_TOKENS generation cap, default 2048 (GLM-5.2 bills reasoning against it) +# TEMP=0.0 greedy, and NOT byte-reproducible here — see tools/probe_accuracy.py +# TOP_P=1.0 +# REPEAT=1 run each eval N times and report the spread +# FULL_EVALS what `full` runs. Default: gsm8k mixed_prefix_gsm8k +# +# Three of run_eval's eleven evals cannot run here: humaneval (no human_eval package in the +# image), math (grades with a second model, needs an OpenAI key) and mmmu (vision). gsm8k, +# mixed_prefix_gsm8k, mmlu, gpqa, aime25 and mgsm_en were each measured against this image; +# mgsm and longbench_v2 were not tried. gpqa and aime25 need long chains of thought and score +# near chance at the 2048 default, so raise MAX_TOKENS a long way before believing them. +set -uo pipefail +DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"; source "$DIR/../common.sh" + +require_env PREFILL_NODE; require_env PREFILL_IP + +SSH_CMD="${SSH_CMD:-ssh -o StrictHostKeyChecking=no}" +on(){ local h="$1"; shift; $SSH_CMD "$h" "$*" /dev/null \ + || die "could not copy $1 into $CTR on $PREFILL_NODE" +} + +ensure_dataset(){ + mkdir -p "$DATA_DIR" || die "could not create $DATA_DIR" + if [ ! -s "$DATA_DIR/test.jsonl" ]; then + log "fetching the GSM8K test split -> $DATA_DIR/test.jsonl" + curl -sSfL -o "$DATA_DIR/test.jsonl" "$GSM8K_URL" \ + || die "could not download GSM8K. Place the file at $DATA_DIR/test.jsonl by hand." + fi + # A short file is a truncated download, which would shrink every gsm8k run and move its score. + local n; n="$(wc -l < "$DATA_DIR/test.jsonl")" + [ "$n" -eq 1319 ] || warn "GSM8K test split has $n rows, expected 1319 — scores will not be comparable" + stage "$DATA_DIR/test.jsonl" "/tmp/gsm8k_test.jsonl" +} + +# ---- probe ----------------------------------------------------------------------------------- +probe(){ + log "=== probe: is a score from this deployment interpretable? ===" + log " ~30 requests, about a minute. Reads answers, not just status codes." + stage "$DIR/tools/probe_accuracy.py" "/tmp/probe_accuracy.py" + on "$PREFILL_NODE" "docker exec $CTR python3 /tmp/probe_accuracy.py \ + --url '$URL' --model '$SERVED' \ + --max-tokens $MAX_TOKENS --temperature $TEMP --top-p $TOP_P" +} + +# ---- one eval --------------------------------------------------------------------------------- +# Questions scored, counted out of run_eval's html report. The interval needs this number and +# the result json does not carry it. Counted rather than assumed because the dataset size is +# the wrong answer: GSM8K ships 1319 rows but gsm8k scores 1314 and mixed_prefix_gsm8k 1299 — +# GSM8KEval slices the few-shot examples off the evaluation set so they cannot leak into it. +# Nor is it a success count: a request that fails past its retries is scored 0 and stays in +# the report. `Correct Answer` is HTML_JINJA's, shared by every eval used here. +count_scored(){ grep -c 'Correct Answer' "$1" 2>/dev/null; } + +# run_eval writes /tmp/_.{json,html} inside $CTR. Both come out: the json is the +# number, the html is every prompt and completion — the only place to see WHY a score moved. +run_one(){ + local name="$1" limit="${NUM_EXAMPLES:-}" n_arg="" data_arg="" scored + local json="/tmp/${name}_${SERVED//\//_}.json" + local html="/tmp/${name}_${SERVED//\//_}.html" + [ -n "$limit" ] && n_arg="--num-examples $limit" + # Both gsm8k variants read the staged file; the others fetch their own. + case "$name" in gsm8k|mixed_prefix_gsm8k) data_arg="--gsm8k-data-path /tmp/gsm8k_test.jsonl" ;; esac + + log "--- $name (${limit:-all} questions, $THREADS concurrent, repeat=$REPEAT) ---" + # Otherwise a run that dies before scoring leaves the previous run's files to be copied out. + on "$PREFILL_NODE" "docker exec $CTR rm -f '$json' '$html'" >/dev/null 2>&1 + # The noise filter runs HERE: piping into grep remotely puts the pipeline in a shell without + # pipefail, so grep's exit code masks a crashed run_eval. Locally, `set -o pipefail` shows it. + on "$PREFILL_NODE" "docker exec $CTR python3 -m sglang.test.run_eval \ + --base-url '$URL' --model '$SERVED' --eval-name $name \ + $n_arg --num-threads $THREADS --repeat $REPEAT \ + --max-tokens $MAX_TOKENS --temperature $TEMP --top-p $TOP_P \ + $THINKING $data_arg 2>&1" \ + | grep --line-buffered -vE '^\[aiter\]|[0-9]+%\|' \ + || { warn " $name failed — see its output above; continuing with the remaining evals" + return 1; } + + mkdir -p "$RUN_DIR" + on "$PREFILL_NODE" "docker cp '$CTR:$json' '$RUN_DIR/'" >/dev/null 2>&1 \ + || { warn " $name wrote no result file — it exited before scoring"; return 1; } + on "$PREFILL_NODE" "docker cp '$CTR:$html' '$RUN_DIR/'" >/dev/null 2>&1 || true + scored="$(count_scored "$RUN_DIR/$(basename "$html")")" + printf '%s\t%s\t%s\n' "$name" "${scored:-}" "$(basename "$json")" >> "$RUN_DIR/.index" +} + +# ---- subcommands ------------------------------------------------------------------------------- +# A separate script rather than an inlined heredoc, unlike the small python fragments elsewhere +# in this kit: it runs on THIS host, and keeping it addressable means any past result directory +# can be re-summarised, or two of them compared, without re-running a 13-minute eval: +# python3 engine/tools/summarise_eval.py lm_eval/ lm_eval/ +summarise(){ + [ -d "$RUN_DIR" ] || { warn "no results to summarise"; return 1; } + python3 "$DIR/tools/summarise_eval.py" "$RUN_DIR" +} + +run_set(){ + local label="$1"; shift + ensure_dataset + log "=== $label: $* (${NUM_EXAMPLES:-all} questions each) ===" + [ -n "${NUM_EXAMPLES:-}" ] || log " the full GSM8K set is ~1300 questions — minutes, not a hang" + for name in "$@"; do run_one "$name"; done + log "=== $label results ===" + summarise + log "artifacts -> $RUN_DIR (the .html files hold every prompt and completion)" +} + +quick(){ NUM_EXAMPLES="${NUM_EXAMPLES:-200}"; run_set quick gsm8k; } + +case "${1:-all}" in + probe) probe ;; + quick) quick ;; + full) shift; [ $# -gt 0 ] && FULL_EVALS="$*"; run_set full $FULL_EVALS ;; + # probe gates quick: the failures it catches produce a NUMBER rather than an error, and that + # number is indistinguishable from a real regression. Two statements rather than + # `probe && quick || warn`, which would report a failing quick as a failing probe. + all) probe || { warn "probe failed — not spending minutes on a score that cannot be read"; exit 1; } + quick ;; + *) die "unknown subcommand '$1'. Use: probe | quick | full [eval ...]" ;; +esac diff --git a/examples/sglang_1p1d_glm5.2/engine/tools/probe_accuracy.py b/examples/sglang_1p1d_glm5.2/engine/tools/probe_accuracy.py new file mode 100755 index 000000000..be872a379 --- /dev/null +++ b/examples/sglang_1p1d_glm5.2/engine/tools/probe_accuracy.py @@ -0,0 +1,342 @@ +#!/usr/bin/env python3 +# Copyright (c) 2026, Advanced Micro Devices, Inc. All rights reserved. +# +# SPDX-License-Identifier: MIT +"""Decide whether a benchmark score from this deployment would mean anything, before paying +for one. + +A GSM8K run against this stack costs tens of minutes and can come back at 0.00 for reasons +that have nothing to do with accuracy -- an empty ``content`` field, a router serving with +one leg registered. It can equally come back at a plausible 0.85 while prefix caching hands +one request another request's KV. Neither is visible in the score. + +So this sends ~30 requests and reports the four things that separate an interpretable score +from a decorative one: + + reachability router healthy and BOTH legs registered + answerability does ``content`` arrive, or does reasoning eat the whole budget + stability the same question, asked repeatedly, must yield the same ANSWER + isolation a shared prefix must not change the answer to the question behind it + +WHY STABILITY IS CHECKED ON THE ANSWER AND NOT ON THE BYTES. This deployment is not +byte-deterministic at ``temperature=0``, and that is expected rather than broken. Measured +here, ten serial repeats of one GSM8K question all returned 72 while their completion +lengths ranged over 214-289 tokens: MTP verification, dp8 attention and batch-dependent +kernel selection each perturb the numerics without changing the argmax that matters. A +byte-for-byte check would fail on every healthy run, which is worse than no check at all. + +Isolation is the failure mode nothing else in this kit can see. ``bench.sh`` builds every +prompt independently, so a radix-cache or kvd lookup returning the wrong block is never +exercised there. + +Usage: + python3 probe_accuracy.py --url http://: --model [options] + +Exit status is 0 when every HARD check passes, 1 otherwise, so it can gate a script. Soft +findings (unstable completion lengths, some empty content when the budget is genuinely +tight) are reported loudly but do NOT fail. +""" + +from __future__ import annotations + +import argparse +import collections +import json +import re +import sys +import urllib.error +import urllib.request +from concurrent.futures import ThreadPoolExecutor + +# Questions whose answer is not in dispute. The point is NOT to measure ability -- a model +# that gets these wrong is not a weaker model, it is a broken deployment. Keeping them +# trivial is what makes a failure unambiguous; a hard question coming back wrong would leave +# "the model is not that good" as a live explanation. +QUESTIONS = [ + ( + "natalia", + "Natalia sold clips to 48 of her friends in April, and then she sold half as many " + "clips in May. How many clips did Natalia sell altogether in April and May?", + "72", + ), + ("multiply", "What is 17 multiplied by 23? Give the number.", "391"), + ( + "speed", + "A train travels 60 miles in 1.5 hours. What is its average speed in miles per hour?", + "40", + ), +] + +# Long enough to span many KV blocks (block size is 64 here), repetitive enough that the +# radix cache will certainly retain it, and semantically inert so that any change in the +# answer is attributable to the cache rather than to the content. +SHARED_PREFIX = ( + "You are a careful assistant. " + + "Background fact number 7 is irrelevant to any question. " * 300 +) + +RED, GRN, YEL, NC = "\033[0;31m", "\033[0;32m", "\033[0;33m", "\033[0m" + + +def mark(ok: bool) -> str: + return f"{GRN}ok{NC}" if ok else f"{RED}FAIL{NC}" + + +class Findings: + """Hard findings fail the probe; soft ones are observations about the deployment.""" + + def __init__(self) -> None: + self.hard: list[str] = [] + self.soft: list[str] = [] + + def fail(self, msg: str) -> None: + self.hard.append(msg) + + def note(self, msg: str) -> None: + self.soft.append(msg) + + +def post(url: str, model: str, prompt: str, args) -> dict: + body = { + "model": model, + "messages": [{"role": "user", "content": prompt}], + "max_tokens": args.max_tokens, + "temperature": args.temperature, + "top_p": args.top_p, + } + req = urllib.request.Request( + url + "/v1/chat/completions", + data=json.dumps(body).encode(), + headers={"Content-Type": "application/json"}, + ) + with urllib.request.urlopen(req, timeout=args.timeout) as fh: + d = json.load(fh) + choice = d["choices"][0] + msg, usage = choice["message"], (d.get("usage") or {}) + # Both fields are None rather than "" when the model produces neither, and `or ""` is the + # difference between a report and a TypeError. + return { + "content": msg.get("content") or "", + "reasoning": msg.get("reasoning_content") or "", + "finish": choice.get("finish_reason"), + "ctok": usage.get("completion_tokens"), + "ptok": usage.get("prompt_tokens"), + } + + +def last_number(text: str) -> str | None: + """The scorer SGLang's own gsm8k eval uses: the last number in the text.""" + found = re.findall(r"-?\d[\d,]*\.?\d*", text.replace(",", "")) + return found[-1].rstrip(".") if found else None + + +def answer_of(resp: dict) -> str | None: + """Read the answer from ``content``, falling back to ``reasoning_content``. + + The fallback is not leniency. With --reasoning-parser glm45 the chain of thought is a + separate field, and a reply that ran out of budget mid-thought has the answer there and + nothing in `content`. Refusing to look would report a budget problem as a wrong answer. + """ + return last_number(resp["content"]) or last_number(resp["reasoning"]) + + +def worst_ngram_repeat(text: str, n: int = 8) -> tuple[int, str]: + """Count of the most-repeated n-gram, i.e. how degenerate the output looks. + + Degeneration is what README note 5 looks like from the client side: the DSA-on-ROCm env + block not taking effect produces repeated tokens rather than an error. An 8-gram is long + enough that ordinary prose does not repeat one. + """ + words = text.split() + if len(words) < n: + return 0, "" + grams = collections.Counter(tuple(words[i : i + n]) for i in range(len(words) - n + 1)) + gram, count = grams.most_common(1)[0] + return count, " ".join(gram) + + +def check_workers(url: str, f: Findings) -> bool: + """Both legs registered. A router with one leg still answers and still produces a score, + but that score describes a different deployment and nothing downstream would say so.""" + print("===== 1. router and workers =====") + try: + with urllib.request.urlopen(url + "/v1/workers", timeout=30) as fh: + workers = json.load(fh).get("workers", []) + except Exception as exc: # noqa: BLE001 + f.fail(f"could not read {url}/v1/workers: {exc}") + print(f" {RED}unreachable: {exc}{NC}") + return False + roles = collections.Counter(w.get("disagg_mode") for w in workers) + for role in ("prefill", "decode"): + n = roles.get(role, 0) + print(f" {role:<8} workers registered: {n}") + if n < 1: + f.fail(f"no {role} worker registered") + print(f" model names: {sorted({w.get('model_name') for w in workers})}") + return True + + +def check_answers(ask, args, f: Findings) -> None: + print("\n===== 2. answers: correct, and the same every time =====") + print( + f" (temperature={args.temperature} max_tokens={args.max_tokens} " + f"repeat={args.repeat} per question)" + ) + empty = capped = total = 0 + for key, prompt, expect in QUESTIONS: + answers, ctoks, worst_rep, worst_gram = [], [], 0, "" + for _ in range(args.repeat): + try: + r = ask(prompt) + except Exception as exc: # noqa: BLE001 + f.fail(f"request failed on '{key}': {exc}") + print(f" {key:<10} {RED}request failed: {exc}{NC}") + break + total += 1 + answers.append(answer_of(r)) + ctoks.append(r["ctok"]) + empty += not r["content"].strip() + capped += r["finish"] == "length" + rep, gram = worst_ngram_repeat(r["reasoning"] + " " + r["content"]) + if rep > worst_rep: + worst_rep, worst_gram = rep, gram + if not answers: + continue + + dist = collections.Counter(answers) + correct = all(a == expect for a in answers) + print( + f" {key:<10} expect={expect:<5} got={dict(dist)!s:<28} " + f"ctok={min(ctoks)}-{max(ctoks):<6} rep8={worst_rep:<4} " + f"{mark(correct and len(dist) == 1)}" + ) + if not correct: + f.fail(f"'{key}' answered {dict(dist)}, expected {expect}") + elif len(dist) > 1: + f.fail(f"'{key}' gave {len(dist)} different answers: {dict(dist)}") + # Soft: a model asked for a table or a list can legitimately repeat a stem, and this + # probe should not fail on style. + if worst_rep >= 4: + f.note(f"'{key}' repeated an 8-gram {worst_rep}x: {worst_gram[:60]!r}") + # Soft by design: byte-instability at temperature 0 is this deployment's normal state + # (see the module docstring). Recorded so a later run can see whether it changed. + if len(set(ctoks)) > 1: + f.note( + f"'{key}' completion length varied {min(ctoks)}-{max(ctoks)} tokens " + f"at temperature {args.temperature}" + ) + + if not total: + return + pct_empty, pct_capped = 100.0 * empty / total, 100.0 * capped / total + print( + f" empty content: {empty}/{total} ({pct_empty:.0f}%) " + f"finish_reason=length: {capped}/{total} ({pct_capped:.0f}%)" + ) + # Empty content plus finish_reason=length is reasoning eating the whole budget, which + # scores 0.00 on every eval while the deployment is healthy. Empty content while stopping + # normally is a different bug. Distinguished here so the fix is obvious. + if pct_empty <= 20.0: + return + if pct_capped > 20.0: + f.fail( + f"{pct_empty:.0f}% of replies had empty content and {pct_capped:.0f}% hit the " + f"token cap — raise --max-tokens above {args.max_tokens} before reading a score" + ) + else: + f.fail( + f"{pct_empty:.0f}% of replies had empty content while stopping normally — " + f"the reasoning parser or chat template is wrong" + ) + + +def check_batch(ask, args, f: Findings) -> None: + """Serial repeats share little with a real eval, which sends dozens of requests at once. + Batch composition changes kernel selection, DP rank assignment and MTP verification, so a + batched answer disagreeing with the serial one points at batching, not at the model.""" + print(f"\n===== 3. same question, {args.concurrency} at once =====") + key, prompt, expect = QUESTIONS[0] + try: + with ThreadPoolExecutor(max_workers=args.concurrency) as pool: + batch = list(pool.map(lambda _: ask(prompt), range(args.concurrency))) + except Exception as exc: # noqa: BLE001 + f.fail(f"batched request failed: {exc}") + print(f" {RED}failed: {exc}{NC}") + return + dist = collections.Counter(answer_of(r) for r in batch) + ok = len(dist) == 1 and next(iter(dist)) == expect + print(f" {key:<10} expect={expect:<5} got={dict(dist)} {mark(ok)}") + if not ok: + f.fail( + f"under concurrency {args.concurrency}, '{key}' gave {dict(dist)}, " + f"expected all {expect}" + ) + + +def check_prefix(ask, f: Findings) -> None: + print("\n===== 4. shared prefix must not change the answer =====") + print(" (the failure this catches is invisible to bench.sh — no shared prefix there)") + for key, prompt, expect in QUESTIONS: + try: + alone, behind = ask(prompt), ask(SHARED_PREFIX + "\n\n" + prompt) + except Exception as exc: # noqa: BLE001 + f.fail(f"prefix check failed on '{key}': {exc}") + print(f" {key:<10} {RED}request failed: {exc}{NC}") + continue + a1, a2 = answer_of(alone), answer_of(behind) + print( + f" {key:<10} standalone={str(a1):<6} behind-prefix={str(a2):<6} " + f"ptok {alone['ptok']}->{behind['ptok']} {mark(a1 == a2 == expect)}" + ) + if not a1 == a2 == expect: + f.fail( + f"'{key}' answered {a1} standalone but {a2} behind a shared prefix " + f"(expected {expect}) — suspect radix cache / kvd / kv-aware routing" + ) + + +def main() -> int: + ap = argparse.ArgumentParser( + description=__doc__, formatter_class=argparse.RawDescriptionHelpFormatter + ) + ap.add_argument("--url", required=True, help="router base url, e.g. http://10.0.0.1:8100") + ap.add_argument("--model", required=True, help="served model name") + ap.add_argument("--max-tokens", type=int, default=2048) + ap.add_argument("--temperature", type=float, default=0.0) + ap.add_argument("--top-p", type=float, default=1.0) + ap.add_argument("--repeat", type=int, default=4, help="repeats per question") + ap.add_argument("--concurrency", type=int, default=8, help="requests in the batched check") + ap.add_argument("--timeout", type=int, default=600) + args = ap.parse_args() + + f = Findings() + if not check_workers(args.url, f): + # Everything below goes through that router; there is nothing to learn by asking + # thirty more times. + print(f"\n{RED}PROBE FAILED{NC} — router unreachable") + return 1 + + ask = lambda prompt: post(args.url, args.model, prompt, args) + check_answers(ask, args, f) + check_batch(ask, args, f) + check_prefix(ask, f) + + print("\n===== verdict =====") + for s in f.soft: + print(f" {YEL}note{NC} {s}") + if not f.soft: + print(" (no soft findings)") + for h in f.hard: + print(f" {RED}FAIL{NC} {h}") + if f.hard: + print(f"\n{RED}PROBE FAILED{NC} — a score measured now would not be interpretable") + return 1 + print(f"\n{GRN}PROBE PASSED{NC} — scores from this deployment are worth measuring") + return 0 + + +if __name__ == "__main__": + try: + sys.exit(main()) + except KeyboardInterrupt: + sys.exit(130) diff --git a/examples/sglang_1p1d_glm5.2/engine/tools/summarise_eval.py b/examples/sglang_1p1d_glm5.2/engine/tools/summarise_eval.py new file mode 100755 index 000000000..9428d6bd9 --- /dev/null +++ b/examples/sglang_1p1d_glm5.2/engine/tools/summarise_eval.py @@ -0,0 +1,257 @@ +#!/usr/bin/env python3 +# Copyright (c) 2026, Advanced Micro Devices, Inc. All rights reserved. +# +# SPDX-License-Identifier: MIT +"""Turn eval result directories into numbers that can actually be compared. + +``sglang.test.run_eval`` writes a score and nothing that bounds it. A bare 0.945 invites +exactly one question -- "is that better or worse than last time" -- and cannot answer it, +because the honest answer depends on how many questions produced it. At 200 questions a +two-point move is noise; at 1300 it is not. So every score here carries its 95% interval, +and every comparison carries the interval of the DIFFERENCE. + + summarise_eval.py one run: scores, intervals, prefix-reuse delta + summarise_eval.py two runs: per-eval delta with its own interval + summarise_eval.py --gate the same, exiting 1 if some eval dropped + +Comparing is the mode that answers the question that comes up. Nobody knows what +GLM-5.2-MXFP4 with an fp8 KV cache "should" score, so an absolute number has no external +baseline -- but a config change, a leg restart or a new image can always be measured +against the same deployment's own earlier run. + +WHY A DIFFERENCE GETS ITS OWN INTERVAL RATHER THAN A COMPARISON OF TWO INTERVALS. +"Do the two intervals overlap" is the intuitive test and it is too conservative: the +uncertainty on a difference is sqrt(se_a^2 + se_b^2), not se_a + se_b, so two runs can +overlap visibly and still differ significantly. Reading overlap hides real regressions. +The difference is estimated directly instead, and the question becomes whether its +interval contains zero. + +Input is the ``.index`` that engine/lm_eval.sh writes beside the results +(``\\t\\t``); the question count lives there because +run_eval's own json does not record it. +""" + +from __future__ import annotations + +import argparse +import json +import math +import os +import sys +from typing import NamedTuple + +# Named because it appears in three places, and a stray 1.65 in one of them would silently +# turn a 95% statement into a 90% one. +Z95 = 1.96 + +RED, GRN, YEL, NC = "\033[0;31m", "\033[0;32m", "\033[0;33m", "\033[0m" + + +class Result(NamedTuple): + name: str + score: float + # Questions scored, counted by lm_eval.sh out of the html report. run_eval divides by + # this, so the interval must too. Smaller than the dataset: GSM8K ships 1319 rows and + # `gsm8k` scores 1314, the few-shot examples being sliced off the evaluation set. + n: int | None + latency: float | None + repeats: list | None + + @property + def interval(self) -> tuple[float, float] | None: + return wilson(self.score, self.n) if self.n else None + + +def wilson(p: float, n: int) -> tuple[float, float]: + """95% Wilson score interval for a proportion. + + Not the textbook Wald interval (p +/- z*sqrt(p(1-p)/n)): that one collapses to zero + width at p=0 and p=1, so a 16-question run that happens to score 1.000 reports + [1.000, 1.000] -- an assertion of certainty drawn from sixteen questions. Wilson stays + finite there (the same run reports [0.806, 1.000]) and is better behaved at the high + scores this deployment produces. The two agree to a few thousandths by 200 questions. + """ + d = 1.0 + Z95 * Z95 / n + centre = (p + Z95 * Z95 / (2 * n)) / d + half = Z95 / d * math.sqrt(max(p * (1.0 - p), 0.0) / n + Z95 * Z95 / (4 * n * n)) + return max(centre - half, 0.0), min(centre + half, 1.0) + + +def load(run_dir: str) -> dict[str, Result]: + index = os.path.join(run_dir, ".index") + if not os.path.isdir(run_dir): + raise SystemExit(f"not a directory: {run_dir}") + if not os.path.exists(index): + raise SystemExit(f"no .index in {run_dir} — was this written by engine/lm_eval.sh?") + + out: dict[str, Result] = {} + for line in open(index): + # Filename taken from the last column so a stray extra field cannot shift it. + cols = [c for c in line.rstrip("\n").split("\t") if c] + if len(cols) < 3: + continue + name, n_raw, path = cols[0], cols[1], os.path.join(run_dir, cols[-1]) + if not os.path.exists(path): + continue + try: + d = json.load(open(path)) + except Exception: # noqa: BLE001 + continue + score = d.get("score", d.get("mean_score")) + if score is None: + continue + out[name] = Result( + name, + float(score), + int(n_raw) if n_raw.isdigit() else None, + d.get("latency"), + d.get("scores"), + ) + return out + + +def diff_interval(a: Result, b: Result) -> tuple[float, float, float] | None: + """(delta, lo, hi) for b.score - a.score, or None when either n is unknown. + + Newcombe's method, which composes the two Wilson intervals rather than adding variances, + so it inherits their behaviour at the extremes instead of degenerating alongside Wald. + """ + if not a.n or not b.n: + return None + lo_a, hi_a = wilson(a.score, a.n) + lo_b, hi_b = wilson(b.score, b.n) + delta = b.score - a.score + return ( + delta, + delta - math.sqrt((b.score - lo_b) ** 2 + (hi_a - a.score) ** 2), + delta + math.sqrt((hi_b - b.score) ** 2 + (a.score - lo_a) ** 2), + ) + + +def report_one(results: dict[str, Result]) -> int: + print(f" {'eval':<22}{'scored':>7}{'score':>8}{'95% CI':>20}{'latency':>10}") + for r in results.values(): + iv = r.interval + ci = f"[{iv[0]:.3f}, {iv[1]:.3f}]" if iv else "" + lat = f"{r.latency:.0f}s" if isinstance(r.latency, (int, float)) else "" + print(f" {r.name:<22}{r.n or '?':>7}{r.score:>8.3f}{ci:>20}{lat:>10}") + if r.repeats: + print(f" {'':<22}repeats: {r.repeats}") + + # The one comparison this kit exists to make. mixed_prefix_gsm8k asks the SAME questions + # as gsm8k behind partially-shared few-shot prefixes, so it is the only load here that + # puts the radix cache, kvd and kv-aware routing on the correctness path. A gap is prefix + # reuse changing answers, which no throughput number would ever show. + base, mixed = results.get("gsm8k"), results.get("mixed_prefix_gsm8k") + if not (base and mixed): + return 0 + print() + d = diff_interval(base, mixed) + if d is None: + print( + f" prefix-reuse delta (mixed_prefix_gsm8k - gsm8k): " + f"{mixed.score - base.score:+.3f} (no interval: question count unknown)" + ) + return 0 + delta, lo, hi = d + print( + f" prefix-reuse delta (mixed_prefix_gsm8k - gsm8k): {delta:+.3f}" + f" 95% CI [{lo:+.3f}, {hi:+.3f}]" + ) + if lo <= 0.0 <= hi: + print( + f" {GRN}the interval contains zero{NC} — no evidence that prefix reuse changes answers" + ) + else: + print(f" {RED}the interval excludes zero{NC} — shared prefixes ARE changing answers.") + print(" suspect the radix cache, kvd L2/L3 or kv-aware routing, not the model:") + print(" re-run with ROUTER_POLICY=round-robin, then with PREFILL_KVD=0,") + print(" and see which one closes the gap.") + return 0 + + +def report_compare( + a: dict[str, Result], b: dict[str, Result], dir_a: str, dir_b: str, gate: bool +) -> int: + print(f" A {dir_a}\n B {dir_b}\n") + print(f" {'eval':<22}{'A':>8}{'B':>9}{'delta':>9}{'95% CI of delta':>22} verdict") + + shared = [k for k in a if k in b] + regressed = False + for name in shared: + ra, rb = a[name], b[name] + d = diff_interval(ra, rb) + if d is None: + print( + f" {name:<22}{ra.score:>8.3f}{rb.score:>9.3f}" + f"{rb.score - ra.score:>+9.3f}{'(n unknown)':>22} ?" + ) + continue + delta, lo, hi = d + if lo <= 0.0 <= hi: + verdict = f"{GRN}same{NC}" + elif delta < 0: + verdict, regressed = f"{RED}REGRESSED{NC}", True + else: + verdict = f"{YEL}improved{NC}" + print( + f" {name:<22}{ra.score:>8.3f}{rb.score:>9.3f}{delta:>+9.3f}" + f"{f'[{lo:+.3f}, {hi:+.3f}]':>22} {verdict}" + ) + # Different sizes are legal and the interval accounts for them, but the two sides are + # then scoring different question sets and a reader would not otherwise notice. + if ra.n != rb.n: + print( + f" {'':<22}{YEL}scored: A={ra.n} B={rb.n}{NC} — different question sets," + f" so this delta is not quite like-for-like" + ) + + for label, only in (("A", [k for k in a if k not in b]), ("B", [k for k in b if k not in a])): + if only: + print(f"\n only in {label}: {', '.join(only)}") + if not shared: + print("\n the two runs share no eval — there is nothing to compare") + return 0 + + print("\n 'same' means the delta's interval contains zero, i.e. this pair of runs cannot") + print(" tell the two configurations apart. It is not proof that they are identical —") + print(" with 200 questions per side the interval is roughly +/-5 points wide.") + if gate and regressed: + print(f"\n{RED}REGRESSION{NC} — at least one eval dropped by more than measurement noise") + return 1 + return 0 + + +def main() -> int: + ap = argparse.ArgumentParser( + description=__doc__, formatter_class=argparse.RawDescriptionHelpFormatter + ) + ap.add_argument( + "dirs", + nargs="+", + metavar="DIR", + help="one result directory to summarise, or two to compare (A then B)", + ) + ap.add_argument( + "--gate", + action="store_true", + help="in compare mode, exit 1 if any eval dropped significantly", + ) + args = ap.parse_args() + + if len(args.dirs) > 2: + raise SystemExit("at most two directories: summarise one, or compare two") + loaded = [load(d) for d in args.dirs] + if not any(loaded): + print(" (no results)") + return 0 + if len(loaded) == 1: + return report_one(loaded[0]) + return report_compare(loaded[0], loaded[1], args.dirs[0], args.dirs[1], args.gate) + + +if __name__ == "__main__": + try: + sys.exit(main()) + except KeyboardInterrupt: + sys.exit(130)