From 180ce1a6c689efd392495b715fe366084ffdfc4c Mon Sep 17 00:00:00 2001 From: lishuoshuo-amd Date: Thu, 27 Aug 2026 11:41:25 +0800 Subject: [PATCH 01/52] Add pre-release E2E CI: full-path release gate with layered per-leg judging. Build the CI wheel, stage it plus the in-pod bootstrap/prompts to NFS, then run 8 demo legs (baremetal/docker x vLLM/SGLang x 3h/12h) as SaFE Authoring workloads through the Claude CLI + setup/demo skills -- the same path a user takes. target_gain=100% is a hard release gate; each leg reports PASS/FAIL on its own terminal (3h legs surface first) via per-leg commit statuses. Triggers: a push to main that bumps pyproject's version runs all 8 legs; a push that only changes this CI's scripts/prompts/workflow (version unchanged) runs the 4 fast 3h legs to validate the logic change; manual dispatch runs a chosen subset (default all 8). All environment values are ${{ secrets.* }} / ${{ vars.* }} references -- none are hard-coded. --- .github/actionlint.yaml | 8 + .github/pre-release/bootstrap-pre-release.sh | 127 +++++++++ .github/pre-release/docker-run-hyperloom.sh | 88 ++++++ .../prompts/pre-release/demo-12h.txt | 28 ++ .../prompts/pre-release/demo-3h.txt | 29 ++ .../pre-release/setup-baremetal-sglang.txt | 29 ++ .../pre-release/setup-baremetal-vllm.txt | 29 ++ .../pre-release/setup-docker-sglang.txt | 37 +++ .../prompts/pre-release/setup-docker-vllm.txt | 37 +++ .github/scripts/pre-release-e2e-dispatch.sh | 245 ++++++++++++++++ .github/scripts/pre-release-e2e-poll.sh | 206 ++++++++++++++ .github/workflows/pre-release-e2e-test.yml | 267 ++++++++++++++++++ 12 files changed, 1130 insertions(+) create mode 100644 .github/actionlint.yaml create mode 100755 .github/pre-release/bootstrap-pre-release.sh create mode 100755 .github/pre-release/docker-run-hyperloom.sh create mode 100644 .github/pre-release/prompts/pre-release/demo-12h.txt create mode 100644 .github/pre-release/prompts/pre-release/demo-3h.txt create mode 100644 .github/pre-release/prompts/pre-release/setup-baremetal-sglang.txt create mode 100644 .github/pre-release/prompts/pre-release/setup-baremetal-vllm.txt create mode 100644 .github/pre-release/prompts/pre-release/setup-docker-sglang.txt create mode 100644 .github/pre-release/prompts/pre-release/setup-docker-vllm.txt create mode 100755 .github/scripts/pre-release-e2e-dispatch.sh create mode 100755 .github/scripts/pre-release-e2e-poll.sh create mode 100644 .github/workflows/pre-release-e2e-test.yml diff --git a/.github/actionlint.yaml b/.github/actionlint.yaml new file mode 100644 index 0000000000..b79f79e155 --- /dev/null +++ b/.github/actionlint.yaml @@ -0,0 +1,8 @@ +# SPDX-FileCopyrightText: 2026 Advanced Micro Devices, Inc. +# SPDX-License-Identifier: MIT +# +# Declare self-hosted runner labels so actionlint does not flag them as unknown. +# Used by ci-e2e.yml and pre-release-e2e-test.yml. +self-hosted-runner: + labels: + - Hyperloom-e2e-ci diff --git a/.github/pre-release/bootstrap-pre-release.sh b/.github/pre-release/bootstrap-pre-release.sh new file mode 100755 index 0000000000..570825bc87 --- /dev/null +++ b/.github/pre-release/bootstrap-pre-release.sh @@ -0,0 +1,127 @@ +#!/usr/bin/env bash +# SPDX-FileCopyrightText: 2026 Advanced Micro Devices, Inc. +# SPDX-License-Identifier: MIT +# +# Pre-release E2E pod bootstrap. Runs as the SaFE Authoring workload entrypoint. It +# reproduces the manual release path: install the packaged wheel, write a pod-local +# .env, install a pinned Claude CLI, then drive setup + demo skills through +# `claude --print`. The poll job (outside the pod) judges the session afterwards. +# See hyperloom-pre-release-e2e-ci-design.md §12. +# +# Two modes, selected by E2E_DOCKER_HOST: +# * unset -> a single baremetal/docker leg in THIS pod (LEG_ID given). +# * "1" -> the privileged 8-GPU host: fan out DOCKER_LEGS to nested containers +# via docker-run-hyperloom.sh , one GPU each. +# +# Inputs (env, injected by the dispatch script): +# CI_VERSION NFS_ROOT +# ANTHROPIC_API_KEY_B64 base64 key; decoded here, written only to pod-local .env +# ANTHROPIC_BASE_URL (optional) CLAUDE_MODEL CLAUDE_CLI_VERSION TARGET_GAIN +# Baremetal leg: LEG_ID HYPERLOOM_RUN_MODE=baremetal HYPERLOOM_BACKEND HYPERLOOM_MODEL_PATH DEMO_HOURS +# Docker host: E2E_DOCKER_HOST=1 DOCKER_LEGS DOCKER_GPU_MAP(json) MODEL_3H MODEL_12H +set -euo pipefail + +: "${CI_VERSION:?}"; : "${NFS_ROOT:?}"; : "${ANTHROPIC_API_KEY_B64:?}" +: "${CLAUDE_MODEL:?}"; : "${CLAUDE_CLI_VERSION:?}" +TARGET_GAIN="${TARGET_GAIN:-100}" + +SELF_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" +PROMPTS_DIR="${NFS_ROOT%/}/bootstrap/${CI_VERSION}/prompts/pre-release" +WHEEL_DIR="${NFS_ROOT%/}/wheels/${CI_VERSION}" + +log() { echo "[bootstrap $(date -u +%H:%M:%S)] $*"; } + +install_claude_cli() { + if command -v claude >/dev/null 2>&1; then log "claude CLI present: $(claude --version 2>/dev/null || true)"; return; fi + log "installing Claude CLI @ ${CLAUDE_CLI_VERSION}" + # Pinned install. The exact channel is environment-specific; keep the version in one + # place (CLAUDE_CLI_VERSION) so the pin is auditable. + npm install -g "@anthropic-ai/claude-code@${CLAUDE_CLI_VERSION}" >/dev/null 2>&1 \ + || { log "ERROR: claude CLI install failed"; return 1; } +} + +# Run ONE leg to completion inside the current filesystem (baremetal pod, or already +# inside a nested docker container). Args: leg backend model_path hours run_mode +run_leg() { + local leg="$1" backend="$2" model_path="$3" hours="$4" run_mode="$5" + local root="${NFS_ROOT%/}/runs/${CI_VERSION}/${leg}" + local session="${root}/session" + mkdir -p "$root" "$session" + log "leg=$leg mode=$run_mode backend=$backend hours=$hours model=$model_path" + + # 1. install the wheel into the leg root (produces importable tree + bundled skills) + local wheels=("$WHEEL_DIR"/hyperloom_inference_optimizer-*.whl) + [ -e "${wheels[0]}" ] || { log "ERROR: no wheel in $WHEEL_DIR"; return 1; } + log "pip install ${wheels[0]} --target $root" + pip install --no-input --target "$root" "${wheels[0]}" >/dev/null + + # 2. decode the key and write the pod-local .env (NEVER on stdout / NEVER to a + # location the poll reads). Restrict perms; scrub on exit. + local envf="${root}/.env" + ( umask 077 + { + echo "ANTHROPIC_API_KEY=$(printf '%s' "$ANTHROPIC_API_KEY_B64" | base64 -d)" + [ -n "${ANTHROPIC_BASE_URL:-}" ] && echo "ANTHROPIC_BASE_URL=${ANTHROPIC_BASE_URL}" + echo "CLAUDE_MODEL=${CLAUDE_MODEL}" + echo "USER_DATA_PATH=${session}" + echo "HYPERLOOM_RUN_MODE=${run_mode}" + echo "FRAMEWORK=${backend}" + echo "MODEL_PATH=${model_path}" + echo "TARGET_GAIN=${TARGET_GAIN}" + echo "DEMO_HOURS=${hours}" + } > "$envf" + ) + trap 'sed -i "/^ANTHROPIC_API_KEY=/d" "'"$envf"'" 2>/dev/null || true' EXIT + + # 3. pin the session dir so the poll finds it without guessing by timestamp (design §9) + export INFERENCE_OPTIMIZER_CURRENT_SESSION_DIR="${session}" + echo "${session}" > "${session}/.session_dir" + + # 4. source env + drive setup, then demo, through the Agent CLI + set -a + # shellcheck disable=SC1090 # envf path is dynamic (per-leg) + . "$envf" + set +a + export PYTHONPATH="${root}:${PYTHONPATH:-}" + + local setup_prompt="${PROMPTS_DIR}/setup-${run_mode}-${backend}.txt" + local demo_prompt; demo_prompt="${PROMPTS_DIR}/demo-${hours}h.txt" + [ -f "$setup_prompt" ] || { log "ERROR: missing $setup_prompt"; return 1; } + [ -f "$demo_prompt" ] || { log "ERROR: missing $demo_prompt"; return 1; } + + log "claude --print (setup)" + claude --print < "$setup_prompt" + log "claude --print (demo ${hours}h)" + claude --print < "$demo_prompt" + log "leg $leg agent turns complete; poll will judge $session/reports/final.json" +} + +# ---- docker host: fan out to nested containers ----------------------------- +run_docker_host() { + : "${DOCKER_LEGS:?}"; : "${DOCKER_GPU_MAP:?}"; : "${MODEL_3H:?}"; : "${MODEL_12H:?}" + local runner="${SELF_DIR}/docker-run-hyperloom.sh" + [ -x "$runner" ] || chmod +x "$runner" 2>/dev/null || true + log "docker host: legs='${DOCKER_LEGS}'" + local pids=() + for leg in $DOCKER_LEGS; do + local idx; idx="$(printf '%s' "$DOCKER_GPU_MAP" | jq -r --arg l "$leg" '.[$l]')" + log "launch nested container: gpu=$idx leg=$leg" + # Each nested container binds one card and runs THIS bootstrap inside, in + # single-leg mode. docker-run-hyperloom.sh enforces GPU + cpu/mem quota (§8). + "$runner" "$idx" "$leg" & + pids+=("$!") + done + local rc=0 + for p in "${pids[@]}"; do wait "$p" || rc=1; done + return "$rc" +} + +# ---- entry ----------------------------------------------------------------- +install_claude_cli || exit 1 + +if [ "${E2E_DOCKER_HOST:-}" = "1" ]; then + run_docker_host +else + : "${LEG_ID:?}"; : "${HYPERLOOM_BACKEND:?}"; : "${HYPERLOOM_MODEL_PATH:?}"; : "${DEMO_HOURS:?}" + run_leg "$LEG_ID" "$HYPERLOOM_BACKEND" "$HYPERLOOM_MODEL_PATH" "$DEMO_HOURS" "${HYPERLOOM_RUN_MODE:-baremetal}" +fi diff --git a/.github/pre-release/docker-run-hyperloom.sh b/.github/pre-release/docker-run-hyperloom.sh new file mode 100755 index 0000000000..4507b73bb7 --- /dev/null +++ b/.github/pre-release/docker-run-hyperloom.sh @@ -0,0 +1,88 @@ +#!/usr/bin/env bash +# SPDX-FileCopyrightText: 2026 Advanced Micro Devices, Inc. +# SPDX-License-Identifier: MIT +# +# Nested docker runner for the privileged 8-GPU pre-release host. This is the ONLY +# docker entry point for docker legs: it force-binds ONE GPU and caps CPU/memory to a +# 1/4 host share so the 4 docker legs are mutually comparable and comparable to the +# baremetal legs (design §8, point E). Prompts are forbidden from running docker +# directly or choosing GPUs via rocm-smi. +# +# Usage: docker-run-hyperloom.sh +# +# Inputs (env, inherited from the host bootstrap): +# NFS_ROOT CI_VERSION MODEL_3H MODEL_12H TARGET_GAIN +# CLAUDE_MODEL CLAUDE_CLI_VERSION ANTHROPIC_API_KEY_B64 [ANTHROPIC_BASE_URL] +# HYPERLOOM_IMAGE backend container image (overrides per-backend default) +# LEG_CPUS LEG_MEM LEG_SHM per-container quota (default 32 / 128g / 64g) +set -euo pipefail + +GPU_INDEX="${1:?usage: docker-run-hyperloom.sh }" +LEG_ID="${2:?usage: docker-run-hyperloom.sh }" + +: "${NFS_ROOT:?}"; : "${CI_VERSION:?}"; : "${ANTHROPIC_API_KEY_B64:?}" +: "${CLAUDE_MODEL:?}"; : "${CLAUDE_CLI_VERSION:?}" +TARGET_GAIN="${TARGET_GAIN:-100}" + +# 1/4 of a 128-core / 512Gi host, matching the baremetal leg envelope. +LEG_CPUS="${LEG_CPUS:-32}" +LEG_MEM="${LEG_MEM:-128g}" +LEG_SHM="${LEG_SHM:-64g}" + +# Resolve per-leg backend / model / hours from the leg id. +case "$LEG_ID" in + *-vllm-*) BACKEND=vllm ;; + *-sglang-*) BACKEND=sglang ;; + *) echo "cannot infer backend from leg '$LEG_ID'" >&2; exit 2 ;; +esac +case "$LEG_ID" in + *-3h) HOURS=3; MODEL_PATH="${MODEL_3H:?MODEL_3H required}" ;; + *-12h) HOURS=12; MODEL_PATH="${MODEL_12H:?MODEL_12H required}" ;; + *) echo "cannot infer duration from leg '$LEG_ID'" >&2; exit 2 ;; +esac + +# Default backend images (overridable via HYPERLOOM_IMAGE); mirrors the demo skill's +# suggested ROCm images. +if [ -n "${HYPERLOOM_IMAGE:-}" ]; then + IMAGE="$HYPERLOOM_IMAGE" +elif [ "$BACKEND" = vllm ]; then + IMAGE="${HYPERLOOM_IMAGE_VLLM:-vllm/vllm-openai-rocm:v0.27.1}" +else + IMAGE="${HYPERLOOM_IMAGE_SGLANG:-lmsysorg/sglang-rocm:v0.5.17-rocm724-mi35x-srt}" +fi + +ROOT="${NFS_ROOT%/}/runs/${CI_VERSION}/${LEG_ID}" +mkdir -p "$ROOT" +BOOTSTRAP="${NFS_ROOT%/}/bootstrap/${CI_VERSION}/bootstrap-pre-release.sh" +NAME="hyperloom-${LEG_ID}" +RD=$((128 + GPU_INDEX)) # renderD node paired with cardN + +echo "[docker-run] leg=$LEG_ID gpu=$GPU_INDEX card$GPU_INDEX/renderD$RD image=$IMAGE cpus=$LEG_CPUS mem=$LEG_MEM" + +docker rm -f "$NAME" >/dev/null 2>&1 || true + +# GPU isolation: expose exactly one card, and set ROCR_VISIBLE_DEVICES=0 so the +# container sees a single device at index 0. CPU/mem hard-capped to the 1/4 share. +exec docker run --rm --name "$NAME" \ + --device "/dev/kfd" \ + --device "/dev/dri/card${GPU_INDEX}" \ + --device "/dev/dri/renderD${RD}" \ + --group-add video \ + --cpus "$LEG_CPUS" --memory "$LEG_MEM" --shm-size "$LEG_SHM" \ + -e ROCR_VISIBLE_DEVICES=0 \ + -e CI_VERSION="$CI_VERSION" \ + -e NFS_ROOT="$NFS_ROOT" \ + -e LEG_ID="$LEG_ID" \ + -e HYPERLOOM_RUN_MODE=docker \ + -e HYPERLOOM_BACKEND="$BACKEND" \ + -e HYPERLOOM_MODEL_PATH="$MODEL_PATH" \ + -e DEMO_HOURS="$HOURS" \ + -e TARGET_GAIN="$TARGET_GAIN" \ + -e CLAUDE_MODEL="$CLAUDE_MODEL" \ + -e CLAUDE_CLI_VERSION="$CLAUDE_CLI_VERSION" \ + -e ANTHROPIC_API_KEY_B64="$ANTHROPIC_API_KEY_B64" \ + ${ANTHROPIC_BASE_URL:+-e ANTHROPIC_BASE_URL="$ANTHROPIC_BASE_URL"} \ + -v "$ROOT:$ROOT" \ + -v "$NFS_ROOT:$NFS_ROOT" \ + --entrypoint bash \ + "$IMAGE" "$BOOTSTRAP" diff --git a/.github/pre-release/prompts/pre-release/demo-12h.txt b/.github/pre-release/prompts/pre-release/demo-12h.txt new file mode 100644 index 0000000000..161997a0a9 --- /dev/null +++ b/.github/pre-release/prompts/pre-release/demo-12h.txt @@ -0,0 +1,28 @@ +You are running the Hyperloom pre-release E2E test non-interactively. Run the 12-hour +demo to completion, then stop. Setup already ran successfully in this workspace. + +Invoke the `hyperloom-qwen3-14b-fp8-12h` demo skill with ONE override and otherwise +its exact default flags: + +- OVERRIDE: use `--target-gain 100` (NOT the skill's default of 50). This is the + pre-release release gate — the run must reach a validated cumulative gain of 100%. +- Keep every other required flag exactly as the skill defines them: `--tp 1 + --conc 64 --isl 1024 --osl 1024 --precision fp8 --max-hours 12 + --max-minutes-framework-pct 0.01 --max-minutes-explore-pct 0.42 + --max-minutes-kernel-pct 0.42`. + +Model path: the skill will ask which model to use. Do NOT ask interactively — use +`MODEL_PATH` from the repository-root `.env` (it is already set to the demo model, +Qwen3-14B-FP8). Verify that path contains `config.json`; if it does, use it and +continue without asking. Load LLM API keys/base URLs and `FRAMEWORK` from `.env`. + +Hard constraints (automated release gate): +- Do NOT modify any GPU-related environment variable or device visibility. +- Do NOT run `docker`, start/exec containers, or choose GPUs via `rocm-smi`. If + HYPERLOOM_RUN_MODE=docker you are already inside the correct single-GPU container. +- Do NOT modify USER_DATA_PATH. +- Do NOT print or copy secret values into output, reports, or logs. + +Let the run proceed to its terminal report (session `reports/final.json` + +`reports/final.md`). When it terminates, stop. The harness judges PASS/FAIL from the +session report — do not fabricate a result. diff --git a/.github/pre-release/prompts/pre-release/demo-3h.txt b/.github/pre-release/prompts/pre-release/demo-3h.txt new file mode 100644 index 0000000000..21e0102628 --- /dev/null +++ b/.github/pre-release/prompts/pre-release/demo-3h.txt @@ -0,0 +1,29 @@ +You are running the Hyperloom pre-release E2E test non-interactively. Run the 3-hour +demo to completion, then stop. Setup already ran successfully in this workspace. + +Invoke the `hyperloom-qwen3-8b-3h` demo skill with ONE override and otherwise its +exact default flags: + +- OVERRIDE: use `--target-gain 100` (NOT the skill's default of 30). This is the + pre-release release gate — the run must reach a validated cumulative gain of 100%. +- Keep every other required flag exactly as the skill defines them: `--tp 1 + --conc 64 --isl 1024 --osl 1024 --precision bf16 --max-hours 3 + --max-minutes-explore-pct 0.39 --max-minutes-sweep-pct 0.01 + --explore-force-exit-budget-pct 0.01 --no-framework-agent --no-kernel + --no-enable-conc-sweep --no-enable-roofline`. + +Model path: the skill will ask which model to use. Do NOT ask interactively — use +`MODEL_PATH` from the repository-root `.env` (it is already set to the demo model). +Verify that path contains `config.json`; if it does, use it and continue without +asking. Load LLM API keys/base URLs and `FRAMEWORK` from `.env`. + +Hard constraints (automated release gate): +- Do NOT modify any GPU-related environment variable or device visibility. +- Do NOT run `docker`, start/exec containers, or choose GPUs via `rocm-smi`. If + HYPERLOOM_RUN_MODE=docker you are already inside the correct single-GPU container. +- Do NOT modify USER_DATA_PATH. +- Do NOT print or copy secret values into output, reports, or logs. + +Let the run proceed to its terminal report (session `reports/final.json` + +`reports/final.md`). When it terminates, stop. The harness judges PASS/FAIL from the +session report — do not fabricate a result. diff --git a/.github/pre-release/prompts/pre-release/setup-baremetal-sglang.txt b/.github/pre-release/prompts/pre-release/setup-baremetal-sglang.txt new file mode 100644 index 0000000000..5302ff40c0 --- /dev/null +++ b/.github/pre-release/prompts/pre-release/setup-baremetal-sglang.txt @@ -0,0 +1,29 @@ +You are running the Hyperloom pre-release E2E test non-interactively. Complete the +setup step for a BAREMETAL + SGLang leg, then stop. Do not run the demo yet. + +Environment is already prepared. A `.env` file exists in the current workspace +(REPO_ROOT) with these values already set: ANTHROPIC_API_KEY, CLAUDE_MODEL, +USER_DATA_PATH, HYPERLOOM_RUN_MODE=baremetal, FRAMEWORK=sglang, MODEL_PATH, +TARGET_GAIN, DEMO_HOURS. The wheel is already installed via `pip install --target .` +so a `hyperloom/` package directory is present. + +Run the hyperloom-setup skill with these fixed decisions — do NOT ask interactive +questions, use the values already in `.env` and the environment: + +- Run mode: baremetal (HYPERLOOM_RUN_MODE is already `baremetal`; keep it). +- Framework: SGLang. Install the framework layer with the setup backend + (`--install-framework sglang`). +- LLM provider / model / USER_DATA_PATH: use the values already in `.env`; do not + change them. + +Hard constraints (this is an automated release gate): +- Do NOT modify any GPU-related environment variable (ROCR_VISIBLE_DEVICES, + HIP_VISIBLE_DEVICES, GPUS_PER_NODE, etc.). The pod already exposes exactly one + GPU; do not override device visibility. +- Do NOT run `docker` and do NOT choose GPUs via `rocm-smi`. This is a baremetal + leg; setup runs on the host. +- Do NOT print, echo, or copy secret values (API keys) into output or logs. +- Do NOT modify USER_DATA_PATH. + +When setup completes successfully, stop. Report only "setup complete: baremetal/sglang". +If setup hard-fails, report the failure and stop. diff --git a/.github/pre-release/prompts/pre-release/setup-baremetal-vllm.txt b/.github/pre-release/prompts/pre-release/setup-baremetal-vllm.txt new file mode 100644 index 0000000000..d0f638f0d0 --- /dev/null +++ b/.github/pre-release/prompts/pre-release/setup-baremetal-vllm.txt @@ -0,0 +1,29 @@ +You are running the Hyperloom pre-release E2E test non-interactively. Complete the +setup step for a BAREMETAL + vLLM leg, then stop. Do not run the demo yet. + +Environment is already prepared. A `.env` file exists in the current workspace +(REPO_ROOT) with these values already set: ANTHROPIC_API_KEY, CLAUDE_MODEL, +USER_DATA_PATH, HYPERLOOM_RUN_MODE=baremetal, FRAMEWORK=vllm, MODEL_PATH, +TARGET_GAIN, DEMO_HOURS. The wheel is already installed via `pip install --target .` +so a `hyperloom/` package directory is present. + +Run the hyperloom-setup skill with these fixed decisions — do NOT ask interactive +questions, use the values already in `.env` and the environment: + +- Run mode: baremetal (HYPERLOOM_RUN_MODE is already `baremetal`; keep it). +- Framework: vLLM. Install the framework layer with the setup backend + (`--install-framework vllm`, isolated framework env). +- LLM provider / model / USER_DATA_PATH: use the values already in `.env`; do not + change them. + +Hard constraints (this is an automated release gate): +- Do NOT modify any GPU-related environment variable (ROCR_VISIBLE_DEVICES, + HIP_VISIBLE_DEVICES, GPUS_PER_NODE, etc.). The pod already exposes exactly one + GPU; do not override device visibility. +- Do NOT run `docker` and do NOT choose GPUs via `rocm-smi`. This is a baremetal + leg; setup runs on the host. +- Do NOT print, echo, or copy secret values (API keys) into output or logs. +- Do NOT modify USER_DATA_PATH. + +When setup completes successfully, stop. Report only "setup complete: baremetal/vllm". +If setup hard-fails, report the failure and stop. diff --git a/.github/pre-release/prompts/pre-release/setup-docker-sglang.txt b/.github/pre-release/prompts/pre-release/setup-docker-sglang.txt new file mode 100644 index 0000000000..439f6dc738 --- /dev/null +++ b/.github/pre-release/prompts/pre-release/setup-docker-sglang.txt @@ -0,0 +1,37 @@ +You are running the Hyperloom pre-release E2E test non-interactively. Complete the +setup step for a DOCKER + SGLang leg, then stop. Do not run the demo yet. + +IMPORTANT: you are ALREADY running inside the backend container. The nested +container was started for you by the test harness (docker-run-hyperloom.sh) and is +bound to exactly one GPU. You must NOT start, run, or exec any further container, +and you must NOT run `docker` at all. Treat this environment as the place where +setup and the demo run directly. + +Environment is already prepared. A `.env` file exists in the current workspace +(REPO_ROOT) with these values already set: ANTHROPIC_API_KEY, CLAUDE_MODEL, +USER_DATA_PATH, HYPERLOOM_RUN_MODE=docker, FRAMEWORK=sglang, MODEL_PATH, +TARGET_GAIN, DEMO_HOURS. The wheel is already installed via `pip install --target .` +so a `hyperloom/` package directory is present. + +Run the hyperloom-setup skill with these fixed decisions — do NOT ask interactive +questions, use the values already in `.env` and the environment: + +- Run mode: docker, but the container already exists and IS the current shell. Do + NOT create a container and do NOT set HYPERLOOM_DOCKER_TARGET_HOST. Run setup + directly in this shell. +- Framework: SGLang. Ensure the SGLang framework layer is available in this + container (install with the setup backend if needed). +- LLM provider / model / USER_DATA_PATH: use the values already in `.env`; do not + change them. + +Hard constraints (this is an automated release gate): +- Do NOT modify any GPU-related environment variable (ROCR_VISIBLE_DEVICES is + already 0 and pins this container to its single card; leave it). Do not override + device visibility. +- Do NOT run `docker`, do NOT start/exec containers, and do NOT choose GPUs via + `rocm-smi`. +- Do NOT print, echo, or copy secret values (API keys) into output or logs. +- Do NOT modify USER_DATA_PATH. + +When setup completes successfully, stop. Report only "setup complete: docker/sglang". +If setup hard-fails, report the failure and stop. diff --git a/.github/pre-release/prompts/pre-release/setup-docker-vllm.txt b/.github/pre-release/prompts/pre-release/setup-docker-vllm.txt new file mode 100644 index 0000000000..05c825f818 --- /dev/null +++ b/.github/pre-release/prompts/pre-release/setup-docker-vllm.txt @@ -0,0 +1,37 @@ +You are running the Hyperloom pre-release E2E test non-interactively. Complete the +setup step for a DOCKER + vLLM leg, then stop. Do not run the demo yet. + +IMPORTANT: you are ALREADY running inside the backend container. The nested +container was started for you by the test harness (docker-run-hyperloom.sh) and is +bound to exactly one GPU. You must NOT start, run, or exec any further container, +and you must NOT run `docker` at all. Treat this environment as the place where +setup and the demo run directly. + +Environment is already prepared. A `.env` file exists in the current workspace +(REPO_ROOT) with these values already set: ANTHROPIC_API_KEY, CLAUDE_MODEL, +USER_DATA_PATH, HYPERLOOM_RUN_MODE=docker, FRAMEWORK=vllm, MODEL_PATH, +TARGET_GAIN, DEMO_HOURS. The wheel is already installed via `pip install --target .` +so a `hyperloom/` package directory is present. + +Run the hyperloom-setup skill with these fixed decisions — do NOT ask interactive +questions, use the values already in `.env` and the environment: + +- Run mode: docker, but the container already exists and IS the current shell. Do + NOT create a container and do NOT set HYPERLOOM_DOCKER_TARGET_HOST. Run setup + directly in this shell. +- Framework: vLLM. Ensure the vLLM framework layer is available in this container + (install with the setup backend if needed). +- LLM provider / model / USER_DATA_PATH: use the values already in `.env`; do not + change them. + +Hard constraints (this is an automated release gate): +- Do NOT modify any GPU-related environment variable (ROCR_VISIBLE_DEVICES is + already 0 and pins this container to its single card; leave it). Do not override + device visibility. +- Do NOT run `docker`, do NOT start/exec containers, and do NOT choose GPUs via + `rocm-smi`. +- Do NOT print, echo, or copy secret values (API keys) into output or logs. +- Do NOT modify USER_DATA_PATH. + +When setup completes successfully, stop. Report only "setup complete: docker/vllm". +If setup hard-fails, report the failure and stop. diff --git a/.github/scripts/pre-release-e2e-dispatch.sh b/.github/scripts/pre-release-e2e-dispatch.sh new file mode 100755 index 0000000000..dd317584b7 --- /dev/null +++ b/.github/scripts/pre-release-e2e-dispatch.sh @@ -0,0 +1,245 @@ +#!/usr/bin/env bash +# SPDX-FileCopyrightText: 2026 Advanced Micro Devices, Inc. +# SPDX-License-Identifier: MIT +# +# Pre-release E2E: create the SaFE Authoring workloads that run the packaged wheel +# through the real user path (Claude CLI + setup skill + demo skill). Unlike the PR +# smoke test (.github/scripts/ci-e2e-dispatch.sh), which uses the orchestration +# endpoint (POST /api/v1/orchestration/workloads, kind=hyperloom) to dispatch a git +# SHA, this dispatches GENERIC Authoring pods (POST /api/v1/workloads, +# kind=Authoring) whose entrypoint is the bootstrap script. See +# hyperloom-pre-release-e2e-ci-design.md §7. +# +# It creates 5 workloads for the 8 legs: +# * 4x non-privileged 1-GPU Authoring (one per baremetal leg) +# * 1x privileged 8-GPU Authoring (docker host; 4 nested containers, GPU 0-3) +# and writes a dispatch map (leg -> workloadId) to $DISPATCH_MAP for the poll step. +# +# Requires: bash, curl, jq on the (self-hosted, in-network) runner. +# +# Inputs (env): +# SAFE_API_BASE SaFE API base url (required) +# SAFE_API_KEY bearer token; privileged pod needs an +# ADMIN token (privileged=true is admin-only) (required) +# SAFE_WORKSPACE_ID workspace that mounts the shared NFS (required) +# CI_VERSION wheel/run version, e.g. 1.0.0b3.dev...+ci (required) +# AUTHORING_IMAGE Authoring base image ref (required) +# NFS_ROOT pre-release test root on shared NFS +# (default /shared_nfs/hyperloom-pre-release-e2e-test) +# MODEL_3H local path to the 3h model (Qwen3-8B) (required) +# MODEL_12H local path to the 12h model (Qwen3-14B-FP8)(required) +# TARGET_GAIN release gate gain %% for every leg (default 100) +# CLAUDE_MODEL model for the Agent turns (required) +# CLAUDE_CLI_VERSION pinned Claude CLI version (required) +# ANTHROPIC_API_KEY Claude CLI auth; injected here as base64 +# into the workload env (never written to NFS)(required) +# ANTHROPIC_BASE_URL optional proxy / base url (optional) +# TASKS comma-separated leg subset (default: all 8) +# DISPATCH_MAP output file: JSON {leg: workloadId} +# (default $RUNNER_TEMP/pre_release_dispatch.json) +# HOST_CPU / HOST_MEM / HOST_SHM privileged host resource request +# (default 128 / 512Gi / 256Gi) +# LEG_CPU / LEG_MEM baremetal leg resource request +# (default 32 / 128Gi) +# SAFE_CACERT / SAFE_INSECURE TLS to the API (CA bundle / skip-verify) +set -euo pipefail + +NFS_ROOT="${NFS_ROOT:-/shared_nfs/hyperloom-pre-release-e2e-test}" +TARGET_GAIN="${TARGET_GAIN:-100}" +HOST_CPU="${HOST_CPU:-128}"; HOST_MEM="${HOST_MEM:-512Gi}"; HOST_SHM="${HOST_SHM:-256Gi}" +LEG_CPU="${LEG_CPU:-32}"; LEG_MEM="${LEG_MEM:-128Gi}" +DISPATCH_MAP="${DISPATCH_MAP:-${RUNNER_TEMP:-/tmp}/pre_release_dispatch.json}" + +: "${SAFE_API_BASE:?SAFE_API_BASE is required}" +: "${SAFE_API_KEY:?SAFE_API_KEY is required}" +: "${SAFE_WORKSPACE_ID:?SAFE_WORKSPACE_ID is required}" +: "${CI_VERSION:?CI_VERSION is required}" +: "${AUTHORING_IMAGE:?AUTHORING_IMAGE is required}" +: "${MODEL_3H:?MODEL_3H is required}" +: "${MODEL_12H:?MODEL_12H is required}" +: "${CLAUDE_MODEL:?CLAUDE_MODEL is required}" +: "${CLAUDE_CLI_VERSION:?CLAUDE_CLI_VERSION is required}" +: "${ANTHROPIC_API_KEY:?ANTHROPIC_API_KEY is required}" + +API="${SAFE_API_BASE%/}/api/v1/workloads" +auth=(-H "Authorization: Bearer ${SAFE_API_KEY}") +tls=() +if [ -n "${SAFE_CACERT:-}" ]; then + tls=(--cacert "$SAFE_CACERT") +elif [ "${SAFE_INSECURE:-0}" = "1" ]; then + tls=(-k) +fi + +summary() { echo "$*" | tee -a "${GITHUB_STEP_SUMMARY:-/dev/null}"; } + +# All 8 legs. Fields: mode backend hours model_path -- gpu index within the docker host +ALL_LEGS="baremetal-vllm-3h baremetal-vllm-12h baremetal-sglang-3h baremetal-sglang-12h \ +docker-vllm-3h docker-vllm-12h docker-sglang-3h docker-sglang-12h" +REQ_TASKS="${TASKS:-$ALL_LEGS}" +REQ_TASKS="${REQ_TASKS//,/ }" + +leg_model_path() { case "$1" in *-3h) echo "$MODEL_3H" ;; *-12h) echo "$MODEL_12H" ;; esac; } +leg_hours() { case "$1" in *-3h) echo "3" ;; *-12h) echo "12" ;; esac; } +leg_backend() { case "$1" in *-vllm-*) echo "vllm" ;; *-sglang-*) echo "sglang" ;; esac; } + +# GPU index a docker leg binds inside the privileged host (design §3). +docker_gpu_index() { + case "$1" in + docker-vllm-3h) echo 0 ;; + docker-vllm-12h) echo 1 ;; + docker-sglang-3h) echo 2 ;; + docker-sglang-12h) echo 3 ;; + *) echo "" ;; + esac +} + +# Common env for every workload. The API key is passed base64 so it is not visible in +# plaintext in the API payload log; bootstrap decodes it and writes it only to the +# pod-local .env (never to NFS). See design §9 (point D). +common_env_json() { + local model_path="$1" hours="$2" backend="$3" + jq -n \ + --arg civ "$CI_VERSION" --arg nfs "$NFS_ROOT" \ + --arg model "$model_path" --arg hours "$hours" --arg backend "$backend" \ + --arg tgain "$TARGET_GAIN" \ + --arg cmodel "$CLAUDE_MODEL" --arg cver "$CLAUDE_CLI_VERSION" \ + --arg keyb64 "$(printf '%s' "$ANTHROPIC_API_KEY" | base64 | tr -d '\n')" \ + --arg baseurl "${ANTHROPIC_BASE_URL:-}" \ + '{ + CI_VERSION: $civ, + NFS_ROOT: $nfs, + HYPERLOOM_MODEL_PATH: $model, + DEMO_HOURS: $hours, + HYPERLOOM_BACKEND: $backend, + TARGET_GAIN: $tgain, + CLAUDE_MODEL: $cmodel, + CLAUDE_CLI_VERSION: $cver, + ANTHROPIC_API_KEY_B64: $keyb64 + } + (if $baseurl == "" then {} else {ANTHROPIC_BASE_URL: $baseurl} end)' +} + +# POST one workload; echo the workloadId. Args: displayName resourcesJson envJson privileged(true|false) +create_workload() { + local name="$1" resources="$2" env="$3" privileged="$4" entry_b64="$5" + local body resp code json wid + body="$(jq -n \ + --arg name "$name" --arg ws "$SAFE_WORKSPACE_ID" --arg img "$AUTHORING_IMAGE" \ + --arg entry "$entry_b64" --argjson res "$resources" --argjson env "$env" \ + --argjson priv "$privileged" \ + '{ + displayName: $name, + workspaceId: $ws, + groupVersionKind: {kind:"Authoring", version:"v1"}, + resources: [$res], + images: [$img], + entryPoints: [$entry], + env: $env, + useWorkspaceStorage: true + } + (if $priv then {privileged:true} else {} end)')" + resp="$(curl -sS "${tls[@]}" -w $'\n%{http_code}' -X POST "$API" \ + "${auth[@]}" -H "Content-Type: application/json" -d "$body")" + code="$(printf '%s' "$resp" | tail -n1)" + json="$(printf '%s' "$resp" | sed '$d')" + if [ "$code" -lt 200 ] || [ "$code" -ge 300 ]; then + summary "❌ create '$name' failed (HTTP $code): $(printf '%s' "$json" | head -c 400)" + return 1 + fi + wid="$(printf '%s' "$json" | jq -r '.workloadId // empty')" + if [ -z "$wid" ]; then + summary "❌ create '$name' returned no workloadId: $(printf '%s' "$json" | head -c 400)" + return 1 + fi + printf '%s' "$wid" +} + +# Base64 the bootstrap entrypoint (SaFE requires base64-encoded entryPoints). +# The bootstrap script is staged to NFS by the build job (from .github/pre-release/) +# and read by the pod at ${NFS_ROOT}/bootstrap/${CI_VERSION}/bootstrap-pre-release.sh. +bootstrap_entry_b64() { + local extra="$1" # extra shell prepended (e.g. E2E_DOCKER_HOST=1) + local cmd + cmd="set -e; ${extra} exec bash \"\${NFS_ROOT}/bootstrap/${CI_VERSION}/bootstrap-pre-release.sh\"" + printf '%s' "$cmd" | base64 | tr -d '\n' +} + +echo "[dispatch] CI_VERSION=$CI_VERSION tasks='$REQ_TASKS'" +declare -A DISPATCH # leg -> workloadId + +# ---- baremetal legs: one non-privileged 1-GPU workload each ---------------- +leg_resources_1gpu="$(jq -n --arg cpu "$LEG_CPU" --arg mem "$LEG_MEM" \ + '{replica:1, gpu:"1", cpu:$cpu, memory:$mem, ephemeralStorage:"100Gi"}')" + +want_docker_host=0 +for leg in $REQ_TASKS; do + case "$leg" in + baremetal-*) + env_json="$(common_env_json "$(leg_model_path "$leg")" "$(leg_hours "$leg")" "$(leg_backend "$leg")" \ + | jq --arg leg "$leg" '. + {LEG_ID:$leg, HYPERLOOM_RUN_MODE:"baremetal"}')" + entry="$(bootstrap_entry_b64 "")" + wid="$(create_workload "e2e-${CI_VERSION}-${leg}" "$leg_resources_1gpu" "$env_json" false "$entry")" + DISPATCH["$leg"]="$wid" + summary "• \`$leg\` → workloadId \`$wid\` (baremetal, 1 GPU)" + ;; + docker-*) + want_docker_host=1 + ;; + *) + summary "⚠️ unknown leg '$leg' ignored" + ;; + esac +done + +# ---- docker legs: one privileged 8-GPU host running all requested docker legs ---- +if [ "$want_docker_host" = 1 ]; then + host_resources="$(jq -n --arg cpu "$HOST_CPU" --arg mem "$HOST_MEM" --arg shm "$HOST_SHM" \ + '{replica:1, gpu:"8", cpu:$cpu, memory:$mem, sharedMemory:$shm, ephemeralStorage:"200Gi"}')" + # The host env carries the per-leg GPU map so the host bootstrap launches the right + # nested containers via docker-run-hyperloom.sh . + docker_legs=""; gpu_map="{}" + for leg in $REQ_TASKS; do + case "$leg" in + docker-*) + idx="$(docker_gpu_index "$leg")" + docker_legs="${docker_legs}${docker_legs:+ }${leg}" + gpu_map="$(printf '%s' "$gpu_map" | jq --arg l "$leg" --arg i "$idx" '. + {($l): $i}')" + ;; + esac + done + # Host env: model paths for both durations, plus the leg->gpu map. Per-leg model/ + # backend are resolved inside the host bootstrap from the leg id. + host_env="$(jq -n \ + --arg civ "$CI_VERSION" --arg nfs "$NFS_ROOT" \ + --arg m3 "$MODEL_3H" --arg m12 "$MODEL_12H" \ + --arg tgain "$TARGET_GAIN" --arg cmodel "$CLAUDE_MODEL" --arg cver "$CLAUDE_CLI_VERSION" \ + --arg keyb64 "$(printf '%s' "$ANTHROPIC_API_KEY" | base64 | tr -d '\n')" \ + --arg baseurl "${ANTHROPIC_BASE_URL:-}" \ + --arg legs "$docker_legs" --argjson gpumap "$gpu_map" \ + '{ + CI_VERSION:$civ, NFS_ROOT:$nfs, + MODEL_3H:$m3, MODEL_12H:$m12, + TARGET_GAIN:$tgain, CLAUDE_MODEL:$cmodel, CLAUDE_CLI_VERSION:$cver, + ANTHROPIC_API_KEY_B64:$keyb64, + HYPERLOOM_RUN_MODE:"docker", + DOCKER_LEGS:$legs, DOCKER_GPU_MAP:($gpumap|tostring) + } + (if $baseurl == "" then {} else {ANTHROPIC_BASE_URL:$baseurl} end)')" + entry="$(bootstrap_entry_b64 "E2E_DOCKER_HOST=1;")" + wid="$(create_workload "e2e-${CI_VERSION}-docker-host" "$host_resources" "$host_env" true "$entry")" + # Every docker leg shares the one host workloadId; the poll distinguishes them by + # reading each leg's own session dir on NFS. + for leg in $docker_legs; do + DISPATCH["$leg"]="$wid" + summary "• \`$leg\` → workloadId \`$wid\` (docker host, GPU $(docker_gpu_index "$leg"))" + done +fi + +# ---- write the dispatch map for the poll step ------------------------------ +map_json="{}" +for leg in "${!DISPATCH[@]}"; do + map_json="$(printf '%s' "$map_json" | jq --arg l "$leg" --arg w "${DISPATCH[$leg]}" '. + {($l):$w}')" +done +printf '%s\n' "$map_json" > "$DISPATCH_MAP" +echo "dispatch_map=$DISPATCH_MAP" >> "${GITHUB_OUTPUT:-/dev/null}" +summary "" +summary "**dispatched $(printf '%s' "$map_json" | jq 'length') legs** → \`$DISPATCH_MAP\`" +echo "[dispatch] wrote $DISPATCH_MAP" +printf '%s\n' "$map_json" | jq . diff --git a/.github/scripts/pre-release-e2e-poll.sh b/.github/scripts/pre-release-e2e-poll.sh new file mode 100755 index 0000000000..ebae898116 --- /dev/null +++ b/.github/scripts/pre-release-e2e-poll.sh @@ -0,0 +1,206 @@ +#!/usr/bin/env bash +# SPDX-FileCopyrightText: 2026 Advanced Micro Devices, Inc. +# SPDX-License-Identifier: MIT +# +# Pre-release E2E: poll the dispatched SaFE Authoring workloads and judge each leg +# independently, reporting PASS/FAIL as soon as that leg reaches a terminal state +# (design §10, point C -- the 3h legs finish ~3-4h and report first; 12h legs later). +# +# A leg PASSes only when ALL hold (design §9): +# 1. its session reports/final.json has stop_reason == "target_reached" +# (equivalently cumulative_gain_validated >= TARGET_GAIN; the gate is 100), +# 2. its owning SaFE workload phase is not Failed/Stopped, +# 3. reports/final.json and reports/final.md both exist, +# 4. crash_count / server_boot_failures are within tolerance. +# Anything else (incl. "ran the full duration without target_reached") is FAIL. +# +# The exit code is 0 only if every requested leg PASSed. +# +# Requires: bash, curl, jq on the (self-hosted, in-network) runner with the NFS +# runs/ tree readable. +# +# Inputs (env): +# SAFE_API_BASE / SAFE_API_KEY SaFE API (required) +# CI_VERSION run version (required) +# DISPATCH_MAP leg->workloadId JSON from dispatch (required) +# NFS_ROOT (default /shared_nfs/hyperloom-pre-release-e2e-test) +# TARGET_GAIN gate %% (default 100) +# POLL_INTERVAL_S seconds between polls (default 120) +# GLOBAL_TIMEOUT_S hard cap; unfinished legs -> FAIL +# (default 50400 = 14h) +# MAX_CRASHES / MAX_BOOT_FAILS tolerance (default 0 / 0) +# Optional GitHub commit status (per-leg context pre-release-e2e/): +# GH_STATUS_TOKEN / GH_STATUS_REPO / GH_STATUS_SHA / GH_STATUS_DETAILS_URL +# SAFE_CACERT / SAFE_INSECURE TLS to the API +set -euo pipefail + +NFS_ROOT="${NFS_ROOT:-/shared_nfs/hyperloom-pre-release-e2e-test}" +TARGET_GAIN="${TARGET_GAIN:-100}" +POLL_INTERVAL_S="${POLL_INTERVAL_S:-120}" +GLOBAL_TIMEOUT_S="${GLOBAL_TIMEOUT_S:-50400}" +MAX_CRASHES="${MAX_CRASHES:-0}" +MAX_BOOT_FAILS="${MAX_BOOT_FAILS:-0}" + +: "${SAFE_API_BASE:?SAFE_API_BASE is required}" +: "${SAFE_API_KEY:?SAFE_API_KEY is required}" +: "${CI_VERSION:?CI_VERSION is required}" +: "${DISPATCH_MAP:?DISPATCH_MAP is required}" +[ -f "$DISPATCH_MAP" ] || { echo "dispatch map $DISPATCH_MAP not found" >&2; exit 2; } + +API="${SAFE_API_BASE%/}/api/v1/workloads" +auth=(-H "Authorization: Bearer ${SAFE_API_KEY}") +tls=() +if [ -n "${SAFE_CACERT:-}" ]; then + tls=(--cacert "$SAFE_CACERT") +elif [ "${SAFE_INSECURE:-0}" = "1" ]; then + tls=(-k) +fi +GH_API="${GH_API:-https://api.github.com}" + +runs_dir="${NFS_ROOT%/}/runs/${CI_VERSION}" +summary() { echo "$*" | tee -a "${GITHUB_STEP_SUMMARY:-/dev/null}"; } + +gh_status_on() { [ -n "${GH_STATUS_TOKEN:-}" ] && [ -n "${GH_STATUS_REPO:-}" ] && [ -n "${GH_STATUS_SHA:-}" ]; } +post_status() { # leg state(pending|success|failure|error) description + gh_status_on || return 0 + local desc="${3:0:139}" + curl -sS -X POST \ + -H "Authorization: Bearer ${GH_STATUS_TOKEN}" \ + -H "Accept: application/vnd.github+json" \ + -H "X-GitHub-Api-Version: 2022-11-28" \ + "${GH_API}/repos/${GH_STATUS_REPO}/statuses/${GH_STATUS_SHA}" \ + -d "$(jq -n --arg s "$2" --arg d "$desc" --arg u "${GH_STATUS_DETAILS_URL:-}" --arg c "pre-release-e2e/$1" \ + '{state:$s, description:$d, context:$c} + (if $u=="" then {} else {target_url:$u} end)')" \ + >/dev/null 2>&1 || true +} + +workload_phase() { # workloadId -> phase string + local wid="$1" detail + detail="$(curl -sS "${tls[@]}" "$API/$wid" "${auth[@]}" 2>/dev/null || true)" + printf '%s' "$detail" | jq -r '.phase // "Unknown"' 2>/dev/null || echo Unknown +} + +# Resolve a leg's session dir. Bootstrap writes the pinned session dir to +# runs///session/.session_dir (design §9: never guess by timestamp). +leg_session_dir() { + local leg="$1" pin + pin="${runs_dir}/${leg}/session/.session_dir" + if [ -f "$pin" ]; then head -n1 "$pin"; return; fi + echo "" +} + +# Judge one leg from its final.json. Echoes "PASS"|"FAIL|". +judge_leg() { + local leg="$1" wphase="$2" sdir final gain stop crashes boots + sdir="$(leg_session_dir "$leg")" + if [ -z "$sdir" ] || [ ! -d "$sdir" ]; then + echo "FAIL|no session dir yet (workload phase=$wphase)"; return + fi + final="${sdir%/}/reports/final.json" + if [ ! -f "$final" ]; then + echo "FAIL|reports/final.json missing (workload phase=$wphase)"; return + fi + if [ ! -f "${sdir%/}/reports/final.md" ]; then + echo "FAIL|reports/final.md missing"; return + fi + stop="$(jq -r '.stop_reason // ""' "$final" 2>/dev/null || echo "")" + gain="$(jq -r '.cumulative_gain_validated // 0' "$final" 2>/dev/null || echo 0)" + crashes="$(jq -r '.crash_count // 0' "$final" 2>/dev/null || echo 0)" + boots="$(jq -r '.server_boot_failures // 0' "$final" 2>/dev/null || echo 0)" + if [ "$crashes" -gt "$MAX_CRASHES" ] 2>/dev/null; then + echo "FAIL|crash_count=$crashes > $MAX_CRASHES"; return + fi + if [ "$boots" -gt "$MAX_BOOT_FAILS" ] 2>/dev/null; then + echo "FAIL|server_boot_failures=$boots > $MAX_BOOT_FAILS"; return + fi + # Primary gate: stop_reason target_reached (== gain >= TARGET_GAIN). + if [ "$stop" = "target_reached" ]; then + echo "PASS|gain=${gain}% stop=${stop}"; return + fi + # Fallback: numeric compare in case stop_reason lags (awk for float). + if awk -v g="$gain" -v t="$TARGET_GAIN" 'BEGIN{exit !(g+0 >= t+0)}'; then + echo "PASS|gain=${gain}% (>= ${TARGET_GAIN})"; return + fi + echo "FAIL|gain=${gain}% < ${TARGET_GAIN} (stop=${stop:-none})" +} + +# ---- poll loop ------------------------------------------------------------- +mapfile -t LEGS < <(jq -r 'keys[]' "$DISPATCH_MAP") +declare -A WID VERDICT +for leg in "${LEGS[@]}"; do + WID["$leg"]="$(jq -r --arg l "$leg" '.[$l]' "$DISPATCH_MAP")" + VERDICT["$leg"]="" + post_status "$leg" pending "dispatched; workload=${WID[$leg]}" +done +summary "## Pre-release E2E — CI_VERSION \`$CI_VERSION\`" +summary "" +summary "Polling ${#LEGS[@]} legs (global timeout $((GLOBAL_TIMEOUT_S/3600))h). Each leg reports on its own terminal (point C)." +summary "" + +start_s="$(date +%s)" +while :; do + pending=0 + for leg in "${LEGS[@]}"; do + [ -n "${VERDICT[$leg]}" ] && continue + wid="${WID[$leg]}" + wphase="$(workload_phase "$wid")" + # Terminal SaFE failure kills the leg immediately. + if [ "$wphase" = "Failed" ] || [ "$wphase" = "Stopped" ]; then + VERDICT["$leg"]="FAIL|workload phase=$wphase" + summary "❌ **$leg** — FAIL (workload $wphase, wid=\`$wid\`)" + post_status "$leg" failure "workload $wphase; wid=$wid" + continue + fi + # Otherwise judge from the on-disk report (present once the leg finishes). + res="$(judge_leg "$leg" "$wphase")" + verdict="${res%%|*}"; detail="${res#*|}" + if [ "$verdict" = "PASS" ]; then + VERDICT["$leg"]="PASS|$detail" + summary "✅ **$leg** — PASS ($detail)" + post_status "$leg" success "PASS — $detail" + elif [ "$wphase" = "Succeeded" ]; then + # Workload ended but the report did not clear the gate -> terminal FAIL. + VERDICT["$leg"]="FAIL|$detail" + summary "❌ **$leg** — FAIL ($detail)" + post_status "$leg" failure "FAIL — $detail" + else + pending=$((pending + 1)) # still running; check again next tick + fi + done + + [ "$pending" -eq 0 ] && break + + elapsed=$(( $(date +%s) - start_s )) + if [ "$elapsed" -ge "$GLOBAL_TIMEOUT_S" ]; then + for leg in "${LEGS[@]}"; do + [ -n "${VERDICT[$leg]}" ] && continue + VERDICT["$leg"]="FAIL|global timeout after ${elapsed}s" + summary "❌ **$leg** — FAIL (global timeout)" + post_status "$leg" failure "global timeout after $((elapsed/3600))h" + done + break + fi + echo "[poll] ${pending} leg(s) still running; elapsed $((elapsed/60))m; sleeping ${POLL_INTERVAL_S}s" + sleep "$POLL_INTERVAL_S" +done + +# ---- aggregate gate -------------------------------------------------------- +summary "" +summary "### Result" +summary "" +summary "| leg | verdict | detail |" +summary "|-----|---------|--------|" +fail=0 +for leg in "${LEGS[@]}"; do + v="${VERDICT[$leg]:-FAIL|no verdict}" + vv="${v%%|*}"; vd="${v#*|}" + icon="✅"; [ "$vv" = "PASS" ] || { icon="❌"; fail=1; } + summary "| \`$leg\` | $icon $vv | $vd |" +done +summary "" +if [ "$fail" -eq 0 ]; then + summary "**GATE: PASS** — all ${#LEGS[@]} legs reached target_gain=${TARGET_GAIN}." + exit 0 +fi +summary "**GATE: FAIL** — one or more legs did not pass. Release blocked." +exit 1 diff --git a/.github/workflows/pre-release-e2e-test.yml b/.github/workflows/pre-release-e2e-test.yml new file mode 100644 index 0000000000..44418600a8 --- /dev/null +++ b/.github/workflows/pre-release-e2e-test.yml @@ -0,0 +1,267 @@ +# SPDX-FileCopyrightText: 2026 Advanced Micro Devices, Inc. +# SPDX-License-Identifier: MIT + +name: Pre-release E2E test + +# Full-path release gate: build the CI wheel, push it to NFS, then run 8 demo legs +# (baremetal/docker x vLLM/SGLang x 3h/12h) as SaFE Authoring workloads driven through +# the Claude CLI + setup/demo skills -- the same path a user takes. Independent of the +# per-PR smoke test (ci-e2e.yml). See hyperloom-pre-release-e2e-ci-design.md. +# +# Triggers: +# * push to main that changes pyproject.toml's `version` field -> FULL run (all 8 legs) +# * push to main that changes this CI's scripts/prompts/workflow (version unchanged) +# -> SCRIPTS-ONLY run (the 4 fast 3h legs, to validate the logic change) +# * manual workflow_dispatch (reuse a wheel, run a leg subset; default all 8) +# +# ── Environment values to fill in (repo Secrets/Variables) ────────────────── +# Secrets (sensitive): SAFE_API_KEY (ADMIN token: privileged 8-GPU pod is admin-only), +# ANTHROPIC_API_KEY. +# Variables (non-secret): SAFE_API_BASE, SAFE_WORKSPACE_ID, AUTHORING_IMAGE, +# PRE_RELEASE_NFS_ROOT, MODEL_3H_PATH, MODEL_12H_PATH, CLAUDE_MODEL, +# CLAUDE_CLI_VERSION, (optional) ANTHROPIC_BASE_URL, SAFE_INSECURE. +# Nothing below hard-codes an environment value; all are `${{ secrets.* }}` / +# `${{ vars.* }}` references. TODO(owner): populate these before first real run. + +on: + push: + branches: [main] + # A push must touch one of these to even start the workflow; `resolve` then + # classifies it into a FULL run (version bump) or a SCRIPTS-ONLY run (CI logic + # changed but version unchanged). + paths: + - "pyproject.toml" # version bump -> full release gate + - ".github/workflows/pre-release-e2e-test.yml" # the workflow itself + - ".github/scripts/pre-release-e2e-*.sh" # dispatch / poll + - ".github/pre-release/**" # bootstrap, docker-run, prompts + workflow_dispatch: + inputs: + reuse_ci_version: + description: "Reuse an existing wheel dir (skip build); e.g. 1.0.0b3.dev202608261530+ci" + required: false + tasks: + description: "Comma-separated subset of leg IDs (default: all 8)" + required: false + +# No cross-version concurrency (design §13, point D): serialize pre-release runs so the +# peak GPU footprint stays at 8. A newer run QUEUES behind the in-flight one. +concurrency: + group: pre-release-e2e + cancel-in-progress: false + +permissions: + contents: read + statuses: write + +jobs: + # 1. resolve: gate on a real version bump (push) or manual input; compute CI_VERSION. + resolve: + runs-on: Hyperloom-e2e-ci + outputs: + run: ${{ steps.decide.outputs.run }} + run_scope: ${{ steps.decide.outputs.run_scope }} + ci_version: ${{ steps.decide.outputs.ci_version }} + base_version: ${{ steps.decide.outputs.base_version }} + reuse: ${{ steps.decide.outputs.reuse }} + tasks: ${{ steps.decide.outputs.tasks }} + steps: + - name: Ensure jq + run: command -v jq >/dev/null 2>&1 || (sudo apt-get update && sudo apt-get install -y jq) + + - uses: actions/checkout@v7 + with: + fetch-depth: 2 # need HEAD~1 to diff the version field on push + + - name: Decide + compute CI_VERSION + id: decide + env: + EVENT: ${{ github.event_name }} + REUSE_IN: ${{ inputs.reuse_ci_version }} + TASKS_IN: ${{ inputs.tasks }} + run: | + set -euo pipefail + # The 4 fast 3h legs used when only CI logic (scripts/prompts/workflow) + # changed but the release version did not -- enough to exercise the changed + # dispatch/bootstrap/poll path without burning a full 14h GPU round. + SCRIPTS_ONLY_TASKS="baremetal-vllm-3h,baremetal-sglang-3h,docker-vllm-3h,docker-sglang-3h" + run=false; run_scope="none"; reuse=""; tasks="${TASKS_IN:-}" + # Extract the [project] version line. The field is a simple quoted string in + # this repo's pyproject.toml, so a scoped grep is deterministic and avoids + # embedding a Python heredoc in the YAML block scalar. + read_version() { # reads TOML on stdin, prints project.version + awk ' + /^\[project\]/{inp=1; next} + /^\[/{inp=0} + inp && /^[[:space:]]*version[[:space:]]*=/{ + gsub(/.*=[[:space:]]*"|".*/,""); print; exit + }' + } + base_version="$(read_version < pyproject.toml)" + if [ "$EVENT" = "workflow_dispatch" ]; then + # Manual runs are full-scope; tasks default to all 8 (empty = all in dispatch). + run=true; run_scope="full"; reuse="${REUSE_IN:-}" + else + # push to main. Classify: version bump -> full; else CI logic changed -> scripts-only. + prev="$(git show HEAD~1:pyproject.toml 2>/dev/null | read_version || true)" + if [ "$base_version" != "$prev" ] && [ -n "$base_version" ]; then + run=true; run_scope="full" + echo "version bump: '${prev}' -> '${base_version}' -> FULL run (all 8 legs)" + else + # Version unchanged. on.push.paths already guaranteed this push touched one + # of the watched paths, and it wasn't the version -> it changed the CI's own + # scripts/prompts/workflow. Run the fast scripts-only scope. (The diff is + # best-effort logging; across a multi-commit push HEAD~1 may not show every + # changed file, so we do NOT gate on it -- the path filter already did.) + run=true; run_scope="scripts-only" + changed="$(git diff --name-only HEAD~1 HEAD 2>/dev/null | grep -E \ + '^\.github/(workflows/pre-release-e2e-test\.yml|scripts/pre-release-e2e-.*\.sh|pre-release/)' \ + | paste -sd',' - || true)" + echo "version unchanged; CI logic changed (${changed:-see path filter}) -> SCRIPTS-ONLY run (4 fast 3h legs)" + fi + fi + + # Default the task set from the scope when the caller didn't pin one. + if [ -z "$tasks" ] && [ "$run_scope" = "scripts-only" ]; then + tasks="$SCRIPTS_ONLY_TASKS" + fi + + if [ -n "$reuse" ]; then + ci_version="$reuse" + else + ci_version="${base_version}.dev$(date -u +%Y%m%d%H%M)+ci" + fi + { + echo "run=$run" + echo "run_scope=$run_scope" + echo "reuse=$reuse" + echo "tasks=$tasks" + echo "base_version=$base_version" + echo "ci_version=$ci_version" + } >> "$GITHUB_OUTPUT" + echo "decision: run=$run scope=$run_scope ci_version=$ci_version reuse='${reuse}' tasks='${tasks}'" + + # 2. build: build the wheel, publish to NFS, stage bootstrap + prompts, write manifest. + build: + needs: resolve + if: needs.resolve.outputs.run == 'true' && needs.resolve.outputs.reuse == '' + runs-on: Hyperloom-e2e-ci + env: + CI_VERSION: ${{ needs.resolve.outputs.ci_version }} + BASE_VERSION: ${{ needs.resolve.outputs.base_version }} + NFS_ROOT: ${{ vars.PRE_RELEASE_NFS_ROOT }} + steps: + - uses: actions/checkout@v7 + + - uses: actions/setup-python@v7 + with: + python-version: "3.11" + + - name: Build wheel with the CI version + id: build + run: | + set -euo pipefail + pip install --upgrade pip + pip install "setuptools>=77" wheel build + # Override the version for this build only (never committed). + SETUPTOOLS_SCM_PRETEND_VERSION="$CI_VERSION" \ + python -m build --wheel --no-isolation --outdir dist \ + -C--global-option=egg_info -C--global-option=--tag-build= 2>&1 | tee build.log || \ + python -m build --wheel --no-isolation --outdir dist + echo "wheel=$(ls dist/*.whl | head -n1)" >> "$GITHUB_OUTPUT" + + - name: Publish wheel + bootstrap + prompts + manifest to NFS + env: + WHEEL: ${{ steps.build.outputs.wheel }} + GIT_SHA: ${{ github.sha }} + RUN_ID: ${{ github.run_id }} + run: | + set -euo pipefail + : "${NFS_ROOT:?PRE_RELEASE_NFS_ROOT var is required}" + wheel_dir="${NFS_ROOT%/}/wheels/${CI_VERSION}" + runs_dir="${NFS_ROOT%/}/runs/${CI_VERSION}" + boot_dir="${NFS_ROOT%/}/bootstrap/${CI_VERSION}" + mkdir -p "$wheel_dir" "$runs_dir" "$boot_dir/prompts/pre-release" + # Wheel named after CI_VERSION per design §5. + cp "$WHEEL" "$wheel_dir/" + # Stage the in-pod scripts + fixed prompts (pods read them from NFS). + cp .github/pre-release/bootstrap-pre-release.sh .github/pre-release/docker-run-hyperloom.sh "$boot_dir/" + cp .github/pre-release/prompts/pre-release/*.txt "$boot_dir/prompts/pre-release/" + chmod +x "$boot_dir"/*.sh + # manifest.json (design §5). + jq -n \ + --arg wd "$wheel_dir" --arg rd "$runs_dir" --arg sha "$GIT_SHA" \ + --arg rid "$RUN_ID" --arg bv "$BASE_VERSION" --arg civ "$CI_VERSION" \ + '{ci_version:$civ, base_version:$bv, wheel_dir:$wd, runs_dir:$rd, + git_sha:$sha, github_run_id:$rid, + tasks:["baremetal-vllm-3h","baremetal-vllm-12h","baremetal-sglang-3h","baremetal-sglang-12h", + "docker-vllm-3h","docker-vllm-12h","docker-sglang-3h","docker-sglang-12h"]}' \ + > "$wheel_dir/manifest.json" + echo "published to $wheel_dir"; cat "$wheel_dir/manifest.json" + + # 3+4. dispatch + poll: create the 5 SaFE workloads, then judge each leg (point C). + run: + needs: [resolve, build] + # Run whenever resolve said go and build didn't fail (build is skipped on reuse). + if: >- + always() && needs.resolve.outputs.run == 'true' && + (needs.build.result == 'success' || needs.build.result == 'skipped') + runs-on: Hyperloom-e2e-ci + timeout-minutes: 900 # ≥14h: covers the 12h leg + bootstrap/setup/agent overhead + env: + CI_VERSION: ${{ needs.resolve.outputs.ci_version }} + TASKS: ${{ needs.resolve.outputs.tasks }} + NFS_ROOT: ${{ vars.PRE_RELEASE_NFS_ROOT }} + TARGET_GAIN: "100" + # SaFE API + SAFE_API_BASE: ${{ vars.SAFE_API_BASE }} + SAFE_API_KEY: ${{ secrets.SAFE_API_KEY }} # ADMIN token (privileged pod) + SAFE_WORKSPACE_ID: ${{ vars.SAFE_WORKSPACE_ID }} + SAFE_INSECURE: ${{ vars.SAFE_INSECURE || '1' }} + AUTHORING_IMAGE: ${{ vars.AUTHORING_IMAGE }} + # models + agent + MODEL_3H: ${{ vars.MODEL_3H_PATH }} + MODEL_12H: ${{ vars.MODEL_12H_PATH }} + CLAUDE_MODEL: ${{ vars.CLAUDE_MODEL }} + CLAUDE_CLI_VERSION: ${{ vars.CLAUDE_CLI_VERSION }} + ANTHROPIC_API_KEY: ${{ secrets.ANTHROPIC_API_KEY }} # GitHub Secret path (design §9 D) + ANTHROPIC_BASE_URL: ${{ vars.ANTHROPIC_BASE_URL }} + # per-leg GitHub commit status + GH_STATUS_TOKEN: ${{ secrets.GITHUB_TOKEN }} + GH_STATUS_REPO: ${{ github.repository }} + GH_STATUS_SHA: ${{ github.sha }} + GH_STATUS_DETAILS_URL: ${{ github.server_url }}/${{ github.repository }}/actions/runs/${{ github.run_id }} + POLL_INTERVAL_S: "120" + GLOBAL_TIMEOUT_S: "50400" + steps: + - uses: actions/checkout@v7 + + - name: Ensure jq + shared dispatch-map path + # RUNNER_TEMP is an env var inside run steps (the `runner.*` context is not + # available in job-level env:), so publish DISPATCH_MAP to the job env here. + run: | + command -v jq >/dev/null 2>&1 || (sudo apt-get update && sudo apt-get install -y jq) + echo "DISPATCH_MAP=${RUNNER_TEMP}/pre_release_dispatch.json" >> "$GITHUB_ENV" + + - name: Dispatch SaFE Authoring workloads + run: | + chmod +x .github/scripts/pre-release-e2e-dispatch.sh + .github/scripts/pre-release-e2e-dispatch.sh + + - name: Poll legs + write release gate + run: | + chmod +x .github/scripts/pre-release-e2e-poll.sh + .github/scripts/pre-release-e2e-poll.sh + + - name: Cancel workloads on job cancel + if: cancelled() + env: + SAFE_API_BASE: ${{ vars.SAFE_API_BASE }} + SAFE_API_KEY: ${{ secrets.SAFE_API_KEY }} + run: | + [ -f "$DISPATCH_MAP" ] || { echo "no dispatch map; nothing to cancel"; exit 0; } + for wid in $(jq -r '.[]' "$DISPATCH_MAP" | sort -u); do + echo "cancelling workload $wid" + curl -sS -k -o /dev/null -w 'DELETE %{http_code}\n' -X DELETE \ + -H "Authorization: Bearer ${SAFE_API_KEY}" \ + "${SAFE_API_BASE%/}/api/v1/workloads/${wid}" || true + done From 61d52fe47024fdb4deacabb8cfbccf1618106901 Mon Sep 17 00:00:00 2001 From: lishuoshuo-amd Date: Thu, 27 Aug 2026 14:20:25 +0800 Subject: [PATCH 02/52] Add sticky PR report comment and per-leg pod deadlines. The poll step now upserts ONE sticky comment (matched by an HTML marker, same pattern as ci-e2e-dispatch.sh) and PATCHes it in place as each leg finishes, so a single comment updates incrementally (design point C). Since this CI runs on push to main (no PR number), it resolves the PR the triggering commit was merged from via GET /commits/{sha}/pulls, falling back to a commit comment when none is found. Widens the workflow token to pull-requests/issues/contents: write. Dispatch now attaches a pod hard-deadline per duration -- 3h legs get 3+1h (14400s), 12h legs get 12+1h (46800s). The shared privileged docker host runs a mix of 3h and 12h nested legs, so its deadline is the MAX over its legs (else a 3h deadline would kill a still-running 12h leg). SaFE terminates the pod at the deadline; the poll then judges that leg FAIL. The deadline field name defaults to activeDeadlineSeconds (k8s convention) and is TODO-flagged for the owner to confirm against the real SaFE Authoring API; DEADLINE_FIELD="" omits it. Co-Authored-By: Claude Opus 4.8 --- .github/scripts/pre-release-e2e-dispatch.sh | 47 ++++++-- .github/scripts/pre-release-e2e-poll.sh | 121 +++++++++++++++++++- .github/workflows/pre-release-e2e-test.yml | 6 +- 3 files changed, 160 insertions(+), 14 deletions(-) diff --git a/.github/scripts/pre-release-e2e-dispatch.sh b/.github/scripts/pre-release-e2e-dispatch.sh index dd317584b7..07064d9b6f 100755 --- a/.github/scripts/pre-release-e2e-dispatch.sh +++ b/.github/scripts/pre-release-e2e-dispatch.sh @@ -41,6 +41,12 @@ # (default 128 / 512Gi / 256Gi) # LEG_CPU / LEG_MEM baremetal leg resource request # (default 32 / 128Gi) +# DEADLINE_3H_S / DEADLINE_12H_S pod hard-timeout per duration +# (default 14400 = 3+1h / 46800 = 12+1h). The docker host +# pod uses the MAX over its legs. SaFE kills the pod at the +# deadline; poll then judges that leg FAIL. +# DEADLINE_FIELD SaFE payload field for the deadline (default +# activeDeadlineSeconds; set "" to omit). TODO(owner): confirm. # SAFE_CACERT / SAFE_INSECURE TLS to the API (CA bundle / skip-verify) set -euo pipefail @@ -50,6 +56,17 @@ HOST_CPU="${HOST_CPU:-128}"; HOST_MEM="${HOST_MEM:-512Gi}"; HOST_SHM="${HOST_SHM LEG_CPU="${LEG_CPU:-32}"; LEG_MEM="${LEG_MEM:-128Gi}" DISPATCH_MAP="${DISPATCH_MAP:-${RUNNER_TEMP:-/tmp}/pre_release_dispatch.json}" +# Pod hard-timeout (design: 3h leg -> 3+1h, 12h leg -> 12+1h). SaFE terminates the +# workload at the deadline; the poll then sees a non-Succeeded terminal / missing +# report and judges that leg FAIL. Given per duration: +DEADLINE_3H_S="${DEADLINE_3H_S:-14400}" # 3h + 1h buffer = 4h +DEADLINE_12H_S="${DEADLINE_12H_S:-46800}" # 12h + 1h buffer = 13h +# The SaFE API field that carries the pod deadline. TODO(owner): confirm the real +# field name/placement against the SaFE Authoring API; k8s convention is +# activeDeadlineSeconds (integer seconds). Set DEADLINE_FIELD="" to omit entirely. +DEADLINE_FIELD="${DEADLINE_FIELD:-activeDeadlineSeconds}" +leg_deadline_s() { case "$1" in *-3h) echo "$DEADLINE_3H_S" ;; *-12h) echo "$DEADLINE_12H_S" ;; esac; } + : "${SAFE_API_BASE:?SAFE_API_BASE is required}" : "${SAFE_API_KEY:?SAFE_API_KEY is required}" : "${SAFE_WORKSPACE_ID:?SAFE_WORKSPACE_ID is required}" @@ -118,14 +135,19 @@ common_env_json() { } + (if $baseurl == "" then {} else {ANTHROPIC_BASE_URL: $baseurl} end)' } -# POST one workload; echo the workloadId. Args: displayName resourcesJson envJson privileged(true|false) +# POST one workload; echo the workloadId. +# Args: displayName resourcesJson envJson privileged(true|false) entry_b64 deadline_s create_workload() { - local name="$1" resources="$2" env="$3" privileged="$4" entry_b64="$5" - local body resp code json wid + local name="$1" resources="$2" env="$3" privileged="$4" entry_b64="$5" deadline_s="${6:-}" + local body resp code json wid dl_json="{}" + # Attach the pod hard-deadline when both a field name and a value are set. + if [ -n "$DEADLINE_FIELD" ] && [ -n "$deadline_s" ]; then + dl_json="$(jq -n --arg k "$DEADLINE_FIELD" --argjson v "$deadline_s" '{($k): $v}')" + fi body="$(jq -n \ --arg name "$name" --arg ws "$SAFE_WORKSPACE_ID" --arg img "$AUTHORING_IMAGE" \ --arg entry "$entry_b64" --argjson res "$resources" --argjson env "$env" \ - --argjson priv "$privileged" \ + --argjson priv "$privileged" --argjson dl "$dl_json" \ '{ displayName: $name, workspaceId: $ws, @@ -135,7 +157,7 @@ create_workload() { entryPoints: [$entry], env: $env, useWorkspaceStorage: true - } + (if $priv then {privileged:true} else {} end)')" + } + (if $priv then {privileged:true} else {} end) + $dl')" resp="$(curl -sS "${tls[@]}" -w $'\n%{http_code}' -X POST "$API" \ "${auth[@]}" -H "Content-Type: application/json" -d "$body")" code="$(printf '%s' "$resp" | tail -n1)" @@ -176,9 +198,10 @@ for leg in $REQ_TASKS; do env_json="$(common_env_json "$(leg_model_path "$leg")" "$(leg_hours "$leg")" "$(leg_backend "$leg")" \ | jq --arg leg "$leg" '. + {LEG_ID:$leg, HYPERLOOM_RUN_MODE:"baremetal"}')" entry="$(bootstrap_entry_b64 "")" - wid="$(create_workload "e2e-${CI_VERSION}-${leg}" "$leg_resources_1gpu" "$env_json" false "$entry")" + dl="$(leg_deadline_s "$leg")" + wid="$(create_workload "e2e-${CI_VERSION}-${leg}" "$leg_resources_1gpu" "$env_json" false "$entry" "$dl")" DISPATCH["$leg"]="$wid" - summary "• \`$leg\` → workloadId \`$wid\` (baremetal, 1 GPU)" + summary "• \`$leg\` → workloadId \`$wid\` (baremetal, 1 GPU, deadline $((dl/3600))h)" ;; docker-*) want_docker_host=1 @@ -223,12 +246,18 @@ if [ "$want_docker_host" = 1 ]; then DOCKER_LEGS:$legs, DOCKER_GPU_MAP:($gpumap|tostring) } + (if $baseurl == "" then {} else {ANTHROPIC_BASE_URL:$baseurl} end)')" entry="$(bootstrap_entry_b64 "E2E_DOCKER_HOST=1;")" - wid="$(create_workload "e2e-${CI_VERSION}-docker-host" "$host_resources" "$host_env" true "$entry")" + # The one host pod runs a mix of 3h and 12h nested legs, so its deadline must be + # the MAX over the legs it hosts (a 3h deadline would kill a still-running 12h leg). + host_dl="$DEADLINE_3H_S" + for leg in $docker_legs; do + case "$leg" in *-12h) host_dl="$DEADLINE_12H_S" ;; esac + done + wid="$(create_workload "e2e-${CI_VERSION}-docker-host" "$host_resources" "$host_env" true "$entry" "$host_dl")" # Every docker leg shares the one host workloadId; the poll distinguishes them by # reading each leg's own session dir on NFS. for leg in $docker_legs; do DISPATCH["$leg"]="$wid" - summary "• \`$leg\` → workloadId \`$wid\` (docker host, GPU $(docker_gpu_index "$leg"))" + summary "• \`$leg\` → workloadId \`$wid\` (docker host, GPU $(docker_gpu_index "$leg"), deadline $((host_dl/3600))h)" done fi diff --git a/.github/scripts/pre-release-e2e-poll.sh b/.github/scripts/pre-release-e2e-poll.sh index ebae898116..ca3137f8af 100755 --- a/.github/scripts/pre-release-e2e-poll.sh +++ b/.github/scripts/pre-release-e2e-poll.sh @@ -74,6 +74,103 @@ post_status() { # leg state(pending|success|failure|error) description >/dev/null 2>&1 || true } +# ---- sticky PR/commit report comment --------------------------------------- +# This CI runs on push to main, which carries no PR number. Resolve the PR that +# the triggering commit was merged from (GET /commits/{sha}/pulls); if none is +# found, fall back to a commit comment. Then upsert ONE sticky comment (matched +# by an HTML marker) and PATCH it in place as legs finish -- so a single comment +# updates incrementally (design §10, point C). Mirrors ci-e2e-dispatch.sh. +REPORT_MARKER="" +PR_NUMBER=""; COMMENT_TARGET="" # COMMENT_TARGET: "pr" | "commit" | "" (disabled) + +gh_report_on() { [ -n "${GH_STATUS_TOKEN:-}" ] && [ -n "${GH_STATUS_REPO:-}" ] && [ -n "${GH_STATUS_SHA:-}" ]; } + +resolve_comment_target() { + gh_report_on || { COMMENT_TARGET=""; return 0; } + # A merged commit usually belongs to exactly one PR; take the first. + PR_NUMBER="$(curl -sS \ + -H "Authorization: Bearer ${GH_STATUS_TOKEN}" \ + -H "Accept: application/vnd.github+json" \ + -H "X-GitHub-Api-Version: 2022-11-28" \ + "${GH_API}/repos/${GH_STATUS_REPO}/commits/${GH_STATUS_SHA}/pulls" 2>/dev/null \ + | jq -r '[.[]|.number][0] // empty' 2>/dev/null || true)" + if [[ "$PR_NUMBER" =~ ^[0-9]+$ ]]; then + COMMENT_TARGET="pr" + echo "[report] commenting on PR #$PR_NUMBER (from commit $GH_STATUS_SHA)" + else + COMMENT_TARGET="commit" + echo "[report] no PR for commit $GH_STATUS_SHA; commenting on the commit" + fi +} + +# GET/PATCH/POST helpers keyed by the marker. The PR path uses the issues API +# (a PR is an issue for comments); the commit path uses the commits API. +_comment_list_url() { + case "$COMMENT_TARGET" in + pr) echo "${GH_API}/repos/${GH_STATUS_REPO}/issues/${PR_NUMBER}/comments?per_page=100" ;; + commit) echo "${GH_API}/repos/${GH_STATUS_REPO}/commits/${GH_STATUS_SHA}/comments?per_page=100" ;; + esac +} +_comment_create_url() { + case "$COMMENT_TARGET" in + pr) echo "${GH_API}/repos/${GH_STATUS_REPO}/issues/${PR_NUMBER}/comments" ;; + commit) echo "${GH_API}/repos/${GH_STATUS_REPO}/commits/${GH_STATUS_SHA}/comments" ;; + esac +} +_comment_patch_url() { # comment id + case "$COMMENT_TARGET" in + pr) echo "${GH_API}/repos/${GH_STATUS_REPO}/issues/comments/$1" ;; + commit) echo "${GH_API}/repos/${GH_STATUS_REPO}/comments/$1" ;; + esac +} + +report_upsert() { # body(markdown, already includes the marker on line 1) + [ -n "$COMMENT_TARGET" ] || return 0 + local body="$1" cid + cid="$(curl -sS -H "Authorization: Bearer ${GH_STATUS_TOKEN}" -H "Accept: application/vnd.github+json" \ + "$(_comment_list_url)" 2>/dev/null \ + | jq -r --arg m "$REPORT_MARKER" '[.[]|select(.body|contains($m))|.id][0] // empty' 2>/dev/null || true)" + if [ -n "$cid" ]; then + curl -sS -X PATCH -H "Authorization: Bearer ${GH_STATUS_TOKEN}" -H "Accept: application/vnd.github+json" \ + "$(_comment_patch_url "$cid")" \ + -d "$(jq -n --arg b "$body" '{body:$b}')" >/dev/null 2>&1 || true + else + curl -sS -X POST -H "Authorization: Bearer ${GH_STATUS_TOKEN}" -H "Accept: application/vnd.github+json" \ + "$(_comment_create_url)" \ + -d "$(jq -n --arg b "$body" '{body:$b}')" >/dev/null 2>&1 || true + fi +} + +# Build the sticky report body from the current VERDICT map. `phase` is a short +# status word (Running|Complete) shown in the heading. Legs with no verdict yet +# render as "⏳ pending". +report_body() { # phase done_count total_count + local phase="$1" done="$2" total="$3" leg v vv vd icon rows="" + for leg in "${LEGS[@]}"; do + v="${VERDICT[$leg]:-}" + if [ -z "$v" ]; then + rows="${rows}| \`${leg}\` | ⏳ pending | running | +" + continue + fi + vv="${v%%|*}"; vd="${v#*|}" + icon="✅"; [ "$vv" = "PASS" ] || icon="❌" + rows="${rows}| \`${leg}\` | ${icon} ${vv} | ${vd} | +" + done + local detail_link="" + [ -n "${GH_STATUS_DETAILS_URL:-}" ] && detail_link="[run details](${GH_STATUS_DETAILS_URL})" + printf '%s\n## Pre-release E2E — %s (%d/%d legs done)\n\nCI_VERSION `%s` · target_gain %s%% · commit `%s`\n\n| leg | verdict | detail |\n|-----|---------|--------|\n%s\n%s\n' \ + "$REPORT_MARKER" "$phase" "$done" "$total" "$CI_VERSION" "$TARGET_GAIN" "$GH_STATUS_SHA" "$rows" "$detail_link" +} + +# Count legs that have a terminal verdict. +done_count() { + local n=0 leg + for leg in "${LEGS[@]}"; do [ -n "${VERDICT[$leg]:-}" ] && n=$((n+1)); done + echo "$n" +} + workload_phase() { # workloadId -> phase string local wid="$1" detail detail="$(curl -sS "${tls[@]}" "$API/$wid" "${auth[@]}" 2>/dev/null || true)" @@ -137,9 +234,15 @@ summary "" summary "Polling ${#LEGS[@]} legs (global timeout $((GLOBAL_TIMEOUT_S/3600))h). Each leg reports on its own terminal (point C)." summary "" +# Resolve the PR (from the triggering commit) or fall back to a commit comment, +# then post the initial "all pending" sticky report. +resolve_comment_target +report_upsert "$(report_body Running "$(done_count)" "${#LEGS[@]}")" + start_s="$(date +%s)" while :; do pending=0 + changed=0 # did any leg reach a verdict this tick? -> refresh the sticky comment for leg in "${LEGS[@]}"; do [ -n "${VERDICT[$leg]}" ] && continue wid="${WID[$leg]}" @@ -149,6 +252,7 @@ while :; do VERDICT["$leg"]="FAIL|workload phase=$wphase" summary "❌ **$leg** — FAIL (workload $wphase, wid=\`$wid\`)" post_status "$leg" failure "workload $wphase; wid=$wid" + changed=1 continue fi # Otherwise judge from the on-disk report (present once the leg finishes). @@ -158,16 +262,21 @@ while :; do VERDICT["$leg"]="PASS|$detail" summary "✅ **$leg** — PASS ($detail)" post_status "$leg" success "PASS — $detail" + changed=1 elif [ "$wphase" = "Succeeded" ]; then # Workload ended but the report did not clear the gate -> terminal FAIL. VERDICT["$leg"]="FAIL|$detail" summary "❌ **$leg** — FAIL ($detail)" post_status "$leg" failure "FAIL — $detail" + changed=1 else pending=$((pending + 1)) # still running; check again next tick fi done + # A leg finished this tick -> refresh the single sticky report comment (point C). + [ "$changed" -eq 1 ] && report_upsert "$(report_body Running "$(done_count)" "${#LEGS[@]}")" + [ "$pending" -eq 0 ] && break elapsed=$(( $(date +%s) - start_s )) @@ -199,8 +308,14 @@ for leg in "${LEGS[@]}"; do done summary "" if [ "$fail" -eq 0 ]; then - summary "**GATE: PASS** — all ${#LEGS[@]} legs reached target_gain=${TARGET_GAIN}." - exit 0 + gate_line="**GATE: PASS** — all ${#LEGS[@]} legs reached target_gain=${TARGET_GAIN}." +else + gate_line="**GATE: FAIL** — one or more legs did not pass. Release blocked." fi -summary "**GATE: FAIL** — one or more legs did not pass. Release blocked." +summary "$gate_line" + +# Final sticky report: the completed table plus the gate verdict. +report_upsert "$(printf '%s\n\n%s\n' "$(report_body Complete "$(done_count)" "${#LEGS[@]}")" "$gate_line")" + +[ "$fail" -eq 0 ] && exit 0 exit 1 diff --git a/.github/workflows/pre-release-e2e-test.yml b/.github/workflows/pre-release-e2e-test.yml index 44418600a8..213a878cbe 100644 --- a/.github/workflows/pre-release-e2e-test.yml +++ b/.github/workflows/pre-release-e2e-test.yml @@ -50,8 +50,10 @@ concurrency: cancel-in-progress: false permissions: - contents: read - statuses: write + contents: write # commit-comment fallback when no PR is associated + statuses: write # per-leg commit statuses (pre-release-e2e/) + pull-requests: write # sticky PR report comment + issues: write # PR comments go through the issues API jobs: # 1. resolve: gate on a real version bump (push) or manual input; compute CI_VERSION. From 37819a38dd8d12c32a427c762f056bc827a77b44 Mon Sep 17 00:00:00 2001 From: lishuoshuo-amd Date: Thu, 27 Aug 2026 14:24:47 +0800 Subject: [PATCH 03/52] Convert pre-release prompts from .txt to .md and polish for markdown. The bootstrap script only reads these as plain text for `claude --print`, so the extension was never load-bearing. Markdown fits better -- they are natural-language instructions, and .md renders headings/lists/code blocks and matches the repo's SKILL.md style. Content is semantically unchanged (same flags, overrides, and hard-constraint wording -- this is an automated release gate); only the formatting was reorganized into sections. Updates the two bootstrap references and the build job's cp glob from *.txt to *.md. Co-Authored-By: Claude Opus 4.8 --- .github/pre-release/bootstrap-pre-release.sh | 4 +- .../prompts/pre-release/demo-12h.md | 40 ++++++++++++++++ .../prompts/pre-release/demo-12h.txt | 28 ----------- .../pre-release/{demo-3h.txt => demo-3h.md} | 38 +++++++++------ .../pre-release/setup-baremetal-sglang.md | 38 +++++++++++++++ .../pre-release/setup-baremetal-sglang.txt | 29 ------------ .../pre-release/setup-baremetal-vllm.md | 38 +++++++++++++++ .../pre-release/setup-baremetal-vllm.txt | 29 ------------ .../pre-release/setup-docker-sglang.md | 46 +++++++++++++++++++ .../pre-release/setup-docker-sglang.txt | 37 --------------- .../prompts/pre-release/setup-docker-vllm.md | 46 +++++++++++++++++++ .../prompts/pre-release/setup-docker-vllm.txt | 37 --------------- .github/workflows/pre-release-e2e-test.yml | 2 +- 13 files changed, 236 insertions(+), 176 deletions(-) create mode 100644 .github/pre-release/prompts/pre-release/demo-12h.md delete mode 100644 .github/pre-release/prompts/pre-release/demo-12h.txt rename .github/pre-release/prompts/pre-release/{demo-3h.txt => demo-3h.md} (50%) create mode 100644 .github/pre-release/prompts/pre-release/setup-baremetal-sglang.md delete mode 100644 .github/pre-release/prompts/pre-release/setup-baremetal-sglang.txt create mode 100644 .github/pre-release/prompts/pre-release/setup-baremetal-vllm.md delete mode 100644 .github/pre-release/prompts/pre-release/setup-baremetal-vllm.txt create mode 100644 .github/pre-release/prompts/pre-release/setup-docker-sglang.md delete mode 100644 .github/pre-release/prompts/pre-release/setup-docker-sglang.txt create mode 100644 .github/pre-release/prompts/pre-release/setup-docker-vllm.md delete mode 100644 .github/pre-release/prompts/pre-release/setup-docker-vllm.txt diff --git a/.github/pre-release/bootstrap-pre-release.sh b/.github/pre-release/bootstrap-pre-release.sh index 570825bc87..cb6d3271d8 100755 --- a/.github/pre-release/bootstrap-pre-release.sh +++ b/.github/pre-release/bootstrap-pre-release.sh @@ -84,8 +84,8 @@ run_leg() { set +a export PYTHONPATH="${root}:${PYTHONPATH:-}" - local setup_prompt="${PROMPTS_DIR}/setup-${run_mode}-${backend}.txt" - local demo_prompt; demo_prompt="${PROMPTS_DIR}/demo-${hours}h.txt" + local setup_prompt="${PROMPTS_DIR}/setup-${run_mode}-${backend}.md" + local demo_prompt; demo_prompt="${PROMPTS_DIR}/demo-${hours}h.md" [ -f "$setup_prompt" ] || { log "ERROR: missing $setup_prompt"; return 1; } [ -f "$demo_prompt" ] || { log "ERROR: missing $demo_prompt"; return 1; } diff --git a/.github/pre-release/prompts/pre-release/demo-12h.md b/.github/pre-release/prompts/pre-release/demo-12h.md new file mode 100644 index 0000000000..906038a2f2 --- /dev/null +++ b/.github/pre-release/prompts/pre-release/demo-12h.md @@ -0,0 +1,40 @@ +# Pre-release E2E — 12h demo leg + +You are running the Hyperloom pre-release E2E test non-interactively. Run the 12-hour +demo to completion, then stop. Setup already ran successfully in this workspace. + +Invoke the `hyperloom-qwen3-14b-fp8-12h` demo skill with **one** override and otherwise +its exact default flags. + +## Flags + +- **OVERRIDE:** use `--target-gain 100` (NOT the skill's default of 50). This is the + pre-release release gate — the run must reach a validated cumulative gain of 100%. +- Keep every other required flag exactly as the skill defines them: + + ``` + --tp 1 --conc 64 --isl 1024 --osl 1024 --precision fp8 --max-hours 12 + --max-minutes-framework-pct 0.01 --max-minutes-explore-pct 0.42 + --max-minutes-kernel-pct 0.42 + ``` + +## Model path + +The skill will ask which model to use. Do **not** ask interactively — use +`MODEL_PATH` from the repository-root `.env` (it is already set to the demo model, +Qwen3-14B-FP8). Verify that path contains `config.json`; if it does, use it and +continue without asking. Load LLM API keys/base URLs and `FRAMEWORK` from `.env`. + +## Hard constraints (automated release gate) + +- Do **not** modify any GPU-related environment variable or device visibility. +- Do **not** run `docker`, start/exec containers, or choose GPUs via `rocm-smi`. If + `HYPERLOOM_RUN_MODE=docker` you are already inside the correct single-GPU container. +- Do **not** modify `USER_DATA_PATH`. +- Do **not** print or copy secret values into output, reports, or logs. + +## Termination + +Let the run proceed to its terminal report (session `reports/final.json` + +`reports/final.md`). When it terminates, stop. The harness judges PASS/FAIL from the +session report — do not fabricate a result. diff --git a/.github/pre-release/prompts/pre-release/demo-12h.txt b/.github/pre-release/prompts/pre-release/demo-12h.txt deleted file mode 100644 index 161997a0a9..0000000000 --- a/.github/pre-release/prompts/pre-release/demo-12h.txt +++ /dev/null @@ -1,28 +0,0 @@ -You are running the Hyperloom pre-release E2E test non-interactively. Run the 12-hour -demo to completion, then stop. Setup already ran successfully in this workspace. - -Invoke the `hyperloom-qwen3-14b-fp8-12h` demo skill with ONE override and otherwise -its exact default flags: - -- OVERRIDE: use `--target-gain 100` (NOT the skill's default of 50). This is the - pre-release release gate — the run must reach a validated cumulative gain of 100%. -- Keep every other required flag exactly as the skill defines them: `--tp 1 - --conc 64 --isl 1024 --osl 1024 --precision fp8 --max-hours 12 - --max-minutes-framework-pct 0.01 --max-minutes-explore-pct 0.42 - --max-minutes-kernel-pct 0.42`. - -Model path: the skill will ask which model to use. Do NOT ask interactively — use -`MODEL_PATH` from the repository-root `.env` (it is already set to the demo model, -Qwen3-14B-FP8). Verify that path contains `config.json`; if it does, use it and -continue without asking. Load LLM API keys/base URLs and `FRAMEWORK` from `.env`. - -Hard constraints (automated release gate): -- Do NOT modify any GPU-related environment variable or device visibility. -- Do NOT run `docker`, start/exec containers, or choose GPUs via `rocm-smi`. If - HYPERLOOM_RUN_MODE=docker you are already inside the correct single-GPU container. -- Do NOT modify USER_DATA_PATH. -- Do NOT print or copy secret values into output, reports, or logs. - -Let the run proceed to its terminal report (session `reports/final.json` + -`reports/final.md`). When it terminates, stop. The harness judges PASS/FAIL from the -session report — do not fabricate a result. diff --git a/.github/pre-release/prompts/pre-release/demo-3h.txt b/.github/pre-release/prompts/pre-release/demo-3h.md similarity index 50% rename from .github/pre-release/prompts/pre-release/demo-3h.txt rename to .github/pre-release/prompts/pre-release/demo-3h.md index 21e0102628..c0c0c56c86 100644 --- a/.github/pre-release/prompts/pre-release/demo-3h.txt +++ b/.github/pre-release/prompts/pre-release/demo-3h.md @@ -1,28 +1,40 @@ +# Pre-release E2E — 3h demo leg + You are running the Hyperloom pre-release E2E test non-interactively. Run the 3-hour demo to completion, then stop. Setup already ran successfully in this workspace. -Invoke the `hyperloom-qwen3-8b-3h` demo skill with ONE override and otherwise its -exact default flags: +Invoke the `hyperloom-qwen3-8b-3h` demo skill with **one** override and otherwise its +exact default flags. + +## Flags -- OVERRIDE: use `--target-gain 100` (NOT the skill's default of 30). This is the +- **OVERRIDE:** use `--target-gain 100` (NOT the skill's default of 30). This is the pre-release release gate — the run must reach a validated cumulative gain of 100%. -- Keep every other required flag exactly as the skill defines them: `--tp 1 - --conc 64 --isl 1024 --osl 1024 --precision bf16 --max-hours 3 +- Keep every other required flag exactly as the skill defines them: + + ``` + --tp 1 --conc 64 --isl 1024 --osl 1024 --precision bf16 --max-hours 3 --max-minutes-explore-pct 0.39 --max-minutes-sweep-pct 0.01 --explore-force-exit-budget-pct 0.01 --no-framework-agent --no-kernel - --no-enable-conc-sweep --no-enable-roofline`. + --no-enable-conc-sweep --no-enable-roofline + ``` -Model path: the skill will ask which model to use. Do NOT ask interactively — use +## Model path + +The skill will ask which model to use. Do **not** ask interactively — use `MODEL_PATH` from the repository-root `.env` (it is already set to the demo model). Verify that path contains `config.json`; if it does, use it and continue without asking. Load LLM API keys/base URLs and `FRAMEWORK` from `.env`. -Hard constraints (automated release gate): -- Do NOT modify any GPU-related environment variable or device visibility. -- Do NOT run `docker`, start/exec containers, or choose GPUs via `rocm-smi`. If - HYPERLOOM_RUN_MODE=docker you are already inside the correct single-GPU container. -- Do NOT modify USER_DATA_PATH. -- Do NOT print or copy secret values into output, reports, or logs. +## Hard constraints (automated release gate) + +- Do **not** modify any GPU-related environment variable or device visibility. +- Do **not** run `docker`, start/exec containers, or choose GPUs via `rocm-smi`. If + `HYPERLOOM_RUN_MODE=docker` you are already inside the correct single-GPU container. +- Do **not** modify `USER_DATA_PATH`. +- Do **not** print or copy secret values into output, reports, or logs. + +## Termination Let the run proceed to its terminal report (session `reports/final.json` + `reports/final.md`). When it terminates, stop. The harness judges PASS/FAIL from the diff --git a/.github/pre-release/prompts/pre-release/setup-baremetal-sglang.md b/.github/pre-release/prompts/pre-release/setup-baremetal-sglang.md new file mode 100644 index 0000000000..9d87f3eab2 --- /dev/null +++ b/.github/pre-release/prompts/pre-release/setup-baremetal-sglang.md @@ -0,0 +1,38 @@ +# Pre-release E2E — setup (baremetal + SGLang) + +You are running the Hyperloom pre-release E2E test non-interactively. Complete the +setup step for a **baremetal + SGLang** leg, then stop. Do not run the demo yet. + +## Environment (already prepared) + +A `.env` file exists in the current workspace (`REPO_ROOT`) with these values already +set: `ANTHROPIC_API_KEY`, `CLAUDE_MODEL`, `USER_DATA_PATH`, +`HYPERLOOM_RUN_MODE=baremetal`, `FRAMEWORK=sglang`, `MODEL_PATH`, `TARGET_GAIN`, +`DEMO_HOURS`. The wheel is already installed via `pip install --target .` so a +`hyperloom/` package directory is present. + +## Fixed decisions + +Run the `hyperloom-setup` skill with these fixed decisions — do **not** ask +interactive questions; use the values already in `.env` and the environment: + +- **Run mode:** baremetal (`HYPERLOOM_RUN_MODE` is already `baremetal`; keep it). +- **Framework:** SGLang. Install the framework layer with the setup backend + (`--install-framework sglang`). +- **LLM provider / model / `USER_DATA_PATH`:** use the values already in `.env`; do + not change them. + +## Hard constraints (automated release gate) + +- Do **not** modify any GPU-related environment variable (`ROCR_VISIBLE_DEVICES`, + `HIP_VISIBLE_DEVICES`, `GPUS_PER_NODE`, etc.). The pod already exposes exactly one + GPU; do not override device visibility. +- Do **not** run `docker` and do **not** choose GPUs via `rocm-smi`. This is a + baremetal leg; setup runs on the host. +- Do **not** print, echo, or copy secret values (API keys) into output or logs. +- Do **not** modify `USER_DATA_PATH`. + +## Termination + +When setup completes successfully, stop. Report only `setup complete: baremetal/sglang`. +If setup hard-fails, report the failure and stop. diff --git a/.github/pre-release/prompts/pre-release/setup-baremetal-sglang.txt b/.github/pre-release/prompts/pre-release/setup-baremetal-sglang.txt deleted file mode 100644 index 5302ff40c0..0000000000 --- a/.github/pre-release/prompts/pre-release/setup-baremetal-sglang.txt +++ /dev/null @@ -1,29 +0,0 @@ -You are running the Hyperloom pre-release E2E test non-interactively. Complete the -setup step for a BAREMETAL + SGLang leg, then stop. Do not run the demo yet. - -Environment is already prepared. A `.env` file exists in the current workspace -(REPO_ROOT) with these values already set: ANTHROPIC_API_KEY, CLAUDE_MODEL, -USER_DATA_PATH, HYPERLOOM_RUN_MODE=baremetal, FRAMEWORK=sglang, MODEL_PATH, -TARGET_GAIN, DEMO_HOURS. The wheel is already installed via `pip install --target .` -so a `hyperloom/` package directory is present. - -Run the hyperloom-setup skill with these fixed decisions — do NOT ask interactive -questions, use the values already in `.env` and the environment: - -- Run mode: baremetal (HYPERLOOM_RUN_MODE is already `baremetal`; keep it). -- Framework: SGLang. Install the framework layer with the setup backend - (`--install-framework sglang`). -- LLM provider / model / USER_DATA_PATH: use the values already in `.env`; do not - change them. - -Hard constraints (this is an automated release gate): -- Do NOT modify any GPU-related environment variable (ROCR_VISIBLE_DEVICES, - HIP_VISIBLE_DEVICES, GPUS_PER_NODE, etc.). The pod already exposes exactly one - GPU; do not override device visibility. -- Do NOT run `docker` and do NOT choose GPUs via `rocm-smi`. This is a baremetal - leg; setup runs on the host. -- Do NOT print, echo, or copy secret values (API keys) into output or logs. -- Do NOT modify USER_DATA_PATH. - -When setup completes successfully, stop. Report only "setup complete: baremetal/sglang". -If setup hard-fails, report the failure and stop. diff --git a/.github/pre-release/prompts/pre-release/setup-baremetal-vllm.md b/.github/pre-release/prompts/pre-release/setup-baremetal-vllm.md new file mode 100644 index 0000000000..7c647ba75f --- /dev/null +++ b/.github/pre-release/prompts/pre-release/setup-baremetal-vllm.md @@ -0,0 +1,38 @@ +# Pre-release E2E — setup (baremetal + vLLM) + +You are running the Hyperloom pre-release E2E test non-interactively. Complete the +setup step for a **baremetal + vLLM** leg, then stop. Do not run the demo yet. + +## Environment (already prepared) + +A `.env` file exists in the current workspace (`REPO_ROOT`) with these values already +set: `ANTHROPIC_API_KEY`, `CLAUDE_MODEL`, `USER_DATA_PATH`, +`HYPERLOOM_RUN_MODE=baremetal`, `FRAMEWORK=vllm`, `MODEL_PATH`, `TARGET_GAIN`, +`DEMO_HOURS`. The wheel is already installed via `pip install --target .` so a +`hyperloom/` package directory is present. + +## Fixed decisions + +Run the `hyperloom-setup` skill with these fixed decisions — do **not** ask +interactive questions; use the values already in `.env` and the environment: + +- **Run mode:** baremetal (`HYPERLOOM_RUN_MODE` is already `baremetal`; keep it). +- **Framework:** vLLM. Install the framework layer with the setup backend + (`--install-framework vllm`, isolated framework env). +- **LLM provider / model / `USER_DATA_PATH`:** use the values already in `.env`; do + not change them. + +## Hard constraints (automated release gate) + +- Do **not** modify any GPU-related environment variable (`ROCR_VISIBLE_DEVICES`, + `HIP_VISIBLE_DEVICES`, `GPUS_PER_NODE`, etc.). The pod already exposes exactly one + GPU; do not override device visibility. +- Do **not** run `docker` and do **not** choose GPUs via `rocm-smi`. This is a + baremetal leg; setup runs on the host. +- Do **not** print, echo, or copy secret values (API keys) into output or logs. +- Do **not** modify `USER_DATA_PATH`. + +## Termination + +When setup completes successfully, stop. Report only `setup complete: baremetal/vllm`. +If setup hard-fails, report the failure and stop. diff --git a/.github/pre-release/prompts/pre-release/setup-baremetal-vllm.txt b/.github/pre-release/prompts/pre-release/setup-baremetal-vllm.txt deleted file mode 100644 index d0f638f0d0..0000000000 --- a/.github/pre-release/prompts/pre-release/setup-baremetal-vllm.txt +++ /dev/null @@ -1,29 +0,0 @@ -You are running the Hyperloom pre-release E2E test non-interactively. Complete the -setup step for a BAREMETAL + vLLM leg, then stop. Do not run the demo yet. - -Environment is already prepared. A `.env` file exists in the current workspace -(REPO_ROOT) with these values already set: ANTHROPIC_API_KEY, CLAUDE_MODEL, -USER_DATA_PATH, HYPERLOOM_RUN_MODE=baremetal, FRAMEWORK=vllm, MODEL_PATH, -TARGET_GAIN, DEMO_HOURS. The wheel is already installed via `pip install --target .` -so a `hyperloom/` package directory is present. - -Run the hyperloom-setup skill with these fixed decisions — do NOT ask interactive -questions, use the values already in `.env` and the environment: - -- Run mode: baremetal (HYPERLOOM_RUN_MODE is already `baremetal`; keep it). -- Framework: vLLM. Install the framework layer with the setup backend - (`--install-framework vllm`, isolated framework env). -- LLM provider / model / USER_DATA_PATH: use the values already in `.env`; do not - change them. - -Hard constraints (this is an automated release gate): -- Do NOT modify any GPU-related environment variable (ROCR_VISIBLE_DEVICES, - HIP_VISIBLE_DEVICES, GPUS_PER_NODE, etc.). The pod already exposes exactly one - GPU; do not override device visibility. -- Do NOT run `docker` and do NOT choose GPUs via `rocm-smi`. This is a baremetal - leg; setup runs on the host. -- Do NOT print, echo, or copy secret values (API keys) into output or logs. -- Do NOT modify USER_DATA_PATH. - -When setup completes successfully, stop. Report only "setup complete: baremetal/vllm". -If setup hard-fails, report the failure and stop. diff --git a/.github/pre-release/prompts/pre-release/setup-docker-sglang.md b/.github/pre-release/prompts/pre-release/setup-docker-sglang.md new file mode 100644 index 0000000000..2d4791939b --- /dev/null +++ b/.github/pre-release/prompts/pre-release/setup-docker-sglang.md @@ -0,0 +1,46 @@ +# Pre-release E2E — setup (docker + SGLang) + +You are running the Hyperloom pre-release E2E test non-interactively. Complete the +setup step for a **docker + SGLang** leg, then stop. Do not run the demo yet. + +> **IMPORTANT:** you are **already** running inside the backend container. The nested +> container was started for you by the test harness (`docker-run-hyperloom.sh`) and is +> bound to exactly one GPU. You must **not** start, run, or exec any further container, +> and you must **not** run `docker` at all. Treat this environment as the place where +> setup and the demo run directly. + +## Environment (already prepared) + +A `.env` file exists in the current workspace (`REPO_ROOT`) with these values already +set: `ANTHROPIC_API_KEY`, `CLAUDE_MODEL`, `USER_DATA_PATH`, +`HYPERLOOM_RUN_MODE=docker`, `FRAMEWORK=sglang`, `MODEL_PATH`, `TARGET_GAIN`, +`DEMO_HOURS`. The wheel is already installed via `pip install --target .` so a +`hyperloom/` package directory is present. + +## Fixed decisions + +Run the `hyperloom-setup` skill with these fixed decisions — do **not** ask +interactive questions; use the values already in `.env` and the environment: + +- **Run mode:** docker, but the container already exists and IS the current shell. Do + **not** create a container and do **not** set `HYPERLOOM_DOCKER_TARGET_HOST`. Run + setup directly in this shell. +- **Framework:** SGLang. Ensure the SGLang framework layer is available in this + container (install with the setup backend if needed). +- **LLM provider / model / `USER_DATA_PATH`:** use the values already in `.env`; do + not change them. + +## Hard constraints (automated release gate) + +- Do **not** modify any GPU-related environment variable (`ROCR_VISIBLE_DEVICES` is + already `0` and pins this container to its single card; leave it). Do not override + device visibility. +- Do **not** run `docker`, do **not** start/exec containers, and do **not** choose + GPUs via `rocm-smi`. +- Do **not** print, echo, or copy secret values (API keys) into output or logs. +- Do **not** modify `USER_DATA_PATH`. + +## Termination + +When setup completes successfully, stop. Report only `setup complete: docker/sglang`. +If setup hard-fails, report the failure and stop. diff --git a/.github/pre-release/prompts/pre-release/setup-docker-sglang.txt b/.github/pre-release/prompts/pre-release/setup-docker-sglang.txt deleted file mode 100644 index 439f6dc738..0000000000 --- a/.github/pre-release/prompts/pre-release/setup-docker-sglang.txt +++ /dev/null @@ -1,37 +0,0 @@ -You are running the Hyperloom pre-release E2E test non-interactively. Complete the -setup step for a DOCKER + SGLang leg, then stop. Do not run the demo yet. - -IMPORTANT: you are ALREADY running inside the backend container. The nested -container was started for you by the test harness (docker-run-hyperloom.sh) and is -bound to exactly one GPU. You must NOT start, run, or exec any further container, -and you must NOT run `docker` at all. Treat this environment as the place where -setup and the demo run directly. - -Environment is already prepared. A `.env` file exists in the current workspace -(REPO_ROOT) with these values already set: ANTHROPIC_API_KEY, CLAUDE_MODEL, -USER_DATA_PATH, HYPERLOOM_RUN_MODE=docker, FRAMEWORK=sglang, MODEL_PATH, -TARGET_GAIN, DEMO_HOURS. The wheel is already installed via `pip install --target .` -so a `hyperloom/` package directory is present. - -Run the hyperloom-setup skill with these fixed decisions — do NOT ask interactive -questions, use the values already in `.env` and the environment: - -- Run mode: docker, but the container already exists and IS the current shell. Do - NOT create a container and do NOT set HYPERLOOM_DOCKER_TARGET_HOST. Run setup - directly in this shell. -- Framework: SGLang. Ensure the SGLang framework layer is available in this - container (install with the setup backend if needed). -- LLM provider / model / USER_DATA_PATH: use the values already in `.env`; do not - change them. - -Hard constraints (this is an automated release gate): -- Do NOT modify any GPU-related environment variable (ROCR_VISIBLE_DEVICES is - already 0 and pins this container to its single card; leave it). Do not override - device visibility. -- Do NOT run `docker`, do NOT start/exec containers, and do NOT choose GPUs via - `rocm-smi`. -- Do NOT print, echo, or copy secret values (API keys) into output or logs. -- Do NOT modify USER_DATA_PATH. - -When setup completes successfully, stop. Report only "setup complete: docker/sglang". -If setup hard-fails, report the failure and stop. diff --git a/.github/pre-release/prompts/pre-release/setup-docker-vllm.md b/.github/pre-release/prompts/pre-release/setup-docker-vllm.md new file mode 100644 index 0000000000..970d57fe59 --- /dev/null +++ b/.github/pre-release/prompts/pre-release/setup-docker-vllm.md @@ -0,0 +1,46 @@ +# Pre-release E2E — setup (docker + vLLM) + +You are running the Hyperloom pre-release E2E test non-interactively. Complete the +setup step for a **docker + vLLM** leg, then stop. Do not run the demo yet. + +> **IMPORTANT:** you are **already** running inside the backend container. The nested +> container was started for you by the test harness (`docker-run-hyperloom.sh`) and is +> bound to exactly one GPU. You must **not** start, run, or exec any further container, +> and you must **not** run `docker` at all. Treat this environment as the place where +> setup and the demo run directly. + +## Environment (already prepared) + +A `.env` file exists in the current workspace (`REPO_ROOT`) with these values already +set: `ANTHROPIC_API_KEY`, `CLAUDE_MODEL`, `USER_DATA_PATH`, +`HYPERLOOM_RUN_MODE=docker`, `FRAMEWORK=vllm`, `MODEL_PATH`, `TARGET_GAIN`, +`DEMO_HOURS`. The wheel is already installed via `pip install --target .` so a +`hyperloom/` package directory is present. + +## Fixed decisions + +Run the `hyperloom-setup` skill with these fixed decisions — do **not** ask +interactive questions; use the values already in `.env` and the environment: + +- **Run mode:** docker, but the container already exists and IS the current shell. Do + **not** create a container and do **not** set `HYPERLOOM_DOCKER_TARGET_HOST`. Run + setup directly in this shell. +- **Framework:** vLLM. Ensure the vLLM framework layer is available in this container + (install with the setup backend if needed). +- **LLM provider / model / `USER_DATA_PATH`:** use the values already in `.env`; do + not change them. + +## Hard constraints (automated release gate) + +- Do **not** modify any GPU-related environment variable (`ROCR_VISIBLE_DEVICES` is + already `0` and pins this container to its single card; leave it). Do not override + device visibility. +- Do **not** run `docker`, do **not** start/exec containers, and do **not** choose + GPUs via `rocm-smi`. +- Do **not** print, echo, or copy secret values (API keys) into output or logs. +- Do **not** modify `USER_DATA_PATH`. + +## Termination + +When setup completes successfully, stop. Report only `setup complete: docker/vllm`. +If setup hard-fails, report the failure and stop. diff --git a/.github/pre-release/prompts/pre-release/setup-docker-vllm.txt b/.github/pre-release/prompts/pre-release/setup-docker-vllm.txt deleted file mode 100644 index 05c825f818..0000000000 --- a/.github/pre-release/prompts/pre-release/setup-docker-vllm.txt +++ /dev/null @@ -1,37 +0,0 @@ -You are running the Hyperloom pre-release E2E test non-interactively. Complete the -setup step for a DOCKER + vLLM leg, then stop. Do not run the demo yet. - -IMPORTANT: you are ALREADY running inside the backend container. The nested -container was started for you by the test harness (docker-run-hyperloom.sh) and is -bound to exactly one GPU. You must NOT start, run, or exec any further container, -and you must NOT run `docker` at all. Treat this environment as the place where -setup and the demo run directly. - -Environment is already prepared. A `.env` file exists in the current workspace -(REPO_ROOT) with these values already set: ANTHROPIC_API_KEY, CLAUDE_MODEL, -USER_DATA_PATH, HYPERLOOM_RUN_MODE=docker, FRAMEWORK=vllm, MODEL_PATH, -TARGET_GAIN, DEMO_HOURS. The wheel is already installed via `pip install --target .` -so a `hyperloom/` package directory is present. - -Run the hyperloom-setup skill with these fixed decisions — do NOT ask interactive -questions, use the values already in `.env` and the environment: - -- Run mode: docker, but the container already exists and IS the current shell. Do - NOT create a container and do NOT set HYPERLOOM_DOCKER_TARGET_HOST. Run setup - directly in this shell. -- Framework: vLLM. Ensure the vLLM framework layer is available in this container - (install with the setup backend if needed). -- LLM provider / model / USER_DATA_PATH: use the values already in `.env`; do not - change them. - -Hard constraints (this is an automated release gate): -- Do NOT modify any GPU-related environment variable (ROCR_VISIBLE_DEVICES is - already 0 and pins this container to its single card; leave it). Do not override - device visibility. -- Do NOT run `docker`, do NOT start/exec containers, and do NOT choose GPUs via - `rocm-smi`. -- Do NOT print, echo, or copy secret values (API keys) into output or logs. -- Do NOT modify USER_DATA_PATH. - -When setup completes successfully, stop. Report only "setup complete: docker/vllm". -If setup hard-fails, report the failure and stop. diff --git a/.github/workflows/pre-release-e2e-test.yml b/.github/workflows/pre-release-e2e-test.yml index 213a878cbe..1b1275e6ae 100644 --- a/.github/workflows/pre-release-e2e-test.yml +++ b/.github/workflows/pre-release-e2e-test.yml @@ -187,7 +187,7 @@ jobs: cp "$WHEEL" "$wheel_dir/" # Stage the in-pod scripts + fixed prompts (pods read them from NFS). cp .github/pre-release/bootstrap-pre-release.sh .github/pre-release/docker-run-hyperloom.sh "$boot_dir/" - cp .github/pre-release/prompts/pre-release/*.txt "$boot_dir/prompts/pre-release/" + cp .github/pre-release/prompts/pre-release/*.md "$boot_dir/prompts/pre-release/" chmod +x "$boot_dir"/*.sh # manifest.json (design §5). jq -n \ From 6a88e39f1af79116ecbc20b12b8376a69badecf4 Mon Sep 17 00:00:00 2001 From: lishuoshuo-amd Date: Thu, 27 Aug 2026 14:35:29 +0800 Subject: [PATCH 04/52] pre-release-e2e: use authoritative SaFE `timeout` field for pod deadline Confirmed against the Primus-SaFE codebase that the create-workload body embeds WorkloadSpec inline, whose top-level integer-seconds `timeout` field is enforced by WorkloadTTLController for all workload kinds (incl. Authoring), counted from dispatch time. Replaces the placeholder `activeDeadlineSeconds` default (a k8s-Job convention that is not the Authoring create-API field). DEADLINE_FIELD stays overridable; "" falls back to workspace maxRuntime / poll-side GLOBAL_TIMEOUT_S. Co-Authored-By: Claude Opus 4.8 --- .github/scripts/pre-release-e2e-dispatch.sh | 22 +++++++++++++-------- 1 file changed, 14 insertions(+), 8 deletions(-) diff --git a/.github/scripts/pre-release-e2e-dispatch.sh b/.github/scripts/pre-release-e2e-dispatch.sh index 07064d9b6f..dd2eb3b362 100755 --- a/.github/scripts/pre-release-e2e-dispatch.sh +++ b/.github/scripts/pre-release-e2e-dispatch.sh @@ -44,9 +44,11 @@ # DEADLINE_3H_S / DEADLINE_12H_S pod hard-timeout per duration # (default 14400 = 3+1h / 46800 = 12+1h). The docker host # pod uses the MAX over its legs. SaFE kills the pod at the -# deadline; poll then judges that leg FAIL. -# DEADLINE_FIELD SaFE payload field for the deadline (default -# activeDeadlineSeconds; set "" to omit). TODO(owner): confirm. +# deadline; poll then judges that leg FAIL. Timing starts when +# the workload is DISPATCHED, not when it is queued. +# DEADLINE_FIELD SaFE payload field for the deadline (default `timeout`, the +# authoritative WorkloadSpec.Timeout field, integer seconds, +# top-level in the create-workload body; set "" to omit). # SAFE_CACERT / SAFE_INSECURE TLS to the API (CA bundle / skip-verify) set -euo pipefail @@ -58,13 +60,17 @@ DISPATCH_MAP="${DISPATCH_MAP:-${RUNNER_TEMP:-/tmp}/pre_release_dispatch.json}" # Pod hard-timeout (design: 3h leg -> 3+1h, 12h leg -> 12+1h). SaFE terminates the # workload at the deadline; the poll then sees a non-Succeeded terminal / missing -# report and judges that leg FAIL. Given per duration: +# report and judges that leg FAIL. The deadline is counted from DISPATCH (not queue) +# time, so the +1h buffer absorbs bootstrap/setup/agent overhead. Given per duration: DEADLINE_3H_S="${DEADLINE_3H_S:-14400}" # 3h + 1h buffer = 4h DEADLINE_12H_S="${DEADLINE_12H_S:-46800}" # 12h + 1h buffer = 13h -# The SaFE API field that carries the pod deadline. TODO(owner): confirm the real -# field name/placement against the SaFE Authoring API; k8s convention is -# activeDeadlineSeconds (integer seconds). Set DEADLINE_FIELD="" to omit entirely. -DEADLINE_FIELD="${DEADLINE_FIELD:-activeDeadlineSeconds}" +# The SaFE API field that carries the pod deadline. Confirmed against the Primus-SaFE +# codebase: the create-workload body embeds WorkloadSpec inline, whose `timeout` +# (integer seconds, top-level, from dispatch time) is enforced by WorkloadTTLController +# for ALL workload kinds incl. Authoring. Set DEADLINE_FIELD="" to omit (then the pod +# survival cap falls back to the workspace's per-scope maxRuntime, or the poll-side +# GLOBAL_TIMEOUT_S if none). Ref: apis/pkg/apis/amd/v1/workload_types.go WorkloadSpec.Timeout. +DEADLINE_FIELD="${DEADLINE_FIELD:-timeout}" leg_deadline_s() { case "$1" in *-3h) echo "$DEADLINE_3H_S" ;; *-12h) echo "$DEADLINE_12H_S" ;; esac; } : "${SAFE_API_BASE:?SAFE_API_BASE is required}" From 3bb65357774db28c0ff67f0263a4a169d9956e7e Mon Sep 17 00:00:00 2001 From: lishuoshuo-amd Date: Thu, 27 Aug 2026 15:07:41 +0800 Subject: [PATCH 05/52] pre-release-e2e: fix nested-docker on privileged pod (live-verified DinD) Verified on a real SaFE privileged Authoring pod (MI355X x8, rocm/pytorch base image) that the base image ships no docker/dockerd, and fixed the nested-container path against what actually works: bootstrap-pre-release.sh * add ensure_dockerd(): apt-get install docker.io, then start a pod-local dockerd detached via setsid with --storage-driver=vfs (no systemd in the pod; overlay-on-overlay fails). Called before fanning out docker legs. docker-run-hyperloom.sh * GPU index -> renderD is stride 8, not +1: RD=128+GPU_INDEX*8 (verified GPU i == renderD(128+8i) via /sys/class/drm + rocm-smi --showbus). * --group-add video FAILS (pod /etc/group has no video/render names); use numeric device-node GIDs from `stat -c %g` instead. * isolate a single card via /dev/kfd + one renderD node only (drop the cardN device; card numbering is not guaranteed aligned to GPU order), plus --security-opt seccomp=unconfined and HIP_VISIBLE_DEVICES=0. Co-Authored-By: Claude Opus 4.8 --- .github/pre-release/bootstrap-pre-release.sh | 29 ++++++++++++++++++++ .github/pre-release/docker-run-hyperloom.sh | 28 +++++++++++++++---- 2 files changed, 51 insertions(+), 6 deletions(-) diff --git a/.github/pre-release/bootstrap-pre-release.sh b/.github/pre-release/bootstrap-pre-release.sh index cb6d3271d8..7ed0fc9103 100755 --- a/.github/pre-release/bootstrap-pre-release.sh +++ b/.github/pre-release/bootstrap-pre-release.sh @@ -96,11 +96,40 @@ run_leg() { log "leg $leg agent turns complete; poll will judge $session/reports/final.json" } +# Install docker + start a pod-local dockerd. VERIFIED on a real privileged MI355X pod +# (2026-08-27): the Authoring base image ships NO docker/dockerd/docker.sock, but the +# pod has full capabilities (CapEff=0x1ffffffffff), so a self-hosted dockerd works. +# Two non-obvious requirements, both confirmed by probing: +# * --storage-driver=vfs -- overlayfs-on-overlayfs fails inside the container rootfs. +# * no systemd in the pod -- start dockerd detached via setsid, then poll the socket. +ensure_dockerd() { + if docker info >/dev/null 2>&1; then log "dockerd already up"; return 0; fi + if ! command -v dockerd >/dev/null 2>&1; then + log "installing docker.io (no docker in the Authoring base image)" + export DEBIAN_FRONTEND=noninteractive + apt-get update -qq >/tmp/apt-docker.log 2>&1 || { log "ERROR: apt-get update failed"; tail -20 /tmp/apt-docker.log; return 1; } + apt-get install -y -qq docker.io >>/tmp/apt-docker.log 2>&1 \ + || { log "ERROR: docker.io install failed"; tail -30 /tmp/apt-docker.log; return 1; } + log "docker installed: $(docker --version 2>&1)" + fi + log "starting pod-local dockerd (vfs, detached via setsid)" + setsid bash -c 'dockerd --host=unix:///var/run/docker.sock --storage-driver=vfs >/var/log/dockerd.log 2>&1' \ + /dev/null 2>&1 & + local i + for i in $(seq 1 60); do + docker info >/dev/null 2>&1 && { log "dockerd up after ${i}s"; return 0; } + sleep 1 + done + log "ERROR: dockerd did not become ready"; tail -40 /var/log/dockerd.log 2>/dev/null || true + return 1 +} + # ---- docker host: fan out to nested containers ----------------------------- run_docker_host() { : "${DOCKER_LEGS:?}"; : "${DOCKER_GPU_MAP:?}"; : "${MODEL_3H:?}"; : "${MODEL_12H:?}" local runner="${SELF_DIR}/docker-run-hyperloom.sh" [ -x "$runner" ] || chmod +x "$runner" 2>/dev/null || true + ensure_dockerd || { log "ERROR: cannot provide docker on the host pod"; return 1; } log "docker host: legs='${DOCKER_LEGS}'" local pids=() for leg in $DOCKER_LEGS; do diff --git a/.github/pre-release/docker-run-hyperloom.sh b/.github/pre-release/docker-run-hyperloom.sh index 4507b73bb7..8d6763fad7 100755 --- a/.github/pre-release/docker-run-hyperloom.sh +++ b/.github/pre-release/docker-run-hyperloom.sh @@ -55,20 +55,36 @@ ROOT="${NFS_ROOT%/}/runs/${CI_VERSION}/${LEG_ID}" mkdir -p "$ROOT" BOOTSTRAP="${NFS_ROOT%/}/bootstrap/${CI_VERSION}/bootstrap-pre-release.sh" NAME="hyperloom-${LEG_ID}" -RD=$((128 + GPU_INDEX)) # renderD node paired with cardN -echo "[docker-run] leg=$LEG_ID gpu=$GPU_INDEX card$GPU_INDEX/renderD$RD image=$IMAGE cpus=$LEG_CPUS mem=$LEG_MEM" +# GPU index -> renderD node. VERIFIED on a real privileged MI355X x8 pod (2026-08-27): +# the 8 physical GPUs map to renderD128,136,144,...,184 -- i.e. stride 8, NOT +1. The +# rocm-smi GPU order matches this render-node order (GPU i == 0002/0003:00:0X.0 == +# renderD(128+8*i)). See project memory. `cardN` numbering is NOT guaranteed to align +# with the GPU order, so we isolate via /dev/kfd + the single renderD node only. +RD=$((128 + GPU_INDEX * 8)) + +# Device group ownership: the pod's /etc/group has NO `video`/`render` NAMES, so +# `--group-add video` FAILS ("no matching entries in group file"). Resolve the numeric +# GIDs of the device nodes and pass those instead (verified working). +KFD_GID="$(stat -c %g /dev/kfd 2>/dev/null || echo 0)" +DRI_GID="$(stat -c %g /dev/dri/renderD${RD} 2>/dev/null || stat -c %g /dev/dri 2>/dev/null || echo 0)" + +echo "[docker-run] leg=$LEG_ID gpu=$GPU_INDEX renderD$RD (kfd_gid=$KFD_GID dri_gid=$DRI_GID) image=$IMAGE cpus=$LEG_CPUS mem=$LEG_MEM" docker rm -f "$NAME" >/dev/null 2>&1 || true -# GPU isolation: expose exactly one card, and set ROCR_VISIBLE_DEVICES=0 so the -# container sees a single device at index 0. CPU/mem hard-capped to the 1/4 share. +# GPU isolation: expose /dev/kfd (shared) + exactly ONE renderD node, so the container +# sees a single device. HIP_VISIBLE_DEVICES=0 pins the app to that one card. CPU/mem +# hard-capped to the 1/4 share. seccomp=unconfined matches how the ROCm images expect +# to run (verified: rocm-smi enumerates the single bound card correctly). exec docker run --rm --name "$NAME" \ --device "/dev/kfd" \ - --device "/dev/dri/card${GPU_INDEX}" \ --device "/dev/dri/renderD${RD}" \ - --group-add video \ + --group-add "$KFD_GID" \ + --group-add "$DRI_GID" \ + --security-opt seccomp=unconfined \ --cpus "$LEG_CPUS" --memory "$LEG_MEM" --shm-size "$LEG_SHM" \ + -e HIP_VISIBLE_DEVICES=0 \ -e ROCR_VISIBLE_DEVICES=0 \ -e CI_VERSION="$CI_VERSION" \ -e NFS_ROOT="$NFS_ROOT" \ From cd86fdaf08e57da9db88ed4b75fe9526334aeae5 Mon Sep 17 00:00:00 2001 From: lishuoshuo-amd Date: Thu, 27 Aug 2026 15:13:01 +0800 Subject: [PATCH 06/52] pre-release-e2e: reclaim workloads via stop, not delete Policy change: after judging (and on job cancel), STOP each SaFE workload instead of DELETE. Stop frees the GPUs immediately -- so a 3h leg that finished early does not idle-hold its card until the `timeout` deadline -- while keeping the workload record and pod filesystem for post-hoc inspection. SaFE exposes no start/restart, so Stopped records are cleaned up manually. Verified 2026-08-27: POST /api/v1/workloads/{id}/stop exists and returns 200. * poll: add stop_workloads() at the end, dedup by workloadId (the 4 docker legs share one host workload). * workflow: cancel step switched from DELETE to POST .../stop. Co-Authored-By: Claude Opus 4.8 --- .github/scripts/pre-release-e2e-poll.sh | 20 ++++++++++++++++++++ .github/workflows/pre-release-e2e-test.yml | 12 +++++++----- 2 files changed, 27 insertions(+), 5 deletions(-) diff --git a/.github/scripts/pre-release-e2e-poll.sh b/.github/scripts/pre-release-e2e-poll.sh index ca3137f8af..38f41aba7b 100755 --- a/.github/scripts/pre-release-e2e-poll.sh +++ b/.github/scripts/pre-release-e2e-poll.sh @@ -317,5 +317,25 @@ summary "$gate_line" # Final sticky report: the completed table plus the gate verdict. report_upsert "$(printf '%s\n\n%s\n' "$(report_body Complete "$(done_count)" "${#LEGS[@]}")" "$gate_line")" +# ---- reclaim: STOP (not delete) every workload ----------------------------- +# Policy: stop, don't delete. This frees the GPUs immediately (so a 3h leg that +# finished early doesn't idle-hold its card until the SaFE `timeout` deadline) while +# KEEPING the workload record + its pod filesystem for post-hoc inspection. SaFE has +# no `start` endpoint, so these are not resumable; clean up Stopped records manually. +# Verified 2026-08-27: POST /api/v1/workloads/{id}/stop exists and returns 200. +stop_workloads() { + local wid seen="" + for leg in "${LEGS[@]}"; do + wid="${WID[$leg]}" + # docker legs share one host workload -> stop each unique id once. + case " $seen " in *" $wid "*) continue ;; esac + seen="${seen} ${wid}" + code="$(curl -sS "${tls[@]}" -o /dev/null -w '%{http_code}' -X POST \ + "$API/$wid/stop" "${auth[@]}" 2>/dev/null || echo 000)" + summary "• stopped workload \`$wid\` (HTTP $code)" + done +} +stop_workloads || true + [ "$fail" -eq 0 ] && exit 0 exit 1 diff --git a/.github/workflows/pre-release-e2e-test.yml b/.github/workflows/pre-release-e2e-test.yml index 1b1275e6ae..1d765122bc 100644 --- a/.github/workflows/pre-release-e2e-test.yml +++ b/.github/workflows/pre-release-e2e-test.yml @@ -254,16 +254,18 @@ jobs: chmod +x .github/scripts/pre-release-e2e-poll.sh .github/scripts/pre-release-e2e-poll.sh - - name: Cancel workloads on job cancel + - name: Stop workloads on job cancel if: cancelled() env: SAFE_API_BASE: ${{ vars.SAFE_API_BASE }} SAFE_API_KEY: ${{ secrets.SAFE_API_KEY }} run: | - [ -f "$DISPATCH_MAP" ] || { echo "no dispatch map; nothing to cancel"; exit 0; } + # Policy: stop, don't delete (frees GPUs, keeps the record + pod fs for + # inspection; SaFE has no restart, so clean up Stopped records manually). + [ -f "$DISPATCH_MAP" ] || { echo "no dispatch map; nothing to stop"; exit 0; } for wid in $(jq -r '.[]' "$DISPATCH_MAP" | sort -u); do - echo "cancelling workload $wid" - curl -sS -k -o /dev/null -w 'DELETE %{http_code}\n' -X DELETE \ + echo "stopping workload $wid" + curl -sS -k -o /dev/null -w 'STOP %{http_code}\n' -X POST \ -H "Authorization: Bearer ${SAFE_API_KEY}" \ - "${SAFE_API_BASE%/}/api/v1/workloads/${wid}" || true + "${SAFE_API_BASE%/}/api/v1/workloads/${wid}/stop" || true done From 8c97cc183d510b5a2230e7ef7c9ef37c1a7519ef Mon Sep 17 00:00:00 2001 From: lishuoshuo-amd Date: Thu, 27 Aug 2026 15:28:54 +0800 Subject: [PATCH 07/52] pre-release-e2e: prefix all repo secrets/variables with PRE_E2E_ Rename every ${{ secrets.* }} / ${{ vars.* }} reference to carry a PRE_E2E_ prefix so this release-gate CI's config never collides with the existing per-PR CI (ci-e2e.yml) secrets/variables in the same repo. GITHUB_TOKEN is left as-is (built-in, cannot be renamed). Only the reference names change; the in-workflow env: keys handed to the dispatch/poll scripts (SAFE_API_BASE, NFS_ROOT, ...) are unchanged, so no script edits are needed. Owner action: create the repo Secrets/Variables under the new PRE_E2E_ names. Co-Authored-By: Claude Opus 4.8 --- .github/workflows/pre-release-e2e-test.yml | 44 ++++++++++++---------- 1 file changed, 24 insertions(+), 20 deletions(-) diff --git a/.github/workflows/pre-release-e2e-test.yml b/.github/workflows/pre-release-e2e-test.yml index 1d765122bc..dae0284246 100644 --- a/.github/workflows/pre-release-e2e-test.yml +++ b/.github/workflows/pre-release-e2e-test.yml @@ -15,11 +15,15 @@ name: Pre-release E2E test # * manual workflow_dispatch (reuse a wheel, run a leg subset; default all 8) # # ── Environment values to fill in (repo Secrets/Variables) ────────────────── -# Secrets (sensitive): SAFE_API_KEY (ADMIN token: privileged 8-GPU pod is admin-only), -# ANTHROPIC_API_KEY. -# Variables (non-secret): SAFE_API_BASE, SAFE_WORKSPACE_ID, AUTHORING_IMAGE, -# PRE_RELEASE_NFS_ROOT, MODEL_3H_PATH, MODEL_12H_PATH, CLAUDE_MODEL, -# CLAUDE_CLI_VERSION, (optional) ANTHROPIC_BASE_URL, SAFE_INSECURE. +# All names carry a PRE_E2E_ prefix so they never collide with the existing per-PR +# CI's secrets/variables (ci-e2e.yml etc.). GITHUB_TOKEN is the one exception -- it is +# GitHub's built-in token and cannot be renamed. +# Secrets (sensitive): PRE_E2E_SAFE_API_KEY (ADMIN token: privileged 8-GPU pod is +# admin-only), PRE_E2E_ANTHROPIC_API_KEY. +# Variables (non-secret): PRE_E2E_SAFE_API_BASE, PRE_E2E_SAFE_WORKSPACE_ID, +# PRE_E2E_AUTHORING_IMAGE, PRE_E2E_NFS_ROOT, PRE_E2E_MODEL_3H_PATH, +# PRE_E2E_MODEL_12H_PATH, PRE_E2E_CLAUDE_MODEL, PRE_E2E_CLAUDE_CLI_VERSION, +# (optional) PRE_E2E_ANTHROPIC_BASE_URL, PRE_E2E_SAFE_INSECURE. # Nothing below hard-codes an environment value; all are `${{ secrets.* }}` / # `${{ vars.* }}` references. TODO(owner): populate these before first real run. @@ -150,7 +154,7 @@ jobs: env: CI_VERSION: ${{ needs.resolve.outputs.ci_version }} BASE_VERSION: ${{ needs.resolve.outputs.base_version }} - NFS_ROOT: ${{ vars.PRE_RELEASE_NFS_ROOT }} + NFS_ROOT: ${{ vars.PRE_E2E_NFS_ROOT }} steps: - uses: actions/checkout@v7 @@ -212,21 +216,21 @@ jobs: env: CI_VERSION: ${{ needs.resolve.outputs.ci_version }} TASKS: ${{ needs.resolve.outputs.tasks }} - NFS_ROOT: ${{ vars.PRE_RELEASE_NFS_ROOT }} + NFS_ROOT: ${{ vars.PRE_E2E_NFS_ROOT }} TARGET_GAIN: "100" # SaFE API - SAFE_API_BASE: ${{ vars.SAFE_API_BASE }} - SAFE_API_KEY: ${{ secrets.SAFE_API_KEY }} # ADMIN token (privileged pod) - SAFE_WORKSPACE_ID: ${{ vars.SAFE_WORKSPACE_ID }} - SAFE_INSECURE: ${{ vars.SAFE_INSECURE || '1' }} - AUTHORING_IMAGE: ${{ vars.AUTHORING_IMAGE }} + SAFE_API_BASE: ${{ vars.PRE_E2E_SAFE_API_BASE }} + SAFE_API_KEY: ${{ secrets.PRE_E2E_SAFE_API_KEY }} # ADMIN token (privileged pod) + SAFE_WORKSPACE_ID: ${{ vars.PRE_E2E_SAFE_WORKSPACE_ID }} + SAFE_INSECURE: ${{ vars.PRE_E2E_SAFE_INSECURE || '1' }} + AUTHORING_IMAGE: ${{ vars.PRE_E2E_AUTHORING_IMAGE }} # models + agent - MODEL_3H: ${{ vars.MODEL_3H_PATH }} - MODEL_12H: ${{ vars.MODEL_12H_PATH }} - CLAUDE_MODEL: ${{ vars.CLAUDE_MODEL }} - CLAUDE_CLI_VERSION: ${{ vars.CLAUDE_CLI_VERSION }} - ANTHROPIC_API_KEY: ${{ secrets.ANTHROPIC_API_KEY }} # GitHub Secret path (design §9 D) - ANTHROPIC_BASE_URL: ${{ vars.ANTHROPIC_BASE_URL }} + MODEL_3H: ${{ vars.PRE_E2E_MODEL_3H_PATH }} + MODEL_12H: ${{ vars.PRE_E2E_MODEL_12H_PATH }} + CLAUDE_MODEL: ${{ vars.PRE_E2E_CLAUDE_MODEL }} + CLAUDE_CLI_VERSION: ${{ vars.PRE_E2E_CLAUDE_CLI_VERSION }} + ANTHROPIC_API_KEY: ${{ secrets.PRE_E2E_ANTHROPIC_API_KEY }} # GitHub Secret path (design §9 D) + ANTHROPIC_BASE_URL: ${{ vars.PRE_E2E_ANTHROPIC_BASE_URL }} # per-leg GitHub commit status GH_STATUS_TOKEN: ${{ secrets.GITHUB_TOKEN }} GH_STATUS_REPO: ${{ github.repository }} @@ -257,8 +261,8 @@ jobs: - name: Stop workloads on job cancel if: cancelled() env: - SAFE_API_BASE: ${{ vars.SAFE_API_BASE }} - SAFE_API_KEY: ${{ secrets.SAFE_API_KEY }} + SAFE_API_BASE: ${{ vars.PRE_E2E_SAFE_API_BASE }} + SAFE_API_KEY: ${{ secrets.PRE_E2E_SAFE_API_KEY }} run: | # Policy: stop, don't delete (frees GPUs, keeps the record + pod fs for # inspection; SaFE has no restart, so clean up Stopped records manually). From c250b12946ad840fc430411adc14ccce9506e37e Mon Sep 17 00:00:00 2001 From: lishuoshuo-amd Date: Thu, 27 Aug 2026 15:45:05 +0800 Subject: [PATCH 08/52] pre-release-e2e: switch gate trigger from push-to-main to pull_request Run the release gate on the OPEN PR (before merge), not after merge. A push-to-main trigger is deliberately removed so a bad release is blocked while the PR is open and merging does not re-run it. resolve.decide now classifies against the PR base branch instead of HEAD~1: version bump vs base -> FULL (8 legs); CI-logic-only change -> SCRIPTS-ONLY (4 fast 3h legs). Uses BASE_SHA/BASE_REF from the event; checkout fetch-depth raised to 0 so the base commit is present for the diff/version compare. poll: PR number now comes straight from the event (PR_NUMBER) with the commit->PR reverse lookup kept as a workflow_dispatch fallback; per-leg commit statuses target the PR head sha (github.sha is the merge commit). Co-Authored-By: Claude Opus 4.8 --- .github/scripts/pre-release-e2e-poll.sh | 15 ++++-- .github/workflows/pre-release-e2e-test.yml | 56 ++++++++++++++-------- 2 files changed, 45 insertions(+), 26 deletions(-) diff --git a/.github/scripts/pre-release-e2e-poll.sh b/.github/scripts/pre-release-e2e-poll.sh index 38f41aba7b..c2f43f5f39 100755 --- a/.github/scripts/pre-release-e2e-poll.sh +++ b/.github/scripts/pre-release-e2e-poll.sh @@ -75,19 +75,24 @@ post_status() { # leg state(pending|success|failure|error) description } # ---- sticky PR/commit report comment --------------------------------------- -# This CI runs on push to main, which carries no PR number. Resolve the PR that -# the triggering commit was merged from (GET /commits/{sha}/pulls); if none is -# found, fall back to a commit comment. Then upsert ONE sticky comment (matched +# This CI runs on pull_request, so the PR number is passed in directly via PR_NUMBER +# (github.event.pull_request.number). If it's absent (e.g. a workflow_dispatch run), +# fall back to resolving the PR from the triggering commit (GET /commits/{sha}/pulls), +# and if that too fails, comment on the commit. Then upsert ONE sticky comment (matched # by an HTML marker) and PATCH it in place as legs finish -- so a single comment # updates incrementally (design §10, point C). Mirrors ci-e2e-dispatch.sh. REPORT_MARKER="" -PR_NUMBER=""; COMMENT_TARGET="" # COMMENT_TARGET: "pr" | "commit" | "" (disabled) +PR_NUMBER="${PR_NUMBER:-}"; COMMENT_TARGET="" # COMMENT_TARGET: "pr" | "commit" | "" (disabled) gh_report_on() { [ -n "${GH_STATUS_TOKEN:-}" ] && [ -n "${GH_STATUS_REPO:-}" ] && [ -n "${GH_STATUS_SHA:-}" ]; } resolve_comment_target() { gh_report_on || { COMMENT_TARGET=""; return 0; } - # A merged commit usually belongs to exactly one PR; take the first. + # Preferred: the pull_request event handed us the PR number directly. + if [[ "$PR_NUMBER" =~ ^[0-9]+$ ]]; then + COMMENT_TARGET="pr"; echo "[report] commenting on PR #$PR_NUMBER (from event)"; return 0 + fi + # Fallback (workflow_dispatch etc.): a commit usually belongs to one PR; take the first. PR_NUMBER="$(curl -sS \ -H "Authorization: Bearer ${GH_STATUS_TOKEN}" \ -H "Accept: application/vnd.github+json" \ diff --git a/.github/workflows/pre-release-e2e-test.yml b/.github/workflows/pre-release-e2e-test.yml index dae0284246..7e4664e8a6 100644 --- a/.github/workflows/pre-release-e2e-test.yml +++ b/.github/workflows/pre-release-e2e-test.yml @@ -8,11 +8,13 @@ name: Pre-release E2E test # the Claude CLI + setup/demo skills -- the same path a user takes. Independent of the # per-PR smoke test (ci-e2e.yml). See hyperloom-pre-release-e2e-ci-design.md. # -# Triggers: -# * push to main that changes pyproject.toml's `version` field -> FULL run (all 8 legs) -# * push to main that changes this CI's scripts/prompts/workflow (version unchanged) -# -> SCRIPTS-ONLY run (the 4 fast 3h legs, to validate the logic change) +# Triggers (gate BEFORE merge, on the PR -- not after merge to main): +# * PR targeting main that bumps pyproject.toml's `version` field -> FULL run (all 8 legs) +# * PR targeting main that only touches this CI's scripts/prompts/ -> SCRIPTS-ONLY run +# workflow (version unchanged) (the 4 fast 3h legs) # * manual workflow_dispatch (reuse a wheel, run a leg subset; default all 8) +# NOTE: there is deliberately NO push-to-main trigger -- the gate runs while the PR is +# open so a bad release is blocked before it lands, and merging does not re-run it. # # ── Environment values to fill in (repo Secrets/Variables) ────────────────── # All names carry a PRE_E2E_ prefix so they never collide with the existing per-PR @@ -28,11 +30,11 @@ name: Pre-release E2E test # `${{ vars.* }}` references. TODO(owner): populate these before first real run. on: - push: + pull_request: branches: [main] - # A push must touch one of these to even start the workflow; `resolve` then - # classifies it into a FULL run (version bump) or a SCRIPTS-ONLY run (CI logic - # changed but version unchanged). + # A PR must touch one of these to even start the workflow; `resolve` then + # classifies it into a FULL run (version bump vs the base branch) or a + # SCRIPTS-ONLY run (CI logic changed but version unchanged). paths: - "pyproject.toml" # version bump -> full release gate - ".github/workflows/pre-release-e2e-test.yml" # the workflow itself @@ -60,7 +62,7 @@ permissions: issues: write # PR comments go through the issues API jobs: - # 1. resolve: gate on a real version bump (push) or manual input; compute CI_VERSION. + # 1. resolve: gate on a real version bump (PR vs base) or manual input; compute CI_VERSION. resolve: runs-on: Hyperloom-e2e-ci outputs: @@ -76,7 +78,9 @@ jobs: - uses: actions/checkout@v7 with: - fetch-depth: 2 # need HEAD~1 to diff the version field on push + # For a PR, actions/checkout checks out the merge ref; fetch-depth 0 gives us + # the base branch too so we can compare the version field against base. + fetch-depth: 0 - name: Decide + compute CI_VERSION id: decide @@ -84,6 +88,8 @@ jobs: EVENT: ${{ github.event_name }} REUSE_IN: ${{ inputs.reuse_ci_version }} TASKS_IN: ${{ inputs.tasks }} + BASE_SHA: ${{ github.event.pull_request.base.sha }} + BASE_REF: ${{ github.event.pull_request.base.ref }} run: | set -euo pipefail # The 4 fast 3h legs used when only CI logic (scripts/prompts/workflow) @@ -102,27 +108,31 @@ jobs: gsub(/.*=[[:space:]]*"|".*/,""); print; exit }' } + # `base_version` here means the version in the PR head (this branch's + # pyproject.toml) -- confusingly it is also compared against the PR *base* + # branch below. Keep the name for downstream outputs (it feeds CI_VERSION). base_version="$(read_version < pyproject.toml)" if [ "$EVENT" = "workflow_dispatch" ]; then # Manual runs are full-scope; tasks default to all 8 (empty = all in dispatch). run=true; run_scope="full"; reuse="${REUSE_IN:-}" else - # push to main. Classify: version bump -> full; else CI logic changed -> scripts-only. - prev="$(git show HEAD~1:pyproject.toml 2>/dev/null | read_version || true)" + # pull_request targeting main. Classify against the PR BASE branch (not HEAD~1: + # actions/checkout gives us the merge ref, whose first parent is the base tip, + # and fetch-depth:0 pulled the base commit BASE_SHA). version bump vs base -> + # FULL; else the PR only touched CI logic -> SCRIPTS-ONLY. + prev="$(git show "${BASE_SHA}:pyproject.toml" 2>/dev/null | read_version || true)" if [ "$base_version" != "$prev" ] && [ -n "$base_version" ]; then run=true; run_scope="full" - echo "version bump: '${prev}' -> '${base_version}' -> FULL run (all 8 legs)" + echo "version bump vs base ${BASE_REF}: '${prev}' -> '${base_version}' -> FULL run (all 8 legs)" else - # Version unchanged. on.push.paths already guaranteed this push touched one - # of the watched paths, and it wasn't the version -> it changed the CI's own - # scripts/prompts/workflow. Run the fast scripts-only scope. (The diff is - # best-effort logging; across a multi-commit push HEAD~1 may not show every - # changed file, so we do NOT gate on it -- the path filter already did.) + # Version unchanged vs base. on.pull_request.paths already guaranteed this PR + # touched one of the watched paths, and it wasn't the version -> it changed + # the CI's own scripts/prompts/workflow. Run the fast scripts-only scope. run=true; run_scope="scripts-only" - changed="$(git diff --name-only HEAD~1 HEAD 2>/dev/null | grep -E \ + changed="$(git diff --name-only "${BASE_SHA}" HEAD 2>/dev/null | grep -E \ '^\.github/(workflows/pre-release-e2e-test\.yml|scripts/pre-release-e2e-.*\.sh|pre-release/)' \ | paste -sd',' - || true)" - echo "version unchanged; CI logic changed (${changed:-see path filter}) -> SCRIPTS-ONLY run (4 fast 3h legs)" + echo "version unchanged vs base ${BASE_REF}; CI logic changed (${changed:-see path filter}) -> SCRIPTS-ONLY run (4 fast 3h legs)" fi fi @@ -234,7 +244,11 @@ jobs: # per-leg GitHub commit status GH_STATUS_TOKEN: ${{ secrets.GITHUB_TOKEN }} GH_STATUS_REPO: ${{ github.repository }} - GH_STATUS_SHA: ${{ github.sha }} + # On pull_request, github.sha is the ephemeral merge commit; per-leg commit + # statuses must land on the PR HEAD sha so they surface on the PR checks tab. + GH_STATUS_SHA: ${{ github.event.pull_request.head.sha }} + # PR number is known directly from the event -- no commit->PR reverse lookup. + PR_NUMBER: ${{ github.event.pull_request.number }} GH_STATUS_DETAILS_URL: ${{ github.server_url }}/${{ github.repository }}/actions/runs/${{ github.run_id }} POLL_INTERVAL_S: "120" GLOBAL_TIMEOUT_S: "50400" From f580aeca8ed1912c591c4d52d62983aa9b199ca8 Mon Sep 17 00:00:00 2001 From: lishuoshuo-amd Date: Thu, 27 Aug 2026 16:42:13 +0800 Subject: [PATCH 09/52] pre-release-e2e: run on dedicated ARC scale set hyperloom-pre-e2e-ci Move the release-gate jobs off the shared Hyperloom-e2e-ci runner (which the per-PR smoke test ci-e2e.yml also uses, causing build to queue behind it) onto their own SaFE-hosted AutoscalingRunnerSet `hyperloom-pre-e2e-ci`. All three jobs (resolve/build/run) now target it, so the pre-release gate no longer contends with the per-PR CI for runner slots. Co-Authored-By: Claude Opus 4.8 --- .github/workflows/pre-release-e2e-test.yml | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/.github/workflows/pre-release-e2e-test.yml b/.github/workflows/pre-release-e2e-test.yml index 7e4664e8a6..c2e880b597 100644 --- a/.github/workflows/pre-release-e2e-test.yml +++ b/.github/workflows/pre-release-e2e-test.yml @@ -64,7 +64,7 @@ permissions: jobs: # 1. resolve: gate on a real version bump (PR vs base) or manual input; compute CI_VERSION. resolve: - runs-on: Hyperloom-e2e-ci + runs-on: hyperloom-pre-e2e-ci outputs: run: ${{ steps.decide.outputs.run }} run_scope: ${{ steps.decide.outputs.run_scope }} @@ -160,7 +160,7 @@ jobs: build: needs: resolve if: needs.resolve.outputs.run == 'true' && needs.resolve.outputs.reuse == '' - runs-on: Hyperloom-e2e-ci + runs-on: hyperloom-pre-e2e-ci env: CI_VERSION: ${{ needs.resolve.outputs.ci_version }} BASE_VERSION: ${{ needs.resolve.outputs.base_version }} @@ -221,7 +221,7 @@ jobs: if: >- always() && needs.resolve.outputs.run == 'true' && (needs.build.result == 'success' || needs.build.result == 'skipped') - runs-on: Hyperloom-e2e-ci + runs-on: hyperloom-pre-e2e-ci timeout-minutes: 900 # ≥14h: covers the 12h leg + bootstrap/setup/agent overhead env: CI_VERSION: ${{ needs.resolve.outputs.ci_version }} From 10f262a1c5ee990440cbf716c7a83dafc6dc7f7c Mon Sep 17 00:00:00 2001 From: lishuoshuo-amd Date: Thu, 27 Aug 2026 16:48:02 +0800 Subject: [PATCH 10/52] pre-release-e2e: use suffixed ARC scale-set name hyperloom-pre-e2e-ci-hfjjz The bare install name hyperloom-pre-e2e-ci did not pick up jobs (queued >2min); the SaFE AutoscalingRunnerSet's listener pod is hyperloom-pre-e2e-ci-hfjjz-*, so the runner scale set name carries the -hfjjz suffix. Point runs-on at it. Co-Authored-By: Claude Opus 4.8 --- .github/workflows/pre-release-e2e-test.yml | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/.github/workflows/pre-release-e2e-test.yml b/.github/workflows/pre-release-e2e-test.yml index c2e880b597..29a722a927 100644 --- a/.github/workflows/pre-release-e2e-test.yml +++ b/.github/workflows/pre-release-e2e-test.yml @@ -64,7 +64,7 @@ permissions: jobs: # 1. resolve: gate on a real version bump (PR vs base) or manual input; compute CI_VERSION. resolve: - runs-on: hyperloom-pre-e2e-ci + runs-on: hyperloom-pre-e2e-ci-hfjjz outputs: run: ${{ steps.decide.outputs.run }} run_scope: ${{ steps.decide.outputs.run_scope }} @@ -160,7 +160,7 @@ jobs: build: needs: resolve if: needs.resolve.outputs.run == 'true' && needs.resolve.outputs.reuse == '' - runs-on: hyperloom-pre-e2e-ci + runs-on: hyperloom-pre-e2e-ci-hfjjz env: CI_VERSION: ${{ needs.resolve.outputs.ci_version }} BASE_VERSION: ${{ needs.resolve.outputs.base_version }} @@ -221,7 +221,7 @@ jobs: if: >- always() && needs.resolve.outputs.run == 'true' && (needs.build.result == 'success' || needs.build.result == 'skipped') - runs-on: hyperloom-pre-e2e-ci + runs-on: hyperloom-pre-e2e-ci-hfjjz timeout-minutes: 900 # ≥14h: covers the 12h leg + bootstrap/setup/agent overhead env: CI_VERSION: ${{ needs.resolve.outputs.ci_version }} From 78f0e04464400330fa6bf19050001f86cd6dac96 Mon Sep 17 00:00:00 2001 From: lishuoshuo-amd Date: Thu, 27 Aug 2026 17:36:56 +0800 Subject: [PATCH 11/52] ci(pre-release): run all 3 jobs on the dedicated bare-metal runner Switch runs-on from the SaFE ARC scale-set (hyperloom-pre-e2e-ci-hfjjz) to a standard self-hosted runner on crsuse2-m2m-061 (label hyperloom-pre-e2e-baremetal). The SaFE runner-proxy image predates the USER_APIKEY auth support (PR #662, 2026-07-10), so it never sends the injected platform key and every runner start 401s. A direct GitHub runner bypasses runner-proxy entirely. Co-Authored-By: Claude Opus 4.8 --- .github/workflows/pre-release-e2e-test.yml | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/.github/workflows/pre-release-e2e-test.yml b/.github/workflows/pre-release-e2e-test.yml index 29a722a927..447bfcfc12 100644 --- a/.github/workflows/pre-release-e2e-test.yml +++ b/.github/workflows/pre-release-e2e-test.yml @@ -64,7 +64,7 @@ permissions: jobs: # 1. resolve: gate on a real version bump (PR vs base) or manual input; compute CI_VERSION. resolve: - runs-on: hyperloom-pre-e2e-ci-hfjjz + runs-on: hyperloom-pre-e2e-baremetal outputs: run: ${{ steps.decide.outputs.run }} run_scope: ${{ steps.decide.outputs.run_scope }} @@ -160,7 +160,7 @@ jobs: build: needs: resolve if: needs.resolve.outputs.run == 'true' && needs.resolve.outputs.reuse == '' - runs-on: hyperloom-pre-e2e-ci-hfjjz + runs-on: hyperloom-pre-e2e-baremetal env: CI_VERSION: ${{ needs.resolve.outputs.ci_version }} BASE_VERSION: ${{ needs.resolve.outputs.base_version }} @@ -221,7 +221,7 @@ jobs: if: >- always() && needs.resolve.outputs.run == 'true' && (needs.build.result == 'success' || needs.build.result == 'skipped') - runs-on: hyperloom-pre-e2e-ci-hfjjz + runs-on: hyperloom-pre-e2e-baremetal timeout-minutes: 900 # ≥14h: covers the 12h leg + bootstrap/setup/agent overhead env: CI_VERSION: ${{ needs.resolve.outputs.ci_version }} From 19a8d0b5f37d4b0418e51639e459f02ed9181e23 Mon Sep 17 00:00:00 2001 From: lishuoshuo-amd Date: Thu, 27 Aug 2026 17:49:41 +0800 Subject: [PATCH 12/52] ci(pre-release): surface dispatch errors + self-heal NFS root perms Two bare-metal fixes: - dispatch: report create_workload failures to stderr, not stdout. The call runs inside wid=$(create_workload ...) command substitution, so a stdout message was captured into $wid and never reached the CI log -- the job failed with exit 1 and no visible reason. stderr surfaces the real HTTP status + SaFE API body. - build: add an idempotent step to mkdir+chown the CI NFS root. The runner runs as 'ubuntu', not in the group owning /shared_nfs, so it cannot create the root. Self-heals across runner reinstall / NFS remount. Co-Authored-By: Claude Opus 4.8 --- .github/scripts/pre-release-e2e-dispatch.sh | 7 +++++-- .github/workflows/pre-release-e2e-test.yml | 14 ++++++++++++++ 2 files changed, 19 insertions(+), 2 deletions(-) diff --git a/.github/scripts/pre-release-e2e-dispatch.sh b/.github/scripts/pre-release-e2e-dispatch.sh index dd2eb3b362..b695fb744e 100755 --- a/.github/scripts/pre-release-e2e-dispatch.sh +++ b/.github/scripts/pre-release-e2e-dispatch.sh @@ -169,12 +169,15 @@ create_workload() { code="$(printf '%s' "$resp" | tail -n1)" json="$(printf '%s' "$resp" | sed '$d')" if [ "$code" -lt 200 ] || [ "$code" -ge 300 ]; then - summary "❌ create '$name' failed (HTTP $code): $(printf '%s' "$json" | head -c 400)" + # Report to stderr: this runs inside wid="$(create_workload ...)" command + # substitution, so a stdout message would be captured into $wid and never + # reach the CI log. stderr surfaces the real HTTP status + API body. + echo "❌ create '$name' failed (HTTP $code): $(printf '%s' "$json" | head -c 400)" >&2 return 1 fi wid="$(printf '%s' "$json" | jq -r '.workloadId // empty')" if [ -z "$wid" ]; then - summary "❌ create '$name' returned no workloadId: $(printf '%s' "$json" | head -c 400)" + echo "❌ create '$name' returned no workloadId: $(printf '%s' "$json" | head -c 400)" >&2 return 1 fi printf '%s' "$wid" diff --git a/.github/workflows/pre-release-e2e-test.yml b/.github/workflows/pre-release-e2e-test.yml index 447bfcfc12..d53fe76cc9 100644 --- a/.github/workflows/pre-release-e2e-test.yml +++ b/.github/workflows/pre-release-e2e-test.yml @@ -168,6 +168,20 @@ jobs: steps: - uses: actions/checkout@v7 + - name: Ensure the CI NFS root is writable by the runner user + # The bare-metal runner runs as `ubuntu`, which is not in the group that owns + # /shared_nfs, so it cannot mkdir the CI root there. Self-heal idempotently + # (survives runner reinstall / NFS remount) instead of relying on a one-off + # manual chown. Needs passwordless sudo on the runner host. + run: | + set -euo pipefail + : "${NFS_ROOT:?PRE_E2E_NFS_ROOT is required}" + if [ ! -w "$NFS_ROOT" ]; then + sudo -n mkdir -p "$NFS_ROOT" + sudo -n chown "$(id -un):$(id -gn)" "$NFS_ROOT" + fi + test -w "$NFS_ROOT" && echo "NFS root writable: $NFS_ROOT" + - uses: actions/setup-python@v7 with: python-version: "3.11" From a941ea8b187466812f605dc31e44ec7158ddcd6b Mon Sep 17 00:00:00 2001 From: lishuoshuo-amd Date: Thu, 27 Aug 2026 17:53:58 +0800 Subject: [PATCH 13/52] ci(pre-release): sanitize workload name to a valid RFC 1123 subdomain SaFE derives the k8s object name from displayName. CI_VERSION contains '+' (1.0.0.dev...+ci), which is not a legal RFC 1123 subdomain char, so every create_workload 422'd ('metadata.name Invalid value'). Fold illegal chars to '-', lowercase, collapse/trim dashes before POSTing. The poll step keys off the DISPATCH_MAP (leg -> workloadId), not the name, so this is safe. Co-Authored-By: Claude Opus 4.8 --- .github/scripts/pre-release-e2e-dispatch.sh | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/.github/scripts/pre-release-e2e-dispatch.sh b/.github/scripts/pre-release-e2e-dispatch.sh index b695fb744e..28c3f2891c 100755 --- a/.github/scripts/pre-release-e2e-dispatch.sh +++ b/.github/scripts/pre-release-e2e-dispatch.sh @@ -146,6 +146,11 @@ common_env_json() { create_workload() { local name="$1" resources="$2" env="$3" privileged="$4" entry_b64="$5" deadline_s="${6:-}" local body resp code json wid dl_json="{}" + # SaFE derives the k8s object name from displayName, which must be an RFC 1123 + # subdomain (lowercase alphanumerics, '-', '.'; start/end alphanumeric). CI_VERSION + # contains '+' (e.g. 1.0.0.dev...+ci), which is illegal and 422s the create. Fold + # any illegal char to '-', lowercase, and collapse/trim leading-trailing dashes. + name="$(printf '%s' "$name" | tr '[:upper:]' '[:lower:]' | sed -E 's/[^a-z0-9.]+/-/g; s/-+/-/g; s/^-+//; s/-+$//')" # Attach the pod hard-deadline when both a field name and a value are set. if [ -n "$DEADLINE_FIELD" ] && [ -n "$deadline_s" ]; then dl_json="$(jq -n --arg k "$DEADLINE_FIELD" --argjson v "$deadline_s" '{($k): $v}')" From d0ccc9415cd8448476a57f40182be7e8c56b5dbc Mon Sep 17 00:00:00 2001 From: lishuoshuo-amd Date: Thu, 27 Aug 2026 18:13:51 +0800 Subject: [PATCH 14/52] fix(pre-release-e2e): cap SaFE workload name at 44 chars, alpha-lead SaFE's vworkload admission webhook enforces a stricter rule than RFC 1123: 1-44 chars, must start with an alphabetic char, end alphanumeric, lowercase [a-z0-9-]. The old 'e2e--' name embedded the full version (1.0.0.dev...+ci) -> 47 chars AND a leading-digit/'.'/'+' after the prefix. Build the name as 'e2e--<6-hex CI_VERSION hash>' so the leg stays intact and never collides across runs, and harden the sanitizer (strip leading non-alpha, cap 44, re-trim trailing dash). Co-Authored-By: Claude Opus 4.8 --- .github/scripts/pre-release-e2e-dispatch.sh | 28 +++++++++++++++------ 1 file changed, 21 insertions(+), 7 deletions(-) diff --git a/.github/scripts/pre-release-e2e-dispatch.sh b/.github/scripts/pre-release-e2e-dispatch.sh index 28c3f2891c..ea479e06d1 100755 --- a/.github/scripts/pre-release-e2e-dispatch.sh +++ b/.github/scripts/pre-release-e2e-dispatch.sh @@ -73,6 +73,15 @@ DEADLINE_12H_S="${DEADLINE_12H_S:-46800}" # 12h + 1h buffer = 13h DEADLINE_FIELD="${DEADLINE_FIELD:-timeout}" leg_deadline_s() { case "$1" in *-3h) echo "$DEADLINE_3H_S" ;; *-12h) echo "$DEADLINE_12H_S" ;; esac; } +# SaFE caps the derived k8s object name at 44 chars (see create_workload), so we can +# NOT embed the full CI_VERSION (e.g. 1.0.0.dev202608270954+ci) in every workload name +# -- it would blow the limit and, once truncated, collide across legs (the leg suffix +# gets cut). Instead build "e2e--": the human-readable leg +# stays intact up front, and a 6-hex digest of CI_VERSION disambiguates across runs +# without length risk. All legs of one run share the same VERSION_TAG. +VERSION_TAG="$(printf '%s' "$CI_VERSION" | sha1sum | cut -c1-6)" +workload_name() { printf 'e2e-%s-%s' "$1" "$VERSION_TAG"; } # $1 = leg (or "docker-host") + : "${SAFE_API_BASE:?SAFE_API_BASE is required}" : "${SAFE_API_KEY:?SAFE_API_KEY is required}" : "${SAFE_WORKSPACE_ID:?SAFE_WORKSPACE_ID is required}" @@ -146,11 +155,16 @@ common_env_json() { create_workload() { local name="$1" resources="$2" env="$3" privileged="$4" entry_b64="$5" deadline_s="${6:-}" local body resp code json wid dl_json="{}" - # SaFE derives the k8s object name from displayName, which must be an RFC 1123 - # subdomain (lowercase alphanumerics, '-', '.'; start/end alphanumeric). CI_VERSION - # contains '+' (e.g. 1.0.0.dev...+ci), which is illegal and 422s the create. Fold - # any illegal char to '-', lowercase, and collapse/trim leading-trailing dashes. - name="$(printf '%s' "$name" | tr '[:upper:]' '[:lower:]' | sed -E 's/[^a-z0-9.]+/-/g; s/-+/-/g; s/^-+//; s/-+$//')" + # SaFE derives the k8s object name from displayName and enforces (via the + # vworkload admission webhook, STRICTER than plain RFC 1123): 1-44 chars, lower + # case alphanumerics or '-', MUST start with an ALPHABETIC char and end with an + # alphanumeric. So a leading digit or a '.'/'+' (both in CI_VERSION, e.g. + # 1.0.0.dev...+ci) is illegal, and the full "e2e--" easily + # exceeds 44. Fold every illegal char to '-', lowercase, collapse/trim dashes, + # then cap at 44 chars re-trimming any trailing dash the cut may leave. + name="$(printf '%s' "$name" | tr '[:upper:]' '[:lower:]' | sed -E 's/[^a-z0-9]+/-/g; s/-+/-/g; s/^[^a-z]+//; s/-+$//')" + name="${name:0:44}"; name="${name%%-}" + [ -n "$name" ] || name="e2e" # Attach the pod hard-deadline when both a field name and a value are set. if [ -n "$DEADLINE_FIELD" ] && [ -n "$deadline_s" ]; then dl_json="$(jq -n --arg k "$DEADLINE_FIELD" --argjson v "$deadline_s" '{($k): $v}')" @@ -213,7 +227,7 @@ for leg in $REQ_TASKS; do | jq --arg leg "$leg" '. + {LEG_ID:$leg, HYPERLOOM_RUN_MODE:"baremetal"}')" entry="$(bootstrap_entry_b64 "")" dl="$(leg_deadline_s "$leg")" - wid="$(create_workload "e2e-${CI_VERSION}-${leg}" "$leg_resources_1gpu" "$env_json" false "$entry" "$dl")" + wid="$(create_workload "$(workload_name "$leg")" "$leg_resources_1gpu" "$env_json" false "$entry" "$dl")" DISPATCH["$leg"]="$wid" summary "• \`$leg\` → workloadId \`$wid\` (baremetal, 1 GPU, deadline $((dl/3600))h)" ;; @@ -266,7 +280,7 @@ if [ "$want_docker_host" = 1 ]; then for leg in $docker_legs; do case "$leg" in *-12h) host_dl="$DEADLINE_12H_S" ;; esac done - wid="$(create_workload "e2e-${CI_VERSION}-docker-host" "$host_resources" "$host_env" true "$entry" "$host_dl")" + wid="$(create_workload "$(workload_name "docker-host")" "$host_resources" "$host_env" true "$entry" "$host_dl")" # Every docker leg shares the one host workloadId; the poll distinguishes them by # reading each leg's own session dir on NFS. for leg in $docker_legs; do From 127a9ee96c861f72ce2b19f9bfd552fa1d9dbc4a Mon Sep 17 00:00:00 2001 From: lishuoshuo-amd Date: Thu, 27 Aug 2026 19:55:34 +0800 Subject: [PATCH 15/52] pre-release-e2e: fix live-run blockers (auth, root perms, workload kind, resources) Six fixes surfaced by the first live run (CI_VERSION 1.0.0.dev202608271014+ci): 1. 401 subscription-key: AMD's APIM gateway (llm-api.amd.com) rejects the bearer key alone -- it needs an Ocp-Apim-Subscription-Key header. Thread a new ANTHROPIC_CUSTOM_HEADERS through workflow -> dispatch env -> bootstrap .env; the CLI expands ${ANTHROPIC_API_KEY} so one secret covers bearer + subscription. 2. kind Authoring -> PyTorchJob: the Authoring mutating webhook overwrites EntryPoints to `sleep infinity`, so our bootstrap never auto-ran. PyTorchJob is not in that mutate switch and honors the submitted entrypoint; privileged/8-GPU/ useWorkspaceStorage/timeout are all kind-agnostic. E2E_DOCKER_HOST now travels in the workload env (a `VAR=1;` command prefix wouldn't export into the entrypoint). 3. root permissions: `claude --print` is fail-closed and dies with no approver -> add --dangerously-skip-permissions; it refuses under root unless IS_SANDBOX=1 (SWSPLAT-42390), matching Hyperloom's own kernel-agent. Also cd into the leg root so claude finds the workspace .env instead of blocking at '/'. 4. minimal base image: install Node/npm (claude CLI) and jq up front -- the SaFE rocm/pytorch image ships neither. 5. host resources: raise the privileged docker host to mem 2048Gi / ephemeral 1792Gi (ref sglang-kimik3-2). The first run was EVICTED at ephemeral 200Gi: vfs storage x 8 ROCm images has no layer dedup. Co-Authored-By: Claude Opus 4.8 --- .github/pre-release/bootstrap-pre-release.sh | 72 ++++++++++++++++++-- .github/scripts/pre-release-e2e-dispatch.sh | 54 +++++++++++---- .github/workflows/pre-release-e2e-test.yml | 9 ++- 3 files changed, 116 insertions(+), 19 deletions(-) diff --git a/.github/pre-release/bootstrap-pre-release.sh b/.github/pre-release/bootstrap-pre-release.sh index 7ed0fc9103..b60ff0a1b2 100755 --- a/.github/pre-release/bootstrap-pre-release.sh +++ b/.github/pre-release/bootstrap-pre-release.sh @@ -31,13 +31,41 @@ WHEEL_DIR="${NFS_ROOT%/}/wheels/${CI_VERSION}" log() { echo "[bootstrap $(date -u +%H:%M:%S)] $*"; } +# The SaFE rocm/pytorch Authoring image ships NO node/npm (verified 2026-08-27 on a +# live pod), so the Claude CLI's `npm install -g` fails with "npm: command not found". +# Install a pinned Node.js LTS from the official binary tarball (both nodejs.org and the +# npm registry are reachable from the pod) into /opt/node and expose it on PATH. Idempotent. +NODE_VERSION="${NODE_VERSION:-v20.18.0}" +NODE_PREFIX="${NODE_PREFIX:-/opt/node}" +ensure_node() { + if command -v node >/dev/null 2>&1 && command -v npm >/dev/null 2>&1; then + log "node present: $(node --version 2>/dev/null) npm $(npm --version 2>/dev/null)"; return 0 + fi + log "installing Node ${NODE_VERSION} (no node/npm in the Authoring base image)" + mkdir -p "$NODE_PREFIX" + curl -fsSL "https://nodejs.org/dist/${NODE_VERSION}/node-${NODE_VERSION}-linux-x64.tar.xz" \ + | tar -xJ -C "$NODE_PREFIX" --strip-components=1 \ + || { log "ERROR: Node download/extract failed"; return 1; } + export PATH="${NODE_PREFIX}/bin:${PATH}" + ln -sf "${NODE_PREFIX}/bin/node" /usr/local/bin/node 2>/dev/null || true + ln -sf "${NODE_PREFIX}/bin/npm" /usr/local/bin/npm 2>/dev/null || true + command -v node >/dev/null 2>&1 || { log "ERROR: node still missing after install"; return 1; } + log "node installed: $(node --version) npm $(npm --version)" +} + install_claude_cli() { if command -v claude >/dev/null 2>&1; then log "claude CLI present: $(claude --version 2>/dev/null || true)"; return; fi + ensure_node || return 1 log "installing Claude CLI @ ${CLAUDE_CLI_VERSION}" # Pinned install. The exact channel is environment-specific; keep the version in one - # place (CLAUDE_CLI_VERSION) so the pin is auditable. - npm install -g "@anthropic-ai/claude-code@${CLAUDE_CLI_VERSION}" >/dev/null 2>&1 \ - || { log "ERROR: claude CLI install failed"; return 1; } + # place (CLAUDE_CLI_VERSION) so the pin is auditable. Surface npm's error (don't + # swallow) so a failure is diagnosable in bootstrap.log. + npm install -g "@anthropic-ai/claude-code@${CLAUDE_CLI_VERSION}" >/tmp/npm-claude.log 2>&1 \ + || { log "ERROR: claude CLI install failed"; tail -20 /tmp/npm-claude.log; return 1; } + # npm's global bin may be under the Node prefix, not on the default PATH. + ln -sf "${NODE_PREFIX}/bin/claude" /usr/local/bin/claude 2>/dev/null || true + command -v claude >/dev/null 2>&1 || { log "ERROR: claude not on PATH after install"; return 1; } + log "claude CLI installed: $(claude --version 2>/dev/null || true)" } # Run ONE leg to completion inside the current filesystem (baremetal pod, or already @@ -62,6 +90,14 @@ run_leg() { { echo "ANTHROPIC_API_KEY=$(printf '%s' "$ANTHROPIC_API_KEY_B64" | base64 -d)" [ -n "${ANTHROPIC_BASE_URL:-}" ] && echo "ANTHROPIC_BASE_URL=${ANTHROPIC_BASE_URL}" + # Gateways like AMD's APIM (llm-api.amd.com) reject the bearer key alone with + # "401 Access denied due to missing subscription key" -- they need an + # Ocp-Apim-Subscription-Key header. The Claude CLI reads ANTHROPIC_CUSTOM_HEADERS + # (newline-delimited "Name: value") and sends it on every request; ${VAR} is + # expanded so the one key covers both the bearer and the subscription header. + # Double-quote in .env (space+colon in an unquoted value is parsed as a command + # on source -> exit 127). See docs/reference/authentication.md "extra headers". + [ -n "${ANTHROPIC_CUSTOM_HEADERS:-}" ] && echo "ANTHROPIC_CUSTOM_HEADERS=\"${ANTHROPIC_CUSTOM_HEADERS}\"" echo "CLAUDE_MODEL=${CLAUDE_MODEL}" echo "USER_DATA_PATH=${session}" echo "HYPERLOOM_RUN_MODE=${run_mode}" @@ -89,10 +125,22 @@ run_leg() { [ -f "$setup_prompt" ] || { log "ERROR: missing $setup_prompt"; return 1; } [ -f "$demo_prompt" ] || { log "ERROR: missing $demo_prompt"; return 1; } + # Run the agent FROM the leg root: the setup prompt refers to "the current + # workspace (REPO_ROOT)" for .env and the installed tree. Bootstrap otherwise runs + # from '/', where claude finds no .env and refuses (it sees ~29 unrelated + # workspaces under /shared_nfs and blocks rather than guess). cd fixes the cwd. + cd "$root" + # --dangerously-skip-permissions: `claude --print` is non-interactive, so the default + # permission mode is fail-closed -- setup's pip/shell steps have no approver and die. + # This is an isolated per-leg CI pod (same posture as the per-PR CI's _incontainer.sh), + # so bypassing the approval gate is acceptable and required for unattended setup/demo. + # IS_SANDBOX=1: the SaFE pod runs as root, and claude refuses to skip permissions under + # root unless IS_SANDBOX=1 (SWSPLAT-42390) -- Hyperloom's own kernel-agent sets the same. + export IS_SANDBOX=1 log "claude --print (setup)" - claude --print < "$setup_prompt" + claude --print --dangerously-skip-permissions < "$setup_prompt" log "claude --print (demo ${hours}h)" - claude --print < "$demo_prompt" + claude --print --dangerously-skip-permissions < "$demo_prompt" log "leg $leg agent turns complete; poll will judge $session/reports/final.json" } @@ -145,7 +193,21 @@ run_docker_host() { return "$rc" } +# The SaFE rocm/pytorch Authoring image is minimal: besides node/npm (see ensure_node) +# it also lacks `jq`, which run_docker_host uses to read DOCKER_GPU_MAP. Install it up +# front (apt works in the pod) so both the host pod and the nested single-leg containers +# -- which re-run THIS script -- have it. Idempotent; cheap when already present. +ensure_base_tools() { + command -v jq >/dev/null 2>&1 && return 0 + log "installing jq (not in the Authoring base image)" + export DEBIAN_FRONTEND=noninteractive + apt-get update -qq >/tmp/apt-jq.log 2>&1 && apt-get install -y -qq jq >>/tmp/apt-jq.log 2>&1 \ + || { log "ERROR: jq install failed"; tail -20 /tmp/apt-jq.log; return 1; } + log "jq installed: $(jq --version 2>&1)" +} + # ---- entry ----------------------------------------------------------------- +ensure_base_tools || exit 1 install_claude_cli || exit 1 if [ "${E2E_DOCKER_HOST:-}" = "1" ]; then diff --git a/.github/scripts/pre-release-e2e-dispatch.sh b/.github/scripts/pre-release-e2e-dispatch.sh index ea479e06d1..671c8e6446 100755 --- a/.github/scripts/pre-release-e2e-dispatch.sh +++ b/.github/scripts/pre-release-e2e-dispatch.sh @@ -2,17 +2,19 @@ # SPDX-FileCopyrightText: 2026 Advanced Micro Devices, Inc. # SPDX-License-Identifier: MIT # -# Pre-release E2E: create the SaFE Authoring workloads that run the packaged wheel +# Pre-release E2E: create the SaFE PyTorchJob workloads that run the packaged wheel # through the real user path (Claude CLI + setup skill + demo skill). Unlike the PR # smoke test (.github/scripts/ci-e2e-dispatch.sh), which uses the orchestration # endpoint (POST /api/v1/orchestration/workloads, kind=hyperloom) to dispatch a git -# SHA, this dispatches GENERIC Authoring pods (POST /api/v1/workloads, -# kind=Authoring) whose entrypoint is the bootstrap script. See +# SHA, this dispatches GENERIC pods (POST /api/v1/workloads) whose entrypoint is the +# bootstrap script. kind=PyTorchJob, NOT Authoring: the Authoring mutating webhook +# rewrites EntryPoints to `sleep infinity`, so an Authoring pod would never run our +# bootstrap; PyTorchJob honors the submitted entrypoint. See # hyperloom-pre-release-e2e-ci-design.md §7. # # It creates 5 workloads for the 8 legs: -# * 4x non-privileged 1-GPU Authoring (one per baremetal leg) -# * 1x privileged 8-GPU Authoring (docker host; 4 nested containers, GPU 0-3) +# * 4x non-privileged 1-GPU PyTorchJob (one per baremetal leg) +# * 1x privileged 8-GPU PyTorchJob (docker host; 4 nested containers, GPU 0-3) # and writes a dispatch map (leg -> workloadId) to $DISPATCH_MAP for the poll step. # # Requires: bash, curl, jq on the (self-hosted, in-network) runner. @@ -37,8 +39,8 @@ # TASKS comma-separated leg subset (default: all 8) # DISPATCH_MAP output file: JSON {leg: workloadId} # (default $RUNNER_TEMP/pre_release_dispatch.json) -# HOST_CPU / HOST_MEM / HOST_SHM privileged host resource request -# (default 128 / 512Gi / 256Gi) +# HOST_CPU / HOST_MEM / HOST_SHM / HOST_EPHEMERAL privileged host resource request +# (default 128 / 2048Gi / 256Gi / 1792Gi -- ref 8-GPU Authoring pod) # LEG_CPU / LEG_MEM baremetal leg resource request # (default 32 / 128Gi) # DEADLINE_3H_S / DEADLINE_12H_S pod hard-timeout per duration @@ -54,7 +56,13 @@ set -euo pipefail NFS_ROOT="${NFS_ROOT:-/shared_nfs/hyperloom-pre-release-e2e-test}" TARGET_GAIN="${TARGET_GAIN:-100}" -HOST_CPU="${HOST_CPU:-128}"; HOST_MEM="${HOST_MEM:-512Gi}"; HOST_SHM="${HOST_SHM:-256Gi}" +# Sized to a proven Running 8-GPU Authoring pod (ref: sglang-kimik3-2): CPU 128, +# mem 2048Gi, ephemeral 1792Gi. The privileged DinD host runs nested docker with the +# vfs storage driver (no layer dedup), so 8 ROCm images x full copies blow past a small +# ephemeral limit -- our first run was EVICTED at ephemeralStorage 200Gi. 1792Gi matches +# the reference host and leaves headroom for vfs image blowup + model/wheel scratch. +HOST_CPU="${HOST_CPU:-128}"; HOST_MEM="${HOST_MEM:-2048Gi}"; HOST_SHM="${HOST_SHM:-256Gi}" +HOST_EPHEMERAL="${HOST_EPHEMERAL:-1792Gi}" LEG_CPU="${LEG_CPU:-32}"; LEG_MEM="${LEG_MEM:-128Gi}" DISPATCH_MAP="${DISPATCH_MAP:-${RUNNER_TEMP:-/tmp}/pre_release_dispatch.json}" @@ -137,6 +145,7 @@ common_env_json() { --arg cmodel "$CLAUDE_MODEL" --arg cver "$CLAUDE_CLI_VERSION" \ --arg keyb64 "$(printf '%s' "$ANTHROPIC_API_KEY" | base64 | tr -d '\n')" \ --arg baseurl "${ANTHROPIC_BASE_URL:-}" \ + --arg cheaders "${ANTHROPIC_CUSTOM_HEADERS:-}" \ '{ CI_VERSION: $civ, NFS_ROOT: $nfs, @@ -147,7 +156,9 @@ common_env_json() { CLAUDE_MODEL: $cmodel, CLAUDE_CLI_VERSION: $cver, ANTHROPIC_API_KEY_B64: $keyb64 - } + (if $baseurl == "" then {} else {ANTHROPIC_BASE_URL: $baseurl} end)' + } + + (if $baseurl == "" then {} else {ANTHROPIC_BASE_URL: $baseurl} end) + + (if $cheaders == "" then {} else {ANTHROPIC_CUSTOM_HEADERS: $cheaders} end)' } # POST one workload; echo the workloadId. @@ -169,6 +180,15 @@ create_workload() { if [ -n "$DEADLINE_FIELD" ] && [ -n "$deadline_s" ]; then dl_json="$(jq -n --arg k "$DEADLINE_FIELD" --argjson v "$deadline_s" '{($k): $v}')" fi + # kind PyTorchJob, NOT Authoring: SaFE's mutating webhook (mutateAuthoring) + # unconditionally overwrites an Authoring workload's EntryPoints to `sleep infinity`, + # so our bootstrap entrypoint would never auto-run -- every leg would idle until + # something exec'd in. PyTorchJob is not in that mutate switch, so it HONORS the + # submitted entryPoints (run via launcher.sh) and bootstrap runs as the pod command. + # version stays "v1"; NO `group` field (webhook clears group, workload_webhook.go:260). + # privileged / 8-GPU / useWorkspaceStorage / timeout are all kind-agnostic (driven by + # request fields, not kind) -- confirmed against Primus-SaFE source. Pod name is + # -master-0, main container `pytorch`. body="$(jq -n \ --arg name "$name" --arg ws "$SAFE_WORKSPACE_ID" --arg img "$AUTHORING_IMAGE" \ --arg entry "$entry_b64" --argjson res "$resources" --argjson env "$env" \ @@ -176,7 +196,7 @@ create_workload() { '{ displayName: $name, workspaceId: $ws, - groupVersionKind: {kind:"Authoring", version:"v1"}, + groupVersionKind: {kind:"PyTorchJob", version:"v1"}, resources: [$res], images: [$img], entryPoints: [$entry], @@ -243,7 +263,8 @@ done # ---- docker legs: one privileged 8-GPU host running all requested docker legs ---- if [ "$want_docker_host" = 1 ]; then host_resources="$(jq -n --arg cpu "$HOST_CPU" --arg mem "$HOST_MEM" --arg shm "$HOST_SHM" \ - '{replica:1, gpu:"8", cpu:$cpu, memory:$mem, sharedMemory:$shm, ephemeralStorage:"200Gi"}')" + --arg eph "$HOST_EPHEMERAL" \ + '{replica:1, gpu:"8", cpu:$cpu, memory:$mem, sharedMemory:$shm, ephemeralStorage:$eph}')" # The host env carries the per-leg GPU map so the host bootstrap launches the right # nested containers via docker-run-hyperloom.sh . docker_legs=""; gpu_map="{}" @@ -264,6 +285,7 @@ if [ "$want_docker_host" = 1 ]; then --arg tgain "$TARGET_GAIN" --arg cmodel "$CLAUDE_MODEL" --arg cver "$CLAUDE_CLI_VERSION" \ --arg keyb64 "$(printf '%s' "$ANTHROPIC_API_KEY" | base64 | tr -d '\n')" \ --arg baseurl "${ANTHROPIC_BASE_URL:-}" \ + --arg cheaders "${ANTHROPIC_CUSTOM_HEADERS:-}" \ --arg legs "$docker_legs" --argjson gpumap "$gpu_map" \ '{ CI_VERSION:$civ, NFS_ROOT:$nfs, @@ -271,9 +293,15 @@ if [ "$want_docker_host" = 1 ]; then TARGET_GAIN:$tgain, CLAUDE_MODEL:$cmodel, CLAUDE_CLI_VERSION:$cver, ANTHROPIC_API_KEY_B64:$keyb64, HYPERLOOM_RUN_MODE:"docker", + E2E_DOCKER_HOST:"1", DOCKER_LEGS:$legs, DOCKER_GPU_MAP:($gpumap|tostring) - } + (if $baseurl == "" then {} else {ANTHROPIC_BASE_URL:$baseurl} end)')" - entry="$(bootstrap_entry_b64 "E2E_DOCKER_HOST=1;")" + } + + (if $baseurl == "" then {} else {ANTHROPIC_BASE_URL:$baseurl} end) + + (if $cheaders == "" then {} else {ANTHROPIC_CUSTOM_HEADERS:$cheaders} end)')" + # E2E_DOCKER_HOST=1 travels in the workload `env` (above), NOT as a command prefix: + # under PyTorchJob the entrypoint actually runs, and a `VAR=1;` prefix followed by a + # separate `exec bash` would NOT export VAR into the bootstrap's environment. + entry="$(bootstrap_entry_b64 "")" # The one host pod runs a mix of 3h and 12h nested legs, so its deadline must be # the MAX over the legs it hosts (a 3h deadline would kill a still-running 12h leg). host_dl="$DEADLINE_3H_S" diff --git a/.github/workflows/pre-release-e2e-test.yml b/.github/workflows/pre-release-e2e-test.yml index d53fe76cc9..dc60985875 100644 --- a/.github/workflows/pre-release-e2e-test.yml +++ b/.github/workflows/pre-release-e2e-test.yml @@ -25,7 +25,8 @@ name: Pre-release E2E test # Variables (non-secret): PRE_E2E_SAFE_API_BASE, PRE_E2E_SAFE_WORKSPACE_ID, # PRE_E2E_AUTHORING_IMAGE, PRE_E2E_NFS_ROOT, PRE_E2E_MODEL_3H_PATH, # PRE_E2E_MODEL_12H_PATH, PRE_E2E_CLAUDE_MODEL, PRE_E2E_CLAUDE_CLI_VERSION, -# (optional) PRE_E2E_ANTHROPIC_BASE_URL, PRE_E2E_SAFE_INSECURE. +# (optional) PRE_E2E_ANTHROPIC_BASE_URL, PRE_E2E_ANTHROPIC_CUSTOM_HEADERS +# (APIM gateways: `Ocp-Apim-Subscription-Key: ${ANTHROPIC_API_KEY}`), PRE_E2E_SAFE_INSECURE. # Nothing below hard-codes an environment value; all are `${{ secrets.* }}` / # `${{ vars.* }}` references. TODO(owner): populate these before first real run. @@ -255,6 +256,12 @@ jobs: CLAUDE_CLI_VERSION: ${{ vars.PRE_E2E_CLAUDE_CLI_VERSION }} ANTHROPIC_API_KEY: ${{ secrets.PRE_E2E_ANTHROPIC_API_KEY }} # GitHub Secret path (design §9 D) ANTHROPIC_BASE_URL: ${{ vars.PRE_E2E_ANTHROPIC_BASE_URL }} + # Extra request headers for gateways that authenticate on a header of their own + # (AMD APIM wants Ocp-Apim-Subscription-Key, else `401 missing subscription key`). + # The Claude CLI reads ANTHROPIC_CUSTOM_HEADERS; ${ANTHROPIC_API_KEY} is expanded + # in-pod so the one secret covers both bearer + subscription-key. Header name is + # non-sensitive -> a Variable. e.g. `Ocp-Apim-Subscription-Key: ${ANTHROPIC_API_KEY}` + ANTHROPIC_CUSTOM_HEADERS: ${{ vars.PRE_E2E_ANTHROPIC_CUSTOM_HEADERS }} # per-leg GitHub commit status GH_STATUS_TOKEN: ${{ secrets.GITHUB_TOKEN }} GH_STATUS_REPO: ${{ github.repository }} From 8542b4d22dbf87315c9b3108fd112357fa24b2d0 Mon Sep 17 00:00:00 2001 From: lishuoshuo-amd Date: Thu, 27 Aug 2026 20:00:02 +0800 Subject: [PATCH 16/52] pre-release-e2e: default the APIM subscription-key header in-pod (no repo var needed) The Ocp-Apim-Subscription-Key header value is just the API key (${ANTHROPIC_API_KEY}, expanded in-pod by Hyperloom's parse_custom_headers), carrying no new secret -- the key already reaches the pod via ANTHROPIC_API_KEY_B64. So instead of requiring a PRE_E2E_ANTHROPIC_CUSTOM_HEADERS repo variable (a "forgot to set it -> 401" trap), bootstrap now defaults ANTHROPIC_CUSTOM_HEADERS to `Ocp-Apim-Subscription-Key: ${ANTHROPIC_API_KEY}` whenever a gateway base URL is set, matching Hyperloom's .env.template and hyperloom-setup SKILL.md exactly. An externally supplied header still overrides; direct api.anthropic.com emits no header. Co-Authored-By: Claude Opus 4.8 --- .github/pre-release/bootstrap-pre-release.sh | 22 ++++++++++++++------ .github/workflows/pre-release-e2e-test.yml | 8 +++++-- 2 files changed, 22 insertions(+), 8 deletions(-) diff --git a/.github/pre-release/bootstrap-pre-release.sh b/.github/pre-release/bootstrap-pre-release.sh index b60ff0a1b2..25c62dc688 100755 --- a/.github/pre-release/bootstrap-pre-release.sh +++ b/.github/pre-release/bootstrap-pre-release.sh @@ -92,12 +92,22 @@ run_leg() { [ -n "${ANTHROPIC_BASE_URL:-}" ] && echo "ANTHROPIC_BASE_URL=${ANTHROPIC_BASE_URL}" # Gateways like AMD's APIM (llm-api.amd.com) reject the bearer key alone with # "401 Access denied due to missing subscription key" -- they need an - # Ocp-Apim-Subscription-Key header. The Claude CLI reads ANTHROPIC_CUSTOM_HEADERS - # (newline-delimited "Name: value") and sends it on every request; ${VAR} is - # expanded so the one key covers both the bearer and the subscription header. - # Double-quote in .env (space+colon in an unquoted value is parsed as a command - # on source -> exit 127). See docs/reference/authentication.md "extra headers". - [ -n "${ANTHROPIC_CUSTOM_HEADERS:-}" ] && echo "ANTHROPIC_CUSTOM_HEADERS=\"${ANTHROPIC_CUSTOM_HEADERS}\"" + # Ocp-Apim-Subscription-Key header whose value is the SAME key. The Claude CLI + # reads ANTHROPIC_CUSTOM_HEADERS (newline-delimited "Name: value") and sends it + # on every request; ${ANTHROPIC_API_KEY} is expanded in-pod so the one key + # covers both bearer + subscription. The header NAME is fixed and non-secret and + # its value carries no NEW secret (the key already arrives via *_B64), so we + # default it here rather than requiring a repo variable -- one less "forgot to + # configure it -> 401" trap. This matches Hyperloom's own llm_config.py contract + # (parse_custom_headers expands ${VAR}). An externally-supplied + # ANTHROPIC_CUSTOM_HEADERS still wins, for a gateway with a different convention. + # Only emitted when a gateway base URL is set (direct api.anthropic.com needs no + # subscription header). Double-quote in .env (space+colon unquoted -> exit 127). + if [ -n "${ANTHROPIC_CUSTOM_HEADERS:-}" ]; then + echo "ANTHROPIC_CUSTOM_HEADERS=\"${ANTHROPIC_CUSTOM_HEADERS}\"" + elif [ -n "${ANTHROPIC_BASE_URL:-}" ]; then + echo 'ANTHROPIC_CUSTOM_HEADERS="Ocp-Apim-Subscription-Key: ${ANTHROPIC_API_KEY}"' + fi echo "CLAUDE_MODEL=${CLAUDE_MODEL}" echo "USER_DATA_PATH=${session}" echo "HYPERLOOM_RUN_MODE=${run_mode}" diff --git a/.github/workflows/pre-release-e2e-test.yml b/.github/workflows/pre-release-e2e-test.yml index dc60985875..abebe2c1da 100644 --- a/.github/workflows/pre-release-e2e-test.yml +++ b/.github/workflows/pre-release-e2e-test.yml @@ -25,8 +25,12 @@ name: Pre-release E2E test # Variables (non-secret): PRE_E2E_SAFE_API_BASE, PRE_E2E_SAFE_WORKSPACE_ID, # PRE_E2E_AUTHORING_IMAGE, PRE_E2E_NFS_ROOT, PRE_E2E_MODEL_3H_PATH, # PRE_E2E_MODEL_12H_PATH, PRE_E2E_CLAUDE_MODEL, PRE_E2E_CLAUDE_CLI_VERSION, -# (optional) PRE_E2E_ANTHROPIC_BASE_URL, PRE_E2E_ANTHROPIC_CUSTOM_HEADERS -# (APIM gateways: `Ocp-Apim-Subscription-Key: ${ANTHROPIC_API_KEY}`), PRE_E2E_SAFE_INSECURE. +# (optional) PRE_E2E_ANTHROPIC_BASE_URL, PRE_E2E_SAFE_INSECURE. +# PRE_E2E_ANTHROPIC_CUSTOM_HEADERS is NOT required: when a gateway base URL is set, +# the bootstrap defaults the AMD APIM header to +# `Ocp-Apim-Subscription-Key: ${ANTHROPIC_API_KEY}` (the value is just the key, +# which already reaches the pod). Set this variable only to OVERRIDE that default +# for a gateway with a different header convention. # Nothing below hard-codes an environment value; all are `${{ secrets.* }}` / # `${{ vars.* }}` references. TODO(owner): populate these before first real run. From fda804d4f2299e3554e31f646624a3634a61c4f8 Mon Sep 17 00:00:00 2001 From: lishuoshuo-amd Date: Thu, 27 Aug 2026 20:02:55 +0800 Subject: [PATCH 17/52] pre-release-e2e: dispatch legs at High priority (Spec.Priority=2) The release-gate legs hold up to 8 GPUs for as long as 14h and block the release, so they must not be starved behind dev workloads. SaFE's scheduler orders the queue by Spec.Priority (int; High=2/Med=1/Low=0 per Primus-SaFE constant.go; the webhook clamps to [0,2]). Add `priority: $prio` to the create-workload body, default 2, overridable via PRIORITY. Co-Authored-By: Claude Opus 4.8 --- .github/scripts/pre-release-e2e-dispatch.sh | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/.github/scripts/pre-release-e2e-dispatch.sh b/.github/scripts/pre-release-e2e-dispatch.sh index 671c8e6446..fa9bfb117b 100755 --- a/.github/scripts/pre-release-e2e-dispatch.sh +++ b/.github/scripts/pre-release-e2e-dispatch.sh @@ -64,6 +64,11 @@ TARGET_GAIN="${TARGET_GAIN:-100}" HOST_CPU="${HOST_CPU:-128}"; HOST_MEM="${HOST_MEM:-2048Gi}"; HOST_SHM="${HOST_SHM:-256Gi}" HOST_EPHEMERAL="${HOST_EPHEMERAL:-1792Gi}" LEG_CPU="${LEG_CPU:-32}"; LEG_MEM="${LEG_MEM:-128Gi}" +# SaFE workload scheduling priority (Spec.Priority, an int): High=2, Med=1, Low=0 +# (Primus-SaFE common/constant.go). The scheduler orders the queue by this value, and +# the webhook clamps it into [0,2]. These release-gate legs hold 8 GPUs for up to 14h +# and block the release, so run them High so they aren't starved behind dev workloads. +PRIORITY="${PRIORITY:-2}" DISPATCH_MAP="${DISPATCH_MAP:-${RUNNER_TEMP:-/tmp}/pre_release_dispatch.json}" # Pod hard-timeout (design: 3h leg -> 3+1h, 12h leg -> 12+1h). SaFE terminates the @@ -193,6 +198,7 @@ create_workload() { --arg name "$name" --arg ws "$SAFE_WORKSPACE_ID" --arg img "$AUTHORING_IMAGE" \ --arg entry "$entry_b64" --argjson res "$resources" --argjson env "$env" \ --argjson priv "$privileged" --argjson dl "$dl_json" \ + --argjson prio "$PRIORITY" \ '{ displayName: $name, workspaceId: $ws, @@ -201,6 +207,7 @@ create_workload() { images: [$img], entryPoints: [$entry], env: $env, + priority: $prio, useWorkspaceStorage: true } + (if $priv then {privileged:true} else {} end) + $dl')" resp="$(curl -sS "${tls[@]}" -w $'\n%{http_code}' -X POST "$API" \ From d9c12b1ee4edc8d4c2f201ee800d10fae32ac832 Mon Sep 17 00:00:00 2001 From: lishuoshuo-amd Date: Thu, 27 Aug 2026 20:11:13 +0800 Subject: [PATCH 18/52] pre-release-e2e: newer push supersedes the in-flight run (auto stop + rerun) Make "push a fix -> the stale run is torn down and the fresh code reruns from scratch" automatic: * concurrency.cancel-in-progress: true, group keyed per-PR. GitHub now cancels the in-flight run on a newer commit to the same PR; the run's `if: cancelled()` step stops its SaFE workloads (freeing GPUs) before the new run dispatches. Previously (cancel-in-progress: false) a new push queued behind a run built from superseded code, and stale legs kept their GPUs until they timed out. * dispatch writes DISPATCH_MAP INCREMENTALLY (record_dispatch appends after every successful create, seeded with an empty map up front) instead of once at the end. With cancel-in-progress a cancel can land mid-dispatch; the end-only write would leave already-created workloads absent from the map and leak their GPUs. Now the cleanup step always sees every workload created so far. Co-Authored-By: Claude Opus 4.8 --- .github/scripts/pre-release-e2e-dispatch.sh | 33 ++++++++++++++------- .github/workflows/pre-release-e2e-test.yml | 13 +++++--- 2 files changed, 32 insertions(+), 14 deletions(-) diff --git a/.github/scripts/pre-release-e2e-dispatch.sh b/.github/scripts/pre-release-e2e-dispatch.sh index fa9bfb117b..c4ec3dff91 100755 --- a/.github/scripts/pre-release-e2e-dispatch.sh +++ b/.github/scripts/pre-release-e2e-dispatch.sh @@ -242,6 +242,23 @@ bootstrap_entry_b64() { echo "[dispatch] CI_VERSION=$CI_VERSION tasks='$REQ_TASKS'" declare -A DISPATCH # leg -> workloadId +# Persist the dispatch map INCREMENTALLY, one entry per created workload -- not just +# once at the end. With concurrency.cancel-in-progress a newer push can cancel THIS run +# mid-dispatch; the job's `if: cancelled()` cleanup then stops whatever is in +# DISPATCH_MAP. If the map were written only after the loop, workloads created before +# the cancel would leak their GPUs. Seed an empty map up front so the file always +# exists, then append after every successful create. +: > "$DISPATCH_MAP" 2>/dev/null || true +printf '{}\n' > "$DISPATCH_MAP" +record_dispatch() { # leg workloadId -- add to the in-memory map AND the on-disk map + local leg="$1" wid="$2" + DISPATCH["$leg"]="$wid" + local tmp="${DISPATCH_MAP}.tmp" + if jq --arg l "$leg" --arg w "$wid" '. + {($l):$w}' "$DISPATCH_MAP" > "$tmp" 2>/dev/null; then + mv "$tmp" "$DISPATCH_MAP" + fi +} + # ---- baremetal legs: one non-privileged 1-GPU workload each ---------------- leg_resources_1gpu="$(jq -n --arg cpu "$LEG_CPU" --arg mem "$LEG_MEM" \ '{replica:1, gpu:"1", cpu:$cpu, memory:$mem, ephemeralStorage:"100Gi"}')" @@ -255,7 +272,7 @@ for leg in $REQ_TASKS; do entry="$(bootstrap_entry_b64 "")" dl="$(leg_deadline_s "$leg")" wid="$(create_workload "$(workload_name "$leg")" "$leg_resources_1gpu" "$env_json" false "$entry" "$dl")" - DISPATCH["$leg"]="$wid" + record_dispatch "$leg" "$wid" summary "• \`$leg\` → workloadId \`$wid\` (baremetal, 1 GPU, deadline $((dl/3600))h)" ;; docker-*) @@ -319,19 +336,15 @@ if [ "$want_docker_host" = 1 ]; then # Every docker leg shares the one host workloadId; the poll distinguishes them by # reading each leg's own session dir on NFS. for leg in $docker_legs; do - DISPATCH["$leg"]="$wid" + record_dispatch "$leg" "$wid" summary "• \`$leg\` → workloadId \`$wid\` (docker host, GPU $(docker_gpu_index "$leg"), deadline $((host_dl/3600))h)" done fi -# ---- write the dispatch map for the poll step ------------------------------ -map_json="{}" -for leg in "${!DISPATCH[@]}"; do - map_json="$(printf '%s' "$map_json" | jq --arg l "$leg" --arg w "${DISPATCH[$leg]}" '. + {($l):$w}')" -done -printf '%s\n' "$map_json" > "$DISPATCH_MAP" +# ---- dispatch map already written incrementally by record_dispatch --------- +# (so a mid-dispatch cancel still leaves a complete-so-far map for cleanup to stop). echo "dispatch_map=$DISPATCH_MAP" >> "${GITHUB_OUTPUT:-/dev/null}" summary "" -summary "**dispatched $(printf '%s' "$map_json" | jq 'length') legs** → \`$DISPATCH_MAP\`" +summary "**dispatched $(jq 'length' "$DISPATCH_MAP") legs** → \`$DISPATCH_MAP\`" echo "[dispatch] wrote $DISPATCH_MAP" -printf '%s\n' "$map_json" | jq . +jq . "$DISPATCH_MAP" diff --git a/.github/workflows/pre-release-e2e-test.yml b/.github/workflows/pre-release-e2e-test.yml index abebe2c1da..cc6f559e82 100644 --- a/.github/workflows/pre-release-e2e-test.yml +++ b/.github/workflows/pre-release-e2e-test.yml @@ -54,11 +54,16 @@ on: description: "Comma-separated subset of leg IDs (default: all 8)" required: false -# No cross-version concurrency (design §13, point D): serialize pre-release runs so the -# peak GPU footprint stays at 8. A newer run QUEUES behind the in-flight one. +# One pre-release run at a time (design §13, point D): the peak GPU footprint stays at +# 8. cancel-in-progress: a NEWER push supersedes the in-flight run -- GitHub cancels the +# old run, whose `if: cancelled()` step stops its SaFE workloads (freeing GPUs) before +# this new run dispatches. This is what makes "push a fix -> the stale run is torn down +# and the fresh code reruns from scratch" automatic, instead of queueing behind a run +# built from superseded code. Per-PR concurrency (group keyed by ref) so two different +# PRs don't cancel each other -- only newer commits on the SAME PR supersede. concurrency: - group: pre-release-e2e - cancel-in-progress: false + group: pre-release-e2e-${{ github.event.pull_request.number || github.ref }} + cancel-in-progress: true permissions: contents: write # commit-comment fallback when no PR is associated From 3f1df200a89e95086203392028742d12b85eb5ba Mon Sep 17 00:00:00 2001 From: lishuoshuo-amd Date: Thu, 27 Aug 2026 20:28:11 +0800 Subject: [PATCH 19/52] pre-release-e2e: bootstrap waits for demo to finish (fix false Succeeded) claude --print is a single non-interactive turn: the demo skill backgrounds `optimize` (setsid nohup) and returns immediately, so run.sh exited 0 and SaFE marked a false "Succeeded" after ~6min for a 3h/12h benchmark that never wrote reports/final.json. run_leg now blocks after the demo turn until the backgrounded optimize reaches a terminal state: it globs the newest nested session dir under $session that holds a state.json (make_session_dir creates $session//-/ and re-pins INFERENCE_OPTIMIZER_CURRENT_SESSION_DIR only inside the CLI process, so the glob is authoritative), re-points the poll's .session_dir pin at it, and polls until reports/final.json exists (or state.json has a terminal stop_reason + a short grace). Returns 0 iff final.json exists (PASS/FAIL by gain stays the poll's job); returns 1 on startup grace (optimize never launched), hard deadline (hours*3600+3600), or a terminal stop_reason with no report -> SaFE marks Failed, not a false Succeeded. Fixing the .session_dir pin also fixes a second latent bug: the poll looked for $session/reports/final.json (parent), but the report lives in the nested run dir. dispatch: widen SaFE pod timeout (DEADLINE_3H_S 14400->16200, DEADLINE_12H_S 46800->48600) so the pod hard-timeout stays strictly greater than bootstrap's own in-pod wait deadline; otherwise SaFE could pre-empt the pod mid-wait and lose the clean return-1 path. Ordering per leg: bootstrap deadline < SaFE pod timeout < poll GLOBAL_TIMEOUT_S (50400s). Co-Authored-By: Claude Opus 4.8 --- .github/pre-release/bootstrap-pre-release.sh | 67 +++++++++++++++++++- .github/scripts/pre-release-e2e-dispatch.sh | 18 ++++-- 2 files changed, 77 insertions(+), 8 deletions(-) diff --git a/.github/pre-release/bootstrap-pre-release.sh b/.github/pre-release/bootstrap-pre-release.sh index 25c62dc688..b0540186ec 100755 --- a/.github/pre-release/bootstrap-pre-release.sh +++ b/.github/pre-release/bootstrap-pre-release.sh @@ -151,7 +151,72 @@ run_leg() { claude --print --dangerously-skip-permissions < "$setup_prompt" log "claude --print (demo ${hours}h)" claude --print --dangerously-skip-permissions < "$demo_prompt" - log "leg $leg agent turns complete; poll will judge $session/reports/final.json" + log "leg $leg demo turn returned; waiting for the background optimize to finish" + + # ---- wait for the backgrounded `optimize` to reach a terminal state -------- + # `claude --print` is ONE non-interactive turn: it returns right after the demo + # skill backgrounds `optimize` (setsid nohup). If we returned now, run.sh would + # exit 0 and SaFE would mark a FALSE "Succeeded" while the benchmark is still + # running. Block here until the run writes reports/final.json (or a deadline). + # + # The real artifacts do NOT live under $session directly: make_session_dir() + # creates a NESTED per-run dir $session//-/ + # and re-pins INFERENCE_OPTIMIZER_CURRENT_SESSION_DIR to it -- but that re-pin + # happens in the CLI's own process and never reaches us. So we discover the real + # dir by globbing the newest one under $session that contains a state.json, and + # re-point the poll's pin (.session_dir) at it. + local wait_interval="${LEG_WAIT_INTERVAL_S:-45}" + local startup_grace="${LEG_STARTUP_GRACE_S:-900}" # 15m for optimize to appear + local final_grace="${LEG_FINAL_GRACE_S:-120}" # state stop_reason -> final.json + local deadline_s=$(( hours * 3600 + 3600 )) # demo budget + 1h margin + local start_ts; start_ts="$(date +%s)" + local real_sdir="" final_json="" state_json="" + + while :; do + local now elapsed; now="$(date +%s)"; elapsed=$(( now - start_ts )) + + if [ -z "$real_sdir" ]; then + # newest dir under $session that actually contains a state.json + real_sdir="$(find "$session" -mindepth 2 -type f -name state.json -printf '%T@ %h\n' 2>/dev/null \ + | sort -rn | head -n1 | cut -d' ' -f2- || true)" + if [ -n "$real_sdir" ]; then + final_json="${real_sdir}/reports/final.json" + state_json="${real_sdir}/state.json" + log "leg $leg real session dir: $real_sdir" + # Re-pin so the poll (leg_session_dir -> head -n1 .session_dir) finds the report. + echo "$real_sdir" > "${session}/.session_dir" + elif [ "$elapsed" -ge "$startup_grace" ]; then + log "ERROR: leg $leg -- no session dir with state.json under $session after ${elapsed}s (optimize never launched?)" + return 1 + fi + fi + + if [ -n "$real_sdir" ]; then + if [ -f "$final_json" ]; then + log "leg $leg final.json present after ${elapsed}s; demo ran to completion" + return 0 + fi + local stop="" + [ -f "$state_json" ] && stop="$(jq -r '.stop_reason // ""' "$state_json" 2>/dev/null || echo "")" + if [ -n "$stop" ]; then + log "leg $leg state.json stop_reason='$stop'; waiting up to ${final_grace}s for final.json" + local g0; g0="$(date +%s)" + while [ ! -f "$final_json" ] && [ $(( $(date +%s) - g0 )) -lt "$final_grace" ]; do sleep 5; done + if [ -f "$final_json" ]; then + log "leg $leg final.json present (stop_reason='$stop'); demo complete" + return 0 + fi + log "ERROR: leg $leg state stop_reason='$stop' but final.json never appeared within ${final_grace}s" + return 1 + fi + fi + + if [ "$elapsed" -ge "$deadline_s" ]; then + log "ERROR: leg $leg deadline ${deadline_s}s reached without reports/final.json (real_sdir='${real_sdir:-}')" + return 1 + fi + sleep "$wait_interval" + done } # Install docker + start a pod-local dockerd. VERIFIED on a real privileged MI355X pod diff --git a/.github/scripts/pre-release-e2e-dispatch.sh b/.github/scripts/pre-release-e2e-dispatch.sh index c4ec3dff91..77ec34673c 100755 --- a/.github/scripts/pre-release-e2e-dispatch.sh +++ b/.github/scripts/pre-release-e2e-dispatch.sh @@ -44,7 +44,7 @@ # LEG_CPU / LEG_MEM baremetal leg resource request # (default 32 / 128Gi) # DEADLINE_3H_S / DEADLINE_12H_S pod hard-timeout per duration -# (default 14400 = 3+1h / 46800 = 12+1h). The docker host +# (default 16200 = 3h+1h+30m / 48600 = 12h+1h+30m). The docker host # pod uses the MAX over its legs. SaFE kills the pod at the # deadline; poll then judges that leg FAIL. Timing starts when # the workload is DISPATCHED, not when it is queued. @@ -71,12 +71,16 @@ LEG_CPU="${LEG_CPU:-32}"; LEG_MEM="${LEG_MEM:-128Gi}" PRIORITY="${PRIORITY:-2}" DISPATCH_MAP="${DISPATCH_MAP:-${RUNNER_TEMP:-/tmp}/pre_release_dispatch.json}" -# Pod hard-timeout (design: 3h leg -> 3+1h, 12h leg -> 12+1h). SaFE terminates the -# workload at the deadline; the poll then sees a non-Succeeded terminal / missing -# report and judges that leg FAIL. The deadline is counted from DISPATCH (not queue) -# time, so the +1h buffer absorbs bootstrap/setup/agent overhead. Given per duration: -DEADLINE_3H_S="${DEADLINE_3H_S:-14400}" # 3h + 1h buffer = 4h -DEADLINE_12H_S="${DEADLINE_12H_S:-46800}" # 12h + 1h buffer = 13h +# Pod hard-timeout. SaFE terminates the workload at the deadline; the poll then sees a +# non-Succeeded terminal / missing report and judges that leg FAIL. Counted from DISPATCH +# (not queue) time. This MUST exceed the bootstrap's own in-pod wait deadline +# (hours*3600+3600, i.e. 3h/12h demo + 1h agent/setup buffer) so SaFE never pre-empts the +# pod mid-wait -- which would lose bootstrap's clean `return 1` + logging and reintroduce +# the premature-teardown race. We add a further +30m pod margin on top of the bootstrap +# deadline. Ordering per leg: bootstrap deadline < SaFE pod timeout < poll GLOBAL_TIMEOUT_S +# (default 50400s=14h, still > 48600s). Given per duration: +DEADLINE_3H_S="${DEADLINE_3H_S:-16200}" # 3h demo + 1h bootstrap buffer + 30m pod margin = 4.5h +DEADLINE_12H_S="${DEADLINE_12H_S:-48600}" # 12h demo + 1h bootstrap buffer + 30m pod margin = 13.5h # The SaFE API field that carries the pod deadline. Confirmed against the Primus-SaFE # codebase: the create-workload body embeds WorkloadSpec inline, whose `timeout` # (integer seconds, top-level, from dispatch time) is enforced by WorkloadTTLController From c626bd89652204ff28cab15f1f17cbe9af6dcdaa Mon Sep 17 00:00:00 2001 From: lishuoshuo-amd Date: Thu, 27 Aug 2026 20:39:09 +0800 Subject: [PATCH 20/52] pre-release-e2e: reap stale SaFE workloads before dispatch (fix GPU leak) The concurrency.cancel-in-progress knob only cancels the GitHub JOB; it does NOT reliably stop the SaFE PyTorchJob pods a superseded/failed run already created. The `if: cancelled()` cleanup only gets a short grace window, and a job that FAILS (not cancels) after dispatch skipped cleanup entirely. Verified: 3 `Running` e2e workloads (incl. an 8-GPU docker host) leaked from a dead run and idle-held their cards, starving the next run. Correct order is the reverse of "cancel job -> hope the pod stops": a NEW run now STOPS every stale e2e-* workload (this workspace, non-terminal phase, not this run's own VERSION_TAG) up front via GET /workloads + POST /stop, frees the GPUs, then dispatches. The old run's poll then sees phase=Stopped and judges those legs FAIL -- the GitHub job ends as a CONSEQUENCE of stopping the pod, not the other way round. reap is resilient to an unreachable API (skips, never aborts under set -e). Also broaden the in-run cleanup step from `if: cancelled()` to `if: cancelled() || failure()` so a job that fails after dispatch stops its own workloads too (idempotent; poll's stop_workloads handles the success path; the next run's reap is the final backstop). Co-Authored-By: Claude Opus 4.8 --- .github/scripts/pre-release-e2e-dispatch.sh | 43 +++++++++++++++++++++ .github/workflows/pre-release-e2e-test.yml | 12 +++++- 2 files changed, 53 insertions(+), 2 deletions(-) diff --git a/.github/scripts/pre-release-e2e-dispatch.sh b/.github/scripts/pre-release-e2e-dispatch.sh index 77ec34673c..099b616296 100755 --- a/.github/scripts/pre-release-e2e-dispatch.sh +++ b/.github/scripts/pre-release-e2e-dispatch.sh @@ -121,6 +121,49 @@ fi summary() { echo "$*" | tee -a "${GITHUB_STEP_SUMMARY:-/dev/null}"; } +# ---- reclaim stale pre-release workloads BEFORE dispatching ----------------- +# The concurrency.cancel-in-progress GitHub knob only cancels the JOB; it does NOT +# reliably stop the SaFE PyTorchJob pods a superseded/failed run already created (the +# `if: cancelled()` cleanup gets a short grace window, and a job that FAILS -- not +# cancels -- after dispatch skips it entirely). Verified 2026-08-27: three `Running` +# e2e workloads (incl. an 8-GPU docker host) leaked from a dead run and idle-held their +# cards. So the correct, self-healing order is the reverse of "cancel job -> hope the +# pod stops": a NEW run STOPS every stale e2e-* workload up front, frees the GPUs, then +# dispatches its own. The old run's poll then sees phase=Stopped and judges those legs +# FAIL -- the GitHub job ends naturally as a consequence of stopping the pod, not the +# other way round. +# +# Scope: only workloads whose displayName starts `e2e-` (this CI's own), in THIS +# workspace, that are NOT already terminal, and NOT this run's own tag (VERSION_TAG, +# whose workloads don't exist yet anyway -- a belt-and-suspenders guard). +reap_stale_workloads() { + local resp + resp="$(curl -sS "${tls[@]}" --max-time 30 "$API" "${auth[@]}" 2>/dev/null || true)" + [ -n "$resp" ] || { echo "[reap] could not list workloads; skipping reclaim" >&2; return 0; } + # Terminal phases we must NOT re-stop; anything else (Running/Pending/Queued/ + # Creating/Unknown/...) is a live pod holding resources. + local stale + stale="$(printf '%s' "$resp" | jq -r --arg ws "$SAFE_WORKSPACE_ID" --arg tag "$VERSION_TAG" ' + (.items // .workloads // .)[]? + | select(((.displayName // .name // "") | startswith("e2e-"))) + | select((.workspaceId // $ws) == $ws) + | select(((.displayName // .name // "") | contains($tag)) | not) + | select((.phase // .status // "") as $p + | (["Stopped","Failed","Succeeded","Completed","Deleted"] | index($p)) | not) + | (.workloadId // .id)' 2>/dev/null || true)" + [ -n "$stale" ] || { summary "• no stale e2e workloads to reclaim"; return 0; } + local wid code n=0 + while IFS= read -r wid; do + [ -n "$wid" ] || continue + code="$(curl -sS "${tls[@]}" --max-time 20 -o /dev/null -w '%{http_code}' \ + -X POST "$API/$wid/stop" "${auth[@]}" 2>/dev/null || echo 000)" + summary "• reclaimed stale workload \`$wid\` (stop HTTP $code)" + n=$((n+1)) + done <<< "$stale" + summary "• reclaimed $n stale e2e workload(s) before dispatch" +} +reap_stale_workloads + # All 8 legs. Fields: mode backend hours model_path -- gpu index within the docker host ALL_LEGS="baremetal-vllm-3h baremetal-vllm-12h baremetal-sglang-3h baremetal-sglang-12h \ docker-vllm-3h docker-vllm-12h docker-sglang-3h docker-sglang-12h" diff --git a/.github/workflows/pre-release-e2e-test.yml b/.github/workflows/pre-release-e2e-test.yml index cc6f559e82..75ad2411fa 100644 --- a/.github/workflows/pre-release-e2e-test.yml +++ b/.github/workflows/pre-release-e2e-test.yml @@ -302,8 +302,16 @@ jobs: chmod +x .github/scripts/pre-release-e2e-poll.sh .github/scripts/pre-release-e2e-poll.sh - - name: Stop workloads on job cancel - if: cancelled() + - name: Stop workloads on abnormal job end + # cancel OR failure OR a crash mid-dispatch/poll all leak pods: the poll step + # stops workloads only when IT runs to completion, so a job that is cancelled + # (superseded push) or fails (dispatch/poll error, runner death) after dispatch + # would idle-hold its GPUs. This idempotent cleanup fires on every abnormal end + # and stops whatever landed in DISPATCH_MAP (stopping an already-terminal + # workload is a harmless no-op). The next run's pre-dispatch reap is the + # backstop for anything even this misses (e.g. the runner dying before this + # step). Success path is handled by the poll script's own stop_workloads(). + if: cancelled() || failure() env: SAFE_API_BASE: ${{ vars.PRE_E2E_SAFE_API_BASE }} SAFE_API_KEY: ${{ secrets.PRE_E2E_SAFE_API_KEY }} From a335130e56066f1739bd39e875551e460ec40fcb Mon Sep 17 00:00:00 2001 From: lishuoshuo-amd Date: Thu, 27 Aug 2026 20:44:09 +0800 Subject: [PATCH 21/52] TEMP TEST: bump version 1.0.0 -> 1.0.1a0 to force full 8-leg pre-release-e2e run Temporary, for testing the pre-release E2E gate end-to-end on all 8 legs. The PR's resolve job classifies a version bump vs main as a FULL run (all 8 legs) instead of the scripts-only 4-leg scope. 1.0.1a0 is a legal PEP 440 alpha so the CI_VERSION (1.0.1a0.dev+ci) and wheel build stay valid. REVERT to 1.0.0 before merge. Co-Authored-By: Claude Opus 4.8 --- pyproject.toml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pyproject.toml b/pyproject.toml index 09a582e357..c9465a4ecc 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -6,7 +6,7 @@ build-backend = "setuptools.build_meta" [project] name = "hyperloom-inference_optimizer" -version = "1.0.0" +version = "1.0.1a0" # TEMP TEST: alpha bump vs main (1.0.0) to force pre-release-e2e FULL run (all 8 legs); REVERT before merge description = "Inference Optimizer — three-role (Orchestration/Critic/Robustness) autonomous LLM inference optimization runtime for AMD GPU platforms. Kernel optimization is handled by programmatic Python handlers." readme = "README.md" requires-python = ">=3.10" From 697c1f93d545aadada111717e84f201068b8c2ea Mon Sep 17 00:00:00 2001 From: lishuoshuo-amd Date: Thu, 27 Aug 2026 21:05:34 +0800 Subject: [PATCH 22/52] fix(pre-release-e2e): mount model dir into nested docker legs Docker legs hung waiting for a final.json that never came: the nested container mounted only $ROOT and $NFS_ROOT (the CI subdir), never the model dir (/shared_nfs/models). So HYPERLOOM_MODEL_PATH resolved to a path absent inside the container and optimize could not boot the server. The demo skill already requires this (examples/*/SKILL.md): "If ... a pre-downloaded model directory is outside the workspace, add matching -v host_path:host_path mounts." docker-run-hyperloom.sh was missing it. Mount the model's PARENT dir (minimal exposure); skip the extra -v when the model already lives under $ROOT/$NFS_ROOT to avoid a duplicate mount. Co-Authored-By: Claude Opus 4.8 --- .github/pre-release/docker-run-hyperloom.sh | 16 ++++++++++++++++ 1 file changed, 16 insertions(+) diff --git a/.github/pre-release/docker-run-hyperloom.sh b/.github/pre-release/docker-run-hyperloom.sh index 8d6763fad7..10b17161f9 100755 --- a/.github/pre-release/docker-run-hyperloom.sh +++ b/.github/pre-release/docker-run-hyperloom.sh @@ -56,6 +56,21 @@ mkdir -p "$ROOT" BOOTSTRAP="${NFS_ROOT%/}/bootstrap/${CI_VERSION}/bootstrap-pre-release.sh" NAME="hyperloom-${LEG_ID}" +# The model dir (e.g. /shared_nfs/models/) lives OUTSIDE $ROOT/$NFS_ROOT, so the +# demo skill's rule applies: "If ... a pre-downloaded model directory is outside the +# workspace, add matching -v host_path:host_path mounts" (examples/*/SKILL.md). Without +# this the container's HYPERLOOM_MODEL_PATH resolves to a path that does not exist and +# optimize can never boot the server -> the leg hangs waiting for a final.json that +# never comes. Mount the model's PARENT dir (minimal exposure). Skip the extra -v when +# the model already lives under a dir we mount ($ROOT or $NFS_ROOT) to avoid a duplicate +# -v that docker rejects. +MODEL_DIR="$(dirname -- "$MODEL_PATH")" +MODEL_MOUNT=() +case "$MODEL_DIR/" in + "${ROOT%/}/"*|"${NFS_ROOT%/}/"*) : ;; # already covered by an existing mount + *) MODEL_MOUNT=(-v "$MODEL_DIR:$MODEL_DIR") ;; +esac + # GPU index -> renderD node. VERIFIED on a real privileged MI355X x8 pod (2026-08-27): # the 8 physical GPUs map to renderD128,136,144,...,184 -- i.e. stride 8, NOT +1. The # rocm-smi GPU order matches this render-node order (GPU i == 0002/0003:00:0X.0 == @@ -100,5 +115,6 @@ exec docker run --rm --name "$NAME" \ ${ANTHROPIC_BASE_URL:+-e ANTHROPIC_BASE_URL="$ANTHROPIC_BASE_URL"} \ -v "$ROOT:$ROOT" \ -v "$NFS_ROOT:$NFS_ROOT" \ + "${MODEL_MOUNT[@]}" \ --entrypoint bash \ "$IMAGE" "$BOOTSTRAP" From b8e63dafaf5a102530320461b3f2b8bc4bb12ec9 Mon Sep 17 00:00:00 2001 From: lishuoshuo-amd Date: Fri, 28 Aug 2026 10:41:04 +0800 Subject: [PATCH 23/52] pre-release-e2e: docker legs follow the demo skill (agent starts its own single-GPU container) The docker legs hung because setup-docker-*.md told the agent "you are already inside the container started by the harness (docker-run-hyperloom.sh); do NOT run docker", stripping the demo skill's docker responsibility away from the agent. docker-run-hyperloom.sh mounted only the model's parent dir, but the model path is a symlink into a DIFFERENT /shared_nfs subtree, so the symlink target was unmounted -> broken symlink in the container -> optimize could not boot -> the leg hung on the wait-for-final.json loop. Refactor so each docker leg = one agent that starts its OWN single-GPU container by following the demo skill (the skill is strictly single-container/single-GPU/single optimize, so one-agent-per-leg is the only skill-aligned shape): - run_docker_host: instead of fanning out to docker-run-hyperloom.sh nested-bootstrap containers, run each leg as a backgrounded `run_leg ... docker` ON THE HOST POD. Session artifacts land under $session on NFS exactly as for baremetal, so the wait-for-final.json loop works unchanged. Each run_leg runs in its own subshell so the per-leg .env-scrub EXIT traps don't clobber each other. - run_leg (docker mode): compute the per-card isolation on the host (renderD = 128+GPU_INDEX*8, numeric KFD/DRI GIDs via stat, per-backend image, whole-/shared_nfs mount root) and inject them into the pod-local .env as HYPERLOOM_IMAGE/ HYPERLOOM_CONTAINER_NAME/HYPERLOOM_SHM_SIZE + E2E_* values. The demo skill's literal `docker run` cannot express single-renderD isolation / numeric GIDs / caps / seccomp, so the harness supplies them and the setup prompt tells the agent to apply them. - setup-docker-{vllm,sglang}.md: rewritten to hand container ownership back to the agent and enumerate the CI hard constraints (single renderD, numeric group-add, cpu/mem/shm caps, seccomp=unconfined, HIP/ROCR_VISIBLE_DEVICES=0, full /shared_nfs mount to resolve the model symlink, secrets never on NFS). - demo-{3h,12h}.md: run optimize inside the container started in setup via docker exec. - delete docker-run-hyperloom.sh and its workflow stage + dispatch comment. Also reconciles MODEL_PATH: the old nested runner passed HYPERLOOM_MODEL_PATH (wrong var); the .env already carries MODEL_PATH, which the demo skill reads. The .env key scrub (EXIT trap) and the wait-for-final.json loop are preserved verbatim; baremetal legs are untouched. Co-Authored-By: Claude Opus 4.8 --- .github/pre-release/bootstrap-pre-release.sh | 85 +++++++++++-- .github/pre-release/docker-run-hyperloom.sh | 120 ------------------ .../prompts/pre-release/demo-12h.md | 7 +- .../prompts/pre-release/demo-3h.md | 7 +- .../pre-release/setup-docker-sglang.md | 73 +++++++---- .../prompts/pre-release/setup-docker-vllm.md | 73 +++++++---- .github/scripts/pre-release-e2e-dispatch.sh | 5 +- .github/workflows/pre-release-e2e-test.yml | 2 +- 8 files changed, 187 insertions(+), 185 deletions(-) delete mode 100755 .github/pre-release/docker-run-hyperloom.sh diff --git a/.github/pre-release/bootstrap-pre-release.sh b/.github/pre-release/bootstrap-pre-release.sh index b0540186ec..58d0f5731f 100755 --- a/.github/pre-release/bootstrap-pre-release.sh +++ b/.github/pre-release/bootstrap-pre-release.sh @@ -9,9 +9,12 @@ # See hyperloom-pre-release-e2e-ci-design.md §12. # # Two modes, selected by E2E_DOCKER_HOST: -# * unset -> a single baremetal/docker leg in THIS pod (LEG_ID given). -# * "1" -> the privileged 8-GPU host: fan out DOCKER_LEGS to nested containers -# via docker-run-hyperloom.sh , one GPU each. +# * unset -> a single baremetal leg in THIS pod (LEG_ID given). +# * "1" -> the privileged 8-GPU host: run each of DOCKER_LEGS as a backgrounded +# run_leg (docker mode) ON THE HOST POD. Each leg's agent follows the +# demo skill to `docker run` its OWN single-GPU container (renderD = +# 128 + gpu_index*8), so the skill owns the container lifecycle. The +# per-leg GPU-isolation values are computed here and injected via .env. # # Inputs (env, injected by the dispatch script): # CI_VERSION NFS_ROOT @@ -25,7 +28,6 @@ set -euo pipefail : "${CLAUDE_MODEL:?}"; : "${CLAUDE_CLI_VERSION:?}" TARGET_GAIN="${TARGET_GAIN:-100}" -SELF_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" PROMPTS_DIR="${NFS_ROOT%/}/bootstrap/${CI_VERSION}/prompts/pre-release" WHEEL_DIR="${NFS_ROOT%/}/wheels/${CI_VERSION}" @@ -85,6 +87,34 @@ run_leg() { # 2. decode the key and write the pod-local .env (NEVER on stdout / NEVER to a # location the poll reads). Restrict perms; scrub on exit. + # + # For a docker leg the agent (not the harness) starts the single-GPU container per the + # demo skill. The demo skill's literal `docker run` cannot express our per-card + # isolation (single renderD node, NUMERIC device GIDs, cpu/mem caps, seccomp), so we + # compute those HERE -- on the host pod, where /dev/kfd + /dev/dri/renderD* exist -- and + # inject them as E2E_* .env values the setup prompt tells the agent to copy verbatim. + # renderD = 128 + GPU_INDEX*8 (stride 8, VERIFIED on a real privileged MI355X x8 pod). + # The pod /etc/group has no `video`/`render` NAMES, so numeric GIDs are required. + local dk_image="" dk_rd="" dk_kfd_gid="" dk_dri_gid="" dk_nfs_mount="" + if [ "$run_mode" = docker ]; then + : "${GPU_INDEX:?docker leg needs GPU_INDEX}" + dk_rd=$(( 128 + GPU_INDEX * 8 )) + dk_kfd_gid="$(stat -c %g /dev/kfd 2>/dev/null || echo 0)" + dk_dri_gid="$(stat -c %g "/dev/dri/renderD${dk_rd}" 2>/dev/null || stat -c %g /dev/dri 2>/dev/null || echo 0)" + if [ "$backend" = vllm ]; then + dk_image="${HYPERLOOM_IMAGE_VLLM:-vllm/vllm-openai-rocm:v0.27.1}" + else + dk_image="${HYPERLOOM_IMAGE_SGLANG:-lmsysorg/sglang-rocm:v0.5.17-rocm724-mi35x-srt}" + fi + # The model path (e.g. /shared_nfs/models/) is a SYMLINK into a DIFFERENT + # /shared_nfs subtree (/shared_nfs/huggingface_models/...). Mounting only NFS_ROOT + # (the CI subdir) or the model's parent leaves the symlink TARGET unmounted -> broken + # symlink in the container -> optimize can't boot. Mount the whole shared-NFS root + # (the top-level dir, e.g. /shared_nfs) so both the model and its symlink target + # resolve at the same absolute path inside the container. Derive it as the first path + # component of NFS_ROOT (override with E2E_SHARED_NFS_ROOT if the layout differs). + dk_nfs_mount="${E2E_SHARED_NFS_ROOT:-/$(printf '%s' "${NFS_ROOT#/}" | cut -d/ -f1)}" + fi local envf="${root}/.env" ( umask 077 { @@ -115,6 +145,20 @@ run_leg() { echo "MODEL_PATH=${model_path}" echo "TARGET_GAIN=${TARGET_GAIN}" echo "DEMO_HOURS=${hours}" + # docker leg: hand the container lifecycle to the agent (demo skill) and carry the + # CI hard constraints it must apply to its `docker run` (see setup-docker-*.md). + if [ "$run_mode" = docker ]; then + echo "HYPERLOOM_IMAGE=${dk_image}" + echo "HYPERLOOM_CONTAINER_NAME=hyperloom-${leg}" # unique per leg (shared host dockerd) + echo "HYPERLOOM_SHM_SIZE=${LEG_SHM:-64g}" + echo "E2E_GPU_INDEX=${GPU_INDEX}" + echo "E2E_RENDERD=${dk_rd}" + echo "E2E_KFD_GID=${dk_kfd_gid}" + echo "E2E_DRI_GID=${dk_dri_gid}" + echo "E2E_LEG_CPUS=${LEG_CPUS:-32}" + echo "E2E_LEG_MEM=${LEG_MEM:-128g}" + echo "E2E_NFS_MOUNT=${dk_nfs_mount}" + fi } > "$envf" ) trap 'sed -i "/^ANTHROPIC_API_KEY=/d" "'"$envf"'" 2>/dev/null || true' EXIT @@ -247,23 +291,36 @@ ensure_dockerd() { return 1 } -# ---- docker host: fan out to nested containers ----------------------------- +# ---- docker host: run each leg (docker mode) ON THIS host pod ---------------- +# The privileged 8-GPU host runs a dockerd, then drives each docker leg as a backgrounded +# run_leg in docker mode. Each leg's agent follows the demo skill to `docker run` its OWN +# single-GPU container (renderD = 128 + gpu_index*8), applying the CI isolation flags that +# run_leg injected into the leg .env. No nested bootstrap: session artifacts land under +# $session on this pod's NFS exactly as for baremetal, so the wait-for-final.json loop in +# run_leg works unchanged. Each `run_leg &` is its own subshell, so their per-leg EXIT +# traps (the .env key scrub) don't clobber each other. run_docker_host() { : "${DOCKER_LEGS:?}"; : "${DOCKER_GPU_MAP:?}"; : "${MODEL_3H:?}"; : "${MODEL_12H:?}" - local runner="${SELF_DIR}/docker-run-hyperloom.sh" - [ -x "$runner" ] || chmod +x "$runner" 2>/dev/null || true ensure_dockerd || { log "ERROR: cannot provide docker on the host pod"; return 1; } log "docker host: legs='${DOCKER_LEGS}'" - local pids=() + local pids=() leg idx backend hours model_path for leg in $DOCKER_LEGS; do - local idx; idx="$(printf '%s' "$DOCKER_GPU_MAP" | jq -r --arg l "$leg" '.[$l]')" - log "launch nested container: gpu=$idx leg=$leg" - # Each nested container binds one card and runs THIS bootstrap inside, in - # single-leg mode. docker-run-hyperloom.sh enforces GPU + cpu/mem quota (§8). - "$runner" "$idx" "$leg" & + idx="$(printf '%s' "$DOCKER_GPU_MAP" | jq -r --arg l "$leg" '.[$l]')" + case "$leg" in + *-vllm-*) backend=vllm ;; + *-sglang-*) backend=sglang ;; + *) log "ERROR: cannot infer backend from leg '$leg'"; return 1 ;; + esac + case "$leg" in + *-3h) hours=3; model_path="$MODEL_3H" ;; + *-12h) hours=12; model_path="$MODEL_12H" ;; + *) log "ERROR: cannot infer duration from leg '$leg'"; return 1 ;; + esac + log "launch docker leg: gpu=$idx renderD$(( 128 + idx * 8 )) leg=$leg backend=$backend hours=$hours" + ( GPU_INDEX="$idx" run_leg "$leg" "$backend" "$model_path" "$hours" docker ) & pids+=("$!") done - local rc=0 + local rc=0 p for p in "${pids[@]}"; do wait "$p" || rc=1; done return "$rc" } diff --git a/.github/pre-release/docker-run-hyperloom.sh b/.github/pre-release/docker-run-hyperloom.sh deleted file mode 100755 index 10b17161f9..0000000000 --- a/.github/pre-release/docker-run-hyperloom.sh +++ /dev/null @@ -1,120 +0,0 @@ -#!/usr/bin/env bash -# SPDX-FileCopyrightText: 2026 Advanced Micro Devices, Inc. -# SPDX-License-Identifier: MIT -# -# Nested docker runner for the privileged 8-GPU pre-release host. This is the ONLY -# docker entry point for docker legs: it force-binds ONE GPU and caps CPU/memory to a -# 1/4 host share so the 4 docker legs are mutually comparable and comparable to the -# baremetal legs (design §8, point E). Prompts are forbidden from running docker -# directly or choosing GPUs via rocm-smi. -# -# Usage: docker-run-hyperloom.sh -# -# Inputs (env, inherited from the host bootstrap): -# NFS_ROOT CI_VERSION MODEL_3H MODEL_12H TARGET_GAIN -# CLAUDE_MODEL CLAUDE_CLI_VERSION ANTHROPIC_API_KEY_B64 [ANTHROPIC_BASE_URL] -# HYPERLOOM_IMAGE backend container image (overrides per-backend default) -# LEG_CPUS LEG_MEM LEG_SHM per-container quota (default 32 / 128g / 64g) -set -euo pipefail - -GPU_INDEX="${1:?usage: docker-run-hyperloom.sh }" -LEG_ID="${2:?usage: docker-run-hyperloom.sh }" - -: "${NFS_ROOT:?}"; : "${CI_VERSION:?}"; : "${ANTHROPIC_API_KEY_B64:?}" -: "${CLAUDE_MODEL:?}"; : "${CLAUDE_CLI_VERSION:?}" -TARGET_GAIN="${TARGET_GAIN:-100}" - -# 1/4 of a 128-core / 512Gi host, matching the baremetal leg envelope. -LEG_CPUS="${LEG_CPUS:-32}" -LEG_MEM="${LEG_MEM:-128g}" -LEG_SHM="${LEG_SHM:-64g}" - -# Resolve per-leg backend / model / hours from the leg id. -case "$LEG_ID" in - *-vllm-*) BACKEND=vllm ;; - *-sglang-*) BACKEND=sglang ;; - *) echo "cannot infer backend from leg '$LEG_ID'" >&2; exit 2 ;; -esac -case "$LEG_ID" in - *-3h) HOURS=3; MODEL_PATH="${MODEL_3H:?MODEL_3H required}" ;; - *-12h) HOURS=12; MODEL_PATH="${MODEL_12H:?MODEL_12H required}" ;; - *) echo "cannot infer duration from leg '$LEG_ID'" >&2; exit 2 ;; -esac - -# Default backend images (overridable via HYPERLOOM_IMAGE); mirrors the demo skill's -# suggested ROCm images. -if [ -n "${HYPERLOOM_IMAGE:-}" ]; then - IMAGE="$HYPERLOOM_IMAGE" -elif [ "$BACKEND" = vllm ]; then - IMAGE="${HYPERLOOM_IMAGE_VLLM:-vllm/vllm-openai-rocm:v0.27.1}" -else - IMAGE="${HYPERLOOM_IMAGE_SGLANG:-lmsysorg/sglang-rocm:v0.5.17-rocm724-mi35x-srt}" -fi - -ROOT="${NFS_ROOT%/}/runs/${CI_VERSION}/${LEG_ID}" -mkdir -p "$ROOT" -BOOTSTRAP="${NFS_ROOT%/}/bootstrap/${CI_VERSION}/bootstrap-pre-release.sh" -NAME="hyperloom-${LEG_ID}" - -# The model dir (e.g. /shared_nfs/models/) lives OUTSIDE $ROOT/$NFS_ROOT, so the -# demo skill's rule applies: "If ... a pre-downloaded model directory is outside the -# workspace, add matching -v host_path:host_path mounts" (examples/*/SKILL.md). Without -# this the container's HYPERLOOM_MODEL_PATH resolves to a path that does not exist and -# optimize can never boot the server -> the leg hangs waiting for a final.json that -# never comes. Mount the model's PARENT dir (minimal exposure). Skip the extra -v when -# the model already lives under a dir we mount ($ROOT or $NFS_ROOT) to avoid a duplicate -# -v that docker rejects. -MODEL_DIR="$(dirname -- "$MODEL_PATH")" -MODEL_MOUNT=() -case "$MODEL_DIR/" in - "${ROOT%/}/"*|"${NFS_ROOT%/}/"*) : ;; # already covered by an existing mount - *) MODEL_MOUNT=(-v "$MODEL_DIR:$MODEL_DIR") ;; -esac - -# GPU index -> renderD node. VERIFIED on a real privileged MI355X x8 pod (2026-08-27): -# the 8 physical GPUs map to renderD128,136,144,...,184 -- i.e. stride 8, NOT +1. The -# rocm-smi GPU order matches this render-node order (GPU i == 0002/0003:00:0X.0 == -# renderD(128+8*i)). See project memory. `cardN` numbering is NOT guaranteed to align -# with the GPU order, so we isolate via /dev/kfd + the single renderD node only. -RD=$((128 + GPU_INDEX * 8)) - -# Device group ownership: the pod's /etc/group has NO `video`/`render` NAMES, so -# `--group-add video` FAILS ("no matching entries in group file"). Resolve the numeric -# GIDs of the device nodes and pass those instead (verified working). -KFD_GID="$(stat -c %g /dev/kfd 2>/dev/null || echo 0)" -DRI_GID="$(stat -c %g /dev/dri/renderD${RD} 2>/dev/null || stat -c %g /dev/dri 2>/dev/null || echo 0)" - -echo "[docker-run] leg=$LEG_ID gpu=$GPU_INDEX renderD$RD (kfd_gid=$KFD_GID dri_gid=$DRI_GID) image=$IMAGE cpus=$LEG_CPUS mem=$LEG_MEM" - -docker rm -f "$NAME" >/dev/null 2>&1 || true - -# GPU isolation: expose /dev/kfd (shared) + exactly ONE renderD node, so the container -# sees a single device. HIP_VISIBLE_DEVICES=0 pins the app to that one card. CPU/mem -# hard-capped to the 1/4 share. seccomp=unconfined matches how the ROCm images expect -# to run (verified: rocm-smi enumerates the single bound card correctly). -exec docker run --rm --name "$NAME" \ - --device "/dev/kfd" \ - --device "/dev/dri/renderD${RD}" \ - --group-add "$KFD_GID" \ - --group-add "$DRI_GID" \ - --security-opt seccomp=unconfined \ - --cpus "$LEG_CPUS" --memory "$LEG_MEM" --shm-size "$LEG_SHM" \ - -e HIP_VISIBLE_DEVICES=0 \ - -e ROCR_VISIBLE_DEVICES=0 \ - -e CI_VERSION="$CI_VERSION" \ - -e NFS_ROOT="$NFS_ROOT" \ - -e LEG_ID="$LEG_ID" \ - -e HYPERLOOM_RUN_MODE=docker \ - -e HYPERLOOM_BACKEND="$BACKEND" \ - -e HYPERLOOM_MODEL_PATH="$MODEL_PATH" \ - -e DEMO_HOURS="$HOURS" \ - -e TARGET_GAIN="$TARGET_GAIN" \ - -e CLAUDE_MODEL="$CLAUDE_MODEL" \ - -e CLAUDE_CLI_VERSION="$CLAUDE_CLI_VERSION" \ - -e ANTHROPIC_API_KEY_B64="$ANTHROPIC_API_KEY_B64" \ - ${ANTHROPIC_BASE_URL:+-e ANTHROPIC_BASE_URL="$ANTHROPIC_BASE_URL"} \ - -v "$ROOT:$ROOT" \ - -v "$NFS_ROOT:$NFS_ROOT" \ - "${MODEL_MOUNT[@]}" \ - --entrypoint bash \ - "$IMAGE" "$BOOTSTRAP" diff --git a/.github/pre-release/prompts/pre-release/demo-12h.md b/.github/pre-release/prompts/pre-release/demo-12h.md index 906038a2f2..13739f4d27 100644 --- a/.github/pre-release/prompts/pre-release/demo-12h.md +++ b/.github/pre-release/prompts/pre-release/demo-12h.md @@ -28,8 +28,11 @@ continue without asking. Load LLM API keys/base URLs and `FRAMEWORK` from `.env` ## Hard constraints (automated release gate) - Do **not** modify any GPU-related environment variable or device visibility. -- Do **not** run `docker`, start/exec containers, or choose GPUs via `rocm-smi`. If - `HYPERLOOM_RUN_MODE=docker` you are already inside the correct single-GPU container. +- Do **not** choose GPUs via `rocm-smi`. +- If `HYPERLOOM_RUN_MODE=docker`, run `optimize` **inside the container you started in + setup** via `docker exec -w "$REPO_ROOT" "$HYPERLOOM_CONTAINER_NAME" …` (per the demo + skill's docker mode). Do **not** start a new container and do **not** change its + device/isolation flags. Otherwise (baremetal) run directly and do not run `docker`. - Do **not** modify `USER_DATA_PATH`. - Do **not** print or copy secret values into output, reports, or logs. diff --git a/.github/pre-release/prompts/pre-release/demo-3h.md b/.github/pre-release/prompts/pre-release/demo-3h.md index c0c0c56c86..d6d4adb75c 100644 --- a/.github/pre-release/prompts/pre-release/demo-3h.md +++ b/.github/pre-release/prompts/pre-release/demo-3h.md @@ -29,8 +29,11 @@ asking. Load LLM API keys/base URLs and `FRAMEWORK` from `.env`. ## Hard constraints (automated release gate) - Do **not** modify any GPU-related environment variable or device visibility. -- Do **not** run `docker`, start/exec containers, or choose GPUs via `rocm-smi`. If - `HYPERLOOM_RUN_MODE=docker` you are already inside the correct single-GPU container. +- Do **not** choose GPUs via `rocm-smi`. +- If `HYPERLOOM_RUN_MODE=docker`, run `optimize` **inside the container you started in + setup** via `docker exec -w "$REPO_ROOT" "$HYPERLOOM_CONTAINER_NAME" …` (per the demo + skill's docker mode). Do **not** start a new container and do **not** change its + device/isolation flags. Otherwise (baremetal) run directly and do not run `docker`. - Do **not** modify `USER_DATA_PATH`. - Do **not** print or copy secret values into output, reports, or logs. diff --git a/.github/pre-release/prompts/pre-release/setup-docker-sglang.md b/.github/pre-release/prompts/pre-release/setup-docker-sglang.md index 2d4791939b..c8fcee6fbe 100644 --- a/.github/pre-release/prompts/pre-release/setup-docker-sglang.md +++ b/.github/pre-release/prompts/pre-release/setup-docker-sglang.md @@ -3,44 +3,73 @@ You are running the Hyperloom pre-release E2E test non-interactively. Complete the setup step for a **docker + SGLang** leg, then stop. Do not run the demo yet. -> **IMPORTANT:** you are **already** running inside the backend container. The nested -> container was started for you by the test harness (`docker-run-hyperloom.sh`) and is -> bound to exactly one GPU. You must **not** start, run, or exec any further container, -> and you must **not** run `docker` at all. Treat this environment as the place where -> setup and the demo run directly. +> **IMPORTANT — you own the container.** You are on the privileged host pod. This is a +> `docker` leg, so **you** must start the backend container yourself by following the +> `hyperloom-setup` skill and the demo skill's **docker mode**: run `docker run` to start +> a long-lived single-GPU container, then run setup **inside** it with `docker exec`. +> Docker is already available on this host (a pod-local `dockerd` is running). ## Environment (already prepared) A `.env` file exists in the current workspace (`REPO_ROOT`) with these values already set: `ANTHROPIC_API_KEY`, `CLAUDE_MODEL`, `USER_DATA_PATH`, `HYPERLOOM_RUN_MODE=docker`, `FRAMEWORK=sglang`, `MODEL_PATH`, `TARGET_GAIN`, -`DEMO_HOURS`. The wheel is already installed via `pip install --target .` so a -`hyperloom/` package directory is present. +`DEMO_HOURS`, and the container/isolation values the demo skill reads: +`HYPERLOOM_IMAGE`, `HYPERLOOM_CONTAINER_NAME`, `HYPERLOOM_SHM_SIZE`, plus the CI +isolation values `E2E_RENDERD`, `E2E_KFD_GID`, `E2E_DRI_GID`, `E2E_LEG_CPUS`, +`E2E_LEG_MEM`, `E2E_NFS_MOUNT`. The wheel is already installed via +`pip install --target .` so a `hyperloom/` package directory is present. ## Fixed decisions -Run the `hyperloom-setup` skill with these fixed decisions — do **not** ask -interactive questions; use the values already in `.env` and the environment: +Follow the `hyperloom-setup` skill in **docker** mode with these fixed decisions — do +**not** ask interactive questions; use the values already in `.env`: -- **Run mode:** docker, but the container already exists and IS the current shell. Do - **not** create a container and do **not** set `HYPERLOOM_DOCKER_TARGET_HOST`. Run - setup directly in this shell. -- **Framework:** SGLang. Ensure the SGLang framework layer is available in this - container (install with the setup backend if needed). +- **Run mode:** docker. Start the container yourself (see the hard constraints below for + the exact `docker run` flags), then `docker exec` the setup inside it. Do **not** set + `HYPERLOOM_DOCKER_TARGET_HOST` (run on the current host). +- **Framework:** SGLang — provided by the container image; run setup with + `--install-framework none --yes` inside the container (do **not** `--install-framework + sglang`). - **LLM provider / model / `USER_DATA_PATH`:** use the values already in `.env`; do not change them. ## Hard constraints (automated release gate) -- Do **not** modify any GPU-related environment variable (`ROCR_VISIBLE_DEVICES` is - already `0` and pins this container to its single card; leave it). Do not override - device visibility. -- Do **not** run `docker`, do **not** start/exec containers, and do **not** choose - GPUs via `rocm-smi`. -- Do **not** print, echo, or copy secret values (API keys) into output or logs. +Your `docker run` **MUST** use exactly the flags below. These **replace** the demo +skill's default `--device /dev/dri` (all GPUs) and `--group-add video` (a group *name* +the pod has no entry for) with single-card isolation and numeric GIDs. Every other +aspect of the skill's docker flow (long-lived `--entrypoint tail … -f /dev/null`, +mounting `$REPO_ROOT:$REPO_ROOT`, `docker exec` setup, running optimize inside) is +unchanged. Read the values from `.env`: + +- **Container name:** `--name "$HYPERLOOM_CONTAINER_NAME"` (already unique per leg; + another leg may share this host's dockerd, so do not rename it to a fixed value). +- **Image:** `"$HYPERLOOM_IMAGE"`. +- **Single-GPU isolation (REPLACES `--device /dev/dri`):** + `--device /dev/kfd --device /dev/dri/renderD${E2E_RENDERD}` +- **Numeric group-add (REPLACES `--group-add video`):** + `--group-add ${E2E_KFD_GID} --group-add ${E2E_DRI_GID}` +- **Resource caps:** `--cpus ${E2E_LEG_CPUS} --memory ${E2E_LEG_MEM} --shm-size ${HYPERLOOM_SHM_SIZE}` +- **Security:** `--security-opt seccomp=unconfined` +- **Device pin:** `-e HIP_VISIBLE_DEVICES=0 -e ROCR_VISIBLE_DEVICES=0` +- **Mounts:** `-v "$REPO_ROOT:$REPO_ROOT" -v "${E2E_NFS_MOUNT}:${E2E_NFS_MOUNT}"` — + mounting **all** of `${E2E_NFS_MOUNT}` at the same absolute path is **required** so + that `MODEL_PATH` (a symlink into another subtree under `${E2E_NFS_MOUNT}`) resolves + inside the container. Do **not** mount only the model's parent directory. + +Also: + +- Do **not** add any other `--device`, do **not** use `--group-add video`, and do **not** + choose GPUs via `rocm-smi`. The single bound `renderD` node + `HIP/ROCR_VISIBLE_DEVICES=0` + are what pin this leg to its one card. +- Do **not** print, echo, or copy secret values (API keys) into output or logs. The key + lives only in the pod-local `.env` (it reaches the container via `-v $REPO_ROOT:$REPO_ROOT`); + do **not** write it anywhere else, and never onto NFS outside that `.env`. - Do **not** modify `USER_DATA_PATH`. ## Termination -When setup completes successfully, stop. Report only `setup complete: docker/sglang`. -If setup hard-fails, report the failure and stop. +When setup completes successfully, stop and report only `setup complete: docker/sglang`. +Leave the container **running** so the demo turn can `docker exec` into it. If setup +hard-fails, report the failure and stop. diff --git a/.github/pre-release/prompts/pre-release/setup-docker-vllm.md b/.github/pre-release/prompts/pre-release/setup-docker-vllm.md index 970d57fe59..ab065e70a7 100644 --- a/.github/pre-release/prompts/pre-release/setup-docker-vllm.md +++ b/.github/pre-release/prompts/pre-release/setup-docker-vllm.md @@ -3,44 +3,73 @@ You are running the Hyperloom pre-release E2E test non-interactively. Complete the setup step for a **docker + vLLM** leg, then stop. Do not run the demo yet. -> **IMPORTANT:** you are **already** running inside the backend container. The nested -> container was started for you by the test harness (`docker-run-hyperloom.sh`) and is -> bound to exactly one GPU. You must **not** start, run, or exec any further container, -> and you must **not** run `docker` at all. Treat this environment as the place where -> setup and the demo run directly. +> **IMPORTANT — you own the container.** You are on the privileged host pod. This is a +> `docker` leg, so **you** must start the backend container yourself by following the +> `hyperloom-setup` skill and the demo skill's **docker mode**: run `docker run` to start +> a long-lived single-GPU container, then run setup **inside** it with `docker exec`. +> Docker is already available on this host (a pod-local `dockerd` is running). ## Environment (already prepared) A `.env` file exists in the current workspace (`REPO_ROOT`) with these values already set: `ANTHROPIC_API_KEY`, `CLAUDE_MODEL`, `USER_DATA_PATH`, `HYPERLOOM_RUN_MODE=docker`, `FRAMEWORK=vllm`, `MODEL_PATH`, `TARGET_GAIN`, -`DEMO_HOURS`. The wheel is already installed via `pip install --target .` so a -`hyperloom/` package directory is present. +`DEMO_HOURS`, and the container/isolation values the demo skill reads: +`HYPERLOOM_IMAGE`, `HYPERLOOM_CONTAINER_NAME`, `HYPERLOOM_SHM_SIZE`, plus the CI +isolation values `E2E_RENDERD`, `E2E_KFD_GID`, `E2E_DRI_GID`, `E2E_LEG_CPUS`, +`E2E_LEG_MEM`, `E2E_NFS_MOUNT`. The wheel is already installed via +`pip install --target .` so a `hyperloom/` package directory is present. ## Fixed decisions -Run the `hyperloom-setup` skill with these fixed decisions — do **not** ask -interactive questions; use the values already in `.env` and the environment: +Follow the `hyperloom-setup` skill in **docker** mode with these fixed decisions — do +**not** ask interactive questions; use the values already in `.env`: -- **Run mode:** docker, but the container already exists and IS the current shell. Do - **not** create a container and do **not** set `HYPERLOOM_DOCKER_TARGET_HOST`. Run - setup directly in this shell. -- **Framework:** vLLM. Ensure the vLLM framework layer is available in this container - (install with the setup backend if needed). +- **Run mode:** docker. Start the container yourself (see the hard constraints below for + the exact `docker run` flags), then `docker exec` the setup inside it. Do **not** set + `HYPERLOOM_DOCKER_TARGET_HOST` (run on the current host). +- **Framework:** vLLM — provided by the container image; run setup with + `--install-framework none --yes` inside the container (do **not** `--install-framework + vllm`). - **LLM provider / model / `USER_DATA_PATH`:** use the values already in `.env`; do not change them. ## Hard constraints (automated release gate) -- Do **not** modify any GPU-related environment variable (`ROCR_VISIBLE_DEVICES` is - already `0` and pins this container to its single card; leave it). Do not override - device visibility. -- Do **not** run `docker`, do **not** start/exec containers, and do **not** choose - GPUs via `rocm-smi`. -- Do **not** print, echo, or copy secret values (API keys) into output or logs. +Your `docker run` **MUST** use exactly the flags below. These **replace** the demo +skill's default `--device /dev/dri` (all GPUs) and `--group-add video` (a group *name* +the pod has no entry for) with single-card isolation and numeric GIDs. Every other +aspect of the skill's docker flow (long-lived `--entrypoint tail … -f /dev/null`, +mounting `$REPO_ROOT:$REPO_ROOT`, `docker exec` setup, running optimize inside) is +unchanged. Read the values from `.env`: + +- **Container name:** `--name "$HYPERLOOM_CONTAINER_NAME"` (already unique per leg; + another leg may share this host's dockerd, so do not rename it to a fixed value). +- **Image:** `"$HYPERLOOM_IMAGE"`. +- **Single-GPU isolation (REPLACES `--device /dev/dri`):** + `--device /dev/kfd --device /dev/dri/renderD${E2E_RENDERD}` +- **Numeric group-add (REPLACES `--group-add video`):** + `--group-add ${E2E_KFD_GID} --group-add ${E2E_DRI_GID}` +- **Resource caps:** `--cpus ${E2E_LEG_CPUS} --memory ${E2E_LEG_MEM} --shm-size ${HYPERLOOM_SHM_SIZE}` +- **Security:** `--security-opt seccomp=unconfined` +- **Device pin:** `-e HIP_VISIBLE_DEVICES=0 -e ROCR_VISIBLE_DEVICES=0` +- **Mounts:** `-v "$REPO_ROOT:$REPO_ROOT" -v "${E2E_NFS_MOUNT}:${E2E_NFS_MOUNT}"` — + mounting **all** of `${E2E_NFS_MOUNT}` at the same absolute path is **required** so + that `MODEL_PATH` (a symlink into another subtree under `${E2E_NFS_MOUNT}`) resolves + inside the container. Do **not** mount only the model's parent directory. + +Also: + +- Do **not** add any other `--device`, do **not** use `--group-add video`, and do **not** + choose GPUs via `rocm-smi`. The single bound `renderD` node + `HIP/ROCR_VISIBLE_DEVICES=0` + are what pin this leg to its one card. +- Do **not** print, echo, or copy secret values (API keys) into output or logs. The key + lives only in the pod-local `.env` (it reaches the container via `-v $REPO_ROOT:$REPO_ROOT`); + do **not** write it anywhere else, and never onto NFS outside that `.env`. - Do **not** modify `USER_DATA_PATH`. ## Termination -When setup completes successfully, stop. Report only `setup complete: docker/vllm`. -If setup hard-fails, report the failure and stop. +When setup completes successfully, stop and report only `setup complete: docker/vllm`. +Leave the container **running** so the demo turn can `docker exec` into it. If setup +hard-fails, report the failure and stop. diff --git a/.github/scripts/pre-release-e2e-dispatch.sh b/.github/scripts/pre-release-e2e-dispatch.sh index 099b616296..de2fb0bcc6 100755 --- a/.github/scripts/pre-release-e2e-dispatch.sh +++ b/.github/scripts/pre-release-e2e-dispatch.sh @@ -336,8 +336,9 @@ if [ "$want_docker_host" = 1 ]; then host_resources="$(jq -n --arg cpu "$HOST_CPU" --arg mem "$HOST_MEM" --arg shm "$HOST_SHM" \ --arg eph "$HOST_EPHEMERAL" \ '{replica:1, gpu:"8", cpu:$cpu, memory:$mem, sharedMemory:$shm, ephemeralStorage:$eph}')" - # The host env carries the per-leg GPU map so the host bootstrap launches the right - # nested containers via docker-run-hyperloom.sh . + # The host env carries the per-leg GPU map so the host bootstrap runs each docker leg + # (run_leg, docker mode) with the right GPU index; each leg's agent then `docker run`s + # its own single-GPU container per the demo skill. docker_legs=""; gpu_map="{}" for leg in $REQ_TASKS; do case "$leg" in diff --git a/.github/workflows/pre-release-e2e-test.yml b/.github/workflows/pre-release-e2e-test.yml index 75ad2411fa..2eaddc255c 100644 --- a/.github/workflows/pre-release-e2e-test.yml +++ b/.github/workflows/pre-release-e2e-test.yml @@ -224,7 +224,7 @@ jobs: # Wheel named after CI_VERSION per design §5. cp "$WHEEL" "$wheel_dir/" # Stage the in-pod scripts + fixed prompts (pods read them from NFS). - cp .github/pre-release/bootstrap-pre-release.sh .github/pre-release/docker-run-hyperloom.sh "$boot_dir/" + cp .github/pre-release/bootstrap-pre-release.sh "$boot_dir/" cp .github/pre-release/prompts/pre-release/*.md "$boot_dir/prompts/pre-release/" chmod +x "$boot_dir"/*.sh # manifest.json (design §5). From fa104b1028351d451caa6c50ab23784a25e4b965 Mon Sep 17 00:00:00 2001 From: lishuoshuo-amd Date: Fri, 28 Aug 2026 10:48:38 +0800 Subject: [PATCH 24/52] pre-release-e2e: preempt stale SaFE workloads in a standalone first job (fix reap deadlock) A newer commit must tear down the in-flight run's SaFE pods. cancel-in-progress only cancels the GitHub JOB, not the PyTorchJob pods, and the existing reap lived in the `dispatch` job -- which needs [resolve, build] on the SINGLE self-hosted baremetal runner. When the old run still held that runner (and its 8 GPUs), the new run's resolve/build queued behind it, so the reap could never run: the cleanup was deadlocked behind the very run it had to reclaim. This is why cancelling didn't actually stop the pods. Add a `preempt` job (job 0) on a GitHub-hosted runner (ubuntu-latest), which does NOT queue behind the busy baremetal runner. It runs a new standalone script pre-release-e2e-reap.sh that LISTs + filters e2e-* non-terminal workloads in this workspace and POSTs /stop to each. Because it runs BEFORE this run dispatches anything, every such workload is necessarily from an older run -> safe to stop wholesale (no VERSION_TAG self-exclusion needed). resolve needs preempt (with if: !cancelled() so a best-effort reap failure never blocks the release gate); build/dispatch chain from resolve unchanged. Stopping the old pods frees the GPUs and makes the old run's poll see phase=Stopped -> its legs FAIL -> the old GitHub job ends as a consequence (the correct causal order). The `if: cancelled()` cleanup step and the dispatch-side reap remain as backstops. Co-Authored-By: Claude Opus 4.8 --- .github/scripts/pre-release-e2e-reap.sh | 78 ++++++++++++++++++++++ .github/workflows/pre-release-e2e-test.yml | 47 +++++++++++-- 2 files changed, 118 insertions(+), 7 deletions(-) create mode 100644 .github/scripts/pre-release-e2e-reap.sh diff --git a/.github/scripts/pre-release-e2e-reap.sh b/.github/scripts/pre-release-e2e-reap.sh new file mode 100644 index 0000000000..66307c3154 --- /dev/null +++ b/.github/scripts/pre-release-e2e-reap.sh @@ -0,0 +1,78 @@ +#!/usr/bin/env bash +# SPDX-FileCopyrightText: 2026 Advanced Micro Devices, Inc. +# SPDX-License-Identifier: MIT +# +# Pre-release E2E: PRE-EMPT stale SaFE workloads at the very START of a run, BEFORE any +# job that touches the single self-hosted GPU runner. +# +# Why a standalone script + its own job (not the reap inside the dispatch script): +# The dispatch reap runs in the `dispatch` job, which `needs: [resolve, build]` and +# `runs-on: hyperloom-pre-e2e-baremetal` -- the SINGLE self-hosted runner. When a newer +# commit supersedes an in-flight run, GitHub's concurrency.cancel-in-progress cancels +# the old JOB but does NOT reliably stop the SaFE PyTorchJob pods it created, and the +# old run may still occupy that one runner (and its 8 GPUs). So the new run's +# resolve/build queue BEHIND the old run and the dispatch reap never gets to run -- a +# deadlock where the cleanup is queued behind the very thing it must clean up. +# +# This script runs in a `preempt` job on a GITHUB-HOSTED runner (ubuntu-latest), which +# does NOT queue behind the busy baremetal runner. It only needs network reach to the +# SaFE API. It fires first and every other job `needs: preempt`, so the stale pods are +# stopped -> GPUs freed -> the old run's poll sees phase=Stopped and its legs FAIL -> +# the old GitHub job ends as a CONSEQUENCE of the pod stopping (the correct causal +# order), and this run's resolve/build/dispatch can then get the runner + GPUs. +# +# Pre-emption semantics: this runs BEFORE this run dispatches ANY workload, so every +# non-terminal `e2e-*` workload in this workspace is necessarily from an OLDER run and is +# safe to stop wholesale -- no VERSION_TAG self-exclusion needed (there is nothing of +# ours to exclude yet). +# +# Inputs (env): +# SAFE_API_BASE SaFE API base url (required) +# SAFE_API_KEY bearer token (ADMIN, to stop privileged pods) (required) +# SAFE_WORKSPACE_ID workspace to scope the reap to (required) +# SAFE_CACERT / SAFE_INSECURE TLS to the API (CA bundle / skip-verify) +set -euo pipefail + +: "${SAFE_API_BASE:?SAFE_API_BASE is required}" +: "${SAFE_API_KEY:?SAFE_API_KEY is required}" +: "${SAFE_WORKSPACE_ID:?SAFE_WORKSPACE_ID is required}" + +API="${SAFE_API_BASE%/}/api/v1/workloads" +auth=(-H "Authorization: Bearer ${SAFE_API_KEY}") +tls=() +if [ -n "${SAFE_CACERT:-}" ]; then + tls=(--cacert "$SAFE_CACERT") +elif [ "${SAFE_INSECURE:-0}" = "1" ]; then + tls=(-k) +fi + +summary() { echo "$*" | tee -a "${GITHUB_STEP_SUMMARY:-/dev/null}"; } + +# List every e2e-* workload in this workspace that is NOT already terminal, and POST +# /stop to each. Resilient: a missing/unreachable API is a skip, never a hard failure +# (we must not block the run just because the reclaim couldn't reach SaFE). +reap_all_stale() { + local resp + resp="$(curl -sS "${tls[@]}" --max-time 30 "$API" "${auth[@]}" 2>/dev/null || true)" + [ -n "$resp" ] || { summary "• [preempt] could not list workloads; skipping reclaim"; return 0; } + local stale + stale="$(printf '%s' "$resp" | jq -r --arg ws "$SAFE_WORKSPACE_ID" ' + (.items // .workloads // .)[]? + | select(((.displayName // .name // "") | startswith("e2e-"))) + | select((.workspaceId // $ws) == $ws) + | select((.phase // .status // "") as $p + | (["Stopped","Failed","Succeeded","Completed","Deleted"] | index($p)) | not) + | (.workloadId // .id)' 2>/dev/null || true)" + [ -n "$stale" ] || { summary "• [preempt] no stale e2e workloads to reclaim"; return 0; } + local wid code n=0 + while IFS= read -r wid; do + [ -n "$wid" ] || continue + code="$(curl -sS "${tls[@]}" --max-time 20 -o /dev/null -w '%{http_code}' \ + -X POST "$API/$wid/stop" "${auth[@]}" 2>/dev/null || echo 000)" + summary "• [preempt] stopped stale workload \`$wid\` (stop HTTP $code)" + n=$((n+1)) + done <<< "$stale" + summary "• [preempt] stopped $n stale e2e workload(s) before this run dispatches" +} + +reap_all_stale diff --git a/.github/workflows/pre-release-e2e-test.yml b/.github/workflows/pre-release-e2e-test.yml index 2eaddc255c..43d458a9ed 100644 --- a/.github/workflows/pre-release-e2e-test.yml +++ b/.github/workflows/pre-release-e2e-test.yml @@ -54,13 +54,19 @@ on: description: "Comma-separated subset of leg IDs (default: all 8)" required: false -# One pre-release run at a time (design §13, point D): the peak GPU footprint stays at -# 8. cancel-in-progress: a NEWER push supersedes the in-flight run -- GitHub cancels the -# old run, whose `if: cancelled()` step stops its SaFE workloads (freeing GPUs) before -# this new run dispatches. This is what makes "push a fix -> the stale run is torn down -# and the fresh code reruns from scratch" automatic, instead of queueing behind a run -# built from superseded code. Per-PR concurrency (group keyed by ref) so two different -# PRs don't cancel each other -- only newer commits on the SAME PR supersede. +# One pre-release run at a time (design §13, point D): the peak GPU footprint stays at 8. +# A NEWER push supersedes the in-flight run. cancel-in-progress cancels the old GitHub +# JOB, but that alone does NOT reliably stop the old run's SaFE PyTorchJob pods (the +# `if: cancelled()` cleanup step gets only a short grace window, and it runs on the single +# busy baremetal runner). So the PRIMARY teardown is the `preempt` job (job 0): it runs +# FIRST on a GitHub-hosted runner and stops every stale e2e-* workload up front, freeing +# the GPUs BEFORE this run's resolve/build/dispatch queue for the baremetal runner. The +# old run's poll then sees phase=Stopped -> its legs FAIL -> the old job ends as a +# consequence. This makes "push a fix -> the stale run is torn down and the fresh code +# reruns" automatic, instead of deadlocking on the reap being queued behind the run it +# must reclaim. The `if: cancelled()` step + the dispatch-side reap remain as backstops. +# Per-PR concurrency (group keyed by ref) so two different PRs don't cancel each other -- +# only newer commits on the SAME PR supersede. concurrency: group: pre-release-e2e-${{ github.event.pull_request.number || github.ref }} cancel-in-progress: true @@ -72,8 +78,35 @@ permissions: issues: write # PR comments go through the issues API jobs: + # 0. preempt: STOP any stale SaFE e2e-* workloads from a superseded/older run BEFORE + # anything touches the single self-hosted GPU runner. Runs on a GitHub-hosted runner so + # it does NOT queue behind that busy baremetal runner (the deadlock we hit: the reap in + # `dispatch` needs [resolve,build] on the one runner, which is still held by the old run + # -> the reclaim can never run). Stopping the old pods frees the GPUs and makes the old + # run's poll see phase=Stopped -> its legs FAIL -> the old job ends as a consequence. + # Every other job needs this, so nothing dispatches until the reclaim has fired. + preempt: + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v7 + - name: Stop stale SaFE e2e workloads + env: + SAFE_API_BASE: ${{ vars.PRE_E2E_SAFE_API_BASE }} + SAFE_API_KEY: ${{ secrets.PRE_E2E_SAFE_API_KEY }} # ADMIN token (stops privileged pods) + SAFE_WORKSPACE_ID: ${{ vars.PRE_E2E_SAFE_WORKSPACE_ID }} + SAFE_INSECURE: ${{ vars.PRE_E2E_SAFE_INSECURE || '1' }} + run: | + command -v jq >/dev/null 2>&1 || (sudo apt-get update && sudo apt-get install -y jq) + chmod +x .github/scripts/pre-release-e2e-reap.sh + .github/scripts/pre-release-e2e-reap.sh + # 1. resolve: gate on a real version bump (PR vs base) or manual input; compute CI_VERSION. resolve: + needs: preempt + # Pre-emption is best-effort reclamation: if it can't reach SaFE it still exits 0, but + # even a hard job failure must NOT block the release gate -- so proceed unless the whole + # run was cancelled (a newer commit superseding us). + if: ${{ !cancelled() }} runs-on: hyperloom-pre-e2e-baremetal outputs: run: ${{ steps.decide.outputs.run }} From 0849a2e84aaf8ee340ebe835c477cd929dbcc781 Mon Sep 17 00:00:00 2001 From: lishuoshuo-amd Date: Fri, 28 Aug 2026 11:11:50 +0800 Subject: [PATCH 25/52] pre-release-e2e: docker legs pick image from the skill list, not hard-coded The bootstrap was hard-coding HYPERLOOM_IMAGE tags, duplicating the demo skill's "Suggested Docker images" list -- and the copies drifted: the sglang default was a nonexistent "-srt" tag and the vllm ref lacked the docker.io/ prefix, so the pod-local dockerd returned "manifest unknown" and every docker leg hung pulling an image that could not be resolved. Remove the image injection entirely. The demo skill's SKILL.md is now the single source of truth for image tags; bootstrap only injects the CI hard constraints the skill cannot know (renderD isolation, numeric GIDs, cpu/mem caps, full /shared_nfs mount, per-leg container name). The setup-docker-*.md prompts now select HYPERLOOM_IMAGE from the skill's list: - vllm: the single architecture-independent vllm row. - sglang: detect the GPU arch via rocminfo (gfx950 -> MI355X, gfx942 -> MI300X) and use the matching sglang row. Detection selects the image tag only; the card is still pinned by the single bound renderD + HIP/ROCR_VISIBLE_DEVICES=0. Co-Authored-By: Claude Opus 4.8 --- .github/pre-release/bootstrap-pre-release.sh | 15 ++++---- .../pre-release/setup-docker-sglang.md | 36 +++++++++++++++---- .../prompts/pre-release/setup-docker-vllm.md | 24 +++++++++---- 3 files changed, 56 insertions(+), 19 deletions(-) diff --git a/.github/pre-release/bootstrap-pre-release.sh b/.github/pre-release/bootstrap-pre-release.sh index 58d0f5731f..dcdfab54de 100755 --- a/.github/pre-release/bootstrap-pre-release.sh +++ b/.github/pre-release/bootstrap-pre-release.sh @@ -95,17 +95,17 @@ run_leg() { # inject them as E2E_* .env values the setup prompt tells the agent to copy verbatim. # renderD = 128 + GPU_INDEX*8 (stride 8, VERIFIED on a real privileged MI355X x8 pod). # The pod /etc/group has no `video`/`render` NAMES, so numeric GIDs are required. - local dk_image="" dk_rd="" dk_kfd_gid="" dk_dri_gid="" dk_nfs_mount="" + local dk_rd="" dk_kfd_gid="" dk_dri_gid="" dk_nfs_mount="" if [ "$run_mode" = docker ]; then : "${GPU_INDEX:?docker leg needs GPU_INDEX}" dk_rd=$(( 128 + GPU_INDEX * 8 )) dk_kfd_gid="$(stat -c %g /dev/kfd 2>/dev/null || echo 0)" dk_dri_gid="$(stat -c %g "/dev/dri/renderD${dk_rd}" 2>/dev/null || stat -c %g /dev/dri 2>/dev/null || echo 0)" - if [ "$backend" = vllm ]; then - dk_image="${HYPERLOOM_IMAGE_VLLM:-vllm/vllm-openai-rocm:v0.27.1}" - else - dk_image="${HYPERLOOM_IMAGE_SGLANG:-lmsysorg/sglang-rocm:v0.5.17-rocm724-mi35x-srt}" - fi + # NOTE: we deliberately do NOT inject HYPERLOOM_IMAGE. The demo skill owns the image + # list (examples/*/SKILL.md "Suggested Docker images") -- duplicating those tags here + # is what caused the wrong sglang tag. Instead the setup-docker-*.md prompt tells the + # agent to pick from the skill's list by backend, and for sglang to detect the GPU + # arch (gfx950 -> mi35x, gfx942 -> mi30x) via rocminfo and choose the matching tag. # The model path (e.g. /shared_nfs/models/) is a SYMLINK into a DIFFERENT # /shared_nfs subtree (/shared_nfs/huggingface_models/...). Mounting only NFS_ROOT # (the CI subdir) or the model's parent leaves the symlink TARGET unmounted -> broken @@ -148,7 +148,8 @@ run_leg() { # docker leg: hand the container lifecycle to the agent (demo skill) and carry the # CI hard constraints it must apply to its `docker run` (see setup-docker-*.md). if [ "$run_mode" = docker ]; then - echo "HYPERLOOM_IMAGE=${dk_image}" + # HYPERLOOM_IMAGE intentionally NOT set: the agent selects it from the skill's + # image list (by backend + detected GPU arch). See setup-docker-*.md. echo "HYPERLOOM_CONTAINER_NAME=hyperloom-${leg}" # unique per leg (shared host dockerd) echo "HYPERLOOM_SHM_SIZE=${LEG_SHM:-64g}" echo "E2E_GPU_INDEX=${GPU_INDEX}" diff --git a/.github/pre-release/prompts/pre-release/setup-docker-sglang.md b/.github/pre-release/prompts/pre-release/setup-docker-sglang.md index c8fcee6fbe..aadc7e122a 100644 --- a/.github/pre-release/prompts/pre-release/setup-docker-sglang.md +++ b/.github/pre-release/prompts/pre-release/setup-docker-sglang.md @@ -14,12 +14,13 @@ setup step for a **docker + SGLang** leg, then stop. Do not run the demo yet. A `.env` file exists in the current workspace (`REPO_ROOT`) with these values already set: `ANTHROPIC_API_KEY`, `CLAUDE_MODEL`, `USER_DATA_PATH`, `HYPERLOOM_RUN_MODE=docker`, `FRAMEWORK=sglang`, `MODEL_PATH`, `TARGET_GAIN`, -`DEMO_HOURS`, and the container/isolation values the demo skill reads: -`HYPERLOOM_IMAGE`, `HYPERLOOM_CONTAINER_NAME`, `HYPERLOOM_SHM_SIZE`, plus the CI +`DEMO_HOURS`, `HYPERLOOM_CONTAINER_NAME`, `HYPERLOOM_SHM_SIZE`, plus the CI isolation values `E2E_RENDERD`, `E2E_KFD_GID`, `E2E_DRI_GID`, `E2E_LEG_CPUS`, `E2E_LEG_MEM`, `E2E_NFS_MOUNT`. The wheel is already installed via `pip install --target .` so a `hyperloom/` package directory is present. +`HYPERLOOM_IMAGE` is **not** in `.env` — you choose it (see "Image selection" below). + ## Fixed decisions Follow the `hyperloom-setup` skill in **docker** mode with these fixed decisions — do @@ -34,6 +35,27 @@ Follow the `hyperloom-setup` skill in **docker** mode with these fixed decisions - **LLM provider / model / `USER_DATA_PATH`:** use the values already in `.env`; do not change them. +## Image selection + +`HYPERLOOM_IMAGE` is not preset — pick it from the demo skill's **"Suggested Docker +images"** list (do **not** ask the user, and do **not** hard-code a tag from memory). +This is a `sglang` leg, and the skill lists a different sglang image per GPU +architecture, so detect the architecture on this host first: + +```bash +gfx="$(/opt/rocm/bin/rocminfo 2>/dev/null | grep -oiE 'gfx9[0-9a-f]+' | head -1)" +``` + +- `gfx950` → **MI355X** → use the skill's `sglang MI355X` image. +- `gfx942` → **MI300X** → use the skill's `sglang MI300X` image. + +Read the exact, fully-qualified `docker.io/...` tag for that row **from the skill file** +(the demo skill's SKILL.md "Suggested Docker images" section) and export it as +`HYPERLOOM_IMAGE` for the `docker run` below. This is architecture **detection only** — +it selects the image tag, not which card the leg runs on (the card is pinned by the +isolation flags below). If `rocminfo` cannot be found or reports no `gfx`, stop and +report the failure rather than guessing a tag. + ## Hard constraints (automated release gate) Your `docker run` **MUST** use exactly the flags below. These **replace** the demo @@ -45,7 +67,7 @@ unchanged. Read the values from `.env`: - **Container name:** `--name "$HYPERLOOM_CONTAINER_NAME"` (already unique per leg; another leg may share this host's dockerd, so do not rename it to a fixed value). -- **Image:** `"$HYPERLOOM_IMAGE"`. +- **Image:** the `HYPERLOOM_IMAGE` you selected above (skill list + detected arch). - **Single-GPU isolation (REPLACES `--device /dev/dri`):** `--device /dev/kfd --device /dev/dri/renderD${E2E_RENDERD}` - **Numeric group-add (REPLACES `--group-add video`):** @@ -60,9 +82,11 @@ unchanged. Read the values from `.env`: Also: -- Do **not** add any other `--device`, do **not** use `--group-add video`, and do **not** - choose GPUs via `rocm-smi`. The single bound `renderD` node + `HIP/ROCR_VISIBLE_DEVICES=0` - are what pin this leg to its one card. +- Do **not** add any other `--device`, and do **not** use `--group-add video`. You may run + `rocminfo`/`rocm-smi` to **detect the GPU architecture** for image selection, but do + **not** use them to **choose which GPU** the leg runs on: the single bound `renderD` node + + `HIP/ROCR_VISIBLE_DEVICES=0` are what pin this leg to its one card. Do not add + `HIP_VISIBLE_DEVICES`/`ROCR_VISIBLE_DEVICES` values other than `0`. - Do **not** print, echo, or copy secret values (API keys) into output or logs. The key lives only in the pod-local `.env` (it reaches the container via `-v $REPO_ROOT:$REPO_ROOT`); do **not** write it anywhere else, and never onto NFS outside that `.env`. diff --git a/.github/pre-release/prompts/pre-release/setup-docker-vllm.md b/.github/pre-release/prompts/pre-release/setup-docker-vllm.md index ab065e70a7..5531c96c15 100644 --- a/.github/pre-release/prompts/pre-release/setup-docker-vllm.md +++ b/.github/pre-release/prompts/pre-release/setup-docker-vllm.md @@ -14,12 +14,13 @@ setup step for a **docker + vLLM** leg, then stop. Do not run the demo yet. A `.env` file exists in the current workspace (`REPO_ROOT`) with these values already set: `ANTHROPIC_API_KEY`, `CLAUDE_MODEL`, `USER_DATA_PATH`, `HYPERLOOM_RUN_MODE=docker`, `FRAMEWORK=vllm`, `MODEL_PATH`, `TARGET_GAIN`, -`DEMO_HOURS`, and the container/isolation values the demo skill reads: -`HYPERLOOM_IMAGE`, `HYPERLOOM_CONTAINER_NAME`, `HYPERLOOM_SHM_SIZE`, plus the CI +`DEMO_HOURS`, `HYPERLOOM_CONTAINER_NAME`, `HYPERLOOM_SHM_SIZE`, plus the CI isolation values `E2E_RENDERD`, `E2E_KFD_GID`, `E2E_DRI_GID`, `E2E_LEG_CPUS`, `E2E_LEG_MEM`, `E2E_NFS_MOUNT`. The wheel is already installed via `pip install --target .` so a `hyperloom/` package directory is present. +`HYPERLOOM_IMAGE` is **not** in `.env` — you choose it (see "Image selection" below). + ## Fixed decisions Follow the `hyperloom-setup` skill in **docker** mode with these fixed decisions — do @@ -34,6 +35,15 @@ Follow the `hyperloom-setup` skill in **docker** mode with these fixed decisions - **LLM provider / model / `USER_DATA_PATH`:** use the values already in `.env`; do not change them. +## Image selection + +`HYPERLOOM_IMAGE` is not preset — pick it from the demo skill's **"Suggested Docker +images"** list (do **not** ask the user, and do **not** hard-code a tag from memory). +This is a `vllm` leg, so use the single fully-qualified `docker.io/...` tag on the +skill's `vllm` row (it is architecture-independent — no GPU detection needed). Read the +exact tag **from the skill file** (the demo skill's SKILL.md "Suggested Docker images" +section) and export it as `HYPERLOOM_IMAGE` for the `docker run` below. + ## Hard constraints (automated release gate) Your `docker run` **MUST** use exactly the flags below. These **replace** the demo @@ -45,7 +55,7 @@ unchanged. Read the values from `.env`: - **Container name:** `--name "$HYPERLOOM_CONTAINER_NAME"` (already unique per leg; another leg may share this host's dockerd, so do not rename it to a fixed value). -- **Image:** `"$HYPERLOOM_IMAGE"`. +- **Image:** the `HYPERLOOM_IMAGE` you selected above (the skill's `vllm` tag). - **Single-GPU isolation (REPLACES `--device /dev/dri`):** `--device /dev/kfd --device /dev/dri/renderD${E2E_RENDERD}` - **Numeric group-add (REPLACES `--group-add video`):** @@ -60,9 +70,11 @@ unchanged. Read the values from `.env`: Also: -- Do **not** add any other `--device`, do **not** use `--group-add video`, and do **not** - choose GPUs via `rocm-smi`. The single bound `renderD` node + `HIP/ROCR_VISIBLE_DEVICES=0` - are what pin this leg to its one card. +- Do **not** add any other `--device`, and do **not** use `--group-add video`. Do **not** + use `rocm-smi`/`rocminfo` to **choose which GPU** the leg runs on: the single bound + `renderD` node + `HIP/ROCR_VISIBLE_DEVICES=0` are what pin this leg to its one card. Do + not add `HIP_VISIBLE_DEVICES`/`ROCR_VISIBLE_DEVICES` values other than `0`. (This vLLM + image is architecture-independent, so no GPU-arch detection is needed.) - Do **not** print, echo, or copy secret values (API keys) into output or logs. The key lives only in the pod-local `.env` (it reaches the container via `-v $REPO_ROOT:$REPO_ROOT`); do **not** write it anywhere else, and never onto NFS outside that `.env`. From 01981d4c1e77c21c229349f462999f8b0033ec5d Mon Sep 17 00:00:00 2001 From: lishuoshuo-amd Date: Fri, 28 Aug 2026 11:26:02 +0800 Subject: [PATCH 26/52] pre-release-e2e: judge leg liveness by stall, not a wall-clock startup budget The wait-for-final.json loop declared a leg dead if `optimize` produced no state.json within a fixed window (900s from when the claude --print turns returned). That judged LIVE legs dead: baremetal SGLang builds from source (gfx950, py3.12), starts a Ray head, then loads the server -- >15m before the first state.json -- so a real, still-progressing leg got "optimize never launched". A pid check can't fix it either: during the build `optimize` is not a process yet, so "not launched yet" and "launched then died" look identical. Replace the wall-clock startup grace with a stall check: while there is no state.json, ask how long since ANY file under $session was last written (USER_DATA_PATH=$session, so build/runtime logs, Ray output, and session artifacts all keep the tree's mtime fresh while anything is progressing). A leg is dead only if the tree has been completely idle for LEG_STALL_GRACE_S (default 600s) and no state.json exists yet. This waits out arbitrarily slow builds as long as they keep writing, but reaps a truly hung/exited launch within ~10m. deadline_s (hours*3600+3600) remains the hard upper bound. Co-Authored-By: Claude Opus 4.8 --- .github/pre-release/bootstrap-pre-release.sh | 29 ++++++++++++++++---- 1 file changed, 24 insertions(+), 5 deletions(-) diff --git a/.github/pre-release/bootstrap-pre-release.sh b/.github/pre-release/bootstrap-pre-release.sh index dcdfab54de..25a1af3e5b 100755 --- a/.github/pre-release/bootstrap-pre-release.sh +++ b/.github/pre-release/bootstrap-pre-release.sh @@ -211,9 +211,19 @@ run_leg() { # dir by globbing the newest one under $session that contains a state.json, and # re-point the poll's pin (.session_dir) at it. local wait_interval="${LEG_WAIT_INTERVAL_S:-45}" - local startup_grace="${LEG_STARTUP_GRACE_S:-900}" # 15m for optimize to appear + # Liveness, NOT a wall-clock startup budget. The prior fixed "optimize must produce a + # state.json within N seconds" judged LIVE legs dead: baremetal SGLang builds from + # source (gfx950, py3.12), starts a Ray head, then loads the server -- >15m before the + # first state.json, and `optimize` isn't even a process yet during the build, so a pid + # check can't tell "not launched yet" from "died". Instead we ask "how long since ANY + # file under $session was last written?": USER_DATA_PATH=$session, so the build/runtime + # logs, Ray output, and session artifacts all land under this tree and keep its mtime + # fresh while anything is making progress. A leg is dead only if the tree has been + # completely IDLE for stall_grace AND no state.json exists yet -- this waits out slow + # builds (as long as they keep writing) but reaps a truly hung/exited launch quickly. + local stall_grace="${LEG_STALL_GRACE_S:-600}" # 10m of NO file writes -> dead local final_grace="${LEG_FINAL_GRACE_S:-120}" # state stop_reason -> final.json - local deadline_s=$(( hours * 3600 + 3600 )) # demo budget + 1h margin + local deadline_s=$(( hours * 3600 + 3600 )) # demo budget + 1h margin (hard cap) local start_ts; start_ts="$(date +%s)" local real_sdir="" final_json="" state_json="" @@ -230,9 +240,18 @@ run_leg() { log "leg $leg real session dir: $real_sdir" # Re-pin so the poll (leg_session_dir -> head -n1 .session_dir) finds the report. echo "$real_sdir" > "${session}/.session_dir" - elif [ "$elapsed" -ge "$startup_grace" ]; then - log "ERROR: leg $leg -- no session dir with state.json under $session after ${elapsed}s (optimize never launched?)" - return 1 + else + # No state.json yet -> judge liveness by how long since ANY file under $session + # was written. newest mtime across the tree; if the tree is empty, fall back to + # $session's own mtime so a brand-new leg isn't reaped on its first iteration. + local last_write idle + last_write="$(find "$session" -type f -printf '%T@\n' 2>/dev/null | sort -rn | head -n1)" + [ -n "$last_write" ] || last_write="$(stat -c %Y "$session" 2>/dev/null || echo "$now")" + idle=$(( now - ${last_write%.*} )) + if [ "$idle" -ge "$stall_grace" ]; then + log "ERROR: leg $leg -- no state.json and no file written under $session for ${idle}s (>= ${stall_grace}s stall; build/optimize hung or exited)" + return 1 + fi fi fi From babd54af4cb77cf256ba451f436b1013363a51f1 Mon Sep 17 00:00:00 2001 From: lishuoshuo-amd Date: Fri, 28 Aug 2026 11:52:14 +0800 Subject: [PATCH 27/52] pre-release-e2e: fix docker-host eviction (vfs data-root on node NVMe) + pin image tag to the skill Docker-host pod was EVICTED live (2026-08-28): "ephemeral local storage usage exceeds the total limit of containers 1792Gi". Root cause: the pod-local dockerd uses --storage-driver=vfs (no layer dedup), and its default data-root /var/lib/docker lives in the container rootfs, which counts against the pod's ephemeralStorage quota. 4 legs pulling+running big ROCm images multiply under vfs and blow past 1792Gi. Fix: this is a privileged host pod, so it sees the node's own disks directly. Point the vfs data-root at /shared-data (node-local ~28T NVMe xfs, a real host filesystem, NOT the container rootfs overlay -> not counted toward ephemeralStorage). Reap stale e2e-docker data-roots from older runs first (/shared-data is host-persistent, so vfs copies would otherwise accumulate and fill the node NVMe). Also (obs1): stop the docker setup agent from freelancing the image tag. The prompts now extract the EXACT tag from the demo skill's SKILL.md via grep and forbid any version bump / substitution. bootstrap injects HYPERLOOM_SKILL_PATH (the leg's demo skill file at $root/.claude/skills//SKILL.md, where pip --target materializes the [tool.setuptools.data-files] example skills) so the grep has a concrete path. Co-Authored-By: Claude Opus 4.8 --- .github/pre-release/bootstrap-pre-release.sh | 54 +++++++++++++++++-- .../pre-release/setup-docker-sglang.md | 51 +++++++++++------- .../prompts/pre-release/setup-docker-vllm.md | 30 ++++++++--- 3 files changed, 108 insertions(+), 27 deletions(-) diff --git a/.github/pre-release/bootstrap-pre-release.sh b/.github/pre-release/bootstrap-pre-release.sh index 25a1af3e5b..0789b163fd 100755 --- a/.github/pre-release/bootstrap-pre-release.sh +++ b/.github/pre-release/bootstrap-pre-release.sh @@ -149,7 +149,20 @@ run_leg() { # CI hard constraints it must apply to its `docker run` (see setup-docker-*.md). if [ "$run_mode" = docker ]; then # HYPERLOOM_IMAGE intentionally NOT set: the agent selects it from the skill's - # image list (by backend + detected GPU arch). See setup-docker-*.md. + # image list (by backend + detected GPU arch). See setup-docker-*.md. To pin the + # tag to the SKILL's exact string (no freelancing), the setup prompt greps it out + # of the demo skill's SKILL.md -- so hand the agent that file's absolute path. + # `pip install --target "$root"` materializes the example demo skills (declared as + # [tool.setuptools.data-files] in pyproject.toml) at + # $root/.claude/skills//SKILL.md + # (empirically verified). The demo skill is chosen by leg duration: + # *-3h -> hyperloom-qwen3-8b-3h ; *-12h -> hyperloom-qwen3-14b-fp8-12h. + local demo_skill="" + case "$leg" in + *-3h) demo_skill="hyperloom-qwen3-8b-3h" ;; + *-12h) demo_skill="hyperloom-qwen3-14b-fp8-12h" ;; + esac + echo "HYPERLOOM_SKILL_PATH=${root}/.claude/skills/${demo_skill}/SKILL.md" echo "HYPERLOOM_CONTAINER_NAME=hyperloom-${leg}" # unique per leg (shared host dockerd) echo "HYPERLOOM_SHM_SIZE=${LEG_SHM:-64g}" echo "E2E_GPU_INDEX=${GPU_INDEX}" @@ -299,8 +312,43 @@ ensure_dockerd() { || { log "ERROR: docker.io install failed"; tail -30 /tmp/apt-docker.log; return 1; } log "docker installed: $(docker --version 2>&1)" fi - log "starting pod-local dockerd (vfs, detached via setsid)" - setsid bash -c 'dockerd --host=unix:///var/run/docker.sock --storage-driver=vfs >/var/log/dockerd.log 2>&1' \ + # --- pick a data-root that does NOT count against the pod's ephemeral limit --- + # The vfs storage driver has NO layer dedup: every image layer + every container is a + # full copy. Left at the default /var/lib/docker (inside the container ROOTFS), 4 legs + # pulling+running big ROCm images blow past the pod's ephemeralStorage quota and the pod + # is EVICTED ("ephemeral local storage usage exceeds the total limit of containers + # 1792Gi" -- observed live 2026-08-28). This is a PRIVILEGED host pod, so it sees the + # NODE's own disks directly: /shared-data is the node-local NVMe (~28T xfs on + # /dev/mapper/nvme_vg-nvme_lv), a real host filesystem -- NOT the container rootfs + # overlay -- so bytes written there are NOT counted toward the pod's ephemeralStorage + # quota. Point the vfs data-root there so image/container copies land on the big host + # disk instead of the small pod-ephemeral quota. + # + # /shared-data is host-persistent (survives this pod), so vfs copies from OLD runs would + # accumulate and eventually fill the node NVMe. Reap stale e2e-docker data-roots (any + # run other than this CI_VERSION) before starting dockerd. dockerd is not up yet, so a + # plain rm of the old data-root dirs is safe. + local docker_data_root="/var/lib/docker" + local dr_base="/shared-data/e2e-docker" + local dr_candidate="${dr_base}/${CI_VERSION:-current}" + if [ -d /shared-data ] && mkdir -p "$dr_candidate" 2>/dev/null && touch "$dr_candidate/.wtest" 2>/dev/null; then + rm -f "$dr_candidate/.wtest" 2>/dev/null || true + # reap other runs' leftovers (best-effort; never abort the run on cleanup failure) + if [ -d "$dr_base" ]; then + for d in "$dr_base"/*; do + [ -d "$d" ] || continue + [ "$d" = "$dr_candidate" ] && continue + log "reaping stale docker data-root from an older run: $d" + rm -rf "$d" 2>/dev/null || true + done + fi + docker_data_root="$dr_candidate" + log "dockerd data-root on node-local NVMe: $docker_data_root (vfs copies stay off the pod ephemeral quota)" + else + log "WARN: /shared-data not writable; dockerd falls back to $docker_data_root (may hit the pod ephemeral limit under vfs)" + fi + log "starting pod-local dockerd (vfs, data-root=$docker_data_root, detached via setsid)" + setsid bash -c "dockerd --host=unix:///var/run/docker.sock --storage-driver=vfs --data-root='$docker_data_root' >/var/log/dockerd.log 2>&1" \ /dev/null 2>&1 & local i for i in $(seq 1 60); do diff --git a/.github/pre-release/prompts/pre-release/setup-docker-sglang.md b/.github/pre-release/prompts/pre-release/setup-docker-sglang.md index aadc7e122a..f39f462109 100644 --- a/.github/pre-release/prompts/pre-release/setup-docker-sglang.md +++ b/.github/pre-release/prompts/pre-release/setup-docker-sglang.md @@ -37,24 +37,39 @@ Follow the `hyperloom-setup` skill in **docker** mode with these fixed decisions ## Image selection -`HYPERLOOM_IMAGE` is not preset — pick it from the demo skill's **"Suggested Docker -images"** list (do **not** ask the user, and do **not** hard-code a tag from memory). -This is a `sglang` leg, and the skill lists a different sglang image per GPU -architecture, so detect the architecture on this host first: - -```bash -gfx="$(/opt/rocm/bin/rocminfo 2>/dev/null | grep -oiE 'gfx9[0-9a-f]+' | head -1)" -``` - -- `gfx950` → **MI355X** → use the skill's `sglang MI355X` image. -- `gfx942` → **MI300X** → use the skill's `sglang MI300X` image. - -Read the exact, fully-qualified `docker.io/...` tag for that row **from the skill file** -(the demo skill's SKILL.md "Suggested Docker images" section) and export it as -`HYPERLOOM_IMAGE` for the `docker run` below. This is architecture **detection only** — -it selects the image tag, not which card the leg runs on (the card is pinned by the -isolation flags below). If `rocminfo` cannot be found or reports no `gfx`, stop and -report the failure rather than guessing a tag. +`HYPERLOOM_IMAGE` is not preset. You **MUST** use the **exact, verbatim** `docker.io/...` +tag string that is written on the demo skill's matching `sglang` row in its **"Suggested +Docker images"** section — nothing else. This is a hard release-gate constraint, not a +suggestion: + +- **Do NOT freelance the tag.** Do not bump the version, do not pick a "newer" or + "latest" build, do not substitute a different tag from your memory, from Docker Hub, or + from anywhere other than the skill file. The pinned tag is the one the release is gated + on; a different tag is a **failure**, even if it also pulls successfully. +- The skill lists a **different `sglang` image per GPU architecture**, so first detect the + architecture on this host, then read the **exact** tag for the matching row **from the + skill file itself** rather than typing it out (the demo skill's path is in + `HYPERLOOM_SKILL_PATH` in `.env`; otherwise it is the `SKILL.md` of the demo skill you + are running): + + ```bash + gfx="$(/opt/rocm/bin/rocminfo 2>/dev/null | grep -oiE 'gfx9[0-9a-f]+' | head -1)" + case "$gfx" in + gfx950) row='MI355X' ;; # MI355X + gfx942) row='MI300X' ;; # MI300X + *) echo "ERROR: could not detect GPU arch (gfx='$gfx')"; exit 1 ;; + esac + HYPERLOOM_IMAGE="$(grep -E "^- \`sglang\` $row" "$HYPERLOOM_SKILL_PATH" | grep -oE 'docker\.io/[^`]+' | head -1)" + echo "arch=$gfx row=$row using image: $HYPERLOOM_IMAGE" + ``` + + This is architecture **detection only** — it selects the image tag, not which card the + leg runs on (the card is pinned by the isolation flags below). +- If the arch cannot be detected, the extracted tag is empty, or the resulting image + cannot be pulled, **stop and report the failure** — do **not** substitute any other tag + to work around it. + +Export the extracted value as `HYPERLOOM_IMAGE` for the `docker run` below. ## Hard constraints (automated release gate) diff --git a/.github/pre-release/prompts/pre-release/setup-docker-vllm.md b/.github/pre-release/prompts/pre-release/setup-docker-vllm.md index 5531c96c15..fd700ddc5b 100644 --- a/.github/pre-release/prompts/pre-release/setup-docker-vllm.md +++ b/.github/pre-release/prompts/pre-release/setup-docker-vllm.md @@ -37,12 +37,30 @@ Follow the `hyperloom-setup` skill in **docker** mode with these fixed decisions ## Image selection -`HYPERLOOM_IMAGE` is not preset — pick it from the demo skill's **"Suggested Docker -images"** list (do **not** ask the user, and do **not** hard-code a tag from memory). -This is a `vllm` leg, so use the single fully-qualified `docker.io/...` tag on the -skill's `vllm` row (it is architecture-independent — no GPU detection needed). Read the -exact tag **from the skill file** (the demo skill's SKILL.md "Suggested Docker images" -section) and export it as `HYPERLOOM_IMAGE` for the `docker run` below. +`HYPERLOOM_IMAGE` is not preset. You **MUST** use the **exact, verbatim** `docker.io/...` +tag string that is written on the demo skill's `vllm` row in its **"Suggested Docker +images"** section — nothing else. This is a hard release-gate constraint, not a +suggestion: + +- **Do NOT freelance the tag.** Do not bump the version, do not pick a "newer" or + "latest" build, do not substitute a different tag from your memory, from Docker Hub, or + from anywhere other than the skill file. The pinned tag is the one the release is gated + on; a different tag is a **failure**, even if it also pulls successfully. +- Read the tag by extracting it **from the skill file itself** rather than typing it out, + e.g. (the demo skill's path is in `HYPERLOOM_SKILL_PATH` in `.env`; otherwise it is the + `SKILL.md` of the demo skill you are running): + + ```bash + HYPERLOOM_IMAGE="$(grep -E '^- `vllm`' "$HYPERLOOM_SKILL_PATH" | grep -oE 'docker\.io/[^`]+' | head -1)" + echo "using image: $HYPERLOOM_IMAGE" + ``` + + This is a `vllm` leg, so use the single arch-independent `vllm` row — no GPU detection + is needed. +- If that command yields an empty string, or the resulting image cannot be pulled, **stop + and report the failure** — do **not** substitute any other tag to work around it. + +Export the extracted value as `HYPERLOOM_IMAGE` for the `docker run` below. ## Hard constraints (automated release gate) From ebcd2a1522d22bfca9d6ae0de17cca3af075f461 Mon Sep 17 00:00:00 2001 From: lishuoshuo-amd Date: Fri, 28 Aug 2026 13:18:07 +0800 Subject: [PATCH 28/52] pre-release-e2e: dedup docker layers (drop vfs) + stop reaping live legs Run 1.0.1a0.dev202608280354+ci failed 6 of 8 legs on two independent causes, both now closed with on-cluster evidence. Docker host EVICTED again ("ephemeral local storage usage exceeds the total limit of containers 1792Gi", ~4min after start, exit 143). Moving the vfs data-root to /shared-data changed nothing: the pod spec shows `shared-data: {"emptyDir":{}}`, and an emptyDir is part of the pod's local ephemeral storage, so it counts against the very quota it was meant to dodge. In the pod `df /shared-data` and `df /` report the identical 28T because both are backed by the same node NVMe, which is what made it look like a hostPath. There is no large hostPath to escape to either: /primus is 123G ext4 on one node and a 12T NFS mount on another. The real cost is vfs itself -- no layer dedup, so every layer and every container is a full copy. /shared-data is a plain xfs mount, not an overlay, and DOES accept an overlay upperdir (probe-verified on a privileged pod: `mount -t overlay` with upperdir/workdir there returns MOUNT_OK, so xfs ftype=1 holds). The original "overlayfs-on-overlayfs fails" reasoning only ever applied to the container rootfs. So try overlay2, fall back to fuse-overlayfs, and FAIL if neither is available -- vfs is not a fallback, it is a guaranteed eviction, and a silent downgrade just reproduces the bug. Each driver gets its own data-root subdir (dockerd refuses a data-root holding another driver's tree) and the effective driver is logged. Both baremetal-sglang legs exited 1 from run_leg's stall check while demonstrably alive. The check measured absolute file mtimes under $session only, so it charged the minutes spent inside the two non-streaming `claude --print` turns to the leg and condemned it on the first loop iteration: each kill lands exactly at last-$session-write + 600s (03:55:38 -> 04:06:07 and 04:03:15 -> 04:13:46). The 12h leg had written $root/setup_sglang_retry.log 26s before being declared hung -- the agent's launcher and setup/install logs live next to the workspace, not under session/, and install.sh writes $root/.cache. Extract the arithmetic into leg_idle_s(root, loop_start, now): scope the liveness scan to $root and measure idleness from the LATER of last write and loop start, so a slow build still keeps the leg alive and the pre-loop gap is never charged to it, while a genuinely hung launch is still reaped in ~10m. Also mirror both agent turns to $session/agent-.log. SaFE deletes a failed leg's PyTorchJob immediately, taking the pod stdout with it; 3 of this run's 5 workloads left no recoverable log and the root cause had to be reconstructed from file mtimes. Add test_pre_release_stall_liveness.py, which runs the real leg_idle_s out of the script (pre-fix logic answers 629s on the observed case, post-fix 5s) and pins that vfs never returns as a storage driver. Co-authored-by: Cursor --- .github/pre-release/bootstrap-pre-release.sh | 167 +++++++++++------- .github/scripts/pre-release-e2e-dispatch.sh | 8 +- .../tests/test_pre_release_stall_liveness.py | 140 +++++++++++++++ 3 files changed, 246 insertions(+), 69 deletions(-) create mode 100644 src/hyperloom/inference_optimizer/tests/test_pre_release_stall_liveness.py diff --git a/.github/pre-release/bootstrap-pre-release.sh b/.github/pre-release/bootstrap-pre-release.sh index 0789b163fd..3da452f79a 100755 --- a/.github/pre-release/bootstrap-pre-release.sh +++ b/.github/pre-release/bootstrap-pre-release.sh @@ -70,6 +70,18 @@ install_claude_cli() { log "claude CLI installed: $(claude --version 2>/dev/null || true)" } +# Seconds a leg has been idle: time since the newest write anywhere under the leg root, +# floored at the wait loop's start so the preceding `claude --print` turns (which can +# leave the tree untouched for longer than the stall grace) are never charged to the leg. +# Args: leg_root loop_start_epoch now_epoch +leg_idle_s() { + local leg_root="$1" loop_start="$2" now="$3" last since + last="$(find "$leg_root" -type f -printf '%T@\n' 2>/dev/null | sort -rn | head -n1)" + since="${last%.*}" + [ -n "$since" ] && [ "$since" -ge "$loop_start" ] 2>/dev/null || since="$loop_start" + echo $(( now - since )) +} + # Run ONE leg to completion inside the current filesystem (baremetal pod, or already # inside a nested docker container). Args: leg backend model_path hours run_mode run_leg() { @@ -205,10 +217,14 @@ run_leg() { # IS_SANDBOX=1: the SaFE pod runs as root, and claude refuses to skip permissions under # root unless IS_SANDBOX=1 (SWSPLAT-42390) -- Hyperloom's own kernel-agent sets the same. export IS_SANDBOX=1 - log "claude --print (setup)" - claude --print --dangerously-skip-permissions < "$setup_prompt" + # Mirror both turns onto NFS. SaFE deletes the PyTorchJob as soon as a leg fails, taking + # the pod's stdout with it, so without this the agent's own account of the failure is + # unrecoverable and a post-mortem is left reconstructing events from file mtimes. + local agent_log="${session}/agent-${leg}.log" + log "claude --print (setup); agent transcript -> $agent_log" + claude --print --dangerously-skip-permissions < "$setup_prompt" 2>&1 | tee -a "$agent_log" log "claude --print (demo ${hours}h)" - claude --print --dangerously-skip-permissions < "$demo_prompt" + claude --print --dangerously-skip-permissions < "$demo_prompt" 2>&1 | tee -a "$agent_log" log "leg $leg demo turn returned; waiting for the background optimize to finish" # ---- wait for the backgrounded `optimize` to reach a terminal state -------- @@ -224,16 +240,19 @@ run_leg() { # dir by globbing the newest one under $session that contains a state.json, and # re-point the poll's pin (.session_dir) at it. local wait_interval="${LEG_WAIT_INTERVAL_S:-45}" - # Liveness, NOT a wall-clock startup budget. The prior fixed "optimize must produce a - # state.json within N seconds" judged LIVE legs dead: baremetal SGLang builds from - # source (gfx950, py3.12), starts a Ray head, then loads the server -- >15m before the - # first state.json, and `optimize` isn't even a process yet during the build, so a pid - # check can't tell "not launched yet" from "died". Instead we ask "how long since ANY - # file under $session was last written?": USER_DATA_PATH=$session, so the build/runtime - # logs, Ray output, and session artifacts all land under this tree and keep its mtime - # fresh while anything is making progress. A leg is dead only if the tree has been - # completely IDLE for stall_grace AND no state.json exists yet -- this waits out slow - # builds (as long as they keep writing) but reaps a truly hung/exited launch quickly. + # Liveness, NOT a wall-clock startup budget: a leg is dead only once NOTHING under the + # leg root has been written for stall_grace and no state.json exists yet. This waits out + # arbitrarily slow builds (baremetal SGLang compiles from source and starts a Ray head, + # >15m before the first state.json) while still reaping a hung launch within ~10m. + # Two properties matter, both learned from legs that were killed while demonstrably + # alive (2026-08-28): + # * scope is $root, not $session -- the agent's launcher and its setup/install logs + # land next to the workspace, and install.sh writes $root/.cache. Watching only + # $session missed all of it and reaped a leg 26s after it last wrote a file. + # * idleness is measured from the LATER of (last write, loop start) -- the two + # `claude --print` turns can leave the tree untouched for longer than stall_grace, + # and that pre-loop gap must not be charged to the leg, or the very first + # iteration condemns it. local stall_grace="${LEG_STALL_GRACE_S:-600}" # 10m of NO file writes -> dead local final_grace="${LEG_FINAL_GRACE_S:-120}" # state stop_reason -> final.json local deadline_s=$(( hours * 3600 + 3600 )) # demo budget + 1h margin (hard cap) @@ -254,15 +273,10 @@ run_leg() { # Re-pin so the poll (leg_session_dir -> head -n1 .session_dir) finds the report. echo "$real_sdir" > "${session}/.session_dir" else - # No state.json yet -> judge liveness by how long since ANY file under $session - # was written. newest mtime across the tree; if the tree is empty, fall back to - # $session's own mtime so a brand-new leg isn't reaped on its first iteration. - local last_write idle - last_write="$(find "$session" -type f -printf '%T@\n' 2>/dev/null | sort -rn | head -n1)" - [ -n "$last_write" ] || last_write="$(stat -c %Y "$session" 2>/dev/null || echo "$now")" - idle=$(( now - ${last_write%.*} )) + local idle + idle="$(leg_idle_s "$root" "$start_ts" "$now")" if [ "$idle" -ge "$stall_grace" ]; then - log "ERROR: leg $leg -- no state.json and no file written under $session for ${idle}s (>= ${stall_grace}s stall; build/optimize hung or exited)" + log "ERROR: leg $leg -- no state.json and no file written under $root for ${idle}s (>= ${stall_grace}s stall; build/optimize hung or exited)" return 1 fi fi @@ -296,12 +310,51 @@ run_leg() { done } -# Install docker + start a pod-local dockerd. VERIFIED on a real privileged MI355X pod -# (2026-08-27): the Authoring base image ships NO docker/dockerd/docker.sock, but the -# pod has full capabilities (CapEff=0x1ffffffffff), so a self-hosted dockerd works. -# Two non-obvious requirements, both confirmed by probing: -# * --storage-driver=vfs -- overlayfs-on-overlayfs fails inside the container rootfs. -# * no systemd in the pod -- start dockerd detached via setsid, then poll the socket. +# Layer-deduplicating storage drivers, tried in order. vfs is deliberately NOT a +# fallback: it copies every image layer and every container in full, which grew the +# host pod past its ephemeralStorage limit and got it EVICTED within minutes +# ("ephemeral local storage usage exceeds the total limit of containers 1792Gi", +# observed live 2026-08-28). Running on vfs is a guaranteed eviction, so a leg that +# cannot get a deduplicating driver must fail loudly instead. +DOCKER_DRIVERS="${DOCKER_DRIVERS:-overlay2 fuse-overlayfs}" + +# Start a detached dockerd on one driver; 0 when the socket answers. Cleans up the +# failed daemon so the next driver starts from a clean socket. +start_dockerd_with_driver() { + local driver="$1" data_root="$2" dlog="/var/log/dockerd-${1}.log" i + mkdir -p "$data_root" || return 1 + log "starting pod-local dockerd (driver=$driver, data-root=$data_root)" + setsid bash -c "dockerd --host=unix:///var/run/docker.sock --storage-driver='$driver' --data-root='$data_root' >'$dlog' 2>&1" \ + /dev/null 2>&1 & + for i in $(seq 1 60); do + docker info >/dev/null 2>&1 && { log "dockerd up after ${i}s (driver=$driver)"; return 0; } + sleep 1 + done + log "WARN: dockerd did not become ready with driver=$driver" + tail -20 "$dlog" 2>/dev/null || true + pkill -f 'dockerd --host=unix:///var/run/docker.sock' 2>/dev/null || true + sleep 3 + rm -f /var/run/docker.sock 2>/dev/null || true + return 1 +} + +# fuse-overlayfs is the userspace fallback when the kernel refuses overlay2 on the +# data-root filesystem. Needs the binary plus /dev/fuse. +ensure_fuse_overlayfs() { + command -v fuse-overlayfs >/dev/null 2>&1 || { + log "installing fuse-overlayfs" + export DEBIAN_FRONTEND=noninteractive + apt-get update -qq >>/tmp/apt-docker.log 2>&1 || true + apt-get install -y -qq fuse-overlayfs >>/tmp/apt-docker.log 2>&1 \ + || { log "WARN: fuse-overlayfs install failed"; return 1; } + } + [ -c /dev/fuse ] || { log "WARN: /dev/fuse missing; fuse-overlayfs unusable"; return 1; } +} + +# Install docker + start a pod-local dockerd. VERIFIED on a real privileged MI355X pod: +# the Authoring base image ships NO docker/dockerd/docker.sock, but the pod has full +# capabilities (CapEff=0x1ffffffffff), so a self-hosted dockerd works. There is no +# systemd in the pod, hence setsid + socket polling rather than a service start. ensure_dockerd() { if docker info >/dev/null 2>&1; then log "dockerd already up"; return 0; fi if ! command -v dockerd >/dev/null 2>&1; then @@ -312,50 +365,34 @@ ensure_dockerd() { || { log "ERROR: docker.io install failed"; tail -30 /tmp/apt-docker.log; return 1; } log "docker installed: $(docker --version 2>&1)" fi - # --- pick a data-root that does NOT count against the pod's ephemeral limit --- - # The vfs storage driver has NO layer dedup: every image layer + every container is a - # full copy. Left at the default /var/lib/docker (inside the container ROOTFS), 4 legs - # pulling+running big ROCm images blow past the pod's ephemeralStorage quota and the pod - # is EVICTED ("ephemeral local storage usage exceeds the total limit of containers - # 1792Gi" -- observed live 2026-08-28). This is a PRIVILEGED host pod, so it sees the - # NODE's own disks directly: /shared-data is the node-local NVMe (~28T xfs on - # /dev/mapper/nvme_vg-nvme_lv), a real host filesystem -- NOT the container rootfs - # overlay -- so bytes written there are NOT counted toward the pod's ephemeralStorage - # quota. Point the vfs data-root there so image/container copies land on the big host - # disk instead of the small pod-ephemeral quota. - # - # /shared-data is host-persistent (survives this pod), so vfs copies from OLD runs would - # accumulate and eventually fill the node NVMe. Reap stale e2e-docker data-roots (any - # run other than this CI_VERSION) before starting dockerd. dockerd is not up yet, so a - # plain rm of the old data-root dirs is safe. + # --- data-root must sit on a filesystem that can back a deduplicating driver --- + # The container rootfs (/) is itself an overlay, and overlayfs cannot use an overlay + # upperdir -- that is why this used to run on vfs. /shared-data is a plain xfs mount + # (/dev/mapper/nvme_vg-nvme_lv) and DOES accept an overlay upperdir (probe-verified: + # `mount -t overlay` with upperdir/workdir under /shared-data succeeds, so xfs ftype=1 + # holds). It is an emptyDir, so it still counts toward the pod's ephemeralStorage quota + # and dies with the pod -- the quota headroom comes from the driver's layer dedup, not + # from the location. local docker_data_root="/var/lib/docker" - local dr_base="/shared-data/e2e-docker" - local dr_candidate="${dr_base}/${CI_VERSION:-current}" - if [ -d /shared-data ] && mkdir -p "$dr_candidate" 2>/dev/null && touch "$dr_candidate/.wtest" 2>/dev/null; then + local dr_candidate="/shared-data/e2e-docker/${CI_VERSION:-current}" + if mkdir -p "$dr_candidate" 2>/dev/null && touch "$dr_candidate/.wtest" 2>/dev/null; then rm -f "$dr_candidate/.wtest" 2>/dev/null || true - # reap other runs' leftovers (best-effort; never abort the run on cleanup failure) - if [ -d "$dr_base" ]; then - for d in "$dr_base"/*; do - [ -d "$d" ] || continue - [ "$d" = "$dr_candidate" ] && continue - log "reaping stale docker data-root from an older run: $d" - rm -rf "$d" 2>/dev/null || true - done - fi docker_data_root="$dr_candidate" - log "dockerd data-root on node-local NVMe: $docker_data_root (vfs copies stay off the pod ephemeral quota)" else - log "WARN: /shared-data not writable; dockerd falls back to $docker_data_root (may hit the pod ephemeral limit under vfs)" + log "WARN: /shared-data not writable; dockerd falls back to $docker_data_root (overlay-on-overlay, overlay2 will likely be refused)" fi - log "starting pod-local dockerd (vfs, data-root=$docker_data_root, detached via setsid)" - setsid bash -c "dockerd --host=unix:///var/run/docker.sock --storage-driver=vfs --data-root='$docker_data_root' >/var/log/dockerd.log 2>&1" \ - /dev/null 2>&1 & - local i - for i in $(seq 1 60); do - docker info >/dev/null 2>&1 && { log "dockerd up after ${i}s"; return 0; } - sleep 1 + # Each driver gets its own subdir: dockerd refuses a data-root that already holds a + # different driver's tree. + local driver + for driver in $DOCKER_DRIVERS; do + if [ "$driver" = fuse-overlayfs ]; then ensure_fuse_overlayfs || continue; fi + if start_dockerd_with_driver "$driver" "${docker_data_root}/${driver}"; then + log "docker storage driver in use: $(docker info -f '{{.Driver}}' 2>/dev/null)" + return 0 + fi done - log "ERROR: dockerd did not become ready"; tail -40 /var/log/dockerd.log 2>/dev/null || true + log "ERROR: no layer-deduplicating docker storage driver available (tried: $DOCKER_DRIVERS)." + log "ERROR: refusing to fall back to vfs -- it has no layer dedup and evicts the pod on ephemeralStorage." return 1 } diff --git a/.github/scripts/pre-release-e2e-dispatch.sh b/.github/scripts/pre-release-e2e-dispatch.sh index de2fb0bcc6..45fbfdded6 100755 --- a/.github/scripts/pre-release-e2e-dispatch.sh +++ b/.github/scripts/pre-release-e2e-dispatch.sh @@ -57,10 +57,10 @@ set -euo pipefail NFS_ROOT="${NFS_ROOT:-/shared_nfs/hyperloom-pre-release-e2e-test}" TARGET_GAIN="${TARGET_GAIN:-100}" # Sized to a proven Running 8-GPU Authoring pod (ref: sglang-kimik3-2): CPU 128, -# mem 2048Gi, ephemeral 1792Gi. The privileged DinD host runs nested docker with the -# vfs storage driver (no layer dedup), so 8 ROCm images x full copies blow past a small -# ephemeral limit -- our first run was EVICTED at ephemeralStorage 200Gi. 1792Gi matches -# the reference host and leaves headroom for vfs image blowup + model/wheel scratch. +# mem 2048Gi, ephemeral 1792Gi. Every writable path the DinD host has -- the container +# rootfs AND the /shared-data emptyDir the nested dockerd stores images in -- counts +# toward this one ephemeralStorage quota, so the host bootstrap requires a +# layer-deduplicating docker storage driver (overlay2) to stay inside it. HOST_CPU="${HOST_CPU:-128}"; HOST_MEM="${HOST_MEM:-2048Gi}"; HOST_SHM="${HOST_SHM:-256Gi}" HOST_EPHEMERAL="${HOST_EPHEMERAL:-1792Gi}" LEG_CPU="${LEG_CPU:-32}"; LEG_MEM="${LEG_MEM:-128Gi}" diff --git a/src/hyperloom/inference_optimizer/tests/test_pre_release_stall_liveness.py b/src/hyperloom/inference_optimizer/tests/test_pre_release_stall_liveness.py new file mode 100644 index 0000000000..f9c32a8ddf --- /dev/null +++ b/src/hyperloom/inference_optimizer/tests/test_pre_release_stall_liveness.py @@ -0,0 +1,140 @@ +# SPDX-FileCopyrightText: 2026 Advanced Micro Devices, Inc. +# SPDX-License-Identifier: MIT + +"""Regression guard for the pre-release E2E leg liveness (stall) check. + +``bootstrap-pre-release.sh`` blocks after the demo turn until ``optimize`` writes +``reports/final.json``, and declares a leg dead when nothing has been written for +``LEG_STALL_GRACE_S``. Two properties of that check killed legs that were provably +still alive (run 1.0.1a0.dev202608280354+ci, both baremetal-sglang legs): + +* the idle window was measured against absolute file mtimes, so the minutes spent + inside the two non-streaming ``claude --print`` turns were charged to the leg and + the very first loop iteration condemned it; +* only ``$session`` was watched, while the agent's launcher, its setup/install logs + and ``install.sh``'s caches land elsewhere under the leg root -- one leg was reaped + 26s after it last wrote a file. + +These tests exercise the real ``leg_idle_s`` helper out of the script. +""" + +from __future__ import annotations + +import os +import subprocess +import time +from pathlib import Path + +import pytest + +_STALL_GRACE_S = 600 + + +def _find_bootstrap() -> Path | None: + """Locate the in-pod bootstrap; None when running from an installed wheel.""" + for parent in Path(__file__).resolve().parents: + candidate = parent / ".github" / "pre-release" / "bootstrap-pre-release.sh" + if candidate.is_file(): + return candidate + return None + + +_BOOTSTRAP = _find_bootstrap() + +pytestmark = pytest.mark.skipif( + _BOOTSTRAP is None, + reason="pre-release liveness guard needs the source checkout (.github/pre-release/)", +) + + +@pytest.fixture(scope="module") +def script() -> str: + assert _BOOTSTRAP is not None + return _BOOTSTRAP.read_text(encoding="utf-8") + + +@pytest.fixture(scope="module") +def leg_idle_fn(script: str) -> str: + """Slice the ``leg_idle_s`` function out of the script so it can be run alone.""" + lines = script.splitlines() + start = next(i for i, line in enumerate(lines) if line.startswith("leg_idle_s() {")) + end = next(i for i, line in enumerate(lines[start:], start) if line == "}") + return "\n".join(lines[start : end + 1]) + + +def _idle(leg_idle_fn: str, root: Path, loop_start: int, now: int) -> int: + """Run leg_idle_s(root, loop_start, now) in bash and return its answer.""" + proc = subprocess.run( + ["bash", "-c", f'{leg_idle_fn}\nleg_idle_s "$1" "$2" "$3"', "_", str(root), str(loop_start), str(now)], + capture_output=True, + text=True, + check=True, + ) + return int(proc.stdout.strip()) + + +def _write(path: Path, age_s: int, now: int) -> None: + path.parent.mkdir(parents=True, exist_ok=True) + path.write_text("x", encoding="utf-8") + stamp = now - age_s + os.utime(path, (stamp, stamp)) + + +def test_pre_loop_gap_is_not_charged_to_the_leg(leg_idle_fn: str, tmp_path: Path) -> None: + """The observed kill: nothing written for 629s, but the loop just started.""" + now = int(time.time()) + _write(tmp_path / "session" / ".session_dir", age_s=629, now=now) + idle = _idle(leg_idle_fn, tmp_path, loop_start=now - 5, now=now) + assert idle == 5 + assert idle < _STALL_GRACE_S + + +def test_writes_outside_the_session_dir_count_as_progress(leg_idle_fn: str, tmp_path: Path) -> None: + """The agent's own logs live next to the workspace, not under session/.""" + now = int(time.time()) + loop_start = now - 700 + _write(tmp_path / "session" / ".session_dir", age_s=700, now=now) + _write(tmp_path / "setup_sglang_retry.log", age_s=26, now=now) + idle = _idle(leg_idle_fn, tmp_path, loop_start=loop_start, now=now) + assert idle == 26 + assert idle < _STALL_GRACE_S + + +def test_a_genuinely_idle_tree_is_still_reaped(leg_idle_fn: str, tmp_path: Path) -> None: + """A hung launch must still be caught once the loop itself has waited it out.""" + now = int(time.time()) + _write(tmp_path / "session" / ".session_dir", age_s=900, now=now) + idle = _idle(leg_idle_fn, tmp_path, loop_start=now - 900, now=now) + assert idle == 900 + assert idle >= _STALL_GRACE_S + + +def test_empty_tree_falls_back_to_the_loop_start(leg_idle_fn: str, tmp_path: Path) -> None: + now = int(time.time()) + assert _idle(leg_idle_fn, tmp_path, loop_start=now - 42, now=now) == 42 + + +def test_stall_check_watches_the_leg_root(script: str) -> None: + """Pin the call site: the check must be scoped to $root, not $session.""" + assert 'idle="$(leg_idle_s "$root" "$start_ts" "$now")"' in script + assert 'no file written under $root' in script + + +def test_agent_turns_are_mirrored_to_nfs(script: str) -> None: + """SaFE deletes a failed leg's pod, so the agent transcript must reach NFS.""" + assert 'agent_log="${session}/agent-${leg}.log"' in script + assert script.count('tee -a "$agent_log"') == 2 + + +def test_dockerd_never_runs_on_vfs(script: str) -> None: + """vfs copies every layer in full and evicted the host pod twice (200Gi, 1792Gi). + + Only deduplicating drivers may be attempted, and a leg with none available has to + fail rather than quietly reproduce the eviction. + """ + assert "--storage-driver=vfs" not in script + drivers = next( + line for line in script.splitlines() if line.startswith("DOCKER_DRIVERS=") + ) + assert "overlay2" in drivers + assert "vfs" not in drivers From c11d626181373bb8527a889cbe8e5db683fcc4a7 Mon Sep 17 00:00:00 2001 From: lishuoshuo-amd Date: Fri, 28 Aug 2026 13:43:45 +0800 Subject: [PATCH 29/52] pre-release-e2e: re-drive agent turns that end before the work is done overlay2 fixed the docker-host eviction (200G plateau vs the 1792Gi limit, pod alive well past the ~4min eviction point) and the stall fix kept the sglang legs alive through a 5min source build. Run 1.0.1a0.dev202608280519+ci then exposed the next layer: most legs never launched `optimize` at all. `claude --print` is one turn, and the agent sometimes ends it with a progress note instead of finishing the job. Live transcripts: baremetal-vllm-3h setup: "Base tooling installed. Waiting on the vLLM ROCm wheel install." (never printed `setup complete: ...`) demo: "Good progress -- torch is in ... Waiting on the monitor." baremetal-vllm-12h demo: "Install step started. Waiting for it to complete." Everything the agent leaves running is a child of that turn, so it dies with it: both legs were left with no claude and no optimize process, and the stall check correctly reaped them 600s later. This is why the previous run's vllm legs passed and this one's did not -- there the demo turn happened to run 16min and got as far as `setsid nohup optimize`; here it returned after 2-4min. Non-deterministic agent behaviour, so the harness has to converge it rather than hope. Setup turn: the prompt's contract is a literal `setup complete: /` line, so grep for it and re-drive the prompt (LEG_TURN_ATTEMPTS, default 3) until the agent reports it, failing the leg only after that. Demo turn: re-drive off the EXISTING stall signal instead of probing "is it launched" right after the turn. The demo skill's launcher runs install.sh before backgrounding optimize, so state.json can legitimately be 10+ minutes out and a short-grace probe would double-launch; whereas "no state.json and nothing written under the leg root for 600s" already means nothing is running. On that signal, ask the agent to finish the job (LEG_DEMO_REDRIVES, default 2) before giving up. Split the wait loop's clocks so the re-drive cannot extend the pod deadline: start_ts still backs deadline_s and is never reassigned, while the new grace_ts backs the stall window and restarts after each re-driven turn. The invariant bootstrap deadline < SaFE pod timeout < poll GLOBAL_TIMEOUT_S is unchanged. Both demo prompts now state that the turn is single and non-interactive, that ending it on "install started" / "waiting on the monitor" kills the work, that optimize must be detached with setsid nohup, and that the run must be confirmed live (nested run dir + state.json + live PID) before the turn ends. Tests: leg_run_started is exercised against a real session tree (a stray state.json at the session top must not count as a launch), the setup marker is cross-checked against all four setup prompts so the grep and the prompts cannot drift, and the deadline clock is pinned against being reset by a re-drive. Co-authored-by: Cursor --- .github/pre-release/bootstrap-pre-release.sh | 53 ++++++++++++- .../prompts/pre-release/demo-12h.md | 21 +++-- .../prompts/pre-release/demo-3h.md | 21 +++-- .../tests/test_pre_release_stall_liveness.py | 76 ++++++++++++++++++- 4 files changed, 154 insertions(+), 17 deletions(-) diff --git a/.github/pre-release/bootstrap-pre-release.sh b/.github/pre-release/bootstrap-pre-release.sh index 3da452f79a..df853a8b7c 100755 --- a/.github/pre-release/bootstrap-pre-release.sh +++ b/.github/pre-release/bootstrap-pre-release.sh @@ -82,6 +82,14 @@ leg_idle_s() { echo $(( now - since )) } +# 0 once the leg has a live run: `optimize` creates a NESTED per-run dir under the +# session and writes state.json into it. Scoped to this leg's own session tree, so it +# stays correct on the shared docker host where four legs run side by side. +# Args: session_dir +leg_run_started() { + [ -n "$(find "$1" -mindepth 2 -type f -name state.json -print -quit 2>/dev/null)" ] +} + # Run ONE leg to completion inside the current filesystem (baremetal pod, or already # inside a nested docker container). Args: leg backend model_path hours run_mode run_leg() { @@ -221,8 +229,26 @@ run_leg() { # the pod's stdout with it, so without this the agent's own account of the failure is # unrecoverable and a post-mortem is left reconstructing events from file mtimes. local agent_log="${session}/agent-${leg}.log" - log "claude --print (setup); agent transcript -> $agent_log" - claude --print --dangerously-skip-permissions < "$setup_prompt" 2>&1 | tee -a "$agent_log" + # A `claude --print` turn can end EARLY with a progress note instead of finishing the + # job -- observed live: a setup turn whose whole answer was "Waiting on the vLLM ROCm + # wheel install", and a demo turn that reported "Install step started" and returned. + # Whatever the agent left running is a child of that turn and dies with it, so an early + # turn silently leaves the leg with nothing running at all. The setup prompt's contract + # is a literal `setup complete: /` line, so re-drive until we see it. + local attempts="${LEG_TURN_ATTEMPTS:-3}" i + for i in $(seq 1 "$attempts"); do + log "claude --print (setup, attempt $i/$attempts); agent transcript -> $agent_log" + claude --print --dangerously-skip-permissions < "$setup_prompt" 2>&1 | tee -a "$agent_log" + if grep -qiE "setup complete: ${run_mode}/${backend}" "$agent_log"; then + log "leg $leg setup reported complete on attempt $i" + break + fi + if [ "$i" -ge "$attempts" ]; then + log "ERROR: leg $leg -- setup never reported 'setup complete: ${run_mode}/${backend}' in $attempts turns" + return 1 + fi + log "WARN: leg $leg -- setup turn $i ended without completing; re-driving the setup prompt" + done log "claude --print (demo ${hours}h)" claude --print --dangerously-skip-permissions < "$demo_prompt" 2>&1 | tee -a "$agent_log" log "leg $leg demo turn returned; waiting for the background optimize to finish" @@ -256,8 +282,19 @@ run_leg() { local stall_grace="${LEG_STALL_GRACE_S:-600}" # 10m of NO file writes -> dead local final_grace="${LEG_FINAL_GRACE_S:-120}" # state stop_reason -> final.json local deadline_s=$(( hours * 3600 + 3600 )) # demo budget + 1h margin (hard cap) + # Two clocks: start_ts backs the hard deadline and is NEVER reset (resetting it would + # let the leg outlive the SaFE pod timeout and lose the clean failure path); grace_ts + # backs the stall window and restarts after each re-driven turn. local start_ts; start_ts="$(date +%s)" + local grace_ts="$start_ts" local real_sdir="" final_json="" state_json="" + # An idle tree with no state.json means the demo turn ended without leaving a running + # `optimize` behind. That is recoverable -- ask the agent to finish the job rather than + # failing the leg -- but only a bounded number of times. Deliberately driven off the + # stall signal instead of an "is it launched yet" probe right after the turn: the demo + # skill's launcher runs install.sh BEFORE backgrounding optimize, so state.json can + # legitimately be 10+ minutes away, and re-driving on a short grace would double-launch. + local demo_redrives=0 max_demo_redrives="${LEG_DEMO_REDRIVES:-2}" while :; do local now elapsed; now="$(date +%s)"; elapsed=$(( now - start_ts )) @@ -274,9 +311,17 @@ run_leg() { echo "$real_sdir" > "${session}/.session_dir" else local idle - idle="$(leg_idle_s "$root" "$start_ts" "$now")" + idle="$(leg_idle_s "$root" "$grace_ts" "$now")" if [ "$idle" -ge "$stall_grace" ]; then - log "ERROR: leg $leg -- no state.json and no file written under $root for ${idle}s (>= ${stall_grace}s stall; build/optimize hung or exited)" + if [ "$demo_redrives" -lt "$max_demo_redrives" ]; then + demo_redrives=$(( demo_redrives + 1 )) + log "WARN: leg $leg -- idle ${idle}s with no state.json; the demo turn left nothing running. Re-driving the demo prompt ($demo_redrives/$max_demo_redrives)" + claude --print --dangerously-skip-permissions < "$demo_prompt" 2>&1 | tee -a "$agent_log" + log "leg $leg demo re-drive $demo_redrives returned" + grace_ts="$(date +%s)" # fresh stall grace for the new turn; deadline unchanged + continue + fi + log "ERROR: leg $leg -- no state.json and no file written under $root for ${idle}s after $demo_redrives demo re-drive(s); giving up" return 1 fi fi diff --git a/.github/pre-release/prompts/pre-release/demo-12h.md b/.github/pre-release/prompts/pre-release/demo-12h.md index 13739f4d27..4e3f88ce2f 100644 --- a/.github/pre-release/prompts/pre-release/demo-12h.md +++ b/.github/pre-release/prompts/pre-release/demo-12h.md @@ -36,8 +36,19 @@ continue without asking. Load LLM API keys/base URLs and `FRAMEWORK` from `.env` - Do **not** modify `USER_DATA_PATH`. - Do **not** print or copy secret values into output, reports, or logs. -## Termination - -Let the run proceed to its terminal report (session `reports/final.json` + -`reports/final.md`). When it terminates, stop. The harness judges PASS/FAIL from the -session report — do not fabricate a result. +## Termination — do not end this turn until the run is launched + +This is a **single non-interactive turn**, and anything still running as a child of it is +killed the moment the turn ends. So: + +1. Finish the install and the launch **inside this turn**. Do **not** end the turn with a + progress note such as "install started", "waiting on the pull", or "waiting on the + monitor" — that kills the work you just started and the leg ends up with nothing + running at all. +2. Start `optimize` **detached** with `setsid nohup` (as the demo skill does) so it + survives the end of this turn. +3. Before you finish, confirm the run is really live and report the paths: the nested + session run dir exists, `state.json` is present in it, and the optimizer PID is alive. + +Only then stop. The harness then waits for the terminal report (`reports/final.json` + +`reports/final.md`) and judges PASS/FAIL from it — do not fabricate a result. diff --git a/.github/pre-release/prompts/pre-release/demo-3h.md b/.github/pre-release/prompts/pre-release/demo-3h.md index d6d4adb75c..4cb32668dc 100644 --- a/.github/pre-release/prompts/pre-release/demo-3h.md +++ b/.github/pre-release/prompts/pre-release/demo-3h.md @@ -37,8 +37,19 @@ asking. Load LLM API keys/base URLs and `FRAMEWORK` from `.env`. - Do **not** modify `USER_DATA_PATH`. - Do **not** print or copy secret values into output, reports, or logs. -## Termination - -Let the run proceed to its terminal report (session `reports/final.json` + -`reports/final.md`). When it terminates, stop. The harness judges PASS/FAIL from the -session report — do not fabricate a result. +## Termination — do not end this turn until the run is launched + +This is a **single non-interactive turn**, and anything still running as a child of it is +killed the moment the turn ends. So: + +1. Finish the install and the launch **inside this turn**. Do **not** end the turn with a + progress note such as "install started", "waiting on the pull", or "waiting on the + monitor" — that kills the work you just started and the leg ends up with nothing + running at all. +2. Start `optimize` **detached** with `setsid nohup` (as the demo skill does) so it + survives the end of this turn. +3. Before you finish, confirm the run is really live and report the paths: the nested + session run dir exists, `state.json` is present in it, and the optimizer PID is alive. + +Only then stop. The harness then waits for the terminal report (`reports/final.json` + +`reports/final.md`) and judges PASS/FAIL from it — do not fabricate a result. diff --git a/src/hyperloom/inference_optimizer/tests/test_pre_release_stall_liveness.py b/src/hyperloom/inference_optimizer/tests/test_pre_release_stall_liveness.py index f9c32a8ddf..e6bcaaa5f6 100644 --- a/src/hyperloom/inference_optimizer/tests/test_pre_release_stall_liveness.py +++ b/src/hyperloom/inference_optimizer/tests/test_pre_release_stall_liveness.py @@ -116,14 +116,84 @@ def test_empty_tree_falls_back_to_the_loop_start(leg_idle_fn: str, tmp_path: Pat def test_stall_check_watches_the_leg_root(script: str) -> None: """Pin the call site: the check must be scoped to $root, not $session.""" - assert 'idle="$(leg_idle_s "$root" "$start_ts" "$now")"' in script + assert 'idle="$(leg_idle_s "$root" "$grace_ts" "$now")"' in script assert 'no file written under $root' in script def test_agent_turns_are_mirrored_to_nfs(script: str) -> None: - """SaFE deletes a failed leg's pod, so the agent transcript must reach NFS.""" + """SaFE deletes a failed leg's pod, so every agent turn must reach NFS.""" assert 'agent_log="${session}/agent-${leg}.log"' in script - assert script.count('tee -a "$agent_log"') == 2 + # setup (retried), the demo turn, and each demo re-drive. + assert script.count('tee -a "$agent_log"') == 3 + + +def _leg_run_started(script: str, session: Path) -> bool: + """Run the real leg_run_started() against a session tree.""" + lines = script.splitlines() + start = next(i for i, line in enumerate(lines) if line.startswith("leg_run_started() {")) + end = next(i for i, line in enumerate(lines[start:], start) if line == "}") + fn = "\n".join(lines[start : end + 1]) + proc = subprocess.run( + ["bash", "-c", f'{fn}\nleg_run_started "$1"', "_", str(session)], + capture_output=True, + text=True, + ) + return proc.returncode == 0 + + +def test_launch_detection_needs_the_nested_run_dir(script: str, tmp_path: Path) -> None: + """optimize writes state.json into $session//-/, never at the top.""" + session = tmp_path / "session" + session.mkdir() + assert not _leg_run_started(script, session) + + # A stray state.json directly under the session must not count as a launch. + (session / "state.json").write_text("{}", encoding="utf-8") + assert not _leg_run_started(script, session) + + nested = session / "Qwen3-8B" / "20260828T053223Z-4351891f" + nested.mkdir(parents=True) + (nested / "state.json").write_text("{}", encoding="utf-8") + assert _leg_run_started(script, session) + + +def test_setup_marker_matches_every_setup_prompt(script: str) -> None: + """The retry gate greps a literal the prompts must actually ask the agent to print.""" + assert 'grep -qiE "setup complete: ${run_mode}/${backend}" "$agent_log"' in script + prompts = _BOOTSTRAP.parent / "prompts" / "pre-release" # type: ignore[union-attr] + for run_mode in ("baremetal", "docker"): + for backend in ("vllm", "sglang"): + prompt = prompts / f"setup-{run_mode}-{backend}.md" + assert f"setup complete: {run_mode}/{backend}" in prompt.read_text(encoding="utf-8") + + +def test_an_early_turn_is_re_driven_not_fatal(script: str) -> None: + """A turn that ends without finishing leaves nothing running; ask again, bounded.""" + assert 'attempts="${LEG_TURN_ATTEMPTS:-3}"' in script + assert 'max_demo_redrives="${LEG_DEMO_REDRIVES:-2}"' in script + assert 'demo_redrives=$(( demo_redrives + 1 ))' in script + + +def test_re_drive_does_not_extend_the_hard_deadline(script: str) -> None: + """The stall grace restarts per turn, but the pod-deadline clock must not. + + bootstrap's own deadline has to stay below the SaFE pod timeout, or SaFE pre-empts + the pod mid-wait and the clean failure path is lost. + """ + body = script.split("run_leg() {", 1)[1] + assert body.count('start_ts="$(date +%s)"') == 1 + assert 'grace_ts="$(date +%s)"' in body + assert 'elapsed=$(( now - start_ts ))' in body + assert 'idle="$(leg_idle_s "$root" "$grace_ts" "$now")"' in body + + +def test_demo_prompts_forbid_ending_the_turn_early(script: str) -> None: + prompts = _BOOTSTRAP.parent / "prompts" / "pre-release" # type: ignore[union-attr] + for hours in (3, 12): + text = (prompts / f"demo-{hours}h.md").read_text(encoding="utf-8") + assert "single non-interactive turn" in text + assert "setsid nohup" in text + assert "state.json" in text def test_dockerd_never_runs_on_vfs(script: str) -> None: From 4cbe0381206167e077e7bca7524bc02cf43cb42f Mon Sep 17 00:00:00 2001 From: lishuoshuo-amd Date: Fri, 28 Aug 2026 14:25:38 +0800 Subject: [PATCH 30/52] pre-release-e2e: resume the agent conversation, budget setup in time, drop preempt Three things, all from run 1.0.1a0.dev202608280605+ci. 1. Setup was budgeted in TURNS, and that was my regression. `claude --print` is "print response and exit", and a framework install (vLLM ROCm wheel into an isolated venv, or SGLang compiled from source for gfx950) runs 10-30min -- far longer than one turn holds. So every agent backgrounds the install, polls its log, and eventually answers with a progress note, which ends the turn with setup incomplete. A cap of 3 turns amounted to ~4min of wall clock and killed two legs whose installs were provably still going: baremetal-vllm-3h died 3.5min in, mid "Phase 2 (isolated vLLM install) is underway" baremetal-sglang-12h died 5.7min in, mid "Still waiting on the kernel build" Budget it in time instead: LEG_SETUP_DEADLINE_S (45m) bounds the whole step, and the existing liveness signal decides failure -- a leg dies only once nothing has been written under the leg root for LEG_SETUP_STALL_S (10m), i.e. nothing is installing. LEG_SETUP_MAX_TURNS stays purely as a token-spend backstop. 2. Follow-up turns now RESUME the leg's conversation instead of re-feeding the prompt as a fresh turn. Re-feeding threw away everything the agent already knew, leaving it to rediscover which install it had launched and which log it was watching. `--session-id` opens a stable per-leg UUID (sha1 of leg+CI_VERSION, shaped into a v4 UUID -- distinct per leg, which matters on the docker host where four agents share one pod) and `--resume` continues it with a one-line nudge. The demo turn resumes the same session too, exactly as a human would keep one chat. Resume failure degrades to a standalone turn rather than failing the leg. That the detached install survives a turn boundary is verified: setup_vllm.log grew 71KB -> 77KB across one, and baremetal-vllm-12h converged to `setup complete: baremetal/vllm` on turn 3 and went on to launch optimize. 3. Remove the `preempt` job and pre-release-e2e-reap.sh. It could never work. A newer run cannot tear down an older one: while the concurrency group is held, GitHub keeps the new run at run-level `pending` with an EMPTY jobs array (run 33145644120 sat pending 20min with zero jobs), so no job of it exists to reclaim anything. And it ran on a GitHub-hosted runner while SAFE_API_BASE is an in-network NodePort, so every run logged `[preempt] could not list workloads; skipping reclaim` after a 30s curl timeout, having stopped nothing -- three runs checked, all identical. Keeping it only made the gate look protected. Teardown runs the other way round instead: poll fails fast on the first FAIL (POLL_FAIL_FAST, default on). It is a release gate -- the first FAIL already blocks the release, so waiting the rest out buys nothing while costing a 12h leg's GPUs and the only self-hosted runner, which is what left the next fix's run stuck at `pending` until the old run was cancelled by hand. Remaining legs are recorded as not judged rather than FAIL. Poll also sleeps in POLL_SLEEP_SLICE_S slices so a cancel lands in seconds instead of at the end of a full interval. Leaked pods stay covered by the dispatch-side reap, which -- unlike preempt -- runs in-network and only executes once this run holds the runner, i.e. once the previous run is provably done. Tests: setup budget must not be turn-based, follow-ups must resume, the session uuid is checked for shape/stability/uniqueness, all agent invocations must go through the single teeing helper, and the orchestration guard pins that no job touching SaFE runs on a GitHub-hosted runner and that fail-fast is armed on every FAIL path. Co-authored-by: Cursor --- .github/pre-release/bootstrap-pre-release.sh | 101 +++++++++++++--- .github/scripts/pre-release-e2e-poll.sh | 34 +++++- .github/scripts/pre-release-e2e-reap.sh | 78 ------------ .github/workflows/pre-release-e2e-test.yml | 58 +++------ .../test_pre_release_gate_orchestration.py | 114 ++++++++++++++++++ .../tests/test_pre_release_stall_liveness.py | 66 +++++++++- 6 files changed, 309 insertions(+), 142 deletions(-) delete mode 100644 .github/scripts/pre-release-e2e-reap.sh create mode 100644 src/hyperloom/inference_optimizer/tests/test_pre_release_gate_orchestration.py diff --git a/.github/pre-release/bootstrap-pre-release.sh b/.github/pre-release/bootstrap-pre-release.sh index df853a8b7c..205a8c949e 100755 --- a/.github/pre-release/bootstrap-pre-release.sh +++ b/.github/pre-release/bootstrap-pre-release.sh @@ -90,6 +90,30 @@ leg_run_started() { [ -n "$(find "$1" -mindepth 2 -type f -name state.json -print -quit 2>/dev/null)" ] } +# A stable per-leg conversation id, so a follow-up turn can --resume the SAME session +# instead of starting from scratch. `--session-id` wants a UUID, so shape a sha1 of +# leg+CI_VERSION into one (version nibble 4, variant nibble 8). +# Args: leg ci_version +leg_session_uuid() { + local h; h="$(printf '%s|%s' "$1" "$2" | sha1sum | cut -c1-32)" + printf '%s-%s-4%s-8%s-%s' "${h:0:8}" "${h:8:4}" "${h:12:3}" "${h:15:3}" "${h:18:12}" +} + +# One agent turn, mirrored to the leg's NFS transcript so it survives pod deletion. +# Args: agent_log [claude flags...] (the prompt/nudge arrives on stdin) +agent_turn() { + local alog="$1"; shift + claude --print --dangerously-skip-permissions "$@" 2>&1 | tee -a "$alog" +} + +# Follow-up nudges for a conversation that ended before finishing. Deliberately short: +# the resumed session still holds the context (what was launched, which log is being +# watched), so restating the whole prompt would only add noise -- and re-feeding it as a +# FRESH turn is worse, because the agent then has to rediscover all of that. +SETUP_RESUME_NUDGE='Your previous turn ended before setup finished. Do NOT restart anything that is already running. Check on the install you launched: if it is still in progress, keep monitoring it and only answer once it has finished. Once it has finished successfully, complete any remaining setup steps and then reply with exactly the completion line the setup instructions asked for. If it has failed, report the failure.' + +DEMO_RESUME_NUDGE='Your previous turn ended without leaving a running optimize behind, and nothing has been written under the workspace since, so the work is not progressing. Do NOT fabricate a result. Finish the launch in THIS turn: complete the install if it is still needed, start optimize detached with setsid nohup so it survives the end of this turn, then confirm the nested session run dir and its state.json exist and report their paths.' + # Run ONE leg to completion inside the current filesystem (baremetal pod, or already # inside a nested docker container). Args: leg backend model_path hours run_mode run_leg() { @@ -229,28 +253,66 @@ run_leg() { # the pod's stdout with it, so without this the agent's own account of the failure is # unrecoverable and a post-mortem is left reconstructing events from file mtimes. local agent_log="${session}/agent-${leg}.log" - # A `claude --print` turn can end EARLY with a progress note instead of finishing the - # job -- observed live: a setup turn whose whole answer was "Waiting on the vLLM ROCm - # wheel install", and a demo turn that reported "Install step started" and returned. - # Whatever the agent left running is a child of that turn and dies with it, so an early - # turn silently leaves the leg with nothing running at all. The setup prompt's contract - # is a literal `setup complete: /` line, so re-drive until we see it. - local attempts="${LEG_TURN_ATTEMPTS:-3}" i - for i in $(seq 1 "$attempts"); do - log "claude --print (setup, attempt $i/$attempts); agent transcript -> $agent_log" - claude --print --dangerously-skip-permissions < "$setup_prompt" 2>&1 | tee -a "$agent_log" + local uuid; uuid="$(leg_session_uuid "$leg" "$CI_VERSION")" + # `claude --print` is "print response and exit": ONE answer per invocation. Setup here + # means installing a framework layer -- a vLLM ROCm wheel into an isolated venv, or + # SGLang compiled from source for gfx950 -- which takes 10-30min, far longer than a + # single turn can hold. Every leg's agent therefore backgrounds the install and polls + # its log, and eventually answers with a progress note ("Phase 2 is installing the + # isolated vLLM env. Still running.") which ENDS the turn with setup incomplete. + # + # The install itself is detached and keeps running (verified: setup_vllm.log grew from + # 71KB to 77KB across a turn boundary), so the fix is to give the agent another turn -- + # RESUMING the same conversation, so it still knows what it started and where the log + # is. Budget it in TIME, not turns: an earlier count-based cap of 3 turns amounted to + # ~4min of wall clock and killed two legs whose installs were demonstrably still + # progressing. A leg only fails here if the tree goes quiet (nothing installing) or the + # whole setup deadline elapses. + local setup_deadline_s="${LEG_SETUP_DEADLINE_S:-2700}" # 45m of setup, then give up + local setup_stall_s="${LEG_SETUP_STALL_S:-600}" # 10m with no writes -> dead + local setup_max_turns="${LEG_SETUP_MAX_TURNS:-30}" # token-spend backstop + local setup_t0; setup_t0="$(date +%s)" + local turn=0 snow sidle + while :; do + turn=$(( turn + 1 )) + if [ "$turn" = 1 ]; then + log "claude --print (setup, turn 1, session $uuid); agent transcript -> $agent_log" + agent_turn "$agent_log" --session-id "$uuid" < "$setup_prompt" + else + log "claude --print (setup, turn $turn, resuming session $uuid)" + if ! printf '%s\n' "$SETUP_RESUME_NUDGE" | agent_turn "$agent_log" --resume "$uuid"; then + log "WARN: leg $leg -- could not resume session $uuid; re-feeding the full setup prompt" + agent_turn "$agent_log" < "$setup_prompt" || true + fi + fi if grep -qiE "setup complete: ${run_mode}/${backend}" "$agent_log"; then - log "leg $leg setup reported complete on attempt $i" + log "leg $leg setup reported complete on turn $turn" break fi - if [ "$i" -ge "$attempts" ]; then - log "ERROR: leg $leg -- setup never reported 'setup complete: ${run_mode}/${backend}' in $attempts turns" + snow="$(date +%s)" + sidle="$(leg_idle_s "$root" "$setup_t0" "$snow")" + if [ "$sidle" -ge "$setup_stall_s" ]; then + log "ERROR: leg $leg -- setup turn $turn ended early and nothing was written under $root for ${sidle}s; the install is not progressing" + return 1 + fi + if [ $(( snow - setup_t0 )) -ge "$setup_deadline_s" ]; then + log "ERROR: leg $leg -- setup never reported 'setup complete: ${run_mode}/${backend}' within ${setup_deadline_s}s ($turn turns)" return 1 fi - log "WARN: leg $leg -- setup turn $i ended without completing; re-driving the setup prompt" + if [ "$turn" -ge "$setup_max_turns" ]; then + log "ERROR: leg $leg -- setup still incomplete after $setup_max_turns turns; giving up" + return 1 + fi + log "WARN: leg $leg -- setup turn $turn ended early (install still progressing, idle ${sidle}s); resuming the conversation" + sleep "${LEG_TURN_GAP_S:-30}" done - log "claude --print (demo ${hours}h)" - claude --print --dangerously-skip-permissions < "$demo_prompt" 2>&1 | tee -a "$agent_log" + log "claude --print (demo ${hours}h, resuming session $uuid)" + # Same conversation as setup: the agent already knows this workspace, which framework + # got installed and where its logs are, exactly like a human continuing the same chat. + if ! agent_turn "$agent_log" --resume "$uuid" < "$demo_prompt"; then + log "WARN: leg $leg -- could not resume session $uuid for the demo; running it standalone" + agent_turn "$agent_log" < "$demo_prompt" + fi log "leg $leg demo turn returned; waiting for the background optimize to finish" # ---- wait for the backgrounded `optimize` to reach a terminal state -------- @@ -315,8 +377,11 @@ run_leg() { if [ "$idle" -ge "$stall_grace" ]; then if [ "$demo_redrives" -lt "$max_demo_redrives" ]; then demo_redrives=$(( demo_redrives + 1 )) - log "WARN: leg $leg -- idle ${idle}s with no state.json; the demo turn left nothing running. Re-driving the demo prompt ($demo_redrives/$max_demo_redrives)" - claude --print --dangerously-skip-permissions < "$demo_prompt" 2>&1 | tee -a "$agent_log" + log "WARN: leg $leg -- idle ${idle}s with no state.json; the demo turn left nothing running. Resuming the conversation to finish the launch ($demo_redrives/$max_demo_redrives)" + if ! printf '%s\n' "$DEMO_RESUME_NUDGE" | agent_turn "$agent_log" --resume "$uuid"; then + log "WARN: leg $leg -- could not resume session $uuid; re-feeding the demo prompt" + agent_turn "$agent_log" < "$demo_prompt" || true + fi log "leg $leg demo re-drive $demo_redrives returned" grace_ts="$(date +%s)" # fresh stall grace for the new turn; deadline unchanged continue diff --git a/.github/scripts/pre-release-e2e-poll.sh b/.github/scripts/pre-release-e2e-poll.sh index c2f43f5f39..1c3734079f 100755 --- a/.github/scripts/pre-release-e2e-poll.sh +++ b/.github/scripts/pre-release-e2e-poll.sh @@ -40,6 +40,15 @@ POLL_INTERVAL_S="${POLL_INTERVAL_S:-120}" GLOBAL_TIMEOUT_S="${GLOBAL_TIMEOUT_S:-50400}" MAX_CRASHES="${MAX_CRASHES:-0}" MAX_BOOT_FAILS="${MAX_BOOT_FAILS:-0}" +# Stop polling as soon as one leg FAILs. This is a release GATE: the first FAIL already +# blocks the release, so the remaining legs cannot change the verdict -- and waiting them +# out costs a 12h leg's GPUs plus the single self-hosted runner, which in turn keeps the +# next fix's run stuck at run-level `pending` (a newer run gets no jobs at all while this +# one holds the concurrency group). Set POLL_FAIL_FAST=0 to judge the whole matrix anyway. +POLL_FAIL_FAST="${POLL_FAIL_FAST:-1}" +# Sleep in short slices instead of one long one so a cancelled job tears down in seconds +# rather than at the end of a full POLL_INTERVAL_S. +POLL_SLEEP_SLICE_S="${POLL_SLEEP_SLICE_S:-5}" : "${SAFE_API_BASE:?SAFE_API_BASE is required}" : "${SAFE_API_KEY:?SAFE_API_KEY is required}" @@ -245,6 +254,7 @@ resolve_comment_target report_upsert "$(report_body Running "$(done_count)" "${#LEGS[@]}")" start_s="$(date +%s)" +fail_seen=0 # has any leg reached a FAIL verdict? -> the gate is already decided while :; do pending=0 changed=0 # did any leg reach a verdict this tick? -> refresh the sticky comment @@ -257,7 +267,7 @@ while :; do VERDICT["$leg"]="FAIL|workload phase=$wphase" summary "❌ **$leg** — FAIL (workload $wphase, wid=\`$wid\`)" post_status "$leg" failure "workload $wphase; wid=$wid" - changed=1 + changed=1; fail_seen=1 continue fi # Otherwise judge from the on-disk report (present once the leg finishes). @@ -273,7 +283,7 @@ while :; do VERDICT["$leg"]="FAIL|$detail" summary "❌ **$leg** — FAIL ($detail)" post_status "$leg" failure "FAIL — $detail" - changed=1 + changed=1; fail_seen=1 else pending=$((pending + 1)) # still running; check again next tick fi @@ -284,6 +294,20 @@ while :; do [ "$pending" -eq 0 ] && break + # Gate already lost -> stop here and free the GPUs + the runner. Remaining legs are + # recorded as unjudged rather than FAIL: they did not fail, we chose not to wait. + if [ "$POLL_FAIL_FAST" = "1" ] && [ "$fail_seen" -eq 1 ]; then + for leg in "${LEGS[@]}"; do + [ -n "${VERDICT[$leg]}" ] && continue + VERDICT["$leg"]="FAIL|not judged (gate already failed; fail-fast)" + summary "⏹ **$leg** — not judged (gate already failed; fail-fast)" + post_status "$leg" failure "not judged — gate already failed" + done + summary "" + summary "⏹ fail-fast: a leg already FAILed, so the gate is decided. Stopping the remaining ${pending} leg(s) instead of holding the GPUs. Set \`POLL_FAIL_FAST=0\` to judge the full matrix." + break + fi + elapsed=$(( $(date +%s) - start_s )) if [ "$elapsed" -ge "$GLOBAL_TIMEOUT_S" ]; then for leg in "${LEGS[@]}"; do @@ -295,7 +319,11 @@ while :; do break fi echo "[poll] ${pending} leg(s) still running; elapsed $((elapsed/60))m; sleeping ${POLL_INTERVAL_S}s" - sleep "$POLL_INTERVAL_S" + slept=0 + while [ "$slept" -lt "$POLL_INTERVAL_S" ]; do + sleep "$POLL_SLEEP_SLICE_S" + slept=$(( slept + POLL_SLEEP_SLICE_S )) + done done # ---- aggregate gate -------------------------------------------------------- diff --git a/.github/scripts/pre-release-e2e-reap.sh b/.github/scripts/pre-release-e2e-reap.sh deleted file mode 100644 index 66307c3154..0000000000 --- a/.github/scripts/pre-release-e2e-reap.sh +++ /dev/null @@ -1,78 +0,0 @@ -#!/usr/bin/env bash -# SPDX-FileCopyrightText: 2026 Advanced Micro Devices, Inc. -# SPDX-License-Identifier: MIT -# -# Pre-release E2E: PRE-EMPT stale SaFE workloads at the very START of a run, BEFORE any -# job that touches the single self-hosted GPU runner. -# -# Why a standalone script + its own job (not the reap inside the dispatch script): -# The dispatch reap runs in the `dispatch` job, which `needs: [resolve, build]` and -# `runs-on: hyperloom-pre-e2e-baremetal` -- the SINGLE self-hosted runner. When a newer -# commit supersedes an in-flight run, GitHub's concurrency.cancel-in-progress cancels -# the old JOB but does NOT reliably stop the SaFE PyTorchJob pods it created, and the -# old run may still occupy that one runner (and its 8 GPUs). So the new run's -# resolve/build queue BEHIND the old run and the dispatch reap never gets to run -- a -# deadlock where the cleanup is queued behind the very thing it must clean up. -# -# This script runs in a `preempt` job on a GITHUB-HOSTED runner (ubuntu-latest), which -# does NOT queue behind the busy baremetal runner. It only needs network reach to the -# SaFE API. It fires first and every other job `needs: preempt`, so the stale pods are -# stopped -> GPUs freed -> the old run's poll sees phase=Stopped and its legs FAIL -> -# the old GitHub job ends as a CONSEQUENCE of the pod stopping (the correct causal -# order), and this run's resolve/build/dispatch can then get the runner + GPUs. -# -# Pre-emption semantics: this runs BEFORE this run dispatches ANY workload, so every -# non-terminal `e2e-*` workload in this workspace is necessarily from an OLDER run and is -# safe to stop wholesale -- no VERSION_TAG self-exclusion needed (there is nothing of -# ours to exclude yet). -# -# Inputs (env): -# SAFE_API_BASE SaFE API base url (required) -# SAFE_API_KEY bearer token (ADMIN, to stop privileged pods) (required) -# SAFE_WORKSPACE_ID workspace to scope the reap to (required) -# SAFE_CACERT / SAFE_INSECURE TLS to the API (CA bundle / skip-verify) -set -euo pipefail - -: "${SAFE_API_BASE:?SAFE_API_BASE is required}" -: "${SAFE_API_KEY:?SAFE_API_KEY is required}" -: "${SAFE_WORKSPACE_ID:?SAFE_WORKSPACE_ID is required}" - -API="${SAFE_API_BASE%/}/api/v1/workloads" -auth=(-H "Authorization: Bearer ${SAFE_API_KEY}") -tls=() -if [ -n "${SAFE_CACERT:-}" ]; then - tls=(--cacert "$SAFE_CACERT") -elif [ "${SAFE_INSECURE:-0}" = "1" ]; then - tls=(-k) -fi - -summary() { echo "$*" | tee -a "${GITHUB_STEP_SUMMARY:-/dev/null}"; } - -# List every e2e-* workload in this workspace that is NOT already terminal, and POST -# /stop to each. Resilient: a missing/unreachable API is a skip, never a hard failure -# (we must not block the run just because the reclaim couldn't reach SaFE). -reap_all_stale() { - local resp - resp="$(curl -sS "${tls[@]}" --max-time 30 "$API" "${auth[@]}" 2>/dev/null || true)" - [ -n "$resp" ] || { summary "• [preempt] could not list workloads; skipping reclaim"; return 0; } - local stale - stale="$(printf '%s' "$resp" | jq -r --arg ws "$SAFE_WORKSPACE_ID" ' - (.items // .workloads // .)[]? - | select(((.displayName // .name // "") | startswith("e2e-"))) - | select((.workspaceId // $ws) == $ws) - | select((.phase // .status // "") as $p - | (["Stopped","Failed","Succeeded","Completed","Deleted"] | index($p)) | not) - | (.workloadId // .id)' 2>/dev/null || true)" - [ -n "$stale" ] || { summary "• [preempt] no stale e2e workloads to reclaim"; return 0; } - local wid code n=0 - while IFS= read -r wid; do - [ -n "$wid" ] || continue - code="$(curl -sS "${tls[@]}" --max-time 20 -o /dev/null -w '%{http_code}' \ - -X POST "$API/$wid/stop" "${auth[@]}" 2>/dev/null || echo 000)" - summary "• [preempt] stopped stale workload \`$wid\` (stop HTTP $code)" - n=$((n+1)) - done <<< "$stale" - summary "• [preempt] stopped $n stale e2e workload(s) before this run dispatches" -} - -reap_all_stale diff --git a/.github/workflows/pre-release-e2e-test.yml b/.github/workflows/pre-release-e2e-test.yml index 43d458a9ed..97a1eb0977 100644 --- a/.github/workflows/pre-release-e2e-test.yml +++ b/.github/workflows/pre-release-e2e-test.yml @@ -54,19 +54,24 @@ on: description: "Comma-separated subset of leg IDs (default: all 8)" required: false -# One pre-release run at a time (design §13, point D): the peak GPU footprint stays at 8. -# A NEWER push supersedes the in-flight run. cancel-in-progress cancels the old GitHub -# JOB, but that alone does NOT reliably stop the old run's SaFE PyTorchJob pods (the -# `if: cancelled()` cleanup step gets only a short grace window, and it runs on the single -# busy baremetal runner). So the PRIMARY teardown is the `preempt` job (job 0): it runs -# FIRST on a GitHub-hosted runner and stops every stale e2e-* workload up front, freeing -# the GPUs BEFORE this run's resolve/build/dispatch queue for the baremetal runner. The -# old run's poll then sees phase=Stopped -> its legs FAIL -> the old job ends as a -# consequence. This makes "push a fix -> the stale run is torn down and the fresh code -# reruns" automatic, instead of deadlocking on the reap being queued behind the run it -# must reclaim. The `if: cancelled()` step + the dispatch-side reap remain as backstops. -# Per-PR concurrency (group keyed by ref) so two different PRs don't cancel each other -- -# only newer commits on the SAME PR supersede. +# One pre-release run at a time: a NEWER push supersedes the in-flight run. Per-PR group +# so two different PRs don't cancel each other -- only newer commits on the SAME PR. +# +# A newer run CANNOT tear down the older one. While the group is held, GitHub keeps the +# new run at run-level `pending` with an EMPTY jobs array (observed 2026-08-28: run +# 33145644120 sat pending for 20min with zero jobs while 33144353030 held the group), so +# no job of the new run exists to do any reclaiming. A short-lived `preempt` job on a +# GitHub-hosted runner used to claim that role and could never work: it was both created +# too late and unable to reach SAFE_API_BASE, an in-network NodePort -- every run logged +# `[preempt] could not list workloads; skipping reclaim` after a 30s curl timeout, so it +# never stopped a single workload. It has been removed rather than left to look effective. +# +# Teardown therefore runs the other way round: the OLD run releases the runner promptly. +# The poll fails fast on the first FAIL (the gate is already lost, so there is nothing to +# gain by holding 8 GPUs and the only self-hosted runner for another 12h) and sleeps in +# short slices so a cancel lands in seconds. Leaked pods are then reclaimed by the +# dispatch-side reap, which -- unlike `preempt` -- runs in-network and only ever executes +# once this run holds the runner, i.e. once the previous run is provably done. concurrency: group: pre-release-e2e-${{ github.event.pull_request.number || github.ref }} cancel-in-progress: true @@ -78,35 +83,8 @@ permissions: issues: write # PR comments go through the issues API jobs: - # 0. preempt: STOP any stale SaFE e2e-* workloads from a superseded/older run BEFORE - # anything touches the single self-hosted GPU runner. Runs on a GitHub-hosted runner so - # it does NOT queue behind that busy baremetal runner (the deadlock we hit: the reap in - # `dispatch` needs [resolve,build] on the one runner, which is still held by the old run - # -> the reclaim can never run). Stopping the old pods frees the GPUs and makes the old - # run's poll see phase=Stopped -> its legs FAIL -> the old job ends as a consequence. - # Every other job needs this, so nothing dispatches until the reclaim has fired. - preempt: - runs-on: ubuntu-latest - steps: - - uses: actions/checkout@v7 - - name: Stop stale SaFE e2e workloads - env: - SAFE_API_BASE: ${{ vars.PRE_E2E_SAFE_API_BASE }} - SAFE_API_KEY: ${{ secrets.PRE_E2E_SAFE_API_KEY }} # ADMIN token (stops privileged pods) - SAFE_WORKSPACE_ID: ${{ vars.PRE_E2E_SAFE_WORKSPACE_ID }} - SAFE_INSECURE: ${{ vars.PRE_E2E_SAFE_INSECURE || '1' }} - run: | - command -v jq >/dev/null 2>&1 || (sudo apt-get update && sudo apt-get install -y jq) - chmod +x .github/scripts/pre-release-e2e-reap.sh - .github/scripts/pre-release-e2e-reap.sh - # 1. resolve: gate on a real version bump (PR vs base) or manual input; compute CI_VERSION. resolve: - needs: preempt - # Pre-emption is best-effort reclamation: if it can't reach SaFE it still exits 0, but - # even a hard job failure must NOT block the release gate -- so proceed unless the whole - # run was cancelled (a newer commit superseding us). - if: ${{ !cancelled() }} runs-on: hyperloom-pre-e2e-baremetal outputs: run: ${{ steps.decide.outputs.run }} diff --git a/src/hyperloom/inference_optimizer/tests/test_pre_release_gate_orchestration.py b/src/hyperloom/inference_optimizer/tests/test_pre_release_gate_orchestration.py new file mode 100644 index 0000000000..15ee1171c3 --- /dev/null +++ b/src/hyperloom/inference_optimizer/tests/test_pre_release_gate_orchestration.py @@ -0,0 +1,114 @@ +# SPDX-FileCopyrightText: 2026 Advanced Micro Devices, Inc. +# SPDX-License-Identifier: MIT + +"""Guards for how the pre-release gate releases its runner and its GPUs. + +The gate owns the only self-hosted baremetal runner, so a run that will not finish +blocks every later run: while the concurrency group is held, GitHub keeps the newer run +at run-level ``pending`` with an EMPTY jobs array, so it has no job with which to +reclaim anything. Two consequences are pinned here. + +A ``preempt`` job on a GitHub-hosted runner used to claim the reclaiming role. It could +never work -- created too late to matter, and unable to reach ``SAFE_API_BASE``, which +is an in-network NodePort: every observed run logged ``[preempt] could not list +workloads; skipping reclaim`` after a 30s curl timeout, having stopped nothing. Nothing +that talks to SaFE may run on a GitHub-hosted runner again. + +Teardown instead relies on the old run leaving promptly: the poll fails fast on the +first FAIL (the gate is already decided) and sleeps in short slices so a cancel lands in +seconds instead of at the end of a full poll interval. + +There is no way to unit-test the scheduling itself short of running the workflow; these +tests pin the invariants it depends on. +""" + +from __future__ import annotations + +from pathlib import Path + +import pytest +import yaml + +_SELF_HOSTED_LABEL = "hyperloom-pre-e2e-baremetal" + + +def _find_github_dir() -> Path | None: + """Locate .github/; None when running from an installed wheel.""" + for parent in Path(__file__).resolve().parents: + candidate = parent / ".github" + if (candidate / "workflows" / "pre-release-e2e-test.yml").is_file(): + return candidate + return None + + +_GITHUB = _find_github_dir() + +pytestmark = pytest.mark.skipif( + _GITHUB is None, + reason="pre-release gate guards need the source checkout (.github/)", +) + + +@pytest.fixture(scope="module") +def workflow() -> dict: + assert _GITHUB is not None + path = _GITHUB / "workflows" / "pre-release-e2e-test.yml" + return yaml.safe_load(path.read_text(encoding="utf-8")) + + +@pytest.fixture(scope="module") +def poll_script() -> str: + assert _GITHUB is not None + return (_GITHUB / "scripts" / "pre-release-e2e-poll.sh").read_text(encoding="utf-8") + + +def test_nothing_that_talks_to_safe_runs_on_a_github_hosted_runner(workflow: dict) -> None: + """SAFE_API_BASE is an in-network NodePort; a hosted runner can only time out.""" + for name, job in workflow["jobs"].items(): + runs_on = job.get("runs-on") + if runs_on == _SELF_HOSTED_LABEL: + continue + rendered = yaml.safe_dump(job) + assert "SAFE_API" not in rendered, f"job {name} on {runs_on} reaches for the SaFE API" + + +def test_the_preempt_job_is_gone(workflow: dict) -> None: + assert "preempt" not in workflow["jobs"] + assert workflow["jobs"] == { + k: v for k, v in workflow["jobs"].items() if k in {"resolve", "build", "run"} + } + assert workflow["jobs"]["resolve"].get("needs") is None + + +def test_the_reap_script_is_gone_and_unreferenced() -> None: + assert _GITHUB is not None + assert not (_GITHUB / "scripts" / "pre-release-e2e-reap.sh").exists() + for wf in (_GITHUB / "workflows").glob("*.yml"): + text = wf.read_text(encoding="utf-8") + for line in text.splitlines(): + if line.lstrip().startswith("#"): + continue # the removal rationale is documented in comments + assert "pre-release-e2e-reap.sh" not in line, f"{wf.name} still runs the reap script" + + +def test_poll_fails_fast_once_the_gate_is_lost(poll_script: str) -> None: + """A decided gate must not keep 8 GPUs and the only runner busy for another 12h.""" + assert 'POLL_FAIL_FAST="${POLL_FAIL_FAST:-1}"' in poll_script + assert 'if [ "$POLL_FAIL_FAST" = "1" ] && [ "$fail_seen" -eq 1 ]; then' in poll_script + # Every path that records a FAIL has to arm the flag, or fail-fast never triggers. + assert poll_script.count("fail_seen=1") == 2 + assert poll_script.count('VERDICT["$leg"]="FAIL|') >= 2 + + +def test_poll_sleeps_in_slices_so_a_cancel_lands_quickly(poll_script: str) -> None: + assert 'POLL_SLEEP_SLICE_S="${POLL_SLEEP_SLICE_S:-5}"' in poll_script + assert 'sleep "$POLL_SLEEP_SLICE_S"' in poll_script + assert 'sleep "$POLL_INTERVAL_S"' not in poll_script + + +def test_abnormal_end_cleanup_still_stops_workloads(workflow: dict) -> None: + """With no preempt job, this step and the dispatch-side reap are the only backstops.""" + steps = workflow["jobs"]["run"]["steps"] + cleanup = [s for s in steps if "cancelled()" in str(s.get("if", ""))] + assert cleanup, "the run job lost its cancel/failure cleanup step" + assert "/stop" in cleanup[0]["run"] diff --git a/src/hyperloom/inference_optimizer/tests/test_pre_release_stall_liveness.py b/src/hyperloom/inference_optimizer/tests/test_pre_release_stall_liveness.py index e6bcaaa5f6..9fb4b741cb 100644 --- a/src/hyperloom/inference_optimizer/tests/test_pre_release_stall_liveness.py +++ b/src/hyperloom/inference_optimizer/tests/test_pre_release_stall_liveness.py @@ -21,6 +21,7 @@ from __future__ import annotations import os +import re import subprocess import time from pathlib import Path @@ -123,8 +124,16 @@ def test_stall_check_watches_the_leg_root(script: str) -> None: def test_agent_turns_are_mirrored_to_nfs(script: str) -> None: """SaFE deletes a failed leg's pod, so every agent turn must reach NFS.""" assert 'agent_log="${session}/agent-${leg}.log"' in script - # setup (retried), the demo turn, and each demo re-drive. - assert script.count('tee -a "$agent_log"') == 3 + # Every turn goes through the one helper, which is the only place that tees, so no + # invocation can bypass the transcript. + assert script.count('tee -a "$alog"') == 1 + invocations = [ + line + for line in script.splitlines() + if "claude --print --dangerously-skip-permissions" in line + and not line.lstrip().startswith("#") + ] + assert invocations == [' claude --print --dangerously-skip-permissions "$@" 2>&1 | tee -a "$alog"'] def _leg_run_started(script: str, session: Path) -> bool: @@ -169,11 +178,62 @@ def test_setup_marker_matches_every_setup_prompt(script: str) -> None: def test_an_early_turn_is_re_driven_not_fatal(script: str) -> None: """A turn that ends without finishing leaves nothing running; ask again, bounded.""" - assert 'attempts="${LEG_TURN_ATTEMPTS:-3}"' in script assert 'max_demo_redrives="${LEG_DEMO_REDRIVES:-2}"' in script assert 'demo_redrives=$(( demo_redrives + 1 ))' in script +def test_setup_is_budgeted_in_time_not_in_turns(script: str) -> None: + """A count-based cap of 3 turns was ~4min of wall clock and killed live installs. + + A framework install runs 10-30min, and each turn ends after ~60-90s, so the budget + has to be a deadline plus a liveness check -- never a small turn count. + """ + assert "LEG_TURN_ATTEMPTS" not in script + assert 'setup_deadline_s="${LEG_SETUP_DEADLINE_S:-2700}"' in script + assert 'setup_stall_s="${LEG_SETUP_STALL_S:-600}"' in script + assert 'sidle="$(leg_idle_s "$root" "$setup_t0" "$snow")"' in script + # The stall check must gate the failure, i.e. a progressing install is never reaped. + assert 'if [ "$sidle" -ge "$setup_stall_s" ]; then' in script + + +def test_follow_up_turns_resume_the_same_conversation(script: str) -> None: + """Re-feeding the prompt as a fresh turn throws away what the agent already knows. + + `--session-id` opens the leg's conversation and `--resume` continues it, so a + follow-up turn still knows what it launched and which log it was watching. + """ + assert 'agent_turn "$agent_log" --session-id "$uuid" < "$setup_prompt"' in script + assert script.count('--resume "$uuid"') == 3 # setup nudge, demo turn, demo re-drive + for nudge in ("SETUP_RESUME_NUDGE", "DEMO_RESUME_NUDGE"): + assert f"{nudge}='" in script + # Resuming can fail (no such session); that must degrade, not kill the leg. + assert script.count("could not resume session") == 3 + + +def test_leg_session_uuid_is_stable_and_well_formed(script: str, tmp_path: Path) -> None: + """--session-id rejects anything that is not a UUID.""" + lines = script.splitlines() + start = next(i for i, line in enumerate(lines) if line.startswith("leg_session_uuid() {")) + end = next(i for i, line in enumerate(lines[start:], start) if line == "}") + fn = "\n".join(lines[start : end + 1]) + + def uuid_for(leg: str, version: str) -> str: + proc = subprocess.run( + ["bash", "-c", f'{fn}\nleg_session_uuid "$1" "$2"', "_", leg, version], + capture_output=True, + text=True, + check=True, + ) + return proc.stdout.strip() + + a = uuid_for("baremetal-vllm-3h", "1.0.0.dev1+ci") + assert re.fullmatch(r"[0-9a-f]{8}-[0-9a-f]{4}-4[0-9a-f]{3}-8[0-9a-f]{3}-[0-9a-f]{12}", a) + assert a == uuid_for("baremetal-vllm-3h", "1.0.0.dev1+ci") # stable across turns + # Distinct per leg (four legs share the docker host) and per run. + assert a != uuid_for("baremetal-vllm-12h", "1.0.0.dev1+ci") + assert a != uuid_for("baremetal-vllm-3h", "1.0.0.dev2+ci") + + def test_re_drive_does_not_extend_the_hard_deadline(script: str) -> None: """The stall grace restarts per turn, but the pod-deadline clock must not. From 10aae6d4c300b57ece62ebb94380899181a27f07 Mon Sep 17 00:00:00 2001 From: lishuoshuo-amd Date: Fri, 28 Aug 2026 15:31:03 +0800 Subject: [PATCH 31/52] pre-release-e2e: leave running legs on fail-fast, raise baremetal memory to 512Gi Fail-fast still marks the gate FAIL and releases the runner, but workloads that were still optimizing are left alive for post-mortem. Dispatch reap on the next run remains the backstop. Baremetal legs now request 512Gi memory and ephemeral storage after sglang-12h OOM at 128Gi. Co-authored-by: Cursor --- .github/scripts/pre-release-e2e-dispatch.sh | 12 ++--- .github/scripts/pre-release-e2e-poll.sh | 44 ++++++++++++++----- .github/workflows/pre-release-e2e-test.yml | 23 ++++++---- .../test_pre_release_gate_orchestration.py | 18 +++++--- 4 files changed, 67 insertions(+), 30 deletions(-) diff --git a/.github/scripts/pre-release-e2e-dispatch.sh b/.github/scripts/pre-release-e2e-dispatch.sh index 45fbfdded6..8c48be5be6 100755 --- a/.github/scripts/pre-release-e2e-dispatch.sh +++ b/.github/scripts/pre-release-e2e-dispatch.sh @@ -41,8 +41,9 @@ # (default $RUNNER_TEMP/pre_release_dispatch.json) # HOST_CPU / HOST_MEM / HOST_SHM / HOST_EPHEMERAL privileged host resource request # (default 128 / 2048Gi / 256Gi / 1792Gi -- ref 8-GPU Authoring pod) -# LEG_CPU / LEG_MEM baremetal leg resource request -# (default 32 / 128Gi) +# LEG_CPU / LEG_MEM / LEG_EPHEMERAL baremetal leg resource request +# (default 32 / 512Gi / 512Gi -- sglang 14B-FP8 + roofline/aiter JIT +# exceeded 128Gi/100Gi on 2026-08-28) # DEADLINE_3H_S / DEADLINE_12H_S pod hard-timeout per duration # (default 16200 = 3h+1h+30m / 48600 = 12h+1h+30m). The docker host # pod uses the MAX over its legs. SaFE kills the pod at the @@ -63,7 +64,8 @@ TARGET_GAIN="${TARGET_GAIN:-100}" # layer-deduplicating docker storage driver (overlay2) to stay inside it. HOST_CPU="${HOST_CPU:-128}"; HOST_MEM="${HOST_MEM:-2048Gi}"; HOST_SHM="${HOST_SHM:-256Gi}" HOST_EPHEMERAL="${HOST_EPHEMERAL:-1792Gi}" -LEG_CPU="${LEG_CPU:-32}"; LEG_MEM="${LEG_MEM:-128Gi}" +LEG_CPU="${LEG_CPU:-32}"; LEG_MEM="${LEG_MEM:-512Gi}" +LEG_EPHEMERAL="${LEG_EPHEMERAL:-512Gi}" # SaFE workload scheduling priority (Spec.Priority, an int): High=2, Med=1, Low=0 # (Primus-SaFE common/constant.go). The scheduler orders the queue by this value, and # the webhook clamps it into [0,2]. These release-gate legs hold 8 GPUs for up to 14h @@ -307,8 +309,8 @@ record_dispatch() { # leg workloadId -- add to the in-memory map AND the on-dis } # ---- baremetal legs: one non-privileged 1-GPU workload each ---------------- -leg_resources_1gpu="$(jq -n --arg cpu "$LEG_CPU" --arg mem "$LEG_MEM" \ - '{replica:1, gpu:"1", cpu:$cpu, memory:$mem, ephemeralStorage:"100Gi"}')" +leg_resources_1gpu="$(jq -n --arg cpu "$LEG_CPU" --arg mem "$LEG_MEM" --arg eph "$LEG_EPHEMERAL" \ + '{replica:1, gpu:"1", cpu:$cpu, memory:$mem, ephemeralStorage:$eph}')" want_docker_host=0 for leg in $REQ_TASKS; do diff --git a/.github/scripts/pre-release-e2e-poll.sh b/.github/scripts/pre-release-e2e-poll.sh index 1c3734079f..d1b0f2ef38 100755 --- a/.github/scripts/pre-release-e2e-poll.sh +++ b/.github/scripts/pre-release-e2e-poll.sh @@ -42,10 +42,13 @@ MAX_CRASHES="${MAX_CRASHES:-0}" MAX_BOOT_FAILS="${MAX_BOOT_FAILS:-0}" # Stop polling as soon as one leg FAILs. This is a release GATE: the first FAIL already # blocks the release, so the remaining legs cannot change the verdict -- and waiting them -# out costs a 12h leg's GPUs plus the single self-hosted runner, which in turn keeps the -# next fix's run stuck at run-level `pending` (a newer run gets no jobs at all while this -# one holds the concurrency group). Set POLL_FAIL_FAST=0 to judge the whole matrix anyway. +# out costs the single self-hosted runner, which in turn keeps the next fix's run stuck at +# run-level `pending` (a newer run gets no jobs at all while this one holds the +# concurrency group). Still-running workloads are LEFT ALIVE (see leave_running_wids) so +# they can finish for debugging; only the poll job exits. Set POLL_FAIL_FAST=0 to keep +# polling until every leg reaches a terminal verdict anyway. POLL_FAIL_FAST="${POLL_FAIL_FAST:-1}" +LEAVE_RUNNING_FILE="${LEAVE_RUNNING_FILE:-${DISPATCH_MAP}.leave_running}" # Sleep in short slices instead of one long one so a cancelled job tears down in seconds # rather than at the end of a full POLL_INTERVAL_S. POLL_SLEEP_SLICE_S="${POLL_SLEEP_SLICE_S:-5}" @@ -294,17 +297,22 @@ while :; do [ "$pending" -eq 0 ] && break - # Gate already lost -> stop here and free the GPUs + the runner. Remaining legs are - # recorded as unjudged rather than FAIL: they did not fail, we chose not to wait. + # Gate already lost -> release the runner, but leave still-running workloads up so they + # can finish on the cluster (useful for debugging infra vs product failures). if [ "$POLL_FAIL_FAST" = "1" ] && [ "$fail_seen" -eq 1 ]; then + leave_wids=() for leg in "${LEGS[@]}"; do [ -n "${VERDICT[$leg]}" ] && continue - VERDICT["$leg"]="FAIL|not judged (gate already failed; fail-fast)" - summary "⏹ **$leg** — not judged (gate already failed; fail-fast)" - post_status "$leg" failure "not judged — gate already failed" + VERDICT["$leg"]="SKIP|still running (gate failed; workload left alive)" + summary "⏳ **$leg** — still running (gate already failed; workload left alive)" + post_status "$leg" pending "gate failed; workload left running" + leave_wids+=( "${WID[$leg]}" ) done - summary "" - summary "⏹ fail-fast: a leg already FAILed, so the gate is decided. Stopping the remaining ${pending} leg(s) instead of holding the GPUs. Set \`POLL_FAIL_FAST=0\` to judge the full matrix." + if [ "${#leave_wids[@]}" -gt 0 ]; then + printf '%s\n' "${leave_wids[@]}" | sort -u | jq -R . | jq -s . > "$LEAVE_RUNNING_FILE" + summary "" + summary "⏹ fail-fast: gate is FAIL. Releasing the runner; ${pending} workload(s) left running for post-mortem. Wids recorded in \`$(basename "$LEAVE_RUNNING_FILE")\`. Set \`POLL_FAIL_FAST=0\` to poll until every leg finishes." + fi break fi @@ -336,7 +344,11 @@ fail=0 for leg in "${LEGS[@]}"; do v="${VERDICT[$leg]:-FAIL|no verdict}" vv="${v%%|*}"; vd="${v#*|}" - icon="✅"; [ "$vv" = "PASS" ] || { icon="❌"; fail=1; } + case "$vv" in + PASS) icon="✅" ;; + SKIP) icon="⏳"; fail=1 ;; + *) icon="❌"; fail=1 ;; + esac summary "| \`$leg\` | $icon $vv | $vd |" done summary "" @@ -356,6 +368,12 @@ report_upsert "$(printf '%s\n\n%s\n' "$(report_body Complete "$(done_count)" "${ # KEEPING the workload record + its pod filesystem for post-hoc inspection. SaFE has # no `start` endpoint, so these are not resumable; clean up Stopped records manually. # Verified 2026-08-27: POST /api/v1/workloads/{id}/stop exists and returns 200. +leave_running_wid() { # wid -> 0 if this workload should stay up + local wid="$1" + [ -f "$LEAVE_RUNNING_FILE" ] || return 1 + jq -e --arg w "$wid" 'index($w) != null' "$LEAVE_RUNNING_FILE" >/dev/null 2>&1 +} + stop_workloads() { local wid seen="" for leg in "${LEGS[@]}"; do @@ -363,6 +381,10 @@ stop_workloads() { # docker legs share one host workload -> stop each unique id once. case " $seen " in *" $wid "*) continue ;; esac seen="${seen} ${wid}" + if leave_running_wid "$wid"; then + summary "• left workload \`$wid\` running (fail-fast; post-mortem)" + continue + fi code="$(curl -sS "${tls[@]}" -o /dev/null -w '%{http_code}' -X POST \ "$API/$wid/stop" "${auth[@]}" 2>/dev/null || echo 000)" summary "• stopped workload \`$wid\` (HTTP $code)" diff --git a/.github/workflows/pre-release-e2e-test.yml b/.github/workflows/pre-release-e2e-test.yml index 97a1eb0977..6484d39b67 100644 --- a/.github/workflows/pre-release-e2e-test.yml +++ b/.github/workflows/pre-release-e2e-test.yml @@ -67,11 +67,11 @@ on: # never stopped a single workload. It has been removed rather than left to look effective. # # Teardown therefore runs the other way round: the OLD run releases the runner promptly. -# The poll fails fast on the first FAIL (the gate is already lost, so there is nothing to -# gain by holding 8 GPUs and the only self-hosted runner for another 12h) and sleeps in -# short slices so a cancel lands in seconds. Leaked pods are then reclaimed by the -# dispatch-side reap, which -- unlike `preempt` -- runs in-network and only ever executes -# once this run holds the runner, i.e. once the previous run is provably done. +# The poll fails fast on the first FAIL (the gate is already lost) but leaves still- +# running workloads up for post-mortem, and sleeps in short slices so a cancel lands in +# seconds. Leaked pods are then reclaimed by the dispatch-side reap, which -- unlike +# `preempt` -- runs in-network and only ever executes once this run holds the runner, +# i.e. once the previous run is provably done. concurrency: group: pre-release-e2e-${{ github.event.pull_request.number || github.ref }} cancel-in-progress: true @@ -319,9 +319,11 @@ jobs: # (superseded push) or fails (dispatch/poll error, runner death) after dispatch # would idle-hold its GPUs. This idempotent cleanup fires on every abnormal end # and stops whatever landed in DISPATCH_MAP (stopping an already-terminal - # workload is a harmless no-op). The next run's pre-dispatch reap is the - # backstop for anything even this misses (e.g. the runner dying before this - # step). Success path is handled by the poll script's own stop_workloads(). + # workload is a harmless no-op). Fail-fast poll failures that intentionally left + # workloads running write DISPATCH_MAP.leave_running; those wids are skipped here + # too. The next run's pre-dispatch reap is the backstop for anything even this + # misses (e.g. the runner dying before this step). Success path is handled by + # the poll script's own stop_workloads(). if: cancelled() || failure() env: SAFE_API_BASE: ${{ vars.PRE_E2E_SAFE_API_BASE }} @@ -330,7 +332,12 @@ jobs: # Policy: stop, don't delete (frees GPUs, keeps the record + pod fs for # inspection; SaFE has no restart, so clean up Stopped records manually). [ -f "$DISPATCH_MAP" ] || { echo "no dispatch map; nothing to stop"; exit 0; } + leave_file="${DISPATCH_MAP}.leave_running" for wid in $(jq -r '.[]' "$DISPATCH_MAP" | sort -u); do + if [ -f "$leave_file" ] && jq -e --arg w "$wid" 'index($w) != null' "$leave_file" >/dev/null 2>&1; then + echo "skipping workload $wid (left running for post-mortem)" + continue + fi echo "stopping workload $wid" curl -sS -k -o /dev/null -w 'STOP %{http_code}\n' -X POST \ -H "Authorization: Bearer ${SAFE_API_KEY}" \ diff --git a/src/hyperloom/inference_optimizer/tests/test_pre_release_gate_orchestration.py b/src/hyperloom/inference_optimizer/tests/test_pre_release_gate_orchestration.py index 15ee1171c3..1a76bd52c9 100644 --- a/src/hyperloom/inference_optimizer/tests/test_pre_release_gate_orchestration.py +++ b/src/hyperloom/inference_optimizer/tests/test_pre_release_gate_orchestration.py @@ -15,8 +15,9 @@ that talks to SaFE may run on a GitHub-hosted runner again. Teardown instead relies on the old run leaving promptly: the poll fails fast on the -first FAIL (the gate is already decided) and sleeps in short slices so a cancel lands in -seconds instead of at the end of a full poll interval. +first FAIL (the gate is already decided), leaves still-running workloads up for post- +mortem, and sleeps in short slices so a cancel lands in seconds instead of at the end +of a full poll interval. There is no way to unit-test the scheduling itself short of running the workflow; these tests pin the invariants it depends on. @@ -92,9 +93,12 @@ def test_the_reap_script_is_gone_and_unreferenced() -> None: def test_poll_fails_fast_once_the_gate_is_lost(poll_script: str) -> None: - """A decided gate must not keep 8 GPUs and the only runner busy for another 12h.""" + """A decided gate must not keep the only runner busy for another 12h.""" assert 'POLL_FAIL_FAST="${POLL_FAIL_FAST:-1}"' in poll_script assert 'if [ "$POLL_FAIL_FAST" = "1" ] && [ "$fail_seen" -eq 1 ]; then' in poll_script + assert "LEAVE_RUNNING_FILE=" in poll_script + assert 'VERDICT["$leg"]="SKIP|still running (gate failed; workload left alive)"' in poll_script + assert "leave_running_wid" in poll_script # Every path that records a FAIL has to arm the flag, or fail-fast never triggers. assert poll_script.count("fail_seen=1") == 2 assert poll_script.count('VERDICT["$leg"]="FAIL|') >= 2 @@ -106,9 +110,11 @@ def test_poll_sleeps_in_slices_so_a_cancel_lands_quickly(poll_script: str) -> No assert 'sleep "$POLL_INTERVAL_S"' not in poll_script -def test_abnormal_end_cleanup_still_stops_workloads(workflow: dict) -> None: - """With no preempt job, this step and the dispatch-side reap are the only backstops.""" +def test_abnormal_end_cleanup_respects_leave_running(workflow: dict) -> None: + """Fail-fast may leave workloads up; cleanup must not stop those wids.""" steps = workflow["jobs"]["run"]["steps"] cleanup = [s for s in steps if "cancelled()" in str(s.get("if", ""))] assert cleanup, "the run job lost its cancel/failure cleanup step" - assert "/stop" in cleanup[0]["run"] + body = cleanup[0]["run"] + assert "/stop" in body + assert "leave_running" in body From 877d61beea3ee2e751c6b93c34067c6d784641aa Mon Sep 17 00:00:00 2001 From: lishuoshuo-amd Date: Fri, 28 Aug 2026 15:35:57 +0800 Subject: [PATCH 32/52] fix: ruff format pre-release gate tests, note leave-running poll behavior Reformat test_pre_release_gate_orchestration and test_pre_release_stall_liveness so ruff format --check passes in CI. Co-authored-by: Cursor --- .github/scripts/pre-release-e2e-poll.sh | 1 + .../tests/test_pre_release_gate_orchestration.py | 4 +--- .../tests/test_pre_release_stall_liveness.py | 13 +++++-------- 3 files changed, 7 insertions(+), 11 deletions(-) diff --git a/.github/scripts/pre-release-e2e-poll.sh b/.github/scripts/pre-release-e2e-poll.sh index d1b0f2ef38..a46f5d1994 100755 --- a/.github/scripts/pre-release-e2e-poll.sh +++ b/.github/scripts/pre-release-e2e-poll.sh @@ -13,6 +13,7 @@ # 3. reports/final.json and reports/final.md both exist, # 4. crash_count / server_boot_failures are within tolerance. # Anything else (incl. "ran the full duration without target_reached") is FAIL. +# Fail-fast leaves still-running optimize legs alive; dispatch reap stops stale e2e-* tags. # # The exit code is 0 only if every requested leg PASSed. # diff --git a/src/hyperloom/inference_optimizer/tests/test_pre_release_gate_orchestration.py b/src/hyperloom/inference_optimizer/tests/test_pre_release_gate_orchestration.py index 1a76bd52c9..ed2011ddff 100644 --- a/src/hyperloom/inference_optimizer/tests/test_pre_release_gate_orchestration.py +++ b/src/hyperloom/inference_optimizer/tests/test_pre_release_gate_orchestration.py @@ -75,9 +75,7 @@ def test_nothing_that_talks_to_safe_runs_on_a_github_hosted_runner(workflow: dic def test_the_preempt_job_is_gone(workflow: dict) -> None: assert "preempt" not in workflow["jobs"] - assert workflow["jobs"] == { - k: v for k, v in workflow["jobs"].items() if k in {"resolve", "build", "run"} - } + assert workflow["jobs"] == {k: v for k, v in workflow["jobs"].items() if k in {"resolve", "build", "run"}} assert workflow["jobs"]["resolve"].get("needs") is None diff --git a/src/hyperloom/inference_optimizer/tests/test_pre_release_stall_liveness.py b/src/hyperloom/inference_optimizer/tests/test_pre_release_stall_liveness.py index 9fb4b741cb..a5546c51f0 100644 --- a/src/hyperloom/inference_optimizer/tests/test_pre_release_stall_liveness.py +++ b/src/hyperloom/inference_optimizer/tests/test_pre_release_stall_liveness.py @@ -118,7 +118,7 @@ def test_empty_tree_falls_back_to_the_loop_start(leg_idle_fn: str, tmp_path: Pat def test_stall_check_watches_the_leg_root(script: str) -> None: """Pin the call site: the check must be scoped to $root, not $session.""" assert 'idle="$(leg_idle_s "$root" "$grace_ts" "$now")"' in script - assert 'no file written under $root' in script + assert "no file written under $root" in script def test_agent_turns_are_mirrored_to_nfs(script: str) -> None: @@ -130,8 +130,7 @@ def test_agent_turns_are_mirrored_to_nfs(script: str) -> None: invocations = [ line for line in script.splitlines() - if "claude --print --dangerously-skip-permissions" in line - and not line.lstrip().startswith("#") + if "claude --print --dangerously-skip-permissions" in line and not line.lstrip().startswith("#") ] assert invocations == [' claude --print --dangerously-skip-permissions "$@" 2>&1 | tee -a "$alog"'] @@ -179,7 +178,7 @@ def test_setup_marker_matches_every_setup_prompt(script: str) -> None: def test_an_early_turn_is_re_driven_not_fatal(script: str) -> None: """A turn that ends without finishing leaves nothing running; ask again, bounded.""" assert 'max_demo_redrives="${LEG_DEMO_REDRIVES:-2}"' in script - assert 'demo_redrives=$(( demo_redrives + 1 ))' in script + assert "demo_redrives=$(( demo_redrives + 1 ))" in script def test_setup_is_budgeted_in_time_not_in_turns(script: str) -> None: @@ -243,7 +242,7 @@ def test_re_drive_does_not_extend_the_hard_deadline(script: str) -> None: body = script.split("run_leg() {", 1)[1] assert body.count('start_ts="$(date +%s)"') == 1 assert 'grace_ts="$(date +%s)"' in body - assert 'elapsed=$(( now - start_ts ))' in body + assert "elapsed=$(( now - start_ts ))" in body assert 'idle="$(leg_idle_s "$root" "$grace_ts" "$now")"' in body @@ -263,8 +262,6 @@ def test_dockerd_never_runs_on_vfs(script: str) -> None: fail rather than quietly reproduce the eviction. """ assert "--storage-driver=vfs" not in script - drivers = next( - line for line in script.splitlines() if line.startswith("DOCKER_DRIVERS=") - ) + drivers = next(line for line in script.splitlines() if line.startswith("DOCKER_DRIVERS=")) assert "overlay2" in drivers assert "vfs" not in drivers From 9eb65ed07b5539812263a7c2ba6bb6bc4a084891 Mon Sep 17 00:00:00 2001 From: lishuoshuo-amd Date: Fri, 28 Aug 2026 15:47:45 +0800 Subject: [PATCH 33/52] pre-release-e2e: per-run workload tags and poll supersede exit VERSION_TAG now hashes CI_VERSION with GITHUB_RUN_ID so dispatch reap can stop pods from a superseded push even when the wheel version is unchanged. Poll detects a newer queued workflow run for the same PR branch, releases the runner without stopping workloads, and leaves teardown to the successor. Co-authored-by: Cursor --- .github/scripts/pre-release-e2e-dispatch.sh | 6 +- .github/scripts/pre-release-e2e-poll.sh | 93 ++++++++++++++++++- .github/workflows/pre-release-e2e-test.yml | 1 + .../test_pre_release_gate_orchestration.py | 24 +++++ 4 files changed, 120 insertions(+), 4 deletions(-) diff --git a/.github/scripts/pre-release-e2e-dispatch.sh b/.github/scripts/pre-release-e2e-dispatch.sh index 8c48be5be6..5079ed6fe7 100755 --- a/.github/scripts/pre-release-e2e-dispatch.sh +++ b/.github/scripts/pre-release-e2e-dispatch.sh @@ -96,9 +96,9 @@ leg_deadline_s() { case "$1" in *-3h) echo "$DEADLINE_3H_S" ;; *-12h) echo "$DEA # NOT embed the full CI_VERSION (e.g. 1.0.0.dev202608270954+ci) in every workload name # -- it would blow the limit and, once truncated, collide across legs (the leg suffix # gets cut). Instead build "e2e--": the human-readable leg -# stays intact up front, and a 6-hex digest of CI_VERSION disambiguates across runs -# without length risk. All legs of one run share the same VERSION_TAG. -VERSION_TAG="$(printf '%s' "$CI_VERSION" | sha1sum | cut -c1-6)" +# stays intact up front, and a 6-hex digest of CI_VERSION+run id disambiguates across +# runs (including repeated pushes with the same wheel version). All legs share VERSION_TAG. +VERSION_TAG="$(printf '%s-%s' "$CI_VERSION" "${GITHUB_RUN_ID:-local}" | sha1sum | cut -c1-6)" workload_name() { printf 'e2e-%s-%s' "$1" "$VERSION_TAG"; } # $1 = leg (or "docker-host") : "${SAFE_API_BASE:?SAFE_API_BASE is required}" diff --git a/.github/scripts/pre-release-e2e-poll.sh b/.github/scripts/pre-release-e2e-poll.sh index a46f5d1994..08a2c8d18a 100755 --- a/.github/scripts/pre-release-e2e-poll.sh +++ b/.github/scripts/pre-release-e2e-poll.sh @@ -32,6 +32,8 @@ # MAX_CRASHES / MAX_BOOT_FAILS tolerance (default 0 / 0) # Optional GitHub commit status (per-leg context pre-release-e2e/): # GH_STATUS_TOKEN / GH_STATUS_REPO / GH_STATUS_SHA / GH_STATUS_DETAILS_URL +# Supersede detection (release runner when a newer pre-release run is queued): +# GITHUB_RUN_ID / PR_NUMBER / HEAD_REF (branch); uses GH_STATUS_TOKEN + GH_STATUS_REPO # SAFE_CACERT / SAFE_INSECURE TLS to the API set -euo pipefail @@ -51,8 +53,10 @@ MAX_BOOT_FAILS="${MAX_BOOT_FAILS:-0}" POLL_FAIL_FAST="${POLL_FAIL_FAST:-1}" LEAVE_RUNNING_FILE="${LEAVE_RUNNING_FILE:-${DISPATCH_MAP}.leave_running}" # Sleep in short slices instead of one long one so a cancelled job tears down in seconds -# rather than at the end of a full POLL_INTERVAL_S. +# rather than at the end of a full POLL_INTERVAL_S. Each slice also re-checks whether a +# newer pre-release run has been queued so this poll can exit and release the runner. POLL_SLEEP_SLICE_S="${POLL_SLEEP_SLICE_S:-5}" +PRE_RELEASE_WORKFLOW_FILE="${PRE_RELEASE_WORKFLOW_FILE:-pre-release-e2e-test.yml}" : "${SAFE_API_BASE:?SAFE_API_BASE is required}" : "${SAFE_API_KEY:?SAFE_API_KEY is required}" @@ -74,6 +78,65 @@ runs_dir="${NFS_ROOT%/}/runs/${CI_VERSION}" summary() { echo "$*" | tee -a "${GITHUB_STEP_SUMMARY:-/dev/null}"; } gh_status_on() { [ -n "${GH_STATUS_TOKEN:-}" ] && [ -n "${GH_STATUS_REPO:-}" ] && [ -n "${GH_STATUS_SHA:-}" ]; } + +# True when a newer pre-release workflow run is queued/in-flight for this PR branch. +# GitHub run ids are monotonic; a pending successor blocks on concurrency until we exit. +supersede_check_on() { + [[ "${GITHUB_RUN_ID:-}" =~ ^[0-9]+$ ]] \ + && [ -n "${GH_STATUS_TOKEN:-}" ] \ + && [ -n "${GH_STATUS_REPO:-}" ] +} + +_supersede_head_ref() { + if [ -n "${HEAD_REF:-}" ]; then + printf '%s' "$HEAD_REF" + return 0 + fi + if [[ "${PR_NUMBER:-}" =~ ^[0-9]+$ ]]; then + curl -sS \ + -H "Authorization: Bearer ${GH_STATUS_TOKEN}" \ + -H "Accept: application/vnd.github+json" \ + -H "X-GitHub-Api-Version: 2022-11-28" \ + "${GH_API}/repos/${GH_STATUS_REPO}/pulls/${PR_NUMBER}" 2>/dev/null \ + | jq -r '.head.ref // empty' 2>/dev/null || true + return 0 + fi + echo "" +} + +superseded_by_newer_run() { + supersede_check_on || return 1 + local head_ref newer + head_ref="$(_supersede_head_ref)" + [ -n "$head_ref" ] || return 1 + newer="$(curl -sS \ + -H "Authorization: Bearer ${GH_STATUS_TOKEN}" \ + -H "Accept: application/vnd.github+json" \ + -H "X-GitHub-Api-Version: 2022-11-28" \ + "${GH_API}/repos/${GH_STATUS_REPO}/actions/workflows/${PRE_RELEASE_WORKFLOW_FILE}/runs?branch=${head_ref}&per_page=10" \ + 2>/dev/null \ + | jq -r --argjson rid "$GITHUB_RUN_ID" ' + [.workflow_runs[]? + | select(.id > $rid) + | select(.status == "queued" or .status == "in_progress" or .status == "pending" or .status == "waiting") + | .id][0] // empty' 2>/dev/null || true)" + [ -n "$newer" ] +} + +mark_superseded_and_exit_poll() { # -> sets superseded=1, marks pending legs SKIP, breaks caller loop + local leg pending=0 + for leg in "${LEGS[@]}"; do + [ -n "${VERDICT[$leg]}" ] && continue + VERDICT["$leg"]="SKIP|superseded by newer run (dispatch reap will stop)" + summary "⏳ **$leg** — superseded (newer run queued; workload left for dispatch reap)" + post_status "$leg" pending "superseded; newer run queued" + pending=$((pending + 1)) + done + summary "" + summary "⏹ superseded: newer pre-release run queued (run_id>${GITHUB_RUN_ID}). Releasing the runner; ${pending} workload(s) left for the successor's dispatch reap." + superseded=1 +} + post_status() { # leg state(pending|success|failure|error) description gh_status_on || return 0 local desc="${3:0:139}" @@ -259,7 +322,12 @@ report_upsert "$(report_body Running "$(done_count)" "${#LEGS[@]}")" start_s="$(date +%s)" fail_seen=0 # has any leg reached a FAIL verdict? -> the gate is already decided +superseded=0 # a newer workflow run is queued -> release runner without stopping pods while :; do + if superseded_by_newer_run; then + mark_superseded_and_exit_poll + break + fi pending=0 changed=0 # did any leg reach a verdict this tick? -> refresh the sticky comment for leg in "${LEGS[@]}"; do @@ -330,11 +398,34 @@ while :; do echo "[poll] ${pending} leg(s) still running; elapsed $((elapsed/60))m; sleeping ${POLL_INTERVAL_S}s" slept=0 while [ "$slept" -lt "$POLL_INTERVAL_S" ]; do + if superseded_by_newer_run; then + mark_superseded_and_exit_poll + break + fi sleep "$POLL_SLEEP_SLICE_S" slept=$(( slept + POLL_SLEEP_SLICE_S )) done + [ "$superseded" -eq 1 ] && break done +if [ "$superseded" -eq 1 ]; then + summary "" + summary "### Result" + summary "" + summary "| leg | verdict | detail |" + summary "|-----|---------|--------|" + for leg in "${LEGS[@]}"; do + v="${VERDICT[$leg]:-SKIP|superseded}" + vv="${v%%|*}"; vd="${v#*|}" + summary "| \`$leg\` | ⏳ $vv | $vd |" + done + summary "" + gate_line="**GATE: SUPERSEDED** — newer run queued; workloads left for dispatch reap." + summary "$gate_line" + report_upsert "$(printf '%s\n\n%s\n' "$(report_body Complete "$(done_count)" "${#LEGS[@]}")" "$gate_line")" + exit 0 +fi + # ---- aggregate gate -------------------------------------------------------- summary "" summary "### Result" diff --git a/.github/workflows/pre-release-e2e-test.yml b/.github/workflows/pre-release-e2e-test.yml index 6484d39b67..a915bc9442 100644 --- a/.github/workflows/pre-release-e2e-test.yml +++ b/.github/workflows/pre-release-e2e-test.yml @@ -290,6 +290,7 @@ jobs: GH_STATUS_SHA: ${{ github.event.pull_request.head.sha }} # PR number is known directly from the event -- no commit->PR reverse lookup. PR_NUMBER: ${{ github.event.pull_request.number }} + HEAD_REF: ${{ github.head_ref }} GH_STATUS_DETAILS_URL: ${{ github.server_url }}/${{ github.repository }}/actions/runs/${{ github.run_id }} POLL_INTERVAL_S: "120" GLOBAL_TIMEOUT_S: "50400" diff --git a/src/hyperloom/inference_optimizer/tests/test_pre_release_gate_orchestration.py b/src/hyperloom/inference_optimizer/tests/test_pre_release_gate_orchestration.py index ed2011ddff..cbfd83f681 100644 --- a/src/hyperloom/inference_optimizer/tests/test_pre_release_gate_orchestration.py +++ b/src/hyperloom/inference_optimizer/tests/test_pre_release_gate_orchestration.py @@ -63,6 +63,12 @@ def poll_script() -> str: return (_GITHUB / "scripts" / "pre-release-e2e-poll.sh").read_text(encoding="utf-8") +@pytest.fixture(scope="module") +def dispatch_script() -> str: + assert _GITHUB is not None + return (_GITHUB / "scripts" / "pre-release-e2e-dispatch.sh").read_text(encoding="utf-8") + + def test_nothing_that_talks_to_safe_runs_on_a_github_hosted_runner(workflow: dict) -> None: """SAFE_API_BASE is an in-network NodePort; a hosted runner can only time out.""" for name, job in workflow["jobs"].items(): @@ -116,3 +122,21 @@ def test_abnormal_end_cleanup_respects_leave_running(workflow: dict) -> None: body = cleanup[0]["run"] assert "/stop" in body assert "leave_running" in body + + +def test_dispatch_version_tag_is_unique_per_run(dispatch_script: str) -> None: + """Reap must distinguish repeated pushes that reuse the same CI_VERSION wheel.""" + assert ( + 'VERSION_TAG="$(printf \'%s-%s\' "$CI_VERSION" "${GITHUB_RUN_ID:-local}" | sha1sum | cut -c1-6)"' + in dispatch_script + ) + + +def test_poll_exits_when_a_newer_run_is_queued(poll_script: str, workflow: dict) -> None: + """A pending successor cannot dispatch until this poll releases the runner.""" + assert "superseded_by_newer_run" in poll_script + assert "mark_superseded_and_exit_poll" in poll_script + assert "GATE: SUPERSEDED" in poll_script + assert "dispatch reap will stop" in poll_script + run_env = yaml.safe_dump(workflow["jobs"]["run"].get("env", {})) + assert "HEAD_REF:" in run_env From 32cee4442437296539e053300e40e5e1c4f17c07 Mon Sep 17 00:00:00 2001 From: lishuoshuo-amd Date: Fri, 28 Aug 2026 15:53:34 +0800 Subject: [PATCH 34/52] pre-release-e2e: trivial dispatch comment for pod-stop supersede test Co-authored-by: Cursor --- .github/scripts/pre-release-e2e-dispatch.sh | 1 + 1 file changed, 1 insertion(+) diff --git a/.github/scripts/pre-release-e2e-dispatch.sh b/.github/scripts/pre-release-e2e-dispatch.sh index 5079ed6fe7..fd4d46ebaf 100755 --- a/.github/scripts/pre-release-e2e-dispatch.sh +++ b/.github/scripts/pre-release-e2e-dispatch.sh @@ -124,6 +124,7 @@ fi summary() { echo "$*" | tee -a "${GITHUB_STEP_SUMMARY:-/dev/null}"; } # ---- reclaim stale pre-release workloads BEFORE dispatching ----------------- +# Stale = any non-terminal e2e-* in this workspace whose VERSION_TAG differs from ours. # The concurrency.cancel-in-progress GitHub knob only cancels the JOB; it does NOT # reliably stop the SaFE PyTorchJob pods a superseded/failed run already created (the # `if: cancelled()` cleanup gets a short grace window, and a job that FAILS -- not From afa1c83de5266d817a1ba7785773849c466128a7 Mon Sep 17 00:00:00 2001 From: lishuoshuo-amd Date: Fri, 28 Aug 2026 16:43:31 +0800 Subject: [PATCH 35/52] pre-release-e2e: judge PASS on clean stop_reason, not target gain Align poll gate with optimize CLI exit 0: final.json must carry a terminal stop_reason in the success set (time_exhausted, target_reached, etc.). TARGET_GAIN still flows to the demo skill at 100 but no longer gates PASS. Co-authored-by: Cursor --- .github/scripts/pre-release-e2e-poll.sh | 33 ++++++++++--------- .../test_pre_release_gate_orchestration.py | 11 +++++++ 2 files changed, 29 insertions(+), 15 deletions(-) diff --git a/.github/scripts/pre-release-e2e-poll.sh b/.github/scripts/pre-release-e2e-poll.sh index 08a2c8d18a..9de019d878 100755 --- a/.github/scripts/pre-release-e2e-poll.sh +++ b/.github/scripts/pre-release-e2e-poll.sh @@ -7,12 +7,11 @@ # (design §10, point C -- the 3h legs finish ~3-4h and report first; 12h legs later). # # A leg PASSes only when ALL hold (design §9): -# 1. its session reports/final.json has stop_reason == "target_reached" -# (equivalently cumulative_gain_validated >= TARGET_GAIN; the gate is 100), -# 2. its owning SaFE workload phase is not Failed/Stopped, -# 3. reports/final.json and reports/final.md both exist, +# 1. reports/final.json and reports/final.md both exist, +# 2. final.json stop_reason is a clean terminal exit (same set as optimize CLI exit 0), +# 3. its owning SaFE workload phase is not Failed/Stopped, # 4. crash_count / server_boot_failures are within tolerance. -# Anything else (incl. "ran the full duration without target_reached") is FAIL. +# TARGET_GAIN still flows to optimize via the demo skill; it is NOT used here to judge PASS. # Fail-fast leaves still-running optimize legs alive; dispatch reap stops stale e2e-* tags. # # The exit code is 0 only if every requested leg PASSed. @@ -25,7 +24,7 @@ # CI_VERSION run version (required) # DISPATCH_MAP leg->workloadId JSON from dispatch (required) # NFS_ROOT (default /shared_nfs/hyperloom-pre-release-e2e-test) -# TARGET_GAIN gate %% (default 100) +# TARGET_GAIN passed to optimize (demo skill); not used to judge PASS # POLL_INTERVAL_S seconds between polls (default 120) # GLOBAL_TIMEOUT_S hard cap; unfinished legs -> FAIL # (default 50400 = 14h) @@ -267,6 +266,15 @@ leg_session_dir() { echo "" } +# Clean terminal stop_reason values (hyperloom.inference_optimizer.cli._SUCCESS_STOP_REASONS). +is_clean_stop_reason() { + case "$1" in + target_reached|global_converged|time_exhausted|max_ticks|sweep_done|conc_sweep_done) + return 0 ;; + *) return 1 ;; + esac +} + # Judge one leg from its final.json. Echoes "PASS"|"FAIL|". judge_leg() { local leg="$1" wphase="$2" sdir final gain stop crashes boots @@ -291,15 +299,10 @@ judge_leg() { if [ "$boots" -gt "$MAX_BOOT_FAILS" ] 2>/dev/null; then echo "FAIL|server_boot_failures=$boots > $MAX_BOOT_FAILS"; return fi - # Primary gate: stop_reason target_reached (== gain >= TARGET_GAIN). - if [ "$stop" = "target_reached" ]; then - echo "PASS|gain=${gain}% stop=${stop}"; return - fi - # Fallback: numeric compare in case stop_reason lags (awk for float). - if awk -v g="$gain" -v t="$TARGET_GAIN" 'BEGIN{exit !(g+0 >= t+0)}'; then - echo "PASS|gain=${gain}% (>= ${TARGET_GAIN})"; return + if is_clean_stop_reason "$stop"; then + echo "PASS|stop=${stop} gain=${gain}%"; return fi - echo "FAIL|gain=${gain}% < ${TARGET_GAIN} (stop=${stop:-none})" + echo "FAIL|stop_reason=${stop:-none} (not a clean terminal exit)" } # ---- poll loop ------------------------------------------------------------- @@ -445,7 +448,7 @@ for leg in "${LEGS[@]}"; do done summary "" if [ "$fail" -eq 0 ]; then - gate_line="**GATE: PASS** — all ${#LEGS[@]} legs reached target_gain=${TARGET_GAIN}." + gate_line="**GATE: PASS** — all ${#LEGS[@]} legs completed with a clean terminal stop_reason." else gate_line="**GATE: FAIL** — one or more legs did not pass. Release blocked." fi diff --git a/src/hyperloom/inference_optimizer/tests/test_pre_release_gate_orchestration.py b/src/hyperloom/inference_optimizer/tests/test_pre_release_gate_orchestration.py index cbfd83f681..ecbc40fdde 100644 --- a/src/hyperloom/inference_optimizer/tests/test_pre_release_gate_orchestration.py +++ b/src/hyperloom/inference_optimizer/tests/test_pre_release_gate_orchestration.py @@ -140,3 +140,14 @@ def test_poll_exits_when_a_newer_run_is_queued(poll_script: str, workflow: dict) assert "dispatch reap will stop" in poll_script run_env = yaml.safe_dump(workflow["jobs"]["run"].get("env", {})) assert "HEAD_REF:" in run_env + + +def test_poll_passes_on_clean_terminal_stop_reason_not_gain(poll_script: str) -> None: + """Gate PASS aligns with optimize CLI exit 0, not cumulative_gain vs TARGET_GAIN.""" + assert "is_clean_stop_reason" in poll_script + assert "target_reached|global_converged|time_exhausted|max_ticks|sweep_done|conc_sweep_done" in poll_script + assert "not used to judge PASS" in poll_script + assert 'echo "PASS|stop=${stop} gain=${gain}%"' in poll_script + assert "gain=${gain}% < ${TARGET_GAIN}" not in poll_script + assert "reached target_gain=" not in poll_script + assert "clean terminal stop_reason" in poll_script From 4aa660c88132b18ac84d63608308c1f911d1593e Mon Sep 17 00:00:00 2001 From: lishuoshuo-amd Date: Fri, 28 Aug 2026 17:06:54 +0800 Subject: [PATCH 36/52] pre-release-e2e: tier docker leg memory and raise host CPU Set nested container caps to 256g/512g for 3h/12h legs (64g shm) to avoid OOM on 14B-FP8 docker paths, and bump the privileged docker-host CPU request to 196 for four parallel agent/setup processes atop 4x32 container caps. Co-authored-by: Cursor --- .github/pre-release/bootstrap-pre-release.sh | 19 ++++++++++++++++-- .github/scripts/pre-release-e2e-dispatch.sh | 21 ++++++++++++++++---- 2 files changed, 34 insertions(+), 6 deletions(-) diff --git a/.github/pre-release/bootstrap-pre-release.sh b/.github/pre-release/bootstrap-pre-release.sh index 205a8c949e..33230b8f57 100755 --- a/.github/pre-release/bootstrap-pre-release.sh +++ b/.github/pre-release/bootstrap-pre-release.sh @@ -208,13 +208,28 @@ run_leg() { esac echo "HYPERLOOM_SKILL_PATH=${root}/.claude/skills/${demo_skill}/SKILL.md" echo "HYPERLOOM_CONTAINER_NAME=hyperloom-${leg}" # unique per leg (shared host dockerd) - echo "HYPERLOOM_SHM_SIZE=${LEG_SHM:-64g}" + local leg_mem leg_shm + case "$leg" in + *-3h) + leg_mem="${DOCKER_LEG_MEM_3H:-256g}" + leg_shm="${DOCKER_LEG_SHM_3H:-64g}" + ;; + *-12h) + leg_mem="${DOCKER_LEG_MEM_12H:-512g}" + leg_shm="${DOCKER_LEG_SHM_12H:-64g}" + ;; + *) + leg_mem="${LEG_MEM:-256g}" + leg_shm="${LEG_SHM:-64g}" + ;; + esac + echo "HYPERLOOM_SHM_SIZE=${leg_shm}" echo "E2E_GPU_INDEX=${GPU_INDEX}" echo "E2E_RENDERD=${dk_rd}" echo "E2E_KFD_GID=${dk_kfd_gid}" echo "E2E_DRI_GID=${dk_dri_gid}" echo "E2E_LEG_CPUS=${LEG_CPUS:-32}" - echo "E2E_LEG_MEM=${LEG_MEM:-128g}" + echo "E2E_LEG_MEM=${leg_mem}" echo "E2E_NFS_MOUNT=${dk_nfs_mount}" fi } > "$envf" diff --git a/.github/scripts/pre-release-e2e-dispatch.sh b/.github/scripts/pre-release-e2e-dispatch.sh index fd4d46ebaf..2f74afbc89 100755 --- a/.github/scripts/pre-release-e2e-dispatch.sh +++ b/.github/scripts/pre-release-e2e-dispatch.sh @@ -40,10 +40,14 @@ # DISPATCH_MAP output file: JSON {leg: workloadId} # (default $RUNNER_TEMP/pre_release_dispatch.json) # HOST_CPU / HOST_MEM / HOST_SHM / HOST_EPHEMERAL privileged host resource request -# (default 128 / 2048Gi / 256Gi / 1792Gi -- ref 8-GPU Authoring pod) +# (default 196 / 2048Gi / 256Gi / 1792Gi -- ref 8-GPU Authoring pod +# uses 128 CPU; +68 for dockerd + 4 parallel agent/setup processes on +# top of 4x32 CPU-capped nested containers) # LEG_CPU / LEG_MEM / LEG_EPHEMERAL baremetal leg resource request # (default 32 / 512Gi / 512Gi -- sglang 14B-FP8 + roofline/aiter JIT # exceeded 128Gi/100Gi on 2026-08-28) +# DOCKER_LEG_MEM_3H / DOCKER_LEG_MEM_12H / DOCKER_LEG_SHM_3H / DOCKER_LEG_SHM_12H +# nested docker container caps (default 256g / 512g / 64g / 64g) # DEADLINE_3H_S / DEADLINE_12H_S pod hard-timeout per duration # (default 16200 = 3h+1h+30m / 48600 = 12h+1h+30m). The docker host # pod uses the MAX over its legs. SaFE kills the pod at the @@ -57,15 +61,20 @@ set -euo pipefail NFS_ROOT="${NFS_ROOT:-/shared_nfs/hyperloom-pre-release-e2e-test}" TARGET_GAIN="${TARGET_GAIN:-100}" -# Sized to a proven Running 8-GPU Authoring pod (ref: sglang-kimik3-2): CPU 128, +# Sized to a proven Running 8-GPU Authoring pod (ref: sglang-kimik3-2): CPU 128 baseline, +# bumped to 196 for four parallel nested legs (4x32 container CPU caps + host/agent headroom). # mem 2048Gi, ephemeral 1792Gi. Every writable path the DinD host has -- the container # rootfs AND the /shared-data emptyDir the nested dockerd stores images in -- counts # toward this one ephemeralStorage quota, so the host bootstrap requires a # layer-deduplicating docker storage driver (overlay2) to stay inside it. -HOST_CPU="${HOST_CPU:-128}"; HOST_MEM="${HOST_MEM:-2048Gi}"; HOST_SHM="${HOST_SHM:-256Gi}" +HOST_CPU="${HOST_CPU:-196}"; HOST_MEM="${HOST_MEM:-2048Gi}"; HOST_SHM="${HOST_SHM:-256Gi}" HOST_EPHEMERAL="${HOST_EPHEMERAL:-1792Gi}" LEG_CPU="${LEG_CPU:-32}"; LEG_MEM="${LEG_MEM:-512Gi}" LEG_EPHEMERAL="${LEG_EPHEMERAL:-512Gi}" +DOCKER_LEG_MEM_3H="${DOCKER_LEG_MEM_3H:-256g}" +DOCKER_LEG_MEM_12H="${DOCKER_LEG_MEM_12H:-512g}" +DOCKER_LEG_SHM_3H="${DOCKER_LEG_SHM_3H:-64g}" +DOCKER_LEG_SHM_12H="${DOCKER_LEG_SHM_12H:-64g}" # SaFE workload scheduling priority (Spec.Priority, an int): High=2, Med=1, Low=0 # (Primus-SaFE common/constant.go). The scheduler orders the queue by this value, and # the webhook clamps it into [0,2]. These release-gate legs hold 8 GPUs for up to 14h @@ -362,6 +371,8 @@ if [ "$want_docker_host" = 1 ]; then --arg baseurl "${ANTHROPIC_BASE_URL:-}" \ --arg cheaders "${ANTHROPIC_CUSTOM_HEADERS:-}" \ --arg legs "$docker_legs" --argjson gpumap "$gpu_map" \ + --arg dm3 "$DOCKER_LEG_MEM_3H" --arg dm12 "$DOCKER_LEG_MEM_12H" \ + --arg ds3 "$DOCKER_LEG_SHM_3H" --arg ds12 "$DOCKER_LEG_SHM_12H" \ '{ CI_VERSION:$civ, NFS_ROOT:$nfs, MODEL_3H:$m3, MODEL_12H:$m12, @@ -369,7 +380,9 @@ if [ "$want_docker_host" = 1 ]; then ANTHROPIC_API_KEY_B64:$keyb64, HYPERLOOM_RUN_MODE:"docker", E2E_DOCKER_HOST:"1", - DOCKER_LEGS:$legs, DOCKER_GPU_MAP:($gpumap|tostring) + DOCKER_LEGS:$legs, DOCKER_GPU_MAP:($gpumap|tostring), + DOCKER_LEG_MEM_3H:$dm3, DOCKER_LEG_MEM_12H:$dm12, + DOCKER_LEG_SHM_3H:$ds3, DOCKER_LEG_SHM_12H:$ds12 } + (if $baseurl == "" then {} else {ANTHROPIC_BASE_URL:$baseurl} end) + (if $cheaders == "" then {} else {ANTHROPIC_CUSTOM_HEADERS:$cheaders} end)')" From e7fd8823042f523f1d0965ceaf678bc41798a825 Mon Sep 17 00:00:00 2001 From: lishuoshuo-amd Date: Sat, 29 Aug 2026 12:14:32 +0800 Subject: [PATCH 37/52] pre-release-e2e: gate on state.json stop_reason and fix 3h/12h demos Judge PASS/FAIL from state.json stop_reason instead of final.json, and complete bootstrap when optimize exits cleanly even if final.json is late. Sync 3h to framework-only (no --no-framework-agent, 0.90 framework pct), align 12h prompts with the 14b skill, and raise demo re-drive budget to 5. Co-authored-by: Cursor --- .github/pre-release/bootstrap-pre-release.sh | 29 ++++++---- .../prompts/pre-release/demo-12h.md | 14 +++-- .../prompts/pre-release/demo-3h.md | 16 ++--- .github/scripts/pre-release-e2e-poll.sh | 58 ++++++++----------- examples/hyperloom-qwen3-8b-3h/SKILL.md | 32 +++++----- .../test_pre_release_gate_orchestration.py | 4 +- .../tests/test_pre_release_stall_liveness.py | 14 ++++- 7 files changed, 87 insertions(+), 80 deletions(-) diff --git a/.github/pre-release/bootstrap-pre-release.sh b/.github/pre-release/bootstrap-pre-release.sh index 33230b8f57..85e238eca6 100755 --- a/.github/pre-release/bootstrap-pre-release.sh +++ b/.github/pre-release/bootstrap-pre-release.sh @@ -114,6 +114,15 @@ SETUP_RESUME_NUDGE='Your previous turn ended before setup finished. Do NOT resta DEMO_RESUME_NUDGE='Your previous turn ended without leaving a running optimize behind, and nothing has been written under the workspace since, so the work is not progressing. Do NOT fabricate a result. Finish the launch in THIS turn: complete the install if it is still needed, start optimize detached with setsid nohup so it survives the end of this turn, then confirm the nested session run dir and its state.json exist and report their paths.' +# Clean terminal stop_reason values (hyperloom.inference_optimizer.cli._SUCCESS_STOP_REASONS). +is_clean_stop_reason() { + case "$1" in + target_reached|global_converged|time_exhausted|max_ticks|sweep_done|conc_sweep_done) + return 0 ;; + *) return 1 ;; + esac +} + # Run ONE leg to completion inside the current filesystem (baremetal pod, or already # inside a nested docker container). Args: leg backend model_path hours run_mode run_leg() { @@ -334,7 +343,7 @@ run_leg() { # `claude --print` is ONE non-interactive turn: it returns right after the demo # skill backgrounds `optimize` (setsid nohup). If we returned now, run.sh would # exit 0 and SaFE would mark a FALSE "Succeeded" while the benchmark is still - # running. Block here until the run writes reports/final.json (or a deadline). + # running. Block here until state.json carries a terminal stop_reason (or a deadline). # # The real artifacts do NOT live under $session directly: make_session_dir() # creates a NESTED per-run dir $session//-/ @@ -357,7 +366,6 @@ run_leg() { # and that pre-loop gap must not be charged to the leg, or the very first # iteration condemns it. local stall_grace="${LEG_STALL_GRACE_S:-600}" # 10m of NO file writes -> dead - local final_grace="${LEG_FINAL_GRACE_S:-120}" # state stop_reason -> final.json local deadline_s=$(( hours * 3600 + 3600 )) # demo budget + 1h margin (hard cap) # Two clocks: start_ts backs the hard deadline and is NEVER reset (resetting it would # let the leg outlive the SaFE pod timeout and lose the clean failure path); grace_ts @@ -371,7 +379,7 @@ run_leg() { # stall signal instead of an "is it launched yet" probe right after the turn: the demo # skill's launcher runs install.sh BEFORE backgrounding optimize, so state.json can # legitimately be 10+ minutes away, and re-driving on a short grace would double-launch. - local demo_redrives=0 max_demo_redrives="${LEG_DEMO_REDRIVES:-2}" + local demo_redrives=0 max_demo_redrives="${LEG_DEMO_REDRIVES:-5}" while :; do local now elapsed; now="$(date +%s)"; elapsed=$(( now - start_ts )) @@ -409,26 +417,23 @@ run_leg() { if [ -n "$real_sdir" ]; then if [ -f "$final_json" ]; then - log "leg $leg final.json present after ${elapsed}s; demo ran to completion" + log "leg $leg final.json present after ${elapsed}s; demo complete" return 0 fi local stop="" [ -f "$state_json" ] && stop="$(jq -r '.stop_reason // ""' "$state_json" 2>/dev/null || echo "")" if [ -n "$stop" ]; then - log "leg $leg state.json stop_reason='$stop'; waiting up to ${final_grace}s for final.json" - local g0; g0="$(date +%s)" - while [ ! -f "$final_json" ] && [ $(( $(date +%s) - g0 )) -lt "$final_grace" ]; do sleep 5; done - if [ -f "$final_json" ]; then - log "leg $leg final.json present (stop_reason='$stop'); demo complete" + if is_clean_stop_reason "$stop"; then + log "leg $leg state.json stop_reason='$stop' after ${elapsed}s; demo complete" return 0 fi - log "ERROR: leg $leg state stop_reason='$stop' but final.json never appeared within ${final_grace}s" + log "ERROR: leg $leg state stop_reason='$stop' (not a clean terminal exit)" return 1 fi fi if [ "$elapsed" -ge "$deadline_s" ]; then - log "ERROR: leg $leg deadline ${deadline_s}s reached without reports/final.json (real_sdir='${real_sdir:-}')" + log "ERROR: leg $leg deadline ${deadline_s}s reached without a terminal stop_reason (real_sdir='${real_sdir:-}')" return 1 fi sleep "$wait_interval" @@ -526,7 +531,7 @@ ensure_dockerd() { # run_leg in docker mode. Each leg's agent follows the demo skill to `docker run` its OWN # single-GPU container (renderD = 128 + gpu_index*8), applying the CI isolation flags that # run_leg injected into the leg .env. No nested bootstrap: session artifacts land under -# $session on this pod's NFS exactly as for baremetal, so the wait-for-final.json loop in +# $session on this pod's NFS exactly as for baremetal, so the wait-for-stop_reason loop in # run_leg works unchanged. Each `run_leg &` is its own subshell, so their per-leg EXIT # traps (the .env key scrub) don't clobber each other. run_docker_host() { diff --git a/.github/pre-release/prompts/pre-release/demo-12h.md b/.github/pre-release/prompts/pre-release/demo-12h.md index 4e3f88ce2f..63a569f463 100644 --- a/.github/pre-release/prompts/pre-release/demo-12h.md +++ b/.github/pre-release/prompts/pre-release/demo-12h.md @@ -8,16 +8,18 @@ its exact default flags. ## Flags -- **OVERRIDE:** use `--target-gain 100` (NOT the skill's default of 50). This is the - pre-release release gate — the run must reach a validated cumulative gain of 100%. +- **OVERRIDE:** use `--target-gain 100` (NOT the skill's default of 50). This shapes + optimize prompts only; the poll gate judges PASS/FAIL from `stop_reason`, not gain. - Keep every other required flag exactly as the skill defines them: ``` --tp 1 --conc 64 --isl 1024 --osl 1024 --precision fp8 --max-hours 12 - --max-minutes-framework-pct 0.01 --max-minutes-explore-pct 0.42 - --max-minutes-kernel-pct 0.42 + --max-minutes-framework-pct 0.43 --max-minutes-kernel-pct 0.42 ``` + Do **not** pass `--no-framework-agent` or `--no-kernel` — the 12h demo runs the full + OPTIMIZE phase (FRAMEWORK_AGENT + KERNEL_AGENT). + ## Model path The skill will ask which model to use. Do **not** ask interactively — use @@ -50,5 +52,5 @@ killed the moment the turn ends. So: 3. Before you finish, confirm the run is really live and report the paths: the nested session run dir exists, `state.json` is present in it, and the optimizer PID is alive. -Only then stop. The harness then waits for the terminal report (`reports/final.json` + -`reports/final.md`) and judges PASS/FAIL from it — do not fabricate a result. +Only then stop. The harness polls `state.json` for a clean terminal `stop_reason` to +judge PASS/FAIL — do not fabricate a result. diff --git a/.github/pre-release/prompts/pre-release/demo-3h.md b/.github/pre-release/prompts/pre-release/demo-3h.md index 4cb32668dc..8a606094c9 100644 --- a/.github/pre-release/prompts/pre-release/demo-3h.md +++ b/.github/pre-release/prompts/pre-release/demo-3h.md @@ -8,17 +8,19 @@ exact default flags. ## Flags -- **OVERRIDE:** use `--target-gain 100` (NOT the skill's default of 30). This is the - pre-release release gate — the run must reach a validated cumulative gain of 100%. +- **OVERRIDE:** use `--target-gain 100` (NOT the skill's default of 30). This shapes + optimize prompts only; the poll gate judges PASS/FAIL from `stop_reason`, not gain. - Keep every other required flag exactly as the skill defines them: ``` --tp 1 --conc 64 --isl 1024 --osl 1024 --precision bf16 --max-hours 3 - --max-minutes-explore-pct 0.39 --max-minutes-sweep-pct 0.01 - --explore-force-exit-budget-pct 0.01 --no-framework-agent --no-kernel - --no-enable-conc-sweep --no-enable-roofline + --max-minutes-framework-pct 0.90 --max-minutes-sweep-pct 0.01 + --no-kernel --no-enable-conc-sweep --no-enable-roofline ``` + Do **not** pass `--no-framework-agent` — the 3h demo runs FRAMEWORK_AGENT (OPTIMIZE) + only; kernel is skipped via `--no-kernel`. + ## Model path The skill will ask which model to use. Do **not** ask interactively — use @@ -51,5 +53,5 @@ killed the moment the turn ends. So: 3. Before you finish, confirm the run is really live and report the paths: the nested session run dir exists, `state.json` is present in it, and the optimizer PID is alive. -Only then stop. The harness then waits for the terminal report (`reports/final.json` + -`reports/final.md`) and judges PASS/FAIL from it — do not fabricate a result. +Only then stop. The harness polls `state.json` for a clean terminal `stop_reason` to +judge PASS/FAIL — do not fabricate a result. diff --git a/.github/scripts/pre-release-e2e-poll.sh b/.github/scripts/pre-release-e2e-poll.sh index 9de019d878..195148703e 100755 --- a/.github/scripts/pre-release-e2e-poll.sh +++ b/.github/scripts/pre-release-e2e-poll.sh @@ -7,10 +7,9 @@ # (design §10, point C -- the 3h legs finish ~3-4h and report first; 12h legs later). # # A leg PASSes only when ALL hold (design §9): -# 1. reports/final.json and reports/final.md both exist, -# 2. final.json stop_reason is a clean terminal exit (same set as optimize CLI exit 0), -# 3. its owning SaFE workload phase is not Failed/Stopped, -# 4. crash_count / server_boot_failures are within tolerance. +# 1. state.json stop_reason is a clean terminal exit (same set as optimize CLI exit 0), +# 2. crash_count is within tolerance (read from state.json). +# final.json is not required; bootstrap may fail waiting for it while optimize succeeded. # TARGET_GAIN still flows to optimize via the demo skill; it is NOT used here to judge PASS. # Fail-fast leaves still-running optimize legs alive; dispatch reap stops stale e2e-* tags. # @@ -275,34 +274,30 @@ is_clean_stop_reason() { esac } -# Judge one leg from its final.json. Echoes "PASS"|"FAIL|". +# Judge one leg from state.json. Echoes "PASS"|"PENDING"|"FAIL|". judge_leg() { - local leg="$1" wphase="$2" sdir final gain stop crashes boots + local leg="$1" wphase="$2" sdir state gain stop crashes sdir="$(leg_session_dir "$leg")" if [ -z "$sdir" ] || [ ! -d "$sdir" ]; then - echo "FAIL|no session dir yet (workload phase=$wphase)"; return + echo "PENDING|no session dir yet (workload phase=$wphase)"; return fi - final="${sdir%/}/reports/final.json" - if [ ! -f "$final" ]; then - echo "FAIL|reports/final.json missing (workload phase=$wphase)"; return + state="${sdir%/}/state.json" + if [ ! -f "$state" ]; then + echo "PENDING|state.json missing (workload phase=$wphase)"; return fi - if [ ! -f "${sdir%/}/reports/final.md" ]; then - echo "FAIL|reports/final.md missing"; return - fi - stop="$(jq -r '.stop_reason // ""' "$final" 2>/dev/null || echo "")" - gain="$(jq -r '.cumulative_gain_validated // 0' "$final" 2>/dev/null || echo 0)" - crashes="$(jq -r '.crash_count // 0' "$final" 2>/dev/null || echo 0)" - boots="$(jq -r '.server_boot_failures // 0' "$final" 2>/dev/null || echo 0)" + stop="$(jq -r '.stop_reason // ""' "$state" 2>/dev/null || echo "")" + gain="$(jq -r '.cumulative_gain_validated // 0' "$state" 2>/dev/null || echo 0)" + crashes="$(jq -r '.crash_count // 0' "$state" 2>/dev/null || echo 0)" if [ "$crashes" -gt "$MAX_CRASHES" ] 2>/dev/null; then echo "FAIL|crash_count=$crashes > $MAX_CRASHES"; return fi - if [ "$boots" -gt "$MAX_BOOT_FAILS" ] 2>/dev/null; then - echo "FAIL|server_boot_failures=$boots > $MAX_BOOT_FAILS"; return + if [ -z "$stop" ]; then + echo "PENDING|state.json stop_reason not set yet (workload phase=$wphase)"; return fi if is_clean_stop_reason "$stop"; then echo "PASS|stop=${stop} gain=${gain}%"; return fi - echo "FAIL|stop_reason=${stop:-none} (not a clean terminal exit)" + echo "FAIL|stop_reason=${stop} (not a clean terminal exit)" } # ---- poll loop ------------------------------------------------------------- @@ -337,15 +332,6 @@ while :; do [ -n "${VERDICT[$leg]}" ] && continue wid="${WID[$leg]}" wphase="$(workload_phase "$wid")" - # Terminal SaFE failure kills the leg immediately. - if [ "$wphase" = "Failed" ] || [ "$wphase" = "Stopped" ]; then - VERDICT["$leg"]="FAIL|workload phase=$wphase" - summary "❌ **$leg** — FAIL (workload $wphase, wid=\`$wid\`)" - post_status "$leg" failure "workload $wphase; wid=$wid" - changed=1; fail_seen=1 - continue - fi - # Otherwise judge from the on-disk report (present once the leg finishes). res="$(judge_leg "$leg" "$wphase")" verdict="${res%%|*}"; detail="${res#*|}" if [ "$verdict" = "PASS" ]; then @@ -353,14 +339,20 @@ while :; do summary "✅ **$leg** — PASS ($detail)" post_status "$leg" success "PASS — $detail" changed=1 - elif [ "$wphase" = "Succeeded" ]; then - # Workload ended but the report did not clear the gate -> terminal FAIL. + elif [ "$verdict" = "PENDING" ]; then + if [ "$wphase" = "Succeeded" ] || [ "$wphase" = "Failed" ] || [ "$wphase" = "Stopped" ]; then + VERDICT["$leg"]="FAIL|$detail (workload phase=$wphase)" + summary "❌ **$leg** — FAIL ($detail; workload $wphase, wid=\`$wid\`)" + post_status "$leg" failure "FAIL — $detail" + changed=1; fail_seen=1 + else + pending=$((pending + 1)) + fi + else VERDICT["$leg"]="FAIL|$detail" summary "❌ **$leg** — FAIL ($detail)" post_status "$leg" failure "FAIL — $detail" changed=1; fail_seen=1 - else - pending=$((pending + 1)) # still running; check again next tick fi done diff --git a/examples/hyperloom-qwen3-8b-3h/SKILL.md b/examples/hyperloom-qwen3-8b-3h/SKILL.md index 95e602931f..0448d215aa 100644 --- a/examples/hyperloom-qwen3-8b-3h/SKILL.md +++ b/examples/hyperloom-qwen3-8b-3h/SKILL.md @@ -1,9 +1,9 @@ --- name: hyperloom-qwen3-8b-3h -description: Run a 3-hour Hyperloom Qwen3-8B optimization session without the Kernel Agent. Use when the user wants a short, no-kernel Hyperloom demo on the local AMD ROCm environment. +description: Run a 3-hour Hyperloom Qwen3-8B FRAMEWORK_AGENT (OPTIMIZE) session without the Kernel Agent. Use when the user wants a short, framework-only Hyperloom demo on the local AMD ROCm environment. --- -# Hyperloom Qwen3-8B 3h No-Kernel Run +# Hyperloom Qwen3-8B 3h Framework-Only (No-Kernel) Run Read `.env` first and resolve `HYPERLOOM_SKILL_PATH`. Read and follow the optimizer skill at `@${HYPERLOOM_SKILL_PATH}` before launching. If `HYPERLOOM_SKILL_PATH` is missing, fall back to `@hyperloom/inference_optimizer/SKILL.md` (wheel install) or `@src/hyperloom/inference_optimizer/SKILL.md` (source checkout). This skill provides the concrete workload and launch constraints for a short Qwen3-8B demo. @@ -84,9 +84,8 @@ Required optimize CLI flags: - `--precision bf16` - `--target-gain 30` - `--max-hours 3` -- `--max-minutes-framework-pct 0.44` +- `--max-minutes-framework-pct 0.90` - `--max-minutes-sweep-pct 0.01` -- `--no-framework-agent` - `--no-kernel` - `--no-enable-conc-sweep` - `--no-enable-roofline` @@ -201,17 +200,14 @@ and the stop reason. Never print API keys, tokens, or custom header values. and critic subprocesses can import `hyperloom.agents` after changing cwd. 3. Run in background with `setsid nohup`. 4. Pass all required optimize CLI flags in the `python -m hyperloom.inference_optimizer.cli optimize` command. Do not rely on `.env` alone for `TP`, `CONC`, `ISL`, `OSL`, or `PRECISION`; CLI defaults can otherwise override the intended workload. -5. Include `--max-minutes-framework-pct 0.44` and `--max-minutes-sweep-pct 0.01` - in the optimize command. With FRAMEWORK_AGENT and KERNEL_AGENT disabled, - Hyperloom redistributes their shares so most of the short run budget is - reserved for OPTIMIZE while still leaving SWEEP/CLOSE time to exit cleanly - near the deadline. -6. Include `--no-framework-agent` in the optimize command so the - FRAMEWORK_AGENT phase is skipped. -7. Include `--no-kernel` in the optimize command so the Kernel Agent phase is skipped. -8. Include `--no-enable-conc-sweep` in the optimize command so the SWEEP-phase post-optimization concurrency sweep is skipped. -9. Include `--no-enable-roofline` in the optimize command so PRELUDE uses the lighter profile path instead of roofline analysis. -10. Report the session ID, log path, PID, and initial health check result. -11. Monitor the process every 300 seconds until work is done. -12. To recover an unexpected crash, only run `optimize --resume-from "$SESSION_DIR"` against the same session dir. After the first launch, never start a new `optimize`; that creates a new `` session and is forbidden. -13. If `stop_reason` in the current session `state.json` is final, stop and exit. +5. Include `--max-minutes-framework-pct 0.90` and `--max-minutes-sweep-pct 0.01` + in the optimize command. With `--no-kernel`, KERNEL_AGENT is disabled and its + budget share is redistributed mostly to FRAMEWORK_AGENT (~99% of wall clock). + Do **not** pass `--no-framework-agent` — that skips OPTIMIZE entirely. +6. Include `--no-kernel` in the optimize command so the Kernel Agent phase is skipped. +7. Include `--no-enable-conc-sweep` in the optimize command so the SWEEP-phase post-optimization concurrency sweep is skipped. +8. Include `--no-enable-roofline` in the optimize command so PRELUDE uses the lighter profile path instead of roofline analysis. +9. Report the session ID, log path, PID, and initial health check result. +10. Monitor the process every 300 seconds until work is done. +11. To recover an unexpected crash, only run `optimize --resume-from "$SESSION_DIR"` against the same session dir. After the first launch, never start a new `optimize`; that creates a new `` session and is forbidden. +12. If `stop_reason` in the current session `state.json` is final, stop and exit. diff --git a/src/hyperloom/inference_optimizer/tests/test_pre_release_gate_orchestration.py b/src/hyperloom/inference_optimizer/tests/test_pre_release_gate_orchestration.py index ecbc40fdde..05bf0667d7 100644 --- a/src/hyperloom/inference_optimizer/tests/test_pre_release_gate_orchestration.py +++ b/src/hyperloom/inference_optimizer/tests/test_pre_release_gate_orchestration.py @@ -150,4 +150,6 @@ def test_poll_passes_on_clean_terminal_stop_reason_not_gain(poll_script: str) -> assert 'echo "PASS|stop=${stop} gain=${gain}%"' in poll_script assert "gain=${gain}% < ${TARGET_GAIN}" not in poll_script assert "reached target_gain=" not in poll_script - assert "clean terminal stop_reason" in poll_script + assert "state.json stop_reason" in poll_script + assert "reports/final.json missing" not in poll_script + assert 'echo "PENDING|state.json stop_reason not set yet' in poll_script diff --git a/src/hyperloom/inference_optimizer/tests/test_pre_release_stall_liveness.py b/src/hyperloom/inference_optimizer/tests/test_pre_release_stall_liveness.py index a5546c51f0..6366cf63fe 100644 --- a/src/hyperloom/inference_optimizer/tests/test_pre_release_stall_liveness.py +++ b/src/hyperloom/inference_optimizer/tests/test_pre_release_stall_liveness.py @@ -3,8 +3,8 @@ """Regression guard for the pre-release E2E leg liveness (stall) check. -``bootstrap-pre-release.sh`` blocks after the demo turn until ``optimize`` writes -``reports/final.json``, and declares a leg dead when nothing has been written for +``bootstrap-pre-release.sh`` blocks after the demo turn until ``optimize`` writes a +clean terminal ``stop_reason`` into ``state.json``, and declares a leg dead when nothing has been written for ``LEG_STALL_GRACE_S``. Two properties of that check killed legs that were provably still alive (run 1.0.1a0.dev202608280354+ci, both baremetal-sglang legs): @@ -177,10 +177,18 @@ def test_setup_marker_matches_every_setup_prompt(script: str) -> None: def test_an_early_turn_is_re_driven_not_fatal(script: str) -> None: """A turn that ends without finishing leaves nothing running; ask again, bounded.""" - assert 'max_demo_redrives="${LEG_DEMO_REDRIVES:-2}"' in script + assert 'max_demo_redrives="${LEG_DEMO_REDRIVES:-5}"' in script assert "demo_redrives=$(( demo_redrives + 1 ))" in script +def test_bootstrap_completes_on_clean_stop_reason_without_final_json(script: str) -> None: + """Align with the poll gate: a clean terminal stop_reason is enough to exit 0.""" + assert "is_clean_stop_reason" in script + assert "final.json never appeared" not in script + assert "LEG_FINAL_GRACE_S" not in script + assert "state.json stop_reason='$stop' after" in script + + def test_setup_is_budgeted_in_time_not_in_turns(script: str) -> None: """A count-based cap of 3 turns was ~4min of wall clock and killed live installs. From a9f16bb9507f6a290c76a82341ff0f9ea77e6767 Mon Sep 17 00:00:00 2001 From: lishuoshuo-amd Date: Sat, 29 Aug 2026 12:16:22 +0800 Subject: [PATCH 38/52] examples: document 12h framework/kernel budget and full OPTIMIZE phase Mirror the 3h skill launch requirements: spell out phase budget flags and forbid --no-framework-agent / --no-kernel. Keep target-gain at 50 for users. Co-authored-by: Cursor --- examples/hyperloom-qwen3-14b-fp8-12h/SKILL.md | 11 +++++++---- 1 file changed, 7 insertions(+), 4 deletions(-) diff --git a/examples/hyperloom-qwen3-14b-fp8-12h/SKILL.md b/examples/hyperloom-qwen3-14b-fp8-12h/SKILL.md index e656a8dbe8..6ce6b0c992 100644 --- a/examples/hyperloom-qwen3-14b-fp8-12h/SKILL.md +++ b/examples/hyperloom-qwen3-14b-fp8-12h/SKILL.md @@ -197,7 +197,10 @@ and the stop reason. Never print API keys, tokens, or custom header values. and critic subprocesses can import `hyperloom.agents` after changing cwd. 3. Run in background with `setsid nohup`. 4. Pass all required optimize CLI flags in the `python -m hyperloom.inference_optimizer.cli optimize` command. Do not rely on `.env` alone for `TP`, `CONC`, `ISL`, `OSL`, or `PRECISION`; CLI defaults can otherwise override the intended workload. -5. Report the session ID, log path, PID, and initial health check result. -6. Monitor the process every 300 seconds until work is done. -7. To recover an unexpected crash, only run `optimize --resume-from "$SESSION_DIR"` against the same session dir. After the first launch, never start a new `optimize`; that creates a new `` session and is forbidden. -8. If `stop_reason` in the current session `state.json` is final, stop and exit. +5. Include `--max-minutes-framework-pct 0.43` and `--max-minutes-kernel-pct 0.42` + in the optimize command. Do **not** pass `--no-framework-agent` or `--no-kernel` — + this demo runs the full OPTIMIZE phase (FRAMEWORK_AGENT + KERNEL_AGENT). +6. Report the session ID, log path, PID, and initial health check result. +7. Monitor the process every 300 seconds until work is done. +8. To recover an unexpected crash, only run `optimize --resume-from "$SESSION_DIR"` against the same session dir. After the first launch, never start a new `optimize`; that creates a new `` session and is forbidden. +9. If `stop_reason` in the current session `state.json` is final, stop and exit. From 4c390c1a0510c4f2651cf814e653c7ddea8f4372 Mon Sep 17 00:00:00 2001 From: lishuoshuo-amd Date: Sat, 29 Aug 2026 12:22:34 +0800 Subject: [PATCH 39/52] pre-release-e2e: dispatch the 8-GPU docker host before baremetal legs Queue the privileged docker host first so dockerd startup and image pulls overlap with baremetal scheduling instead of starting after four 1-GPU pods. Co-authored-by: Cursor --- .github/scripts/pre-release-e2e-dispatch.sh | 52 ++++++++++--------- .../test_pre_release_gate_orchestration.py | 7 +++ 2 files changed, 34 insertions(+), 25 deletions(-) diff --git a/.github/scripts/pre-release-e2e-dispatch.sh b/.github/scripts/pre-release-e2e-dispatch.sh index 2f74afbc89..a584293b86 100755 --- a/.github/scripts/pre-release-e2e-dispatch.sh +++ b/.github/scripts/pre-release-e2e-dispatch.sh @@ -318,27 +318,21 @@ record_dispatch() { # leg workloadId -- add to the in-memory map AND the on-dis fi } -# ---- baremetal legs: one non-privileged 1-GPU workload each ---------------- leg_resources_1gpu="$(jq -n --arg cpu "$LEG_CPU" --arg mem "$LEG_MEM" --arg eph "$LEG_EPHEMERAL" \ '{replica:1, gpu:"1", cpu:$cpu, memory:$mem, ephemeralStorage:$eph}')" +# Discover requested docker legs up front so the 8-GPU host can be dispatched first. +# It schedules more slowly and spends minutes on dockerd + image pulls before the +# nested legs even start setup, so queue it before the four 1-GPU baremetal pods. want_docker_host=0 +docker_legs=""; gpu_map="{}" for leg in $REQ_TASKS; do case "$leg" in - baremetal-*) - env_json="$(common_env_json "$(leg_model_path "$leg")" "$(leg_hours "$leg")" "$(leg_backend "$leg")" \ - | jq --arg leg "$leg" '. + {LEG_ID:$leg, HYPERLOOM_RUN_MODE:"baremetal"}')" - entry="$(bootstrap_entry_b64 "")" - dl="$(leg_deadline_s "$leg")" - wid="$(create_workload "$(workload_name "$leg")" "$leg_resources_1gpu" "$env_json" false "$entry" "$dl")" - record_dispatch "$leg" "$wid" - summary "• \`$leg\` → workloadId \`$wid\` (baremetal, 1 GPU, deadline $((dl/3600))h)" - ;; docker-*) want_docker_host=1 - ;; - *) - summary "⚠️ unknown leg '$leg' ignored" + idx="$(docker_gpu_index "$leg")" + docker_legs="${docker_legs}${docker_legs:+ }${leg}" + gpu_map="$(printf '%s' "$gpu_map" | jq --arg l "$leg" --arg i "$idx" '. + {($l): $i}')" ;; esac done @@ -351,18 +345,6 @@ if [ "$want_docker_host" = 1 ]; then # The host env carries the per-leg GPU map so the host bootstrap runs each docker leg # (run_leg, docker mode) with the right GPU index; each leg's agent then `docker run`s # its own single-GPU container per the demo skill. - docker_legs=""; gpu_map="{}" - for leg in $REQ_TASKS; do - case "$leg" in - docker-*) - idx="$(docker_gpu_index "$leg")" - docker_legs="${docker_legs}${docker_legs:+ }${leg}" - gpu_map="$(printf '%s' "$gpu_map" | jq --arg l "$leg" --arg i "$idx" '. + {($l): $i}')" - ;; - esac - done - # Host env: model paths for both durations, plus the leg->gpu map. Per-leg model/ - # backend are resolved inside the host bootstrap from the leg id. host_env="$(jq -n \ --arg civ "$CI_VERSION" --arg nfs "$NFS_ROOT" \ --arg m3 "$MODEL_3H" --arg m12 "$MODEL_12H" \ @@ -405,6 +387,26 @@ if [ "$want_docker_host" = 1 ]; then done fi +# ---- baremetal legs: one non-privileged 1-GPU workload each ---------------- +for leg in $REQ_TASKS; do + case "$leg" in + baremetal-*) + env_json="$(common_env_json "$(leg_model_path "$leg")" "$(leg_hours "$leg")" "$(leg_backend "$leg")" \ + | jq --arg leg "$leg" '. + {LEG_ID:$leg, HYPERLOOM_RUN_MODE:"baremetal"}')" + entry="$(bootstrap_entry_b64 "")" + dl="$(leg_deadline_s "$leg")" + wid="$(create_workload "$(workload_name "$leg")" "$leg_resources_1gpu" "$env_json" false "$entry" "$dl")" + record_dispatch "$leg" "$wid" + summary "• \`$leg\` → workloadId \`$wid\` (baremetal, 1 GPU, deadline $((dl/3600))h)" + ;; + docker-*) + ;; + *) + summary "⚠️ unknown leg '$leg' ignored" + ;; + esac +done + # ---- dispatch map already written incrementally by record_dispatch --------- # (so a mid-dispatch cancel still leaves a complete-so-far map for cleanup to stop). echo "dispatch_map=$DISPATCH_MAP" >> "${GITHUB_OUTPUT:-/dev/null}" diff --git a/src/hyperloom/inference_optimizer/tests/test_pre_release_gate_orchestration.py b/src/hyperloom/inference_optimizer/tests/test_pre_release_gate_orchestration.py index 05bf0667d7..8ce47c9d1b 100644 --- a/src/hyperloom/inference_optimizer/tests/test_pre_release_gate_orchestration.py +++ b/src/hyperloom/inference_optimizer/tests/test_pre_release_gate_orchestration.py @@ -132,6 +132,13 @@ def test_dispatch_version_tag_is_unique_per_run(dispatch_script: str) -> None: ) +def test_docker_host_is_dispatched_before_baremetal(dispatch_script: str) -> None: + """The 8-GPU docker host schedules slowly; queue it before the 1-GPU baremetal pods.""" + docker_pos = dispatch_script.index("queue it before the four 1-GPU baremetal pods") + bare_pos = dispatch_script.index("# ---- baremetal legs: one non-privileged 1-GPU workload each") + assert docker_pos < bare_pos + + def test_poll_exits_when_a_newer_run_is_queued(poll_script: str, workflow: dict) -> None: """A pending successor cannot dispatch until this poll releases the runner.""" assert "superseded_by_newer_run" in poll_script From 261198944b41703a8b6a899f5ea8a5fbb726c95b Mon Sep 17 00:00:00 2001 From: lishuoshuo-amd Date: Sat, 29 Aug 2026 22:25:45 +0800 Subject: [PATCH 40/52] Fix pre-release poll state.json access and keep polling all legs. Read root-only state.json via sudo on the baremetal runner, publish readable permissions from bootstrap, continue per-leg polling after gate FAIL, and use distinct report icons for SKIP vs FAIL. Co-authored-by: Cursor --- .github/pre-release/bootstrap-pre-release.sh | 12 +++ .github/scripts/pre-release-e2e-poll.sh | 92 +++++++++++-------- .../test_pre_release_gate_orchestration.py | 40 +++++--- .../tests/test_pre_release_stall_liveness.py | 7 ++ 4 files changed, 100 insertions(+), 51 deletions(-) diff --git a/.github/pre-release/bootstrap-pre-release.sh b/.github/pre-release/bootstrap-pre-release.sh index 85e238eca6..150ecdddaa 100755 --- a/.github/pre-release/bootstrap-pre-release.sh +++ b/.github/pre-release/bootstrap-pre-release.sh @@ -123,6 +123,14 @@ is_clean_stop_reason() { esac } +# Optimize writes state.json root-only; the poll runner reads as ubuntu. +publish_state_for_poll() { + local state_json="$1" sdir="$2" session="$3" + [ -f "$state_json" ] || return 0 + chmod a+r "$state_json" 2>/dev/null || true + chmod a+X "$sdir" "$session" 2>/dev/null || true +} + # Run ONE leg to completion inside the current filesystem (baremetal pod, or already # inside a nested docker container). Args: leg backend model_path hours run_mode run_leg() { @@ -394,6 +402,7 @@ run_leg() { log "leg $leg real session dir: $real_sdir" # Re-pin so the poll (leg_session_dir -> head -n1 .session_dir) finds the report. echo "$real_sdir" > "${session}/.session_dir" + publish_state_for_poll "$state_json" "$real_sdir" "$session" else local idle idle="$(leg_idle_s "$root" "$grace_ts" "$now")" @@ -416,6 +425,9 @@ run_leg() { fi if [ -n "$real_sdir" ]; then + if [ -f "$state_json" ]; then + publish_state_for_poll "$state_json" "$real_sdir" "$session" + fi if [ -f "$final_json" ]; then log "leg $leg final.json present after ${elapsed}s; demo complete" return 0 diff --git a/.github/scripts/pre-release-e2e-poll.sh b/.github/scripts/pre-release-e2e-poll.sh index 195148703e..282abf0533 100755 --- a/.github/scripts/pre-release-e2e-poll.sh +++ b/.github/scripts/pre-release-e2e-poll.sh @@ -11,12 +11,15 @@ # 2. crash_count is within tolerance (read from state.json). # final.json is not required; bootstrap may fail waiting for it while optimize succeeded. # TARGET_GAIN still flows to optimize via the demo skill; it is NOT used here to judge PASS. -# Fail-fast leaves still-running optimize legs alive; dispatch reap stops stale e2e-* tags. +# When the gate is already FAIL, poll keeps running until every leg reaches a terminal +# verdict so per-leg GitHub checks and the sticky report stay aligned with optimize. +# Superseded runs still exit early and leave workloads for dispatch reap. # # The exit code is 0 only if every requested leg PASSed. # -# Requires: bash, curl, jq on the (self-hosted, in-network) runner with the NFS -# runs/ tree readable. +# Requires: bash, curl, jq on the (self-hosted, in-network) runner. state.json is +# written root-only inside pods; poll reads it via sudo -n when needed (passwordless +# sudo on the baremetal runner, same as the NFS-root setup step). # # Inputs (env): # SAFE_API_BASE / SAFE_API_KEY SaFE API (required) @@ -41,14 +44,6 @@ POLL_INTERVAL_S="${POLL_INTERVAL_S:-120}" GLOBAL_TIMEOUT_S="${GLOBAL_TIMEOUT_S:-50400}" MAX_CRASHES="${MAX_CRASHES:-0}" MAX_BOOT_FAILS="${MAX_BOOT_FAILS:-0}" -# Stop polling as soon as one leg FAILs. This is a release GATE: the first FAIL already -# blocks the release, so the remaining legs cannot change the verdict -- and waiting them -# out costs the single self-hosted runner, which in turn keeps the next fix's run stuck at -# run-level `pending` (a newer run gets no jobs at all while this one holds the -# concurrency group). Still-running workloads are LEFT ALIVE (see leave_running_wids) so -# they can finish for debugging; only the poll job exits. Set POLL_FAIL_FAST=0 to keep -# polling until every leg reaches a terminal verdict anyway. -POLL_FAIL_FAST="${POLL_FAIL_FAST:-1}" LEAVE_RUNNING_FILE="${LEAVE_RUNNING_FILE:-${DISPATCH_MAP}.leave_running}" # Sleep in short slices instead of one long one so a cancelled job tears down in seconds # rather than at the end of a full POLL_INTERVAL_S. Each slice also re-checks whether a @@ -122,14 +117,18 @@ superseded_by_newer_run() { } mark_superseded_and_exit_poll() { # -> sets superseded=1, marks pending legs SKIP, breaks caller loop - local leg pending=0 + local leg pending=0 leave_wids=() for leg in "${LEGS[@]}"; do [ -n "${VERDICT[$leg]}" ] && continue VERDICT["$leg"]="SKIP|superseded by newer run (dispatch reap will stop)" summary "⏳ **$leg** — superseded (newer run queued; workload left for dispatch reap)" post_status "$leg" pending "superseded; newer run queued" + leave_wids+=( "${WID[$leg]}" ) pending=$((pending + 1)) done + if [ "${#leave_wids[@]}" -gt 0 ]; then + printf '%s\n' "${leave_wids[@]}" | sort -u | jq -R . | jq -s . > "$LEAVE_RUNNING_FILE" + fi summary "" summary "⏹ superseded: newer pre-release run queued (run_id>${GITHUB_RUN_ID}). Releasing the runner; ${pending} workload(s) left for the successor's dispatch reap." superseded=1 @@ -220,6 +219,15 @@ report_upsert() { # body(markdown, already includes the marker on line 1) fi } +# Icon for a leg verdict in the sticky report table. +verdict_icon() { + case "$1" in + PASS) printf '%s' "✅" ;; + SKIP|PENDING) printf '%s' "⏳" ;; + *) printf '%s' "❌" ;; + esac +} + # Build the sticky report body from the current VERDICT map. `phase` is a short # status word (Running|Complete) shown in the heading. Legs with no verdict yet # render as "⏳ pending". @@ -233,7 +241,7 @@ report_body() { # phase done_count total_count continue fi vv="${v%%|*}"; vd="${v#*|}" - icon="✅"; [ "$vv" = "PASS" ] || icon="❌" + icon="$(verdict_icon "$vv")" rows="${rows}| \`${leg}\` | ${icon} ${vv} | ${vd} | " done @@ -265,6 +273,25 @@ leg_session_dir() { echo "" } +# Read one jq filter from state.json. Pods write it root-only (mode 600); the runner +# user reads via sudo -n when needed. Echoes "__UNREADABLE__" when the file exists +# but cannot be opened. +state_json_query() { + local file="$1" filter="$2" + if [ ! -f "$file" ]; then + echo ""; return 0 + fi + if [ -r "$file" ]; then + jq -r "$filter" "$file" 2>/dev/null || echo "" + return 0 + fi + if sudo -n test -r "$file" 2>/dev/null; then + sudo -n jq -r "$filter" "$file" 2>/dev/null || echo "" + return 0 + fi + echo "__UNREADABLE__" +} + # Clean terminal stop_reason values (hyperloom.inference_optimizer.cli._SUCCESS_STOP_REASONS). is_clean_stop_reason() { case "$1" in @@ -285,9 +312,12 @@ judge_leg() { if [ ! -f "$state" ]; then echo "PENDING|state.json missing (workload phase=$wphase)"; return fi - stop="$(jq -r '.stop_reason // ""' "$state" 2>/dev/null || echo "")" - gain="$(jq -r '.cumulative_gain_validated // 0' "$state" 2>/dev/null || echo 0)" - crashes="$(jq -r '.crash_count // 0' "$state" 2>/dev/null || echo 0)" + stop="$(state_json_query "$state" '.stop_reason // ""')" + gain="$(state_json_query "$state" '.cumulative_gain_validated // 0')" + crashes="$(state_json_query "$state" '.crash_count // 0')" + if [ "$stop" = "__UNREADABLE__" ] || [ "$gain" = "__UNREADABLE__" ] || [ "$crashes" = "__UNREADABLE__" ]; then + echo "PENDING|state.json not readable (workload phase=$wphase)"; return + fi if [ "$crashes" -gt "$MAX_CRASHES" ] 2>/dev/null; then echo "FAIL|crash_count=$crashes > $MAX_CRASHES"; return fi @@ -320,6 +350,7 @@ report_upsert "$(report_body Running "$(done_count)" "${#LEGS[@]}")" start_s="$(date +%s)" fail_seen=0 # has any leg reached a FAIL verdict? -> the gate is already decided +gate_fail_announced=0 superseded=0 # a newer workflow run is queued -> release runner without stopping pods while :; do if superseded_by_newer_run; then @@ -341,7 +372,7 @@ while :; do changed=1 elif [ "$verdict" = "PENDING" ]; then if [ "$wphase" = "Succeeded" ] || [ "$wphase" = "Failed" ] || [ "$wphase" = "Stopped" ]; then - VERDICT["$leg"]="FAIL|$detail (workload phase=$wphase)" + VERDICT["$leg"]="FAIL|$detail" summary "❌ **$leg** — FAIL ($detail; workload $wphase, wid=\`$wid\`)" post_status "$leg" failure "FAIL — $detail" changed=1; fail_seen=1 @@ -359,27 +390,14 @@ while :; do # A leg finished this tick -> refresh the single sticky report comment (point C). [ "$changed" -eq 1 ] && report_upsert "$(report_body Running "$(done_count)" "${#LEGS[@]}")" - [ "$pending" -eq 0 ] && break - - # Gate already lost -> release the runner, but leave still-running workloads up so they - # can finish on the cluster (useful for debugging infra vs product failures). - if [ "$POLL_FAIL_FAST" = "1" ] && [ "$fail_seen" -eq 1 ]; then - leave_wids=() - for leg in "${LEGS[@]}"; do - [ -n "${VERDICT[$leg]}" ] && continue - VERDICT["$leg"]="SKIP|still running (gate failed; workload left alive)" - summary "⏳ **$leg** — still running (gate already failed; workload left alive)" - post_status "$leg" pending "gate failed; workload left running" - leave_wids+=( "${WID[$leg]}" ) - done - if [ "${#leave_wids[@]}" -gt 0 ]; then - printf '%s\n' "${leave_wids[@]}" | sort -u | jq -R . | jq -s . > "$LEAVE_RUNNING_FILE" - summary "" - summary "⏹ fail-fast: gate is FAIL. Releasing the runner; ${pending} workload(s) left running for post-mortem. Wids recorded in \`$(basename "$LEAVE_RUNNING_FILE")\`. Set \`POLL_FAIL_FAST=0\` to poll until every leg finishes." - fi - break + if [ "$fail_seen" -eq 1 ] && [ "$gate_fail_announced" -eq 0 ]; then + summary "" + summary "⚠️ **GATE: FAIL** — at least one leg did not pass. Continuing to poll until every leg reaches a terminal verdict." + gate_fail_announced=1 fi + [ "$pending" -eq 0 ] && break + elapsed=$(( $(date +%s) - start_s )) if [ "$elapsed" -ge "$GLOBAL_TIMEOUT_S" ]; then for leg in "${LEGS[@]}"; do @@ -469,7 +487,7 @@ stop_workloads() { case " $seen " in *" $wid "*) continue ;; esac seen="${seen} ${wid}" if leave_running_wid "$wid"; then - summary "• left workload \`$wid\` running (fail-fast; post-mortem)" + summary "• left workload \`$wid\` running (superseded; dispatch reap)" continue fi code="$(curl -sS "${tls[@]}" -o /dev/null -w '%{http_code}' -X POST \ diff --git a/src/hyperloom/inference_optimizer/tests/test_pre_release_gate_orchestration.py b/src/hyperloom/inference_optimizer/tests/test_pre_release_gate_orchestration.py index 8ce47c9d1b..0c58338d3a 100644 --- a/src/hyperloom/inference_optimizer/tests/test_pre_release_gate_orchestration.py +++ b/src/hyperloom/inference_optimizer/tests/test_pre_release_gate_orchestration.py @@ -14,10 +14,9 @@ workloads; skipping reclaim`` after a 30s curl timeout, having stopped nothing. Nothing that talks to SaFE may run on a GitHub-hosted runner again. -Teardown instead relies on the old run leaving promptly: the poll fails fast on the -first FAIL (the gate is already decided), leaves still-running workloads up for post- -mortem, and sleeps in short slices so a cancel lands in seconds instead of at the end -of a full poll interval. +Teardown instead relies on the old run leaving promptly when superseded: the poll +sleeps in short slices so a cancel lands in seconds instead of at the end of a full +poll interval. After the gate is lost it keeps polling until every leg reports. There is no way to unit-test the scheduling itself short of running the workflow; these tests pin the invariants it depends on. @@ -96,26 +95,39 @@ def test_the_reap_script_is_gone_and_unreferenced() -> None: assert "pre-release-e2e-reap.sh" not in line, f"{wf.name} still runs the reap script" -def test_poll_fails_fast_once_the_gate_is_lost(poll_script: str) -> None: - """A decided gate must not keep the only runner busy for another 12h.""" - assert 'POLL_FAIL_FAST="${POLL_FAIL_FAST:-1}"' in poll_script - assert 'if [ "$POLL_FAIL_FAST" = "1" ] && [ "$fail_seen" -eq 1 ]; then' in poll_script - assert "LEAVE_RUNNING_FILE=" in poll_script - assert 'VERDICT["$leg"]="SKIP|still running (gate failed; workload left alive)"' in poll_script - assert "leave_running_wid" in poll_script - # Every path that records a FAIL has to arm the flag, or fail-fast never triggers. +def test_poll_keeps_polling_after_the_gate_is_lost(poll_script: str) -> None: + """Each leg must reach a terminal verdict even after the gate is already FAIL.""" + assert "gate_fail_announced" in poll_script + assert "Continuing to poll until every leg reaches a terminal verdict" in poll_script + assert 'VERDICT["$leg"]="SKIP|still running (gate failed; workload left alive)"' not in poll_script + assert "POLL_FAIL_FAST" not in poll_script assert poll_script.count("fail_seen=1") == 2 assert poll_script.count('VERDICT["$leg"]="FAIL|') >= 2 +def test_poll_reads_root_only_state_json_via_sudo(poll_script: str) -> None: + """Pods write state.json mode 600; the runner user reads it with sudo -n.""" + assert "state_json_query" in poll_script + assert "sudo -n jq" in poll_script + assert "state.json not readable" in poll_script + assert "__UNREADABLE__" in poll_script + + +def test_poll_report_icons_distinguish_skip_from_fail(poll_script: str) -> None: + assert "verdict_icon" in poll_script + assert 'SKIP|PENDING' in poll_script + + def test_poll_sleeps_in_slices_so_a_cancel_lands_quickly(poll_script: str) -> None: assert 'POLL_SLEEP_SLICE_S="${POLL_SLEEP_SLICE_S:-5}"' in poll_script assert 'sleep "$POLL_SLEEP_SLICE_S"' in poll_script assert 'sleep "$POLL_INTERVAL_S"' not in poll_script -def test_abnormal_end_cleanup_respects_leave_running(workflow: dict) -> None: - """Fail-fast may leave workloads up; cleanup must not stop those wids.""" +def test_abnormal_end_cleanup_respects_leave_running(workflow: dict, poll_script: str) -> None: + """Superseded runs may leave workloads up; cleanup must not stop those wids.""" + assert "LEAVE_RUNNING_FILE=" in poll_script + assert "leave_running_wid" in poll_script steps = workflow["jobs"]["run"]["steps"] cleanup = [s for s in steps if "cancelled()" in str(s.get("if", ""))] assert cleanup, "the run job lost its cancel/failure cleanup step" diff --git a/src/hyperloom/inference_optimizer/tests/test_pre_release_stall_liveness.py b/src/hyperloom/inference_optimizer/tests/test_pre_release_stall_liveness.py index 6366cf63fe..2d33879e6c 100644 --- a/src/hyperloom/inference_optimizer/tests/test_pre_release_stall_liveness.py +++ b/src/hyperloom/inference_optimizer/tests/test_pre_release_stall_liveness.py @@ -189,6 +189,13 @@ def test_bootstrap_completes_on_clean_stop_reason_without_final_json(script: str assert "state.json stop_reason='$stop' after" in script +def test_bootstrap_publishes_state_json_for_poll_reader(script: str) -> None: + """The poll runner reads state.json as ubuntu; bootstrap opens it after each write.""" + assert "publish_state_for_poll" in script + assert 'chmod a+r "$state_json"' in script + assert 'publish_state_for_poll "$state_json" "$real_sdir" "$session"' in script + + def test_setup_is_budgeted_in_time_not_in_turns(script: str) -> None: """A count-based cap of 3 turns was ~4min of wall clock and killed live installs. From 09917b998206072cf36c244ae8b1beffaa75cd74 Mon Sep 17 00:00:00 2001 From: lishuoshuo-amd Date: Sun, 30 Aug 2026 20:31:27 +0800 Subject: [PATCH 41/52] pre-release-e2e: clarify poll global timeout comment Document that legs with an empty stop_reason rely on GLOBAL_TIMEOUT_S rather than an NFS stall check. Co-authored-by: Cursor --- .github/scripts/pre-release-e2e-poll.sh | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/.github/scripts/pre-release-e2e-poll.sh b/.github/scripts/pre-release-e2e-poll.sh index 282abf0533..3a43106f80 100755 --- a/.github/scripts/pre-release-e2e-poll.sh +++ b/.github/scripts/pre-release-e2e-poll.sh @@ -29,7 +29,8 @@ # TARGET_GAIN passed to optimize (demo skill); not used to judge PASS # POLL_INTERVAL_S seconds between polls (default 120) # GLOBAL_TIMEOUT_S hard cap; unfinished legs -> FAIL -# (default 50400 = 14h) +# (default 50400 = 14h; zombie legs with an empty +# stop_reason wait here rather than a stall check) # MAX_CRASHES / MAX_BOOT_FAILS tolerance (default 0 / 0) # Optional GitHub commit status (per-leg context pre-release-e2e/): # GH_STATUS_TOKEN / GH_STATUS_REPO / GH_STATUS_SHA / GH_STATUS_DETAILS_URL From 77ff230a753923c1463a87fa1d2327bad50e6ace Mon Sep 17 00:00:00 2001 From: lishuoshuo-amd Date: Sun, 30 Aug 2026 20:32:47 +0800 Subject: [PATCH 42/52] Fix ruff format in pre-release gate orchestration test Normalize quote style so ruff format --check passes in CI. Co-authored-by: Cursor --- .../tests/test_pre_release_gate_orchestration.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/hyperloom/inference_optimizer/tests/test_pre_release_gate_orchestration.py b/src/hyperloom/inference_optimizer/tests/test_pre_release_gate_orchestration.py index 0c58338d3a..e0a648a2fd 100644 --- a/src/hyperloom/inference_optimizer/tests/test_pre_release_gate_orchestration.py +++ b/src/hyperloom/inference_optimizer/tests/test_pre_release_gate_orchestration.py @@ -115,7 +115,7 @@ def test_poll_reads_root_only_state_json_via_sudo(poll_script: str) -> None: def test_poll_report_icons_distinguish_skip_from_fail(poll_script: str) -> None: assert "verdict_icon" in poll_script - assert 'SKIP|PENDING' in poll_script + assert "SKIP|PENDING" in poll_script def test_poll_sleeps_in_slices_so_a_cancel_lands_quickly(poll_script: str) -> None: From 8153224a6ac34306dc87450397d6f646b3d85062 Mon Sep 17 00:00:00 2001 From: lishuoshuo-amd Date: Tue, 1 Sep 2026 11:12:07 +0800 Subject: [PATCH 43/52] Revert pre-release E2E temp version bump to 1.0.0 Co-authored-by: Cursor --- pyproject.toml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pyproject.toml b/pyproject.toml index c9465a4ecc..09a582e357 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -6,7 +6,7 @@ build-backend = "setuptools.build_meta" [project] name = "hyperloom-inference_optimizer" -version = "1.0.1a0" # TEMP TEST: alpha bump vs main (1.0.0) to force pre-release-e2e FULL run (all 8 legs); REVERT before merge +version = "1.0.0" description = "Inference Optimizer — three-role (Orchestration/Critic/Robustness) autonomous LLM inference optimization runtime for AMD GPU platforms. Kernel optimization is handled by programmatic Python handlers." readme = "README.md" requires-python = ">=3.10" From c2b7b44b74ab713a73b028c12b5b2d9bc72fa5a8 Mon Sep 17 00:00:00 2001 From: lishuoshuo-amd Date: Tue, 1 Sep 2026 15:25:23 +0800 Subject: [PATCH 44/52] examples: expose the local-exploration toggle in the advanced skill The skill listed framework local exploration among the default-on phase toggles but never collected it or passed it, so the choice was unreachable. Also split the OPTIMIZE guardrail: --no-framework-agent drops the whole phase, while --no-framework-local-explore drops only its authoring arm. Co-authored-by: Cursor --- examples/hyperloom-custom-advanced/SKILL.md | 13 ++++++++++--- 1 file changed, 10 insertions(+), 3 deletions(-) diff --git a/examples/hyperloom-custom-advanced/SKILL.md b/examples/hyperloom-custom-advanced/SKILL.md index 18b2fcb349..6f7d07a9c6 100644 --- a/examples/hyperloom-custom-advanced/SKILL.md +++ b/examples/hyperloom-custom-advanced/SKILL.md @@ -160,7 +160,8 @@ resolved values in the launch plan before starting the optimizer. Collect these optional advanced values: - Phase toggles: `--no-kernel`, `--no-framework-agent`, - `--no-enable-conc-sweep`, `--no-enable-roofline`. + `--no-framework-local-explore`, `--no-enable-conc-sweep`, + `--no-enable-roofline`. - Phase budget percentages: `PHASE_BUDGET_PRELUDE_PCT`, `PHASE_BUDGET_FRAMEWORK_PCT`, `PHASE_BUDGET_KERNEL_PCT`, @@ -179,8 +180,13 @@ Guardrails: pass explicit CLI flags in the optimize command. - Omit `--gpu-type` unless the user explicitly chooses a hint; otherwise let Hyperloom auto-detect from ROCm/system info. -- Warn the user when both `--no-framework-agent` and `--no-kernel` are selected; that - collapses the run mostly to baseline and sweep validation. +- `--no-framework-agent` skips the entire OPTIMIZE phase (PRELUDE goes straight + to KERNEL_AGENT), dropping both of its arms: upstream-PR landing and local + source authoring. Warn before applying it; combined with `--no-kernel` it + leaves only baseline and sweep validation. +- `--no-framework-local-explore` keeps OPTIMIZE but drops only its local + authoring arm, so the phase exits after three empty upstream discoveries + instead of authoring a patch from live source. No effect under diff-only mode. - Phase budget percentages are caps, not guaranteed time usage. A phase may end earlier, and disabled work phases have their share redistributed by the optimizer. @@ -327,6 +333,7 @@ OPT_FLAGS=( [ -n "${PHASE_BUDGET_CLOSE_PCT:-}" ] && OPT_FLAGS+=(--max-minutes-close-pct "$PHASE_BUDGET_CLOSE_PCT") [ "${NO_KERNEL:-0}" = "1" ] && OPT_FLAGS+=(--no-kernel) [ "${NO_FRAMEWORK_AGENT:-0}" = "1" ] && OPT_FLAGS+=(--no-framework-agent) +[ "${NO_FRAMEWORK_LOCAL_EXPLORE:-0}" = "1" ] && OPT_FLAGS+=(--no-framework-local-explore) [ "${NO_CONC_SWEEP:-0}" = "1" ] && OPT_FLAGS+=(--no-enable-conc-sweep) [ "${NO_ROOFLINE:-0}" = "1" ] && OPT_FLAGS+=(--no-enable-roofline) From 3376fdff383c691f3086f7e09d6a7d3c6cc4b5d7 Mon Sep 17 00:00:00 2001 From: lishuoshuo-amd Date: Tue, 1 Sep 2026 15:54:29 +0800 Subject: [PATCH 45/52] pre-release-e2e: stop a reused CI_VERSION from passing the gate on stale artifacts A reused CI_VERSION (reuse_ci_version, or a job re-run) puts a run on the paths a finished run already wrote, and nothing proved which run an artifact belonged to. Verdicts are recorded once and the poll loop breaks as soon as nothing is pending, so one stale read on the first tick declared the whole gate PASS before any pod had booted; bootstrap likewise saw an old "setup complete" in the appended agent log and an old state.json as its own. Dispatch now hands its VERSION_TAG to the pods and to the poll, bootstrap stamps it into .session_dir, and the poll ignores a pin carrying any other tag. The wait loop only considers a state.json newer than the leg start, and the agent transcript is rotated instead of appended -- both keep the previous run's files for post-mortem. The pod hard-timeout counted only the demo wait, leaving it 15m BELOW what bootstrap can spend once the 45m setup budget is included: a leg using its full setup budget was killed by SaFE mid-wait, losing the clean return and the logs. Size both pod deadlines off setup + demo wait + margin, raise the poll global timeout above them, and pin the whole ladder in a test so it cannot drift again. Co-authored-by: Cursor --- .github/pre-release/bootstrap-pre-release.sh | 25 ++++- .github/scripts/pre-release-e2e-dispatch.sh | 25 +++-- .github/scripts/pre-release-e2e-poll.sh | 27 +++-- .github/workflows/pre-release-e2e-test.yml | 2 +- .../test_pre_release_gate_orchestration.py | 106 ++++++++++++++++++ 5 files changed, 166 insertions(+), 19 deletions(-) diff --git a/.github/pre-release/bootstrap-pre-release.sh b/.github/pre-release/bootstrap-pre-release.sh index 150ecdddaa..855386053a 100755 --- a/.github/pre-release/bootstrap-pre-release.sh +++ b/.github/pre-release/bootstrap-pre-release.sh @@ -124,6 +124,13 @@ is_clean_stop_reason() { } # Optimize writes state.json root-only; the poll runner reads as ubuntu. +# Pin the session dir for the poll, stamped with this run's tag (dispatch passes RUN_TAG). +# On a reused CI_VERSION the previous run's pin is still on disk pointing at a finished +# session; the tag is how the poll tells that leftover from ours. Args: dir session +pin_session_dir() { + { printf '%s\n' "$1"; printf '%s\n' "${RUN_TAG:-}"; } > "${2}/.session_dir" +} + publish_state_for_poll() { local state_json="$1" sdir="$2" session="$3" [ -f "$state_json" ] || return 0 @@ -138,6 +145,10 @@ run_leg() { local root="${NFS_ROOT%/}/runs/${CI_VERSION}/${leg}" local session="${root}/session" mkdir -p "$root" "$session" + # A reused CI_VERSION (workflow_dispatch reuse_ci_version, or a job re-run) lands on the + # same paths, so the previous run's artifacts are still here. Nothing older than leg_t0 + # belongs to this run; the artifacts are kept for post-mortem, never trusted as ours. + local leg_t0; leg_t0="$(date +%s)" log "leg=$leg mode=$run_mode backend=$backend hours=$hours model=$model_path" # 1. install the wheel into the leg root (produces importable tree + bundled skills) @@ -255,7 +266,7 @@ run_leg() { # 3. pin the session dir so the poll finds it without guessing by timestamp (design §9) export INFERENCE_OPTIMIZER_CURRENT_SESSION_DIR="${session}" - echo "${session}" > "${session}/.session_dir" + pin_session_dir "${session}" "${session}" # 4. source env + drive setup, then demo, through the Agent CLI set -a @@ -285,6 +296,12 @@ run_leg() { # the pod's stdout with it, so without this the agent's own account of the failure is # unrecoverable and a post-mortem is left reconstructing events from file mtimes. local agent_log="${session}/agent-${leg}.log" + # Rotate rather than append: the setup loop below decides it is done by grepping this + # file for the completion marker, and a previous run's marker would satisfy it on turn 1 + # with nothing installed. Rotating keeps the old transcript for post-mortem. + if [ -f "$agent_log" ]; then + mv "$agent_log" "${agent_log%.log}.prev-$(date -u +%Y%m%dT%H%M%SZ).log" 2>/dev/null || true + fi local uuid; uuid="$(leg_session_uuid "$leg" "$CI_VERSION")" # `claude --print` is "print response and exit": ONE answer per invocation. Setup here # means installing a framework layer -- a vLLM ROCm wheel into an isolated venv, or @@ -394,14 +411,16 @@ run_leg() { if [ -z "$real_sdir" ]; then # newest dir under $session that actually contains a state.json - real_sdir="$(find "$session" -mindepth 2 -type f -name state.json -printf '%T@ %h\n' 2>/dev/null \ + # -newermt leg_t0 excludes a previous run's nested dirs, which sit under the same + # $session and would otherwise win "newest" and end this wait with their state.json. + real_sdir="$(find "$session" -mindepth 2 -type f -name state.json -newermt "@$leg_t0" -printf '%T@ %h\n' 2>/dev/null \ | sort -rn | head -n1 | cut -d' ' -f2- || true)" if [ -n "$real_sdir" ]; then final_json="${real_sdir}/reports/final.json" state_json="${real_sdir}/state.json" log "leg $leg real session dir: $real_sdir" # Re-pin so the poll (leg_session_dir -> head -n1 .session_dir) finds the report. - echo "$real_sdir" > "${session}/.session_dir" + pin_session_dir "$real_sdir" "$session" publish_state_for_poll "$state_json" "$real_sdir" "$session" else local idle diff --git a/.github/scripts/pre-release-e2e-dispatch.sh b/.github/scripts/pre-release-e2e-dispatch.sh index a584293b86..ed303e5884 100755 --- a/.github/scripts/pre-release-e2e-dispatch.sh +++ b/.github/scripts/pre-release-e2e-dispatch.sh @@ -84,14 +84,16 @@ DISPATCH_MAP="${DISPATCH_MAP:-${RUNNER_TEMP:-/tmp}/pre_release_dispatch.json}" # Pod hard-timeout. SaFE terminates the workload at the deadline; the poll then sees a # non-Succeeded terminal / missing report and judges that leg FAIL. Counted from DISPATCH -# (not queue) time. This MUST exceed the bootstrap's own in-pod wait deadline -# (hours*3600+3600, i.e. 3h/12h demo + 1h agent/setup buffer) so SaFE never pre-empts the -# pod mid-wait -- which would lose bootstrap's clean `return 1` + logging and reintroduce -# the premature-teardown race. We add a further +30m pod margin on top of the bootstrap -# deadline. Ordering per leg: bootstrap deadline < SaFE pod timeout < poll GLOBAL_TIMEOUT_S -# (default 50400s=14h, still > 48600s). Given per duration: -DEADLINE_3H_S="${DEADLINE_3H_S:-16200}" # 3h demo + 1h bootstrap buffer + 30m pod margin = 4.5h -DEADLINE_12H_S="${DEADLINE_12H_S:-48600}" # 12h demo + 1h bootstrap buffer + 30m pod margin = 13.5h +# (not queue) time. This MUST exceed everything bootstrap can spend in-pod, which is the +# setup budget (LEG_SETUP_DEADLINE_S, 45m) PLUS the demo wait deadline (hours*3600+3600, +# i.e. 3h/12h demo + 1h agent buffer). An earlier version counted only the demo wait and +# so sat 15m BELOW the bootstrap total: a leg that used its full setup budget was killed +# by SaFE mid-wait, losing bootstrap's clean `return 1` + logs. We add a further +30m pod +# margin on top of that total. Ordering per leg: +# bootstrap total (setup + demo wait) < SaFE pod timeout < poll GLOBAL_TIMEOUT_S +# 3h: 2700 + 14400 = 17100 < 18900 < 52200 ; 12h: 2700 + 46800 = 49500 < 51300 < 52200 +DEADLINE_3H_S="${DEADLINE_3H_S:-18900}" # 45m setup + 3h demo + 1h buffer + 30m pod margin = 5.25h +DEADLINE_12H_S="${DEADLINE_12H_S:-51300}" # 45m setup + 12h demo + 1h buffer + 30m pod margin = 14.25h # The SaFE API field that carries the pod deadline. Confirmed against the Primus-SaFE # codebase: the create-workload body embeds WorkloadSpec inline, whose `timeout` # (integer seconds, top-level, from dispatch time) is enforced by WorkloadTTLController @@ -210,6 +212,7 @@ common_env_json() { --arg keyb64 "$(printf '%s' "$ANTHROPIC_API_KEY" | base64 | tr -d '\n')" \ --arg baseurl "${ANTHROPIC_BASE_URL:-}" \ --arg cheaders "${ANTHROPIC_CUSTOM_HEADERS:-}" \ + --arg rtag "$VERSION_TAG" \ '{ CI_VERSION: $civ, NFS_ROOT: $nfs, @@ -219,6 +222,7 @@ common_env_json() { TARGET_GAIN: $tgain, CLAUDE_MODEL: $cmodel, CLAUDE_CLI_VERSION: $cver, + RUN_TAG: $rtag, ANTHROPIC_API_KEY_B64: $keyb64 } + (if $baseurl == "" then {} else {ANTHROPIC_BASE_URL: $baseurl} end) @@ -309,6 +313,9 @@ declare -A DISPATCH # leg -> workloadId # exists, then append after every successful create. : > "$DISPATCH_MAP" 2>/dev/null || true printf '{}\n' > "$DISPATCH_MAP" +# Hand the poll this run's tag out-of-band rather than re-deriving it there: the pods +# stamp it into their session pin, and the poll rejects a pin carrying any other tag. +printf '%s\n' "$VERSION_TAG" > "${DISPATCH_MAP}.version_tag" record_dispatch() { # leg workloadId -- add to the in-memory map AND the on-disk map local leg="$1" wid="$2" DISPATCH["$leg"]="$wid" @@ -355,10 +362,12 @@ if [ "$want_docker_host" = 1 ]; then --arg legs "$docker_legs" --argjson gpumap "$gpu_map" \ --arg dm3 "$DOCKER_LEG_MEM_3H" --arg dm12 "$DOCKER_LEG_MEM_12H" \ --arg ds3 "$DOCKER_LEG_SHM_3H" --arg ds12 "$DOCKER_LEG_SHM_12H" \ + --arg rtag "$VERSION_TAG" \ '{ CI_VERSION:$civ, NFS_ROOT:$nfs, MODEL_3H:$m3, MODEL_12H:$m12, TARGET_GAIN:$tgain, CLAUDE_MODEL:$cmodel, CLAUDE_CLI_VERSION:$cver, + RUN_TAG:$rtag, ANTHROPIC_API_KEY_B64:$keyb64, HYPERLOOM_RUN_MODE:"docker", E2E_DOCKER_HOST:"1", diff --git a/.github/scripts/pre-release-e2e-poll.sh b/.github/scripts/pre-release-e2e-poll.sh index 3a43106f80..a434064ae8 100755 --- a/.github/scripts/pre-release-e2e-poll.sh +++ b/.github/scripts/pre-release-e2e-poll.sh @@ -28,9 +28,11 @@ # NFS_ROOT (default /shared_nfs/hyperloom-pre-release-e2e-test) # TARGET_GAIN passed to optimize (demo skill); not used to judge PASS # POLL_INTERVAL_S seconds between polls (default 120) -# GLOBAL_TIMEOUT_S hard cap; unfinished legs -> FAIL -# (default 50400 = 14h; zombie legs with an empty -# stop_reason wait here rather than a stall check) +# GLOBAL_TIMEOUT_S hard cap; unfinished legs -> FAIL (default 52200 = +# 14.5h, above the 14.25h 12h-leg pod timeout so the +# pod's own death is observed instead of timing out +# first; zombie legs with an empty stop_reason wait +# here rather than a stall check) # MAX_CRASHES / MAX_BOOT_FAILS tolerance (default 0 / 0) # Optional GitHub commit status (per-leg context pre-release-e2e/): # GH_STATUS_TOKEN / GH_STATUS_REPO / GH_STATUS_SHA / GH_STATUS_DETAILS_URL @@ -42,10 +44,13 @@ set -euo pipefail NFS_ROOT="${NFS_ROOT:-/shared_nfs/hyperloom-pre-release-e2e-test}" TARGET_GAIN="${TARGET_GAIN:-100}" POLL_INTERVAL_S="${POLL_INTERVAL_S:-120}" -GLOBAL_TIMEOUT_S="${GLOBAL_TIMEOUT_S:-50400}" +GLOBAL_TIMEOUT_S="${GLOBAL_TIMEOUT_S:-52200}" MAX_CRASHES="${MAX_CRASHES:-0}" MAX_BOOT_FAILS="${MAX_BOOT_FAILS:-0}" LEAVE_RUNNING_FILE="${LEAVE_RUNNING_FILE:-${DISPATCH_MAP}.leave_running}" +# This run's dispatch tag, written by dispatch beside the map. Pods stamp it into their +# session pin; an untagged/foreign pin is a leftover from an earlier run on these paths. +RUN_TAG="${RUN_TAG:-$(cat "${DISPATCH_MAP}.version_tag" 2>/dev/null || true)}" # Sleep in short slices instead of one long one so a cancelled job tears down in seconds # rather than at the end of a full POLL_INTERVAL_S. Each slice also re-checks whether a # newer pre-release run has been queued so this poll can exit and release the runner. @@ -268,10 +273,18 @@ workload_phase() { # workloadId -> phase string # Resolve a leg's session dir. Bootstrap writes the pinned session dir to # runs///session/.session_dir (design §9: never guess by timestamp). leg_session_dir() { - local leg="$1" pin + local leg="$1" pin tag pin="${runs_dir}/${leg}/session/.session_dir" - if [ -f "$pin" ]; then head -n1 "$pin"; return; fi - echo "" + [ -f "$pin" ] || { echo ""; return; } + # Line 2 carries the writing run's tag. A pin from an earlier run that reused this + # CI_VERSION points at a FINISHED session, so honouring it would judge that run's + # state.json as ours -- a clean stop_reason there would pass the gate before this + # run's pod has even booted. Treat a foreign or missing tag as "not pinned yet". + if [ -n "${RUN_TAG:-}" ]; then + tag="$(sed -n 2p "$pin" 2>/dev/null || true)" + [ "$tag" = "$RUN_TAG" ] || { echo ""; return; } + fi + head -n1 "$pin" } # Read one jq filter from state.json. Pods write it root-only (mode 600); the runner diff --git a/.github/workflows/pre-release-e2e-test.yml b/.github/workflows/pre-release-e2e-test.yml index a915bc9442..2af196d478 100644 --- a/.github/workflows/pre-release-e2e-test.yml +++ b/.github/workflows/pre-release-e2e-test.yml @@ -293,7 +293,7 @@ jobs: HEAD_REF: ${{ github.head_ref }} GH_STATUS_DETAILS_URL: ${{ github.server_url }}/${{ github.repository }}/actions/runs/${{ github.run_id }} POLL_INTERVAL_S: "120" - GLOBAL_TIMEOUT_S: "50400" + GLOBAL_TIMEOUT_S: "52200" steps: - uses: actions/checkout@v7 diff --git a/src/hyperloom/inference_optimizer/tests/test_pre_release_gate_orchestration.py b/src/hyperloom/inference_optimizer/tests/test_pre_release_gate_orchestration.py index e0a648a2fd..e181a27141 100644 --- a/src/hyperloom/inference_optimizer/tests/test_pre_release_gate_orchestration.py +++ b/src/hyperloom/inference_optimizer/tests/test_pre_release_gate_orchestration.py @@ -24,6 +24,8 @@ from __future__ import annotations +import re +import subprocess from pathlib import Path import pytest @@ -68,6 +70,12 @@ def dispatch_script() -> str: return (_GITHUB / "scripts" / "pre-release-e2e-dispatch.sh").read_text(encoding="utf-8") +@pytest.fixture(scope="module") +def bootstrap_script() -> str: + assert _GITHUB is not None + return (_GITHUB / "pre-release" / "bootstrap-pre-release.sh").read_text(encoding="utf-8") + + def test_nothing_that_talks_to_safe_runs_on_a_github_hosted_runner(workflow: dict) -> None: """SAFE_API_BASE is an in-network NodePort; a hosted runner can only time out.""" for name, job in workflow["jobs"].items(): @@ -172,3 +180,101 @@ def test_poll_passes_on_clean_terminal_stop_reason_not_gain(poll_script: str) -> assert "state.json stop_reason" in poll_script assert "reports/final.json missing" not in poll_script assert 'echo "PENDING|state.json stop_reason not set yet' in poll_script + + +# ---- reusing a CI_VERSION must not let the previous run's artifacts pass the gate ---- +# A reused CI_VERSION (workflow_dispatch reuse_ci_version, or a job re-run) puts this run +# on the paths a finished run already wrote. Verdicts are recorded once and never revisited +# and the loop breaks as soon as nothing is pending, so a single stale read on the first +# tick is enough to declare the whole gate PASS before a pod has booted. + + +def _leg_session_dir(poll_script: str, runs_dir: Path, leg: str, run_tag: str) -> str: + """Run the real leg_session_dir() out of the poll script.""" + lines = poll_script.splitlines() + start = next(i for i, line in enumerate(lines) if line.startswith("leg_session_dir() {")) + end = next(i for i in range(start, len(lines)) if lines[i] == "}") + fn = "\n".join(lines[start : end + 1]) + proc = subprocess.run( + [ + "bash", + "-c", + f'runs_dir="$1"; RUN_TAG="$2"\n{fn}\nleg_session_dir "$3"', + "_", + str(runs_dir), + run_tag, + leg, + ], + capture_output=True, + text=True, + check=True, + ) + return proc.stdout.strip() + + +def test_a_stale_session_pin_cannot_pass_the_gate(poll_script: str, tmp_path: Path) -> None: + """A pin written by an earlier run on these paths must not resolve to a session dir.""" + leg = "baremetal-vllm-3h" + session = tmp_path / leg / "session" + finished = session / "Qwen3-8B" / "20260830T000000Z-deadbeef" + finished.mkdir(parents=True) + pin = session / ".session_dir" + + pin.write_text(f"{finished}\nold-run\n", encoding="utf-8") + assert _leg_session_dir(poll_script, tmp_path, leg, "this-run") == "" + + # An untagged pin predates the stamping and carries no proof of ownership either. + pin.write_text(f"{finished}\n", encoding="utf-8") + assert _leg_session_dir(poll_script, tmp_path, leg, "this-run") == "" + + pin.write_text(f"{finished}\nthis-run\n", encoding="utf-8") + assert _leg_session_dir(poll_script, tmp_path, leg, "this-run") == str(finished) + + +def test_the_run_tag_reaches_the_pod_and_the_poll(dispatch_script: str, bootstrap_script: str) -> None: + """Dispatch is the single source of the tag: pods stamp it, the poll compares it.""" + assert "RUN_TAG: $rtag" in dispatch_script + assert "RUN_TAG:$rtag" in dispatch_script + assert '"${DISPATCH_MAP}.version_tag"' in dispatch_script + assert "printf '%s\\n' \"${RUN_TAG:-}\"" in bootstrap_script + assert bootstrap_script.count("pin_session_dir ") == 2 + assert 'echo "$real_sdir" > "${session}/.session_dir"' not in bootstrap_script + + +def test_bootstrap_rotates_the_agent_log_instead_of_appending(bootstrap_script: str) -> None: + """The setup marker grep reads this file; a previous run's marker must not satisfy it.""" + assert ".prev-$(date -u +%Y%m%dT%H%M%SZ).log" in bootstrap_script + assert 'grep -qiE "setup complete: ${run_mode}/${backend}" "$agent_log"' in bootstrap_script + + +def test_bootstrap_ignores_a_state_json_older_than_the_leg(bootstrap_script: str) -> None: + """The wait loop picks the newest state.json under $session -- scope it to this run.""" + assert 'local leg_t0; leg_t0="$(date +%s)"' in bootstrap_script + assert '-name state.json -newermt "@$leg_t0"' in bootstrap_script + + +# ---- layered timeouts: bootstrap total < SaFE pod timeout < poll global timeout ---- + + +def _shell_default(script: str, name: str) -> int: + match = re.search(rf"\$\{{{name}:-(\d+)\}}", script) + assert match, f"{name} default not found" + return int(match.group(1)) + + +def test_pod_timeout_covers_the_whole_bootstrap_budget( + dispatch_script: str, bootstrap_script: str, poll_script: str, workflow: dict +) -> None: + """Setup is a separate budget; leaving it out of the pod cap gets legs killed mid-wait.""" + setup_s = _shell_default(bootstrap_script, "LEG_SETUP_DEADLINE_S") + # The workflow env wins over the script default, so the effective value is the one + # the ladder has to hold for. + global_s = int(workflow["jobs"]["run"]["env"]["GLOBAL_TIMEOUT_S"]) + assert global_s == _shell_default(poll_script, "GLOBAL_TIMEOUT_S") + assert global_s < int(workflow["jobs"]["run"]["timeout-minutes"]) * 60 + assert "local deadline_s=$(( hours * 3600 + 3600 ))" in bootstrap_script + for hours, name in ((3, "DEADLINE_3H_S"), (12, "DEADLINE_12H_S")): + pod_s = _shell_default(dispatch_script, name) + bootstrap_s = setup_s + hours * 3600 + 3600 + assert bootstrap_s < pod_s, f"{name}={pod_s} is below the {bootstrap_s}s bootstrap budget" + assert pod_s < global_s, f"{name}={pod_s} outlives the poll's {global_s}s global timeout" From 64ce0acd2e6c2c0de80e3b31c17a62d700e52862 Mon Sep 17 00:00:00 2001 From: lishuoshuo-amd Date: Tue, 1 Sep 2026 16:06:29 +0800 Subject: [PATCH 46/52] pre-release-e2e: retry the first setup turn, and stop the comments overstating the gate Three claims the code did not back, and one asymmetry that cost a leg. The first setup turn was called bare while every later one is wrapped, so under set -e a single transient CLI error (a 429, a dropped connection) killed a multi-hour leg outright -- the failure mode the retry loop exists to absorb. Guard it like the others and let the stall check and the setup deadline decide. MAX_BOOT_FAILS was declared and documented as a tolerance, but nothing ever read it: the boot-failure count lives in the report journal, not state.json. Drop the knob rather than leave a criterion the gate does not enforce. Three comments promised the API key never reaches NFS. It does -- the leg's .env sits beside the workspace the agent reads it from, and the EXIT trap that scrubs it cannot run on a SIGKILL. Say that, so the next reader knows it still needs hardening instead of trusting a guarantee that was never true. Declare hyperloom-pre-e2e-baremetal in actionlint.yaml, which already claimed to cover this workflow; undeclared, it fails pre-commit on unrelated work. Co-authored-by: Cursor --- .github/actionlint.yaml | 1 + .github/pre-release/bootstrap-pre-release.sh | 19 +++++++++++--- .github/scripts/pre-release-e2e-dispatch.sh | 7 ++--- .github/scripts/pre-release-e2e-poll.sh | 6 +++-- .../test_pre_release_gate_orchestration.py | 26 +++++++++++++++++++ 5 files changed, 50 insertions(+), 9 deletions(-) diff --git a/.github/actionlint.yaml b/.github/actionlint.yaml index b79f79e155..f249292750 100644 --- a/.github/actionlint.yaml +++ b/.github/actionlint.yaml @@ -6,3 +6,4 @@ self-hosted-runner: labels: - Hyperloom-e2e-ci + - hyperloom-pre-e2e-baremetal diff --git a/.github/pre-release/bootstrap-pre-release.sh b/.github/pre-release/bootstrap-pre-release.sh index 855386053a..96993d88d3 100755 --- a/.github/pre-release/bootstrap-pre-release.sh +++ b/.github/pre-release/bootstrap-pre-release.sh @@ -18,7 +18,10 @@ # # Inputs (env, injected by the dispatch script): # CI_VERSION NFS_ROOT -# ANTHROPIC_API_KEY_B64 base64 key; decoded here, written only to pod-local .env +# ANTHROPIC_API_KEY_B64 base64 key; decoded into the leg's .env, which lives on +# NFS with the workspace the agent reads it from. Kept off +# stdout and out of the API payload, but NOT off NFS: the +# EXIT trap scrubs the key, so a SIGKILL leaves it on disk. # ANTHROPIC_BASE_URL (optional) CLAUDE_MODEL CLAUDE_CLI_VERSION TARGET_GAIN # Baremetal leg: LEG_ID HYPERLOOM_RUN_MODE=baremetal HYPERLOOM_BACKEND HYPERLOOM_MODEL_PATH DEMO_HOURS # Docker host: E2E_DOCKER_HOST=1 DOCKER_LEGS DOCKER_GPU_MAP(json) MODEL_3H MODEL_12H @@ -157,8 +160,10 @@ run_leg() { log "pip install ${wheels[0]} --target $root" pip install --no-input --target "$root" "${wheels[0]}" >/dev/null - # 2. decode the key and write the pod-local .env (NEVER on stdout / NEVER to a - # location the poll reads). Restrict perms; scrub on exit. + # 2. decode the key and write the leg's .env. It lands on NFS because the agent reads it + # from the workspace it runs in; umask 077 restricts it and the EXIT trap scrubs the + # key, but a SIGKILL (eviction, SaFE hard timeout) leaves it readable on the share. + # Never echoed to stdout. # # For a docker leg the agent (not the harness) starts the single-GPU container per the # demo skill. The demo skill's literal `docker run` cannot express our per-card @@ -326,7 +331,13 @@ run_leg() { turn=$(( turn + 1 )) if [ "$turn" = 1 ]; then log "claude --print (setup, turn 1, session $uuid); agent transcript -> $agent_log" - agent_turn "$agent_log" --session-id "$uuid" < "$setup_prompt" + # Tolerated exactly like a resumed turn: under `set -e` a bare call would let one + # transient CLI/gateway error (a 429, a dropped connection) kill a multi-hour leg, + # while the same failure from turn 2 on is only a warning. The stall check and the + # setup deadline below are what decide the leg is dead. + if ! agent_turn "$agent_log" --session-id "$uuid" < "$setup_prompt"; then + log "WARN: leg $leg -- setup turn 1 exited non-zero; anything it detached keeps running" + fi else log "claude --print (setup, turn $turn, resuming session $uuid)" if ! printf '%s\n' "$SETUP_RESUME_NUDGE" | agent_turn "$agent_log" --resume "$uuid"; then diff --git a/.github/scripts/pre-release-e2e-dispatch.sh b/.github/scripts/pre-release-e2e-dispatch.sh index ed303e5884..1a89b33e27 100755 --- a/.github/scripts/pre-release-e2e-dispatch.sh +++ b/.github/scripts/pre-release-e2e-dispatch.sh @@ -34,7 +34,8 @@ # CLAUDE_MODEL model for the Agent turns (required) # CLAUDE_CLI_VERSION pinned Claude CLI version (required) # ANTHROPIC_API_KEY Claude CLI auth; injected here as base64 -# into the workload env (never written to NFS)(required) +# into the workload env; bootstrap decodes it +# into the leg's .env, which is on NFS (required) # ANTHROPIC_BASE_URL optional proxy / base url (optional) # TASKS comma-separated leg subset (default: all 8) # DISPATCH_MAP output file: JSON {leg: workloadId} @@ -200,8 +201,8 @@ docker_gpu_index() { } # Common env for every workload. The API key is passed base64 so it is not visible in -# plaintext in the API payload log; bootstrap decodes it and writes it only to the -# pod-local .env (never to NFS). See design §9 (point D). +# plaintext in the API payload log; bootstrap decodes it into the leg's .env, which sits +# on NFS beside the workspace and is scrubbed by an EXIT trap. See design §9 (point D). common_env_json() { local model_path="$1" hours="$2" backend="$3" jq -n \ diff --git a/.github/scripts/pre-release-e2e-poll.sh b/.github/scripts/pre-release-e2e-poll.sh index a434064ae8..95294db26d 100755 --- a/.github/scripts/pre-release-e2e-poll.sh +++ b/.github/scripts/pre-release-e2e-poll.sh @@ -33,7 +33,10 @@ # pod's own death is observed instead of timing out # first; zombie legs with an empty stop_reason wait # here rather than a stall check) -# MAX_CRASHES / MAX_BOOT_FAILS tolerance (default 0 / 0) +# MAX_CRASHES crash_count tolerance (default 0). Server boot +# failures are NOT a pass criterion: the count lives in +# the report journal, not state.json, so nothing here +# reads it -- do not re-add a knob that judges nothing. # Optional GitHub commit status (per-leg context pre-release-e2e/): # GH_STATUS_TOKEN / GH_STATUS_REPO / GH_STATUS_SHA / GH_STATUS_DETAILS_URL # Supersede detection (release runner when a newer pre-release run is queued): @@ -46,7 +49,6 @@ TARGET_GAIN="${TARGET_GAIN:-100}" POLL_INTERVAL_S="${POLL_INTERVAL_S:-120}" GLOBAL_TIMEOUT_S="${GLOBAL_TIMEOUT_S:-52200}" MAX_CRASHES="${MAX_CRASHES:-0}" -MAX_BOOT_FAILS="${MAX_BOOT_FAILS:-0}" LEAVE_RUNNING_FILE="${LEAVE_RUNNING_FILE:-${DISPATCH_MAP}.leave_running}" # This run's dispatch tag, written by dispatch beside the map. Pods stamp it into their # session pin; an untagged/foreign pin is a leftover from an earlier run on these paths. diff --git a/src/hyperloom/inference_optimizer/tests/test_pre_release_gate_orchestration.py b/src/hyperloom/inference_optimizer/tests/test_pre_release_gate_orchestration.py index e181a27141..2c2b020cc4 100644 --- a/src/hyperloom/inference_optimizer/tests/test_pre_release_gate_orchestration.py +++ b/src/hyperloom/inference_optimizer/tests/test_pre_release_gate_orchestration.py @@ -180,6 +180,32 @@ def test_poll_passes_on_clean_terminal_stop_reason_not_gain(poll_script: str) -> assert "state.json stop_reason" in poll_script assert "reports/final.json missing" not in poll_script assert 'echo "PENDING|state.json stop_reason not set yet' in poll_script + # A tolerance knob nothing reads reads as a criterion the gate enforces; it did not. + assert "MAX_BOOT_FAILS=" not in poll_script + + +def test_every_runner_label_is_declared_for_actionlint(workflow: dict) -> None: + """An undeclared self-hosted label fails pre-commit on work that never touched it.""" + assert _GITHUB is not None + declared = yaml.safe_load((_GITHUB / "actionlint.yaml").read_text(encoding="utf-8")) + labels = set(declared["self-hosted-runner"]["labels"]) + for job in workflow["jobs"].values(): + runs_on = job["runs-on"] + if isinstance(runs_on, str) and "${{" not in runs_on: + assert runs_on in labels, f"runs-on {runs_on!r} is not in .github/actionlint.yaml" + + +def test_the_first_setup_turn_tolerates_a_transient_failure(bootstrap_script: str) -> None: + """A bare first call let one 429 kill a multi-hour leg; turn 2 was already guarded.""" + assert 'if ! agent_turn "$agent_log" --session-id "$uuid" < "$setup_prompt"; then' in bootstrap_script + + +def test_the_env_file_does_not_claim_to_stay_off_nfs(dispatch_script: str, bootstrap_script: str) -> None: + """The key lands on the share; comments saying otherwise stop anyone from hardening it.""" + for script in (dispatch_script, bootstrap_script): + assert "never written to NFS" not in script + assert "never to NFS" not in script + assert "only to pod-local" not in script # ---- reusing a CI_VERSION must not let the previous run's artifacts pass the gate ---- From 770018f358887ca4cd0ca9a44c7c141c9f3bfd7d Mon Sep 17 00:00:00 2001 From: lishuoshuo-amd Date: Tue, 1 Sep 2026 16:06:29 +0800 Subject: [PATCH 47/52] pre-release-e2e: say why target-gain is 100, and let a CA bundle replace -k The demo prompts stated that the 100% target is not the gate but never why it is set out of reach: to stop the loop converging on the skill's own target so the full phase sequence runs. Without that a reader takes it for a performance goal. The scripts have always honoured SAFE_CACERT ahead of skip-verify, matching ci-e2e and forge-e2e, but the workflow wired no variable into it -- so the only reachable mode was SAFE_INSECURE=1, sending an admin token over an unverified connection with no way to opt out. Inject PRE_E2E_SAFE_CACERT and document it. Unset, behaviour is unchanged. Co-authored-by: Cursor --- .github/pre-release/prompts/pre-release/demo-12h.md | 6 ++++-- .github/pre-release/prompts/pre-release/demo-3h.md | 6 ++++-- .github/workflows/pre-release-e2e-test.yml | 8 +++++++- 3 files changed, 15 insertions(+), 5 deletions(-) diff --git a/.github/pre-release/prompts/pre-release/demo-12h.md b/.github/pre-release/prompts/pre-release/demo-12h.md index 63a569f463..f1db57811f 100644 --- a/.github/pre-release/prompts/pre-release/demo-12h.md +++ b/.github/pre-release/prompts/pre-release/demo-12h.md @@ -8,8 +8,10 @@ its exact default flags. ## Flags -- **OVERRIDE:** use `--target-gain 100` (NOT the skill's default of 50). This shapes - optimize prompts only; the poll gate judges PASS/FAIL from `stop_reason`, not gain. +- **OVERRIDE:** use `--target-gain 100` (NOT the skill's default of 50). It is set out of + reach on purpose: the run must not converge early on the skill's own target, so the full + phase sequence gets exercised. This shapes optimize prompts only; the poll gate judges + PASS/FAIL from `stop_reason`, not gain, so 100 is not a performance goal to chase. - Keep every other required flag exactly as the skill defines them: ``` diff --git a/.github/pre-release/prompts/pre-release/demo-3h.md b/.github/pre-release/prompts/pre-release/demo-3h.md index 8a606094c9..aab72c1dd9 100644 --- a/.github/pre-release/prompts/pre-release/demo-3h.md +++ b/.github/pre-release/prompts/pre-release/demo-3h.md @@ -8,8 +8,10 @@ exact default flags. ## Flags -- **OVERRIDE:** use `--target-gain 100` (NOT the skill's default of 30). This shapes - optimize prompts only; the poll gate judges PASS/FAIL from `stop_reason`, not gain. +- **OVERRIDE:** use `--target-gain 100` (NOT the skill's default of 30). It is set out of + reach on purpose: the run must not converge early on the skill's own target, so the full + phase sequence gets exercised. This shapes optimize prompts only; the poll gate judges + PASS/FAIL from `stop_reason`, not gain, so 100 is not a performance goal to chase. - Keep every other required flag exactly as the skill defines them: ``` diff --git a/.github/workflows/pre-release-e2e-test.yml b/.github/workflows/pre-release-e2e-test.yml index 2af196d478..a81be30554 100644 --- a/.github/workflows/pre-release-e2e-test.yml +++ b/.github/workflows/pre-release-e2e-test.yml @@ -25,7 +25,9 @@ name: Pre-release E2E test # Variables (non-secret): PRE_E2E_SAFE_API_BASE, PRE_E2E_SAFE_WORKSPACE_ID, # PRE_E2E_AUTHORING_IMAGE, PRE_E2E_NFS_ROOT, PRE_E2E_MODEL_3H_PATH, # PRE_E2E_MODEL_12H_PATH, PRE_E2E_CLAUDE_MODEL, PRE_E2E_CLAUDE_CLI_VERSION, -# (optional) PRE_E2E_ANTHROPIC_BASE_URL, PRE_E2E_SAFE_INSECURE. +# (optional) PRE_E2E_ANTHROPIC_BASE_URL, PRE_E2E_SAFE_INSECURE, +# PRE_E2E_SAFE_CACERT (path to a CA bundle on the runner; takes precedence over +# PRE_E2E_SAFE_INSECURE, which defaults to 1 and skips certificate verification). # PRE_E2E_ANTHROPIC_CUSTOM_HEADERS is NOT required: when a gateway base URL is set, # the bootstrap defaults the AMD APIM header to # `Ocp-Apim-Subscription-Key: ${ANTHROPIC_API_KEY}` (the value is just the key, @@ -267,6 +269,10 @@ jobs: SAFE_API_BASE: ${{ vars.PRE_E2E_SAFE_API_BASE }} SAFE_API_KEY: ${{ secrets.PRE_E2E_SAFE_API_KEY }} # ADMIN token (privileged pod) SAFE_WORKSPACE_ID: ${{ vars.PRE_E2E_SAFE_WORKSPACE_ID }} + # A CA bundle wins over skip-verify (same precedence as ci-e2e / forge-e2e). Unset + # leaves today's behaviour untouched; setting it is how this stops sending an admin + # token over an unverified connection, which is what SAFE_INSECURE=1 does. + SAFE_CACERT: ${{ vars.PRE_E2E_SAFE_CACERT }} SAFE_INSECURE: ${{ vars.PRE_E2E_SAFE_INSECURE || '1' }} AUTHORING_IMAGE: ${{ vars.PRE_E2E_AUTHORING_IMAGE }} # models + agent From 22e19f7e62650c6839e059ec3d572b86f1884882 Mon Sep 17 00:00:00 2001 From: lishuoshuo-amd Date: Tue, 1 Sep 2026 16:16:39 +0800 Subject: [PATCH 48/52] pre-release-e2e: stop a dependency edit from buying a four-leg round pyproject.toml has to stay in on.pull_request.paths, or a PR that only bumps the version would never start the workflow. The scope decision then read "no version bump" as "the CI's own logic changed" -- but a plain dependency or tooling edit satisfies that same path filter, so it took the scripts-only branch and spent four 3h legs on a change with no bearing on the release path. Decide on the CI paths instead. The file list was already being computed for the log line; use it, and run nothing when it comes back empty. This makes the run=false output reachable for the first time: resolve still reports, while build and run skip on their existing conditions. The classification had no test despite being described as one -- add it, running the real decide step over a scratch git repo across five scenarios: version bump, CI-path change, dependency-only edit, dependency edit alongside a CI change, and manual dispatch. Co-authored-by: Cursor --- .github/workflows/pre-release-e2e-test.yml | 16 ++- .../test_pre_release_gate_orchestration.py | 114 ++++++++++++++++++ 2 files changed, 125 insertions(+), 5 deletions(-) diff --git a/.github/workflows/pre-release-e2e-test.yml b/.github/workflows/pre-release-e2e-test.yml index a81be30554..23f0840350 100644 --- a/.github/workflows/pre-release-e2e-test.yml +++ b/.github/workflows/pre-release-e2e-test.yml @@ -148,14 +148,20 @@ jobs: run=true; run_scope="full" echo "version bump vs base ${BASE_REF}: '${prev}' -> '${base_version}' -> FULL run (all 8 legs)" else - # Version unchanged vs base. on.pull_request.paths already guaranteed this PR - # touched one of the watched paths, and it wasn't the version -> it changed - # the CI's own scripts/prompts/workflow. Run the fast scripts-only scope. - run=true; run_scope="scripts-only" + # Version unchanged vs base. pyproject.toml is itself a watched path -- it has + # to be, or a PR that ONLY bumps the version would never start this workflow -- + # so "no version bump" does NOT imply the CI's own logic changed. A plain + # dependency or tooling edit lands here and used to buy a 4-leg round it has no + # use for. Decide on the CI paths themselves; anything else is not our concern. changed="$(git diff --name-only "${BASE_SHA}" HEAD 2>/dev/null | grep -E \ '^\.github/(workflows/pre-release-e2e-test\.yml|scripts/pre-release-e2e-.*\.sh|pre-release/)' \ | paste -sd',' - || true)" - echo "version unchanged vs base ${BASE_REF}; CI logic changed (${changed:-see path filter}) -> SCRIPTS-ONLY run (4 fast 3h legs)" + if [ -n "$changed" ]; then + run=true; run_scope="scripts-only" + echo "version unchanged vs base ${BASE_REF}; CI logic changed (${changed}) -> SCRIPTS-ONLY run (4 fast 3h legs)" + else + echo "version unchanged vs base ${BASE_REF} and no pre-release CI path touched -> no run" + fi fi fi diff --git a/src/hyperloom/inference_optimizer/tests/test_pre_release_gate_orchestration.py b/src/hyperloom/inference_optimizer/tests/test_pre_release_gate_orchestration.py index 2c2b020cc4..1263b71b85 100644 --- a/src/hyperloom/inference_optimizer/tests/test_pre_release_gate_orchestration.py +++ b/src/hyperloom/inference_optimizer/tests/test_pre_release_gate_orchestration.py @@ -24,6 +24,7 @@ from __future__ import annotations +import os import re import subprocess from pathlib import Path @@ -208,6 +209,119 @@ def test_the_env_file_does_not_claim_to_stay_off_nfs(dispatch_script: str, boots assert "only to pod-local" not in script +# ---- trigger classification: what a PR costs in GPU hours ---- +# pyproject.toml must stay in on.pull_request.paths, or a PR that only bumps the version +# would never start the workflow. So the scope decision cannot infer "not a version bump, +# therefore CI logic changed" -- a dependency edit satisfies the path filter too. + +_PYPROJECT = '[project]\nname = "x"\nversion = "{version}"\ndependencies = [{deps}]\n' +_CI_PATHS = ( + ".github/workflows/pre-release-e2e-test.yml", + ".github/scripts/pre-release-e2e-poll.sh", + ".github/pre-release/bootstrap-pre-release.sh", +) + + +def _decide_step_script(workflow: dict) -> str: + steps = workflow["jobs"]["resolve"]["steps"] + return next(s["run"] for s in steps if s.get("id") == "decide") + + +def _git(repo: Path, *args: str) -> str: + return subprocess.run(["git", *args], cwd=repo, capture_output=True, text=True, check=True).stdout.strip() + + +def _decide(workflow: dict, tmp_path: Path, *, edits: dict[str, str], event: str = "pull_request") -> dict[str, str]: + """Commit a base tree, apply `edits`, then run the real decide step over the diff.""" + repo = tmp_path / "repo" + repo.mkdir() + _git(repo, "init", "-q", "-b", "main") + _git(repo, "config", "user.email", "t@t") + _git(repo, "config", "user.name", "t") + (repo / "pyproject.toml").write_text(_PYPROJECT.format(version="1.0.0", deps=""), encoding="utf-8") + for rel in _CI_PATHS: + path = repo / rel + path.parent.mkdir(parents=True, exist_ok=True) + path.write_text("base\n", encoding="utf-8") + _git(repo, "add", "-A") + _git(repo, "commit", "-qm", "base") + base_sha = _git(repo, "rev-parse", "HEAD") + + for rel, body in edits.items(): + (repo / rel).write_text(body, encoding="utf-8") + if edits: + _git(repo, "add", "-A") + _git(repo, "commit", "-qm", "head") + + out = tmp_path / "gh_output" + out.write_text("", encoding="utf-8") + subprocess.run( + ["bash", "-c", _decide_step_script(workflow)], + cwd=repo, + env={ + "PATH": os.environ["PATH"], + "HOME": str(tmp_path), + "EVENT": event, + "REUSE_IN": "", + "TASKS_IN": "", + "BASE_SHA": base_sha, + "BASE_REF": "main", + "GITHUB_OUTPUT": str(out), + }, + capture_output=True, + text=True, + check=True, + ) + return dict(line.split("=", 1) for line in out.read_text(encoding="utf-8").splitlines() if "=" in line) + + +def test_a_version_bump_runs_every_leg(workflow: dict, tmp_path: Path) -> None: + got = _decide(workflow, tmp_path, edits={"pyproject.toml": _PYPROJECT.format(version="1.0.1", deps="")}) + assert got["run"] == "true" + assert got["run_scope"] == "full" + assert got["tasks"] == "" # empty = all 8 in dispatch + assert got["ci_version"].startswith("1.0.1.dev") + + +def test_touching_this_ci_runs_the_fast_legs(workflow: dict, tmp_path: Path) -> None: + got = _decide(workflow, tmp_path, edits={".github/scripts/pre-release-e2e-poll.sh": "changed\n"}) + assert got["run"] == "true" + assert got["run_scope"] == "scripts-only" + assert got["tasks"].split(",") == [ + "baremetal-vllm-3h", + "baremetal-sglang-3h", + "docker-vllm-3h", + "docker-sglang-3h", + ] + + +def test_a_dependency_edit_costs_no_gpu_time(workflow: dict, tmp_path: Path) -> None: + """A pyproject change that is not a version bump must not buy a 4-leg round.""" + got = _decide(workflow, tmp_path, edits={"pyproject.toml": _PYPROJECT.format(version="1.0.0", deps='"requests"')}) + assert got["run"] == "false" + assert got["run_scope"] == "none" + + +def test_a_dependency_edit_alongside_a_ci_change_still_runs(workflow: dict, tmp_path: Path) -> None: + got = _decide( + workflow, + tmp_path, + edits={ + "pyproject.toml": _PYPROJECT.format(version="1.0.0", deps='"requests"'), + ".github/pre-release/bootstrap-pre-release.sh": "changed\n", + }, + ) + assert got["run"] == "true" + assert got["run_scope"] == "scripts-only" + + +def test_a_manual_run_is_always_full_scope(workflow: dict, tmp_path: Path) -> None: + got = _decide(workflow, tmp_path, edits={}, event="workflow_dispatch") + assert got["run"] == "true" + assert got["run_scope"] == "full" + assert got["tasks"] == "" + + # ---- reusing a CI_VERSION must not let the previous run's artifacts pass the gate ---- # A reused CI_VERSION (workflow_dispatch reuse_ci_version, or a job re-run) puts this run # on the paths a finished run already wrote. Verdicts are recorded once and never revisited From 051c9451fa2074e3234121468c79dabde911baf7 Mon Sep 17 00:00:00 2001 From: lishuoshuo-amd Date: Tue, 1 Sep 2026 16:21:31 +0800 Subject: [PATCH 49/52] pre-release-e2e: let an unreachable SaFE API say so instead of stalling for 14h workload_phase sent curl's stderr to /dev/null and swallowed its exit status, so jq read an empty body and produced an empty phase. Empty is not one of the terminal phases, so every leg stayed PENDING and the poll held 8 GPUs and the only self-hosted runner until GLOBAL_TIMEOUT_S -- with nothing in the log to say the API had refused the connection, rejected the token, or gone away. It now returns __APIERR__ with the HTTP status and keeps curl's diagnosis, which the loop reports once per tick rather than once per leg. After API_FAIL_ABORT consecutive polls in which EVERY query failed (~20min by default) it stops waiting and fails the unjudged legs with that reason. Only a total outage counts: one workload disappearing must not end a run whose other legs are progressing, and state.json still comes off NFS, so a leg that exits cleanly is judged either way. What an unreachable API costs is telling a zombie leg from a slow one, which is the only thing the remaining wait was for. Co-authored-by: Cursor --- .github/scripts/pre-release-e2e-poll.sh | 56 +++++++++++++++++-- .../test_pre_release_gate_orchestration.py | 44 +++++++++++++++ 2 files changed, 96 insertions(+), 4 deletions(-) diff --git a/.github/scripts/pre-release-e2e-poll.sh b/.github/scripts/pre-release-e2e-poll.sh index 95294db26d..76fea5cf57 100755 --- a/.github/scripts/pre-release-e2e-poll.sh +++ b/.github/scripts/pre-release-e2e-poll.sh @@ -50,6 +50,11 @@ POLL_INTERVAL_S="${POLL_INTERVAL_S:-120}" GLOBAL_TIMEOUT_S="${GLOBAL_TIMEOUT_S:-52200}" MAX_CRASHES="${MAX_CRASHES:-0}" LEAVE_RUNNING_FILE="${LEAVE_RUNNING_FILE:-${DISPATCH_MAP}.leave_running}" +API_ERR_FILE="${API_ERR_FILE:-${DISPATCH_MAP}.api_err}" +# Consecutive polls in which EVERY workload query failed. A poll that cannot reach the +# API learns nothing from waiting, so give up rather than hold 8 GPUs and the only +# self-hosted runner until GLOBAL_TIMEOUT_S. Default ~20min at POLL_INTERVAL_S=120. +API_FAIL_ABORT="${API_FAIL_ABORT:-10}" # This run's dispatch tag, written by dispatch beside the map. Pods stamp it into their # session pin; an untagged/foreign pin is a leftover from an earlier run on these paths. RUN_TAG="${RUN_TAG:-$(cat "${DISPATCH_MAP}.version_tag" 2>/dev/null || true)}" @@ -266,10 +271,20 @@ done_count() { echo "$n" } -workload_phase() { # workloadId -> phase string - local wid="$1" detail - detail="$(curl -sS "${tls[@]}" "$API/$wid" "${auth[@]}" 2>/dev/null || true)" - printf '%s' "$detail" | jq -r '.phase // "Unknown"' 2>/dev/null || echo Unknown +# workloadId -> phase string, or __APIERR__ when the query itself did not answer. +# Discarding curl's stderr here used to make an unreachable or unauthorized API +# indistinguishable from a workload that simply had no phase yet: every leg read +# `Unknown`, which is not terminal, so the poll waited out GLOBAL_TIMEOUT_S with nothing +# in the log to say why. The error text goes to $API_ERR_FILE for the caller to report +# once per tick; this runs inside a command substitution, so it cannot count anything. +workload_phase() { + local wid="$1" body code + body="$(curl -sS "${tls[@]}" -w $'\n%{http_code}' "$API/$wid" "${auth[@]}" 2>"$API_ERR_FILE")" || true + code="$(printf '%s' "$body" | tail -n1)" + case "$code" in + 2*) printf '%s' "$body" | sed '$d' | jq -r '.phase // "Unknown"' 2>/dev/null || echo Unknown ;; + *) printf '%s' "__APIERR__ (HTTP ${code:-none})" ;; + esac } # Resolve a leg's session dir. Bootstrap writes the pinned session dir to @@ -368,6 +383,8 @@ start_s="$(date +%s)" fail_seen=0 # has any leg reached a FAIL verdict? -> the gate is already decided gate_fail_announced=0 superseded=0 # a newer workflow run is queued -> release runner without stopping pods +api_dead=0 # the API has been unreachable for API_FAIL_ABORT consecutive polls +api_fail_streak=0 while :; do if superseded_by_newer_run; then mark_superseded_and_exit_poll @@ -375,10 +392,13 @@ while :; do fi pending=0 changed=0 # did any leg reach a verdict this tick? -> refresh the sticky comment + api_err=0; api_queried=0 for leg in "${LEGS[@]}"; do [ -n "${VERDICT[$leg]}" ] && continue wid="${WID[$leg]}" wphase="$(workload_phase "$wid")" + api_queried=$(( api_queried + 1 )) + case "$wphase" in __APIERR__*) api_err=$(( api_err + 1 )) ;; esac res="$(judge_leg "$leg" "$wphase")" verdict="${res%%|*}"; detail="${res#*|}" if [ "$verdict" = "PASS" ]; then @@ -403,6 +423,34 @@ while :; do fi done + # Report an API outage once per tick rather than once per leg, and stop waiting on one + # that is total: state.json still comes off NFS, so a leg that exits cleanly is judged + # either way -- what an unreachable API costs is the ability to tell a zombie leg from + # a slow one, which is exactly what the remaining wait would have been for. + if [ "$api_err" -gt 0 ] && [ "$api_err" -eq "$api_queried" ]; then + api_fail_streak=$(( api_fail_streak + 1 )) + api_err_text="$(tr '\r\n' ' ' < "$API_ERR_FILE" 2>/dev/null | head -c 300 || true)" + echo "WARN: all $api_queried workload queries failed (streak ${api_fail_streak}/${API_FAIL_ABORT}): ${api_err_text:-no stderr from curl}" >&2 + if [ "$api_fail_streak" -ge "$API_FAIL_ABORT" ]; then + api_dead=1 + summary "" + summary "❌ **SaFE API unreachable** for ${api_fail_streak} consecutive polls — abandoning the wait instead of holding the runner to the ${GLOBAL_TIMEOUT_S}s timeout." + summary "" + summary "\`\`\`" + summary "${api_err_text:-no stderr from curl}" + summary "\`\`\`" + for leg in "${LEGS[@]}"; do + [ -n "${VERDICT[$leg]}" ] && continue + VERDICT["$leg"]="FAIL|SaFE API unreachable for ${api_fail_streak} polls" + post_status "$leg" error "SaFE API unreachable" + done + break + fi + else + [ "$api_fail_streak" -gt 0 ] && echo "[poll] workload queries recovered after ${api_fail_streak} failed poll(s)" + api_fail_streak=0 + fi + # A leg finished this tick -> refresh the single sticky report comment (point C). [ "$changed" -eq 1 ] && report_upsert "$(report_body Running "$(done_count)" "${#LEGS[@]}")" diff --git a/src/hyperloom/inference_optimizer/tests/test_pre_release_gate_orchestration.py b/src/hyperloom/inference_optimizer/tests/test_pre_release_gate_orchestration.py index 1263b71b85..437f305a90 100644 --- a/src/hyperloom/inference_optimizer/tests/test_pre_release_gate_orchestration.py +++ b/src/hyperloom/inference_optimizer/tests/test_pre_release_gate_orchestration.py @@ -322,6 +322,50 @@ def test_a_manual_run_is_always_full_scope(workflow: dict, tmp_path: Path) -> No assert got["tasks"] == "" +# ---- an unreachable API must not read as "no phase yet" ---- + + +def _workload_phase(poll_script: str, tmp_path: Path, api: str) -> tuple[str, str]: + """Run the real workload_phase() against `api`; return (stdout, captured curl stderr).""" + lines = poll_script.splitlines() + start = next(i for i, line in enumerate(lines) if line.startswith("workload_phase() {")) + end = next(i for i in range(start, len(lines)) if lines[i] == "}") + fn = "\n".join(lines[start : end + 1]) + err_file = tmp_path / "api_err" + proc = subprocess.run( + [ + "bash", + "-c", + f'set -euo pipefail\nAPI="$1"; API_ERR_FILE="$2"; tls=(); auth=()\n{fn}\nworkload_phase wid-1', + "_", + api, + str(err_file), + ], + capture_output=True, + text=True, + check=True, + ) + return proc.stdout, err_file.read_text(encoding="utf-8") if err_file.is_file() else "" + + +def test_an_unreachable_api_is_not_mistaken_for_a_missing_phase(poll_script: str, tmp_path: Path) -> None: + """`Unknown` is not terminal, so swallowing the error waited out the global timeout.""" + # Port 1 on loopback refuses instantly -- no network egress, no timeout to wait on. + out, err = _workload_phase(poll_script, tmp_path, "https://127.0.0.1:1") + assert out.startswith("__APIERR__"), out + assert err.strip(), "curl's diagnosis must be kept, not sent to /dev/null" + + +def test_the_poll_gives_up_on_a_total_api_outage(poll_script: str) -> None: + """Waiting out 14h holds 8 GPUs to learn nothing the first failed poll did not say.""" + assert 'API_FAIL_ABORT="${API_FAIL_ABORT:-10}"' in poll_script + assert 'case "$wphase" in __APIERR__*) api_err=$(( api_err + 1 )) ;; esac' in poll_script + # Only a TOTAL outage counts: one leg's workload going missing must not abort the run. + assert '[ "$api_err" -gt 0 ] && [ "$api_err" -eq "$api_queried" ]' in poll_script + assert 'VERDICT["$leg"]="FAIL|SaFE API unreachable' in poll_script + assert "api_fail_streak=0" in poll_script + + # ---- reusing a CI_VERSION must not let the previous run's artifacts pass the gate ---- # A reused CI_VERSION (workflow_dispatch reuse_ci_version, or a job re-run) puts this run # on the paths a finished run already wrote. Verdicts are recorded once and never revisited From 4ba72bfa3fe7285beae6933d7b60936abb6411af Mon Sep 17 00:00:00 2001 From: lishuoshuo-amd Date: Tue, 1 Sep 2026 16:54:17 +0800 Subject: [PATCH 50/52] pre-release-e2e: derive a docker leg's GPU from its position, and drop the map DOCKER_GPU_MAP travelled to the host pod alongside DOCKER_LEGS carrying a leg->index mapping. Being a second copy of an ordering the list already has, the only thing it could contribute was disagreeing with it: `jq -r '.[$l]'` answers the string "null" for a missing key, arithmetic reads that as 0, and two legs bind renderD128 and the same card. Both lists were built in one loop, so nothing in this repo could trigger it -- but the map bought no safety for that risk either. The host now numbers DOCKER_LEGS as it walks it, which is the assignment. Dispatch numbers the same list for its summary line and cannot drift, because it is the same list. A leg name that is not in it yields empty rather than 0, and run_leg's existing GPU_INDEX guard still rejects that. Assignment is now positional rather than fixed per leg name, so a subset run packs onto the low cards instead of leaving gaps; each leg's card is logged on launch. Co-authored-by: Cursor --- .github/pre-release/bootstrap-pre-release.sh | 21 +++++++----- .github/scripts/pre-release-e2e-dispatch.sh | 32 +++++++++---------- .../test_pre_release_gate_orchestration.py | 14 ++++++++ 3 files changed, 42 insertions(+), 25 deletions(-) diff --git a/.github/pre-release/bootstrap-pre-release.sh b/.github/pre-release/bootstrap-pre-release.sh index 96993d88d3..34219d3dd5 100755 --- a/.github/pre-release/bootstrap-pre-release.sh +++ b/.github/pre-release/bootstrap-pre-release.sh @@ -24,7 +24,7 @@ # EXIT trap scrubs the key, so a SIGKILL leaves it on disk. # ANTHROPIC_BASE_URL (optional) CLAUDE_MODEL CLAUDE_CLI_VERSION TARGET_GAIN # Baremetal leg: LEG_ID HYPERLOOM_RUN_MODE=baremetal HYPERLOOM_BACKEND HYPERLOOM_MODEL_PATH DEMO_HOURS -# Docker host: E2E_DOCKER_HOST=1 DOCKER_LEGS DOCKER_GPU_MAP(json) MODEL_3H MODEL_12H +# Docker host: E2E_DOCKER_HOST=1 DOCKER_LEGS MODEL_3H MODEL_12H set -euo pipefail : "${CI_VERSION:?}"; : "${NFS_ROOT:?}"; : "${ANTHROPIC_API_KEY_B64:?}" @@ -577,12 +577,17 @@ ensure_dockerd() { # run_leg works unchanged. Each `run_leg &` is its own subshell, so their per-leg EXIT # traps (the .env key scrub) don't clobber each other. run_docker_host() { - : "${DOCKER_LEGS:?}"; : "${DOCKER_GPU_MAP:?}"; : "${MODEL_3H:?}"; : "${MODEL_12H:?}" + : "${DOCKER_LEGS:?}"; : "${MODEL_3H:?}"; : "${MODEL_12H:?}" ensure_dockerd || { log "ERROR: cannot provide docker on the host pod"; return 1; } log "docker host: legs='${DOCKER_LEGS}'" - local pids=() leg idx backend hours model_path + # The index is the leg's position in DOCKER_LEGS. A leg->index map used to travel + # alongside this list; being a second copy of the same ordering, its only possible + # contribution was to disagree with it -- and a lookup miss yielded the string "null", + # which arithmetic reads as 0, quietly binding two legs to the same card. Dispatch + # numbers the same list the same way for its summary line. + local pids=() leg idx=-1 backend hours model_path for leg in $DOCKER_LEGS; do - idx="$(printf '%s' "$DOCKER_GPU_MAP" | jq -r --arg l "$leg" '.[$l]')" + idx=$(( idx + 1 )) case "$leg" in *-vllm-*) backend=vllm ;; *-sglang-*) backend=sglang ;; @@ -602,10 +607,10 @@ run_docker_host() { return "$rc" } -# The SaFE rocm/pytorch Authoring image is minimal: besides node/npm (see ensure_node) -# it also lacks `jq`, which run_docker_host uses to read DOCKER_GPU_MAP. Install it up -# front (apt works in the pod) so both the host pod and the nested single-leg containers -# -- which re-run THIS script -- have it. Idempotent; cheap when already present. +# The SaFE rocm/pytorch Authoring image is minimal: besides node/npm (see ensure_node) it +# also lacks `jq`, which the wait loop needs to read stop_reason out of state.json. Install +# it up front (apt works in the pod) so both the host pod and the nested single-leg +# containers -- which re-run THIS script -- have it. Idempotent; cheap when present. ensure_base_tools() { command -v jq >/dev/null 2>&1 && return 0 log "installing jq (not in the Authoring base image)" diff --git a/.github/scripts/pre-release-e2e-dispatch.sh b/.github/scripts/pre-release-e2e-dispatch.sh index 1a89b33e27..7c96e5972f 100755 --- a/.github/scripts/pre-release-e2e-dispatch.sh +++ b/.github/scripts/pre-release-e2e-dispatch.sh @@ -189,17 +189,6 @@ leg_model_path() { case "$1" in *-3h) echo "$MODEL_3H" ;; *-12h) echo "$MODEL_12 leg_hours() { case "$1" in *-3h) echo "3" ;; *-12h) echo "12" ;; esac; } leg_backend() { case "$1" in *-vllm-*) echo "vllm" ;; *-sglang-*) echo "sglang" ;; esac; } -# GPU index a docker leg binds inside the privileged host (design §3). -docker_gpu_index() { - case "$1" in - docker-vllm-3h) echo 0 ;; - docker-vllm-12h) echo 1 ;; - docker-sglang-3h) echo 2 ;; - docker-sglang-12h) echo 3 ;; - *) echo "" ;; - esac -} - # Common env for every workload. The API key is passed base64 so it is not visible in # plaintext in the API payload log; bootstrap decodes it into the leg's .env, which sits # on NFS beside the workspace and is scrubbed by an EXIT trap. See design §9 (point D). @@ -333,17 +322,26 @@ leg_resources_1gpu="$(jq -n --arg cpu "$LEG_CPU" --arg mem "$LEG_MEM" --arg eph # It schedules more slowly and spends minutes on dockerd + image pulls before the # nested legs even start setup, so queue it before the four 1-GPU baremetal pods. want_docker_host=0 -docker_legs=""; gpu_map="{}" +docker_legs="" for leg in $REQ_TASKS; do case "$leg" in docker-*) want_docker_host=1 - idx="$(docker_gpu_index "$leg")" docker_legs="${docker_legs}${docker_legs:+ }${leg}" - gpu_map="$(printf '%s' "$gpu_map" | jq --arg l "$leg" --arg i "$idx" '. + {($l): $i}')" ;; esac done +# The host pod binds each leg to the GPU at its position in DOCKER_LEGS (design §3), so +# there is nothing to send: the ordered list IS the assignment. Numbering it here too is +# only for the summary below, and cannot disagree because it is the same list. +docker_leg_gpu_index() { # leg -> its position in $docker_legs, or "" when absent + local want="$1" i=0 leg + for leg in $docker_legs; do + [ "$leg" = "$want" ] && { printf '%s' "$i"; return 0; } + i=$(( i + 1 )) + done + printf '' +} # ---- docker legs: one privileged 8-GPU host running all requested docker legs ---- if [ "$want_docker_host" = 1 ]; then @@ -360,7 +358,7 @@ if [ "$want_docker_host" = 1 ]; then --arg keyb64 "$(printf '%s' "$ANTHROPIC_API_KEY" | base64 | tr -d '\n')" \ --arg baseurl "${ANTHROPIC_BASE_URL:-}" \ --arg cheaders "${ANTHROPIC_CUSTOM_HEADERS:-}" \ - --arg legs "$docker_legs" --argjson gpumap "$gpu_map" \ + --arg legs "$docker_legs" \ --arg dm3 "$DOCKER_LEG_MEM_3H" --arg dm12 "$DOCKER_LEG_MEM_12H" \ --arg ds3 "$DOCKER_LEG_SHM_3H" --arg ds12 "$DOCKER_LEG_SHM_12H" \ --arg rtag "$VERSION_TAG" \ @@ -372,7 +370,7 @@ if [ "$want_docker_host" = 1 ]; then ANTHROPIC_API_KEY_B64:$keyb64, HYPERLOOM_RUN_MODE:"docker", E2E_DOCKER_HOST:"1", - DOCKER_LEGS:$legs, DOCKER_GPU_MAP:($gpumap|tostring), + DOCKER_LEGS:$legs, DOCKER_LEG_MEM_3H:$dm3, DOCKER_LEG_MEM_12H:$dm12, DOCKER_LEG_SHM_3H:$ds3, DOCKER_LEG_SHM_12H:$ds12 } @@ -393,7 +391,7 @@ if [ "$want_docker_host" = 1 ]; then # reading each leg's own session dir on NFS. for leg in $docker_legs; do record_dispatch "$leg" "$wid" - summary "• \`$leg\` → workloadId \`$wid\` (docker host, GPU $(docker_gpu_index "$leg"), deadline $((host_dl/3600))h)" + summary "• \`$leg\` → workloadId \`$wid\` (docker host, GPU $(docker_leg_gpu_index "$leg"), deadline $((host_dl/3600))h)" done fi diff --git a/src/hyperloom/inference_optimizer/tests/test_pre_release_gate_orchestration.py b/src/hyperloom/inference_optimizer/tests/test_pre_release_gate_orchestration.py index 437f305a90..8cdca14c37 100644 --- a/src/hyperloom/inference_optimizer/tests/test_pre_release_gate_orchestration.py +++ b/src/hyperloom/inference_optimizer/tests/test_pre_release_gate_orchestration.py @@ -356,6 +356,20 @@ def test_an_unreachable_api_is_not_mistaken_for_a_missing_phase(poll_script: str assert err.strip(), "curl's diagnosis must be kept, not sent to /dev/null" +def test_the_gpu_assignment_is_the_leg_order_not_a_second_copy_of_it( + dispatch_script: str, bootstrap_script: str +) -> None: + """A parallel leg->GPU map could only ever disagree with DOCKER_LEGS; a miss read as 0.""" + for script in (dispatch_script, bootstrap_script): + assert "DOCKER_GPU_MAP" not in script + assert "gpu_map" not in script + # Both sides number the same ordered list: dispatch for its summary, the host for the + # binding. Same list, same counting, so they cannot drift apart. + assert "docker_leg_gpu_index() {" in dispatch_script + assert "local pids=() leg idx=-1 backend hours model_path" in bootstrap_script + assert "idx=$(( idx + 1 ))" in bootstrap_script + + def test_the_poll_gives_up_on_a_total_api_outage(poll_script: str) -> None: """Waiting out 14h holds 8 GPUs to learn nothing the first failed poll did not say.""" assert 'API_FAIL_ABORT="${API_FAIL_ABORT:-10}"' in poll_script From cb6b386f6ad3e43e8f13cab72e2ca365dec14ad1 Mon Sep 17 00:00:00 2001 From: lishuoshuo-amd Date: Tue, 1 Sep 2026 17:07:39 +0800 Subject: [PATCH 51/52] pre-release-e2e: give a manually dispatched run its per-leg checks back GH_STATUS_SHA came only from the pull_request event, so on workflow_dispatch it was empty and gh_status_on / gh_report_on returned early without logging: no per-leg pre-release-e2e/ statuses, no sticky report, nothing saying why. Manual dispatch is the entry point that carries reuse_ci_version and a leg subset, so the run most likely to happen at release time was the one with no visibility. Fall back to github.sha, the dispatched ref's real commit. The pull_request path is unchanged -- statuses still land on the PR head, not the ephemeral merge commit -- and a populated SHA also revives the poll's commit->PR lookup, which was dead code in exactly the case it was written for. Co-authored-by: Cursor --- .github/workflows/pre-release-e2e-test.yml | 10 ++++++++-- .../tests/test_pre_release_gate_orchestration.py | 12 ++++++++++++ 2 files changed, 20 insertions(+), 2 deletions(-) diff --git a/.github/workflows/pre-release-e2e-test.yml b/.github/workflows/pre-release-e2e-test.yml index 23f0840350..ef397e30d0 100644 --- a/.github/workflows/pre-release-e2e-test.yml +++ b/.github/workflows/pre-release-e2e-test.yml @@ -299,8 +299,14 @@ jobs: GH_STATUS_REPO: ${{ github.repository }} # On pull_request, github.sha is the ephemeral merge commit; per-leg commit # statuses must land on the PR HEAD sha so they surface on the PR checks tab. - GH_STATUS_SHA: ${{ github.event.pull_request.head.sha }} - # PR number is known directly from the event -- no commit->PR reverse lookup. + # workflow_dispatch carries no pull_request object, so without the fallback the + # SHA was empty and every status/report call returned early without a word -- + # leaving the entry point most likely to be used at release time with no checks + # at all. github.sha is the dispatched ref's real commit, which statuses can + # attach to, and it also revives the poll's commit->PR lookup for the report. + GH_STATUS_SHA: ${{ github.event.pull_request.head.sha || github.sha }} + # Known directly from a pull_request event; empty on dispatch, where the poll + # falls back to resolving the PR from the commit. PR_NUMBER: ${{ github.event.pull_request.number }} HEAD_REF: ${{ github.head_ref }} GH_STATUS_DETAILS_URL: ${{ github.server_url }}/${{ github.repository }}/actions/runs/${{ github.run_id }} diff --git a/src/hyperloom/inference_optimizer/tests/test_pre_release_gate_orchestration.py b/src/hyperloom/inference_optimizer/tests/test_pre_release_gate_orchestration.py index 8cdca14c37..f628ba060f 100644 --- a/src/hyperloom/inference_optimizer/tests/test_pre_release_gate_orchestration.py +++ b/src/hyperloom/inference_optimizer/tests/test_pre_release_gate_orchestration.py @@ -356,6 +356,18 @@ def test_an_unreachable_api_is_not_mistaken_for_a_missing_phase(poll_script: str assert err.strip(), "curl's diagnosis must be kept, not sent to /dev/null" +def test_a_manual_run_still_gets_per_leg_checks(workflow: dict, poll_script: str) -> None: + """An empty SHA silently disabled every status and the report, with nothing logged.""" + sha = workflow["jobs"]["run"]["env"]["GH_STATUS_SHA"] + assert "github.event.pull_request.head.sha" in sha + assert "github.sha" in sha, "workflow_dispatch has no pull_request object" + # The gates that a missing SHA short-circuits, including the report's PR lookup that + # only becomes reachable once the SHA is populated. + assert "gh_status_on || return 0" in poll_script + assert "statuses/${GH_STATUS_SHA}" in poll_script + assert "commits/${GH_STATUS_SHA}/pulls" in poll_script + + def test_the_gpu_assignment_is_the_leg_order_not_a_second_copy_of_it( dispatch_script: str, bootstrap_script: str ) -> None: From 98f406577b9d4f242820021a7bf9789aa1c4ed53 Mon Sep 17 00:00:00 2001 From: lishuoshuo-amd Date: Tue, 1 Sep 2026 17:56:39 +0800 Subject: [PATCH 52/52] pre-release-e2e: check the staged artifacts before claiming any GPU reuse_ci_version skips the build job that publishes the wheel, the bootstrap script and the prompts, and nothing on the CI side looked at whether they were there. A wrong version string or a cleaned-up NFS dir was therefore discovered only inside the pods, and only after each had been scheduled, pulled a ROCm image, apt-installed jq and npm-installed the Claude CLI -- the wheel check sits behind all of that in bootstrap's entry. All eight legs then reported the same "state.json missing (workload phase=Failed)", which names none of it, and SaFE deletes a failed leg's pod before its stdout can be read. Stat the four things the pods need before dispatch and fail with the list of what is absent. It costs nothing and no GPU is claimed. Also name the provenance on a reused version: skipping build means the pods run the bootstrap and prompts staged by THAT build, not this branch's. Reusing a wheel to retest a script change would otherwise pass silently on the old script -- a quiet wrong answer rather than a loud failure. The step now prints the commit and run id the staged copies came from. Co-authored-by: Cursor --- .github/workflows/pre-release-e2e-test.yml | 51 +++++++++++++++++++ .../test_pre_release_gate_orchestration.py | 23 +++++++++ 2 files changed, 74 insertions(+) diff --git a/.github/workflows/pre-release-e2e-test.yml b/.github/workflows/pre-release-e2e-test.yml index ef397e30d0..72e202e083 100644 --- a/.github/workflows/pre-release-e2e-test.yml +++ b/.github/workflows/pre-release-e2e-test.yml @@ -322,6 +322,57 @@ jobs: command -v jq >/dev/null 2>&1 || (sudo apt-get update && sudo apt-get install -y jq) echo "DISPATCH_MAP=${RUNNER_TEMP}/pre_release_dispatch.json" >> "$GITHUB_ENV" + - name: Verify the staged wheel, bootstrap and prompts + # The pods read all three off NFS, so anything missing is only discovered inside + # them -- and only AFTER the pod has been scheduled, the ROCm image pulled, jq + # apt-installed and the Claude CLI npm-installed, because the wheel check sits + # behind those in bootstrap's entry. All 8 legs then report the same + # "state.json missing (workload phase=Failed)", which names none of this, and SaFE + # deletes a failed leg's pod (with its stdout) before anyone can look. `reuse` + # skips the build job that publishes them, so it is the way to arrive here with + # nothing staged: a wrong version string, or a cleaned-up NFS dir. Cost of + # checking: four stats, no GPU claimed. + env: + NFS_ROOT: ${{ vars.PRE_E2E_NFS_ROOT }} + REUSE: ${{ needs.resolve.outputs.reuse }} + run: | + set -euo pipefail + : "${NFS_ROOT:?PRE_RELEASE_NFS_ROOT var is required}" + wheel_dir="${NFS_ROOT%/}/wheels/${CI_VERSION}" + boot_dir="${NFS_ROOT%/}/bootstrap/${CI_VERSION}" + missing="" + [ -f "$wheel_dir/manifest.json" ] || missing="${missing}\n - $wheel_dir/manifest.json" + ls "$wheel_dir"/hyperloom_inference_optimizer-*.whl >/dev/null 2>&1 \ + || missing="${missing}\n - a wheel in $wheel_dir" + [ -f "$boot_dir/bootstrap-pre-release.sh" ] || missing="${missing}\n - $boot_dir/bootstrap-pre-release.sh" + for p in demo-3h demo-12h setup-baremetal-vllm setup-baremetal-sglang \ + setup-docker-vllm setup-docker-sglang; do + [ -f "$boot_dir/prompts/pre-release/${p}.md" ] \ + || missing="${missing}\n - $boot_dir/prompts/pre-release/${p}.md" + done + if [ -n "$missing" ]; then + { + echo "❌ **Nothing to run:** \`CI_VERSION=${CI_VERSION}\` is not staged on NFS." + echo "" + echo "Missing:" + printf '%b\n' "$missing" + } | tee -a "${GITHUB_STEP_SUMMARY:-/dev/null}" >&2 + exit 1 + fi + # A reused version brings that build's bootstrap and prompts with it, not this + # branch's -- the build job that would restage them is skipped. Say whose code + # is about to run, so "I reused a wheel to test my script change" cannot pass + # silently on the old script. + if [ -n "${REUSE:-}" ]; then + sha="$(jq -r '.git_sha // "unknown"' "$wheel_dir/manifest.json")" + rid="$(jq -r '.github_run_id // "unknown"' "$wheel_dir/manifest.json")" + { + echo "♻️ Reusing \`${CI_VERSION}\`: the pods will run the bootstrap and prompts" + echo "staged by run \`${rid}\` from commit \`${sha}\`, NOT this branch's copies." + } | tee -a "${GITHUB_STEP_SUMMARY:-/dev/null}" + fi + echo "staged wheel + bootstrap + 6 prompts present for $CI_VERSION" + - name: Dispatch SaFE Authoring workloads run: | chmod +x .github/scripts/pre-release-e2e-dispatch.sh diff --git a/src/hyperloom/inference_optimizer/tests/test_pre_release_gate_orchestration.py b/src/hyperloom/inference_optimizer/tests/test_pre_release_gate_orchestration.py index f628ba060f..99c0659d11 100644 --- a/src/hyperloom/inference_optimizer/tests/test_pre_release_gate_orchestration.py +++ b/src/hyperloom/inference_optimizer/tests/test_pre_release_gate_orchestration.py @@ -356,6 +356,29 @@ def test_an_unreachable_api_is_not_mistaken_for_a_missing_phase(poll_script: str assert err.strip(), "curl's diagnosis must be kept, not sent to /dev/null" +def test_nothing_is_dispatched_before_the_staged_artifacts_are_checked(workflow: dict) -> None: + """`reuse` skips the build job, so the wheel/bootstrap/prompts may not be there at all. + + In-pod the wheel check sits behind an apt and an npm install, and a failed leg's pod + (with its stdout) is deleted, so the whole set fails slowly and says nothing useful. + """ + steps = workflow["jobs"]["run"]["steps"] + names = [s.get("name") or s.get("uses") or "" for s in steps] + verify = next(i for i, n in enumerate(names) if n.startswith("Verify the staged")) + dispatch = next(i for i, n in enumerate(names) if n.startswith("Dispatch")) + assert verify < dispatch, "the check must gate dispatch, not follow it" + body = steps[verify]["run"] + assert "manifest.json" in body + assert "hyperloom_inference_optimizer-*.whl" in body + assert "bootstrap-pre-release.sh" in body + for prompt in ("demo-3h", "demo-12h", "setup-baremetal-vllm", "setup-docker-sglang"): + assert prompt in body + assert "exit 1" in body + # A reused version carries that build's scripts, not this branch's; say so. + assert "git_sha" in body + assert 'if [ -n "${REUSE:-}" ]; then' in body + + def test_a_manual_run_still_gets_per_leg_checks(workflow: dict, poll_script: str) -> None: """An empty SHA silently disabled every status and the report, with nothing logged.""" sha = workflow["jobs"]["run"]["env"]["GH_STATUS_SHA"]