diff --git a/.github/actionlint.yaml b/.github/actionlint.yaml new file mode 100644 index 0000000000..b79f79e155 --- /dev/null +++ b/.github/actionlint.yaml @@ -0,0 +1,8 @@ +# SPDX-FileCopyrightText: 2026 Advanced Micro Devices, Inc. +# SPDX-License-Identifier: MIT +# +# Declare self-hosted runner labels so actionlint does not flag them as unknown. +# Used by ci-e2e.yml and pre-release-e2e-test.yml. +self-hosted-runner: + labels: + - Hyperloom-e2e-ci diff --git a/.github/pre-release/bootstrap-pre-release.sh b/.github/pre-release/bootstrap-pre-release.sh new file mode 100755 index 0000000000..150ecdddaa --- /dev/null +++ b/.github/pre-release/bootstrap-pre-release.sh @@ -0,0 +1,597 @@ +#!/usr/bin/env bash +# SPDX-FileCopyrightText: 2026 Advanced Micro Devices, Inc. +# SPDX-License-Identifier: MIT +# +# Pre-release E2E pod bootstrap. Runs as the SaFE Authoring workload entrypoint. It +# reproduces the manual release path: install the packaged wheel, write a pod-local +# .env, install a pinned Claude CLI, then drive setup + demo skills through +# `claude --print`. The poll job (outside the pod) judges the session afterwards. +# See hyperloom-pre-release-e2e-ci-design.md §12. +# +# Two modes, selected by E2E_DOCKER_HOST: +# * unset -> a single baremetal leg in THIS pod (LEG_ID given). +# * "1" -> the privileged 8-GPU host: run each of DOCKER_LEGS as a backgrounded +# run_leg (docker mode) ON THE HOST POD. Each leg's agent follows the +# demo skill to `docker run` its OWN single-GPU container (renderD = +# 128 + gpu_index*8), so the skill owns the container lifecycle. The +# per-leg GPU-isolation values are computed here and injected via .env. +# +# Inputs (env, injected by the dispatch script): +# CI_VERSION NFS_ROOT +# ANTHROPIC_API_KEY_B64 base64 key; decoded here, written only to pod-local .env +# ANTHROPIC_BASE_URL (optional) CLAUDE_MODEL CLAUDE_CLI_VERSION TARGET_GAIN +# Baremetal leg: LEG_ID HYPERLOOM_RUN_MODE=baremetal HYPERLOOM_BACKEND HYPERLOOM_MODEL_PATH DEMO_HOURS +# Docker host: E2E_DOCKER_HOST=1 DOCKER_LEGS DOCKER_GPU_MAP(json) MODEL_3H MODEL_12H +set -euo pipefail + +: "${CI_VERSION:?}"; : "${NFS_ROOT:?}"; : "${ANTHROPIC_API_KEY_B64:?}" +: "${CLAUDE_MODEL:?}"; : "${CLAUDE_CLI_VERSION:?}" +TARGET_GAIN="${TARGET_GAIN:-100}" + +PROMPTS_DIR="${NFS_ROOT%/}/bootstrap/${CI_VERSION}/prompts/pre-release" +WHEEL_DIR="${NFS_ROOT%/}/wheels/${CI_VERSION}" + +log() { echo "[bootstrap $(date -u +%H:%M:%S)] $*"; } + +# The SaFE rocm/pytorch Authoring image ships NO node/npm (verified 2026-08-27 on a +# live pod), so the Claude CLI's `npm install -g` fails with "npm: command not found". +# Install a pinned Node.js LTS from the official binary tarball (both nodejs.org and the +# npm registry are reachable from the pod) into /opt/node and expose it on PATH. Idempotent. +NODE_VERSION="${NODE_VERSION:-v20.18.0}" +NODE_PREFIX="${NODE_PREFIX:-/opt/node}" +ensure_node() { + if command -v node >/dev/null 2>&1 && command -v npm >/dev/null 2>&1; then + log "node present: $(node --version 2>/dev/null) npm $(npm --version 2>/dev/null)"; return 0 + fi + log "installing Node ${NODE_VERSION} (no node/npm in the Authoring base image)" + mkdir -p "$NODE_PREFIX" + curl -fsSL "https://nodejs.org/dist/${NODE_VERSION}/node-${NODE_VERSION}-linux-x64.tar.xz" \ + | tar -xJ -C "$NODE_PREFIX" --strip-components=1 \ + || { log "ERROR: Node download/extract failed"; return 1; } + export PATH="${NODE_PREFIX}/bin:${PATH}" + ln -sf "${NODE_PREFIX}/bin/node" /usr/local/bin/node 2>/dev/null || true + ln -sf "${NODE_PREFIX}/bin/npm" /usr/local/bin/npm 2>/dev/null || true + command -v node >/dev/null 2>&1 || { log "ERROR: node still missing after install"; return 1; } + log "node installed: $(node --version) npm $(npm --version)" +} + +install_claude_cli() { + if command -v claude >/dev/null 2>&1; then log "claude CLI present: $(claude --version 2>/dev/null || true)"; return; fi + ensure_node || return 1 + log "installing Claude CLI @ ${CLAUDE_CLI_VERSION}" + # Pinned install. The exact channel is environment-specific; keep the version in one + # place (CLAUDE_CLI_VERSION) so the pin is auditable. Surface npm's error (don't + # swallow) so a failure is diagnosable in bootstrap.log. + npm install -g "@anthropic-ai/claude-code@${CLAUDE_CLI_VERSION}" >/tmp/npm-claude.log 2>&1 \ + || { log "ERROR: claude CLI install failed"; tail -20 /tmp/npm-claude.log; return 1; } + # npm's global bin may be under the Node prefix, not on the default PATH. + ln -sf "${NODE_PREFIX}/bin/claude" /usr/local/bin/claude 2>/dev/null || true + command -v claude >/dev/null 2>&1 || { log "ERROR: claude not on PATH after install"; return 1; } + log "claude CLI installed: $(claude --version 2>/dev/null || true)" +} + +# Seconds a leg has been idle: time since the newest write anywhere under the leg root, +# floored at the wait loop's start so the preceding `claude --print` turns (which can +# leave the tree untouched for longer than the stall grace) are never charged to the leg. +# Args: leg_root loop_start_epoch now_epoch +leg_idle_s() { + local leg_root="$1" loop_start="$2" now="$3" last since + last="$(find "$leg_root" -type f -printf '%T@\n' 2>/dev/null | sort -rn | head -n1)" + since="${last%.*}" + [ -n "$since" ] && [ "$since" -ge "$loop_start" ] 2>/dev/null || since="$loop_start" + echo $(( now - since )) +} + +# 0 once the leg has a live run: `optimize` creates a NESTED per-run dir under the +# session and writes state.json into it. Scoped to this leg's own session tree, so it +# stays correct on the shared docker host where four legs run side by side. +# Args: session_dir +leg_run_started() { + [ -n "$(find "$1" -mindepth 2 -type f -name state.json -print -quit 2>/dev/null)" ] +} + +# A stable per-leg conversation id, so a follow-up turn can --resume the SAME session +# instead of starting from scratch. `--session-id` wants a UUID, so shape a sha1 of +# leg+CI_VERSION into one (version nibble 4, variant nibble 8). +# Args: leg ci_version +leg_session_uuid() { + local h; h="$(printf '%s|%s' "$1" "$2" | sha1sum | cut -c1-32)" + printf '%s-%s-4%s-8%s-%s' "${h:0:8}" "${h:8:4}" "${h:12:3}" "${h:15:3}" "${h:18:12}" +} + +# One agent turn, mirrored to the leg's NFS transcript so it survives pod deletion. +# Args: agent_log [claude flags...] (the prompt/nudge arrives on stdin) +agent_turn() { + local alog="$1"; shift + claude --print --dangerously-skip-permissions "$@" 2>&1 | tee -a "$alog" +} + +# Follow-up nudges for a conversation that ended before finishing. Deliberately short: +# the resumed session still holds the context (what was launched, which log is being +# watched), so restating the whole prompt would only add noise -- and re-feeding it as a +# FRESH turn is worse, because the agent then has to rediscover all of that. +SETUP_RESUME_NUDGE='Your previous turn ended before setup finished. Do NOT restart anything that is already running. Check on the install you launched: if it is still in progress, keep monitoring it and only answer once it has finished. Once it has finished successfully, complete any remaining setup steps and then reply with exactly the completion line the setup instructions asked for. If it has failed, report the failure.' + +DEMO_RESUME_NUDGE='Your previous turn ended without leaving a running optimize behind, and nothing has been written under the workspace since, so the work is not progressing. Do NOT fabricate a result. Finish the launch in THIS turn: complete the install if it is still needed, start optimize detached with setsid nohup so it survives the end of this turn, then confirm the nested session run dir and its state.json exist and report their paths.' + +# Clean terminal stop_reason values (hyperloom.inference_optimizer.cli._SUCCESS_STOP_REASONS). +is_clean_stop_reason() { + case "$1" in + target_reached|global_converged|time_exhausted|max_ticks|sweep_done|conc_sweep_done) + return 0 ;; + *) return 1 ;; + esac +} + +# Optimize writes state.json root-only; the poll runner reads as ubuntu. +publish_state_for_poll() { + local state_json="$1" sdir="$2" session="$3" + [ -f "$state_json" ] || return 0 + chmod a+r "$state_json" 2>/dev/null || true + chmod a+X "$sdir" "$session" 2>/dev/null || true +} + +# Run ONE leg to completion inside the current filesystem (baremetal pod, or already +# inside a nested docker container). Args: leg backend model_path hours run_mode +run_leg() { + local leg="$1" backend="$2" model_path="$3" hours="$4" run_mode="$5" + local root="${NFS_ROOT%/}/runs/${CI_VERSION}/${leg}" + local session="${root}/session" + mkdir -p "$root" "$session" + log "leg=$leg mode=$run_mode backend=$backend hours=$hours model=$model_path" + + # 1. install the wheel into the leg root (produces importable tree + bundled skills) + local wheels=("$WHEEL_DIR"/hyperloom_inference_optimizer-*.whl) + [ -e "${wheels[0]}" ] || { log "ERROR: no wheel in $WHEEL_DIR"; return 1; } + log "pip install ${wheels[0]} --target $root" + pip install --no-input --target "$root" "${wheels[0]}" >/dev/null + + # 2. decode the key and write the pod-local .env (NEVER on stdout / NEVER to a + # location the poll reads). Restrict perms; scrub on exit. + # + # For a docker leg the agent (not the harness) starts the single-GPU container per the + # demo skill. The demo skill's literal `docker run` cannot express our per-card + # isolation (single renderD node, NUMERIC device GIDs, cpu/mem caps, seccomp), so we + # compute those HERE -- on the host pod, where /dev/kfd + /dev/dri/renderD* exist -- and + # inject them as E2E_* .env values the setup prompt tells the agent to copy verbatim. + # renderD = 128 + GPU_INDEX*8 (stride 8, VERIFIED on a real privileged MI355X x8 pod). + # The pod /etc/group has no `video`/`render` NAMES, so numeric GIDs are required. + local dk_rd="" dk_kfd_gid="" dk_dri_gid="" dk_nfs_mount="" + if [ "$run_mode" = docker ]; then + : "${GPU_INDEX:?docker leg needs GPU_INDEX}" + dk_rd=$(( 128 + GPU_INDEX * 8 )) + dk_kfd_gid="$(stat -c %g /dev/kfd 2>/dev/null || echo 0)" + dk_dri_gid="$(stat -c %g "/dev/dri/renderD${dk_rd}" 2>/dev/null || stat -c %g /dev/dri 2>/dev/null || echo 0)" + # NOTE: we deliberately do NOT inject HYPERLOOM_IMAGE. The demo skill owns the image + # list (examples/*/SKILL.md "Suggested Docker images") -- duplicating those tags here + # is what caused the wrong sglang tag. Instead the setup-docker-*.md prompt tells the + # agent to pick from the skill's list by backend, and for sglang to detect the GPU + # arch (gfx950 -> mi35x, gfx942 -> mi30x) via rocminfo and choose the matching tag. + # The model path (e.g. /shared_nfs/models/) is a SYMLINK into a DIFFERENT + # /shared_nfs subtree (/shared_nfs/huggingface_models/...). Mounting only NFS_ROOT + # (the CI subdir) or the model's parent leaves the symlink TARGET unmounted -> broken + # symlink in the container -> optimize can't boot. Mount the whole shared-NFS root + # (the top-level dir, e.g. /shared_nfs) so both the model and its symlink target + # resolve at the same absolute path inside the container. Derive it as the first path + # component of NFS_ROOT (override with E2E_SHARED_NFS_ROOT if the layout differs). + dk_nfs_mount="${E2E_SHARED_NFS_ROOT:-/$(printf '%s' "${NFS_ROOT#/}" | cut -d/ -f1)}" + fi + local envf="${root}/.env" + ( umask 077 + { + echo "ANTHROPIC_API_KEY=$(printf '%s' "$ANTHROPIC_API_KEY_B64" | base64 -d)" + [ -n "${ANTHROPIC_BASE_URL:-}" ] && echo "ANTHROPIC_BASE_URL=${ANTHROPIC_BASE_URL}" + # Gateways like AMD's APIM (llm-api.amd.com) reject the bearer key alone with + # "401 Access denied due to missing subscription key" -- they need an + # Ocp-Apim-Subscription-Key header whose value is the SAME key. The Claude CLI + # reads ANTHROPIC_CUSTOM_HEADERS (newline-delimited "Name: value") and sends it + # on every request; ${ANTHROPIC_API_KEY} is expanded in-pod so the one key + # covers both bearer + subscription. The header NAME is fixed and non-secret and + # its value carries no NEW secret (the key already arrives via *_B64), so we + # default it here rather than requiring a repo variable -- one less "forgot to + # configure it -> 401" trap. This matches Hyperloom's own llm_config.py contract + # (parse_custom_headers expands ${VAR}). An externally-supplied + # ANTHROPIC_CUSTOM_HEADERS still wins, for a gateway with a different convention. + # Only emitted when a gateway base URL is set (direct api.anthropic.com needs no + # subscription header). Double-quote in .env (space+colon unquoted -> exit 127). + if [ -n "${ANTHROPIC_CUSTOM_HEADERS:-}" ]; then + echo "ANTHROPIC_CUSTOM_HEADERS=\"${ANTHROPIC_CUSTOM_HEADERS}\"" + elif [ -n "${ANTHROPIC_BASE_URL:-}" ]; then + echo 'ANTHROPIC_CUSTOM_HEADERS="Ocp-Apim-Subscription-Key: ${ANTHROPIC_API_KEY}"' + fi + echo "CLAUDE_MODEL=${CLAUDE_MODEL}" + echo "USER_DATA_PATH=${session}" + echo "HYPERLOOM_RUN_MODE=${run_mode}" + echo "FRAMEWORK=${backend}" + echo "MODEL_PATH=${model_path}" + echo "TARGET_GAIN=${TARGET_GAIN}" + echo "DEMO_HOURS=${hours}" + # docker leg: hand the container lifecycle to the agent (demo skill) and carry the + # CI hard constraints it must apply to its `docker run` (see setup-docker-*.md). + if [ "$run_mode" = docker ]; then + # HYPERLOOM_IMAGE intentionally NOT set: the agent selects it from the skill's + # image list (by backend + detected GPU arch). See setup-docker-*.md. To pin the + # tag to the SKILL's exact string (no freelancing), the setup prompt greps it out + # of the demo skill's SKILL.md -- so hand the agent that file's absolute path. + # `pip install --target "$root"` materializes the example demo skills (declared as + # [tool.setuptools.data-files] in pyproject.toml) at + # $root/.claude/skills//SKILL.md + # (empirically verified). The demo skill is chosen by leg duration: + # *-3h -> hyperloom-qwen3-8b-3h ; *-12h -> hyperloom-qwen3-14b-fp8-12h. + local demo_skill="" + case "$leg" in + *-3h) demo_skill="hyperloom-qwen3-8b-3h" ;; + *-12h) demo_skill="hyperloom-qwen3-14b-fp8-12h" ;; + esac + echo "HYPERLOOM_SKILL_PATH=${root}/.claude/skills/${demo_skill}/SKILL.md" + echo "HYPERLOOM_CONTAINER_NAME=hyperloom-${leg}" # unique per leg (shared host dockerd) + local leg_mem leg_shm + case "$leg" in + *-3h) + leg_mem="${DOCKER_LEG_MEM_3H:-256g}" + leg_shm="${DOCKER_LEG_SHM_3H:-64g}" + ;; + *-12h) + leg_mem="${DOCKER_LEG_MEM_12H:-512g}" + leg_shm="${DOCKER_LEG_SHM_12H:-64g}" + ;; + *) + leg_mem="${LEG_MEM:-256g}" + leg_shm="${LEG_SHM:-64g}" + ;; + esac + echo "HYPERLOOM_SHM_SIZE=${leg_shm}" + echo "E2E_GPU_INDEX=${GPU_INDEX}" + echo "E2E_RENDERD=${dk_rd}" + echo "E2E_KFD_GID=${dk_kfd_gid}" + echo "E2E_DRI_GID=${dk_dri_gid}" + echo "E2E_LEG_CPUS=${LEG_CPUS:-32}" + echo "E2E_LEG_MEM=${leg_mem}" + echo "E2E_NFS_MOUNT=${dk_nfs_mount}" + fi + } > "$envf" + ) + trap 'sed -i "/^ANTHROPIC_API_KEY=/d" "'"$envf"'" 2>/dev/null || true' EXIT + + # 3. pin the session dir so the poll finds it without guessing by timestamp (design §9) + export INFERENCE_OPTIMIZER_CURRENT_SESSION_DIR="${session}" + echo "${session}" > "${session}/.session_dir" + + # 4. source env + drive setup, then demo, through the Agent CLI + set -a + # shellcheck disable=SC1090 # envf path is dynamic (per-leg) + . "$envf" + set +a + export PYTHONPATH="${root}:${PYTHONPATH:-}" + + local setup_prompt="${PROMPTS_DIR}/setup-${run_mode}-${backend}.md" + local demo_prompt; demo_prompt="${PROMPTS_DIR}/demo-${hours}h.md" + [ -f "$setup_prompt" ] || { log "ERROR: missing $setup_prompt"; return 1; } + [ -f "$demo_prompt" ] || { log "ERROR: missing $demo_prompt"; return 1; } + + # Run the agent FROM the leg root: the setup prompt refers to "the current + # workspace (REPO_ROOT)" for .env and the installed tree. Bootstrap otherwise runs + # from '/', where claude finds no .env and refuses (it sees ~29 unrelated + # workspaces under /shared_nfs and blocks rather than guess). cd fixes the cwd. + cd "$root" + # --dangerously-skip-permissions: `claude --print` is non-interactive, so the default + # permission mode is fail-closed -- setup's pip/shell steps have no approver and die. + # This is an isolated per-leg CI pod (same posture as the per-PR CI's _incontainer.sh), + # so bypassing the approval gate is acceptable and required for unattended setup/demo. + # IS_SANDBOX=1: the SaFE pod runs as root, and claude refuses to skip permissions under + # root unless IS_SANDBOX=1 (SWSPLAT-42390) -- Hyperloom's own kernel-agent sets the same. + export IS_SANDBOX=1 + # Mirror both turns onto NFS. SaFE deletes the PyTorchJob as soon as a leg fails, taking + # the pod's stdout with it, so without this the agent's own account of the failure is + # unrecoverable and a post-mortem is left reconstructing events from file mtimes. + local agent_log="${session}/agent-${leg}.log" + local uuid; uuid="$(leg_session_uuid "$leg" "$CI_VERSION")" + # `claude --print` is "print response and exit": ONE answer per invocation. Setup here + # means installing a framework layer -- a vLLM ROCm wheel into an isolated venv, or + # SGLang compiled from source for gfx950 -- which takes 10-30min, far longer than a + # single turn can hold. Every leg's agent therefore backgrounds the install and polls + # its log, and eventually answers with a progress note ("Phase 2 is installing the + # isolated vLLM env. Still running.") which ENDS the turn with setup incomplete. + # + # The install itself is detached and keeps running (verified: setup_vllm.log grew from + # 71KB to 77KB across a turn boundary), so the fix is to give the agent another turn -- + # RESUMING the same conversation, so it still knows what it started and where the log + # is. Budget it in TIME, not turns: an earlier count-based cap of 3 turns amounted to + # ~4min of wall clock and killed two legs whose installs were demonstrably still + # progressing. A leg only fails here if the tree goes quiet (nothing installing) or the + # whole setup deadline elapses. + local setup_deadline_s="${LEG_SETUP_DEADLINE_S:-2700}" # 45m of setup, then give up + local setup_stall_s="${LEG_SETUP_STALL_S:-600}" # 10m with no writes -> dead + local setup_max_turns="${LEG_SETUP_MAX_TURNS:-30}" # token-spend backstop + local setup_t0; setup_t0="$(date +%s)" + local turn=0 snow sidle + while :; do + turn=$(( turn + 1 )) + if [ "$turn" = 1 ]; then + log "claude --print (setup, turn 1, session $uuid); agent transcript -> $agent_log" + agent_turn "$agent_log" --session-id "$uuid" < "$setup_prompt" + else + log "claude --print (setup, turn $turn, resuming session $uuid)" + if ! printf '%s\n' "$SETUP_RESUME_NUDGE" | agent_turn "$agent_log" --resume "$uuid"; then + log "WARN: leg $leg -- could not resume session $uuid; re-feeding the full setup prompt" + agent_turn "$agent_log" < "$setup_prompt" || true + fi + fi + if grep -qiE "setup complete: ${run_mode}/${backend}" "$agent_log"; then + log "leg $leg setup reported complete on turn $turn" + break + fi + snow="$(date +%s)" + sidle="$(leg_idle_s "$root" "$setup_t0" "$snow")" + if [ "$sidle" -ge "$setup_stall_s" ]; then + log "ERROR: leg $leg -- setup turn $turn ended early and nothing was written under $root for ${sidle}s; the install is not progressing" + return 1 + fi + if [ $(( snow - setup_t0 )) -ge "$setup_deadline_s" ]; then + log "ERROR: leg $leg -- setup never reported 'setup complete: ${run_mode}/${backend}' within ${setup_deadline_s}s ($turn turns)" + return 1 + fi + if [ "$turn" -ge "$setup_max_turns" ]; then + log "ERROR: leg $leg -- setup still incomplete after $setup_max_turns turns; giving up" + return 1 + fi + log "WARN: leg $leg -- setup turn $turn ended early (install still progressing, idle ${sidle}s); resuming the conversation" + sleep "${LEG_TURN_GAP_S:-30}" + done + log "claude --print (demo ${hours}h, resuming session $uuid)" + # Same conversation as setup: the agent already knows this workspace, which framework + # got installed and where its logs are, exactly like a human continuing the same chat. + if ! agent_turn "$agent_log" --resume "$uuid" < "$demo_prompt"; then + log "WARN: leg $leg -- could not resume session $uuid for the demo; running it standalone" + agent_turn "$agent_log" < "$demo_prompt" + fi + log "leg $leg demo turn returned; waiting for the background optimize to finish" + + # ---- wait for the backgrounded `optimize` to reach a terminal state -------- + # `claude --print` is ONE non-interactive turn: it returns right after the demo + # skill backgrounds `optimize` (setsid nohup). If we returned now, run.sh would + # exit 0 and SaFE would mark a FALSE "Succeeded" while the benchmark is still + # running. Block here until state.json carries a terminal stop_reason (or a deadline). + # + # The real artifacts do NOT live under $session directly: make_session_dir() + # creates a NESTED per-run dir $session//-/ + # and re-pins INFERENCE_OPTIMIZER_CURRENT_SESSION_DIR to it -- but that re-pin + # happens in the CLI's own process and never reaches us. So we discover the real + # dir by globbing the newest one under $session that contains a state.json, and + # re-point the poll's pin (.session_dir) at it. + local wait_interval="${LEG_WAIT_INTERVAL_S:-45}" + # Liveness, NOT a wall-clock startup budget: a leg is dead only once NOTHING under the + # leg root has been written for stall_grace and no state.json exists yet. This waits out + # arbitrarily slow builds (baremetal SGLang compiles from source and starts a Ray head, + # >15m before the first state.json) while still reaping a hung launch within ~10m. + # Two properties matter, both learned from legs that were killed while demonstrably + # alive (2026-08-28): + # * scope is $root, not $session -- the agent's launcher and its setup/install logs + # land next to the workspace, and install.sh writes $root/.cache. Watching only + # $session missed all of it and reaped a leg 26s after it last wrote a file. + # * idleness is measured from the LATER of (last write, loop start) -- the two + # `claude --print` turns can leave the tree untouched for longer than stall_grace, + # and that pre-loop gap must not be charged to the leg, or the very first + # iteration condemns it. + local stall_grace="${LEG_STALL_GRACE_S:-600}" # 10m of NO file writes -> dead + local deadline_s=$(( hours * 3600 + 3600 )) # demo budget + 1h margin (hard cap) + # Two clocks: start_ts backs the hard deadline and is NEVER reset (resetting it would + # let the leg outlive the SaFE pod timeout and lose the clean failure path); grace_ts + # backs the stall window and restarts after each re-driven turn. + local start_ts; start_ts="$(date +%s)" + local grace_ts="$start_ts" + local real_sdir="" final_json="" state_json="" + # An idle tree with no state.json means the demo turn ended without leaving a running + # `optimize` behind. That is recoverable -- ask the agent to finish the job rather than + # failing the leg -- but only a bounded number of times. Deliberately driven off the + # stall signal instead of an "is it launched yet" probe right after the turn: the demo + # skill's launcher runs install.sh BEFORE backgrounding optimize, so state.json can + # legitimately be 10+ minutes away, and re-driving on a short grace would double-launch. + local demo_redrives=0 max_demo_redrives="${LEG_DEMO_REDRIVES:-5}" + + while :; do + local now elapsed; now="$(date +%s)"; elapsed=$(( now - start_ts )) + + if [ -z "$real_sdir" ]; then + # newest dir under $session that actually contains a state.json + real_sdir="$(find "$session" -mindepth 2 -type f -name state.json -printf '%T@ %h\n' 2>/dev/null \ + | sort -rn | head -n1 | cut -d' ' -f2- || true)" + if [ -n "$real_sdir" ]; then + final_json="${real_sdir}/reports/final.json" + state_json="${real_sdir}/state.json" + log "leg $leg real session dir: $real_sdir" + # Re-pin so the poll (leg_session_dir -> head -n1 .session_dir) finds the report. + echo "$real_sdir" > "${session}/.session_dir" + publish_state_for_poll "$state_json" "$real_sdir" "$session" + else + local idle + idle="$(leg_idle_s "$root" "$grace_ts" "$now")" + if [ "$idle" -ge "$stall_grace" ]; then + if [ "$demo_redrives" -lt "$max_demo_redrives" ]; then + demo_redrives=$(( demo_redrives + 1 )) + log "WARN: leg $leg -- idle ${idle}s with no state.json; the demo turn left nothing running. Resuming the conversation to finish the launch ($demo_redrives/$max_demo_redrives)" + if ! printf '%s\n' "$DEMO_RESUME_NUDGE" | agent_turn "$agent_log" --resume "$uuid"; then + log "WARN: leg $leg -- could not resume session $uuid; re-feeding the demo prompt" + agent_turn "$agent_log" < "$demo_prompt" || true + fi + log "leg $leg demo re-drive $demo_redrives returned" + grace_ts="$(date +%s)" # fresh stall grace for the new turn; deadline unchanged + continue + fi + log "ERROR: leg $leg -- no state.json and no file written under $root for ${idle}s after $demo_redrives demo re-drive(s); giving up" + return 1 + fi + fi + fi + + if [ -n "$real_sdir" ]; then + if [ -f "$state_json" ]; then + publish_state_for_poll "$state_json" "$real_sdir" "$session" + fi + if [ -f "$final_json" ]; then + log "leg $leg final.json present after ${elapsed}s; demo complete" + return 0 + fi + local stop="" + [ -f "$state_json" ] && stop="$(jq -r '.stop_reason // ""' "$state_json" 2>/dev/null || echo "")" + if [ -n "$stop" ]; then + if is_clean_stop_reason "$stop"; then + log "leg $leg state.json stop_reason='$stop' after ${elapsed}s; demo complete" + return 0 + fi + log "ERROR: leg $leg state stop_reason='$stop' (not a clean terminal exit)" + return 1 + fi + fi + + if [ "$elapsed" -ge "$deadline_s" ]; then + log "ERROR: leg $leg deadline ${deadline_s}s reached without a terminal stop_reason (real_sdir='${real_sdir:-}')" + return 1 + fi + sleep "$wait_interval" + done +} + +# Layer-deduplicating storage drivers, tried in order. vfs is deliberately NOT a +# fallback: it copies every image layer and every container in full, which grew the +# host pod past its ephemeralStorage limit and got it EVICTED within minutes +# ("ephemeral local storage usage exceeds the total limit of containers 1792Gi", +# observed live 2026-08-28). Running on vfs is a guaranteed eviction, so a leg that +# cannot get a deduplicating driver must fail loudly instead. +DOCKER_DRIVERS="${DOCKER_DRIVERS:-overlay2 fuse-overlayfs}" + +# Start a detached dockerd on one driver; 0 when the socket answers. Cleans up the +# failed daemon so the next driver starts from a clean socket. +start_dockerd_with_driver() { + local driver="$1" data_root="$2" dlog="/var/log/dockerd-${1}.log" i + mkdir -p "$data_root" || return 1 + log "starting pod-local dockerd (driver=$driver, data-root=$data_root)" + setsid bash -c "dockerd --host=unix:///var/run/docker.sock --storage-driver='$driver' --data-root='$data_root' >'$dlog' 2>&1" \ + /dev/null 2>&1 & + for i in $(seq 1 60); do + docker info >/dev/null 2>&1 && { log "dockerd up after ${i}s (driver=$driver)"; return 0; } + sleep 1 + done + log "WARN: dockerd did not become ready with driver=$driver" + tail -20 "$dlog" 2>/dev/null || true + pkill -f 'dockerd --host=unix:///var/run/docker.sock' 2>/dev/null || true + sleep 3 + rm -f /var/run/docker.sock 2>/dev/null || true + return 1 +} + +# fuse-overlayfs is the userspace fallback when the kernel refuses overlay2 on the +# data-root filesystem. Needs the binary plus /dev/fuse. +ensure_fuse_overlayfs() { + command -v fuse-overlayfs >/dev/null 2>&1 || { + log "installing fuse-overlayfs" + export DEBIAN_FRONTEND=noninteractive + apt-get update -qq >>/tmp/apt-docker.log 2>&1 || true + apt-get install -y -qq fuse-overlayfs >>/tmp/apt-docker.log 2>&1 \ + || { log "WARN: fuse-overlayfs install failed"; return 1; } + } + [ -c /dev/fuse ] || { log "WARN: /dev/fuse missing; fuse-overlayfs unusable"; return 1; } +} + +# Install docker + start a pod-local dockerd. VERIFIED on a real privileged MI355X pod: +# the Authoring base image ships NO docker/dockerd/docker.sock, but the pod has full +# capabilities (CapEff=0x1ffffffffff), so a self-hosted dockerd works. There is no +# systemd in the pod, hence setsid + socket polling rather than a service start. +ensure_dockerd() { + if docker info >/dev/null 2>&1; then log "dockerd already up"; return 0; fi + if ! command -v dockerd >/dev/null 2>&1; then + log "installing docker.io (no docker in the Authoring base image)" + export DEBIAN_FRONTEND=noninteractive + apt-get update -qq >/tmp/apt-docker.log 2>&1 || { log "ERROR: apt-get update failed"; tail -20 /tmp/apt-docker.log; return 1; } + apt-get install -y -qq docker.io >>/tmp/apt-docker.log 2>&1 \ + || { log "ERROR: docker.io install failed"; tail -30 /tmp/apt-docker.log; return 1; } + log "docker installed: $(docker --version 2>&1)" + fi + # --- data-root must sit on a filesystem that can back a deduplicating driver --- + # The container rootfs (/) is itself an overlay, and overlayfs cannot use an overlay + # upperdir -- that is why this used to run on vfs. /shared-data is a plain xfs mount + # (/dev/mapper/nvme_vg-nvme_lv) and DOES accept an overlay upperdir (probe-verified: + # `mount -t overlay` with upperdir/workdir under /shared-data succeeds, so xfs ftype=1 + # holds). It is an emptyDir, so it still counts toward the pod's ephemeralStorage quota + # and dies with the pod -- the quota headroom comes from the driver's layer dedup, not + # from the location. + local docker_data_root="/var/lib/docker" + local dr_candidate="/shared-data/e2e-docker/${CI_VERSION:-current}" + if mkdir -p "$dr_candidate" 2>/dev/null && touch "$dr_candidate/.wtest" 2>/dev/null; then + rm -f "$dr_candidate/.wtest" 2>/dev/null || true + docker_data_root="$dr_candidate" + else + log "WARN: /shared-data not writable; dockerd falls back to $docker_data_root (overlay-on-overlay, overlay2 will likely be refused)" + fi + # Each driver gets its own subdir: dockerd refuses a data-root that already holds a + # different driver's tree. + local driver + for driver in $DOCKER_DRIVERS; do + if [ "$driver" = fuse-overlayfs ]; then ensure_fuse_overlayfs || continue; fi + if start_dockerd_with_driver "$driver" "${docker_data_root}/${driver}"; then + log "docker storage driver in use: $(docker info -f '{{.Driver}}' 2>/dev/null)" + return 0 + fi + done + log "ERROR: no layer-deduplicating docker storage driver available (tried: $DOCKER_DRIVERS)." + log "ERROR: refusing to fall back to vfs -- it has no layer dedup and evicts the pod on ephemeralStorage." + return 1 +} + +# ---- docker host: run each leg (docker mode) ON THIS host pod ---------------- +# The privileged 8-GPU host runs a dockerd, then drives each docker leg as a backgrounded +# run_leg in docker mode. Each leg's agent follows the demo skill to `docker run` its OWN +# single-GPU container (renderD = 128 + gpu_index*8), applying the CI isolation flags that +# run_leg injected into the leg .env. No nested bootstrap: session artifacts land under +# $session on this pod's NFS exactly as for baremetal, so the wait-for-stop_reason loop in +# run_leg works unchanged. Each `run_leg &` is its own subshell, so their per-leg EXIT +# traps (the .env key scrub) don't clobber each other. +run_docker_host() { + : "${DOCKER_LEGS:?}"; : "${DOCKER_GPU_MAP:?}"; : "${MODEL_3H:?}"; : "${MODEL_12H:?}" + ensure_dockerd || { log "ERROR: cannot provide docker on the host pod"; return 1; } + log "docker host: legs='${DOCKER_LEGS}'" + local pids=() leg idx backend hours model_path + for leg in $DOCKER_LEGS; do + idx="$(printf '%s' "$DOCKER_GPU_MAP" | jq -r --arg l "$leg" '.[$l]')" + case "$leg" in + *-vllm-*) backend=vllm ;; + *-sglang-*) backend=sglang ;; + *) log "ERROR: cannot infer backend from leg '$leg'"; return 1 ;; + esac + case "$leg" in + *-3h) hours=3; model_path="$MODEL_3H" ;; + *-12h) hours=12; model_path="$MODEL_12H" ;; + *) log "ERROR: cannot infer duration from leg '$leg'"; return 1 ;; + esac + log "launch docker leg: gpu=$idx renderD$(( 128 + idx * 8 )) leg=$leg backend=$backend hours=$hours" + ( GPU_INDEX="$idx" run_leg "$leg" "$backend" "$model_path" "$hours" docker ) & + pids+=("$!") + done + local rc=0 p + for p in "${pids[@]}"; do wait "$p" || rc=1; done + return "$rc" +} + +# The SaFE rocm/pytorch Authoring image is minimal: besides node/npm (see ensure_node) +# it also lacks `jq`, which run_docker_host uses to read DOCKER_GPU_MAP. Install it up +# front (apt works in the pod) so both the host pod and the nested single-leg containers +# -- which re-run THIS script -- have it. Idempotent; cheap when already present. +ensure_base_tools() { + command -v jq >/dev/null 2>&1 && return 0 + log "installing jq (not in the Authoring base image)" + export DEBIAN_FRONTEND=noninteractive + apt-get update -qq >/tmp/apt-jq.log 2>&1 && apt-get install -y -qq jq >>/tmp/apt-jq.log 2>&1 \ + || { log "ERROR: jq install failed"; tail -20 /tmp/apt-jq.log; return 1; } + log "jq installed: $(jq --version 2>&1)" +} + +# ---- entry ----------------------------------------------------------------- +ensure_base_tools || exit 1 +install_claude_cli || exit 1 + +if [ "${E2E_DOCKER_HOST:-}" = "1" ]; then + run_docker_host +else + : "${LEG_ID:?}"; : "${HYPERLOOM_BACKEND:?}"; : "${HYPERLOOM_MODEL_PATH:?}"; : "${DEMO_HOURS:?}" + run_leg "$LEG_ID" "$HYPERLOOM_BACKEND" "$HYPERLOOM_MODEL_PATH" "$DEMO_HOURS" "${HYPERLOOM_RUN_MODE:-baremetal}" +fi diff --git a/.github/pre-release/prompts/pre-release/demo-12h.md b/.github/pre-release/prompts/pre-release/demo-12h.md new file mode 100644 index 0000000000..63a569f463 --- /dev/null +++ b/.github/pre-release/prompts/pre-release/demo-12h.md @@ -0,0 +1,56 @@ +# Pre-release E2E — 12h demo leg + +You are running the Hyperloom pre-release E2E test non-interactively. Run the 12-hour +demo to completion, then stop. Setup already ran successfully in this workspace. + +Invoke the `hyperloom-qwen3-14b-fp8-12h` demo skill with **one** override and otherwise +its exact default flags. + +## Flags + +- **OVERRIDE:** use `--target-gain 100` (NOT the skill's default of 50). This shapes + optimize prompts only; the poll gate judges PASS/FAIL from `stop_reason`, not gain. +- Keep every other required flag exactly as the skill defines them: + + ``` + --tp 1 --conc 64 --isl 1024 --osl 1024 --precision fp8 --max-hours 12 + --max-minutes-framework-pct 0.43 --max-minutes-kernel-pct 0.42 + ``` + + Do **not** pass `--no-framework-agent` or `--no-kernel` — the 12h demo runs the full + OPTIMIZE phase (FRAMEWORK_AGENT + KERNEL_AGENT). + +## Model path + +The skill will ask which model to use. Do **not** ask interactively — use +`MODEL_PATH` from the repository-root `.env` (it is already set to the demo model, +Qwen3-14B-FP8). Verify that path contains `config.json`; if it does, use it and +continue without asking. Load LLM API keys/base URLs and `FRAMEWORK` from `.env`. + +## Hard constraints (automated release gate) + +- Do **not** modify any GPU-related environment variable or device visibility. +- Do **not** choose GPUs via `rocm-smi`. +- If `HYPERLOOM_RUN_MODE=docker`, run `optimize` **inside the container you started in + setup** via `docker exec -w "$REPO_ROOT" "$HYPERLOOM_CONTAINER_NAME" …` (per the demo + skill's docker mode). Do **not** start a new container and do **not** change its + device/isolation flags. Otherwise (baremetal) run directly and do not run `docker`. +- Do **not** modify `USER_DATA_PATH`. +- Do **not** print or copy secret values into output, reports, or logs. + +## Termination — do not end this turn until the run is launched + +This is a **single non-interactive turn**, and anything still running as a child of it is +killed the moment the turn ends. So: + +1. Finish the install and the launch **inside this turn**. Do **not** end the turn with a + progress note such as "install started", "waiting on the pull", or "waiting on the + monitor" — that kills the work you just started and the leg ends up with nothing + running at all. +2. Start `optimize` **detached** with `setsid nohup` (as the demo skill does) so it + survives the end of this turn. +3. Before you finish, confirm the run is really live and report the paths: the nested + session run dir exists, `state.json` is present in it, and the optimizer PID is alive. + +Only then stop. The harness polls `state.json` for a clean terminal `stop_reason` to +judge PASS/FAIL — do not fabricate a result. diff --git a/.github/pre-release/prompts/pre-release/demo-3h.md b/.github/pre-release/prompts/pre-release/demo-3h.md new file mode 100644 index 0000000000..8a606094c9 --- /dev/null +++ b/.github/pre-release/prompts/pre-release/demo-3h.md @@ -0,0 +1,57 @@ +# Pre-release E2E — 3h demo leg + +You are running the Hyperloom pre-release E2E test non-interactively. Run the 3-hour +demo to completion, then stop. Setup already ran successfully in this workspace. + +Invoke the `hyperloom-qwen3-8b-3h` demo skill with **one** override and otherwise its +exact default flags. + +## Flags + +- **OVERRIDE:** use `--target-gain 100` (NOT the skill's default of 30). This shapes + optimize prompts only; the poll gate judges PASS/FAIL from `stop_reason`, not gain. +- Keep every other required flag exactly as the skill defines them: + + ``` + --tp 1 --conc 64 --isl 1024 --osl 1024 --precision bf16 --max-hours 3 + --max-minutes-framework-pct 0.90 --max-minutes-sweep-pct 0.01 + --no-kernel --no-enable-conc-sweep --no-enable-roofline + ``` + + Do **not** pass `--no-framework-agent` — the 3h demo runs FRAMEWORK_AGENT (OPTIMIZE) + only; kernel is skipped via `--no-kernel`. + +## Model path + +The skill will ask which model to use. Do **not** ask interactively — use +`MODEL_PATH` from the repository-root `.env` (it is already set to the demo model). +Verify that path contains `config.json`; if it does, use it and continue without +asking. Load LLM API keys/base URLs and `FRAMEWORK` from `.env`. + +## Hard constraints (automated release gate) + +- Do **not** modify any GPU-related environment variable or device visibility. +- Do **not** choose GPUs via `rocm-smi`. +- If `HYPERLOOM_RUN_MODE=docker`, run `optimize` **inside the container you started in + setup** via `docker exec -w "$REPO_ROOT" "$HYPERLOOM_CONTAINER_NAME" …` (per the demo + skill's docker mode). Do **not** start a new container and do **not** change its + device/isolation flags. Otherwise (baremetal) run directly and do not run `docker`. +- Do **not** modify `USER_DATA_PATH`. +- Do **not** print or copy secret values into output, reports, or logs. + +## Termination — do not end this turn until the run is launched + +This is a **single non-interactive turn**, and anything still running as a child of it is +killed the moment the turn ends. So: + +1. Finish the install and the launch **inside this turn**. Do **not** end the turn with a + progress note such as "install started", "waiting on the pull", or "waiting on the + monitor" — that kills the work you just started and the leg ends up with nothing + running at all. +2. Start `optimize` **detached** with `setsid nohup` (as the demo skill does) so it + survives the end of this turn. +3. Before you finish, confirm the run is really live and report the paths: the nested + session run dir exists, `state.json` is present in it, and the optimizer PID is alive. + +Only then stop. The harness polls `state.json` for a clean terminal `stop_reason` to +judge PASS/FAIL — do not fabricate a result. diff --git a/.github/pre-release/prompts/pre-release/setup-baremetal-sglang.md b/.github/pre-release/prompts/pre-release/setup-baremetal-sglang.md new file mode 100644 index 0000000000..9d87f3eab2 --- /dev/null +++ b/.github/pre-release/prompts/pre-release/setup-baremetal-sglang.md @@ -0,0 +1,38 @@ +# Pre-release E2E — setup (baremetal + SGLang) + +You are running the Hyperloom pre-release E2E test non-interactively. Complete the +setup step for a **baremetal + SGLang** leg, then stop. Do not run the demo yet. + +## Environment (already prepared) + +A `.env` file exists in the current workspace (`REPO_ROOT`) with these values already +set: `ANTHROPIC_API_KEY`, `CLAUDE_MODEL`, `USER_DATA_PATH`, +`HYPERLOOM_RUN_MODE=baremetal`, `FRAMEWORK=sglang`, `MODEL_PATH`, `TARGET_GAIN`, +`DEMO_HOURS`. The wheel is already installed via `pip install --target .` so a +`hyperloom/` package directory is present. + +## Fixed decisions + +Run the `hyperloom-setup` skill with these fixed decisions — do **not** ask +interactive questions; use the values already in `.env` and the environment: + +- **Run mode:** baremetal (`HYPERLOOM_RUN_MODE` is already `baremetal`; keep it). +- **Framework:** SGLang. Install the framework layer with the setup backend + (`--install-framework sglang`). +- **LLM provider / model / `USER_DATA_PATH`:** use the values already in `.env`; do + not change them. + +## Hard constraints (automated release gate) + +- Do **not** modify any GPU-related environment variable (`ROCR_VISIBLE_DEVICES`, + `HIP_VISIBLE_DEVICES`, `GPUS_PER_NODE`, etc.). The pod already exposes exactly one + GPU; do not override device visibility. +- Do **not** run `docker` and do **not** choose GPUs via `rocm-smi`. This is a + baremetal leg; setup runs on the host. +- Do **not** print, echo, or copy secret values (API keys) into output or logs. +- Do **not** modify `USER_DATA_PATH`. + +## Termination + +When setup completes successfully, stop. Report only `setup complete: baremetal/sglang`. +If setup hard-fails, report the failure and stop. diff --git a/.github/pre-release/prompts/pre-release/setup-baremetal-vllm.md b/.github/pre-release/prompts/pre-release/setup-baremetal-vllm.md new file mode 100644 index 0000000000..7c647ba75f --- /dev/null +++ b/.github/pre-release/prompts/pre-release/setup-baremetal-vllm.md @@ -0,0 +1,38 @@ +# Pre-release E2E — setup (baremetal + vLLM) + +You are running the Hyperloom pre-release E2E test non-interactively. Complete the +setup step for a **baremetal + vLLM** leg, then stop. Do not run the demo yet. + +## Environment (already prepared) + +A `.env` file exists in the current workspace (`REPO_ROOT`) with these values already +set: `ANTHROPIC_API_KEY`, `CLAUDE_MODEL`, `USER_DATA_PATH`, +`HYPERLOOM_RUN_MODE=baremetal`, `FRAMEWORK=vllm`, `MODEL_PATH`, `TARGET_GAIN`, +`DEMO_HOURS`. The wheel is already installed via `pip install --target .` so a +`hyperloom/` package directory is present. + +## Fixed decisions + +Run the `hyperloom-setup` skill with these fixed decisions — do **not** ask +interactive questions; use the values already in `.env` and the environment: + +- **Run mode:** baremetal (`HYPERLOOM_RUN_MODE` is already `baremetal`; keep it). +- **Framework:** vLLM. Install the framework layer with the setup backend + (`--install-framework vllm`, isolated framework env). +- **LLM provider / model / `USER_DATA_PATH`:** use the values already in `.env`; do + not change them. + +## Hard constraints (automated release gate) + +- Do **not** modify any GPU-related environment variable (`ROCR_VISIBLE_DEVICES`, + `HIP_VISIBLE_DEVICES`, `GPUS_PER_NODE`, etc.). The pod already exposes exactly one + GPU; do not override device visibility. +- Do **not** run `docker` and do **not** choose GPUs via `rocm-smi`. This is a + baremetal leg; setup runs on the host. +- Do **not** print, echo, or copy secret values (API keys) into output or logs. +- Do **not** modify `USER_DATA_PATH`. + +## Termination + +When setup completes successfully, stop. Report only `setup complete: baremetal/vllm`. +If setup hard-fails, report the failure and stop. diff --git a/.github/pre-release/prompts/pre-release/setup-docker-sglang.md b/.github/pre-release/prompts/pre-release/setup-docker-sglang.md new file mode 100644 index 0000000000..f39f462109 --- /dev/null +++ b/.github/pre-release/prompts/pre-release/setup-docker-sglang.md @@ -0,0 +1,114 @@ +# Pre-release E2E — setup (docker + SGLang) + +You are running the Hyperloom pre-release E2E test non-interactively. Complete the +setup step for a **docker + SGLang** leg, then stop. Do not run the demo yet. + +> **IMPORTANT — you own the container.** You are on the privileged host pod. This is a +> `docker` leg, so **you** must start the backend container yourself by following the +> `hyperloom-setup` skill and the demo skill's **docker mode**: run `docker run` to start +> a long-lived single-GPU container, then run setup **inside** it with `docker exec`. +> Docker is already available on this host (a pod-local `dockerd` is running). + +## Environment (already prepared) + +A `.env` file exists in the current workspace (`REPO_ROOT`) with these values already +set: `ANTHROPIC_API_KEY`, `CLAUDE_MODEL`, `USER_DATA_PATH`, +`HYPERLOOM_RUN_MODE=docker`, `FRAMEWORK=sglang`, `MODEL_PATH`, `TARGET_GAIN`, +`DEMO_HOURS`, `HYPERLOOM_CONTAINER_NAME`, `HYPERLOOM_SHM_SIZE`, plus the CI +isolation values `E2E_RENDERD`, `E2E_KFD_GID`, `E2E_DRI_GID`, `E2E_LEG_CPUS`, +`E2E_LEG_MEM`, `E2E_NFS_MOUNT`. The wheel is already installed via +`pip install --target .` so a `hyperloom/` package directory is present. + +`HYPERLOOM_IMAGE` is **not** in `.env` — you choose it (see "Image selection" below). + +## Fixed decisions + +Follow the `hyperloom-setup` skill in **docker** mode with these fixed decisions — do +**not** ask interactive questions; use the values already in `.env`: + +- **Run mode:** docker. Start the container yourself (see the hard constraints below for + the exact `docker run` flags), then `docker exec` the setup inside it. Do **not** set + `HYPERLOOM_DOCKER_TARGET_HOST` (run on the current host). +- **Framework:** SGLang — provided by the container image; run setup with + `--install-framework none --yes` inside the container (do **not** `--install-framework + sglang`). +- **LLM provider / model / `USER_DATA_PATH`:** use the values already in `.env`; do + not change them. + +## Image selection + +`HYPERLOOM_IMAGE` is not preset. You **MUST** use the **exact, verbatim** `docker.io/...` +tag string that is written on the demo skill's matching `sglang` row in its **"Suggested +Docker images"** section — nothing else. This is a hard release-gate constraint, not a +suggestion: + +- **Do NOT freelance the tag.** Do not bump the version, do not pick a "newer" or + "latest" build, do not substitute a different tag from your memory, from Docker Hub, or + from anywhere other than the skill file. The pinned tag is the one the release is gated + on; a different tag is a **failure**, even if it also pulls successfully. +- The skill lists a **different `sglang` image per GPU architecture**, so first detect the + architecture on this host, then read the **exact** tag for the matching row **from the + skill file itself** rather than typing it out (the demo skill's path is in + `HYPERLOOM_SKILL_PATH` in `.env`; otherwise it is the `SKILL.md` of the demo skill you + are running): + + ```bash + gfx="$(/opt/rocm/bin/rocminfo 2>/dev/null | grep -oiE 'gfx9[0-9a-f]+' | head -1)" + case "$gfx" in + gfx950) row='MI355X' ;; # MI355X + gfx942) row='MI300X' ;; # MI300X + *) echo "ERROR: could not detect GPU arch (gfx='$gfx')"; exit 1 ;; + esac + HYPERLOOM_IMAGE="$(grep -E "^- \`sglang\` $row" "$HYPERLOOM_SKILL_PATH" | grep -oE 'docker\.io/[^`]+' | head -1)" + echo "arch=$gfx row=$row using image: $HYPERLOOM_IMAGE" + ``` + + This is architecture **detection only** — it selects the image tag, not which card the + leg runs on (the card is pinned by the isolation flags below). +- If the arch cannot be detected, the extracted tag is empty, or the resulting image + cannot be pulled, **stop and report the failure** — do **not** substitute any other tag + to work around it. + +Export the extracted value as `HYPERLOOM_IMAGE` for the `docker run` below. + +## Hard constraints (automated release gate) + +Your `docker run` **MUST** use exactly the flags below. These **replace** the demo +skill's default `--device /dev/dri` (all GPUs) and `--group-add video` (a group *name* +the pod has no entry for) with single-card isolation and numeric GIDs. Every other +aspect of the skill's docker flow (long-lived `--entrypoint tail … -f /dev/null`, +mounting `$REPO_ROOT:$REPO_ROOT`, `docker exec` setup, running optimize inside) is +unchanged. Read the values from `.env`: + +- **Container name:** `--name "$HYPERLOOM_CONTAINER_NAME"` (already unique per leg; + another leg may share this host's dockerd, so do not rename it to a fixed value). +- **Image:** the `HYPERLOOM_IMAGE` you selected above (skill list + detected arch). +- **Single-GPU isolation (REPLACES `--device /dev/dri`):** + `--device /dev/kfd --device /dev/dri/renderD${E2E_RENDERD}` +- **Numeric group-add (REPLACES `--group-add video`):** + `--group-add ${E2E_KFD_GID} --group-add ${E2E_DRI_GID}` +- **Resource caps:** `--cpus ${E2E_LEG_CPUS} --memory ${E2E_LEG_MEM} --shm-size ${HYPERLOOM_SHM_SIZE}` +- **Security:** `--security-opt seccomp=unconfined` +- **Device pin:** `-e HIP_VISIBLE_DEVICES=0 -e ROCR_VISIBLE_DEVICES=0` +- **Mounts:** `-v "$REPO_ROOT:$REPO_ROOT" -v "${E2E_NFS_MOUNT}:${E2E_NFS_MOUNT}"` — + mounting **all** of `${E2E_NFS_MOUNT}` at the same absolute path is **required** so + that `MODEL_PATH` (a symlink into another subtree under `${E2E_NFS_MOUNT}`) resolves + inside the container. Do **not** mount only the model's parent directory. + +Also: + +- Do **not** add any other `--device`, and do **not** use `--group-add video`. You may run + `rocminfo`/`rocm-smi` to **detect the GPU architecture** for image selection, but do + **not** use them to **choose which GPU** the leg runs on: the single bound `renderD` node + + `HIP/ROCR_VISIBLE_DEVICES=0` are what pin this leg to its one card. Do not add + `HIP_VISIBLE_DEVICES`/`ROCR_VISIBLE_DEVICES` values other than `0`. +- Do **not** print, echo, or copy secret values (API keys) into output or logs. The key + lives only in the pod-local `.env` (it reaches the container via `-v $REPO_ROOT:$REPO_ROOT`); + do **not** write it anywhere else, and never onto NFS outside that `.env`. +- Do **not** modify `USER_DATA_PATH`. + +## Termination + +When setup completes successfully, stop and report only `setup complete: docker/sglang`. +Leave the container **running** so the demo turn can `docker exec` into it. If setup +hard-fails, report the failure and stop. diff --git a/.github/pre-release/prompts/pre-release/setup-docker-vllm.md b/.github/pre-release/prompts/pre-release/setup-docker-vllm.md new file mode 100644 index 0000000000..fd700ddc5b --- /dev/null +++ b/.github/pre-release/prompts/pre-release/setup-docker-vllm.md @@ -0,0 +1,105 @@ +# Pre-release E2E — setup (docker + vLLM) + +You are running the Hyperloom pre-release E2E test non-interactively. Complete the +setup step for a **docker + vLLM** leg, then stop. Do not run the demo yet. + +> **IMPORTANT — you own the container.** You are on the privileged host pod. This is a +> `docker` leg, so **you** must start the backend container yourself by following the +> `hyperloom-setup` skill and the demo skill's **docker mode**: run `docker run` to start +> a long-lived single-GPU container, then run setup **inside** it with `docker exec`. +> Docker is already available on this host (a pod-local `dockerd` is running). + +## Environment (already prepared) + +A `.env` file exists in the current workspace (`REPO_ROOT`) with these values already +set: `ANTHROPIC_API_KEY`, `CLAUDE_MODEL`, `USER_DATA_PATH`, +`HYPERLOOM_RUN_MODE=docker`, `FRAMEWORK=vllm`, `MODEL_PATH`, `TARGET_GAIN`, +`DEMO_HOURS`, `HYPERLOOM_CONTAINER_NAME`, `HYPERLOOM_SHM_SIZE`, plus the CI +isolation values `E2E_RENDERD`, `E2E_KFD_GID`, `E2E_DRI_GID`, `E2E_LEG_CPUS`, +`E2E_LEG_MEM`, `E2E_NFS_MOUNT`. The wheel is already installed via +`pip install --target .` so a `hyperloom/` package directory is present. + +`HYPERLOOM_IMAGE` is **not** in `.env` — you choose it (see "Image selection" below). + +## Fixed decisions + +Follow the `hyperloom-setup` skill in **docker** mode with these fixed decisions — do +**not** ask interactive questions; use the values already in `.env`: + +- **Run mode:** docker. Start the container yourself (see the hard constraints below for + the exact `docker run` flags), then `docker exec` the setup inside it. Do **not** set + `HYPERLOOM_DOCKER_TARGET_HOST` (run on the current host). +- **Framework:** vLLM — provided by the container image; run setup with + `--install-framework none --yes` inside the container (do **not** `--install-framework + vllm`). +- **LLM provider / model / `USER_DATA_PATH`:** use the values already in `.env`; do + not change them. + +## Image selection + +`HYPERLOOM_IMAGE` is not preset. You **MUST** use the **exact, verbatim** `docker.io/...` +tag string that is written on the demo skill's `vllm` row in its **"Suggested Docker +images"** section — nothing else. This is a hard release-gate constraint, not a +suggestion: + +- **Do NOT freelance the tag.** Do not bump the version, do not pick a "newer" or + "latest" build, do not substitute a different tag from your memory, from Docker Hub, or + from anywhere other than the skill file. The pinned tag is the one the release is gated + on; a different tag is a **failure**, even if it also pulls successfully. +- Read the tag by extracting it **from the skill file itself** rather than typing it out, + e.g. (the demo skill's path is in `HYPERLOOM_SKILL_PATH` in `.env`; otherwise it is the + `SKILL.md` of the demo skill you are running): + + ```bash + HYPERLOOM_IMAGE="$(grep -E '^- `vllm`' "$HYPERLOOM_SKILL_PATH" | grep -oE 'docker\.io/[^`]+' | head -1)" + echo "using image: $HYPERLOOM_IMAGE" + ``` + + This is a `vllm` leg, so use the single arch-independent `vllm` row — no GPU detection + is needed. +- If that command yields an empty string, or the resulting image cannot be pulled, **stop + and report the failure** — do **not** substitute any other tag to work around it. + +Export the extracted value as `HYPERLOOM_IMAGE` for the `docker run` below. + +## Hard constraints (automated release gate) + +Your `docker run` **MUST** use exactly the flags below. These **replace** the demo +skill's default `--device /dev/dri` (all GPUs) and `--group-add video` (a group *name* +the pod has no entry for) with single-card isolation and numeric GIDs. Every other +aspect of the skill's docker flow (long-lived `--entrypoint tail … -f /dev/null`, +mounting `$REPO_ROOT:$REPO_ROOT`, `docker exec` setup, running optimize inside) is +unchanged. Read the values from `.env`: + +- **Container name:** `--name "$HYPERLOOM_CONTAINER_NAME"` (already unique per leg; + another leg may share this host's dockerd, so do not rename it to a fixed value). +- **Image:** the `HYPERLOOM_IMAGE` you selected above (the skill's `vllm` tag). +- **Single-GPU isolation (REPLACES `--device /dev/dri`):** + `--device /dev/kfd --device /dev/dri/renderD${E2E_RENDERD}` +- **Numeric group-add (REPLACES `--group-add video`):** + `--group-add ${E2E_KFD_GID} --group-add ${E2E_DRI_GID}` +- **Resource caps:** `--cpus ${E2E_LEG_CPUS} --memory ${E2E_LEG_MEM} --shm-size ${HYPERLOOM_SHM_SIZE}` +- **Security:** `--security-opt seccomp=unconfined` +- **Device pin:** `-e HIP_VISIBLE_DEVICES=0 -e ROCR_VISIBLE_DEVICES=0` +- **Mounts:** `-v "$REPO_ROOT:$REPO_ROOT" -v "${E2E_NFS_MOUNT}:${E2E_NFS_MOUNT}"` — + mounting **all** of `${E2E_NFS_MOUNT}` at the same absolute path is **required** so + that `MODEL_PATH` (a symlink into another subtree under `${E2E_NFS_MOUNT}`) resolves + inside the container. Do **not** mount only the model's parent directory. + +Also: + +- Do **not** add any other `--device`, and do **not** use `--group-add video`. Do **not** + use `rocm-smi`/`rocminfo` to **choose which GPU** the leg runs on: the single bound + `renderD` node + `HIP/ROCR_VISIBLE_DEVICES=0` are what pin this leg to its one card. Do + not add `HIP_VISIBLE_DEVICES`/`ROCR_VISIBLE_DEVICES` values other than `0`. (This vLLM + image is architecture-independent, so no GPU-arch detection is needed.) +- Do **not** print, echo, or copy secret values (API keys) into output or logs. The key + lives only in the pod-local `.env` (it reaches the container via `-v $REPO_ROOT:$REPO_ROOT`); + do **not** write it anywhere else, and never onto NFS outside that `.env`. +- Do **not** modify `USER_DATA_PATH`. + +## Termination + +When setup completes successfully, stop and report only `setup complete: docker/vllm`. +Leave the container **running** so the demo turn can `docker exec` into it. If setup +hard-fails, report the failure and stop. diff --git a/.github/scripts/pre-release-e2e-dispatch.sh b/.github/scripts/pre-release-e2e-dispatch.sh new file mode 100755 index 0000000000..a584293b86 --- /dev/null +++ b/.github/scripts/pre-release-e2e-dispatch.sh @@ -0,0 +1,416 @@ +#!/usr/bin/env bash +# SPDX-FileCopyrightText: 2026 Advanced Micro Devices, Inc. +# SPDX-License-Identifier: MIT +# +# Pre-release E2E: create the SaFE PyTorchJob workloads that run the packaged wheel +# through the real user path (Claude CLI + setup skill + demo skill). Unlike the PR +# smoke test (.github/scripts/ci-e2e-dispatch.sh), which uses the orchestration +# endpoint (POST /api/v1/orchestration/workloads, kind=hyperloom) to dispatch a git +# SHA, this dispatches GENERIC pods (POST /api/v1/workloads) whose entrypoint is the +# bootstrap script. kind=PyTorchJob, NOT Authoring: the Authoring mutating webhook +# rewrites EntryPoints to `sleep infinity`, so an Authoring pod would never run our +# bootstrap; PyTorchJob honors the submitted entrypoint. See +# hyperloom-pre-release-e2e-ci-design.md §7. +# +# It creates 5 workloads for the 8 legs: +# * 4x non-privileged 1-GPU PyTorchJob (one per baremetal leg) +# * 1x privileged 8-GPU PyTorchJob (docker host; 4 nested containers, GPU 0-3) +# and writes a dispatch map (leg -> workloadId) to $DISPATCH_MAP for the poll step. +# +# Requires: bash, curl, jq on the (self-hosted, in-network) runner. +# +# Inputs (env): +# SAFE_API_BASE SaFE API base url (required) +# SAFE_API_KEY bearer token; privileged pod needs an +# ADMIN token (privileged=true is admin-only) (required) +# SAFE_WORKSPACE_ID workspace that mounts the shared NFS (required) +# CI_VERSION wheel/run version, e.g. 1.0.0b3.dev...+ci (required) +# AUTHORING_IMAGE Authoring base image ref (required) +# NFS_ROOT pre-release test root on shared NFS +# (default /shared_nfs/hyperloom-pre-release-e2e-test) +# MODEL_3H local path to the 3h model (Qwen3-8B) (required) +# MODEL_12H local path to the 12h model (Qwen3-14B-FP8)(required) +# TARGET_GAIN release gate gain %% for every leg (default 100) +# CLAUDE_MODEL model for the Agent turns (required) +# CLAUDE_CLI_VERSION pinned Claude CLI version (required) +# ANTHROPIC_API_KEY Claude CLI auth; injected here as base64 +# into the workload env (never written to NFS)(required) +# ANTHROPIC_BASE_URL optional proxy / base url (optional) +# TASKS comma-separated leg subset (default: all 8) +# DISPATCH_MAP output file: JSON {leg: workloadId} +# (default $RUNNER_TEMP/pre_release_dispatch.json) +# HOST_CPU / HOST_MEM / HOST_SHM / HOST_EPHEMERAL privileged host resource request +# (default 196 / 2048Gi / 256Gi / 1792Gi -- ref 8-GPU Authoring pod +# uses 128 CPU; +68 for dockerd + 4 parallel agent/setup processes on +# top of 4x32 CPU-capped nested containers) +# LEG_CPU / LEG_MEM / LEG_EPHEMERAL baremetal leg resource request +# (default 32 / 512Gi / 512Gi -- sglang 14B-FP8 + roofline/aiter JIT +# exceeded 128Gi/100Gi on 2026-08-28) +# DOCKER_LEG_MEM_3H / DOCKER_LEG_MEM_12H / DOCKER_LEG_SHM_3H / DOCKER_LEG_SHM_12H +# nested docker container caps (default 256g / 512g / 64g / 64g) +# DEADLINE_3H_S / DEADLINE_12H_S pod hard-timeout per duration +# (default 16200 = 3h+1h+30m / 48600 = 12h+1h+30m). The docker host +# pod uses the MAX over its legs. SaFE kills the pod at the +# deadline; poll then judges that leg FAIL. Timing starts when +# the workload is DISPATCHED, not when it is queued. +# DEADLINE_FIELD SaFE payload field for the deadline (default `timeout`, the +# authoritative WorkloadSpec.Timeout field, integer seconds, +# top-level in the create-workload body; set "" to omit). +# SAFE_CACERT / SAFE_INSECURE TLS to the API (CA bundle / skip-verify) +set -euo pipefail + +NFS_ROOT="${NFS_ROOT:-/shared_nfs/hyperloom-pre-release-e2e-test}" +TARGET_GAIN="${TARGET_GAIN:-100}" +# Sized to a proven Running 8-GPU Authoring pod (ref: sglang-kimik3-2): CPU 128 baseline, +# bumped to 196 for four parallel nested legs (4x32 container CPU caps + host/agent headroom). +# mem 2048Gi, ephemeral 1792Gi. Every writable path the DinD host has -- the container +# rootfs AND the /shared-data emptyDir the nested dockerd stores images in -- counts +# toward this one ephemeralStorage quota, so the host bootstrap requires a +# layer-deduplicating docker storage driver (overlay2) to stay inside it. +HOST_CPU="${HOST_CPU:-196}"; HOST_MEM="${HOST_MEM:-2048Gi}"; HOST_SHM="${HOST_SHM:-256Gi}" +HOST_EPHEMERAL="${HOST_EPHEMERAL:-1792Gi}" +LEG_CPU="${LEG_CPU:-32}"; LEG_MEM="${LEG_MEM:-512Gi}" +LEG_EPHEMERAL="${LEG_EPHEMERAL:-512Gi}" +DOCKER_LEG_MEM_3H="${DOCKER_LEG_MEM_3H:-256g}" +DOCKER_LEG_MEM_12H="${DOCKER_LEG_MEM_12H:-512g}" +DOCKER_LEG_SHM_3H="${DOCKER_LEG_SHM_3H:-64g}" +DOCKER_LEG_SHM_12H="${DOCKER_LEG_SHM_12H:-64g}" +# SaFE workload scheduling priority (Spec.Priority, an int): High=2, Med=1, Low=0 +# (Primus-SaFE common/constant.go). The scheduler orders the queue by this value, and +# the webhook clamps it into [0,2]. These release-gate legs hold 8 GPUs for up to 14h +# and block the release, so run them High so they aren't starved behind dev workloads. +PRIORITY="${PRIORITY:-2}" +DISPATCH_MAP="${DISPATCH_MAP:-${RUNNER_TEMP:-/tmp}/pre_release_dispatch.json}" + +# Pod hard-timeout. SaFE terminates the workload at the deadline; the poll then sees a +# non-Succeeded terminal / missing report and judges that leg FAIL. Counted from DISPATCH +# (not queue) time. This MUST exceed the bootstrap's own in-pod wait deadline +# (hours*3600+3600, i.e. 3h/12h demo + 1h agent/setup buffer) so SaFE never pre-empts the +# pod mid-wait -- which would lose bootstrap's clean `return 1` + logging and reintroduce +# the premature-teardown race. We add a further +30m pod margin on top of the bootstrap +# deadline. Ordering per leg: bootstrap deadline < SaFE pod timeout < poll GLOBAL_TIMEOUT_S +# (default 50400s=14h, still > 48600s). Given per duration: +DEADLINE_3H_S="${DEADLINE_3H_S:-16200}" # 3h demo + 1h bootstrap buffer + 30m pod margin = 4.5h +DEADLINE_12H_S="${DEADLINE_12H_S:-48600}" # 12h demo + 1h bootstrap buffer + 30m pod margin = 13.5h +# The SaFE API field that carries the pod deadline. Confirmed against the Primus-SaFE +# codebase: the create-workload body embeds WorkloadSpec inline, whose `timeout` +# (integer seconds, top-level, from dispatch time) is enforced by WorkloadTTLController +# for ALL workload kinds incl. Authoring. Set DEADLINE_FIELD="" to omit (then the pod +# survival cap falls back to the workspace's per-scope maxRuntime, or the poll-side +# GLOBAL_TIMEOUT_S if none). Ref: apis/pkg/apis/amd/v1/workload_types.go WorkloadSpec.Timeout. +DEADLINE_FIELD="${DEADLINE_FIELD:-timeout}" +leg_deadline_s() { case "$1" in *-3h) echo "$DEADLINE_3H_S" ;; *-12h) echo "$DEADLINE_12H_S" ;; esac; } + +# SaFE caps the derived k8s object name at 44 chars (see create_workload), so we can +# NOT embed the full CI_VERSION (e.g. 1.0.0.dev202608270954+ci) in every workload name +# -- it would blow the limit and, once truncated, collide across legs (the leg suffix +# gets cut). Instead build "e2e--": the human-readable leg +# stays intact up front, and a 6-hex digest of CI_VERSION+run id disambiguates across +# runs (including repeated pushes with the same wheel version). All legs share VERSION_TAG. +VERSION_TAG="$(printf '%s-%s' "$CI_VERSION" "${GITHUB_RUN_ID:-local}" | sha1sum | cut -c1-6)" +workload_name() { printf 'e2e-%s-%s' "$1" "$VERSION_TAG"; } # $1 = leg (or "docker-host") + +: "${SAFE_API_BASE:?SAFE_API_BASE is required}" +: "${SAFE_API_KEY:?SAFE_API_KEY is required}" +: "${SAFE_WORKSPACE_ID:?SAFE_WORKSPACE_ID is required}" +: "${CI_VERSION:?CI_VERSION is required}" +: "${AUTHORING_IMAGE:?AUTHORING_IMAGE is required}" +: "${MODEL_3H:?MODEL_3H is required}" +: "${MODEL_12H:?MODEL_12H is required}" +: "${CLAUDE_MODEL:?CLAUDE_MODEL is required}" +: "${CLAUDE_CLI_VERSION:?CLAUDE_CLI_VERSION is required}" +: "${ANTHROPIC_API_KEY:?ANTHROPIC_API_KEY is required}" + +API="${SAFE_API_BASE%/}/api/v1/workloads" +auth=(-H "Authorization: Bearer ${SAFE_API_KEY}") +tls=() +if [ -n "${SAFE_CACERT:-}" ]; then + tls=(--cacert "$SAFE_CACERT") +elif [ "${SAFE_INSECURE:-0}" = "1" ]; then + tls=(-k) +fi + +summary() { echo "$*" | tee -a "${GITHUB_STEP_SUMMARY:-/dev/null}"; } + +# ---- reclaim stale pre-release workloads BEFORE dispatching ----------------- +# Stale = any non-terminal e2e-* in this workspace whose VERSION_TAG differs from ours. +# The concurrency.cancel-in-progress GitHub knob only cancels the JOB; it does NOT +# reliably stop the SaFE PyTorchJob pods a superseded/failed run already created (the +# `if: cancelled()` cleanup gets a short grace window, and a job that FAILS -- not +# cancels -- after dispatch skips it entirely). Verified 2026-08-27: three `Running` +# e2e workloads (incl. an 8-GPU docker host) leaked from a dead run and idle-held their +# cards. So the correct, self-healing order is the reverse of "cancel job -> hope the +# pod stops": a NEW run STOPS every stale e2e-* workload up front, frees the GPUs, then +# dispatches its own. The old run's poll then sees phase=Stopped and judges those legs +# FAIL -- the GitHub job ends naturally as a consequence of stopping the pod, not the +# other way round. +# +# Scope: only workloads whose displayName starts `e2e-` (this CI's own), in THIS +# workspace, that are NOT already terminal, and NOT this run's own tag (VERSION_TAG, +# whose workloads don't exist yet anyway -- a belt-and-suspenders guard). +reap_stale_workloads() { + local resp + resp="$(curl -sS "${tls[@]}" --max-time 30 "$API" "${auth[@]}" 2>/dev/null || true)" + [ -n "$resp" ] || { echo "[reap] could not list workloads; skipping reclaim" >&2; return 0; } + # Terminal phases we must NOT re-stop; anything else (Running/Pending/Queued/ + # Creating/Unknown/...) is a live pod holding resources. + local stale + stale="$(printf '%s' "$resp" | jq -r --arg ws "$SAFE_WORKSPACE_ID" --arg tag "$VERSION_TAG" ' + (.items // .workloads // .)[]? + | select(((.displayName // .name // "") | startswith("e2e-"))) + | select((.workspaceId // $ws) == $ws) + | select(((.displayName // .name // "") | contains($tag)) | not) + | select((.phase // .status // "") as $p + | (["Stopped","Failed","Succeeded","Completed","Deleted"] | index($p)) | not) + | (.workloadId // .id)' 2>/dev/null || true)" + [ -n "$stale" ] || { summary "• no stale e2e workloads to reclaim"; return 0; } + local wid code n=0 + while IFS= read -r wid; do + [ -n "$wid" ] || continue + code="$(curl -sS "${tls[@]}" --max-time 20 -o /dev/null -w '%{http_code}' \ + -X POST "$API/$wid/stop" "${auth[@]}" 2>/dev/null || echo 000)" + summary "• reclaimed stale workload \`$wid\` (stop HTTP $code)" + n=$((n+1)) + done <<< "$stale" + summary "• reclaimed $n stale e2e workload(s) before dispatch" +} +reap_stale_workloads + +# All 8 legs. Fields: mode backend hours model_path -- gpu index within the docker host +ALL_LEGS="baremetal-vllm-3h baremetal-vllm-12h baremetal-sglang-3h baremetal-sglang-12h \ +docker-vllm-3h docker-vllm-12h docker-sglang-3h docker-sglang-12h" +REQ_TASKS="${TASKS:-$ALL_LEGS}" +REQ_TASKS="${REQ_TASKS//,/ }" + +leg_model_path() { case "$1" in *-3h) echo "$MODEL_3H" ;; *-12h) echo "$MODEL_12H" ;; esac; } +leg_hours() { case "$1" in *-3h) echo "3" ;; *-12h) echo "12" ;; esac; } +leg_backend() { case "$1" in *-vllm-*) echo "vllm" ;; *-sglang-*) echo "sglang" ;; esac; } + +# GPU index a docker leg binds inside the privileged host (design §3). +docker_gpu_index() { + case "$1" in + docker-vllm-3h) echo 0 ;; + docker-vllm-12h) echo 1 ;; + docker-sglang-3h) echo 2 ;; + docker-sglang-12h) echo 3 ;; + *) echo "" ;; + esac +} + +# Common env for every workload. The API key is passed base64 so it is not visible in +# plaintext in the API payload log; bootstrap decodes it and writes it only to the +# pod-local .env (never to NFS). See design §9 (point D). +common_env_json() { + local model_path="$1" hours="$2" backend="$3" + jq -n \ + --arg civ "$CI_VERSION" --arg nfs "$NFS_ROOT" \ + --arg model "$model_path" --arg hours "$hours" --arg backend "$backend" \ + --arg tgain "$TARGET_GAIN" \ + --arg cmodel "$CLAUDE_MODEL" --arg cver "$CLAUDE_CLI_VERSION" \ + --arg keyb64 "$(printf '%s' "$ANTHROPIC_API_KEY" | base64 | tr -d '\n')" \ + --arg baseurl "${ANTHROPIC_BASE_URL:-}" \ + --arg cheaders "${ANTHROPIC_CUSTOM_HEADERS:-}" \ + '{ + CI_VERSION: $civ, + NFS_ROOT: $nfs, + HYPERLOOM_MODEL_PATH: $model, + DEMO_HOURS: $hours, + HYPERLOOM_BACKEND: $backend, + TARGET_GAIN: $tgain, + CLAUDE_MODEL: $cmodel, + CLAUDE_CLI_VERSION: $cver, + ANTHROPIC_API_KEY_B64: $keyb64 + } + + (if $baseurl == "" then {} else {ANTHROPIC_BASE_URL: $baseurl} end) + + (if $cheaders == "" then {} else {ANTHROPIC_CUSTOM_HEADERS: $cheaders} end)' +} + +# POST one workload; echo the workloadId. +# Args: displayName resourcesJson envJson privileged(true|false) entry_b64 deadline_s +create_workload() { + local name="$1" resources="$2" env="$3" privileged="$4" entry_b64="$5" deadline_s="${6:-}" + local body resp code json wid dl_json="{}" + # SaFE derives the k8s object name from displayName and enforces (via the + # vworkload admission webhook, STRICTER than plain RFC 1123): 1-44 chars, lower + # case alphanumerics or '-', MUST start with an ALPHABETIC char and end with an + # alphanumeric. So a leading digit or a '.'/'+' (both in CI_VERSION, e.g. + # 1.0.0.dev...+ci) is illegal, and the full "e2e--" easily + # exceeds 44. Fold every illegal char to '-', lowercase, collapse/trim dashes, + # then cap at 44 chars re-trimming any trailing dash the cut may leave. + name="$(printf '%s' "$name" | tr '[:upper:]' '[:lower:]' | sed -E 's/[^a-z0-9]+/-/g; s/-+/-/g; s/^[^a-z]+//; s/-+$//')" + name="${name:0:44}"; name="${name%%-}" + [ -n "$name" ] || name="e2e" + # Attach the pod hard-deadline when both a field name and a value are set. + if [ -n "$DEADLINE_FIELD" ] && [ -n "$deadline_s" ]; then + dl_json="$(jq -n --arg k "$DEADLINE_FIELD" --argjson v "$deadline_s" '{($k): $v}')" + fi + # kind PyTorchJob, NOT Authoring: SaFE's mutating webhook (mutateAuthoring) + # unconditionally overwrites an Authoring workload's EntryPoints to `sleep infinity`, + # so our bootstrap entrypoint would never auto-run -- every leg would idle until + # something exec'd in. PyTorchJob is not in that mutate switch, so it HONORS the + # submitted entryPoints (run via launcher.sh) and bootstrap runs as the pod command. + # version stays "v1"; NO `group` field (webhook clears group, workload_webhook.go:260). + # privileged / 8-GPU / useWorkspaceStorage / timeout are all kind-agnostic (driven by + # request fields, not kind) -- confirmed against Primus-SaFE source. Pod name is + # -master-0, main container `pytorch`. + body="$(jq -n \ + --arg name "$name" --arg ws "$SAFE_WORKSPACE_ID" --arg img "$AUTHORING_IMAGE" \ + --arg entry "$entry_b64" --argjson res "$resources" --argjson env "$env" \ + --argjson priv "$privileged" --argjson dl "$dl_json" \ + --argjson prio "$PRIORITY" \ + '{ + displayName: $name, + workspaceId: $ws, + groupVersionKind: {kind:"PyTorchJob", version:"v1"}, + resources: [$res], + images: [$img], + entryPoints: [$entry], + env: $env, + priority: $prio, + useWorkspaceStorage: true + } + (if $priv then {privileged:true} else {} end) + $dl')" + resp="$(curl -sS "${tls[@]}" -w $'\n%{http_code}' -X POST "$API" \ + "${auth[@]}" -H "Content-Type: application/json" -d "$body")" + code="$(printf '%s' "$resp" | tail -n1)" + json="$(printf '%s' "$resp" | sed '$d')" + if [ "$code" -lt 200 ] || [ "$code" -ge 300 ]; then + # Report to stderr: this runs inside wid="$(create_workload ...)" command + # substitution, so a stdout message would be captured into $wid and never + # reach the CI log. stderr surfaces the real HTTP status + API body. + echo "❌ create '$name' failed (HTTP $code): $(printf '%s' "$json" | head -c 400)" >&2 + return 1 + fi + wid="$(printf '%s' "$json" | jq -r '.workloadId // empty')" + if [ -z "$wid" ]; then + echo "❌ create '$name' returned no workloadId: $(printf '%s' "$json" | head -c 400)" >&2 + return 1 + fi + printf '%s' "$wid" +} + +# Base64 the bootstrap entrypoint (SaFE requires base64-encoded entryPoints). +# The bootstrap script is staged to NFS by the build job (from .github/pre-release/) +# and read by the pod at ${NFS_ROOT}/bootstrap/${CI_VERSION}/bootstrap-pre-release.sh. +bootstrap_entry_b64() { + local extra="$1" # extra shell prepended (e.g. E2E_DOCKER_HOST=1) + local cmd + cmd="set -e; ${extra} exec bash \"\${NFS_ROOT}/bootstrap/${CI_VERSION}/bootstrap-pre-release.sh\"" + printf '%s' "$cmd" | base64 | tr -d '\n' +} + +echo "[dispatch] CI_VERSION=$CI_VERSION tasks='$REQ_TASKS'" +declare -A DISPATCH # leg -> workloadId + +# Persist the dispatch map INCREMENTALLY, one entry per created workload -- not just +# once at the end. With concurrency.cancel-in-progress a newer push can cancel THIS run +# mid-dispatch; the job's `if: cancelled()` cleanup then stops whatever is in +# DISPATCH_MAP. If the map were written only after the loop, workloads created before +# the cancel would leak their GPUs. Seed an empty map up front so the file always +# exists, then append after every successful create. +: > "$DISPATCH_MAP" 2>/dev/null || true +printf '{}\n' > "$DISPATCH_MAP" +record_dispatch() { # leg workloadId -- add to the in-memory map AND the on-disk map + local leg="$1" wid="$2" + DISPATCH["$leg"]="$wid" + local tmp="${DISPATCH_MAP}.tmp" + if jq --arg l "$leg" --arg w "$wid" '. + {($l):$w}' "$DISPATCH_MAP" > "$tmp" 2>/dev/null; then + mv "$tmp" "$DISPATCH_MAP" + fi +} + +leg_resources_1gpu="$(jq -n --arg cpu "$LEG_CPU" --arg mem "$LEG_MEM" --arg eph "$LEG_EPHEMERAL" \ + '{replica:1, gpu:"1", cpu:$cpu, memory:$mem, ephemeralStorage:$eph}')" + +# Discover requested docker legs up front so the 8-GPU host can be dispatched first. +# It schedules more slowly and spends minutes on dockerd + image pulls before the +# nested legs even start setup, so queue it before the four 1-GPU baremetal pods. +want_docker_host=0 +docker_legs=""; gpu_map="{}" +for leg in $REQ_TASKS; do + case "$leg" in + docker-*) + want_docker_host=1 + idx="$(docker_gpu_index "$leg")" + docker_legs="${docker_legs}${docker_legs:+ }${leg}" + gpu_map="$(printf '%s' "$gpu_map" | jq --arg l "$leg" --arg i "$idx" '. + {($l): $i}')" + ;; + esac +done + +# ---- docker legs: one privileged 8-GPU host running all requested docker legs ---- +if [ "$want_docker_host" = 1 ]; then + host_resources="$(jq -n --arg cpu "$HOST_CPU" --arg mem "$HOST_MEM" --arg shm "$HOST_SHM" \ + --arg eph "$HOST_EPHEMERAL" \ + '{replica:1, gpu:"8", cpu:$cpu, memory:$mem, sharedMemory:$shm, ephemeralStorage:$eph}')" + # The host env carries the per-leg GPU map so the host bootstrap runs each docker leg + # (run_leg, docker mode) with the right GPU index; each leg's agent then `docker run`s + # its own single-GPU container per the demo skill. + host_env="$(jq -n \ + --arg civ "$CI_VERSION" --arg nfs "$NFS_ROOT" \ + --arg m3 "$MODEL_3H" --arg m12 "$MODEL_12H" \ + --arg tgain "$TARGET_GAIN" --arg cmodel "$CLAUDE_MODEL" --arg cver "$CLAUDE_CLI_VERSION" \ + --arg keyb64 "$(printf '%s' "$ANTHROPIC_API_KEY" | base64 | tr -d '\n')" \ + --arg baseurl "${ANTHROPIC_BASE_URL:-}" \ + --arg cheaders "${ANTHROPIC_CUSTOM_HEADERS:-}" \ + --arg legs "$docker_legs" --argjson gpumap "$gpu_map" \ + --arg dm3 "$DOCKER_LEG_MEM_3H" --arg dm12 "$DOCKER_LEG_MEM_12H" \ + --arg ds3 "$DOCKER_LEG_SHM_3H" --arg ds12 "$DOCKER_LEG_SHM_12H" \ + '{ + CI_VERSION:$civ, NFS_ROOT:$nfs, + MODEL_3H:$m3, MODEL_12H:$m12, + TARGET_GAIN:$tgain, CLAUDE_MODEL:$cmodel, CLAUDE_CLI_VERSION:$cver, + ANTHROPIC_API_KEY_B64:$keyb64, + HYPERLOOM_RUN_MODE:"docker", + E2E_DOCKER_HOST:"1", + DOCKER_LEGS:$legs, DOCKER_GPU_MAP:($gpumap|tostring), + DOCKER_LEG_MEM_3H:$dm3, DOCKER_LEG_MEM_12H:$dm12, + DOCKER_LEG_SHM_3H:$ds3, DOCKER_LEG_SHM_12H:$ds12 + } + + (if $baseurl == "" then {} else {ANTHROPIC_BASE_URL:$baseurl} end) + + (if $cheaders == "" then {} else {ANTHROPIC_CUSTOM_HEADERS:$cheaders} end)')" + # E2E_DOCKER_HOST=1 travels in the workload `env` (above), NOT as a command prefix: + # under PyTorchJob the entrypoint actually runs, and a `VAR=1;` prefix followed by a + # separate `exec bash` would NOT export VAR into the bootstrap's environment. + entry="$(bootstrap_entry_b64 "")" + # The one host pod runs a mix of 3h and 12h nested legs, so its deadline must be + # the MAX over the legs it hosts (a 3h deadline would kill a still-running 12h leg). + host_dl="$DEADLINE_3H_S" + for leg in $docker_legs; do + case "$leg" in *-12h) host_dl="$DEADLINE_12H_S" ;; esac + done + wid="$(create_workload "$(workload_name "docker-host")" "$host_resources" "$host_env" true "$entry" "$host_dl")" + # Every docker leg shares the one host workloadId; the poll distinguishes them by + # reading each leg's own session dir on NFS. + for leg in $docker_legs; do + record_dispatch "$leg" "$wid" + summary "• \`$leg\` → workloadId \`$wid\` (docker host, GPU $(docker_gpu_index "$leg"), deadline $((host_dl/3600))h)" + done +fi + +# ---- baremetal legs: one non-privileged 1-GPU workload each ---------------- +for leg in $REQ_TASKS; do + case "$leg" in + baremetal-*) + env_json="$(common_env_json "$(leg_model_path "$leg")" "$(leg_hours "$leg")" "$(leg_backend "$leg")" \ + | jq --arg leg "$leg" '. + {LEG_ID:$leg, HYPERLOOM_RUN_MODE:"baremetal"}')" + entry="$(bootstrap_entry_b64 "")" + dl="$(leg_deadline_s "$leg")" + wid="$(create_workload "$(workload_name "$leg")" "$leg_resources_1gpu" "$env_json" false "$entry" "$dl")" + record_dispatch "$leg" "$wid" + summary "• \`$leg\` → workloadId \`$wid\` (baremetal, 1 GPU, deadline $((dl/3600))h)" + ;; + docker-*) + ;; + *) + summary "⚠️ unknown leg '$leg' ignored" + ;; + esac +done + +# ---- dispatch map already written incrementally by record_dispatch --------- +# (so a mid-dispatch cancel still leaves a complete-so-far map for cleanup to stop). +echo "dispatch_map=$DISPATCH_MAP" >> "${GITHUB_OUTPUT:-/dev/null}" +summary "" +summary "**dispatched $(jq 'length' "$DISPATCH_MAP") legs** → \`$DISPATCH_MAP\`" +echo "[dispatch] wrote $DISPATCH_MAP" +jq . "$DISPATCH_MAP" diff --git a/.github/scripts/pre-release-e2e-poll.sh b/.github/scripts/pre-release-e2e-poll.sh new file mode 100755 index 0000000000..3a43106f80 --- /dev/null +++ b/.github/scripts/pre-release-e2e-poll.sh @@ -0,0 +1,502 @@ +#!/usr/bin/env bash +# SPDX-FileCopyrightText: 2026 Advanced Micro Devices, Inc. +# SPDX-License-Identifier: MIT +# +# Pre-release E2E: poll the dispatched SaFE Authoring workloads and judge each leg +# independently, reporting PASS/FAIL as soon as that leg reaches a terminal state +# (design §10, point C -- the 3h legs finish ~3-4h and report first; 12h legs later). +# +# A leg PASSes only when ALL hold (design §9): +# 1. state.json stop_reason is a clean terminal exit (same set as optimize CLI exit 0), +# 2. crash_count is within tolerance (read from state.json). +# final.json is not required; bootstrap may fail waiting for it while optimize succeeded. +# TARGET_GAIN still flows to optimize via the demo skill; it is NOT used here to judge PASS. +# When the gate is already FAIL, poll keeps running until every leg reaches a terminal +# verdict so per-leg GitHub checks and the sticky report stay aligned with optimize. +# Superseded runs still exit early and leave workloads for dispatch reap. +# +# The exit code is 0 only if every requested leg PASSed. +# +# Requires: bash, curl, jq on the (self-hosted, in-network) runner. state.json is +# written root-only inside pods; poll reads it via sudo -n when needed (passwordless +# sudo on the baremetal runner, same as the NFS-root setup step). +# +# Inputs (env): +# SAFE_API_BASE / SAFE_API_KEY SaFE API (required) +# CI_VERSION run version (required) +# DISPATCH_MAP leg->workloadId JSON from dispatch (required) +# NFS_ROOT (default /shared_nfs/hyperloom-pre-release-e2e-test) +# TARGET_GAIN passed to optimize (demo skill); not used to judge PASS +# POLL_INTERVAL_S seconds between polls (default 120) +# GLOBAL_TIMEOUT_S hard cap; unfinished legs -> FAIL +# (default 50400 = 14h; zombie legs with an empty +# stop_reason wait here rather than a stall check) +# MAX_CRASHES / MAX_BOOT_FAILS tolerance (default 0 / 0) +# Optional GitHub commit status (per-leg context pre-release-e2e/): +# GH_STATUS_TOKEN / GH_STATUS_REPO / GH_STATUS_SHA / GH_STATUS_DETAILS_URL +# Supersede detection (release runner when a newer pre-release run is queued): +# GITHUB_RUN_ID / PR_NUMBER / HEAD_REF (branch); uses GH_STATUS_TOKEN + GH_STATUS_REPO +# SAFE_CACERT / SAFE_INSECURE TLS to the API +set -euo pipefail + +NFS_ROOT="${NFS_ROOT:-/shared_nfs/hyperloom-pre-release-e2e-test}" +TARGET_GAIN="${TARGET_GAIN:-100}" +POLL_INTERVAL_S="${POLL_INTERVAL_S:-120}" +GLOBAL_TIMEOUT_S="${GLOBAL_TIMEOUT_S:-50400}" +MAX_CRASHES="${MAX_CRASHES:-0}" +MAX_BOOT_FAILS="${MAX_BOOT_FAILS:-0}" +LEAVE_RUNNING_FILE="${LEAVE_RUNNING_FILE:-${DISPATCH_MAP}.leave_running}" +# Sleep in short slices instead of one long one so a cancelled job tears down in seconds +# rather than at the end of a full POLL_INTERVAL_S. Each slice also re-checks whether a +# newer pre-release run has been queued so this poll can exit and release the runner. +POLL_SLEEP_SLICE_S="${POLL_SLEEP_SLICE_S:-5}" +PRE_RELEASE_WORKFLOW_FILE="${PRE_RELEASE_WORKFLOW_FILE:-pre-release-e2e-test.yml}" + +: "${SAFE_API_BASE:?SAFE_API_BASE is required}" +: "${SAFE_API_KEY:?SAFE_API_KEY is required}" +: "${CI_VERSION:?CI_VERSION is required}" +: "${DISPATCH_MAP:?DISPATCH_MAP is required}" +[ -f "$DISPATCH_MAP" ] || { echo "dispatch map $DISPATCH_MAP not found" >&2; exit 2; } + +API="${SAFE_API_BASE%/}/api/v1/workloads" +auth=(-H "Authorization: Bearer ${SAFE_API_KEY}") +tls=() +if [ -n "${SAFE_CACERT:-}" ]; then + tls=(--cacert "$SAFE_CACERT") +elif [ "${SAFE_INSECURE:-0}" = "1" ]; then + tls=(-k) +fi +GH_API="${GH_API:-https://api.github.com}" + +runs_dir="${NFS_ROOT%/}/runs/${CI_VERSION}" +summary() { echo "$*" | tee -a "${GITHUB_STEP_SUMMARY:-/dev/null}"; } + +gh_status_on() { [ -n "${GH_STATUS_TOKEN:-}" ] && [ -n "${GH_STATUS_REPO:-}" ] && [ -n "${GH_STATUS_SHA:-}" ]; } + +# True when a newer pre-release workflow run is queued/in-flight for this PR branch. +# GitHub run ids are monotonic; a pending successor blocks on concurrency until we exit. +supersede_check_on() { + [[ "${GITHUB_RUN_ID:-}" =~ ^[0-9]+$ ]] \ + && [ -n "${GH_STATUS_TOKEN:-}" ] \ + && [ -n "${GH_STATUS_REPO:-}" ] +} + +_supersede_head_ref() { + if [ -n "${HEAD_REF:-}" ]; then + printf '%s' "$HEAD_REF" + return 0 + fi + if [[ "${PR_NUMBER:-}" =~ ^[0-9]+$ ]]; then + curl -sS \ + -H "Authorization: Bearer ${GH_STATUS_TOKEN}" \ + -H "Accept: application/vnd.github+json" \ + -H "X-GitHub-Api-Version: 2022-11-28" \ + "${GH_API}/repos/${GH_STATUS_REPO}/pulls/${PR_NUMBER}" 2>/dev/null \ + | jq -r '.head.ref // empty' 2>/dev/null || true + return 0 + fi + echo "" +} + +superseded_by_newer_run() { + supersede_check_on || return 1 + local head_ref newer + head_ref="$(_supersede_head_ref)" + [ -n "$head_ref" ] || return 1 + newer="$(curl -sS \ + -H "Authorization: Bearer ${GH_STATUS_TOKEN}" \ + -H "Accept: application/vnd.github+json" \ + -H "X-GitHub-Api-Version: 2022-11-28" \ + "${GH_API}/repos/${GH_STATUS_REPO}/actions/workflows/${PRE_RELEASE_WORKFLOW_FILE}/runs?branch=${head_ref}&per_page=10" \ + 2>/dev/null \ + | jq -r --argjson rid "$GITHUB_RUN_ID" ' + [.workflow_runs[]? + | select(.id > $rid) + | select(.status == "queued" or .status == "in_progress" or .status == "pending" or .status == "waiting") + | .id][0] // empty' 2>/dev/null || true)" + [ -n "$newer" ] +} + +mark_superseded_and_exit_poll() { # -> sets superseded=1, marks pending legs SKIP, breaks caller loop + local leg pending=0 leave_wids=() + for leg in "${LEGS[@]}"; do + [ -n "${VERDICT[$leg]}" ] && continue + VERDICT["$leg"]="SKIP|superseded by newer run (dispatch reap will stop)" + summary "⏳ **$leg** — superseded (newer run queued; workload left for dispatch reap)" + post_status "$leg" pending "superseded; newer run queued" + leave_wids+=( "${WID[$leg]}" ) + pending=$((pending + 1)) + done + if [ "${#leave_wids[@]}" -gt 0 ]; then + printf '%s\n' "${leave_wids[@]}" | sort -u | jq -R . | jq -s . > "$LEAVE_RUNNING_FILE" + fi + summary "" + summary "⏹ superseded: newer pre-release run queued (run_id>${GITHUB_RUN_ID}). Releasing the runner; ${pending} workload(s) left for the successor's dispatch reap." + superseded=1 +} + +post_status() { # leg state(pending|success|failure|error) description + gh_status_on || return 0 + local desc="${3:0:139}" + curl -sS -X POST \ + -H "Authorization: Bearer ${GH_STATUS_TOKEN}" \ + -H "Accept: application/vnd.github+json" \ + -H "X-GitHub-Api-Version: 2022-11-28" \ + "${GH_API}/repos/${GH_STATUS_REPO}/statuses/${GH_STATUS_SHA}" \ + -d "$(jq -n --arg s "$2" --arg d "$desc" --arg u "${GH_STATUS_DETAILS_URL:-}" --arg c "pre-release-e2e/$1" \ + '{state:$s, description:$d, context:$c} + (if $u=="" then {} else {target_url:$u} end)')" \ + >/dev/null 2>&1 || true +} + +# ---- sticky PR/commit report comment --------------------------------------- +# This CI runs on pull_request, so the PR number is passed in directly via PR_NUMBER +# (github.event.pull_request.number). If it's absent (e.g. a workflow_dispatch run), +# fall back to resolving the PR from the triggering commit (GET /commits/{sha}/pulls), +# and if that too fails, comment on the commit. Then upsert ONE sticky comment (matched +# by an HTML marker) and PATCH it in place as legs finish -- so a single comment +# updates incrementally (design §10, point C). Mirrors ci-e2e-dispatch.sh. +REPORT_MARKER="" +PR_NUMBER="${PR_NUMBER:-}"; COMMENT_TARGET="" # COMMENT_TARGET: "pr" | "commit" | "" (disabled) + +gh_report_on() { [ -n "${GH_STATUS_TOKEN:-}" ] && [ -n "${GH_STATUS_REPO:-}" ] && [ -n "${GH_STATUS_SHA:-}" ]; } + +resolve_comment_target() { + gh_report_on || { COMMENT_TARGET=""; return 0; } + # Preferred: the pull_request event handed us the PR number directly. + if [[ "$PR_NUMBER" =~ ^[0-9]+$ ]]; then + COMMENT_TARGET="pr"; echo "[report] commenting on PR #$PR_NUMBER (from event)"; return 0 + fi + # Fallback (workflow_dispatch etc.): a commit usually belongs to one PR; take the first. + PR_NUMBER="$(curl -sS \ + -H "Authorization: Bearer ${GH_STATUS_TOKEN}" \ + -H "Accept: application/vnd.github+json" \ + -H "X-GitHub-Api-Version: 2022-11-28" \ + "${GH_API}/repos/${GH_STATUS_REPO}/commits/${GH_STATUS_SHA}/pulls" 2>/dev/null \ + | jq -r '[.[]|.number][0] // empty' 2>/dev/null || true)" + if [[ "$PR_NUMBER" =~ ^[0-9]+$ ]]; then + COMMENT_TARGET="pr" + echo "[report] commenting on PR #$PR_NUMBER (from commit $GH_STATUS_SHA)" + else + COMMENT_TARGET="commit" + echo "[report] no PR for commit $GH_STATUS_SHA; commenting on the commit" + fi +} + +# GET/PATCH/POST helpers keyed by the marker. The PR path uses the issues API +# (a PR is an issue for comments); the commit path uses the commits API. +_comment_list_url() { + case "$COMMENT_TARGET" in + pr) echo "${GH_API}/repos/${GH_STATUS_REPO}/issues/${PR_NUMBER}/comments?per_page=100" ;; + commit) echo "${GH_API}/repos/${GH_STATUS_REPO}/commits/${GH_STATUS_SHA}/comments?per_page=100" ;; + esac +} +_comment_create_url() { + case "$COMMENT_TARGET" in + pr) echo "${GH_API}/repos/${GH_STATUS_REPO}/issues/${PR_NUMBER}/comments" ;; + commit) echo "${GH_API}/repos/${GH_STATUS_REPO}/commits/${GH_STATUS_SHA}/comments" ;; + esac +} +_comment_patch_url() { # comment id + case "$COMMENT_TARGET" in + pr) echo "${GH_API}/repos/${GH_STATUS_REPO}/issues/comments/$1" ;; + commit) echo "${GH_API}/repos/${GH_STATUS_REPO}/comments/$1" ;; + esac +} + +report_upsert() { # body(markdown, already includes the marker on line 1) + [ -n "$COMMENT_TARGET" ] || return 0 + local body="$1" cid + cid="$(curl -sS -H "Authorization: Bearer ${GH_STATUS_TOKEN}" -H "Accept: application/vnd.github+json" \ + "$(_comment_list_url)" 2>/dev/null \ + | jq -r --arg m "$REPORT_MARKER" '[.[]|select(.body|contains($m))|.id][0] // empty' 2>/dev/null || true)" + if [ -n "$cid" ]; then + curl -sS -X PATCH -H "Authorization: Bearer ${GH_STATUS_TOKEN}" -H "Accept: application/vnd.github+json" \ + "$(_comment_patch_url "$cid")" \ + -d "$(jq -n --arg b "$body" '{body:$b}')" >/dev/null 2>&1 || true + else + curl -sS -X POST -H "Authorization: Bearer ${GH_STATUS_TOKEN}" -H "Accept: application/vnd.github+json" \ + "$(_comment_create_url)" \ + -d "$(jq -n --arg b "$body" '{body:$b}')" >/dev/null 2>&1 || true + fi +} + +# Icon for a leg verdict in the sticky report table. +verdict_icon() { + case "$1" in + PASS) printf '%s' "✅" ;; + SKIP|PENDING) printf '%s' "⏳" ;; + *) printf '%s' "❌" ;; + esac +} + +# Build the sticky report body from the current VERDICT map. `phase` is a short +# status word (Running|Complete) shown in the heading. Legs with no verdict yet +# render as "⏳ pending". +report_body() { # phase done_count total_count + local phase="$1" done="$2" total="$3" leg v vv vd icon rows="" + for leg in "${LEGS[@]}"; do + v="${VERDICT[$leg]:-}" + if [ -z "$v" ]; then + rows="${rows}| \`${leg}\` | ⏳ pending | running | +" + continue + fi + vv="${v%%|*}"; vd="${v#*|}" + icon="$(verdict_icon "$vv")" + rows="${rows}| \`${leg}\` | ${icon} ${vv} | ${vd} | +" + done + local detail_link="" + [ -n "${GH_STATUS_DETAILS_URL:-}" ] && detail_link="[run details](${GH_STATUS_DETAILS_URL})" + printf '%s\n## Pre-release E2E — %s (%d/%d legs done)\n\nCI_VERSION `%s` · target_gain %s%% · commit `%s`\n\n| leg | verdict | detail |\n|-----|---------|--------|\n%s\n%s\n' \ + "$REPORT_MARKER" "$phase" "$done" "$total" "$CI_VERSION" "$TARGET_GAIN" "$GH_STATUS_SHA" "$rows" "$detail_link" +} + +# Count legs that have a terminal verdict. +done_count() { + local n=0 leg + for leg in "${LEGS[@]}"; do [ -n "${VERDICT[$leg]:-}" ] && n=$((n+1)); done + echo "$n" +} + +workload_phase() { # workloadId -> phase string + local wid="$1" detail + detail="$(curl -sS "${tls[@]}" "$API/$wid" "${auth[@]}" 2>/dev/null || true)" + printf '%s' "$detail" | jq -r '.phase // "Unknown"' 2>/dev/null || echo Unknown +} + +# Resolve a leg's session dir. Bootstrap writes the pinned session dir to +# runs///session/.session_dir (design §9: never guess by timestamp). +leg_session_dir() { + local leg="$1" pin + pin="${runs_dir}/${leg}/session/.session_dir" + if [ -f "$pin" ]; then head -n1 "$pin"; return; fi + echo "" +} + +# Read one jq filter from state.json. Pods write it root-only (mode 600); the runner +# user reads via sudo -n when needed. Echoes "__UNREADABLE__" when the file exists +# but cannot be opened. +state_json_query() { + local file="$1" filter="$2" + if [ ! -f "$file" ]; then + echo ""; return 0 + fi + if [ -r "$file" ]; then + jq -r "$filter" "$file" 2>/dev/null || echo "" + return 0 + fi + if sudo -n test -r "$file" 2>/dev/null; then + sudo -n jq -r "$filter" "$file" 2>/dev/null || echo "" + return 0 + fi + echo "__UNREADABLE__" +} + +# Clean terminal stop_reason values (hyperloom.inference_optimizer.cli._SUCCESS_STOP_REASONS). +is_clean_stop_reason() { + case "$1" in + target_reached|global_converged|time_exhausted|max_ticks|sweep_done|conc_sweep_done) + return 0 ;; + *) return 1 ;; + esac +} + +# Judge one leg from state.json. Echoes "PASS"|"PENDING"|"FAIL|". +judge_leg() { + local leg="$1" wphase="$2" sdir state gain stop crashes + sdir="$(leg_session_dir "$leg")" + if [ -z "$sdir" ] || [ ! -d "$sdir" ]; then + echo "PENDING|no session dir yet (workload phase=$wphase)"; return + fi + state="${sdir%/}/state.json" + if [ ! -f "$state" ]; then + echo "PENDING|state.json missing (workload phase=$wphase)"; return + fi + stop="$(state_json_query "$state" '.stop_reason // ""')" + gain="$(state_json_query "$state" '.cumulative_gain_validated // 0')" + crashes="$(state_json_query "$state" '.crash_count // 0')" + if [ "$stop" = "__UNREADABLE__" ] || [ "$gain" = "__UNREADABLE__" ] || [ "$crashes" = "__UNREADABLE__" ]; then + echo "PENDING|state.json not readable (workload phase=$wphase)"; return + fi + if [ "$crashes" -gt "$MAX_CRASHES" ] 2>/dev/null; then + echo "FAIL|crash_count=$crashes > $MAX_CRASHES"; return + fi + if [ -z "$stop" ]; then + echo "PENDING|state.json stop_reason not set yet (workload phase=$wphase)"; return + fi + if is_clean_stop_reason "$stop"; then + echo "PASS|stop=${stop} gain=${gain}%"; return + fi + echo "FAIL|stop_reason=${stop} (not a clean terminal exit)" +} + +# ---- poll loop ------------------------------------------------------------- +mapfile -t LEGS < <(jq -r 'keys[]' "$DISPATCH_MAP") +declare -A WID VERDICT +for leg in "${LEGS[@]}"; do + WID["$leg"]="$(jq -r --arg l "$leg" '.[$l]' "$DISPATCH_MAP")" + VERDICT["$leg"]="" + post_status "$leg" pending "dispatched; workload=${WID[$leg]}" +done +summary "## Pre-release E2E — CI_VERSION \`$CI_VERSION\`" +summary "" +summary "Polling ${#LEGS[@]} legs (global timeout $((GLOBAL_TIMEOUT_S/3600))h). Each leg reports on its own terminal (point C)." +summary "" + +# Resolve the PR (from the triggering commit) or fall back to a commit comment, +# then post the initial "all pending" sticky report. +resolve_comment_target +report_upsert "$(report_body Running "$(done_count)" "${#LEGS[@]}")" + +start_s="$(date +%s)" +fail_seen=0 # has any leg reached a FAIL verdict? -> the gate is already decided +gate_fail_announced=0 +superseded=0 # a newer workflow run is queued -> release runner without stopping pods +while :; do + if superseded_by_newer_run; then + mark_superseded_and_exit_poll + break + fi + pending=0 + changed=0 # did any leg reach a verdict this tick? -> refresh the sticky comment + for leg in "${LEGS[@]}"; do + [ -n "${VERDICT[$leg]}" ] && continue + wid="${WID[$leg]}" + wphase="$(workload_phase "$wid")" + res="$(judge_leg "$leg" "$wphase")" + verdict="${res%%|*}"; detail="${res#*|}" + if [ "$verdict" = "PASS" ]; then + VERDICT["$leg"]="PASS|$detail" + summary "✅ **$leg** — PASS ($detail)" + post_status "$leg" success "PASS — $detail" + changed=1 + elif [ "$verdict" = "PENDING" ]; then + if [ "$wphase" = "Succeeded" ] || [ "$wphase" = "Failed" ] || [ "$wphase" = "Stopped" ]; then + VERDICT["$leg"]="FAIL|$detail" + summary "❌ **$leg** — FAIL ($detail; workload $wphase, wid=\`$wid\`)" + post_status "$leg" failure "FAIL — $detail" + changed=1; fail_seen=1 + else + pending=$((pending + 1)) + fi + else + VERDICT["$leg"]="FAIL|$detail" + summary "❌ **$leg** — FAIL ($detail)" + post_status "$leg" failure "FAIL — $detail" + changed=1; fail_seen=1 + fi + done + + # A leg finished this tick -> refresh the single sticky report comment (point C). + [ "$changed" -eq 1 ] && report_upsert "$(report_body Running "$(done_count)" "${#LEGS[@]}")" + + if [ "$fail_seen" -eq 1 ] && [ "$gate_fail_announced" -eq 0 ]; then + summary "" + summary "⚠️ **GATE: FAIL** — at least one leg did not pass. Continuing to poll until every leg reaches a terminal verdict." + gate_fail_announced=1 + fi + + [ "$pending" -eq 0 ] && break + + elapsed=$(( $(date +%s) - start_s )) + if [ "$elapsed" -ge "$GLOBAL_TIMEOUT_S" ]; then + for leg in "${LEGS[@]}"; do + [ -n "${VERDICT[$leg]}" ] && continue + VERDICT["$leg"]="FAIL|global timeout after ${elapsed}s" + summary "❌ **$leg** — FAIL (global timeout)" + post_status "$leg" failure "global timeout after $((elapsed/3600))h" + done + break + fi + echo "[poll] ${pending} leg(s) still running; elapsed $((elapsed/60))m; sleeping ${POLL_INTERVAL_S}s" + slept=0 + while [ "$slept" -lt "$POLL_INTERVAL_S" ]; do + if superseded_by_newer_run; then + mark_superseded_and_exit_poll + break + fi + sleep "$POLL_SLEEP_SLICE_S" + slept=$(( slept + POLL_SLEEP_SLICE_S )) + done + [ "$superseded" -eq 1 ] && break +done + +if [ "$superseded" -eq 1 ]; then + summary "" + summary "### Result" + summary "" + summary "| leg | verdict | detail |" + summary "|-----|---------|--------|" + for leg in "${LEGS[@]}"; do + v="${VERDICT[$leg]:-SKIP|superseded}" + vv="${v%%|*}"; vd="${v#*|}" + summary "| \`$leg\` | ⏳ $vv | $vd |" + done + summary "" + gate_line="**GATE: SUPERSEDED** — newer run queued; workloads left for dispatch reap." + summary "$gate_line" + report_upsert "$(printf '%s\n\n%s\n' "$(report_body Complete "$(done_count)" "${#LEGS[@]}")" "$gate_line")" + exit 0 +fi + +# ---- aggregate gate -------------------------------------------------------- +summary "" +summary "### Result" +summary "" +summary "| leg | verdict | detail |" +summary "|-----|---------|--------|" +fail=0 +for leg in "${LEGS[@]}"; do + v="${VERDICT[$leg]:-FAIL|no verdict}" + vv="${v%%|*}"; vd="${v#*|}" + case "$vv" in + PASS) icon="✅" ;; + SKIP) icon="⏳"; fail=1 ;; + *) icon="❌"; fail=1 ;; + esac + summary "| \`$leg\` | $icon $vv | $vd |" +done +summary "" +if [ "$fail" -eq 0 ]; then + gate_line="**GATE: PASS** — all ${#LEGS[@]} legs completed with a clean terminal stop_reason." +else + gate_line="**GATE: FAIL** — one or more legs did not pass. Release blocked." +fi +summary "$gate_line" + +# Final sticky report: the completed table plus the gate verdict. +report_upsert "$(printf '%s\n\n%s\n' "$(report_body Complete "$(done_count)" "${#LEGS[@]}")" "$gate_line")" + +# ---- reclaim: STOP (not delete) every workload ----------------------------- +# Policy: stop, don't delete. This frees the GPUs immediately (so a 3h leg that +# finished early doesn't idle-hold its card until the SaFE `timeout` deadline) while +# KEEPING the workload record + its pod filesystem for post-hoc inspection. SaFE has +# no `start` endpoint, so these are not resumable; clean up Stopped records manually. +# Verified 2026-08-27: POST /api/v1/workloads/{id}/stop exists and returns 200. +leave_running_wid() { # wid -> 0 if this workload should stay up + local wid="$1" + [ -f "$LEAVE_RUNNING_FILE" ] || return 1 + jq -e --arg w "$wid" 'index($w) != null' "$LEAVE_RUNNING_FILE" >/dev/null 2>&1 +} + +stop_workloads() { + local wid seen="" + for leg in "${LEGS[@]}"; do + wid="${WID[$leg]}" + # docker legs share one host workload -> stop each unique id once. + case " $seen " in *" $wid "*) continue ;; esac + seen="${seen} ${wid}" + if leave_running_wid "$wid"; then + summary "• left workload \`$wid\` running (superseded; dispatch reap)" + continue + fi + code="$(curl -sS "${tls[@]}" -o /dev/null -w '%{http_code}' -X POST \ + "$API/$wid/stop" "${auth[@]}" 2>/dev/null || echo 000)" + summary "• stopped workload \`$wid\` (HTTP $code)" + done +} +stop_workloads || true + +[ "$fail" -eq 0 ] && exit 0 +exit 1 diff --git a/.github/workflows/pre-release-e2e-test.yml b/.github/workflows/pre-release-e2e-test.yml new file mode 100644 index 0000000000..a915bc9442 --- /dev/null +++ b/.github/workflows/pre-release-e2e-test.yml @@ -0,0 +1,346 @@ +# SPDX-FileCopyrightText: 2026 Advanced Micro Devices, Inc. +# SPDX-License-Identifier: MIT + +name: Pre-release E2E test + +# Full-path release gate: build the CI wheel, push it to NFS, then run 8 demo legs +# (baremetal/docker x vLLM/SGLang x 3h/12h) as SaFE Authoring workloads driven through +# the Claude CLI + setup/demo skills -- the same path a user takes. Independent of the +# per-PR smoke test (ci-e2e.yml). See hyperloom-pre-release-e2e-ci-design.md. +# +# Triggers (gate BEFORE merge, on the PR -- not after merge to main): +# * PR targeting main that bumps pyproject.toml's `version` field -> FULL run (all 8 legs) +# * PR targeting main that only touches this CI's scripts/prompts/ -> SCRIPTS-ONLY run +# workflow (version unchanged) (the 4 fast 3h legs) +# * manual workflow_dispatch (reuse a wheel, run a leg subset; default all 8) +# NOTE: there is deliberately NO push-to-main trigger -- the gate runs while the PR is +# open so a bad release is blocked before it lands, and merging does not re-run it. +# +# ── Environment values to fill in (repo Secrets/Variables) ────────────────── +# All names carry a PRE_E2E_ prefix so they never collide with the existing per-PR +# CI's secrets/variables (ci-e2e.yml etc.). GITHUB_TOKEN is the one exception -- it is +# GitHub's built-in token and cannot be renamed. +# Secrets (sensitive): PRE_E2E_SAFE_API_KEY (ADMIN token: privileged 8-GPU pod is +# admin-only), PRE_E2E_ANTHROPIC_API_KEY. +# Variables (non-secret): PRE_E2E_SAFE_API_BASE, PRE_E2E_SAFE_WORKSPACE_ID, +# PRE_E2E_AUTHORING_IMAGE, PRE_E2E_NFS_ROOT, PRE_E2E_MODEL_3H_PATH, +# PRE_E2E_MODEL_12H_PATH, PRE_E2E_CLAUDE_MODEL, PRE_E2E_CLAUDE_CLI_VERSION, +# (optional) PRE_E2E_ANTHROPIC_BASE_URL, PRE_E2E_SAFE_INSECURE. +# PRE_E2E_ANTHROPIC_CUSTOM_HEADERS is NOT required: when a gateway base URL is set, +# the bootstrap defaults the AMD APIM header to +# `Ocp-Apim-Subscription-Key: ${ANTHROPIC_API_KEY}` (the value is just the key, +# which already reaches the pod). Set this variable only to OVERRIDE that default +# for a gateway with a different header convention. +# Nothing below hard-codes an environment value; all are `${{ secrets.* }}` / +# `${{ vars.* }}` references. TODO(owner): populate these before first real run. + +on: + pull_request: + branches: [main] + # A PR must touch one of these to even start the workflow; `resolve` then + # classifies it into a FULL run (version bump vs the base branch) or a + # SCRIPTS-ONLY run (CI logic changed but version unchanged). + paths: + - "pyproject.toml" # version bump -> full release gate + - ".github/workflows/pre-release-e2e-test.yml" # the workflow itself + - ".github/scripts/pre-release-e2e-*.sh" # dispatch / poll + - ".github/pre-release/**" # bootstrap, docker-run, prompts + workflow_dispatch: + inputs: + reuse_ci_version: + description: "Reuse an existing wheel dir (skip build); e.g. 1.0.0b3.dev202608261530+ci" + required: false + tasks: + description: "Comma-separated subset of leg IDs (default: all 8)" + required: false + +# One pre-release run at a time: a NEWER push supersedes the in-flight run. Per-PR group +# so two different PRs don't cancel each other -- only newer commits on the SAME PR. +# +# A newer run CANNOT tear down the older one. While the group is held, GitHub keeps the +# new run at run-level `pending` with an EMPTY jobs array (observed 2026-08-28: run +# 33145644120 sat pending for 20min with zero jobs while 33144353030 held the group), so +# no job of the new run exists to do any reclaiming. A short-lived `preempt` job on a +# GitHub-hosted runner used to claim that role and could never work: it was both created +# too late and unable to reach SAFE_API_BASE, an in-network NodePort -- every run logged +# `[preempt] could not list workloads; skipping reclaim` after a 30s curl timeout, so it +# never stopped a single workload. It has been removed rather than left to look effective. +# +# Teardown therefore runs the other way round: the OLD run releases the runner promptly. +# The poll fails fast on the first FAIL (the gate is already lost) but leaves still- +# running workloads up for post-mortem, and sleeps in short slices so a cancel lands in +# seconds. Leaked pods are then reclaimed by the dispatch-side reap, which -- unlike +# `preempt` -- runs in-network and only ever executes once this run holds the runner, +# i.e. once the previous run is provably done. +concurrency: + group: pre-release-e2e-${{ github.event.pull_request.number || github.ref }} + cancel-in-progress: true + +permissions: + contents: write # commit-comment fallback when no PR is associated + statuses: write # per-leg commit statuses (pre-release-e2e/) + pull-requests: write # sticky PR report comment + issues: write # PR comments go through the issues API + +jobs: + # 1. resolve: gate on a real version bump (PR vs base) or manual input; compute CI_VERSION. + resolve: + runs-on: hyperloom-pre-e2e-baremetal + outputs: + run: ${{ steps.decide.outputs.run }} + run_scope: ${{ steps.decide.outputs.run_scope }} + ci_version: ${{ steps.decide.outputs.ci_version }} + base_version: ${{ steps.decide.outputs.base_version }} + reuse: ${{ steps.decide.outputs.reuse }} + tasks: ${{ steps.decide.outputs.tasks }} + steps: + - name: Ensure jq + run: command -v jq >/dev/null 2>&1 || (sudo apt-get update && sudo apt-get install -y jq) + + - uses: actions/checkout@v7 + with: + # For a PR, actions/checkout checks out the merge ref; fetch-depth 0 gives us + # the base branch too so we can compare the version field against base. + fetch-depth: 0 + + - name: Decide + compute CI_VERSION + id: decide + env: + EVENT: ${{ github.event_name }} + REUSE_IN: ${{ inputs.reuse_ci_version }} + TASKS_IN: ${{ inputs.tasks }} + BASE_SHA: ${{ github.event.pull_request.base.sha }} + BASE_REF: ${{ github.event.pull_request.base.ref }} + run: | + set -euo pipefail + # The 4 fast 3h legs used when only CI logic (scripts/prompts/workflow) + # changed but the release version did not -- enough to exercise the changed + # dispatch/bootstrap/poll path without burning a full 14h GPU round. + SCRIPTS_ONLY_TASKS="baremetal-vllm-3h,baremetal-sglang-3h,docker-vllm-3h,docker-sglang-3h" + run=false; run_scope="none"; reuse=""; tasks="${TASKS_IN:-}" + # Extract the [project] version line. The field is a simple quoted string in + # this repo's pyproject.toml, so a scoped grep is deterministic and avoids + # embedding a Python heredoc in the YAML block scalar. + read_version() { # reads TOML on stdin, prints project.version + awk ' + /^\[project\]/{inp=1; next} + /^\[/{inp=0} + inp && /^[[:space:]]*version[[:space:]]*=/{ + gsub(/.*=[[:space:]]*"|".*/,""); print; exit + }' + } + # `base_version` here means the version in the PR head (this branch's + # pyproject.toml) -- confusingly it is also compared against the PR *base* + # branch below. Keep the name for downstream outputs (it feeds CI_VERSION). + base_version="$(read_version < pyproject.toml)" + if [ "$EVENT" = "workflow_dispatch" ]; then + # Manual runs are full-scope; tasks default to all 8 (empty = all in dispatch). + run=true; run_scope="full"; reuse="${REUSE_IN:-}" + else + # pull_request targeting main. Classify against the PR BASE branch (not HEAD~1: + # actions/checkout gives us the merge ref, whose first parent is the base tip, + # and fetch-depth:0 pulled the base commit BASE_SHA). version bump vs base -> + # FULL; else the PR only touched CI logic -> SCRIPTS-ONLY. + prev="$(git show "${BASE_SHA}:pyproject.toml" 2>/dev/null | read_version || true)" + if [ "$base_version" != "$prev" ] && [ -n "$base_version" ]; then + run=true; run_scope="full" + echo "version bump vs base ${BASE_REF}: '${prev}' -> '${base_version}' -> FULL run (all 8 legs)" + else + # Version unchanged vs base. on.pull_request.paths already guaranteed this PR + # touched one of the watched paths, and it wasn't the version -> it changed + # the CI's own scripts/prompts/workflow. Run the fast scripts-only scope. + run=true; run_scope="scripts-only" + changed="$(git diff --name-only "${BASE_SHA}" HEAD 2>/dev/null | grep -E \ + '^\.github/(workflows/pre-release-e2e-test\.yml|scripts/pre-release-e2e-.*\.sh|pre-release/)' \ + | paste -sd',' - || true)" + echo "version unchanged vs base ${BASE_REF}; CI logic changed (${changed:-see path filter}) -> SCRIPTS-ONLY run (4 fast 3h legs)" + fi + fi + + # Default the task set from the scope when the caller didn't pin one. + if [ -z "$tasks" ] && [ "$run_scope" = "scripts-only" ]; then + tasks="$SCRIPTS_ONLY_TASKS" + fi + + if [ -n "$reuse" ]; then + ci_version="$reuse" + else + ci_version="${base_version}.dev$(date -u +%Y%m%d%H%M)+ci" + fi + { + echo "run=$run" + echo "run_scope=$run_scope" + echo "reuse=$reuse" + echo "tasks=$tasks" + echo "base_version=$base_version" + echo "ci_version=$ci_version" + } >> "$GITHUB_OUTPUT" + echo "decision: run=$run scope=$run_scope ci_version=$ci_version reuse='${reuse}' tasks='${tasks}'" + + # 2. build: build the wheel, publish to NFS, stage bootstrap + prompts, write manifest. + build: + needs: resolve + if: needs.resolve.outputs.run == 'true' && needs.resolve.outputs.reuse == '' + runs-on: hyperloom-pre-e2e-baremetal + env: + CI_VERSION: ${{ needs.resolve.outputs.ci_version }} + BASE_VERSION: ${{ needs.resolve.outputs.base_version }} + NFS_ROOT: ${{ vars.PRE_E2E_NFS_ROOT }} + steps: + - uses: actions/checkout@v7 + + - name: Ensure the CI NFS root is writable by the runner user + # The bare-metal runner runs as `ubuntu`, which is not in the group that owns + # /shared_nfs, so it cannot mkdir the CI root there. Self-heal idempotently + # (survives runner reinstall / NFS remount) instead of relying on a one-off + # manual chown. Needs passwordless sudo on the runner host. + run: | + set -euo pipefail + : "${NFS_ROOT:?PRE_E2E_NFS_ROOT is required}" + if [ ! -w "$NFS_ROOT" ]; then + sudo -n mkdir -p "$NFS_ROOT" + sudo -n chown "$(id -un):$(id -gn)" "$NFS_ROOT" + fi + test -w "$NFS_ROOT" && echo "NFS root writable: $NFS_ROOT" + + - uses: actions/setup-python@v7 + with: + python-version: "3.11" + + - name: Build wheel with the CI version + id: build + run: | + set -euo pipefail + pip install --upgrade pip + pip install "setuptools>=77" wheel build + # Override the version for this build only (never committed). + SETUPTOOLS_SCM_PRETEND_VERSION="$CI_VERSION" \ + python -m build --wheel --no-isolation --outdir dist \ + -C--global-option=egg_info -C--global-option=--tag-build= 2>&1 | tee build.log || \ + python -m build --wheel --no-isolation --outdir dist + echo "wheel=$(ls dist/*.whl | head -n1)" >> "$GITHUB_OUTPUT" + + - name: Publish wheel + bootstrap + prompts + manifest to NFS + env: + WHEEL: ${{ steps.build.outputs.wheel }} + GIT_SHA: ${{ github.sha }} + RUN_ID: ${{ github.run_id }} + run: | + set -euo pipefail + : "${NFS_ROOT:?PRE_RELEASE_NFS_ROOT var is required}" + wheel_dir="${NFS_ROOT%/}/wheels/${CI_VERSION}" + runs_dir="${NFS_ROOT%/}/runs/${CI_VERSION}" + boot_dir="${NFS_ROOT%/}/bootstrap/${CI_VERSION}" + mkdir -p "$wheel_dir" "$runs_dir" "$boot_dir/prompts/pre-release" + # Wheel named after CI_VERSION per design §5. + cp "$WHEEL" "$wheel_dir/" + # Stage the in-pod scripts + fixed prompts (pods read them from NFS). + cp .github/pre-release/bootstrap-pre-release.sh "$boot_dir/" + cp .github/pre-release/prompts/pre-release/*.md "$boot_dir/prompts/pre-release/" + chmod +x "$boot_dir"/*.sh + # manifest.json (design §5). + jq -n \ + --arg wd "$wheel_dir" --arg rd "$runs_dir" --arg sha "$GIT_SHA" \ + --arg rid "$RUN_ID" --arg bv "$BASE_VERSION" --arg civ "$CI_VERSION" \ + '{ci_version:$civ, base_version:$bv, wheel_dir:$wd, runs_dir:$rd, + git_sha:$sha, github_run_id:$rid, + tasks:["baremetal-vllm-3h","baremetal-vllm-12h","baremetal-sglang-3h","baremetal-sglang-12h", + "docker-vllm-3h","docker-vllm-12h","docker-sglang-3h","docker-sglang-12h"]}' \ + > "$wheel_dir/manifest.json" + echo "published to $wheel_dir"; cat "$wheel_dir/manifest.json" + + # 3+4. dispatch + poll: create the 5 SaFE workloads, then judge each leg (point C). + run: + needs: [resolve, build] + # Run whenever resolve said go and build didn't fail (build is skipped on reuse). + if: >- + always() && needs.resolve.outputs.run == 'true' && + (needs.build.result == 'success' || needs.build.result == 'skipped') + runs-on: hyperloom-pre-e2e-baremetal + timeout-minutes: 900 # ≥14h: covers the 12h leg + bootstrap/setup/agent overhead + env: + CI_VERSION: ${{ needs.resolve.outputs.ci_version }} + TASKS: ${{ needs.resolve.outputs.tasks }} + NFS_ROOT: ${{ vars.PRE_E2E_NFS_ROOT }} + TARGET_GAIN: "100" + # SaFE API + SAFE_API_BASE: ${{ vars.PRE_E2E_SAFE_API_BASE }} + SAFE_API_KEY: ${{ secrets.PRE_E2E_SAFE_API_KEY }} # ADMIN token (privileged pod) + SAFE_WORKSPACE_ID: ${{ vars.PRE_E2E_SAFE_WORKSPACE_ID }} + SAFE_INSECURE: ${{ vars.PRE_E2E_SAFE_INSECURE || '1' }} + AUTHORING_IMAGE: ${{ vars.PRE_E2E_AUTHORING_IMAGE }} + # models + agent + MODEL_3H: ${{ vars.PRE_E2E_MODEL_3H_PATH }} + MODEL_12H: ${{ vars.PRE_E2E_MODEL_12H_PATH }} + CLAUDE_MODEL: ${{ vars.PRE_E2E_CLAUDE_MODEL }} + CLAUDE_CLI_VERSION: ${{ vars.PRE_E2E_CLAUDE_CLI_VERSION }} + ANTHROPIC_API_KEY: ${{ secrets.PRE_E2E_ANTHROPIC_API_KEY }} # GitHub Secret path (design §9 D) + ANTHROPIC_BASE_URL: ${{ vars.PRE_E2E_ANTHROPIC_BASE_URL }} + # Extra request headers for gateways that authenticate on a header of their own + # (AMD APIM wants Ocp-Apim-Subscription-Key, else `401 missing subscription key`). + # The Claude CLI reads ANTHROPIC_CUSTOM_HEADERS; ${ANTHROPIC_API_KEY} is expanded + # in-pod so the one secret covers both bearer + subscription-key. Header name is + # non-sensitive -> a Variable. e.g. `Ocp-Apim-Subscription-Key: ${ANTHROPIC_API_KEY}` + ANTHROPIC_CUSTOM_HEADERS: ${{ vars.PRE_E2E_ANTHROPIC_CUSTOM_HEADERS }} + # per-leg GitHub commit status + GH_STATUS_TOKEN: ${{ secrets.GITHUB_TOKEN }} + GH_STATUS_REPO: ${{ github.repository }} + # On pull_request, github.sha is the ephemeral merge commit; per-leg commit + # statuses must land on the PR HEAD sha so they surface on the PR checks tab. + GH_STATUS_SHA: ${{ github.event.pull_request.head.sha }} + # PR number is known directly from the event -- no commit->PR reverse lookup. + PR_NUMBER: ${{ github.event.pull_request.number }} + HEAD_REF: ${{ github.head_ref }} + GH_STATUS_DETAILS_URL: ${{ github.server_url }}/${{ github.repository }}/actions/runs/${{ github.run_id }} + POLL_INTERVAL_S: "120" + GLOBAL_TIMEOUT_S: "50400" + steps: + - uses: actions/checkout@v7 + + - name: Ensure jq + shared dispatch-map path + # RUNNER_TEMP is an env var inside run steps (the `runner.*` context is not + # available in job-level env:), so publish DISPATCH_MAP to the job env here. + run: | + command -v jq >/dev/null 2>&1 || (sudo apt-get update && sudo apt-get install -y jq) + echo "DISPATCH_MAP=${RUNNER_TEMP}/pre_release_dispatch.json" >> "$GITHUB_ENV" + + - name: Dispatch SaFE Authoring workloads + run: | + chmod +x .github/scripts/pre-release-e2e-dispatch.sh + .github/scripts/pre-release-e2e-dispatch.sh + + - name: Poll legs + write release gate + run: | + chmod +x .github/scripts/pre-release-e2e-poll.sh + .github/scripts/pre-release-e2e-poll.sh + + - name: Stop workloads on abnormal job end + # cancel OR failure OR a crash mid-dispatch/poll all leak pods: the poll step + # stops workloads only when IT runs to completion, so a job that is cancelled + # (superseded push) or fails (dispatch/poll error, runner death) after dispatch + # would idle-hold its GPUs. This idempotent cleanup fires on every abnormal end + # and stops whatever landed in DISPATCH_MAP (stopping an already-terminal + # workload is a harmless no-op). Fail-fast poll failures that intentionally left + # workloads running write DISPATCH_MAP.leave_running; those wids are skipped here + # too. The next run's pre-dispatch reap is the backstop for anything even this + # misses (e.g. the runner dying before this step). Success path is handled by + # the poll script's own stop_workloads(). + if: cancelled() || failure() + env: + SAFE_API_BASE: ${{ vars.PRE_E2E_SAFE_API_BASE }} + SAFE_API_KEY: ${{ secrets.PRE_E2E_SAFE_API_KEY }} + run: | + # Policy: stop, don't delete (frees GPUs, keeps the record + pod fs for + # inspection; SaFE has no restart, so clean up Stopped records manually). + [ -f "$DISPATCH_MAP" ] || { echo "no dispatch map; nothing to stop"; exit 0; } + leave_file="${DISPATCH_MAP}.leave_running" + for wid in $(jq -r '.[]' "$DISPATCH_MAP" | sort -u); do + if [ -f "$leave_file" ] && jq -e --arg w "$wid" 'index($w) != null' "$leave_file" >/dev/null 2>&1; then + echo "skipping workload $wid (left running for post-mortem)" + continue + fi + echo "stopping workload $wid" + curl -sS -k -o /dev/null -w 'STOP %{http_code}\n' -X POST \ + -H "Authorization: Bearer ${SAFE_API_KEY}" \ + "${SAFE_API_BASE%/}/api/v1/workloads/${wid}/stop" || true + done diff --git a/examples/hyperloom-qwen3-14b-fp8-12h/SKILL.md b/examples/hyperloom-qwen3-14b-fp8-12h/SKILL.md index 38cc0396b3..780d78b0fa 100644 --- a/examples/hyperloom-qwen3-14b-fp8-12h/SKILL.md +++ b/examples/hyperloom-qwen3-14b-fp8-12h/SKILL.md @@ -190,7 +190,10 @@ and the stop reason. Never print API keys, tokens, or custom header values. and critic subprocesses can import `hyperloom.agents` after changing cwd. 3. Run in background with `setsid nohup`. 4. Pass all required optimize CLI flags in the `python -m hyperloom.inference_optimizer.cli optimize` command. Do not rely on `.env` alone for `TP`, `CONC`, `ISL`, `OSL`, or `PRECISION`; CLI defaults can otherwise override the intended workload. -5. Report the session ID, log path, PID, and initial health check result. -6. Monitor the process every 300 seconds until work is done. -7. To recover an unexpected crash, only run `optimize --resume-from "$SESSION_DIR"` against the same session dir. After the first launch, never start a new `optimize`; that creates a new `` session and is forbidden. -8. If `stop_reason` in the current session `state.json` is final, stop and exit. +5. Include `--max-minutes-framework-pct 0.43` and `--max-minutes-kernel-pct 0.42` + in the optimize command. Do **not** pass `--no-framework-agent` or `--no-kernel` — + this demo runs the full OPTIMIZE phase (FRAMEWORK_AGENT + KERNEL_AGENT). +6. Report the session ID, log path, PID, and initial health check result. +7. Monitor the process every 300 seconds until work is done. +8. To recover an unexpected crash, only run `optimize --resume-from "$SESSION_DIR"` against the same session dir. After the first launch, never start a new `optimize`; that creates a new `` session and is forbidden. +9. If `stop_reason` in the current session `state.json` is final, stop and exit. diff --git a/examples/hyperloom-qwen3-8b-3h/SKILL.md b/examples/hyperloom-qwen3-8b-3h/SKILL.md index d5f16a5ee1..6f9cc63b66 100644 --- a/examples/hyperloom-qwen3-8b-3h/SKILL.md +++ b/examples/hyperloom-qwen3-8b-3h/SKILL.md @@ -1,9 +1,9 @@ --- name: hyperloom-qwen3-8b-3h -description: Run a 3-hour Hyperloom Qwen3-8B optimization session without the Kernel Agent. Use when the user wants a short, no-kernel Hyperloom demo on the local AMD ROCm environment. +description: Run a 3-hour Hyperloom Qwen3-8B FRAMEWORK_AGENT (OPTIMIZE) session without the Kernel Agent. Use when the user wants a short, framework-only Hyperloom demo on the local AMD ROCm environment. --- -# Hyperloom Qwen3-8B 3h No-Kernel Run +# Hyperloom Qwen3-8B 3h Framework-Only (No-Kernel) Run Read `.env` first and resolve `HYPERLOOM_SKILL_PATH`. Read and follow the optimizer skill at `@${HYPERLOOM_SKILL_PATH}` before launching. If `HYPERLOOM_SKILL_PATH` is missing, fall back to `@hyperloom/inference_optimizer/SKILL.md` (wheel install) or `@src/hyperloom/inference_optimizer/SKILL.md` (source checkout). This skill provides the concrete workload and launch constraints for a short Qwen3-8B demo. @@ -77,9 +77,8 @@ Required optimize CLI flags: - `--precision bf16` - `--target-gain 30` - `--max-hours 3` -- `--max-minutes-framework-pct 0.44` +- `--max-minutes-framework-pct 0.90` - `--max-minutes-sweep-pct 0.01` -- `--no-framework-agent` - `--no-kernel` - `--no-enable-conc-sweep` - `--no-enable-roofline` @@ -194,17 +193,14 @@ and the stop reason. Never print API keys, tokens, or custom header values. and critic subprocesses can import `hyperloom.agents` after changing cwd. 3. Run in background with `setsid nohup`. 4. Pass all required optimize CLI flags in the `python -m hyperloom.inference_optimizer.cli optimize` command. Do not rely on `.env` alone for `TP`, `CONC`, `ISL`, `OSL`, or `PRECISION`; CLI defaults can otherwise override the intended workload. -5. Include `--max-minutes-framework-pct 0.44` and `--max-minutes-sweep-pct 0.01` - in the optimize command. With FRAMEWORK_AGENT and KERNEL_AGENT disabled, - Hyperloom redistributes their shares so most of the short run budget is - reserved for OPTIMIZE while still leaving SWEEP/CLOSE time to exit cleanly - near the deadline. -6. Include `--no-framework-agent` in the optimize command so the - FRAMEWORK_AGENT phase is skipped. -7. Include `--no-kernel` in the optimize command so the Kernel Agent phase is skipped. -8. Include `--no-enable-conc-sweep` in the optimize command so the SWEEP-phase post-optimization concurrency sweep is skipped. -9. Include `--no-enable-roofline` in the optimize command so PRELUDE uses the lighter profile path instead of roofline analysis. -10. Report the session ID, log path, PID, and initial health check result. -11. Monitor the process every 300 seconds until work is done. -12. To recover an unexpected crash, only run `optimize --resume-from "$SESSION_DIR"` against the same session dir. After the first launch, never start a new `optimize`; that creates a new `` session and is forbidden. -13. If `stop_reason` in the current session `state.json` is final, stop and exit. +5. Include `--max-minutes-framework-pct 0.90` and `--max-minutes-sweep-pct 0.01` + in the optimize command. With `--no-kernel`, KERNEL_AGENT is disabled and its + budget share is redistributed mostly to FRAMEWORK_AGENT (~99% of wall clock). + Do **not** pass `--no-framework-agent` — that skips OPTIMIZE entirely. +6. Include `--no-kernel` in the optimize command so the Kernel Agent phase is skipped. +7. Include `--no-enable-conc-sweep` in the optimize command so the SWEEP-phase post-optimization concurrency sweep is skipped. +8. Include `--no-enable-roofline` in the optimize command so PRELUDE uses the lighter profile path instead of roofline analysis. +9. Report the session ID, log path, PID, and initial health check result. +10. Monitor the process every 300 seconds until work is done. +11. To recover an unexpected crash, only run `optimize --resume-from "$SESSION_DIR"` against the same session dir. After the first launch, never start a new `optimize`; that creates a new `` session and is forbidden. +12. If `stop_reason` in the current session `state.json` is final, stop and exit. diff --git a/pyproject.toml b/pyproject.toml index 809066e32b..9e056a5f4d 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -6,7 +6,7 @@ build-backend = "setuptools.build_meta" [project] name = "hyperloom-inference_optimizer" -version = "1.0.0" +version = "1.0.1a0" # TEMP TEST: alpha bump vs main (1.0.0) to force pre-release-e2e FULL run (all 8 legs); REVERT before merge description = "Inference Optimizer — three-role (Orchestration/Critic/Robustness) autonomous LLM inference optimization runtime for AMD GPU platforms. Kernel optimization is handled by programmatic Python handlers." readme = "README.md" requires-python = ">=3.10" diff --git a/src/hyperloom/inference_optimizer/tests/test_pre_release_gate_orchestration.py b/src/hyperloom/inference_optimizer/tests/test_pre_release_gate_orchestration.py new file mode 100644 index 0000000000..e0a648a2fd --- /dev/null +++ b/src/hyperloom/inference_optimizer/tests/test_pre_release_gate_orchestration.py @@ -0,0 +1,174 @@ +# SPDX-FileCopyrightText: 2026 Advanced Micro Devices, Inc. +# SPDX-License-Identifier: MIT + +"""Guards for how the pre-release gate releases its runner and its GPUs. + +The gate owns the only self-hosted baremetal runner, so a run that will not finish +blocks every later run: while the concurrency group is held, GitHub keeps the newer run +at run-level ``pending`` with an EMPTY jobs array, so it has no job with which to +reclaim anything. Two consequences are pinned here. + +A ``preempt`` job on a GitHub-hosted runner used to claim the reclaiming role. It could +never work -- created too late to matter, and unable to reach ``SAFE_API_BASE``, which +is an in-network NodePort: every observed run logged ``[preempt] could not list +workloads; skipping reclaim`` after a 30s curl timeout, having stopped nothing. Nothing +that talks to SaFE may run on a GitHub-hosted runner again. + +Teardown instead relies on the old run leaving promptly when superseded: the poll +sleeps in short slices so a cancel lands in seconds instead of at the end of a full +poll interval. After the gate is lost it keeps polling until every leg reports. + +There is no way to unit-test the scheduling itself short of running the workflow; these +tests pin the invariants it depends on. +""" + +from __future__ import annotations + +from pathlib import Path + +import pytest +import yaml + +_SELF_HOSTED_LABEL = "hyperloom-pre-e2e-baremetal" + + +def _find_github_dir() -> Path | None: + """Locate .github/; None when running from an installed wheel.""" + for parent in Path(__file__).resolve().parents: + candidate = parent / ".github" + if (candidate / "workflows" / "pre-release-e2e-test.yml").is_file(): + return candidate + return None + + +_GITHUB = _find_github_dir() + +pytestmark = pytest.mark.skipif( + _GITHUB is None, + reason="pre-release gate guards need the source checkout (.github/)", +) + + +@pytest.fixture(scope="module") +def workflow() -> dict: + assert _GITHUB is not None + path = _GITHUB / "workflows" / "pre-release-e2e-test.yml" + return yaml.safe_load(path.read_text(encoding="utf-8")) + + +@pytest.fixture(scope="module") +def poll_script() -> str: + assert _GITHUB is not None + return (_GITHUB / "scripts" / "pre-release-e2e-poll.sh").read_text(encoding="utf-8") + + +@pytest.fixture(scope="module") +def dispatch_script() -> str: + assert _GITHUB is not None + return (_GITHUB / "scripts" / "pre-release-e2e-dispatch.sh").read_text(encoding="utf-8") + + +def test_nothing_that_talks_to_safe_runs_on_a_github_hosted_runner(workflow: dict) -> None: + """SAFE_API_BASE is an in-network NodePort; a hosted runner can only time out.""" + for name, job in workflow["jobs"].items(): + runs_on = job.get("runs-on") + if runs_on == _SELF_HOSTED_LABEL: + continue + rendered = yaml.safe_dump(job) + assert "SAFE_API" not in rendered, f"job {name} on {runs_on} reaches for the SaFE API" + + +def test_the_preempt_job_is_gone(workflow: dict) -> None: + assert "preempt" not in workflow["jobs"] + assert workflow["jobs"] == {k: v for k, v in workflow["jobs"].items() if k in {"resolve", "build", "run"}} + assert workflow["jobs"]["resolve"].get("needs") is None + + +def test_the_reap_script_is_gone_and_unreferenced() -> None: + assert _GITHUB is not None + assert not (_GITHUB / "scripts" / "pre-release-e2e-reap.sh").exists() + for wf in (_GITHUB / "workflows").glob("*.yml"): + text = wf.read_text(encoding="utf-8") + for line in text.splitlines(): + if line.lstrip().startswith("#"): + continue # the removal rationale is documented in comments + assert "pre-release-e2e-reap.sh" not in line, f"{wf.name} still runs the reap script" + + +def test_poll_keeps_polling_after_the_gate_is_lost(poll_script: str) -> None: + """Each leg must reach a terminal verdict even after the gate is already FAIL.""" + assert "gate_fail_announced" in poll_script + assert "Continuing to poll until every leg reaches a terminal verdict" in poll_script + assert 'VERDICT["$leg"]="SKIP|still running (gate failed; workload left alive)"' not in poll_script + assert "POLL_FAIL_FAST" not in poll_script + assert poll_script.count("fail_seen=1") == 2 + assert poll_script.count('VERDICT["$leg"]="FAIL|') >= 2 + + +def test_poll_reads_root_only_state_json_via_sudo(poll_script: str) -> None: + """Pods write state.json mode 600; the runner user reads it with sudo -n.""" + assert "state_json_query" in poll_script + assert "sudo -n jq" in poll_script + assert "state.json not readable" in poll_script + assert "__UNREADABLE__" in poll_script + + +def test_poll_report_icons_distinguish_skip_from_fail(poll_script: str) -> None: + assert "verdict_icon" in poll_script + assert "SKIP|PENDING" in poll_script + + +def test_poll_sleeps_in_slices_so_a_cancel_lands_quickly(poll_script: str) -> None: + assert 'POLL_SLEEP_SLICE_S="${POLL_SLEEP_SLICE_S:-5}"' in poll_script + assert 'sleep "$POLL_SLEEP_SLICE_S"' in poll_script + assert 'sleep "$POLL_INTERVAL_S"' not in poll_script + + +def test_abnormal_end_cleanup_respects_leave_running(workflow: dict, poll_script: str) -> None: + """Superseded runs may leave workloads up; cleanup must not stop those wids.""" + assert "LEAVE_RUNNING_FILE=" in poll_script + assert "leave_running_wid" in poll_script + steps = workflow["jobs"]["run"]["steps"] + cleanup = [s for s in steps if "cancelled()" in str(s.get("if", ""))] + assert cleanup, "the run job lost its cancel/failure cleanup step" + body = cleanup[0]["run"] + assert "/stop" in body + assert "leave_running" in body + + +def test_dispatch_version_tag_is_unique_per_run(dispatch_script: str) -> None: + """Reap must distinguish repeated pushes that reuse the same CI_VERSION wheel.""" + assert ( + 'VERSION_TAG="$(printf \'%s-%s\' "$CI_VERSION" "${GITHUB_RUN_ID:-local}" | sha1sum | cut -c1-6)"' + in dispatch_script + ) + + +def test_docker_host_is_dispatched_before_baremetal(dispatch_script: str) -> None: + """The 8-GPU docker host schedules slowly; queue it before the 1-GPU baremetal pods.""" + docker_pos = dispatch_script.index("queue it before the four 1-GPU baremetal pods") + bare_pos = dispatch_script.index("# ---- baremetal legs: one non-privileged 1-GPU workload each") + assert docker_pos < bare_pos + + +def test_poll_exits_when_a_newer_run_is_queued(poll_script: str, workflow: dict) -> None: + """A pending successor cannot dispatch until this poll releases the runner.""" + assert "superseded_by_newer_run" in poll_script + assert "mark_superseded_and_exit_poll" in poll_script + assert "GATE: SUPERSEDED" in poll_script + assert "dispatch reap will stop" in poll_script + run_env = yaml.safe_dump(workflow["jobs"]["run"].get("env", {})) + assert "HEAD_REF:" in run_env + + +def test_poll_passes_on_clean_terminal_stop_reason_not_gain(poll_script: str) -> None: + """Gate PASS aligns with optimize CLI exit 0, not cumulative_gain vs TARGET_GAIN.""" + assert "is_clean_stop_reason" in poll_script + assert "target_reached|global_converged|time_exhausted|max_ticks|sweep_done|conc_sweep_done" in poll_script + assert "not used to judge PASS" in poll_script + assert 'echo "PASS|stop=${stop} gain=${gain}%"' in poll_script + assert "gain=${gain}% < ${TARGET_GAIN}" not in poll_script + assert "reached target_gain=" not in poll_script + assert "state.json stop_reason" in poll_script + assert "reports/final.json missing" not in poll_script + assert 'echo "PENDING|state.json stop_reason not set yet' in poll_script diff --git a/src/hyperloom/inference_optimizer/tests/test_pre_release_stall_liveness.py b/src/hyperloom/inference_optimizer/tests/test_pre_release_stall_liveness.py new file mode 100644 index 0000000000..2d33879e6c --- /dev/null +++ b/src/hyperloom/inference_optimizer/tests/test_pre_release_stall_liveness.py @@ -0,0 +1,282 @@ +# SPDX-FileCopyrightText: 2026 Advanced Micro Devices, Inc. +# SPDX-License-Identifier: MIT + +"""Regression guard for the pre-release E2E leg liveness (stall) check. + +``bootstrap-pre-release.sh`` blocks after the demo turn until ``optimize`` writes a +clean terminal ``stop_reason`` into ``state.json``, and declares a leg dead when nothing has been written for +``LEG_STALL_GRACE_S``. Two properties of that check killed legs that were provably +still alive (run 1.0.1a0.dev202608280354+ci, both baremetal-sglang legs): + +* the idle window was measured against absolute file mtimes, so the minutes spent + inside the two non-streaming ``claude --print`` turns were charged to the leg and + the very first loop iteration condemned it; +* only ``$session`` was watched, while the agent's launcher, its setup/install logs + and ``install.sh``'s caches land elsewhere under the leg root -- one leg was reaped + 26s after it last wrote a file. + +These tests exercise the real ``leg_idle_s`` helper out of the script. +""" + +from __future__ import annotations + +import os +import re +import subprocess +import time +from pathlib import Path + +import pytest + +_STALL_GRACE_S = 600 + + +def _find_bootstrap() -> Path | None: + """Locate the in-pod bootstrap; None when running from an installed wheel.""" + for parent in Path(__file__).resolve().parents: + candidate = parent / ".github" / "pre-release" / "bootstrap-pre-release.sh" + if candidate.is_file(): + return candidate + return None + + +_BOOTSTRAP = _find_bootstrap() + +pytestmark = pytest.mark.skipif( + _BOOTSTRAP is None, + reason="pre-release liveness guard needs the source checkout (.github/pre-release/)", +) + + +@pytest.fixture(scope="module") +def script() -> str: + assert _BOOTSTRAP is not None + return _BOOTSTRAP.read_text(encoding="utf-8") + + +@pytest.fixture(scope="module") +def leg_idle_fn(script: str) -> str: + """Slice the ``leg_idle_s`` function out of the script so it can be run alone.""" + lines = script.splitlines() + start = next(i for i, line in enumerate(lines) if line.startswith("leg_idle_s() {")) + end = next(i for i, line in enumerate(lines[start:], start) if line == "}") + return "\n".join(lines[start : end + 1]) + + +def _idle(leg_idle_fn: str, root: Path, loop_start: int, now: int) -> int: + """Run leg_idle_s(root, loop_start, now) in bash and return its answer.""" + proc = subprocess.run( + ["bash", "-c", f'{leg_idle_fn}\nleg_idle_s "$1" "$2" "$3"', "_", str(root), str(loop_start), str(now)], + capture_output=True, + text=True, + check=True, + ) + return int(proc.stdout.strip()) + + +def _write(path: Path, age_s: int, now: int) -> None: + path.parent.mkdir(parents=True, exist_ok=True) + path.write_text("x", encoding="utf-8") + stamp = now - age_s + os.utime(path, (stamp, stamp)) + + +def test_pre_loop_gap_is_not_charged_to_the_leg(leg_idle_fn: str, tmp_path: Path) -> None: + """The observed kill: nothing written for 629s, but the loop just started.""" + now = int(time.time()) + _write(tmp_path / "session" / ".session_dir", age_s=629, now=now) + idle = _idle(leg_idle_fn, tmp_path, loop_start=now - 5, now=now) + assert idle == 5 + assert idle < _STALL_GRACE_S + + +def test_writes_outside_the_session_dir_count_as_progress(leg_idle_fn: str, tmp_path: Path) -> None: + """The agent's own logs live next to the workspace, not under session/.""" + now = int(time.time()) + loop_start = now - 700 + _write(tmp_path / "session" / ".session_dir", age_s=700, now=now) + _write(tmp_path / "setup_sglang_retry.log", age_s=26, now=now) + idle = _idle(leg_idle_fn, tmp_path, loop_start=loop_start, now=now) + assert idle == 26 + assert idle < _STALL_GRACE_S + + +def test_a_genuinely_idle_tree_is_still_reaped(leg_idle_fn: str, tmp_path: Path) -> None: + """A hung launch must still be caught once the loop itself has waited it out.""" + now = int(time.time()) + _write(tmp_path / "session" / ".session_dir", age_s=900, now=now) + idle = _idle(leg_idle_fn, tmp_path, loop_start=now - 900, now=now) + assert idle == 900 + assert idle >= _STALL_GRACE_S + + +def test_empty_tree_falls_back_to_the_loop_start(leg_idle_fn: str, tmp_path: Path) -> None: + now = int(time.time()) + assert _idle(leg_idle_fn, tmp_path, loop_start=now - 42, now=now) == 42 + + +def test_stall_check_watches_the_leg_root(script: str) -> None: + """Pin the call site: the check must be scoped to $root, not $session.""" + assert 'idle="$(leg_idle_s "$root" "$grace_ts" "$now")"' in script + assert "no file written under $root" in script + + +def test_agent_turns_are_mirrored_to_nfs(script: str) -> None: + """SaFE deletes a failed leg's pod, so every agent turn must reach NFS.""" + assert 'agent_log="${session}/agent-${leg}.log"' in script + # Every turn goes through the one helper, which is the only place that tees, so no + # invocation can bypass the transcript. + assert script.count('tee -a "$alog"') == 1 + invocations = [ + line + for line in script.splitlines() + if "claude --print --dangerously-skip-permissions" in line and not line.lstrip().startswith("#") + ] + assert invocations == [' claude --print --dangerously-skip-permissions "$@" 2>&1 | tee -a "$alog"'] + + +def _leg_run_started(script: str, session: Path) -> bool: + """Run the real leg_run_started() against a session tree.""" + lines = script.splitlines() + start = next(i for i, line in enumerate(lines) if line.startswith("leg_run_started() {")) + end = next(i for i, line in enumerate(lines[start:], start) if line == "}") + fn = "\n".join(lines[start : end + 1]) + proc = subprocess.run( + ["bash", "-c", f'{fn}\nleg_run_started "$1"', "_", str(session)], + capture_output=True, + text=True, + ) + return proc.returncode == 0 + + +def test_launch_detection_needs_the_nested_run_dir(script: str, tmp_path: Path) -> None: + """optimize writes state.json into $session//-/, never at the top.""" + session = tmp_path / "session" + session.mkdir() + assert not _leg_run_started(script, session) + + # A stray state.json directly under the session must not count as a launch. + (session / "state.json").write_text("{}", encoding="utf-8") + assert not _leg_run_started(script, session) + + nested = session / "Qwen3-8B" / "20260828T053223Z-4351891f" + nested.mkdir(parents=True) + (nested / "state.json").write_text("{}", encoding="utf-8") + assert _leg_run_started(script, session) + + +def test_setup_marker_matches_every_setup_prompt(script: str) -> None: + """The retry gate greps a literal the prompts must actually ask the agent to print.""" + assert 'grep -qiE "setup complete: ${run_mode}/${backend}" "$agent_log"' in script + prompts = _BOOTSTRAP.parent / "prompts" / "pre-release" # type: ignore[union-attr] + for run_mode in ("baremetal", "docker"): + for backend in ("vllm", "sglang"): + prompt = prompts / f"setup-{run_mode}-{backend}.md" + assert f"setup complete: {run_mode}/{backend}" in prompt.read_text(encoding="utf-8") + + +def test_an_early_turn_is_re_driven_not_fatal(script: str) -> None: + """A turn that ends without finishing leaves nothing running; ask again, bounded.""" + assert 'max_demo_redrives="${LEG_DEMO_REDRIVES:-5}"' in script + assert "demo_redrives=$(( demo_redrives + 1 ))" in script + + +def test_bootstrap_completes_on_clean_stop_reason_without_final_json(script: str) -> None: + """Align with the poll gate: a clean terminal stop_reason is enough to exit 0.""" + assert "is_clean_stop_reason" in script + assert "final.json never appeared" not in script + assert "LEG_FINAL_GRACE_S" not in script + assert "state.json stop_reason='$stop' after" in script + + +def test_bootstrap_publishes_state_json_for_poll_reader(script: str) -> None: + """The poll runner reads state.json as ubuntu; bootstrap opens it after each write.""" + assert "publish_state_for_poll" in script + assert 'chmod a+r "$state_json"' in script + assert 'publish_state_for_poll "$state_json" "$real_sdir" "$session"' in script + + +def test_setup_is_budgeted_in_time_not_in_turns(script: str) -> None: + """A count-based cap of 3 turns was ~4min of wall clock and killed live installs. + + A framework install runs 10-30min, and each turn ends after ~60-90s, so the budget + has to be a deadline plus a liveness check -- never a small turn count. + """ + assert "LEG_TURN_ATTEMPTS" not in script + assert 'setup_deadline_s="${LEG_SETUP_DEADLINE_S:-2700}"' in script + assert 'setup_stall_s="${LEG_SETUP_STALL_S:-600}"' in script + assert 'sidle="$(leg_idle_s "$root" "$setup_t0" "$snow")"' in script + # The stall check must gate the failure, i.e. a progressing install is never reaped. + assert 'if [ "$sidle" -ge "$setup_stall_s" ]; then' in script + + +def test_follow_up_turns_resume_the_same_conversation(script: str) -> None: + """Re-feeding the prompt as a fresh turn throws away what the agent already knows. + + `--session-id` opens the leg's conversation and `--resume` continues it, so a + follow-up turn still knows what it launched and which log it was watching. + """ + assert 'agent_turn "$agent_log" --session-id "$uuid" < "$setup_prompt"' in script + assert script.count('--resume "$uuid"') == 3 # setup nudge, demo turn, demo re-drive + for nudge in ("SETUP_RESUME_NUDGE", "DEMO_RESUME_NUDGE"): + assert f"{nudge}='" in script + # Resuming can fail (no such session); that must degrade, not kill the leg. + assert script.count("could not resume session") == 3 + + +def test_leg_session_uuid_is_stable_and_well_formed(script: str, tmp_path: Path) -> None: + """--session-id rejects anything that is not a UUID.""" + lines = script.splitlines() + start = next(i for i, line in enumerate(lines) if line.startswith("leg_session_uuid() {")) + end = next(i for i, line in enumerate(lines[start:], start) if line == "}") + fn = "\n".join(lines[start : end + 1]) + + def uuid_for(leg: str, version: str) -> str: + proc = subprocess.run( + ["bash", "-c", f'{fn}\nleg_session_uuid "$1" "$2"', "_", leg, version], + capture_output=True, + text=True, + check=True, + ) + return proc.stdout.strip() + + a = uuid_for("baremetal-vllm-3h", "1.0.0.dev1+ci") + assert re.fullmatch(r"[0-9a-f]{8}-[0-9a-f]{4}-4[0-9a-f]{3}-8[0-9a-f]{3}-[0-9a-f]{12}", a) + assert a == uuid_for("baremetal-vllm-3h", "1.0.0.dev1+ci") # stable across turns + # Distinct per leg (four legs share the docker host) and per run. + assert a != uuid_for("baremetal-vllm-12h", "1.0.0.dev1+ci") + assert a != uuid_for("baremetal-vllm-3h", "1.0.0.dev2+ci") + + +def test_re_drive_does_not_extend_the_hard_deadline(script: str) -> None: + """The stall grace restarts per turn, but the pod-deadline clock must not. + + bootstrap's own deadline has to stay below the SaFE pod timeout, or SaFE pre-empts + the pod mid-wait and the clean failure path is lost. + """ + body = script.split("run_leg() {", 1)[1] + assert body.count('start_ts="$(date +%s)"') == 1 + assert 'grace_ts="$(date +%s)"' in body + assert "elapsed=$(( now - start_ts ))" in body + assert 'idle="$(leg_idle_s "$root" "$grace_ts" "$now")"' in body + + +def test_demo_prompts_forbid_ending_the_turn_early(script: str) -> None: + prompts = _BOOTSTRAP.parent / "prompts" / "pre-release" # type: ignore[union-attr] + for hours in (3, 12): + text = (prompts / f"demo-{hours}h.md").read_text(encoding="utf-8") + assert "single non-interactive turn" in text + assert "setsid nohup" in text + assert "state.json" in text + + +def test_dockerd_never_runs_on_vfs(script: str) -> None: + """vfs copies every layer in full and evicted the host pod twice (200Gi, 1792Gi). + + Only deduplicating drivers may be attempted, and a leg with none available has to + fail rather than quietly reproduce the eviction. + """ + assert "--storage-driver=vfs" not in script + drivers = next(line for line in script.splitlines() if line.startswith("DOCKER_DRIVERS=")) + assert "overlay2" in drivers + assert "vfs" not in drivers