[codex] Support packing students into GPU pods

.agents/skills/senpai-status-check/SKILL.md

@@ -114,16 +114,39 @@ PY
 
 # Fleet overview and real training process sweep
 kubectl --context pai-2 get pods -l app=senpai -o wide
-for podref in $(kubectl --context pai-2 get pods -l app=senpai,role=student -o name | sort); do
-  pod=${podref#pod/}
-  student=${pod#senpai-}; student=${student%-*}; student=${student%-*}
-  out=$(kubectl --context pai-2 exec "$pod" -- sh -lc "ps -eo pid,etime,comm,args | awk '\$3 ~ /python/ && /train.py/ {print; exit}'" 2>/dev/null || true)
+kubectl --context pai-2 get pods -l app=senpai,role=student -o json > /tmp/student_pods.json
+uv run python - <<'PY' </tmp/student_pods.json >/tmp/student_runtime.tsv
+import json, sys
+for item in json.load(sys.stdin).get("items", []):
+    meta = item.get("metadata", {})
+    labels = meta.get("labels", {})
+    annotations = meta.get("annotations", {})
+    pod = meta.get("name", "")
+    names = [n.strip() for n in annotations.get("senpai/student-names", "").split(",") if n.strip()]
+    if not names and labels.get("student"):
+        names = [labels["student"]]
+    grouped = "student" not in labels
+    for student in names:
+        runner = f"/workspace/senpai-{student}" if grouped else "/workspace/senpai"
+        target = f"{runner}/target"
+        home = f"/workspace/home-{student}" if grouped else "/root"
+        print(f"{student}\t{pod}\t{target}\t{home}\t{runner}")
+PY
+while IFS="$(printf '\t')" read -r student pod target home runner; do
+  out=$(kubectl --context pai-2 exec "$pod" -- sh -lc '
+    target="${1%/}"
+    runner="${2%/}"
+    . "$runner/k8s/senpai-processes.sh"
+    pid=$(senpai_training_pids_for_target "$target" target | head -1)
+    [ -n "$pid" ] || exit 0
+    ps -p "$pid" -o pid=,etime=,comm=,args= 2>/dev/null
+  ' sh "$target" "$runner" 2>/dev/null || true)
   if [ -n "$out" ]; then
     printf "%-10s PYTRAIN %s\n" "$student" "$(printf "%s" "$out" | sed 's/^[[:space:]]*//; s/[[:space:]][[:space:]]*/ /g' | cut -c1-180)"
   else
     printf "%-10s NO_PYTRAIN\n" "$student"
   fi
-done
+done </tmp/student_runtime.tsv
 
 # Advisor state and raw log locations
 gh api 'repos/wandb/senpai/contents/research/CURRENT_RESEARCH_STATE.md?ref=radford' \
@@ -132,12 +155,11 @@ kubectl --context pai-2 exec deploy/senpai-advisor -- sh -lc \
   "find /root/.claude -type f -name '*.jsonl' -printf '%T@ %p %s\n' | sort -nr | head -10"
 
 # Student raw log locations and latest tails
-for podref in $(kubectl --context pai-2 get pods -l app=senpai,role=student -o name | sort); do
-  pod=${podref#pod/}
-  echo "### $pod"
+while IFS="$(printf '\t')" read -r student pod target home runner; do
+  echo "### $student ($pod)"
   kubectl --context pai-2 exec "$pod" -- sh -lc \
-    "latest=\$(find /root/.claude -type f -name '*.jsonl' -printf '%T@ %p\n' | sort -nr | awk 'NR==1{sub(/^[^ ]+ /,\"\"); print}'); [ -n \"\$latest\" ] && tail -40 \"\$latest\""
-done
+    "latest=\$(find \"\$1/.claude\" -type f -name '*.jsonl' -printf '%T@ %p\n' 2>/dev/null | sort -nr | awk 'NR==1{sub(/^[^ ]+ /,\"\"); print}'); [ -n \"\$latest\" ] && tail -40 \"\$latest\"" sh "$home"
+done </tmp/student_runtime.tsv
 ```
 
 ## W&B metric scan pattern

.claude/skills/senpai-status-check/SKILL.md

@@ -114,16 +114,39 @@ PY
 
 # Fleet overview and real training process sweep
 kubectl --context pai-2 get pods -l app=senpai -o wide
-for podref in $(kubectl --context pai-2 get pods -l app=senpai,role=student -o name | sort); do
-  pod=${podref#pod/}
-  student=${pod#senpai-}; student=${student%-*}; student=${student%-*}
-  out=$(kubectl --context pai-2 exec "$pod" -- sh -lc "ps -eo pid,etime,comm,args | awk '\$3 ~ /python/ && /train.py/ {print; exit}'" 2>/dev/null || true)
+kubectl --context pai-2 get pods -l app=senpai,role=student -o json > /tmp/student_pods.json
+uv run python - <<'PY' </tmp/student_pods.json >/tmp/student_runtime.tsv
+import json, sys
+for item in json.load(sys.stdin).get("items", []):
+    meta = item.get("metadata", {})
+    labels = meta.get("labels", {})
+    annotations = meta.get("annotations", {})
+    pod = meta.get("name", "")
+    names = [n.strip() for n in annotations.get("senpai/student-names", "").split(",") if n.strip()]
+    if not names and labels.get("student"):
+        names = [labels["student"]]
+    grouped = "student" not in labels
+    for student in names:
+        runner = f"/workspace/senpai-{student}" if grouped else "/workspace/senpai"
+        target = f"{runner}/target"
+        home = f"/workspace/home-{student}" if grouped else "/root"
+        print(f"{student}\t{pod}\t{target}\t{home}\t{runner}")
+PY
+while IFS="$(printf '\t')" read -r student pod target home runner; do
+  out=$(kubectl --context pai-2 exec "$pod" -- sh -lc '
+    target="${1%/}"
+    runner="${2%/}"
+    . "$runner/k8s/senpai-processes.sh"
+    pid=$(senpai_training_pids_for_target "$target" target | head -1)
+    [ -n "$pid" ] || exit 0
+    ps -p "$pid" -o pid=,etime=,comm=,args= 2>/dev/null
+  ' sh "$target" "$runner" 2>/dev/null || true)
   if [ -n "$out" ]; then
     printf "%-10s PYTRAIN %s\n" "$student" "$(printf "%s" "$out" | sed 's/^[[:space:]]*//; s/[[:space:]][[:space:]]*/ /g' | cut -c1-180)"
   else
     printf "%-10s NO_PYTRAIN\n" "$student"
   fi
-done
+done </tmp/student_runtime.tsv
 
 # Advisor state and raw log locations
 gh api 'repos/wandb/senpai/contents/research/CURRENT_RESEARCH_STATE.md?ref=radford' \
@@ -132,12 +155,11 @@ kubectl --context pai-2 exec deploy/senpai-advisor -- sh -lc \
   "find /root/.claude -type f -name '*.jsonl' -printf '%T@ %p %s\n' | sort -nr | head -10"
 
 # Student raw log locations and latest tails
-for podref in $(kubectl --context pai-2 get pods -l app=senpai,role=student -o name | sort); do
-  pod=${podref#pod/}
-  echo "### $pod"
+while IFS="$(printf '\t')" read -r student pod target home runner; do
+  echo "### $student ($pod)"
   kubectl --context pai-2 exec "$pod" -- sh -lc \
-    "latest=\$(find /root/.claude -type f -name '*.jsonl' -printf '%T@ %p\n' | sort -nr | awk 'NR==1{sub(/^[^ ]+ /,\"\"); print}'); [ -n \"\$latest\" ] && tail -40 \"\$latest\""
-done
+    "latest=\$(find \"\$1/.claude\" -type f -name '*.jsonl' -printf '%T@ %p\n' 2>/dev/null | sort -nr | awk 'NR==1{sub(/^[^ ]+ /,\"\"); print}'); [ -n \"\$latest\" ] && tail -40 \"\$latest\"" sh "$home"
+done </tmp/student_runtime.tsv
 ```
 
 ## W&B metric scan pattern

README.md

@@ -118,13 +118,25 @@ student_assignment_drift_grace_s: 1800
 n_students: 4
 student_prefix: ""
 gpus_per_student: 8
+students_per_gpu_pod: 1
 cpu_per_gpu: 15
 memory_gi_per_gpu: 120
 preflight_only: false
 ```
 
 `launch.py` reads this via `simple_parsing` — every field can be overridden on the CLI.
 
+### Student pod packing
+
+By default, `students_per_gpu_pod: 1` preserves the normal deployment topology:
+one logical student loop per GPU pod. Set `students_per_gpu_pod` above 1 to
+pack multiple logical students into each pod while still requesting
+`gpus_per_student` GPUs for that pod. This is useful for single-GPU targets
+where the advisor coordinates heavy runs and students can still prepare
+launchers, inspect logs, run smoke checks, or take turns on the shared GPU.
+In packed mode, `gpus_per_student` is the shared pod GPU request, not a
+per-logical-student allocation.
+
 ### Responsiveness knobs
 
 Advisor and student entrypoints poll GitHub before invoking Claude Code. By

k8s/entrypoint-advisor.sh

@@ -114,6 +114,9 @@ mkdir -p "$LOGDIR"
 # --- Install checked-in Claude Code config into user scope ---
 mkdir -p "$HOME/.claude"
 cp -a "$WORKDIR/.claude/." "$HOME/.claude/"
+if ! python3 -m json.tool "$HOME/.claude.json" >/dev/null 2>&1; then
+    printf '{}\n' > "$HOME/.claude.json"
+fi
 
 echo "=== Claude config installed ==="
 ls "$HOME/.claude/skills/wandb-primary/SKILL.md" "$HOME/.claude/agents/researcher-agent.md"
@@ -147,7 +150,7 @@ if [ -n "${EXTRA_INSTRUCTIONS_B64:-}" ]; then
 fi
 
 # Add "$KEY_INFO" (reminder of student names etc) to PROMPT
-KEY_INFO=$'\n\n Key information:\n\n Students: '"$STUDENT_NAMES"' | GPUs per Student: '"$GPUS_PER_STUDENT"' | Tag: '"$RESEARCH_TAG"' | Target repo: '"$GH_REPO"' | Target base branch: '"${TARGET_REPO_BRANCH:-<default>}"' | Advisor Branch: '"$ADVISOR_BRANCH"' | W&B entity/project: '"$WANDB_ENTITY"'/'"$WANDB_PROJECT"$'\n'
+KEY_INFO=$'\n\n Key information:\n\n Students: '"$STUDENT_NAMES"' | GPU request: '"$GPUS_PER_STUDENT"' | Tag: '"$RESEARCH_TAG"' | Target repo: '"$GH_REPO"' | Target base branch: '"${TARGET_REPO_BRANCH:-<default>}"' | Advisor Branch: '"$ADVISOR_BRANCH"' | W&B entity/project: '"$WANDB_ENTITY"'/'"$WANDB_PROJECT"$'\n'
 FULL_PROMPT="${PROMPT}"$'\n\n'"${KEY_INFO}"
 
 # Heartbeat prompt for polling

k8s/entrypoint-student-group.sh

@@ -0,0 +1,83 @@
+#!/bin/bash
+
+# SPDX-FileCopyrightText: 2026 CoreWeave, Inc.
+# SPDX-License-Identifier: Apache-2.0
+# SPDX-PackageName: senpai
+
+set -e
+set -o pipefail
+
+echo "=== Senpai Student Group: ${STUDENT_GROUP_NAME:-group} ==="
+echo "Students: ${STUDENT_NAMES}"
+
+repo_auth_url="$(printf '%s' "$REPO_URL" | sed "s#https://github.com/#https://${GITHUB_TOKEN}@github.com/#")"
+
+student_extra_instructions() {
+    local student="$1"
+    [ -n "${STUDENT_EXTRA_INSTRUCTIONS_B64_JSON_B64:-}" ] || return 0
+    printf '%s' "$STUDENT_EXTRA_INSTRUCTIONS_B64_JSON_B64" | base64 -d |
+        STUDENT_FOR_EXTRA="$student" python3 -c '
+import json
+import os
+import sys
+
+mapping = json.load(sys.stdin)
+print(mapping.get(os.environ["STUDENT_FOR_EXTRA"], ""))
+'
+}
+
+terminate_group() {
+    local pid
+    echo "=== Terminating grouped student processes ==="
+    for pid in "${student_pids[@]:-}"; do
+        [ -n "$pid" ] || continue
+        kill -TERM "$pid" 2>/dev/null || true
+    done
+    wait 2>/dev/null || true
+}
+
+IFS=',' read -r -a raw_students <<< "$STUDENT_NAMES"
+students=()
+for raw_student in "${raw_students[@]}"; do
+    student="$(printf '%s' "$raw_student" | xargs)"
+    [ -n "$student" ] && students+=("$student")
+done
+
+if [ "${#students[@]}" -eq 0 ]; then
+    echo "ERROR: STUDENT_NAMES is empty" >&2
+    exit 2
+fi
+
+student_pids=()
+trap terminate_group INT TERM
+
+for student in "${students[@]}"; do
+    (
+        set -e
+        export STUDENT_NAME="$student"
+        export WORKDIR="/workspace/senpai-${student}"
+        export HOME="/workspace/home-${student}"
+        export SENPAI_GROUPED_STUDENT_POD=1
+        export EXTRA_INSTRUCTIONS_B64
+
+        EXTRA_INSTRUCTIONS_B64="$(student_extra_instructions "$student")"
+
+        mkdir -p "$HOME"
+        git clone --branch "$REPO_BRANCH" --single-branch --depth 1 --no-tags "$repo_auth_url" "$WORKDIR"
+        git -C "$WORKDIR" remote set-url origin "$REPO_URL"
+        cd "$WORKDIR"
+        bash k8s/entrypoint-student.sh
+    ) &
+    pid="$!"
+    student_pids+=("$pid")
+    echo "Started student ${student} as pid ${pid}"
+done
+
+set +e
+wait -n "${student_pids[@]}"
+status=$?
+set -e
+
+echo "=== A grouped student process exited with status ${status}; restarting the group pod ==="
+terminate_group
+exit "$status"

k8s/entrypoint-student.sh

@@ -7,14 +7,18 @@
 set -e
 set -o pipefail
 
-WORKDIR="/workspace/senpai"
+WORKDIR="${WORKDIR:-/workspace/senpai}"
 GH_HISTORY_SCOPE="${GH_HISTORY_SCOPE:-branch}"
 TARGET_REPO_BRANCH="${TARGET_REPO_BRANCH:-}"
 export SENPAI_ROLE="student"
 export TARGET_WORKDIR="$WORKDIR/$PROBLEM_DIR"
 GIT_CREDENTIAL_FILE="$WORKDIR/.git-credentials"
 SENPAI_PLUGIN="$WORKDIR/plugins/senpai"
 
+if [ -x /usr/bin/nvidia-smi ]; then
+    export PATH="/usr/bin:$PATH"
+fi
+
 echo "=== Senpai Student: $STUDENT_NAME ==="
 echo "Runner repo:  $REPO_URL (branch: $REPO_BRANCH)"
 echo "Target repo:  $TARGET_REPO_URL (base branch: ${TARGET_REPO_BRANCH:-<default>}; advisor branch: $ADVISOR_BRANCH)"
@@ -62,6 +66,9 @@ fi
 # --- Install checked-in Claude Code config into user scope ---
 mkdir -p "$HOME/.claude"
 cp -a "$WORKDIR/.claude/." "$HOME/.claude/"
+if ! python3 -m json.tool "$HOME/.claude.json" >/dev/null 2>&1; then
+    printf '{}\n' > "$HOME/.claude.json"
+fi
 
 echo "=== Claude config installed ==="
 ls "$HOME/.claude/skills/wandb-primary/SKILL.md" "$HOME/.claude/agents/researcher-agent.md"
@@ -91,7 +98,7 @@ if [ -n "${EXTRA_INSTRUCTIONS_B64:-}" ]; then
     PROMPT="${PROMPT}"$'\n\n# Finally, some additional instructions\n\n'"$(printf '%s' "$EXTRA_INSTRUCTIONS_B64" | base64 -d)"
 fi
 
-KEY_INFO=$'\n\nKey information:\n\nStudent: '"$STUDENT_NAME"' | GPUs per Student: '"$GPUS_PER_STUDENT"' | Target repo: '"$GH_REPO"' | Target base branch: '"${TARGET_REPO_BRANCH:-<default>}"' | Advisor Branch: '"$ADVISOR_BRANCH"' | W&B entity/project: '"$WANDB_ENTITY"'/'"$WANDB_PROJECT"$'\n'
+KEY_INFO=$'\n\nKey information:\n\nStudent: '"$STUDENT_NAME"' | GPU request: '"$GPUS_PER_STUDENT"' | Target repo: '"$GH_REPO"' | Target base branch: '"${TARGET_REPO_BRANCH:-<default>}"' | Advisor Branch: '"$ADVISOR_BRANCH"' | W&B entity/project: '"$WANDB_ENTITY"'/'"$WANDB_PROJECT"$'\n'
 FULL_PROMPT="${PROMPT}"$'\n\n'"${KEY_INFO}"
 
 HEARTBEAT_PROMPT="Continue your student loop using the assigned PRs and GitHub issues listed in the Student research state below. The entrypoint owns assignment polling; do not start persistent GitHub polling monitors. For active training, use sparse wakeups plus training_log_status; do not stream per-epoch logs into Monitor."

k8s/install-weave-cc-plugin.sh

@@ -18,6 +18,11 @@
 # already-token-authenticated HTTPS form set up in the deployment entrypoint.
 git config --global url."https://${GITHUB_TOKEN}@github.com/".insteadOf "git@github.com:"
 
+mkdir -p "$HOME/.weave_claude_plugin"
+if ! python3 -m json.tool "$HOME/.weave_claude_plugin/settings.json" >/dev/null 2>&1; then
+    rm -f "$HOME/.weave_claude_plugin/settings.json"
+fi
+
 WEAVE_PROJECT="${WANDB_ENTITY}/${WANDB_PROJECT}" \
     weave-claude-plugin install --non-interactive