[codex] Fix safe student checkout and idle routing

k8s/entrypoint-student.sh

@@ -105,16 +105,87 @@ SLEEP_TIME_S="${SENPAI_STUDENT_POLL_INTERVAL_S:-${SENPAI_POLL_INTERVAL_S:-600}}"
 POLL_JITTER_S="${SENPAI_STUDENT_POLL_JITTER_S:-${SENPAI_POLL_JITTER_S:-120}}"
 MAX_TURNS=100000
 
+quarantine_dirty_target_worktree() {
+    local context="$1" status ts message
+    status=$(git status --porcelain --untracked-files=all)
+    [ -z "$status" ] && return 0
+
+    ts=$(date -u +%Y%m%dT%H%M%SZ)
+    message="senpai-${STUDENT_NAME}-${ts}: ${context}"
+    echo "WARN: dirty target worktree before ${context}; preserving changes in git stash: ${message}"
+    git status --short
+    git stash push --include-untracked -m "$message"
+}
+
+mark_advisor_refresh_failure() {
+    local reason="$1" marker="STUDENT-ADVISOR-REFRESH-FAILED" body assigned_json
+    body="${marker}: student:${STUDENT_NAME} could not refresh advisor branch ${ADVISOR_BRANCH} in ${PROBLEM_DIR}. ${reason}. Dirty or untracked local work was preserved in a timestamped git stash if present; the pod will retry after the poll sleep."
+    echo "ERROR: ${body}" >&2
+
+    assigned_json=$(student_poll_for_work "$STUDENT_NAME" 2>/dev/null || printf '[]')
+    printf '%s' "$assigned_json" | python3 -c 'import json,sys; print("\n".join(str(pr["number"]) for pr in json.load(sys.stdin)))' |
+    while IFS= read -r pr_number; do
+        [ -n "$pr_number" ] || continue
+        if ! pr_issue_comments "$pr_number" 2>/dev/null | grep -Fq "${marker}: student:${STUDENT_NAME}"; then
+            comment_on_pr "$pr_number" "$body" || true
+        fi
+    done
+}
+
+safe_checkout_advisor_branch() {
+    cd "$WORKDIR/$PROBLEM_DIR"
+    if ! quarantine_dirty_target_worktree "refresh advisor branch ${ADVISOR_BRANCH}"; then
+        mark_advisor_refresh_failure "Failed to preserve the existing worktree before refreshing ${ADVISOR_BRANCH}"
+        return 1
+    fi
+
+    if ! git checkout "$ADVISOR_BRANCH"; then
+        mark_advisor_refresh_failure "git checkout ${ADVISOR_BRANCH} failed"
+        return 1
+    fi
+    if ! git pull --ff-only origin "$ADVISOR_BRANCH"; then
+        mark_advisor_refresh_failure "git pull --ff-only origin ${ADVISOR_BRANCH} failed"
+        return 1
+    fi
+}
+
+mark_assignment_checkout_failure() {
+    local pr_number="$1" head_ref="$2" reason="$3" marker="STUDENT-CHECKOUT-FAILED" body
+    body="${marker}: student:${STUDENT_NAME} could not prepare branch ${head_ref} in ${PROBLEM_DIR}. ${reason}. Dirty or untracked local work was preserved in a timestamped git stash if present; the pod will retry after the poll sleep."
+    echo "ERROR: ${body}" >&2
+    if ! pr_issue_comments "$pr_number" 2>/dev/null | grep -Fq "${marker}: student:${STUDENT_NAME}"; then
+        comment_on_pr "$pr_number" "$body" || true
+    fi
+}
+
+safe_checkout_assignment() {
+    local pr_number="$1" head_ref="$2"
+    quarantine_dirty_target_worktree "checkout assignment #${pr_number} ${head_ref}" || {
+        mark_assignment_checkout_failure "$pr_number" "$head_ref" "Failed to preserve the existing worktree before checkout"
+        return 1
+    }
+
+    if ! git fetch origin "$head_ref"; then
+        mark_assignment_checkout_failure "$pr_number" "$head_ref" "git fetch origin ${head_ref} failed"
+        return 1
+    fi
+    if ! git checkout -B "$head_ref" FETCH_HEAD; then
+        mark_assignment_checkout_failure "$pr_number" "$head_ref" "git checkout -B ${head_ref} FETCH_HEAD failed"
+        return 1
+    fi
+}
+
 ITERATION=0
 while true; do
     ITERATION=$((ITERATION + 1))
     LOGFILE="$LOGDIR/iteration_${ITERATION}_$(date +%Y%m%d_%H%M%S).log"
     echo "=== Student Heartbeat iteration $ITERATION ($(date)) ==="
 
     # Return to latest advisor branch so student starts from the current baseline
-    cd "$WORKDIR/$PROBLEM_DIR"
-    git checkout "$ADVISOR_BRANCH" 2>/dev/null || true
-    git pull --ff-only origin "$ADVISOR_BRANCH" 2>/dev/null || true
+    if ! safe_checkout_advisor_branch; then
+        senpai_sleep_with_jitter "$SLEEP_TIME_S" "$POLL_JITTER_S"
+        continue
+    fi
 
     # Overwrite CLAUDE.md with the student role instructions — git checkout/pull clobbers it with the developer copy.
     # Whitelist vars so envsubst substitutes pod env vars but leaves Claude Code runtime vars
@@ -135,8 +206,11 @@ while true; do
 
     if [ "$ASSIGNED_COUNT" -eq 1 ]; then
         ASSIGNED_HEAD=$(printf '%s' "$ASSIGNED_JSON" | python3 -c 'import json,sys; print(json.load(sys.stdin)[0]["headRefName"])')
-        git fetch origin "$ASSIGNED_HEAD"
-        git checkout -B "$ASSIGNED_HEAD" FETCH_HEAD
+        ASSIGNED_PR=$(printf '%s' "$ASSIGNED_JSON" | python3 -c 'import json,sys; print(json.load(sys.stdin)[0]["number"])')
+        if ! safe_checkout_assignment "$ASSIGNED_PR" "$ASSIGNED_HEAD"; then
+            senpai_sleep_with_jitter "$SLEEP_TIME_S" "$POLL_JITTER_S"
+            continue
+        fi
     fi
 
     # --- Build triage info ---

plugins/senpai/scripts/senpai-gh.sh

@@ -991,6 +991,7 @@ duplicate_wips = {
 }
 
 conflict_re = re.compile(r"merge conflict|rebase conflict|cannot automatically merge|can.t automatically merge|conflicts? with", re.I)
+checkout_failure_re = re.compile(r"STUDENT-(?:CHECKOUT|ADVISOR-REFRESH)-FAILED", re.I)
 prs_requiring_advisor_action = []
 for pr in prs:
     labels, students, routed_students, unknown_students = metadata[pr["number"]]
@@ -1018,6 +1019,8 @@ for pr in prs:
     comment_text = "\n".join(comment.get("body", "") for comment in comments_by_pr.get(pr["number"], []))
     if conflict_re.search(comment_text):
         reasons.append("merge_conflict_comment")
+    if checkout_failure_re.search(comment_text):
+        reasons.append("student_checkout_failed")
     if pr.get("isDraft") and "status:review" in labels:
         reasons.append("draft_but_claimed_mergeable")
     if reasons:
@@ -1059,16 +1062,44 @@ print(json.dumps([
 }
 
 # Compute which students are idle (have no status:wip PR).
+# In cluster mode, a student is idle only when the GitHub assignment state says
+# it has no WIP PR and Kubernetes says its worker pod is running and ready.
+# Local/dev shells without Kubernetes keep the historical assignment-only check.
 # Expects a comma-separated student list and the advisor branch.
 # Returns a JSON array of idle student names.
 #   list_idle_students <student_names_csv> <branch>
+student_worker_liveness_available() {
+    [ -n "${RESEARCH_TAG:-}" ] && command -v kubectl >/dev/null 2>&1
+}
+
+student_worker_is_available() {
+    local student="$1" tag="${RESEARCH_TAG:-}" pod
+    student_worker_liveness_available || return 0
+    [ -n "$student" ] && [ -n "$tag" ] || return 1
+
+    pod=$(
+        kubectl get pods -l "app=senpai,research-tag=${tag},student=${student}" \
+            -o jsonpath='{range .items[?(@.status.phase=="Running")]}{.metadata.name}{"\t"}{range .status.conditions[?(@.type=="Ready")]}{.status}{end}{"\n"}{end}' 2>/dev/null |
+            awk '$2 == "True" {print $1; exit}'
+    ) || return 1
+    [ -n "$pod" ]
+}
+
 list_idle_students() {
     local students_csv="$1" branch="$2"
-    local all_prs
+    local all_prs available_students="" student
+    local -a students
     all_prs=$(rest_labeled_pull_details "${branch},status:wip")
+    IFS=',' read -r -a students <<< "$students_csv"
+    for student in "${students[@]}"; do
+        student="${student//[[:space:]]/}"
+        [ -n "$student" ] || continue
+        student_worker_is_available "$student" || continue
+        available_students+="${student}"$'\n'
+    done
     printf '%s' "$all_prs" | python3 -c "
 import json, sys
-students = [s.strip() for s in sys.argv[1].split(',') if s.strip()]
+students = [s.strip() for s in sys.argv[1].splitlines() if s.strip()]
 prs = json.loads(sys.stdin.read())
 busy = set()
 for pr in prs:
@@ -1077,5 +1108,5 @@ for pr in prs:
         if name.startswith('student:'):
             busy.add(name.split(':', 1)[1])
 print(json.dumps([s for s in students if s not in busy]))
-" "$students_csv"
+" "$available_students"
 }

plugins/senpai/skills/senpai-gh/SKILL.md

@@ -69,7 +69,7 @@ The default repo for `gh` is set via the injected `GH_REPO` env var, so no `--re
 | `list_prs_requiring_advisor_action <branch> [stale_wip_seconds] [student_names_csv]` | List open branch PRs requiring advisor action, with `reasons` such as stale WIP, duplicate student WIP, missing, unknown, or unroutable student labels, or rebase conflicts. Returns JSON array. |
 | `list_all_prs <branch>` | List all open PRs on a branch (any status). Returns JSON array. |
 | `student_poll_for_work <student_name> [branch]` | List branch-scoped WIP PRs assigned to a student. Returns JSON array. |
-| `list_idle_students <names_csv> <branch>` | Print names of students with no `status:wip` PR, one per line. |
+| `list_idle_students <names_csv> <branch>` | Return JSON names of students with no `status:wip` PR whose worker pod is running and ready when Kubernetes liveness is available. |
 | `pr_body <pr#>` | Read one PR body through REST. Returns JSON. |
 | `pr_issue_comments <pr#>` | Read all PR conversation comments through REST pagination. Returns JSON array. |
 | `pr_reviews <pr#>` | Read all PR review submissions through REST pagination. Returns JSON array. |

tests/test_senpai_gh_idle.sh

@@ -0,0 +1,52 @@
+#!/bin/bash
+
+set -e
+set -o pipefail
+
+WORKDIR="$(cd "$(dirname "$0")/.." && pwd)"
+cd "$WORKDIR"
+
+source "$WORKDIR/plugins/senpai/scripts/senpai-gh.sh"
+
+assert_json_eq() {
+    local expected="$1" actual="$2"
+    python3 - "$expected" "$actual" <<'PY'
+import json
+import sys
+
+expected = json.loads(sys.argv[1])
+actual = json.loads(sys.argv[2])
+if actual != expected:
+    raise SystemExit(f"expected {expected!r}, got {actual!r}")
+PY
+}
+
+rest_labeled_pull_details() {
+    printf '%s\n' '[{"number":1,"labels":[{"name":"status:wip"},{"name":"student:busy"}]}]'
+}
+
+mock_bin=$(mktemp -d "${TMPDIR:-/tmp}/senpai-idle-test.XXXXXX")
+trap 'rm -rf "$mock_bin"' EXIT
+
+cat > "$mock_bin/kubectl" <<'SH'
+#!/bin/sh
+
+case "$*" in
+    *student=busy*) printf 'busy-pod\tTrue\n' ;;
+    *student=ready*) printf 'ready-pod\tTrue\n' ;;
+    *student=notready*) printf 'notready-pod\tFalse\n' ;;
+    *student=dead*) : ;;
+    *) exit 1 ;;
+esac
+SH
+chmod +x "$mock_bin/kubectl"
+
+export PATH="$mock_bin:$PATH"
+export RESEARCH_TAG="test"
+
+actual=$(list_idle_students "busy,ready,notready,dead" "advisor")
+assert_json_eq '["ready"]' "$actual"
+
+unset RESEARCH_TAG
+actual=$(list_idle_students "busy,ready,notready,dead" "advisor")
+assert_json_eq '["ready","notready","dead"]' "$actual"