diff --git a/k8s/entrypoint-student.sh b/k8s/entrypoint-student.sh index 2d4fdc13f..fca29226a 100644 --- a/k8s/entrypoint-student.sh +++ b/k8s/entrypoint-student.sh @@ -105,6 +105,76 @@ SLEEP_TIME_S="${SENPAI_STUDENT_POLL_INTERVAL_S:-${SENPAI_POLL_INTERVAL_S:-600}}" POLL_JITTER_S="${SENPAI_STUDENT_POLL_JITTER_S:-${SENPAI_POLL_JITTER_S:-120}}" MAX_TURNS=100000 +quarantine_dirty_target_worktree() { + local context="$1" status ts message + status=$(git status --porcelain --untracked-files=all) + [ -z "$status" ] && return 0 + + ts=$(date -u +%Y%m%dT%H%M%SZ) + message="senpai-${STUDENT_NAME}-${ts}: ${context}" + echo "WARN: dirty target worktree before ${context}; preserving changes in git stash: ${message}" + git status --short + git stash push --include-untracked -m "$message" +} + +mark_advisor_refresh_failure() { + local reason="$1" marker="STUDENT-ADVISOR-REFRESH-FAILED" body assigned_json + body="${marker}: student:${STUDENT_NAME} could not refresh advisor branch ${ADVISOR_BRANCH} in ${PROBLEM_DIR}. ${reason}. Dirty or untracked local work was preserved in a timestamped git stash if present; the pod will retry after the poll sleep." + echo "ERROR: ${body}" >&2 + + assigned_json=$(student_poll_for_work "$STUDENT_NAME" 2>/dev/null || printf '[]') + printf '%s' "$assigned_json" | python3 -c 'import json,sys; print("\n".join(str(pr["number"]) for pr in json.load(sys.stdin)))' | + while IFS= read -r pr_number; do + [ -n "$pr_number" ] || continue + if ! pr_issue_comments "$pr_number" 2>/dev/null | grep -Fq "${marker}: student:${STUDENT_NAME}"; then + comment_on_pr "$pr_number" "$body" || true + fi + done +} + +safe_checkout_advisor_branch() { + cd "$WORKDIR/$PROBLEM_DIR" + if ! quarantine_dirty_target_worktree "refresh advisor branch ${ADVISOR_BRANCH}"; then + mark_advisor_refresh_failure "Failed to preserve the existing worktree before refreshing ${ADVISOR_BRANCH}" + return 1 + fi + + if ! git checkout "$ADVISOR_BRANCH"; then + mark_advisor_refresh_failure "git checkout ${ADVISOR_BRANCH} failed" + return 1 + fi + if ! git pull --ff-only origin "$ADVISOR_BRANCH"; then + mark_advisor_refresh_failure "git pull --ff-only origin ${ADVISOR_BRANCH} failed" + return 1 + fi +} + +mark_assignment_checkout_failure() { + local pr_number="$1" head_ref="$2" reason="$3" marker="STUDENT-CHECKOUT-FAILED" body + body="${marker}: student:${STUDENT_NAME} could not prepare branch ${head_ref} in ${PROBLEM_DIR}. ${reason}. Dirty or untracked local work was preserved in a timestamped git stash if present; the pod will retry after the poll sleep." + echo "ERROR: ${body}" >&2 + if ! pr_issue_comments "$pr_number" 2>/dev/null | grep -Fq "${marker}: student:${STUDENT_NAME}"; then + comment_on_pr "$pr_number" "$body" || true + fi +} + +safe_checkout_assignment() { + local pr_number="$1" head_ref="$2" + quarantine_dirty_target_worktree "checkout assignment #${pr_number} ${head_ref}" || { + mark_assignment_checkout_failure "$pr_number" "$head_ref" "Failed to preserve the existing worktree before checkout" + return 1 + } + + if ! git fetch origin "$head_ref"; then + mark_assignment_checkout_failure "$pr_number" "$head_ref" "git fetch origin ${head_ref} failed" + return 1 + fi + if ! git checkout -B "$head_ref" FETCH_HEAD; then + mark_assignment_checkout_failure "$pr_number" "$head_ref" "git checkout -B ${head_ref} FETCH_HEAD failed" + return 1 + fi +} + ITERATION=0 while true; do ITERATION=$((ITERATION + 1)) @@ -112,9 +182,10 @@ while true; do echo "=== Student Heartbeat iteration $ITERATION ($(date)) ===" # Return to latest advisor branch so student starts from the current baseline - cd "$WORKDIR/$PROBLEM_DIR" - git checkout "$ADVISOR_BRANCH" 2>/dev/null || true - git pull --ff-only origin "$ADVISOR_BRANCH" 2>/dev/null || true + if ! safe_checkout_advisor_branch; then + senpai_sleep_with_jitter "$SLEEP_TIME_S" "$POLL_JITTER_S" + continue + fi # Overwrite CLAUDE.md with the student role instructions — git checkout/pull clobbers it with the developer copy. # Whitelist vars so envsubst substitutes pod env vars but leaves Claude Code runtime vars @@ -135,8 +206,11 @@ while true; do if [ "$ASSIGNED_COUNT" -eq 1 ]; then ASSIGNED_HEAD=$(printf '%s' "$ASSIGNED_JSON" | python3 -c 'import json,sys; print(json.load(sys.stdin)[0]["headRefName"])') - git fetch origin "$ASSIGNED_HEAD" - git checkout -B "$ASSIGNED_HEAD" FETCH_HEAD + ASSIGNED_PR=$(printf '%s' "$ASSIGNED_JSON" | python3 -c 'import json,sys; print(json.load(sys.stdin)[0]["number"])') + if ! safe_checkout_assignment "$ASSIGNED_PR" "$ASSIGNED_HEAD"; then + senpai_sleep_with_jitter "$SLEEP_TIME_S" "$POLL_JITTER_S" + continue + fi fi # --- Build triage info --- diff --git a/plugins/senpai/scripts/senpai-gh.sh b/plugins/senpai/scripts/senpai-gh.sh index 6632a6279..2ebef35b2 100644 --- a/plugins/senpai/scripts/senpai-gh.sh +++ b/plugins/senpai/scripts/senpai-gh.sh @@ -991,6 +991,7 @@ duplicate_wips = { } conflict_re = re.compile(r"merge conflict|rebase conflict|cannot automatically merge|can.t automatically merge|conflicts? with", re.I) +checkout_failure_re = re.compile(r"STUDENT-(?:CHECKOUT|ADVISOR-REFRESH)-FAILED", re.I) prs_requiring_advisor_action = [] for pr in prs: labels, students, routed_students, unknown_students = metadata[pr["number"]] @@ -1018,6 +1019,8 @@ for pr in prs: comment_text = "\n".join(comment.get("body", "") for comment in comments_by_pr.get(pr["number"], [])) if conflict_re.search(comment_text): reasons.append("merge_conflict_comment") + if checkout_failure_re.search(comment_text): + reasons.append("student_checkout_failed") if pr.get("isDraft") and "status:review" in labels: reasons.append("draft_but_claimed_mergeable") if reasons: @@ -1059,16 +1062,44 @@ print(json.dumps([ } # Compute which students are idle (have no status:wip PR). +# In cluster mode, a student is idle only when the GitHub assignment state says +# it has no WIP PR and Kubernetes says its worker pod is running and ready. +# Local/dev shells without Kubernetes keep the historical assignment-only check. # Expects a comma-separated student list and the advisor branch. # Returns a JSON array of idle student names. # list_idle_students +student_worker_liveness_available() { + [ -n "${RESEARCH_TAG:-}" ] && command -v kubectl >/dev/null 2>&1 +} + +student_worker_is_available() { + local student="$1" tag="${RESEARCH_TAG:-}" pod + student_worker_liveness_available || return 0 + [ -n "$student" ] && [ -n "$tag" ] || return 1 + + pod=$( + kubectl get pods -l "app=senpai,research-tag=${tag},student=${student}" \ + -o jsonpath='{range .items[?(@.status.phase=="Running")]}{.metadata.name}{"\t"}{range .status.conditions[?(@.type=="Ready")]}{.status}{end}{"\n"}{end}' 2>/dev/null | + awk '$2 == "True" {print $1; exit}' + ) || return 1 + [ -n "$pod" ] +} + list_idle_students() { local students_csv="$1" branch="$2" - local all_prs + local all_prs available_students="" student + local -a students all_prs=$(rest_labeled_pull_details "${branch},status:wip") + IFS=',' read -r -a students <<< "$students_csv" + for student in "${students[@]}"; do + student="${student//[[:space:]]/}" + [ -n "$student" ] || continue + student_worker_is_available "$student" || continue + available_students+="${student}"$'\n' + done printf '%s' "$all_prs" | python3 -c " import json, sys -students = [s.strip() for s in sys.argv[1].split(',') if s.strip()] +students = [s.strip() for s in sys.argv[1].splitlines() if s.strip()] prs = json.loads(sys.stdin.read()) busy = set() for pr in prs: @@ -1077,5 +1108,5 @@ for pr in prs: if name.startswith('student:'): busy.add(name.split(':', 1)[1]) print(json.dumps([s for s in students if s not in busy])) -" "$students_csv" +" "$available_students" } diff --git a/plugins/senpai/skills/senpai-gh/SKILL.md b/plugins/senpai/skills/senpai-gh/SKILL.md index cd8f88636..b10e9e6e1 100644 --- a/plugins/senpai/skills/senpai-gh/SKILL.md +++ b/plugins/senpai/skills/senpai-gh/SKILL.md @@ -69,7 +69,7 @@ The default repo for `gh` is set via the injected `GH_REPO` env var, so no `--re | `list_prs_requiring_advisor_action [stale_wip_seconds] [student_names_csv]` | List open branch PRs requiring advisor action, with `reasons` such as stale WIP, duplicate student WIP, missing, unknown, or unroutable student labels, or rebase conflicts. Returns JSON array. | | `list_all_prs ` | List all open PRs on a branch (any status). Returns JSON array. | | `student_poll_for_work [branch]` | List branch-scoped WIP PRs assigned to a student. Returns JSON array. | -| `list_idle_students ` | Print names of students with no `status:wip` PR, one per line. | +| `list_idle_students ` | Return JSON names of students with no `status:wip` PR whose worker pod is running and ready when Kubernetes liveness is available. | | `pr_body ` | Read one PR body through REST. Returns JSON. | | `pr_issue_comments ` | Read all PR conversation comments through REST pagination. Returns JSON array. | | `pr_reviews ` | Read all PR review submissions through REST pagination. Returns JSON array. | diff --git a/tests/test_senpai_gh_idle.sh b/tests/test_senpai_gh_idle.sh new file mode 100755 index 000000000..16d2b5dd1 --- /dev/null +++ b/tests/test_senpai_gh_idle.sh @@ -0,0 +1,52 @@ +#!/bin/bash + +set -e +set -o pipefail + +WORKDIR="$(cd "$(dirname "$0")/.." && pwd)" +cd "$WORKDIR" + +source "$WORKDIR/plugins/senpai/scripts/senpai-gh.sh" + +assert_json_eq() { + local expected="$1" actual="$2" + python3 - "$expected" "$actual" <<'PY' +import json +import sys + +expected = json.loads(sys.argv[1]) +actual = json.loads(sys.argv[2]) +if actual != expected: + raise SystemExit(f"expected {expected!r}, got {actual!r}") +PY +} + +rest_labeled_pull_details() { + printf '%s\n' '[{"number":1,"labels":[{"name":"status:wip"},{"name":"student:busy"}]}]' +} + +mock_bin=$(mktemp -d "${TMPDIR:-/tmp}/senpai-idle-test.XXXXXX") +trap 'rm -rf "$mock_bin"' EXIT + +cat > "$mock_bin/kubectl" <<'SH' +#!/bin/sh + +case "$*" in + *student=busy*) printf 'busy-pod\tTrue\n' ;; + *student=ready*) printf 'ready-pod\tTrue\n' ;; + *student=notready*) printf 'notready-pod\tFalse\n' ;; + *student=dead*) : ;; + *) exit 1 ;; +esac +SH +chmod +x "$mock_bin/kubectl" + +export PATH="$mock_bin:$PATH" +export RESEARCH_TAG="test" + +actual=$(list_idle_students "busy,ready,notready,dead" "advisor") +assert_json_eq '["ready"]' "$actual" + +unset RESEARCH_TAG +actual=$(list_idle_students "busy,ready,notready,dead" "advisor") +assert_json_eq '["ready","notready","dead"]' "$actual"