diff --git a/k8s/advisor-claude-watchdog.sh b/k8s/advisor-claude-watchdog.sh index 5fc9e6455..1ebd547f8 100644 --- a/k8s/advisor-claude-watchdog.sh +++ b/k8s/advisor-claude-watchdog.sh @@ -17,6 +17,7 @@ ADVISOR_CLAUDE_STALE_LOG_S="${ADVISOR_CLAUDE_STALE_LOG_S:-1200}" ADVISOR_CLAUDE_KILL_GRACE_S="${ADVISOR_CLAUDE_KILL_GRACE_S:-15}" ADVISOR_CLAUDE_SELF_PGREP_STALE_S="${ADVISOR_CLAUDE_SELF_PGREP_STALE_S:-300}" ADVISOR_CLAUDE_SELF_PGREP_PATTERNS="${ADVISOR_CLAUDE_SELF_PGREP_PATTERNS:-wandb_sparse_val.py}" +ADVISOR_CLAUDE_TASK_OUTPUT_WAIT_STALE_S="${ADVISOR_CLAUDE_TASK_OUTPUT_WAIT_STALE_S:-300}" # Read log mtimes portably so stale-output checks work on Linux and macOS. advisor_file_mtime_s() { @@ -113,11 +114,87 @@ advisor_self_pgrep_wait_pids() { done | sort -nu } +# Utility-only descendants, such as sleep/grep/cat, do not mean real work is +# still running below a Claude task-output waiter. +advisor_real_descendant_process_exists() { + local root_pid="$1" pid comm + + for pid in $(advisor_descendant_pids "$root_pid"); do + comm=$(ps -o comm= -p "$pid" 2>/dev/null | awk '{print $1}' || true) + [ -n "$comm" ] || continue + case "$comm" in + awk|basename|cat|cut|dirname|grep|head|pgrep|ps|sed|sleep|sort|stat|tail|tee|test|tr|wc) + ;; + *) + return 0 + ;; + esac + done + + return 1 +} + +advisor_is_task_output_wait_line() { + local line="$1" + + case "$line" in + *"/tmp/claude-"*"/tasks/"*".output"*) ;; + *) return 1 ;; + esac + + case "$line" in + *"until "*"grep -q "*"sleep "*"done"*) return 0 ;; + *) return 1 ;; + esac +} + +advisor_extract_task_output_path() { + local line="$1" + + printf '%s\n' "$line" | + awk ' + { + for (i = 1; i <= NF; i++) { + token = $i + sub(/[;)]*$/, "", token) + if (token ~ /^\/tmp\/claude-[^[:space:]]*\/tasks\/[^[:space:]]*[.]output$/) { + print token + exit + } + } + } + ' +} + +# Find Claude tool-output waiters whose output file stopped changing and whose +# real worker process is gone. This catches stale TaskOutput waits without +# binding the watchdog to one specific command or one specific error string. +advisor_stale_task_output_wait_pids() { + local root_pid="$1" now_ts="$2" + local pid line output_path output_mtime output_age + + for pid in $(advisor_tree_pids "$root_pid"); do + line=$(ps -o comm= -o args= -p "$pid" 2>/dev/null || true) + [ -n "$line" ] || continue + advisor_is_task_output_wait_line "$line" || continue + + output_path=$(advisor_extract_task_output_path "$line") + [ -n "$output_path" ] && [ -f "$output_path" ] || continue + + output_mtime=$(advisor_file_mtime_s "$output_path" 2>/dev/null || printf '%s' "$now_ts") + output_age=$((now_ts - output_mtime)) + [ "$output_age" -ge "$ADVISOR_CLAUDE_TASK_OUTPUT_WAIT_STALE_S" ] || continue + + advisor_real_descendant_process_exists "$pid" && continue + printf '%s\n' "$pid" + done | sort -nu +} + # Run Claude under supervision and return 124 when the outer loop should re-poll. run_advisor_claude_with_watchdog() { run_senpai_claude "$@" & local claude_pid=$! - local start_ts now_ts runtime log_mtime log_age reason rc self_pgrep_wait_pids + local start_ts now_ts runtime log_mtime log_age reason rc self_pgrep_wait_pids task_output_wait_pids local watchdog_fired=0 start_ts=$(date +%s) @@ -140,6 +217,13 @@ run_advisor_claude_with_watchdog() { fi fi + task_output_wait_pids=$(advisor_stale_task_output_wait_pids "$claude_pid" "$now_ts") + if [ -n "$task_output_wait_pids" ]; then + advisor_log_watchdog_trigger "=== Advisor Claude watchdog: stopping stale task-output wait children after ${runtime}s: $(printf '%s' "$task_output_wait_pids" | tr '\n' ' ')===" + advisor_stop_process_trees "$task_output_wait_pids" + watchdog_fired=1 + fi + [ "$runtime" -lt "$ADVISOR_CLAUDE_MIN_RUNTIME_S" ] && continue log_mtime=$(advisor_file_mtime_s "$LOGFILE" 2>/dev/null || printf '%s' "$now_ts") diff --git a/tests/test_advisor_claude_watchdog.sh b/tests/test_advisor_claude_watchdog.sh new file mode 100644 index 000000000..5b9fa0a8d --- /dev/null +++ b/tests/test_advisor_claude_watchdog.sh @@ -0,0 +1,130 @@ +#!/bin/bash + +# SPDX-FileCopyrightText: 2026 CoreWeave, Inc. +# SPDX-License-Identifier: Apache-2.0 +# SPDX-PackageName: senpai + +set -euo pipefail + +ROOT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")/.." && pwd)" +source "$ROOT_DIR/k8s/advisor-claude-watchdog.sh" + +ADVISOR_CLAUDE_TASK_OUTPUT_WAIT_STALE_S=0 + +ROOT_PID="" +REAL_WORK_PID="" +TMPDIR="" + +cleanup() { + set +e + [ -n "${ROOT_PID:-}" ] && kill -TERM "$ROOT_PID" 2>/dev/null + [ -n "${REAL_WORK_PID:-}" ] && kill -TERM "$REAL_WORK_PID" 2>/dev/null + sleep 0.2 + [ -n "${ROOT_PID:-}" ] && kill -KILL "$ROOT_PID" 2>/dev/null + [ -n "${REAL_WORK_PID:-}" ] && kill -KILL "$REAL_WORK_PID" 2>/dev/null + [ -n "${TMPDIR:-}" ] && rm -rf "$TMPDIR" +} +trap cleanup EXIT + +make_task_output() { + TMPDIR=$(mktemp -d /tmp/claude-watchdog-test.XXXXXX) + mkdir -p "$TMPDIR/-workspace-target/session/tasks" + TASK_OUTPUT="$TMPDIR/-workspace-target/session/tasks/bn7xfp7ad.output" + printf 'wandb: [wandb.Api()] Loaded credentials\nTerminated\n' > "$TASK_OUTPUT" +} + +start_waiter_without_real_worker() { + make_task_output + ROOT_PID="" + bash -c ' + task_output="$1" + bash -c '"'"' + task_output="$1" + until [ -s "$task_output" ] && grep -q "ERROR\|===" "$task_output" 2>/dev/null; do + sleep 3 + done + wait $(pgrep -P $$) + cat "$task_output" + '"'"' _ "$task_output" & + wait + ' _ "$TASK_OUTPUT" & + ROOT_PID=$! + sleep 0.5 +} + +start_waiter_with_real_worker() { + make_task_output + ROOT_PID="" + bash -c ' + task_output="$1" + bash -c '"'"' + task_output="$1" + python3 -c "import time; time.sleep(30)" & + until [ -s "$task_output" ] && grep -q "ERROR\|===" "$task_output" 2>/dev/null; do + sleep 3 + done + wait $(pgrep -P $$) + cat "$task_output" + '"'"' _ "$task_output" & + wait + ' _ "$TASK_OUTPUT" & + ROOT_PID=$! + sleep 0.5 +} + +start_custom_grep_waiter_without_real_worker() { + make_task_output + ROOT_PID="" + bash -c ' + task_output="$1" + bash -c '"'"' + task_output="$1" + until grep -q "no val yet\|metric .*: state\|metric .*: error" "$task_output" 2>/dev/null; do + sleep 3 + done + tail -40 "$task_output" + '"'"' _ "$task_output" & + wait + ' _ "$TASK_OUTPUT" & + ROOT_PID=$! + sleep 0.5 +} + +test_detects_stale_waiter_without_worker() { + local pids + start_waiter_without_real_worker + pids=$(advisor_stale_task_output_wait_pids "$ROOT_PID" "$(date +%s)") + if [ -z "$pids" ]; then + echo "expected stale task-output waiter to be detected" >&2 + exit 1 + fi + cleanup +} + +test_detects_custom_grep_waiter_without_worker() { + local pids + start_custom_grep_waiter_without_real_worker + pids=$(advisor_stale_task_output_wait_pids "$ROOT_PID" "$(date +%s)") + if [ -z "$pids" ]; then + echo "expected custom grep task-output waiter to be detected" >&2 + exit 1 + fi + cleanup +} + +test_ignores_waiter_with_real_worker() { + local pids + start_waiter_with_real_worker + pids=$(advisor_stale_task_output_wait_pids "$ROOT_PID" "$(date +%s)") + if [ -n "$pids" ]; then + echo "expected waiter with real worker descendant to be ignored, got: $pids" >&2 + exit 1 + fi + cleanup +} + +test_detects_stale_waiter_without_worker +test_detects_custom_grep_waiter_without_worker +test_ignores_waiter_with_real_worker + +echo "advisor Claude watchdog tests passed"