fix(bin): keep confirming a slow watcher start while it makes progress

bin/fm-claude-stop-autoarm.sh

@@ -360,13 +360,16 @@ run_arm() {  # <output file, or empty for none>
 # successor receives the closed arm's pid as FM_WATCH_PREDECESSOR_ARM_PID; it
 # must outlive this hook's exit, so it is detached three ways: nohup, stdio
 # away from the hook's pipes, and its own process group. Its one status line
-# is awaited within the arm's own confirmation budget plus slack. Sets
+# is awaited until the successor exits or GRACE plus startup slack elapses:
+# the arm keeps extending its confirmation windows for a watcher still making
+# progress, up to the FM_GUARD_GRACE passed to it, so FM_ARM_CONFIRM_TIMEOUT
+# alone would report a slow but valid start as unconfirmed. Sets
 # SUCCESSOR_FAILURE to the banner line for an unconfirmed successor.
 SUCCESSOR_FAILURE=
 start_handling_successor() {  # <closed-arm-pid>
   local out pid deadline budget line monitor_was_on=0
-  budget=${FM_ARM_CONFIRM_TIMEOUT:-30}
-  case "$budget" in ''|*[!0-9]*) budget=30 ;; esac
+  budget=$GRACE
+  case "$budget" in ''|*[!0-9]*) budget=300 ;; esac
   if ! out=$(mktemp "$STATE/.claude-autoarm-successor.XXXXXX"); then
     SUCCESSOR_FAILURE='The handling successor did not confirm a live watcher (its output file could not be created); this handling turn runs uncovered until the next turn end re-arms.'
     return 1
@@ -377,13 +380,19 @@ start_handling_successor() {  # <closed-arm-pid>
     nohup "$SCRIPT_DIR/fm-watch-arm.sh" >"$out" 2>&1 </dev/null &
   pid=$!
   [ "$monitor_was_on" -eq 1 ] || set +m 2>/dev/null || true
-  deadline=$(( $(date +%s) + budget + 2 ))
+  deadline=$(( $(date +%s) + budget + 5 ))
   while :; do
     if grep -Eq '^watcher: (started|attached) pid=[0-9]+' "$out" 2>/dev/null; then
       rm -f "$out" 2>/dev/null || true
       return 0
     fi
     grep -q '^watcher: FAILED' "$out" 2>/dev/null && break
+    if ! kill -0 "$pid" 2>/dev/null; then
+      # An arm that confirmed and then exited still leaves its started line.
+      grep -Eq '^watcher: (started|attached) pid=[0-9]+' "$out" 2>/dev/null \
+        && { rm -f "$out" 2>/dev/null || true; return 0; }
+      break
+    fi
     [ "$(date +%s)" -ge "$deadline" ] && break
     sleep 0.2
   done

bin/fm-watch-arm.sh

@@ -153,6 +153,8 @@ GRACE=${FM_GUARD_GRACE:-300}
 # How long to wait for a freshly forked watcher to acquire the lock and beat.
 # Git Bash/MSYS pays a much higher fork cost while the watcher completes its
 # required pre-lock migration, so its bounded default covers that cold start.
+# A child still consuming CPU when a window ends is slow, not wedged, so it is
+# granted further windows up to GRACE after its start (owned_child_progressed).
 case "${OSTYPE:-}" in
   msys*|mingw*|cygwin*) ARM_CONFIRM_DEFAULT=30 ;;
   *) ARM_CONFIRM_DEFAULT=10 ;;
@@ -704,7 +706,7 @@ handle_arm_signal() {
     # publishing the home-bound lock identity. Do not TERM it in the middle of
     # stale-lock acquisition: that can abandon the steal mutex. Let startup
     # reach that cleanup-ready point (or exit naturally) before forwarding TERM,
-    # but never past the startup confirmation deadline.
+    # but never past the first confirmation window's deadline.
     while fm_pid_alive "$child"; do
       if fm_watcher_lock_matches_pid "$STATE" "$WATCH" "$child" "$FM_HOME" \
         || [ "$(date +%s)" -ge "$deadline" ]; then
@@ -730,7 +732,10 @@ child_out=$(mktemp "$STATE/.watch-arm-output.XXXXXX") || {
 }
 # date(1) exposes whole seconds. Keep the configured confirmation budget from
 # collapsing when startup begins just before the next second boundary.
-deadline=$(( $(date +%s) + CONFIRM_TIMEOUT + 1 ))
+confirm_start=$(date +%s)
+deadline=$(( confirm_start + CONFIRM_TIMEOUT + 1 ))
+window_end=$deadline
+confirm_limit=$(( confirm_start + GRACE ))
 if [ -n "${FM_WATCH_PREDECESSOR_ARM_PID:-}" ]; then
   FM_WATCH_HANDLING_SUCCESSOR=1 "$WATCH" >"$child_out" &
 else
@@ -740,6 +745,39 @@ child=$!
 cycle_begin "$child" started "$(fm_pid_identity "$child" 2>/dev/null || true)"
 child_done=0
 
+# Print the CPU time the child has consumed, as an opaque value that changes
+# only while it runs: utime+stime ticks from a Linux-compatible /proc (Linux,
+# Git Bash/MSYS), else ps(1)'s cumulative time. Empty when neither is readable.
+child_cpu_time() {
+  local stat_line
+  local -a stat_fields
+  if [ -r "/proc/$child/stat" ]; then
+    stat_line=$(cat "/proc/$child/stat" 2>/dev/null) || return 0
+    # After the final comm delimiter, indexes 11 and 12 are stat fields 14 and 15.
+    read -r -a stat_fields <<< "${stat_line##*)}"
+    printf '%s+%s\n' "${stat_fields[11]:-}" "${stat_fields[12]:-}"
+  else
+    ps -o time= -p "$child" 2>/dev/null
+  fi
+}
+child_cpu=$(child_cpu_time)
+
+# At the end of a confirmation window, true when the child consumed CPU during
+# it and GRACE has not elapsed since its start; it then opens the next window.
+# A window with no CPU progress, or the grace running out, ends the wait. Only
+# the first window bounds a signal's wait (handle_arm_signal), so an owner that
+# stops this arm past it still sees it exit promptly.
+owned_child_progressed() {
+  local now cpu
+  now=$(date +%s)
+  [ "$now" -lt "$confirm_limit" ] || return 1
+  cpu=$(child_cpu_time)
+  [ -n "$cpu" ] && [ "$cpu" != "$child_cpu" ] || return 1
+  child_cpu=$cpu
+  window_end=$(( now + CONFIRM_TIMEOUT + 1 ))
+  [ "$window_end" -le "$confirm_limit" ] || window_end=$confirm_limit
+}
+
 owned_child_finished() {
   local rc=$1 signal reason_type status
   signal=$(cycle_signal_name "$rc")
@@ -850,7 +888,9 @@ while :; do
     fail_arm_ground_gone
     exit 1
   fi
-  [ "$(date +%s)" -ge "$deadline" ] && break
+  if [ "$(date +%s)" -ge "$window_end" ]; then
+    owned_child_progressed || break
+  fi
   sleep 0.2
 done
 

docs/configuration.md

@@ -3062,10 +3062,10 @@ FM_CLAUDE_AUTOARM_ATTEMPTS=2   # bounded Stop-owned arm attempts per Claude auto
 FM_CLAUDE_AUTOARM_SYNC_WAIT_MS=800   # milliseconds the --claude turn-end guard waits for watcher health, an open Stop auto-arm generation claim, or a fresh epoch before deciding recovery ownership or failure progression
 FM_CLAUDE_AUTOARM_EPOCH_FRESH=15   # seconds a recorded auto-arm outcome remains eligible for the current event epoch's recovery or failure decision
 FM_CLAUDE_TURNEND_BLOCK_BUDGET=3   # consecutive --claude guard re-blocks before the verified one-time attended fail-open; safely below Claude Code's 8-block override
-FM_ARM_CONFIRM_TIMEOUT=10   # seconds fm-watch-arm waits to confirm a fresh watcher before reporting FAILED; default 30 on Git Bash/MSYS
+FM_ARM_CONFIRM_TIMEOUT=10   # seconds of each window fm-watch-arm waits to confirm a fresh watcher; a watcher still consuming CPU when a window ends gets another, up to FM_GUARD_GRACE after its start, before the arm reports FAILED; default 30 on Git Bash/MSYS
 FM_ARM_ATTACH_POLL=0.5  # seconds between checks while fm-watch-arm follows an attached watcher cycle (bin/fm-watch-arm.sh header)
 FM_OPENCODE_ARM_READY_TIMEOUT_MS=12000   # milliseconds the OpenCode primary watcher plugin waits for an arm attempt to report started, healthy, wake, or failure; default 35000 on Windows to stay above the MSYS confirm budget
-FM_PI_ARM_READY_TIMEOUT_MS=12000   # milliseconds the Pi watcher extension waits for a successor arm to report started or attached; default 35000 on Windows to stay above the MSYS confirm budget
+FM_PI_ARM_READY_TIMEOUT_MS=12000   # milliseconds the Pi watcher extension waits for a successor arm to report started or attached; default 35000 on Windows to stay above the MSYS confirm budget; the OpenCode and Pi primaries keep these fixed ready bounds rather than waiting out FM_ARM_CONFIRM_TIMEOUT's extended windows, so a slower start is still stopped and retried there (extending them is a possible follow-up)
 FM_WATCH_ARM_RETIRE_TIMEOUT_MS=1000   # milliseconds Pi/OpenCode wait for an unready successor arm to exit before abandoning retries
 FM_WATCH_REARM_RETRY_BASE_MS=250   # Pi/OpenCode adapter base delay for continuity restoration retries
 FM_WATCH_REARM_RETRY_MAX_MS=4000   # Pi/OpenCode adapter cap for exponential continuity retry delay

docs/watcher-continuity.md

@@ -172,7 +172,7 @@ Claude's Stop hook also starts one handling successor before notification.
 After an actionable foreground close, including an attached peer cycle that ended, the hook:
 
 1. Launches `bin/fm-watch-arm.sh` with the closed arm's pid as `FM_WATCH_PREDECESSOR_ARM_PID`.
-2. Waits for that arm's one status line.
+2. Waits for that arm's one status line, until the arm exits or the hook's guard grace plus a few seconds elapses, because the arm extends its confirmation windows for a watcher still making progress up to that grace.
 3. Only then exits 2 with the wake.
 
 A child of the hook cannot outlive its exit-2 rewake.
@@ -513,7 +513,7 @@ It checks that a newly appended keyed decision is classified without rereading e
 - One-notice failure episodes.
 - Exit-2 translation.
 - The handling successor an ended attached cycle starts with the closed arm as its predecessor and that outlives the rewake.
-- An unconfirmed successor reported in the banner without withholding the wake.
+- An unconfirmed successor reported in the banner without withholding the wake, a successor confirming past `FM_ARM_CONFIRM_TIMEOUT` still confirmed, and a silent successor bounded by the grace.
 - Host-timeout HUP/TERM/INT translation into the same durable failure handoff.
 
 It also covers generation-claim single-flight, stuck-claim supersession, superseded-owner silence, notice-marker refusal and retry, ownership-atomic episode reset, and the legacy upgrade shim.

tests/fm-claude-stop-autoarm.test.sh

@@ -97,7 +97,8 @@ write_arm_fixture() {
   # successor (FM_WATCH_PREDECESSOR_ARM_PID set) is recorded apart in
   # state/successor-ran so attempt counts stay about the foreground; it confirms
   # a started watcher and exits, parks while state/successor-park exists, or
-  # fails while state/successor-fail exists.
+  # fails while state/successor-fail exists. state/successor-delay holds seconds
+  # to wait before reporting; state/successor-silent parks it without a report.
   cat > "$dir/bin/fm-watch-arm.sh" <<'SH'
 #!/usr/bin/env bash
 if [ -n "${FM_WATCH_PREDECESSOR_ARM_PID:-}" ]; then
@@ -106,6 +107,8 @@ if [ -n "${FM_WATCH_PREDECESSOR_ARM_PID:-}" ]; then
     printf 'watcher: FAILED - no live watcher with a fresh beacon\n'
     exit 1
   fi
+  while [ -e "$FM_HOME/state/successor-silent" ]; do sleep 0.05; done
+  [ ! -e "$FM_HOME/state/successor-delay" ] || sleep "$(cat "$FM_HOME/state/successor-delay")"
   printf 'watcher: started pid=%s (beacon fresh)\n' "$$"
   while [ -e "$FM_HOME/state/successor-park" ]; do sleep 0.05; done
   exit 0
@@ -570,6 +573,38 @@ test_unconfirmed_handling_successor_still_rewakes() {
   pass "auto-arm: an unconfirmed handling successor is reported in the rewake instead of blocking it"
 }
 
+# The arm keeps extending its confirmation windows for a watcher still making
+# progress, up to FM_GUARD_GRACE, so the hook must not give up on a successor at
+# FM_ARM_CONFIRM_TIMEOUT.
+test_late_reporting_handling_successor_is_confirmed() {
+  local dir out status
+  dir=$(make_primary_dir "$TMP_ROOT/successor-late")
+  : > "$dir/state/task.meta"
+  write_arm_fixture "$dir" attached-delivered
+  printf '5\n' > "$dir/state/successor-delay"
+  out=$(FM_ARM_CONFIRM_TIMEOUT=1 FM_GUARD_GRACE=30 run_autoarm "$dir" 2>/dev/null); status=$?
+  expect_code 2 "$status" "a late-confirming handling successor must still rewake"
+  assert_contains "$out" "signal: task.status done: fixture peer cycle ended" "rewake must carry the delivered reason"
+  assert_not_contains "$out" "did not confirm" "a successor confirming past FM_ARM_CONFIRM_TIMEOUT but within grace is confirmed"
+  pass "auto-arm: a handling successor confirming past FM_ARM_CONFIRM_TIMEOUT is still confirmed"
+}
+
+test_silent_handling_successor_is_bounded_by_grace() {
+  local dir out status start elapsed
+  dir=$(make_primary_dir "$TMP_ROOT/successor-silent")
+  : > "$dir/state/task.meta"
+  write_arm_fixture "$dir" attached-delivered
+  : > "$dir/state/successor-silent"
+  start=$(date +%s)
+  out=$(FM_ARM_CONFIRM_TIMEOUT=1 FM_GUARD_GRACE=2 run_autoarm "$dir" 2>/dev/null); status=$?
+  elapsed=$(( $(date +%s) - start ))
+  rm -f "$dir/state/successor-silent"
+  expect_code 2 "$status" "a silent handling successor must never withhold the delivered wake"
+  assert_contains "$out" "did not confirm a live watcher" "the rewake must say this turn runs uncovered"
+  [ "$elapsed" -lt 20 ] || fail "the successor wait must stay bounded by FM_GUARD_GRACE plus slack, took ${elapsed}s"
+  pass "auto-arm: a silent handling successor is bounded by the grace cap and reported"
+}
+
 test_failed_close_rewakes_with_failure_banner() {
   local dir out status
   dir=$(make_primary_dir "$TMP_ROOT/failed")
@@ -2008,6 +2043,8 @@ test_actionable_close_rewakes_with_reason
 test_actionable_close_with_live_successor_rewakes_once
 test_attached_cycle_end_starts_handling_successor
 test_unconfirmed_handling_successor_still_rewakes
+test_late_reporting_handling_successor_is_confirmed
+test_silent_handling_successor_is_bounded_by_grace
 test_failed_close_rewakes_with_failure_banner
 test_failed_cycles_notify_once_and_keep_retrying
 test_failure_notice_marker_write_refuses_delivery_and_retries

tests/fm-watch-arm.test.sh

@@ -1565,6 +1565,109 @@ test_arm_refuses_a_disposable_validation_checkout() {
   pass "watch-arm: a disposable validation checkout refuses to arm"
 }
 
+# Arm a fresh watcher whose own startup first runs <startup-snippet>, standing
+# in for a host so loaded that the forked watcher needs longer than the arm's
+# one-second confirmation window to take its lock and beat. BASH_ENV injects the
+# snippet, gated on the watcher's script name so the arm and every other bash
+# process start normally. Extra VAR=value assignments reach the arm unchanged.
+start_slowed_arm() {  # <dir> <startup-snippet> [VAR=value...]
+  local dir=$1 snippet=$2
+  shift 2
+  # shellcheck disable=SC2016 # $0 is expanded by the watcher, not here.
+  printf 'case "$0" in */fm-watch.sh) %s ;; esac\n' "$snippet" > "$dir/watcher-start.sh"
+  env PATH="$dir/fakebin:$PATH" FM_HOME="$dir/home" FM_STATE_OVERRIDE="$dir/state" \
+    FM_POLL=1 FM_SIGNAL_GRACE=0 FM_CHECK_INTERVAL=999999 FM_HEARTBEAT=999999 \
+    FM_ARM_CONFIRM_TIMEOUT=1 BASH_ENV="$dir/watcher-start.sh" "$@" "$WATCH_ARM" > "$dir/arm.out" 2>&1 &
+  ARM_PID=$!
+}
+
+# A heavily loaded host can keep a freshly forked watcher from beating within
+# the confirmation window while it is still working through startup. The arm
+# used to TERM that live child at the deadline and report confirmation-timeout,
+# so every retry repeated the same cold start and supervision stayed down. A
+# start that keeps consuming CPU must instead be confirmed once it beats.
+test_arm_confirms_a_slow_watcher_start_that_keeps_progressing() {
+  local dir i status
+  dir=$(make_case slow-progressing-start)
+  mkdir -p "$dir/home/data"
+  # shellcheck disable=SC2016 # The watcher's own shell expands the snippet.
+  start_slowed_arm "$dir" 'end=$((SECONDS + 4)); while [ "$SECONDS" -lt "$end" ]; do :; done'
+  i=0
+  while [ "$i" -lt 300 ]; do
+    grep -q '^watcher: ' "$dir/arm.out" 2>/dev/null && break
+    is_live_non_zombie "$ARM_PID" || break
+    sleep 0.1
+    i=$((i + 1))
+  done
+  grep -q '^watcher: started pid=' "$dir/arm.out" \
+    || fail "arm gave up on a slow watcher start that kept progressing: $(cat "$dir/arm.out")"
+  ! grep -q 'reason=confirmation-timeout' "$dir/state/.watch-cycle-exits.log" 2>/dev/null \
+    || fail "arm ledgered a confirmation timeout for a start it went on to confirm"
+  FM_HOME="$dir/home" FM_STATE_OVERRIDE="$dir/state" "$WATCH_ARM" --stop >/dev/null 2>&1
+  wait_for_exit "$ARM_PID" 100
+  status=$?
+  [ "$status" -ne 124 ] || fail "arm did not close after its confirmed watcher stopped"
+  pass "watch-arm: a slow watcher start that keeps progressing is confirmed, not killed at the window"
+}
+
+# The extension is earned by progress, so a start that stops consuming CPU
+# still fails at the window rather than holding the arm while it waits.
+test_arm_fails_a_watcher_start_that_stops_progressing() {
+  local dir status
+  dir=$(make_case stalled-start)
+  mkdir -p "$dir/home/data"
+  start_slowed_arm "$dir" 'exec sleep 30'
+  wait_for_exit "$ARM_PID" 150
+  status=$?
+  [ "$status" -ne 124 ] || fail "arm kept waiting on a watcher start with no progress: $(cat "$dir/arm.out")"
+  [ "$status" -ne 0 ] || fail "arm reported success for a stalled watcher start: $(cat "$dir/arm.out")"
+  grep -qF 'watcher: FAILED - no live watcher with a fresh beacon' "$dir/arm.out" \
+    || fail "arm did not report the typed confirmation failure: $(cat "$dir/arm.out")"
+  grep -q 'reason=confirmation-timeout' "$dir/state/.watch-cycle-exits.log" \
+    || fail "the stalled start was not ledgered as a confirmation timeout"
+  pass "watch-arm: a watcher start that stops progressing still fails at the confirmation window"
+}
+
+# Owners stop an arm that misses their own readiness budget and allow it only a
+# short bound to exit (Pi/OpenCode 1s, then the supervision host's KILL that
+# would orphan the watcher). An extended window must not lengthen that exit, so
+# a signal past the first window stops the slow watcher at once.
+test_arm_exits_promptly_when_stopped_during_an_extended_window() {
+  local dir status watcher_pid
+  dir=$(make_case signalled-extended-start)
+  mkdir -p "$dir/home/data"
+  start_slowed_arm "$dir" 'while :; do :; done' FM_ARM_CONFIRM_TIMEOUT=5
+  # Past the first six-second window, early in the second.
+  sleep 8
+  is_live_non_zombie "$ARM_PID" \
+    || fail "arm ended before its extended window: $(cat "$dir/arm.out")"
+  kill -TERM "$ARM_PID" 2>/dev/null || true
+  wait_for_exit "$ARM_PID" 25
+  status=$?
+  [ "$status" -ne 124 ] || fail "a stopped arm kept waiting on its slow watcher's extended window"
+  watcher_pid=$(tr '\t' '\n' < "$dir/state/.watch-cycle-exits.log" | sed -n 's/^watcher_pid=//p' | tail -1)
+  [ -n "$watcher_pid" ] || fail "the interrupted cycle was not ledgered"
+  ! is_live_non_zombie "$watcher_pid" || fail "a stopped arm left its watcher pid $watcher_pid running"
+  pass "watch-arm: an arm stopped during an extended window exits at once and stops its watcher"
+}
+
+# Progress never extends the wait past the liveness grace: a start that spins
+# without ever beating is reported once FM_GUARD_GRACE has elapsed.
+test_arm_bounds_a_progressing_watcher_start_by_the_grace() {
+  local dir status
+  dir=$(make_case spinning-start)
+  mkdir -p "$dir/home/data"
+  start_slowed_arm "$dir" 'while :; do :; done' FM_GUARD_GRACE=4
+  wait_for_exit "$ARM_PID" 150
+  status=$?
+  [ "$status" -ne 124 ] || fail "arm followed a spinning watcher start past the grace: $(cat "$dir/arm.out")"
+  grep -qF 'watcher: FAILED - no live watcher with a fresh beacon' "$dir/arm.out" \
+    || fail "arm did not report the typed confirmation failure: $(cat "$dir/arm.out")"
+  grep -q 'reason=confirmation-timeout' "$dir/state/.watch-cycle-exits.log" \
+    || fail "the spinning start was not ledgered as a confirmation timeout"
+  pass "watch-arm: progress never extends a watcher start's confirmation past the grace"
+}
+
 # Start a real watcher through the real arm for a temporary home and set
 # WATCH_PID from the arm's started line. Both stdout and stderr land in <arm-out>
 # so the watcher's own exit reason, which it logs to stderr, is readable there.
@@ -1959,6 +2062,10 @@ test_attached_arm_reports_the_delivered_wake
 test_attached_arm_reports_the_delivered_wake_after_drain
 test_arm_refuses_an_unusable_launch_confirm_window
 test_arm_refuses_a_disposable_validation_checkout
+test_arm_confirms_a_slow_watcher_start_that_keeps_progressing
+test_arm_fails_a_watcher_start_that_stops_progressing
+test_arm_exits_promptly_when_stopped_during_an_extended_window
+test_arm_bounds_a_progressing_watcher_start_by_the_grace
 test_watcher_exits_when_its_state_directory_is_removed
 test_watcher_exits_when_its_home_is_removed
 test_attached_arm_exits_when_its_root_is_removed