fix(bin): reconcile remote secondmate delivery and wake mates on unread parent instructions

bin/fm-classify-lib.sh

@@ -815,7 +815,7 @@ _fm_decision_key_transition_allowed() {  # <key> <note>
 _fm_is_pending_reply_escalation() {  # <key> <note>
   case "$1" in pending-reply-*) ;; *) return 1 ;; esac
   case "$2" in
-    pending-reply-missed:*|pending-reply-delivery-unknown:*|pending-reply-recovery-delivery-failed:*|pending-reply-recovery-delivery-unknown:*) return 0 ;;
+    pending-reply-missed:*|pending-reply-delivery-unknown:*|pending-reply-unread:*|pending-reply-recovery-delivery-failed:*|pending-reply-recovery-delivery-unknown:*) return 0 ;;
     *) return 1 ;;
   esac
 }

bin/fm-remote-secondmate-control.sh

@@ -10,6 +10,7 @@
 #   fm-remote-secondmate-control.sh key <id> <key>
 #   fm-remote-secondmate-control.sh capture <id> [lines]
 #   fm-remote-secondmate-control.sh observe <id>
+#   fm-remote-secondmate-control.sh inbox-status <id> [<corr-id>]
 #   fm-remote-secondmate-control.sh sync <id> [<parent-commit>]
 #   fm-remote-secondmate-control.sh update <id>
 #   fm-remote-secondmate-control.sh retire <id> [--force]
@@ -62,7 +63,7 @@ CONTROL_DATA="$TARGET_HOME/data/.parent-route"
 REMOTE_HERDR_SESSION=fm-remote
 
 die() { printf 'error: %s\n' "$1" >&2; exit 1; }
-usage() { sed -n '2,24p' "$0" | sed 's/^# \{0,1\}//'; exit 2; }
+usage() { sed -n '2,25p' "$0" | sed 's/^# \{0,1\}//'; exit 2; }
 validate_id() { case "$1" in ''|*[!A-Za-z0-9._-]*) die "invalid secondmate id: $1" ;; esac; }
 
 # Sourcing fm-wake-lib.sh through the libraries below creates "$FM_HOME/state",
@@ -350,6 +351,29 @@ cmd_observe() {
   printf '\n'
 }
 
+# Read-only delivery facts for the parent's reply tracker: how many records sit
+# unhandled in this secondmate's host-local steering inbox and how old the
+# oldest is, where one correlated request stands there (unhandled, handled, or
+# absent), and the recorded pane's composer verdict. The parent cannot read this
+# inbox itself; bin/fm-pending-reply-lib.sh owns the output format and what the
+# tracker does with it. An unreadable endpoint reports composer=unknown rather
+# than failing, because the inbox facts stand on their own.
+cmd_inbox_status() {
+  local id=$1 corr=${2:-} composer=unknown
+  validate_id "$id"
+  case "$corr" in
+    '') ;;
+    *[!0-9a-f]*) die "invalid correlation id: $corr" ;;
+    *) [ "${#corr}" -eq 16 ] || die "invalid correlation id: $corr" ;;
+  esac
+  validate_home "$id"
+  if [ -f "$(meta_path "$id")" ] && remote_endpoint_load "$id"; then
+    composer=$(fm_backend_composer_state "$REMOTE_ENDPOINT_BACKEND" "$REMOTE_ENDPOINT_TARGET" "fm-$id" 2>/dev/null) \
+      || composer=unknown
+  fi
+  fm_pending_reply_inbox_status_report "$CONTROL_STATE" "$id" "$corr" "$composer"
+}
+
 # Make <commit> readable in this home's own object store without moving any other
 # checkout. Ordered by cost: already present, then this host's Firstmate copy (a
 # read-only fetch of that one commit, which never advances that copy's HEAD), then
@@ -449,6 +473,7 @@ case "${1:-}" in
   key) shift; [ "$#" -eq 2 ] || usage; cmd_key "$@" ;;
   capture) shift; [ "$#" -ge 1 ] && [ "$#" -le 2 ] || usage; cmd_capture "$@" ;;
   observe) shift; [ "$#" -eq 1 ] || usage; cmd_observe "$@" ;;
+  inbox-status) shift; [ "$#" -ge 1 ] && [ "$#" -le 2 ] || usage; cmd_inbox_status "$@" ;;
   sync) shift; [ "$#" -ge 1 ] && [ "$#" -le 2 ] || usage; cmd_sync "$@" ;;
   update) shift; [ "$#" -eq 1 ] || usage; cmd_update "$@" ;;
   retire) shift; [ "$#" -ge 1 ] && [ "$#" -le 2 ] || usage; cmd_retire "$@" ;;

bin/fm-send.sh

@@ -141,10 +141,12 @@
 # seconds (default 30, and any override must be a positive integer): a bound
 # hit is completion-unknown and exits through this same unconfirmed contract
 # instead of waiting out a busy remote queue.
-# The remote host runs no re-ring ladder of its own: a swallowed ordinary
-# doorbell surfaces through the parent's pending-reply recovery and escalation,
-# whose recovery request re-rings the remote doorbell when it is enqueued;
-# fire-and-forget delivery deliberately arms neither mechanism. Internal
+# The remote host runs no re-ring ladder of its own: the mate's own watcher
+# wakes its session for any unhandled record there (bin/fm-watch.sh
+# parent_inbox_tick), and a still-unread ordinary request surfaces through the
+# parent's pending-reply tracker, which reads the remote inbox and escalates it
+# as unread (bin/fm-pending-reply-lib.sh); a delivery left unknown here is
+# reconciled against that inbox the same way once the mate is reachable. Internal
 # semantic callers may set FM_SEND_EXPECTED_SPAWN_GEN or
 # FM_SEND_EXPECTED_REMOTE_HOST to require that sampled identity to still match
 # during the final locked remote-route validation; unset or empty guards do not
@@ -961,6 +963,12 @@ else
       fi
       if [ -n "$PENDING_REPLY_CORR" ]; then
         fm_pending_reply_mark_delivery_unknown "$STATE" "$PENDING_REPLY_CORR" || true
+        # Kept for the tracker's one same-correlation resend once the mate is
+        # reachable and its inbox proves the record never landed. A keyed
+        # answer stores none: only the printed resend closes its keys, so the
+        # tracker leaves that record to it.
+        [ -n "$RESOLVE_KEYS" ] ||
+          fm_pending_reply_store_undelivered_text "$STATE" "$PENDING_REPLY_CORR" "$RESOLVE_ANSWER_TEXT" || true
       fi
       if [ "$remote_rc" -eq 255 ]; then
         echo "error: steer to remote secondmate $TARGET_REMOTE_ID is unconfirmed (transport lost twice; remote completion unknown). Only the correlation-reusing resend below is idempotent and lands on the same remote inbox record:" >&2

bin/fm-task-inbox-lib.sh

@@ -33,6 +33,10 @@
 #                              so later polls suppress another escalation
 #   <task>.inbox/.retry-ring   name of a fire-and-forget record still owed its
 #                              one retry ring (fm_task_inbox_mark_retry)
+#   <task>.inbox/.ring-log     bounded diagnostic log of every ring decision:
+#                              "<utc>\t<msg>\t<decision>\t<detail>", decision one
+#                              of rang, unconfirmed, skipped:other-text,
+#                              skipped:busy, failed, dead (fm_task_inbox_ring)
 #
 # Record format (fm_task_inbox_write / fm_task_inbox_body):
 #   schema=fm-task-inbox.v1
@@ -80,7 +84,8 @@
 # without this retry the record could sit unread until a checkpoint. A pending ordinary record's
 # ladder rings the same inbox, so the retry waits behind it, and an
 # acknowledged record drops its mark. The remote steer leg has no watcher
-# ladder and owes no retry.
+# ladder and owes no retry; there the receiving mate's own watcher wakes its
+# session for any unhandled record instead (bin/fm-watch.sh parent_inbox_tick).
 #
 # Inbox names containing bytes outside printable ASCII are unsupported. The
 # doorbell refuses them rather than sending terminal control bytes to a pane.
@@ -371,24 +376,44 @@ fm_task_inbox_doorbell_line() {  # <record-path>
 # whose Enter never landed, so on an agent not reported busy it is submitted
 # rather than skipped; skipping it would block every later ring. On both paths
 # a lost first Enter gets one confirmed retry.
+# Every decision is appended to the inbox's .ring-log (see the header), because
+# no caller records why a ring was skipped and a stuck doorbell is otherwise
+# undiagnosable after the fact; the log never changes the return value.
 fm_task_inbox_ring() {  # <backend> <target> <record-path> [expected-label]
-  local backend=$1 target=$2 rec=$3 label=${4:-} line cstate verdict
-  case "$(fm_backend_agent_state "$backend" "$target" 2>/dev/null || true)" in
-    dead|missing) return 3 ;;
+  local backend=$1 target=$2 rec=$3 label=${4:-} line cstate verdict astate
+  astate=$(fm_backend_agent_state "$backend" "$target" 2>/dev/null || true)
+  case "$astate" in
+    dead|missing) _fm_task_inbox_ring_log "$rec" dead "agent=$astate"; return 3 ;;
   esac
   if ! line=$(fm_task_inbox_doorbell_line "$rec"); then
+    _fm_task_inbox_ring_log "$rec" failed "doorbell-line-refused"
     return 2
   fi
   cstate=$(fm_backend_composer_state "$backend" "$target" "$label" 2>/dev/null) || cstate=unknown
   case "$cstate" in
     pending)
-      fm_task_inbox_composer_holds "$backend" "$target" "$line" "$label" \
-        && [ "$(fm_backend_busy_state "$backend" "$target" 2>/dev/null)" != busy ] \
-        || return 1
-      fm_backend_send_key "$backend" "$target" Enter "$label" >/dev/null 2>&1 || return 2
+      if ! fm_task_inbox_composer_holds "$backend" "$target" "$line" "$label"; then
+        _fm_task_inbox_ring_log "$rec" skipped:other-text "composer=pending"
+        return 1
+      fi
+      if [ "$(fm_backend_busy_state "$backend" "$target" 2>/dev/null)" = busy ]; then
+        _fm_task_inbox_ring_log "$rec" skipped:busy "composer=own-doorbell"
+        return 1
+      fi
+      if ! fm_backend_send_key "$backend" "$target" Enter "$label" >/dev/null 2>&1; then
+        _fm_task_inbox_ring_log "$rec" failed "composer=own-doorbell enter"
+        return 2
+      fi
       sleep 0.3
-      fm_task_inbox_composer_holds "$backend" "$target" "$line" "$label" || return 0
-      fm_backend_send_key "$backend" "$target" Enter "$label" >/dev/null 2>&1 || return 2
+      if ! fm_task_inbox_composer_holds "$backend" "$target" "$line" "$label"; then
+        _fm_task_inbox_ring_log "$rec" rang "composer=own-doorbell submitted"
+        return 0
+      fi
+      if ! fm_backend_send_key "$backend" "$target" Enter "$label" >/dev/null 2>&1; then
+        _fm_task_inbox_ring_log "$rec" failed "composer=own-doorbell second-enter"
+        return 2
+      fi
+      _fm_task_inbox_ring_log "$rec" unconfirmed "composer=own-doorbell second-enter"
       return 0
       ;;
   esac
@@ -397,18 +422,77 @@ fm_task_inbox_ring() {  # <backend> <target> <record-path> [expected-label]
   # shell only a suffix; the `: ` prefix protects complete lines only. Do not
   # add process-bound atomic delivery here unless an incident reopens this.
   if ! verdict=$(fm_backend_send_text_submit "$backend" "$target" "$line" 2 0.4 0.3 "$label" 2>/dev/null); then
+    _fm_task_inbox_ring_log "$rec" failed "composer=$cstate submit"
     return 2
   fi
   # The verdict is read only to report a failed keystroke; every other value
   # (empty, pending, unknown, ...) is deliberately ignored, never proof.
-  [ "$verdict" != send-failed ] || return 2
+  if [ "$verdict" = send-failed ]; then
+    _fm_task_inbox_ring_log "$rec" failed "composer=$cstate verdict=send-failed"
+    return 2
+  fi
+  if [ "$verdict" = empty ]; then
+    _fm_task_inbox_ring_log "$rec" rang "composer=$cstate verdict=empty"
+  else
+    _fm_task_inbox_ring_log "$rec" unconfirmed "composer=$cstate verdict=${verdict:-none}"
+  fi
+  return 0
+}
+
+# Append one ring decision to the inbox's .ring-log, best-effort and bounded
+# (trimmed to its newest 200 lines past 64 KiB). Never fails its caller.
+_fm_task_inbox_ring_log() {  # <record-path> <decision> [detail]
+  local dir=${1%/*} log sz
+  dir=${dir%/handled}
+  [ -d "$dir" ] || return 0
+  log="$dir/.ring-log"
+  printf '%s\t%s\t%s\t%s\n' "$(date -u +%Y-%m-%dT%H:%M:%SZ)" "${1##*/}" "$2" "${3:-}" \
+    >> "$log" 2>/dev/null || return 0
+  sz=$(wc -c < "$log" 2>/dev/null | tr -d '[:space:]')
+  case "$sz" in ''|*[!0-9]*) return 0 ;; esac
+  if [ "$sz" -ge 65536 ]; then
+    tail -n 200 "$log" > "$log.tmp.$$" 2>/dev/null && mv -f "$log.tmp.$$" "$log" 2>/dev/null
+    rm -f "$log.tmp.$$" 2>/dev/null
+  fi
   return 0
 }
 
-# Whether the composer's content, ignoring line wrapping, is exactly <line>.
+# Whether a Herdr composer read <held> is our own doorbell <line>: equal to it
+# ignoring whitespace and wrapping, or the shape Claude leaves when it collapses
+# the burst typed by the Herdr typing path - one bare `[Pasted text #N]`
+# placeholder, optionally followed by a literal tail that is the end of <line>.
+# A `+M lines` placeholder or several placeholders are an operator's paste, not
+# a single-line doorbell, and are never recognised.
+_fm_task_inbox_herdr_holds_doorbell() {  # <line> <held>
+  local line=$1 held=$2 rest
+  fm_composer_normalize_spaces_var line
+  fm_composer_normalize_spaces_var held
+  line=${line//[$' \t\r\n\v\f']/}
+  line=${line//$'\xE2\x81\xA3'/}
+  held=${held//[$' \t\r\n\v\f']/}
+  held=${held//$'\xE2\x81\xA3'/}
+  [ -n "$line" ] && [ -n "$held" ] || return 1
+  [ "$held" = "$line" ] && return 0
+  [[ $held =~ ^\[Pastedtext#[0-9]+\](.*)$ ]] || return 1
+  rest=${BASH_REMATCH[1]}
+  [ -n "$rest" ] || return 0
+  case "$rest" in *'[Pastedtext#'*) return 1 ;; esac
+  [[ $line == *"$rest" ]]
+}
+
+# Whether the composer's content is exactly <line>, ignoring line wrapping.
+# Herdr reads its composer with the same visible-viewport reader its typing
+# path uses (fm_backend_herdr_composer_content), so a doorbell that path typed
+# - wrapped, or collapsed into a placeholder - is recognised as our own rather
+# than skipped as someone else's draft. Other backends compare a plain capture.
 fm_task_inbox_composer_holds() {  # <backend> <target> <line> [expected-label]
   local cap held
   fm_backend_source "$1" || return 1
+  if [ "$1" = herdr ]; then
+    held=$(fm_backend_herdr_composer_content "$2" 2>/dev/null) || return 1
+    _fm_task_inbox_herdr_holds_doorbell "$3" "$held"
+    return
+  fi
   cap=$(fm_backend_capture "$1" "$2" "$FM_COMPOSER_CAPTURE_LINES" "${4:-}" 2>/dev/null) || return 1
   held=$(fm_composer_extract_selected_content styled=0 "$cap") || return 1
   [ -n "$held" ] && [ "$(printf '%s' "$held" | tr -d '[:space:]')" = "$(printf '%s' "$3" | tr -d '[:space:]')" ]
@@ -427,6 +511,27 @@ fm_task_inbox_is_fire_and_forget() {  # <record-path>
   ' "$rec"
 }
 
+# Every unhandled record, ordinary and fire-and-forget alike: prints
+# "<count> <oldest-basename> <oldest-age-seconds>" with the oldest chosen by
+# sequence, or fails when the inbox holds none. The receiving side's own
+# watcher (bin/fm-watch.sh parent_inbox_tick) and the parent's reply tracker
+# (bin/fm-pending-reply-lib.sh, through the remote inbox-status leg) read it.
+fm_task_inbox_unhandled_summary() {  # <state-dir> <task-id>
+  local dir best='' best_n=0 count=0 f n
+  dir=$(fm_task_inbox_dir "$1" "$2")
+  for f in "$dir"/*.msg; do
+    [ -e "$f" ] || continue
+    n=$(fm_task_inbox_seq_of "${f##*/}") || continue
+    count=$((count + 1))
+    if [ -z "$best" ] || [ "$n" -lt "$best_n" ]; then
+      best=$f
+      best_n=$n
+    fi
+  done
+  [ -n "$best" ] || return 1
+  printf '%s %s %s' "$count" "${best##*/}" "$(fm_path_age "$best")"
+}
+
 # Oldest escalation-tracked unhandled record, or fail when none is due.
 fm_task_inbox_oldest_unhandled() {  # <state-dir> <task-id>
   local dir best='' best_n=0 f n

bin/fm-test-run.sh

@@ -830,7 +830,7 @@ tests/fm-on.test.sh 11473
 tests/fm-opencode-primary-live-e2e.test.sh 47
 tests/fm-operational-input.test.sh 2404
 tests/fm-peek-remote.test.sh 1082
-tests/fm-pending-reply.test.sh 41090
+tests/fm-pending-reply.test.sh 46200
 tests/fm-pi-branch-extension.test.sh 77218
 tests/fm-pi-branch-live-e2e.test.sh 48
 tests/fm-pi-branch-responsiveness-live-e2e.test.sh 12834
@@ -878,7 +878,7 @@ tests/fm-secondmate-sync.test.sh 55501
 tests/fm-send-agy-confirm.test.sh 4440
 tests/fm-send-inbox-doorbell-live-e2e.test.sh 108
 tests/fm-send-inbox.test.sh 41713
-tests/fm-send-remote-delivery.test.sh 31964
+tests/fm-send-remote-delivery.test.sh 66000
 tests/fm-send-resolve-key.test.sh 47317
 tests/fm-send-secondmate-marker-herdr-e2e.test.sh 80
 tests/fm-send-secondmate-marker.test.sh 7574
@@ -908,7 +908,7 @@ tests/fm-supervision-host-park.test.sh 329326
 tests/fm-supervision-host.test.sh 435581
 tests/fm-tangle-guard.test.sh 8501
 tests/fm-task-delivery.test.sh 32789
-tests/fm-task-inbox.test.sh 31965
+tests/fm-task-inbox.test.sh 35000
 tests/fm-tasks-axi.test.sh 2293
 tests/fm-teardown-endpoint-safety.test.sh 40851
 tests/fm-teardown.test.sh 202132

bin/fm-watch.sh

@@ -165,6 +165,13 @@
 #                          inbox and a fresh child beacon are not idle proof;
 #                          the foreign queue itself stays read-only, and one
 #                          parent notification covers each no-progress episode
+#   check: parent instruction waiting: <n> unhandled in <inbox>, oldest <s>s; ...
+#                          this home is a secondmate and its own steering inbox
+#                          from its parent (state/parent-route/<id>.inbox, or a
+#                          local mate's $FM_TASK_INBOX) holds a record older than
+#                          FM_TASK_INBOX_GRACE_SECS; queued once per oldest record
+#                          so a parked mate session wakes and reads it without
+#                          depending on a typed doorbell (parent_inbox_tick)
 #   check: secondmate <id> auto-relaunched after <cause> (<where>)
 #                          the liveness tick probed a registered secondmate's
 #                          recorded endpoint, got the recovery-grade `dead` or
@@ -1099,6 +1106,50 @@ EOF
   return 0
 }
 
+# A secondmate's own watcher is a wake source for the steering inbox its parent
+# writes: a parked mate session wakes on that queued check exactly as it does
+# for a child event, so reading a parent instruction never depends on composer
+# typing (a doorbell whose Enter never lands leaves an idle mate deaf, and the
+# remote host runs no re-ring ladder). The inbox is the remote placement's
+# state/parent-route/<id>.inbox in this home, else a local mate's own
+# $FM_TASK_INBOX when it names this home's id. Every unhandled record counts,
+# fire-and-forget included, once the oldest is past the inbox grace; one wake
+# per oldest record (.parent-inbox-woken), queued before that marker is written
+# so a crash can only duplicate a wake, never lose one. The inbox itself stays
+# read-only here: reading and acknowledging records is the woken session's job.
+parent_inbox_tick() {
+  local id inbox_state summary count oldest age woken_marker="$STATE/.parent-inbox-woken"
+  local key reason queued
+  [ -f "$FM_HOME/.fm-secondmate-home" ] && [ ! -L "$FM_HOME/.fm-secondmate-home" ] || return 0
+  id=$(cat "$FM_HOME/.fm-secondmate-home" 2>/dev/null || true)
+  case "$id" in ''|*[!A-Za-z0-9._-]*) return 0 ;; esac
+  inbox_state="$STATE/parent-route"
+  if [ ! -d "$(fm_task_inbox_dir "$inbox_state" "$id")" ]; then
+    case "${FM_TASK_INBOX:-}" in
+      /*/"$id.inbox") [ -d "$FM_TASK_INBOX" ] || return 0; inbox_state=${FM_TASK_INBOX%/*} ;;
+      *) return 0 ;;
+    esac
+  fi
+  if ! summary=$(fm_task_inbox_unhandled_summary "$inbox_state" "$id"); then
+    rm -f "$woken_marker" 2>/dev/null || true
+    return 0
+  fi
+  read -r count oldest age <<EOF
+$summary
+EOF
+  case "$age" in ''|*[!0-9]*) return 0 ;; esac
+  [ "$age" -ge "$(fm_task_inbox_grace_secs)" ] || return 0
+  [ "$(cat "$woken_marker" 2>/dev/null || true)" != "$oldest" ] || return 0
+  key="parent-inbox-$id-${oldest%.msg}"
+  reason="check: parent instruction waiting: $count unhandled in $(fm_task_inbox_dir "$inbox_state" "$id"), oldest ${age}s; read and act on each in numeric order, then mv each into its handled/"
+  queued=$(fm_wake_queued_keys check)
+  if ! printf '%s\n' "$queued" | grep -Fx "$key" >/dev/null 2>&1; then
+    fm_wake_append check "$key" "$reason" || return 1
+  fi
+  printf '%s\n' "$oldest" > "$woken_marker" || return 1
+  wake "$reason"
+}
+
 # The ordinary-supervision half of the secondmate liveness guarantee, paired
 # with bin/fm-bootstrap.sh's session-start sweep over the shared library in
 # bin/fm-secondmate-liveness-lib.sh (which owns the state contract, the remote
@@ -3620,6 +3671,12 @@ while :; do
     exit 1
   }
 
+  # A secondmate home wakes its own session for an unread parent instruction.
+  parent_inbox_tick || {
+    echo "watcher: parent-route inbox wake could not be recorded" >&2
+    exit 1
+  }
+
   # Process-to-event liveness repair. This never discovers a result by polling:
   # each registered source has its own child blocking on that source, and this
   # only republishes results already captured durably and restarts a source