fix(bin): reconcile remote secondmate delivery and wake mates on unread parent instructions

bin/fm-classify-lib.sh

@@ -815,7 +815,7 @@ _fm_decision_key_transition_allowed() {  # <key> <note>
 _fm_is_pending_reply_escalation() {  # <key> <note>
   case "$1" in pending-reply-*) ;; *) return 1 ;; esac
   case "$2" in
-    pending-reply-missed:*|pending-reply-delivery-unknown:*|pending-reply-recovery-delivery-failed:*|pending-reply-recovery-delivery-unknown:*) return 0 ;;
+    pending-reply-missed:*|pending-reply-delivery-unknown:*|pending-reply-unread:*|pending-reply-recovery-delivery-failed:*|pending-reply-recovery-delivery-unknown:*) return 0 ;;
     *) return 1 ;;
   esac
 }

bin/fm-remote-secondmate-control.sh

@@ -11,6 +11,7 @@
 #   fm-remote-secondmate-control.sh key <id> <key>
 #   fm-remote-secondmate-control.sh capture <id> [lines]
 #   fm-remote-secondmate-control.sh observe <id>
+#   fm-remote-secondmate-control.sh inbox-status <id> [<corr-id>]
 #   fm-remote-secondmate-control.sh sync <id> [<parent-commit>]
 #   fm-remote-secondmate-control.sh update <id>
 #   fm-remote-secondmate-control.sh retire <id> [--force]
@@ -63,7 +64,7 @@ CONTROL_DATA="$TARGET_HOME/data/.parent-route"
 REMOTE_HERDR_SESSION=fm-remote
 
 die() { printf 'error: %s\n' "$1" >&2; exit 1; }
-usage() { sed -n '2,24p' "$0" | sed 's/^# \{0,1\}//'; exit 2; }
+usage() { sed -n '2,25p' "$0" | sed 's/^# \{0,1\}//'; exit 2; }
 validate_id() { case "$1" in ''|*[!A-Za-z0-9._-]*) die "invalid secondmate id: $1" ;; esac; }
 
 # Sourcing fm-wake-lib.sh through the libraries below creates "$FM_HOME/state",
@@ -365,6 +366,29 @@ cmd_observe() {
   printf '\n'
 }
 
+# Read-only delivery facts for the parent's reply tracker: how many records sit
+# unhandled in this secondmate's host-local steering inbox and how old the
+# oldest is, where one correlated request stands there (unhandled, handled, or
+# absent), and the recorded pane's composer verdict. The parent cannot read this
+# inbox itself; bin/fm-pending-reply-lib.sh owns the output format and what the
+# tracker does with it. An unreadable endpoint reports composer=unknown rather
+# than failing, because the inbox facts stand on their own.
+cmd_inbox_status() {
+  local id=$1 corr=${2:-} composer=unknown
+  validate_id "$id"
+  case "$corr" in
+    '') ;;
+    *[!0-9a-f]*) die "invalid correlation id: $corr" ;;
+    *) [ "${#corr}" -eq 16 ] || die "invalid correlation id: $corr" ;;
+  esac
+  validate_home "$id"
+  if [ -f "$(meta_path "$id")" ] && remote_endpoint_load "$id"; then
+    composer=$(fm_backend_composer_state "$REMOTE_ENDPOINT_BACKEND" "$REMOTE_ENDPOINT_TARGET" "fm-$id" 2>/dev/null) \
+      || composer=unknown
+  fi
+  fm_pending_reply_inbox_status_report "$CONTROL_STATE" "$id" "$corr" "$composer"
+}
+
 # Make <commit> readable in this home's own object store without moving any other
 # checkout. Ordered by cost: already present, then this host's Firstmate copy (a
 # read-only fetch of that one commit, which never advances that copy's HEAD), then
@@ -465,6 +489,7 @@ case "${1:-}" in
   key) shift; [ "$#" -eq 2 ] || usage; cmd_key "$@" ;;
   capture) shift; [ "$#" -ge 1 ] && [ "$#" -le 2 ] || usage; cmd_capture "$@" ;;
   observe) shift; [ "$#" -eq 1 ] || usage; cmd_observe "$@" ;;
+  inbox-status) shift; [ "$#" -ge 1 ] && [ "$#" -le 2 ] || usage; cmd_inbox_status "$@" ;;
   sync) shift; [ "$#" -ge 1 ] && [ "$#" -le 2 ] || usage; cmd_sync "$@" ;;
   update) shift; [ "$#" -eq 1 ] || usage; cmd_update "$@" ;;
   retire) shift; [ "$#" -ge 1 ] && [ "$#" -le 2 ] || usage; cmd_retire "$@" ;;

bin/fm-send.sh

@@ -145,10 +145,12 @@
 # seconds (default 30, and any override must be a positive integer): a bound
 # hit is completion-unknown and exits through this same unconfirmed contract
 # instead of waiting out a busy remote queue.
-# The remote host runs no re-ring ladder of its own: a swallowed ordinary
-# doorbell surfaces through the parent's pending-reply recovery and escalation,
-# whose recovery request re-rings the remote doorbell when it is enqueued;
-# fire-and-forget delivery deliberately arms neither mechanism. Internal
+# The remote host runs no re-ring ladder of its own: the mate's own watcher
+# wakes its session for any unhandled record there (bin/fm-watch.sh
+# parent_inbox_tick), and a still-unread ordinary request surfaces through the
+# parent's pending-reply tracker, which reads the remote inbox and escalates it
+# as unread (bin/fm-pending-reply-lib.sh); a delivery left unknown here is
+# reconciled against that inbox the same way once the mate is reachable. Internal
 # semantic callers may set FM_SEND_EXPECTED_SPAWN_GEN or
 # FM_SEND_EXPECTED_REMOTE_HOST to require that sampled identity to still match
 # during the final locked remote-route validation; unset or empty guards do not
@@ -999,6 +1001,12 @@ else
       fi
       if [ -n "$PENDING_REPLY_CORR" ]; then
         fm_pending_reply_mark_delivery_unknown "$STATE" "$PENDING_REPLY_CORR" || true
+        # Kept for the tracker's one same-correlation resend once the mate is
+        # reachable and its inbox proves the record never landed. A keyed
+        # answer stores none: only the printed resend closes its keys, so the
+        # tracker leaves that record to it.
+        [ -n "$RESOLVE_KEYS" ] ||
+          fm_pending_reply_store_undelivered_text "$STATE" "$PENDING_REPLY_CORR" "$RESOLVE_ANSWER_TEXT" || true
       fi
       if [ "$remote_rc" -eq 255 ]; then
         echo "error: steer to remote secondmate $TARGET_REMOTE_ID is unconfirmed (transport lost twice; remote completion unknown). Only the correlation-reusing resend below is idempotent and lands on the same remote inbox record:" >&2

bin/fm-task-inbox-lib.sh

@@ -33,6 +33,10 @@
 #                              so later polls suppress another escalation
 #   <task>.inbox/.retry-ring   name of a fire-and-forget record still owed its
 #                              one retry ring (fm_task_inbox_mark_retry)
+#   <task>.inbox/.ring-log     bounded diagnostic log of every ring decision:
+#                              "<utc>\t<msg>\t<decision>\t<detail>", decision one
+#                              of rang, unconfirmed, skipped:other-text,
+#                              skipped:busy, failed, dead (fm_task_inbox_ring)
 #
 # Record format (fm_task_inbox_write / fm_task_inbox_body):
 #   schema=fm-task-inbox.v1
@@ -80,7 +84,8 @@
 # without this retry the record could sit unread until a checkpoint. A pending ordinary record's
 # ladder rings the same inbox, so the retry waits behind it, and an
 # acknowledged record drops its mark. The remote steer leg has no watcher
-# ladder and owes no retry.
+# ladder and owes no retry; there the receiving mate's own watcher wakes its
+# session for any unhandled record instead (bin/fm-watch.sh parent_inbox_tick).
 #
 # Inbox names containing bytes outside printable ASCII are unsupported. The
 # doorbell refuses them rather than sending terminal control bytes to a pane.
@@ -115,6 +120,13 @@ fm_task_inbox_grace_secs() {
   printf '%s' "$g"
 }
 
+# The watcher poll interval the ladder runs at (bin/fm-watch.sh FM_POLL).
+fm_task_inbox_poll_secs() {
+  local p=${FM_POLL:-15}
+  case "$p" in ''|*[!0-9]*|0) p=15 ;; esac
+  printf '%s' "$p"
+}
+
 fm_task_inbox_ring_max() {
   local m=${FM_TASK_INBOX_RING_MAX:-$FM_TASK_INBOX_RING_MAX_DEFAULT}
   case "$m" in ''|*[!0-9]*) m=$FM_TASK_INBOX_RING_MAX_DEFAULT ;; esac
@@ -386,24 +398,44 @@ fm_task_inbox_doorbell_line() {  # <record-path>
 # whose Enter never landed, so on an agent not reported busy it is submitted
 # rather than skipped; skipping it would block every later ring. On both paths
 # a lost first Enter gets one confirmed retry.
+# Every decision is appended to the inbox's .ring-log (see the header), because
+# no caller records why a ring was skipped and a stuck doorbell is otherwise
+# undiagnosable after the fact; the log never changes the return value.
 fm_task_inbox_ring() {  # <backend> <target> <record-path> [expected-label]
-  local backend=$1 target=$2 rec=$3 label=${4:-} line cstate verdict
-  case "$(fm_backend_agent_state "$backend" "$target" 2>/dev/null || true)" in
-    dead|missing) return 3 ;;
+  local backend=$1 target=$2 rec=$3 label=${4:-} line cstate verdict astate
+  astate=$(fm_backend_agent_state "$backend" "$target" 2>/dev/null || true)
+  case "$astate" in
+    dead|missing) _fm_task_inbox_ring_log "$rec" dead "agent=$astate"; return 3 ;;
   esac
   if ! line=$(fm_task_inbox_doorbell_line "$rec"); then
+    _fm_task_inbox_ring_log "$rec" failed "doorbell-line-refused"
     return 2
   fi
   cstate=$(fm_backend_composer_state "$backend" "$target" "$label" 2>/dev/null) || cstate=unknown
   case "$cstate" in
     pending)
-      fm_task_inbox_composer_holds "$backend" "$target" "$line" "$label" \
-        && [ "$(fm_backend_busy_state "$backend" "$target" 2>/dev/null)" != busy ] \
-        || return 1
-      fm_backend_send_key "$backend" "$target" Enter "$label" >/dev/null 2>&1 || return 2
+      if ! fm_task_inbox_composer_holds "$backend" "$target" "$line" "$rec" "$label"; then
+        _fm_task_inbox_ring_log "$rec" skipped:other-text "composer=pending"
+        return 1
+      fi
+      if [ "$(fm_backend_busy_state "$backend" "$target" 2>/dev/null)" = busy ]; then
+        _fm_task_inbox_ring_log "$rec" skipped:busy "composer=own-doorbell"
+        return 1
+      fi
+      if ! fm_backend_send_key "$backend" "$target" Enter "$label" >/dev/null 2>&1; then
+        _fm_task_inbox_ring_log "$rec" failed "composer=own-doorbell enter"
+        return 2
+      fi
       sleep 0.3
-      fm_task_inbox_composer_holds "$backend" "$target" "$line" "$label" || return 0
-      fm_backend_send_key "$backend" "$target" Enter "$label" >/dev/null 2>&1 || return 2
+      if ! fm_task_inbox_composer_holds "$backend" "$target" "$line" "$rec" "$label"; then
+        _fm_task_inbox_ring_log "$rec" rang "composer=own-doorbell submitted"
+        return 0
+      fi
+      if ! fm_backend_send_key "$backend" "$target" Enter "$label" >/dev/null 2>&1; then
+        _fm_task_inbox_ring_log "$rec" failed "composer=own-doorbell second-enter"
+        return 2
+      fi
+      _fm_task_inbox_ring_log "$rec" unconfirmed "composer=own-doorbell second-enter"
       return 0
       ;;
   esac
@@ -412,19 +444,94 @@ fm_task_inbox_ring() {  # <backend> <target> <record-path> [expected-label]
   # shell only a suffix; the `: ` prefix protects complete lines only. Do not
   # add process-bound atomic delivery here unless an incident reopens this.
   if ! verdict=$(fm_backend_send_text_submit "$backend" "$target" "$line" 2 0.4 0.3 "$label" 2>/dev/null); then
+    _fm_task_inbox_ring_log "$rec" failed "composer=$cstate submit"
     return 2
   fi
   # The verdict is read only to report a failed keystroke; every other value
   # (empty, pending, unknown, ...) is deliberately ignored, never proof.
-  [ "$verdict" != send-failed ] || return 2
+  if [ "$verdict" = send-failed ]; then
+    _fm_task_inbox_ring_log "$rec" failed "composer=$cstate verdict=send-failed"
+    return 2
+  fi
+  if [ "$verdict" = empty ]; then
+    _fm_task_inbox_ring_log "$rec" rang "composer=$cstate verdict=empty"
+  else
+    _fm_task_inbox_ring_log "$rec" unconfirmed "composer=$cstate verdict=${verdict:-none}"
+  fi
+  return 0
+}
+
+# Append one ring decision to the inbox's .ring-log, best-effort and bounded
+# (trimmed to its newest 200 lines past 64 KiB). Never fails its caller.
+_fm_task_inbox_ring_log() {  # <record-path> <decision> [detail]
+  local dir=${1%/*} log sz
+  dir=${dir%/handled}
+  [ -d "$dir" ] || return 0
+  log="$dir/.ring-log"
+  printf '%s\t%s\t%s\t%s\n' "$(date -u +%Y-%m-%dT%H:%M:%SZ)" "${1##*/}" "$2" "${3:-}" \
+    >> "$log" 2>/dev/null || return 0
+  sz=$(wc -c < "$log" 2>/dev/null | tr -d '[:space:]')
+  case "$sz" in ''|*[!0-9]*) return 0 ;; esac
+  if [ "$sz" -ge 65536 ]; then
+    tail -n 200 "$log" > "$log.tmp.$$" 2>/dev/null && mv -f "$log.tmp.$$" "$log" 2>/dev/null
+    rm -f "$log.tmp.$$" 2>/dev/null
+  fi
   return 0
 }
 
-# Whether the composer's content, ignoring line wrapping, is exactly <line>.
-fm_task_inbox_composer_holds() {  # <backend> <target> <line> [expected-label]
+# Whether a Herdr composer read <held> is our own doorbell <line> for
+# <record-path>: equal to it ignoring whitespace and wrapping, or the shape
+# Claude leaves when it collapses the burst typed by the Herdr typing path - one
+# bare `[Pasted text #N]` placeholder, optionally followed by a literal tail that
+# is the end of <line>. A placeholder hides its content, so an operator's paste
+# has the same shape: it is recognised only while the inbox's .ring-log already
+# shows a recent earlier ring of this record that typed the doorbell (rang or
+# unconfirmed). A `+M lines` placeholder or several placeholders are an
+# operator's paste, not a single-line doorbell, and are never recognised.
+_fm_task_inbox_herdr_holds_doorbell() {  # <line> <held> <record-path>
+  local line=$1 held=$2 rec=$3 rest dir now window cutoff
+  fm_composer_normalize_spaces_var line
+  fm_composer_normalize_spaces_var held
+  line=${line//[$' \t\r\n\v\f']/}
+  line=${line//$'\xE2\x81\xA3'/}
+  held=${held//[$' \t\r\n\v\f']/}
+  held=${held//$'\xE2\x81\xA3'/}
+  [ -n "$line" ] && [ -n "$held" ] || return 1
+  [ "$held" = "$line" ] && return 0
+  [[ $held =~ ^\[Pastedtext#[0-9]+\](.*)$ ]] || return 1
+  rest=${BASH_REMATCH[1]}
+  case "$rest" in *'[Pastedtext#'*) return 1 ;; esac
+  [ -z "$rest" ] || [[ $line == *"$rest" ]] || return 1
+  dir=${rec%/*}
+  dir=${dir%/handled}
+  # Recent means within one re-ring spacing (the inbox grace) plus three watcher
+  # passes (FM_POLL, default 15 s): the next re-ring of a doorbell whose Enter
+  # was lost falls inside it, an operator paste long after the last ring does
+  # not. The log's UTC stamps sort as text.
+  now=$(date +%s)
+  window=$(( $(fm_task_inbox_grace_secs) + 3 * $(fm_task_inbox_poll_secs) ))
+  cutoff=$(date -u -r "$((now - window))" +%Y-%m-%dT%H:%M:%SZ 2>/dev/null \
+    || date -u -d "@$((now - window))" +%Y-%m-%dT%H:%M:%SZ 2>/dev/null) || return 1
+  awk -F '\t' -v msg="${rec##*/}" -v cutoff="$cutoff" '
+    $2 == msg && ($3 == "rang" || $3 == "unconfirmed") && $1 >= cutoff { found = 1; exit }
+    END { exit(found ? 0 : 1) }
+  ' "$dir/.ring-log" 2>/dev/null
+}
+
+# Whether the composer's content is exactly <line>, ignoring line wrapping.
+# Herdr reads its composer with the same visible-viewport reader its typing
+# path uses (fm_backend_herdr_composer_content), so a doorbell that path typed
+# - wrapped, or collapsed into a placeholder - is recognised as our own rather
+# than skipped as someone else's draft. Other backends compare a plain capture.
+fm_task_inbox_composer_holds() {  # <backend> <target> <line> <record-path> [expected-label]
   local cap held
   fm_backend_source "$1" || return 1
-  cap=$(fm_backend_capture "$1" "$2" "$FM_COMPOSER_CAPTURE_LINES" "${4:-}" 2>/dev/null) || return 1
+  if [ "$1" = herdr ]; then
+    held=$(fm_backend_herdr_composer_content "$2" 2>/dev/null) || return 1
+    _fm_task_inbox_herdr_holds_doorbell "$3" "$held" "$4"
+    return
+  fi
+  cap=$(fm_backend_capture "$1" "$2" "$FM_COMPOSER_CAPTURE_LINES" "${5:-}" 2>/dev/null) || return 1
   held=$(fm_composer_extract_selected_content styled=0 "$cap") || return 1
   [ -n "$held" ] && [ "$(printf '%s' "$held" | tr -d '[:space:]')" = "$(printf '%s' "$3" | tr -d '[:space:]')" ]
 }
@@ -442,6 +549,27 @@ fm_task_inbox_is_fire_and_forget() {  # <record-path>
   ' "$rec"
 }
 
+# Every unhandled record, ordinary and fire-and-forget alike: prints
+# "<count> <oldest-basename> <oldest-age-seconds>" with the oldest chosen by
+# sequence, or fails when the inbox holds none. The receiving side's own
+# watcher (bin/fm-watch.sh parent_inbox_tick) and the parent's reply tracker
+# (bin/fm-pending-reply-lib.sh, through the remote inbox-status leg) read it.
+fm_task_inbox_unhandled_summary() {  # <state-dir> <task-id>
+  local dir best='' best_n=0 count=0 f n
+  dir=$(fm_task_inbox_dir "$1" "$2")
+  for f in "$dir"/*.msg; do
+    [ -e "$f" ] || continue
+    n=$(fm_task_inbox_seq_of "${f##*/}") || continue
+    count=$((count + 1))
+    if [ -z "$best" ] || [ "$n" -lt "$best_n" ]; then
+      best=$f
+      best_n=$n
+    fi
+  done
+  [ -n "$best" ] || return 1
+  printf '%s %s %s' "$count" "${best##*/}" "$(fm_path_age "$best")"
+}
+
 # Oldest escalation-tracked unhandled record, or fail when none is due.
 fm_task_inbox_oldest_unhandled() {  # <state-dir> <task-id>
   local dir best='' best_n=0 f n

bin/fm-teardown.sh

@@ -1090,8 +1090,8 @@ remote_recovery_paths_validate() {
 }
 
 # Remove every parent pending-reply record for $ID, plus its delivery
-# confirmation when present. Shared by local and remote secondmate retirement
-# after the home/route is safely gone.
+# confirmation and saved undelivered text when present. Shared by local and
+# remote secondmate retirement after the home/route is safely gone.
 pending_replies_cleanup_for_task() {
   local pending_dir=$1 expected_real=${2-} rec base corr task_id
   [ -d "$pending_dir" ] || return 0
@@ -1109,7 +1109,7 @@ pending_replies_cleanup_for_task() {
       printf '%s' "$base" | grep -Eq '^[a-f0-9]{16}$' || exit 1
       corr=$(fm_meta_get "$rec" corr_id)
       [ -z "$corr" ] || [ "$corr" = "$base" ] || exit 1
-      rm -f -- "./.delivery-confirmed-$base" "$rec" || exit 1
+      rm -f -- "./.delivery-confirmed-$base" "./.undelivered-text-$base" "$rec" || exit 1
     done
   )
 }

bin/fm-test-run.sh

@@ -839,7 +839,7 @@ tests/fm-on.test.sh 11473
 tests/fm-opencode-primary-live-e2e.test.sh 47
 tests/fm-operational-input.test.sh 2404
 tests/fm-peek-remote.test.sh 1082
-tests/fm-pending-reply.test.sh 41090
+tests/fm-pending-reply.test.sh 46200
 tests/fm-pi-branch-extension.test.sh 77218
 tests/fm-pi-branch-live-e2e.test.sh 48
 tests/fm-pi-branch-responsiveness-live-e2e.test.sh 12834
@@ -889,7 +889,7 @@ tests/fm-secondmate-sync.test.sh 55501
 tests/fm-send-agy-confirm.test.sh 4440
 tests/fm-send-inbox-doorbell-live-e2e.test.sh 108
 tests/fm-send-inbox.test.sh 41713
-tests/fm-send-remote-delivery.test.sh 31964
+tests/fm-send-remote-delivery.test.sh 66000
 tests/fm-send-resolve-key.test.sh 47317
 tests/fm-send-secondmate-marker-herdr-e2e.test.sh 80
 tests/fm-send-secondmate-marker.test.sh 7574
@@ -919,7 +919,7 @@ tests/fm-supervision-host-park.test.sh 329326
 tests/fm-supervision-host.test.sh 435581
 tests/fm-tangle-guard.test.sh 8501
 tests/fm-task-delivery.test.sh 32789
-tests/fm-task-inbox.test.sh 31965
+tests/fm-task-inbox.test.sh 35000
 tests/fm-tasks-axi.test.sh 2293
 tests/fm-teardown-endpoint-safety.test.sh 40851
 tests/fm-teardown.test.sh 202132