Repository navigation
Expand file tree
/
Copy pathfm-control.sh
More file actions
executable file
·883 lines (831 loc) · 38.4 KB
/
Copy pathfm-control.sh
File metadata and controls
executable file
·883 lines (831 loc) · 38.4 KB
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
221
222
223
224
225
226
227
228
229
230
231
232
233
234
235
236
237
238
239
240
241
242
243
244
245
246
247
248
249
250
251
252
253
254
255
256
257
258
259
260
261
262
263
264
265
266
267
268
269
270
271
272
273
274
275
276
277
278
279
280
281
282
283
284
285
286
287
288
289
290
291
292
293
294
295
296
297
298
299
300
301
302
303
304
305
306
307
308
309
310
311
312
313
314
315
316
317
318
319
320
321
322
323
324
325
326
327
328
329
330
331
332
333
334
335
336
337
338
339
340
341
342
343
344
345
346
347
348
349
350
351
352
353
354
355
356
357
358
359
360
361
362
363
364
365
366
367
368
369
370
371
372
373
374
375
376
377
378
379
380
381
382
383
384
385
386
387
388
389
390
391
392
393
394
395
396
397
398
399
400
401
402
403
404
405
406
407
408
409
410
411
412
413
414
415
416
417
418
419
420
421
422
423
424
425
426
427
428
429
430
431
432
433
434
435
436
437
438
439
440
441
442
443
444
445
446
447
448
449
450
451
452
453
454
455
456
457
458
459
460
461
462
463
464
465
466
467
468
469
470
471
472
473
474
475
476
477
478
479
480
481
482
483
484
485
486
487
488
489
490
491
492
493
494
495
496
497
498
499
500
501
502
503
504
505
506
507
508
509
510
511
512
513
514
515
516
517
518
519
520
521
522
523
524
525
526
527
528
529
530
531
532
533
534
535
536
537
538
539
540
541
542
543
544
545
546
547
548
549
550
551
552
553
554
555
556
557
558
559
560
561
562
563
564
565
566
567
568
569
570
571
572
573
574
575
576
577
578
579
580
581
582
583
584
585
586
587
588
589
590
591
592
593
594
595
596
597
598
599
600
601
602
603
604
605
606
607
608
609
610
611
612
613
614
615
616
617
618
619
620
621
622
623
624
625
626
627
628
629
630
631
632
633
634
635
636
637
638
639
640
641
642
643
644
645
646
647
648
649
650
651
652
653
654
655
656
657
658
659
660
661
662
663
664
665
666
667
668
669
670
671
672
673
674
675
676
677
678
679
680
681
682
683
684
685
686
687
688
689
690
691
692
693
694
695
696
697
698
699
700
701
702
703
704
705
706
707
708
709
710
711
712
713
714
715
716
717
718
719
720
721
722
723
724
725
726
727
728
729
730
731
732
733
734
735
736
737
738
739
740
741
742
743
744
745
746
747
748
749
750
751
752
753
754
755
756
757
758
759
760
761
762
763
764
765
766
767
768
769
770
771
772
773
774
775
776
777
778
779
780
781
782
783
784
785
786
787
788
789
790
791
792
793
794
795
796
797
798
799
800
801
802
803
804
805
806
807
808
809
810
811
812
813
814
815
816
817
818
819
820
821
822
823
824
825
826
827
828
829
830
831
832
833
834
835
836
837
838
839
840
841
842
843
844
845
846
847
848
849
850
851
852
853
854
855
856
857
858
859
860
861
862
863
864
865
866
867
868
869
870
871
872
873
874
875
876
877
878
879
880
881
882
883
#!/usr/bin/env bash
# fm-control.sh - the CONTROL PLANE for a firstmate-owned agent: allowlisted
# lifecycle verbs addressed to an exact task id.
#
# Usage: fm-control.sh <task-id> interrupt
# fm-control.sh <task-id> exit
# fm-control.sh <task-id> relaunch [--harness <name>] [--model <name>]
# [--effort <level>]
# (--note <text> | --note-file <path>)
#
# Why this exists, and how it differs from fm-send.sh. bin/fm-send.sh is the
# DATA plane: conversational text for the agent to read, always routing-marked
# for a kind=secondmate target so the reply returns through the status path.
# That marking is right for a message and wrong for a lifecycle command - a
# marked "/quit" arrives as ordinary chat the agent reasons ABOUT instead of
# executing. This script is the control plane: semantic process control with a
# closed verb list, per-harness mechanics owned by an executable adapter
# (bin/fm-control-lib.sh) rather than improvised in agent prose, and a verified
# postcondition for every action. There is deliberately NO arbitrary-text and
# NO generic raw-key entry point here; fm-send remains the only way to send an
# agent something to read.
#
# interrupt Deliver the harness's verified interrupt sequence. The agent
# keeps running. Postcondition: delivery succeeded, the endpoint
# still exists, and the agent is still alive where the backend can
# classify that. Cancellation is confirmed only from an adapter-
# owned acknowledgement and otherwise reported unconfirmed. Busy
# state is never rewritten as proof of the action.
# exit Stop the agent, preserving its terminal endpoint, worktree, and
# every uncommitted change. Interrupts first when the task reads
# busy, then submits the harness's exit command. Postcondition:
# the backend's recovery-grade classifier reports the agent gone.
# Already-stopped is success (idempotent).
# relaunch Transactionally replace the running agent with a new one, in the
# SAME endpoint and SAME worktree, on the same or a newly chosen
# harness/model/effort - so switching harness is one ordinary use
# of this verb. An explicit `default` model or effort clears that
# axis for the replacement. With no explicit axis, a secondmate
# re-resolves its durable config/secondmate-harness pin (harness
# plus its optional model and effort tokens) exactly as any other
# respawn does, while a ship or scout keeps the exact adapter
# already recorded for it.
# A prefixed raw-command basename cannot reconstruct its launch
# command, so relaunch requires an explicit --harness for it.
# --note is required for a ship or scout, whose replacement
# inherits the local copy but none of the conversation; a
# secondmate reconciles its own home's records at startup, so its
# standing charter is never rewritten.
# Records a durable checkpoint and that note, exits the old agent,
# then delegates the launch to its single owner,
# bin/fm-spawn.sh --relaunch. A failure before publication keeps
# the prior durable record in place and reports the concrete
# state; it never leaves a half-transitioned task claiming to be
# running.
#
# Teardown and discard are NOT verbs here and never will be. `exit` stops an
# agent and preserves everything else; removing a worktree, killing an
# endpoint, or discarding work stays with bin/fm-teardown.sh, which owns the
# landed-work test.
#
# `resume` is not a verb: it is not deterministic across the verified adapters
# (bin/fm-control-lib.sh's header owns that reasoning). `relaunch` covers the
# same need for every adapter because the brief on disk, not a harness-private
# session, is the durable instruction.
#
# Targeting is EXACT: only a bare task id with a state/<id>.meta record in
# THIS home is accepted, and the record must pass the shared endpoint-identity
# validation (bin/fm-backend.sh's fm_backend_validate_task_endpoint). A legacy
# fm-<id> label, an explicit session:window endpoint, and a bare window name
# are all refused - a lifecycle command delivered to the wrong endpoint is far
# worse than a loud refusal.
#
# A remotely placed secondmate is refused by name: its agent runs on another
# host, so no postcondition this plane verifies could be read for it here.
#
# Fail-closed boundaries:
# - An unverified harness, or a harness whose control mechanics are unknown,
# is refused rather than guessed at.
# - A backend that cannot deliver the harness's interrupt key is refused
# (Orca's terminal API has no Escape).
# - `exit` and `relaunch` require a backend with a recovery-grade agent-state
# classifier (tmux, herdr), because without one the "the agent stopped"
# postcondition cannot be proven. zellij, orca, and cmux are refused rather
# than reported as successful blind.
# - An ambiguous or unreadable endpoint state refuses; only a positively
# classified state acts.
#
# Environment knobs (all bounded waits, seconds):
# FM_CONTROL_POLL poll interval for postcondition waits (0.5)
# FM_CONTROL_SETTLE_WAIT adapter acknowledgement wait after interrupt (5)
# FM_CONTROL_EXIT_WAIT alive->dead wait after the exit command (30)
# FM_CONTROL_LAUNCH_WAIT dead->alive wait after a relaunch (90)
# FM_CONTROL_EXIT_RETRIES Enter retries for the exit command (3)
set -eu
SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"
FM_ROOT="${FM_ROOT_OVERRIDE:-$(cd "$SCRIPT_DIR/.." && pwd)}"
usage() {
# The whole leading comment block, ending at the first non-comment line.
sed -n '2,${/^#/!q;p;}' "$0" | sed 's/^# \{0,1\}//'
}
case "${1:-}" in
-h|--help) usage; exit 0 ;;
esac
# shellcheck source=bin/fm-gate-refuse-lib.sh
. "$SCRIPT_DIR/fm-gate-refuse-lib.sh"
# Fail closed before any fleet mutation: a no-mistakes gate agent must never
# drive a crewmate's lifecycle (see bin/fm-gate-refuse-lib.sh).
fm_refuse_if_gate_agent
if [ -z "${FM_HOME+x}" ] || [ -z "${FM_HOME:-}" ]; then
echo "error: FM_HOME is not set; fm-control refuses to resolve a task without an explicit firstmate home" >&2
exit 1
fi
[ -d "$FM_HOME" ] || {
echo "error: FM_HOME '$FM_HOME' is not a directory" >&2
exit 1
}
STATE="${FM_STATE_OVERRIDE:-$FM_HOME/state}"
DATA="${FM_DATA_OVERRIDE:-$FM_HOME/data}"
[ -d "$STATE" ] || {
echo "error: state dir '$STATE' is missing; fm-control cannot resolve tasks for FM_HOME '$FM_HOME'" >&2
exit 1
}
# shellcheck source=bin/fm-backend.sh
. "$SCRIPT_DIR/fm-backend.sh"
# shellcheck source=bin/fm-busy-lib.sh
. "$SCRIPT_DIR/fm-busy-lib.sh"
# shellcheck source=bin/fm-control-lib.sh
. "$SCRIPT_DIR/fm-control-lib.sh"
# shellcheck source=bin/fm-pr-lib.sh
. "$SCRIPT_DIR/fm-pr-lib.sh"
# shellcheck source=bin/fm-wake-lib.sh
. "$SCRIPT_DIR/fm-wake-lib.sh"
POLL=${FM_CONTROL_POLL:-0.5}
SETTLE_WAIT=${FM_CONTROL_SETTLE_WAIT:-5}
EXIT_WAIT=${FM_CONTROL_EXIT_WAIT:-30}
LAUNCH_WAIT=${FM_CONTROL_LAUNCH_WAIT:-90}
EXIT_RETRIES=${FM_CONTROL_EXIT_RETRIES:-3}
die() { # <message>
echo "error: $1" >&2
exit 1
}
CONTROL_LOCK=
CONTROL_LOCK_HELD=0
RELAUNCH_ACTIVE=0
RELAUNCH_PHASE=start
control_cleanup() {
local status=$?
if [ "$RELAUNCH_ACTIVE" = 1 ] \
&& declare -F relaunch_rollback >/dev/null 2>&1; then
relaunch_rollback || true
fi
if [ "$CONTROL_LOCK_HELD" = 1 ]; then
CONTROL_LOCK_HELD=0
fm_lock_release "$CONTROL_LOCK" || true
fi
if declare -F fm_lease_guard_release >/dev/null 2>&1; then
fm_lease_guard_release || true
fi
return "$status"
}
# --- argument parsing -------------------------------------------------------
RAW_ID=${1:-}
VERB=${2:-}
[ -n "$RAW_ID" ] && [ -n "$VERB" ] || { usage >&2; exit 2; }
shift 2
if ! fm_control_verb_allowed "$VERB"; then
{
if [ "$VERB" = resume ]; then
echo "error: 'resume' is not a control verb: resuming an exited agent is not deterministic across the verified adapters (codex and grok need a session id printed at exit, opencode continues the most recent session for the cwd, and claude, pi, pi-signed, and kimi have no verified pane-resume contract). Use 'relaunch', which carries the brief plus a progress note into a fresh agent on any adapter."
else
echo "error: '$VERB' is not a control verb"
fi
echo "allowed verbs:"
fm_control_verbs | sed 's/^/ /'
} >&2
exit 2
fi
NEW_HARNESS=
NEW_MODEL=
NEW_EFFORT=
HARNESS_SET=0
MODEL_SET=0
EFFORT_SET=0
NOTE=
NOTE_SET=0
control_want_value=
for control_arg in "$@"; do
if [ -n "$control_want_value" ]; then
case "$control_arg" in
--*) die "--$control_want_value requires a value" ;;
esac
case "$control_want_value" in
harness) NEW_HARNESS=$control_arg; HARNESS_SET=1 ;;
model) NEW_MODEL=$control_arg; MODEL_SET=1 ;;
effort) NEW_EFFORT=$control_arg; EFFORT_SET=1 ;;
note) NOTE=$control_arg; NOTE_SET=1 ;;
note_file)
[ -f "$control_arg" ] || die "--note-file '$control_arg' is not a readable file"
NOTE=$(cat "$control_arg")
NOTE_SET=1
;;
esac
control_want_value=
continue
fi
case "$control_arg" in
--harness) control_want_value=harness ;;
--harness=*) NEW_HARNESS=${control_arg#--harness=}; HARNESS_SET=1 ;;
--model) control_want_value=model ;;
--model=*) NEW_MODEL=${control_arg#--model=}; MODEL_SET=1 ;;
--effort) control_want_value=effort ;;
--effort=*) NEW_EFFORT=${control_arg#--effort=}; EFFORT_SET=1 ;;
--note) control_want_value=note ;;
--note=*) NOTE=${control_arg#--note=}; NOTE_SET=1 ;;
--note-file) control_want_value=note_file ;;
--note-file=*)
[ -f "${control_arg#--note-file=}" ] || die "--note-file '${control_arg#--note-file=}' is not a readable file"
NOTE=$(cat "${control_arg#--note-file=}")
NOTE_SET=1
;;
*) die "unexpected argument '$control_arg'" ;;
esac
done
if [ -n "$control_want_value" ]; then
[ "$control_want_value" = note_file ] && die "--note-file requires a value"
die "--$control_want_value requires a value"
fi
if [ "$VERB" != relaunch ]; then
[ "$HARNESS_SET" = 0 ] && [ "$MODEL_SET" = 0 ] && [ "$EFFORT_SET" = 0 ] && [ "$NOTE_SET" = 0 ] \
|| die "--harness, --model, --effort, and --note apply to 'relaunch' only"
fi
[ "$HARNESS_SET" = 0 ] || [ -n "$NEW_HARNESS" ] || die "--harness requires a non-empty value"
[ "$MODEL_SET" = 0 ] || [ -n "$NEW_MODEL" ] || die "--model requires a non-empty value"
[ "$EFFORT_SET" = 0 ] || [ -n "$NEW_EFFORT" ] || die "--effort requires a non-empty value"
case "$NEW_EFFORT" in
''|default|low|medium|high|xhigh|max|ultra) ;;
*) die "--effort must be one of default, low, medium, high, xhigh, max, ultra" ;;
esac
# --- exact task-id resolution ----------------------------------------------
case "$RAW_ID" in
*:*) die "'$RAW_ID' is an explicit backend endpoint; fm-control accepts an exact task id only, so a lifecycle command can never land on an endpoint this home does not own" ;;
esac
if ! fm_task_id_creation_valid "$RAW_ID"; then
die "'$RAW_ID' is not a valid task id"
fi
ID=$RAW_ID
# Supervision lease guard: lifecycle control is overlap territory between the
# two Pi supervision actors; refuse while the OTHER actor holds this task's
# live lease (contract: bin/fm-lease-lib.sh; no-op in homes without leases).
# shellcheck source=bin/fm-lease-lib.sh
. "$SCRIPT_DIR/fm-lease-lib.sh"
fm_lease_guard "$ID" "lifecycle control (fm-control)"
CONTROL_LOCK="$STATE/.control-$ID.lock"
trap control_cleanup EXIT
fm_lock_try_acquire "$CONTROL_LOCK" \
|| die "another lifecycle action is already running for task $ID"
CONTROL_LOCK_HELD=1
META="$STATE/$ID.meta"
if [ ! -f "$META" ]; then
case "$RAW_ID" in
fm-*)
if [ -f "$STATE/${RAW_ID#fm-}.meta" ]; then
die "'$RAW_ID' is a window label, not a task id; pass the exact task id '${RAW_ID#fm-}'"
fi
;;
esac
die "no task '$ID' in $STATE (fm-control resolves an exact task id only)"
fi
# A remotely placed secondmate records its endpoint on ANOTHER host, so every
# postcondition this plane verifies - the agent-state classification, the busy
# verdict, the endpoint's existence - would be read here for an endpoint that
# does not live here. Endpoint validation already refuses such a record, since
# `window=remote:<id>` can never match a local backend's required shape, so
# nothing can be delivered to a wrong endpoint either way. What that refusal
# cannot say is WHY, and "malformed metadata" is the wrong thing to tell an
# operator about a correctly configured remote route. Name the placement
# instead, using the same `remote_host` signal bin/fm-send.sh routes on.
if [ -n "$(fm_meta_get "$META" remote_host)" ]; then
die "task $ID is a remotely placed secondmate on $(fm_meta_get "$META" remote_host); its agent runs outside this home, so no lifecycle action here could verify that it interrupted, stopped, or came back. Drive its lifecycle on that host, and reconcile it through the secondmate recovery path rather than this plane"
fi
fm_backend_validate_task_endpoint "$META" "$ID" || exit 1
BACKEND=$FM_BACKEND_VALIDATED_BACKEND
T=$FM_BACKEND_VALIDATED_TARGET
LABEL="fm-$ID"
RECORDED_HARNESS=$(fm_meta_get "$META" harness)
KIND=$(fm_meta_get "$META" kind)
WT=$(fm_meta_get "$META" worktree)
[ -n "$KIND" ] || KIND=ship
HARNESS=$(fm_control_harness_family "$RECORDED_HARNESS") \
|| die "task $ID records harness '${RECORDED_HARNESS:-none}', which has no verified control mechanics; fm-control refuses to guess an interrupt key or exit command"
fm_control_harness_supported "$HARNESS" \
|| die "task $ID records harness '${RECORDED_HARNESS:-none}', which has no verified control mechanics; fm-control refuses to guess an interrupt key or exit command"
fm_backend_validate "$BACKEND" || exit 1
# --- shared helpers ---------------------------------------------------------
agent_state() {
fm_backend_agent_state "$BACKEND" "$T"
}
busy_verdict() {
fm_busy_classify_meta "$META" "$ID" "$STATE"
}
# wait_agent_state <wanted...> <timeout>: poll until agent_state prints one of
# the wanted values. Prints the final observed state; returns 0 on a match.
wait_agent_state() { # <timeout> <wanted>...
local timeout=$1 state want elapsed=0
shift
while :; do
state=$(agent_state)
for want in "$@"; do
if [ "$state" = "$want" ]; then
printf '%s' "$state"
return 0
fi
done
awk -v e="$elapsed" -v t="$timeout" 'BEGIN{exit !(e < t)}' || break
sleep "$POLL"
elapsed=$(awk -v e="$elapsed" -v p="$POLL" 'BEGIN{printf "%.3f", e + p}')
done
printf '%s' "$state"
return 1
}
require_state_verified_backend() { # <verb>
fm_control_backend_state_verified "$BACKEND" && return 0
die "task $ID runs on the $BACKEND backend, which has no recovery-grade agent-state classifier, so '$1' cannot prove the agent actually stopped; refusing rather than reporting an unproven transition as done"
}
# send_interrupt_keys: deliver the harness's interrupt key the verified number
# of times, then the composer-clear key when the adapter needs one. Refuses
# before sending anything when the backend cannot deliver either key, because
# an interrupt that cancels the turn but leaves the restored prompt in the
# composer would make the next submitted line concatenate onto it.
send_interrupt_keys() {
local key repeat clear i=0
key=$(fm_control_interrupt_key "$HARNESS")
repeat=$(fm_control_interrupt_repeat "$HARNESS")
clear=$(fm_control_interrupt_clear_key "$HARNESS")
fm_control_backend_supports_key "$BACKEND" "$key" \
|| die "harness $HARNESS interrupts with $key, which the $BACKEND backend cannot deliver; refusing to send a different key"
[ -z "$clear" ] || fm_control_backend_supports_key "$BACKEND" "$clear" \
|| die "harness $HARNESS needs $clear to clear its composer after an interrupt, which the $BACKEND backend cannot deliver; refusing to leave the cancelled prompt where the next submitted line would concatenate onto it"
while [ "$i" -lt "$repeat" ]; do
fm_backend_send_key "$BACKEND" "$T" "$key" "$LABEL" \
|| die "interrupt key $key was not delivered to task $ID on $BACKEND"
i=$((i + 1))
[ "$i" -ge "$repeat" ] || sleep 0.2
done
[ -z "$clear" ] || fm_backend_send_key "$BACKEND" "$T" "$clear" "$LABEL" \
|| die "interrupt key $key reached task $ID, but $clear did not, so its composer still holds the cancelled prompt; clear it before the next lifecycle action"
}
prepare_interrupt_ack() {
INTERRUPT_ACK_SOURCE=$(fm_control_interrupt_ack_source "$HARNESS")
INTERRUPT_ACK_LOG=
INTERRUPT_ACK_RUN=
case "$INTERRUPT_ACK_SOURCE" in
muse-session-terminal)
INTERRUPT_ACK_LOG=$(fm_busy_muse_session_log "$STATE" "$ID" 2>/dev/null || true)
[ -n "$INTERRUPT_ACK_LOG" ] || return 0
INTERRUPT_ACK_RUN=$(fm_busy_muse_active_run_id "$INTERRUPT_ACK_LOG" 2>/dev/null || true)
;;
esac
}
interrupt_cancel_claim() {
local elapsed=0 terminal=
case "$INTERRUPT_ACK_SOURCE:$INTERRUPT_ACK_RUN" in
muse-session-terminal:?*) ;;
*) printf 'unconfirmed'; return 0 ;;
esac
while :; do
terminal=$(fm_busy_muse_run_terminal "$INTERRUPT_ACK_LOG" "$INTERRUPT_ACK_RUN" 2>/dev/null || true)
case "$terminal" in
cancelled) printf 'confirmed'; return 0 ;;
?*) printf 'unconfirmed'; return 0 ;;
esac
awk -v e="$elapsed" -v t="$SETTLE_WAIT" 'BEGIN{exit !(e < t)}' || break
sleep "$POLL"
elapsed=$(awk -v e="$elapsed" -v p="$POLL" 'BEGIN{printf "%.3f", e + p}')
done
printf 'unconfirmed'
}
# deliver_interrupt: deliver and observe the strongest adapter-owned
# cancellation claim available after delivery.
deliver_interrupt() {
local cancel
prepare_interrupt_ack
send_interrupt_keys
cancel=$(interrupt_cancel_claim)
printf '%s' "$cancel"
}
verify_interrupt_running() {
local proof after
fm_backend_target_exists "$BACKEND" "$T" "$LABEL" \
|| die "task $ID's endpoint disappeared while interrupting it; no further control action is safe"
proof=endpoint
if fm_control_backend_state_verified "$BACKEND"; then
# An interrupt cancels a turn; it must never have stopped the agent. This
# is the postcondition that separates a landed interrupt from an accident.
after=$(agent_state)
[ "$after" = alive ] \
|| die "task $ID's agent is '$after' after its interrupt key; an interrupt must leave the agent running"
proof=agent-alive
fi
printf '%s' "$proof"
}
do_interrupt() {
local proof cancel
cancel=$(deliver_interrupt) || return $?
proof=$(verify_interrupt_running) || return $?
printf '%s cancel=%s' "$proof" "$cancel"
}
retire_busy_incarnation() {
if [ -f "$STATE/$ID.busy-gen" ]; then
"$SCRIPT_DIR/fm-busy-event.sh" retire "$STATE" "$ID" --current-gen >/dev/null 2>&1 || true
fi
}
# do_exit: stop the running agent, preserving endpoint and worktree. Prints
# `already-stopped` or `stopped`.
do_exit() {
local state cmd verdict cancel interrupt_result=not-needed
require_state_verified_backend exit
state=$(agent_state)
case "$state" in
dead)
printf 'already-stopped'
return 0
;;
alive) ;;
missing) die "task $ID's recorded endpoint is gone, so there is no agent to stop; reconcile the task before any further control action" ;;
*) die "task $ID's endpoint reads '$state' rather than a positively classified state; refusing to send a lifecycle command into an unattributed endpoint" ;;
esac
# A busy agent is interrupted first before the exit command is submitted.
case "$(busy_verdict)" in
busy*)
cancel=$(deliver_interrupt) || return $?
state=$(agent_state)
case "$state" in
dead)
retire_busy_incarnation
printf 'stopped'
return 0
;;
alive) interrupt_result="delivered verified=agent-alive cancel=$cancel" ;;
missing) die "task $ID's recorded endpoint disappeared after interrupt delivery, so exit cannot prove whether the agent stopped" ;;
*) die "task $ID's endpoint reads '$state' after interrupt delivery rather than a positively classified state; exit cannot prove whether the agent stopped" ;;
esac
;;
esac
cmd=$(fm_control_exit_command "$HARNESS")
# The submit verdict is NOT the postcondition here: a successful exit command
# destroys the composer the verdict is read from, so a post-exit read can
# legitimately report anything. Only a hard transport failure aborts; the
# authoritative proof is the agent-state wait below. The retried Enter still
# matters, because a slash command opens a completion popup on some TUIs that
# swallows the first Enter.
verdict=$(fm_backend_send_text_submit "$BACKEND" "$T" "$cmd" "$EXIT_RETRIES" "$POLL" 1.2 "$LABEL") \
|| die "the exit command could not be sent to task $ID on $BACKEND"
[ "$verdict" != send-failed ] \
|| die "the exit command could not be sent to task $ID on $BACKEND"
state=$(wait_agent_state "$EXIT_WAIT" dead) || {
die "exit-delivered $ID interrupt=$interrupt_result exit-command=delivered agent-state=$state exit=unconfirmed; the agent did not stop within ${EXIT_WAIT}s"
}
# The incarnation is over: retire its busy wiring so no stale record or
# orphaned generation survives the agent that produced it.
retire_busy_incarnation
printf 'stopped'
}
# --- transactional relaunch -------------------------------------------------
#
# The transaction's durable record is state/<id>.control-relaunch, with the
# prior metadata and brief preserved beside it. Every failure path runs through
# relaunch_rollback (an EXIT trap, so a refusal raised deep inside a shared
# helper is covered too) and leaves either the pre-relaunch durable record or a
# concrete, named partial state - never a task whose record claims an agent
# that is not running.
JOURNAL="$STATE/$ID.control-relaunch"
META_PRIOR="$JOURNAL.meta-prior"
BRIEF_PRIOR="$JOURNAL.brief-prior"
NOTE_FILE="$JOURNAL.note"
RELAUNCH_META_PUBLISHED=0
RELAUNCH_AGENT_CONFIRMED=0
RELAUNCH_TX=
RELAUNCH_BRIEF=
PRIOR_HARNESS=$HARNESS
PRIOR_RECORDED_HARNESS=$RECORDED_HARNESS
CONFIG_HARNESS=
CONFIG_MODEL=
CONFIG_EFFORT=
PRIOR_MODEL=
PRIOR_EFFORT=
TARGET_HARNESS=$HARNESS
TARGET_MODEL=
TARGET_EFFORT=
journal_write() { # <phase> [extra-line]...
local phase=$1
shift
if {
echo "v1"
echo "task=$ID"
echo "phase=$phase"
echo "ts=$(date -u +%Y-%m-%dT%H:%M:%SZ)"
echo "backend=$BACKEND"
echo "endpoint=$T"
echo "worktree=$WT"
echo "kind=$KIND"
echo "from_harness=$PRIOR_RECORDED_HARNESS"
echo "from_model=$PRIOR_MODEL"
echo "from_effort=$PRIOR_EFFORT"
echo "to_harness=$TARGET_HARNESS"
echo "to_model=$TARGET_MODEL"
echo "to_effort=$TARGET_EFFORT"
local line
for line in "$@"; do
echo "$line"
done
} > "$JOURNAL.tmp" && mv -f "$JOURNAL.tmp" "$JOURNAL"; then
RELAUNCH_PHASE=$phase
return 0
fi
return 1
}
relaunch_rollback() {
local state
[ "$RELAUNCH_ACTIVE" = 1 ] || return 0
[ "$RELAUNCH_PHASE" != complete ] || return 0
RELAUNCH_ACTIVE=0
case "$RELAUNCH_PHASE" in
checkpoint|noted)
# The old agent was never touched. Restore the instructions byte-exact so
# a refused relaunch leaves nothing behind.
if [ -n "$RELAUNCH_BRIEF" ] && [ -f "$BRIEF_PRIOR" ]; then
cp -p "$BRIEF_PRIOR" "$RELAUNCH_BRIEF" 2>/dev/null || true
fi
journal_write "failed:$RELAUNCH_PHASE" "rollback=instructions-restored" || true
echo "error: relaunch of $ID was refused before its agent was touched; nothing changed" >&2
;;
stopping)
state=$(agent_state 2>/dev/null || printf unknown)
case "$state" in
alive)
if [ -n "$RELAUNCH_BRIEF" ] && [ -f "$BRIEF_PRIOR" ]; then
cp -p "$BRIEF_PRIOR" "$RELAUNCH_BRIEF" 2>/dev/null || true
fi
journal_write "failed:$RELAUNCH_PHASE" "rollback=instructions-restored-agent-alive" || true
echo "error: relaunch of $ID failed while stopping the old agent, which is still running; its original instructions were restored" >&2
;;
dead)
journal_write "failed:$RELAUNCH_PHASE" "rollback=prior-record-kept-agent-dead" || true
echo "error: $ID's agent stopped but relaunch did not reach replacement launch; no agent is running, and its work plus progress note are preserved at $WT" >&2
;;
*)
journal_write "failed:$RELAUNCH_PHASE" "rollback=none-agent-state-$state" || true
echo "error: relaunch of $ID failed while stopping the old agent and its state is '$state'; the durable record and progress note were retained for recovery" >&2
;;
esac
;;
exited|launching)
if [ "$RELAUNCH_AGENT_CONFIRMED" = 1 ]; then
journal_write "failed:$RELAUNCH_PHASE" "rollback=none-new-agent-confirmed" || true
echo "error: $ID's replacement is running on $TARGET_HARNESS, but transaction completion could not be persisted; its published record was retained for reconciliation" >&2
elif [ "$RELAUNCH_META_PUBLISHED" = 1 ] \
|| { [ -n "$RELAUNCH_TX" ] \
&& [ "$(fm_meta_get "$META" control_relaunch_tx)" = "$RELAUNCH_TX" ]; }; then
# The launch owner published the new incarnation's record. Leaving it
# in place is the honest state: the task is now recorded on the new
# harness with no agent confirmed, which is exactly what recovery
# reconciles. Rewriting it back to the old harness would be a second,
# worse inaccuracy.
journal_write "failed:$RELAUNCH_PHASE" "rollback=none-new-record-kept" || true
echo "error: $ID was relaunched on $TARGET_HARNESS but no running agent could be confirmed; its work is preserved at $WT" >&2
else
journal_write "failed:$RELAUNCH_PHASE" "rollback=prior-record-kept" || true
echo "error: $ID's agent was stopped but the replacement did not launch; no agent is running, and its work plus the recorded progress note are preserved at $WT" >&2
fi
;;
esac
return 0
}
resolve_relaunch_profile() {
PRIOR_HARNESS=$HARNESS
PRIOR_RECORDED_HARNESS=$RECORDED_HARNESS
PRIOR_MODEL=$(fm_meta_get "$META" model)
PRIOR_EFFORT=$(fm_meta_get "$META" effort)
[ -n "$PRIOR_MODEL" ] || PRIOR_MODEL=default
[ -n "$PRIOR_EFFORT" ] || PRIOR_EFFORT=default
if [ "$HARNESS_SET" = 0 ] \
&& [ "$PRIOR_RECORDED_HARNESS" != "$PRIOR_HARNESS" ]; then
die "task $ID records harness '$PRIOR_RECORDED_HARNESS', whose original launch command cannot be reconstructed from its recorded basename; relaunching without --harness would substitute the canonical adapter '$PRIOR_HARNESS' for the command actually running. Pass an explicit --harness to choose the replacement runtime deliberately"
fi
CONFIG_HARNESS=
CONFIG_MODEL=
CONFIG_EFFORT=
if [ "$KIND" = secondmate ]; then
# A secondmate's harness, model, and effort are a durable configured pin
# that every respawn re-resolves (the secondmate-provisioning contract), so
# a relaunch with no explicit harness picks up a newly configured one
# instead of freezing whatever this incarnation happens to run. Crewmates
# and scouts deliberately do NOT resolve config here: their harness comes
# from firstmate's own dispatch-profile judgment at intake, and silently
# re-resolving it would bypass that consultation.
CONFIG_HARNESS=$("$SCRIPT_DIR/fm-harness.sh" secondmate 2>/dev/null || true)
CONFIG_MODEL=$("$SCRIPT_DIR/fm-harness.sh" secondmate-model 2>/dev/null || true)
CONFIG_EFFORT=$("$SCRIPT_DIR/fm-harness.sh" secondmate-effort 2>/dev/null || true)
case "$CONFIG_EFFORT" in
''|low|medium|high|xhigh|max|ultra) ;;
*)
echo "warning: config/secondmate-harness effort token '$CONFIG_EFFORT' is not one of low, medium, high, xhigh, max, ultra; ignoring" >&2
CONFIG_EFFORT=
;;
esac
fi
if [ "$HARNESS_SET" = 1 ]; then
fm_control_harness_supported "$NEW_HARNESS" \
|| die "'$NEW_HARNESS' is not a verified harness; fm-control refuses to relaunch onto an adapter with no verified control or launch mechanics"
TARGET_HARNESS=$NEW_HARNESS
elif [ "$HARNESS_SET" = 0 ] && [ -n "$CONFIG_HARNESS" ]; then
fm_control_harness_supported "$CONFIG_HARNESS" \
|| die "the configured secondmate harness '$CONFIG_HARNESS' is not verified; fm-control refuses to relaunch onto an adapter with no verified control or launch mechanics"
TARGET_HARNESS=$CONFIG_HARNESS
else
TARGET_HARNESS=$PRIOR_HARNESS
fi
# The launch owner refuses an adapter that cannot run this task's kind, but it
# is only reached after the old agent has been stopped. Asking the same
# capability table here keeps that refusal on the pre-stop side of the
# transaction, where nothing has changed yet.
fm_control_harness_supports_kind "$TARGET_HARNESS" "$KIND" \
|| die "'$TARGET_HARNESS' is not verified to run a $KIND task, so relaunching $ID onto it would stop the running agent for a launch that must be refused; choose an adapter verified for this kind"
# A model or effort chosen for the previous harness does not transfer to a
# different one, so an explicit harness change resets both axes unless the
# caller names them too.
if [ "$MODEL_SET" = 1 ]; then
TARGET_MODEL=$NEW_MODEL
elif [ "$HARNESS_SET" = 0 ] && [ -n "$CONFIG_HARNESS" ]; then
TARGET_MODEL=${CONFIG_MODEL:-default}
elif [ "$TARGET_HARNESS" = "$PRIOR_HARNESS" ]; then
TARGET_MODEL=$PRIOR_MODEL
else
TARGET_MODEL=default
fi
if [ "$EFFORT_SET" = 1 ]; then
TARGET_EFFORT=$NEW_EFFORT
elif [ "$HARNESS_SET" = 0 ] && [ -n "$CONFIG_HARNESS" ]; then
TARGET_EFFORT=${CONFIG_EFFORT:-default}
elif [ "$TARGET_HARNESS" = "$PRIOR_HARNESS" ]; then
TARGET_EFFORT=$PRIOR_EFFORT
else
TARGET_EFFORT=default
fi
if [ "$TARGET_EFFORT" = ultra ]; then
"$SCRIPT_DIR/fm-harness.sh" validate-native-effort "$TARGET_HARNESS" "$TARGET_MODEL" "$TARGET_EFFORT" || return 1
fi
}
# safe_checkpoint: prove, before anything is stopped, that the work a relaunch
# must preserve is actually there and recoverable afterwards. Fills
# CHECKPOINT_LINES with the journal lines describing what it proved, and
# refuses outright when any of it cannot be established.
CHECKPOINT_LINES=()
safe_checkpoint() {
local wt_real wt_top wt_top_real head head_ref head_ref_status status_output dirty children marker child_meta
CHECKPOINT_LINES=()
[ -n "$WT" ] || die "task $ID has no recorded worktree; refusing to relaunch without a recorded local copy to preserve"
[ -d "$WT" ] || die "task $ID's recorded worktree $WT is missing; refusing to relaunch and lose track of its work"
wt_real=$(cd "$WT" 2>/dev/null && pwd -P) || die "task $ID's recorded worktree $WT cannot be resolved"
wt_top=$(git -C "$WT" rev-parse --show-toplevel 2>/dev/null) \
|| die "task $ID's recorded worktree $WT is not a git worktree; refusing to relaunch without a checkout whose unlanded work can be accounted for"
wt_top_real=$(cd "$wt_top" 2>/dev/null && pwd -P) || wt_top_real=$wt_top
[ "$wt_real" = "$wt_top_real" ] \
|| die "task $ID's recorded worktree $WT is not a worktree root (root is $wt_top); refusing to relaunch against an ambiguous checkout"
if head=$(git -C "$WT" rev-parse --verify HEAD 2>/dev/null); then
:
elif head_ref=$(git -C "$WT" symbolic-ref -q HEAD 2>/dev/null); then
if git -C "$WT" show-ref --verify --quiet "$head_ref" 2>/dev/null; then
die "task $ID's worktree HEAD exists but cannot be resolved; refusing to relaunch from an unreadable checkout"
else
head_ref_status=$?
[ "$head_ref_status" -eq 1 ] \
|| die "task $ID's worktree HEAD cannot be inspected; refusing to relaunch from an unreadable checkout"
head=unborn
fi
else
die "task $ID's worktree HEAD cannot be inspected; refusing to relaunch from an unreadable checkout"
fi
status_output=$(git -C "$WT" status --porcelain 2>/dev/null) \
|| die "task $ID's worktree status cannot be inspected; refusing to relaunch without accounting for local changes"
if [ -n "$status_output" ]; then
dirty=yes
else
dirty=no
fi
CHECKPOINT_LINES+=("worktree_head=$head" "worktree_dirty=$dirty")
if [ "$KIND" = secondmate ]; then
# A secondmate's own crewmates outlive its relaunch: they run in their own
# endpoints, and the relaunched secondmate reconciles them from its home's
# durable records at startup. The checkpoint proves those records are
# readable BEFORE the agent stops, so a relaunch can never strand child
# work behind an unreadable home.
marker=$(cat "$WT/.fm-secondmate-home" 2>/dev/null || true)
[ "$marker" = "$ID" ] \
|| die "task $ID's home $WT is not marked as its own seeded secondmate home (marker: ${marker:-none}); refusing to relaunch"
[ -d "$WT/state" ] \
|| die "secondmate $ID's home has no readable state directory, so its child work cannot be accounted for; refusing to relaunch"
find "$WT/state" -mindepth 1 -maxdepth 1 -print >/dev/null 2>&1 \
|| die "secondmate $ID's child records cannot be traversed; refusing to relaunch"
children=0
for child_meta in "$WT/state"/*.meta; do
if [ ! -e "$child_meta" ] && [ ! -L "$child_meta" ]; then
continue
fi
if [ ! -f "$child_meta" ] || [ -L "$child_meta" ] \
|| ! cat "$child_meta" >/dev/null 2>&1; then
die "secondmate $ID's child record $child_meta is not a readable regular file; refusing to relaunch"
fi
children=$((children + 1))
done
CHECKPOINT_LINES+=("children=$children")
fi
}
# record_note: put the required progress note somewhere durable, and - for a
# ship or scout, whose only record of the interrupted reasoning is the
# conversation about to be discarded - into the instructions the replacement
# actually reads. A secondmate's charter is a durable standing document and is
# never rewritten: a secondmate reconciles its own home's records at startup,
# so the note stays parent-side audit evidence.
record_note() {
local stamp
[ -n "$NOTE" ] || return 0
stamp=$(date -u +%Y-%m-%dT%H:%M:%SZ)
printf '%s\n' "$NOTE" > "$NOTE_FILE"
case "$KIND" in
ship|scout)
cp -p "$RELAUNCH_BRIEF" "$BRIEF_PRIOR" \
|| die "could not preserve task $ID's instructions before recording the progress note"
{
echo
echo "## Progress note ($stamp)"
echo
echo "This task was relaunched. Continue from here; the local copy and every"
echo "uncommitted change are exactly as the previous worker left them."
echo
echo "First, check your instruction inbox: list $STATE/$ID.inbox/*.msg, act on"
echo "each message in numeric order, then mv each handled file into"
echo "$STATE/$ID.inbox/handled/. A steer sent before the relaunch survives there."
echo
printf '%s\n' "$NOTE"
} >> "$RELAUNCH_BRIEF" \
|| die "could not append the progress note to task $ID's instructions"
;;
esac
}
do_relaunch() {
local exit_result state note_line
local -a spawn_args
require_state_verified_backend relaunch
resolve_relaunch_profile
case "$KIND" in
ship|scout)
RELAUNCH_BRIEF="$DATA/$ID/brief.md"
[ -f "$RELAUNCH_BRIEF" ] \
|| die "task $ID has no instructions at $RELAUNCH_BRIEF; refusing to relaunch a worker with nothing to work from"
[ "$NOTE_SET" = 1 ] && [ -n "$NOTE" ] \
|| die "relaunch of a $KIND task requires --note (or --note-file): the replacement worker inherits the local copy but none of the conversation, so it must be told what happened"
;;
secondmate)
# The charter in the secondmate's own home is its instruction source and
# stays untouched.
RELAUNCH_BRIEF=
;;
*)
die "task $ID records kind '$KIND', which has no defined relaunch shape"
;;
esac
if [ -n "$NOTE" ]; then
note_line="note_file=$NOTE_FILE"
else
note_line="note=none"
fi
safe_checkpoint
cp -p "$META" "$META_PRIOR" || die "could not preserve task $ID's durable record before relaunching"
RELAUNCH_ACTIVE=1
journal_write checkpoint "${CHECKPOINT_LINES[@]}" "$note_line"
record_note
journal_write noted "${CHECKPOINT_LINES[@]}" "$note_line"
journal_write stopping "${CHECKPOINT_LINES[@]}" "$note_line"
exit_result=$(do_exit)
journal_write exited "${CHECKPOINT_LINES[@]}" "$note_line" "exit_result=$exit_result"
# The launch owner (fm-spawn --relaunch) clears the previous incarnation's
# per-task harness wiring before arming the new one, so nothing to do here.
RELAUNCH_TX="${BASHPID:-$$}.$(date -u +%Y%m%dT%H%M%SZ).$RANDOM"
journal_write launching "${CHECKPOINT_LINES[@]}" "$note_line" "relaunch_tx=$RELAUNCH_TX"
spawn_args=("$ID" --relaunch --harness "$TARGET_HARNESS")
[ "$TARGET_MODEL" = default ] || spawn_args+=(--model "$TARGET_MODEL")
[ "$TARGET_EFFORT" = default ] || spawn_args+=(--effort "$TARGET_EFFORT")
if FM_CONTROL_RELAUNCH_TX="$RELAUNCH_TX" \
"$SCRIPT_DIR/fm-spawn.sh" "${spawn_args[@]}" >/dev/null; then
RELAUNCH_META_PUBLISHED=1
else
[ "$(fm_meta_get "$META" control_relaunch_tx)" != "$RELAUNCH_TX" ] \
|| RELAUNCH_META_PUBLISHED=1
die "the replacement agent for $ID could not be launched on $TARGET_HARNESS"
fi
state=$(wait_agent_state "$LAUNCH_WAIT" alive) || {
die "the replacement agent for $ID did not come up within ${LAUNCH_WAIT}s (endpoint reads '$state')"
}
RELAUNCH_AGENT_CONFIRMED=1
journal_write complete "${CHECKPOINT_LINES[@]}" "$note_line" "exit_result=$exit_result"
RELAUNCH_ACTIVE=0
echo "relaunched $ID harness=$TARGET_HARNESS from=$PRIOR_RECORDED_HARNESS model=$TARGET_MODEL effort=$TARGET_EFFORT backend=$BACKEND endpoint=$T worktree=$WT"
}
# --- verbs ------------------------------------------------------------------
case "$VERB" in
interrupt)
state=$(agent_state)
case "$state" in
alive) ;;
unverified)
# No recovery-grade classifier on this backend. Interrupt is
# non-destructive and its endpoint-existence postcondition is still
# real, so it proceeds - the printed proof names exactly what was
# verified rather than implying more.
;;
dead|missing) die "no agent is running at task $ID's recorded endpoint (state: $state); there is nothing to interrupt" ;;
*) die "task $ID's endpoint reads '$state' rather than a positively classified state; refusing to send a lifecycle key into an unattributed endpoint" ;;
esac
proof=$(do_interrupt)
echo "interrupt-delivered $ID harness=$HARNESS backend=$BACKEND verified=$proof"
;;
exit)
result=$(do_exit)
echo "$result $ID harness=$HARNESS backend=$BACKEND endpoint=$T worktree=$WT"
;;
relaunch)
do_relaunch
;;
esac