Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
117 changes: 83 additions & 34 deletions bin/fm-control-lib.sh
Original file line number Diff line number Diff line change
Expand Up @@ -197,32 +197,42 @@ fm_control_exit_command() { # <harness>
fm_control_omp_home_owner_pid() { # <home-state>
local home_state=$1 pid=""
if [ -f "$home_state/.lock" ] && [ ! -L "$home_state/.lock" ]; then
IFS= read -r pid < "$home_state/.lock" || pid=""
pid=$(cat "$home_state/.lock" 2>/dev/null) || return 1
fi
case "$pid" in
''|*[!0-9]*|0|1) return 1 ;;
''|*[!0-9]*|0*|1) return 1 ;;
esac
printf '%s' "$pid"
}

# Retire the session artifacts an OMP second mate leaves behind, so its
# replacement launch is not refused by its own predecessor's leftovers.
#
# The order is the contract. Both owners have to be proven gone FIRST - the
# recorded endpoint, and the process holding the home session - because an
# artifact removed under a live agent is indistinguishable from a fresh one, and
# the launch owner would then accept a duplicate against a still-running mate.
# Only ESRCH proves absence. A failed probe can also mean EPERM or an
# unavailable probe command; keep those owners and their artifacts untouched.
fm_control_omp_owner_absent() { # <probe-command> <pid> <owner-description>
local probe_cmd=$1 pid=$2 description=$3 probe_output
if probe_output=$(LC_ALL=C "$probe_cmd" -0 "$pid" 2>&1); then
printf 'error: OMP %s owner is still live; refusing relaunch\n' "$description" >&2
return 1
fi
case "$probe_output" in
*'No such process'*) return 0 ;;
*)
printf 'error: OMP %s owner is unverifiable; refusing relaunch\n' "$description" >&2
return 1
;;
esac
}

# Retire OMP session artifacts only after the prior session is absent and its
# primary marker is tied to this home, this runtime, and a known extension
# closure. A stale marker is not disposable just because its recorded PID is
# dead: foreign, malformed, or unexplained state must keep refusing relaunch.
#
# FM_RESTART_STOP_CMD replaces the signal and the liveness probe for the home
# session owner (default: kill; invoked as "<cmd> <pid>" to stop it and
# "<cmd> -0 <pid>" to probe it, so one override covers both halves). It exists so
# tests/fm-secondmate-restart.test.sh can drive this sequence deterministically;
# unset, this is an ordinary kill and the behavior is unchanged.
# FM_RESTART_STOP_WAIT seconds to wait for the owner to go (default 30)
# FM_RESTART_STOP_POLL seconds between absence probes (default 1)
# FM_RESTART_STOP_CMD overrides kill for absence probes only, invoked as
# "<cmd> -0 <pid>"; it never receives a stop signal request here.
fm_control_omp_secondmate_prepare_relaunch() { # <state> <id> <worktree> <backend> <target> <meta>
local state=$1 id=$2 worktree=$3 backend=$4 target=$5 meta=$6 omp_state
local stop_cmd stop_wait stop_poll owner_pid elapsed proven
local stop_cmd owner_pid marker marker_pid
local expected_bun expected_bin current_version head committed_version lib_dir lock_present
omp_state=$(fm_backend_agent_state "$backend" "$target" "$meta" 2>/dev/null || printf 'unreadable')
case "$omp_state" in
missing) ;;
Expand All @@ -234,24 +244,63 @@ fm_control_omp_secondmate_prepare_relaunch() { # <state> <id> <worktree> <backe
*) return 1 ;;
esac
stop_cmd=${FM_RESTART_STOP_CMD:-kill}
stop_wait=${FM_RESTART_STOP_WAIT:-30}
stop_poll=${FM_RESTART_STOP_POLL:-1}
case "$stop_wait" in ''|*[!0-9]*) return 1 ;; esac
case "$stop_poll" in ''|*[!0-9]*|0) return 1 ;; esac
[ -d "$state" ] && [ ! -L "$state" ] \
&& [ -d "$worktree/state" ] && [ ! -L "$worktree/state" ] \
|| { printf 'error: OMP home state is not an ordinary directory; refusing relaunch\n' >&2; return 1; }

marker="$worktree/state/.omp-primary-extension-loaded"
marker_pid=
if [ -e "$marker" ] || [ -L "$marker" ]; then
[ -f "$marker" ] && [ ! -L "$marker" ] \
|| { printf 'error: OMP primary marker is not an ordinary file; refusing relaunch\n' >&2; return 1; }
lib_dir=$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd -P) || return 1
# shellcheck source=bin/fm-primary-watch-version-lib.sh
. "$lib_dir/fm-primary-watch-version-lib.sh" || return 1
# shellcheck source=bin/fm-omp-process-lib.sh
. "$lib_dir/fm-omp-process-lib.sh" || return 1
fm_omp_primary_marker_read "$marker" \
|| { printf 'error: OMP primary marker is malformed; refusing relaunch\n' >&2; return 1; }
marker_pid=$FM_OMP_MARKER_PID
case "$marker_pid" in
0|1|0*|*[!0-9]*)
printf 'error: OMP primary marker has an invalid owner PID; refusing relaunch\n' >&2
return 1
;;
esac
expected_bun=$(fm_meta_get "$meta" omp_bun 2>/dev/null || true)
expected_bin=$(fm_meta_get "$meta" omp_bin 2>/dev/null || true)
if [ -z "$expected_bun" ] || [ -z "$expected_bin" ] \
|| [ "$FM_OMP_MARKER_BUN" != "$expected_bun" ] \
|| [ "$FM_OMP_MARKER_BIN" != "$expected_bin" ]; then
printf 'error: OMP primary marker runtime does not match the recorded home; refusing relaunch\n' >&2
return 1
fi
current_version=$(fm_primary_watch_version "$worktree/.omp/extensions/fm-primary-omp.ts" "$worktree" 2>/dev/null) \
|| { printf 'error: current OMP primary closure is unreadable; refusing relaunch\n' >&2; return 1; }
if [ "$FM_OMP_MARKER_VERSION" != "$current_version" ]; then
head=$(git -C "$worktree" rev-parse --verify 'HEAD^{commit}' 2>/dev/null) \
|| { printf 'error: OMP home HEAD is unverifiable; refusing relaunch\n' >&2; return 1; }
committed_version=$(fm_omp_primary_version_at_commit "$worktree" "$head" 2>/dev/null) \
|| { printf 'error: committed OMP primary closure is unreadable; refusing relaunch\n' >&2; return 1; }
[ "$committed_version" = "$current_version" ] \
|| { printf 'error: OMP primary closure differs from the recorded home revision; refusing relaunch\n' >&2; return 1; }
fm_omp_primary_version_fast_forwarded_from "$worktree" "$FM_OMP_MARKER_VERSION" >/dev/null 2>&1 \
|| { printf 'error: OMP primary marker version is not explained by recorded fast-forward history; refusing relaunch\n' >&2; return 1; }
fi
fi

lock_present=0
if [ -e "$worktree/state/.lock" ] || [ -L "$worktree/state/.lock" ]; then
lock_present=1
fi
if owner_pid=$(fm_control_omp_home_owner_pid "$worktree/state"); then
"$stop_cmd" "$owner_pid" >/dev/null 2>&1 || true
proven=0
elapsed=0
while :; do
if ! "$stop_cmd" -0 "$owner_pid" >/dev/null 2>&1; then
proven=1
break
fi
[ "$elapsed" -lt "$stop_wait" ] || break
sleep "$stop_poll"
elapsed=$((elapsed + stop_poll))
done
[ "$proven" -eq 1 ] || return 1
if [ -n "$marker_pid" ]; then
fm_control_omp_owner_absent "$stop_cmd" "$marker_pid" 'primary marker' || return 1
fi
fm_control_omp_owner_absent "$stop_cmd" "$owner_pid" 'home lock' || return 1
elif [ "$lock_present" -eq 1 ] || [ -n "$marker_pid" ]; then
printf 'error: OMP home lock owner is unverifiable; refusing relaunch\n' >&2
return 1
fi
rm -f -- "$state/$id.omp-ext.ts" "$state/$id.omp-ready" \
"$state/$id.omp-started" "$worktree/state/.omp-primary-extension-loaded"
Expand Down
2 changes: 2 additions & 0 deletions bin/fm-spawn.sh
Original file line number Diff line number Diff line change
Expand Up @@ -3203,6 +3203,8 @@ if [ "$KIND" = secondmate ]; then
if ! fm_omp_primary_marker_read "$OMP_PRIMARY_MARKER" \
|| [ -z "$OMP_EXPECTED_MARKER_VERSION" ] \
|| [ "$FM_OMP_MARKER_VERSION" != "$OMP_EXPECTED_MARKER_VERSION" ] \
|| [ "$FM_OMP_MARKER_BUN" != "$OMP_BUN_CANON" ] \
|| [ "$FM_OMP_MARKER_BIN" != "$OMP_BIN_CANON" ] \
|| [ "$FM_OMP_MARKER_PID" != "$OMP_STALE_LOCK_PID" ]; then
echo "error: OMP secondmate home has an unowned or malformed primary-integration marker at $OMP_PRIMARY_MARKER; refusing duplicate launch" >&2
exit 1
Expand Down
2 changes: 1 addition & 1 deletion docs/agent-control.md
Original file line number Diff line number Diff line change
Expand Up @@ -70,7 +70,7 @@ It is not deterministic across the verified adapters: codex and grok resume only
A secondmate relaunch does not require one and never rewrites its standing charter.
4. **Stop the old agent** through the `exit` verb, with its postcondition.
An endpoint the backend authoritatively reports missing counts as already stopped - there is no agent to stop - so the transaction journals that verdict and continues instead of refusing.
5. **Retire an OMP second mate's session artifacts**, only for that case, after both its endpoint and its home session owner are proven gone; `bin/fm-control-lib.sh`'s `fm_control_omp_secondmate_prepare_relaunch` owns that sequence and the two overrides that let a test drive it.
5. **Retire an OMP second mate's session artifacts** through `bin/fm-control-lib.sh`'s `fm_control_omp_secondmate_prepare_relaunch`, subject to the recovery checks owned by [configuration's harness support](configuration.md#harness-support).
Without it the launch owner would refuse the replacement over its own predecessor's leftovers, so an OMP second mate launches fresh with `--secondmate` and its new endpoint is revalidated from the record the launch published rather than the retired one.
6. **Launch the replacement** through its single owner, `bin/fm-spawn.sh --relaunch`, which adopts the recorded worktree and normally the endpoint - and recreates the endpoint inside that same worktree when the backend proves the recorded one gone - while allowing OMP secondmates to publish a fresh endpoint after clearing the previous harness's per-task wiring and arming a fresh busy generation.
A ship relaunch also recovers the recorded mode and yolo contract when the caller does not repeat them.
Expand Down
7 changes: 6 additions & 1 deletion docs/configuration.md
Original file line number Diff line number Diff line change
Expand Up @@ -326,7 +326,12 @@ For OMP secondmate launches, it explicitly loads that home's `.omp/extensions/fm
After proving a failed launch's owned endpoint stopped, cleanup retires its generation artifacts while preserving the persistent home, endpoint metadata, durable inbox, and real session files.
Without a live marker or lock owner, cleanup preserves an already-correct session binding or atomically repairs it to the pre-launch retained session, or to the unique saved session when no valid binding exists; an empty store permits pointer removal, while ambiguous saved sessions require explicit reconciliation.
A relaunch whose recorded endpoint is proven dead or missing retires the prior generation's runtime markers (adapter, ready, started, doorbell markers and request receipts) only after the endpoint, session-lock, and integration-marker live-owner checks pass, so a wake-on-delivery restore after a successful generation is accepted.
[`tests/fm-omp-secondmate.test.sh`](../tests/fm-omp-secondmate.test.sh) and [`tests/fm-remote-secondmate-lifecycle-e2e.test.sh`](../tests/fm-remote-secondmate-lifecycle-e2e.test.sh) cover failed-bind recovery and post-generation restore relaunch for local and remote homes.
For OMP secondmates, the `fm-control.sh` relaunch owner parses the four-line primary marker and requires its Bun and entrypoint paths to match the recorded home runtime.
It proves both the `.lock` owner and the marker owner absent without signaling either process; only an explicit no-such-process probe result proves absence.
If the marker hash differs from the current closure, the current working closure must match `HEAD` and the marker hash must occur on that home's recorded fast-forward history.
The owner path removes only stale runtime markers, preserving `state/.omp-session` and saved session files; direct `fm-spawn.sh` still refuses an old-version marker, PID mismatch, or foreign runtime that the owner path has not safely reconciled.
Malformed, unverifiable, symlinked, non-file, foreign, live-owner, or unexplained-history state refuses recovery.
[`tests/fm-omp-secondmate.test.sh`](../tests/fm-omp-secondmate.test.sh), [`tests/fm-secondmate-restart.test.sh`](../tests/fm-secondmate-restart.test.sh), and [`tests/fm-remote-secondmate-lifecycle-e2e.test.sh`](../tests/fm-remote-secondmate-lifecycle-e2e.test.sh) cover direct-launch refusals, owner-path safety, retained-session preservation, and updated-extension relaunch in fixture homes.
Firstmate's tracked `.omp/config.yml` keeps OMP's todo tool enabled but disables its end-of-turn incomplete-todo reminders because the session todo is a projection of long-lived fleet work rather than work that must finish within the current captain turn.
This prevents OMP from scheduling a second model turn after a captain-facing answer solely because projected fleet work remains incomplete.
The separately discovered `.omp/extensions/fm-fleet-hooks.ts` fails open while redacting credential-shaped values from text chunks in tool results, reporting valid `fm-todo-project.sh --check` drift through hidden next-turn context, and injecting a bounded roster, OPEN DECISIONS, in-flight PR, and Ready/In-flight backlog snapshot during compaction.
Expand Down
75 changes: 70 additions & 5 deletions tests/fm-omp-secondmate.test.sh
Original file line number Diff line number Diff line change
Expand Up @@ -892,7 +892,7 @@ test_legacy_launch_rejects_runtime_fallback() {
}

test_duplicate_recovery_states() {
local out before after mode expected version invalid_lock
local out before after mode expected version invalid_lock lock_pid marker_pid old_version

for row in 'live already has a live agent' 'ambiguous has an ambiguous agent process' 'unreadable endpoint state is unreadable'; do
mode=${row%% *}
Expand Down Expand Up @@ -933,13 +933,78 @@ test_duplicate_recovery_states() {
assert_contains "$out" 'unowned or malformed primary-integration marker' "trailing integration marker refusal was not actionable"
[ "$(count_new_windows)" = 0 ] || fail "trailing integration marker refusal created another endpoint"

setup_case duplicate-owned-stale-marker
setup_case duplicate-old-version-marker-refused
write_meta
version=$(fm_primary_watch_version "$HOME_DIR/.omp/extensions/fm-primary-omp.ts" "$HOME_DIR")
printf '%s\n99999999\n%s\n%s\n' "$version" "$TEST_OMP_BUN" "$TEST_OMP_BIN" > "$HOME_DIR/state/.omp-primary-extension-loaded"
sleep 0.05 &
lock_pid=$!
sleep 0.05 &
marker_pid=$!
wait "$lock_pid" || fail "fixture lock owner did not exit"
wait "$marker_pid" || fail "fixture marker owner did not exit"
old_version="${version%?}0"
[ "$old_version" != "$version" ] || old_version="${version%?}1"
[ "$lock_pid" != "$marker_pid" ] || fail "fixture lock and marker owners unexpectedly share a PID"
if kill -0 "$lock_pid" 2>/dev/null || kill -0 "$marker_pid" 2>/dev/null; then
fail "fixture lock or marker owner is still alive"
fi
printf '%s\n%s\n%s\n%s\n' "$old_version" "$marker_pid" "$TEST_OMP_BUN" "$TEST_OMP_BIN" \
> "$HOME_DIR/state/.omp-primary-extension-loaded"
printf '%s\n' "$lock_pid" > "$HOME_DIR/state/.lock"
out=$(FM_TEST_STATE_MODE=missing run_spawn 2>&1) \
&& fail "direct OMP launch bypassed the owner relaunch path for an old-version marker"
assert_contains "$out" 'unowned or malformed primary-integration marker' \
"direct OMP launch did not preserve its refusal for an old-version marker"
[ "$(count_new_windows)" = 0 ] || fail "old-version marker refusal created another endpoint"
printf 'fixture stale marker: version=%s current=%s lock_pid=%s alive=no marker_pid=%s alive=no\n' \
"$old_version" "$version" "$lock_pid" "$marker_pid"

setup_case duplicate-live-marker-owner
write_meta
version=$(fm_primary_watch_version "$HOME_DIR/.omp/extensions/fm-primary-omp.ts" "$HOME_DIR")
printf '%s\n%s\n%s\n%s\n' "$version" "$AGENT_PID" "$TEST_OMP_BUN" "$TEST_OMP_BIN" \
> "$HOME_DIR/state/.omp-primary-extension-loaded"
printf '%s\n' "$AGENT_PID" > "$HOME_DIR/state/.lock"
out=$(FM_TEST_STATE_MODE=missing run_spawn 2>&1) \
&& fail "OMP secondmate replaced a marker owned by a live process"
assert_contains "$out" 'already has a live session-lock owner' "live marker-owner refusal was not actionable"
[ "$(count_new_windows)" = 0 ] || fail "live marker refusal created another endpoint"

setup_case duplicate-foreign-marker
write_meta
version=$(fm_primary_watch_version "$HOME_DIR/.omp/extensions/fm-primary-omp.ts" "$HOME_DIR")
printf '%s\n99999999\n%s\n%s\n' "$version" "$TEST_OMP_BUN" "$CASE/foreign-omp" \
> "$HOME_DIR/state/.omp-primary-extension-loaded"
printf '99999999\n' > "$HOME_DIR/state/.lock"
out=$(FM_TEST_STATE_MODE=missing run_spawn 2>&1) || fail "OMP secondmate did not recover from its proven stale integration marker: $out"
assert_contains "$(cat "$TMUX_LOG")" 'new-window' "proven stale integration marker did not permit recovery"
out=$(FM_TEST_STATE_MODE=missing run_spawn 2>&1) \
&& fail "OMP secondmate replaced a foreign integration marker"
assert_contains "$out" 'unowned or malformed primary-integration marker' \
"foreign marker refusal was not actionable"
[ "$(count_new_windows)" = 0 ] || fail "foreign marker refusal created another endpoint"

for marker_shape in symlink directory reserved-pid; do
setup_case "duplicate-marker-$marker_shape"
write_meta
version=$(fm_primary_watch_version "$HOME_DIR/.omp/extensions/fm-primary-omp.ts" "$HOME_DIR")
case "$marker_shape" in
symlink)
printf 'outside marker target\n' > "$CASE/marker-target"
ln -s "$CASE/marker-target" "$HOME_DIR/state/.omp-primary-extension-loaded"
;;
directory)
mkdir "$HOME_DIR/state/.omp-primary-extension-loaded"
;;
reserved-pid)
printf '%s\n1\n%s\n%s\n' "$version" "$TEST_OMP_BUN" "$TEST_OMP_BIN" \
> "$HOME_DIR/state/.omp-primary-extension-loaded"
;;
esac
printf '99999999\n' > "$HOME_DIR/state/.lock"
out=$(FM_TEST_STATE_MODE=missing run_spawn 2>&1) \
&& fail "OMP secondmate accepted $marker_shape primary marker"
assert_contains "$out" 'primary-integration marker' "$marker_shape marker refusal was not actionable"
[ "$(count_new_windows)" = 0 ] || fail "$marker_shape marker refusal created another endpoint"
done

setup_case duplicate-dead
write_meta
Expand Down
Loading
Loading