fix(bin): retire a stalled owned watcher child with a bounded TERM/KILL sequence

.gitignore

@@ -3,6 +3,7 @@ state/
 data/
 .no-mistakes/
 .lavish/
+.claude/hooks/
 .fm-secondmate-home
 .fm-secondmate-parent
 .DS_Store

bin/fm-watch-arm.sh

@@ -35,8 +35,12 @@
 # It NEVER reports started/attached/healthy off a stale beacon or a dead/reused pid: a
 # stale-beacon or dead-pid holder either self-heals (the fresh child steals the
 # dead lock per the singleton self-eviction/steal path and is confirmed) or this
-# returns the FAILED line. On started it waits the child and propagates the wake
-# reason; on attached it stays live across identity-matched successors. A cycle
+# returns the FAILED line. After started it keeps verifying the owned child instead
+# of reducing liveness to process existence: a live identity-matched child whose
+# beacon reaches the shared grace is retired with a bounded TERM/KILL sequence,
+# its stale ownership is released through the watcher-down recovery transition,
+# and this arm fails loudly so a persistent adapter can retry without a primary
+# session restart. On attached it stays live across identity-matched successors. A cycle
 # that ends with no reason line and no healthy successor is resolved against the
 # watcher's identity-bound delivery record: a matching record reports that wake
 # and exits 0, and only a cycle that delivered nothing is the typed nonzero
@@ -77,8 +81,15 @@ case "${OSTYPE:-}" in
   *) ARM_CONFIRM_DEFAULT=10 ;;
 esac
 CONFIRM_TIMEOUT=${FM_ARM_CONFIRM_TIMEOUT:-$ARM_CONFIRM_DEFAULT}
-# Poll interval while attached to an existing healthy watcher.
+# Poll interval while attached to an existing healthy watcher; also the
+# cadence for the owned-child wait loop and the watchdog's stale-beacon recheck.
 ATTACH_POLL=${FM_ARM_ATTACH_POLL:-0.5}
+# Seconds allowed for a stalled owned watcher to retire after TERM before its
+# isolated process group receives KILL. This is deliberately much shorter than
+# the liveness grace: once that grace has elapsed, keeping stale singleton
+# ownership longer cannot restore supervision.
+STALL_RETIRE_TIMEOUT=${FM_WATCH_STALL_RETIRE_TIMEOUT:-2}
+case "$STALL_RETIRE_TIMEOUT" in ''|*[!0-9]*|0) STALL_RETIRE_TIMEOUT=2 ;; esac
 CYCLE_LOG="$STATE/.watch-cycle-exits.log"
 CYCLE_LOG_LOCK="$STATE/.watch-cycle-exits.lock"
 CYCLE_LOG_MAX_BYTES=${FM_WATCH_CYCLE_LOG_MAX_BYTES:-262144}
@@ -223,7 +234,12 @@ cycle_mark_predecessor_successor() {
 }
 
 clear_stale_recorded_watcher_lock() {
-  local lock_home lock_path lock_identity
+  local expected_pid=${1:-} lock_pid lock_home lock_path lock_identity
+  lock_pid=$(cat "$WATCH_LOCK/pid" 2>/dev/null || true)
+  if [ -n "$expected_pid" ]; then
+    [ "$lock_pid" = "$expected_pid" ] || return 0
+    fm_pid_alive "$lock_pid" && return 1
+  fi
   lock_home=$(cat "$WATCH_LOCK/fm-home" 2>/dev/null || true)
   lock_path=$(cat "$WATCH_LOCK/watcher-path" 2>/dev/null || true)
   lock_identity=$(cat "$WATCH_LOCK/pid-identity" 2>/dev/null || true)
@@ -447,11 +463,132 @@ fi
 # harness-tracked task) tears the watcher down too, and the watcher's eventual
 # wake exit propagates out so the harness re-notifies firstmate.
 child=
+child_group=
 child_out=
-cleanup_child() {
-  if [ -n "$child" ] && fm_pid_alive "$child"; then
-    kill -TERM "$child" 2>/dev/null || true
+watchdog_pid=
+watchdog_status=
+
+watch_child_running() {
+  local proc_root stat state_line
+  local -a stat_fields
+  [ -n "$child" ] || return 1
+  fm_pid_alive "$child" || return 1
+  proc_root=${FM_PROC_ROOT_OVERRIDE:-/proc}
+  if [ -r "$proc_root/$child/stat" ]; then
+    state_line=$(cat "$proc_root/$child/stat" 2>/dev/null) || return 0
+    read -r -a stat_fields <<< "${state_line##*)}"
+    stat=${stat_fields[0]:-}
+  else
+    stat=$(ps -p "$child" -o stat= 2>/dev/null | sed 's/^[[:space:]]*//' || true)
+  fi
+  case "$stat" in
+    Z*) return 1 ;;
+  esac
+  return 0
+}
+
+signal_watch_child() {  # <signal>
+  local signal=$1
+  if [ -n "$child_group" ]; then
+    kill -"$signal" -- "-$child_group" 2>/dev/null || true
+  elif [ -n "$child" ]; then
+    kill -"$signal" "$child" 2>/dev/null || true
+  fi
+}
+
+# Retire the owned watcher without ever waiting indefinitely on the same child
+# that caused the liveness failure. WATCH_CHILD_RC records the reaped status, or
+# 124 if even KILL could not make the direct child waitable inside the bound.
+WATCH_CHILD_RC=0
+retire_watch_child() {
+  local deadline
+  WATCH_CHILD_RC=0
+  [ -n "$child" ] || return 0
+  if watch_child_running; then
+    signal_watch_child TERM
+    deadline=$(( $(date +%s) + STALL_RETIRE_TIMEOUT + 1 ))
+    while watch_child_running && [ "$(date +%s)" -lt "$deadline" ]; do
+      sleep 0.05
+    done
   fi
+  # Sweep the whole isolated group even when the watcher shell honored TERM:
+  # a descendant that ignored it must not survive as an orphaned vendor wait.
+  signal_watch_child KILL
+  deadline=$(( $(date +%s) + 2 ))
+  while watch_child_running && [ "$(date +%s)" -lt "$deadline" ]; do
+    sleep 0.05
+  done
+  if watch_child_running; then
+    WATCH_CHILD_RC=124
+    return 1
+  fi
+  if wait "$child" 2>/dev/null; then
+    WATCH_CHILD_RC=0
+  else
+    WATCH_CHILD_RC=$?
+  fi
+  child=
+  child_group=
+  return 0
+}
+
+owned_child_has_stale_beacon() {
+  local lock_pid age
+  age=$(fm_path_age "$BEAT")
+  [ "$age" -ge "$GRACE" ] || return 1
+  lock_pid=$(cat "$WATCH_LOCK/pid" 2>/dev/null || true)
+  [ "$lock_pid" = "$child" ] || return 1
+  fm_watcher_lock_matches_pid "$STATE" "$WATCH" "$child" "$FM_HOME" || return 1
+  return 0
+}
+
+stop_owned_watchdog() {
+  [ -n "$watchdog_pid" ] || return 0
+  kill -TERM "$watchdog_pid" 2>/dev/null || true
+  wait "$watchdog_pid" 2>/dev/null || true
+  watchdog_pid=
+}
+
+# Keep the arm following the child through a short poll so an actionable child
+# close still propagates promptly, and once the watchdog has retired the
+# stalled group the same poll is bounded by the retire deadline below instead
+# of hanging on a child KILL cannot make waitable. A separate arm-owned
+# watchdog performs only the stale-beacon check and retires the isolated
+# watcher group when the shared grace expires.
+start_owned_watchdog() {
+  watchdog_status="$child_out.liveness"
+  rm -f "$watchdog_status" 2>/dev/null || true
+  (
+    watchdog_sleep_pid=
+    # shellcheck disable=SC2329 # Invoked indirectly by the signal trap below.
+    stop_watchdog_sleep() {
+      [ -z "$watchdog_sleep_pid" ] || kill -TERM "$watchdog_sleep_pid" 2>/dev/null || true
+      exit 0
+    }
+    trap stop_watchdog_sleep HUP TERM INT
+    while fm_pid_alive "$child"; do
+      if owned_child_has_stale_beacon; then
+        fm_path_age "$BEAT" > "$watchdog_status"
+        signal_watch_child TERM
+        sleep "$STALL_RETIRE_TIMEOUT"
+        signal_watch_child KILL
+        exit 0
+      fi
+      # Wait through a background sleep so the arm's stop signal interrupts the
+      # wait immediately; a foreground sleep defers Bash's trap and delays every
+      # healthy actionable close by the full polling interval.
+      sleep "$ATTACH_POLL" &
+      watchdog_sleep_pid=$!
+      wait "$watchdog_sleep_pid" 2>/dev/null || true
+      watchdog_sleep_pid=
+    done
+  ) &
+  watchdog_pid=$!
+}
+
+cleanup_child() {
+  stop_owned_watchdog
+  retire_watch_child || true
   if [ -n "$child_out" ]; then
     rm -f "$child_out" 2>/dev/null || true
   fi
@@ -461,10 +598,6 @@ cleanup_child() {
 handle_arm_signal() {
   local signal=$1 rc=$2
   trap - HUP TERM INT
-  if [ -n "$child" ] && fm_pid_alive "$child"; then
-    kill -TERM "$child" 2>/dev/null || true
-    wait "$child" 2>/dev/null || true
-  fi
   cycle_log_append "$rc" "$signal" arm-interrupted none
   cleanup_child
   exit "$rc"
@@ -478,12 +611,21 @@ child_out=$(mktemp "$STATE/.watch-arm-output.XXXXXX") || {
   echo "watcher: FAILED - no live watcher with a fresh beacon"
   exit 1
 }
+# Give the owned watcher a separate process group. The stale-beacon path can
+# then retire a hung backend helper together with the watcher instead of killing
+# only the lock holder and orphaning the subprocess it was blocked on.
+monitor_was_on=0
+case $- in *m*) monitor_was_on=1 ;; esac
+set -m
 if [ -n "${FM_WATCH_PREDECESSOR_ARM_PID:-}" ]; then
-  FM_WATCH_HANDLING_SUCCESSOR=1 "$WATCH" >"$child_out" &
+  ( set +m; export FM_WATCH_HANDLING_SUCCESSOR=1; exec "$WATCH" ) >"$child_out" &
 else
-  "$WATCH" >"$child_out" &
+  ( set +m; exec "$WATCH" ) >"$child_out" &
 fi
 child=$!
+[ "$monitor_was_on" -eq 1 ] || set +m
+child_group=$(ps -p "$child" -o pgid= 2>/dev/null | tr -d '[:space:]' || true)
+[ "$child_group" = "$child" ] || child_group=
 cycle_begin "$child" started "$(fm_pid_identity "$child" 2>/dev/null || true)"
 child_done=0
 
@@ -540,6 +682,56 @@ owned_child_finished() {
   return "$status"
 }
 
+# Follow a watcher this arm actually forked while rechecking the same strict
+# identity+beacon predicate used for initial readiness. The old raw `wait`
+# could never observe an alive-but-stalled watcher, so Pi/OpenCode kept an arm
+# claim forever and every repair call became an ownership no-op.
+wait_owned_child() {
+  local stalled_pid age rc retire_deadline
+  stalled_pid=$child
+  start_owned_watchdog
+  rc=124
+  retire_deadline=
+  while watch_child_running; do
+    if [ -z "$retire_deadline" ] && [ -s "$watchdog_status" ]; then
+      retire_deadline=$(( $(date +%s) + STALL_RETIRE_TIMEOUT + 3 ))
+    fi
+    if [ -n "$retire_deadline" ] && [ "$(date +%s)" -ge "$retire_deadline" ]; then
+      break
+    fi
+    sleep "$ATTACH_POLL"
+  done
+  if watch_child_running; then
+    WATCH_CHILD_RC=124
+  elif wait "$child" 2>/dev/null; then
+    rc=0
+  else
+    rc=$?
+  fi
+  stop_owned_watchdog
+  if [ -s "$watchdog_status" ]; then
+    age=$(cat "$watchdog_status" 2>/dev/null || fm_path_age "$BEAT")
+    signal_watch_child KILL
+    if ! fm_recovery_marker_publish "$STATE/.watcher-down" downtime \
+      || ! clear_stale_recorded_watcher_lock "$stalled_pid"; then
+      cycle_log_append "$rc" "$(cycle_signal_name "$rc")" stale-beacon-release-failed none
+      echo "watcher: FAILED - watcher pid=$stalled_pid stopped advancing its beacon for ${age}s; recovery state could not release stale ownership"
+      return 1
+    fi
+    cycle_log_append "$rc" "$(cycle_signal_name "$rc")" stale-beacon-retired none
+    rm -f "$child_out" "$watchdog_status" 2>/dev/null || true
+    child=
+    child_group=
+    child_out=
+    watchdog_status=
+    echo "watcher: FAILED - watcher pid=$stalled_pid stopped advancing its beacon for ${age}s; retired the stalled cycle and released stale ownership for bounded recovery"
+    return 1
+  fi
+  rm -f "$watchdog_status" 2>/dev/null || true
+  watchdog_status=
+  owned_child_finished "$rc"
+}
+
 # Verify the outcome: poll until this child is the confirmed healthy watcher, or
 # until some other watcher legitimately holds the singleton (a startup race), or
 # until the child gives up. Only then print the honest line.
@@ -552,7 +744,6 @@ while :; do
       cycle_refresh_lock_before
       if ! handling_generation=$(handling_successor_generation); then
         cleanup_child
-        wait "$child" 2>/dev/null || true
         cycle_log_append 1 none handling-handoff-failed none
         echo "watcher: FAILED - established successor could not inspect handling state"
         exit 1
@@ -563,9 +754,7 @@ while :; do
       else
         echo "watcher: started pid=$child (beacon fresh)"
       fi
-      wait "$child"
-      rc=$?
-      owned_child_finished "$rc"
+      wait_owned_child
       exit $?
     fi
     # Another watcher won the singleton; our child stood down.
@@ -588,8 +777,7 @@ done
 trap - HUP TERM INT
 print_watch_output "$child_out"
 cleanup_child
-wait "$child" 2>/dev/null
-rc=$?
+rc=$WATCH_CHILD_RC
 cycle_log_append "$rc" "$(cycle_signal_name "$rc")" confirmation-timeout none
 echo "watcher: FAILED - no live watcher with a fresh beacon"
 exit 1

bin/fm-watch.sh

@@ -815,6 +815,7 @@ if [ "${FM_WATCH_HANDLING_SUCCESSOR:-0}" = 1 ]; then
   touch "$STATE/.last-watcher-beat"
   handling_wait=0
   while [ "$handling_wait" -lt 600 ]; do
+    touch "$STATE/.last-watcher-beat"
     fm_recovery_marker_snapshot "$WATCHER_DOWNTIME_MARKER" || true
     case "$FM_RECOVERY_MARKER_TOKEN" in
       pending:downtime:*) ;;

docs/configuration.md

@@ -563,7 +563,8 @@ FM_CLAUDE_AUTOARM_SYNC_WAIT_MS=800   # milliseconds the --claude turn-end guard
 FM_CLAUDE_AUTOARM_EPOCH_FRESH=15   # seconds a recorded auto-arm outcome remains eligible for the current event epoch's recovery or failure decision
 FM_CLAUDE_TURNEND_BLOCK_BUDGET=3   # consecutive --claude guard re-blocks before the verified one-time attended fail-open; safely below Claude Code's 8-block override
 FM_ARM_CONFIRM_TIMEOUT=10   # seconds fm-watch-arm waits to confirm a fresh watcher before reporting FAILED; default 30 on Git Bash/MSYS
-FM_ARM_ATTACH_POLL=0.5  # seconds between checks while fm-watch-arm is attached to an existing healthy watcher cycle
+FM_ARM_ATTACH_POLL=0.5  # seconds between checks while fm-watch-arm follows a healthy watcher cycle: attached to a peer, or polling its forked child's liveness and the watchdog's stale-beacon recheck
+FM_WATCH_STALL_RETIRE_TIMEOUT=2   # seconds an owned live-but-stale watcher gets after TERM before fm-watch-arm kills its isolated process group and releases stale ownership; a child that survives the bound keeps its lock, and invalid or zero values use 2
 FM_OPENCODE_ARM_READY_TIMEOUT_MS=12000   # milliseconds the OpenCode primary watcher plugin waits for an arm attempt to report started, healthy, wake, or failure; default 35000 on Windows to stay above the MSYS confirm budget
 FM_PI_ARM_READY_TIMEOUT_MS=12000   # milliseconds the Pi watcher extension waits for a successor arm to report started or attached; default 35000 on Windows to stay above the MSYS confirm budget
 FM_WATCH_ARM_RETIRE_TIMEOUT_MS=1000   # milliseconds Pi/OpenCode wait for an unready successor arm to exit before abandoning retries

docs/watcher-continuity.md

@@ -63,6 +63,10 @@ An attached arm follows verified identity-matched successors and resolves the sa
 Before releasing its singleton lock after printing an actionable reason, the watcher records that reason with its PID and process identity in `state/.watch-deliveries.log`.
 A matching PID and identity lets an attached arm report the delivered reason and exit zero even after its durable wake was handled and acknowledged, while an unrelated queue producer or a recycled PID cannot satisfy the match.
 Only a cycle with no matching delivery record emits `watcher: FAILED - cycle ended without an actionable reason` and exits nonzero.
+After initial readiness, an arm that forked the watcher keeps applying the same identity-bound beacon predicate instead of treating a live PID as permanent health.
+When that owned watcher reaches the shared stale-beacon grace, the arm sends TERM and then KILL to the watcher's isolated process group within `FM_WATCH_STALL_RETIRE_TIMEOUT`, publishes the existing watcher-down recovery episode, and exits with a typed failure.
+The stale lock is removed only once the child is dead and still matches the expected PID, so a child that survives the bound keeps its lock and the ledger records the refused release.
+Persistent adapters therefore lose their owned-child no-op when the child is no longer healthy and can run their existing bounded retry without restarting the primary session.
 
 The arm layer appends one tab-separated record per observed cycle to `state/.watch-cycle-exits.log`.
 Each record includes arm and watcher PIDs, start and end timestamps, exit code and signal, classified reason, beacon age, lock identity before and after close, and successor disposition.
@@ -77,7 +81,7 @@ Only the watcher process touches `state/.last-watcher-beat`; no helper process c
 `tests/fm-pi-watch-extension.test.sh` checks Pi's first-cycle-or-explicit-repair tool metadata and ownership-based redundant-call no-ops, then simulates actionable and empty child closes against the actual Pi and OpenCode close handlers, blocks prompt delivery to prove the successor launches first, verifies single-flight behavior, changes the session lock before close to prove ownership is rechecked, and hangs each successor arm to prove bounded fallback delivery includes the typed restoration failure.
 The same suite covers ordinary same-process session replacement for `/new`, `/resume`, and `/fork`, same-instance shutdown-plus-start, stale prior-generation callbacks, repeated transitions with exactly one live cycle, disappearance of the shutting-down refusal after a valid replacement activates, and terminal quit still refusing late rearm.
 `tests/fm-watch-arm.test.sh` covers durable queue replay, real remote parent-replies ingestion into the authoritative status log, decision-only OPEN DECISIONS recovery, interrupted handling replay, generation-bound acknowledgement, a persistent live successor after recovery, a watcher close inside the handling window that must leave the printed acknowledgement valid, and the self-healing moved-generation acknowledgement that consumes its handled rows and names its remedy.
-`tests/fm-watcher-lock.test.sh` covers verified-successor attach, recovery publication before stale-lock removal, the typed self-eviction failure, bounded and successor-linked lifecycle rows, and a SIGSTOP counterfactual that distinguishes a live PID from a stale beacon before classifying termination.
+`tests/fm-watcher-lock.test.sh` covers verified-successor attach, recovery publication before stale-lock removal, the typed self-eviction failure, bounded and successor-linked lifecycle rows, and a real-process SIGSTOP counterfactual that proves the bounded retirement contract on both platforms: the Linux refusal shape with the live holder's lock retained, and on other hosts the relinquished lock, same-session re-arm, and a healthy successor whose beacon keeps advancing.
 `tests/fm-subagent-pretool-check.test.sh` proves Claude retains only the non-status Bash seatbelts.
 `tests/fm-claude-stop-autoarm.test.sh` covers the auto-arm's scope, stale and live session owners, unchanged AFK and need boundaries, single-flight, bounded failure retries, benign live-watcher cycle ends, one-notice failure episodes, and exit-2 translation.
 `FM_CLAUDE_LIVE_E2E=1 tests/fm-claude-stop-autoarm-live-e2e.test.sh` starts with the reproduced stale-lock state, runs session start first, completes two tokenless cycles, and checks the competing-live-owner negative control.

tests/fm-watch-arm.test.sh

@@ -141,7 +141,8 @@ drain_ack_pair() {  # <drain-stderr>
 start_rearm_arm() {  # <home> <state> <fakebin> <arm-out> [predecessor-arm-pid]
   local home=$1 state=$2 fakebin=$3 armout=$4 predecessor=${5:-} i
   PATH="$fakebin:$PATH" FM_HOME="$home" FM_STATE_OVERRIDE="$state" \
-    FM_POLL=1 FM_SIGNAL_GRACE=0 FM_CHECK_INTERVAL=999999 FM_HEARTBEAT=999999 \
+    FM_ARM_CONFIRM_TIMEOUT=60 FM_POLL=1 FM_SIGNAL_GRACE=0 \
+    FM_CHECK_INTERVAL=999999 FM_HEARTBEAT=999999 \
     FM_WATCH_PREDECESSOR_ARM_PID="$predecessor" \
     "$WATCH_ARM" --restart > "$armout" &
   ARM_PID=$!
@@ -287,16 +288,14 @@ test_rearm_resurfaces_durable_queue_and_remote_open_decision() {
   append_wake "$state" check startup-network 'check: startup-network'
 
   start_rearm_arm "$home" "$state" "$fakebin" "$armout"
-  sleep 0.25
-  if is_live_non_zombie "$ARM_PID"; then
-    # End the fixture through an ordinary actionable status transition so this
-    # failing pre-fix path leaves no child behind.
-    printf 'done: fixture cleanup\n' > "$state/cleanup.status"
-    wait_for_exit "$ARM_PID" 80 || true
+  # The arm reaps its liveness watchdog before returning the watcher reason, so
+  # assert the recovery's bounded outcome instead of a scheduler-sensitive
+  # quarter-second process snapshot.
+  wait_for_exit "$ARM_PID" 100
+  status=$?
+  if [ "$status" -eq 124 ]; then
     fail "re-arm stayed live instead of surfacing durable wakes and the still-open remote decision"
   fi
-  wait "$ARM_PID"
-  status=$?
   expect_code 0 "$status" "re-arm re-surface wake must close successfully"
   grep -F 'check: rearm-resurface' "$armout" >/dev/null \
     || fail "re-arm did not report the durable recovery wake: $(cat "$armout")"