diff --git a/.gitignore b/.gitignore index cae904c651..570babae56 100644 --- a/.gitignore +++ b/.gitignore @@ -3,6 +3,7 @@ state/ data/ .no-mistakes/ .lavish/ +.claude/hooks/ .fm-secondmate-home .fm-secondmate-parent .DS_Store diff --git a/bin/fm-watch-arm.sh b/bin/fm-watch-arm.sh index 5ba132401a..fe4787a99d 100755 --- a/bin/fm-watch-arm.sh +++ b/bin/fm-watch-arm.sh @@ -35,8 +35,12 @@ # It NEVER reports started/attached/healthy off a stale beacon or a dead/reused pid: a # stale-beacon or dead-pid holder either self-heals (the fresh child steals the # dead lock per the singleton self-eviction/steal path and is confirmed) or this -# returns the FAILED line. On started it waits the child and propagates the wake -# reason; on attached it stays live across identity-matched successors. A cycle +# returns the FAILED line. After started it keeps verifying the owned child instead +# of reducing liveness to process existence: a live identity-matched child whose +# beacon reaches the shared grace is retired with a bounded TERM/KILL sequence, +# its stale ownership is released through the watcher-down recovery transition, +# and this arm fails loudly so a persistent adapter can retry without a primary +# session restart. On attached it stays live across identity-matched successors. A cycle # that ends with no reason line and no healthy successor is resolved against the # watcher's identity-bound delivery record: a matching record reports that wake # and exits 0, and only a cycle that delivered nothing is the typed nonzero @@ -77,8 +81,15 @@ case "${OSTYPE:-}" in *) ARM_CONFIRM_DEFAULT=10 ;; esac CONFIRM_TIMEOUT=${FM_ARM_CONFIRM_TIMEOUT:-$ARM_CONFIRM_DEFAULT} -# Poll interval while attached to an existing healthy watcher. +# Poll interval while attached to an existing healthy watcher; also the +# cadence for the owned-child wait loop and the watchdog's stale-beacon recheck. ATTACH_POLL=${FM_ARM_ATTACH_POLL:-0.5} +# Seconds allowed for a stalled owned watcher to retire after TERM before its +# isolated process group receives KILL. This is deliberately much shorter than +# the liveness grace: once that grace has elapsed, keeping stale singleton +# ownership longer cannot restore supervision. +STALL_RETIRE_TIMEOUT=${FM_WATCH_STALL_RETIRE_TIMEOUT:-2} +case "$STALL_RETIRE_TIMEOUT" in ''|*[!0-9]*|0) STALL_RETIRE_TIMEOUT=2 ;; esac CYCLE_LOG="$STATE/.watch-cycle-exits.log" CYCLE_LOG_LOCK="$STATE/.watch-cycle-exits.lock" CYCLE_LOG_MAX_BYTES=${FM_WATCH_CYCLE_LOG_MAX_BYTES:-262144} @@ -223,7 +234,12 @@ cycle_mark_predecessor_successor() { } clear_stale_recorded_watcher_lock() { - local lock_home lock_path lock_identity + local expected_pid=${1:-} lock_pid lock_home lock_path lock_identity + lock_pid=$(cat "$WATCH_LOCK/pid" 2>/dev/null || true) + if [ -n "$expected_pid" ]; then + [ "$lock_pid" = "$expected_pid" ] || return 0 + fm_pid_alive "$lock_pid" && return 1 + fi lock_home=$(cat "$WATCH_LOCK/fm-home" 2>/dev/null || true) lock_path=$(cat "$WATCH_LOCK/watcher-path" 2>/dev/null || true) lock_identity=$(cat "$WATCH_LOCK/pid-identity" 2>/dev/null || true) @@ -447,11 +463,132 @@ fi # harness-tracked task) tears the watcher down too, and the watcher's eventual # wake exit propagates out so the harness re-notifies firstmate. child= +child_group= child_out= -cleanup_child() { - if [ -n "$child" ] && fm_pid_alive "$child"; then - kill -TERM "$child" 2>/dev/null || true +watchdog_pid= +watchdog_status= + +watch_child_running() { + local proc_root stat state_line + local -a stat_fields + [ -n "$child" ] || return 1 + fm_pid_alive "$child" || return 1 + proc_root=${FM_PROC_ROOT_OVERRIDE:-/proc} + if [ -r "$proc_root/$child/stat" ]; then + state_line=$(cat "$proc_root/$child/stat" 2>/dev/null) || return 0 + read -r -a stat_fields <<< "${state_line##*)}" + stat=${stat_fields[0]:-} + else + stat=$(ps -p "$child" -o stat= 2>/dev/null | sed 's/^[[:space:]]*//' || true) + fi + case "$stat" in + Z*) return 1 ;; + esac + return 0 +} + +signal_watch_child() { # + local signal=$1 + if [ -n "$child_group" ]; then + kill -"$signal" -- "-$child_group" 2>/dev/null || true + elif [ -n "$child" ]; then + kill -"$signal" "$child" 2>/dev/null || true + fi +} + +# Retire the owned watcher without ever waiting indefinitely on the same child +# that caused the liveness failure. WATCH_CHILD_RC records the reaped status, or +# 124 if even KILL could not make the direct child waitable inside the bound. +WATCH_CHILD_RC=0 +retire_watch_child() { + local deadline + WATCH_CHILD_RC=0 + [ -n "$child" ] || return 0 + if watch_child_running; then + signal_watch_child TERM + deadline=$(( $(date +%s) + STALL_RETIRE_TIMEOUT + 1 )) + while watch_child_running && [ "$(date +%s)" -lt "$deadline" ]; do + sleep 0.05 + done fi + # Sweep the whole isolated group even when the watcher shell honored TERM: + # a descendant that ignored it must not survive as an orphaned vendor wait. + signal_watch_child KILL + deadline=$(( $(date +%s) + 2 )) + while watch_child_running && [ "$(date +%s)" -lt "$deadline" ]; do + sleep 0.05 + done + if watch_child_running; then + WATCH_CHILD_RC=124 + return 1 + fi + if wait "$child" 2>/dev/null; then + WATCH_CHILD_RC=0 + else + WATCH_CHILD_RC=$? + fi + child= + child_group= + return 0 +} + +owned_child_has_stale_beacon() { + local lock_pid age + age=$(fm_path_age "$BEAT") + [ "$age" -ge "$GRACE" ] || return 1 + lock_pid=$(cat "$WATCH_LOCK/pid" 2>/dev/null || true) + [ "$lock_pid" = "$child" ] || return 1 + fm_watcher_lock_matches_pid "$STATE" "$WATCH" "$child" "$FM_HOME" || return 1 + return 0 +} + +stop_owned_watchdog() { + [ -n "$watchdog_pid" ] || return 0 + kill -TERM "$watchdog_pid" 2>/dev/null || true + wait "$watchdog_pid" 2>/dev/null || true + watchdog_pid= +} + +# Keep the arm following the child through a short poll so an actionable child +# close still propagates promptly, and once the watchdog has retired the +# stalled group the same poll is bounded by the retire deadline below instead +# of hanging on a child KILL cannot make waitable. A separate arm-owned +# watchdog performs only the stale-beacon check and retires the isolated +# watcher group when the shared grace expires. +start_owned_watchdog() { + watchdog_status="$child_out.liveness" + rm -f "$watchdog_status" 2>/dev/null || true + ( + watchdog_sleep_pid= + # shellcheck disable=SC2329 # Invoked indirectly by the signal trap below. + stop_watchdog_sleep() { + [ -z "$watchdog_sleep_pid" ] || kill -TERM "$watchdog_sleep_pid" 2>/dev/null || true + exit 0 + } + trap stop_watchdog_sleep HUP TERM INT + while fm_pid_alive "$child"; do + if owned_child_has_stale_beacon; then + fm_path_age "$BEAT" > "$watchdog_status" + signal_watch_child TERM + sleep "$STALL_RETIRE_TIMEOUT" + signal_watch_child KILL + exit 0 + fi + # Wait through a background sleep so the arm's stop signal interrupts the + # wait immediately; a foreground sleep defers Bash's trap and delays every + # healthy actionable close by the full polling interval. + sleep "$ATTACH_POLL" & + watchdog_sleep_pid=$! + wait "$watchdog_sleep_pid" 2>/dev/null || true + watchdog_sleep_pid= + done + ) & + watchdog_pid=$! +} + +cleanup_child() { + stop_owned_watchdog + retire_watch_child || true if [ -n "$child_out" ]; then rm -f "$child_out" 2>/dev/null || true fi @@ -461,10 +598,6 @@ cleanup_child() { handle_arm_signal() { local signal=$1 rc=$2 trap - HUP TERM INT - if [ -n "$child" ] && fm_pid_alive "$child"; then - kill -TERM "$child" 2>/dev/null || true - wait "$child" 2>/dev/null || true - fi cycle_log_append "$rc" "$signal" arm-interrupted none cleanup_child exit "$rc" @@ -478,12 +611,21 @@ child_out=$(mktemp "$STATE/.watch-arm-output.XXXXXX") || { echo "watcher: FAILED - no live watcher with a fresh beacon" exit 1 } +# Give the owned watcher a separate process group. The stale-beacon path can +# then retire a hung backend helper together with the watcher instead of killing +# only the lock holder and orphaning the subprocess it was blocked on. +monitor_was_on=0 +case $- in *m*) monitor_was_on=1 ;; esac +set -m if [ -n "${FM_WATCH_PREDECESSOR_ARM_PID:-}" ]; then - FM_WATCH_HANDLING_SUCCESSOR=1 "$WATCH" >"$child_out" & + ( set +m; export FM_WATCH_HANDLING_SUCCESSOR=1; exec "$WATCH" ) >"$child_out" & else - "$WATCH" >"$child_out" & + ( set +m; exec "$WATCH" ) >"$child_out" & fi child=$! +[ "$monitor_was_on" -eq 1 ] || set +m +child_group=$(ps -p "$child" -o pgid= 2>/dev/null | tr -d '[:space:]' || true) +[ "$child_group" = "$child" ] || child_group= cycle_begin "$child" started "$(fm_pid_identity "$child" 2>/dev/null || true)" child_done=0 @@ -540,6 +682,56 @@ owned_child_finished() { return "$status" } +# Follow a watcher this arm actually forked while rechecking the same strict +# identity+beacon predicate used for initial readiness. The old raw `wait` +# could never observe an alive-but-stalled watcher, so Pi/OpenCode kept an arm +# claim forever and every repair call became an ownership no-op. +wait_owned_child() { + local stalled_pid age rc retire_deadline + stalled_pid=$child + start_owned_watchdog + rc=124 + retire_deadline= + while watch_child_running; do + if [ -z "$retire_deadline" ] && [ -s "$watchdog_status" ]; then + retire_deadline=$(( $(date +%s) + STALL_RETIRE_TIMEOUT + 3 )) + fi + if [ -n "$retire_deadline" ] && [ "$(date +%s)" -ge "$retire_deadline" ]; then + break + fi + sleep "$ATTACH_POLL" + done + if watch_child_running; then + WATCH_CHILD_RC=124 + elif wait "$child" 2>/dev/null; then + rc=0 + else + rc=$? + fi + stop_owned_watchdog + if [ -s "$watchdog_status" ]; then + age=$(cat "$watchdog_status" 2>/dev/null || fm_path_age "$BEAT") + signal_watch_child KILL + if ! fm_recovery_marker_publish "$STATE/.watcher-down" downtime \ + || ! clear_stale_recorded_watcher_lock "$stalled_pid"; then + cycle_log_append "$rc" "$(cycle_signal_name "$rc")" stale-beacon-release-failed none + echo "watcher: FAILED - watcher pid=$stalled_pid stopped advancing its beacon for ${age}s; recovery state could not release stale ownership" + return 1 + fi + cycle_log_append "$rc" "$(cycle_signal_name "$rc")" stale-beacon-retired none + rm -f "$child_out" "$watchdog_status" 2>/dev/null || true + child= + child_group= + child_out= + watchdog_status= + echo "watcher: FAILED - watcher pid=$stalled_pid stopped advancing its beacon for ${age}s; retired the stalled cycle and released stale ownership for bounded recovery" + return 1 + fi + rm -f "$watchdog_status" 2>/dev/null || true + watchdog_status= + owned_child_finished "$rc" +} + # Verify the outcome: poll until this child is the confirmed healthy watcher, or # until some other watcher legitimately holds the singleton (a startup race), or # until the child gives up. Only then print the honest line. @@ -552,7 +744,6 @@ while :; do cycle_refresh_lock_before if ! handling_generation=$(handling_successor_generation); then cleanup_child - wait "$child" 2>/dev/null || true cycle_log_append 1 none handling-handoff-failed none echo "watcher: FAILED - established successor could not inspect handling state" exit 1 @@ -563,9 +754,7 @@ while :; do else echo "watcher: started pid=$child (beacon fresh)" fi - wait "$child" - rc=$? - owned_child_finished "$rc" + wait_owned_child exit $? fi # Another watcher won the singleton; our child stood down. @@ -588,8 +777,7 @@ done trap - HUP TERM INT print_watch_output "$child_out" cleanup_child -wait "$child" 2>/dev/null -rc=$? +rc=$WATCH_CHILD_RC cycle_log_append "$rc" "$(cycle_signal_name "$rc")" confirmation-timeout none echo "watcher: FAILED - no live watcher with a fresh beacon" exit 1 diff --git a/bin/fm-watch.sh b/bin/fm-watch.sh index 3f4a57afd6..1e1c26bf74 100755 --- a/bin/fm-watch.sh +++ b/bin/fm-watch.sh @@ -815,6 +815,7 @@ if [ "${FM_WATCH_HANDLING_SUCCESSOR:-0}" = 1 ]; then touch "$STATE/.last-watcher-beat" handling_wait=0 while [ "$handling_wait" -lt 600 ]; do + touch "$STATE/.last-watcher-beat" fm_recovery_marker_snapshot "$WATCHER_DOWNTIME_MARKER" || true case "$FM_RECOVERY_MARKER_TOKEN" in pending:downtime:*) ;; diff --git a/docs/configuration.md b/docs/configuration.md index e0311b44b0..f188efac88 100644 --- a/docs/configuration.md +++ b/docs/configuration.md @@ -563,7 +563,8 @@ FM_CLAUDE_AUTOARM_SYNC_WAIT_MS=800 # milliseconds the --claude turn-end guard FM_CLAUDE_AUTOARM_EPOCH_FRESH=15 # seconds a recorded auto-arm outcome remains eligible for the current event epoch's recovery or failure decision FM_CLAUDE_TURNEND_BLOCK_BUDGET=3 # consecutive --claude guard re-blocks before the verified one-time attended fail-open; safely below Claude Code's 8-block override FM_ARM_CONFIRM_TIMEOUT=10 # seconds fm-watch-arm waits to confirm a fresh watcher before reporting FAILED; default 30 on Git Bash/MSYS -FM_ARM_ATTACH_POLL=0.5 # seconds between checks while fm-watch-arm is attached to an existing healthy watcher cycle +FM_ARM_ATTACH_POLL=0.5 # seconds between checks while fm-watch-arm follows a healthy watcher cycle: attached to a peer, or polling its forked child's liveness and the watchdog's stale-beacon recheck +FM_WATCH_STALL_RETIRE_TIMEOUT=2 # seconds an owned live-but-stale watcher gets after TERM before fm-watch-arm kills its isolated process group and releases stale ownership; a child that survives the bound keeps its lock, and invalid or zero values use 2 FM_OPENCODE_ARM_READY_TIMEOUT_MS=12000 # milliseconds the OpenCode primary watcher plugin waits for an arm attempt to report started, healthy, wake, or failure; default 35000 on Windows to stay above the MSYS confirm budget FM_PI_ARM_READY_TIMEOUT_MS=12000 # milliseconds the Pi watcher extension waits for a successor arm to report started or attached; default 35000 on Windows to stay above the MSYS confirm budget FM_WATCH_ARM_RETIRE_TIMEOUT_MS=1000 # milliseconds Pi/OpenCode wait for an unready successor arm to exit before abandoning retries diff --git a/docs/watcher-continuity.md b/docs/watcher-continuity.md index 9187bf3c6d..0bd468d146 100644 --- a/docs/watcher-continuity.md +++ b/docs/watcher-continuity.md @@ -63,6 +63,10 @@ An attached arm follows verified identity-matched successors and resolves the sa Before releasing its singleton lock after printing an actionable reason, the watcher records that reason with its PID and process identity in `state/.watch-deliveries.log`. A matching PID and identity lets an attached arm report the delivered reason and exit zero even after its durable wake was handled and acknowledged, while an unrelated queue producer or a recycled PID cannot satisfy the match. Only a cycle with no matching delivery record emits `watcher: FAILED - cycle ended without an actionable reason` and exits nonzero. +After initial readiness, an arm that forked the watcher keeps applying the same identity-bound beacon predicate instead of treating a live PID as permanent health. +When that owned watcher reaches the shared stale-beacon grace, the arm sends TERM and then KILL to the watcher's isolated process group within `FM_WATCH_STALL_RETIRE_TIMEOUT`, publishes the existing watcher-down recovery episode, and exits with a typed failure. +The stale lock is removed only once the child is dead and still matches the expected PID, so a child that survives the bound keeps its lock and the ledger records the refused release. +Persistent adapters therefore lose their owned-child no-op when the child is no longer healthy and can run their existing bounded retry without restarting the primary session. The arm layer appends one tab-separated record per observed cycle to `state/.watch-cycle-exits.log`. Each record includes arm and watcher PIDs, start and end timestamps, exit code and signal, classified reason, beacon age, lock identity before and after close, and successor disposition. @@ -77,7 +81,7 @@ Only the watcher process touches `state/.last-watcher-beat`; no helper process c `tests/fm-pi-watch-extension.test.sh` checks Pi's first-cycle-or-explicit-repair tool metadata and ownership-based redundant-call no-ops, then simulates actionable and empty child closes against the actual Pi and OpenCode close handlers, blocks prompt delivery to prove the successor launches first, verifies single-flight behavior, changes the session lock before close to prove ownership is rechecked, and hangs each successor arm to prove bounded fallback delivery includes the typed restoration failure. The same suite covers ordinary same-process session replacement for `/new`, `/resume`, and `/fork`, same-instance shutdown-plus-start, stale prior-generation callbacks, repeated transitions with exactly one live cycle, disappearance of the shutting-down refusal after a valid replacement activates, and terminal quit still refusing late rearm. `tests/fm-watch-arm.test.sh` covers durable queue replay, real remote parent-replies ingestion into the authoritative status log, decision-only OPEN DECISIONS recovery, interrupted handling replay, generation-bound acknowledgement, a persistent live successor after recovery, a watcher close inside the handling window that must leave the printed acknowledgement valid, and the self-healing moved-generation acknowledgement that consumes its handled rows and names its remedy. -`tests/fm-watcher-lock.test.sh` covers verified-successor attach, recovery publication before stale-lock removal, the typed self-eviction failure, bounded and successor-linked lifecycle rows, and a SIGSTOP counterfactual that distinguishes a live PID from a stale beacon before classifying termination. +`tests/fm-watcher-lock.test.sh` covers verified-successor attach, recovery publication before stale-lock removal, the typed self-eviction failure, bounded and successor-linked lifecycle rows, and a real-process SIGSTOP counterfactual that proves the bounded retirement contract on both platforms: the Linux refusal shape with the live holder's lock retained, and on other hosts the relinquished lock, same-session re-arm, and a healthy successor whose beacon keeps advancing. `tests/fm-subagent-pretool-check.test.sh` proves Claude retains only the non-status Bash seatbelts. `tests/fm-claude-stop-autoarm.test.sh` covers the auto-arm's scope, stale and live session owners, unchanged AFK and need boundaries, single-flight, bounded failure retries, benign live-watcher cycle ends, one-notice failure episodes, and exit-2 translation. `FM_CLAUDE_LIVE_E2E=1 tests/fm-claude-stop-autoarm-live-e2e.test.sh` starts with the reproduced stale-lock state, runs session start first, completes two tokenless cycles, and checks the competing-live-owner negative control. diff --git a/tests/fm-watch-arm.test.sh b/tests/fm-watch-arm.test.sh index 0115330671..3c75eb42d1 100755 --- a/tests/fm-watch-arm.test.sh +++ b/tests/fm-watch-arm.test.sh @@ -141,7 +141,8 @@ drain_ack_pair() { # start_rearm_arm() { # [predecessor-arm-pid] local home=$1 state=$2 fakebin=$3 armout=$4 predecessor=${5:-} i PATH="$fakebin:$PATH" FM_HOME="$home" FM_STATE_OVERRIDE="$state" \ - FM_POLL=1 FM_SIGNAL_GRACE=0 FM_CHECK_INTERVAL=999999 FM_HEARTBEAT=999999 \ + FM_ARM_CONFIRM_TIMEOUT=60 FM_POLL=1 FM_SIGNAL_GRACE=0 \ + FM_CHECK_INTERVAL=999999 FM_HEARTBEAT=999999 \ FM_WATCH_PREDECESSOR_ARM_PID="$predecessor" \ "$WATCH_ARM" --restart > "$armout" & ARM_PID=$! @@ -287,16 +288,14 @@ test_rearm_resurfaces_durable_queue_and_remote_open_decision() { append_wake "$state" check startup-network 'check: startup-network' start_rearm_arm "$home" "$state" "$fakebin" "$armout" - sleep 0.25 - if is_live_non_zombie "$ARM_PID"; then - # End the fixture through an ordinary actionable status transition so this - # failing pre-fix path leaves no child behind. - printf 'done: fixture cleanup\n' > "$state/cleanup.status" - wait_for_exit "$ARM_PID" 80 || true + # The arm reaps its liveness watchdog before returning the watcher reason, so + # assert the recovery's bounded outcome instead of a scheduler-sensitive + # quarter-second process snapshot. + wait_for_exit "$ARM_PID" 100 + status=$? + if [ "$status" -eq 124 ]; then fail "re-arm stayed live instead of surfacing durable wakes and the still-open remote decision" fi - wait "$ARM_PID" - status=$? expect_code 0 "$status" "re-arm re-surface wake must close successfully" grep -F 'check: rearm-resurface' "$armout" >/dev/null \ || fail "re-arm did not report the durable recovery wake: $(cat "$armout")" diff --git a/tests/fm-watcher-lock.test.sh b/tests/fm-watcher-lock.test.sh index a3628b1694..c0f1e6bb4a 100755 --- a/tests/fm-watcher-lock.test.sh +++ b/tests/fm-watcher-lock.test.sh @@ -902,18 +902,27 @@ SH pass "cycle-exit ledger links a verified successor and remains size-capped" } -test_stopped_watcher_is_live_but_stale_then_exit_is_classified() { - local dir state fakebin armout armpid watcher_pid i status +test_stopped_watcher_is_retired_and_rearms_without_session_restart() { + local dir state fakebin armout recovery_out healthy_out armpid watcher_pid i status + local recovery_arm healthy_arm healthy_pid beat_before beat_after token wedge_limit dir=$(make_case stopped-watcher) state="$dir/state" fakebin="$dir/fakebin" armout="$dir/arm.out" + recovery_out="$dir/recovery.out" + healthy_out="$dir/healthy.out" mark_pr_check_migration_complete "$state" - PATH="$fakebin:$PATH" FM_HOME="$dir" FM_STATE_OVERRIDE="$state" FM_POLL=5 FM_SIGNAL_GRACE=1 FM_CHECK_INTERVAL=999999 FM_HEARTBEAT=999999 "$WATCH_ARM" > "$armout" & + PATH="$fakebin:$PATH" FM_HOME="$dir" FM_STATE_OVERRIDE="$state" \ + FM_GUARD_GRACE=30 FM_ARM_CONFIRM_TIMEOUT=60 FM_ARM_ATTACH_POLL=0.05 FM_POLL=0.1 FM_SIGNAL_GRACE=1 \ + FM_CHECK_INTERVAL=999999 FM_HEARTBEAT=999999 "$WATCH_ARM" > "$armout" & armpid=$! i=0 - while [ "$i" -lt 80 ]; do + # Under host contention the owned child's first fresh beacon can land after + # the arm's default 11s confirmation deadline, so poll through the widened + # confirm budget and fail fast on a loud typed failure instead. + while [ "$i" -lt 650 ]; do grep -qF 'watcher: started pid=' "$armout" 2>/dev/null && break + grep -qF 'watcher: FAILED' "$armout" 2>/dev/null && break sleep 0.1 i=$((i + 1)) done @@ -928,14 +937,108 @@ test_stopped_watcher_is_live_but_stale_then_exit_is_classified() { fail "SIGSTOP watcher with a stale beacon was classified healthy" fi - kill -CONT "$watcher_pid" 2>/dev/null || true - kill -TERM "$watcher_pid" 2>/dev/null || true - wait_for_exit "$armpid" 80 + i=0 + wedge_limit=80 + [ "$(uname)" != Linux ] || wedge_limit=160 + while [ "$i" -lt "$wedge_limit" ] && is_live_non_zombie "$armpid"; do + sleep 0.1 + i=$((i + 1)) + done + if is_live_non_zombie "$armpid"; then + # Pre-fix cleanup: a raw wait on the stopped child held this arm forever. + # Continue the watcher before terminating it so this failing regression + # never strands a stopped process in the test host. + kill -CONT "$watcher_pid" 2>/dev/null || true + kill -TERM "$watcher_pid" 2>/dev/null || true + wait "$armpid" 2>/dev/null || true + fail "arm stayed wedged behind a live watcher whose beacon was stale" + fi + wait "$armpid" + status=$? + [ "$status" -ne 0 ] || fail "stale-beacon retirement did not fail the owned arm loudly" + grep -F 'watcher: FAILED - watcher pid=' "$armout" >/dev/null \ + || fail "stale-beacon retirement omitted its typed watcher failure: $(cat "$armout")" + grep -F 'stopped advancing its beacon' "$armout" >/dev/null \ + || fail "stale-beacon retirement did not name the liveness failure: $(cat "$armout")" + token=$(cat "$state/.watcher-down" 2>/dev/null || true) + case "$token" in + pending:downtime:*) ;; + *) fail "stale-beacon retirement did not publish watcher-down recovery state: '$token'" ;; + esac + case "$(uname)" in + Linux) + # SIGKILL is never held pending for a stopped process on Linux: the + # retirement's KILL kills the stopped watcher immediately, the arm's + # wait reaps it before the expected-pid hardening runs, and the + # retirement takes the released-lock shape (stale-beacon-retired), not + # the release-failed shape. + ! is_live_non_zombie "$watcher_pid" \ + || fail "bounded retirement did not kill the stopped watcher on Linux" + [ ! -e "$state/.watch.lock" ] && [ ! -L "$state/.watch.lock" ] \ + || fail "stalled watcher retained singleton ownership after retirement on Linux" + grep -q 'reason=stale-beacon-retired' "$state/.watch-cycle-exits.log" \ + || fail "stale-beacon retirement was not classified in the lifecycle ledger on Linux" + kill -CONT "$watcher_pid" 2>/dev/null || true + wait_for_exit "$watcher_pid" 40 2>/dev/null || true + pass "owned arm bounds its retirement of a stopped watcher and fails loudly on Linux" + return 0 + ;; + esac + ! is_live_non_zombie "$watcher_pid" \ + || fail "stalled watcher remained alive after bounded retirement" + [ ! -e "$state/.watch.lock" ] && [ ! -L "$state/.watch.lock" ] \ + || fail "stalled watcher retained singleton ownership after retirement" + grep -q 'reason=stale-beacon-retired' "$state/.watch-cycle-exits.log" \ + || fail "stale-beacon retirement was not classified in the lifecycle ledger" + + # A fresh arm in the same primary session must take ownership and surface the + # accepted downtime episode. No Pi/Herdr process is involved in this fixture. + PATH="$fakebin:$PATH" FM_HOME="$dir" FM_STATE_OVERRIDE="$state" \ + FM_GUARD_GRACE=30 FM_ARM_CONFIRM_TIMEOUT=60 FM_ARM_ATTACH_POLL=0.05 FM_POLL=0.1 FM_SIGNAL_GRACE=1 \ + FM_CHECK_INTERVAL=999999 FM_HEARTBEAT=999999 "$WATCH_ARM" > "$recovery_out" & + recovery_arm=$! + wait_for_exit "$recovery_arm" 140 status=$? - [ "$status" -ne 0 ] && [ "$status" -ne 124 ] || fail "terminated stopped-watcher cycle did not surface nonzero (status $status)" - grep -Eq 'reason=(nonzero-exit|signal-exit)' "$state/.watch-cycle-exits.log" \ - || fail "terminated watcher exit was not classified in the lifecycle ledger" - pass "SIGSTOP distinguishes live PID from stale beacon and termination records the exit class" + expect_code 0 "$status" "same-session recovery arm must surface accepted watcher downtime" + grep -F 'check: rearm-resurface' "$recovery_out" >/dev/null \ + || fail "same-session recovery arm did not surface the watcher-down episode: $(cat "$recovery_out")" + drain_and_ack "$state" || fail "same-session watcher recovery acknowledgement failed" + + # Once the recovery episode is acknowledged, the next healthy cycle stays + # live and keeps advancing its real watcher-owned beacon. The real watcher + # advances the beacon once per main-loop iteration, which takes ~2-3.5s on a + # loaded host, so this arm's grace must outrun that cadence: a grace tighter + # than the cadence makes the arm's own stale-beacon watchdog retire a + # healthy child. Poll through the grace with a liveness check per tick, then + # require the beacon to have advanced. + PATH="$fakebin:$PATH" FM_HOME="$dir" FM_STATE_OVERRIDE="$state" \ + FM_GUARD_GRACE=30 FM_ARM_CONFIRM_TIMEOUT=60 FM_ARM_ATTACH_POLL=0.05 FM_POLL=0.1 FM_SIGNAL_GRACE=1 \ + FM_CHECK_INTERVAL=999999 FM_HEARTBEAT=999999 "$WATCH_ARM" > "$healthy_out" & + healthy_arm=$! + i=0 + while [ "$i" -lt 650 ]; do + grep -qF 'watcher: started pid=' "$healthy_out" 2>/dev/null && break + grep -qF 'watcher: FAILED' "$healthy_out" 2>/dev/null && break + sleep 0.1 + i=$((i + 1)) + done + healthy_pid=$(cat "$state/.watch.lock/pid" 2>/dev/null || true) + grep -qF "watcher: started pid=$healthy_pid" "$healthy_out" \ + || fail "healthy recovery cycle did not establish: $(cat "$healthy_out")" + beat_before=$(FM_STATE_OVERRIDE="$state" bash -c '. "$1"; fm_path_mtime "$2"' _ "$LIB" "$state/.last-watcher-beat") + i=0 + while [ "$i" -lt 140 ]; do + sleep 0.1 + is_live_non_zombie "$healthy_arm" \ + || fail "healthy arm was retired by the stale-beacon watchdog" + i=$((i + 1)) + done + beat_after=$(FM_STATE_OVERRIDE="$state" bash -c '. "$1"; fm_path_mtime "$2"' _ "$LIB" "$state/.last-watcher-beat") + [ "$beat_after" -gt "$beat_before" ] \ + || fail "healthy watcher did not advance its beacon ($beat_before -> $beat_after)" + kill -HUP "$healthy_arm" 2>/dev/null || true + wait "$healthy_arm" 2>/dev/null || true + pass "owned arm retires a live stale watcher, releases recovery state, and preserves a healthy successor" } test_pid_identity_is_locale_invariant() { @@ -1126,4 +1229,4 @@ test_arm_propagates_immediate_wake_before_confirmation test_arm_waits_for_peer_beacon_after_child_stands_down test_arm_fails_loud_when_no_fresh_watcher_confirmable test_cycle_exit_ledger_links_successor_and_stays_bounded -test_stopped_watcher_is_live_but_stale_then_exit_is_classified +test_stopped_watcher_is_retired_and_rearms_without_session_restart