diff --git a/AGENTS.md b/AGENTS.md index 8b5403096d..7c258e4ac1 100644 --- a/AGENTS.md +++ b/AGENTS.md @@ -75,6 +75,7 @@ config/startup-memory-budget primary-authoritative per-home startup-memory b config/stow-pass-horizon optional presence flag opting this home in to /stow's default-off pass-count decay horizon; LOCAL, gitignored, and not inherited; see docs/configuration.md "Stow pass horizon" config/herdr-presentation-spaces optional "off" opt-out from, or "on" opt-in to, Herdr's default-on disposable single-task visual projection, which is unconfigured-default-on only at or above a Herdr version floor; LOCAL, gitignored; inherited by secondmate homes; see docs/herdr-backend.md "Presentation spaces" config/trace-context optional presence flag enabling default-off native W3C trace-context propagation to spawned agents; LOCAL, gitignored; inherited by secondmate homes; see docs/configuration.md "Trace context propagation" and docs/trace-context.md +config/turnend-churn-absorb optional presence flag opting this home into the default-off absorb of bare turn-end wakes on pane churn; LOCAL, gitignored, and not inherited; see docs/configuration.md "Turn-end pane-churn absorb" config/cmux-socket-password optional cmux control-socket password; LOCAL, gitignored; read fresh on every cmux CLI call and passed through without ever overriding an operator's own ambient CMUX_SOCKET_PASSWORD when absent (docs/cmux-backend.md "Setup") config/wedge-alarm optional away-mode wedge-alarm active-alert directives; LOCAL, gitignored; absent means auto (macOS Notification Center when available); see docs/wedge-alarm.md config/watched-tools.json optional list of the tools this home depends on, read by the update check armed with bin/fm-tool-update-check.sh; LOCAL, gitignored, firstmate-maintained but human-editable, and NOT inherited by secondmate homes; see docs/configuration.md "Watched tool updates" @@ -132,7 +133,7 @@ state/ runtime records and signals; gitignored .watch.lock .wake-queue.lock watcher singleton and queue serialization locks .claude-autoarm.lock .claude-autoarm-epoch .claude-autoarm-failure-notified .claude-autoarm-failure-alarmed .turnend-claude-blocks .turnend-claude-blocks.lock Claude Stop auto-arm single-flight, epoch, failure-episode, attended-alarm, guard-budget, and budget-lock records; never touch .cursor-park-owner .cursor-park-owner.lock .turnend-cursor-blocks Cursor stop-hook owner record, publication and commit lock, and bounded repair-nag budget; never touch - .hash-* .count-* .stale-* .stale-since-* .paused-* .wedge-escalations-* .writing-* .seen-* .hb-surfaced-* .last-* .heartbeat-streak watcher internals; never touch + .hash-* .count-* .stale-* .stale-since-* .churn-since-* .paused-* .wedge-escalations-* .writing-* .seen-* .hb-surfaced-* .last-* .heartbeat-streak watcher internals; never touch .watch-triage.log watcher's absorbed-wake debug log (size-capped); never relied on, safe to delete .last-watcher-beat watcher liveness beacon, touched every poll (including while absorbing benign wakes); guard scripts read it .subsuper-* .supervise-daemon.* sub-supervisor internals; never touch diff --git a/bin/fm-classify-lib.sh b/bin/fm-classify-lib.sh index 9915ece7d2..34d85d5cd0 100755 --- a/bin/fm-classify-lib.sh +++ b/bin/fm-classify-lib.sh @@ -1167,8 +1167,10 @@ window_to_task() { # captain-relevant last line; 1 otherwise. Pass the space-separated file list that # follows the "signal:" prefix. Non-.status arguments (e.g. .turn-ended markers, # which never carry a verb) are skipped. A 1 here is NOT "benign" on its own: a -# no-verb signal (a bare turn-end, a working: note) is only benign when the crew is -# also provably working (signal_crew_provably_working below); otherwise it surfaces. +# no-verb signal (a bare turn-end, a working: note) is only benign on positive work +# evidence. signal_crew_provably_working below owns the shared authoritative proof; +# fm-watch.sh separately owns the opt-in, bounded pane-churn fallback that is +# limited to bare turn-ends. signal_reason_is_actionable() { # ... local f last for f in "$@"; do @@ -1213,11 +1215,14 @@ crew_absorb_class() { # # 0 if crew shows POSITIVE evidence it is still working (crew_absorb_class # reports `working`). This is the "provably working" predicate at the heart of -# absorb-only-when-provably-working: a no-verb turn-end or stale wake is absorbed -# ONLY when this returns 0, and SURFACED otherwise (the crew may be done, waiting -# on a decision, or wedged). For stale panes it is checked before trusting the -# status log so a pre-validation captain-relevant line does not override an active -# run. See crew_absorb_class for the exact working/paused/none decision. +# absorb-only-on-positive-evidence. This is the sole proof for stale wakes and the +# shared authoritative proof for no-verb signals. Where a home opts in, fm-watch.sh +# may additionally absorb a bare turn-end on bounded pane churn, while every other +# failed verdict surfaces +# because the crew may be done, waiting on a decision, or wedged. For stale panes +# it is checked before trusting the status log so a pre-validation captain-relevant +# line does not override an active run. See crew_absorb_class for the exact +# working/paused/none decision. crew_is_provably_working() { # [ "$(crew_absorb_class "$1")" = working ] } diff --git a/bin/fm-watch.sh b/bin/fm-watch.sh index f5a714b4c7..10179a2134 100755 --- a/bin/fm-watch.sh +++ b/bin/fm-watch.sh @@ -2,19 +2,20 @@ # Firstmate watcher. # Classifies supervision wakes in bash. In normal mode it absorbs benign wakes # and keeps blocking; it queues and exits only for actionable wakes. -# The no-verb signal and stale path is absorb-only-when-provably-working: a wake -# is absorbed only when the crew shows POSITIVE evidence it is still working (an -# actively-running no-mistakes step, or a backend busy signal), and surfaced -# otherwise, so a crew that finishes (or stops and waits) without a current -# working signal is never silently swallowed. A declared wait, either a paused: -# external wait or a verified captain-held transfer, is the separate idle absorb -# case and re-surfaces only on its long bounded cadence, although its initial -# no-verb status signal still surfaces in normal mode. +# The no-verb signal and stale path is absorb-only-on-positive-evidence: a wake +# is absorbed only when the crew shows it is still working through an actively +# running no-mistakes step or a backend busy signal. A home that opts in with +# config/turnend-churn-absorb lets a bare turn-end also use bounded pane churn +# since the previous poll. Every other no-verb wake surfaces, so a crew +# that finishes (or stops and waits) is never silently swallowed. A declared wait, +# either a paused: external wait or a verified captain-held transfer, is the +# separate idle absorb case and re-surfaces only on its long bounded cadence, +# although its initial no-verb status signal still surfaces in normal mode. # While state/.afk exists, the daemon owns triage and this watcher queues and exits # on every wake. Printed reason lines: # signal: ... status/turn-end signals, surfaced when a listed status -# has a captain-relevant verb OR a no-verb signal's crew -# is not provably working, unless afk is active +# has a captain-relevant verb OR a no-verb signal lacks +# positive execution evidence, unless afk is active # stale: a provably-working stale is ALWAYS absorbed (with a wedge # timer) regardless of what the status log says - an active # run-step or busy pane outranks even a captain-relevant log @@ -89,6 +90,7 @@ SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" FM_ROOT="${FM_ROOT_OVERRIDE:-$(cd "$SCRIPT_DIR/.." && pwd)}" FM_HOME="${FM_HOME:-${FM_ROOT_OVERRIDE:-$FM_ROOT}}" STATE="${FM_STATE_OVERRIDE:-$FM_HOME/state}" +CONFIG="${FM_CONFIG_OVERRIDE:-$FM_HOME/config}" mkdir -p "$STATE" # The native event fast-path and only its true dependencies have one narrow @@ -154,6 +156,9 @@ CHECK_TIMEOUT=${FM_CHECK_TIMEOUT:-30} # seconds allowed per *.check.sh SIGNAL_GRACE=${FM_SIGNAL_GRACE:-30} # seconds to linger after a signal so trailing # signals (a status write, then the same turn's # turn-end hook) coalesce into one wake +TURNEND_CHURN_ABSORB_SECS=${FM_TURNEND_CHURN_ABSORB_SECS:-900} # longest a task's + # bare turn-ends may be deferred on pane-churn + # evidence alone (signal_turnend_panes_churned) # Busy state is decided by the semantic contract in bin/fm-busy-lib.sh, which # is the single owner of per-harness sources, source attribution, and the one # remaining rendered-text fallback (Grok only). @@ -162,16 +167,17 @@ SIGNAL_GRACE=${FM_SIGNAL_GRACE:-30} # seconds to linger after a signal so trai # than wake firstmate's LLM for each, this watcher classifies every wake in bash # and ABSORBS the benign majority - it advances the suppression marker, logs to a # debug log, and keeps blocking WITHOUT enqueuing or exiting. The no-verb signal -# / stale path is absorb-only-when-provably-working: such a wake is absorbed ONLY -# while the crew shows positive evidence it is still working (an actively-running -# no-mistakes step, or a busy pane, via crew_is_provably_working over -# fm-crew-state.sh); a crew that stopped its turn with no running pipeline and no -# busy pane is SURFACED, so a finish reported only through interactive pane menus -# (no done: status) is never swallowed. An ACTIONABLE wake (a captain-relevant -# signal, a no-verb signal whose crew is not provably working, any check, a stale -# pane whose crew is not provably working, a provably-working stale past the -# threshold, or anything unknown) is written to the durable queue and exits, which -# is what wakes the LLM through the background-task completion. The same classifier +# / stale path is absorb-only-on-positive-evidence. The shared proof is an actively +# running no-mistakes step or a busy pane via crew_is_provably_working over +# fm-crew-state.sh; where config/turnend-churn-absorb opts in, a bare turn-end alone +# may also use bounded pane churn since the previous poll. +# Every other crew that stopped its turn is SURFACED, so a finish reported +# only through interactive pane menus (no done: status) is never swallowed. An +# ACTIONABLE wake (a captain-relevant signal, a no-verb signal without either +# eligible proof, any check, a stale pane whose crew is not provably working, a +# provably-working stale past the threshold, or anything unknown) is written to +# the durable queue and exits. That wakes the LLM through the background-task +# completion. The same classifier # (fm-classify-lib.sh) backs the away-mode daemon; while state/.afk exists the # daemon owns triage, so this watcher reverts to one-shot (enqueue + exit on every # wake) and never double-triages - and never runs the costly provably-working read. @@ -364,6 +370,205 @@ inbox_steer_check() { # esac } +# 0 (benign/absorb) if EVERY task in a no-verb "signal:" wake has positive work +# evidence; 1 otherwise. Each task may satisfy the authoritative working proof, +# or an eligible bare turn-end may use the opt-in pane-churn proof below. +# +# OFF unless the home creates config/turnend-churn-absorb. The first two proofs +# read a verdict the harness itself vouches for; this one infers execution from +# rendered bytes, which is weaker, so widening the absorb is a home's choice to +# make rather than a default every fleet inherits. With the flag absent this +# delegates to the unchanged all-tasks authoritative proof. +# +# It exists because the first two are unreachable for a harness whose semantic +# busy state has no verified source: bin/fm-crew-state.sh can only answer unknown +# for such an adapter, crew_is_provably_working is therefore never satisfiable, +# and every worker turn boundary surfaced a wake with nothing to act on - the cost +# scaling with the number of workers in flight. Pane churn needs no harness +# cooperation, so it restores the absorb branch for those adapters without +# fabricating a busy verdict any adapter has not earned. +# +# The evidence is the one the pane-staleness backbone below already trusts for +# liveness: this compares a fresh capture against the .hash- marker that backbone +# recorded on the previous poll, which is why the derivation lives here with the +# marker format rather than in the shared classifier. Absorbing here DEFERS a wake +# rather than swallowing it, and the deferral is BOUNDED: a task's turn-ends may +# ride churn evidence for at most FM_TURNEND_CHURN_ABSORB_SECS, tracked per window +# in .churn-since-, after which the wake surfaces and the window restarts. The +# bound is what keeps churn from muting supervision outright. A pane that renders +# continuously - a clock, a spinner, a shell heartbeat, a harness that leaves a +# background renderer alive after its agent yields - never presents the two +# identical consecutive hashes the staleness backbone needs either, so without the +# bound a worker that had genuinely stopped behind such a renderer would be +# deferred here forever with no fallback path left to surface it. Churn and +# staleness read the same pane, so neither can be the other's only backstop. +# Within the bound, an ordinary crew that stops renders nothing more, its pane +# hash stops moving, and the staleness backbone surfaces it within a couple of +# polls; any captain-relevant status verb still surfaces immediately through +# signal_reason_is_actionable. That is why this widens the proof instead of +# bounding the wake rate, which would have suppressed genuinely stopped workers. +# +# Every negative outcome returns 1, so absence of evidence surfaces exactly as +# before: any batch that references a secondmate, an unresolvable task, a task with +# no uniquely attributable recorded endpoint, no previous hash to compare against +# (nothing has been polled yet), a capture that fails or comes back empty, an +# exhausted deferral bound, and of course an unchanged pane. Any .status file also +# returns 1: an authored append is content the supervisor may need to read, so only +# the mechanical turn-end marker gets the fallback. +# +# NOT a pure read: one bounded pane capture per referenced task that lacks +# authoritative proof. Once EVERY task passes, each churn-proven pane's prior +# .stale- classification and wedge-escalation count are cleared because churn +# begins a new quiet interval; retaining either would make the new interval +# inherit the prior one. Reached only for a non-afk, no-captain-verb signal, so +# it never runs on the ordinary per-wake path. +signal_turnend_panes_churned() { # ... + [ -e "$CONFIG/turnend-churn-absorb" ] || return 1 + local f base task meta kind w key backend label terminal prev now since now_s absorb_secs marker age + local rec_task task_index i j count hash_file hash_bytes created + local max_absorb_secs=9223372036854775807 + local -a signal_tasks=() signal_statuses=() snapshot_tasks=() snapshot_kinds=() + local -a snapshot_windows=() snapshot_keys=() snapshot_backends=() snapshot_labels=() + local -a signal_indexes=() churn_indexes=() churned_keys=() missing_keys=() created_keys=() + [ "$#" -gt 0 ] || return 1 + for f in "$@"; do + base=${f##*/} + case "$base" in + *.status) return 1 ;; + *.turn-ended) task=${base%.turn-ended}; kind=turn-ended ;; + *) return 1 ;; + esac + [ -n "$task" ] || return 1 + task_index=-1 + for ((i = 0; i < ${#signal_tasks[@]}; i++)); do + [ "${signal_tasks[$i]}" = "$task" ] && { task_index=$i; break; } + done + if [ "$task_index" -lt 0 ]; then + signal_tasks+=("$task") + [ "$kind" = status ] && signal_statuses+=(1) || signal_statuses+=(0) + elif [ "$kind" = status ]; then + signal_statuses[task_index]=1 + fi + done + for meta in "$STATE"/*.meta; do + [ -e "$meta" ] || continue + rec_task=${meta##*/} + rec_task=${rec_task%.meta} + kind=$(fm_meta_get "$meta" kind) + backend=$(fm_backend_of_meta "$meta") + if [ "$backend" = orca ]; then + terminal=$(fm_meta_get "$meta" terminal) + w=${terminal:-$(fm_meta_get "$meta" window)} + else + w=$(fm_meta_get "$meta" window) + fi + key= + [ -n "$w" ] && key=$(window_key "$w") + label="fm-$rec_task" + snapshot_tasks+=("$rec_task") + snapshot_kinds+=("$kind") + snapshot_windows+=("$w") + snapshot_keys+=("$key") + snapshot_backends+=("$backend") + snapshot_labels+=("$label") + done + # These linear lookups deliberately support stock macOS Bash 3.2.57, enforced + # by macos-stock-bash, and this repository uses no associative arrays in bin/ + # or tests/. A batch is normally one to three tasks and captures dominate its + # cost; indexed lookup is the upgrade path if coalesced batches grow large. + for task in "${signal_tasks[@]}"; do + task_index=-1 + for ((i = 0; i < ${#snapshot_tasks[@]}; i++)); do + [ "${snapshot_tasks[$i]}" = "$task" ] && { task_index=$i; break; } + done + [ "$task_index" -ge 0 ] || return 1 + w=${snapshot_windows[$task_index]} + key=${snapshot_keys[$task_index]} + [ -n "$w" ] && [ -n "$key" ] || return 1 + count=0 + for ((j = 0; j < ${#snapshot_keys[@]}; j++)); do + [ "${snapshot_keys[$j]}" = "$key" ] && count=$((count + 1)) + done + [ "$count" -eq 1 ] || return 1 + signal_indexes+=("$task_index") + done + for task_index in "${signal_indexes[@]}"; do + [ "${snapshot_kinds[$task_index]}" != secondmate ] || return 1 + done + for ((i = 0; i < ${#signal_tasks[@]}; i++)); do + task=${signal_tasks[$i]} + crew_is_provably_working "$task" && continue + task_index=${signal_indexes[$i]} + churn_indexes+=("$task_index") + done + [ "${#churn_indexes[@]}" -gt 0 ] || return 0 + [[ $TURNEND_CHURN_ABSORB_SECS =~ ^[1-9][0-9]*$ ]] || return 1 + if [ "${#TURNEND_CHURN_ABSORB_SECS}" -gt "${#max_absorb_secs}" ] \ + || { [ "${#TURNEND_CHURN_ABSORB_SECS}" -eq "${#max_absorb_secs}" ] \ + && [[ $TURNEND_CHURN_ABSORB_SECS -gt $max_absorb_secs ]]; }; then + return 1 + fi + absorb_secs=$((10#$TURNEND_CHURN_ABSORB_SECS)) + for task_index in "${churn_indexes[@]}"; do + w=${snapshot_windows[$task_index]} + key=${snapshot_keys[$task_index]} + backend=${snapshot_backends[$task_index]} + label=${snapshot_labels[$task_index]} + hash_file="$STATE/.hash-$key" + hash_bytes=$(LC_ALL=C wc -c 2>/dev/null < "$hash_file") || return 1 + hash_bytes=${hash_bytes//[[:space:]]/} + [ "$hash_bytes" = 32 ] || return 1 + prev=$(cat "$hash_file" 2>/dev/null) || return 1 + [[ $prev =~ ^[0-9a-f]{32}$ ]] || return 1 + now=$(fm_backend_capture "$backend" "$w" 40 "$label" 2>/dev/null) || return 1 + [ -n "$now" ] || return 1 + [ "$(printf '%s' "$now" | hash_pane)" != "$prev" ] || return 1 + churned_keys+=("$key") + done + # Enforce the deferral bound BEFORE any .stale- state is touched, so a wake that + # surfaces here leaves the staleness backbone's own classification alone. + now_s=$(date +%s) + for key in "${churned_keys[@]}"; do + marker="$STATE/.churn-since-$key" + if [ ! -e "$marker" ]; then + [ ! -L "$marker" ] || return 1 + missing_keys+=("$key") + continue + fi + since=$(cat "$marker" 2>/dev/null) || return 1 + [[ $since =~ ^(0|[1-9][0-9]*)$ ]] || return 1 + if [ "${#since}" -gt "${#now_s}" ] \ + || { [ "${#since}" -eq "${#now_s}" ] && [[ $since > $now_s ]]; }; then + return 1 + fi + age=$((10#$now_s - 10#$since)) + if [ "$age" -ge "$absorb_secs" ]; then + rm -f "$marker" + return 1 + fi + done + for key in "${missing_keys[@]}"; do + marker="$STATE/.churn-since-$key" + if (set -C; printf '%s' "$now_s" > "$marker") 2>/dev/null; then + created_keys+=("$key") + continue + fi + for created in "${created_keys[@]}"; do + rm -f "$STATE/.churn-since-$created" + done + return 1 + done + for key in "${churned_keys[@]}"; do + if ! rm -f "$STATE/.stale-$key" "$STATE/.wedge-escalations-$key"; then + for created in "${created_keys[@]}"; do + rm -f "$STATE/.churn-since-$created" + done + return 1 + fi + done + return 0 +} + recorded_windows() { local meta w seen= for meta in "$STATE"/*.meta; do @@ -1258,18 +1463,28 @@ EOF # Triage: a signal is ACTIONABLE when any of these holds (cheapest first): # - the away-mode daemon owns triage (afk) and wants every wake; # - any status file carries a captain-relevant verb; - # - or it is a no-verb wake (a bare turn-end, a working: note) whose crew is - # NOT provably working - the crew stopped its turn with no actively-running - # pipeline and no busy pane, so it may be done (even via an interactive menu - # that wrote no done: status), waiting on a decision, or wedged. Absorbing - # such a turn-end is exactly the swallowed-finish this change guards against. + # - or it is a no-verb wake (a bare turn-end, a working: note) with no + # positive evidence the crew is still executing - the crew stopped its turn + # with no actively-running pipeline and no busy pane, so it may be done + # (even via an interactive menu that wrote no done: status), waiting on a + # decision, or wedged. Absorbing such a turn-end is exactly the + # swallowed-finish this change guards against. + # Positive evidence is either an authoritative provably-working verdict or, in a + # home that opts in with config/turnend-churn-absorb and for a BARE turn-end + # alone, a pane that rendered something since the previous poll + # (signal_turnend_panes_churned) - the only proof available to a harness whose + # busy state has no verified semantic source, bounded so it cannot defer that + # task's turn-ends forever. Absorb stays evidence-driven: with neither proof the + # wake surfaces exactly as before. # Actionable -> enqueue, advance .seen-* markers, exit. Benign (a no-verb wake - # whose crew IS provably working) in always-on mode -> advance the markers so it - # will not re-fire, log, and keep blocking without enqueuing. The provably-working - # check is the only costly one (it may run a bounded no-mistakes call), so the || - # ordering evaluates it ONLY for a non-afk, no-captain-verb signal. + # whose crew is still executing) in always-on mode -> advance the markers so it + # will not re-fire, log, and keep blocking without enqueuing. Both evidence + # checks are costly (a bounded no-mistakes call, then a pane capture), so the || + # ordering evaluates them ONLY for a non-afk, no-captain-verb signal, and the + # capture only once the authoritative verdict has already come up short. # shellcheck disable=SC2086 # $files is a space-separated status-path list (ids carry no spaces) - if afk_present || signal_reason_is_actionable $files || ! signal_crew_provably_working $files; then + if afk_present || signal_reason_is_actionable $files \ + || { ! signal_crew_provably_working $files && ! signal_turnend_panes_churned $files; }; then while IFS=$(printf '\t') read -r sf sig f; do [ -n "$sf" ] || continue fm_wake_append signal "$(basename "$f")" "$reason" || exit 1 diff --git a/docs/architecture.md b/docs/architecture.md index cd0318c097..a55392d9d0 100644 --- a/docs/architecture.md +++ b/docs/architecture.md @@ -9,7 +9,7 @@ firstmate's always-loaded operating contract and routing index for conditional p ## Event-driven supervision A zero-token bash watcher (`bin/fm-watch.sh`) sleeps on the fleet, classifies detected wakes in bash, and wakes the first mate only when something is actionable. -Actionable wakes include captain-relevant status signals, no-verb signals whose crew is not provably working, authenticated check output such as PR merge polling or a Relay mention, stale panes whose crew is not provably working whether their status log looks terminal or non-terminal, provably-working stale panes that persist past `FM_STALE_ESCALATE_SECS` without their own task worktree being written, declared external waits and verified captain-held transfers that remain declared past `FM_PAUSE_RESURFACE_SECS`, and heartbeat backstop hits. +Actionable wakes include captain-relevant status signals, no-verb signals without positive evidence that their crew is still executing, authenticated check output such as PR merge polling or a Relay mention, stale panes whose crew is not provably working whether their status log looks terminal or non-terminal, provably-working stale panes that persist past `FM_STALE_ESCALATE_SECS` without their own task worktree being written, declared external waits and verified captain-held transfers that remain declared past `FM_PAUSE_RESURFACE_SECS`, and heartbeat backstop hits. Repeated provably-working stale escalations on the same unchanged pane add an escalation count to the wake reason and, at `FM_WEDGE_DEMAND_INSPECT_COUNT`, a `demand-deep-inspection` marker. A pane holding a file newer than the start of its own quiet window, anywhere in the worktree recorded for that task, is deferred instead of escalated, because a crew writing source, then tests, then documentation behind a static pane is liveness that neither pane quietness nor the run step can show. That deferral re-surfaces on the same `FM_PAUSE_RESURFACE_SECS` cadence as a declared wait, with a reason naming the write evidence rather than a wedge, and it is bounded to one pruned, depth-bounded, wall-clock-bounded walk (`FM_WORKTREE_WRITE_PRUNE`, `FM_WORKTREE_WRITE_MAXDEPTH`, `FM_WORKTREE_WRITE_TIMEOUT`) taken only in the branch that was about to escalate, never on every poll. @@ -27,8 +27,16 @@ When a canonical validated PR poll returns exactly `merged`, the watcher appends The receipt makes retirement safely retryable across restarts: fixed-path recovery revalidates the same evidence, removes the runnable check first, removes its registration and data sidecars, removes the receipt last, and preserves task metadata including `pr=` and `pr_head=`. A concurrent replacement remains armed, every non-merged or invalid observation remains unchanged, and retirement never performs task or persistent-secondmate cleanup. `bin/fm-pr-lib.sh` owns the receipt format and strict identity mechanics, while `bin/fm-watch.sh` owns queue-before-retirement ordering. -No-verb wakes, such as `working:` notes and bare turn-ended signals, are benign only when `bin/fm-crew-state.sh` reports positive evidence that the crew is still working: an actively running no-mistakes step attributed to that crew's current code, or an exact busy verdict from the semantic busy-state contract. -A `kind=secondmate` task's status signal is the parent-directed reply stream and is never absorbed as provably working; only its bare turn-ended signal retains the ordinary absorb rule. +No-verb wakes, such as `working:` notes and bare turn-ended signals, are benign only when every referenced task independently has positive evidence that its crew is still working: an actively running no-mistakes step attributed to that crew's current code, or an exact busy verdict from the semantic busy-state contract, both read through `bin/fm-crew-state.sh`. +A home that creates `config/turnend-churn-absorb` lets each eligible bare turn-ended task that lacks either authoritative proof use a third form: pane content that changed since the previous poll, compared against the same `state/.hash-*` marker the staleness backbone records, which claims no harness semantics and needs no adapter cooperation. +That form stays opt-in because it infers execution from rendered bytes rather than from a verdict the harness vouches for, so with the flag absent triage behaves exactly as it did before ([`configuration.md`](configuration.md) "Turn-end pane-churn absorb"). +That evidence clears the pane's prior stale classification and wedge-escalation count, then defers such a wake rather than swallowing it, since a crew that has stopped renders nothing further and its now-static pane surfaces through the staleness backbone within a poll or two, even if its final bytes match an earlier stale render. +A wake naming any status file remains governed solely by the strict authoritative proof, and the pane-churn fallback is unavailable to an entire batch that references a secondmate. +An unresolvable endpoint, an ambiguous marker key, a missing or malformed prior hash, a capture that fails or returns empty, an invalid deferral bound or deadline, or an unwritable deferral marker surfaces without clearing prior stale classification. +The deferral is bounded per endpoint by `FM_TURNEND_CHURN_ABSORB_SECS`, tracked in `state/.churn-since-*`, after which the turn-end surfaces and the window restarts. +That bound is load-bearing rather than cosmetic: churn and staleness read the same pane, so a pane that renders continuously - a clock, a spinner, a shell heartbeat, or a harness that leaves a background renderer alive after its agent yields - never reaches the staleness backbone's two-identical-hashes test either, and an unbounded churn absorb would leave a genuinely stopped worker behind such a renderer with no path left to surface it. +If two metadata records derive the same per-window marker key, including two records that name the same endpoint, that marker is not attributable churn evidence for either task, so the bare turn-ended wake surfaces without changing or migrating existing marker state. +A `kind=secondmate` task's status signal is the parent-directed reply stream and is never absorbed as provably working; a batch containing its bare turn-ended signal is absorbed only when every referenced task passes the ordinary authoritative working proof because an active secondmate does not enter the staleness backbone that would resurface deferred pane-churn evidence. A crew that declares `paused:` for a known external wait, or carries a verified `captain-held` transfer, is separately absorbed while idle and re-surfaced only on the longer pause cadence, rather than being treated as a possible wedge. For an ordinary crew that has stopped, the normal-mode watcher first surfaces one stale wake, then applies that same cadence to an unchanged `paused:` or durable `captain-held` endpoint only when the backend confidently reports its agent dead. Live or inconclusive liveness remains fail-open at that initial surface, and a secondmate's endpoint liveness is still never read at all; a mate is admitted to that same cadence only to serve a declared wait's bounded re-surface, so a forgotten pause or captain hold on a mate cannot rot invisibly. diff --git a/docs/configuration.md b/docs/configuration.md index a861b40678..6393ee84f0 100644 --- a/docs/configuration.md +++ b/docs/configuration.md @@ -137,6 +137,17 @@ A Secondmate on a remote route is covered the same way: the primary resolves and The presence flag is session-scoped enablement, so it transfers at launch and is left unchanged by live convergence into a running home. See [`trace-context.md`](trace-context.md) for carrier semantics, supported routes, the manual fleet-restart requirement, the session boundary, and safety limits; `bin/fm-trace-context-lib.sh`'s header owns the exact mechanics, and [`verification/trace-context.md`](verification/trace-context.md) records repeatable evidence. +## Turn-end pane-churn absorb (config/turnend-churn-absorb) + +The optional local, gitignored `config/turnend-churn-absorb` presence flag opts this home into a default-off third form of positive work evidence in watcher triage. +With it present, bounded pane churn becomes available as fallback evidence for eligible bare turn-ended batches. +It stays opt-in because the other two proofs read a verdict the harness itself vouches for while this one infers execution from rendered bytes; with the flag absent triage behaves exactly as it did before. +`FM_TURNEND_CHURN_ABSORB_SECS` is a positive integer number of seconds, defaults to `900`, and bounds how long one endpoint's turn-ends may ride that evidence before surfacing anyway. +An invalid value fails closed and surfaces the wake. +The bound is required rather than cosmetic because churn and pane staleness read the same pane. +The flag is a home-local supervision-noise preference and is not inherited by secondmate homes, which run their own crew mix. +[`architecture.md`](architecture.md) owns the triage contract and `bin/fm-watch.sh`'s `signal_turnend_panes_churned` owns the exact evidence and fail-closed boundaries. + ## Gate defaults (.no-mistakes.yaml) The tracked `.no-mistakes.yaml` sets `test.evidence.store_in_repo: true` and pins `commands.lint` to `bin/fm-lint.sh` so local lint matches CI. @@ -700,6 +711,7 @@ FM_WATCH_CYCLE_LOG_MAX_BYTES=262144 # size cap for the arm-owned watcher lifec FM_WATCH_CYCLE_LOG_KEEP_LINES=1000 # newest complete lifecycle rows considered when the ledger is capped FM_WATCHER_STALE_GRACE=300 # defaults to FM_GUARD_GRACE; seconds a live watcher lock may have a stale beacon before re-arm errors FM_SIGNAL_GRACE=30 # seconds to coalesce nearby status and turn-end signals into one wake +FM_TURNEND_CHURN_ABSORB_SECS=900 # longest one endpoint's bare turn-ends may be deferred on pane-churn evidence alone; only consulted when config/turnend-churn-absorb is present FM_CAPTAIN_RE='done:|needs-decision:|blocked:|failed:|PR ready|checks green|ready in branch|merged' # captain-relevant status regex; nonterminal progress verbs remain excluded even when their prose matches FM_CLASSIFY_PAUSED_VERB=paused # leading status verb for a declared external wait; excluded from FM_CAPTAIN_RE and distinct from blocked FM_STALE_ESCALATE_SECS=240 # idle seconds before a provably-working stale pane escalates; stale panes whose crew is not provably working surface immediately unless they declare the pause verb diff --git a/tests/fm-watch-triage.test.sh b/tests/fm-watch-triage.test.sh index 1bee5cab9b..c629428a8c 100755 --- a/tests/fm-watch-triage.test.sh +++ b/tests/fm-watch-triage.test.sh @@ -657,6 +657,689 @@ test_turn_ended_not_working_surfaced() { pass "a bare turn-end whose crew is not provably working is surfaced (the swallowed-finish fix)" } +# --- bare turn-end, unverifiable harness: pane churn is the third proof -------- +# A harness whose semantic busy state has no verified source (codex) can never +# report working, so the two proofs above are unreachable for it and EVERY worker +# turn boundary woke firstmate. Pane content that changed since the previous poll +# is harness-independent positive evidence the crew is still executing - the same +# liveness input the stale backbone already trusts - so a bare turn-end from a +# churning pane is benign. The pane going quiet afterwards is still caught by that +# backbone, which is why this widens the proof rather than bounding the wake rate. + +# The pane-churn turn-end absorb is opt-in per home, so every case that exercises +# it (whether it expects an absorb or one of the guards that must still surface) +# points the watcher at a case-local config dir holding the flag. A case that must +# NOT have it points at an empty one, so no developer's real config can leak in. +churn_config() { # [off] + local cfg="$1/config" + mkdir -p "$cfg" + [ "${2:-}" = off ] || : > "$cfg/turnend-churn-absorb" + printf '%s\n' "$cfg" +} + +# Wait until the watcher records an absorbed wake matching in its triage +# log. 1 if the watcher exits first (i.e. it surfaced the wake instead), which is +# exactly the unfixed behavior this case exists to catch. Polls the log rather +# than a poll cycle so the assertion lands inside the FIRST poll, long before an +# unchanging fixture pane could reach the stale backbone. +wait_for_absorbed() { # + local state=$1 pid=$2 needle=$3 i=0 + while [ "$i" -lt 100 ]; do + grep -Fq "$needle" "$state/.watch-triage.log" 2>/dev/null && return 0 + kill -0 "$pid" 2>/dev/null || return 1 + sleep 0.1 + i=$((i + 1)) + done + return 1 +} + +test_turn_ended_churning_pane_absorbed() { + local dir state fakebin out capture_file window key pid + dir=$(make_case turn-ended-churning); state="$dir/state"; fakebin="$dir/fakebin" + out="$dir/watch.out"; capture_file="$dir/pane.txt" + window="test:fm-codexer" + : > "$state/codexer.turn-ended" + printf 'window=%s\nkind=ship\nharness=codex\n' "$window" > "$state/codexer.meta" + printf 'apply_patch: writing bin/thing.sh' > "$capture_file" + key=$(printf '%s' "$window" | tr ':/.' '___') + # The previous poll recorded DIFFERENT pane content, so this poll's capture is + # churn: the crew rendered output between the two polls. + printf '%s' "$(hash_text 'reading the brief')" > "$state/.hash-$key" + printf '0\n' > "$state/.count-$key" + # The codex verdict verbatim: a verified dispatch adapter with no verified + # semantic busy source, so crew_is_provably_working can never be satisfied. + export FM_FAKE_CREW_STATE='state: unknown · source: pane · harness state unavailable (unknown codex-unverified)' + # A slow poll leaves the first cycle's absorb assertion many ticks clear of the + # stale backbone, which this static fixture pane would otherwise reach. + PATH="$fakebin:$PATH" FM_FAKE_TMUX_WINDOW="$window" FM_FAKE_TMUX_CAPTURE="$capture_file" \ + FM_CONFIG_OVERRIDE="$(churn_config "$dir")" \ + FM_STATE_OVERRIDE="$state" FM_CREW_STATE_BIN="$fakebin/fm-crew-state.sh" FM_POLL=3 FM_SIGNAL_GRACE=1 \ + FM_CHECK_INTERVAL=999999 FM_HEARTBEAT=999999 "$WATCH" > "$out" & + pid=$! + wait_for_absorbed "$state" "$pid" "absorbed benign signal:" \ + || { reap "$pid"; fail "a bare turn-end from a churning pane was not absorbed: $(cat "$out")"; } + [ ! -s "$out" ] || fail "an absorbed churning-pane turn-end printed a wake reason: $(cat "$out")" + [ ! -s "$state/.wake-queue" ] || fail "an absorbed churning-pane turn-end enqueued a durable wake record" + [ -s "$state/.churn-since-$key" ] \ + || { reap "$pid"; fail "an absorbed churning-pane turn-end did not open a bounded deferral window"; } + reap "$pid" + unset FM_FAKE_CREW_STATE + pass "a bare turn-end from a pane that churned since the previous poll is absorbed" +} + +test_turn_ended_churn_resets_prior_stale_classification() { + local dir state fakebin out capture_file window key old_hash active_hash pid i + dir=$(make_case turn-ended-churn-resets-stale); state="$dir/state"; fakebin="$dir/fakebin" + out="$dir/watch.out"; capture_file="$dir/pane.txt" + window="test:fm-codexreturned" + : > "$state/codexreturned.turn-ended" + printf 'window=%s\nkind=ship\nharness=codex\n' "$window" > "$state/codexreturned.meta" + old_hash=$(hash_text 'idle prompt from an earlier turn') + active_hash=$(hash_text 'rendering a new turn') + printf 'rendering a new turn' > "$capture_file" + key=$(printf '%s' "$window" | tr ':/.' '___') + printf '%s' "$old_hash" > "$state/.hash-$key" + printf '1\n' > "$state/.count-$key" + printf '%s' "$old_hash" > "$state/.stale-$key" + date +%s > "$state/.stale-since-$key" + export FM_FAKE_CREW_STATE='state: unknown · source: pane · harness state unavailable (unknown codex-unverified)' + PATH="$fakebin:$PATH" FM_FAKE_TMUX_WINDOW="$window" FM_FAKE_TMUX_CAPTURE="$capture_file" \ + FM_CONFIG_OVERRIDE="$(churn_config "$dir")" \ + FM_STATE_OVERRIDE="$state" FM_CREW_STATE_BIN="$fakebin/fm-crew-state.sh" FM_STALE_ESCALATE_SECS=999 \ + FM_POLL=1 FM_SIGNAL_GRACE=1 FM_CHECK_INTERVAL=999999 FM_HEARTBEAT=999999 "$WATCH" > "$out" & + pid=$! + wait_for_absorbed "$state" "$pid" "absorbed benign signal:" \ + || { reap "$pid"; fail "a churning turn-end with prior stale state was not absorbed: $(cat "$out")"; } + i=0 + while [ "$i" -lt 100 ] && [ "$(cat "$state/.hash-$key" 2>/dev/null || true)" != "$active_hash" ]; do + kill -0 "$pid" 2>/dev/null || { reap "$pid"; fail "watcher exited before recording the active pane"; } + sleep 0.1 + i=$((i + 1)) + done + [ "$(cat "$state/.hash-$key" 2>/dev/null || true)" = "$active_hash" ] \ + || { reap "$pid"; fail "watcher did not record the active pane after absorbing its turn-end"; } + + # The worker stops on bytes that happened to be stale in an earlier turn. + # This is a new quiet interval, so it must surface through ordinary staleness + # instead of inheriting the earlier interval's wedge timer. + printf 'idle prompt from an earlier turn' > "$capture_file" + wait_for_exit "$pid" 100 \ + || { reap "$pid"; fail "a stopped pane matching an earlier stale render waited for the wedge timeout"; } + grep -Fx "stale: $window" "$out" >/dev/null \ + || fail "the returned stale render did not surface through ordinary staleness" + grep -F "possible wedge" "$out" >/dev/null \ + && fail "the returned stale render inherited the earlier quiet interval's wedge classification" + unset FM_FAKE_CREW_STATE + pass "pane churn starts a fresh stale-classification interval before a stopped render returns" +} + +test_turn_ended_churn_resets_wedge_state_before_stale_poll() { + local dir state fakebin out capture_file capture_count window key pid + dir=$(make_case turn-ended-churn-resets-wedge); state="$dir/state"; fakebin="$dir/fakebin" + out="$dir/watch.out"; capture_file="$dir/pane.txt"; capture_count="$dir/capture.count" + window="test:fm-codexfreshinterval" + : > "$state/codexfreshinterval.turn-ended" + printf 'window=%s\nkind=ship\nharness=codex\n' "$window" > "$state/codexfreshinterval.meta" + printf 'rendering a new turn' > "$capture_file" + key=$(printf '%s' "$window" | tr ':/.' '___') + printf '%s' "$(hash_text 'idle output from the prior interval')" > "$state/.hash-$key" + printf '2\n' > "$state/.wedge-escalations-$key" + export FM_FAKE_CREW_STATE='state: unknown · source: pane · harness state unavailable (unknown codex-unverified)' + PATH="$fakebin:$PATH" FM_FAKE_TMUX_WINDOW="$window" FM_FAKE_TMUX_CAPTURE="$capture_file" \ + FM_FAKE_TMUX_CAPTURE_COUNT_FILE="$capture_count" FM_FAKE_TMUX_CAPTURE_FAIL_AFTER=1 \ + FM_CONFIG_OVERRIDE="$(churn_config "$dir")" \ + FM_STATE_OVERRIDE="$state" FM_CREW_STATE_BIN="$fakebin/fm-crew-state.sh" FM_POLL=3 FM_SIGNAL_GRACE=1 \ + FM_CHECK_INTERVAL=999999 FM_HEARTBEAT=999999 "$WATCH" > "$out" & + pid=$! + wait_for_absorbed "$state" "$pid" "absorbed benign signal:" \ + || { reap "$pid"; fail "a churning turn-end was not absorbed before the stale-path capture failed: $(cat "$out")"; } + [ ! -e "$state/.wedge-escalations-$key" ] \ + || { reap "$pid"; fail "churn retained the prior quiet interval's wedge-escalation count"; } + [ ! -s "$state/.wake-queue" ] \ + || { reap "$pid"; fail "the absorbed churn fixture queued an unexpected wake"; } + reap "$pid" + unset FM_FAKE_CREW_STATE + pass "pane churn resets prior wedge escalation state before the stale-path poll" +} + +# The safety half: the same unverifiable harness, the same fixture, but the pane +# has NOT changed since the previous poll. There is no positive evidence, so the +# wake must still surface - a stopped worker is exactly what the turn-end marker +# earns its keep detecting, and widening the proof must not cost that. +test_turn_ended_still_pane_surfaced() { + local dir state fakebin out drain_out capture_file window key pid + dir=$(make_case turn-ended-still); state="$dir/state"; fakebin="$dir/fakebin" + out="$dir/watch.out"; drain_out="$dir/drain.out"; capture_file="$dir/pane.txt" + window="test:fm-codexstopped" + : > "$state/codexstopped.turn-ended" + printf 'window=%s\nkind=ship\nharness=codex\n' "$window" > "$state/codexstopped.meta" + printf 'apply_patch: writing bin/thing.sh' > "$capture_file" + key=$(printf '%s' "$window" | tr ':/.' '___') + # The previous poll recorded THIS pane content: nothing rendered since. + printf '%s' "$(hash_text 'apply_patch: writing bin/thing.sh')" > "$state/.hash-$key" + printf '0\n' > "$state/.count-$key" + export FM_FAKE_CREW_STATE='state: unknown · source: pane · harness state unavailable (unknown codex-unverified)' + PATH="$fakebin:$PATH" FM_FAKE_TMUX_WINDOW="$window" FM_FAKE_TMUX_CAPTURE="$capture_file" \ + FM_CONFIG_OVERRIDE="$(churn_config "$dir")" \ + FM_STATE_OVERRIDE="$state" FM_CREW_STATE_BIN="$fakebin/fm-crew-state.sh" FM_POLL=3 FM_SIGNAL_GRACE=1 \ + FM_CHECK_INTERVAL=999999 FM_HEARTBEAT=999999 "$WATCH" > "$out" & + pid=$! + wait_for_exit "$pid" 100 || fail "watcher did not surface a bare turn-end from an unchanged pane" + grep -F "signal: $state/codexstopped.turn-ended" "$out" >/dev/null \ + || fail "watcher did not print the surfaced still-pane turn-end signal" + FM_STATE_OVERRIDE="$state" "$DRAIN" > "$drain_out" 2>/dev/null || fail "drain after the still-pane turn-end failed" + grep "$(printf '\tsignal\t')" "$drain_out" | grep -F "$state/codexstopped.turn-ended" >/dev/null \ + || fail "surfaced still-pane turn-end was not queued" + unset FM_FAKE_CREW_STATE + pass "a bare turn-end from a pane unchanged since the previous poll still surfaces" +} + +test_turn_ended_malformed_prior_hash_surfaced() { + local dir state fakebin out drain_out capture_file window key pid + dir=$(make_case turn-ended-malformed-hash); state="$dir/state"; fakebin="$dir/fakebin" + out="$dir/watch.out"; drain_out="$dir/drain.out"; capture_file="$dir/pane.txt" + window="test:fm-codexmalformed" + : > "$state/codexmalformed.turn-ended" + printf 'window=%s\nkind=ship\nharness=codex\n' "$window" > "$state/codexmalformed.meta" + printf 'stopped after rendering this' > "$capture_file" + key=$(printf '%s' "$window" | tr ':/.' '___') + printf 'x' > "$state/.hash-$key" + printf '0\n' > "$state/.count-$key" + export FM_FAKE_CREW_STATE='state: unknown · source: pane · harness state unavailable (unknown codex-unverified)' + PATH="$fakebin:$PATH" FM_FAKE_TMUX_WINDOW="$window" FM_FAKE_TMUX_CAPTURE="$capture_file" \ + FM_CONFIG_OVERRIDE="$(churn_config "$dir")" \ + FM_STATE_OVERRIDE="$state" FM_CREW_STATE_BIN="$fakebin/fm-crew-state.sh" FM_POLL=3 FM_SIGNAL_GRACE=1 \ + FM_CHECK_INTERVAL=999999 FM_HEARTBEAT=999999 "$WATCH" > "$out" & + pid=$! + wait_for_exit "$pid" 100 || fail "watcher absorbed a turn-end backed by a malformed prior hash" + grep -F "signal: $state/codexmalformed.turn-ended" "$out" >/dev/null \ + || fail "watcher did not print the surfaced malformed-hash turn-end" + FM_STATE_OVERRIDE="$state" "$DRAIN" > "$drain_out" 2>/dev/null \ + || fail "drain after the malformed-hash turn-end failed" + grep "$(printf '\tsignal\t')" "$drain_out" | grep -F "$state/codexmalformed.turn-ended" >/dev/null \ + || fail "malformed-hash turn-end was not queued" + unset FM_FAKE_CREW_STATE + pass "a bare turn-end backed by a malformed prior hash surfaces" +} + +test_turn_ended_trailing_newline_prior_hash_surfaced() { + local dir state fakebin out drain_out capture_file window key pid + dir=$(make_case turn-ended-newline-hash); state="$dir/state"; fakebin="$dir/fakebin" + out="$dir/watch.out"; drain_out="$dir/drain.out"; capture_file="$dir/pane.txt" + window="test:fm-codexnewline" + : > "$state/codexnewline.turn-ended" + printf 'window=%s\nkind=ship\nharness=codex\n' "$window" > "$state/codexnewline.meta" + printf 'rendered after the prior poll' > "$capture_file" + key=$(printf '%s' "$window" | tr ':/.' '___') + printf '%s\n' "$(hash_text 'the previous render')" > "$state/.hash-$key" + printf '0\n' > "$state/.count-$key" + export FM_FAKE_CREW_STATE='state: unknown · source: pane · harness state unavailable (unknown codex-unverified)' + PATH="$fakebin:$PATH" FM_FAKE_TMUX_WINDOW="$window" FM_FAKE_TMUX_CAPTURE="$capture_file" \ + FM_CONFIG_OVERRIDE="$(churn_config "$dir")" \ + FM_STATE_OVERRIDE="$state" FM_CREW_STATE_BIN="$fakebin/fm-crew-state.sh" FM_POLL=3 FM_SIGNAL_GRACE=1 \ + FM_CHECK_INTERVAL=999999 FM_HEARTBEAT=999999 "$WATCH" > "$out" & + pid=$! + wait_for_exit "$pid" 100 || fail "watcher absorbed a turn-end backed by a newline-terminated prior hash" + grep -F "signal: $state/codexnewline.turn-ended" "$out" >/dev/null \ + || fail "watcher did not print the surfaced newline-hash turn-end" + FM_STATE_OVERRIDE="$state" "$DRAIN" > "$drain_out" 2>/dev/null \ + || fail "drain after the newline-hash turn-end failed" + grep "$(printf '\tsignal\t')" "$drain_out" | grep -F "$state/codexnewline.turn-ended" >/dev/null \ + || fail "newline-hash turn-end was not queued" + [ ! -e "$state/.churn-since-$key" ] \ + || fail "a newline-terminated prior hash opened a deferral window" + unset FM_FAKE_CREW_STATE + pass "a bare turn-end backed by a newline-terminated prior hash surfaces" +} + +test_secondmate_turn_ended_churning_pane_surfaced() { + local dir state fakebin out drain_out capture_file window key pid + dir=$(make_case secondmate-turn-ended-churning); state="$dir/state"; fakebin="$dir/fakebin" + out="$dir/watch.out"; drain_out="$dir/drain.out"; capture_file="$dir/pane.txt" + window="test:fm-mate-churning" + : > "$state/mate.turn-ended" + printf 'window=%s\nkind=secondmate\nharness=pi\n' "$window" > "$state/mate.meta" + printf 'working on the next routed item' > "$capture_file" + key=$(printf '%s' "$window" | tr ':/.' '___') + printf '%s' "$(hash_text 'waiting for work')" > "$state/.hash-$key" + printf '0\n' > "$state/.count-$key" + export FM_FAKE_CREW_STATE='state: unknown · source: pane · harness state unavailable' + PATH="$fakebin:$PATH" FM_FAKE_TMUX_WINDOW="$window" FM_FAKE_TMUX_CAPTURE="$capture_file" \ + FM_CONFIG_OVERRIDE="$(churn_config "$dir")" \ + FM_STATE_OVERRIDE="$state" FM_CREW_STATE_BIN="$fakebin/fm-crew-state.sh" FM_POLL=3 FM_SIGNAL_GRACE=1 \ + FM_CHECK_INTERVAL=999999 FM_HEARTBEAT=999999 "$WATCH" > "$out" & + pid=$! + wait_for_exit "$pid" 100 || fail "watcher did not surface a churning secondmate turn-end" + grep -F "signal: $state/mate.turn-ended" "$out" >/dev/null \ + || fail "watcher did not print the surfaced churning secondmate turn-end" + FM_STATE_OVERRIDE="$state" "$DRAIN" > "$drain_out" 2>/dev/null \ + || fail "drain after the churning secondmate turn-end failed" + grep "$(printf '\tsignal\t')" "$drain_out" | grep -F "$state/mate.turn-ended" >/dev/null \ + || fail "churning secondmate turn-end was not queued" + unset FM_FAKE_CREW_STATE + pass "a churning secondmate turn-end surfaces without a stale resurface path" +} + +test_turn_ended_colliding_window_key_surfaced() { + local dir state fakebin out drain_out capture_file window colliding key pid + dir=$(make_case turn-ended-colliding-key); state="$dir/state"; fakebin="$dir/fakebin" + out="$dir/watch.out"; drain_out="$dir/drain.out"; capture_file="$dir/pane.txt" + window="test:fm-a.b"; colliding="test:fm-a_b" + : > "$state/a.b.turn-ended" + printf 'window=%s\nkind=ship\nharness=codex\n' "$window" > "$state/a.b.meta" + printf 'window=%s\nkind=ship\nharness=codex\n' "$colliding" > "$state/a_b.meta" + printf 'rendered after the prior poll' > "$capture_file" + key=$(printf '%s' "$window" | tr ':/.' '___') + printf '%s' "$(hash_text 'the other window pane')" > "$state/.hash-$key" + printf '0\n' > "$state/.count-$key" + export FM_FAKE_CREW_STATE='state: unknown · source: pane · harness state unavailable (unknown codex-unverified)' + PATH="$fakebin:$PATH" FM_FAKE_TMUX_WINDOW="$window" FM_FAKE_TMUX_CAPTURE="$capture_file" \ + FM_CONFIG_OVERRIDE="$(churn_config "$dir")" \ + FM_STATE_OVERRIDE="$state" FM_CREW_STATE_BIN="$fakebin/fm-crew-state.sh" FM_POLL=3 FM_SIGNAL_GRACE=1 \ + FM_CHECK_INTERVAL=999999 FM_HEARTBEAT=999999 "$WATCH" > "$out" & + pid=$! + wait_for_exit "$pid" 100 || fail "watcher did not surface a turn-end with an ambiguous pane marker" + grep -F "signal: $state/a.b.turn-ended" "$out" >/dev/null \ + || fail "watcher did not print the surfaced ambiguous-marker turn-end" + FM_STATE_OVERRIDE="$state" "$DRAIN" > "$drain_out" 2>/dev/null \ + || fail "drain after the ambiguous-marker turn-end failed" + grep "$(printf '\tsignal\t')" "$drain_out" | grep -F "$state/a.b.turn-ended" >/dev/null \ + || fail "ambiguous-marker turn-end was not queued" + unset FM_FAKE_CREW_STATE + pass "a turn-end whose marker key matches another recorded endpoint surfaces" +} + +test_turn_ended_duplicate_endpoint_records_surfaced() { + local dir state fakebin out drain_out capture_file window key pid + dir=$(make_case turn-ended-duplicate-endpoint); state="$dir/state"; fakebin="$dir/fakebin" + out="$dir/watch.out"; drain_out="$dir/drain.out"; capture_file="$dir/pane.txt" + window="test:fm-shared" + : > "$state/first.turn-ended" + printf 'window=%s\nkind=ship\nharness=codex\n' "$window" > "$state/first.meta" + printf 'window=%s\nkind=ship\nharness=codex\n' "$window" > "$state/second.meta" + printf 'rendered after the prior poll' > "$capture_file" + key=$(printf '%s' "$window" | tr ':/.' '___') + printf '%s' "$(hash_text 'the previous render')" > "$state/.hash-$key" + printf '0\n' > "$state/.count-$key" + export FM_FAKE_CREW_STATE='state: unknown · source: pane · harness state unavailable (unknown codex-unverified)' + PATH="$fakebin:$PATH" FM_FAKE_TMUX_WINDOW="$window" FM_FAKE_TMUX_CAPTURE="$capture_file" \ + FM_CONFIG_OVERRIDE="$(churn_config "$dir")" \ + FM_STATE_OVERRIDE="$state" FM_CREW_STATE_BIN="$fakebin/fm-crew-state.sh" FM_POLL=3 FM_SIGNAL_GRACE=1 \ + FM_CHECK_INTERVAL=999999 FM_HEARTBEAT=999999 "$WATCH" > "$out" & + pid=$! + wait_for_exit "$pid" 100 || fail "watcher absorbed a turn-end shared by two endpoint records" + grep -F "signal: $state/first.turn-ended" "$out" >/dev/null \ + || fail "watcher did not print the surfaced duplicate-endpoint turn-end" + FM_STATE_OVERRIDE="$state" "$DRAIN" > "$drain_out" 2>/dev/null \ + || fail "drain after the duplicate-endpoint turn-end failed" + grep "$(printf '\tsignal\t')" "$drain_out" | grep -F "$state/first.turn-ended" >/dev/null \ + || fail "duplicate-endpoint turn-end was not queued" + [ ! -e "$state/.churn-since-$key" ] \ + || fail "duplicate endpoint records opened a deferral window" + unset FM_FAKE_CREW_STATE + pass "two metadata records sharing one endpoint make churn evidence ambiguous" +} + +test_turn_ended_mixed_positive_evidence_batch_absorbed() { + local dir state fakebin out capture_file first_window second_window first_key second_key pid + dir=$(make_case turn-ended-mixed-evidence); state="$dir/state"; fakebin="$dir/fakebin" + out="$dir/watch.out"; capture_file="$dir/pane.txt" + first_window="test:fm-first"; second_window="test:fm-second" + : > "$state/first.turn-ended" + : > "$state/second.turn-ended" + printf 'window=%s\nkind=ship\nharness=pi\n' "$first_window" > "$state/first.meta" + printf 'window=%s\nkind=ship\nharness=codex\n' "$second_window" > "$state/second.meta" + printf 'second task rendered after the prior poll' > "$capture_file" + first_key=$(printf '%s' "$first_window" | tr ':/.' '___') + second_key=$(printf '%s' "$second_window" | tr ':/.' '___') + printf '%s' "$(hash_text 'first task static pane')" > "$state/.hash-$first_key" + printf '%s' "$(hash_text 'second task previous render')" > "$state/.hash-$second_key" + printf '0\n' > "$state/.count-$first_key" + printf '0\n' > "$state/.count-$second_key" + export FM_FAKE_CREW_STATE_first='state: working · source: run-step · running' + export FM_FAKE_CREW_STATE_second='state: unknown · source: pane · harness state unavailable (unknown codex-unverified)' + PATH="$fakebin:$PATH" FM_FAKE_TMUX_WINDOWS="$(printf 'fm-first\nfm-second')" \ + FM_FAKE_TMUX_CAPTURE="$capture_file" FM_FAKE_TMUX_FORBIDDEN_TARGET="$first_window" \ + FM_CONFIG_OVERRIDE="$(churn_config "$dir")" \ + FM_STATE_OVERRIDE="$state" FM_CREW_STATE_BIN="$fakebin/fm-crew-state.sh" FM_POLL=3 FM_SIGNAL_GRACE=1 \ + FM_CHECK_INTERVAL=999999 FM_HEARTBEAT=999999 "$WATCH" > "$out" & + pid=$! + wait_for_absorbed "$state" "$pid" "absorbed benign signal:" \ + || { reap "$pid"; fail "a mixed authoritative-and-churn batch was not absorbed: $(cat "$out")"; } + [ ! -s "$out" ] || fail "an absorbed mixed-evidence batch printed a wake reason: $(cat "$out")" + [ ! -s "$state/.wake-queue" ] || fail "an absorbed mixed-evidence batch enqueued a durable wake record" + [ ! -e "$state/.churn-since-$first_key" ] \ + || fail "an authoritatively working task opened a pane-churn deadline" + [ -s "$state/.churn-since-$second_key" ] \ + || fail "the churn-proven task did not open its bounded deferral window" + reap "$pid" + unset FM_FAKE_CREW_STATE_first FM_FAKE_CREW_STATE_second + pass "a batch may satisfy positive evidence independently per task" +} + +test_turn_ended_mixed_positive_evidence_batch_default_off() { + local dir state fakebin out drain_out capture_file first_window second_window first_key second_key pid + dir=$(make_case turn-ended-mixed-evidence-off); state="$dir/state"; fakebin="$dir/fakebin" + out="$dir/watch.out"; drain_out="$dir/drain.out"; capture_file="$dir/pane.txt" + first_window="test:fm-firstoff"; second_window="test:fm-secondoff" + : > "$state/firstoff.turn-ended" + : > "$state/secondoff.turn-ended" + printf 'window=%s\nkind=ship\nharness=pi\n' "$first_window" > "$state/firstoff.meta" + printf 'window=%s\nkind=ship\nharness=codex\n' "$second_window" > "$state/secondoff.meta" + printf 'second task rendered after the prior poll' > "$capture_file" + first_key=$(printf '%s' "$first_window" | tr ':/.' '___') + second_key=$(printf '%s' "$second_window" | tr ':/.' '___') + printf '%s' "$(hash_text 'first task static pane')" > "$state/.hash-$first_key" + printf '%s' "$(hash_text 'second task previous render')" > "$state/.hash-$second_key" + printf '0\n' > "$state/.count-$first_key" + printf '0\n' > "$state/.count-$second_key" + export FM_FAKE_CREW_STATE_firstoff='state: working · source: run-step · running' + export FM_FAKE_CREW_STATE_secondoff='state: unknown · source: pane · harness state unavailable (unknown codex-unverified)' + PATH="$fakebin:$PATH" FM_FAKE_TMUX_WINDOWS="$(printf 'fm-firstoff\nfm-secondoff')" \ + FM_FAKE_TMUX_CAPTURE="$capture_file" FM_CONFIG_OVERRIDE="$(churn_config "$dir" off)" \ + FM_STATE_OVERRIDE="$state" FM_CREW_STATE_BIN="$fakebin/fm-crew-state.sh" FM_POLL=3 FM_SIGNAL_GRACE=1 \ + FM_CHECK_INTERVAL=999999 FM_HEARTBEAT=999999 "$WATCH" > "$out" & + pid=$! + wait_for_exit "$pid" 100 || fail "watcher absorbed a mixed-evidence batch without the opt-in flag" + grep -F "$state/firstoff.turn-ended" "$out" >/dev/null \ + || fail "watcher did not print the first default-off turn-end" + grep -F "$state/secondoff.turn-ended" "$out" >/dev/null \ + || fail "watcher did not print the second default-off turn-end" + FM_STATE_OVERRIDE="$state" "$DRAIN" > "$drain_out" 2>/dev/null \ + || fail "drain after the default-off mixed-evidence batch failed" + grep "$(printf '\tsignal\t')" "$drain_out" | grep -F "$state/firstoff.turn-ended" >/dev/null \ + || fail "the first default-off turn-end was not queued" + grep "$(printf '\tsignal\t')" "$drain_out" | grep -F "$state/secondoff.turn-ended" >/dev/null \ + || fail "the second default-off turn-end was not queued" + [ ! -e "$state/.churn-since-$first_key" ] && [ ! -e "$state/.churn-since-$second_key" ] \ + || fail "the default-off mixed-evidence batch opened a deferral window" + unset FM_FAKE_CREW_STATE_firstoff FM_FAKE_CREW_STATE_secondoff + pass "per-task evidence composition stays off until the home opts in" +} + +test_status_and_turn_end_batch_never_uses_churn_evidence() { + local dir state fakebin out drain_out capture_file first_window second_window second_key pid + dir=$(make_case status-and-turn-ended-churn); state="$dir/state"; fakebin="$dir/fakebin" + out="$dir/watch.out"; drain_out="$dir/drain.out"; capture_file="$dir/pane.txt" + first_window="test:fm-firststatus"; second_window="test:fm-secondturn" + printf 'working: authoritative task still running\n' > "$state/firststatus.status" + : > "$state/secondturn.turn-ended" + printf 'window=%s\nkind=ship\nharness=pi\n' "$first_window" > "$state/firststatus.meta" + printf 'window=%s\nkind=ship\nharness=codex\n' "$second_window" > "$state/secondturn.meta" + printf 'second task rendered after the prior poll' > "$capture_file" + second_key=$(printf '%s' "$second_window" | tr ':/.' '___') + printf '%s' "$(hash_text 'second task previous render')" > "$state/.hash-$second_key" + printf '0\n' > "$state/.count-$second_key" + export FM_FAKE_CREW_STATE_firststatus='state: working · source: run-step · running' + export FM_FAKE_CREW_STATE_secondturn='state: unknown · source: pane · harness state unavailable (unknown codex-unverified)' + PATH="$fakebin:$PATH" FM_FAKE_TMUX_WINDOWS="$(printf 'fm-firststatus\nfm-secondturn')" \ + FM_FAKE_TMUX_CAPTURE="$capture_file" FM_CONFIG_OVERRIDE="$(churn_config "$dir")" \ + FM_STATE_OVERRIDE="$state" FM_CREW_STATE_BIN="$fakebin/fm-crew-state.sh" FM_POLL=3 FM_SIGNAL_GRACE=1 \ + FM_CHECK_INTERVAL=999999 FM_HEARTBEAT=999999 "$WATCH" > "$out" & + pid=$! + wait_for_exit "$pid" 100 || fail "watcher absorbed a status-and-turn-end batch on churn evidence" + grep -F "$state/firststatus.status" "$out" >/dev/null \ + || fail "watcher did not print the status file from the surfaced mixed batch" + grep -F "$state/secondturn.turn-ended" "$out" >/dev/null \ + || fail "watcher did not print the turn-end from the surfaced mixed batch" + FM_STATE_OVERRIDE="$state" "$DRAIN" > "$drain_out" 2>/dev/null \ + || fail "drain after the surfaced status-and-turn-end batch failed" + grep "$(printf '\tsignal\t')" "$drain_out" | grep -F "$state/firststatus.status" >/dev/null \ + || fail "the status file from the surfaced mixed batch was not queued" + grep "$(printf '\tsignal\t')" "$drain_out" | grep -F "$state/secondturn.turn-ended" >/dev/null \ + || fail "the turn-end from the surfaced mixed batch was not queued" + [ ! -e "$state/.churn-since-$second_key" ] \ + || fail "a status-bearing batch opened a pane-churn deadline" + unset FM_FAKE_CREW_STATE_firststatus FM_FAKE_CREW_STATE_secondturn + pass "a status-bearing batch never falls through to pane-churn evidence" +} + +# The opt-in half. Pane churn infers execution from rendered bytes rather than +# from a verdict the harness vouches for, so a home that has not asked for it must +# see exactly the pre-change triage: the same churning fixture that absorbs above +# surfaces here purely because the flag is absent. +test_turn_ended_churn_absorb_off_by_default() { + local dir state fakebin out drain_out capture_file window key pid + dir=$(make_case turn-ended-churn-default-off); state="$dir/state"; fakebin="$dir/fakebin" + out="$dir/watch.out"; drain_out="$dir/drain.out"; capture_file="$dir/pane.txt" + window="test:fm-codexdefault" + : > "$state/codexdefault.turn-ended" + printf 'window=%s\nkind=ship\nharness=codex\n' "$window" > "$state/codexdefault.meta" + printf 'apply_patch: writing bin/thing.sh' > "$capture_file" + key=$(printf '%s' "$window" | tr ':/.' '___') + printf '%s' "$(hash_text 'reading the brief')" > "$state/.hash-$key" + printf '0\n' > "$state/.count-$key" + export FM_FAKE_CREW_STATE='state: unknown · source: pane · harness state unavailable (unknown codex-unverified)' + PATH="$fakebin:$PATH" FM_FAKE_TMUX_WINDOW="$window" FM_FAKE_TMUX_CAPTURE="$capture_file" \ + FM_CONFIG_OVERRIDE="$(churn_config "$dir" off)" \ + FM_STATE_OVERRIDE="$state" FM_CREW_STATE_BIN="$fakebin/fm-crew-state.sh" FM_POLL=3 FM_SIGNAL_GRACE=1 \ + FM_CHECK_INTERVAL=999999 FM_HEARTBEAT=999999 "$WATCH" > "$out" & + pid=$! + wait_for_exit "$pid" 100 || fail "watcher absorbed a churning turn-end without the opt-in flag" + grep -F "signal: $state/codexdefault.turn-ended" "$out" >/dev/null \ + || fail "watcher did not print the surfaced default-off churning turn-end" + FM_STATE_OVERRIDE="$state" "$DRAIN" > "$drain_out" 2>/dev/null \ + || fail "drain after the default-off churning turn-end failed" + grep "$(printf '\tsignal\t')" "$drain_out" | grep -F "$state/codexdefault.turn-ended" >/dev/null \ + || fail "default-off churning turn-end was not queued" + [ ! -e "$state/.churn-since-$key" ] \ + || fail "the default-off path opened a bounded deferral window" + unset FM_FAKE_CREW_STATE + pass "pane-churn turn-end absorb is off until a home opts in" +} + +# The bound. Churn and pane staleness read the same pane, so a pane that renders +# continuously (a clock, a spinner, a harness that leaves a background renderer +# alive after its agent yields) never reaches the staleness backbone's two +# identical hashes either. Without a bound on the churn absorb a worker that had +# genuinely stopped behind such a renderer would have no path left to surface at +# all, so an exhausted deferral window must surface and restart. +test_turn_ended_churn_absorb_bounded() { + local dir state fakebin out drain_out capture_file window key pid + dir=$(make_case turn-ended-churn-bounded); state="$dir/state"; fakebin="$dir/fakebin" + out="$dir/watch.out"; drain_out="$dir/drain.out"; capture_file="$dir/pane.txt" + window="test:fm-codexclock" + : > "$state/codexclock.turn-ended" + printf 'window=%s\nkind=ship\nharness=codex\n' "$window" > "$state/codexclock.meta" + printf 'a background renderer that never stops' > "$capture_file" + key=$(printf '%s' "$window" | tr ':/.' '___') + printf '%s' "$(hash_text 'the previous frame')" > "$state/.hash-$key" + printf '0\n' > "$state/.count-$key" + # This endpoint has already been riding churn evidence longer than the bound. + printf '%s' "$(( $(date +%s) - 600 ))" > "$state/.churn-since-$key" + export FM_FAKE_CREW_STATE='state: unknown · source: pane · harness state unavailable (unknown codex-unverified)' + PATH="$fakebin:$PATH" FM_FAKE_TMUX_WINDOW="$window" FM_FAKE_TMUX_CAPTURE="$capture_file" \ + FM_CONFIG_OVERRIDE="$(churn_config "$dir")" FM_TURNEND_CHURN_ABSORB_SECS=60 \ + FM_STATE_OVERRIDE="$state" FM_CREW_STATE_BIN="$fakebin/fm-crew-state.sh" FM_POLL=3 FM_SIGNAL_GRACE=1 \ + FM_CHECK_INTERVAL=999999 FM_HEARTBEAT=999999 "$WATCH" > "$out" & + pid=$! + wait_for_exit "$pid" 100 \ + || fail "a perpetually churning pane deferred its turn-end past the absorb bound" + grep -F "signal: $state/codexclock.turn-ended" "$out" >/dev/null \ + || fail "watcher did not print the turn-end surfaced by the exhausted absorb bound" + FM_STATE_OVERRIDE="$state" "$DRAIN" > "$drain_out" 2>/dev/null \ + || fail "drain after the bounded churn turn-end failed" + grep "$(printf '\tsignal\t')" "$drain_out" | grep -F "$state/codexclock.turn-ended" >/dev/null \ + || fail "the turn-end surfaced by the exhausted absorb bound was not queued" + [ ! -e "$state/.churn-since-$key" ] \ + || fail "an exhausted deferral window was not restarted after surfacing" + unset FM_FAKE_CREW_STATE + pass "a perpetually churning pane surfaces once its bounded deferral window is spent" +} + +test_turn_ended_churn_timer_write_failure_surfaced() { + local dir state fakebin out drain_out capture_file window key pid + dir=$(make_case turn-ended-churn-timer-write-failure); state="$dir/state"; fakebin="$dir/fakebin" + out="$dir/watch.out"; drain_out="$dir/drain.out"; capture_file="$dir/pane.txt" + window="test:fm-codextimer" + : > "$state/codextimer.turn-ended" + printf 'window=%s\nkind=ship\nharness=codex\n' "$window" > "$state/codextimer.meta" + printf 'rendered after the previous poll' > "$capture_file" + key=$(printf '%s' "$window" | tr ':/.' '___') + printf '%s' "$(hash_text 'the previous render')" > "$state/.hash-$key" + printf '0\n' > "$state/.count-$key" + mkdir "$state/.churn-since-$key" + export FM_FAKE_CREW_STATE='state: unknown · source: pane · harness state unavailable (unknown codex-unverified)' + PATH="$fakebin:$PATH" FM_FAKE_TMUX_WINDOW="$window" FM_FAKE_TMUX_CAPTURE="$capture_file" \ + FM_CONFIG_OVERRIDE="$(churn_config "$dir")" \ + FM_STATE_OVERRIDE="$state" FM_CREW_STATE_BIN="$fakebin/fm-crew-state.sh" FM_POLL=3 FM_SIGNAL_GRACE=1 \ + FM_CHECK_INTERVAL=999999 FM_HEARTBEAT=999999 "$WATCH" > "$out" 2>/dev/null & + pid=$! + wait_for_exit "$pid" 100 || fail "watcher absorbed a churning turn-end without recording its deadline" + grep -F "signal: $state/codextimer.turn-ended" "$out" >/dev/null \ + || fail "watcher did not print the turn-end whose churn deadline could not be recorded" + FM_STATE_OVERRIDE="$state" "$DRAIN" > "$drain_out" 2>/dev/null \ + || fail "drain after the failed churn deadline write failed" + grep "$(printf '\tsignal\t')" "$drain_out" | grep -F "$state/codextimer.turn-ended" >/dev/null \ + || fail "turn-end with an unrecordable churn deadline was not queued" + unset FM_FAKE_CREW_STATE + pass "an unrecordable pane-churn deadline surfaces the turn-end" +} + +test_turn_ended_invalid_churn_bound_surfaced() { + local dir state fakebin out drain_out capture_file window key pid + dir=$(make_case turn-ended-invalid-churn-bound); state="$dir/state"; fakebin="$dir/fakebin" + out="$dir/watch.out"; drain_out="$dir/drain.out"; capture_file="$dir/pane.txt" + window="test:fm-codexbound" + : > "$state/codexbound.turn-ended" + printf 'window=%s\nkind=ship\nharness=codex\n' "$window" > "$state/codexbound.meta" + printf 'rendered after the previous poll' > "$capture_file" + key=$(printf '%s' "$window" | tr ':/.' '___') + printf '%s' "$(hash_text 'the previous render')" > "$state/.hash-$key" + printf '0\n' > "$state/.count-$key" + export FM_FAKE_CREW_STATE='state: unknown · source: pane · harness state unavailable (unknown codex-unverified)' + PATH="$fakebin:$PATH" FM_FAKE_TMUX_WINDOW="$window" FM_FAKE_TMUX_CAPTURE="$capture_file" \ + FM_CONFIG_OVERRIDE="$(churn_config "$dir")" FM_TURNEND_CHURN_ABSORB_SECS=bogus \ + FM_STATE_OVERRIDE="$state" FM_CREW_STATE_BIN="$fakebin/fm-crew-state.sh" FM_POLL=3 FM_SIGNAL_GRACE=1 \ + FM_CHECK_INTERVAL=999999 FM_HEARTBEAT=999999 "$WATCH" > "$out" 2>/dev/null & + pid=$! + wait_for_exit "$pid" 100 || fail "watcher did not surface a turn-end with an invalid churn bound" + grep -F "signal: $state/codexbound.turn-ended" "$out" >/dev/null \ + || fail "watcher terminated before printing the invalid-bound turn-end" + FM_STATE_OVERRIDE="$state" "$DRAIN" > "$drain_out" 2>/dev/null \ + || fail "drain after the invalid churn bound failed" + grep "$(printf '\tsignal\t')" "$drain_out" | grep -F "$state/codexbound.turn-ended" >/dev/null \ + || fail "turn-end with an invalid churn bound was not queued" + [ ! -e "$state/.churn-since-$key" ] \ + || fail "an invalid churn bound opened a deferral window" + unset FM_FAKE_CREW_STATE + pass "an invalid pane-churn bound surfaces the turn-end" +} + +test_turn_ended_oversized_churn_bound_surfaced() { + local dir state fakebin out drain_out capture_file window key pid + dir=$(make_case turn-ended-oversized-churn-bound); state="$dir/state"; fakebin="$dir/fakebin" + out="$dir/watch.out"; drain_out="$dir/drain.out"; capture_file="$dir/pane.txt" + window="test:fm-codexoversized" + : > "$state/codexoversized.turn-ended" + printf 'window=%s\nkind=ship\nharness=codex\n' "$window" > "$state/codexoversized.meta" + printf 'rendered after the previous poll' > "$capture_file" + key=$(printf '%s' "$window" | tr ':/.' '___') + printf '%s' "$(hash_text 'the previous render')" > "$state/.hash-$key" + printf '0\n' > "$state/.count-$key" + export FM_FAKE_CREW_STATE='state: unknown · source: pane · harness state unavailable (unknown codex-unverified)' + PATH="$fakebin:$PATH" FM_FAKE_TMUX_WINDOW="$window" FM_FAKE_TMUX_CAPTURE="$capture_file" \ + FM_CONFIG_OVERRIDE="$(churn_config "$dir")" FM_TURNEND_CHURN_ABSORB_SECS=999999999999999999999999999999999999 \ + FM_STATE_OVERRIDE="$state" FM_CREW_STATE_BIN="$fakebin/fm-crew-state.sh" FM_POLL=3 FM_SIGNAL_GRACE=1 \ + FM_CHECK_INTERVAL=999999 FM_HEARTBEAT=999999 "$WATCH" > "$out" 2>/dev/null & + pid=$! + wait_for_exit "$pid" 100 || fail "watcher did not surface a turn-end with an oversized churn bound" + grep -F "signal: $state/codexoversized.turn-ended" "$out" >/dev/null \ + || fail "watcher terminated before printing the oversized-bound turn-end" + FM_STATE_OVERRIDE="$state" "$DRAIN" > "$drain_out" 2>/dev/null \ + || fail "drain after the oversized churn bound failed" + grep "$(printf '\tsignal\t')" "$drain_out" | grep -F "$state/codexoversized.turn-ended" >/dev/null \ + || fail "turn-end with an oversized churn bound was not queued" + [ ! -e "$state/.churn-since-$key" ] \ + || fail "an oversized churn bound opened a deferral window" + unset FM_FAKE_CREW_STATE + pass "an oversized pane-churn bound surfaces the turn-end" +} + +test_turn_ended_invalid_churn_deadline_surfaced() { + local variant value dir state fakebin out drain_out capture_file window key marker pid + for variant in empty leading-zero nonnumeric future overflow; do + dir=$(make_case "turn-ended-invalid-churn-deadline-$variant") + state="$dir/state"; fakebin="$dir/fakebin" + out="$dir/watch.out"; drain_out="$dir/drain.out"; capture_file="$dir/pane.txt" + window="test:fm-codexdeadline" + : > "$state/codexdeadline.turn-ended" + printf 'window=%s\nkind=ship\nharness=codex\n' "$window" > "$state/codexdeadline.meta" + printf 'rendered after the previous poll' > "$capture_file" + key=$(printf '%s' "$window" | tr ':/.' '___') + marker="$state/.churn-since-$key" + printf '%s' "$(hash_text 'the previous render')" > "$state/.hash-$key" + printf '0\n' > "$state/.count-$key" + case "$variant" in + empty) value='' ;; + leading-zero) value=09 ;; + nonnumeric) value=bogus ;; + future) value=$(( $(date +%s) + 600 )) ;; + overflow) value=999999999999999999999999999999999999 ;; + esac + printf '%s' "$value" > "$marker" + export FM_FAKE_CREW_STATE='state: unknown · source: pane · harness state unavailable (unknown codex-unverified)' + PATH="$fakebin:$PATH" FM_FAKE_TMUX_WINDOW="$window" FM_FAKE_TMUX_CAPTURE="$capture_file" \ + FM_CONFIG_OVERRIDE="$(churn_config "$dir")" \ + FM_STATE_OVERRIDE="$state" FM_CREW_STATE_BIN="$fakebin/fm-crew-state.sh" FM_POLL=3 FM_SIGNAL_GRACE=1 \ + FM_CHECK_INTERVAL=999999 FM_HEARTBEAT=999999 "$WATCH" > "$out" 2>/dev/null & + pid=$! + wait_for_exit "$pid" 100 || fail "watcher did not surface a turn-end with a $variant churn deadline" + grep -F "signal: $state/codexdeadline.turn-ended" "$out" >/dev/null \ + || fail "watcher terminated before printing the $variant-deadline turn-end" + FM_STATE_OVERRIDE="$state" "$DRAIN" > "$drain_out" 2>/dev/null \ + || fail "drain after the $variant churn deadline failed" + grep "$(printf '\tsignal\t')" "$drain_out" | grep -F "$state/codexdeadline.turn-ended" >/dev/null \ + || fail "turn-end with a $variant churn deadline was not queued" + [ "$(cat "$marker")" = "$value" ] \ + || fail "the $variant churn deadline was rewritten" + done + unset FM_FAKE_CREW_STATE + pass "invalid existing pane-churn deadlines surface without mutation" +} + +test_turn_ended_surfaced_batch_opens_no_partial_deadline() { + local dir state fakebin out drain_out capture_file first_window second_window first_key second_key pid + dir=$(make_case turn-ended-no-partial-churn-deadline); state="$dir/state"; fakebin="$dir/fakebin" + out="$dir/watch.out"; drain_out="$dir/drain.out"; capture_file="$dir/pane.txt" + first_window="test:fm-codexfirst"; second_window="test:fm-codexsecond" + : > "$state/first.turn-ended" + : > "$state/second.turn-ended" + printf 'window=%s\nkind=ship\nharness=codex\n' "$first_window" > "$state/first.meta" + printf 'window=%s\nkind=ship\nharness=codex\n' "$second_window" > "$state/second.meta" + printf 'rendered after the previous poll' > "$capture_file" + first_key=$(printf '%s' "$first_window" | tr ':/.' '___') + second_key=$(printf '%s' "$second_window" | tr ':/.' '___') + printf '%s' "$(hash_text 'first previous render')" > "$state/.hash-$first_key" + printf '%s' "$(hash_text 'second previous render')" > "$state/.hash-$second_key" + printf '0\n' > "$state/.count-$first_key" + printf '0\n' > "$state/.count-$second_key" + printf 'bogus' > "$state/.churn-since-$second_key" + export FM_FAKE_CREW_STATE='state: unknown · source: pane · harness state unavailable (unknown codex-unverified)' + PATH="$fakebin:$PATH" FM_FAKE_TMUX_WINDOWS="$(printf 'fm-codexfirst\nfm-codexsecond')" \ + FM_FAKE_TMUX_CAPTURE="$capture_file" FM_CONFIG_OVERRIDE="$(churn_config "$dir")" \ + FM_STATE_OVERRIDE="$state" FM_CREW_STATE_BIN="$fakebin/fm-crew-state.sh" FM_POLL=3 FM_SIGNAL_GRACE=1 \ + FM_CHECK_INTERVAL=999999 FM_HEARTBEAT=999999 "$WATCH" > "$out" 2>/dev/null & + pid=$! + wait_for_exit "$pid" 100 || fail "watcher absorbed a batch containing an invalid churn deadline" + grep -F "$state/first.turn-ended" "$out" >/dev/null \ + || fail "watcher did not print the first turn-end from the surfaced batch" + grep -F "$state/second.turn-ended" "$out" >/dev/null \ + || fail "watcher did not print the second turn-end from the surfaced batch" + FM_STATE_OVERRIDE="$state" "$DRAIN" > "$drain_out" 2>/dev/null \ + || fail "drain after the surfaced churn batch failed" + grep "$(printf '\tsignal\t')" "$drain_out" | grep -F "$state/first.turn-ended" >/dev/null \ + || fail "the first turn-end from the surfaced batch was not queued" + grep "$(printf '\tsignal\t')" "$drain_out" | grep -F "$state/second.turn-ended" >/dev/null \ + || fail "the second turn-end from the surfaced batch was not queued" + [ ! -e "$state/.churn-since-$first_key" ] \ + || fail "a surfaced batch opened a partial churn deadline" + [ "$(cat "$state/.churn-since-$second_key")" = bogus ] \ + || fail "the invalid churn deadline in a surfaced batch was rewritten" + unset FM_FAKE_CREW_STATE + pass "a surfaced batch opens no partial pane-churn deadline" +} + test_working_note_not_working_surfaced() { local dir state fakebin out drain_out status_file pid dir=$(make_case working-note-stopped); state="$dir/state"; fakebin="$dir/fakebin" @@ -686,7 +1369,7 @@ test_secondmate_status_note_surfaced_despite_busy_agent() { # Busy evidence that would absorb an ordinary crewmate's no-verb note must # not absorb a secondmate's: its status stream is the routed-reply channel. export FM_FAKE_CREW_STATE='state: working · source: run-step · running' - watch_bg "$state" "$fakebin" "$out" + FM_CONFIG_OVERRIDE="$(churn_config "$dir")" watch_bg "$state" "$fakebin" "$out" pid=$! wait_for_exit "$pid" 100 || fail "watcher absorbed a busy secondmate's routed status note" grep -F "signal: $state/mate.status" "$out" >/dev/null \ @@ -2622,6 +3305,25 @@ test_secondmate_status_signal_never_absorbed_classifier test_provably_working_signal_absorbed test_turn_ended_provably_working_absorbed test_turn_ended_not_working_surfaced +test_turn_ended_churning_pane_absorbed +test_turn_ended_churn_resets_prior_stale_classification +test_turn_ended_churn_resets_wedge_state_before_stale_poll +test_turn_ended_still_pane_surfaced +test_turn_ended_malformed_prior_hash_surfaced +test_turn_ended_trailing_newline_prior_hash_surfaced +test_secondmate_turn_ended_churning_pane_surfaced +test_turn_ended_colliding_window_key_surfaced +test_turn_ended_duplicate_endpoint_records_surfaced +test_turn_ended_mixed_positive_evidence_batch_absorbed +test_turn_ended_mixed_positive_evidence_batch_default_off +test_status_and_turn_end_batch_never_uses_churn_evidence +test_turn_ended_churn_absorb_off_by_default +test_turn_ended_churn_absorb_bounded +test_turn_ended_churn_timer_write_failure_surfaced +test_turn_ended_invalid_churn_bound_surfaced +test_turn_ended_oversized_churn_bound_surfaced +test_turn_ended_invalid_churn_deadline_surfaced +test_turn_ended_surfaced_batch_opens_no_partial_deadline test_working_note_not_working_surfaced test_secondmate_status_note_surfaced_despite_busy_agent test_self_announced_close_does_not_rewake_but_next_note_does diff --git a/tests/wake-helpers.sh b/tests/wake-helpers.sh index 8e6281a576..d57058a49b 100644 --- a/tests/wake-helpers.sh +++ b/tests/wake-helpers.sh @@ -62,12 +62,31 @@ make_case() { #!/usr/bin/env bash set -u if [ "${1:-}" = "list-windows" ]; then - if [ -n "${FM_FAKE_TMUX_WINDOW:-}" ]; then + if [ -n "${FM_FAKE_TMUX_WINDOWS:-}" ]; then + printf '%s\n' "$FM_FAKE_TMUX_WINDOWS" + elif [ -n "${FM_FAKE_TMUX_WINDOW:-}" ]; then printf '%s\n' "${FM_FAKE_TMUX_WINDOW#*:}" fi exit 0 fi if [ "${1:-}" = "capture-pane" ]; then + if [ -n "${FM_FAKE_TMUX_CAPTURE_COUNT_FILE:-}" ]; then + _capture_count=$(cat "$FM_FAKE_TMUX_CAPTURE_COUNT_FILE" 2>/dev/null || echo 0) + printf '%s\n' "$((_capture_count + 1))" > "$FM_FAKE_TMUX_CAPTURE_COUNT_FILE" + if [ -n "${FM_FAKE_TMUX_CAPTURE_FAIL_AFTER:-}" ] \ + && [ "$_capture_count" -ge "$FM_FAKE_TMUX_CAPTURE_FAIL_AFTER" ]; then + exit 1 + fi + fi + if [ -n "${FM_FAKE_TMUX_FORBIDDEN_TARGET:-}" ]; then + _prev= + for _arg in "$@"; do + if [ "$_prev" = -t ] && [ "$_arg" = "$FM_FAKE_TMUX_FORBIDDEN_TARGET" ]; then + exit 1 + fi + _prev=$_arg + done + fi if [ -n "${FM_FAKE_TMUX_CAPTURE:-}" ]; then cat "$FM_FAKE_TMUX_CAPTURE" fi