Skip to content
Open
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
94 changes: 67 additions & 27 deletions bin/fm-spawn.sh
Original file line number Diff line number Diff line change
Expand Up @@ -48,7 +48,7 @@
# then tmux.
# Spawn-capable backends are the reference tmux adapter and experimental
# herdr, zellij, orca, and cmux. Orca owns both the task worktree and
# terminal, so ship/scout Orca spawns do not run treehouse get; cmux is a
# terminal, so ship/scout Orca spawns do not lease a treehouse slot; cmux is a
# session provider only, exactly like herdr/zellij, so it does. An
# auto-detected herdr or cmux spawn prints a loud stderr notice;
# auto-detected tmux stays silent; zellij and orca are never auto-detected.
Expand Down Expand Up @@ -665,6 +665,8 @@ SPAWN_META_LOCK_HELD=0
SPAWN_META_PUBLISH_STARTED=0
SPAWN_TASK_SET_LOCK=
SPAWN_TASK_SET_LOCK_HELD=0
TREEHOUSE_LEASE_ABORT_CLEANUP=0
WT_LEASED=
RELAUNCH_REPLACEMENT_PENDING=0
RELAUNCH_REPLACEMENT_BUSY_GEN=
RELAUNCH_REPLACEMENT_HARNESS=
Expand Down Expand Up @@ -692,6 +694,15 @@ parse_orca_worktree_result() {

spawn_abort_cleanup() {
local status=$?
if [ "$TREEHOUSE_LEASE_ABORT_CLEANUP" = 1 ]; then
TREEHOUSE_LEASE_ABORT_CLEANUP=0
if [ -n "$WT_LEASED" ] && [ -n "$PROJ_ABS" ]; then
if ! ( cd "$PROJ_ABS" && treehouse return --force "$WT_LEASED" >/dev/null 2>&1 ); then
echo "warning: could not release the treehouse lease $WT_LEASED for aborted spawn $ID; the pool slot stays reserved until it is returned by hand" >&2
fi
fi
WT_LEASED=
fi
if [ "$RELAUNCH_REPLACEMENT_PENDING" = 1 ] \
&& [ "$SPAWN_META_PUBLISH_STARTED" = 1 ] \
&& [ -n "$SPAWN_META_TMP" ] \
Expand Down Expand Up @@ -1865,7 +1876,7 @@ case "$BACKEND" in
# #134 robustness (tmux): fm_backend_tmux_create_task captures a stable window
# id and pins the window name (automatic-rename/allow-rename off) so a captain's
# non-default tmux config cannot rename the window away from fm-<id> once
# treehouse cd's into the worktree. WT_TARGET carries that stable id for the
# the pane cds into its leased worktree. WT_TARGET carries that stable id for the
# rename-critical worktree-detection steps below; the persisted window= handle
# stays $T (the name form), which is safe now that rename is disabled.
WID=$(fm_backend_tmux_create_task "$SES" "$W" "$PROJ_ABS") || exit 1
Expand Down Expand Up @@ -2097,8 +2108,8 @@ fi
# #134 robustness: only tmux needs a worktree-detection target distinct from $T -
# its rename-safe stable window id, set as WT_TARGET=$WID in the tmux branch above.
# Every other backend addresses its pane/surface by the id already in $T, so default
# WT_TARGET to $T for them (and for any future backend) - the shared treehouse-get +
# worktree-detection steps below must never reference an unbound WT_TARGET under set -u.
# WT_TARGET to $T for them (and for any future backend) - the shared worktree-lease +
# worktree-settle steps below must never reference an unbound WT_TARGET under set -u.
: "${WT_TARGET:=$T}"
spawn_send_text_line() { # <target> <text>
case "$BACKEND" in
Expand Down Expand Up @@ -2212,36 +2223,58 @@ if [ "$RELAUNCH" -eq 1 ]; then
fi
[ "$KIND" = secondmate ] || validate_spawn_worktree "relaunch" "$T"
elif [ "$KIND" != secondmate ] && [ "$BACKEND" != orca ]; then
spawn_send_text_line "$WT_TARGET" 'treehouse get'
# Durable slot ownership: firstmate itself leases the worktree with
# `treehouse get --lease --lease-holder <task-id>` instead of asking the
# pane's shell to run an interactive `treehouse get`. An interactive get
# holds its slot only while processes still run inside it, so a parked
# worker whose occupant processes went quiet could see its slot handed to a
# later spawn while both tasks stayed recorded as live - two workers in one
# worktree (#2754). A lease survives process death until teardown's
# `treehouse return --force` releases it, which is exactly this task's
# ownership lifetime. Run from the project clone: treehouse resolves the
# pool from the working directory. Stderr passes through so an update-check
# banner or allocator error is visible in the spawn output; stdout must be
# only the leased absolute path.
if ! WT=$(cd "$PROJ_ABS" && treehouse get --lease --lease-holder "$ID"); then
echo "error: treehouse get --lease failed for $ID; refusing to launch into an unleased pool slot; inspect window $T" >&2
exit 1
fi
WT=${WT%%$'\n'*}
if [ -z "$WT" ]; then
echo "error: treehouse get --lease did not report a worktree path for $ID; refusing to launch; inspect window $T" >&2
exit 1
fi
# From here until the task record is published, an abort must release the
# lease again - an unreleased lease would silently shrink the pool, since
# nothing else knows the slot was taken (spawn_abort_cleanup owns it).
WT_LEASED=$WT
TREEHOUSE_LEASE_ABORT_CLEANUP=1

# Wait for the treehouse subshell: the pane's cwd moves from the project to the worktree.
# Move the pane into the exact slot this task leased, then wait for the
# shell to settle there before anything records or launches against it.
spawn_send_text_line "$WT_TARGET" "cd $(shell_quote "$WT")"
# Target the stable window id, not the name: if the name is ever lost (e.g. an
# automatic-rename slips through), display-message -t <bad-name> falls back to the
# active client's window, which would misread firstmate's OWN pane path as the
# worktree and tangle a hook into the primary checkout. The window id never lies.
# Compare against PROJ_ABS_REAL (physical), not PROJ_ABS: a symlinked project
# prefix would otherwise make the pane's OS-level cwd read differ from
# PROJ_ABS on the very first poll, before the pane has actually moved.
#
# A single read that already differs from PROJ_ABS_REAL is not proof the pane
# settled there: on some tmux/WSL setups a brand-new window's pane_current_path
# transiently reports an unrelated stale path (seen live as another real git
# checkout entirely) before the shell catches up with treehouse get's cd. That
# stale path still passes the PROJ_ABS_REAL comparison and validate_spawn_worktree
# below (it resolves to a real, distinct worktree top-level too), so accepting it
# on one read alone silently records the wrong worktree= in state/<id>.meta. Require
# two consecutive reads to agree on the same non-project path before accepting it;
# a mismatch just becomes the new candidate rather than resetting the wait, so a
# pane that is already settled by the first real read only costs the one existing
# inter-poll sleep as confirmation, not a whole extra cycle on top.
# A single read equal to the leased path is not proof the pane settled:
# on some tmux/WSL setups a brand-new window's pane_current_path transiently
# reports an unrelated stale path (seen live as another real git checkout
# entirely) before the shell catches up with the cd. Require two consecutive
# reads to agree on the leased path before accepting it; any other read resets
# the candidate, so a pane that is already settled by the first real read only
# costs the one existing inter-poll sleep as confirmation, not a whole extra
# cycle on top. Physical comparison via real_path_or_raw keeps a symlinked
# project prefix from reading as a different path than the pane's OS-level cwd.
wt_leased_real=$(real_path_or_raw "$WT")
candidate=""
for _ in $(seq 1 60); do
p=$(spawn_current_path "$WT_TARGET" || true)
if [ -n "$p" ]; then
p_real=$(real_path_or_raw "$p")
if [ "$p_real" != "$PROJ_ABS_REAL" ]; then
if [ -n "$candidate" ] && [ "$p_real" = "$candidate" ]; then
WT="$p"
if [ "$p_real" = "$wt_leased_real" ]; then
if [ -n "$candidate" ] && [ "$candidate" = "$p_real" ]; then
break
fi
candidate="$p_real"
Expand All @@ -2253,12 +2286,12 @@ elif [ "$KIND" != secondmate ] && [ "$BACKEND" != orca ]; then
fi
sleep 1
done
if [ -z "$WT" ]; then
echo "error: treehouse get did not enter a worktree within 60s; inspect window $T" >&2
if [ -z "$candidate" ] || [ "$candidate" != "$wt_leased_real" ]; then
echo "error: pane did not enter its leased worktree '$WT' within 60s; inspect window $T" >&2
exit 1
fi

validate_spawn_worktree "treehouse get" "$T"
validate_spawn_worktree "treehouse lease" "$T"
fi
if [ "$RELAUNCH" -eq 0 ] && [ "$KIND" != secondmate ]; then
freshen_spawn_worktree_base "$WT" || exit 1
Expand Down Expand Up @@ -2690,6 +2723,14 @@ preserve_relaunch_meta() {
if [ "$RELAUNCH" -eq 1 ]; then
SPAWN_META_PUBLISH_STARTED=1
mv -f "$SPAWN_META_TMP" "$STATE/$ID.meta"
fi
# The published record now owns the slot through its teardown's
# `treehouse return --force`, so an abort here must NOT release the lease -
# that would hand a recorded live task's worktree back to the pool (#2754).
TREEHOUSE_LEASE_ABORT_CLEANUP=0
WT_LEASED=
[ "$BACKEND" = orca ] && ORCA_ABORT_CLEANUP=0
if [ "$RELAUNCH" -eq 1 ]; then
RELAUNCH_REPLACEMENT_PENDING=0
SPAWN_META_PUBLISH_STARTED=0
SPAWN_META_TMP=
Expand All @@ -2703,7 +2744,6 @@ if [ "$SPAWN_TASK_SET_LOCK_HELD" = 1 ]; then
SPAWN_TASK_SET_LOCK_HELD=0
fm_lock_release "$SPAWN_TASK_SET_LOCK"
fi
[ "$BACKEND" = orca ] && ORCA_ABORT_CLEANUP=0

sq_brief=$(shell_quote "$BRIEF")
sq_turnend=$(shell_quote "$TURNEND")
Expand Down
1 change: 1 addition & 0 deletions bin/fm-test-run.sh
Original file line number Diff line number Diff line change
Expand Up @@ -204,6 +204,7 @@ family_for_basename() {
fm-herdr-session-cleanup.test.sh|fm-send-resolve-key.test.sh|fm-send-strict.test.sh|fm-spawn-batch.test.sh|\
fm-spawn-dispatch-profile.test.sh|\
fm-trace-context-spawn.test.sh|fm-spawn-worktree-settle.test.sh|\
fm-spawn-worktree-lease.test.sh|\
fm-teardown-endpoint-safety.test.sh)
printf '%s\n' backend-dispatch
;;
Expand Down
2 changes: 1 addition & 1 deletion docs/cmux-backend.md
Original file line number Diff line number Diff line change
Expand Up @@ -87,7 +87,7 @@ A genuinely fresh surface returns an internal error from `read-screen` until som
Target readiness therefore uses the structural `list-panes` response instead of a content read.
Capture remains bounded and locally trimmed after `read-screen` becomes available.

`current_directory` follows a top-level shell `cd` but not the foreground subshell opened by `treehouse get`.
`current_directory` follows a top-level shell `cd`, which is how spawn moves the pane into its leased worktree; it does not follow nested foreground subshells.
Spawn-time worktree discovery sends begin and end markers around `pwd`, captures the marked block, and joins wrapped path lines.

Literal send and Enter are separate calls.
Expand Down
2 changes: 1 addition & 1 deletion docs/zellij-backend.md
Original file line number Diff line number Diff line change
Expand Up @@ -66,7 +66,7 @@ A pane can still disappear between verification and the operation; downstream su

Every pane operation passes an explicit `--pane-id` because a new session can focus its release-notes plugin pane, whose numeric plugin id is in a separate namespace from terminal pane ids.

`pane_cwd` follows a top-level shell `cd` but not the foreground subshell opened by `treehouse get`.
`pane_cwd` follows a top-level shell `cd`, which is how spawn moves the pane into its leased worktree; it does not follow nested foreground subshells.
Worktree discovery therefore sends begin and end markers around `pwd`, captures the marked block, and joins wrapped path lines.
This active probe is scoped to spawn-time worktree discovery and is not advertised as a general live-cwd API.

Expand Down
21 changes: 19 additions & 2 deletions tests/fm-backend-herdr-presentation-e2e.test.sh
Original file line number Diff line number Diff line change
Expand Up @@ -207,8 +207,25 @@ set -u
done
printf '\n'
} >> "$TREEHOUSE_CALL_LOG"
if [ -d "$POST_CREATE_ABORT_CONTROL" ] && [ "${1:-}" = get ]; then
exit 0
if [ -d "$POST_CREATE_ABORT_CONTROL" ]; then
case "${1:-}" in
get)
# fm-spawn executes the pool acquire itself since the slot-lease fix:
# answer `get --lease` with exactly the bogus cwd the wrapped pane get
# reports for these tasks, so the armed abort still happens at worktree
# validation ("did not yield an isolated worktree") instead of early at
# an empty acquisition.
if [ "${2:-}" = --lease ]; then
printf '%s\n' "$POST_CREATE_ABORT_CONTROL/not-a-worktree"
fi
exit 0
;;
return)
# The post-validation abort releases its lease best-effort; there is no
# real pool slot behind the bogus path to return.
exit 0
;;
esac
fi
exec "$REAL_TREEHOUSE" "$@"
SH
Expand Down
30 changes: 28 additions & 2 deletions tests/fm-backend.test.sh
Original file line number Diff line number Diff line change
Expand Up @@ -785,7 +785,20 @@ esac
exit 0
SH
chmod +x "$fb/tmux"
fm_fake_exit0 "$fb" treehouse
cat > "$fb/treehouse" <<SH
#!/usr/bin/env bash
set -u
case "\${1:-}" in
get)
shift
[ "\${1:-}" = --lease ] || exit 0
printf '%s\\n' "$wt"
exit 0
;;
esac
exit 0
SH
chmod +x "$fb/treehouse"
printf '%s\n' "$fb"
}

Expand Down Expand Up @@ -855,7 +868,20 @@ esac
exit 0
SH
chmod +x "$fb/tmux"
fm_fake_exit0 "$fb" treehouse
cat > "$fb/treehouse" <<SH
#!/usr/bin/env bash
set -u
case "\${1:-}" in
get)
shift
[ "\${1:-}" = --lease ] || exit 0
printf '%s\\n' "$wt"
exit 0
;;
esac
exit 0
SH
chmod +x "$fb/treehouse"
printf '%s\n' "$fb"
}

Expand Down
3 changes: 2 additions & 1 deletion tests/fm-busy-adapter-wiring.test.sh
Original file line number Diff line number Diff line change
Expand Up @@ -35,7 +35,8 @@ esac
exit 0
SH
chmod +x "$fakebin/tmux"
fm_fake_exit0 "$fakebin" treehouse pi opencode claude codex
fm_fake_exit0 "$fakebin" pi opencode claude codex
fm_fake_treehouse_lease "$fakebin"
printf '%s\n' "$fakebin"
}

Expand Down
6 changes: 4 additions & 2 deletions tests/fm-captain-hold-lifecycle.test.sh
Original file line number Diff line number Diff line change
Expand Up @@ -29,7 +29,8 @@ make_home() { # <name>
## Done
EOF
fakebin=$(fm_fakebin "$home")
fm_fake_exit0 "$fakebin" tmux treehouse no-mistakes gh gh-axi
fm_fake_exit0 "$fakebin" tmux no-mistakes gh gh-axi
fm_fake_treehouse_lease "$fakebin"
printf '%s\n' "$home"
}

Expand Down Expand Up @@ -591,7 +592,8 @@ test_secondmate_hold_stays_in_authoritative_home() {
## Done
EOF
fakebin=$(fm_fakebin "$mate")
fm_fake_exit0 "$fakebin" tmux treehouse no-mistakes gh gh-axi
fm_fake_exit0 "$fakebin" tmux no-mistakes gh gh-axi
fm_fake_treehouse_lease "$fakebin"
origin=sample-mate-review
mkdir -p "$mate/data/$origin"
tasks_in "$mate" add "$origin" "Investigate secondmate sample" --kind scout --repo sample --start >/dev/null
Expand Down
2 changes: 1 addition & 1 deletion tests/fm-gate-refuse.test.sh
Original file line number Diff line number Diff line change
Expand Up @@ -152,7 +152,7 @@ esac
exit 0
SH
chmod +x "$fakebin/tmux"
fm_fake_exit0 "$fakebin" treehouse
fm_fake_treehouse_lease "$fakebin"
printf '%s\n' "$fakebin"
}

Expand Down
3 changes: 2 additions & 1 deletion tests/fm-grok-harness.test.sh
Original file line number Diff line number Diff line change
Expand Up @@ -26,7 +26,8 @@ esac
exit 0
SH
chmod +x "$fakebin/tmux"
fm_fake_exit0 "$fakebin" treehouse gh-axi gh
fm_fake_exit0 "$fakebin" gh-axi gh
fm_fake_treehouse_lease "$fakebin"
printf '%s\n' "$fakebin"
}

Expand Down
3 changes: 2 additions & 1 deletion tests/fm-kimi-harness.test.sh
Original file line number Diff line number Diff line change
Expand Up @@ -131,7 +131,8 @@ esac
exit 0
SH
chmod +x "$fakebin/tmux"
fm_fake_exit0 "$fakebin" treehouse gh-axi gh
fm_fake_exit0 "$fakebin" gh-axi gh
fm_fake_treehouse_lease "$fakebin"
fm_fake_exit0 "$fakebin" kimi
ln -s "$JQ_BIN" "$fakebin/jq"
printf '%s\n' "$fakebin"
Expand Down
3 changes: 2 additions & 1 deletion tests/fm-muse-harness.test.sh
Original file line number Diff line number Diff line change
Expand Up @@ -112,7 +112,8 @@ set -u
exec "$FM_FAKE_MUSE_VERSIONED" -c 'result=$($FM_FAKE_HARNESS_PROBE); printf "%s" "$result" > "$FM_FAKE_HARNESS_RESULT"'
SH
chmod +x "$fakebin/muse"
fm_fake_exit0 "$fakebin" treehouse gh-axi gh
fm_fake_exit0 "$fakebin" gh-axi gh
fm_fake_treehouse_lease "$fakebin"
printf '%s\n' "$fakebin"
}

Expand Down
Loading
Loading