From 1f346018bf18c5d41a4f53f1fa82a7ef1d4265f2 Mon Sep 17 00:00:00 2001 From: Minh Trinh Date: Wed, 22 Jul 2026 22:42:06 +0000 Subject: [PATCH 01/15] Grow the live browser to fill the viewer pane; report resolution on resume The fleet rendered every browser at a fixed 1280x800. Now the viewer reports its pane size (debounced ResizeObserver) and the browser grows to fill it, clamped to [1280x800 .. 1920x1080]. The size is frozen while an agent drives (the resize is gated on the same input-enabled flag as human input, so an agent's cached `state` element indices can't shift mid-task), and reported back on resume/handback -- with a re-`state` nudge when it changed. Co-Authored-By: Claude Opus 4.8 (1M context) --- .agents/skills/agentic-browser-fleet/SKILL.md | 2 +- libs/browser/src/browser/assets/index.html | 14 ++++ libs/browser/src/browser/session.py | 65 ++++++++++++++++--- 3 files changed, 72 insertions(+), 9 deletions(-) diff --git a/.agents/skills/agentic-browser-fleet/SKILL.md b/.agents/skills/agentic-browser-fleet/SKILL.md index e11f0d508..afa88da91 100644 --- a/.agents/skills/agentic-browser-fleet/SKILL.md +++ b/.agents/skills/agentic-browser-fleet/SKILL.md @@ -168,7 +168,7 @@ Every browser has exactly one controller; every command's output names the owner - You switch to a different browser for the rest of the task -> release the one you're leaving. - Driving several at once -> keep them until fully done, then release each. - If you forget, an idle lease auto-frees after ~90s; if a later command says you no longer hold it, just acquire it again. -- **The human always wins.** If a human takes control, your next command comes back with status `busy_human`/`lost_control` (exit 2). You lost control: **stop, tell the user the human took the wheel, and end your turn.** Do not retry, poll, or `--reclaim` on your own. You're queued to resume first; you'll be messaged when they hand it back. On resume, **re-run `state ` first** (the page changed), then continue. Resume early only on an explicit "keep going": `acquire --reclaim`, then `state `. +- **The human always wins.** If a human takes control, your next command comes back with status `busy_human`/`lost_control` (exit 2). You lost control: **stop, tell the user the human took the wheel, and end your turn.** Do not retry, poll, or `--reclaim` on your own. You're queued to resume first; you'll be messaged when they hand it back. On resume, **re-run `state ` first** (the page changed -- and the view may have been resized while they held it, reflowing the layout, so treat every element number as stale), then continue. Resume early only on an explicit "keep going": `acquire --reclaim`, then `state `. - **Agents never preempt each other.** A browser another agent holds returns (exit 3): ```text diff --git a/libs/browser/src/browser/assets/index.html b/libs/browser/src/browser/assets/index.html index d348c64fe..57229034a 100644 --- a/libs/browser/src/browser/assets/index.html +++ b/libs/browser/src/browser/assets/index.html @@ -108,6 +108,7 @@ (function () { "use strict"; const app = document.getElementById("app"); + const stage = document.getElementById("stage"); const canvas = document.getElementById("screen"); const ctx = canvas.getContext("2d"); const placeholder = document.getElementById("placeholder"); @@ -325,6 +326,18 @@ function sendCast(obj) { if (castWs && castWs.readyState === 1) castWs.send(JSON.stringify(obj)); } + // Report our pane size so the server grows the browser to fill it (it clamps to + // [1280x800 .. 1920x1080] and ignores this while an agent drives). Debounced -- a + // drag fires a burst and each server resize re-attaches the screencast. + let resizeTimer = null; + function reportSize() { + clearTimeout(resizeTimer); + resizeTimer = setTimeout(() => { + sendCast({ type: "resize", width: Math.round(stage.clientWidth), height: Math.round(stage.clientHeight) }); + }, 150); + } + new ResizeObserver(reportSize).observe(stage); + // --- coordinate scaling: canvas CSS px -> frame device px --- function scaled(e) { const r = canvas.getBoundingClientRect(); @@ -407,6 +420,7 @@ function connect(id) { browserId = id; castWs = new WebSocket(wsUrl("browsers/" + id + "/cast")); + castWs.onopen = () => reportSize(); // send our pane size once the socket is live castWs.onmessage = (ev) => { let m; try { m = JSON.parse(ev.data); } catch (_) { return; } if (m.type === "crashed") showCrashed(); diff --git a/libs/browser/src/browser/session.py b/libs/browser/src/browser/session.py index aa0c92c4b..bb41ead02 100644 --- a/libs/browser/src/browser/session.py +++ b/libs/browser/src/browser/session.py @@ -101,6 +101,12 @@ _SCREENCAST_QUALITY = 55 _SCREENCAST_MAX_WIDTH = 1280 _SCREENCAST_MAX_HEIGHT = 800 +# The live render size floats between the floor above (also a sane desktop baseline -- +# smaller reflows sites to mobile layouts) and this cap: the viewer reports its pane +# size and we grow the browser to fill it, never above the cap (memory -- up to +# _MAX_SESSIONS headless Chromiums render concurrently). +_RENDER_MAX_WIDTH = 1920 +_RENDER_MAX_HEIGHT = 1080 # Every frame: the first frame after a tab switch arrives sooner, so clicking a # tab feels snappier. Slightly more bandwidth than skipping frames. _SCREENCAST_EVERY_NTH_FRAME = 1 @@ -454,6 +460,15 @@ class LiveBrowser(MutableModel): _selector_map: dict[int, Any] = PrivateAttr(default_factory=dict) _lease_touched_at: float = PrivateAttr(default=0.0) _screenshot_seq: int = PrivateAttr(default=0) + # Live render size (viewport / device-metrics / screencast), grown to the human's + # pane by _apply_resize between the floor (_SCREENCAST_MAX_*) and cap (_RENDER_MAX_*). + # Frozen while an agent drives so its `state` element indices don't shift mid-task. + _render_w: int = PrivateAttr(default=_SCREENCAST_MAX_WIDTH) + _render_h: int = PrivateAttr(default=_SCREENCAST_MAX_HEIGHT) + # The render size the current agent started with -- compared on resume to tell it + # if the human resized (reflowing the page) while they held control (see _wake_agent). + _agent_render_w: int = PrivateAttr(default=_SCREENCAST_MAX_WIDTH) + _agent_render_h: int = PrivateAttr(default=_SCREENCAST_MAX_HEIGHT) # Direct-control resume queue: agents whose command was rejected (a human or # another agent held the browser). They ended their turns; when the browser # frees they are handed it FIFO and messaged to resume (see _wake_agent). This @@ -694,18 +709,19 @@ async def _set_active_page(self, page: Page) -> None: self._active_target_id = info["targetInfo"]["targetId"] except _BROWSER_ERRORS: self._active_target_id = None - # Force a uniform render size on EVERY tab. browser-use pins the + # Force the current render size on EVERY tab. browser-use pins the # viewport on the first page, but tabs opened later (by the agent or # by the site) can come up at a different size, so their frames would # stream at a different resolution and the viewer would letterbox them # inconsistently. Overriding the device metrics on each screencast - # target makes every tab stream at exactly the screencast cap. + # target makes every tab stream at exactly _render_w x _render_h, and + # re-applies the human's latest resize (see _apply_resize). try: await cdp.send( "Emulation.setDeviceMetricsOverride", { - "width": _SCREENCAST_MAX_WIDTH, - "height": _SCREENCAST_MAX_HEIGHT, + "width": self._render_w, + "height": self._render_h, "deviceScaleFactor": 1, "mobile": False, }, @@ -718,8 +734,8 @@ async def _set_active_page(self, page: Page) -> None: { "format": _SCREENCAST_FORMAT, "quality": _SCREENCAST_QUALITY, - "maxWidth": _SCREENCAST_MAX_WIDTH, - "maxHeight": _SCREENCAST_MAX_HEIGHT, + "maxWidth": self._render_w, + "maxHeight": self._render_h, "everyNthFrame": _SCREENCAST_EVERY_NTH_FRAME, }, ) @@ -965,7 +981,15 @@ async def handle_cast_message(self, message: dict[str, Any]) -> None: stale human input land after the handoff (the input/control TOCTOU). """ kind = message.get("type") - if kind in ("mouse", "key", "tab", "navigate", "back", "forward", "reload"): + if kind == "resize": + # Same gate as input: _input_enabled is set iff the human (or an idle-free + # browser) owns it, so resizes are dropped while an agent drives -- that's + # the "aspect locked during agent control" freeze, for free. + async with self._control_lock: + if not self._input_enabled.is_set(): + return + await self._apply_resize(message) + elif kind in ("mouse", "key", "tab", "navigate", "back", "forward", "reload"): async with self._control_lock: if not self._input_enabled.is_set(): return @@ -992,6 +1016,18 @@ async def _dispatch_input(self, message: dict[str, Any]) -> None: except _BROWSER_ERRORS as e: logger.debug("cast input ignored ({})", e) + async def _apply_resize(self, message: dict[str, Any]) -> None: + """Human/idle resized their pane: re-render the browser to fill it, clamped to + [floor .. cap]. Reached only while input is enabled (human owns it), so an + agent's cached `state` indices never shift mid-task. Reuses _set_active_page, + which re-applies the new size to the device-metrics override + screencast.""" + w = max(_SCREENCAST_MAX_WIDTH, min(_RENDER_MAX_WIDTH, int(message.get("width", 0)))) + h = max(_SCREENCAST_MAX_HEIGHT, min(_RENDER_MAX_HEIGHT, int(message.get("height", 0)))) + if (w, h) == (self._render_w, self._render_h) or self._active_page is None: + return + self._render_w, self._render_h = w, h + await self._set_active_page(self._active_page) + async def _handle_tab_control(self, message: dict[str, Any]) -> None: if self._context is None: return @@ -1036,6 +1072,9 @@ async def _write_control_locked( else: self._input_enabled.clear() self._lease_touched_at = time.monotonic() # start the sticky-lease idle clock + # Remember the size the agent starts at, so a human resize during a later + # takeover can be reported back to it on resume (see _wake_agent). + self._agent_render_w, self._agent_render_h = self._render_w, self._render_h self._broadcast(self._control_message()) def _waiting_names(self) -> list[str]: @@ -1081,6 +1120,7 @@ def _control_state(self) -> dict[str, Any]: "owner_agent_id": self.owner_agent_id, "owner_name": self.owner_agent_name, "human_pinned": self.human_pinned, + "resolution": [self._render_w, self._render_h], } async def acquire_with_state( @@ -1220,11 +1260,20 @@ async def _wake_agent(self, agent_id: str, agent_name: str | None) -> None: """Message a queued agent that the browser is its again, so it resumes in a fresh turn (it ended its turn when it lost control). If it fails, or the agent never shows, the claim window passes the browser on.""" + if (self._render_w, self._render_h) != (self._agent_render_w, self._agent_render_h): + size_note = ( + f" The view is now {self._render_w}x{self._render_h} " + f"(was {self._agent_render_w}x{self._agent_render_h} when you left) -- the page reflowed, " + f"so your earlier element numbers are void; recompute from the fresh `state` list." + ) + else: + size_note = f" The view is {self._render_w}x{self._render_h} (unchanged)." await self._message_agent( agent_id, agent_name, f"Browser {self.browser_id} was handed back to you (the human finished with it). " - f"Re-run `state {self.browser_id}` to re-read the page, then continue where you left off.", + f"Re-run `state {self.browser_id}` to re-read the page, then continue where you left off." + f"{size_note}", ) async def _abandon_queues_locked(self, reason: str) -> None: From eaf6853dbf5f561f61cd1e7ba5c2a8166becb752 Mon Sep 17 00:00:00 2001 From: Minh Trinh Date: Wed, 22 Jul 2026 22:45:53 +0000 Subject: [PATCH 02/15] Run the browser fleet headful under an Xvfb virtual display Headless Chromium has no reachable clipboard, which blocks native copy/paste (and images entirely). Give it a real X11 session instead: deferred-install now apt-installs xvfb + xclip (own marker), a new [program:xvfb] runs an Xvfb server at :99 sized to the max render (1920x1080), and the browser service exports DISPLAY=:99 so Chromium launches headful into it. _HEADLESS now defaults to headful when a DISPLAY is present and headless otherwise, so tests and bare dev boxes still run without a display. Browser readiness also waits on the xvfb marker when headful. This is prerequisite infra for native clipboard (next commits); anti-bot fidelity is incidental and marginal (CDP, not the window, is the dominant signal). Co-Authored-By: Claude Opus 4.8 (1M context) --- libs/browser/src/browser/session.py | 23 +++++++++++++++------- scripts/deferred_install.sh | 23 ++++++++++++++++++++++ supervisord.conf | 30 +++++++++++++++++++++++++++-- 3 files changed, 67 insertions(+), 9 deletions(-) diff --git a/libs/browser/src/browser/session.py b/libs/browser/src/browser/session.py index bb41ead02..12e07f737 100644 --- a/libs/browser/src/browser/session.py +++ b/libs/browser/src/browser/session.py @@ -111,10 +111,11 @@ # tab feels snappier. Slightly more bandwidth than skipping frames. _SCREENCAST_EVERY_NTH_FRAME = 1 -# Deferred-install marker (see scripts/deferred_install.sh). Chromium installs -# asynchronously on first container boot; launching a browser before it exists -# fails, so callers gate on this. No Xvfb: CDP streaming/input are headless. +# Deferred-install markers (see scripts/deferred_install.sh). Chromium and the +# Xvfb virtual display install asynchronously on first container boot; launching +# a browser before they exist fails, so callers gate on these. _PLAYWRIGHT_MARKER = Path("/var/lib/minds/deferred-install/done.playwright") +_XVFB_MARKER = Path("/var/lib/minds/deferred-install/done.xvfb") # Default model. browser-use's own default LLM is ChatBrowserUse (its hosted # model), so to drive with the user's Anthropic key we pass ChatAnthropic @@ -122,10 +123,14 @@ # API as-is (browser-use accepts an arbitrary model string). _DEFAULT_MODEL = os.environ.get("BROWSER_USE_MODEL", "claude-sonnet-4-6") -# Headless by default. CDP screencast + input are display-independent (they work -# in headless Chromium), so no Xvfb is needed. Set BROWSER_HEADLESS=0 to run -# headful (stronger anti-bot fidelity) if a site blocks headless. -_HEADLESS = os.environ.get("BROWSER_HEADLESS", "1") != "0" +# Headful under a virtual display (Xvfb) by default: the browser service runs an +# Xvfb server and exports DISPLAY, so Chromium renders into a real X11 session. +# That is what makes the OS clipboard usable -- xclip populates/reads the X11 +# clipboard for native copy/paste (images included), which a headless Chromium +# has no reachable clipboard for. Falls back to headless where no DISPLAY exists +# (tests, bare dev boxes) so those still run. Force either mode with +# BROWSER_HEADLESS=1/0. +_HEADLESS = os.environ.get("BROWSER_HEADLESS", "0" if os.environ.get("DISPLAY") else "1") != "0" # Page the browser opens on, and the default for "New tab". _HOME_URL = os.environ.get("BROWSER_HOME_URL", "https://www.google.com") @@ -359,6 +364,10 @@ def deferred_install_ready() -> tuple[bool, str]: return True, "ready" # host/CI testing without the deferred-install marker if not _PLAYWRIGHT_MARKER.exists(): return False, "Chromium is still installing in this workspace; try again in a minute." + # Headful needs the Xvfb display present; wait for its install too (headless + # runs -- tests, bare dev boxes -- don't need it). + if not _HEADLESS and not _XVFB_MARKER.exists(): + return False, "The virtual display is still installing in this workspace; try again in a minute." return True, "ready" diff --git a/scripts/deferred_install.sh b/scripts/deferred_install.sh index c1e1bf8ec..9d5e475ef 100755 --- a/scripts/deferred_install.sh +++ b/scripts/deferred_install.sh @@ -91,10 +91,33 @@ _install_playwright() { fi } +_install_xvfb() { + local marker + marker="$(_marker_for xvfb)" + if [ -f "$marker" ]; then + _log "xvfb: marker present at $marker, skipping" + return 0 + fi + # Headful Chromium needs a display; Xvfb is a headless X server that gives it + # one (the browser runs headful under it -- see session.py's _HEADLESS). xclip + # bridges the resulting X11 clipboard to/from the user for native copy/paste + # (images included). Recover any interrupted dpkg first, same as playwright. + _recover_interrupted_dpkg + _log "xvfb: installing xvfb + xclip" + if apt-get update -y && apt-get install -y --no-install-recommends xvfb xclip; then + touch "$marker" + _log "xvfb: install complete, marker written to $marker" + else + _log "xvfb: install FAILED; marker not written so the next boot retries" + return 1 + fi +} + main() { mkdir -p "$MARKER_DIR" local rc=0 _install_playwright || rc=$? + _install_xvfb || rc=$? if [ "$rc" -eq 0 ]; then _log "all deferred installs complete" else diff --git a/supervisord.conf b/supervisord.conf index 594e1ace0..960b9018b 100644 --- a/supervisord.conf +++ b/supervisord.conf @@ -214,8 +214,33 @@ stderr_logfile_maxbytes=10MB stdout_logfile_backups=3 stderr_logfile_backups=3 -# Live-browser web service: a headless Chromium streamed via CDP screencast with -# CDP input + a browser-use agent, at /service/browser/. No virtual display needed. +# Virtual display for the browser fleet. Chromium runs HEADFUL under this Xvfb +# server (see session.py's _HEADLESS) so it has a real X11 clipboard, which xclip +# reads/writes for native copy/paste (images included). Xvfb and xclip install via +# deferred-install (minutes on first boot), so this retries until the binary exists +# (autorestart + high startretries), same pattern as the browser service below. The +# screen is sized to the browser's max render size (see _RENDER_MAX_* in session.py) +# so a maximized viewport never exceeds the display. Shed-safe: tiny (~40MB) and +# rarely an earlyoom target; if it does die, supervisord respawns it. +[program:xvfb] +command=python3 scripts/oom_tag_service.py xvfb Xvfb :99 -screen 0 1920x1080x24 -nolisten tcp +directory=/mngr/code +autostart=true +autorestart=true +startretries=1000000 +startsecs=2 +stopasgroup=true +killasgroup=true +stdout_logfile=/var/log/supervisor/xvfb-stdout.log +stderr_logfile=/var/log/supervisor/xvfb-stderr.log +stdout_logfile_maxbytes=10MB +stderr_logfile_maxbytes=10MB +stdout_logfile_backups=3 +stderr_logfile_backups=3 + +# Live-browser web service: a headful Chromium (under the Xvfb display above) +# streamed via CDP screencast with CDP input + a browser-use agent, at +# /service/browser/. DISPLAY=:99 points Chromium (and xclip, for clipboard) at Xvfb. # # OOM: shared browsers are the most expendable thing in the workspace, so this is # the one service that opts OUT of the protected band. It raises its own @@ -234,6 +259,7 @@ stderr_logfile_backups=3 [program:browser] command=bash -c "echo 1000 > /proc/self/oom_score_adj || true && ROOT_PATH=/service/browser python3 scripts/forward_port.py --url http://localhost:8081 --name browser && uv run browser-service" directory=/mngr/code +environment=DISPLAY=":99" autostart=true autorestart=true startretries=1000000 From a0c506f7a6befc4d657049f38fbe380693a5c711 Mon Sep 17 00:00:00 2001 From: Minh Trinh Date: Wed, 22 Jul 2026 22:52:51 +0000 Subject: [PATCH 03/15] Native clipboard bridge: copy/paste between the user and the browser The browser now runs headful with a real X11 clipboard, so copy/paste is bridged natively via xclip -- no page-origin Async Clipboard API, no https/activation constraints, and text or images the same way. - session.py: clipboard_paste (xclip -i then a native "paste" editing command) and clipboard_copy/cut (native "copy"/"cut" then xclip -o), gated on human control. Server side is mime-agnostic -- images already work here. - runner.py: GET/POST /browsers//clipboard (blobs over HTTP, since the cast socket has a ~1 MiB cap); MAX_CONTENT_LENGTH bounds a paste at 32 MB. - viewer: Ctrl OR Cmd + C/V/X read/write the user's real navigator.clipboard (Mac + Linux), bridged over the new endpoint instead of forwarded as keystrokes. - IframePanel: allow=clipboard-read/write so the embedded viewer can reach it. This wires text end-to-end; the viewer's image capture + a paste-progress indicator are the next commits (the server already handles image bytes). Co-Authored-By: Claude Opus 4.8 (1M context) --- .../frontend/src/views/IframePanel.ts | 3 + libs/browser/src/browser/assets/index.html | 29 ++++++ libs/browser/src/browser/runner.py | 27 ++++++ libs/browser/src/browser/session.py | 95 +++++++++++++++++++ 4 files changed, 154 insertions(+) diff --git a/apps/system_interface/frontend/src/views/IframePanel.ts b/apps/system_interface/frontend/src/views/IframePanel.ts index 711c353b4..f5abc6158 100644 --- a/apps/system_interface/frontend/src/views/IframePanel.ts +++ b/apps/system_interface/frontend/src/views/IframePanel.ts @@ -18,6 +18,9 @@ export const IframePanel: m.Component = { title, style: "width: 100%; height: 100%; border: none;", sandbox: "allow-scripts allow-same-origin allow-forms allow-popups", + // Let embedded services (e.g. the browser fleet viewer) reach the user's + // clipboard via navigator.clipboard for copy/paste into the remote browser. + allow: "clipboard-read; clipboard-write", }; if (serviceName) { attrs[IFRAME_PANEL_SERVICE_NAME_ATTR] = serviceName; diff --git a/libs/browser/src/browser/assets/index.html b/libs/browser/src/browser/assets/index.html index 57229034a..1eb221a52 100644 --- a/libs/browser/src/browser/assets/index.html +++ b/libs/browser/src/browser/assets/index.html @@ -377,8 +377,37 @@ sendCast({ type: "mouse", event: { type: "mouseWheel", x: p.x, y: p.y, button: "none", deltaX: e.deltaX, deltaY: e.deltaY, modifiers: modifiers(e) } }); }, { passive: false }); + // --- clipboard bridge ------------------------------------------------------- + // The remote browser runs headful with a real OS clipboard; these sync it with + // the USER's clipboard over HTTP (not the cast socket -- images exceed its cap). + // Triggered by the standard shortcuts (Ctrl OR Cmd + C/V/X, so Mac and Linux both + // work), reading/writing navigator.clipboard = the user's actual clipboard. + function clipUrl() { return new URL("browsers/" + browserId + "/clipboard", document.baseURI).toString(); } + + async function clipboard(kind) { + if (!navigator.clipboard) return; // needs a secure context (https/localhost) + try { + if (kind === "v") { // paste: user's clipboard -> remote browser + const text = await navigator.clipboard.readText(); + if (text) await fetch(clipUrl(), { method: "POST", headers: { "Content-Type": "text/plain" }, body: text }); + } else { // c (copy) or x (cut): remote selection -> user's clipboard + const r = await fetch(clipUrl() + (kind === "x" ? "?cut=1" : "")); + const j = await r.json(); + if (j.ok && j.mime && j.mime.indexOf("text/") === 0 && j.text) await navigator.clipboard.writeText(j.text); + } + } catch (_) { /* permission denied / not controlling / nothing selected: ignore */ } + } + function key(type, e) { if (controlOwner === "agent" || !hasFrame) return; + // Clipboard shortcuts are bridged to the user's real OS clipboard (clipboard() + // above), not forwarded as keystrokes -- the remote browser has its own + // clipboard. Handle on keyDown; swallow the matching keyUp too. + if ((e.ctrlKey || e.metaKey) && !e.altKey && ["c", "v", "x"].includes(e.key.toLowerCase())) { + e.preventDefault(); + if (type === "keyDown") clipboard(e.key.toLowerCase()); + return; + } e.preventDefault(); const isChar = type === "keyDown" && e.key.length === 1; sendCast({ type: "key", event: { diff --git a/libs/browser/src/browser/runner.py b/libs/browser/src/browser/runner.py index 44d839ffa..9809a3f01 100644 --- a/libs/browser/src/browser/runner.py +++ b/libs/browser/src/browser/runner.py @@ -109,6 +109,9 @@ application = Flask(__name__, static_folder=None) application.config["SOCK_SERVER_OPTIONS"] = {"ping_interval": 25} +# Clipboard paste bodies carry raw image bytes (the WS proxy's ~1 MiB cap is why +# clipboard rides HTTP, not the cast socket). Bound it so a giant paste can't OOM. +application.config["MAX_CONTENT_LENGTH"] = 32 * 1024 * 1024 sock = Sock(application) # Init gate: cleared at import, set when startup restore finishes (always, even on @@ -734,6 +737,28 @@ def cmd_tab(browser_id: str) -> Response: ) +def cmd_clipboard_copy(browser_id: str) -> Response: + """Human viewer copies (or cuts, ``?cut=1``) the browser's current selection to + their local clipboard. Not agent-gated -- the session gates on human control. + Returns ``{ok, mime, text|data}``; ``mime`` is null when nothing is selected.""" + resolved = _resolve_sync(browser_id) + if isinstance(resolved, Response): + return resolved + cut = request.args.get("cut") == "1" + return jsonify(bridge.run(resolved.clipboard_copy(cut=cut), timeout=_DIRECT_ACTION_TIMEOUT)) + + +def cmd_clipboard_paste(browser_id: str) -> Response: + """Human viewer pastes their local clipboard into the browser. Body is the raw + clipboard bytes; Content-Type is the mime (text/plain or image/*).""" + resolved = _resolve_sync(browser_id) + if isinstance(resolved, Response): + return resolved + data = request.get_data() + mime = (request.content_type or "text/plain").split(";")[0].strip() or "text/plain" + return jsonify(bridge.run(resolved.clipboard_paste(data, mime), timeout=_DIRECT_ACTION_TIMEOUT)) + + # --- screencast WebSocket ---------------------------------------------------- @@ -876,6 +901,8 @@ def _register_routes() -> None: application.add_url_rule("/browsers//keys", view_func=cmd_keys, methods=["POST"]) application.add_url_rule("/browsers//screenshot", view_func=cmd_screenshot, methods=["POST"]) application.add_url_rule("/browsers//tab", view_func=cmd_tab, methods=["POST"]) + application.add_url_rule("/browsers//clipboard", view_func=cmd_clipboard_copy, methods=["GET"], endpoint="clipboard_copy") + application.add_url_rule("/browsers//clipboard", view_func=cmd_clipboard_paste, methods=["POST"], endpoint="clipboard_paste") sock.route("/browsers//cast")(cast_socket) diff --git a/libs/browser/src/browser/session.py b/libs/browser/src/browser/session.py index 12e07f737..d920a83ea 100644 --- a/libs/browser/src/browser/session.py +++ b/libs/browser/src/browser/session.py @@ -1055,6 +1055,101 @@ async def _handle_tab_control(self, message: dict[str, Any]) -> None: if 0 <= index < len(self._context.pages): await self._context.pages[index].close() + # --- clipboard bridge (human viewer <-> the browser's X11 clipboard) ------ + # The browser runs headful under Xvfb (see _HEADLESS), so it has a real X11 + # clipboard. xclip reads/writes it from OUTSIDE the page, so a paste/copy is + # fully native -- no page-origin Async Clipboard API, no https/user-activation + # constraints, and images work the same as text. Gated on _input_enabled: only + # the controlling human, never an agent mid-task, drives the clipboard. + + async def clipboard_paste(self, data: bytes, mime: str) -> dict[str, Any]: + """Write the user's clipboard payload into the browser's X11 clipboard, then + fire a native paste into the focused element. ``mime`` is text/* or image/*.""" + async with self._control_lock: + if not self._input_enabled.is_set(): + return {"ok": False, "status": "not_controlling"} + cdp = self._active_cdp + if cdp is None: + return {"ok": False, "status": "no_page"} + if not await self._xclip_write(data, mime): + return {"ok": False, "status": "clipboard_error"} + # The "paste" editing command reads the X11 clipboard we just populated, + # independent of the user's keymap. + try: + await cdp.send("Input.dispatchKeyEvent", {"type": "keyDown", "key": "v", "code": "KeyV", "windowsVirtualKeyCode": 86, "modifiers": 2, "commands": ["paste"]}) + await cdp.send("Input.dispatchKeyEvent", {"type": "keyUp", "key": "v", "code": "KeyV", "windowsVirtualKeyCode": 86, "modifiers": 2}) + except _BROWSER_ERRORS as e: + logger.debug("clipboard paste dispatch ignored ({})", e) + return {"ok": False, "status": "error"} + return {"ok": True} + + async def clipboard_copy(self, *, cut: bool = False) -> dict[str, Any]: + """Fire a native copy (or cut) of the current selection, then read the X11 + clipboard out for the user. Returns ``{ok, mime, text|data}``; ``mime`` is None + when nothing is selected. Binary payloads come back base64 in ``data``, text in + ``text``.""" + async with self._control_lock: + if not self._input_enabled.is_set(): + return {"ok": False, "status": "not_controlling"} + cdp = self._active_cdp + if cdp is None: + return {"ok": False, "status": "no_page"} + command = "cut" if cut else "copy" + key, code, vk = ("x", "KeyX", 88) if cut else ("c", "KeyC", 67) + try: + await cdp.send("Input.dispatchKeyEvent", {"type": "keyDown", "key": key, "code": code, "windowsVirtualKeyCode": vk, "modifiers": 2, "commands": [command]}) + await cdp.send("Input.dispatchKeyEvent", {"type": "keyUp", "key": key, "code": code, "windowsVirtualKeyCode": vk, "modifiers": 2}) + except _BROWSER_ERRORS as e: + logger.debug("clipboard {} dispatch ignored ({})", command, e) + return {"ok": False, "status": "error"} + data, mime = await self._xclip_read() + if data is None or mime is None: + return {"ok": True, "mime": None} # nothing selected / empty clipboard + if mime.startswith("text/"): + return {"ok": True, "mime": mime, "text": data.decode("utf-8", "replace")} + return {"ok": True, "mime": mime, "data": base64.b64encode(data).decode("ascii")} + + async def _xclip_write(self, data: bytes, mime: str) -> bool: + """Load ``data`` into the X11 CLIPBOARD selection. xclip forks a background owner + that serves the selection until another app claims it, so it persists for the + paste. DISPLAY is inherited from the browser service's env (:99).""" + args = ["xclip", "-selection", "clipboard"] + if not mime.startswith("text/"): + args += ["-t", mime] + args += ["-i"] + try: + proc = await asyncio.create_subprocess_exec( + *args, stdin=asyncio.subprocess.PIPE, stdout=asyncio.subprocess.DEVNULL, stderr=asyncio.subprocess.DEVNULL + ) + await proc.communicate(data) + return proc.returncode == 0 + except OSError as e: + logger.warning("xclip write failed for browser {} ({})", self.browser_id, e) + return False + + async def _xclip_read(self) -> tuple[bytes | None, str | None]: + """Read the X11 CLIPBOARD selection, preferring an image if present. A short + wait lets the just-issued copy command land in the clipboard first.""" + await asyncio.sleep(0.12) + targets = await self._xclip_out("TARGETS") + mime = "image/png" if targets and b"image/png" in targets else "text/plain" + data = await self._xclip_out(mime) + if not data: + return None, None + return data, mime + + async def _xclip_out(self, target: str) -> bytes | None: + try: + proc = await asyncio.create_subprocess_exec( + "xclip", "-selection", "clipboard", "-o", "-t", target, + stdout=asyncio.subprocess.PIPE, stderr=asyncio.subprocess.DEVNULL, + ) + out, _ = await proc.communicate() + return out if proc.returncode == 0 else None + except OSError as e: + logger.warning("xclip read failed for browser {} ({})", self.browser_id, e) + return None + # --- ownership state machine ---------------------------------------------- def _state_tuple(self) -> tuple[ControlOwner, str | None, bool]: From 17ed91b561934a89b8c8ad3c83b69871864db4ed Mon Sep 17 00:00:00 2001 From: Minh Trinh Date: Wed, 22 Jul 2026 22:54:56 +0000 Subject: [PATCH 04/15] Clipboard: image copy/paste in the viewer + a Pasting/Copying toast MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Extends the viewer's clipboard bridge to images and adds the non-blocking progress pill the server side was already ready for. - Paste reads the user's clipboard with navigator.clipboard.read() (image, else text) and POSTs the raw bytes; copy/cut writes an image ClipboardItem (or text) back to the user's clipboard from the server's base64 payload. - A top-center pill (Pasting…/Copying…) appears after a 200ms delay -- so instant text ops don't flicker, but image/long transfers show progress. pointer-events: none, and paste is server-owned once uploaded, so the user can navigate away and it still lands. Co-Authored-By: Claude Opus 4.8 (1M context) --- libs/browser/src/browser/assets/index.html | 47 ++++++++++++++++++++-- 1 file changed, 44 insertions(+), 3 deletions(-) diff --git a/libs/browser/src/browser/assets/index.html b/libs/browser/src/browser/assets/index.html index 1eb221a52..e172509b1 100644 --- a/libs/browser/src/browser/assets/index.html +++ b/libs/browser/src/browser/assets/index.html @@ -71,6 +71,12 @@ /* While init/crashed cover the pane, hide the tab + nav chrome so a half-built browser never peeks through. Toggled by the `chrome-hidden` class on #app. */ #app.chrome-hidden #tabbar, #app.chrome-hidden #navbar, #app.chrome-hidden #returnbar { display: none; } + + /* Non-blocking top-center pill shown while a larger clipboard op is in flight + (image paste/copy, long text). pointer-events:none so it never traps the user + -- the paste is already server-owned, so they can navigate away freely. */ + #cliptoast { position: absolute; top: 12px; left: 50%; transform: translateX(-50%); z-index: 30; background: rgba(14, 99, 156, 0.92); color: #fff; padding: 6px 16px; border-radius: 999px; font-size: 13px; pointer-events: none; opacity: 0; transition: opacity .15s; } + #cliptoast.show { opacity: 1; } @@ -89,6 +95,7 @@
Starting browser…
+
An agent has control.
@@ -109,6 +116,7 @@ "use strict"; const app = document.getElementById("app"); const stage = document.getElementById("stage"); + const cliptoast = document.getElementById("cliptoast"); const canvas = document.getElementById("screen"); const ctx = canvas.getContext("2d"); const placeholder = document.getElementById("placeholder"); @@ -384,18 +392,51 @@ // work), reading/writing navigator.clipboard = the user's actual clipboard. function clipUrl() { return new URL("browsers/" + browserId + "/clipboard", document.baseURI).toString(); } + // Non-blocking progress pill for larger clipboard ops. Delayed so instant text + // ops don't flicker; returns a done() that clears it. pointer-events:none (CSS) + // means it never traps the user -- the paste is server-owned once uploaded, so + // they can navigate away and it still lands. + function withClipToast(label) { + const t = setTimeout(() => { cliptoast.textContent = label; cliptoast.classList.add("show"); }, 200); + return () => { clearTimeout(t); cliptoast.classList.remove("show"); }; + } + + function b64ToBlob(b64, mime) { + const bin = atob(b64), arr = new Uint8Array(bin.length); + for (let i = 0; i < bin.length; i++) arr[i] = bin.charCodeAt(i); + return new Blob([arr], { type: mime }); + } + + // Read the user's clipboard as {mime, body}: prefer an image, fall back to text. + async function readLocalClipboard() { + try { + const items = await navigator.clipboard.read(); + for (const it of items) { + const imgType = it.types.find((t) => t.indexOf("image/") === 0); + if (imgType) return { mime: imgType, body: await it.getType(imgType) }; + } + } catch (_) { /* read() unsupported/denied -> text fallback below */ } + const text = await navigator.clipboard.readText(); + return text ? { mime: "text/plain", body: text } : null; + } + async function clipboard(kind) { if (!navigator.clipboard) return; // needs a secure context (https/localhost) + const done = withClipToast(kind === "v" ? "Pasting…" : "Copying…"); try { if (kind === "v") { // paste: user's clipboard -> remote browser - const text = await navigator.clipboard.readText(); - if (text) await fetch(clipUrl(), { method: "POST", headers: { "Content-Type": "text/plain" }, body: text }); + const c = await readLocalClipboard(); + if (c) await fetch(clipUrl(), { method: "POST", headers: { "Content-Type": c.mime }, body: c.body }); } else { // c (copy) or x (cut): remote selection -> user's clipboard const r = await fetch(clipUrl() + (kind === "x" ? "?cut=1" : "")); const j = await r.json(); - if (j.ok && j.mime && j.mime.indexOf("text/") === 0 && j.text) await navigator.clipboard.writeText(j.text); + if (j.ok && j.mime) { + if (j.mime.indexOf("text/") === 0) { if (j.text) await navigator.clipboard.writeText(j.text); } + else if (j.data) await navigator.clipboard.write([new ClipboardItem({ [j.mime]: b64ToBlob(j.data, j.mime) })]); + } } } catch (_) { /* permission denied / not controlling / nothing selected: ignore */ } + finally { done(); } } function key(type, e) { From 5f754b45d83dfce29f14521eb2017f2b7d343cb6 Mon Sep 17 00:00:00 2001 From: Minh Trinh Date: Wed, 22 Jul 2026 22:56:31 +0000 Subject: [PATCH 05/15] docs: browser README reflects headful-under-Xvfb + native clipboard Co-Authored-By: Claude Opus 4.8 (1M context) --- libs/browser/README.md | 8 +++++--- 1 file changed, 5 insertions(+), 3 deletions(-) diff --git a/libs/browser/README.md b/libs/browser/README.md index ad9153548..33ab1971a 100644 --- a/libs/browser/README.md +++ b/libs/browser/README.md @@ -8,9 +8,11 @@ agent, identified by its `MNGR_AGENT_ID`, or the human). thread-per-connection) that owns every browser. browser_use, Playwright (async), and the per-browser ownership state machine run on one background asyncio event loop, reached from the Flask threads through a single `run_coroutine_threadsafe` - bridge. Each browser is a headless Chromium driven by `browser_use.BrowserSession`, observed - over the same CDP endpoint to stream a live view (`Page.startScreencast` -> - base64 JPEG frames over a WebSocket) and inject human input. Each browser is + bridge. Each browser is a **headful** Chromium (under an Xvfb virtual display, so + it has a real X11 clipboard for native copy/paste -- see `_HEADLESS` in + `session.py`) driven by `browser_use.BrowserSession`, observed over the same CDP + endpoint to stream a live view (`Page.startScreencast` -> base64 JPEG frames over + a WebSocket) and inject human input. Each browser is addressed by a random ~2-word english NAME (e.g. `alex-smith`), generated on demand and never reused; the fleet starts empty and there is no default browser. From a429978feeadd9914a67a49d0bd8920453ac6116 Mon Sep 17 00:00:00 2001 From: Minh Trinh Date: Wed, 22 Jul 2026 23:22:37 +0000 Subject: [PATCH 06/15] Clipboard: drive copy/paste/cut from the real events, not keystrokes The previous cut watched for Ctrl/Cmd+C/V/X keystrokes -- which missed alternative bindings (Ctrl/Shift+Insert, Shift+Delete), right-click menu, and the Edit menu, so it wasn't OS/keymap-agnostic. Switch to document copy/paste/cut EVENT listeners, which fire regardless of how the user triggered them. - paste reads the event's clipboardData directly (text + images) -- and this drops the navigator.clipboard.readText path, so no clipboard-read permission prompt. - copy/cut pull the remote selection and write the user's clipboard via navigator.clipboard (writeText / ClipboardItem). - key() no longer drives the clipboard; it just stops forwarding the clipboard shortcuts to the remote (and lets the browser's default fire the events) so nothing pastes/cuts twice. Contract note: relies on document-level clipboard events firing over the canvas viewer (the Excalidraw/Figma pattern) -- solid in Chromium; needs a cross-browser live check. A hidden focusable capture element is the fallback if a browser balks. Co-Authored-By: Claude Opus 4.8 (1M context) --- libs/browser/src/browser/assets/index.html | 77 +++++++++++++--------- 1 file changed, 45 insertions(+), 32 deletions(-) diff --git a/libs/browser/src/browser/assets/index.html b/libs/browser/src/browser/assets/index.html index e172509b1..b06cd189d 100644 --- a/libs/browser/src/browser/assets/index.html +++ b/libs/browser/src/browser/assets/index.html @@ -388,8 +388,9 @@ // --- clipboard bridge ------------------------------------------------------- // The remote browser runs headful with a real OS clipboard; these sync it with // the USER's clipboard over HTTP (not the cast socket -- images exceed its cap). - // Triggered by the standard shortcuts (Ctrl OR Cmd + C/V/X, so Mac and Linux both - // work), reading/writing navigator.clipboard = the user's actual clipboard. + // Driven by the real copy/paste/cut EVENTS (below), so it's OS- and keymap- + // agnostic: fires on Ctrl+C, Cmd+C, Ctrl+Insert, the right-click menu, or the Edit + // menu alike -- never tied to guessing specific keystrokes. function clipUrl() { return new URL("browsers/" + browserId + "/clipboard", document.baseURI).toString(); } // Non-blocking progress pill for larger clipboard ops. Delayed so instant text @@ -407,46 +408,58 @@ return new Blob([arr], { type: mime }); } - // Read the user's clipboard as {mime, body}: prefer an image, fall back to text. - async function readLocalClipboard() { - try { - const items = await navigator.clipboard.read(); - for (const it of items) { - const imgType = it.types.find((t) => t.indexOf("image/") === 0); - if (imgType) return { mime: imgType, body: await it.getType(imgType) }; - } - } catch (_) { /* read() unsupported/denied -> text fallback below */ } - const text = await navigator.clipboard.readText(); - return text ? { mime: "text/plain", body: text } : null; + function clipActive() { return controlOwner !== "agent" && hasFrame && castWs && castWs.readyState === 1; } + + // PASTE: the event's own clipboardData carries text + images with no permission + // prompt (it's a user-initiated paste gesture). Send the raw bytes to the remote. + async function sendPaste(body, mime) { + const done = withClipToast("Pasting…"); + try { await fetch(clipUrl(), { method: "POST", headers: { "Content-Type": mime }, body }); } + catch (_) { /* not controlling / network: ignore */ } + finally { done(); } } + document.addEventListener("paste", (e) => { + if (!clipActive()) return; + e.preventDefault(); + for (const it of e.clipboardData.items) { + if (it.type.indexOf("image/") === 0) { + const blob = it.getAsFile(); + if (blob) { sendPaste(blob, it.type); return; } + } + } + const text = e.clipboardData.getData("text/plain"); + if (text) sendPaste(new Blob([text], { type: "text/plain" }), "text/plain"); + }); - async function clipboard(kind) { + // COPY/CUT: pull the remote browser's selection and write it to the user's real + // clipboard (text via writeText, image via ClipboardItem). + async function copyFromRemote(cut) { if (!navigator.clipboard) return; // needs a secure context (https/localhost) - const done = withClipToast(kind === "v" ? "Pasting…" : "Copying…"); + const done = withClipToast("Copying…"); try { - if (kind === "v") { // paste: user's clipboard -> remote browser - const c = await readLocalClipboard(); - if (c) await fetch(clipUrl(), { method: "POST", headers: { "Content-Type": c.mime }, body: c.body }); - } else { // c (copy) or x (cut): remote selection -> user's clipboard - const r = await fetch(clipUrl() + (kind === "x" ? "?cut=1" : "")); - const j = await r.json(); - if (j.ok && j.mime) { - if (j.mime.indexOf("text/") === 0) { if (j.text) await navigator.clipboard.writeText(j.text); } - else if (j.data) await navigator.clipboard.write([new ClipboardItem({ [j.mime]: b64ToBlob(j.data, j.mime) })]); - } + const r = await fetch(clipUrl() + (cut ? "?cut=1" : "")); + const j = await r.json(); + if (j.ok && j.mime) { + if (j.mime.indexOf("text/") === 0) { if (j.text) await navigator.clipboard.writeText(j.text); } + else if (j.data) await navigator.clipboard.write([new ClipboardItem({ [j.mime]: b64ToBlob(j.data, j.mime) })]); } - } catch (_) { /* permission denied / not controlling / nothing selected: ignore */ } + } catch (_) { /* denied / nothing selected: ignore */ } finally { done(); } } + document.addEventListener("copy", (e) => { if (clipActive()) { e.preventDefault(); copyFromRemote(false); } }); + document.addEventListener("cut", (e) => { if (clipActive()) { e.preventDefault(); copyFromRemote(true); } }); function key(type, e) { if (controlOwner === "agent" || !hasFrame) return; - // Clipboard shortcuts are bridged to the user's real OS clipboard (clipboard() - // above), not forwarded as keystrokes -- the remote browser has its own - // clipboard. Handle on keyDown; swallow the matching keyUp too. - if ((e.ctrlKey || e.metaKey) && !e.altKey && ["c", "v", "x"].includes(e.key.toLowerCase())) { - e.preventDefault(); - if (type === "keyDown") clipboard(e.key.toLowerCase()); + // Clipboard shortcuts are handled by the copy/paste/cut EVENT listeners above. + // Let the browser's DEFAULT fire those events -- so DON'T preventDefault and + // DON'T forward, or (a) the paste/copy event never fires and (b) the remote + // would paste/cut a second time. Covers Ctrl/Cmd+C/V/X plus the legacy + // Ctrl/Shift+Insert and Shift+Delete bindings; every other key passes through. + const kl = e.key.toLowerCase(); + if (((e.ctrlKey || e.metaKey) && !e.altKey && ["c", "v", "x"].includes(kl)) + || (e.key === "Insert" && (e.ctrlKey || e.shiftKey)) + || (e.key === "Delete" && e.shiftKey)) { return; } e.preventDefault(); From efbde0d4d8b511907449bdd739331a32f442cf29 Mon Sep 17 00:00:00 2001 From: Minh Trinh Date: Thu, 23 Jul 2026 08:04:46 +0000 Subject: [PATCH 07/15] Resize: lower the clamp floor to 640x480 + log the resize path The 1280x800 floor was larger than typical panels, so sub-1280 panes pinned to it and never changed. Lower it to 640x480 so real panels track their size, and log every resize request (raw, clamped, applied, headless flag) + when the input gate drops it -- to pin down whether the stuck-at-1280x800 report is the floor, the human-control gate, or device-metrics not driving the frame under headful. Co-Authored-By: Claude Opus 4.8 (1M context) --- libs/browser/src/browser/session.py | 14 ++++++++++++-- 1 file changed, 12 insertions(+), 2 deletions(-) diff --git a/libs/browser/src/browser/session.py b/libs/browser/src/browser/session.py index d920a83ea..1d0a7fb22 100644 --- a/libs/browser/src/browser/session.py +++ b/libs/browser/src/browser/session.py @@ -107,6 +107,10 @@ # _MAX_SESSIONS headless Chromiums render concurrently). _RENDER_MAX_WIDTH = 1920 _RENDER_MAX_HEIGHT = 1080 +# Floor for the clamp -- small enough that a typical (sub-1280) panel actually +# tracks its size instead of pinning to a too-big minimum, but not degenerate. +_RENDER_MIN_WIDTH = 640 +_RENDER_MIN_HEIGHT = 480 # Every frame: the first frame after a tab switch arrives sooner, so clicking a # tab feels snappier. Slightly more bandwidth than skipping frames. _SCREENCAST_EVERY_NTH_FRAME = 1 @@ -996,6 +1000,7 @@ async def handle_cast_message(self, message: dict[str, Any]) -> None: # the "aspect locked during agent control" freeze, for free. async with self._control_lock: if not self._input_enabled.is_set(): + logger.info("browser {} resize ignored: input not enabled (an agent controls it)", self.browser_id) return await self._apply_resize(message) elif kind in ("mouse", "key", "tab", "navigate", "back", "forward", "reload"): @@ -1030,8 +1035,13 @@ async def _apply_resize(self, message: dict[str, Any]) -> None: [floor .. cap]. Reached only while input is enabled (human owns it), so an agent's cached `state` indices never shift mid-task. Reuses _set_active_page, which re-applies the new size to the device-metrics override + screencast.""" - w = max(_SCREENCAST_MAX_WIDTH, min(_RENDER_MAX_WIDTH, int(message.get("width", 0)))) - h = max(_SCREENCAST_MAX_HEIGHT, min(_RENDER_MAX_HEIGHT, int(message.get("height", 0)))) + raw_w, raw_h = int(message.get("width", 0)), int(message.get("height", 0)) + w = max(_RENDER_MIN_WIDTH, min(_RENDER_MAX_WIDTH, raw_w)) + h = max(_RENDER_MIN_HEIGHT, min(_RENDER_MAX_HEIGHT, raw_h)) + logger.info( + "browser {} resize request {}x{} -> clamped {}x{} (was {}x{}, headless={})", + self.browser_id, raw_w, raw_h, w, h, self._render_w, self._render_h, _HEADLESS, + ) if (w, h) == (self._render_w, self._render_h) or self._active_page is None: return self._render_w, self._render_h = w, h From 19a8d635afe9ef048607ea73d1f6d4fcdad84111 Mon Sep 17 00:00:00 2001 From: Minh Trinh Date: Thu, 23 Jul 2026 18:25:46 +0000 Subject: [PATCH 08/15] Reject session-less service:browser opens; soften the pane-pull fallback Two fixes so a browser pane never orphans and its absence never reads as an error: - server.py: the layout open/split broadcast handler now rejects a bare service:browser (no ?session=) with a 400 that guides the caller to a real browser name -- stopping an agent from spawning the dead 'Open a browser from the + menu' placeholder pane. The fleet's own pane-pull always carries a session, so it's unaffected (new is_sessionless_browser_ref helper + tests). - fleet.py: when _pull_in_pane can't surface the pane (background/sub-agent with no chat in view), report it as a neutral, optional nudge ('browser X is ready; open it from the + menu to watch') instead of 'I couldn't open it' -- the browser is up and fully drivable regardless; the pane is just a convenience. Co-Authored-By: Claude Opus 4.8 (1M context) --- .../imbue/system_interface/layout_ops.py | 22 +++++++++++++++ .../imbue/system_interface/layout_ops_test.py | 15 +++++++++++ .../imbue/system_interface/server.py | 17 ++++++++++++ .../imbue/system_interface/server_test.py | 27 +++++++++++++++++++ libs/browser/src/browser/fleet.py | 9 ++++--- 5 files changed, 87 insertions(+), 3 deletions(-) diff --git a/apps/system_interface/imbue/system_interface/layout_ops.py b/apps/system_interface/imbue/system_interface/layout_ops.py index 72e274412..8e20536e9 100644 --- a/apps/system_interface/imbue/system_interface/layout_ops.py +++ b/apps/system_interface/imbue/system_interface/layout_ops.py @@ -328,6 +328,28 @@ def _service_session_suffix(url: Any) -> str: return f"?{_BROWSER_SESSION_QUERY_KEY}={session_values[0]}" +def is_sessionless_browser_ref(ref: Any) -> bool: + """True if ``ref`` addresses the browser fleet viewer with no ``?session=``. + + A browser pane must name a specific fleet browser + (``service:browser?session=``); the bare ``service:browser`` opens a + session-less viewer bound to nothing (the dead "Open a browser from the + + menu" placeholder). The open/split broadcast handler rejects it so agents + can't erroneously spawn an orphan pane -- the fleet's own pane-pull always + carries a session, so it is unaffected. + """ + if not isinstance(ref, str): + return False + prefix = "service:" + if not ref.startswith(prefix): + return False + name, _, query = ref[len(prefix) :].partition("?") + if name != "browser": + return False + session_values = urllib.parse.parse_qs(query).get(_BROWSER_SESSION_QUERY_KEY, []) + return not any(value for value in session_values) + + def _resolve_ref( panel_id: str, params: dict[str, Any] | None, diff --git a/apps/system_interface/imbue/system_interface/layout_ops_test.py b/apps/system_interface/imbue/system_interface/layout_ops_test.py index 7f8e5d030..1687c6743 100644 --- a/apps/system_interface/imbue/system_interface/layout_ops_test.py +++ b/apps/system_interface/imbue/system_interface/layout_ops_test.py @@ -14,11 +14,26 @@ from imbue.system_interface.layout_ops import is_destroyable_terminal_session from imbue.system_interface.layout_ops import is_known_op from imbue.system_interface.layout_ops import is_mutating_op +from imbue.system_interface.layout_ops import is_sessionless_browser_ref from imbue.system_interface.layout_ops import layout_inspect from imbue.system_interface.layout_ops import layout_list from imbue.system_interface.layout_ops import parse_tmux_sessions_output +def test_is_sessionless_browser_ref() -> None: + # Bare browser ref (or an empty session) is the orphan-pane case -> rejected. + assert is_sessionless_browser_ref("service:browser") is True + assert is_sessionless_browser_ref("service:browser?session=") is True + assert is_sessionless_browser_ref("service:browser?foo=bar") is True + # A real session name is fine. + assert is_sessionless_browser_ref("service:browser?session=alex-smith") is False + # Not a browser ref (or not a service ref, or non-string) -> not our concern. + assert is_sessionless_browser_ref("service:web") is False + assert is_sessionless_browser_ref("service:browserfoo") is False + assert is_sessionless_browser_ref("chat:alex-smith") is False + assert is_sessionless_browser_ref(None) is False + + def test_known_ops_cover_the_full_surface() -> None: for op in ( "list", diff --git a/apps/system_interface/imbue/system_interface/server.py b/apps/system_interface/imbue/system_interface/server.py index a4e0a56b8..fe47b96c6 100644 --- a/apps/system_interface/imbue/system_interface/server.py +++ b/apps/system_interface/imbue/system_interface/server.py @@ -49,6 +49,7 @@ from imbue.system_interface.layout_ops import is_destroyable_terminal_session from imbue.system_interface.layout_ops import is_known_op from imbue.system_interface.layout_ops import is_mutating_op +from imbue.system_interface.layout_ops import is_sessionless_browser_ref from imbue.system_interface.layout_ops import layout_inspect from imbue.system_interface.layout_ops import layout_list from imbue.system_interface.layout_ops import parse_tmux_sessions_output @@ -1404,6 +1405,22 @@ def _layout_broadcast_endpoint() -> Response: # in the HTTP response. Every other ref kind either dedups against # the existing panel set or is discoverable via a subsequent # ``inspect``. + # A browser pane must name a specific fleet browser. Reject a session-less + # ``service:browser`` open/split before it broadcasts, so an agent can't spawn + # the orphan "Open a browser from the + menu" placeholder pane. Guides the caller + # to the right form rather than erroring opaquely. The fleet's own pane-pull + # always carries ``?session=``, so it is unaffected. + if op in {"open", "split"} and is_sessionless_browser_ref(args_raw.get("ref")): + error = ErrorResponse( + detail=( + "A browser pane needs a specific browser name: use " + "'service:browser?session=', or the agentic-browser-fleet 'new'/'task' " + "commands, which open the pane for you. The bare 'service:browser' opens a " + "viewer bound to no browser." + ) + ) + return _json_response(error.model_dump(), status_code=400) + allocated_ref: str | None = None if op in {"open", "split"} and args_raw.get("ref") == "service:terminal": panel_id, allocated_ref = allocate_terminal_panel_id() diff --git a/apps/system_interface/imbue/system_interface/server_test.py b/apps/system_interface/imbue/system_interface/server_test.py index b1f3e97bd..c4b650051 100644 --- a/apps/system_interface/imbue/system_interface/server_test.py +++ b/apps/system_interface/imbue/system_interface/server_test.py @@ -935,6 +935,33 @@ def test_layout_broadcast_mutating_op_without_matching_client_is_412(app: Flask) assert "No connected client has layout" in response.get_json()["detail"] +def test_layout_broadcast_sessionless_browser_is_rejected(app: Flask) -> None: + """A bare ``service:browser`` open (no ``?session=``) is a 400 -- it would spawn + the orphan session-less viewer pane. A session-qualified ref goes through.""" + matching_queue = _register_fake_client(app, "client-1", "desktop") + client = app.test_client() + # Bare browser ref -> rejected with a guiding message (fires before the layout checks). + bare = client.post( + "/api/layout/broadcast", + json={"op": "open", "args": {"ref": "service:browser", "layout": "desktop"}, "agent_id": "agent-42"}, + ) + assert bare.status_code == 400 + assert "needs a specific browser name" in bare.get_json()["detail"] + assert matching_queue.empty() # nothing broadcast + # A session-qualified browser ref is allowed and reaches the client. + ok = client.post( + "/api/layout/broadcast", + json={ + "op": "open", + "args": {"ref": "service:browser?session=alex-smith", "layout": "desktop"}, + "agent_id": "agent-42", + }, + ) + assert ok.status_code == 200 + msg = _next_broadcast_message(matching_queue) + assert msg["args"]["ref"] == "service:browser?session=alex-smith" + + def test_layout_broadcast_mutating_op_unknown_layout_is_404(app: Flask) -> None: client = app.test_client() response = client.post( diff --git a/libs/browser/src/browser/fleet.py b/libs/browser/src/browser/fleet.py index 0615b2536..9d2557f9a 100644 --- a/libs/browser/src/browser/fleet.py +++ b/libs/browser/src/browser/fleet.py @@ -199,9 +199,12 @@ def _pull_in_pane(browser_name: str) -> None: return if _layout("split", ref, "--relative-to", "self", "--direction", "right", "--new-group", quiet=True): return - _err(f"browser {browser_name} is running, but I couldn't open its live pane here. " - 'If you are the workspace\'s main agent, open it from the "+" menu; a background ' - "or sub-agent can't show panes (have the main agent drive the browser).") + # Not an error -- the browser is up and fully drivable from the CLI; the pane is + # only a live-view convenience. Optimistic: the split lands when a client is + # watching this agent's chat, and otherwise (background/sub-agent, no chat in + # view) we just offer the manual route without implying anything broke. + _out(f"browser {browser_name} is ready. To watch it live, open it from the " + '"+" menu (New browser -> ' + f"{browser_name}) in the side panel.") # --- commands ----------------------------------------------------------------- From 116385eeffa3389a65165e43d0a72fda7050b31c Mon Sep 17 00:00:00 2001 From: Minh Trinh Date: Thu, 23 Jul 2026 18:53:18 +0000 Subject: [PATCH 09/15] Add changelog entries + drop two trailing comments (ratchet) - Changelog entries for all four touched projects (agents, browser, dev, system_interface), documenting the fill-the-pane sizing, headful+Xvfb native clipboard, the session-less service:browser gate, and the neutral pane-pull fallback. - Move two trailing comments to their own lines (server_test.py, session.py) so the trailing-comments ratchet stops firing. Co-Authored-By: Claude Opus 4.8 (1M context) --- .agents/changelog/browser-fleet-improvements.md | 1 + .../system_interface/changelog/browser-fleet-improvements.md | 3 +++ apps/system_interface/imbue/system_interface/server_test.py | 3 ++- dev/changelog/browser-fleet-improvements.md | 1 + libs/browser/changelog/browser-fleet-improvements.md | 5 +++++ libs/browser/src/browser/session.py | 3 ++- 6 files changed, 14 insertions(+), 2 deletions(-) create mode 100644 .agents/changelog/browser-fleet-improvements.md create mode 100644 apps/system_interface/changelog/browser-fleet-improvements.md create mode 100644 dev/changelog/browser-fleet-improvements.md create mode 100644 libs/browser/changelog/browser-fleet-improvements.md diff --git a/.agents/changelog/browser-fleet-improvements.md b/.agents/changelog/browser-fleet-improvements.md new file mode 100644 index 000000000..59b07a775 --- /dev/null +++ b/.agents/changelog/browser-fleet-improvements.md @@ -0,0 +1 @@ +The `agentic-browser-fleet` skill now notes that, on resume after a human held the browser, the live view may have been resized and the page reflowed -- so every cached element number should be treated as stale and `state` re-run before acting. diff --git a/apps/system_interface/changelog/browser-fleet-improvements.md b/apps/system_interface/changelog/browser-fleet-improvements.md new file mode 100644 index 000000000..d1689052d --- /dev/null +++ b/apps/system_interface/changelog/browser-fleet-improvements.md @@ -0,0 +1,3 @@ +Opening a bare `service:browser` (with no `?session=`) is now rejected with a clear message pointing at the right form, instead of spawning an orphan browser pane bound to no browser -- the dead "Open a browser from the + menu" placeholder. Browser panes must name a specific fleet browser; the fleet's own `new`/`task` commands already do, so only a stray manual `layout.py open service:browser` was ever affected. + +The embedded browser-viewer iframe is now granted `clipboard-read`/`clipboard-write` permission so it can sync copy/paste with your local clipboard. diff --git a/apps/system_interface/imbue/system_interface/server_test.py b/apps/system_interface/imbue/system_interface/server_test.py index c4b650051..7d3b1e148 100644 --- a/apps/system_interface/imbue/system_interface/server_test.py +++ b/apps/system_interface/imbue/system_interface/server_test.py @@ -947,7 +947,8 @@ def test_layout_broadcast_sessionless_browser_is_rejected(app: Flask) -> None: ) assert bare.status_code == 400 assert "needs a specific browser name" in bare.get_json()["detail"] - assert matching_queue.empty() # nothing broadcast + # nothing should have been broadcast to the client + assert matching_queue.empty() # A session-qualified browser ref is allowed and reaches the client. ok = client.post( "/api/layout/broadcast", diff --git a/dev/changelog/browser-fleet-improvements.md b/dev/changelog/browser-fleet-improvements.md new file mode 100644 index 000000000..b3a4d9ee7 --- /dev/null +++ b/dev/changelog/browser-fleet-improvements.md @@ -0,0 +1 @@ +First-boot deferred install now also installs `xvfb` and `xclip`, and a new `xvfb` supervised service provides the virtual display the browser fleet now runs headful under (the browser service is pointed at it via `DISPLAY=:99`). This is the infrastructure that enables the live browser's native clipboard copy/paste. diff --git a/libs/browser/changelog/browser-fleet-improvements.md b/libs/browser/changelog/browser-fleet-improvements.md new file mode 100644 index 000000000..681f9b785 --- /dev/null +++ b/libs/browser/changelog/browser-fleet-improvements.md @@ -0,0 +1,5 @@ +The live browser now fills its viewer pane instead of a fixed 1280x800 box: the viewer reports its size and the browser grows to fill it (clamped 640x480 .. 1920x1080), frozen while an agent is driving so its element numbers don't shift mid-task, and the resolution is reported back on resume so the agent knows to re-read the page if it changed. + +The fleet now runs headful under an Xvfb virtual display, which gives it a real OS clipboard: copy and paste -- text and images -- between your machine and the remote browser now work natively, driven by the real copy/paste/cut events (so any keybinding, right-click, or the Edit menu works), with a non-blocking "Pasting.../Copying..." indicator for larger transfers. + +When a background or sub-agent can't surface the browser's live pane, it now reports that the browser is ready and offers to open it from the "+" menu, rather than framing it as a failure -- the browser is fully drivable from the CLI either way; the pane is only a live-view convenience. diff --git a/libs/browser/src/browser/session.py b/libs/browser/src/browser/session.py index 1d0a7fb22..03429689e 100644 --- a/libs/browser/src/browser/session.py +++ b/libs/browser/src/browser/session.py @@ -1114,7 +1114,8 @@ async def clipboard_copy(self, *, cut: bool = False) -> dict[str, Any]: return {"ok": False, "status": "error"} data, mime = await self._xclip_read() if data is None or mime is None: - return {"ok": True, "mime": None} # nothing selected / empty clipboard + # nothing selected / empty clipboard + return {"ok": True, "mime": None} if mime.startswith("text/"): return {"ok": True, "mime": mime, "text": data.decode("utf-8", "replace")} return {"ok": True, "mime": mime, "data": base64.b64encode(data).decode("ascii")} From 5e1839452b7071c11448293ee7dcdfa4bc453377 Mon Sep 17 00:00:00 2001 From: Minh Trinh Date: Fri, 24 Jul 2026 23:51:11 +0000 Subject: [PATCH 10/15] Fix the browser pane auto-open: resolve the active layout via context _pull_in_pane never passed the --layout that split has required since named layouts landed, so the split was always rejected and the pane never appeared -- every browser command silently fell through to the '+' menu message. Now it runs 'layout.py context' (read-only) to resolve the layout the requester's client is viewing and passes --layout to the split. Fully optimistic: - unreachable layout server (isolated launch-task sub-agent) -> skip silently - reachable but the split can't land (human not viewing that layout) -> the neutral '+' menu nudge Same-container agents (primary, + New agent, native Task subagents that share the parent's identity) surface the pane reliably when their chat is on-screen. Co-Authored-By: Claude Opus 4.8 (1M context) --- .../changelog/browser-fleet-improvements.md | 2 +- libs/browser/fleet_test.py | 41 ++++++- libs/browser/src/browser/fleet.py | 100 +++++++++++++----- 3 files changed, 115 insertions(+), 28 deletions(-) diff --git a/libs/browser/changelog/browser-fleet-improvements.md b/libs/browser/changelog/browser-fleet-improvements.md index 681f9b785..ec771ee84 100644 --- a/libs/browser/changelog/browser-fleet-improvements.md +++ b/libs/browser/changelog/browser-fleet-improvements.md @@ -2,4 +2,4 @@ The live browser now fills its viewer pane instead of a fixed 1280x800 box: the The fleet now runs headful under an Xvfb virtual display, which gives it a real OS clipboard: copy and paste -- text and images -- between your machine and the remote browser now work natively, driven by the real copy/paste/cut events (so any keybinding, right-click, or the Edit menu works), with a non-blocking "Pasting.../Copying..." indicator for larger transfers. -When a background or sub-agent can't surface the browser's live pane, it now reports that the browser is ready and offers to open it from the "+" menu, rather than framing it as a failure -- the browser is fully drivable from the CLI either way; the pane is only a live-view convenience. +The browser's live pane now actually auto-opens beside your chat when an agent works a browser (`new` / `task` / any direct command): the fleet resolves which layout you're currently viewing (via `layout.py context`) and surfaces the pane there -- previously it never named a layout, so the split was silently rejected and the pane never appeared. If the layout server is unreachable (an isolated sub-agent in its own container) it skips silently; if your screen is reachable but the pane can't land (you're not viewing that layout) it offers the manual "+"-menu route instead of implying anything broke. Either way the browser is fully drivable from the CLI; the pane is only a live-view convenience. diff --git a/libs/browser/fleet_test.py b/libs/browser/fleet_test.py index 14dbc7a5f..4ceb142f4 100644 --- a/libs/browser/fleet_test.py +++ b/libs/browser/fleet_test.py @@ -133,24 +133,59 @@ def test_parser_accepts_direct_verbs() -> None: def test_pull_in_pane_opens_each_browser_in_its_own_pane(monkeypatch: pytest.MonkeyPatch) -> None: # A user-started agent surfaces each browser as its OWN pane (--new-group), beside # its own chat (--relative-to self), not tabbed into an existing browser pane. The - # session ref keys on the NAME. + # split carries the resolved --layout, and the session ref keys on the NAME. calls: list[tuple] = [] + monkeypatch.setattr(fleet, "_resolve_active_layout", lambda: (True, "desktop")) monkeypatch.setattr(fleet, "_layout", lambda *a, **k: calls.append(a) or True) monkeypatch.delenv("BROWSER_FLEET_ANCHOR", raising=False) fleet._pull_in_pane("alex-smith") assert calls and "--new-group" in calls[0] and "right" in calls[0] and "self" in calls[0] + assert "--layout" in calls[0] and "desktop" in calls[0] assert any("session=alex-smith" in arg for arg in calls[0]) def test_pull_in_pane_warns_cleanly_when_it_cant_show_a_pane(monkeypatch: pytest.MonkeyPatch) -> None: - # A background/sub-agent (no chat in this UI) can't land the split: we attempt it, - # then warn in one clean line -- never crash, never leak the raw 5s "not registered" + # Reachable layout server, but the split never lands (human isn't viewing that layout): + # we attempt it, then warn in one clean line -- never crash, never leak layout.py's raw # error (the browser is still running). + monkeypatch.setattr(fleet, "_resolve_active_layout", lambda: (True, "desktop")) monkeypatch.setattr(fleet, "_layout", lambda *a, **k: False) # layout never lands monkeypatch.delenv("BROWSER_FLEET_ANCHOR", raising=False) fleet._pull_in_pane("riley-jones") # must not raise +def test_pull_in_pane_skips_silently_when_layout_server_unreachable(monkeypatch: pytest.MonkeyPatch) -> None: + # An isolated launch-task sub-agent can't reach the layout server: _resolve_active_layout + # returns (False, None). We must NOT attempt the split and NOT print anything. + attempted: list[tuple] = [] + printed: list[str] = [] + monkeypatch.setattr(fleet, "_resolve_active_layout", lambda: (False, None)) + monkeypatch.setattr(fleet, "_layout", lambda *a, **k: attempted.append(a) or True) + monkeypatch.setattr(fleet, "_out", lambda msg: printed.append(msg)) + fleet._pull_in_pane("riley-jones") + assert attempted == [] and printed == [] + + +def test_resolve_active_layout_prefers_client_that_messaged_this_agent(monkeypatch: pytest.MonkeyPatch) -> None: + # Two connected clients on different layouts; pick the one whose recent messages named + # OUR agent (context exposes agent_name, not id). + stdout = ( + '[{"is_connected": true, "current_layout": "mobile",' + ' "recent_messages": [{"agent_name": "someone-else"}]},' + ' {"is_connected": true, "current_layout": "desktop",' + ' "recent_messages": [{"agent_name": "riley-jones"}]}]' + ) + monkeypatch.setenv("MNGR_AGENT_NAME", "riley-jones") + monkeypatch.setattr(fleet.subprocess, "run", lambda *a, **k: fleet.subprocess.CompletedProcess([], 0, stdout, "")) + assert fleet._resolve_active_layout() == (True, "desktop") + + +def test_resolve_active_layout_unreachable_returns_false(monkeypatch: pytest.MonkeyPatch) -> None: + # A non-zero context exit (isolated sub-agent / no daemon) -> (False, None): skip silently. + monkeypatch.setattr(fleet.subprocess, "run", lambda *a, **k: fleet.subprocess.CompletedProcess([], 1, "", "boom")) + assert fleet._resolve_active_layout() == (False, None) + + def test_cmd_new_pulls_a_pane_by_name(monkeypatch: pytest.MonkeyPatch) -> None: # "Open a new browser" should visibly open its pane (by the returned name), not wait # for the first command. The daemon returns the chosen name as `name`. diff --git a/libs/browser/src/browser/fleet.py b/libs/browser/src/browser/fleet.py index 9d2557f9a..25e888f26 100644 --- a/libs/browser/src/browser/fleet.py +++ b/libs/browser/src/browser/fleet.py @@ -176,33 +176,85 @@ def _layout(*args: str, quiet: bool = False) -> bool: return result.returncode == 0 +def _resolve_active_layout() -> tuple[bool, str | None]: + """Resolve the layout to surface a browser pane into, via ``layout.py context``. + + ``split`` requires a ``--layout`` (mutating ops only apply on clients that have that + named layout active), so the pane-pull must name the layout the human is actually + viewing. ``context`` is a read-only query over the client-activity log. + + Returns ``(reachable, layout)``: + * ``reachable`` is False when the layout server can't be reached at all -- an isolated + ``launch-task`` sub-agent in its own container, or no daemon. The caller skips + silently: there is no screen of ours to surface into. + * When reachable, ``layout`` is the active layout to target -- the current layout of + the connected client that most recently messaged THIS agent (matched by name, since + the context summary carries ``agent_name`` not id), else the most-recently-active + connected client's layout, else None (reachable but nothing to place it on). + """ + root = _repo_root() + script = root / "scripts" / "layout.py" + if not script.exists(): + return (False, None) + result = subprocess.run( + [sys.executable, str(script), "context", "--json"], cwd=str(root), capture_output=True, text=True + ) + if result.returncode != 0: + return (False, None) # unreachable (isolated sub-agent / no daemon) + try: + clients = json.loads(result.stdout or "[]") + except json.JSONDecodeError: + return (True, None) + # ``context`` lists clients most-recently-active first; keep connected ones reporting a layout. + connected = [ + client + for client in clients + if isinstance(client, dict) and client.get("is_connected") and client.get("current_layout") + ] + my_name = os.environ.get("MNGR_AGENT_NAME") + if my_name: + for client in connected: + if any(msg.get("agent_name") == my_name for msg in client.get("recent_messages", [])): + return (True, str(client["current_layout"])) + if connected: + return (True, str(connected[0]["current_layout"])) + return (True, None) + + def _pull_in_pane(browser_name: str) -> None: - """Surface browser ``browser_name`` as its OWN pane to the right of the requesting - agent's chat (each browser in a separate pane). - - ``--new-group`` forces a fresh pane rather than tabbing the browser into an - existing pane group, so opening a second browser lands beside the first, not as a - tab inside it. Splitting an already-open browser is a no-op that just focuses it, - so this is safe to call repeatedly. - - Any agent the user started -- the primary, or one opened via "+ New agent" -- - surfaces the pane next to its OWN chat (``--relative-to self``). A launch-task / - background agent has no chat in this workspace's UI (and may be a separate - container), so the split can't land; we then say so in one clear line rather than - leaking layout.py's raw 5s "service not registered" error. Either way the browser - is running and reachable -- the pane is just a viewing convenience. + """Surface browser ``browser_name`` as its OWN pane beside the requesting agent's chat, + optimistically. + + Resolves the layout the requester's client is viewing via ``layout.py context`` (see + ``_resolve_active_layout``). If the layout server is unreachable -- an isolated + ``launch-task`` sub-agent in its own container -- we **skip silently**: there is no + screen of ours to surface into. Otherwise we split the browser into that layout next + to the agent's own chat (``--relative-to self``), or the parent's chat when a parent + handed a sub-agent its ref via ``$BROWSER_FLEET_ANCHOR``. ``--new-group`` makes each + browser its own pane; splitting an already-open one just focuses it, so this is safe + to call repeatedly. + + If the split can't land (no target layout, or the human isn't currently viewing it), + we fall back to one neutral line offering the manual "+"-menu route -- the browser is + up and fully drivable from the CLI either way; the pane is only a live-view convenience. """ + reachable, layout = _resolve_active_layout() + if not reachable: + return # isolated sub-agent / no layout server -- nothing of ours to surface into ref = f"service:browser?session={browser_name}" - # A parent may hand a sub-agent its chat as an anchor; otherwise anchor on our own. - anchor = os.environ.get(_ENV_ANCHOR) - if anchor and _layout("split", ref, "--relative-to", anchor, "--direction", "right", "--new-group", quiet=True): - return - if _layout("split", ref, "--relative-to", "self", "--direction", "right", "--new-group", quiet=True): - return - # Not an error -- the browser is up and fully drivable from the CLI; the pane is - # only a live-view convenience. Optimistic: the split lands when a client is - # watching this agent's chat, and otherwise (background/sub-agent, no chat in - # view) we just offer the manual route without implying anything broke. + if layout is not None: + # A parent may hand a sub-agent its chat as an anchor; otherwise anchor on our own. + anchor = os.environ.get(_ENV_ANCHOR) + if anchor and _layout( + "split", ref, "--relative-to", anchor, "--direction", "right", "--new-group", "--layout", layout, quiet=True + ): + return + if _layout( + "split", ref, "--relative-to", "self", "--direction", "right", "--new-group", "--layout", layout, quiet=True + ): + return + # Reachable but couldn't place the pane. Not an error -- offer the manual route + # without implying anything broke. _out(f"browser {browser_name} is ready. To watch it live, open it from the " '"+" menu (New browser -> ' + f"{browser_name}) in the side panel.") From 41cf2fd0ad4855e598da08ac0d09f5d70e592b91 Mon Sep 17 00:00:00 2001 From: Minh Trinh Date: Sat, 25 Jul 2026 01:40:14 +0000 Subject: [PATCH 11/15] CLAUDE.md: document the browser as a tool (agentic-browser-fleet vs Playwright) Adds a 'Browser is available as a tool' section to the workspace agent instructions: use the agentic-browser-fleet skill for collaborative, human-shareable browsing (with the control handoff/queue), or Playwright directly for lightweight integration testing on the same stealth Chromium. Co-Authored-By: Claude Opus 4.8 (1M context) --- CLAUDE.md | 7 +++++++ dev/changelog/browser-fleet-improvements.md | 2 ++ 2 files changed, 9 insertions(+) diff --git a/CLAUDE.md b/CLAUDE.md index bf8e4299b..f596909a2 100644 --- a/CLAUDE.md +++ b/CLAUDE.md @@ -177,6 +177,13 @@ They are inherently flaky due to timing and useless in CI, but valuable for agen If the user talks to you about files or directories on disk, assume (unless context indicates otherwise) they mean their local disk, not the one in your sandbox -- use the `file-sharing` skill to bridge the two. +# Browser is available as a tool + +A stealth build of Chromium designed to look like an ordinary human browser is installed in this workspace and can be used to complete browser-related tasks. + +1. When the user requests any browser-related tasks to be complete or a browser to be opened, use the `agentic-browser-fleet` skill, which allows you to drive many Chromium browsers. These are collaborative browsers which all agents and human users can use, though there is a mutually-exclusive control handoff and queuing system so only one is using a browser at a time. The skill has more information. Remember to hand off control to user when help is needed in the browser, such as anti-bot detection tests, and also release control when you are finished with a task so other agents and the user can use it. +2. If you'd like to do integration testing/small-scale web app scripting, use Playwright instead of spinning up an entire browser through the agentic-browser-fleet skill. This uses the same Chromium, just more lightweight. The user and other agents won't be able to collaborate on this; this is for quicker rendering and interaction tasks on the web. + # Work delegation You can delegate larger tasks to sub-agents using the `launch-task` skill. diff --git a/dev/changelog/browser-fleet-improvements.md b/dev/changelog/browser-fleet-improvements.md index b3a4d9ee7..716283d2a 100644 --- a/dev/changelog/browser-fleet-improvements.md +++ b/dev/changelog/browser-fleet-improvements.md @@ -1 +1,3 @@ First-boot deferred install now also installs `xvfb` and `xclip`, and a new `xvfb` supervised service provides the virtual display the browser fleet now runs headful under (the browser service is pointed at it via `DISPLAY=:99`). This is the infrastructure that enables the live browser's native clipboard copy/paste. + +The workspace agent instructions (`CLAUDE.md`) now include a "Browser is available as a tool" section: use the `agentic-browser-fleet` skill for collaborative, human-shareable browsing, or Playwright directly for lightweight integration testing / scripting on the same Chromium. From f783e97b81a773d0d283d5087882c933e0864169 Mon Sep 17 00:00:00 2001 From: Minh Trinh Date: Sat, 25 Jul 2026 04:27:22 +0000 Subject: [PATCH 12/15] skills: tell agents the browser pane is automatic, never open it by hand An agent was seen running 'layout.py open browser --layout desktop' -- the exact orphan-pane anti-pattern (no session -> dead placeholder, now gated). The pane already auto-opens on new/task/first-command with a graceful fallback line, so: - agentic-browser-fleet skill: explicit 'the pane is automatic; never open it yourself with layout.py; relay the fallback line instead of retrying'. - manage-layout skill: the generic 'open ' row now excludes 'browser' (the fleet auto-surfaces its own panes). Co-Authored-By: Claude Opus 4.8 (1M context) --- .agents/changelog/browser-fleet-improvements.md | 2 ++ .agents/skills/agentic-browser-fleet/SKILL.md | 4 +++- .agents/skills/manage-layout/SKILL.md | 2 +- 3 files changed, 6 insertions(+), 2 deletions(-) diff --git a/.agents/changelog/browser-fleet-improvements.md b/.agents/changelog/browser-fleet-improvements.md index 59b07a775..5288b9e9a 100644 --- a/.agents/changelog/browser-fleet-improvements.md +++ b/.agents/changelog/browser-fleet-improvements.md @@ -1 +1,3 @@ The `agentic-browser-fleet` skill now notes that, on resume after a human held the browser, the live view may have been resized and the page reflowed -- so every cached element number should be treated as stale and `state` re-run before acting. + +The `agentic-browser-fleet` and `manage-layout` skills now tell agents the browser pane is surfaced automatically and must never be opened by hand (`layout.py open browser` / `split browser`) -- a bare `service:browser` has no browser bound and is rejected, and a session-qualified open is redundant with the auto-pane. diff --git a/.agents/skills/agentic-browser-fleet/SKILL.md b/.agents/skills/agentic-browser-fleet/SKILL.md index afa88da91..310995fe7 100644 --- a/.agents/skills/agentic-browser-fleet/SKILL.md +++ b/.agents/skills/agentic-browser-fleet/SKILL.md @@ -204,7 +204,9 @@ uv run agentic-browser-fleet handoff alex-smith "solve the CAPTCHA on the sign-i ## Live view vs. your output -The browser shows up live in a UI pane next to your chat so the human can watch you operate it. That pane is **viewer only** -- your actual output (the `state` listings, the `ok:`/error lines, the screenshot paths) is in your CLI output here in the chat. Read and relay the CLI output; don't tell the user to "check the tab" for results. +The browser shows up live in a UI pane next to your chat so the human can watch you operate it. **This pane is automatic** -- `new`, `task`, and your first direct command each surface (and focus) it for you. **Never open it yourself** with `layout.py open`/`split browser` (or any other tool): a bare `service:browser` has no browser bound and the daemon rejects it, and a session-qualified open is redundant with the auto-pane. If the auto-pane can't land (you're a background/sub-agent with no chat in view), the CLI already prints a one-line "open it from the + menu" note -- relay that; don't retry with layout.py. + +That pane is **viewer only** -- your actual output (the `state` listings, the `ok:`/error lines, the screenshot paths) is in your CLI output here in the chat. Read and relay the CLI output; don't tell the user to "check the tab" for results. ## Multiple browsers, tabs, sub-agents diff --git a/.agents/skills/manage-layout/SKILL.md b/.agents/skills/manage-layout/SKILL.md index e0e1a9d34..46d02775e 100644 --- a/.agents/skills/manage-layout/SKILL.md +++ b/.agents/skills/manage-layout/SKILL.md @@ -65,7 +65,7 @@ Consequences for you: | Locate one panel + its tab-mates + cardinal neighbors | `python3 scripts/layout.py where [--layout ]` | | List everything addressable (services + agents) with open/running flags | `python3 scripts/layout.py list` | | Switch a client onto a named layout | `python3 scripts/layout.py load [--client ]` | -| Surface a service / URL / terminal / chat alongside your chat | `python3 scripts/layout.py open --layout ` | +| Surface a service / URL / terminal / chat alongside your chat (NOT `browser` -- the agentic-browser-fleet auto-surfaces its own panes; never `open browser`) | `python3 scripts/layout.py open --layout ` | | Put a new terminal in the same tab group as your chat | `python3 scripts/layout.py split terminal --relative-to=self --direction=within --layout ` | | Close a tab | `python3 scripts/layout.py close --layout ` | From eae368abba63fc802be40704e5bb160e101fee2d Mon Sep 17 00:00:00 2001 From: Minh Trinh Date: Sat, 25 Jul 2026 04:32:05 +0000 Subject: [PATCH 13/15] skills: tighten the pane guidance (qualify auto-open, allow named open) Correct + condense the earlier note: the pane auto-opens only when the user is watching that chat; if they explicitly ask to open one, use service:browser?session=; otherwise point them at the '+ -> browser' menu. One tight paragraph in agentic-browser-fleet, two sentences in manage-layout. Co-Authored-By: Claude Opus 4.8 (1M context) --- .agents/changelog/browser-fleet-improvements.md | 2 +- .agents/skills/agentic-browser-fleet/SKILL.md | 4 ++-- .agents/skills/manage-layout/SKILL.md | 6 +++++- 3 files changed, 8 insertions(+), 4 deletions(-) diff --git a/.agents/changelog/browser-fleet-improvements.md b/.agents/changelog/browser-fleet-improvements.md index 5288b9e9a..566edd197 100644 --- a/.agents/changelog/browser-fleet-improvements.md +++ b/.agents/changelog/browser-fleet-improvements.md @@ -1,3 +1,3 @@ The `agentic-browser-fleet` skill now notes that, on resume after a human held the browser, the live view may have been resized and the page reflowed -- so every cached element number should be treated as stale and `state` re-run before acting. -The `agentic-browser-fleet` and `manage-layout` skills now tell agents the browser pane is surfaced automatically and must never be opened by hand (`layout.py open browser` / `split browser`) -- a bare `service:browser` has no browser bound and is rejected, and a session-qualified open is redundant with the auto-pane. +The `agentic-browser-fleet` and `manage-layout` skills now tell agents the browser pane is surfaced automatically when the user is watching that chat, so they shouldn't manage it by hand: never open a bare `service:browser` (rejected -- no browser bound); if a user explicitly asks to open one, use `service:browser?session=`; otherwise point them at the "+ -> browser" menu. diff --git a/.agents/skills/agentic-browser-fleet/SKILL.md b/.agents/skills/agentic-browser-fleet/SKILL.md index 310995fe7..c2c7cb829 100644 --- a/.agents/skills/agentic-browser-fleet/SKILL.md +++ b/.agents/skills/agentic-browser-fleet/SKILL.md @@ -204,9 +204,9 @@ uv run agentic-browser-fleet handoff alex-smith "solve the CAPTCHA on the sign-i ## Live view vs. your output -The browser shows up live in a UI pane next to your chat so the human can watch you operate it. **This pane is automatic** -- `new`, `task`, and your first direct command each surface (and focus) it for you. **Never open it yourself** with `layout.py open`/`split browser` (or any other tool): a bare `service:browser` has no browser bound and the daemon rejects it, and a session-qualified open is redundant with the auto-pane. If the auto-pane can't land (you're a background/sub-agent with no chat in view), the CLI already prints a one-line "open it from the + menu" note -- relay that; don't retry with layout.py. +The browser streams to a UI pane next to your chat. `new`/`task`/your first command surface it automatically -- **but only when the user is currently watching your chat** (it lands beside that chat on their active layout). So don't manage panes yourself: never open a bare `service:browser` (no browser bound -- the daemon rejects it). If the user explicitly asks you to open a browser that isn't showing, run `layout.py open service:browser?session=` **with the name**; otherwise just tell them to open it from the workspace **+ -> browser** menu -- don't fuss over it. -That pane is **viewer only** -- your actual output (the `state` listings, the `ok:`/error lines, the screenshot paths) is in your CLI output here in the chat. Read and relay the CLI output; don't tell the user to "check the tab" for results. +The pane is **viewer only** -- your real output (`state` listings, `ok:`/error lines, screenshot paths) is here in the CLI. Read and relay that; don't tell the user to "check the tab" for results. ## Multiple browsers, tabs, sub-agents diff --git a/.agents/skills/manage-layout/SKILL.md b/.agents/skills/manage-layout/SKILL.md index 46d02775e..c4a1675bf 100644 --- a/.agents/skills/manage-layout/SKILL.md +++ b/.agents/skills/manage-layout/SKILL.md @@ -65,7 +65,7 @@ Consequences for you: | Locate one panel + its tab-mates + cardinal neighbors | `python3 scripts/layout.py where [--layout ]` | | List everything addressable (services + agents) with open/running flags | `python3 scripts/layout.py list` | | Switch a client onto a named layout | `python3 scripts/layout.py load [--client ]` | -| Surface a service / URL / terminal / chat alongside your chat (NOT `browser` -- the agentic-browser-fleet auto-surfaces its own panes; never `open browser`) | `python3 scripts/layout.py open --layout ` | +| Surface a service / URL / terminal / chat alongside your chat | `python3 scripts/layout.py open --layout ` | | Put a new terminal in the same tab group as your chat | `python3 scripts/layout.py split terminal --relative-to=self --direction=within --layout ` | | Close a tab | `python3 scripts/layout.py close --layout ` | @@ -96,6 +96,10 @@ Targets `open` accepts: than creating a duplicate. This is the same terminal the chat panel's "Open agent terminal" button mounts. +Not `browser`. The agentic-browser-fleet surfaces its own panes; a bare +`open browser` binds to no browser and is rejected. Only open one when a +user explicitly asks, and always name it: `open service:browser?session=`. + ## Refs: how every panel is addressed Every panel has a stable, type-prefixed ref returned by `inspect`: From 90702476ace3700879594a5e872e13f7fd288479 Mon Sep 17 00:00:00 2001 From: Minh Trinh Date: Sat, 25 Jul 2026 04:49:44 +0000 Subject: [PATCH 14/15] deferred_install: symlink Fortress into Playwright's default chromium path A bare playwright.chromium.launch() (no executable_path) looks in Playwright's own browser cache, which the Fortress swap leaves empty (install-deps, not install -> no managed Chromium downloaded), so ad-hoc Playwright calls error 'Chromium not present'. Symlink Fortress into that expected path -- resolved from Playwright itself so it tracks the pinned version's revision/layout -- so the default resolves to the same one engine. Chromium finds its resources via /proc/self/exe (the real Fortress dir), so symlinking the binary suffices. Not end-to-end tested with Fortress here (no Fortress on this box); the path resolution is verified. Wants a workspace smoke test of a bare chromium.launch(). Co-Authored-By: Claude Opus 4.8 (1M context) --- dev/changelog/browser-fleet-improvements.md | 2 ++ scripts/deferred_install.sh | 19 +++++++++++++++++++ 2 files changed, 21 insertions(+) diff --git a/dev/changelog/browser-fleet-improvements.md b/dev/changelog/browser-fleet-improvements.md index 716283d2a..f02689d0f 100644 --- a/dev/changelog/browser-fleet-improvements.md +++ b/dev/changelog/browser-fleet-improvements.md @@ -1,3 +1,5 @@ First-boot deferred install now also installs `xvfb` and `xclip`, and a new `xvfb` supervised service provides the virtual display the browser fleet now runs headful under (the browser service is pointed at it via `DISPLAY=:99`). This is the infrastructure that enables the live browser's native clipboard copy/paste. The workspace agent instructions (`CLAUDE.md`) now include a "Browser is available as a tool" section: use the `agentic-browser-fleet` skill for collaborative, human-shareable browsing, or Playwright directly for lightweight integration testing / scripting on the same Chromium. + +The Fortress deferred install now also symlinks Fortress into Playwright's default browser-cache path, so a bare `playwright.chromium.launch()` (no `executable_path`) resolves to Fortress instead of erroring on the managed Chromium the Fortress swap intentionally stopped downloading. One engine, no second Chromium download. diff --git a/scripts/deferred_install.sh b/scripts/deferred_install.sh index f879c0389..5a10291f0 100755 --- a/scripts/deferred_install.sh +++ b/scripts/deferred_install.sh @@ -124,6 +124,25 @@ _install_fortress() { return 1 fi chmod +x "$_FORTRESS_INSTALL_DIR/tilion-fortress/tilion" + # Point Playwright's DEFAULT chromium at Fortress too. A bare `chromium.launch()` + # (no executable_path) looks in Playwright's own browser cache, which this install + # deliberately leaves empty (we run `install-deps`, not `install`, so no managed + # Chromium is downloaded). Symlink Fortress into that expected path -- resolved + # from Playwright itself so it tracks the pinned version's revision/layout -- so + # ad-hoc Playwright calls use the same one engine instead of erroring on a missing + # build. Chromium finds its resources via /proc/self/exe (the real Fortress dir), + # so symlinking just the binary is enough. + local pw_chrome + pw_chrome="$(cd "$REPO_ROOT" && uv run python -c 'from playwright.sync_api import sync_playwright; p=sync_playwright().start(); print(p.chromium.executable_path); p.stop()' 2>/dev/null)" + if [ -n "$pw_chrome" ]; then + mkdir -p "$(dirname "$pw_chrome")" + ln -sf "$_FORTRESS_INSTALL_DIR/tilion-fortress/tilion" "$pw_chrome" + # Some Playwright versions gate launch() on a per-browser install marker. + touch "$(dirname "$(dirname "$pw_chrome")")/INSTALLATION_COMPLETE" 2>/dev/null || true + _log "fortress: pointed Playwright's default chromium at Fortress ($pw_chrome)" + else + _log "fortress: WARNING could not resolve Playwright's chromium path; a bare chromium.launch() will still need an explicit executable_path" + fi touch "$marker" _log "fortress: install complete (${_FORTRESS_INSTALL_DIR}/tilion-fortress/tilion), marker written to $marker" } From 5df781ae334f1c72a52f265692258be11df3e1e5 Mon Sep 17 00:00:00 2001 From: Minh Trinh Date: Thu, 30 Jul 2026 01:50:22 +0000 Subject: [PATCH 15/15] Point _resolve_active_layout at system/scripts/layout.py post-declutter Co-Authored-By: Claude Fable 5 --- system/apps/browser/src/browser/fleet.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/system/apps/browser/src/browser/fleet.py b/system/apps/browser/src/browser/fleet.py index 440456ad1..d14b4eb1d 100644 --- a/system/apps/browser/src/browser/fleet.py +++ b/system/apps/browser/src/browser/fleet.py @@ -193,7 +193,7 @@ def _resolve_active_layout() -> tuple[bool, str | None]: connected client's layout, else None (reachable but nothing to place it on). """ root = _repo_root() - script = root / "scripts" / "layout.py" + script = root / "system" / "scripts" / "layout.py" if not script.exists(): return (False, None) result = subprocess.run(