diff --git a/CHANGELOG.md b/CHANGELOG.md index 3f5d02677..f42d5a408 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -4,6 +4,7 @@ ### Fixed +- Browser: open the ChatGPT attachment menu through the exact composer control and trusted keyboard activation, avoiding broad coordinate clicks that could start a nearby Work suggestion; stop before prompt submission if activation enters Work or another conversation. - Browser: attach to running Chrome when `DevToolsActivePort` metadata is absent, with IPv6 support and bounded endpoint retries that include response-body reads. Fixes #414. Thanks @devYRPauli! - Browser: recognize collision-renamed attachment chips, including short filenames, while keeping Unicode filename boundaries and visible extensions distinct across upload and send checks. Fixes #393. Thanks @devYRPauli! - Browser: select and verify thinking effort in ChatGPT's direct-slider picker without an Advanced submenu, keeping explicit Pro requests fail-closed. Fixes #422. diff --git a/docs/manual-tests.md b/docs/manual-tests.md index a4aaf28b8..d094c1dbe 100644 --- a/docs/manual-tests.md +++ b/docs/manual-tests.md @@ -134,6 +134,12 @@ Debug note: when you have a live ChatGPT tab open under a DevTools port and need - Remember: the browser composer now pastes only the user prompt (plus any inline file blocks). If you see the default “You are Oracle…” text or other system-prefixed content in the ChatGPT composer, something regressed in `assembleBrowserPrompt` and you should stop and file a bug. - Heartbeats: Browser runs emit `--heartbeat` status while waiting. Long Thinking/Pro runs should show `[browser] ChatGPT thinking ...` or `[browser] Waiting for ChatGPT response ...`; the log must not include reasoning text from the side panel. +4b. **Chat/Work Attachment Safety** + +- Leave ChatGPT on the Work home, then run an Instant browser consult with one small attachment. +- Oracle must switch to Chat before upload, open only the exact composer `+` control, and submit the requested prompt. +- Compare the ChatGPT sidebar before and after: the run may create its expected ordinary Chat conversation, but it must not start a Work suggestion/task. If attachment activation changes to Work or another conversation, Oracle must stop before submitting the prompt. + ## Post-Run Validation - `oracle session ` should replay the transcript with markdown. diff --git a/docs/windows-work.md b/docs/windows-work.md index 5f9125d9a..d4c85cf3b 100644 --- a/docs/windows-work.md +++ b/docs/windows-work.md @@ -12,3 +12,4 @@ Read this file whenever you're working from Windows and add new findings so the Future Windows gotchas belong here. Update this doc when you learn something new. - ChatGPT sidebar/history labels can include phrases like "Login setup instruction"; login probes must match exact auth CTAs, not any visible text starting with login, or manual-login automation loops forever before typing. +- ChatGPT's Work home can place suggestion cards close to the composer attachment control. Attachment automation must target only `#composer-plus-btn` / `data-testid="composer-plus-btn"`, prefer focused trusted-key activation over page coordinates, and verify that the click did not create a Work or unexpected conversation before uploading. diff --git a/src/browser/actions/attachments.ts b/src/browser/actions/attachments.ts index 22be2e88d..745e69d3f 100644 --- a/src/browser/actions/attachments.ts +++ b/src/browser/actions/attachments.ts @@ -5,6 +5,7 @@ import { buildConversationTurnListExpression } from "../conversationTurns.js"; import { delay } from "../utils.js"; import { logDomFailure } from "../domDebug.js"; import { transferAttachmentViaDataTransfer } from "./attachmentDataTransfer.js"; +import { BrowserAutomationError } from "../../oracle/errors.js"; export function buildAttachmentNamePattern( expectedName: string, @@ -31,6 +32,186 @@ export function buildAttachmentNamePattern( return new RegExp(`(?:^|[^${filenameChars}])${name}`, "iu"); } +export interface ComposerPlusActivationResult { + method: "trusted-keyboard" | "synthetic" | "unavailable"; + startUrl: string; +} + +type ComposerPlusProbe = { + status?: "focused" | "missing" | "work-selected"; + startUrl?: string; + focused?: boolean; +}; + +export async function activateComposerPlus( + runtime: ChromeClient["Runtime"], + input?: ChromeClient["Input"], +): Promise { + const probe = await Promise.resolve( + runtime.evaluate({ + expression: `(() => { + const startUrl = location.href; + const normalize = (value) => String(value || '').replace(/\\s+/g, ' ').trim().toLowerCase(); + const isWorkLabel = (value) => ['work', '工作'].includes(normalize(value)); + const selected = (node) => + node?.getAttribute?.('aria-checked') === 'true' || + node?.getAttribute?.('aria-selected') === 'true' || + node?.getAttribute?.('data-state') === 'on'; + const workToggleSelected = Array.from(document.querySelectorAll('button[role="radio"]')).some( + (node) => isWorkLabel(node.textContent) && selected(node), + ); + const conversationId = location.pathname.match(/\\/c\\/([a-zA-Z0-9-]+)/)?.[1] || null; + const activeWorkConversation = conversationId + ? Array.from(document.querySelectorAll('a.__menu-item[href*="/c/"]')).some((node) => { + let candidateId = null; + try { + candidateId = new URL(node.getAttribute('href') || '', location.origin).pathname.match( + /\\/c\\/([a-zA-Z0-9-]+)/, + )?.[1] || null; + } catch { + return false; + } + if (candidateId !== conversationId) return false; + return Array.from(node.querySelectorAll('span')).some((span) => + span instanceof HTMLElement && + span.tagName === 'SPAN' && + isWorkLabel(span.textContent) && + span.childElementCount === 0 && + !span.hasAttribute('dir') && + span.classList.contains('shrink-0') && + span.parentElement?.matches('span.flex.items-center'), + ); + }) + : false; + if (workToggleSelected || activeWorkConversation) { + return { status: 'work-selected', startUrl }; + } + const selectors = ['#composer-plus-btn', 'button[data-testid="composer-plus-btn"]']; + for (const selector of selectors) { + const node = document.querySelector(selector); + if (!(node instanceof HTMLElement)) continue; + const rect = node.getBoundingClientRect(); + if (rect.width <= 0 || rect.height <= 0) continue; + node.focus({ preventScroll: true }); + return { status: 'focused', startUrl, focused: document.activeElement === node }; + } + return { status: 'missing', startUrl }; + })()`, + returnByValue: true, + }), + ) + .then((result) => result?.result?.value as ComposerPlusProbe | undefined) + .catch(() => undefined); + + const startUrl = typeof probe?.startUrl === "string" ? probe.startUrl : ""; + if (probe?.status === "work-selected") { + throw new BrowserAutomationError( + "Oracle refused to open the attachment menu because ChatGPT is in Work mode.", + { + stage: "upload-attachment", + code: "attachment-control-work-mode", + details: { startUrl }, + }, + ); + } + if (probe?.status !== "focused") { + return { method: "unavailable", startUrl }; + } + + if (probe.focused && input && typeof input.dispatchKeyEvent === "function") { + try { + const enter = { + key: "Enter", + code: "Enter", + windowsVirtualKeyCode: 13, + nativeVirtualKeyCode: 13, + } as const; + await input.dispatchKeyEvent({ + type: "keyDown", + ...enter, + text: "\r", + unmodifiedText: "\r", + }); + await input.dispatchKeyEvent({ type: "keyUp", ...enter }); + return { method: "trusted-keyboard", startUrl }; + } catch { + // Fall through to an exact-selector synthetic click. Never use page coordinates here. + } + } + + const clicked = await Promise.resolve( + runtime.evaluate({ + expression: `(() => { + const selectors = ['#composer-plus-btn', 'button[data-testid="composer-plus-btn"]']; + for (const selector of selectors) { + const node = document.querySelector(selector); + if (!(node instanceof HTMLElement)) continue; + const rect = node.getBoundingClientRect(); + if (rect.width <= 0 || rect.height <= 0) continue; + node.click(); + return true; + } + return false; + })()`, + returnByValue: true, + }), + ) + .then((result) => Boolean(result?.result?.value)) + .catch(() => false); + return { method: clicked ? "synthetic" : "unavailable", startUrl }; +} + +export async function assertComposerPlusStayedInPlace( + runtime: ChromeClient["Runtime"], + startUrl: string, +): Promise { + const result = await runtime.evaluate({ + expression: `(() => { + const normalize = (value) => String(value || '').replace(/\\s+/g, ' ').trim().toLowerCase(); + const isWorkLabel = (value) => ['work', '工作'].includes(normalize(value)); + const selected = (node) => + node?.getAttribute?.('aria-checked') === 'true' || + node?.getAttribute?.('aria-selected') === 'true' || + node?.getAttribute?.('data-state') === 'on'; + const workSelected = Array.from(document.querySelectorAll('button[role="radio"]')).some( + (node) => isWorkLabel(node.textContent) && selected(node), + ); + return { currentUrl: location.href, workSelected }; + })()`, + returnByValue: true, + }); + const value = result?.result?.value as + | { currentUrl?: string; workSelected?: boolean } + | undefined; + const currentUrl = typeof value?.currentUrl === "string" ? value.currentUrl : ""; + const conversationId = (value: string): string | null => { + try { + return new URL(value).pathname.match(/^\/c\/([^/?#]+)/)?.[1] ?? null; + } catch { + return null; + } + }; + const startConversationId = conversationId(startUrl); + const currentConversationId = conversationId(currentUrl); + const unexpectedConversation = startConversationId + ? currentConversationId !== startConversationId + : currentConversationId !== null; + if (value?.workSelected || unexpectedConversation) { + throw new BrowserAutomationError( + "ChatGPT navigated to Work or another conversation while Oracle opened the attachment menu; upload was stopped before prompt submission.", + { + stage: "upload-attachment", + code: "attachment-control-unexpected-navigation", + details: { + startUrl, + currentUrl, + workSelected: Boolean(value?.workSelected), + }, + }, + ); + } +} + export async function uploadAttachmentFile( deps: { runtime: ChromeClient["Runtime"]; @@ -325,73 +506,13 @@ export async function uploadAttachmentFile( }; }; - // New ChatGPT UI hides the real file input behind a composer "+" menu; click it pre-emptively. - // Learned: synthetic `.click()` is sometimes ignored (isTrusted checks). Prefer a CDP mouse click when possible. - const clickPlusTrusted = async (): Promise => { - if (!input || typeof input.dispatchMouseEvent !== "function") return false; - const locate = await runtime - .evaluate({ - expression: `(() => { - const selectors = [ - '#composer-plus-btn', - 'button[data-testid="composer-plus-btn"]', - '[data-testid*="plus"]', - 'button[aria-label*="add"]', - 'button[aria-label*="attachment"]', - 'button[aria-label*="file"]', - ]; - for (const selector of selectors) { - const el = document.querySelector(selector); - if (!(el instanceof HTMLElement)) continue; - const rect = el.getBoundingClientRect(); - if (rect.width <= 0 || rect.height <= 0) continue; - el.scrollIntoView({ block: 'center', inline: 'center' }); - const nextRect = el.getBoundingClientRect(); - return { ok: true, x: nextRect.left + nextRect.width / 2, y: nextRect.top + nextRect.height / 2 }; - } - return { ok: false }; - })()`, - returnByValue: true, - }) - .then((res) => res?.result?.value as { ok?: boolean; x?: number; y?: number } | undefined) - .catch(() => undefined); - if (!locate?.ok || typeof locate.x !== "number" || typeof locate.y !== "number") return false; - const x = locate.x; - const y = locate.y; - await input.dispatchMouseEvent({ type: "mouseMoved", x, y }); - await input.dispatchMouseEvent({ type: "mousePressed", x, y, button: "left", clickCount: 1 }); - await input.dispatchMouseEvent({ type: "mouseReleased", x, y, button: "left", clickCount: 1 }); - return true; - }; - - const clickedTrusted = await clickPlusTrusted().catch(() => false); - if (!clickedTrusted) { - await Promise.resolve( - runtime.evaluate({ - expression: `(() => { - const selectors = [ - '#composer-plus-btn', - 'button[data-testid="composer-plus-btn"]', - '[data-testid*="plus"]', - 'button[aria-label*="add"]', - 'button[aria-label*="attachment"]', - 'button[aria-label*="file"]', - ]; - for (const selector of selectors) { - const el = document.querySelector(selector); - if (el instanceof HTMLElement) { - el.click(); - return true; - } - } - return false; - })()`, - returnByValue: true, - }), - ).catch(() => undefined); - } - + // The Work home places suggestion cards close to the composer. Never use a broad selector or + // coordinate click here: focus the exact composer control and activate it with a trusted key. + const plusActivation = await activateComposerPlus(runtime, input); await delay(350); + if (plusActivation.method !== "unavailable") { + await assertComposerPlusStayedInPlace(runtime, plusActivation.startUrl); + } const normalizeForMatch = (value: string): string => String(value || "") diff --git a/tests/browser/pageActions.test.ts b/tests/browser/pageActions.test.ts index d6a6b2dcc..856fffc02 100644 --- a/tests/browser/pageActions.test.ts +++ b/tests/browser/pageActions.test.ts @@ -1735,6 +1735,80 @@ describe("waitForAssistantResponse", () => { }); }); +describe("activateComposerPlus", () => { + test("uses the exact composer control and a trusted keyboard activation", async () => { + const evaluate = vi.fn().mockResolvedValue({ + result: { + value: { + status: "focused", + startUrl: "https://chatgpt.com/", + focused: true, + }, + }, + }); + const dispatchKeyEvent = vi.fn().mockResolvedValue(undefined); + const dispatchMouseEvent = vi.fn().mockResolvedValue(undefined); + const runtime = { evaluate } as unknown as ChromeClient["Runtime"]; + const input = { dispatchKeyEvent, dispatchMouseEvent } as unknown as ChromeClient["Input"]; + + await expect(attachments.activateComposerPlus(runtime, input)).resolves.toEqual({ + method: "trusted-keyboard", + startUrl: "https://chatgpt.com/", + }); + + const expression = String(evaluate.mock.calls[0]?.[0]?.expression ?? ""); + expect(expression).toContain("#composer-plus-btn"); + expect(expression).toContain('button[data-testid="composer-plus-btn"]'); + expect(expression).not.toContain('[data-testid*="plus"]'); + expect(expression).not.toContain('button[aria-label*="add"]'); + expect(dispatchKeyEvent).toHaveBeenNthCalledWith( + 1, + expect.objectContaining({ type: "keyDown", key: "Enter" }), + ); + expect(dispatchKeyEvent).toHaveBeenNthCalledWith( + 2, + expect.objectContaining({ type: "keyUp", key: "Enter" }), + ); + expect(dispatchMouseEvent).not.toHaveBeenCalled(); + }); + + test("fails closed when ChatGPT is already in Work mode", async () => { + const runtime = { + evaluate: vi.fn().mockResolvedValue({ + result: { + value: { + status: "work-selected", + startUrl: "https://chatgpt.com/", + }, + }, + }), + } as unknown as ChromeClient["Runtime"]; + + await expect(attachments.activateComposerPlus(runtime)).rejects.toMatchObject({ + details: { code: "attachment-control-work-mode" }, + }); + }); + + test("rejects an unexpected conversation created by attachment activation", async () => { + const runtime = { + evaluate: vi.fn().mockResolvedValue({ + result: { + value: { + currentUrl: "https://chatgpt.com/c/unexpected-work-task", + workSelected: true, + }, + }, + }), + } as unknown as ChromeClient["Runtime"]; + + await expect( + attachments.assertComposerPlusStayedInPlace(runtime, "https://chatgpt.com/"), + ).rejects.toMatchObject({ + details: { code: "attachment-control-unexpected-navigation" }, + }); + }); +}); + describe("uploadAttachmentFile", () => { let transferSpy: ReturnType;