diff --git a/CHANGELOG.md b/CHANGELOG.md index 3f5d02677..f841ffb2b 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -4,6 +4,7 @@ ### Fixed +- Browser: recover when ChatGPT leaves an attachment-bearing image prompt staged after a no-op send-button click by submitting it once via Enter; normalize ProseMirror paragraph whitespace and persist `promptSubmitted` only after the user turn is observable. - Browser: attach to running Chrome when `DevToolsActivePort` metadata is absent, with IPv6 support and bounded endpoint retries that include response-body reads. Fixes #414. Thanks @devYRPauli! - Browser: recognize collision-renamed attachment chips, including short filenames, while keeping Unicode filename boundaries and visible extensions distinct across upload and send checks. Fixes #393. Thanks @devYRPauli! - Browser: select and verify thinking effort in ChatGPT's direct-slider picker without an Advanced submenu, keeping explicit Pro requests fail-closed. Fixes #422. diff --git a/docs/manual-tests.md b/docs/manual-tests.md index a4aaf28b8..47e224b9f 100644 --- a/docs/manual-tests.md +++ b/docs/manual-tests.md @@ -134,6 +134,12 @@ Debug note: when you have a live ChatGPT tab open under a DevTools port and need - Remember: the browser composer now pastes only the user prompt (plus any inline file blocks). If you see the default “You are Oracle…” text or other system-prefixed content in the ChatGPT composer, something regressed in `assembleBrowserPrompt` and you should stop and file a bug. - Heartbeats: Browser runs emit `--heartbeat` status while waiting. Long Thinking/Pro runs should show `[browser] ChatGPT thinking ...` or `[browser] Waiting for ChatGPT response ...`; the log must not include reasoning text from the side panel. +4b. **ChatGPT image generation with a reference image** + +- Run `oracle --engine browser --browser-manual-login --model gpt-5.6-sol --browser-thinking-time light --browser-attachments always --file /tmp/reference.png --generate-image /tmp/oracle-image-smoke.png --prompt "Generate a simple square image based on the reference."`. +- Confirm the session runtime records `promptSubmitted: true`, ChatGPT shows the submitted user turn (not a prompt left in the composer), and `/tmp/oracle-image-smoke.png` is a valid downloaded image. +- If the send-button click is a no-op, expect `Send click left the prompt staged; submitting once via Enter` before the user turn appears. + ## Post-Run Validation - `oracle session ` should replay the transcript with markdown. diff --git a/docs/windows-work.md b/docs/windows-work.md index 5f9125d9a..6f36e46ce 100644 --- a/docs/windows-work.md +++ b/docs/windows-work.md @@ -12,3 +12,4 @@ Read this file whenever you're working from Windows and add new findings so the Future Windows gotchas belong here. Update this doc when you learn something new. - ChatGPT sidebar/history labels can include phrases like "Login setup instruction"; login probes must match exact auth CTAs, not any visible text starting with login, or manual-login automation loops forever before typing. +- On the Windows ChatGPT home composer, a reference-image upload can leave the prompt staged even after a trusted click on the visible enabled `data-testid="send-button"`. An Enter fallback must compare canonical whitespace because ProseMirror inserts extra paragraph newlines; gate the fallback on no new turn/no stop control, and record `promptSubmitted` only after the user turn is observable. diff --git a/src/browser/actions/promptComposer.ts b/src/browser/actions/promptComposer.ts index 7f04f745e..fa28291ad 100644 --- a/src/browser/actions/promptComposer.ts +++ b/src/browser/actions/promptComposer.ts @@ -5,6 +5,7 @@ import { PROMPT_FALLBACK_SELECTOR, SEND_BUTTON_SELECTORS, STOP_BUTTON_SELECTOR, + STOP_BUTTON_SELECTORS, ASSISTANT_ROLE_SELECTOR, } from "../constants.js"; import { @@ -235,18 +236,107 @@ export async function submitPrompt( logger("Submitted prompt via Enter key"); } else { logger("Clicked send button"); + await submitStagedPromptViaEnter( + runtime, + input, + prompt, + deps.baselineTurns ?? undefined, + logger, + ); } - await deps.onPromptSubmitted?.(); const commitTimeoutMs = Math.max(60_000, deps.inputTimeoutMs ?? 0); // Learned: the send button can succeed but the turn doesn't appear immediately; verify commit via turns/stop button. - return await verifyPromptCommitted( + const committedTurns = await verifyPromptCommitted( runtime, prompt, commitTimeoutMs, logger, deps.baselineTurns ?? undefined, ); + // A click is only an attempt: persist promptSubmitted after ChatGPT exposes + // the user turn so no-op controls cannot make an unsubmitted session look live. + await deps.onPromptSubmitted?.(); + return committedTurns; +} + +async function submitStagedPromptViaEnter( + Runtime: ChromeClient["Runtime"], + Input: ChromeClient["Input"], + prompt: string, + baselineTurns: number | undefined, + logger: BrowserLogger, +): Promise { + // ChatGPT can leave the composer untouched after a trusted click, especially + // when an image attachment is present. Give the click a chance to commit, then + // use Enter only while the same prompt is still staged and no generation/turn + // signal has appeared. ProseMirror inserts extra paragraph newlines, so compare + // canonical whitespace rather than raw innerText. + await delay(750); + const normalizedPrompt = prompt.replace(/\s+/gu, " ").trim(); + const normalizedPromptLiteral = JSON.stringify(normalizedPrompt); + const inputSelectorsLiteral = JSON.stringify(INPUT_SELECTORS); + const stopSelectorsLiteral = JSON.stringify(STOP_BUTTON_SELECTORS); + const baselineLiteral = + typeof baselineTurns === "number" && Number.isFinite(baselineTurns) + ? Math.max(0, Math.floor(baselineTurns)) + : -1; + const outcome = await Runtime.evaluate({ + expression: `(() => { + const selectors = ${inputSelectorsLiteral}; + const stopSelectors = ${stopSelectorsLiteral}; + const normalize = (value) => String(value ?? '').replace(/\\s+/gu, ' ').trim(); + const isVisible = (node) => { + if (!(node instanceof HTMLElement)) return false; + const rect = node.getBoundingClientRect(); + return rect.width > 0 && rect.height > 0; + }; + const readValue = (node) => { + if (node instanceof HTMLTextAreaElement || node instanceof HTMLInputElement) { + return node.value ?? ''; + } + return node.innerText ?? node.textContent ?? ''; + }; + const candidates = selectors + .flatMap((selector) => Array.from(document.querySelectorAll(selector))) + .filter(isVisible); + const editor = candidates.find( + (node) => normalize(readValue(node)) === ${normalizedPromptLiteral}, + ) ?? null; + const stopVisible = stopSelectors.some((selector) => + Array.from(document.querySelectorAll(selector)).some(isVisible), + ); + const turns = ${buildConversationTurnListExpression()}; + const baseline = ${baselineLiteral}; + const hasNewTurn = baseline >= 0 && turns.length > baseline; + if (!(editor instanceof HTMLElement) || stopVisible || hasNewTurn) { + return { staged: false, focused: false, stopVisible, hasNewTurn }; + } + editor.focus(); + return { + staged: true, + focused: document.activeElement === editor, + stopVisible, + hasNewTurn, + }; + })()`, + returnByValue: true, + }).catch(() => null); + if (!outcome?.result?.value?.staged || !outcome.result.value.focused) { + return false; + } + logger("Send click left the prompt staged; submitting once via Enter"); + await Input.dispatchKeyEvent({ + type: "keyDown", + ...ENTER_KEY_EVENT, + text: ENTER_KEY_TEXT, + unmodifiedText: ENTER_KEY_TEXT, + }); + await Input.dispatchKeyEvent({ + type: "keyUp", + ...ENTER_KEY_EVENT, + }); + return true; } export async function clearPromptComposer(Runtime: ChromeClient["Runtime"], logger: BrowserLogger) { @@ -940,5 +1030,6 @@ function summarizeCommitProbe(probe: CommitProbeState): Record export const __test__ = { attemptSendButton, sendButtonTimeoutMs, + submitStagedPromptViaEnter, verifyPromptCommitted, }; diff --git a/tests/browser/promptComposer.test.ts b/tests/browser/promptComposer.test.ts index 6bd8edb11..3a4cf6d28 100644 --- a/tests/browser/promptComposer.test.ts +++ b/tests/browser/promptComposer.test.ts @@ -242,7 +242,7 @@ describe("promptComposer", () => { expect(promptComposer.sendButtonTimeoutMs(["oracle-attach-verify.txt"], 120_000)).toBe(120_000); }); - test("marks prompt submitted before commit verification finishes", async () => { + test("marks prompt submitted after commit verification succeeds", async () => { const onPromptSubmitted = vi.fn(); const runtime = { evaluate: vi.fn(async ({ expression }: { expression: string }) => { @@ -260,6 +260,7 @@ describe("promptComposer", () => { if (expression.includes("button.scrollIntoView")) { return { result: { value: { status: "clicked" } } }; } + expect(onPromptSubmitted).not.toHaveBeenCalled(); return { result: { value: { @@ -295,6 +296,71 @@ describe("promptComposer", () => { expect(onPromptSubmitted).toHaveBeenCalledTimes(1); }); + test("uses Enter when a no-op click leaves a ProseMirror prompt staged with extra newlines", async () => { + vi.useFakeTimers(); + try { + let focused = false; + class FakeHTMLElement { + innerText = "Line 1\n\n\nLine 2"; + textContent = this.innerText; + + getBoundingClientRect() { + return { width: 100, height: 30 }; + } + + focus() { + focused = true; + } + } + class FakeTextAreaElement extends FakeHTMLElement { + value = ""; + } + class FakeInputElement extends FakeHTMLElement { + value = ""; + } + const editor = new FakeHTMLElement(); + const document = { + get activeElement() { + return focused ? editor : null; + }, + querySelectorAll: (selector: string) => + selector === "#prompt-textarea" || selector === ".ProseMirror" ? [editor] : [], + }; + const runtime = { + evaluate: vi.fn(async ({ expression }: { expression: string }) => ({ + result: { + value: Function( + "document", + "HTMLElement", + "HTMLTextAreaElement", + "HTMLInputElement", + `return ${expression};`, + )(document, FakeHTMLElement, FakeTextAreaElement, FakeInputElement), + }, + })), + }; + const input = { dispatchKeyEvent: vi.fn() }; + const logger = Object.assign(vi.fn(), { verbose: false }); + + const result = promptComposer.submitStagedPromptViaEnter( + runtime as never, + input as never, + "Line 1\n\nLine 2", + 0, + logger as never, + ); + await vi.advanceTimersByTimeAsync(750); + + await expect(result).resolves.toBe(true); + expect(input.dispatchKeyEvent).toHaveBeenCalledTimes(2); + expect(logger).toHaveBeenCalledWith( + "Send click left the prompt staged; submitting once via Enter", + ); + } finally { + vi.useRealTimers(); + } + }); + test("waits for a delayed trusted click without issuing a second send", async () => { vi.useFakeTimers(); try {