diff --git a/CHANGELOG.md b/CHANGELOG.md index 9edf60788..223b12266 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,5 +1,15 @@ # Changelog +## Unreleased + +### Added + +- Browser: persist thinking-effort selection evidence as `browser.thinkingSelection`, parallel to `browser.modelSelection`, and render it in `oracle status`. `ensureThinkingTime` verified the requested tier and then discarded the result, and the model evidence cannot stand in: for a Pro-capable target the picker reports the requested model string as the resolved label, so `resolvedLabel === requestedModel; verified: true` is byte-identical whether or not the Pro row was ever selected. `verified` is true only for the two statuses that positively observed the option's selected state, and strict (fail-closed) requests still throw before submit on every other outcome, so a persisted strict record is proof by refusal that no degraded tier was used. + +### Fixed + +- Remote: stop stripping run identity and selection evidence from bridged results. `sanitizeResult` correctly keeps host detail (pids, ports, profile paths) from crossing to a client on another machine, but the whitelist had drifted narrower than that rule and also dropped `modelSelection`, `thinkingSelection`, `archive`, `tabUrl`, `conversationId`, and `promptSubmitted` — so a remote caller could not tell which model or effort answered their prompt, nor bind the answer to a durable ChatGPT URL. The fields are optional, so nothing failed; the answer simply arrived unattributable. + ## 0.18.0 — 2026-08-14 ### Changed diff --git a/src/browser/actions/thinkingTime.ts b/src/browser/actions/thinkingTime.ts index 6ed7c383f..18e29f19f 100644 --- a/src/browser/actions/thinkingTime.ts +++ b/src/browser/actions/thinkingTime.ts @@ -1,5 +1,9 @@ import type { ChromeClient, BrowserLogger } from "../types.js"; import type { ThinkingTimeLevel } from "../../oracle/types.js"; +import type { + BrowserThinkingSelectionEvidence, + BrowserThinkingSelectionStatus, +} from "../../sessionManager.js"; import { MENU_CONTAINER_SELECTOR, MENU_ITEM_SELECTOR, @@ -103,13 +107,18 @@ function logPickerDiagnostic(result: ThinkingTimeOutcome | undefined, logger: Br * * Missing controls remain best-effort except Pro Extended, which fails closed * unless the selected option is confirmed. + * + * Returns the evidence record for what was actually confirmed in the picker, so + * a caller can persist proof that a run submitted at the requested effort rather + * than inheriting whatever tier the composer already had. Strict (fail-closed) + * requests never return an unverified record — they throw before submit. */ export async function ensureThinkingTime( Runtime: ChromeClient["Runtime"], level: ThinkingTimeLevel, logger: BrowserLogger, desiredModel?: string | null, -) { +): Promise { const result = await evaluateThinkingTimeSelection(Runtime, level, desiredModel); const capitalizedLevel = level.charAt(0).toUpperCase() + level.slice(1); const targetModelKind = inferThinkingTargetModelKind(desiredModel); @@ -120,14 +129,28 @@ export async function ensureThinkingTime( const strictProEffort = level === "pro" || ((targetModelKind === "pro" || observedModelKind === "pro") && level === "extended"); + const evidence = ( + status: BrowserThinkingSelectionStatus, + resolvedLabel: string | null, + ): BrowserThinkingSelectionEvidence => ({ + requestedLevel: level, + status, + resolvedLabel, + verified: status === "already-selected" || status === "switched", + strictFailClosed: strictProEffort, + targetModelKind: targetModelKind ?? null, + observedModelKind: observedModelKind ?? null, + source: "chatgpt-thinking-picker", + capturedAt: new Date().toISOString(), + }); switch (result?.status) { case "already-selected": logger(formatBrowserThinkingLog(`${result.label ?? capitalizedLevel} (already selected)`)); - return; + return evidence("already-selected", result.label ?? null); case "switched": logger(formatBrowserThinkingLog(result.label ?? capitalizedLevel)); - return; + return evidence("switched", result.label ?? null); case "option-disabled": { await logDomFailure(Runtime, logger, "thinking-option-disabled"); logPickerDiagnostic(result, logger); @@ -148,7 +171,7 @@ export async function ensureThinkingTime( `${result.label ?? capitalizedLevel} is unavailable on this account (${result.notice ?? "no reason given"}); keeping the effort already selected in ChatGPT.`, ), ); - return; + return evidence("unverified", result.label ?? null); } case "chip-not-found": case "menu-not-found": @@ -176,7 +199,7 @@ export async function ensureThinkingTime( ? "the effort in ChatGPT is unconfirmed" : "keeping the effort already selected in ChatGPT"; logger(formatBrowserThinkingLog(`${message}; ${outcome}.`)); - return; + return evidence("unverified", null); } default: { await logDomFailure(Runtime, logger, "thinking-time-unknown"); @@ -192,7 +215,7 @@ export async function ensureThinkingTime( `unknown outcome selecting ${capitalizedLevel}; continuing with ChatGPT default.`, ), ); - return; + return evidence("unverified", null); } } } diff --git a/src/browser/index.ts b/src/browser/index.ts index cf970b129..f0b776394 100644 --- a/src/browser/index.ts +++ b/src/browser/index.ts @@ -56,7 +56,10 @@ import { } from "./actions/deepResearch.js"; import { estimateTokenCount, withRetries, delay } from "./utils.js"; import { formatElapsed } from "../oracle/format.js"; -import type { BrowserModelSelectionEvidence } from "../sessionStore.js"; +import type { + BrowserModelSelectionEvidence, + BrowserThinkingSelectionEvidence, +} from "../sessionStore.js"; import { CHATGPT_URL, DEFAULT_MODEL_STRATEGY } from "./constants.js"; import type { LaunchedChrome } from "chrome-launcher"; import { BrowserAutomationError } from "../oracle/errors.js"; @@ -951,6 +954,7 @@ export async function runBrowserMode(options: BrowserRunOptions): Promise => { @@ -1507,7 +1511,7 @@ export async function runBrowserMode(options: BrowserRunOptions): Promise ensureThinkingTime(Runtime, thinkingTime, logger, thinkingTargetModel), { retries: 2, delayMs: 300, @@ -1752,6 +1756,7 @@ export async function runBrowserMode(options: BrowserRunOptions): Promise ensureThinkingTime(Runtime, thinkingTime, logger, thinkingTargetModel), { retries: 2, @@ -3292,6 +3299,7 @@ async function runRemoteBrowserMode( artifacts: savedArtifacts, archive, modelSelection: modelSelectionEvidence, + thinkingSelection: thinkingSelectionEvidence, tookMs: durationMs, answerTokens: tokens, answerChars: researchResult.text.length, @@ -3772,6 +3780,7 @@ async function runRemoteBrowserMode( savedFiles: fileArtifacts.savedFiles, archive, modelSelection: modelSelectionEvidence, + thinkingSelection: thinkingSelectionEvidence, controllerPid: process.pid, }; } catch (error) { diff --git a/src/browser/modelDisplay.ts b/src/browser/modelDisplay.ts index f14e839bc..dca196388 100644 --- a/src/browser/modelDisplay.ts +++ b/src/browser/modelDisplay.ts @@ -1,4 +1,8 @@ -import type { BrowserModelSelectionEvidence, SessionMetadata } from "../sessionStore.js"; +import type { + BrowserModelSelectionEvidence, + BrowserThinkingSelectionEvidence, + SessionMetadata, +} from "../sessionStore.js"; import type { BrowserModelStrategy } from "./types.js"; interface BrowserModelDisplayInput { @@ -100,3 +104,22 @@ export function formatBrowserModelSelectionEvidence( const verified = evidence.verified ? "yes" : "no"; return `requestedKey=${requestedKey}; target=${target}; resolvedLabel=${resolvedLabel}; status=${evidence.status}; strategy=${strategy}; verified=${verified}; source=${evidence.source}; capturedAt=${evidence.capturedAt}`; } + +/** + * Renders thinking-effort evidence. Kept beside the model-selection formatter + * because the two answer different questions: the model formatter says WHICH + * model the picker resolved, this one says whether the requested effort tier was + * actually confirmed before submit. For Pro-capable models the model evidence + * cannot answer the second question — its resolvedLabel is the requested model + * string either way. + */ +export function formatBrowserThinkingSelectionEvidence( + evidence: BrowserThinkingSelectionEvidence, +): string { + const resolvedLabel = cleanLabel(evidence.resolvedLabel) ?? "(none)"; + const verified = evidence.verified ? "yes" : "no"; + const failClosed = evidence.strictFailClosed ? "yes" : "no"; + const targetKind = evidence.targetModelKind ?? "(none)"; + const observedKind = evidence.observedModelKind ?? "(none)"; + return `requestedLevel=${evidence.requestedLevel}; status=${evidence.status}; resolvedLabel=${resolvedLabel}; verified=${verified}; failClosed=${failClosed}; targetModelKind=${targetKind}; observedModelKind=${observedKind}; source=${evidence.source}; capturedAt=${evidence.capturedAt}`; +} diff --git a/src/browser/sessionRunner.ts b/src/browser/sessionRunner.ts index 6d2b1b5f0..246236421 100644 --- a/src/browser/sessionRunner.ts +++ b/src/browser/sessionRunner.ts @@ -4,6 +4,7 @@ import { formatTokenCount } from "../oracle/runUtils.js"; import { formatFinishLine } from "../oracle/finishLine.js"; import type { BrowserModelSelectionEvidence, + BrowserThinkingSelectionEvidence, BrowserRunWarning, BrowserSessionConfig, BrowserRuntimeMetadata, @@ -21,6 +22,7 @@ import { } from "./artifacts.js"; import { formatBrowserModelSelectionEvidence, + formatBrowserThinkingSelectionEvidence, formatBrowserModelTarget, resolveBrowserModelDisplayName, } from "./modelDisplay.js"; @@ -36,6 +38,7 @@ export interface BrowserExecutionResult { runtime: BrowserRuntimeMetadata; archive?: BrowserArchiveResult; modelSelection?: BrowserModelSelectionEvidence; + thinkingSelection?: BrowserThinkingSelectionEvidence; warnings?: BrowserRunWarning[]; answerText: string; artifacts?: SessionArtifact[]; @@ -246,6 +249,12 @@ export async function runBrowserSessionExecution( `[browser] Model selection evidence: ${formatBrowserModelSelectionEvidence(modelSelection, runOptions.model)}`, ); } + const thinkingSelection = browserResult.thinkingSelection; + if (thinkingSelection) { + log( + `[browser] Thinking effort evidence: ${formatBrowserThinkingSelectionEvidence(thinkingSelection)}`, + ); + } const warnings = buildBrowserRunWarnings({ runOptions, browserConfig, @@ -321,6 +330,7 @@ export async function runBrowserSessionExecution( }, archive: browserResult.archive, modelSelection, + thinkingSelection, warnings, answerText, artifacts: savedArtifacts, diff --git a/src/browser/types.ts b/src/browser/types.ts index 58c701a62..7c7e5a152 100644 --- a/src/browser/types.ts +++ b/src/browser/types.ts @@ -2,6 +2,7 @@ import type CDP from "chrome-remote-interface"; import type Protocol from "devtools-protocol"; import type { BrowserModelSelectionEvidence, + BrowserThinkingSelectionEvidence, BrowserRunWarning, BrowserRuntimeMetadata, } from "../sessionStore.js"; @@ -173,6 +174,7 @@ export interface BrowserRunResult { savedFiles?: SavedBrowserFile[]; archive?: BrowserArchiveResult; modelSelection?: BrowserModelSelectionEvidence; + thinkingSelection?: BrowserThinkingSelectionEvidence; warnings?: BrowserRunWarning[]; tookMs: number; answerTokens: number; diff --git a/src/cli/sessionDisplay.ts b/src/cli/sessionDisplay.ts index 8402c5890..e193ab4eb 100644 --- a/src/cli/sessionDisplay.ts +++ b/src/cli/sessionDisplay.ts @@ -33,6 +33,7 @@ import { import { formatSessionExecutionLabel } from "./sessionLifecycle.js"; import { formatBrowserModelSelectionEvidence, + formatBrowserThinkingSelectionEvidence, formatSessionBrowserModelWithRequestedKey, resolveSessionBrowserModelDisplayName, } from "../browser/modelDisplay.js"; @@ -373,6 +374,7 @@ export async function attachSession( config: metadata.browser?.config, runtime, modelSelection: metadata.browser?.modelSelection, + thinkingSelection: metadata.browser?.thinkingSelection, warnings: metadata.browser?.warnings, }, artifacts, @@ -743,7 +745,11 @@ export function formatUserErrorMetadata(metadata?: SessionUserErrorMetadata): st export function formatBrowserEvidence(metadata: SessionMetadata): string[] | null { const browser = metadata.browser; - if (!browser?.modelSelection && (!browser?.warnings || browser.warnings.length === 0)) { + if ( + !browser?.modelSelection && + !browser?.thinkingSelection && + (!browser?.warnings || browser.warnings.length === 0) + ) { return null; } const lines: string[] = []; @@ -751,6 +757,10 @@ export function formatBrowserEvidence(metadata: SessionMetadata): string[] | nul if (evidence) { lines.push(`model ${formatBrowserModelSelectionEvidence(evidence, metadata.model)}`); } + const thinkingEvidence = browser.thinkingSelection; + if (thinkingEvidence) { + lines.push(`effort ${formatBrowserThinkingSelectionEvidence(thinkingEvidence)}`); + } for (const warning of browser.warnings ?? []) { lines.push(`warning ${warning.code}: ${warning.message}`); } diff --git a/src/cli/sessionRunner.ts b/src/cli/sessionRunner.ts index 49074d875..5a96fb037 100644 --- a/src/cli/sessionRunner.ts +++ b/src/cli/sessionRunner.ts @@ -172,6 +172,7 @@ export async function performSessionRun({ runtime: result.runtime, archive: result.archive, modelSelection: result.modelSelection, + thinkingSelection: result.thinkingSelection, warnings: result.warnings, }, artifacts: mergeArtifacts(sessionMeta.artifacts, result.artifacts), diff --git a/src/remote/server.ts b/src/remote/server.ts index 397d9e354..2187ea1f4 100644 --- a/src/remote/server.ts +++ b/src/remote/server.ts @@ -723,6 +723,17 @@ function sanitizeName(raw: string): string { return raw.replace(/[^a-zA-Z0-9._-]/g, "_"); } +/** + * Whitelist rather than blacklist: a bridged result must never carry host detail + * (pids, ports, profile paths) to a client on another machine. + * + * The fields below are on the safe side of that line and are load-bearing for the + * caller. Selection evidence is the caller's only proof of WHICH model and effort + * answered their prompt — dropping it left a remote run indistinguishable from one + * that silently inherited whatever the composer had selected. The conversation + * identity is what binds an answer to a durable ChatGPT URL the caller can revisit; + * without it a bridged answer is unattributable. None of it describes the host. + */ function sanitizeResult( result: BrowserRunResult, warnings: BrowserRunWarning[] = [], @@ -734,6 +745,12 @@ function sanitizeResult( tookMs: result.tookMs, answerTokens: result.answerTokens, answerChars: result.answerChars, + modelSelection: result.modelSelection, + thinkingSelection: result.thinkingSelection, + archive: result.archive, + tabUrl: result.tabUrl, + conversationId: result.conversationId, + promptSubmitted: result.promptSubmitted, warnings: warnings.length > 0 ? warnings : undefined, chromePid: undefined, chromePort: undefined, diff --git a/src/sessionManager.ts b/src/sessionManager.ts index 26de6be19..3f3512151 100644 --- a/src/sessionManager.ts +++ b/src/sessionManager.ts @@ -137,6 +137,36 @@ export interface BrowserModelSelectionEvidence { capturedAt: string; } +export type BrowserThinkingSelectionStatus = "already-selected" | "switched" | "unverified"; + +/** + * Machine-checkable evidence that a requested thinking-effort tier was actually + * confirmed in ChatGPT's composer before the prompt was submitted. + * + * This exists because {@link BrowserModelSelectionEvidence} cannot carry it: for + * a Pro-capable model the picker deliberately reports the requested model string + * as the resolved label, so `resolvedLabel === requestedModel; verified: true` is + * byte-identical whether or not the Pro effort row was selected. Without a + * separate record, "this run answered at Pro effort" is unprovable after the fact. + * + * `verified` is true only for statuses that positively observed the option's + * selected state (aria-checked/aria-selected/data-state, or a composer pill whose + * label matches the target tier). `strictFailClosed` records that the run was in + * the fail-closed regime, where every non-confirming outcome throws before submit + * — so a submitted strict run is itself evidence that no degraded tier was used. + */ +export interface BrowserThinkingSelectionEvidence { + requestedLevel: ThinkingTimeLevel; + status: BrowserThinkingSelectionStatus; + resolvedLabel?: string | null; + verified: boolean; + strictFailClosed: boolean; + targetModelKind?: string | null; + observedModelKind?: string | null; + source: "chatgpt-thinking-picker"; + capturedAt: string; +} + export interface BrowserRunWarning { code: string; severity: "warning"; @@ -150,6 +180,7 @@ export interface BrowserMetadata { harvest?: BrowserHarvestMetadata; archive?: BrowserArchiveResult; modelSelection?: BrowserModelSelectionEvidence; + thinkingSelection?: BrowserThinkingSelectionEvidence; warnings?: BrowserRunWarning[]; } diff --git a/src/sessionStore.ts b/src/sessionStore.ts index ae95a400a..6eb6ef19b 100644 --- a/src/sessionStore.ts +++ b/src/sessionStore.ts @@ -143,6 +143,8 @@ export type { BrowserHarvestMetadata, BrowserModelSelectionEvidence, BrowserModelSelectionEvidenceStatus, + BrowserThinkingSelectionEvidence, + BrowserThinkingSelectionStatus, BrowserRunWarning, SessionTransportMetadata, SessionUserErrorMetadata, diff --git a/tests/browser/thinkingTime.test.ts b/tests/browser/thinkingTime.test.ts index 169d54cd3..45c4c4b74 100644 --- a/tests/browser/thinkingTime.test.ts +++ b/tests/browser/thinkingTime.test.ts @@ -327,6 +327,8 @@ describe("browser thinking-time selection expression", () => { }; const logs: string[] = []; + // Best-effort resolution still yields an evidence record, but one that + // refuses to claim the tier was confirmed. await expect( ensureThinkingTime( runtime as never, @@ -334,7 +336,13 @@ describe("browser thinking-time selection expression", () => { ((message: string) => logs.push(message)) as never, null, ), - ).resolves.toBeUndefined(); + ).resolves.toMatchObject({ + requestedLevel: "extended", + status: "unverified", + verified: false, + strictFailClosed: false, + source: "chatgpt-thinking-picker", + }); expect(logs.at(-1)).toContain("keeping the effort already selected in ChatGPT"); }); @@ -3489,7 +3497,7 @@ describe("unified Intelligence picker with Advanced -> Effort submenu", () => { ((line: string) => logs.push(line)) as never, null, ), - ).resolves.toBeUndefined(); + ).resolves.toMatchObject({ status: "unverified", verified: false }); expect(logs.join(" ")).toContain("Limit reached. Try again after Aug 16, 2026."); }); @@ -3543,3 +3551,92 @@ describe("unified Intelligence picker with Advanced -> Effort submenu", () => { expect(logged).not.toContain("continuing with default"); }); }); + +describe("thinking-effort selection evidence", () => { + // Why this exists: for a Pro-capable model the model picker reports the + // requested model string as the resolved label, so browser.modelSelection is + // byte-identical whether or not Pro effort was actually selected. Without this + // record, "the run answered at Pro effort" is unprovable after the fact. + it("records a verified record when the Pro row was already selected", async () => { + const runtime = { + evaluate: async () => ({ + result: { value: { status: "already-selected", label: "Pro", modelKind: "pro" } }, + }), + }; + const evidence = await ensureThinkingTime( + runtime as never, + "pro", + (() => {}) as never, + "gpt-5.6-sol", + ); + expect(evidence).toMatchObject({ + requestedLevel: "pro", + status: "already-selected", + resolvedLabel: "Pro", + verified: true, + strictFailClosed: true, + observedModelKind: "pro", + source: "chatgpt-thinking-picker", + }); + expect(Date.parse(evidence.capturedAt)).not.toBeNaN(); + }); + + it("records a verified record when the picker switched to Pro", async () => { + const runtime = { + evaluate: async () => ({ + result: { value: { status: "switched", label: "Pro" } }, + }), + }; + const evidence = await ensureThinkingTime( + runtime as never, + "pro", + (() => {}) as never, + "gpt-5.6-sol", + ); + expect(evidence).toMatchObject({ status: "switched", verified: true, strictFailClosed: true }); + }); + + it("never returns an unverified record for a strict Pro request", async () => { + // The whole point of fail-closed: a strict request either produces confirmed + // evidence or throws before submit. It must never resolve to verified:false, + // because a persisted unverified record would still read as "the run happened". + const degraded = [ + "option-disabled", + "chip-not-found", + "menu-not-found", + "option-not-found", + "selection-unverified", + "model-kind-not-found", + "unknown-status", + undefined, + ] as const; + for (const status of degraded) { + const runtime = { + evaluate: async () => ({ + result: { value: status === undefined ? undefined : { status } }, + }), + }; + await expect( + ensureThinkingTime(runtime as never, "pro", (() => {}) as never, "gpt-5.6-sol"), + ).rejects.toThrow(); + } + }); + + it("marks a non-strict degraded selection unverified rather than silently succeeding", async () => { + const runtime = { + evaluate: async () => ({ result: { value: { status: "selection-unverified" } } }), + }; + const evidence = await ensureThinkingTime( + runtime as never, + "standard", + (() => {}) as never, + null, + ); + expect(evidence).toMatchObject({ + requestedLevel: "standard", + status: "unverified", + verified: false, + strictFailClosed: false, + }); + }); +}); diff --git a/tests/remote/server.test.ts b/tests/remote/server.test.ts index 28a1f7b25..8e7c3c6c3 100644 --- a/tests/remote/server.test.ts +++ b/tests/remote/server.test.ts @@ -581,3 +581,77 @@ async function httpGetJson({ req.end(); }); } + +describe("bridged result sanitization", () => { + test.skipIf(!CAN_LISTEN_LOCALHOST)( + "carries selection evidence and conversation identity, never host detail", + async () => { + // Two properties in one test because they are the same decision seen from + // both sides: the whitelist must pass what makes a remote answer + // attributable, and must still refuse anything describing this machine. + const server = await createRemoteServer( + { host: "127.0.0.1", port: 0, token: "secret", logger: () => {} }, + { + runBrowser: async () => { + const result: BrowserRunResult = { + answerText: "hi", + answerMarkdown: "hi", + tookMs: 1, + answerTokens: 1, + answerChars: 2, + modelSelection: { + requestedModel: "gpt-5.6-sol", + resolvedLabel: "GPT-5.6 Sol", + strategy: "select", + status: "switched", + verified: true, + source: "chatgpt-model-picker", + capturedAt: "2026-08-18T00:00:00.000Z", + }, + thinkingSelection: { + requestedLevel: "pro", + status: "switched", + resolvedLabel: "Pro", + verified: true, + strictFailClosed: true, + source: "chatgpt-thinking-picker", + capturedAt: "2026-08-18T00:00:00.000Z", + }, + tabUrl: "https://chatgpt.com/c/abc-123", + conversationId: "abc-123", + promptSubmitted: true, + chromePid: 4242, + chromePort: 9222, + userDataDir: "/Users/someone/.oracle/browser-profile", + }; + return result; + }, + }, + ); + + const executor = createRemoteBrowserExecutor({ + host: `127.0.0.1:${server.port}`, + token: "secret", + }); + const result = await executor({ prompt: "remote", config: {} }); + + // Without these a bridged run cannot be proven to have answered at the + // requested model and effort, and its answer cannot be bound to a URL. + expect(result.thinkingSelection).toMatchObject({ + requestedLevel: "pro", + verified: true, + strictFailClosed: true, + }); + expect(result.modelSelection?.resolvedLabel).toBe("GPT-5.6 Sol"); + expect(result.conversationId).toBe("abc-123"); + expect(result.tabUrl).toBe("https://chatgpt.com/c/abc-123"); + + // Host detail stays on the host. + expect(result.chromePid).toBeUndefined(); + expect(result.chromePort).toBeUndefined(); + expect(result.userDataDir).toBeUndefined(); + + await server.close(); + }, + ); +});