From 308c80153b30a81dc8085d87c525032d3ae2903c Mon Sep 17 00:00:00 2001 From: Caleb Sowers Date: Tue, 18 Aug 2026 03:02:32 -0400 Subject: [PATCH 1/4] feat(browser): capture ChatGPT's own conversation record as evidence MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Oracle's answer capture is a rendering of what ChatGPT displayed. For most answers that is the same thing as the answer; for notation it is not. On a live Pro run in a ChatGPT project, the captured Markdown for a math-heavy turn differed from the provider's own record of that same turn: `\,` lost its backslash and `\mathcal{F}_s` came back as `\mathcal{F}*s`. Nothing failed, no fallback fired, and every keyword-level check passed — the text simply was not what the model wrote. `--browser-capture-provider-native` (off by default) additionally fetches `/backend-api/conversation/` from inside the authenticated page and saves two files beside the run's other artifacts: - the conversation document, verbatim — the bytes are written from the same string that was hashed, with no parse-and-reserialize in between; - an evidence file from a SECOND, independent fetch, normalized and hashed in the page so only digests cross the boundary. Node never sees the second body, so a mistake on this side cannot make the two agree by construction. The run's own answer is then compared to those digests, and the result recorded as matched / divergent / unknown. That replaces a length heuristic with the provider's bytes: "this transcript is the provider's text" becomes checkable rather than assumed. Deliberate limits: - Capture never gates an answer. `/backend-api/*` sits behind bot mitigation that can return 403 to an in-page fetch while the user is logged in, so every failure is a typed reason and a normal result. A conversation with no id — temporary chats, or a run whose URL never settled — is `unavailable`, not an error. - Document-level hashes of the two fetches are recorded, never gated: the backend document carries volatile nested metadata and can differ between fetches at identical turn content. The per-turn comparison is the load-bearing one. - The bearer token from /api/auth/session is used in the page and never returned or logged, per the existing note in navigation.ts. - Payload is drained in bounded chunks with a ceiling and a timeout, and `exceptionDetails` is checked, so an in-page throw is reported rather than collapsed into an empty result. The in-page normalization is checked byte-for-byte against the reference implementation it must agree with, over a fixture whose expected digests that reference produced — including the JSON-fallback branches where Python's sort_keys/ensure_ascii dumps and its int-vs-float rendering diverge from JSON.stringify. Claude-Session: https://claude.ai/code/session_01HsXirqcfqtr1Cae9zYCLDk --- src/browser/chatgptConversation.ts | 831 ++++++++++++++++++ src/browser/config.ts | 3 + src/browser/index.ts | 106 ++- src/browser/types.ts | 2 + src/cli/browserConfig.ts | 2 + src/config.ts | 7 + src/sessionManager.ts | 2 + tests/browser/chatgptConversation.test.ts | 128 +++ .../provider-conversation-normalization.json | 40 + 9 files changed, 1112 insertions(+), 9 deletions(-) create mode 100644 src/browser/chatgptConversation.ts create mode 100644 tests/browser/chatgptConversation.test.ts create mode 100644 tests/fixtures/provider-conversation-normalization.json diff --git a/src/browser/chatgptConversation.ts b/src/browser/chatgptConversation.ts new file mode 100644 index 000000000..dc6e961bd --- /dev/null +++ b/src/browser/chatgptConversation.ts @@ -0,0 +1,831 @@ +import { createHash } from "node:crypto"; +import path from "node:path"; +import { mkdir, writeFile } from "node:fs/promises"; +import type { SessionArtifact } from "../sessionManager.js"; +import { resolveSessionArtifactsDir, resolveUniqueArtifactPath } from "./artifacts.js"; +import type { BrowserLogger, ChromeClient } from "./types.js"; + +/** + * Provider-native conversation capture. + * + * Oracle's answer capture is a rendering of what ChatGPT displayed: copy-button + * Markdown when it works, DOM text when it does not. Neither is the provider's + * own record of the conversation, and for notation-heavy answers the difference + * is not cosmetic — rendered KaTeX loses the LaTeX source it was rendered from. + * + * This module fetches ChatGPT's own conversation document from + * `/backend-api/conversation/` inside the authenticated page, so a caller can + * hold the provider's bytes rather than a re-rendering of them. + * + * Two properties make the result usable as evidence rather than as a second + * opinion: + * + * A. The document is materialized verbatim — the page returns `response.text()` + * and those exact bytes are what gets written and hashed. Nothing is parsed + * and re-serialized on the way to disk. + * B. A second, independent fetch is normalized and hashed *in the page*, and + * only the digests come back. A Node-side mistake therefore cannot make B + * agree with A by construction, because Node never sees B's body. + * + * Document-level hashes of A and B are expected to differ: the backend document + * carries volatile nested metadata that changes between fetches at identical byte + * length and turn count. That is recorded, never gated. The per-turn comparison is + * the load-bearing one. + * + * Capture is best-effort by design and must never gate an answer. `/backend-api/*` + * sits behind bot mitigation that can return 403 to an in-page fetch while the + * user is perfectly well logged in, so every failure is typed and reported rather + * than thrown. + */ + +export type ProviderNativeFailureReason = + | "no-conversation-id" + | "auth-session-unavailable" + | "challenged" + | "http-error" + | "empty-document" + | "evaluate-failed" + | "digest-unavailable"; + +export interface ProviderNativeTurnDigest { + /** Position among non-system turns, in conversation order. */ + index: number; + role: string; + contentType: string; + /** UTF-8 byte length of the normalized turn body. */ + bytes: number; + /** SHA-256 of the normalized turn body, as decimal bytes. */ + sha256Decimal: number[]; +} + +export interface ProviderNativeCaptureFailure { + reason: ProviderNativeFailureReason; + detail?: string; + httpStatus?: number; +} + +export interface ProviderNativeCapture { + conversationId: string; + /** Verbatim bytes of fetch A. */ + rawText: string; + rawSha256: string; + rawBytes: number; + /** Digests derived in-page from the independent fetch B. */ + evidence: { + documentSha256Decimal: number[]; + documentBytes: number; + perTurn: ProviderNativeTurnDigest[]; + fetchedAt: string; + } | null; + evidenceFailure?: ProviderNativeCaptureFailure; + /** Recorded, never gated: the backend document mutates between fetches. */ + documentHashesMatch: boolean | null; +} + +export type ProviderNativeCaptureOutcome = + | { status: "captured"; capture: ProviderNativeCapture } + | { status: "unavailable"; failure: ProviderNativeCaptureFailure }; + +const STASH_KEY = "__oracleConversationCapture"; +const DRAIN_CHUNK_CHARS = 500_000; +/** + * Ceilings, not guesses about what Chrome will tolerate. A conversation document + * is normally well under a megabyte; anything past this is a sign the fetch + * returned something other than a conversation (a challenge page, an error body), + * and draining it would spend minutes proving that. + */ +const MAX_DOCUMENT_CHARS = 64 * 1024 * 1024; +const CAPTURE_TIMEOUT_MS = 120_000; + +async function withTimeout(operation: Promise, timeoutMs: number, label: string): Promise { + let timer: NodeJS.Timeout | undefined; + try { + return await Promise.race([ + operation, + new Promise((_resolve, reject) => { + timer = setTimeout( + () => reject(new Error(`${label} timed out after ${timeoutMs}ms`)), + timeoutMs, + ); + }), + ]); + } finally { + if (timer) clearTimeout(timer); + } +} + +/** + * Canonical turn normalization, ported from the downstream proof-grade + * normalizer so digests computed here are directly comparable to digests + * recomputed there from the same bytes. + * + * The JSON fallback branches must reproduce Python's + * `json.dumps(sort_keys=True, ensure_ascii=False)` exactly, which is why numbers + * carry their source literal through the parse: JSON `1.0` and `1` both become + * the JS number 1, but Python renders them `1.0` and `1`. A literal-aware parser + * keeps that distinction, and `pyNumber` applies Python's float repr rules on top + * (integral floats gain a trailing `.0`; exponent forms are left alone). + */ +function buildNormalizerSource(): string { + return ` + const PY_INT = Symbol.for('oracle.pyInt'); + const PY_FLOAT = Symbol.for('oracle.pyFloat'); + // Literal-preserving JSON parse: numbers become boxed values that remember + // whether the source literal was an integer or a float. + const parseJsonPreservingNumbers = (text) => { + let i = 0; + const err = (msg) => { throw new Error(msg + ' at ' + i); }; + const ws = () => { while (i < text.length && ' \\t\\n\\r'.includes(text[i])) i += 1; }; + const parseValue = () => { + ws(); + const ch = text[i]; + if (ch === '{') return parseObject(); + if (ch === '[') return parseArray(); + if (ch === '"') return parseString(); + if (ch === 't') { i += 4; return true; } + if (ch === 'f') { i += 5; return false; } + if (ch === 'n') { i += 4; return null; } + return parseNumber(); + }; + const parseObject = () => { + const out = {}; i += 1; ws(); + if (text[i] === '}') { i += 1; return out; } + for (;;) { + ws(); + if (text[i] !== '"') err('expected key'); + const key = parseString(); + ws(); + if (text[i] !== ':') err('expected colon'); + i += 1; + out[key] = parseValue(); + ws(); + if (text[i] === ',') { i += 1; continue; } + if (text[i] === '}') { i += 1; return out; } + err('expected , or }'); + } + }; + const parseArray = () => { + const out = []; i += 1; ws(); + if (text[i] === ']') { i += 1; return out; } + for (;;) { + out.push(parseValue()); + ws(); + if (text[i] === ',') { i += 1; continue; } + if (text[i] === ']') { i += 1; return out; } + err('expected , or ]'); + } + }; + const parseString = () => { + const start = i; i += 1; + while (i < text.length) { + const ch = text[i]; + if (ch === '\\\\') { i += 2; continue; } + if (ch === '"') { i += 1; return JSON.parse(text.slice(start, i)); } + i += 1; + } + err('unterminated string'); + }; + const parseNumber = () => { + const start = i; + while (i < text.length && '-+.eE0123456789'.includes(text[i])) i += 1; + const literal = text.slice(start, i); + if (!literal) err('expected value'); + const value = Number(literal); + const isFloat = /[.eE]/.test(literal); + return { [isFloat ? PY_FLOAT : PY_INT]: true, value, literal }; + }; + const result = parseValue(); + return result; + }; + const isBoxedNumber = (v) => v !== null && typeof v === 'object' && (v[PY_INT] === true || v[PY_FLOAT] === true); + // Python repr for a float: shortest round-trip, but always visibly a float. + const pyFloatRepr = (value) => { + if (!Number.isFinite(value)) return value > 0 ? 'Infinity' : (value < 0 ? '-Infinity' : 'NaN'); + const text = String(value); + return /[.eEn]/.test(text) ? text : text + '.0'; + }; + const pyNumber = (boxed) => { + if (boxed[PY_FLOAT] === true) return pyFloatRepr(boxed.value); + // Python int repr; normalizes JSON's permitted "-0". + return String(BigInt(boxed.literal)); + }; + // json.dumps(sort_keys=True, ensure_ascii=False): keys sorted by code unit, + // ", " between items and ": " after keys. + const pyDumps = (value) => { + if (value === null) return 'null'; + if (value === true) return 'true'; + if (value === false) return 'false'; + if (isBoxedNumber(value)) return pyNumber(value); + if (typeof value === 'string') return JSON.stringify(value); + if (Array.isArray(value)) return '[' + value.map(pyDumps).join(', ') + ']'; + if (typeof value === 'object') { + const keys = Object.keys(value).sort(); + return '{' + keys.map((k) => JSON.stringify(k) + ': ' + pyDumps(value[k])).join(', ') + '}'; + } + return 'null'; + }; + const plainString = (value) => (typeof value === 'string' ? value : ''); + // Content extraction, per content type. + const contentText = (content) => { + const contentType = typeof content?.content_type === 'string' ? content.content_type : 'text'; + if (contentType === 'text') { + const parts = Array.isArray(content.parts) ? content.parts : []; + return [contentType, parts.filter((p) => typeof p === 'string').join('\\n\\n')]; + } + if (contentType === 'code' || contentType === 'execution_output') { + return [contentType, plainString(content.text)]; + } + if (contentType === 'thoughts') { + const thoughts = Array.isArray(content.thoughts) ? content.thoughts : []; + const chunks = thoughts.map((thought) => { + if (thought !== null && typeof thought === 'object' && !Array.isArray(thought) && !isBoxedNumber(thought)) { + const inner = thought.content; + return inner === undefined ? 'None' : pyStr(inner); + } + return pyStr(thought); + }); + return [contentType, chunks.join('\\n\\n')]; + } + if (contentType === 'reasoning_recap') { + const inner = content.content; + return [contentType, inner ? pyStr(inner) : '']; + } + if (contentType === 'multimodal_text') { + const parts = Array.isArray(content.parts) ? content.parts : []; + const chunks = parts.map((part) => (typeof part === 'string' ? part : pyDumps(part))); + return [contentType, chunks.join('\\n\\n')]; + } + return [contentType, pyDumps(content)]; + }; + // Python str() for the scalar cases the normalizer can reach. + const pyStr = (value) => { + if (typeof value === 'string') return value; + if (value === null) return 'None'; + if (value === true) return 'True'; + if (value === false) return 'False'; + if (isBoxedNumber(value)) return pyNumber(value); + return pyDumps(value); + }; + // Conversation order: prefer the current-node chain, else the first root's + // first-child chain. + const nodeOrder = (document) => { + const mapping = document.mapping; + const current = document.current_node; + if (typeof current === 'string' && mapping[current]) { + const chain = []; + const seen = new Set(); + let nodeId = current; + while (nodeId && mapping[nodeId] && !seen.has(nodeId)) { + seen.add(nodeId); + chain.push(nodeId); + nodeId = mapping[nodeId].parent; + } + return chain.reverse(); + } + const roots = Object.keys(mapping).filter((key) => !mapping[key].parent); + if (roots.length === 0) throw new Error('backend-api mapping has no root node'); + const order = []; + const seen = new Set(); + let nodeId = roots[0]; + while (nodeId && !seen.has(nodeId)) { + seen.add(nodeId); + order.push(nodeId); + const children = mapping[nodeId].children || []; + nodeId = children.length > 0 ? children[0] : null; + } + return order; + }; + const normalizeTurns = (document) => { + if (!document || typeof document !== 'object' || !document.mapping) { + throw new Error('backend-api JSON has no mapping'); + } + const turns = []; + for (const nodeId of nodeOrder(document)) { + const node = document.mapping[nodeId]; + const message = node && node.message; + if (!message) continue; + const role = (message.author && typeof message.author.role === 'string') ? message.author.role : 'unknown'; + if (role === 'system') continue; + const [contentType, body] = contentText(message.content || {}); + turns.push({ index: turns.length, role, contentType, body }); + } + return turns; + }; + `; +} + +function buildAuthAndFetchSource(conversationId: string): string { + return ` + const conversationId = ${JSON.stringify(conversationId)}; + // The conversation endpoint needs the bearer token that /api/auth/session + // issues to the logged-in page. The token is used here and never returned. + const fetchConversationText = async () => { + const sessionResponse = await fetch('/api/auth/session', { credentials: 'include' }); + if (!sessionResponse.ok) { + return { ok: false, reason: 'auth-session-unavailable', httpStatus: sessionResponse.status }; + } + const session = await sessionResponse.json().catch(() => null); + const accessToken = session && typeof session.accessToken === 'string' ? session.accessToken : null; + if (!accessToken) { + return { ok: false, reason: 'auth-session-unavailable', detail: 'session carries no accessToken' }; + } + const response = await fetch('/backend-api/conversation/' + encodeURIComponent(conversationId), { + credentials: 'include', + headers: { Authorization: 'Bearer ' + accessToken, Accept: 'application/json' }, + }); + if (!response.ok) { + // Bot mitigation answers with an HTML challenge rather than JSON, and it + // means "retry later from a human-looking page", not "you are logged out". + const contentType = response.headers.get('content-type') || ''; + const challenged = response.status === 403 || contentType.includes('text/html'); + return { + ok: false, + reason: challenged ? 'challenged' : 'http-error', + httpStatus: response.status, + }; + } + const text = await response.text(); + if (!text) return { ok: false, reason: 'empty-document' }; + return { ok: true, text }; + }; + `; +} + +function buildFetchDocumentExpression(conversationId: string): string { + return `(async () => { + ${buildAuthAndFetchSource(conversationId)} + const result = await fetchConversationText(); + if (!result.ok) return result; + // Stashed rather than returned whole: a conversation document can be several + // megabytes, and one oversized evaluate response is a worse failure mode than + // a handful of bounded ones. + globalThis[${JSON.stringify(STASH_KEY)}] = result.text; + return { ok: true, length: result.text.length }; + })()`; +} + +function buildDrainExpression(offset: number): string { + return `(() => { + const stash = globalThis[${JSON.stringify(STASH_KEY)}]; + if (typeof stash !== 'string') return null; + return stash.slice(${offset}, ${offset + DRAIN_CHUNK_CHARS}); + })()`; +} + +function buildReleaseExpression(): string { + return `(() => { delete globalThis[${JSON.stringify(STASH_KEY)}]; return true; })()`; +} + +/** + * Normalize-and-digest, shared by the live evidence path and its test double. + * `sourceExpression` must evaluate to `{ok:true,text}` or a typed failure. + */ +function buildDigestSource(sourceExpression: string): string { + return ` + ${buildNormalizerSource()} + if (!globalThis.crypto || !globalThis.crypto.subtle || typeof globalThis.crypto.subtle.digest !== 'function') { + return { ok: false, reason: 'digest-unavailable' }; + } + const result = await (${sourceExpression}); + if (!result.ok) return result; + const encoder = new TextEncoder(); + const digestDecimal = async (value) => { + const bytes = encoder.encode(value); + const digest = await globalThis.crypto.subtle.digest('SHA-256', bytes); + return { digest: Array.from(new Uint8Array(digest)), bytes: bytes.length }; + }; + let document; + try { + document = parseJsonPreservingNumbers(result.text); + } catch (error) { + return { ok: false, reason: 'evaluate-failed', detail: String(error && error.message ? error.message : error) }; + } + let turns; + try { + turns = normalizeTurns(document); + } catch (error) { + return { ok: false, reason: 'evaluate-failed', detail: String(error && error.message ? error.message : error) }; + } + const perTurn = []; + for (const turn of turns) { + const hashed = await digestDecimal(turn.body); + perTurn.push({ + index: turn.index, + role: turn.role, + contentType: turn.contentType, + bytes: hashed.bytes, + sha256Decimal: hashed.digest, + }); + } + const documentDigest = await digestDecimal(result.text); + return { + ok: true, + documentSha256Decimal: documentDigest.digest, + documentBytes: documentDigest.bytes, + perTurn, + fetchedAt: new Date().toISOString(), + }; + `; +} + +/** + * Fetch B: independent, normalized and hashed without leaving the page. Only + * digests cross the boundary, so this cannot be an echo of fetch A. + */ +function buildEvidenceExpression(conversationId: string): string { + return `(async () => { + ${buildAuthAndFetchSource(conversationId)} + ${buildDigestSource("fetchConversationText()")} + })()`; +} + +/** + * The same normalization and hashing the page performs, over a caller-supplied + * document instead of a fetched one. Exists so the normalizer can be checked + * against the reference implementation it must agree with, without a browser. + */ +export function buildNormalizeAndDigestExpressionForTest(rawText: string): string { + return `(async () => { + ${buildDigestSource(`Promise.resolve({ ok: true, text: ${JSON.stringify(rawText)} })`)} + })()`; +} + +async function evaluateInPage( + Runtime: ChromeClient["Runtime"], + expression: string, + awaitPromise: boolean, +): Promise { + const evaluated = await withTimeout( + Runtime.evaluate({ expression, awaitPromise, returnByValue: true }), + CAPTURE_TIMEOUT_MS, + "in-page evaluation", + ); + const exception = (evaluated as { exceptionDetails?: { text?: string } }).exceptionDetails; + if (exception) { + throw new Error(exception.text ?? "in-page evaluation threw"); + } + return (evaluated.result?.value ?? null) as T | null; +} + +interface InPageFailure { + ok: false; + reason: ProviderNativeFailureReason; + detail?: string; + httpStatus?: number; +} + +function toFailure(value: InPageFailure): ProviderNativeCaptureFailure { + return { reason: value.reason, detail: value.detail, httpStatus: value.httpStatus }; +} + +export async function captureProviderNativeConversation(params: { + Runtime: ChromeClient["Runtime"]; + conversationId: string | null | undefined; + logger?: BrowserLogger; +}): Promise { + const { Runtime, logger } = params; + const conversationId = params.conversationId?.trim(); + if (!conversationId) { + return { status: "unavailable", failure: { reason: "no-conversation-id" } }; + } + + let head: ({ ok: true; length: number } | InPageFailure) | null; + try { + head = await evaluateInPage(Runtime, buildFetchDocumentExpression(conversationId), true); + } catch (error) { + return { + status: "unavailable", + failure: { + reason: "evaluate-failed", + detail: error instanceof Error ? error.message : String(error), + }, + }; + } + if (!head) { + return { status: "unavailable", failure: { reason: "evaluate-failed" } }; + } + if (!head.ok) { + return { status: "unavailable", failure: toFailure(head) }; + } + if (head.length > MAX_DOCUMENT_CHARS) { + await evaluateInPage(Runtime, buildReleaseExpression(), false).catch(() => null); + return { + status: "unavailable", + failure: { + reason: "http-error", + detail: `document of ${head.length} chars exceeds the ${MAX_DOCUMENT_CHARS}-char capture ceiling`, + }, + }; + } + + let rawText = ""; + try { + while (rawText.length < head.length) { + const chunk = await evaluateInPage( + Runtime, + buildDrainExpression(rawText.length), + false, + ); + if (chunk === null || chunk === "") { + break; + } + rawText += chunk; + } + } finally { + await evaluateInPage(Runtime, buildReleaseExpression(), false).catch(() => null); + } + + if (rawText.length !== head.length) { + return { + status: "unavailable", + failure: { + reason: "evaluate-failed", + detail: `document drained ${rawText.length} of ${head.length} chars`, + }, + }; + } + + const rawBuffer = Buffer.from(rawText, "utf8"); + const capture: ProviderNativeCapture = { + conversationId, + rawText, + rawSha256: createHash("sha256").update(rawBuffer).digest("hex"), + rawBytes: rawBuffer.byteLength, + evidence: null, + documentHashesMatch: null, + }; + + let evidence: + | ( + | { + ok: true; + documentSha256Decimal: number[]; + documentBytes: number; + perTurn: ProviderNativeTurnDigest[]; + fetchedAt: string; + } + | InPageFailure + ) + | null; + try { + evidence = await evaluateInPage(Runtime, buildEvidenceExpression(conversationId), true); + } catch (error) { + evidence = { + ok: false, + reason: "evaluate-failed", + detail: error instanceof Error ? error.message : String(error), + }; + } + + if (evidence && evidence.ok) { + capture.evidence = { + documentSha256Decimal: evidence.documentSha256Decimal, + documentBytes: evidence.documentBytes, + perTurn: evidence.perTurn, + fetchedAt: evidence.fetchedAt, + }; + const evidenceHex = Buffer.from(evidence.documentSha256Decimal).toString("hex"); + capture.documentHashesMatch = evidenceHex === capture.rawSha256; + if (!capture.documentHashesMatch) { + // Expected: the backend document carries volatile nested metadata. Logged + // so it is visible, recorded so it is auditable, never treated as failure. + logger?.( + `[capture] provider document hash differs between fetches (expected: volatile metadata); per-turn digests are the comparison that counts`, + ); + } + } else if (evidence) { + capture.evidenceFailure = toFailure(evidence); + } else { + capture.evidenceFailure = { reason: "evaluate-failed" }; + } + + return { status: "captured", capture }; +} + +/** + * What a run records about its own provider-native capture: enough to know + * whether proof-grade material exists and where, without carrying the material. + */ +/** + * How the run's own captured answer compares to the provider's record of it. + * + * `matched` means the answer Oracle captured is byte-identical to one of the + * turns the provider reports, verified against digests derived by the + * independent second fetch. `divergent` means it is not — which is not a failed + * run, but is a run whose transcript must not be treated as the provider's text. + * Notation is where this bites: a markdown round-trip that renders `_s` as `*s` + * or drops the escape in `\,` reads fine and is wrong. + */ +export type AnswerFidelity = "matched" | "divergent" | "unknown"; + +export interface ProviderNativeCaptureSummary { + status: "captured" | "unavailable"; + /** Whether the run's captured answer matches the provider's own bytes. */ + answerFidelity?: AnswerFidelity; + /** Which normalization of the captured answer matched, when one did. */ + answerMatch?: "exact" | "trimmed"; + conversationId?: string; + rawSha256?: string; + rawBytes?: number; + turnCount?: number; + /** Recorded, not gated — the backend document mutates between fetches. */ + documentHashesMatch?: boolean | null; + failure?: ProviderNativeCaptureFailure; + evidenceFailure?: ProviderNativeCaptureFailure; + capturedAt?: string; +} + +function decimalToHex(bytes: number[]): string { + return Buffer.from(bytes).toString("hex"); +} + +/** + * Compares the run's captured answer against the provider's turns by digest. + * + * Deliberately a digest membership test rather than a second normalizer: the + * digests come from the independent fetch, so a match is evidence the captured + * answer is the provider's bytes, and no second implementation of the + * normalization can drift away from the one that produced them. + */ +function compareAnswerToProviderTurns( + answerMarkdown: string | undefined, + perTurn: ProviderNativeTurnDigest[] | undefined, +): { fidelity: AnswerFidelity; match?: "exact" | "trimmed" } { + if (!answerMarkdown || !perTurn || perTurn.length === 0) { + return { fidelity: "unknown" }; + } + const digests = new Set(perTurn.map((turn) => decimalToHex(turn.sha256Decimal))); + const exact = createHash("sha256").update(Buffer.from(answerMarkdown, "utf8")).digest("hex"); + if (digests.has(exact)) { + return { fidelity: "matched", match: "exact" }; + } + // Transcript writers trim; a trailing newline is not a fidelity failure. + const trimmed = createHash("sha256") + .update(Buffer.from(answerMarkdown.trim(), "utf8")) + .digest("hex"); + if (digests.has(trimmed)) { + return { fidelity: "matched", match: "trimmed" }; + } + return { fidelity: "divergent" }; +} + +/** + * Captures the provider's own conversation document and writes it beside the + * run's other artifacts, along with the independently-derived digests. + * + * Two files rather than one, because they answer different questions and a + * downstream verifier must be able to tell them apart: the raw document is the + * material, the evidence file is the independent observation of it. Merging them + * would make the evidence self-certifying. + * + * Never throws. A run whose capture failed is still a run whose answer is + * perfectly good — it simply is not proof-grade, and says so. + */ +export async function finalizeProviderNativeCapture(params: { + Runtime: ChromeClient["Runtime"]; + conversationId: string | null | undefined; + conversationUrl?: string | null; + sessionId?: string; + /** The answer this run captured, for comparison against the provider's record. */ + answerMarkdown?: string; + logger?: BrowserLogger; +}): Promise<{ summary: ProviderNativeCaptureSummary; artifacts: SessionArtifact[] }> { + const { logger } = params; + let outcome: ProviderNativeCaptureOutcome; + try { + outcome = await captureProviderNativeConversation({ + Runtime: params.Runtime, + conversationId: params.conversationId, + logger, + }); + } catch (error) { + return { + summary: { + status: "unavailable", + failure: { + reason: "evaluate-failed", + detail: error instanceof Error ? error.message : String(error), + }, + }, + artifacts: [], + }; + } + + if (outcome.status === "unavailable") { + if (outcome.failure.reason !== "no-conversation-id") { + logger?.( + `[capture] Provider-native conversation capture unavailable (${outcome.failure.reason}); the answer is unaffected.`, + ); + } + return { summary: { status: "unavailable", failure: outcome.failure }, artifacts: [] }; + } + + const capture = outcome.capture; + const capturedAt = new Date().toISOString(); + const { fidelity, match } = compareAnswerToProviderTurns( + params.answerMarkdown, + capture.evidence?.perTurn, + ); + if (fidelity === "divergent") { + logger?.( + "[capture] The captured answer does not match any provider turn byte-for-byte; treat this transcript as a rendering, not as the provider's text.", + ); + } + const summary: ProviderNativeCaptureSummary = { + status: "captured", + answerFidelity: fidelity, + answerMatch: match, + conversationId: capture.conversationId, + rawSha256: capture.rawSha256, + rawBytes: capture.rawBytes, + turnCount: capture.evidence?.perTurn.length, + documentHashesMatch: capture.documentHashesMatch, + evidenceFailure: capture.evidenceFailure, + capturedAt, + }; + + if (!params.sessionId) { + return { summary, artifacts: [] }; + } + + const artifacts: SessionArtifact[] = []; + try { + const dir = resolveSessionArtifactsDir(params.sessionId); + await mkdir(dir, { recursive: true }); + + const rawPath = await resolveUniqueArtifactPath( + path.join(dir, `conversation-${capture.conversationId}-raw.json`), + ); + // Written from the same string that was hashed, so the file on disk is the + // thing the digest describes. + await writeFile(rawPath, capture.rawText, "utf8"); + artifacts.push({ + kind: "file", + path: rawPath, + label: "provider-native-conversation-raw", + mimeType: "application/json", + sizeBytes: capture.rawBytes, + sha256: capture.rawSha256, + sourceUrl: params.conversationUrl ?? undefined, + }); + + if (capture.evidence) { + const evidenceDocument = { + schema: "oracle.provider-native-capture-evidence/v1", + conversation_id: capture.conversationId, + chatgpt_url: params.conversationUrl ?? null, + captured_at: capturedAt, + fetched_at: capture.evidence.fetchedAt, + raw_backend_api_json: { + // False by construction: this file describes the SECOND fetch, whose + // body never left the page. The first fetch is the one on disk. + materialized_to_disk: false, + sha256_decimal_bytes: capture.evidence.documentSha256Decimal, + bytes: capture.evidence.documentBytes, + }, + materialized_document: { + path: path.basename(rawPath), + sha256: capture.rawSha256, + bytes: capture.rawBytes, + }, + document_hashes_match: capture.documentHashesMatch, + answer_fidelity: fidelity, + answer_match: match ?? null, + per_turn: capture.evidence.perTurn.map((turn) => ({ + i: turn.index, + role: turn.role, + ct: turn.contentType, + blen: turn.bytes, + sha256_dec: turn.sha256Decimal, + sha256_hex: decimalToHex(turn.sha256Decimal), + })), + }; + const evidencePath = await resolveUniqueArtifactPath( + path.join(dir, `conversation-${capture.conversationId}-evidence.json`), + ); + const serialized = `${JSON.stringify(evidenceDocument, null, 2)}\n`; + await writeFile(evidencePath, serialized, "utf8"); + artifacts.push({ + kind: "file", + path: evidencePath, + label: "provider-native-conversation-evidence", + mimeType: "application/json", + sizeBytes: Buffer.byteLength(serialized, "utf8"), + sha256: createHash("sha256").update(serialized).digest("hex"), + }); + } + logger?.( + `[capture] Provider-native conversation captured: ${capture.rawBytes} bytes, ${ + capture.evidence?.perTurn.length ?? 0 + } turns independently hashed.`, + ); + } catch (error) { + logger?.( + `[capture] Provider-native conversation captured but could not be written: ${ + error instanceof Error ? error.message : String(error) + }`, + ); + } + + return { summary, artifacts }; +} diff --git a/src/browser/config.ts b/src/browser/config.ts index b0e5c86eb..fda5c9adb 100644 --- a/src/browser/config.ts +++ b/src/browser/config.ts @@ -67,6 +67,7 @@ export const DEFAULT_BROWSER_CONFIG: ResolvedBrowserConfig = { researchMode: "off", archiveConversations: "auto", resumeConversationUrl: null, + captureProviderNative: false, }; export function resolveBrowserConfig( @@ -154,6 +155,8 @@ export function resolveBrowserConfig( archiveConversations, resumeConversationUrl: config?.resumeConversationUrl ?? DEFAULT_BROWSER_CONFIG.resumeConversationUrl, + captureProviderNative: + config?.captureProviderNative ?? DEFAULT_BROWSER_CONFIG.captureProviderNative, manualLogin, manualLoginProfileDir: manualLogin ? resolvedProfileDir : null, manualLoginCookieSync: diff --git a/src/browser/index.ts b/src/browser/index.ts index cf970b129..cccf8e3c2 100644 --- a/src/browser/index.ts +++ b/src/browser/index.ts @@ -47,6 +47,10 @@ import { import { INPUT_SELECTORS } from "./constants.js"; import { uploadAttachmentViaDataTransfer } from "./actions/remoteFileTransfer.js"; import { ensureThinkingTime } from "./actions/thinkingTime.js"; +import { + finalizeProviderNativeCapture, + type ProviderNativeCaptureSummary, +} from "./chatgptConversation.js"; import { startThinkingStatusMonitor } from "./actions/thinkingStatus.js"; import { activateDeepResearch, @@ -56,7 +60,7 @@ import { } from "./actions/deepResearch.js"; import { estimateTokenCount, withRetries, delay } from "./utils.js"; import { formatElapsed } from "../oracle/format.js"; -import type { BrowserModelSelectionEvidence } from "../sessionStore.js"; +import type { BrowserModelSelectionEvidence, SessionArtifact } from "../sessionStore.js"; import { CHATGPT_URL, DEFAULT_MODEL_STRATEGY } from "./constants.js"; import type { LaunchedChrome } from "chrome-launcher"; import { BrowserAutomationError } from "../oracle/errors.js"; @@ -896,6 +900,38 @@ function shouldCleanupBlankTabsAfterLastLease(options: { ); } +/** + * Provider-native capture, gated on explicit opt-in. + * + * Off by default because it costs two extra authenticated requests per run and + * only matters when a caller intends to treat the transcript as evidence rather + * than as an answer. When it is on and it fails, the run is unaffected: the + * summary records why, and nothing throws. + */ +async function runProviderNativeCapture(params: { + Runtime: ChromeClient["Runtime"]; + config: ResolvedBrowserConfig; + conversationUrl?: string | null; + sessionId?: string; + answerMarkdown?: string; + logger: BrowserLogger; +}): Promise<{ summary?: ProviderNativeCaptureSummary; artifacts: SessionArtifact[] }> { + if (!params.config.captureProviderNative) { + return { artifacts: [] }; + } + const conversationId = params.conversationUrl + ? extractConversationIdFromUrl(params.conversationUrl) + : undefined; + return finalizeProviderNativeCapture({ + Runtime: params.Runtime, + conversationId, + conversationUrl: params.conversationUrl, + sessionId: params.sessionId, + answerMarkdown: params.answerMarkdown, + logger: params.logger, + }); +} + function buildSkippedModelSelectionEvidence( desiredModel: string | null | undefined, strategy: BrowserModelSelectionEvidence["strategy"], @@ -1724,6 +1760,14 @@ export async function runBrowserMode(options: BrowserRunOptions): Promise saveBrowserTranscriptArtifact({ @@ -1731,12 +1775,18 @@ export async function runBrowserMode(options: BrowserRunOptions): Promise saveBrowserTranscriptArtifact({ @@ -2236,12 +2298,12 @@ export async function runBrowserMode(options: BrowserRunOptions): Promise saveBrowserTranscriptArtifact({ @@ -3270,12 +3340,18 @@ async function runRemoteBrowserMode( prompt: promptText, answerMarkdown: researchResult.text, conversationUrl: lastUrl, - artifacts: appendArtifacts(undefined, [reportArtifact]), + artifacts: appendArtifacts( + appendArtifacts(undefined, [reportArtifact]), + providerCapture.artifacts, + ), logger, }), logger, ); - const savedArtifacts = appendArtifacts(undefined, [reportArtifact, transcriptArtifact]); + const savedArtifacts = appendArtifacts( + appendArtifacts(undefined, [reportArtifact, transcriptArtifact]), + providerCapture.artifacts, + ); const archive = await maybeArchiveCompletedConversation({ Runtime, logger, @@ -3718,6 +3794,18 @@ async function runRemoteBrowserMode( }); const savedImageArtifacts = appendArtifacts(undefined, imageArtifacts.savedImages); const savedBrowserArtifacts = appendArtifacts(savedImageArtifacts, fileArtifacts.savedFiles); + const providerCapture = await runProviderNativeCapture({ + Runtime, + config, + conversationUrl: lastUrl, + sessionId: options.sessionId, + answerMarkdown, + logger, + }); + const browserArtifactsWithCapture = appendArtifacts( + savedBrowserArtifacts, + providerCapture.artifacts, + ); const transcriptArtifact = await saveOptionalArtifact( () => saveBrowserTranscriptArtifact({ @@ -3725,12 +3813,12 @@ async function runRemoteBrowserMode( prompt: promptText, answerMarkdown, conversationUrl: lastUrl, - artifacts: savedBrowserArtifacts, + artifacts: browserArtifactsWithCapture, logger, }), logger, ); - const savedArtifacts = appendArtifacts(savedBrowserArtifacts, [transcriptArtifact]); + const savedArtifacts = appendArtifacts(browserArtifactsWithCapture, [transcriptArtifact]); const archive = await maybeArchiveCompletedConversation({ Runtime, logger, diff --git a/src/browser/types.ts b/src/browser/types.ts index 58c701a62..9446d54fa 100644 --- a/src/browser/types.ts +++ b/src/browser/types.ts @@ -118,6 +118,8 @@ export interface BrowserAutomationConfig { archiveConversations?: BrowserArchiveMode; /** Existing ChatGPT conversation URL to open before submitting the prompt. */ resumeConversationUrl?: string | null; + /** Capture ChatGPT's own conversation document plus independent per-turn digests. */ + captureProviderNative?: boolean; } export interface BrowserRunOptions { diff --git a/src/cli/browserConfig.ts b/src/cli/browserConfig.ts index 47fe96b9c..a41513c59 100644 --- a/src/cli/browserConfig.ts +++ b/src/cli/browserConfig.ts @@ -92,6 +92,7 @@ export interface BrowserFlagOptions { browserThinkingTime?: ThinkingTimeLevel; browserResearch?: BrowserResearchMode; browserArchive?: BrowserArchiveMode; + browserCaptureProviderNative?: boolean; browserModelLabel?: string; /** Original model request before browser alias normalization. */ browserRequestedModel?: ModelName; @@ -328,6 +329,7 @@ export async function buildBrowserConfig( thinkingTime, researchMode: options.browserResearch === "deep" ? "deep" : "off", archiveConversations: options.browserArchive, + captureProviderNative: options.browserCaptureProviderNative, }; } diff --git a/src/config.ts b/src/config.ts index dada15544..c87c704f7 100644 --- a/src/config.ts +++ b/src/config.ts @@ -71,6 +71,13 @@ export interface BrowserConfigDefaults { manualLoginProfileDir?: string | null; /** Seed a manual-login profile from configured Chrome/inline cookies. */ manualLoginCookieSync?: boolean; + /** + * Also fetch ChatGPT's own conversation document and an independent set of + * per-turn digests, saved beside the run's other artifacts. Off by default: + * it costs two extra authenticated requests and only matters when the + * transcript is meant to be evidence rather than an answer. + */ + captureProviderNative?: boolean; } export interface AzureConfig { diff --git a/src/sessionManager.ts b/src/sessionManager.ts index 26de6be19..bb80ae023 100644 --- a/src/sessionManager.ts +++ b/src/sessionManager.ts @@ -85,6 +85,8 @@ export interface BrowserSessionConfig { archiveConversations?: BrowserArchiveMode; /** Browser-only: existing ChatGPT conversation URL to resume before submitting. */ resumeConversationUrl?: string | null; + /** Capture ChatGPT's own conversation document plus independent per-turn digests. */ + captureProviderNative?: boolean; } export interface BrowserRuntimeMetadata { diff --git a/tests/browser/chatgptConversation.test.ts b/tests/browser/chatgptConversation.test.ts new file mode 100644 index 000000000..36279d605 --- /dev/null +++ b/tests/browser/chatgptConversation.test.ts @@ -0,0 +1,128 @@ +import { describe, expect, it } from "vitest"; +import { readFile } from "node:fs/promises"; +import path from "node:path"; +import { + buildNormalizeAndDigestExpressionForTest, + captureProviderNativeConversation, +} from "../../src/browser/chatgptConversation.js"; + +interface FixtureTurn { + index: number; + role: string; + content_type: string; + bytes: number; + sha256: string; +} + +async function evaluateInNode(expression: string): Promise { + // The capture expression is written to run in the page. Node provides the same + // primitives it depends on (TextEncoder, crypto.subtle), so the normalizer can + // be checked here without a browser. + return await (0, eval)(expression); +} + +describe("provider conversation normalization", () => { + it("reproduces the reference normalizer byte-for-byte across every content type", async () => { + // The expected digests in this fixture were produced by the downstream + // proof-grade normalizer itself, not by this implementation. That is the + // point: these two must agree, and only one of them is authoritative. + // + // The awkward cases are deliberate. `multimodal_text` and unknown content + // types fall back to Python's json.dumps(sort_keys=True, ensure_ascii=False), + // which writes ", " and ": " separators, sorts keys, leaves non-ASCII + // unescaped, and — the part JSON.parse destroys — renders 1.0 as "1.0" and 1 + // as "1". A literal-preserving parse is what keeps those apart. + const fixturePath = path.join( + process.cwd(), + "tests/fixtures/provider-conversation-normalization.json", + ); + const fixture = JSON.parse(await readFile(fixturePath, "utf8")) as { + raw: string; + expected: FixtureTurn[]; + }; + + const result = (await evaluateInNode( + buildNormalizeAndDigestExpressionForTest(fixture.raw), + )) as { + ok: boolean; + perTurn: { + index: number; + role: string; + contentType: string; + bytes: number; + sha256Decimal: number[]; + }[]; + }; + + expect(result.ok).toBe(true); + expect(result.perTurn).toHaveLength(fixture.expected.length); + for (const [position, expected] of fixture.expected.entries()) { + const actual = result.perTurn[position]; + expect({ + index: actual?.index, + role: actual?.role, + content_type: actual?.contentType, + bytes: actual?.bytes, + sha256: Buffer.from(actual?.sha256Decimal ?? []).toString("hex"), + }).toEqual(expected); + } + }); + + it("skips system turns and follows the current-node chain", async () => { + const fixturePath = path.join( + process.cwd(), + "tests/fixtures/provider-conversation-normalization.json", + ); + const fixture = JSON.parse(await readFile(fixturePath, "utf8")) as { + raw: string; + expected: FixtureTurn[]; + }; + const document = JSON.parse(fixture.raw) as { mapping: Record }; + // Six nodes carry messages; one of them is the system turn that must not + // appear, and the root carries none. + expect(Object.keys(document.mapping)).toHaveLength(fixture.expected.length + 2); + expect(fixture.expected.some((turn) => turn.role === "system")).toBe(false); + }); +}); + +describe("provider capture failure handling", () => { + it("treats a conversation with no id as a normal unavailable result, not an error", async () => { + const outcome = await captureProviderNativeConversation({ + Runtime: { + evaluate: async () => { + throw new Error("should never be called"); + }, + } as never, + conversationId: undefined, + }); + expect(outcome).toEqual({ status: "unavailable", failure: { reason: "no-conversation-id" } }); + }); + + it("reports a bot-mitigation challenge as its own reason rather than a generic failure", async () => { + // A 403 here means "retry later from a page that looks human", not "you are + // logged out" — and it must never be mistaken for a failed run. + const outcome = await captureProviderNativeConversation({ + Runtime: { + evaluate: async () => ({ + result: { value: { ok: false, reason: "challenged", httpStatus: 403 } }, + }), + } as never, + conversationId: "abc-123", + }); + expect(outcome).toEqual({ + status: "unavailable", + failure: { reason: "challenged", detail: undefined, httpStatus: 403 }, + }); + }); + + it("surfaces an in-page exception instead of silently returning nothing", async () => { + const outcome = await captureProviderNativeConversation({ + Runtime: { + evaluate: async () => ({ exceptionDetails: { text: "TypeError: boom" }, result: {} }), + } as never, + conversationId: "abc-123", + }); + expect(outcome.status).toBe("unavailable"); + expect(outcome).toMatchObject({ failure: { reason: "evaluate-failed" } }); + }); +}); diff --git a/tests/fixtures/provider-conversation-normalization.json b/tests/fixtures/provider-conversation-normalization.json new file mode 100644 index 000000000..df5d78cf6 --- /dev/null +++ b/tests/fixtures/provider-conversation-normalization.json @@ -0,0 +1,40 @@ +{ + "raw": "{\"title\": \"Fidelity fixture\", \"conversation_id\": \"fixture-0001\", \"current_node\": \"n5\", \"mapping\": {\"root\": {\"id\": \"root\", \"parent\": null, \"children\": [\"n0\"], \"message\": null}, \"n0\": {\"id\": \"n0\", \"parent\": \"root\", \"children\": [\"n1\"], \"message\": {\"author\": {\"role\": \"system\"}, \"content\": {\"content_type\": \"text\", \"parts\": [\"ignored\"]}}}, \"n1\": {\"id\": \"n1\", \"parent\": \"n0\", \"children\": [\"n2\"], \"message\": {\"author\": {\"role\": \"user\"}, \"content\": {\"content_type\": \"multimodal_text\", \"parts\": [{\"content_type\": \"image_asset_pointer\", \"asset_pointer\": \"file-service://file-abc\", \"size_bytes\": 123456, \"width\": 1024, \"height\": 768, \"fovea\": null, \"metadata\": {\"dalle\": null, \"gizmo\": null}, \"downsampled\": 1.0, \"ratio\": 0.75, \"huge\": 1e+30}, \"Compare $\\\\Delta(G)$ against $\\\\chi'(G)$ — see attachment ≤ ∞.\"]}}}, \"n2\": {\"id\": \"n2\", \"parent\": \"n1\", \"children\": [\"n3\"], \"message\": {\"author\": {\"role\": \"assistant\"}, \"content\": {\"content_type\": \"thoughts\", \"thoughts\": [{\"summary\": \"s\", \"content\": \"First thought ∀ε>0\"}, {\"summary\": \"t\", \"content\": \"Second thought\"}]}}}, \"n3\": {\"id\": \"n3\", \"parent\": \"n2\", \"children\": [\"n4\"], \"message\": {\"author\": {\"role\": \"assistant\"}, \"content\": {\"content_type\": \"code\", \"language\": \"python\", \"text\": \"def f(x):\\n return x ** 2 # $ and \\\\ and `\"}}}, \"n4\": {\"id\": \"n4\", \"parent\": \"n3\", \"children\": [\"n5\"], \"message\": {\"author\": {\"role\": \"assistant\"}, \"content\": {\"content_type\": \"reasoning_recap\", \"content\": \"Recapped ⌈n⌉\"}}}, \"n5\": {\"id\": \"n5\", \"parent\": \"n4\", \"children\": [], \"message\": {\"author\": {\"role\": \"assistant\"}, \"content\": {\"content_type\": \"some_future_type\", \"payload\": {\"z\": 1, \"a\": [1, 2.0, true, null], \"m\": \"unicode ≡ ok\"}, \"n\": 7}}}}}", + "expected": [ + { + "index": 0, + "role": "user", + "content_type": "multimodal_text", + "bytes": 312, + "sha256": "c340b1301608951c3c9f30fcfcb3549988168397c20599e7c99312c40c8680b0" + }, + { + "index": 1, + "role": "assistant", + "content_type": "thoughts", + "bytes": 37, + "sha256": "0da6f512884d7b4f90e33fa87821c389f486064039183d835939ba805f52c9e1" + }, + { + "index": 2, + "role": "assistant", + "content_type": "code", + "bytes": 44, + "sha256": "700f20751ff52acebe46193c1cc4d84488b4231fb72d00b5e0e595810e39fed3" + }, + { + "index": 3, + "role": "assistant", + "content_type": "reasoning_recap", + "bytes": 16, + "sha256": "7cbbfb8e46caf3661d65b2c94dd78c20ec2e5a279aab31e641d903560262bfa9" + }, + { + "index": 4, + "role": "assistant", + "content_type": "some_future_type", + "bytes": 115, + "sha256": "f1831bf03183b036297da807f85b6ad26563b06e3e556442d43286666db31929" + } + ] +} From 956d386269c1df998fd5ab197d1da18177b55867 Mon Sep 17 00:00:00 2001 From: Caleb Sowers Date: Tue, 18 Aug 2026 03:10:26 -0400 Subject: [PATCH 2/4] fix(browser): describe the materialized document in capture evidence, not the second fetch MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The evidence file reported the second fetch's document hash as though it described the raw JSON sitting next to it. Verifiers use that field to confirm the file on disk was not altered between capture and ingest, so pointing it at a different fetch made the check fail for a reason unrelated to what it tests — and fail intermittently, since the two fetches sometimes agree and sometimes do not. Both Quiet conversations were captured twice while working on this: the first pass produced matching document hashes for one and differing hashes for the other; the second pass produced differing hashes for both, over identical turn content. That is the nested-metadata volatility this format already expects, and it is exactly why document-level equality is a poor fidelity criterion. `raw_backend_api_json` now describes the document actually on disk. The second fetch moves to its own `independent_fetch` block, where its document hash is a volatility record rather than a criterion, and its per-turn digests remain what the fidelity comparison is built on. Claude-Session: https://claude.ai/code/session_01HsXirqcfqtr1Cae9zYCLDk --- src/browser/chatgptConversation.ts | 29 ++++++++++++++++++++++++----- 1 file changed, 24 insertions(+), 5 deletions(-) diff --git a/src/browser/chatgptConversation.ts b/src/browser/chatgptConversation.ts index dc6e961bd..2a2cc1279 100644 --- a/src/browser/chatgptConversation.ts +++ b/src/browser/chatgptConversation.ts @@ -639,6 +639,10 @@ function decimalToHex(bytes: number[]): string { return Buffer.from(bytes).toString("hex"); } +function hexToDecimalString(hex: string): string { + return Array.from(Buffer.from(hex, "hex")).join(" "); +} + /** * Compares the run's captured answer against the provider's turns by digest. * @@ -777,11 +781,23 @@ export async function finalizeProviderNativeCapture(params: { captured_at: capturedAt, fetched_at: capture.evidence.fetchedAt, raw_backend_api_json: { - // False by construction: this file describes the SECOND fetch, whose - // body never left the page. The first fetch is the one on disk. + // This block describes the document this evidence accompanies — the + // one on disk — so a verifier can confirm the file was not altered + // between capture and ingest. materialized_to_disk: false, - sha256_decimal_bytes: capture.evidence.documentSha256Decimal, - bytes: capture.evidence.documentBytes, + sha256_decimal_bytes: hexToDecimalString(capture.rawSha256), + bytes: capture.rawBytes, + }, + // The independent second fetch, kept separate on purpose. Its per-turn + // digests are the evidence; its document hash is only a volatility + // record. Document-level equality is NOT a fidelity criterion: the + // backend document carries nested metadata that changes between fetches + // at identical turn content, so gating on it would fail honest captures + // and pass nothing extra. + independent_fetch: { + document_sha256_decimal_bytes: capture.evidence.documentSha256Decimal.join(" "), + document_bytes: capture.evidence.documentBytes, + fetched_at: capture.evidence.fetchedAt, }, materialized_document: { path: path.basename(rawPath), @@ -796,7 +812,10 @@ export async function finalizeProviderNativeCapture(params: { role: turn.role, ct: turn.contentType, blen: turn.bytes, - sha256_dec: turn.sha256Decimal, + // Space-separated decimal bytes: the transport-safe encoding verifiers + // of this format expect, and one that survives copy/paste through + // channels that mangle hex or JSON arrays. + sha256_dec: turn.sha256Decimal.join(" "), sha256_hex: decimalToHex(turn.sha256Decimal), })), }; From c5410e122411cf89e4b4ecdc0095d3aaa9d5b705 Mon Sep 17 00:00:00 2001 From: Caleb Sowers Date: Tue, 18 Aug 2026 04:22:03 -0400 Subject: [PATCH 3/4] feat(cli): expose --browser-capture-provider-native The flag's plumbing landed with the capture feature but its option registration did not, so the config field existed and nothing could set it from the command line. Belongs squashed into the capture commit before this goes upstream. Claude-Session: https://claude.ai/code/session_01HsXirqcfqtr1Cae9zYCLDk --- bin/oracle-cli.ts | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/bin/oracle-cli.ts b/bin/oracle-cli.ts index 491df6156..3390f1f4b 100755 --- a/bin/oracle-cli.ts +++ b/bin/oracle-cli.ts @@ -152,6 +152,7 @@ interface CliOptions extends OptionValues { browserManualLoginProfileDir?: string; copyProfile?: string; browserThinkingTime?: "light" | "standard" | "extended" | "extra-high" | "pro" | "heavy"; + browserCaptureProviderNative?: boolean; browserResearch?: "off" | "deep"; browserFollowUp?: string[]; browserAllowCookieErrors?: boolean; @@ -819,6 +820,10 @@ program "Browser research mode: deep activates ChatGPT Deep Research.", ).choices(["off", "deep"]), ) + .option( + "--browser-capture-provider-native", + "Also save ChatGPT's own conversation document plus an independent set of per-turn digests alongside the run's artifacts, for runs whose transcript must be evidence rather than an answer.", + ) .addOption( new Option( "--browser-archive ", From 09dc763714fe16df4ef055d0c42ea6dac5b7f789 Mon Sep 17 00:00:00 2001 From: Caleb Sowers Date: Tue, 18 Aug 2026 14:58:25 -0400 Subject: [PATCH 4/4] docs: changelog for provider-native conversation capture --- CHANGELOG.md | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/CHANGELOG.md b/CHANGELOG.md index 9edf60788..46f934ced 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,5 +1,11 @@ # Changelog +## Unreleased + +### Added + +- Browser: `--browser-capture-provider-native` saves ChatGPT's own record of the conversation alongside the run's artifacts, for runs whose transcript has to be evidence rather than an answer. Oracle's answer capture is a rendering of what ChatGPT displayed, and for notation that is not the same thing: on a live Pro run the captured Markdown for a math-heavy turn differed from the provider's record of that same turn — `\,` lost its backslash and `\mathcal{F}_s` came back as `\mathcal{F}*s` — with no fallback fired and every keyword-level check passing. Two files are written: the conversation document verbatim, and an evidence file from a second, independent fetch normalized and hashed **in the page** so only digests cross the boundary. The run's own answer is then compared against those digests and recorded as matched / divergent / unknown. Capture never gates an answer: `/backend-api/*` sits behind bot mitigation that can refuse an in-page fetch while the user is logged in, so every failure is a typed reason and a normal result, and a conversation with no id (a temporary chat, or a run whose URL never settled) is `unavailable` rather than an error. Document-level hashes of the two fetches are recorded but never gated — the backend document carries nested metadata that changes between fetches at identical turn content. + ## 0.18.0 — 2026-08-14 ### Changed @@ -10,6 +16,7 @@ ### Fixed - Browser: detect a disabled ChatGPT effort tier (e.g. an exhausted Pro allotment) before clicking it, and report the account's own reset notice instead of a misleading "selection unverified" failure. Thanks @enieuwy! + ## 0.17.3 — 2026-08-13 **Highlight:** browser-mode answers and recovery are reliable again — no more