diff --git a/packages/core/src/agent/model-input-recorder.ts b/packages/core/src/agent/model-input-recorder.ts new file mode 100644 index 0000000000..fd1ef37edd --- /dev/null +++ b/packages/core/src/agent/model-input-recorder.ts @@ -0,0 +1,62 @@ +import { Buffer } from 'node:buffer'; +import type { ModelRuntime } from '@/ai-model/models'; +import { ScreenshotItem } from '@/screenshot-item'; +import type { ExecutionTask } from '@/types'; +import { parseBase64 } from '@midscene/shared/img'; +import { sha256Hex } from '@midscene/shared/utils'; + +const MODEL_INPUT_TIMING = 'model-input'; + +function screenshotContentHash(imageBase64: string): string { + const { body } = parseBase64(imageBase64); + return sha256Hex(Buffer.from(body, 'base64')); +} + +/** + * Bind a model runtime to one report task so the report retains the exact + * data-URI bytes passed to the provider after padding/cropping/resizing. + */ +export function recordModelInputsForTask( + modelRuntime: ModelRuntime, + task: ExecutionTask, + sourceScreenshot?: ScreenshotItem, +): ModelRuntime { + return { + ...modelRuntime, + onModelInputImages: (images) => { + modelRuntime.onModelInputImages?.(images); + + for (const [index, imageBase64] of images.entries()) { + if (!imageBase64.startsWith('data:image/')) { + continue; + } + + const contentHash = screenshotContentHash(imageBase64); + const alreadyRecorded = task.recorder?.some( + (item) => + item.timing === MODEL_INPUT_TIMING && + item.screenshot && + screenshotContentHash(item.screenshot.base64) === contentHash, + ); + if (alreadyRecorded) { + continue; + } + + const uiScreenshot = task.uiContext?.screenshot ?? sourceScreenshot; + const screenshot = + uiScreenshot && + screenshotContentHash(uiScreenshot.base64) === contentHash + ? uiScreenshot + : ScreenshotItem.create(imageBase64, Date.now()); + const recorderItem = { + type: 'screenshot' as const, + ts: Date.now(), + screenshot, + timing: MODEL_INPUT_TIMING, + description: `Model input ${index + 1} (exact bytes, sha256: ${contentHash})`, + }; + task.recorder = [...(task.recorder ?? []), recorderItem]; + } + }, + }; +} diff --git a/packages/core/src/agent/task-builder.ts b/packages/core/src/agent/task-builder.ts index d8cdb71e46..790b489ba2 100644 --- a/packages/core/src/agent/task-builder.ts +++ b/packages/core/src/agent/task-builder.ts @@ -25,6 +25,7 @@ import { sleep } from '@/utils'; import { generateElementByRect } from '@midscene/shared/extractor'; import { getDebug } from '@midscene/shared/logger'; import { assert } from '@midscene/shared/utils'; +import { recordModelInputsForTask } from './model-input-recorder'; import type { TaskCache } from './task-cache'; import { withUsageIntent } from './usage-intent'; import { @@ -554,7 +555,11 @@ export class TaskBuilder { context: uiContext, planLocatedElement, }, - defaultModel, + recordModelInputsForTask( + defaultModel, + task, + uiContext.screenshot, + ), abortSignal, ); applyDump(locateResult.dump); diff --git a/packages/core/src/agent/tasks.ts b/packages/core/src/agent/tasks.ts index 3821f9d28a..592dd5f492 100644 --- a/packages/core/src/agent/tasks.ts +++ b/packages/core/src/agent/tasks.ts @@ -38,6 +38,7 @@ import { ServiceError, aiActProgressScope } from '@/types'; import { getDebug } from '@midscene/shared/logger'; import { assert } from '@midscene/shared/utils'; import { ExecutionSession } from './execution-session'; +import { recordModelInputsForTask } from './model-input-recorder'; import { type AgentProgressPublisher, createAiActActionReporter, @@ -587,7 +588,11 @@ export class TaskExecutor { context: planningUiContext, actionContext: param.aiActContext, actionSpace, - modelRuntime: planningModel, + modelRuntime: recordModelInputsForTask( + planningModel, + executorContext.task, + planningUiContext.screenshot, + ), conversationHistory, includeLocateInPlanning, imagesIncludeCount, @@ -917,7 +922,7 @@ export class TaskExecutor { try { extractResult = await this.service.extract( demandInput, - modelRuntime, + recordModelInputsForTask(modelRuntime, task, uiContext.screenshot), opt, extraPageDescription, multimodalPrompt, diff --git a/packages/core/src/ai-model/model-adapter/types.ts b/packages/core/src/ai-model/model-adapter/types.ts index bae75251fd..d93953cee9 100644 --- a/packages/core/src/ai-model/model-adapter/types.ts +++ b/packages/core/src/ai-model/model-adapter/types.ts @@ -245,6 +245,8 @@ export interface ModelRuntime { * such as order-sensitive judging and deep-locate search-area calls). */ onUsage?: (usage: AIUsageInfo) => void; + /** Exact image URLs included in a model request, after all preprocessing. */ + onModelInputImages?: (images: readonly string[]) => void; } export interface ModelAdapterDefinition { diff --git a/packages/core/src/ai-model/service-caller/index.ts b/packages/core/src/ai-model/service-caller/index.ts index 1c02963008..71312d6ffd 100644 --- a/packages/core/src/ai-model/service-caller/index.ts +++ b/packages/core/src/ai-model/service-caller/index.ts @@ -345,6 +345,22 @@ export async function callAI( }> { const { config: modelConfig, adapter } = modelRuntime; + if (modelRuntime.onModelInputImages) { + const imageUrls = messages.flatMap((message) => { + if (!Array.isArray(message.content)) { + return []; + } + return message.content.flatMap((part) => + part.type === 'image_url' && typeof part.image_url?.url === 'string' + ? [part.image_url.url] + : [], + ); + }); + if (imageUrls.length > 0) { + modelRuntime.onModelInputImages(imageUrls); + } + } + // Stable internal ID for this call, used by the agent to deduplicate usage // across the onUsage callback and the task-dump-based collectUsageMetrics() // path when the provider does not return a request_id. diff --git a/packages/core/tests/unit-test/gpt-image-detail.test.ts b/packages/core/tests/unit-test/gpt-image-detail.test.ts index 8820872d88..373a9f6f19 100644 --- a/packages/core/tests/unit-test/gpt-image-detail.test.ts +++ b/packages/core/tests/unit-test/gpt-image-detail.test.ts @@ -183,4 +183,16 @@ describe('GPT image detail handling', () => { expect(mockCreate.mock.calls[0][0]).toHaveProperty('temperature', 0.7); }); + + it('reports the exact image URL included in the model request', async () => { + const onModelInputImages = vi.fn(); + await callAI(imageMessage, { + ...getModelRuntime(baseModelConfig), + onModelInputImages, + }); + + expect(onModelInputImages).toHaveBeenCalledWith([ + 'https://example.com/shot.png', + ]); + }); }); diff --git a/packages/core/tests/unit-test/model-input-recorder.test.ts b/packages/core/tests/unit-test/model-input-recorder.test.ts new file mode 100644 index 0000000000..59e9197eed --- /dev/null +++ b/packages/core/tests/unit-test/model-input-recorder.test.ts @@ -0,0 +1,81 @@ +import { recordModelInputsForTask } from '@/agent/model-input-recorder'; +import type { ModelRuntime } from '@/ai-model/models'; +import { ScreenshotItem } from '@/screenshot-item'; +import type { ExecutionTask } from '@/types'; +import { describe, expect, it, vi } from 'vitest'; + +const pngBase64 = 'data:image/png;base64,iVBORw0KGgoAAAANSUhEUgAAAAUA'; +const webpBase64 = + 'data:image/webp;base64,UklGRjQAAABXRUJQVlA4ICgAAACQAQCdASoCAAMAAMASJQBOl0AAjNAA/v4icv1difCfoP7mxzi2QwAA'; + +function createTask(): ExecutionTask { + const screenshot = ScreenshotItem.create(pngBase64, 100); + return { + taskId: 'task-1', + type: 'Planning', + subType: 'Plan', + status: 'running', + executor: vi.fn(), + uiContext: { + screenshot, + shotSize: { width: 5, height: 1 }, + shrunkShotToLogicalRatio: 1, + }, + }; +} + +describe('recordModelInputsForTask', () => { + it('reuses the report screenshot when the exact bytes are sent to AI', () => { + const task = createTask(); + const runtime = recordModelInputsForTask({} as ModelRuntime, task); + + runtime.onModelInputImages?.([pngBase64]); + + expect(task.recorder).toHaveLength(1); + expect(task.recorder?.[0].screenshot).toBe(task.uiContext?.screenshot); + expect(task.recorder?.[0].timing).toBe('model-input'); + expect(task.recorder?.[0].description).toMatch( + /^Model input 1 \(exact bytes, sha256: [a-f0-9]{64}\)$/, + ); + }); + + it('records a transformed model image and deduplicates request retries', () => { + const task = createTask(); + const parentCallback = vi.fn(); + const parentRuntime = {} as ModelRuntime; + parentRuntime.onModelInputImages = parentCallback; + const runtime = recordModelInputsForTask(parentRuntime, task); + + runtime.onModelInputImages?.([webpBase64]); + runtime.onModelInputImages?.([webpBase64]); + + expect(parentCallback).toHaveBeenCalledTimes(2); + expect(task.recorder).toHaveLength(1); + expect(task.recorder?.[0].screenshot?.base64).toBe(webpBase64); + expect(task.recorder?.[0].screenshot).not.toBe(task.uiContext?.screenshot); + }); + + it('reuses the executor context screenshot before it is bound to the task', () => { + const task = createTask(); + const sourceScreenshot = task.uiContext!.screenshot; + task.uiContext = undefined; + const runtime = recordModelInputsForTask( + {} as ModelRuntime, + task, + sourceScreenshot, + ); + + runtime.onModelInputImages?.([pngBase64]); + + expect(task.recorder?.[0].screenshot).toBe(sourceScreenshot); + }); + + it('does not turn remote reference image URLs into screenshot items', () => { + const task = createTask(); + const runtime = recordModelInputsForTask({} as ModelRuntime, task); + + runtime.onModelInputImages?.(['https://example.com/reference.png']); + + expect(task.recorder).toBeUndefined(); + }); +}); diff --git a/packages/core/tests/unit-test/tasks-null-data.test.ts b/packages/core/tests/unit-test/tasks-null-data.test.ts index 83c5e6861a..ebd52451a6 100644 --- a/packages/core/tests/unit-test/tasks-null-data.test.ts +++ b/packages/core/tests/unit-test/tasks-null-data.test.ts @@ -41,6 +41,12 @@ const expectEmptyUIContext = () => shrunkShotToLogicalRatio: 1, }); +const expectRecordedModelRuntime = (config: IModelConfig) => + expect.objectContaining({ + config: expect.objectContaining(config), + onModelInputImages: expect.any(Function), + }); + const createMockUsage = (totalTokens: number): AIUsageInfo => ({ prompt_tokens: 0, completion_tokens: 0, @@ -214,7 +220,7 @@ describe('TaskExecutor - Null Data Handling', () => { StatementIsTruthy: 'Boolean, based on the current screenshot and its contents if provided, unless the user explicitly asks to compare with reference images, whether the following statement is true: Page title is correct', }, - getModelRuntime(mockModelConfig), + expectRecordedModelRuntime(mockModelConfig), {}, '', undefined, @@ -269,7 +275,7 @@ describe('TaskExecutor - Null Data Handling', () => { StatementIsTruthy: "Boolean, the user wants to do some 'wait for' operation. based on the current screenshot and its contents if provided, unless the user explicitly asks to compare with reference images, please check whether the following statement is true: Element is visible", }, - getModelRuntime(mockModelConfig), + expectRecordedModelRuntime(mockModelConfig), {}, '', undefined, @@ -636,7 +642,7 @@ describe('TaskExecutor - Null Data Handling', () => { Number: 'Number, based on the current screenshot and its contents if provided, unless the user explicitly asks to compare with reference images, Extract the price', }, - getModelRuntime(mockModelConfig), + expectRecordedModelRuntime(mockModelConfig), {}, '', undefined, @@ -731,7 +737,7 @@ describe('TaskExecutor - Null Data Handling', () => { Number: 'Number, based on the current screenshot and its contents if provided, unless the user explicitly asks to compare with reference images, Extract the price', }, - getModelRuntime(mockModelConfig), + expectRecordedModelRuntime(mockModelConfig), {}, '', undefined, @@ -787,7 +793,7 @@ describe('TaskExecutor - Null Data Handling', () => { Boolean: 'Boolean, based on the current screenshot and its contents if provided, unless the user explicitly asks to compare with reference images, there is a like button', }, - getModelRuntime(mockModelConfig), + expectRecordedModelRuntime(mockModelConfig), {}, '', undefined, diff --git a/packages/shared/src/utils.ts b/packages/shared/src/utils.ts index dec7ef4eb3..31b5d54c10 100644 --- a/packages/shared/src/utils.ts +++ b/packages/shared/src/utils.ts @@ -12,6 +12,11 @@ export function uuid(): string { return generateUUID(); } +/** Return the lowercase SHA-256 digest of UTF-8 text or raw bytes. */ +export function sha256Hex(input: string | Uint8Array): string { + return sha256.create().update(input).hex(); +} + const hashMap: Record = {}; // id - combined export function generateHashId(rect: any, content = ''): string {