feat(core): record exact model input screenshots

packages/core/src/agent/model-input-recorder.ts

@@ -0,0 +1,62 @@
+import { Buffer } from 'node:buffer';
+import type { ModelRuntime } from '@/ai-model/models';
+import { ScreenshotItem } from '@/screenshot-item';
+import type { ExecutionTask } from '@/types';
+import { parseBase64 } from '@midscene/shared/img';
+import { sha256Hex } from '@midscene/shared/utils';
+
+const MODEL_INPUT_TIMING = 'model-input';
+
+function screenshotContentHash(imageBase64: string): string {
+  const { body } = parseBase64(imageBase64);
+  return sha256Hex(Buffer.from(body, 'base64'));
+}
+
+/**
+ * Bind a model runtime to one report task so the report retains the exact
+ * data-URI bytes passed to the provider after padding/cropping/resizing.
+ */
+export function recordModelInputsForTask(
+  modelRuntime: ModelRuntime,
+  task: ExecutionTask,
+  sourceScreenshot?: ScreenshotItem,
+): ModelRuntime {
+  return {
+    ...modelRuntime,
+    onModelInputImages: (images) => {
+      modelRuntime.onModelInputImages?.(images);
+
+      for (const [index, imageBase64] of images.entries()) {
+        if (!imageBase64.startsWith('data:image/')) {
+          continue;
+        }
+
+        const contentHash = screenshotContentHash(imageBase64);
+        const alreadyRecorded = task.recorder?.some(
+          (item) =>
+            item.timing === MODEL_INPUT_TIMING &&
+            item.screenshot &&
+            screenshotContentHash(item.screenshot.base64) === contentHash,
+        );
+        if (alreadyRecorded) {
+          continue;
+        }
+
+        const uiScreenshot = task.uiContext?.screenshot ?? sourceScreenshot;
+        const screenshot =
+          uiScreenshot &&
+          screenshotContentHash(uiScreenshot.base64) === contentHash
+            ? uiScreenshot
+            : ScreenshotItem.create(imageBase64, Date.now());
+        const recorderItem = {
+          type: 'screenshot' as const,
+          ts: Date.now(),
+          screenshot,
+          timing: MODEL_INPUT_TIMING,
+          description: `Model input ${index + 1} (exact bytes, sha256: ${contentHash})`,
+        };
+        task.recorder = [...(task.recorder ?? []), recorderItem];
+      }
+    },
+  };
+}

packages/core/src/agent/task-builder.ts

@@ -25,6 +25,7 @@ import { sleep } from '@/utils';
 import { generateElementByRect } from '@midscene/shared/extractor';
 import { getDebug } from '@midscene/shared/logger';
 import { assert } from '@midscene/shared/utils';
+import { recordModelInputsForTask } from './model-input-recorder';
 import type { TaskCache } from './task-cache';
 import { withUsageIntent } from './usage-intent';
 import {
@@ -554,7 +555,11 @@ export class TaskBuilder {
                 context: uiContext,
                 planLocatedElement,
               },
-              defaultModel,
+              recordModelInputsForTask(
+                defaultModel,
+                task,
+                uiContext.screenshot,
+              ),
               abortSignal,
             );
             applyDump(locateResult.dump);

packages/core/src/agent/tasks.ts

@@ -38,6 +38,7 @@ import { ServiceError, aiActProgressScope } from '@/types';
 import { getDebug } from '@midscene/shared/logger';
 import { assert } from '@midscene/shared/utils';
 import { ExecutionSession } from './execution-session';
+import { recordModelInputsForTask } from './model-input-recorder';
 import {
   type AgentProgressPublisher,
   createAiActActionReporter,
@@ -587,7 +588,11 @@ export class TaskExecutor {
                 context: planningUiContext,
                 actionContext: param.aiActContext,
                 actionSpace,
-                modelRuntime: planningModel,
+                modelRuntime: recordModelInputsForTask(
+                  planningModel,
+                  executorContext.task,
+                  planningUiContext.screenshot,
+                ),
                 conversationHistory,
                 includeLocateInPlanning,
                 imagesIncludeCount,
@@ -917,7 +922,7 @@ export class TaskExecutor {
         try {
           extractResult = await this.service.extract<any>(
             demandInput,
-            modelRuntime,
+            recordModelInputsForTask(modelRuntime, task, uiContext.screenshot),
             opt,
             extraPageDescription,
             multimodalPrompt,

packages/core/src/ai-model/model-adapter/types.ts

@@ -245,6 +245,8 @@ export interface ModelRuntime {
    * such as order-sensitive judging and deep-locate search-area calls).
    */
   onUsage?: (usage: AIUsageInfo) => void;
+  /** Exact image URLs included in a model request, after all preprocessing. */
+  onModelInputImages?: (images: readonly string[]) => void;
 }
 
 export interface ModelAdapterDefinition {

packages/core/src/ai-model/service-caller/index.ts

@@ -345,6 +345,22 @@ export async function callAI(
 }> {
   const { config: modelConfig, adapter } = modelRuntime;
 
+  if (modelRuntime.onModelInputImages) {
+    const imageUrls = messages.flatMap((message) => {
+      if (!Array.isArray(message.content)) {
+        return [];
+      }
+      return message.content.flatMap((part) =>
+        part.type === 'image_url' && typeof part.image_url?.url === 'string'
+          ? [part.image_url.url]
+          : [],
+      );
+    });
+    if (imageUrls.length > 0) {
+      modelRuntime.onModelInputImages(imageUrls);
+    }
+  }
+
   // Stable internal ID for this call, used by the agent to deduplicate usage
   // across the onUsage callback and the task-dump-based collectUsageMetrics()
   // path when the provider does not return a request_id.

packages/core/tests/unit-test/gpt-image-detail.test.ts

@@ -183,4 +183,16 @@ describe('GPT image detail handling', () => {
 
     expect(mockCreate.mock.calls[0][0]).toHaveProperty('temperature', 0.7);
   });
+
+  it('reports the exact image URL included in the model request', async () => {
+    const onModelInputImages = vi.fn();
+    await callAI(imageMessage, {
+      ...getModelRuntime(baseModelConfig),
+      onModelInputImages,
+    });
+
+    expect(onModelInputImages).toHaveBeenCalledWith([
+      'https://example.com/shot.png',
+    ]);
+  });
 });

packages/core/tests/unit-test/model-input-recorder.test.ts

@@ -0,0 +1,81 @@
+import { recordModelInputsForTask } from '@/agent/model-input-recorder';
+import type { ModelRuntime } from '@/ai-model/models';
+import { ScreenshotItem } from '@/screenshot-item';
+import type { ExecutionTask } from '@/types';
+import { describe, expect, it, vi } from 'vitest';
+
+const pngBase64 = 'data:image/png;base64,iVBORw0KGgoAAAANSUhEUgAAAAUA';
+const webpBase64 =
+  'data:image/webp;base64,UklGRjQAAABXRUJQVlA4ICgAAACQAQCdASoCAAMAAMASJQBOl0AAjNAA/v4icv1difCfoP7mxzi2QwAA';
+
+function createTask(): ExecutionTask {
+  const screenshot = ScreenshotItem.create(pngBase64, 100);
+  return {
+    taskId: 'task-1',
+    type: 'Planning',
+    subType: 'Plan',
+    status: 'running',
+    executor: vi.fn(),
+    uiContext: {
+      screenshot,
+      shotSize: { width: 5, height: 1 },
+      shrunkShotToLogicalRatio: 1,
+    },
+  };
+}
+
+describe('recordModelInputsForTask', () => {
+  it('reuses the report screenshot when the exact bytes are sent to AI', () => {
+    const task = createTask();
+    const runtime = recordModelInputsForTask({} as ModelRuntime, task);
+
+    runtime.onModelInputImages?.([pngBase64]);
+
+    expect(task.recorder).toHaveLength(1);
+    expect(task.recorder?.[0].screenshot).toBe(task.uiContext?.screenshot);
+    expect(task.recorder?.[0].timing).toBe('model-input');
+    expect(task.recorder?.[0].description).toMatch(
+      /^Model input 1 \(exact bytes, sha256: [a-f0-9]{64}\)$/,
+    );
+  });
+
+  it('records a transformed model image and deduplicates request retries', () => {
+    const task = createTask();
+    const parentCallback = vi.fn();
+    const parentRuntime = {} as ModelRuntime;
+    parentRuntime.onModelInputImages = parentCallback;
+    const runtime = recordModelInputsForTask(parentRuntime, task);
+
+    runtime.onModelInputImages?.([webpBase64]);
+    runtime.onModelInputImages?.([webpBase64]);
+
+    expect(parentCallback).toHaveBeenCalledTimes(2);
+    expect(task.recorder).toHaveLength(1);
+    expect(task.recorder?.[0].screenshot?.base64).toBe(webpBase64);
+    expect(task.recorder?.[0].screenshot).not.toBe(task.uiContext?.screenshot);
+  });
+
+  it('reuses the executor context screenshot before it is bound to the task', () => {
+    const task = createTask();
+    const sourceScreenshot = task.uiContext!.screenshot;
+    task.uiContext = undefined;
+    const runtime = recordModelInputsForTask(
+      {} as ModelRuntime,
+      task,
+      sourceScreenshot,
+    );
+
+    runtime.onModelInputImages?.([pngBase64]);
+
+    expect(task.recorder?.[0].screenshot).toBe(sourceScreenshot);
+  });
+
+  it('does not turn remote reference image URLs into screenshot items', () => {
+    const task = createTask();
+    const runtime = recordModelInputsForTask({} as ModelRuntime, task);
+
+    runtime.onModelInputImages?.(['https://example.com/reference.png']);
+
+    expect(task.recorder).toBeUndefined();
+  });
+});

packages/core/tests/unit-test/tasks-null-data.test.ts

@@ -41,6 +41,12 @@ const expectEmptyUIContext = () =>
     shrunkShotToLogicalRatio: 1,
   });
 
+const expectRecordedModelRuntime = (config: IModelConfig) =>
+  expect.objectContaining({
+    config: expect.objectContaining(config),
+    onModelInputImages: expect.any(Function),
+  });
+
 const createMockUsage = (totalTokens: number): AIUsageInfo => ({
   prompt_tokens: 0,
   completion_tokens: 0,
@@ -214,7 +220,7 @@ describe('TaskExecutor - Null Data Handling', () => {
           StatementIsTruthy:
             'Boolean, based on the current screenshot and its contents if provided, unless the user explicitly asks to compare with reference images, whether the following statement is true: Page title is correct',
         },
-        getModelRuntime(mockModelConfig),
+        expectRecordedModelRuntime(mockModelConfig),
         {},
         '',
         undefined,
@@ -269,7 +275,7 @@ describe('TaskExecutor - Null Data Handling', () => {
           StatementIsTruthy:
             "Boolean, the user wants to do some 'wait for' operation. based on the current screenshot and its contents if provided, unless the user explicitly asks to compare with reference images, please check whether the following statement is true: Element is visible",
         },
-        getModelRuntime(mockModelConfig),
+        expectRecordedModelRuntime(mockModelConfig),
         {},
         '',
         undefined,
@@ -636,7 +642,7 @@ describe('TaskExecutor - Null Data Handling', () => {
           Number:
             'Number, based on the current screenshot and its contents if provided, unless the user explicitly asks to compare with reference images, Extract the price',
         },
-        getModelRuntime(mockModelConfig),
+        expectRecordedModelRuntime(mockModelConfig),
         {},
         '',
         undefined,
@@ -731,7 +737,7 @@ describe('TaskExecutor - Null Data Handling', () => {
           Number:
             'Number, based on the current screenshot and its contents if provided, unless the user explicitly asks to compare with reference images, Extract the price',
         },
-        getModelRuntime(mockModelConfig),
+        expectRecordedModelRuntime(mockModelConfig),
         {},
         '',
         undefined,
@@ -787,7 +793,7 @@ describe('TaskExecutor - Null Data Handling', () => {
           Boolean:
             'Boolean, based on the current screenshot and its contents if provided, unless the user explicitly asks to compare with reference images, there is a like button',
         },
-        getModelRuntime(mockModelConfig),
+        expectRecordedModelRuntime(mockModelConfig),
         {},
         '',
         undefined,

packages/shared/src/utils.ts

@@ -12,6 +12,11 @@ export function uuid(): string {
   return generateUUID();
 }
 
+/** Return the lowercase SHA-256 digest of UTF-8 text or raw bytes. */
+export function sha256Hex(input: string | Uint8Array): string {
+  return sha256.create().update(input).hex();
+}
+
 const hashMap: Record<string, string> = {}; // id - combined
 
 export function generateHashId(rect: any, content = ''): string {