feat(core): activate WebP screenshots with JPEG fallback

apps/site/docs/en/model-config.mdx

@@ -28,6 +28,7 @@ If you configure a dedicated Insight or Planning model, model-related `MIDSCENE_
 | `MIDSCENE_MODEL_REASONING_EFFORT` | Controls model-native thinking effort, supported by some models. Common values: `low`, `medium`, `high`. See [Model-native thinking](#model-native-reasoning) |
 | `MIDSCENE_MODEL_REASONING_BUDGET` | Thinking token budget (number), supported by some models. See [Model-native thinking](#model-native-reasoning) |
 | `MIDSCENE_MODEL_RESPONSE_FORMAT` | Structured response strategy: `auto` (default) lets Midscene automatically use `response_format` in appropriate scenarios to specify a structured output format (usually JSON), making the model response as suitable for structured parsing as possible; `none` does not set `response_format`, for models that do not support structured output. |
+| `MIDSCENE_MODEL_IMAGE_INPUT_FORMAT` | Encoding for inline images sent to the model: `webp` (default) or `jpeg`. Use `jpeg` as a compatibility fallback when an OpenAI-compatible provider rejects WebP. This only changes the request payload; persisted screenshots and reports remain WebP. |
 | `MIDSCENE_MODEL_HTTP_PROXY` | HTTP/HTTPS proxy, e.g., `http://127.0.0.1:8080` or `https://proxy.example.com:8080`. Takes precedence over `MIDSCENE_MODEL_SOCKS_PROXY` |
 | `MIDSCENE_MODEL_SOCKS_PROXY` | SOCKS proxy, e.g., `socks5://127.0.0.1:1080` |
 | `MIDSCENE_MODEL_INIT_CONFIG_JSON` | JSON blob that overrides the OpenAI SDK initialization config. Use `defaultHeaders` for custom auth headers; `extra_headers` and `extraHeaders` are accepted as aliases |
@@ -54,6 +55,7 @@ Set the following if the Insight intent needs a different model:
 | `MIDSCENE_INSIGHT_MODEL_INIT_CONFIG_JSON` | Optional; same effect as `MIDSCENE_MODEL_INIT_CONFIG_JSON` |
 | `MIDSCENE_INSIGHT_MODEL_EXTRA_BODY_JSON` | Optional; same effect as `MIDSCENE_MODEL_EXTRA_BODY_JSON` |
 | `MIDSCENE_INSIGHT_MODEL_RESPONSE_FORMAT` | Optional; controls the structured response strategy in appropriate Insight scenarios |
+| `MIDSCENE_INSIGHT_MODEL_IMAGE_INPUT_FORMAT` | Optional; same effect as `MIDSCENE_MODEL_IMAGE_INPUT_FORMAT` for the dedicated Insight model |
 
 ### Configure a dedicated Planning model
 
@@ -74,6 +76,7 @@ Set the following if the Planning intent needs a different model:
 | `MIDSCENE_PLANNING_MODEL_INIT_CONFIG_JSON` | Optional; same effect as `MIDSCENE_MODEL_INIT_CONFIG_JSON` |
 | `MIDSCENE_PLANNING_MODEL_EXTRA_BODY_JSON` | Optional; same effect as `MIDSCENE_MODEL_EXTRA_BODY_JSON` |
 | `MIDSCENE_PLANNING_MODEL_RESPONSE_FORMAT` | Optional; controls the structured response strategy in appropriate Planning scenarios |
+| `MIDSCENE_PLANNING_MODEL_IMAGE_INPUT_FORMAT` | Optional; same effect as `MIDSCENE_MODEL_IMAGE_INPUT_FORMAT` for the dedicated Planning model |
 
 ### Model-native thinking {#model-native-reasoning}
 

apps/site/docs/en/reference/index.mdx

@@ -1366,7 +1366,7 @@ interface RecordToReportOptions {
   screenshotBase64?: string;
   screenshots?: {
     /**
-     * PNG/JPEG data URI, or raw PNG base64 body.
+     * PNG/JPEG/WebP data URI, or raw PNG/JPEG/WebP base64 body.
      */
     base64: string;
     description?: string;
@@ -1384,7 +1384,7 @@ function recordToReport(
   - `title?: string` — Optional title for the report entry. Default: `'untitled'`.
   - `options?: RecordToReportOptions` — Optional configuration:
     - `content?: string` — Description of the screenshot.
-    - `screenshots?: Array<{ base64: string; description?: string }>` — One or more screenshots to record under the same report entry. When this option is set, Midscene does not capture another screenshot automatically. `base64` accepts a PNG/JPEG data URI such as `data:image/png;base64,...` or a raw base64 body, which Midscene treats as PNG.
+    - `screenshots?: Array<{ base64: string; description?: string }>` — One or more screenshots to record under the same report entry. When this option is set, Midscene does not capture another screenshot automatically. `base64` accepts a PNG/JPEG/WebP data URI such as `data:image/webp;base64,...` or a raw PNG/JPEG/WebP base64 body. Midscene infers known raw image signatures and treats an unrecognized raw body as PNG for compatibility.
 
 - Compatibility:
 

apps/site/docs/zh/model-config.mdx

@@ -27,6 +27,7 @@
 | `MIDSCENE_MODEL_REASONING_EFFORT` | 控制模型的原生思考力度，部分模型支持。常用值：`low`、`medium`、`high`。详见[模型原生思考](#model-native-reasoning) |
 | `MIDSCENE_MODEL_REASONING_BUDGET` | 思考 Token 预算（数字），部分模型支持。详见[模型原生思考](#model-native-reasoning) |
 | `MIDSCENE_MODEL_RESPONSE_FORMAT` | 结构化响应策略：`auto`（默认）表示 Midscene 会在合适的场景自动使用 `response_format` 参数指定模型输出的结构化格式（一般是 JSON），来尽可能保证模型返回值能够被结构化解析；`none` 表示不指定 `response_format` 参数，适用于模型不支持结构化输出的情况。 |
+| `MIDSCENE_MODEL_IMAGE_INPUT_FORMAT` | 发送给模型的内联图片编码：`webp`（默认）或 `jpeg`。当 OpenAI 兼容服务不接受 WebP 时，可设为 `jpeg` 作为兼容回退。该配置只改变请求体，落盘截图和报告仍保持 WebP。 |
 | `MIDSCENE_MODEL_HTTP_PROXY` | HTTP/HTTPS 代理配置，如 `http://127.0.0.1:8080` 或 `https://proxy.example.com:8080`，优先级高于 `MIDSCENE_MODEL_SOCKS_PROXY` |
 | `MIDSCENE_MODEL_SOCKS_PROXY` | SOCKS 代理配置，如 `socks5://127.0.0.1:1080` |
 | `MIDSCENE_MODEL_INIT_CONFIG_JSON` | 覆盖 OpenAI SDK 初始化配置的 JSON。自定义鉴权 header 请使用 `defaultHeaders`；`extra_headers` 和 `extraHeaders` 也会作为别名兼容 |
@@ -53,6 +54,7 @@
 | `MIDSCENE_INSIGHT_MODEL_INIT_CONFIG_JSON` | 可选，效果等同于 `MIDSCENE_MODEL_INIT_CONFIG_JSON`  |
 | `MIDSCENE_INSIGHT_MODEL_EXTRA_BODY_JSON` | 可选，效果等同于 `MIDSCENE_MODEL_EXTRA_BODY_JSON`  |
 | `MIDSCENE_INSIGHT_MODEL_RESPONSE_FORMAT` | 可选，在合适的 Insight 场景中控制结构化响应策略 |
+| `MIDSCENE_INSIGHT_MODEL_IMAGE_INPUT_FORMAT` | 可选，为独立 Insight 模型配置图片输入格式，效果等同于 `MIDSCENE_MODEL_IMAGE_INPUT_FORMAT` |
 
 ### 为 Planning 意图单独配置模型
 
@@ -73,6 +75,7 @@
 | `MIDSCENE_PLANNING_MODEL_INIT_CONFIG_JSON` | 可选，效果等同于 `MIDSCENE_MODEL_INIT_CONFIG_JSON`  |
 | `MIDSCENE_PLANNING_MODEL_EXTRA_BODY_JSON` | 可选，效果等同于 `MIDSCENE_MODEL_EXTRA_BODY_JSON`  |
 | `MIDSCENE_PLANNING_MODEL_RESPONSE_FORMAT` | 可选，在合适的 Planning 场景中控制结构化响应策略 |
+| `MIDSCENE_PLANNING_MODEL_IMAGE_INPUT_FORMAT` | 可选，为独立 Planning 模型配置图片输入格式，效果等同于 `MIDSCENE_MODEL_IMAGE_INPUT_FORMAT` |
 
 ### 模型原生思考 {#model-native-reasoning}
 

apps/site/docs/zh/reference/index.mdx

@@ -1349,7 +1349,7 @@ interface RecordToReportOptions {
   screenshotBase64?: string;
   screenshots?: {
     /**
-     * PNG/JPEG data URI，或裸 PNG base64 body。
+     * PNG/JPEG/WebP data URI，或裸 PNG/JPEG/WebP base64 body。
      */
     base64: string;
     description?: string;
@@ -1367,7 +1367,7 @@ function recordToReport(
   - `title?: string` - 可选，截图的标题，如果未提供，则标题为 'untitled'。
   - `options?: RecordToReportOptions` - 可选，一个配置对象，包含：
     - `content?: string` - 截图的描述。
-    - `screenshots?: Array<{ base64: string; description?: string }>` - 在同一个报告条目下记录一张或多张传入的截图。设置该选项后，Midscene 不会再自动截图。`base64` 推荐使用 PNG/JPEG data URI，例如 `data:image/png;base64,...`；也可以传裸 base64 body，此时会按 PNG 处理。
+    - `screenshots?: Array<{ base64: string; description?: string }>` - 在同一个报告条目下记录一张或多张传入的截图。设置该选项后，Midscene 不会再自动截图。`base64` 支持 PNG/JPEG/WebP data URI，例如 `data:image/webp;base64,...`，也支持裸 PNG/JPEG/WebP base64 body。Midscene 会根据已知图片签名推断裸数据格式；无法识别时为保持兼容仍按 PNG 处理。
 
 - 兼容性：
 

packages/core/src/agent/screenshot-preparation.ts

@@ -1,19 +1,20 @@
 import {
-  type JpegBase64DataUrl,
-  convertBase64ImageToJpeg,
+  type WebpBase64DataUrl,
+  convertBase64ImageToWebp,
   imageInfoOfBase64,
-  resizeBase64ImageToJpeg,
+  resizeBase64ImageToWebp,
 } from '@midscene/shared/img';
 import type { Size } from '../types';
 
-const SCREENSHOT_JPEG_QUALITY = 90;
+const SCREENSHOT_WEBP_QUALITY = 90;
+const SCREENSHOT_WEBP_EFFORT = 1;
 
 export interface PrepareRawScreenshotOptions {
   shrinkFactor?: number;
 }
 
 export interface PreparedScreenshot {
-  base64: JpegBase64DataUrl;
+  base64: WebpBase64DataUrl;
   originalSize: Size;
   shotSize: Size;
 }
@@ -58,10 +59,11 @@ export async function prepareRawScreenshot(
       : { ...originalSize };
   assertValidSize(shotSize, 'prepared screenshot dimensions');
 
-  const base64 = await resizeBase64ImageToJpeg(screenshotBase64, {
+  const base64 = await resizeBase64ImageToWebp(screenshotBase64, {
     sourceSize: originalSize,
     targetSize: shotSize,
-    jpegQuality: SCREENSHOT_JPEG_QUALITY,
+    webpQuality: SCREENSHOT_WEBP_QUALITY,
+    webpEffort: SCREENSHOT_WEBP_EFFORT,
   });
 
   return {
@@ -72,16 +74,19 @@ export async function prepareRawScreenshot(
 }
 
 /**
- * Prepare a screenshot for persistence when only JPEG output is required.
+ * Prepare a screenshot for persistence when only WebP output is required.
  * Unscaled frames avoid dimension reads; scaled frames use the full pipeline.
  */
 export async function prepareScreenshotForPersistence(
   screenshotBase64: string,
   options?: PrepareRawScreenshotOptions,
-): Promise<JpegBase64DataUrl> {
+): Promise<WebpBase64DataUrl> {
   const shrinkFactor = options?.shrinkFactor ?? 1;
   if (shrinkFactor === 1) {
-    return convertBase64ImageToJpeg(screenshotBase64, SCREENSHOT_JPEG_QUALITY);
+    return convertBase64ImageToWebp(screenshotBase64, {
+      webpQuality: SCREENSHOT_WEBP_QUALITY,
+      webpEffort: SCREENSHOT_WEBP_EFFORT,
+    });
   }
   return (await prepareRawScreenshot(screenshotBase64, options)).base64;
 }

packages/core/src/agent/ui-observer.ts

@@ -186,7 +186,7 @@ interface BufferedFrame extends DeviceFrameRef {
 function isImageDataUrl(value: unknown): value is string {
   return (
     typeof value === 'string' &&
-    /^data:image\/(?:png|jpe?g);base64,/i.test(value)
+    /^data:image\/(?:png|jpe?g|webp);base64,/i.test(value)
   );
 }
 

packages/core/src/agent/utils.ts

@@ -21,10 +21,7 @@ import {
   globalConfigManager,
 } from '@midscene/shared/env';
 import { generateElementByRect } from '@midscene/shared/extractor';
-import {
-  imageInfoOfBase64,
-  normalizeScreenshotBase64,
-} from '@midscene/shared/img';
+import { normalizeScreenshotBase64 } from '@midscene/shared/img';
 import { getDebug } from '@midscene/shared/logger';
 import { _keyDefinitions } from '@midscene/shared/us-keyboard-layout';
 import { assert, ifInBrowser, logMsg, uuid } from '@midscene/shared/utils';
@@ -164,9 +161,10 @@ export async function createScreenshotBoundUIContext(
 ): Promise<UIContext> {
   const normalizedScreenshotBase64 =
     normalizeScreenshotBase64(screenshotBase64);
-  const actualScreenshotSize = await imageInfoOfBase64(
+  const preparedScreenshot = await prepareRawScreenshot(
     normalizedScreenshotBase64,
   );
+  const actualScreenshotSize = preparedScreenshot.originalSize;
   if (
     opt.screenshotSize &&
     (opt.screenshotSize.width !== actualScreenshotSize.width ||
@@ -182,8 +180,8 @@ export async function createScreenshotBoundUIContext(
   }
 
   return {
-    screenshot: ScreenshotItem.create(normalizedScreenshotBase64, Date.now()),
-    shotSize: actualScreenshotSize,
+    screenshot: ScreenshotItem.create(preparedScreenshot.base64, Date.now()),
+    shotSize: preparedScreenshot.shotSize,
     shrunkShotToLogicalRatio: 1,
     _isFrozen: true,
   };

packages/core/src/ai-model/model-adapter/image-preprocess.ts

@@ -37,10 +37,15 @@ export async function prepareModelImage(options: {
   let modelWidth = width;
   let modelHeight = height;
 
-  if (policy.padBlockSize !== undefined) {
+  const padBlockSize = policy.padBlockSize;
+  const requiresPadding =
+    padBlockSize !== undefined &&
+    (width % padBlockSize !== 0 || height % padBlockSize !== 0);
+  if (requiresPadding) {
     const paddedResult = await paddingToMatchBlockByBase64(
       imageBase64,
-      policy.padBlockSize,
+      padBlockSize,
+      'webp',
     );
     preparedImageBase64 = paddedResult.imageBase64;
     modelWidth = paddedResult.width;

packages/core/src/ai-model/service-caller/index.ts

@@ -52,6 +52,7 @@ import {
   isModelCallRecordingEnabled,
   recordModelCallEvent,
 } from './model-call-recorder';
+import { prepareModelMessagesImageInput } from './model-image-input';
 import {
   type OpenAIErrorResponseContext,
   formatOpenAIAPIErrorDetails,
@@ -399,6 +400,10 @@ export async function callAI(
         });
       }
     : undefined;
+  const modelMessages = await prepareModelMessagesImageInput(
+    messages,
+    modelConfig.imageInputFormat,
+  );
   const chatCompletionInput = {
     intent: modelConfig.intent,
     userConfig: {
@@ -441,7 +446,7 @@ export async function callAI(
 
     try {
       const codexResult = await callAIWithCodexAppServer(
-        messages,
+        modelMessages,
         modelConfig,
         {
           stream: options?.stream,
@@ -595,10 +600,10 @@ export async function callAI(
   // resolution for localization-sensitive tasks.
   const messagesWithImageDetail: ChatCompletionMessageParam[] = (() => {
     if (!imageDetail) {
-      return messages;
+      return modelMessages;
     }
 
-    return messages.map((msg) => {
+    return modelMessages.map((msg) => {
       if (!Array.isArray(msg.content)) {
         return msg;
       }

packages/core/src/ai-model/service-caller/model-image-input.ts

@@ -0,0 +1,99 @@
+import type { TModelImageInputFormat } from '@midscene/shared/env';
+import {
+  convertBase64ImageToJpeg,
+  convertBase64ImageToWebp,
+} from '@midscene/shared/img';
+import type { ChatCompletionMessageParam } from 'openai/resources/index';
+
+const supportedBase64ImageUrlPattern =
+  /^data:image\/(?:png|jpe?g|webp);base64,/i;
+type MessageContentPart = Exclude<
+  NonNullable<ChatCompletionMessageParam['content']>,
+  string
+>[number];
+
+async function convertModelImageUrl(
+  url: string,
+  imageInputFormat: TModelImageInputFormat,
+  conversionCache: Map<string, Promise<string>>,
+): Promise<string> {
+  if (!supportedBase64ImageUrlPattern.test(url)) {
+    return url;
+  }
+
+  const cacheKey = `${imageInputFormat}\0${url}`;
+  const cachedConversion = conversionCache.get(cacheKey);
+  if (cachedConversion) {
+    return cachedConversion;
+  }
+
+  const conversion =
+    imageInputFormat === 'jpeg'
+      ? convertBase64ImageToJpeg(url)
+      : convertBase64ImageToWebp(url);
+  conversionCache.set(cacheKey, conversion);
+  return conversion;
+}
+
+/**
+ * Applies the selected encoding to inline screenshot inputs at the final model
+ * request boundary. Remote URLs are intentionally left unchanged.
+ */
+export async function prepareModelMessagesImageInput(
+  messages: ChatCompletionMessageParam[],
+  imageInputFormat: TModelImageInputFormat = 'webp',
+): Promise<ChatCompletionMessageParam[]> {
+  let messagesChanged = false;
+  const preparedMessages: ChatCompletionMessageParam[] = [];
+  const conversionCache = new Map<string, Promise<string>>();
+
+  // Process sequentially to bound memory when one request contains many
+  // full-resolution observation frames.
+  for (const message of messages) {
+    if (!Array.isArray(message.content)) {
+      preparedMessages.push(message);
+      continue;
+    }
+
+    let contentChanged = false;
+    const preparedContent: MessageContentPart[] = [];
+    for (const part of message.content) {
+      if (part.type !== 'image_url' || !part.image_url?.url) {
+        preparedContent.push(part);
+        continue;
+      }
+
+      const preparedUrl = await convertModelImageUrl(
+        part.image_url.url,
+        imageInputFormat,
+        conversionCache,
+      );
+      if (preparedUrl === part.image_url.url) {
+        preparedContent.push(part);
+        continue;
+      }
+
+      contentChanged = true;
+      preparedContent.push({
+        ...part,
+        image_url: {
+          ...part.image_url,
+          url: preparedUrl,
+        },
+      });
+    }
+
+    if (!contentChanged) {
+      preparedMessages.push(message);
+      continue;
+    }
+
+    messagesChanged = true;
+    preparedMessages.push({
+      ...message,
+      content: preparedContent,
+    } as ChatCompletionMessageParam);
+  }
+
+  return messagesChanged ? preparedMessages : messages;
+}

packages/core/src/ai-model/workflows/grounding/search-area.ts

@@ -78,9 +78,14 @@ export async function buildSearchAreaConfig(options: {
   const croppedResult = await cropByRect(
     context.screenshot.base64,
     sectionRect,
+    'webp',
   );
 
-  const scaledResult = await scaleImage(croppedResult.imageBase64, scaleRatio);
+  const scaledResult = await scaleImage(
+    croppedResult.imageBase64,
+    scaleRatio,
+    'webp',
+  );
   return {
     sourceRect: sectionRect,
     image: {

packages/core/src/ai-model/workflows/recorder-generation/markdown.ts

@@ -2,7 +2,7 @@ import type { IModelConfig } from '@midscene/shared/env';
 import {
   imageInfoOfBase64,
   parseBase64,
-  resizeBase64ImageToJpeg,
+  resizeBase64ImageToWebp,
 } from '@midscene/shared/img';
 import { getDebug } from '@midscene/shared/logger';
 import {
@@ -54,7 +54,7 @@ async function compressScreenshotAssetForMarkdownReplay(
   }
 
   const scale = MARKDOWN_REPLAY_SCREENSHOT_MAX_EDGE / longestEdge;
-  const dataUrl = await resizeBase64ImageToJpeg(asset.dataUrl, {
+  const dataUrl = await resizeBase64ImageToWebp(asset.dataUrl, {
     sourceSize: { width, height },
     targetSize: {
       width: Math.max(1, Math.round(width * scale)),