feat(shared): add WebP image primitives

packages/core/tests/unit-test/agent-dump-update.test.ts

@@ -255,7 +255,7 @@ describe('Agent dump update screenshot serialization', () => {
         screenshots: [{ base64: 'data:image/svg+xml;base64,custom' }],
       }),
     ).rejects.toThrow(
-      'recordToReport: screenshot #1 base64 must be a PNG/JPEG data URI or raw PNG base64 string',
+      'recordToReport: screenshot #1 base64 must be a PNG/JPEG/WebP data URI or raw PNG/WebP base64 string',
     );
 
     expect(screenshotBase64).not.toHaveBeenCalled();

packages/shared/src/img/box-select.ts

@@ -758,9 +758,11 @@ async function encodeRgbaWithSharp(
   const output = await Sharp(Buffer.from(pixels), {
     raw: { width, height, channels: 4 },
   })
-    .jpeg({ quality: 90, chromaSubsampling: '4:4:4' })
+    // Keep synthetic marker edges and colors exact; these pixels carry model
+    // semantics and are more important than the small extra payload.
+    .webp({ lossless: true, effort: 1 })
     .toBuffer();
-  return createImgBase64ByFormat('jpeg', output.toString('base64'));
+  return createImgBase64ByFormat('webp', output.toString('base64'));
 }
 
 export const compositeElementInfoImg = async (options: {

packages/shared/src/img/browser-webp-encoder.ts

@@ -0,0 +1,114 @@
+export interface BrowserWebpEncodeInput {
+  pixels: ArrayLike<number>;
+  width: number;
+  height: number;
+  /** Encoder quality from 0 to 100. Defaults to 90. */
+  quality?: number;
+}
+
+/**
+ * Encode RGBA pixels with the WebP encoder provided by the browser.
+ *
+ * Keep this function self-contained so browser contract tests can execute the
+ * production implementation in a page or Worker without a test-only copy.
+ */
+export async function encodeRgbaToWebp({
+  pixels,
+  width,
+  height,
+  quality = 90,
+}: BrowserWebpEncodeInput): Promise<Uint8Array> {
+  if (
+    !Number.isSafeInteger(width) ||
+    !Number.isSafeInteger(height) ||
+    width <= 0 ||
+    height <= 0
+  ) {
+    throw new Error('WebP image dimensions must be positive safe integers');
+  }
+
+  if (!Number.isFinite(quality) || quality < 0 || quality > 100) {
+    throw new Error('WebP quality must be between 0 and 100');
+  }
+
+  const expectedPixelCount = width * height * 4;
+  if (
+    !Number.isSafeInteger(expectedPixelCount) ||
+    pixels.length !== expectedPixelCount
+  ) {
+    throw new Error(
+      `WebP RGBA pixel length must be ${expectedPixelCount}, got ${pixels.length}`,
+    );
+  }
+
+  const normalizedQuality = quality / 100;
+  let outputBlob: Blob;
+
+  if (typeof OffscreenCanvas !== 'undefined') {
+    const canvas = new OffscreenCanvas(width, height);
+    const context = canvas.getContext('2d');
+    if (!context) {
+      throw new Error('Failed to get an OffscreenCanvas 2d context');
+    }
+
+    const imageData = context.createImageData(width, height);
+    imageData.data.set(pixels);
+    context.putImageData(imageData, 0, 0);
+    outputBlob = await canvas.convertToBlob({
+      type: 'image/webp',
+      quality: normalizedQuality,
+    });
+  } else if (typeof document !== 'undefined') {
+    const canvas = document.createElement('canvas');
+    canvas.width = width;
+    canvas.height = height;
+    const context = canvas.getContext('2d');
+    if (!context) {
+      throw new Error('Failed to get an HTMLCanvasElement 2d context');
+    }
+
+    const imageData = context.createImageData(width, height);
+    imageData.data.set(pixels);
+    context.putImageData(imageData, 0, 0);
+    outputBlob = await new Promise<Blob>((resolve, reject) => {
+      canvas.toBlob(
+        (blob) => {
+          if (blob) {
+            resolve(blob);
+          } else {
+            reject(new Error('HTMLCanvasElement failed to encode WebP'));
+          }
+        },
+        'image/webp',
+        normalizedQuality,
+      );
+    });
+  } else {
+    throw new Error(
+      'WebP encoding requires OffscreenCanvas or HTMLCanvasElement',
+    );
+  }
+
+  if (outputBlob.type.toLowerCase() !== 'image/webp') {
+    throw new Error(
+      `Browser WebP encoder returned ${outputBlob.type || 'an unknown MIME type'}`,
+    );
+  }
+
+  const output = new Uint8Array(await outputBlob.arrayBuffer());
+  const isWebp =
+    output.length >= 12 &&
+    output[0] === 0x52 &&
+    output[1] === 0x49 &&
+    output[2] === 0x46 &&
+    output[3] === 0x46 &&
+    output[8] === 0x57 &&
+    output[9] === 0x45 &&
+    output[10] === 0x42 &&
+    output[11] === 0x50;
+  if (!isWebp) {
+    throw new Error('Browser WebP encoder returned invalid WebP bytes');
+  }
+
+  return output;
+}

packages/shared/src/img/canvas-fallback.ts

@@ -4,6 +4,11 @@
  */
 
 import { getDebug } from '../logger';
+import {
+  detectScreenshotImageFormatFromBuffer,
+  inferScreenshotImageFormatFromBase64,
+  screenshotImageMimeType,
+} from './image-format';
 
 const debug = getDebug('img:canvas-fallback');
 
@@ -42,6 +47,10 @@ export class CanvasImage {
 
   get_bytes_jpeg(quality: number): Uint8Array {
     const dataUrl = this.canvas.toDataURL('image/jpeg', quality / 100);
+    return CanvasImage.bytesFromDataUrl(dataUrl);
+  }
+
+  private static bytesFromDataUrl(dataUrl: string): Uint8Array {
     const base64 = dataUrl.split(',')[1];
     const binary = atob(base64);
     const bytes = new Uint8Array(binary.length);
@@ -89,7 +98,9 @@ export class CanvasImage {
       if (base64Body.startsWith('data:')) {
         img.src = base64Body;
       } else {
-        img.src = `data:image/png;base64,${base64Body}`;
+        const format =
+          inferScreenshotImageFormatFromBase64(base64Body) ?? 'png';
+        img.src = `data:${screenshotImageMimeType(format)};base64,${base64Body}`;
       }
     });
   }
@@ -99,7 +110,10 @@ export class CanvasImage {
    */
   static async new_from_byteslice(bytes: Uint8Array): Promise<CanvasImage> {
     return new Promise((resolve, reject) => {
-      const blob = new Blob([bytes], { type: 'image/png' });
+      const format = detectScreenshotImageFormatFromBuffer(bytes) ?? 'png';
+      const blob = new Blob([bytes], {
+        type: screenshotImageMimeType(format),
+      });
       const url = URL.createObjectURL(blob);
       const img = new Image();
 

packages/shared/src/img/image-format.ts

@@ -0,0 +1,131 @@
+export type ScreenshotImageFormat = 'png' | 'jpeg' | 'webp';
+
+export type ScreenshotImageMimeType = 'image/png' | 'image/jpeg' | 'image/webp';
+
+const mimeTypeByFormat: Record<ScreenshotImageFormat, ScreenshotImageMimeType> =
+  {
+    png: 'image/png',
+    jpeg: 'image/jpeg',
+    webp: 'image/webp',
+  };
+
+export function screenshotImageMimeType(
+  format: ScreenshotImageFormat,
+): ScreenshotImageMimeType {
+  return mimeTypeByFormat[format];
+}
+
+export function screenshotImageExtension(
+  format: ScreenshotImageFormat,
+): ScreenshotImageFormat {
+  return format;
+}
+
+export function screenshotImageFormatFromExtension(
+  extension: unknown,
+): ScreenshotImageFormat | undefined {
+  if (typeof extension !== 'string') {
+    return undefined;
+  }
+
+  switch (extension.toLowerCase()) {
+    case 'png':
+      return 'png';
+    case 'jpeg':
+    case 'jpg':
+      return 'jpeg';
+    case 'webp':
+      return 'webp';
+    default:
+      return undefined;
+  }
+}
+
+export function screenshotImageFormatFromMimeType(
+  mimeType: unknown,
+): ScreenshotImageFormat | undefined {
+  if (typeof mimeType !== 'string') {
+    return undefined;
+  }
+
+  switch (mimeType.toLowerCase()) {
+    case 'image/png':
+      return 'png';
+    case 'image/jpeg':
+    case 'image/jpg':
+      return 'jpeg';
+    case 'image/webp':
+      return 'webp';
+    default:
+      return undefined;
+  }
+}
+
+export function isScreenshotImageMimeType(
+  mimeType: unknown,
+): mimeType is ScreenshotImageMimeType {
+  return (
+    mimeType === 'image/png' ||
+    mimeType === 'image/jpeg' ||
+    mimeType === 'image/webp'
+  );
+}
+
+export function inferScreenshotImageFormatFromBase64(
+  base64Body: string,
+): ScreenshotImageFormat | undefined {
+  const normalizedBody = base64Body.replace(/\s/g, '');
+  if (normalizedBody.startsWith('iVBORw0KGgo')) {
+    return 'png';
+  }
+  if (normalizedBody.startsWith('/9j/')) {
+    return 'jpeg';
+  }
+  if (normalizedBody.startsWith('UklGR')) {
+    return 'webp';
+  }
+  return undefined;
+}
+
+export function detectScreenshotImageFormatFromBuffer(
+  buffer: Uint8Array,
+): ScreenshotImageFormat | undefined {
+  if (
+    buffer.length >= 8 &&
+    buffer[0] === 0x89 &&
+    buffer[1] === 0x50 &&
+    buffer[2] === 0x4e &&
+    buffer[3] === 0x47 &&
+    buffer[4] === 0x0d &&
+    buffer[5] === 0x0a &&
+    buffer[6] === 0x1a &&
+    buffer[7] === 0x0a
+  ) {
+    return 'png';
+  }
+
+  if (
+    buffer.length >= 3 &&
+    buffer[0] === 0xff &&
+    buffer[1] === 0xd8 &&
+    buffer[2] === 0xff
+  ) {
+    return 'jpeg';
+  }
+
+  if (
+    buffer.length >= 12 &&
+    buffer[0] === 0x52 &&
+    buffer[1] === 0x49 &&
+    buffer[2] === 0x46 &&
+    buffer[3] === 0x46 &&
+    buffer[8] === 0x57 &&
+    buffer[9] === 0x45 &&
+    buffer[10] === 0x42 &&
+    buffer[11] === 0x50
+  ) {
+    return 'webp';
+  }
+
+  return undefined;
+}

packages/shared/src/img/index.ts

@@ -2,13 +2,29 @@ export {
   imageInfoOfBase64,
   isValidPNGImageBuffer,
   isValidJPEGImageBuffer,
+  isValidWebPImageBuffer,
   isValidImageBuffer,
   validateScreenshotBuffer,
   type ValidateScreenshotBufferOptions,
 } from './info';
+export {
+  detectScreenshotImageFormatFromBuffer,
+  inferScreenshotImageFormatFromBase64,
+  isScreenshotImageMimeType,
+  screenshotImageExtension,
+  screenshotImageFormatFromExtension,
+  screenshotImageFormatFromMimeType,
+  screenshotImageMimeType,
+  type ScreenshotImageFormat,
+  type ScreenshotImageMimeType,
+} from './image-format';
 export {
   resizeAndConvertImgBuffer,
   convertImgBufferToJpeg,
+  convertImgBufferToWebp,
+  canonicalizeScreenshotBase64,
+  DEFAULT_WEBP_SCREENSHOT_EFFORT,
+  DEFAULT_WEBP_SCREENSHOT_QUALITY,
   resizeImgBase64,
   zoomForGPT4o,
   saveBase64Image,
@@ -24,10 +40,16 @@ export {
   normalizeBase64Image,
   normalizeScreenshotBase64,
   type NormalizeScreenshotBase64Options,
+  type CanonicalizeScreenshotOptions,
+  type WebpScreenshotEncodeOptions,
 } from './transform';
 export {
   processImageElementInfo,
   compositeElementInfoImg,
   compositePointMarkerImg,
   annotateRects,
 } from './box-select';
+export {
+  encodeRgbaToWebp,
+  type BrowserWebpEncodeInput,
+} from './browser-webp-encoder';

packages/shared/src/img/info.ts

@@ -4,6 +4,7 @@ import type { Size } from '../types';
 import { ifInNode } from '../utils';
 import getPhoton from './get-photon';
 import getSharp from './get-sharp';
+import { detectScreenshotImageFormatFromBuffer } from './image-format';
 
 export interface ImageInfo extends Size {}
 
@@ -118,12 +119,25 @@ export function isValidJPEGImageBuffer(buffer: Buffer): boolean {
 }
 
 /**
- * Check if the Buffer is a valid image (PNG or JPEG)
+ * Check if the Buffer has a WebP signature.
  * @param buffer The Buffer to check
- * @returns true if the Buffer is a valid PNG or JPEG image, otherwise false
+ * @returns true if the Buffer has a WebP signature, otherwise false
+ */
+export function isValidWebPImageBuffer(buffer: Buffer): boolean {
+  return detectScreenshotImageFormatFromBuffer(buffer) === 'webp';
+}
+
+/**
+ * Check if the Buffer is a supported screenshot image (PNG, JPEG, or WebP)
+ * @param buffer The Buffer to check
+ * @returns true if the Buffer has a supported image signature, otherwise false
  */
 export function isValidImageBuffer(buffer: Buffer): boolean {
-  return isValidPNGImageBuffer(buffer) || isValidJPEGImageBuffer(buffer);
+  return (
+    isValidPNGImageBuffer(buffer) ||
+    isValidJPEGImageBuffer(buffer) ||
+    isValidWebPImageBuffer(buffer)
+  );
 }
 
 export interface ValidateScreenshotBufferOptions {