From 96adcb1aa353eb6d67de8080d52e4dde226c2bb7 Mon Sep 17 00:00:00 2001 From: quanruzhuoxiu Date: Thu, 23 Jul 2026 14:49:09 +0800 Subject: [PATCH 1/2] feat(shared): add WebP image primitives --- packages/shared/src/img/box-select.ts | 6 +- .../shared/src/img/browser-webp-encoder.ts | 114 +++++++++ packages/shared/src/img/canvas-fallback.ts | 18 +- packages/shared/src/img/image-format.ts | 131 ++++++++++ packages/shared/src/img/index.ts | 22 ++ packages/shared/src/img/info.ts | 20 +- packages/shared/src/img/transform.ts | 228 +++++++++++++----- .../tests/unit-test/image/index.test.ts | 27 ++- .../shared/tests/unit-test/transform.test.ts | 88 ++++++- 9 files changed, 579 insertions(+), 75 deletions(-) create mode 100644 packages/shared/src/img/browser-webp-encoder.ts create mode 100644 packages/shared/src/img/image-format.ts diff --git a/packages/shared/src/img/box-select.ts b/packages/shared/src/img/box-select.ts index 1b76deb366..0afa332281 100644 --- a/packages/shared/src/img/box-select.ts +++ b/packages/shared/src/img/box-select.ts @@ -758,9 +758,11 @@ async function encodeRgbaWithSharp( const output = await Sharp(Buffer.from(pixels), { raw: { width, height, channels: 4 }, }) - .jpeg({ quality: 90, chromaSubsampling: '4:4:4' }) + // Keep synthetic marker edges and colors exact; these pixels carry model + // semantics and are more important than the small extra payload. + .webp({ lossless: true, effort: 1 }) .toBuffer(); - return createImgBase64ByFormat('jpeg', output.toString('base64')); + return createImgBase64ByFormat('webp', output.toString('base64')); } export const compositeElementInfoImg = async (options: { diff --git a/packages/shared/src/img/browser-webp-encoder.ts b/packages/shared/src/img/browser-webp-encoder.ts new file mode 100644 index 0000000000..be2cdb38e0 --- /dev/null +++ b/packages/shared/src/img/browser-webp-encoder.ts @@ -0,0 +1,114 @@ +export interface BrowserWebpEncodeInput { + pixels: ArrayLike; + width: number; + height: number; + /** Encoder quality from 0 to 100. Defaults to 90. */ + quality?: number; +} + +/** + * Encode RGBA pixels with the WebP encoder provided by the browser. + * + * Keep this function self-contained so browser contract tests can execute the + * production implementation in a page or Worker without a test-only copy. + */ +export async function encodeRgbaToWebp({ + pixels, + width, + height, + quality = 90, +}: BrowserWebpEncodeInput): Promise { + if ( + !Number.isSafeInteger(width) || + !Number.isSafeInteger(height) || + width <= 0 || + height <= 0 + ) { + throw new Error('WebP image dimensions must be positive safe integers'); + } + + if (!Number.isFinite(quality) || quality < 0 || quality > 100) { + throw new Error('WebP quality must be between 0 and 100'); + } + + const expectedPixelCount = width * height * 4; + if ( + !Number.isSafeInteger(expectedPixelCount) || + pixels.length !== expectedPixelCount + ) { + throw new Error( + `WebP RGBA pixel length must be ${expectedPixelCount}, got ${pixels.length}`, + ); + } + + const normalizedQuality = quality / 100; + let outputBlob: Blob; + + if (typeof OffscreenCanvas !== 'undefined') { + const canvas = new OffscreenCanvas(width, height); + const context = canvas.getContext('2d'); + if (!context) { + throw new Error('Failed to get an OffscreenCanvas 2d context'); + } + + const imageData = context.createImageData(width, height); + imageData.data.set(pixels); + context.putImageData(imageData, 0, 0); + outputBlob = await canvas.convertToBlob({ + type: 'image/webp', + quality: normalizedQuality, + }); + } else if (typeof document !== 'undefined') { + const canvas = document.createElement('canvas'); + canvas.width = width; + canvas.height = height; + const context = canvas.getContext('2d'); + if (!context) { + throw new Error('Failed to get an HTMLCanvasElement 2d context'); + } + + const imageData = context.createImageData(width, height); + imageData.data.set(pixels); + context.putImageData(imageData, 0, 0); + outputBlob = await new Promise((resolve, reject) => { + canvas.toBlob( + (blob) => { + if (blob) { + resolve(blob); + } else { + reject(new Error('HTMLCanvasElement failed to encode WebP')); + } + }, + 'image/webp', + normalizedQuality, + ); + }); + } else { + throw new Error( + 'WebP encoding requires OffscreenCanvas or HTMLCanvasElement', + ); + } + + if (outputBlob.type.toLowerCase() !== 'image/webp') { + throw new Error( + `Browser WebP encoder returned ${outputBlob.type || 'an unknown MIME type'}`, + ); + } + + const output = new Uint8Array(await outputBlob.arrayBuffer()); + const isWebp = + output.length >= 12 && + output[0] === 0x52 && + output[1] === 0x49 && + output[2] === 0x46 && + output[3] === 0x46 && + output[8] === 0x57 && + output[9] === 0x45 && + output[10] === 0x42 && + output[11] === 0x50; + if (!isWebp) { + throw new Error('Browser WebP encoder returned invalid WebP bytes'); + } + + return output; +} diff --git a/packages/shared/src/img/canvas-fallback.ts b/packages/shared/src/img/canvas-fallback.ts index bfc0f4bf02..62267aaed4 100644 --- a/packages/shared/src/img/canvas-fallback.ts +++ b/packages/shared/src/img/canvas-fallback.ts @@ -4,6 +4,11 @@ */ import { getDebug } from '../logger'; +import { + detectScreenshotImageFormatFromBuffer, + inferScreenshotImageFormatFromBase64, + screenshotImageMimeType, +} from './image-format'; const debug = getDebug('img:canvas-fallback'); @@ -42,6 +47,10 @@ export class CanvasImage { get_bytes_jpeg(quality: number): Uint8Array { const dataUrl = this.canvas.toDataURL('image/jpeg', quality / 100); + return CanvasImage.bytesFromDataUrl(dataUrl); + } + + private static bytesFromDataUrl(dataUrl: string): Uint8Array { const base64 = dataUrl.split(',')[1]; const binary = atob(base64); const bytes = new Uint8Array(binary.length); @@ -89,7 +98,9 @@ export class CanvasImage { if (base64Body.startsWith('data:')) { img.src = base64Body; } else { - img.src = `data:image/png;base64,${base64Body}`; + const format = + inferScreenshotImageFormatFromBase64(base64Body) ?? 'png'; + img.src = `data:${screenshotImageMimeType(format)};base64,${base64Body}`; } }); } @@ -99,7 +110,10 @@ export class CanvasImage { */ static async new_from_byteslice(bytes: Uint8Array): Promise { return new Promise((resolve, reject) => { - const blob = new Blob([bytes], { type: 'image/png' }); + const format = detectScreenshotImageFormatFromBuffer(bytes) ?? 'png'; + const blob = new Blob([bytes], { + type: screenshotImageMimeType(format), + }); const url = URL.createObjectURL(blob); const img = new Image(); diff --git a/packages/shared/src/img/image-format.ts b/packages/shared/src/img/image-format.ts new file mode 100644 index 0000000000..256fadbdef --- /dev/null +++ b/packages/shared/src/img/image-format.ts @@ -0,0 +1,131 @@ +export type ScreenshotImageFormat = 'png' | 'jpeg' | 'webp'; + +export type ScreenshotImageMimeType = 'image/png' | 'image/jpeg' | 'image/webp'; + +const mimeTypeByFormat: Record = + { + png: 'image/png', + jpeg: 'image/jpeg', + webp: 'image/webp', + }; + +export function screenshotImageMimeType( + format: ScreenshotImageFormat, +): ScreenshotImageMimeType { + return mimeTypeByFormat[format]; +} + +export function screenshotImageExtension( + format: ScreenshotImageFormat, +): ScreenshotImageFormat { + return format; +} + +export function screenshotImageFormatFromExtension( + extension: unknown, +): ScreenshotImageFormat | undefined { + if (typeof extension !== 'string') { + return undefined; + } + + switch (extension.toLowerCase()) { + case 'png': + return 'png'; + case 'jpeg': + case 'jpg': + return 'jpeg'; + case 'webp': + return 'webp'; + default: + return undefined; + } +} + +export function screenshotImageFormatFromMimeType( + mimeType: unknown, +): ScreenshotImageFormat | undefined { + if (typeof mimeType !== 'string') { + return undefined; + } + + switch (mimeType.toLowerCase()) { + case 'image/png': + return 'png'; + case 'image/jpeg': + case 'image/jpg': + return 'jpeg'; + case 'image/webp': + return 'webp'; + default: + return undefined; + } +} + +export function isScreenshotImageMimeType( + mimeType: unknown, +): mimeType is ScreenshotImageMimeType { + return ( + mimeType === 'image/png' || + mimeType === 'image/jpeg' || + mimeType === 'image/webp' + ); +} + +export function inferScreenshotImageFormatFromBase64( + base64Body: string, +): ScreenshotImageFormat | undefined { + const normalizedBody = base64Body.replace(/\s/g, ''); + if (normalizedBody.startsWith('iVBORw0KGgo')) { + return 'png'; + } + if (normalizedBody.startsWith('/9j/')) { + return 'jpeg'; + } + if (normalizedBody.startsWith('UklGR')) { + return 'webp'; + } + return undefined; +} + +export function detectScreenshotImageFormatFromBuffer( + buffer: Uint8Array, +): ScreenshotImageFormat | undefined { + if ( + buffer.length >= 8 && + buffer[0] === 0x89 && + buffer[1] === 0x50 && + buffer[2] === 0x4e && + buffer[3] === 0x47 && + buffer[4] === 0x0d && + buffer[5] === 0x0a && + buffer[6] === 0x1a && + buffer[7] === 0x0a + ) { + return 'png'; + } + + if ( + buffer.length >= 3 && + buffer[0] === 0xff && + buffer[1] === 0xd8 && + buffer[2] === 0xff + ) { + return 'jpeg'; + } + + if ( + buffer.length >= 12 && + buffer[0] === 0x52 && + buffer[1] === 0x49 && + buffer[2] === 0x46 && + buffer[3] === 0x46 && + buffer[8] === 0x57 && + buffer[9] === 0x45 && + buffer[10] === 0x42 && + buffer[11] === 0x50 + ) { + return 'webp'; + } + + return undefined; +} diff --git a/packages/shared/src/img/index.ts b/packages/shared/src/img/index.ts index 9079c4e7f6..4754cc8a9e 100644 --- a/packages/shared/src/img/index.ts +++ b/packages/shared/src/img/index.ts @@ -2,13 +2,29 @@ export { imageInfoOfBase64, isValidPNGImageBuffer, isValidJPEGImageBuffer, + isValidWebPImageBuffer, isValidImageBuffer, validateScreenshotBuffer, type ValidateScreenshotBufferOptions, } from './info'; +export { + detectScreenshotImageFormatFromBuffer, + inferScreenshotImageFormatFromBase64, + isScreenshotImageMimeType, + screenshotImageExtension, + screenshotImageFormatFromExtension, + screenshotImageFormatFromMimeType, + screenshotImageMimeType, + type ScreenshotImageFormat, + type ScreenshotImageMimeType, +} from './image-format'; export { resizeAndConvertImgBuffer, convertImgBufferToJpeg, + convertImgBufferToWebp, + canonicalizeScreenshotBase64, + DEFAULT_WEBP_SCREENSHOT_EFFORT, + DEFAULT_WEBP_SCREENSHOT_QUALITY, resizeImgBase64, zoomForGPT4o, saveBase64Image, @@ -24,6 +40,8 @@ export { normalizeBase64Image, normalizeScreenshotBase64, type NormalizeScreenshotBase64Options, + type CanonicalizeScreenshotOptions, + type WebpScreenshotEncodeOptions, } from './transform'; export { processImageElementInfo, @@ -31,3 +49,7 @@ export { compositePointMarkerImg, annotateRects, } from './box-select'; +export { + encodeRgbaToWebp, + type BrowserWebpEncodeInput, +} from './browser-webp-encoder'; diff --git a/packages/shared/src/img/info.ts b/packages/shared/src/img/info.ts index a06d1b42fe..75cc13d656 100644 --- a/packages/shared/src/img/info.ts +++ b/packages/shared/src/img/info.ts @@ -4,6 +4,7 @@ import type { Size } from '../types'; import { ifInNode } from '../utils'; import getPhoton from './get-photon'; import getSharp from './get-sharp'; +import { detectScreenshotImageFormatFromBuffer } from './image-format'; export interface ImageInfo extends Size {} @@ -118,12 +119,25 @@ export function isValidJPEGImageBuffer(buffer: Buffer): boolean { } /** - * Check if the Buffer is a valid image (PNG or JPEG) + * Check if the Buffer has a WebP signature. * @param buffer The Buffer to check - * @returns true if the Buffer is a valid PNG or JPEG image, otherwise false + * @returns true if the Buffer has a WebP signature, otherwise false + */ +export function isValidWebPImageBuffer(buffer: Buffer): boolean { + return detectScreenshotImageFormatFromBuffer(buffer) === 'webp'; +} + +/** + * Check if the Buffer is a supported screenshot image (PNG, JPEG, or WebP) + * @param buffer The Buffer to check + * @returns true if the Buffer has a supported image signature, otherwise false */ export function isValidImageBuffer(buffer: Buffer): boolean { - return isValidPNGImageBuffer(buffer) || isValidJPEGImageBuffer(buffer); + return ( + isValidPNGImageBuffer(buffer) || + isValidJPEGImageBuffer(buffer) || + isValidWebPImageBuffer(buffer) + ); } export interface ValidateScreenshotBufferOptions { diff --git a/packages/shared/src/img/transform.ts b/packages/shared/src/img/transform.ts index caeb583015..c44c9c3064 100644 --- a/packages/shared/src/img/transform.ts +++ b/packages/shared/src/img/transform.ts @@ -7,11 +7,62 @@ import type { PhotonImage as PhotonImageType } from '@silvia-odwyer/photon'; import { getDebug } from '../logger'; import type { Rect } from '../types'; import { ifInNode } from '../utils'; +import { encodeRgbaToWebp } from './browser-webp-encoder'; import getPhoton from './get-photon'; import getSharp from './get-sharp'; +import { + type ScreenshotImageFormat, + detectScreenshotImageFormatFromBuffer, + inferScreenshotImageFormatFromBase64, + screenshotImageMimeType, +} from './image-format'; const imgDebug = getDebug('img'); +export const DEFAULT_WEBP_SCREENSHOT_QUALITY = 90; +export const DEFAULT_WEBP_SCREENSHOT_EFFORT = 1; + +export interface WebpScreenshotEncodeOptions { + /** Encoder quality from 0 to 100. Defaults to 90. */ + quality?: number; + /** Sharp encoder CPU effort from 0 to 6. Defaults to 1. */ + effort?: number; +} + +export interface CanonicalizeScreenshotOptions + extends WebpScreenshotEncodeOptions { + /** Keep a valid JPEG source byte-for-byte instead of applying another lossy encode. */ + preserveJpeg?: boolean; +} + +function assertWebpBuffer(buffer: Uint8Array, label: string): void { + if (detectScreenshotImageFormatFromBuffer(buffer) !== 'webp') { + throw new Error(`${label} did not produce a valid WebP image`); + } +} + +interface BrowserImagePixels { + get_raw_pixels(): Uint8Array; + get_width(): number; + get_height(): number; +} + +async function encodeBrowserImageToWebp( + image: BrowserImagePixels, + quality = DEFAULT_WEBP_SCREENSHOT_QUALITY, +): Promise { + const output = Buffer.from( + await encodeRgbaToWebp({ + pixels: image.get_raw_pixels(), + width: image.get_width(), + height: image.get_height(), + quality, + }), + ); + assertWebpBuffer(output, 'Browser image encoder'); + return output; +} + /** * Saves a Base64-encoded image to a file * @@ -33,7 +84,7 @@ export async function saveBase64Image(options: { /** * Resizes an image from Buffer, maybe return a new format - * - If the image is Resized, the returned format will be jpg. + * - If the image is resized, the returned format will be WebP. * - If the image is not Resized, it will return to its original format. * @returns { buffer: resized buffer, format: the new format} */ @@ -78,8 +129,12 @@ export async function resizeAndConvertImgBuffer( const resizedBuffer = await Sharp(inputData) .resize(newSize.width, newSize.height) - .jpeg({ quality: 90 }) + .webp({ + quality: DEFAULT_WEBP_SCREENSHOT_QUALITY, + effort: DEFAULT_WEBP_SCREENSHOT_EFFORT, + }) .toBuffer(); + assertWebpBuffer(resizedBuffer, 'Sharp resize'); const resizeEndTime = Date.now(); imgDebug( @@ -88,8 +143,7 @@ export async function resizeAndConvertImgBuffer( return { buffer: resizedBuffer, - // by Sharp.jpeg() - format: 'jpeg', + format: 'webp', }; } @@ -126,8 +180,7 @@ export async function resizeAndConvertImgBuffer( SamplingFilter.CatmullRom, ); - const outputBytes = outputImage.get_bytes_jpeg(90); - const resizedBuffer = Buffer.from(outputBytes); + const resizedBuffer = await encodeBrowserImageToWebp(outputImage); // Free memory inputImage.free(); @@ -141,8 +194,7 @@ export async function resizeAndConvertImgBuffer( return { buffer: resizedBuffer, - // by Photon.get_bytes_jpeg() - format: 'jpeg', + format: 'webp', }; } @@ -173,50 +225,53 @@ export async function convertImgBufferToJpeg( } } +/** Convert an image buffer to a validated WebP image without resizing it. */ +export async function convertImgBufferToWebp( + inputData: Buffer, + options: WebpScreenshotEncodeOptions = {}, +): Promise { + const quality = options.quality ?? DEFAULT_WEBP_SCREENSHOT_QUALITY; + const effort = options.effort ?? DEFAULT_WEBP_SCREENSHOT_EFFORT; + + if (ifInNode) { + const Sharp = await getSharp(); + const output = await Sharp(inputData).webp({ quality, effort }).toBuffer(); + assertWebpBuffer(output, 'Sharp'); + return output; + } + + const mimeType = detectImageMimeTypeFromBuffer(inputData); + if (!mimeType) { + throw new Error('Cannot encode WebP from an unsupported image buffer'); + } + const photonImage = await photonFromBase64( + `data:${mimeType};base64,${inputData.toString('base64')}`, + ); + try { + return await encodeBrowserImageToWebp(photonImage, quality); + } finally { + photonImage.free(); + } +} + const base64ImageDataUrlPattern = /^data:image\/[a-zA-Z0-9.+-]+;base64,/i; const supportedScreenshotDataUriPattern = - /^data:image\/(png|jpe?g);base64,([\s\S]*)$/i; + /^data:image\/(png|jpe?g|webp);base64,([\s\S]*)$/i; const rawBase64BodyPattern = /^[A-Za-z0-9+/=\s]+$/; -export const inferBase64ImageFormat = (base64Body: string) => { - if (base64Body.startsWith('iVBORw0KGgo')) { - return 'png'; - } - return 'jpeg'; -}; +export const inferBase64ImageFormat = ( + base64Body: string, +): ScreenshotImageFormat => + inferScreenshotImageFormatFromBase64(base64Body) ?? 'jpeg'; function detectImageMimeTypeFromBuffer(buffer: Buffer): string | undefined { - if ( - buffer.length >= 8 && - buffer[0] === 0x89 && - buffer[1] === 0x50 && - buffer[2] === 0x4e && - buffer[3] === 0x47 && - buffer[4] === 0x0d && - buffer[5] === 0x0a && - buffer[6] === 0x1a && - buffer[7] === 0x0a - ) { - return 'image/png'; - } - if ( - buffer.length >= 3 && - buffer[0] === 0xff && - buffer[1] === 0xd8 && - buffer[2] === 0xff - ) { - return 'image/jpeg'; + const screenshotFormat = detectScreenshotImageFormatFromBuffer(buffer); + if (screenshotFormat) { + return screenshotImageMimeType(screenshotFormat); } if (buffer.length >= 6 && buffer.subarray(0, 3).toString('ascii') === 'GIF') { return 'image/gif'; } - if ( - buffer.length >= 12 && - buffer.subarray(0, 4).toString('ascii') === 'RIFF' && - buffer.subarray(8, 12).toString('ascii') === 'WEBP' - ) { - return 'image/webp'; - } if (buffer.length >= 2 && buffer[0] === 0x42 && buffer[1] === 0x4d) { return 'image/bmp'; } @@ -243,10 +298,10 @@ export const normalizeScreenshotBase64 = ( const dataUriMatch = trimmedBase64.match(supportedScreenshotDataUriPattern); if (dataUriMatch) { - const imageFormat = + const imageFormat: ScreenshotImageFormat = dataUriMatch[1].toLowerCase() === 'jpg' ? 'jpeg' - : dataUriMatch[1].toLowerCase(); + : (dataUriMatch[1].toLowerCase() as ScreenshotImageFormat); const body = dataUriMatch[2]; if (!normalizeBase64Body(body)) { throw new Error(`${label} cannot be empty`); @@ -256,17 +311,22 @@ export const normalizeScreenshotBase64 = ( if (trimmedBase64.startsWith('data:')) { throw new Error( - `${label} must be a PNG/JPEG data URI or raw PNG base64 string`, + `${label} must be a PNG/JPEG/WebP data URI or raw PNG/WebP base64 string`, ); } if (!rawBase64BodyPattern.test(trimmedBase64)) { throw new Error( - `${label} must be a PNG/JPEG data URI or raw PNG base64 string`, + `${label} must be a PNG/JPEG/WebP data URI or raw PNG/WebP base64 string`, ); } - return createImgBase64ByFormat('png', trimmedBase64); + const base64Body = normalizeBase64Body(trimmedBase64); + const inferredFormat = inferScreenshotImageFormatFromBase64(base64Body); + return createImgBase64ByFormat( + inferredFormat === 'webp' ? 'webp' : 'png', + base64Body, + ); }; export const normalizeBase64Image = (base64: string) => { @@ -283,6 +343,38 @@ export const normalizeBase64Image = (base64: string) => { ); }; +/** + * Normalize a screenshot at the AI/report boundary. + * + * Valid WebP is passed through byte-for-byte. Callers can also preserve JPEG + * sources to avoid a second lossy encode for native MJPEG/HDC streams. + */ +export async function canonicalizeScreenshotBase64( + inputBase64: string, + options: CanonicalizeScreenshotOptions = {}, +): Promise { + const { body } = parseBase64(inputBase64); + const inputBuffer = Buffer.from(body, 'base64'); + const inputFormat = detectScreenshotImageFormatFromBuffer(inputBuffer); + if (!inputFormat) { + throw new Error('Cannot canonicalize an unsupported screenshot image'); + } + + if ( + inputFormat === 'webp' || + (inputFormat === 'jpeg' && options.preserveJpeg) + ) { + return createImgBase64ByFormat(inputFormat, body); + } + + const startedAt = Date.now(); + const output = await convertImgBufferToWebp(inputBuffer, options); + imgDebug( + `canonicalizeScreenshot done, ${inputFormat}->webp, bytes: ${inputBuffer.length}->${output.length}, cost: ${Date.now() - startedAt}ms`, + ); + return createImgBase64ByFormat('webp', output.toString('base64')); +} + export async function resizeImgBase64( inputBase64: string, newSize: { @@ -425,12 +517,16 @@ export async function paddingToMatchBlockByBase64( bottom: targetHeight - height, background: { r: 255, g: 255, b: 255, alpha: 1 }, }) - .jpeg({ quality: 90 }) + .webp({ + quality: DEFAULT_WEBP_SCREENSHOT_QUALITY, + effort: DEFAULT_WEBP_SCREENSHOT_EFFORT, + }) .toBuffer(); + assertWebpBuffer(output, 'Sharp padding'); return { width: targetWidth, height: targetHeight, - imageBase64: createImgBase64ByFormat('jpeg', output.toString('base64')), + imageBase64: createImgBase64ByFormat('webp', output.toString('base64')), }; } @@ -473,12 +569,16 @@ export async function cropByRect( width, height, }) - .jpeg({ quality: 90 }) + .webp({ + quality: DEFAULT_WEBP_SCREENSHOT_QUALITY, + effort: DEFAULT_WEBP_SCREENSHOT_EFFORT, + }) .toBuffer(); + assertWebpBuffer(output, 'Sharp crop'); return { width, height, - imageBase64: createImgBase64ByFormat('jpeg', output.toString('base64')), + imageBase64: createImgBase64ByFormat('webp', output.toString('base64')), }; } @@ -503,11 +603,10 @@ export async function cropByRect( export async function photonToBase64( image: PhotonImageType, - quality = 90, + quality = DEFAULT_WEBP_SCREENSHOT_QUALITY, ): Promise { - const bytes = image.get_bytes_jpeg(quality); - const base64Body = Buffer.from(bytes).toString('base64'); - return `data:image/jpeg;base64,${base64Body}`; + const bytes = await encodeBrowserImageToWebp(image, quality); + return createImgBase64ByFormat('webp', bytes.toString('base64')); } export const httpImg2Base64 = async (url: string): Promise => { @@ -657,17 +756,22 @@ export async function scaleImage( kernel: 'lanczos3', fit: 'fill', }) - .jpeg({ - quality: 90, + .webp({ + quality: DEFAULT_WEBP_SCREENSHOT_QUALITY, + effort: DEFAULT_WEBP_SCREENSHOT_EFFORT, }) .toBuffer(); + assertWebpBuffer(resizedBuffer, 'Sharp scale'); const scaleEndTime = Date.now(); imgDebug( `scaleImage done (Sharp): ${originalWidth}x${originalHeight} -> ${newWidth}x${newHeight} (scale=${scale}), cost: ${scaleEndTime - scaleStartTime}ms`, ); - const base64 = `data:image/jpeg;base64,${resizedBuffer.toString('base64')}`; + const base64 = createImgBase64ByFormat( + 'webp', + resizedBuffer.toString('base64'), + ); return { width: newWidth, @@ -703,8 +807,7 @@ export async function scaleImage( SamplingFilter.CatmullRom, ); - const outputBytes = outputImage.get_bytes_jpeg(90); - const resizedBuffer = Buffer.from(outputBytes); + const resizedBuffer = await encodeBrowserImageToWebp(outputImage); // Free memory inputImage.free(); @@ -715,7 +818,10 @@ export async function scaleImage( `scaleImage done (Photon): ${originalWidth}x${originalHeight} -> ${newWidth}x${newHeight} (scale=${scale}), cost: ${scaleEndTime - scaleStartTime}ms`, ); - const base64 = `data:image/jpeg;base64,${resizedBuffer.toString('base64')}`; + const base64 = createImgBase64ByFormat( + 'webp', + resizedBuffer.toString('base64'), + ); return { width: newWidth, diff --git a/packages/shared/tests/unit-test/image/index.test.ts b/packages/shared/tests/unit-test/image/index.test.ts index f5f9adba62..a20c4c93cd 100644 --- a/packages/shared/tests/unit-test/image/index.test.ts +++ b/packages/shared/tests/unit-test/image/index.test.ts @@ -8,6 +8,7 @@ import { httpImg2Base64, imageInfoOfBase64, isValidPNGImageBuffer, + isValidWebPImageBuffer, localImg2Base64, resizeAndConvertImgBuffer, resizeImgBase64, @@ -22,6 +23,10 @@ import { } from '../../../src/img/transform'; import { getFixture } from '../../utils'; +const webpBase64 = + 'UklGRjQAAABXRUJQVlA4ICgAAACQAQCdASoCAAMAAMASJQBOl0AAjNAA/v4icv1difCfoP7mxzi2QwAA'; +const webpDataUri = `data:image/webp;base64,${webpBase64}`; + describe('imageInfoOfBase64', () => { it('returns correct dimensions for PNG image', async () => { const image = getFixture('icon.png'); @@ -41,6 +46,13 @@ describe('imageInfoOfBase64', () => { expect(info.height).toBe(905); }); + it('returns correct dimensions for WebP image', async () => { + await expect(imageInfoOfBase64(webpDataUri)).resolves.toEqual({ + width: 2, + height: 3, + }); + }); + it('works with base64 string without data URI header', async () => { const image = getFixture('icon.png'); const base64WithHeader = localImg2Base64(image); @@ -320,6 +332,15 @@ describe('image utils', () => { ).toThrow('Screenshot buffer has invalid image format'); }); + it('isValidWebPImageBuffer accepts WebP and rejects malformed RIFF data', () => { + expect(isValidWebPImageBuffer(Buffer.from(webpBase64, 'base64'))).toBe( + true, + ); + expect(isValidWebPImageBuffer(Buffer.from('RIFF1234NOPE', 'ascii'))).toBe( + false, + ); + }); + it('validateScreenshotBuffer accepts valid screenshots above the minimum size', () => { const buffer = readFileSync(getFixture('icon.png')); @@ -466,7 +487,7 @@ describe('resizeAndConvertImgBuffer', () => { ); expect(format).toBe('png'); }); - it('Sharp resize will get jpeg format', async () => { + it('Sharp resize will get WebP format', async () => { const { format, buffer } = await resizeAndConvertImgBuffer( 'png', imageBuffer, @@ -475,7 +496,9 @@ describe('resizeAndConvertImgBuffer', () => { height: 1, }, ); - expect(format).toBe('jpeg'); + expect(format).toBe('webp'); + expect(buffer.subarray(0, 4).toString('ascii')).toBe('RIFF'); + expect(buffer.subarray(8, 12).toString('ascii')).toBe('WEBP'); }); }); diff --git a/packages/shared/tests/unit-test/transform.test.ts b/packages/shared/tests/unit-test/transform.test.ts index cb7bbedf9f..e595175d7e 100644 --- a/packages/shared/tests/unit-test/transform.test.ts +++ b/packages/shared/tests/unit-test/transform.test.ts @@ -1,6 +1,8 @@ import path from 'node:path'; import { describe, expect, it, vi } from 'vitest'; +import { encodeRgbaToWebp } from '../../src/img/browser-webp-encoder'; import { + canonicalizeScreenshotBase64, inferBase64ImageFormat, normalizeBase64Image, normalizeScreenshotBase64, @@ -8,6 +10,39 @@ import { scaleImage, } from '../../src/img/transform'; +describe('encodeRgbaToWebp', () => { + it('validates dimensions before selecting a browser encoder', async () => { + await expect( + encodeRgbaToWebp({ + pixels: [], + width: 0, + height: 1, + }), + ).rejects.toThrow('WebP image dimensions must be positive safe integers'); + }); + + it('validates the RGBA pixel length', async () => { + await expect( + encodeRgbaToWebp({ + pixels: [255, 255, 255], + width: 1, + height: 1, + }), + ).rejects.toThrow('WebP RGBA pixel length must be 4, got 3'); + }); + + it('validates encoder quality', async () => { + await expect( + encodeRgbaToWebp({ + pixels: [255, 255, 255, 255], + width: 1, + height: 1, + quality: 101, + }), + ).rejects.toThrow('WebP quality must be between 0 and 100'); + }); +}); + describe('preapareImageUrl', () => { it('url is not a string will throw an error', async () => { await expect(preProcessImageUrl(1 as any, false)).rejects.toThrowError( @@ -95,16 +130,25 @@ describe('normalizeBase64Image', () => { 'data:image/jpeg;base64,/9j/4AAQSkZJRg==', ); }); + + it('wraps bare WebP base64 with the WebP MIME type', () => { + expect(normalizeBase64Image(' UklGRjQAAABXRUJQ VlA4IA== ')).toBe( + 'data:image/webp;base64,UklGRjQAAABXRUJQVlA4IA==', + ); + }); }); describe('normalizeScreenshotBase64', () => { - it('accepts PNG and JPEG data urls', () => { + it('accepts PNG, JPEG, and WebP data urls', () => { expect( normalizeScreenshotBase64(' data:image/png;base64,aaa\r\nbbb '), ).toBe('data:image/png;base64,aaabbb'); expect(normalizeScreenshotBase64('data:image/jpeg;base64,/9j/4AAQ')).toBe( 'data:image/jpeg;base64,/9j/4AAQ', ); + expect( + normalizeScreenshotBase64('data:image/webp;base64,UklGRjQAAA=='), + ).toBe('data:image/webp;base64,UklGRjQAAA=='); }); it('normalizes jpg data urls to jpeg', () => { @@ -119,6 +163,12 @@ describe('normalizeScreenshotBase64', () => { ); }); + it('recognizes raw WebP base64', () => { + expect(normalizeScreenshotBase64(' UklGRjQAAA BXRUJQ ')).toBe( + 'data:image/webp;base64,UklGRjQAAABXRUJQ', + ); + }); + it('uses the provided label in validation errors', () => { expect(() => normalizeScreenshotBase64(' ', { label: 'custom screenshot' }), @@ -129,14 +179,15 @@ describe('normalizeScreenshotBase64', () => { label: 'custom screenshot', }), ).toThrow( - 'custom screenshot must be a PNG/JPEG data URI or raw PNG base64 string', + 'custom screenshot must be a PNG/JPEG/WebP data URI or raw PNG/WebP base64 string', ); }); }); describe('inferBase64ImageFormat', () => { - it('detects png payloads and otherwise falls back to jpeg', () => { + it('detects PNG and WebP payloads and otherwise falls back to JPEG', () => { expect(inferBase64ImageFormat('iVBORw0KGgoaaa')).toBe('png'); + expect(inferBase64ImageFormat('UklGRjQAAABXRUJQ')).toBe('webp'); expect(inferBase64ImageFormat('/9j/4AAQSkZJRg==')).toBe('jpeg'); }); }); @@ -152,7 +203,7 @@ describe('scaleImage', () => { expect(result.width).toBe(2); expect(result.height).toBe(2); expect(result.imageBase64).toMatchInlineSnapshot( - `"data:image/jpeg;base64,/9j/2wBDAAMCAgMCAgMDAwMEAwMEBQgFBQQEBQoHBwYIDAoMDAsKCwsNDhIQDQ4RDgsLEBYQERMUFRUVDA8XGBYUGBIUFRT/2wBDAQMEBAUEBQkFBQkUDQsNFBQUFBQUFBQUFBQUFBQUFBQUFBQUFBQUFBQUFBQUFBQUFBQUFBQUFBQUFBQUFBQUFBT/wAARCAACAAIDASIAAhEBAxEB/8QAFQABAQAAAAAAAAAAAAAAAAAAAAn/xAAUEAEAAAAAAAAAAAAAAAAAAAAA/8QAFAEBAAAAAAAAAAAAAAAAAAAAAP/EABQRAQAAAAAAAAAAAAAAAAAAAAD/2gAMAwEAAhEDEQA/AKpgA//Z"`, + `"data:image/webp;base64,UklGRioAAABXRUJQVlA4IB4AAACQAQCdASoCAAIAAMASJaQAAzoO0gAA/v/+JwoMAAA="`, ); }); @@ -162,7 +213,7 @@ describe('scaleImage', () => { expect(result.width).toBe(3); expect(result.height).toBe(3); expect(result.imageBase64).toMatchInlineSnapshot( - `"data:image/jpeg;base64,/9j/2wBDAAMCAgMCAgMDAwMEAwMEBQgFBQQEBQoHBwYIDAoMDAsKCwsNDhIQDQ4RDgsLEBYQERMUFRUVDA8XGBYUGBIUFRT/2wBDAQMEBAUEBQkFBQkUDQsNFBQUFBQUFBQUFBQUFBQUFBQUFBQUFBQUFBQUFBQUFBQUFBQUFBQUFBQUFBQUFBQUFBT/wAARCAADAAMDASIAAhEBAxEB/8QAFQABAQAAAAAAAAAAAAAAAAAAAAn/xAAUEAEAAAAAAAAAAAAAAAAAAAAA/8QAFAEBAAAAAAAAAAAAAAAAAAAAAP/EABQRAQAAAAAAAAAAAAAAAAAAAAD/2gAMAwEAAhEDEQA/AKpgA//Z"`, + `"data:image/webp;base64,UklGRioAAABXRUJQVlA4IB4AAACQAQCdASoDAAMAAMASJaQAAzoO0gAA/v/+JwoMAAA="`, ); }); @@ -209,3 +260,30 @@ describe('scaleImage', () => { vi.restoreAllMocks(); }); }); + +describe('canonicalizeScreenshotBase64', () => { + const onePixelWhiteImage = + 'data:image/jpeg;base64,/9j/4AAQSkZJRgABAQAAAQABAAD/2wCEAAMDAwMDAwMDAwMEBAQEBAYFBQUFBgkGBwYHBgkOCAoICAoIDgwPDAsMDwwWEQ8PERYZFRQVGR4bGx4mJCYyMkMBAwMDAwMDAwMDAwQEBAQEBgUFBQUGCQYHBgcGCQ4ICggICggODA8MCwwPDBYRDw8RFhkVFBUZHhsbHiYkJjIyQ//CABEIAAEAAQMBIgACEQEDEQH/xAAnAAEBAAAAAAAAAAAAAAAAAAAACQEBAAAAAAAAAAAAAAAAAAAAAP/aAAwDAQACEAMQAAAAqmD/xAAUEAEAAAAAAAAAAAAAAAAAAAAA/9oACAEBAAE/AH//xAAUEQEAAAAAAAAAAAAAAAAAAAAA/9oACAECAQE/AH//xAAUEQEAAAAAAAAAAAAAAAAAAAAA/9oACAEDAQE/AH//2Q=='; + + it('passes an existing WebP through byte-for-byte', async () => { + const webp = await canonicalizeScreenshotBase64(onePixelWhiteImage); + expect(await canonicalizeScreenshotBase64(webp)).toBe(webp); + }); + + it('preserves native JPEG only when auto policy requests it', async () => { + expect( + await canonicalizeScreenshotBase64(onePixelWhiteImage, { + preserveJpeg: true, + }), + ).toBe(onePixelWhiteImage); + expect(await canonicalizeScreenshotBase64(onePixelWhiteImage)).toMatch( + /^data:image\/webp;base64,UklGR/, + ); + }); + + it('rejects unsupported image bytes instead of returning a blank image', async () => { + await expect( + canonicalizeScreenshotBase64('data:image/png;base64,aGVsbG8='), + ).rejects.toThrow('unsupported screenshot image'); + }); +}); From 1f21c1ddeff6b808c5dc5677a8c59c7c7787b98a Mon Sep 17 00:00:00 2001 From: quanruzhuoxiu Date: Thu, 23 Jul 2026 19:55:31 +0800 Subject: [PATCH 2/2] test(core): align screenshot format assertion --- packages/core/tests/unit-test/agent-dump-update.test.ts | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/packages/core/tests/unit-test/agent-dump-update.test.ts b/packages/core/tests/unit-test/agent-dump-update.test.ts index 9f9bda4066..77f2a5c809 100644 --- a/packages/core/tests/unit-test/agent-dump-update.test.ts +++ b/packages/core/tests/unit-test/agent-dump-update.test.ts @@ -255,7 +255,7 @@ describe('Agent dump update screenshot serialization', () => { screenshots: [{ base64: 'data:image/svg+xml;base64,custom' }], }), ).rejects.toThrow( - 'recordToReport: screenshot #1 base64 must be a PNG/JPEG data URI or raw PNG base64 string', + 'recordToReport: screenshot #1 base64 must be a PNG/JPEG/WebP data URI or raw PNG/WebP base64 string', ); expect(screenshotBase64).not.toHaveBeenCalled();