diff --git a/apps/chrome-extension/src/extension/recorder/screenshot-export.ts b/apps/chrome-extension/src/extension/recorder/screenshot-export.ts new file mode 100644 index 0000000000..02415ec442 --- /dev/null +++ b/apps/chrome-extension/src/extension/recorder/screenshot-export.ts @@ -0,0 +1,97 @@ +import { + type ScreenshotImageFormat, + type ScreenshotImageMimeType, + inferScreenshotImageFormatFromBase64, + screenshotImageExtension, + screenshotImageMimeType, +} from '@midscene/shared/img/image-format'; +import type { RecordingSession } from '../../store'; + +export interface RecorderScreenshotAsset { + body: string; + extension: ScreenshotImageFormat; + mimeType: ScreenshotImageMimeType; +} + +export const recorderScreenshotAsset = ( + screenshotBase64: string, +): RecorderScreenshotAsset => { + const separator = ';base64,'; + const separatorIndex = screenshotBase64.indexOf(separator); + const body = ( + separatorIndex === -1 + ? screenshotBase64 + : screenshotBase64.slice(separatorIndex + separator.length) + ).replace(/\s/g, ''); + const format = inferScreenshotImageFormatFromBase64(body); + if (!format) { + throw new Error('Unsupported recorder screenshot image format'); + } + + return { + body, + extension: screenshotImageExtension(format), + mimeType: screenshotImageMimeType(format), + }; +}; + +export const generateEventsMarkdownTable = ( + sessions: RecordingSession[], +): string => { + let markdown = '# Test Events Report\n\n'; + + sessions.forEach((session, sessionIndex) => { + if (session.events.length === 0) return; + + markdown += `## ${session.name}\n\n`; + if (session.description) { + markdown += `**Description:** ${session.description}\n\n`; + } + markdown += `**Created:** ${new Date(session.createdAt).toLocaleString()}\n\n`; + + markdown += '| Page | Screenshot Before | Screenshot After | Action |\n'; + markdown += '|------|------------|------------|--------|\n'; + + session.events.forEach((event, eventIndex) => { + const page = event.title || event.url || ''; + const screenshotBefore = event.screenshotBefore + ? `![](./images/screenshot_${sessionIndex}_${eventIndex}_before.${recorderScreenshotAsset(event.screenshotBefore).extension})` + : 'N/A'; + const screenshotAfter = event.screenshotAfter + ? `![](./images/screenshot_${sessionIndex}_${eventIndex}_after.${recorderScreenshotAsset(event.screenshotAfter).extension})` + : 'N/A'; + let action = ''; + switch (event.type) { + case 'click': + action = `Click on ${event.elementDescription || 'element'}`; + break; + case 'input': + action = `Input "${event.value}" into ${event.elementDescription || 'field'}`; + break; + case 'navigation': + action = `Navigate to ${event.url}`; + break; + default: + action = `${event.type} on ${event.elementDescription || 'element'}`; + } + + markdown += `| ${page} | ${screenshotBefore} | ${screenshotAfter} | ${action} |\n`; + }); + + if (session.generatedCode?.yaml || session.generatedCode?.playwright) { + markdown += '## Generated Code\n\n'; + if (session.generatedCode?.yaml) { + markdown += '### YAML\n\n'; + markdown += `\`\`\`yaml\n${session.generatedCode.yaml}\n\`\`\`\n\n`; + } + if (session.generatedCode?.playwright) { + markdown += '### Playwright\n\n'; + markdown += `\`\`\`playwright\n${session.generatedCode.playwright}\n\`\`\`\n\n`; + } + } + + markdown += '\n\n\n'; + }); + + return markdown; +}; diff --git a/apps/chrome-extension/src/extension/recorder/utils.ts b/apps/chrome-extension/src/extension/recorder/utils.ts index 628b77865e..133473e0e8 100644 --- a/apps/chrome-extension/src/extension/recorder/utils.ts +++ b/apps/chrome-extension/src/extension/recorder/utils.ts @@ -12,6 +12,11 @@ import JSZip from 'jszip'; import type { ChatCompletionContentPart } from 'openai/resources/index'; import type { RecordingSession } from '../../store'; import { recordLogger } from './logger'; +import { + type RecorderScreenshotAsset, + generateEventsMarkdownTable, + recorderScreenshotAsset, +} from './screenshot-export'; import { isChromeExtension, safeChromeAPI } from './types'; // Generate default session name with current time @@ -713,74 +718,12 @@ const generateDetailedSequentialMindmap = ( return mermaid; }; -// Generate markdown table for event details -const generateEventsMarkdownTable = (sessions: RecordingSession[]): string => { - let markdown = '# Test Events Report\n\n'; - - sessions.forEach((session, sessionIndex) => { - if (session.events.length === 0) return; - - markdown += `## ${session.name}\n\n`; - if (session.description) { - markdown += `**Description:** ${session.description}\n\n`; - } - markdown += `**Created:** ${new Date(session.createdAt).toLocaleString()}\n\n`; - - markdown += '| Page | Screenshot Before | Screenshot After | Action |\n'; - markdown += '|------|------------|------------|--------|\n'; - - session.events.forEach((event, eventIndex) => { - let expected = 'N/A'; - const page = event.title || event.url || ''; - const screenshotBefore = event.screenshotBefore - ? `![](./images/screenshot_${sessionIndex}_${eventIndex}_before.png)` - : 'N/A'; - const screenshotAfter = event.screenshotAfter - ? `![](./images/screenshot_${sessionIndex}_${eventIndex}_after.png)` - : 'N/A'; - if (event.type === 'navigation') { - expected = `Navigate to ${event.url}`; - } - - let action = ''; - switch (event.type) { - case 'click': - action = `Click on ${event.elementDescription || 'element'}`; - break; - case 'input': - action = `Input "${event.value}" into ${event.elementDescription || 'field'}`; - break; - case 'navigation': - action = `Navigate to ${event.url}`; - break; - default: - action = `${event.type} on ${event.elementDescription || 'element'}`; - } - - markdown += `| ${page} | ${screenshotBefore} | ${screenshotAfter} | ${action} |\n`; - }); - - if (session.generatedCode?.yaml || session.generatedCode?.playwright) { - markdown += '## Generated Code\n\n'; - if (session.generatedCode?.yaml) { - markdown += '### YAML\n\n'; - markdown += `\`\`\`yaml\n${session.generatedCode.yaml}\n\`\`\`\n\n`; - } - if (session.generatedCode?.playwright) { - markdown += '### Playwright\n\n'; - markdown += `\`\`\`playwright\n${session.generatedCode.playwright}\n\`\`\`\n\n`; - } - } - - markdown += '\n\n\n'; - }); - - return markdown; -}; - // Convert base64 to blob -const base64ToBlob = (base64: string, mimeType: string): Blob => { - const byteCharacters = atob(base64.split(',')[1]); +const base64BodyToBlob = ( + base64Body: string, + mimeType: RecorderScreenshotAsset['mimeType'], +): Blob => { + const byteCharacters = atob(base64Body); const byteNumbers = new Array(byteCharacters.length); for (let i = 0; i < byteCharacters.length; i++) { byteNumbers[i] = byteCharacters.charCodeAt(i); @@ -810,15 +753,16 @@ export const exportAllEventsToZip = async (sessions: RecordingSession[]) => { // Process each session and extract images sessionsWithEvents.forEach((session, sessionIndex) => { session.events.forEach((event, eventIndex) => { - const ext = 'png'; if (event.screenshotBefore) { - const fileName = `screenshot_${sessionIndex}_${eventIndex}_before.${ext}`; - const blob = base64ToBlob(event.screenshotBefore, `image/${ext}`); + const asset = recorderScreenshotAsset(event.screenshotBefore); + const fileName = `screenshot_${sessionIndex}_${eventIndex}_before.${asset.extension}`; + const blob = base64BodyToBlob(asset.body, asset.mimeType); imagesFolder?.file(fileName, blob); } if (event.screenshotAfter) { - const fileName = `screenshot_${sessionIndex}_${eventIndex}_after.${ext}`; - const blob = base64ToBlob(event.screenshotAfter, `image/${ext}`); + const asset = recorderScreenshotAsset(event.screenshotAfter); + const fileName = `screenshot_${sessionIndex}_${eventIndex}_after.${asset.extension}`; + const blob = base64BodyToBlob(asset.body, asset.mimeType); imagesFolder?.file(fileName, blob); } }); diff --git a/apps/chrome-extension/src/scripts/worker.ts b/apps/chrome-extension/src/scripts/worker.ts index 48b7fe1be0..295d7b42ca 100644 --- a/apps/chrome-extension/src/scripts/worker.ts +++ b/apps/chrome-extension/src/scripts/worker.ts @@ -4,6 +4,7 @@ import type { UIContext } from '@midscene/core'; import { uuid } from '@midscene/shared/utils'; import { BridgeConnector, type BridgeStatus } from '../utils/bridgeConnector'; import { registerAlarmListener, safeSetupKeepalive } from '../utils/keepalive'; +import { canonicalizeRecorderScreenshot } from '../utils/screenshot'; import { workerMessageTypes } from '../utils/workerMessageTypes'; // save screenshot @@ -448,8 +449,22 @@ chrome.runtime.onMessage.addListener((request, sender, sendResponse) => { chrome.runtime.lastError, ); sendResponse(null); + } else if (!dataUrl) { + console.error( + '[ServiceWorker] Screenshot capture returned empty data', + ); + sendResponse(null); } else { - sendResponse(dataUrl); + void canonicalizeRecorderScreenshot(dataUrl).then( + (webpDataUrl) => sendResponse(webpDataUrl), + (error) => { + console.error( + '[ServiceWorker] Failed to encode recorder screenshot as WebP:', + error, + ); + sendResponse(null); + }, + ); } }, ); diff --git a/apps/chrome-extension/src/utils/screenshot.ts b/apps/chrome-extension/src/utils/screenshot.ts new file mode 100644 index 0000000000..d9ca300f83 --- /dev/null +++ b/apps/chrome-extension/src/utils/screenshot.ts @@ -0,0 +1,77 @@ +const WEBP_QUALITY = 0.9; +const DATA_URL_PATTERN = /^data:image\/(png|jpe?g|webp);base64,([\s\S]+)$/i; + +function decodeScreenshotDataUrl(dataUrl: string): { + bytes: Uint8Array; + mimeType: string; +} { + const match = DATA_URL_PATTERN.exec(dataUrl); + if (!match) { + throw new Error( + 'Recorder screenshot must be a PNG, JPEG, or WebP data URL', + ); + } + + const binary = atob(match[2]); + const bytes = Uint8Array.from(binary, (character) => character.charCodeAt(0)); + return { + bytes, + mimeType: `image/${match[1].toLowerCase().replace('jpg', 'jpeg')}`, + }; +} + +function isWebp(bytes: Uint8Array): boolean { + return ( + bytes.length >= 12 && + String.fromCharCode(...bytes.subarray(0, 4)) === 'RIFF' && + String.fromCharCode(...bytes.subarray(8, 12)) === 'WEBP' + ); +} + +function encodeBase64(bytes: Uint8Array): string { + let binary = ''; + const chunkSize = 0x8000; + for (let offset = 0; offset < bytes.length; offset += chunkSize) { + binary += String.fromCharCode( + ...bytes.subarray(offset, offset + chunkSize), + ); + } + return btoa(binary); +} + +/** Convert captureVisibleTab's PNG output to the recorder's canonical WebP. */ +export async function canonicalizeRecorderScreenshot( + dataUrl: string, +): Promise { + const source = decodeScreenshotDataUrl(dataUrl); + if (source.mimeType === 'image/webp') { + if (!isWebp(source.bytes)) { + throw new Error('Recorder screenshot has an invalid WebP signature'); + } + return dataUrl; + } + + const bitmap = await createImageBitmap( + new Blob([source.bytes], { type: source.mimeType }), + ); + try { + const canvas = new OffscreenCanvas(bitmap.width, bitmap.height); + const context = canvas.getContext('2d'); + if (!context) { + throw new Error('Recorder screenshot WebP encoder is unavailable'); + } + context.drawImage(bitmap, 0, 0); + + const output = await canvas.convertToBlob({ + type: 'image/webp', + quality: WEBP_QUALITY, + }); + const bytes = new Uint8Array(await output.arrayBuffer()); + if (output.type !== 'image/webp' || !isWebp(bytes)) { + throw new Error('Recorder screenshot encoder returned invalid WebP'); + } + return `data:image/webp;base64,${encodeBase64(bytes)}`; + } finally { + bitmap.close(); + } +} diff --git a/apps/chrome-extension/tests/recorder-export-image-format.test.ts b/apps/chrome-extension/tests/recorder-export-image-format.test.ts new file mode 100644 index 0000000000..039b97348a --- /dev/null +++ b/apps/chrome-extension/tests/recorder-export-image-format.test.ts @@ -0,0 +1,57 @@ +import { describe, expect, it } from 'vitest'; +import { + generateEventsMarkdownTable, + recorderScreenshotAsset, +} from '../src/extension/recorder/screenshot-export'; +import type { RecordingSession } from '../src/store'; + +const webpBody = + 'UklGRioAAABXRUJQVlA4IB4AAAAwAQCdASoBAAEAAUAmJQBOgCHwAP7+hNQAAAA='; +const pngBody = + 'iVBORw0KGgoAAAANSUhEUgAAAAEAAAABCAQAAAC1HAwCAAAAC0lEQVR42mNk+M9QDwADhgGAWjR9awAAAABJRU5ErkJggg=='; + +describe('Chrome recorder screenshot export', () => { + it('derives the exported extension and MIME type from the image bytes', () => { + expect( + recorderScreenshotAsset(`data:image/png;base64,${webpBody}`), + ).toEqual({ + body: webpBody, + extension: 'webp', + mimeType: 'image/webp', + }); + expect(recorderScreenshotAsset(`data:image/png;base64,${pngBody}`)).toEqual( + { + body: pngBody, + extension: 'png', + mimeType: 'image/png', + }, + ); + }); + + it('uses each screenshot actual extension in the Markdown table', () => { + const sessions = [ + { + id: 'session-1', + name: 'WebP export', + createdAt: 1, + updatedAt: 1, + status: 'completed', + events: [ + { + type: 'click', + timestamp: 1, + hashId: 'event-1', + screenshotBefore: `data:image/webp;base64,${webpBody}`, + screenshotAfter: `data:image/png;base64,${pngBody}`, + }, + ], + }, + ] as RecordingSession[]; + + const markdown = generateEventsMarkdownTable(sessions); + + expect(markdown).toContain('![](./images/screenshot_0_0_before.webp)'); + expect(markdown).toContain('![](./images/screenshot_0_0_after.png)'); + expect(markdown).not.toContain('screenshot_0_0_before.png'); + }); +}); diff --git a/apps/chrome-extension/tests/screenshot.test.ts b/apps/chrome-extension/tests/screenshot.test.ts new file mode 100644 index 0000000000..83b27d84c1 --- /dev/null +++ b/apps/chrome-extension/tests/screenshot.test.ts @@ -0,0 +1,58 @@ +import { Buffer } from 'node:buffer'; +import { afterEach, beforeEach, describe, expect, it, vi } from 'vitest'; +import { canonicalizeRecorderScreenshot } from '../src/utils/screenshot'; + +describe('canonicalizeRecorderScreenshot', () => { + const pngDataUrl = + 'data:image/png;base64,iVBORw0KGgoAAAANSUhEUgAAAAEAAAABCAQAAAC1HAwCAAAAC0lEQVR42mNk+M9QDwADhgGAWjR9awAAAABJRU5ErkJggg=='; + const webpBytes = Buffer.from( + 'UklGRioAAABXRUJQVlA4IB4AAAAwAQCdASoBAAEAAUAmJQBOgCHwAP7+hNQAAAA=', + 'base64', + ); + const drawImage = vi.fn(); + const close = vi.fn(); + const convertToBlob = vi.fn( + async () => new Blob([webpBytes], { type: 'image/webp' }), + ); + + beforeEach(() => { + vi.stubGlobal( + 'createImageBitmap', + vi.fn(async () => ({ width: 1, height: 1, close })), + ); + vi.stubGlobal( + 'OffscreenCanvas', + class { + getContext() { + return { drawImage }; + } + + convertToBlob = convertToBlob; + }, + ); + }); + + afterEach(() => { + vi.unstubAllGlobals(); + vi.clearAllMocks(); + }); + + it('turns captureVisibleTab PNG output into a real WebP image', async () => { + const result = await canonicalizeRecorderScreenshot(pngDataUrl); + const bytes = Buffer.from(result.split(',')[1], 'base64'); + + expect(result).toMatch(/^data:image\/webp;base64,/); + expect(bytes.subarray(0, 4).toString('ascii')).toBe('RIFF'); + expect(bytes.subarray(8, 12).toString('ascii')).toBe('WEBP'); + expect(convertToBlob).toHaveBeenCalledWith({ + type: 'image/webp', + quality: 0.9, + }); + }); + + it('passes an existing WebP through without re-encoding', async () => { + const webp = `data:image/webp;base64,${webpBytes.toString('base64')}`; + await expect(canonicalizeRecorderScreenshot(webp)).resolves.toBe(webp); + expect(convertToBlob).not.toHaveBeenCalled(); + }); +}); diff --git a/packages/playground/src/mjpeg-stream-handler.ts b/packages/playground/src/mjpeg-stream-handler.ts index db105b4751..7a8f6f7992 100644 --- a/packages/playground/src/mjpeg-stream-handler.ts +++ b/packages/playground/src/mjpeg-stream-handler.ts @@ -1,5 +1,9 @@ import http from 'node:http'; import type { Agent as PageAgent } from '@midscene/core/agent'; +import { + inferScreenshotImageFormatFromBase64, + screenshotImageMimeType, +} from '@midscene/shared/img/image-format'; import { getDebug } from '@midscene/shared/logger'; import type { Request, Response } from 'express'; import { @@ -29,6 +33,18 @@ function toMjpegFrameDataUrl(data: string, contentType?: string) { return `data:${contentType || 'image/jpeg'};base64,${data}`; } +function screenshotContentType(data: string): string { + const dataUriMatch = data.match(/^data:image\/(png|jpe?g|webp);base64,/i); + if (dataUriMatch) { + const format = dataUriMatch[1].toLowerCase(); + return format === 'jpg' ? 'image/jpeg' : `image/${format}`; + } + + const body = data.replace(/^data:image\/[^;]+;base64,/i, ''); + const format = inferScreenshotImageFormatFromBase64(body); + return format ? screenshotImageMimeType(format) : 'image/jpeg'; +} + /** * Inputs the handler reads on every request, late-bound through callbacks * so a single handler instance can survive across device reconnects without @@ -263,7 +279,7 @@ export class MjpegStreamHandler { writeMjpegFrame(res, boundary, { data: base64, - contentType: 'image/jpeg', + contentType: screenshotContentType(base64), }); } catch (err) { if (stopped) break; diff --git a/packages/playground/src/server.ts b/packages/playground/src/server.ts index 01c8afb687..e944c8b3b0 100644 --- a/packages/playground/src/server.ts +++ b/packages/playground/src/server.ts @@ -11,7 +11,7 @@ import { } from 'node:fs'; import { stat } from 'node:fs/promises'; import type { Server } from 'node:http'; -import { basename, dirname, join, resolve, sep } from 'node:path'; +import { basename, dirname, extname, join, resolve, sep } from 'node:path'; import { fileURLToPath } from 'node:url'; import type { AgentDescribeElementAtPointResult, @@ -37,7 +37,14 @@ import { overrideAIConfig, } from '@midscene/shared/env'; import { generateElementByPoint } from '@midscene/shared/extractor'; -import { annotateRects, imageInfoOfBase64 } from '@midscene/shared/img'; +import { + annotateRects, + imageInfoOfBase64, + screenshotImageExtension, + screenshotImageFormatFromExtension, + screenshotImageFormatFromMimeType, + screenshotImageMimeType, +} from '@midscene/shared/img'; import { getDebug } from '@midscene/shared/logger'; import type { MidsceneRecorderScreenshotAssetRef, @@ -350,7 +357,7 @@ function writeRecorderAiDescribeScreenshot( const { base64, mimeType } = extractBase64Payload(imageBase64); const bytes = Buffer.from(base64, 'base64'); const sha256 = createHash('sha256').update(bytes).digest('hex'); - const extension = mimeType?.includes('jpeg') ? 'jpg' : 'png'; + const extension = recorderScreenshotAssetExtension(mimeType); const pathParts = formatRecorderAiDescribeScreenshotPathParts(); const dumpRoot = resolve( getMidsceneRunSubDir('dump'), @@ -393,7 +400,11 @@ function assertRecorderScreenshotAssetId(assetId: string) { } function recorderScreenshotAssetExtension(mimeType?: string) { - return mimeType?.includes('jpeg') ? 'jpg' : 'png'; + const format = screenshotImageFormatFromMimeType(mimeType); + if (!format) { + return 'png'; + } + return format === 'jpeg' ? 'jpg' : screenshotImageExtension(format); } function recorderScreenshotAssetPath(assetId: string, mimeType?: string) { @@ -428,7 +439,7 @@ function initializeRecorderScreenshotAssetUsage() { continue; } const match = entry.name.match( - /^(.*)-[0-9a-f]{8}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{12}\.(?:png|jpg)$/i, + /^(.*)-[0-9a-f]{8}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{12}\.(?:png|jpg|webp)$/i, ); if (!match?.[1]) { continue; @@ -508,7 +519,7 @@ function readRecorderScreenshotAsset( function findRecorderScreenshotAssetPath(assetId: string) { const safeAssetId = assertRecorderScreenshotAssetId(assetId); const root = getRecorderScreenshotAssetRoot(); - for (const extension of ['png', 'jpg']) { + for (const extension of ['png', 'jpg', 'webp']) { const filePath = assertPathInsideDirectory( root, join(root, `${safeAssetId}.${extension}`), @@ -554,7 +565,7 @@ function pruneRecorderScreenshotAssetsForSession( if (!entry.isFile() || !entry.name.startsWith(prefix)) { continue; } - const assetId = entry.name.replace(/\.(?:png|jpg)$/i, ''); + const assetId = entry.name.replace(/\.(?:png|jpg|webp)$/i, ''); const filePath = join(root, entry.name); if (!retained.has(assetId)) { rmSync(filePath, { force: true }); @@ -3095,7 +3106,7 @@ class PlaygroundServer { return res.status(404).json({ error: 'Report not found or expired' }); } - const match = /^([A-Za-z0-9_-]+)\.(png|jpe?g)$/.exec( + const match = /^([A-Za-z0-9_-]+)\.(png|jpe?g|webp)$/.exec( req.params.assetName, ); if (!match) { @@ -3125,7 +3136,11 @@ class PlaygroundServer { if (commaIndex === -1) { throw new Error('Invalid screenshot data URI'); } - const mimeType = extension === 'png' ? 'image/png' : 'image/jpeg'; + const format = screenshotImageFormatFromExtension(extension); + if (!format) { + throw new Error(`Unsupported screenshot extension: ${extension}`); + } + const mimeType = screenshotImageMimeType(format); return res .type(mimeType) .send(Buffer.from(dataUri.slice(commaIndex + 1), 'base64')); @@ -3769,7 +3784,13 @@ class PlaygroundServer { .status(404) .json({ error: 'Recorder screenshot not found' }); } - res.type(filePath.endsWith('.jpg') ? 'image/jpeg' : 'image/png'); + const format = screenshotImageFormatFromExtension( + extname(filePath).slice(1), + ); + if (!format) { + throw new Error('Unsupported recorder screenshot format'); + } + res.type(screenshotImageMimeType(format)); return res.send(readFileSync(filePath)); } catch (error) { return res.status(400).json({ diff --git a/packages/playground/tests/unit/server-interact.test.ts b/packages/playground/tests/unit/server-interact.test.ts index d67a1e7554..dc3fe20afb 100644 --- a/packages/playground/tests/unit/server-interact.test.ts +++ b/packages/playground/tests/unit/server-interact.test.ts @@ -1,4 +1,4 @@ -import { createReadStream } from 'node:fs'; +import { createReadStream, existsSync } from 'node:fs'; import { mkdir, mkdtemp, rm, writeFile } from 'node:fs/promises'; import { tmpdir } from 'node:os'; import { join } from 'node:path'; @@ -20,12 +20,15 @@ vi.mock('@midscene/core', async (importOriginal) => { const VALID_PNG_BASE64 = 'data:image/png;base64,iVBORw0KGgoAAAANSUhEUgAAABQAAAAUCAYAAACNiR0NAAAACXBIWXMAAAPoAAAD6AG1e1JrAAAAKklEQVR4nO3MIQEAAAzDsPo3/ePhDi4CwpWxUMMXaaFH4QgLPQpHWHg6fOdROhs7ULsmAAAAAElFTkSuQmCC'; +const VALID_WEBP_BASE64 = + 'data:image/webp;base64,UklGRjQAAABXRUJQVlA4ICgAAACQAQCdASoCAAMAAMASJQBOl0AAjNAA/v4icv1difCfoP7mxzi2QwAA'; function createMockResponse() { return { statusCode: 200, body: undefined as unknown, headers: {} as Record, + contentType: undefined as string | undefined, status(code: number) { this.statusCode = code; return this; @@ -38,7 +41,8 @@ function createMockResponse() { this.body = payload; return this; }, - type() { + type(contentType: string) { + this.contentType = contentType; return this; }, setHeader(name: string, value: string) { @@ -202,6 +206,7 @@ describe('PlaygroundServer manual interaction APIs', () => { }); beforeEach(() => { + vi.mocked(existsSync).mockImplementation(() => true); vi.mocked(coreDescribeElementAtPoint).mockReset(); vi.mocked(coreDescribeElementAtPoint).mockRejectedValue( new Error('Active agent does not support describeElementAtPoint.'), @@ -436,7 +441,7 @@ describe('PlaygroundServer manual interaction APIs', () => { type: 'midscene_screenshot_ref', id: 'shot-1', capturedAt: 1, - mimeType: 'image/png', + mimeType: 'image/webp', storage: 'inline', }, }, @@ -445,7 +450,7 @@ describe('PlaygroundServer manual interaction APIs', () => { }, ], }; - const reportHTML = `\n\n`; + const reportHTML = `\n\n`; vi.mocked(createReadStream).mockImplementation( () => ({ @@ -499,12 +504,13 @@ describe('PlaygroundServer manual interaction APIs', () => { const screenshotResponse = createMockResponse(); await screenshotHandler( { - params: { reportId: report.id, assetName: 'shot-1.png' }, + params: { reportId: report.id, assetName: 'shot-1.webp' }, }, screenshotResponse, ); expect(Buffer.isBuffer(screenshotResponse.body)).toBe(true); expect((screenshotResponse.body as Buffer).length).toBeGreaterThan(0); + expect(screenshotResponse.contentType).toBe('image/webp'); } finally { await server.close(); } @@ -866,7 +872,7 @@ describe('PlaygroundServer manual interaction APIs', () => { interfaceType: 'ios', actionSpace: () => [], inputPrimitives, - screenshotBase64: async () => VALID_PNG_BASE64, + screenshotBase64: async () => VALID_WEBP_BASE64, size: async () => ({ width: 390, height: 844 }), }, } as any); @@ -915,7 +921,7 @@ describe('PlaygroundServer manual interaction APIs', () => { expect(rawEvents[0]).toMatchObject({ screenshotAsset: { id: expect.stringMatching(/^session-preview-/), - mimeType: 'image/png', + mimeType: 'image/webp', bytes: expect.any(Number), }, }); @@ -929,11 +935,16 @@ describe('PlaygroundServer manual interaction APIs', () => { '/recorder/assets/:assetId', ); const assetResponse = createMockResponse(); + vi.mocked(existsSync).mockImplementation((filePath) => + String(filePath).endsWith('.webp'), + ); await assetHandler( { params: { assetId: rawEvents[0].screenshotAsset.id } }, assetResponse, ); + vi.mocked(existsSync).mockImplementation(() => true); expect(assetResponse.statusCode).toBe(200); + expect(assetResponse.contentType).toBe('image/webp'); const describeResponse = await describeRecorderEvent(server, rawEvents[0]); expect(describeResponse.body).toMatchObject({ @@ -989,7 +1000,7 @@ describe('PlaygroundServer manual interaction APIs', () => { [10, 20], expect.objectContaining({ verifyPrompt: false, - screenshotBase64: expect.stringMatching(/^data:image\/png;base64,/), + screenshotBase64: expect.stringMatching(/^data:image\/webp;base64,/), coordinateSpace: 'logical', logicalSize: { width: 390, height: 844 }, onProgress: expect.any(Function), diff --git a/packages/playground/tests/unit/server-mjpeg-stream.test.ts b/packages/playground/tests/unit/server-mjpeg-stream.test.ts index f752ff98bb..ad4dbdba84 100644 --- a/packages/playground/tests/unit/server-mjpeg-stream.test.ts +++ b/packages/playground/tests/unit/server-mjpeg-stream.test.ts @@ -385,6 +385,35 @@ describe('PlaygroundServer MJPEG streaming', () => { ); }); + test('GET /mjpeg labels polling WebP bytes with image/webp', async () => { + const webpBase64 = + 'data:image/webp;base64,UklGRjQAAABXRUJQVlA4ICgAAACQAQCdASoCAAMAAMASJQBOl0AAjNAA/v4icv1difCfoP7mxzi2QwAA'; + const screenshotBase64 = vi.fn(async () => webpBase64); + const server = new PlaygroundServer({ + interface: { + interfaceType: 'web', + actionSpace: () => [], + screenshotBase64, + size: async () => ({ width: 2, height: 3 }), + }, + } as any); + + await server.launch(6128); + const mjpegHandler = getRouteHandler(server, 'get', '/mjpeg'); + const request = createMockRequest(); + const response = createMockStreamResponse(); + const streamPromise = mjpegHandler(request, response); + for (let i = 0; i < 10 && screenshotBase64.mock.calls.length === 0; i++) { + await new Promise((resolve) => setTimeout(resolve, 10)); + } + request.listeners.get('close')?.(); + await streamPromise; + + expect(response.chunks.map((chunk) => chunk.toString()).join('')).toContain( + 'Content-Type: image/webp', + ); + }); + test('GET /mjpeg falls back to screenshot polling when producer emits no initial frame', async () => { const stop = vi.fn(); const screenshotBase64 = vi.fn(async () => diff --git a/packages/web-integration/src/chrome-extension/page.ts b/packages/web-integration/src/chrome-extension/page.ts index 126a51f56e..ace6d2c697 100644 --- a/packages/web-integration/src/chrome-extension/page.ts +++ b/packages/web-integration/src/chrome-extension/page.ts @@ -21,7 +21,6 @@ import type { } from '@midscene/core/device'; import type { ElementInfo } from '@midscene/shared/extractor'; import { treeToList } from '@midscene/shared/extractor'; -import { createImgBase64ByFormat } from '@midscene/shared/img'; import { getDebug } from '@midscene/shared/logger'; import { assert } from '@midscene/shared/utils'; import type { Protocol as CDPTypes } from 'devtools-protocol'; @@ -43,6 +42,10 @@ import { injectStopWaterFlowAnimation, injectWaterFlowAnimation, } from './dynamic-scripts'; +import { + type ChromeExtensionScreenshotFormat, + captureChromeExtensionScreenshot, +} from './screenshot'; const debug = getDebug('web:chrome-extension:page'); @@ -768,15 +771,19 @@ export default class ChromeExtensionProxyPage implements AbstractInterface { return sizeInfo; } - async screenshotBase64() { + async screenshotBase64( + format?: ChromeExtensionScreenshotFormat, + ): Promise { // screenshot by cdp await this.hideMousePointer(); - const format = 'jpeg'; - const base64 = await this.sendCommandToDebugger('Page.captureScreenshot', { + return captureChromeExtensionScreenshot( + (params) => + this.sendCommandToDebugger< + CDPTypes.Page.CaptureScreenshotResponse, + CDPTypes.Page.CaptureScreenshotRequest + >('Page.captureScreenshot', params), format, - quality: 90, - }); - return createImgBase64ByFormat(format, base64.data); + ); } async url() { diff --git a/packages/web-integration/src/chrome-extension/screenshot.ts b/packages/web-integration/src/chrome-extension/screenshot.ts new file mode 100644 index 0000000000..8707eb5993 --- /dev/null +++ b/packages/web-integration/src/chrome-extension/screenshot.ts @@ -0,0 +1,19 @@ +import type { Protocol as CDPTypes } from 'devtools-protocol'; + +export type ChromeExtensionScreenshotFormat = 'webp' | 'jpeg'; + +export const DEFAULT_CHROME_EXTENSION_SCREENSHOT_FORMAT: ChromeExtensionScreenshotFormat = + 'webp'; + +export async function captureChromeExtensionScreenshot( + sendCaptureCommand: ( + params: CDPTypes.Page.CaptureScreenshotRequest, + ) => Promise, + format: ChromeExtensionScreenshotFormat = DEFAULT_CHROME_EXTENSION_SCREENSHOT_FORMAT, +): Promise { + const result = await sendCaptureCommand({ + format, + quality: 90, + }); + return `data:image/${format};base64,${result.data}`; +} diff --git a/packages/web-integration/src/puppeteer/base-page.ts b/packages/web-integration/src/puppeteer/base-page.ts index b2199eed81..91f00045ae 100644 --- a/packages/web-integration/src/puppeteer/base-page.ts +++ b/packages/web-integration/src/puppeteer/base-page.ts @@ -24,6 +24,7 @@ import { import type { ElementInfo } from '@midscene/shared/extractor'; import { treeToList } from '@midscene/shared/extractor'; import { + canonicalizeScreenshotBase64, createImgBase64ByFormat, imageInfoOfBase64, } from '@midscene/shared/img'; @@ -457,7 +458,7 @@ export class Page< } async screenshotBase64(): Promise { - const imgType = 'jpeg' as const; + const imgType = 'webp' as const; const quality = 90; const startTime = Date.now(); debugPage('screenshotBase64 begin'); @@ -473,26 +474,41 @@ export class Page< } else if (this.interfaceType === 'playwright') { const page = this.underlyingPage as PlaywrightPage; try { - const buffer = await page.screenshot({ - type: imgType, - quality, - timeout: 10 * 1000, - }); - base64 = createImgBase64ByFormat(imgType, buffer.toString('base64')); + // Playwright's public API only exposes PNG/JPEG. Chromium can avoid an + // extra conversion by using CDP's native WebP encoder. + base64 = await this.screenshotBase64ByPlaywrightCdp(imgType, quality); } catch (error) { if (isClosedPageError(error) || page.isClosed()) { throw error; } const errorMsg = error instanceof Error ? error.message : String(error); - console.warn( - `[Midscene] Playwright screenshot failed: ${errorMsg}. Falling back to CDP screenshot.`, - ); - debugPage( - 'playwright screenshot failed, trying CDP fallback: %s', - error, + warnPage( + `Playwright CDP WebP screenshot failed: ${errorMsg}. Falling back to PNG capture and WebP encoding.`, ); - base64 = await this.screenshotBase64ByPlaywrightCdp(imgType, quality); + try { + const buffer = await page.screenshot({ + type: 'png', + timeout: 10 * 1000, + }); + base64 = await canonicalizeScreenshotBase64( + createImgBase64ByFormat('png', buffer.toString('base64')), + { quality }, + ); + } catch (fallbackError) { + debugPage( + 'Playwright PNG screenshot fallback also failed: %s', + fallbackError, + ); + const fallbackErrorMessage = + fallbackError instanceof Error + ? fallbackError.message + : String(fallbackError); + throw new Error( + `Playwright screenshot failed through both CDP WebP and PNG fallback (CDP: ${errorMsg}; PNG fallback: ${fallbackErrorMessage})`, + { cause: fallbackError }, + ); + } } } else { throw new Error('Unsupported page type for screenshot'); @@ -503,7 +519,7 @@ export class Page< } private async screenshotBase64ByPlaywrightCdp( - imgType: 'jpeg' | 'png', + imgType: 'jpeg' | 'png' | 'webp', quality?: number, ) { const client = await this.createPageCdpSession('CDP screenshot fallback'); @@ -685,9 +701,15 @@ export class Page< ); // MjpegStreamFrame.data is contractually bare base64; screenshotBase64() // returns a `data:image/...;base64,...` URL, so strip the prefix here. + const contentType = dataUrl.match(/^data:(image\/\w+);base64,/)?.[1]; + if (!contentType) { + throw new Error( + 'Screenshot fallback returned an invalid image data URL', + ); + } activeStream.onFrame({ data: dataUrl.replace(DATA_URL_BASE64_PREFIX, ''), - contentType: 'image/jpeg', + contentType, }); } catch (error) { debugPage('screencast visual refresh failed: %s', error); diff --git a/packages/web-integration/tests/ai/web/playwright/browser-webp-encoder.spec.ts b/packages/web-integration/tests/ai/web/playwright/browser-webp-encoder.spec.ts new file mode 100644 index 0000000000..e7d91f7518 --- /dev/null +++ b/packages/web-integration/tests/ai/web/playwright/browser-webp-encoder.spec.ts @@ -0,0 +1,182 @@ +import { Buffer } from 'node:buffer'; +import { + type BrowserWebpEncodeInput, + encodeRgbaToWebp, +} from '@midscene/shared/img'; +import { type Page, expect, test } from '@playwright/test'; + +type BrowserWebpEncoder = ( + input: BrowserWebpEncodeInput, +) => Promise; + +function createRgbaFixture(width: number, height: number): number[] { + const pixels = new Array(width * height * 4); + let randomState = 0x1234abcd; + + for (let y = 0; y < height; y++) { + for (let x = 0; x < width; x++) { + randomState = (randomState * 1664525 + 1013904223) >>> 0; + const offset = (y * width + x) * 4; + const noise = randomState & 0xff; + pixels[offset] = (x * 11 + y * 3 + noise) & 0xff; + pixels[offset + 1] = (x * 5 + y * 17 + (noise >> 1)) & 0xff; + pixels[offset + 2] = (x * 19 + y * 7 + (noise >> 2)) & 0xff; + pixels[offset + 3] = 255; + } + } + + return pixels; +} + +function expectWebpSignature(bytes: number[]): void { + const buffer = Buffer.from(bytes); + expect(buffer.subarray(0, 4).toString('ascii')).toBe('RIFF'); + expect(buffer.subarray(8, 12).toString('ascii')).toBe('WEBP'); +} + +async function decodeDimensions( + page: Page, + bytes: number[], +): Promise<{ width: number; height: number }> { + return page.evaluate(async (encodedBytes) => { + const bitmap = await createImageBitmap( + new Blob([new Uint8Array(encodedBytes)], { type: 'image/webp' }), + ); + const dimensions = { width: bitmap.width, height: bitmap.height }; + bitmap.close(); + return dimensions; + }, bytes); +} + +test.describe('browser WebP encoder', () => { + const width = 128; + const height = 96; + const pixels = createRgbaFixture(width, height); + const encoderSource = encodeRgbaToWebp.toString(); + + test.beforeEach(async ({ page }) => { + await page.goto('data:text/html,browser WebP encoder'); + await page.addScriptTag({ + content: `globalThis.__midsceneEncodeRgbaToWebp = ${encoderSource};`, + }); + }); + + test('uses HTML Canvas quality on the browser main thread', async ({ + page, + }) => { + await page.evaluate(() => { + Object.defineProperty(globalThis, 'OffscreenCanvas', { + configurable: true, + value: undefined, + }); + }); + + const [quality10, quality90] = await page.evaluate( + async ({ fixturePixels, fixtureWidth, fixtureHeight }) => { + const encode = ( + globalThis as typeof globalThis & { + __midsceneEncodeRgbaToWebp: BrowserWebpEncoder; + } + ).__midsceneEncodeRgbaToWebp; + const lowQuality = await encode({ + pixels: fixturePixels, + width: fixtureWidth, + height: fixtureHeight, + quality: 10, + }); + const highQuality = await encode({ + pixels: fixturePixels, + width: fixtureWidth, + height: fixtureHeight, + quality: 90, + }); + return [Array.from(lowQuality), Array.from(highQuality)]; + }, + { + fixturePixels: pixels, + fixtureWidth: width, + fixtureHeight: height, + }, + ); + + expectWebpSignature(quality10); + expectWebpSignature(quality90); + expect(quality10).not.toEqual(quality90); + expect(quality10.length).toBeLessThan(quality90.length); + await expect(decodeDimensions(page, quality90)).resolves.toEqual({ + width, + height, + }); + }); + + test('encodes WebP with OffscreenCanvas inside a Worker', async ({ + page, + }) => { + const workerBytes = await page.evaluate( + async ({ + productionEncoderSource, + fixturePixels, + fixtureWidth, + fixtureHeight, + }) => { + const workerSource = ` + const encodeRgbaToWebp = ${productionEncoderSource}; + self.onmessage = async (event) => { + try { + const output = await encodeRgbaToWebp(event.data); + self.postMessage({ bytes: Array.from(output) }); + } catch (error) { + self.postMessage({ + error: error instanceof Error ? error.message : String(error), + }); + } + }; + `; + const workerUrl = URL.createObjectURL( + new Blob([workerSource], { type: 'text/javascript' }), + ); + const worker = new Worker(workerUrl); + + try { + return await new Promise((resolve, reject) => { + worker.onmessage = ( + event: MessageEvent<{ bytes?: number[]; error?: string }>, + ) => { + if (event.data.error) { + reject(new Error(event.data.error)); + return; + } + if (!event.data.bytes) { + reject(new Error('Worker returned no WebP bytes')); + return; + } + resolve(event.data.bytes); + }; + worker.onerror = (event) => reject(new Error(event.message)); + worker.postMessage({ + pixels: fixturePixels, + width: fixtureWidth, + height: fixtureHeight, + quality: 90, + }); + }); + } finally { + worker.terminate(); + URL.revokeObjectURL(workerUrl); + } + }, + { + productionEncoderSource: encoderSource, + fixturePixels: pixels, + fixtureWidth: width, + fixtureHeight: height, + }, + ); + + expectWebpSignature(workerBytes); + await expect(decodeDimensions(page, workerBytes)).resolves.toEqual({ + width, + height, + }); + }); +}); diff --git a/packages/web-integration/tests/ai/web/playwright/screenshot-cdp-fallback.spec.ts b/packages/web-integration/tests/ai/web/playwright/screenshot-cdp-fallback.spec.ts index 4c2d2bc6a9..2601becb27 100644 --- a/packages/web-integration/tests/ai/web/playwright/screenshot-cdp-fallback.spec.ts +++ b/packages/web-integration/tests/ai/web/playwright/screenshot-cdp-fallback.spec.ts @@ -3,7 +3,7 @@ import type { AddressInfo } from 'node:net'; import { PlaywrightWebPage } from '@/playwright'; import { expect, test } from '@playwright/test'; -test.describe('playwright screenshot CDP fallback', () => { +test.describe('playwright CDP screenshot', () => { test.setTimeout(30 * 1000); let slowServer: ReturnType; @@ -31,7 +31,7 @@ test.describe('playwright screenshot CDP fallback', () => { slowServer?.close(); }); - test('should fall back to CDP when a hanging web font blocks screenshot', async ({ + test('should capture WebP when a hanging web font blocks Playwright screenshot', async ({ page, }) => { await page.setViewportSize({ width: 1280, height: 900 }); @@ -40,8 +40,8 @@ test.describe('playwright screenshot CDP fallback', () => { timeout: 15_000, }); - // Inject a @font-face pointing to the local slow server, - // causing Playwright's screenshot to hang on "waiting for fonts to load" + // Inject a @font-face pointing to the local slow server. Playwright's + // public screenshot API waits for fonts, while CDP capture should not. const fontUrl = `${slowServerUrl}/hanging-font.woff2`; await page.evaluate((url: string) => { const style = document.createElement('style'); @@ -63,7 +63,7 @@ test.describe('playwright screenshot CDP fallback', () => { const webPage = new PlaywrightWebPage(page); const screenshotBase64 = await webPage.screenshotBase64(); - expect(screenshotBase64).toContain('data:image/jpeg;base64,'); + expect(screenshotBase64).toContain('data:image/webp;base64,'); expect(screenshotBase64.length).toBeGreaterThan(1_000); }); }); diff --git a/packages/web-integration/tests/ai/web/puppeteer/webp-report.test.ts b/packages/web-integration/tests/ai/web/puppeteer/webp-report.test.ts new file mode 100644 index 0000000000..82ba0669fd --- /dev/null +++ b/packages/web-integration/tests/ai/web/puppeteer/webp-report.test.ts @@ -0,0 +1,109 @@ +import { createHash } from 'node:crypto'; +import { + existsSync, + readFileSync, + rmSync, + statSync, + writeFileSync, +} from 'node:fs'; +import { basename, dirname, join } from 'node:path'; +import { PuppeteerAgent } from '@/puppeteer'; +import { reportFileToMarkdown } from '@midscene/core'; +import { afterEach, describe, expect, it, vi } from 'vitest'; +import { launchPage } from './utils'; + +const FIXTURES_DIR = join(__dirname, '../../fixtures'); +const REPORT_NAME = + process.env.MIDSCENE_WEBP_REPORT_NAME || 'webp-model-input-validation'; + +vi.setConfig({ + testTimeout: 180 * 1000, +}); + +describe('WebP model input report', () => { + let reset: (() => Promise) | undefined; + let agent: PuppeteerAgent | undefined; + + afterEach(async () => { + if (agent) { + await agent.destroy(); + } + await reset?.(); + }); + + it('keeps the exact WebP model input in HTML and Markdown reports', async () => { + const launched = await launchPage( + `file://${join(FIXTURES_DIR, 'search-engine.html')}`, + { viewport: { width: 1120, height: 700 } }, + ); + reset = launched.reset; + agent = new PuppeteerAgent(launched.originPage, { + generateReport: true, + reportFileName: REPORT_NAME, + }); + + const captured = await agent.interface.screenshotBase64(); + expect(captured).toMatch(/^data:image\/webp;base64,UklGR/); + + await agent.aiAssert('A search input box and a search button are visible'); + await agent.destroy(); + const reportFile = agent.reportFile; + agent = undefined; + + expect(reportFile).toBeTruthy(); + expect(existsSync(reportFile!)).toBe(true); + + const markdownDir = join(dirname(reportFile!), `${REPORT_NAME}-markdown`); + rmSync(markdownDir, { recursive: true, force: true }); + const markdownResult = await reportFileToMarkdown({ + htmlPath: reportFile!, + outputDir: markdownDir, + }); + const markdown = readFileSync(markdownResult.markdownFiles[0], 'utf8'); + expect(markdown).not.toContain('No model metadata recorded.'); + expect(markdown).not.toContain('No token usage recorded.'); + expect(markdown).toContain('timing=model-input'); + + const modelInputHash = markdown.match( + /Model input \d+ \(exact bytes, sha256: ([a-f0-9]{64})\)/, + )?.[1]; + expect(modelInputHash).toBeTruthy(); + expect(markdownResult.screenshotFiles.length).toBeGreaterThan(0); + expect( + markdownResult.screenshotFiles.every((file) => file.endsWith('.webp')), + ).toBe(true); + + const screenshots = markdownResult.screenshotFiles.map((file) => { + const bytes = readFileSync(file); + expect(bytes.subarray(0, 4).toString('ascii')).toBe('RIFF'); + expect(bytes.subarray(8, 12).toString('ascii')).toBe('WEBP'); + return { + file: basename(file), + bytes: statSync(file).size, + sha256: createHash('sha256').update(bytes).digest('hex'), + }; + }); + expect(screenshots.some((item) => item.sha256 === modelInputHash)).toBe( + true, + ); + + const manifestFile = join(markdownDir, 'validation-manifest.json'); + writeFileSync( + manifestFile, + `${JSON.stringify( + { + reportFile, + markdownFile: markdownResult.markdownFiles[0], + modelInputHash, + screenshots, + }, + null, + 2, + )}\n`, + 'utf8', + ); + + console.log(`WebP validation report: ${reportFile}`); + console.log(`WebP validation manifest: ${manifestFile}`); + }); +}); diff --git a/packages/web-integration/tests/unit-test/base-page-screencast.test.ts b/packages/web-integration/tests/unit-test/base-page-screencast.test.ts index bf081c07d9..9580c26ac1 100644 --- a/packages/web-integration/tests/unit-test/base-page-screencast.test.ts +++ b/packages/web-integration/tests/unit-test/base-page-screencast.test.ts @@ -76,6 +76,9 @@ function jpegBase64(width: number, height: number): string { ]).toString('base64'); } +const webpBase64 = + 'UklGRioAAABXRUJQVlA4IB4AAAAwAQCdASoBAAEAAUAmJQBOgCHwAP7+hNQAAAA='; + describe('Page startMjpegStream', () => { it('starts a Puppeteer CDP screencast and ACKs incoming frames', async () => { const handlers = new Map unknown>(); @@ -154,7 +157,7 @@ describe('Page startMjpegStream', () => { const mockPage = { bringToFront: vi.fn().mockResolvedValue(undefined), evaluate: vi.fn().mockResolvedValue({ width: 1280, height: 720 }), - screenshot: vi.fn().mockResolvedValue(jpegBase64(1280, 720)), + screenshot: vi.fn().mockResolvedValue(webpBase64), url: () => 'http://example.com', target: () => ({ createCDPSession: vi.fn().mockResolvedValue(client), @@ -179,8 +182,8 @@ describe('Page startMjpegStream', () => { }); await vi.waitFor(() => { expect(onFrame).toHaveBeenCalledWith({ - data: jpegBase64(1280, 720), - contentType: 'image/jpeg', + data: webpBase64, + contentType: 'image/webp', }); }); @@ -256,14 +259,14 @@ describe('Page startMjpegStream', () => { await page.flushPendingVisualUpdate(); expect(mockPage.screenshot).toHaveBeenCalledWith({ - type: 'jpeg', + type: 'webp', quality: 90, encoding: 'base64', }); // Hub contract: MjpegStreamFrame.data is bare base64, never a data URL. expect(onFrame).toHaveBeenCalledWith({ data: 'cmVmcmVzaA==', - contentType: 'image/jpeg', + contentType: 'image/webp', }); await handle.stop(); @@ -317,7 +320,7 @@ describe('Page startMjpegStream', () => { it('force-pushes a final screenshot after navigation replaces the preview with a transient frame', async () => { const mockPage = { evaluate: vi.fn().mockResolvedValue({ width: 1280, height: 720 }), - screenshot: vi.fn().mockResolvedValue(jpegBase64(1280, 720)), + screenshot: vi.fn().mockResolvedValue(webpBase64), url: () => 'http://example.com', } as any; const page = new Page(mockPage, 'puppeteer'); @@ -336,13 +339,13 @@ describe('Page startMjpegStream', () => { await page.flushPendingVisualUpdate(true); expect(mockPage.screenshot).toHaveBeenCalledWith({ - type: 'jpeg', + type: 'webp', quality: 90, encoding: 'base64', }); expect(onFrame).toHaveBeenCalledWith({ - data: jpegBase64(1280, 720), - contentType: 'image/jpeg', + data: webpBase64, + contentType: 'image/webp', }); }); diff --git a/packages/web-integration/tests/unit-test/base-page-screenshot.test.ts b/packages/web-integration/tests/unit-test/base-page-screenshot.test.ts index c4d7355e5f..1024bb781f 100644 --- a/packages/web-integration/tests/unit-test/base-page-screenshot.test.ts +++ b/packages/web-integration/tests/unit-test/base-page-screenshot.test.ts @@ -24,9 +24,36 @@ vi.mock('@/web-page', () => ({ })); describe('Page screenshotBase64', () => { - it('uses the regular playwright screenshot path when it succeeds', async () => { + const webpBody = + 'UklGRjQAAABXRUJQVlA4ICgAAACQAQCdASoCAAMAAMASJQBOl0AAjNAA/v4icv1difCfoP7mxzi2QwAA'; + const pngBody = + 'iVBORw0KGgoAAAANSUhEUgAAAAEAAAABCAQAAAC1HAwCAAAAC0lEQVR42mNk+M9QDwADhgGAWjR9awAAAABJRU5ErkJggg=='; + + it('uses Puppeteer native WebP capture', async () => { + const screenshot = vi.fn().mockResolvedValue(webpBody); + const page = new Page( + { + url: () => 'http://example.com', + screenshot, + } as any, + 'puppeteer', + ); + + await expect(page.screenshotBase64()).resolves.toBe( + `data:image/webp;base64,${webpBody}`, + ); + expect(screenshot).toHaveBeenCalledWith({ + type: 'webp', + quality: 90, + encoding: 'base64', + }); + }); + + it('uses Chromium CDP native WebP when available', async () => { const screenshot = vi.fn().mockResolvedValue(Buffer.from('plain-shot')); - const newCDPSession = vi.fn(); + const detach = vi.fn().mockResolvedValue(undefined); + const send = vi.fn().mockResolvedValue({ data: webpBody }); + const newCDPSession = vi.fn().mockResolvedValue({ send, detach }); const mockPage = { url: () => 'http://example.com', isClosed: () => false, @@ -44,23 +71,21 @@ describe('Page screenshotBase64', () => { const page = new Page(mockPage, 'playwright'); const result = await page.screenshotBase64(); - expect(result).toContain('data:image/jpeg;base64,'); - expect(screenshot).toHaveBeenCalledTimes(1); - expect(newCDPSession).not.toHaveBeenCalled(); + expect(result).toBe(`data:image/webp;base64,${webpBody}`); + expect(screenshot).not.toHaveBeenCalled(); + expect(send).toHaveBeenCalledWith('Page.captureScreenshot', { + format: 'webp', + quality: 90, + }); }); - it('falls back to a CDP screenshot when playwright screenshot times out', async () => { + it('falls back to public PNG capture and returns WebP without CDP', async () => { const screenshot = vi .fn() - .mockRejectedValue( - new Error('page.screenshot: Timeout 10000ms exceeded.'), - ); - const detach = vi.fn().mockResolvedValue(undefined); - const send = vi.fn().mockResolvedValue({ data: 'Y2RwLXNob3Q=' }); - const newCDPSession = vi.fn().mockResolvedValue({ - send, - detach, - }); + .mockResolvedValue(Buffer.from(pngBody, 'base64')); + const newCDPSession = vi + .fn() + .mockRejectedValue(new Error('CDP unavailable')); const mockPage = { url: () => 'http://example.com', isClosed: () => false, @@ -78,13 +103,12 @@ describe('Page screenshotBase64', () => { const page = new Page(mockPage, 'playwright'); const result = await page.screenshotBase64(); - expect(result).toContain('data:image/jpeg;base64,'); + expect(result).toMatch(/^data:image\/webp;base64,UklGR/); expect(newCDPSession).toHaveBeenCalledTimes(1); - expect(send).toHaveBeenCalledWith('Page.captureScreenshot', { - format: 'jpeg', - quality: 90, + expect(screenshot).toHaveBeenCalledWith({ + type: 'png', + timeout: 10 * 1000, }); - expect(detach).toHaveBeenCalledTimes(1); }); it('times out when the CDP screenshot fallback does not return in time', async () => { @@ -124,11 +148,13 @@ describe('Page screenshotBase64', () => { await vi.advanceTimersByTimeAsync(10 * 1000); await expect(resultPromise).resolves.toMatchObject({ - message: 'CDP screenshot timeout after 10000ms.', + message: expect.stringContaining( + 'Playwright screenshot failed through both CDP WebP and PNG fallback', + ), }); expect(newCDPSession).toHaveBeenCalledTimes(1); expect(send).toHaveBeenCalledWith('Page.captureScreenshot', { - format: 'jpeg', + format: 'webp', quality: 90, }); expect(detach).toHaveBeenCalledTimes(1); @@ -173,7 +199,9 @@ describe('Page screenshotBase64', () => { await vi.advanceTimersByTimeAsync(10 * 1000); await expect(resultPromise).resolves.toMatchObject({ - message: 'CDP screenshot timeout after 10000ms.', + message: expect.stringContaining( + 'Playwright screenshot failed through both CDP WebP and PNG fallback', + ), }); expect(detach).toHaveBeenCalledTimes(1); diff --git a/packages/web-integration/tests/unit-test/chrome-extension-screenshot.test.ts b/packages/web-integration/tests/unit-test/chrome-extension-screenshot.test.ts new file mode 100644 index 0000000000..390dacd24c --- /dev/null +++ b/packages/web-integration/tests/unit-test/chrome-extension-screenshot.test.ts @@ -0,0 +1,157 @@ +import { Buffer } from 'node:buffer'; +import { mkdtempSync, readFileSync, rmSync } from 'node:fs'; +import { tmpdir } from 'node:os'; +import { basename, join } from 'node:path'; +import { + ExecutionDump, + ReportGenerator, + ScreenshotItem, + parseImageScripts, + reportFileToMarkdown, +} from '@midscene/core'; +import { describe, expect, it, vi } from 'vitest'; +import { captureChromeExtensionScreenshot } from '../../src/chrome-extension/screenshot'; +import { launchPage } from '../ai/web/puppeteer/utils'; + +describe('Chrome extension screenshot format', () => { + it('captures WebP at quality 90 by default', async () => { + const sendCaptureCommand = vi + .fn() + .mockResolvedValue({ data: 'SCREENSHOT_BASE64' }); + + await expect( + captureChromeExtensionScreenshot(sendCaptureCommand), + ).resolves.toBe('data:image/webp;base64,SCREENSHOT_BASE64'); + expect(sendCaptureCommand).toHaveBeenCalledWith({ + format: 'webp', + quality: 90, + }); + }); + + it('allows an explicit JPEG fallback', async () => { + const sendCaptureCommand = vi + .fn() + .mockResolvedValue({ data: 'SCREENSHOT_BASE64' }); + + await expect( + captureChromeExtensionScreenshot(sendCaptureCommand, 'jpeg'), + ).resolves.toBe('data:image/jpeg;base64,SCREENSHOT_BASE64'); + expect(sendCaptureCommand).toHaveBeenCalledWith({ + format: 'jpeg', + quality: 90, + }); + }); + + it('keeps a real CDP WebP capture intact through report and Markdown export', async () => { + const tmpDir = mkdtempSync( + join(tmpdir(), 'midscene-chrome-extension-webp-'), + ); + + try { + const pageHtml = ` + + +
+
WebP report fixture
+
+ + `; + const { originPage, reset } = await launchPage( + `data:text/html,${encodeURIComponent(pageHtml)}`, + { + viewport: { + width: 320, + height: 180, + deviceScaleFactor: 1, + }, + }, + ); + + try { + const cdpSession = await originPage.createCDPSession(); + + try { + const screenshotBase64 = await captureChromeExtensionScreenshot( + (params) => cdpSession.send('Page.captureScreenshot', params), + ); + const screenshotBytes = Buffer.from( + screenshotBase64.split(',')[1], + 'base64', + ); + + expect(screenshotBase64).toMatch(/^data:image\/webp;base64,/); + expect(screenshotBytes.subarray(0, 4).toString('ascii')).toBe('RIFF'); + expect(screenshotBytes.subarray(8, 12).toString('ascii')).toBe( + 'WEBP', + ); + + const capturedAt = Date.now(); + const screenshot = ScreenshotItem.create( + screenshotBase64, + capturedAt, + ); + const execution = new ExecutionDump({ + id: 'chrome-extension-webp-report', + logTime: capturedAt, + name: 'Chrome extension WebP report', + tasks: [ + { + taskId: 'capture-webp', + type: 'Insight', + subType: 'Locate', + param: { prompt: 'Capture the fixture page' }, + uiContext: { + screenshot, + shotSize: { width: 320, height: 180 }, + shrunkShotToLogicalRatio: 1, + }, + executor: async () => undefined, + recorder: [], + status: 'finished', + }, + ], + }); + const reportPath = join(tmpDir, 'chrome-extension-webp.html'); + const reportGenerator = new ReportGenerator({ + reportPath, + screenshotMode: 'inline', + autoPrint: false, + }); + + reportGenerator.onExecutionUpdate(execution, { + groupName: 'Chrome extension WebP integration test', + sdkVersion: 'test', + modelBriefs: [], + }); + await expect(reportGenerator.finalize()).resolves.toBe(reportPath); + + const reportHtml = readFileSync(reportPath, 'utf-8'); + expect(parseImageScripts(reportHtml)[screenshot.id]).toBe( + screenshotBase64, + ); + expect(reportHtml).toContain('"mimeType":"image/webp"'); + + const markdownDir = join(tmpDir, 'markdown'); + const markdownResult = await reportFileToMarkdown({ + htmlPath: reportPath, + outputDir: markdownDir, + }); + + expect(markdownResult.screenshotFiles).toHaveLength(1); + const [exportedScreenshotPath] = markdownResult.screenshotFiles; + expect(exportedScreenshotPath).toMatch(/\.webp$/); + expect(readFileSync(exportedScreenshotPath)).toEqual(screenshotBytes); + expect( + readFileSync(join(markdownDir, 'report.md'), 'utf-8'), + ).toContain(`./screenshots/${basename(exportedScreenshotPath)}`); + } finally { + await cdpSession.detach(); + } + } finally { + await reset(); + } + } finally { + rmSync(tmpDir, { recursive: true, force: true }); + } + }); +});