feat(device): use WebP for device screenshot producers

packages/android/src/device.ts

@@ -31,6 +31,7 @@ import {
 } from '@midscene/shared/env';
 import type { ElementInfo } from '@midscene/shared/extractor';
 import {
+  canonicalizeScreenshotBase64,
   createImgBase64ByFormat,
   validateScreenshotBuffer,
 } from '@midscene/shared/img';
@@ -518,7 +519,7 @@ ${Object.keys(size)
   /**
    * Continuous frame source backed by the scrcpy video stream.
    *
-   * Decoding H.264 to JPEG costs an ffmpeg process per frame (~100ms measured
+   * Decoding H.264 to WebP costs an ffmpeg process plus one Sharp encode per
    * on-device), so `latest()` hands out RAW keyframe handles (near-zero cost —
    * the stream is already flowing) and `decode()` pays the ffmpeg cost only
    * for the frames the observer actually sampled, at the end of the window.
@@ -551,7 +552,7 @@ ${Object.keys(size)
         const images: string[] = [];
         for (const frameRef of refs) {
           images.push(
-            await adapter.decodeRawKeyframeToJpegBase64(
+            await adapter.decodeRawKeyframeToWebpBase64(
               frameRef.ref as RawKeyframe,
             ),
           );
@@ -1269,9 +1270,8 @@ ${Object.keys(size)
     }
 
     debugDevice('Converting to base64');
-    const result = createImgBase64ByFormat(
-      'png',
-      screenshotBuffer.toString('base64'),
+    const result = await canonicalizeScreenshotBase64(
+      createImgBase64ByFormat('png', screenshotBuffer.toString('base64')),
     );
     if (localScreenshotPath) {
       debugDevice(`Deleting local screenshot: ${localScreenshotPath}`);

packages/android/src/scrcpy-device-adapter.ts

@@ -207,11 +207,11 @@ export class ScrcpyDeviceAdapter {
 
     try {
       const manager = await this.ensureManager(deviceInfo);
-      const screenshotBuffer = await manager.getScreenshotJpeg();
+      const screenshotBuffer = await manager.getScreenshotWebp();
       this.clearFailure();
 
       return createImgBase64ByFormat(
-        'jpeg',
+        'webp',
         screenshotBuffer.toString('base64'),
       );
     } catch (error) {
@@ -249,15 +249,15 @@ export class ScrcpyDeviceAdapter {
   }
 
   /**
-   * Decode a raw keyframe to a JPEG data URL. Deferred, per-frame-expensive
+   * Decode a raw keyframe to a WebP data URL. Deferred, per-frame-expensive
    * step (one ffmpeg process per call) — only call on sampled frames.
    */
-  async decodeRawKeyframeToJpegBase64(frame: RawKeyframe): Promise<string> {
+  async decodeRawKeyframeToWebpBase64(frame: RawKeyframe): Promise<string> {
     if (!this.manager) {
       throw new Error('scrcpy manager is not initialized');
     }
-    const jpegBuffer = await this.manager.decodeRawKeyframeToJpeg(frame);
-    return createImgBase64ByFormat('jpeg', jpegBuffer.toString('base64'));
+    const webpBuffer = await this.manager.decodeRawKeyframeToWebp(frame);
+    return createImgBase64ByFormat('webp', webpBuffer.toString('base64'));
   }
 
   /**

packages/android/src/scrcpy-manager.ts

@@ -50,7 +50,7 @@ export interface ScrcpyScreenshotOptions {
 
 /**
  * A raw (not yet decoded) H.264 keyframe emitted by the scrcpy stream.
- * Holding these is cheap — decoding to JPEG costs an ffmpeg run per frame, so
+ * Holding these is cheap — decoding to WebP costs an ffmpeg run per frame, so
  * consumers (e.g. UI observers) buffer raw keyframes and decode only
  * the frames they actually need, after sampling.
  */
@@ -465,7 +465,7 @@ export class ScrcpyScreenshotManager {
    * one subscriber is active, incoming keyframes keep resetting the idle timer
    * so the connection is not torn down mid-capture. Returns an unsubscribe fn.
    *
-   * Frames are emitted RAW (no decoding). Use {@link decodeRawKeyframeToJpeg}
+   * Frames are emitted RAW (no decoding). Use {@link decodeRawKeyframeToWebp}
    * on the frames you actually need — one ffmpeg run per unique frame.
    */
   subscribeKeyframes(listener: (frame: RawKeyframe) => void): () => void {
@@ -492,20 +492,20 @@ export class ScrcpyScreenshotManager {
 
   /**
    * Decode a raw keyframe (from {@link subscribeKeyframes} or
-   * {@link getLatestRawKeyframe}) to a JPEG buffer. This is the deferred,
+   * {@link getLatestRawKeyframe}) to a WebP buffer. This is the deferred,
    * per-frame-expensive step (one ffmpeg process per call) — call it only on
    * sampled frames, never inside a capture loop.
    */
-  async decodeRawKeyframeToJpeg(frame: RawKeyframe): Promise<Buffer> {
-    return this.decodeH264ToJpeg(Buffer.concat([frame.header, frame.data]));
+  async decodeRawKeyframeToWebp(frame: RawKeyframe): Promise<Buffer> {
+    return this.decodeH264ToWebp(Buffer.concat([frame.header, frame.data]));
   }
 
   /**
-   * Get screenshot as JPEG.
+   * Get screenshot as WebP.
    * Tries to get a fresh frame within a short timeout. If the screen is static
    * (no new frames arrive), falls back to the latest cached keyframe.
    */
-  async getScreenshotJpeg(): Promise<Buffer> {
+  async getScreenshotWebp(): Promise<Buffer> {
     const perfStart = Date.now();
 
     const t1 = Date.now();
@@ -542,7 +542,7 @@ export class ScrcpyScreenshotManager {
     );
 
     const t4 = Date.now();
-    const result = await this.decodeH264ToJpeg(keyframeBuffer);
+    const result = await this.decodeH264ToWebp(keyframeBuffer);
     const decodeTime = Date.now() - t4;
 
     const totalTime = Date.now() - perfStart;
@@ -662,10 +662,18 @@ export class ScrcpyScreenshotManager {
   }
 
   /**
-   * Decode H.264 data to JPEG using ffmpeg
+   * Decode H.264 to raw RGB with ffmpeg, then encode the selected frame once
+   * as WebP with Sharp. The bundled ffmpeg does not include libwebp.
    */
-  private async decodeH264ToJpeg(h264Buffer: Buffer): Promise<Buffer> {
+  private async decodeH264ToWebp(h264Buffer: Buffer): Promise<Buffer> {
     const { spawn } = await import('node:child_process');
+    const { default: sharp } = await import('sharp');
+    const resolution = this.videoResolution;
+    if (!resolution?.width || !resolution.height) {
+      throw new Error(
+        'Cannot decode scrcpy frame before video resolution is known',
+      );
+    }
 
     return new Promise((resolve, reject) => {
       const ffmpegArgs = [
@@ -676,11 +684,9 @@ export class ScrcpyScreenshotManager {
         '-vframes',
         '1',
         '-f',
-        'image2pipe',
-        '-vcodec',
-        'mjpeg',
-        '-q:v',
-        '5',
+        'rawvideo',
+        '-pix_fmt',
+        'rgb24',
         '-loglevel',
         'error',
         'pipe:1',
@@ -691,32 +697,68 @@ export class ScrcpyScreenshotManager {
         stdio: ['pipe', 'pipe', 'pipe'],
       });
 
-      const chunks: Buffer[] = [];
       let stderrOutput = '';
-
-      ffmpeg.stdout.on('data', (chunk: Buffer) => {
-        chunks.push(chunk);
-      });
+      let settled = false;
+
+      const encoder = sharp({
+        raw: {
+          width: resolution.width,
+          height: resolution.height,
+          channels: 3,
+        },
+      }).webp({ quality: 90, effort: 1 });
+      // Resolve failures into a value immediately so an ffmpeg spawn/exit
+      // failure cannot leave Sharp with a temporarily unhandled rejection.
+      const encodedWebpResult = encoder.toBuffer().then(
+        (buffer) => ({ ok: true as const, buffer }),
+        (error: unknown) => ({ ok: false as const, error }),
+      );
+      ffmpeg.stdout.pipe(encoder);
 
       ffmpeg.stderr.on('data', (data: Buffer) => {
         stderrOutput += data.toString();
       });
 
-      ffmpeg.on('close', (code) => {
-        if (code === 0 && chunks.length > 0) {
-          const jpegBuffer = Buffer.concat(chunks);
-          debugScrcpy(
-            `FFmpeg decode successful, JPEG size: ${jpegBuffer.length} bytes`,
-          );
-          resolve(jpegBuffer);
-        } else {
+      ffmpeg.on('close', async (code) => {
+        if (settled) return;
+        if (code !== 0) {
+          settled = true;
           const errorMsg = stderrOutput || `FFmpeg exited with code ${code}`;
           debugScrcpy(`FFmpeg decode failed: ${errorMsg}`);
-          reject(new Error(`H.264 to JPEG decode failed: ${errorMsg}`));
+          reject(new Error(`H.264 frame decode failed: ${errorMsg}`));
+          return;
+        }
+
+        try {
+          const encodeResult = await encodedWebpResult;
+          if (!encodeResult.ok) {
+            throw encodeResult.error;
+          }
+          const webpBuffer = encodeResult.buffer;
+          if (
+            webpBuffer.subarray(0, 4).toString('ascii') !== 'RIFF' ||
+            webpBuffer.subarray(8, 12).toString('ascii') !== 'WEBP'
+          ) {
+            throw new Error('Sharp returned invalid WebP bytes');
+          }
+          settled = true;
+          debugScrcpy(
+            `H.264 decode and WebP encode successful, WebP size: ${webpBuffer.length} bytes`,
+          );
+          resolve(webpBuffer);
+        } catch (error) {
+          settled = true;
+          reject(
+            new Error(
+              `H.264 to WebP encode failed: ${error instanceof Error ? error.message : String(error)}`,
+            ),
+          );
         }
       });
 
       ffmpeg.on('error', (error) => {
+        if (settled) return;
+        settled = true;
         reject(new Error(`Failed to spawn ffmpeg process: ${error.message}`));
       });
 

packages/android/tests/ai/android-emulator-smoke.test.ts

@@ -347,7 +347,7 @@ describe.skipIf(!RUN_LIVE_SMOKE)('Android Emulator live smoke', () => {
     );
     const searchXmlFile = path.join(diagnosticsDir, 'settings-search.xml');
     const finalXmlFile = path.join(diagnosticsDir, 'settings-final.xml');
-    const screenshotFile = path.join(diagnosticsDir, 'settings-search.png');
+    const screenshotFile = path.join(diagnosticsDir, 'settings-search.webp');
     const dumpFile = path.join(diagnosticsDir, 'agent-dump.json');
     const evidenceFile = path.join(diagnosticsDir, 'evidence.json');
     const runDir = path.resolve(process.env.MIDSCENE_RUN_DIR || 'midscene_run');
@@ -395,6 +395,7 @@ describe.skipIf(!RUN_LIVE_SMOKE)('Android Emulator live smoke', () => {
       const logicalSize = await device.size();
       const screenshot = await device.screenshotBase64();
       const screenshotSize = await imageInfoOfBase64(screenshot);
+      expect(screenshot).toMatch(/^data:image\/webp;base64,/);
       const density = await adb.getScreenDensity();
       evidence.device = {
         id: devices[0].udid,
@@ -528,6 +529,7 @@ describe.skipIf(!RUN_LIVE_SMOKE)('Android Emulator live smoke', () => {
       evidence.keyboardShownBeforeBack = await waitForKeyboardState(adb, true);
 
       const searchScreenshot = await device.screenshotBase64();
+      expect(searchScreenshot).toMatch(/^data:image\/webp;base64,/);
       await writeFile(screenshotFile, screenshotBuffer(searchScreenshot));
 
       await agent.back();

packages/android/tests/ai/todo.test.ts

@@ -224,14 +224,13 @@ describe('Test todo list', () => {
         const resolvedDiagnosticsDir = path.resolve(diagnosticsDir);
         await mkdir(resolvedDiagnosticsDir, { recursive: true });
         await Promise.all([
-          agent.interface
-            .screenshotBase64()
-            .then((base64) =>
-              writeFile(
-                path.join(resolvedDiagnosticsDir, 'todo-final.png'),
-                screenshotBuffer(base64),
-              ),
-            ),
+          agent.interface.screenshotBase64().then((base64) => {
+            expect(base64).toMatch(/^data:image\/webp;base64,/);
+            return writeFile(
+              path.join(resolvedDiagnosticsDir, 'todo-final.webp'),
+              screenshotBuffer(base64),
+            );
+          }),
           writeFile(
             path.join(resolvedDiagnosticsDir, 'todo-agent-dump.json'),
             `${agent.dumpDataString()}\n`,

packages/android/tests/unit-test/open-frame-source.test.ts

@@ -117,7 +117,7 @@ describe('AndroidDevice frame-source capability', () => {
         listener = cb;
         return unsubscribe;
       }),
-      decodeRawKeyframeToJpegBase64: decode,
+      decodeRawKeyframeToWebpBase64: decode,
     };
     (device as any).getDevicePhysicalInfo = vi.fn().mockResolvedValue({});
 

packages/android/tests/unit-test/page.test.ts

@@ -108,6 +108,7 @@ vi.mock('@midscene/shared/img', async (importOriginal) => {
     });
   return {
     ...original,
+    canonicalizeScreenshotBase64: vi.fn(),
     createImgBase64ByFormat: vi.fn(),
     resizeAndConvertImgBuffer: vi.fn(),
     validateScreenshotBuffer,
@@ -546,6 +547,9 @@ Stdout:
           format,
         }),
       );
+      vi.mocked(ImgUtils.canonicalizeScreenshotBase64).mockImplementation(
+        async (dataUrl) => dataUrl.replace('image/png', 'image/webp'),
+      );
     });
 
     it('should take screenshot successfully with takeScreenshot', async () => {
@@ -558,6 +562,7 @@ Stdout:
       );
 
       const result = await device.screenshotBase64();
+      expect(result).toContain('data:image/webp;base64,');
       expect(result).toContain(mockBuffer.toString('base64'));
       expect(mockAdb.shell).not.toHaveBeenCalled();
     });
@@ -581,6 +586,7 @@ Stdout:
       expect(mockAdb.pull).toHaveBeenCalled();
       expect(fs.promises.readFile).toHaveBeenCalled();
       expect(result).toContain(mockBuffer.toString('base64'));
+      expect(result).toContain('data:image/webp;base64,');
       // rm is now executed via execFile (fire-and-forget), not adb.shell
     });
 
@@ -601,6 +607,7 @@ Stdout:
       const result = await defaultDevice.screenshotBase64();
 
       expect(result).toContain(smallValidPng.toString('base64'));
+      expect(result).toContain('data:image/webp;base64,');
       expect(mockAdb.pull).toHaveBeenCalled();
     });