From 4f6ed549f8281f222521a4757cd0e7a00c4d86b1 Mon Sep 17 00:00:00 2001 From: zubeyralmaho Date: Tue, 21 Jul 2026 16:48:30 +0300 Subject: [PATCH 1/2] feat(ios): add clipboard/pasteboard get and set via WDA Add IOSWebDriverClient.getPasteboard()/.setPasteboard(), the low-level POST .../wda/getPasteboard and .../wda/setPasteboard calls (base64 content, contentType: "plaintext"), and IOSDevice.getClipboardText()/ .setClipboardText() as the public, ergonomic wrapper. There was previously no way to read a value that an app only exposes through a native "Copy" action (e.g. a share sheet's "Copy Link" for a generated invite/meeting link) -- WDA's own GET /source also cannot help there, since it 500s while a native share sheet (UIActivityViewController) is presented and its accessibility tree cannot be walked. The pasteboard is WDA-native and system-wide, so it works regardless. The request shape (POST + JSON body, not GET + query string) is cross-checked against the unmerged #2628, which shipped passing unit tests against this exact endpoint contract. Test plan: - npx nx test ios (151 passed) - npx nx build ios --- packages/ios/src/device.ts | 14 ++++ packages/ios/src/ios-webdriver-client.ts | 51 ++++++++++++++ packages/ios/tests/unit-test/device.test.ts | 17 +++++ .../ios/tests/unit-test/wda-backend.test.ts | 67 ++++++++++++++++++- 4 files changed, 148 insertions(+), 1 deletion(-) diff --git a/packages/ios/src/device.ts b/packages/ios/src/device.ts index fb964ed6b0..cd46f52b5a 100644 --- a/packages/ios/src/device.ts +++ b/packages/ios/src/device.ts @@ -860,6 +860,20 @@ ScreenSize: ${size.width}x${size.height} (DPR: ${size.scale}) await this.wdaBackend.pressHomeButton(); } + /** + * Read the system pasteboard (clipboard). The only way to retrieve a value that an app + * exposes solely through a native "Copy" action -- e.g. a share sheet's "Copy Link" -- + * since that value has no other on-screen representation to read. + */ + async getClipboardText(): Promise { + return await this.wdaBackend.getPasteboard(); + } + + /** Write plain text to the system pasteboard (clipboard). */ + async setClipboardText(text: string): Promise { + await this.wdaBackend.setPasteboard(text); + } + async appSwitcher(): Promise { try { // For iOS, use swipe up with slower/longer duration to trigger app switcher diff --git a/packages/ios/src/ios-webdriver-client.ts b/packages/ios/src/ios-webdriver-client.ts index 9699f56303..9c0a831be4 100644 --- a/packages/ios/src/ios-webdriver-client.ts +++ b/packages/ios/src/ios-webdriver-client.ts @@ -341,6 +341,57 @@ export class IOSWebDriverClient extends WebDriverClient { } } + /** + * Read the iOS pasteboard (system clipboard) via WDA's `wda/getPasteboard`. + * Useful for values only exposed through a native "Copy" action (e.g. a share + * sheet's "Copy Link"), which have no other readable representation on screen. + * @param contentType Pasteboard content type WDA should decode as; 'plaintext' covers + * the common case (copied text/URLs). + */ + async getPasteboard(contentType = 'plaintext'): Promise { + this.ensureSession(); + + try { + const response = await this.makeRequest( + 'POST', + `/session/${this.sessionId}/wda/getPasteboard`, + { contentType }, + ); + const value = response?.value; + if (!value) { + return ''; + } + return Buffer.from(value, 'base64').toString('utf8'); + } catch (error) { + debugIOS(`Failed to read pasteboard: ${error}`); + throw new Error(`Failed to read pasteboard: ${error}`); + } + } + + /** + * Write to the iOS pasteboard (system clipboard) via WDA's `wda/setPasteboard`. + * @param text Plain text to place on the pasteboard. + * @param contentType Pasteboard content type; 'plaintext' covers the common case. + */ + async setPasteboard(text: string, contentType = 'plaintext'): Promise { + this.ensureSession(); + + try { + await this.makeRequest( + 'POST', + `/session/${this.sessionId}/wda/setPasteboard`, + { + content: Buffer.from(text, 'utf8').toString('base64'), + contentType, + }, + ); + debugIOS(`Wrote to pasteboard: "${text}"`); + } catch (error) { + debugIOS(`Failed to write pasteboard "${text}": ${error}`); + throw new Error(`Failed to write pasteboard: ${error}`); + } + } + async tap(x: number, y: number): Promise { this.ensureSession(); diff --git a/packages/ios/tests/unit-test/device.test.ts b/packages/ios/tests/unit-test/device.test.ts index 4f60cde22b..f9d9c62f39 100644 --- a/packages/ios/tests/unit-test/device.test.ts +++ b/packages/ios/tests/unit-test/device.test.ts @@ -43,6 +43,8 @@ describe('IOSDevice', () => { clearActiveElement: vi.fn().mockResolvedValue(true), pressKey: vi.fn().mockResolvedValue(undefined), pressHomeButton: vi.fn().mockResolvedValue(undefined), + getPasteboard: vi.fn().mockResolvedValue('clipboard-text'), + setPasteboard: vi.fn().mockResolvedValue(undefined), launchApp: vi.fn().mockResolvedValue(undefined), terminateApp: vi.fn().mockResolvedValue(undefined), openUrl: vi.fn().mockResolvedValue(undefined), @@ -413,6 +415,21 @@ describe('IOSDevice', () => { expect(mockWdaClient.swipe).toHaveBeenCalled(); }); + it('should read the clipboard via the WDA pasteboard', async () => { + await device.connect(); + + const text = await device.getClipboardText(); + expect(mockWdaClient.getPasteboard).toHaveBeenCalled(); + expect(text).toBe('clipboard-text'); + }); + + it('should write the clipboard via the WDA pasteboard', async () => { + await device.connect(); + + await device.setClipboardText('new-text'); + expect(mockWdaClient.setPasteboard).toHaveBeenCalledWith('new-text'); + }); + it('should handle keyboard dismissal', async () => { await device.connect(); diff --git a/packages/ios/tests/unit-test/wda-backend.test.ts b/packages/ios/tests/unit-test/wda-backend.test.ts index 9168c1708d..ade860a1c8 100644 --- a/packages/ios/tests/unit-test/wda-backend.test.ts +++ b/packages/ios/tests/unit-test/wda-backend.test.ts @@ -1,7 +1,70 @@ import { DEFAULT_WDA_PORT } from '@midscene/shared/constants'; -import { describe, expect, it } from 'vitest'; +import { describe, expect, it, vi } from 'vitest'; +import { IOSWebDriverClient } from '../../src/ios-webdriver-client'; import type { IOSWebDriverClient as IOSWebDriverClientType } from '../../src/ios-webdriver-client'; +function createClientWithSession() { + const client = new IOSWebDriverClient({ + port: DEFAULT_WDA_PORT, + host: 'localhost', + }); + // Bypass createSession() — we only want to observe outbound HTTP calls. + (client as any).sessionId = 'session-under-test'; + const makeRequest = vi + .spyOn(client as any, 'makeRequest') + .mockResolvedValue(undefined); + return { client, makeRequest }; +} + +describe('IOSWebDriverClient pasteboard', () => { + it('getPasteboard POSTs contentType and base64-decodes the response', async () => { + const { client, makeRequest } = createClientWithSession(); + makeRequest.mockResolvedValue({ + value: Buffer.from('Hello 世界', 'utf8').toString('base64'), + }); + + const text = await client.getPasteboard(); + + expect(makeRequest).toHaveBeenCalledWith( + 'POST', + '/session/session-under-test/wda/getPasteboard', + { contentType: 'plaintext' }, + ); + expect(text).toBe('Hello 世界'); + }); + + it('getPasteboard returns an empty string when WDA reports no value', async () => { + const { client, makeRequest } = createClientWithSession(); + makeRequest.mockResolvedValue({ value: '' }); + + await expect(client.getPasteboard()).resolves.toBe(''); + }); + + it('getPasteboard rejects when the request itself fails', async () => { + const { client, makeRequest } = createClientWithSession(); + makeRequest.mockRejectedValue(new Error('WDA HTTP error: 500')); + + await expect(client.getPasteboard()).rejects.toThrow( + 'Failed to read pasteboard', + ); + }); + + it('setPasteboard POSTs base64-encoded content and contentType', async () => { + const { client, makeRequest } = createClientWithSession(); + + await client.setPasteboard('Hello 世界'); + + expect(makeRequest).toHaveBeenCalledWith( + 'POST', + '/session/session-under-test/wda/setPasteboard', + { + content: Buffer.from('Hello 世界', 'utf8').toString('base64'), + contentType: 'plaintext', + }, + ); + }); +}); + describe('IOSWebDriverClient - Simple Tests', () => { describe('Module Structure', () => { it('should export IOSWebDriverClient class', async () => { @@ -38,6 +101,8 @@ describe('IOSWebDriverClient - Simple Tests', () => { 'tap', 'swipe', 'typeText', + 'getPasteboard', + 'setPasteboard', 'pressKey', 'launchApp', 'openUrl', From 6c519fddd80d611336762fd0528e4a350d5b57ab Mon Sep 17 00:00:00 2001 From: zubeyralmaho Date: Tue, 11 Aug 2026 14:53:50 +0300 Subject: [PATCH 2/2] feat(ios): expose clipboard get/set in the iOS action space Wire IOSDevice.getClipboardText()/.setClipboardText() into actionSpace() as the IOSGetClipboard and IOSSetClipboard actions, following the existing IOSHomeButton / IOSAppSwitcher pattern for iOS-specific actions. Without this the pasteboard methods are reachable only from hand-written code: the planner has no tool for them, so a prompt like "copy the invite link and read it back" cannot work. IOSGetClipboard is what makes a value that exists only behind a native "Copy" action (e.g. a share sheet's "Copy Link") usable inside a plan, since such a value is rendered nowhere on screen and cannot be reached by locating or extraction. Both actions are registered locally rather than as canonical cross-platform actions in @midscene/core, matching how the other iOS-specific actions are declared. interfaceAlias also exposes them as agent.getClipboardText() and agent.setClipboardText(). Test plan: - npx nx test ios (163 passed) - npx nx build ios - pnpm run lint --- packages/ios/src/device.ts | 38 +++++++++++++++++++ packages/ios/tests/unit-test/device.test.ts | 27 +++++++++++++ .../ios/tests/unit-test/structure.test.ts | 2 + 3 files changed, 67 insertions(+) diff --git a/packages/ios/src/device.ts b/packages/ios/src/device.ts index cd46f52b5a..76e86e8184 100644 --- a/packages/ios/src/device.ts +++ b/packages/ios/src/device.ts @@ -1109,6 +1109,14 @@ type TerminateParam = z.infer; export type DeviceActionTerminate = DeviceAction; +const setClipboardParamSchema = z.object({ + text: z + .string() + .describe('Plain text to write to the iOS system pasteboard (clipboard)'), +}); + +type SetClipboardParam = z.infer; + /** * Platform-specific action definitions for iOS * Single source of truth for both runtime behavior and type definitions @@ -1180,9 +1188,39 @@ const createPlatformActions = (device: IOSDevice) => { await device.appSwitcher(); }, }), + IOSGetClipboard: defineAction({ + name: 'IOSGetClipboard', + description: + 'Read the iOS system pasteboard (clipboard) and return its text. Use this to obtain a value that an app only exposes through a native "Copy" action -- e.g. a share sheet\'s "Copy Link" -- and that is therefore not rendered anywhere on screen.', + interfaceAlias: 'getClipboardText', + call: async () => { + return await device.getClipboardText(); + }, + }), + IOSSetClipboard: defineAction< + typeof setClipboardParamSchema, + SetClipboardParam, + void + >({ + name: 'IOSSetClipboard', + description: + 'Write plain text to the iOS system pasteboard (clipboard) so it can be pasted into an app.', + interfaceAlias: 'setClipboardText', + paramSchema: setClipboardParamSchema, + sample: { + text: 'https://example.com/invite/abc123', + }, + call: async (param) => { + await device.setClipboardText(param.text); + }, + }), } as const; }; export type DeviceActionIOSHomeButton = DeviceAction; export type DeviceActionIOSAppSwitcher = DeviceAction; + +export type DeviceActionIOSGetClipboard = DeviceAction; + +export type DeviceActionIOSSetClipboard = DeviceAction; diff --git a/packages/ios/tests/unit-test/device.test.ts b/packages/ios/tests/unit-test/device.test.ts index f9d9c62f39..9c20529ae8 100644 --- a/packages/ios/tests/unit-test/device.test.ts +++ b/packages/ios/tests/unit-test/device.test.ts @@ -430,6 +430,33 @@ describe('IOSDevice', () => { expect(mockWdaClient.setPasteboard).toHaveBeenCalledWith('new-text'); }); + it('should expose IOSGetClipboard in the action space and return its text', async () => { + await device.connect(); + + const action = device + .actionSpace() + .find((candidate) => candidate.name === 'IOSGetClipboard'); + expect(action).toBeDefined(); + + const text = await action!.call(undefined as never); + expect(mockWdaClient.getPasteboard).toHaveBeenCalled(); + expect(text).toBe('clipboard-text'); + }); + + it('should expose IOSSetClipboard in the action space and forward its text', async () => { + await device.connect(); + + const action = device + .actionSpace() + .find((candidate) => candidate.name === 'IOSSetClipboard'); + expect(action).toBeDefined(); + + await action!.call({ text: 'from-action-space' } as never); + expect(mockWdaClient.setPasteboard).toHaveBeenCalledWith( + 'from-action-space', + ); + }); + it('should handle keyboard dismissal', async () => { await device.connect(); diff --git a/packages/ios/tests/unit-test/structure.test.ts b/packages/ios/tests/unit-test/structure.test.ts index 1373faeb45..6a95036634 100644 --- a/packages/ios/tests/unit-test/structure.test.ts +++ b/packages/ios/tests/unit-test/structure.test.ts @@ -38,6 +38,8 @@ describe('iOS Package Structure', () => { expect(actionNames).toContain('LongPress'); expect(actionNames).toContain('Swipe'); expect(actionNames).toContain('IOSAppSwitcher'); + expect(actionNames).toContain('IOSGetClipboard'); + expect(actionNames).toContain('IOSSetClipboard'); }); it('should respect configuration options', () => {