diff --git a/packages/computer/README.md b/packages/computer/README.md index e89daf672a..2b5dc667fa 100644 --- a/packages/computer/README.md +++ b/packages/computer/README.md @@ -7,6 +7,30 @@ Midscene.js Computer Desktop Automation - AI-powered desktop automation for: See . +## VNC keyboard input on macOS + +When Midscene runs on macOS and controls a foreground VNC client, enable +physical keyboard events so modifier keys are sent as explicit key-down and +key-up transitions. Text must also use sequential input; a positive +`keyboardTypeDelay` enables that behavior in the default `legacy` input mode: + +```ts +import { agentForComputer } from '@midscene/computer'; + +const agent = await agentForComputer({ + keyboardEventMode: 'physical', + keyboardTypeDelay: 80, +}); +``` + +The default `keyboardEventMode: 'logical'` keeps the standard AppleScript +behavior for non-VNC applications. This option is ignored outside macOS and +when `keyboardDriver` is set to `libnut`. + +Use `physical` only for a VNC client with matching en-US keyboard layouts. Its +shifted-punctuation mapping is not layout-independent, and native macOS apps +may interpret the base key directly—for example, `!@#` can become `123`. + ## RDP support Use `agentForRDPComputer()`: diff --git a/packages/computer/src/agent-tools.ts b/packages/computer/src/agent-tools.ts index d3046faa3a..74e6fcb2da 100644 --- a/packages/computer/src/agent-tools.ts +++ b/packages/computer/src/agent-tools.ts @@ -62,6 +62,12 @@ const computerInitArgShape = { .describe( 'Text input strategy. "legacy" (default) preserves current Computer behavior, "sequential" sends one Unicode code point at a time, and "bulk" uses one backend input operation. "bulk" requires keyboardTypeDelay to be omitted or set to 0.', ), + keyboardEventMode: z + .enum(['logical', 'physical']) + .optional() + .describe( + 'macOS AppleScript keyboard event mode for local control. "logical" (default) targets native apps; "physical" is only for VNC clients, assumes an en-US layout for shifted punctuation, and requires sequential text input or a positive keyboardTypeDelay. Ignored in RDP mode and outside the macOS AppleScript driver.', + ), // RDP options. Providing `host` switches connect into RDP mode and routes // the session through the RDP helper binary instead of the local desktop. // All other RDP options below are silently ignored unless `host` is set. @@ -116,7 +122,10 @@ const computerInitArgShape = { export type ComputerLocalInitArgs = { mode: 'local'; } & Pick & - Pick & + Pick< + ComputerDeviceOpt, + 'inputStrategy' | 'keyboardTypeDelay' | 'keyboardEventMode' + > & AgentBehaviorInitArgs; /** Init args for the RDP remote-desktop agent. */ @@ -136,7 +145,11 @@ export type ComputerInitArgs = ComputerLocalInitArgs | ComputerRDPInitArgs; type ExtractedComputerInitArgs = Partial< Pick< ComputerDeviceOpt, - 'displayId' | 'headless' | 'inputStrategy' | 'keyboardTypeDelay' + | 'displayId' + | 'headless' + | 'inputStrategy' + | 'keyboardTypeDelay' + | 'keyboardEventMode' > & RDPConnectionConfig & AgentBehaviorInitArgs @@ -155,7 +168,12 @@ function adaptComputerInitArgs( } if (extracted.host) { // Drop local-only fields; they're meaningless in RDP mode. - const { displayId: _d, headless: _h, ...rdpFields } = extracted; + const { + displayId: _d, + headless: _h, + keyboardEventMode: _k, + ...rdpFields + } = extracted; const host = normalizeRdpHost(extracted.host); return { mode: 'rdp', @@ -169,6 +187,7 @@ function adaptComputerInitArgs( headless: extracted.headless, keyboardTypeDelay: extracted.keyboardTypeDelay, inputStrategy: extracted.inputStrategy, + keyboardEventMode: extracted.keyboardEventMode, ...(extractAgentBehaviorInitArgs(extracted) ?? {}), }; } @@ -263,12 +282,15 @@ export class ComputerMidsceneTools extends BaseMidsceneTools< const headless = opts?.mode === 'local' ? opts.headless : undefined; const keyboardTypeDelay = opts?.keyboardTypeDelay; const inputStrategy = opts?.inputStrategy; + const keyboardEventMode = + opts?.mode === 'local' ? opts.keyboardEventMode : undefined; debug('Creating Computer agent with displayId:', displayId || 'primary'); const agentOpts = { ...(displayId ? { displayId } : {}), ...(headless !== undefined ? { headless } : {}), ...(keyboardTypeDelay !== undefined ? { keyboardTypeDelay } : {}), ...(inputStrategy !== undefined ? { inputStrategy } : {}), + ...(keyboardEventMode !== undefined ? { keyboardEventMode } : {}), ...(this.options.keepXvfbAliveUntilProcessExit ? { keepXvfbAliveUntilProcessExit: true } : {}), diff --git a/packages/computer/src/agent.ts b/packages/computer/src/agent.ts index 4cd993be28..b9fe98c441 100644 --- a/packages/computer/src/agent.ts +++ b/packages/computer/src/agent.ts @@ -32,6 +32,7 @@ function createLocalComputerDevice( keyboardTypeDelay: opts?.keyboardTypeDelay, inputStrategy: opts?.inputStrategy, keyboardDriver: opts?.keyboardDriver, + keyboardEventMode: opts?.keyboardEventMode, headless: opts?.headless, xvfbResolution: opts?.xvfbResolution, keepXvfbAliveUntilProcessExit: opts?.keepXvfbAliveUntilProcessExit, diff --git a/packages/computer/src/apple-script-keyboard.ts b/packages/computer/src/apple-script-keyboard.ts new file mode 100644 index 0000000000..9774f074ad --- /dev/null +++ b/packages/computer/src/apple-script-keyboard.ts @@ -0,0 +1,153 @@ +import { execFileSync } from 'node:child_process'; +import { getDebug } from '@midscene/shared/logger'; +import { US_SHIFTED_CHARACTER_KEYS } from './keyboard-layout'; + +const debugKeyboard = getDebug('computer:keyboard'); + +const APPLE_SCRIPT_KEY_CODES: Readonly>> = { + return: 36, + enter: 36, + tab: 48, + space: 49, + backspace: 51, + delete: 51, + escape: 53, + forwarddelete: 117, + left: 123, + right: 124, + down: 125, + up: 126, + home: 115, + end: 119, + pageup: 116, + pagedown: 121, + f1: 122, + f2: 120, + f3: 99, + f4: 118, + f5: 96, + f6: 97, + f7: 98, + f8: 100, + f9: 101, + f10: 109, + f11: 103, + f12: 111, +}; + +const APPLE_SCRIPT_MODIFIER_KEYS: Readonly>> = { + command: 'command', + cmd: 'command', + control: 'control', + ctrl: 'control', + shift: 'shift', + alt: 'option', + option: 'option', + meta: 'command', +}; + +/** + * Modifier delivery mode for the macOS AppleScript keyboard backend. + * + * `logical` is the default for native macOS applications. `physical` emits + * explicit modifier transitions for VNC clients and assumes an en-US mapping + * for shifted punctuation; it should not be enabled for native applications. + */ +export type KeyboardEventMode = 'logical' | 'physical'; + +function buildKeyCommand(key: string): string { + const keyCode = APPLE_SCRIPT_KEY_CODES[key.toLowerCase()]; + if (keyCode !== undefined) { + return `key code ${keyCode}`; + } + + const escapedKey = key.replace(/\\/g, '\\\\').replace(/"/g, '\\"'); + return `keystroke "${escapedKey}"`; +} + +function resolveModifierKeys(modifiers: string[]): string[] { + return modifiers + .map((modifier) => APPLE_SCRIPT_MODIFIER_KEYS[modifier.toLowerCase()]) + .filter((modifier): modifier is string => modifier !== undefined); +} + +function buildLogicalKeyPress(key: string, modifiers: string[]): string { + const modifierKeys = resolveModifierKeys(modifiers); + const modifierClause = modifierKeys.length + ? ` using {${modifierKeys + .map((modifier) => `${modifier} down`) + .join(', ')}}` + : ''; + return `tell application "System Events" to ${buildKeyCommand(key)}${modifierClause}`; +} + +function resolvePhysicalKey( + key: string, + modifiers: string[], +): { key: string; modifiers: string[] } { + const resolvedModifiers = [...modifiers]; + const shiftedBaseKey = US_SHIFTED_CHARACTER_KEYS.get(key); + + if (/^[A-Z]$/.test(key)) { + resolvedModifiers.push('shift'); + return { key: key.toLowerCase(), modifiers: resolvedModifiers }; + } + if (shiftedBaseKey !== undefined) { + resolvedModifiers.push('shift'); + return { key: shiftedBaseKey, modifiers: resolvedModifiers }; + } + return { key, modifiers: resolvedModifiers }; +} + +function buildPhysicalKeyPress(key: string, modifiers: string[]): string { + const resolved = resolvePhysicalKey(key, modifiers); + const modifierKeys = [...new Set(resolveModifierKeys(resolved.modifiers))]; + const keyCommand = buildKeyCommand(resolved.key); + + if (modifierKeys.length === 0) { + return `tell application "System Events" to ${keyCommand}`; + } + + const releaseCommands = [...modifierKeys] + .reverse() + .map((modifier) => `key up ${modifier}`); + return [ + 'tell application "System Events"', + 'try', + ...modifierKeys.map((modifier) => `key down ${modifier}`), + keyCommand, + 'on error errorMessage number errorNumber', + ...releaseCommands, + 'error errorMessage number errorNumber', + 'end try', + ...releaseCommands, + 'end tell', + ].join('\n'); +} + +/** @internal exported for focused unit tests */ +export function buildAppleScriptKeyPress( + key: string, + modifiers: string[] = [], + eventMode: KeyboardEventMode = 'logical', +): string { + return eventMode === 'physical' + ? buildPhysicalKeyPress(key, modifiers) + : buildLogicalKeyPress(key, modifiers); +} + +/** Send one key press through macOS System Events without invoking a shell. */ +export function sendKeyViaAppleScript( + key: string, + modifiers: string[] = [], + eventMode: KeyboardEventMode = 'logical', +): void { + const script = buildAppleScriptKeyPress(key, modifiers, eventMode); + debugKeyboard('sendKeyViaAppleScript', { + key, + modifiers, + eventMode, + script, + }); + execFileSync('osascript', ['-e', script]); +} diff --git a/packages/computer/src/device.ts b/packages/computer/src/device.ts index eac5271f41..d047d1f07a 100644 --- a/packages/computer/src/device.ts +++ b/packages/computer/src/device.ts @@ -26,11 +26,16 @@ import { sleep } from '@midscene/core/utils'; import { createImgBase64ByFormat } from '@midscene/shared/img'; import { getDebug } from '@midscene/shared/logger'; import screenshot from 'screenshot-desktop'; +import { + type KeyboardEventMode, + sendKeyViaAppleScript, +} from './apple-script-keyboard'; import { ComputerInputDriver, type LibNut, type ScrollDirection, } from './input-driver'; +import { US_SHIFTED_CHARACTER_KEYS } from './keyboard-layout'; import { runWindowsPhysicalPixelPowershell } from './windows-dpi'; import { WindowsPointerDriver, @@ -153,34 +158,6 @@ const LIBNUT_FALLBACK_PIXELS_PER_DETENT = 100; const LIBNUT_FALLBACK_TICK_DELAY_MS = 30; const LIBNUT_FALLBACK_MAX_DETENTS = 200; const LIBNUT_FALLBACK_DETENT_AMOUNT = process.platform === 'win32' ? 120 : 1; -// Work around libnut's Linux shifted-punctuation behavior for en-US layouts. -// This is intentionally not a universal keyboard-layout map: non-US layouts -// can place these characters on different keys or modifier levels (for example, -// AltGr). Layout-independent support should resolve characters against the -// active layout in the input backend instead of extending this table. -const LINUX_SHIFTED_CHARACTER_KEYS = new Map([ - ['~', '`'], - ['!', '1'], - ['@', '2'], - ['#', '3'], - ['$', '4'], - ['%', '5'], - ['^', '6'], - ['&', '7'], - ['*', '8'], - ['(', '9'], - [')', '0'], - ['_', '-'], - ['+', '='], - ['{', '['], - ['}', ']'], - ['|', '\\'], - [':', ';'], - ['"', "'"], - ['<', ','], - ['>', '.'], - ['?', '/'], -]); // Edge scrolls (scrollToTop / scrollToBottom / ...) must drive all the way to // the boundary on every backend. The phased path requests EDGE_SCROLL_TOTAL_PX // (50_000 px); the libnut fallback aims for the same distance, capped at @@ -225,87 +202,6 @@ const EDGE_SCROLL_SPEC: Record = { scrollToRight: { direction: 'right', key: 'end', libnut: [1, 0] }, }; -// macOS AppleScript key code mapping -// Reference: https://eastmanreference.com/complete-list-of-applescript-key-codes -const APPLESCRIPT_KEY_CODE_MAP: Record = { - // Special keys - return: 36, - enter: 36, - tab: 48, - space: 49, - backspace: 51, - delete: 51, - escape: 53, - forwarddelete: 117, - - // Arrow keys - left: 123, - right: 124, - down: 125, - up: 126, - - // Navigation keys - home: 115, - end: 119, - pageup: 116, - pagedown: 121, - - // Function keys - f1: 122, - f2: 120, - f3: 99, - f4: 118, - f5: 96, - f6: 97, - f7: 98, - f8: 100, - f9: 101, - f10: 109, - f11: 103, - f12: 111, -}; - -// Modifier key mapping for AppleScript -const APPLESCRIPT_MODIFIER_MAP: Record = { - command: 'command down', - cmd: 'command down', - control: 'control down', - ctrl: 'control down', - shift: 'shift down', - alt: 'option down', - option: 'option down', - meta: 'command down', -}; - -/** - * Send a key press using AppleScript (macOS only) - * More reliable than libnut for TUI applications like Bubble Tea - */ -function sendKeyViaAppleScript(key: string, modifiers: string[] = []): void { - const lowerKey = key.toLowerCase(); - const keyCode = APPLESCRIPT_KEY_CODE_MAP[lowerKey]; - - // Build modifier string - const modifierParts = modifiers - .map((m) => APPLESCRIPT_MODIFIER_MAP[m.toLowerCase()]) - .filter(Boolean); - const modifierStr = - modifierParts.length > 0 ? ` using {${modifierParts.join(', ')}}` : ''; - - let script: string; - - if (keyCode !== undefined) { - // Use key code for special keys - script = `tell application "System Events" to key code ${keyCode}${modifierStr}`; - } else { - const escapedKey = key.replace(/\\/g, '\\\\').replace(/"/g, '\\"'); - script = `tell application "System Events" to keystroke "${escapedKey}"${modifierStr}`; - } - - debugDevice('sendKeyViaAppleScript', { key, modifiers, script }); - execFileSync('osascript', ['-e', script]); -} - function escapePowershellSingleQuoted(value: string): string { return value.replace(/'/g, "''"); } @@ -821,6 +717,19 @@ export interface ComputerDeviceOpt extends ComputerDeviceInputOpt { * - 'libnut': Use libnut's keyTap (faster but may not work with some TUI apps) */ keyboardDriver?: 'applescript' | 'libnut'; + /** + * How the macOS AppleScript keyboard driver represents modifier keys. + * `logical` keeps the default compact `keystroke ... using` behavior. + * `physical` emits explicit modifier key-down/key-up transitions for apps + * such as VNC clients that forward physical keyboard events. Physical mode + * assumes an en-US layout for shifted punctuation and may type base keys in + * native macOS applications. Text input must use sequential input or a + * positive `keyboardTypeDelay` to emit individual keys. + * + * Ignored outside macOS and when `keyboardDriver` is `libnut`. + * @default 'logical' + */ + keyboardEventMode?: KeyboardEventMode; /** * Headless mode via Xvfb (Linux only). * - true: start Xvfb virtual display @@ -855,7 +764,12 @@ export class ComputerDevice implements AbstractInterface { private readonly inputDriver = new ComputerInputDriver({ getLibnut: () => libnut, useAppleScript: () => this.useAppleScript, - sendKeyViaAppleScript, + sendKeyViaAppleScript: (key, modifiers) => + sendKeyViaAppleScript( + key, + modifiers, + this.options?.keyboardEventMode ?? 'logical', + ), runPhasedScroll, debug: (message) => debugDevice(message), }); @@ -1577,7 +1491,7 @@ $g.Dispose(); $bmp.Dispose(); $ms.Dispose() sendCharacter: (character) => { const linuxShiftedKey = process.platform === 'linux' - ? LINUX_SHIFTED_CHARACTER_KEYS.get(character) + ? US_SHIFTED_CHARACTER_KEYS.get(character) : undefined; if (character === '\n') { this.inputDriver.sendKey('return'); diff --git a/packages/computer/src/index.ts b/packages/computer/src/index.ts index 5ead953547..45cff7a888 100644 --- a/packages/computer/src/index.ts +++ b/packages/computer/src/index.ts @@ -1,4 +1,5 @@ export { ComputerDevice } from './device'; +export type { KeyboardEventMode } from './apple-script-keyboard'; export type { ComputerDeviceOpt, DisplayInfo } from './device'; export { diff --git a/packages/computer/src/keyboard-layout.ts b/packages/computer/src/keyboard-layout.ts new file mode 100644 index 0000000000..4e94dd732e --- /dev/null +++ b/packages/computer/src/keyboard-layout.ts @@ -0,0 +1,31 @@ +/** + * Physical base keys for shifted characters on en-US keyboard layouts. + * + * This is intentionally not a universal keyboard-layout map. Other layouts + * can place these characters on different keys or modifier levels, such as + * AltGr. Layout-independent support should resolve characters against the + * active layout in the input backend instead of extending this table. + */ +export const US_SHIFTED_CHARACTER_KEYS: ReadonlyMap = new Map([ + ['~', '`'], + ['!', '1'], + ['@', '2'], + ['#', '3'], + ['$', '4'], + ['%', '5'], + ['^', '6'], + ['&', '7'], + ['*', '8'], + ['(', '9'], + [')', '0'], + ['_', '-'], + ['+', '='], + ['{', '['], + ['}', ']'], + ['|', '\\'], + [':', ';'], + ['"', "'"], + ['<', ','], + ['>', '.'], + ['?', '/'], +]); diff --git a/packages/computer/tests/unit-test/agent-tools.test.ts b/packages/computer/tests/unit-test/agent-tools.test.ts index a0173a7076..ae66916280 100644 --- a/packages/computer/tests/unit-test/agent-tools.test.ts +++ b/packages/computer/tests/unit-test/agent-tools.test.ts @@ -68,6 +68,7 @@ describe('ComputerMidsceneTools', () => { headless: true, 'keyboard-type-delay': 80, 'input-strategy': 'sequential', + 'keyboard-event-mode': 'physical', }, }); @@ -76,6 +77,7 @@ describe('ComputerMidsceneTools', () => { headless: true, keyboardTypeDelay: 80, inputStrategy: 'sequential', + keyboardEventMode: 'physical', }); }); @@ -170,6 +172,7 @@ describe('ComputerMidsceneTools', () => { 'computer.displayId': expect.anything(), 'computer.headless': expect.anything(), 'computer.inputStrategy': expect.anything(), + 'computer.keyboardEventMode': expect.anything(), 'computer.keyboardTypeDelay': expect.anything(), 'computer.waitAfterAction': expect.anything(), 'computer.replanningCycleLimit': expect.anything(), @@ -182,6 +185,7 @@ describe('ComputerMidsceneTools', () => { 'computer.headless': expect.anything(), 'computer.host': expect.anything(), 'computer.inputStrategy': expect.anything(), + 'computer.keyboardEventMode': expect.anything(), 'computer.keyboardTypeDelay': expect.anything(), 'computer.waitAfterAction': expect.anything(), 'computer.port': expect.anything(), @@ -213,6 +217,7 @@ describe('ComputerMidsceneTools', () => { 'ignore-certificate': true, 'input-strategy': 'sequential', 'keyboard-type-delay': 80, + 'keyboard-event-mode': 'physical', }); expect(agentForRDPComputer).toHaveBeenCalledWith( @@ -228,6 +233,9 @@ describe('ComputerMidsceneTools', () => { keyboardTypeDelay: 80, }), ); + expect(rs.mocked(agentForRDPComputer).mock.calls[0][0]).not.toHaveProperty( + 'keyboardEventMode', + ); expect(agentFromComputer).not.toHaveBeenCalled(); }); diff --git a/packages/computer/tests/unit-test/apple-script-keyboard.test.ts b/packages/computer/tests/unit-test/apple-script-keyboard.test.ts new file mode 100644 index 0000000000..a3ed4c4ef0 --- /dev/null +++ b/packages/computer/tests/unit-test/apple-script-keyboard.test.ts @@ -0,0 +1,69 @@ +import { describe, expect, it } from '@rstest/core'; +import { buildAppleScriptKeyPress } from '../../src/apple-script-keyboard'; + +describe('AppleScript keyboard event modes', () => { + it('keeps compact logical events as the default', () => { + expect(buildAppleScriptKeyPress('s', ['control'])).toBe( + 'tell application "System Events" to keystroke "s" using {control down}', + ); + expect(buildAppleScriptKeyPress('K')).toBe( + 'tell application "System Events" to keystroke "K"', + ); + }); + + it.each([ + ['K', 'k'], + ['#', '3'], + ])('decomposes physical %s into Shift plus %s', (key, baseKey) => { + expect(buildAppleScriptKeyPress(key, [], 'physical')).toBe( + [ + 'tell application "System Events"', + 'try', + 'key down shift', + `keystroke "${baseKey}"`, + 'on error errorMessage number errorNumber', + 'key up shift', + 'error errorMessage number errorNumber', + 'end try', + 'key up shift', + 'end tell', + ].join('\n'), + ); + }); + + it('deduplicates aliases and releases multiple modifiers in reverse order', () => { + expect( + buildAppleScriptKeyPress( + 'K', + ['meta', 'alt', 'command', 'shift'], + 'physical', + ), + ).toBe( + [ + 'tell application "System Events"', + 'try', + 'key down command', + 'key down option', + 'key down shift', + 'keystroke "k"', + 'on error errorMessage number errorNumber', + 'key up shift', + 'key up option', + 'key up command', + 'error errorMessage number errorNumber', + 'end try', + 'key up shift', + 'key up option', + 'key up command', + 'end tell', + ].join('\n'), + ); + }); + + it('uses AppleScript key codes for modified special keys', () => { + const script = buildAppleScriptKeyPress('Enter', ['ctrl'], 'physical'); + + expect(script).toContain('key down control\nkey code 36'); + expect(script).not.toContain('keystroke "Enter"'); + }); +}); diff --git a/packages/computer/tests/unit-test/device-security.test.ts b/packages/computer/tests/unit-test/device-security.test.ts index 0a7bf1ace6..4d714a5e50 100644 --- a/packages/computer/tests/unit-test/device-security.test.ts +++ b/packages/computer/tests/unit-test/device-security.test.ts @@ -177,15 +177,20 @@ afterEach(() => { Object.defineProperty(process, 'platform', { value: originalPlatform }); }); -async function createConnectedDevice() { +async function createConnectedDevice(options?: { + keyboardEventMode?: 'logical' | 'physical'; +}) { const { ComputerDevice } = await import('../../src/device'); - const device = new ComputerDevice({}); + const device = new ComputerDevice(options); await device.connect(); return device; } -async function runKeyboardPress(keyName: string): Promise { - const device = await createConnectedDevice(); +async function runKeyboardPress( + keyName: string, + keyboardEventMode?: 'logical' | 'physical', +): Promise { + const device = await createConnectedDevice({ keyboardEventMode }); const keyboardPress = device .actionSpace() @@ -235,6 +240,35 @@ describe('ComputerDevice AppleScript security', () => { 'tell application "System Events" to keystroke "a\\"\\\\b"', ]); }); + + it('keeps logical modifier events by default', async () => { + await runKeyboardPress('Control+s'); + + expect(mockState.execFileSync).toHaveBeenCalledWith('osascript', [ + '-e', + 'tell application "System Events" to keystroke "s" using {control down}', + ]); + }); + + it('holds shortcut modifiers explicitly for VNC clients', async () => { + await runKeyboardPress('Control+s', 'physical'); + + expect(mockState.execFileSync).toHaveBeenCalledWith('osascript', [ + '-e', + [ + 'tell application "System Events"', + 'try', + 'key down control', + 'keystroke "s"', + 'on error errorMessage number errorNumber', + 'key up control', + 'error errorMessage number errorNumber', + 'end try', + 'key up control', + 'end tell', + ].join('\n'), + ]); + }); }); describe('ComputerDevice destroy input gate', () => {