fix(computer): preserve modifier keys through VNC

packages/computer/README.md

@@ -7,6 +7,30 @@ Midscene.js Computer Desktop Automation - AI-powered desktop automation for:
 
 See <https://midscenejs.com/platforms/desktop>.
 
+## VNC keyboard input on macOS
+
+When Midscene runs on macOS and controls a foreground VNC client, enable
+physical keyboard events so modifier keys are sent as explicit key-down and
+key-up transitions. Text must also use sequential input; a positive
+`keyboardTypeDelay` enables that behavior in the default `legacy` input mode:
+
+```ts
+import { agentForComputer } from '@midscene/computer';
+
+const agent = await agentForComputer({
+  keyboardEventMode: 'physical',
+  keyboardTypeDelay: 80,
+});
+```
+
+The default `keyboardEventMode: 'logical'` keeps the standard AppleScript
+behavior for non-VNC applications. This option is ignored outside macOS and
+when `keyboardDriver` is set to `libnut`.
+
+Use `physical` only for a VNC client with matching en-US keyboard layouts. Its
+shifted-punctuation mapping is not layout-independent, and native macOS apps
+may interpret the base key directly—for example, `!@#` can become `123`.
+
 ## RDP support
 
 Use `agentForRDPComputer()`:

packages/computer/src/agent-tools.ts

@@ -62,6 +62,12 @@ const computerInitArgShape = {
     .describe(
       'Text input strategy. "legacy" (default) preserves current Computer behavior, "sequential" sends one Unicode code point at a time, and "bulk" uses one backend input operation. "bulk" requires keyboardTypeDelay to be omitted or set to 0.',
     ),
+  keyboardEventMode: z
+    .enum(['logical', 'physical'])
+    .optional()
+    .describe(
+      'macOS AppleScript keyboard event mode for local control. "logical" (default) targets native apps; "physical" is only for VNC clients, assumes an en-US layout for shifted punctuation, and requires sequential text input or a positive keyboardTypeDelay. Ignored in RDP mode and outside the macOS AppleScript driver.',
+    ),
   // RDP options. Providing `host` switches connect into RDP mode and routes
   // the session through the RDP helper binary instead of the local desktop.
   // All other RDP options below are silently ignored unless `host` is set.
@@ -116,7 +122,10 @@ const computerInitArgShape = {
 export type ComputerLocalInitArgs = {
   mode: 'local';
 } & Pick<ComputerDeviceOpt, 'displayId' | 'headless'> &
-  Pick<ComputerDeviceOpt, 'inputStrategy' | 'keyboardTypeDelay'> &
+  Pick<
+    ComputerDeviceOpt,
+    'inputStrategy' | 'keyboardTypeDelay' | 'keyboardEventMode'
+  > &
   AgentBehaviorInitArgs;
 
 /** Init args for the RDP remote-desktop agent. */
@@ -136,7 +145,11 @@ export type ComputerInitArgs = ComputerLocalInitArgs | ComputerRDPInitArgs;
 type ExtractedComputerInitArgs = Partial<
   Pick<
     ComputerDeviceOpt,
-    'displayId' | 'headless' | 'inputStrategy' | 'keyboardTypeDelay'
+    | 'displayId'
+    | 'headless'
+    | 'inputStrategy'
+    | 'keyboardTypeDelay'
+    | 'keyboardEventMode'
   > &
     RDPConnectionConfig &
     AgentBehaviorInitArgs
@@ -155,7 +168,12 @@ function adaptComputerInitArgs(
   }
   if (extracted.host) {
     // Drop local-only fields; they're meaningless in RDP mode.
-    const { displayId: _d, headless: _h, ...rdpFields } = extracted;
+    const {
+      displayId: _d,
+      headless: _h,
+      keyboardEventMode: _k,
+      ...rdpFields
+    } = extracted;
     const host = normalizeRdpHost(extracted.host);
     return {
       mode: 'rdp',
@@ -169,6 +187,7 @@ function adaptComputerInitArgs(
     headless: extracted.headless,
     keyboardTypeDelay: extracted.keyboardTypeDelay,
     inputStrategy: extracted.inputStrategy,
+    keyboardEventMode: extracted.keyboardEventMode,
     ...(extractAgentBehaviorInitArgs(extracted) ?? {}),
   };
 }
@@ -263,12 +282,15 @@ export class ComputerMidsceneTools extends BaseMidsceneTools<
     const headless = opts?.mode === 'local' ? opts.headless : undefined;
     const keyboardTypeDelay = opts?.keyboardTypeDelay;
     const inputStrategy = opts?.inputStrategy;
+    const keyboardEventMode =
+      opts?.mode === 'local' ? opts.keyboardEventMode : undefined;
     debug('Creating Computer agent with displayId:', displayId || 'primary');
     const agentOpts = {
       ...(displayId ? { displayId } : {}),
       ...(headless !== undefined ? { headless } : {}),
       ...(keyboardTypeDelay !== undefined ? { keyboardTypeDelay } : {}),
       ...(inputStrategy !== undefined ? { inputStrategy } : {}),
+      ...(keyboardEventMode !== undefined ? { keyboardEventMode } : {}),
       ...(this.options.keepXvfbAliveUntilProcessExit
         ? { keepXvfbAliveUntilProcessExit: true }
         : {}),

packages/computer/src/agent.ts

@@ -32,6 +32,7 @@ function createLocalComputerDevice(
     keyboardTypeDelay: opts?.keyboardTypeDelay,
     inputStrategy: opts?.inputStrategy,
     keyboardDriver: opts?.keyboardDriver,
+    keyboardEventMode: opts?.keyboardEventMode,
     headless: opts?.headless,
     xvfbResolution: opts?.xvfbResolution,
     keepXvfbAliveUntilProcessExit: opts?.keepXvfbAliveUntilProcessExit,

packages/computer/src/apple-script-keyboard.ts

@@ -0,0 +1,153 @@
+import { execFileSync } from 'node:child_process';
+import { getDebug } from '@midscene/shared/logger';
+import { US_SHIFTED_CHARACTER_KEYS } from './keyboard-layout';
+
+const debugKeyboard = getDebug('computer:keyboard');
+
+const APPLE_SCRIPT_KEY_CODES: Readonly<Partial<Record<string, number>>> = {
+  return: 36,
+  enter: 36,
+  tab: 48,
+  space: 49,
+  backspace: 51,
+  delete: 51,
+  escape: 53,
+  forwarddelete: 117,
+  left: 123,
+  right: 124,
+  down: 125,
+  up: 126,
+  home: 115,
+  end: 119,
+  pageup: 116,
+  pagedown: 121,
+  f1: 122,
+  f2: 120,
+  f3: 99,
+  f4: 118,
+  f5: 96,
+  f6: 97,
+  f7: 98,
+  f8: 100,
+  f9: 101,
+  f10: 109,
+  f11: 103,
+  f12: 111,
+};
+
+const APPLE_SCRIPT_MODIFIER_KEYS: Readonly<Partial<Record<string, string>>> = {
+  command: 'command',
+  cmd: 'command',
+  control: 'control',
+  ctrl: 'control',
+  shift: 'shift',
+  alt: 'option',
+  option: 'option',
+  meta: 'command',
+};
+
+/**
+ * Modifier delivery mode for the macOS AppleScript keyboard backend.
+ *
+ * `logical` is the default for native macOS applications. `physical` emits
+ * explicit modifier transitions for VNC clients and assumes an en-US mapping
+ * for shifted punctuation; it should not be enabled for native applications.
+ */
+export type KeyboardEventMode = 'logical' | 'physical';
+
+function buildKeyCommand(key: string): string {
+  const keyCode = APPLE_SCRIPT_KEY_CODES[key.toLowerCase()];
+  if (keyCode !== undefined) {
+    return `key code ${keyCode}`;
+  }
+
+  const escapedKey = key.replace(/\\/g, '\\\\').replace(/"/g, '\\"');
+  return `keystroke "${escapedKey}"`;
+}
+
+function resolveModifierKeys(modifiers: string[]): string[] {
+  return modifiers
+    .map((modifier) => APPLE_SCRIPT_MODIFIER_KEYS[modifier.toLowerCase()])
+    .filter((modifier): modifier is string => modifier !== undefined);
+}
+
+function buildLogicalKeyPress(key: string, modifiers: string[]): string {
+  const modifierKeys = resolveModifierKeys(modifiers);
+  const modifierClause = modifierKeys.length
+    ? ` using {${modifierKeys
+        .map((modifier) => `${modifier} down`)
+        .join(', ')}}`
+    : '';
+  return `tell application "System Events" to ${buildKeyCommand(key)}${modifierClause}`;
+}
+
+function resolvePhysicalKey(
+  key: string,
+  modifiers: string[],
+): { key: string; modifiers: string[] } {
+  const resolvedModifiers = [...modifiers];
+  const shiftedBaseKey = US_SHIFTED_CHARACTER_KEYS.get(key);
+
+  if (/^[A-Z]$/.test(key)) {
+    resolvedModifiers.push('shift');
+    return { key: key.toLowerCase(), modifiers: resolvedModifiers };
+  }
+  if (shiftedBaseKey !== undefined) {
+    resolvedModifiers.push('shift');
+    return { key: shiftedBaseKey, modifiers: resolvedModifiers };
+  }
+  return { key, modifiers: resolvedModifiers };
+}
+
+function buildPhysicalKeyPress(key: string, modifiers: string[]): string {
+  const resolved = resolvePhysicalKey(key, modifiers);
+  const modifierKeys = [...new Set(resolveModifierKeys(resolved.modifiers))];
+  const keyCommand = buildKeyCommand(resolved.key);
+
+  if (modifierKeys.length === 0) {
+    return `tell application "System Events" to ${keyCommand}`;
+  }
+
+  const releaseCommands = [...modifierKeys]
+    .reverse()
+    .map((modifier) => `key up ${modifier}`);
+  return [
+    'tell application "System Events"',
+    'try',
+    ...modifierKeys.map((modifier) => `key down ${modifier}`),
+    keyCommand,
+    'on error errorMessage number errorNumber',
+    ...releaseCommands,
+    'error errorMessage number errorNumber',
+    'end try',
+    ...releaseCommands,
+    'end tell',
+  ].join('\n');
+}
+
+/** @internal exported for focused unit tests */
+export function buildAppleScriptKeyPress(
+  key: string,
+  modifiers: string[] = [],
+  eventMode: KeyboardEventMode = 'logical',
+): string {
+  return eventMode === 'physical'
+    ? buildPhysicalKeyPress(key, modifiers)
+    : buildLogicalKeyPress(key, modifiers);
+}
+
+/** Send one key press through macOS System Events without invoking a shell. */
+export function sendKeyViaAppleScript(
+  key: string,
+  modifiers: string[] = [],
+  eventMode: KeyboardEventMode = 'logical',
+): void {
+  const script = buildAppleScriptKeyPress(key, modifiers, eventMode);
+  debugKeyboard('sendKeyViaAppleScript', {
+    key,
+    modifiers,
+    eventMode,
+    script,
+  });
+  execFileSync('osascript', ['-e', script]);
+}