[1/3] feat(desktop): add desktop observation and overlay baseline

apps/stage-tamagotchi/src/main/index.ts

@@ -34,6 +34,7 @@ import { setupAboutWindowReusable } from './windows/about'
 import { setupBeatSync } from './windows/beat-sync'
 import { setupCaptionWindowManager } from './windows/caption'
 import { setupChatWindowReusableFunc } from './windows/chat'
+import { isDesktopOverlayEnabled, setupDesktopOverlayWindow } from './windows/desktop-overlay'
 import { setupDevtoolsWindow } from './windows/devtools'
 import { setupMainWindow } from './windows/main'
 import { setupNoticeWindowManager } from './windows/notice'
@@ -191,6 +192,22 @@ app.whenReady().then(async () => {
     build: async ({ dependsOn }) => setupTray(dependsOn),
   })
 
+  // Desktop grounding overlay — gated by AIRI_DESKTOP_OVERLAY=1
+  if (isDesktopOverlayEnabled()) {
+    const desktopOverlay = injeca.provide('windows:desktop-overlay', {
+      dependsOn: { mcpStdioManager, serverChannel, i18n },
+      build: async ({ dependsOn }) => setupDesktopOverlayWindow(dependsOn),
+    })
+
+    // NOTICE: Separate invoke ensures the overlay is eagerly built.
+    // Without this, injeca.start() would skip it because no other
+    // provider depends on 'windows:desktop-overlay'.
+    injeca.invoke({
+      dependsOn: { desktopOverlay },
+      callback: noop,
+    })
+  }
+
   injeca.invoke({
     dependsOn: { mainWindow, tray, serverChannel, airiHttpServer, pluginHost, mcpStdioManager, onboardingWindow: onboardingWindowManager },
     callback: noop,

apps/stage-tamagotchi/src/main/windows/desktop-overlay/index.ts

@@ -0,0 +1,148 @@
+/**
+ * Desktop Grounding Overlay — transparent always-on-top window
+ *
+ * Renders:
+ * - Ghost pointer dot at the snap-resolved click position
+ * - Bounding box around the matched target candidate
+ * - Source label + confidence badge
+ * - Stale flags
+ *
+ * Gated by AIRI_DESKTOP_OVERLAY=1 environment variable.
+ * When disabled, this module is a no-op.
+ *
+ * Data flow (v1):
+ * - The overlay renderer polls `computer_use::desktop_get_state` via the MCP bridge
+ * - No IPC push from main process to renderer
+ * - No Eventa channels or server push
+ *
+ * The overlay is click-through (setIgnoreMouseEvents) so it never
+ * intercepts real user or OS-level click events.
+ */
+
+import type { I18n } from '../../libs/i18n'
+import type { ServerChannel } from '../../services/airi/channel-server'
+import type { McpStdioManager } from '../../services/airi/mcp-servers'
+
+import { join, resolve } from 'node:path'
+
+import { BrowserWindow, screen } from 'electron'
+
+import { baseUrl, getElectronMainDirname, load, withHashRoute } from '../../libs/electron/location'
+import { setupDesktopOverlayElectronInvokes } from './rpc/index.electron'
+
+/** Whether the desktop overlay feature is enabled */
+export function isDesktopOverlayEnabled(): boolean {
+  return process.env.AIRI_DESKTOP_OVERLAY === '1'
+}
+
+let overlayWindow: BrowserWindow | null = null
+
+/**
+ * Create the transparent overlay window covering the full primary display.
+ * The window is:
+ * - Always on top (screen level)
+ * - Click-through (ignoreMouseEvents)
+ * - Transparent and frameless
+ * - Not shown in taskbar / dock
+ *
+ * Returns null if AIRI_DESKTOP_OVERLAY is not set.
+ */
+export async function setupDesktopOverlayWindow(params: {
+  mcpStdioManager: McpStdioManager
+  serverChannel: ServerChannel
+  i18n: I18n
+}): Promise<BrowserWindow | null> {
+  if (!isDesktopOverlayEnabled()) {
+    return null
+  }
+
+  // Use primary display bounds (not just size) — the origin may be non-zero
+  // when multiple displays are arranged in macOS Display Preferences.
+  const primaryDisplay = screen.getPrimaryDisplay()
+  const { x, y, width, height } = primaryDisplay.bounds
+
+  overlayWindow = new BrowserWindow({
+    title: 'AIRI Desktop Overlay',
+    width,
+    height,
+    x,
+    y,
+    show: false,
+    frame: false,
+    transparent: true,
+    alwaysOnTop: true,
+    skipTaskbar: true,
+    hasShadow: false,
+    // Round corners off for pixel-accurate overlay
+    roundedCorners: false,
+    // Prevent the overlay from stealing focus
+    focusable: false,
+    webPreferences: {
+      preload: join(getElectronMainDirname(), '../preload/index.mjs'),
+      sandbox: false,
+      // Disable background throttling so animations stay smooth
+      backgroundThrottling: false,
+    },
+  })
+
+  // Make click-through: all mouse events pass through to the desktop
+  overlayWindow.setIgnoreMouseEvents(true, { forward: true })
+
+  // Set to screen level (above all other windows)
+  overlayWindow.setAlwaysOnTop(true, 'screen-saver')
+
+  // Prevent the window from appearing in screenshots/recordings if possible
+  overlayWindow.setContentProtection(true)
+
+  // Hide from Mission Control / Exposé on macOS
+  overlayWindow.setVisibleOnAllWorkspaces(true, { visibleOnFullScreen: true })
+
+  overlayWindow.on('ready-to-show', () => {
+    overlayWindow?.show()
+  })
+
+  overlayWindow.on('closed', () => {
+    overlayWindow = null
+  })
+
+  // NOTICE: Wire eventa RPC BEFORE loading the renderer page.
+  // The overlay's onMounted fires during load() and immediately starts
+  // polling via callTool. If the handlers aren't registered yet, the
+  // first eventa invoke hangs forever (no response dispatched back to
+  // this window), and all subsequent poll cycles never fire because
+  // the poll loop awaits each call sequentially.
+  await setupDesktopOverlayElectronInvokes({
+    window: overlayWindow,
+    mcpStdioManager: params.mcpStdioManager,
+    serverChannel: params.serverChannel,
+    i18n: params.i18n,
+  })
+
+  // Load the overlay renderer page
+  await load(
+    overlayWindow,
+    withHashRoute(
+      baseUrl(resolve(getElectronMainDirname(), '..', 'renderer')),
+      '/desktop-overlay',
+    ),
+  )
+
+  return overlayWindow
+}
+
+/**
+ * Get the current overlay window instance (if active).
+ */
+export function getDesktopOverlayWindow(): BrowserWindow | null {
+  return overlayWindow
+}
+
+/**
+ * Tear down the overlay window.
+ */
+export function destroyDesktopOverlay(): void {
+  if (overlayWindow && !overlayWindow.isDestroyed()) {
+    overlayWindow.close()
+    overlayWindow = null
+  }
+}

apps/stage-tamagotchi/src/main/windows/desktop-overlay/rpc/index.electron.ts

@@ -0,0 +1,39 @@
+/**
+ * Desktop Overlay Window — Electron RPC bootstrap
+ *
+ * Minimal eventa context setup for the overlay BrowserWindow.
+ * Only registers base window services and MCP tool services —
+ * the overlay only needs callTool/listTools for polling
+ * `computer_use::desktop_get_state`.
+ *
+ * Follows the same pattern as main/chat/settings window RPC setups.
+ */
+
+import type { BrowserWindow } from 'electron'
+
+import type { I18n } from '../../../libs/i18n'
+import type { ServerChannel } from '../../../services/airi/channel-server'
+import type { McpStdioManager } from '../../../services/airi/mcp-servers'
+
+import { createContext } from '@moeru/eventa/adapters/electron/main'
+import { ipcMain } from 'electron'
+
+import { createMcpServersService } from '../../../services/airi/mcp-servers'
+import { setupBaseWindowElectronInvokes } from '../../shared/window'
+
+export async function setupDesktopOverlayElectronInvokes(params: {
+  window: BrowserWindow
+  mcpStdioManager: McpStdioManager
+  serverChannel: ServerChannel
+  i18n: I18n
+}) {
+  // TODO: once we refactored eventa to support window-namespaced contexts,
+  // we can remove the setMaxListeners call below since eventa will be able to dispatch and
+  // manage events within eventa's context system.
+  ipcMain.setMaxListeners(0)
+
+  const { context } = createContext(ipcMain, params.window)
+
+  await setupBaseWindowElectronInvokes({ context, window: params.window, i18n: params.i18n, serverChannel: params.serverChannel })
+  createMcpServersService({ context, manager: params.mcpStdioManager })
+}

apps/stage-tamagotchi/src/renderer/pages/desktop-overlay-coordinates.test.ts

@@ -0,0 +1,120 @@
+import { describe, expect, it } from 'vitest'
+
+import {
+  pointInOverlay,
+  rectIntersectsOverlay,
+  screenRectToLocal,
+  screenToLocal,
+} from './desktop-overlay-coordinates'
+
+// ---------------------------------------------------------------------------
+// screenToLocal
+// ---------------------------------------------------------------------------
+
+describe('screenToLocal', () => {
+  it('subtracts overlay origin from screen point', () => {
+    const result = screenToLocal({ x: 500, y: -800 }, { x: 0, y: -1080 })
+    expect(result).toEqual({ x: 500, y: 280 })
+  })
+
+  it('is identity when overlay origin is (0,0)', () => {
+    const result = screenToLocal({ x: 100, y: 200 }, { x: 0, y: 0 })
+    expect(result).toEqual({ x: 100, y: 200 })
+  })
+
+  it('handles negative overlay origin', () => {
+    const result = screenToLocal({ x: 441, y: -1037 }, { x: 0, y: -1080 })
+    expect(result).toEqual({ x: 441, y: 43 })
+  })
+})
+
+// ---------------------------------------------------------------------------
+// screenRectToLocal
+// ---------------------------------------------------------------------------
+
+describe('screenRectToLocal', () => {
+  it('shifts rect origin, preserves size', () => {
+    const result = screenRectToLocal(
+      { x: 100, y: -1000, width: 80, height: 30 },
+      { x: 0, y: -1080 },
+    )
+    expect(result).toEqual({ x: 100, y: 80, width: 80, height: 30 })
+  })
+
+  it('is identity when overlay origin is (0,0)', () => {
+    const rect = { x: 50, y: 100, width: 200, height: 150 }
+    const result = screenRectToLocal(rect, { x: 0, y: 0 })
+    expect(result).toEqual(rect)
+  })
+})
+
+// ---------------------------------------------------------------------------
+// rectIntersectsOverlay
+// ---------------------------------------------------------------------------
+
+describe('rectIntersectsOverlay', () => {
+  const overlay = { x: 0, y: -1080, width: 1440, height: 900 }
+
+  it('returns true for rect fully inside overlay', () => {
+    expect(rectIntersectsOverlay(
+      { x: 100, y: -1000, width: 80, height: 30 },
+      overlay,
+    )).toBe(true)
+  })
+
+  it('returns true for rect partially overlapping', () => {
+    expect(rectIntersectsOverlay(
+      { x: 1400, y: -1080, width: 100, height: 50 },
+      overlay,
+    )).toBe(true)
+  })
+
+  it('returns false for rect entirely above overlay', () => {
+    expect(rectIntersectsOverlay(
+      { x: 100, y: -2000, width: 80, height: 30 },
+      overlay,
+    )).toBe(false)
+  })
+
+  it('returns false for rect entirely below overlay', () => {
+    expect(rectIntersectsOverlay(
+      { x: 100, y: 0, width: 80, height: 30 },
+      overlay,
+    )).toBe(false)
+  })
+
+  it('returns false for rect entirely to the right', () => {
+    expect(rectIntersectsOverlay(
+      { x: 1500, y: -500, width: 80, height: 30 },
+      overlay,
+    )).toBe(false)
+  })
+})
+
+// ---------------------------------------------------------------------------
+// pointInOverlay
+// ---------------------------------------------------------------------------
+
+describe('pointInOverlay', () => {
+  const overlay = { x: 0, y: -1080, width: 1440, height: 900 }
+
+  it('returns true for point inside', () => {
+    expect(pointInOverlay({ x: 720, y: -540 }, overlay)).toBe(true)
+  })
+
+  it('returns true for point at top-left corner', () => {
+    expect(pointInOverlay({ x: 0, y: -1080 }, overlay)).toBe(true)
+  })
+
+  it('returns false for point outside (below)', () => {
+    expect(pointInOverlay({ x: 720, y: 0 }, overlay)).toBe(false)
+  })
+
+  it('returns false for point outside (above)', () => {
+    expect(pointInOverlay({ x: 720, y: -1200 }, overlay)).toBe(false)
+  })
+
+  it('returns false for point outside (right)', () => {
+    expect(pointInOverlay({ x: 1500, y: -540 }, overlay)).toBe(false)
+  })
+})