fix(computer): smooth native drag trajectory

.github/workflows/headless-linux.yml

@@ -21,6 +21,14 @@ on:
         required: false
         default: 'main'
         type: string
+      suite:
+        description: 'Jobs to run'
+        required: false
+        default: 'all'
+        type: choice
+        options:
+          - all
+          - drag-only
 
 permissions:
   contents: read
@@ -33,7 +41,46 @@ env:
   ELECTRON_SKIP_BINARY_DOWNLOAD: "1"
 
 jobs:
+  headless-linux-drag-and-drop:
+    runs-on: ubuntu-22.04
+    env:
+      MIDSCENE_COMPUTER_HEADLESS_LINUX: "true"
+      CI: "1"
+
+    steps:
+    - uses: actions/checkout@34e114876b0b11c390a56381ad16ebd13914f8d5 # v4.3.1
+      with:
+        ref: ${{ github.event.inputs.branch || github.ref }}
+
+    - name: Setup Node.js and pnpm
+      uses: ./.github/actions/setup-node-pnpm
+
+    - name: Cache pnpm store
+      id: pnpm-cache
+      uses: ./.github/actions/pnpm-cache/restore
+
+    - name: Install dependencies
+      run: pnpm install --frozen-lockfile --ignore-scripts
+
+    - name: Save pnpm store
+      if: steps.pnpm-cache.outputs.cache-hit != 'true'
+      uses: ./.github/actions/pnpm-cache/save
+
+    - name: Setup headless browser
+      uses: ./.github/actions/setup-headless-browser
+
+    - name: Build computer projects
+      run: pnpm exec nx run-many --target=build --projects="@midscene/report,@midscene/computer" --skip-nx-cache
+
+    - name: Run Linux drag-and-drop regression
+      # libnut keeps a process-wide X11 connection. Retrying after the test
+      # tears down Xvfb would terminate the worker before it can report the
+      # original assertion, so this live-display regression runs once.
+      run: AI_TEST_TYPE=computer npx nx test @midscene/computer -- tests/ai/linux-drag-and-drop.test.ts --retry 0
+      timeout-minutes: 10
+
   headless-linux-ai-todo:
+    if: github.event_name != 'workflow_dispatch' || inputs.suite != 'drag-only'
     runs-on: ${{ fromJSON(vars.LINUX_SELF_HOSTED_RUNNER_LABELS || '"ubuntu-22.04"') }}
     env:
       MIDSCENE_MODEL_API_KEY: ${{ secrets.MIDSCENE_MODEL_API_KEY }}
@@ -103,6 +150,7 @@ jobs:
       run: exit 1
 
   chrome-extension:
+    if: github.event_name != 'workflow_dispatch' || inputs.suite != 'drag-only'
     runs-on: ${{ fromJSON(vars.LINUX_SELF_HOSTED_RUNNER_LABELS || '"ubuntu-22.04"') }}
     env:
       # Env vars for computer agent (screenshot + AI actions)
@@ -162,6 +210,7 @@ jobs:
       run: exit 1
 
   chrome-extension-bridge:
+    if: github.event_name != 'workflow_dispatch' || inputs.suite != 'drag-only'
     runs-on: ${{ fromJSON(vars.LINUX_SELF_HOSTED_RUNNER_LABELS || '"ubuntu-22.04"') }}
     env:
       MIDSCENE_MODEL_API_KEY: ${{ secrets.MIDSCENE_MODEL_API_KEY }}
@@ -219,6 +268,7 @@ jobs:
       run: exit 1
 
   chrome-extension-playground:
+    if: github.event_name != 'workflow_dispatch' || inputs.suite != 'drag-only'
     runs-on: ${{ fromJSON(vars.LINUX_SELF_HOSTED_RUNNER_LABELS || '"ubuntu-22.04"') }}
     strategy:
       fail-fast: false
@@ -284,6 +334,7 @@ jobs:
       run: exit 1
 
   chrome-extension-recorder:
+    if: github.event_name != 'workflow_dispatch' || inputs.suite != 'drag-only'
     runs-on: ${{ fromJSON(vars.LINUX_SELF_HOSTED_RUNNER_LABELS || '"ubuntu-22.04"') }}
     env:
       MIDSCENE_MODEL_API_KEY: ${{ secrets.MIDSCENE_MODEL_API_KEY }}
@@ -341,6 +392,7 @@ jobs:
       run: exit 1
 
   chrome-extension-settings:
+    if: github.event_name != 'workflow_dispatch' || inputs.suite != 'drag-only'
     runs-on: ${{ fromJSON(vars.LINUX_SELF_HOSTED_RUNNER_LABELS || '"ubuntu-22.04"') }}
     env:
       MIDSCENE_MODEL_API_KEY: ${{ secrets.MIDSCENE_MODEL_API_KEY }}

packages/computer/src/device.ts

@@ -116,8 +116,10 @@ export interface Point {
 // Constants
 const SMOOTH_MOVE_STEPS_TAP = 8;
 const SMOOTH_MOVE_STEPS_MOUSE_MOVE = 10;
+const SMOOTH_MOVE_STEPS_DRAG = 20;
 const SMOOTH_MOVE_DELAY_TAP = 8;
 const SMOOTH_MOVE_DELAY_MOUSE_MOVE = 10;
+const SMOOTH_MOVE_DELAY_DRAG = 10;
 const MOUSE_MOVE_EFFECT_WAIT = 300;
 const CLICK_SETTLE_DELAY = 50;
 const CLICK_HOLD_DURATION = 100;
@@ -971,6 +973,14 @@ export class ComputerDevice implements AbstractInterface {
           await this.moveDisplayPointer(
             to,
             'Mouse did not reach the drag end target',
+            {
+              // Native desktop toolkits commonly use the first motion beyond
+              // their threshold to enter drag mode. Keep emitting held-button
+              // motions so the drop target can observe the active drag before
+              // the button is released.
+              smoothSteps: SMOOTH_MOVE_STEPS_DRAG,
+              smoothDelay: SMOOTH_MOVE_DELAY_DRAG,
+            },
           );
           await this.inputDriver.delay(100);
         });

packages/computer/tests/ai/fixtures/linux-drag-and-drop.html

@@ -0,0 +1,145 @@
+<!doctype html>
+<html lang="en">
+  <head>
+    <meta charset="UTF-8" />
+    <title>Linux drag-and-drop regression fixture</title>
+    <style>
+      html,
+      body {
+        width: 100%;
+        height: 100%;
+        margin: 0;
+        overflow: hidden;
+        background: #f4f4f5;
+        font: 24px sans-serif;
+      }
+
+      .item {
+        position: absolute;
+        left: 240px;
+        display: grid;
+        width: 240px;
+        height: 140px;
+        place-items: center;
+        border: 4px solid #18181b;
+        border-radius: 12px;
+        user-select: none;
+      }
+
+      #target {
+        top: 60px;
+        background: #bbf7d0;
+      }
+
+      #source {
+        top: 330px;
+        background: #bfdbfe;
+      }
+
+      #status {
+        position: absolute;
+        top: 520px;
+        left: 240px;
+      }
+    </style>
+  </head>
+  <body>
+    <div id="target" class="item">Target folder</div>
+    <div id="source" class="item">Source file</div>
+    <div id="status">Waiting</div>
+
+    <script>
+      const source = document.querySelector('#source');
+      const target = document.querySelector('#target');
+      const status = document.querySelector('#status');
+      let pointerDown = false;
+      let dragStarted = false;
+      let targetObservedAfterDragStart = false;
+      let startX = 0;
+      let startY = 0;
+
+      const report = (name, event) => {
+        const params = new URLSearchParams({ name });
+        if (event) {
+          params.set('x', event.clientX);
+          params.set('y', event.clientY);
+          params.set('buttons', event.buttons);
+          params.set('element', document.elementFromPoint(event.clientX, event.clientY)?.id || '');
+        }
+        fetch(`/event?${params}`, { method: 'POST' }).catch(() => {});
+      };
+
+      source.addEventListener('mousedown', (event) => {
+        pointerDown = true;
+        startX = event.clientX;
+        startY = event.clientY;
+        status.textContent = 'Pressed';
+        report('pressed', event);
+      });
+
+      document.addEventListener('mousemove', (event) => {
+        if (!pointerDown || event.buttons !== 1) return;
+
+        const distance = Math.hypot(event.clientX - startX, event.clientY - startY);
+        if (!dragStarted && distance >= 8) {
+          // Desktop toolkits use the first motion beyond their threshold to
+          // enter drag mode. The drop target can only observe later motions.
+          dragStarted = true;
+          status.textContent = 'Dragging';
+          report('drag-started', event);
+          return;
+        }
+
+        if (dragStarted && target.contains(document.elementFromPoint(event.clientX, event.clientY))) {
+          targetObservedAfterDragStart = true;
+          target.style.background = '#4ade80';
+          status.textContent = 'Target observed';
+          report('target-observed', event);
+        }
+      });
+
+      document.addEventListener('mouseup', (event) => {
+        if (!pointerDown) return;
+        pointerDown = false;
+
+        const releasedOnTarget = target.contains(
+          document.elementFromPoint(event.clientX, event.clientY),
+        );
+        if (dragStarted && targetObservedAfterDragStart && releasedOnTarget) {
+          status.textContent = 'Dropped';
+          report('dropped', event);
+        } else {
+          status.textContent = 'Drop missed';
+          report('drop-missed', event);
+        }
+      });
+
+      for (const type of ['mousedown', 'mousemove', 'mouseup']) {
+        document.addEventListener(type, (event) => report(`raw-${type}`, event), true);
+      }
+
+      // Chromium can report its initial, pre-maximize window geometry during
+      // the first animation frames on Xvfb. Wait for the native window to
+      // settle, then send element centers in display coordinates to the test.
+      setTimeout(() => {
+        const viewportLeft = window.screenX + (window.outerWidth - window.innerWidth) / 2;
+        const viewportTop = window.screenY + window.outerHeight - window.innerHeight;
+        const sourceRect = source.getBoundingClientRect();
+        const targetRect = target.getBoundingClientRect();
+        const params = new URLSearchParams({
+          left: viewportLeft,
+          top: viewportTop,
+          sourceX: viewportLeft + sourceRect.left + sourceRect.width / 2,
+          sourceY: viewportTop + sourceRect.top + sourceRect.height / 2,
+          targetX: viewportLeft + targetRect.left + targetRect.width / 2,
+          targetY: viewportTop + targetRect.top + targetRect.height / 2,
+          innerWidth: window.innerWidth,
+          innerHeight: window.innerHeight,
+          outerWidth: window.outerWidth,
+          outerHeight: window.outerHeight,
+        });
+        fetch(`/ready?${params}`, { method: 'POST' }).catch(() => {});
+      }, 1000);
+    </script>
+  </body>
+</html>