feat(browser): sharper computer use for the desktop browser agent

apps/desktop/src/main/browser-agent/cdp.test.ts

@@ -1,4 +1,5 @@
 import { getErrorMessage } from '@sim/utils/errors'
+import { toRecord } from '@sim/utils/object'
 import { afterEach, beforeEach, describe, expect, it, vi } from 'vitest'
 
 vi.mock('electron', () => import('@/test/electron-mock'))
@@ -13,9 +14,11 @@ import {
 import {
   captureScreenshot,
   clickAt,
+  consumeAgentContextMenu,
   ensureInstrumented,
   evaluateInIsolatedFrame,
   insertText,
+  PRIMARY_CLICK,
   releaseFileInput,
   resolveFileInput,
   setColorScheme,
@@ -255,6 +258,77 @@ describe('browser-agent CDP instrumentation', () => {
     ])
   })
 
+  it('holds the button down for holdMs before releasing it', async () => {
+    const contents = new WebContentsView().webContents
+    const types = () =>
+      vi.mocked(contents.debugger.sendCommand).mock.calls.map(([, params]) => toRecord(params).type)
+    vi.useFakeTimers()
+    try {
+      const click = clickAt(contents, 5, 6, false, { ...PRIMARY_CLICK, holdMs: 1500 })
+      await vi.advanceTimersByTimeAsync(1000)
+      expect(types()).toEqual(['mousePressed'])
+
+      await vi.advanceTimersByTimeAsync(500)
+      await click
+      expect(types()).toEqual(['mousePressed', 'mouseReleased'])
+    } finally {
+      vi.useRealTimers()
+    }
+  })
+
+  it('presses nothing when its click was aborted before dispatch', async () => {
+    const contents = new WebContentsView().webContents
+    const controller = new AbortController()
+    controller.abort()
+
+    await expect(
+      clickAt(contents, 5, 6, false, PRIMARY_CLICK, controller.signal)
+    ).rejects.toMatchObject({ name: 'AbortError' })
+    expect(contents.debugger.sendCommand).not.toHaveBeenCalled()
+  })
+
+  it('releases a held button as soon as its click is aborted', async () => {
+    const contents = new WebContentsView().webContents
+    const types = () =>
+      vi.mocked(contents.debugger.sendCommand).mock.calls.map(([, params]) => toRecord(params).type)
+    vi.useFakeTimers()
+    try {
+      const controller = new AbortController()
+      const hold = { ...PRIMARY_CLICK, holdMs: 10_000 }
+      const click = clickAt(contents, 5, 6, false, hold, controller.signal)
+      await vi.advanceTimersByTimeAsync(100)
+      expect(types()).toEqual(['mousePressed'])
+
+      controller.abort()
+      await expect(click).rejects.toMatchObject({ name: 'AbortError' })
+      expect(types()).toEqual(['mousePressed', 'mouseReleased'])
+    } finally {
+      vi.useRealTimers()
+    }
+  })
+
+  it('keeps a held right-click marked as the agent context menu until release', async () => {
+    const contents = new WebContentsView().webContents
+    vi.useFakeTimers()
+    try {
+      const rightHold = { ...PRIMARY_CLICK, button: 'right' as const, holdMs: 1500 }
+      await Promise.all([
+        clickAt(contents, 5, 6, false, rightHold),
+        vi.advanceTimersByTimeAsync(1500),
+      ])
+      expect(consumeAgentContextMenu(contents)).toBe(true)
+
+      const click = clickAt(contents, 5, 6, false, rightHold)
+      await vi.advanceTimersByTimeAsync(0)
+      expect(consumeAgentContextMenu(contents)).toBe(true)
+      await vi.advanceTimersByTimeAsync(1500)
+      await click
+      expect(consumeAgentContextMenu(contents)).toBe(false)
+    } finally {
+      vi.useRealTimers()
+    }
+  })
+
   it('releases the mouse after a partial click failure', async () => {
     const contents = new WebContentsView().webContents
     vi.mocked(contents.debugger.sendCommand)

apps/desktop/src/main/browser-agent/cdp.ts

@@ -11,7 +11,7 @@
 import type { BrowserTheme } from '@sim/browser-protocol'
 import { createLogger } from '@sim/logger'
 import { getErrorMessage } from '@sim/utils/errors'
-import { sleep } from '@sim/utils/helpers'
+import { interruptibleSleep, sleep } from '@sim/utils/helpers'
 import { isRecordLike } from '@sim/utils/object'
 import type { NativeImage, WebContents, WebFrameMain } from 'electron'
 
@@ -954,9 +954,16 @@ export interface PointerClick {
   clickCount: 1 | 2 | 3
   /** CDP modifier bitmask (Alt=1, Ctrl=2, Meta=4, Shift=8). */
   modifiers: number
+  /** How long the button stays down before release; press-and-hold controls need it. */
+  holdMs: number
 }
 
-export const PRIMARY_CLICK: PointerClick = { button: 'left', clickCount: 1, modifiers: 0 }
+export const PRIMARY_CLICK: PointerClick = {
+  button: 'left',
+  clickCount: 1,
+  modifiers: 0,
+  holdMs: 0,
+}
 
 const BUTTON_MASKS: Record<PointerClick['button'], number> = { left: 1, right: 2, middle: 4 }
 const agentContextClicks = new WeakMap<WebContents, number>()
@@ -976,15 +983,23 @@ export function clearAgentContextMenu(contents: WebContents): void {
   agentContextClicks.delete(contents)
 }
 
+/**
+ * Clicks at viewport coordinates. An already-aborted `signal` rejects before anything is pressed.
+ * During a press-and-hold it ends the hold early: the click rejects with the abort reason and the
+ * button is released at once, so a cancelled or timed-out click cannot stay held into the next
+ * action. That release can still activate the control under the pointer.
+ */
 export async function clickAt(
   contents: WebContents,
   x: number,
   y: number,
   moveBeforePress = true,
-  click: PointerClick = PRIMARY_CLICK
+  click: PointerClick = PRIMARY_CLICK,
+  signal?: AbortSignal
 ): Promise<void> {
   if (moveBeforePress) await moveMouse(contents, x, y)
-  const { button, clickCount, modifiers } = click
+  signal?.throwIfAborted()
+  const { button, clickCount, modifiers, holdMs } = click
   const buttons = BUTTON_MASKS[button]
   let pressed = false
   try {
@@ -1005,6 +1020,15 @@ export async function clickAt(
         modifiers,
         clickCount: count,
       })
+      if (holdMs > 0) {
+        await interruptibleSleep(holdMs, signal)
+        // Windows opens the context menu on release, after the hold; renew a marker a
+        // press-time menu has not already consumed.
+        if (button === 'right' && agentContextClicks.has(contents)) {
+          agentContextClicks.set(contents, Date.now())
+        }
+        signal?.throwIfAborted()
+      }
       await sendInput(contents, 'Input.dispatchMouseEvent', {
         type: 'mouseReleased',
         x,

apps/desktop/src/main/browser-agent/context-menu.test.ts

@@ -4,7 +4,7 @@ import { describe, expect, it, vi } from 'vitest'
 vi.mock('electron', () => import('@/test/electron-mock'))
 
 import { Menu, WebContentsView } from 'electron'
-import { clickAt } from '@/main/browser-agent/cdp'
+import { clickAt, PRIMARY_CLICK } from '@/main/browser-agent/cdp'
 import {
   attachAgentContextMenu,
   BASE_ZOOM_FACTOR,
@@ -228,7 +228,7 @@ describe('attachAgentContextMenu', () => {
       ContextMenuListener,
     ][]
     const onContextMenu = listeners.find(([event]) => event === 'context-menu')![1]
-    await clickAt(contents, 10, 20, false, { button: 'right', clickCount: 1, modifiers: 0 })
+    await clickAt(contents, 10, 20, false, { ...PRIMARY_CLICK, button: 'right' })
     vi.mocked(Menu.buildFromTemplate).mockClear()
 
     onContextMenu({}, params())
@@ -252,7 +252,7 @@ describe('attachAgentContextMenu', () => {
       ][]
       const onInput = listeners.find(([event]) => event === 'input-event')?.[1]
       const onContextMenu = listeners.find(([event]) => event === 'context-menu')![1]
-      await clickAt(contents, 10, 20, false, { button: 'right', clickCount: 1, modifiers: 0 })
+      await clickAt(contents, 10, 20, false, { ...PRIMARY_CLICK, button: 'right' })
       vi.mocked(Menu.buildFromTemplate).mockClear()
 
       onInput?.({}, { type: inputEvent })

apps/desktop/src/main/browser-agent/driver.test.ts

@@ -3603,6 +3603,34 @@ describe('credential protection', () => {
     })
   })
 
+  it('reports a press-and-hold cancelled mid-hold as an unknown outcome', async () => {
+    const contents = await openPage()
+    respondWith(contents, {})
+
+    const pending = driver.executeTool(
+      'chat-test',
+      'browser_click',
+      { elementId: 0, holdMs: 5_000 },
+      'hold-call'
+    )
+    await vi.waitFor(() =>
+      expect(
+        cdpCalls(contents, 'Input.dispatchMouseEvent').some(
+          ([, params]) => toRecord(params).type === 'mousePressed'
+        )
+      ).toBe(true)
+    )
+    driver.cancelTool('chat-test', 'hold-call')
+
+    await expect(pending).resolves.toMatchObject({
+      ok: true,
+      result: { outcomeUnknown: true, doNotRetry: true },
+    })
+    expect(
+      cdpCalls(contents, 'Input.dispatchMouseEvent').map(([, params]) => toRecord(params).type)
+    ).toContain('mouseReleased')
+  })
+
   it('rejects batches that name non-action tools or observe per action', async () => {
     await openPage()
 
@@ -3624,6 +3652,17 @@ describe('credential protection', () => {
       error: expect.stringContaining('Batch action 0'),
     })
     expect(observed).toMatchObject({ ok: false, error: expect.stringContaining('cannot observe') })
+
+    const held = await driver.executeTool('chat-test', 'browser_batch', {
+      actions: [
+        { tool: 'browser_click', args: { elementId: 0, holdMs: 2000 } },
+        { tool: 'browser_click', args: { elementId: 0 } },
+      ],
+    })
+    expect(held).toMatchObject({
+      ok: false,
+      error: expect.stringContaining('cannot press and hold'),
+    })
   })
 
   it('keeps element ids valid when an observed action is refused before dispatch', async () => {

apps/desktop/src/main/browser-agent/driver.ts

@@ -76,6 +76,7 @@ import {
   resolveFileInputTarget,
   scrollPage,
   selectOptionInElement,
+  serializePageCall,
   setFocusedInputValue,
   typeIntoElement,
 } from '@/main/browser-agent/page-functions'
@@ -187,6 +188,9 @@ function parseBatchActions(params: Record<string, unknown>): BatchAction[] {
     if ('observe' in action.args) {
       throw new ToolError(`Batch action ${index} cannot observe; pass observe on the batch itself.`)
     }
+    if (num(action.args, 'holdMs')) {
+      throw new ToolError(`Batch action ${index} cannot press and hold; run it as its own click.`)
+    }
     return { tool: action.tool, args: action.args }
   })
 }
@@ -1121,6 +1125,9 @@ const POINTER_BUTTONS: ReadonlySet<string> = new Set(['left', 'right', 'middle']
 /** Enough to walk a slider or list by keyboard in one call without flooding the page. */
 const MAX_KEY_REPEAT = 50
 
+/** Longest press-and-hold a click may request; well inside the click tool's watchdog. */
+const MAX_POINTER_HOLD_MS = 10_000
+
 /** The optional click gesture shared by `browser_click` and `browser_click_at`. */
 function pointerClick(params: Record<string, unknown>): cdp.PointerClick {
   const button = str(params, 'button') ?? 'left'
@@ -1133,10 +1140,20 @@ function pointerClick(params: Record<string, unknown>): cdp.PointerClick {
   if (!Array.isArray(names) || names.length > 4 || names.some((name) => typeof name !== 'string')) {
     throw new ToolError('modifiers must be a list of modifier names such as ["Shift"] or ["Mod"].')
   }
+  const holdMs = num(params, 'holdMs') ?? 0
+  if (!Number.isInteger(holdMs) || holdMs < 0 || holdMs > MAX_POINTER_HOLD_MS) {
+    throw new ToolError(
+      `holdMs must be a whole number of milliseconds from 0 to ${MAX_POINTER_HOLD_MS}.`
+    )
+  }
+  if (holdMs > 0 && clickCount !== 1) {
+    throw new ToolError('holdMs applies to a single press; use clickCount 1.')
+  }
   return {
     button: button as cdp.PointerClick['button'],
     clickCount,
     modifiers: cdpModifiers(parseModifiers(names)),
+    holdMs,
   }
 }
 
@@ -1163,7 +1180,9 @@ function uploadPaths(params: Record<string, unknown>): string[] {
 }
 
 function isPrimaryClick(click: cdp.PointerClick): boolean {
-  return click.button === 'left' && click.clickCount === 1 && click.modifiers === 0
+  return (
+    click.button === 'left' && click.clickCount === 1 && click.modifiers === 0 && click.holdMs === 0
+  )
 }
 
 const DIALOG_ANSWERING_TOOLS: ReadonlySet<BrowserToolName> = new Set([
@@ -1274,7 +1293,7 @@ async function execInPage<Args extends unknown[], Result>(
       'The active tab is blank. Call browser_navigate before using page inspection or interaction tools.'
     )
   }
-  const invocation = `(${String(fn)}).apply(null, ${JSON.stringify(args)})`
+  const invocation = serializePageCall(fn as (...args: never[]) => unknown, args)
   const expression =
     typeof notAfter === 'number'
       ? `(Date.now() >= ${Math.floor(notAfter)} ? ({error: "expired"}) : ${invocation})`
@@ -3204,7 +3223,10 @@ async function executeToolInner(
         try {
           assertCurrentExecution()
           assertElementActionCurrent(contents, elementId, target)
-          await cdp.clickAt(contents, x, y, false, click)
+          // A hold keeps the press in flight for seconds; cancelling it mid-gesture must read as
+          // an outcome that may have acted, never as a click that did not start.
+          if (click.holdMs > 0) onActionOutcome?.({ status: 'pending' })
+          await cdp.clickAt(contents, x, y, false, click, signal)
           trusted = true
           activation = 'native-pointer'
         } catch (error) {
@@ -3244,7 +3266,8 @@ async function executeToolInner(
           try {
             assertCurrentExecution()
             assertElementActionCurrent(contents, elementId, target)
-            await cdp.clickAt(contents, finalTopPoint.x, finalTopPoint.y, false, click)
+            if (click.holdMs > 0) onActionOutcome?.({ status: 'pending' })
+            await cdp.clickAt(contents, finalTopPoint.x, finalTopPoint.y, false, click, signal)
             trusted = true
             activation = 'native-pointer'
             prepared = finalSurface
@@ -3261,7 +3284,7 @@ async function executeToolInner(
         } else {
           if (!isPrimaryClick(click)) {
             throw new ToolError(
-              'This framed control has no reliable pointer position, so only a plain left click can activate it. Use browser_screenshot and browser_click_at for other buttons, click counts, or modifiers.'
+              'This framed control has no reliable pointer position, so only a plain left click can activate it. Use browser_screenshot and browser_click_at for other buttons, click counts, holds, or modifiers.'
             )
           }
           const activationKey = prepared.activationKey
@@ -4622,8 +4645,9 @@ async function executeToolInner(
       const beforeElement = await activeElementState(contents)
       assertCurrentExecution()
       assertActiveContents(contents, clickNavigationEpoch)
+      if (click.holdMs > 0) onActionOutcome?.({ status: 'pending' })
       try {
-        await cdp.clickAt(contents, x, y, true, click)
+        await cdp.clickAt(contents, x, y, true, click, signal)
       } catch (error) {
         const rescued = navigationRescue(contents, clickNavigationEpoch, urlAtDispatch, {
           trusted: true,