From a2a54730a7796d4751beb1d85574b1f649320929 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Micha=C5=82=20Pierzcha=C5=82a?= Date: Sat, 3 Oct 2026 22:25:55 +0200 Subject: [PATCH] feat(macos): draw a ghost cursor for native app backend actions Each helper action on a native-backend app session draws its own pointer, glides it onto the target from a short fixed offset, and pulses on delivery, so a person can follow the agent while keeping the real pointer. The cursor keeps no state between helper processes and costs about 0.3 s per action. AGENT_DEVICE_MACOS_GHOST_CURSOR=0 turns it off; only a drawn cursor adds its budget to the helper deadline. Co-Authored-By: Claude Opus 5.5 (1M context) --- .../BackgroundInteraction.swift | 24 ++- .../AgentDeviceMacOSHelper/GhostCursor.swift | 141 ++++++++++++++++++ .../Sources/AgentDeviceMacOSHelper/main.swift | 35 +++-- docs/adr/0031-macos-native-app-backend.md | 5 + docs/adr/README.md | 2 +- .../platform-apple/src/os/macos/helper.ts | 39 ++++- .../os/macos/native-app-interactor.test.ts | 76 +++++++++- website/docs/docs/commands.md | 2 +- website/docs/docs/configuration.md | 2 +- 9 files changed, 300 insertions(+), 26 deletions(-) create mode 100644 apple/macos-helper/Sources/AgentDeviceMacOSHelper/GhostCursor.swift diff --git a/apple/macos-helper/Sources/AgentDeviceMacOSHelper/BackgroundInteraction.swift b/apple/macos-helper/Sources/AgentDeviceMacOSHelper/BackgroundInteraction.swift index 30e9e69573..73d3adf200 100644 --- a/apple/macos-helper/Sources/AgentDeviceMacOSHelper/BackgroundInteraction.swift +++ b/apple/macos-helper/Sources/AgentDeviceMacOSHelper/BackgroundInteraction.swift @@ -84,7 +84,8 @@ func requireSessionApplication(bundleId: String?) throws -> NSRunningApplication func pressInBackground( _ request: MouseClickRequest, - app: NSRunningApplication + app: NSRunningApplication, + cursor: GhostCursor? ) throws -> BackgroundPressResponse { guard !request.doubleClick, request.holdMs <= 0 else { throw refusal( @@ -94,6 +95,7 @@ func pressInBackground( ) } let point = CGPoint(x: request.x, y: request.y) + cursor?.move(to: point) guard let target = resolvePressTarget(app: app, point: point), let mechanism = perform(target) else { @@ -105,6 +107,7 @@ func pressInBackground( guard perform(target) != nil else { break } clicks += 1 } + cursor?.pulse() return BackgroundPressResponse( x: request.x, y: request.y, @@ -121,14 +124,19 @@ func pressInBackground( func typeInBackground( text: String, delayMs: Int, - app: NSRunningApplication + app: NSRunningApplication, + cursor: GhostCursor? ) throws -> BackgroundTextResponse { let appElement = AXUIElementCreateApplication(app.processIdentifier) let focused = elementAttribute(appElement, attribute: kAXFocusedUIElementAttribute as String) + if let focused, let rect = rectAttribute(focused) { + cursor?.move(to: CGPoint(x: rect.x + rect.width / 2, y: rect.y + rect.height / 2)) + } let focusedRole = focused.map(role(of:)) if text == "\n", let focused, actionNames(of: focused).contains(kAXConfirmAction as String), AXUIElementPerformAction(focused, kAXConfirmAction as CFString) == .success { + cursor?.pulse() return BackgroundTextResponse( bundleId: app.bundleIdentifier, mechanism: .axConfirm, role: focusedRole, windowTitle: windowTitle(of: focused)) @@ -138,11 +146,13 @@ func typeInBackground( AXUIElementSetAttributeValue(focused, kAXSelectedTextAttribute as CFString, text as CFString) == .success { + cursor?.pulse() return BackgroundTextResponse( bundleId: app.bundleIdentifier, mechanism: .axSelectedText, role: focusedRole, windowTitle: windowTitle(of: focused)) } try postText(text, delayMs: delayMs, pid: app.processIdentifier) + cursor?.pulse() return BackgroundTextResponse( bundleId: app.bundleIdentifier, mechanism: .keyEvents, role: focusedRole, windowTitle: focused.flatMap(windowTitle(of:))) @@ -152,8 +162,10 @@ func typeInBackground( func fillInBackground( point: CGPoint, text: String, - app: NSRunningApplication + app: NSRunningApplication, + cursor: GhostCursor? ) throws -> BackgroundTextResponse { + cursor?.move(to: point) let hit = elementAtPoint(in: app, point: point) let chain = hit.map(pressSearchChain) ?? [] let fallback = actionWindow(app: app, hit: hit).flatMap { window in @@ -169,6 +181,7 @@ func fillInBackground( else { throw refusal(.noTextInput, "the text input refused the new value", app: app) } + cursor?.pulse() return BackgroundTextResponse( bundleId: app.bundleIdentifier, mechanism: .axValue, role: role(of: input), windowTitle: windowTitle(of: input)) @@ -179,7 +192,8 @@ func scrollInBackground( direction: String, amount: Double?, pixels: Double?, - app: NSRunningApplication + app: NSRunningApplication, + cursor: GhostCursor? ) throws -> BackgroundScrollResponse { let isVertical = direction == "up" || direction == "down" guard isVertical || direction == "left" || direction == "right" else { @@ -197,6 +211,7 @@ func scrollInBackground( pixels: pixels ) let center = CGPoint(x: frame.midX, y: frame.midY) + cursor?.move(to: center) let revealsLaterContent = direction == "down" || direction == "right" guard performScrollBarScroll( @@ -208,6 +223,7 @@ func scrollInBackground( else { throw refusal(.noScrollBar, "no scroll area with a settable scroll bar under the window center", app: app) } + cursor?.pulse() // Reported as the equivalent drag, start to end, the way the runner reports a desktop scroll. let half = travel / 2 let sign: Double = revealsLaterContent ? 1 : -1 diff --git a/apple/macos-helper/Sources/AgentDeviceMacOSHelper/GhostCursor.swift b/apple/macos-helper/Sources/AgentDeviceMacOSHelper/GhostCursor.swift new file mode 100644 index 0000000000..32a7889b42 --- /dev/null +++ b/apple/macos-helper/Sources/AgentDeviceMacOSHelper/GhostCursor.swift @@ -0,0 +1,141 @@ +import AppKit +import QuartzCore + +/// A drawn pointer that shows where a background action lands while the user keeps the real +/// one. It appears just above and left of the target, glides onto it, and pulses when the action +/// is delivered: about 0.3 s per action. It holds no state between helper processes. +final class GhostCursor { + private static let size = CGSize(width: 64, height: 64) + /// Where the arrow's tip sits inside the panel, from its top-left corner; it leaves room for + /// the pulse ring's full radius. + private static let tipOffset = CGPoint(x: 22, y: 22) + /// Where the glide starts, relative to the target. + private static let approach = CGVector(dx: -36, dy: -36) + + private let panel: NSPanel + private let view: GhostCursorView + private let primaryScreenHeight: CGFloat + + /// Nil when no display is attached; the action still runs, just unseen. + static func show() -> GhostCursor? { + _ = NSApplication.shared + NSApp.setActivationPolicy(.accessory) + guard let primary = NSScreen.screens.first else { return nil } + return GhostCursor(primaryScreenHeight: primary.frame.height) + } + + private init(primaryScreenHeight: CGFloat) { + self.primaryScreenHeight = primaryScreenHeight + view = GhostCursorView(frame: CGRect(origin: .zero, size: Self.size), tip: Self.tipOffset) + panel = NSPanel( + contentRect: CGRect(origin: .zero, size: Self.size), + styleMask: [.borderless, .nonactivatingPanel], + backing: .buffered, + defer: false + ) + panel.isOpaque = false + panel.backgroundColor = .clear + panel.hasShadow = false + panel.ignoresMouseEvents = true + panel.level = .screenSaver + panel.collectionBehavior = [.canJoinAllSpaces, .stationary, .ignoresCycle, .fullScreenAuxiliary] + panel.contentView = view + } + + /// Glides onto a point in global top-left coordinates, the space AX and CGEvent share. + func move(to target: CGPoint, duration: TimeInterval = 0.15) { + let start = CGPoint(x: target.x + Self.approach.dx, y: target.y + Self.approach.dy) + place(at: start) + panel.orderFrontRegardless() + let steps = max(1, Int(duration / 0.016)) + for step in 1...steps { + let t = Double(step) / Double(steps) + let eased = 1 - pow(1 - t, 3) + place(at: CGPoint(x: start.x + (target.x - start.x) * eased, y: start.y + (target.y - start.y) * eased)) + flush(for: 0.016) + } + } + + /// A ring at the tip, marking the moment the action was delivered; nothing when the cursor never + /// moved onto a target. + func pulse(duration: TimeInterval = 0.12) { + guard panel.isVisible else { return } + let steps = max(1, Int(duration / 0.016)) + for step in 1...steps { + view.ringProgress = CGFloat(step) / CGFloat(steps) + view.display() + flush(for: 0.016) + } + } + + func hide() { + panel.orderOut(nil) + CATransaction.flush() + } + + private func place(at tip: CGPoint) { + panel.setFrameOrigin( + CGPoint( + x: tip.x - Self.tipOffset.x, + y: primaryScreenHeight - tip.y - Self.size.height + Self.tipOffset.y + ) + ) + } + + private func flush(for interval: TimeInterval) { + CATransaction.flush() + RunLoop.current.run(until: Date(timeIntervalSinceNow: interval)) + } +} + +private final class GhostCursorView: NSView { + private let tip: CGPoint + var ringProgress: CGFloat? + + init(frame: CGRect, tip: CGPoint) { + self.tip = tip + super.init(frame: frame) + } + + required init?(coder: NSCoder) { + nil + } + + override var isFlipped: Bool { true } + + override func draw(_ dirtyRect: NSRect) { + NSColor.clear.setFill() + dirtyRect.fill() + if let progress = ringProgress { + let radius = 6 + 14 * progress + let ring = NSBezierPath( + ovalIn: CGRect(x: tip.x - radius, y: tip.y - radius, width: radius * 2, height: radius * 2) + ) + ring.lineWidth = 3 + NSColor.systemPurple.withAlphaComponent(0.85 * (1 - progress)).setStroke() + ring.stroke() + } + let arrow = NSBezierPath() + arrow.move(to: tip) + arrow.line(to: CGPoint(x: tip.x, y: tip.y + 22)) + arrow.line(to: CGPoint(x: tip.x + 6, y: tip.y + 17)) + arrow.line(to: CGPoint(x: tip.x + 10, y: tip.y + 26)) + arrow.line(to: CGPoint(x: tip.x + 14, y: tip.y + 24)) + arrow.line(to: CGPoint(x: tip.x + 10, y: tip.y + 15)) + arrow.line(to: CGPoint(x: tip.x + 17, y: tip.y + 15)) + arrow.close() + arrow.lineJoinStyle = .round + let shadow = NSShadow() + shadow.shadowBlurRadius = 3 + shadow.shadowOffset = CGSize(width: 0, height: -1) + shadow.shadowColor = NSColor.black.withAlphaComponent(0.35) + NSGraphicsContext.saveGraphicsState() + shadow.set() + NSColor.systemPurple.setFill() + arrow.fill() + NSGraphicsContext.restoreGraphicsState() + arrow.lineWidth = 1.5 + NSColor.white.setStroke() + arrow.stroke() + } +} diff --git a/apple/macos-helper/Sources/AgentDeviceMacOSHelper/main.swift b/apple/macos-helper/Sources/AgentDeviceMacOSHelper/main.swift index 9d0aa0c545..4e93d84c99 100644 --- a/apple/macos-helper/Sources/AgentDeviceMacOSHelper/main.swift +++ b/apple/macos-helper/Sources/AgentDeviceMacOSHelper/main.swift @@ -419,7 +419,9 @@ struct AgentDeviceMacOSHelper { if surface == "app" { let app = try requireSessionApplication(bundleId: bundleId) return SuccessEnvelope( - data: try pressInBackground(request, app: app) + data: try withGhostCursor(arguments) { cursor in + try pressInBackground(request, app: app, cursor: cursor) + } ) } try pressAtPosition(request) @@ -450,7 +452,9 @@ struct AgentDeviceMacOSHelper { bundleId: try optionValue(arguments: arguments, name: "--bundle-id").map(validatedBundleId) ) return SuccessEnvelope( - data: try typeInBackground(text: text, delayMs: delayMs, app: app) + data: try withGhostCursor(arguments) { cursor in + try typeInBackground(text: text, delayMs: delayMs, app: app, cursor: cursor) + } ) } @@ -467,7 +471,10 @@ struct AgentDeviceMacOSHelper { bundleId: try optionValue(arguments: arguments, name: "--bundle-id").map(validatedBundleId) ) return SuccessEnvelope( - data: try fillInBackground(point: CGPoint(x: x, y: y), text: text, app: app) + data: try withGhostCursor(arguments) { cursor in + try fillInBackground( + point: CGPoint(x: x, y: y), text: text, app: app, cursor: cursor) + } ) } @@ -484,12 +491,15 @@ struct AgentDeviceMacOSHelper { bundleId: try optionValue(arguments: arguments, name: "--bundle-id").map(validatedBundleId) ) return SuccessEnvelope( - data: try scrollInBackground( - direction: direction, - amount: amount, - pixels: pixels, - app: app - ) + data: try withGhostCursor(arguments) { cursor in + try scrollInBackground( + direction: direction, + amount: amount, + pixels: pixels, + app: app, + cursor: cursor + ) + } ) } @@ -542,6 +552,13 @@ private func optionValue(arguments: [String], name: String) -> String? { return arguments[index + 1] } +/// Runs a background action under the ghost cursor when the host asked for one. +private func withGhostCursor(_ arguments: [String], _ body: (GhostCursor?) throws -> T) throws -> T { + let cursor = arguments.contains("--ghost-cursor") ? GhostCursor.show() : nil + defer { cursor?.hide() } + return try body(cursor) +} + private func positiveDoubleOption(arguments: [String], name: String) throws -> Double? { guard let raw = optionValue(arguments: arguments, name: name) else { return nil } guard let value = Double(raw), value.isFinite, value > 0 else { diff --git a/docs/adr/0031-macos-native-app-backend.md b/docs/adr/0031-macos-native-app-backend.md index 1b8390cfd3..3d90fae226 100644 --- a/docs/adr/0031-macos-native-app-backend.md +++ b/docs/adr/0031-macos-native-app-backend.md @@ -67,6 +67,11 @@ point. Sending an element identity instead would be a new dispatch path with its row, and an `AXUIElement` cannot outlive the one-shot helper process that resolved it, so an identity would be a tree path re-resolved against a tree that may have changed. +**Ghost cursor.** Each helper action draws its own pointer, glides onto the target from a short +fixed offset, and pulses on delivery, so a person can follow the agent without losing the real +pointer. It costs about 0.3 s per action and keeps no state between helper processes; a persistent +pointer would need a long-lived helper. `AGENT_DEVICE_MACOS_GHOST_CURSOR=0` disables it. + ## Rejected alternatives - **Suppressing Automation Mode.** `automationmodetool` removes the authentication prompt, not the diff --git a/docs/adr/README.md b/docs/adr/README.md index 7694aab843..dabc53d7b1 100644 --- a/docs/adr/README.md +++ b/docs/adr/README.md @@ -32,7 +32,7 @@ | [0028 Capability-Family Cell Vocabulary — One Runtime Source (Proposed)](0028-capability-family-cell-vocabulary.md) | adding a capability operation family, `UnavailablePlatformRuntimeFacts` / `UNAVAILABLE_CELLS`, `INTERACTOR_OPERATIONS`, and why an eight-package fan-out recurs | | [0029 Daemon Policy](0029-daemon-policy.md) | `AGENT_DEVICE_DAEMON_POLICY`, confining a daemon's commands, devices, or device shutdown, and where operator rules are enforced for batch/replay steps | | [0030 Process Lock Exclusion](0030-process-lock-exclusion.md) | process-lock publication/reclaim/release, retained mutation guards, and the single-protocol upgrade boundary | -| [0031 macOS Native App Backend](0031-macos-native-app-backend.md) | `AGENT_DEVICE_MACOS_APP_BACKEND`, driving macOS app sessions without XCTest Automation Mode, and why pointer actions are accessibility actions only | +| [0031 macOS Native App Backend](0031-macos-native-app-backend.md) | `AGENT_DEVICE_MACOS_APP_BACKEND`, driving macOS app sessions without XCTest Automation Mode, why pointer actions are accessibility actions only, and the ghost cursor | ADRs record *why*; the registries and gates they describe are the living source of truth — when prose and a registry disagree, the registry wins and the ADR needs a follow-up. diff --git a/packages/platform-apple/src/os/macos/helper.ts b/packages/platform-apple/src/os/macos/helper.ts index 135f65f3f4..97cc51f146 100644 --- a/packages/platform-apple/src/os/macos/helper.ts +++ b/packages/platform-apple/src/os/macos/helper.ts @@ -112,6 +112,26 @@ function appendMacOsHelperContextArgs( } } +const MACOS_GHOST_CURSOR_ENV = 'AGENT_DEVICE_MACOS_GHOST_CURSOR'; + +/** + * A background action on an app session draws its own pointer so the user can follow it while + * keeping the real one; `AGENT_DEVICE_MACOS_GHOST_CURSOR=0` turns the drawing off. + */ +function appendGhostCursorArg(args: string[], surface: SessionSurface | undefined): void { + if (surface !== 'app') return; + if (readHostEnvironmentVariable(MACOS_GHOST_CURSOR_ENV)?.trim() === '0') return; + args.push('--ghost-cursor'); +} + +/** + * The ghost cursor's glide and pulse take about 0.3 s; the budget also covers the helper's first + * WindowServer connection. Only a drawn cursor earns it. + */ +function ghostCursorBudgetMs(args: readonly string[]): number { + return args.includes('--ghost-cursor') ? 1_000 : 0; +} + /** * The helper's app-surface vocabulary, pinned with its Swift enums by * `contracts/fixtures/macos-native-helper-outcomes.json`. @@ -492,9 +512,10 @@ export async function runMacOsPressAction( args.push('--double-click'); } appendMacOsHelperContextArgs(args, options); + appendGhostCursorArg(args, options.surface); return await runMacOsHelper(args, { signal: options.signal, - timeoutMs: macOsClickScheduleMs(options) + MACOS_HELPER_TIMEOUT_MS, + timeoutMs: macOsClickScheduleMs(options) + MACOS_HELPER_TIMEOUT_MS + ghostCursorBudgetMs(args), }); } @@ -506,9 +527,11 @@ export async function runMacOsTypeAction( const args = ['type', '--text', text]; if (options.delayMs && options.delayMs > 0) args.push('--delay-ms', String(options.delayMs)); appendMacOsHelperContextArgs(args, { bundleId: options.bundleId }); + appendGhostCursorArg(args, 'app'); return await runMacOsHelper(args, { signal: options.signal, - timeoutMs: MACOS_HELPER_TIMEOUT_MS + text.length * (options.delayMs ?? 0), + timeoutMs: + MACOS_HELPER_TIMEOUT_MS + ghostCursorBudgetMs(args) + text.length * (options.delayMs ?? 0), }); } @@ -521,7 +544,11 @@ export async function runMacOsFillAction( ): Promise<{ mechanism?: MacOsDeliveryMechanism; role?: string; windowTitle?: string }> { const args = ['fill', '--x', String(x), '--y', String(y), '--text', text]; appendMacOsHelperContextArgs(args, { bundleId: options.bundleId }); - return await runMacOsHelper(args, { signal: options.signal }); + appendGhostCursorArg(args, 'app'); + return await runMacOsHelper(args, { + signal: options.signal, + timeoutMs: MACOS_HELPER_TIMEOUT_MS + ghostCursorBudgetMs(args), + }); } /** Scrolls the scroll area at the center of the app session's front window. */ @@ -538,7 +565,11 @@ export async function runMacOsScrollAction( if (options.amount !== undefined) args.push('--amount', String(options.amount)); if (options.pixels !== undefined) args.push('--pixels', String(options.pixels)); appendMacOsHelperContextArgs(args, { bundleId: options.bundleId }); - return await runMacOsHelper(args, { signal: options.signal }); + appendGhostCursorArg(args, 'app'); + return await runMacOsHelper(args, { + signal: options.signal, + timeoutMs: MACOS_HELPER_TIMEOUT_MS + ghostCursorBudgetMs(args), + }); } export async function runMacOsScreenshotAction( diff --git a/packages/platform-apple/src/os/macos/native-app-interactor.test.ts b/packages/platform-apple/src/os/macos/native-app-interactor.test.ts index 6380d86f34..e72ab77afe 100644 --- a/packages/platform-apple/src/os/macos/native-app-interactor.test.ts +++ b/packages/platform-apple/src/os/macos/native-app-interactor.test.ts @@ -1,5 +1,5 @@ import assert from 'node:assert/strict'; -import { test } from 'vitest'; +import { afterEach, beforeEach, test, vi } from 'vitest'; import type { Interactor, RunnerContext } from '@agent-device/contracts/interactor-types'; import { AppError } from '@agent-device/kernel/errors'; import { createLocalAppleToolProvider, withAppleToolProvider } from '../../core/tool-provider.ts'; @@ -7,6 +7,14 @@ import { macOsNativeAppInteractor } from './native-app-interactor.ts'; const context: RunnerContext = { appBundleId: 'com.apple.TextEdit' }; +// The drawn cursor is the default; pin it so an operator's opt-out does not change the argv. +beforeEach(() => { + vi.stubEnv('AGENT_DEVICE_MACOS_GHOST_CURSOR', ''); +}); +afterEach(() => { + vi.unstubAllEnvs(); +}); + /** The XCTest-backed interactor the native one draws from; any member it calls is recorded. */ function runnerInteractor(reached: string[] = []): Interactor { return new Proxy({} as Interactor, { @@ -44,13 +52,24 @@ async function recordHelperCalls( return { calls, result }; } -test('a native tap presses the app surface and reports the mechanism', async () => { +test('a native tap presses the app surface under the ghost cursor and reports the mechanism', async () => { const { calls, result } = await recordHelperCalls( { x: 10, y: 20, mechanism: 'ax-press', windowTitle: 'Untitled' }, async (interactor) => await interactor.tap(10, 20), ); assert.deepEqual(calls, [ - ['press', '--x', '10', '--y', '20', '--bundle-id', 'com.apple.TextEdit', '--surface', 'app'], + [ + 'press', + '--x', + '10', + '--y', + '20', + '--bundle-id', + 'com.apple.TextEdit', + '--surface', + 'app', + '--ghost-cursor', + ], ]); assert.deepEqual(result, { mechanism: 'ax-press', windowTitle: 'Untitled' }); }); @@ -61,8 +80,19 @@ test('native type and fill address the session app, not the frontmost one', asyn await interactor.fill(5, 6, 'world'); }); assert.deepEqual(calls, [ - ['type', '--text', 'hello', '--bundle-id', 'com.apple.TextEdit'], - ['fill', '--x', '5', '--y', '6', '--text', 'world', '--bundle-id', 'com.apple.TextEdit'], + ['type', '--text', 'hello', '--bundle-id', 'com.apple.TextEdit', '--ghost-cursor'], + [ + 'fill', + '--x', + '5', + '--y', + '6', + '--text', + 'world', + '--bundle-id', + 'com.apple.TextEdit', + '--ghost-cursor', + ], ]); }); @@ -81,7 +111,16 @@ test('a native scroll reports travel from the window frame the helper resolved', async (interactor) => await interactor.scroll('down', { pixels: 300 }), ); assert.deepEqual(calls, [ - ['scroll', '--direction', 'down', '--pixels', '300', '--bundle-id', 'com.apple.TextEdit'], + [ + 'scroll', + '--direction', + 'down', + '--pixels', + '300', + '--bundle-id', + 'com.apple.TextEdit', + '--ghost-cursor', + ], ]); assert.deepEqual(result, { x1: 480, @@ -95,6 +134,31 @@ test('a native scroll reports travel from the window frame the helper resolved', }); }); +test('AGENT_DEVICE_MACOS_GHOST_CURSOR=0 sends the action without a drawn cursor', async () => { + vi.stubEnv('AGENT_DEVICE_MACOS_GHOST_CURSOR', '0'); + const { calls } = await recordHelperCalls({ mechanism: 'ax-press' }, async (interactor) => { + await interactor.tap(10, 20); + await interactor.type('hello'); + }); + assert.ok(calls.every((argv) => !argv.includes('--ghost-cursor'))); +}); + +test('only a drawn cursor extends the helper deadline', async () => { + const deadlines: Array = []; + const provider = createLocalAppleToolProvider({ + macosHelper: { + run: async (_args, options) => { + deadlines.push(options?.timeoutMs); + return { exitCode: 0, stdout: JSON.stringify({ ok: true, data: {} }), stderr: '' }; + }, + }, + }); + await withAppleToolProvider(provider, async () => await nativeInteractor().fill(1, 2, 'x')); + vi.stubEnv('AGENT_DEVICE_MACOS_GHOST_CURSOR', '0'); + await withAppleToolProvider(provider, async () => await nativeInteractor().fill(1, 2, 'x')); + assert.equal(deadlines[0]! - deadlines[1]!, 1_000); +}); + test('runner-only members refuse with the backend reason instead of starting XCTest', async () => { const reached: string[] = []; const interactor = nativeInteractor(context, reached); diff --git a/website/docs/docs/commands.md b/website/docs/docs/commands.md index da9b5f5870..7fbff37e5e 100644 --- a/website/docs/docs/commands.md +++ b/website/docs/docs/commands.md @@ -344,7 +344,7 @@ agent-device snapshot -i --platform apple --target desktop - Prefer selector or `@ref`-driven interactions on macOS. Window position can shift between runs, so raw x/y point commands are less stable than snapshot-derived targets. - Use `click --button secondary` for context menus on macOS, then run `snapshot -i` again. - On `frontmost-app` and `menubar` surfaces, `press` and `click` post synthetic mouse events through the macOS helper (the `desktop` surface inspects only): `--hold-ms` is how long the button stays down (at least 40 ms, 60 ms by default, because AppKit drops a release posted in the same tick as its press), `--count` is that many independent clicks (apps that detect a double-click by timing, such as Finder, still read two clicks at the default interval as one; pass an `--interval-ms` longer than the system double-click time to keep them apart), and `--double-tap` posts each click as a double-click pair. A long schedule such as `--hold-ms 10000 --count 4` is given the time it needs, and a helper stopped mid-hold — by a cancelled request, a dropped client, or its deadline — releases the button before it exits. `--jitter-px` is not applied on these surfaces. -- With `AGENT_DEVICE_MACOS_APP_BACKEND=native`, macOS app sessions run without XCTest: no Automation Mode overlay, the app can stay behind other windows, and the real pointer stays with you. `click`, `press`, and `fill` use accessibility actions on the element at their target point; `type` inserts at the focused control and falls back to key events sent to the app; `scroll` moves the accessibility scroll bar under the front window's center. Chromium-based apps expose their full tree. The daemon then never starts the XCTest runner on macOS: `record`, `prepare`, `back`, `longpress` and `--hold-ms`, `--double-tap`, `--button secondary`, and gestures are refused with `UNSUPPORTED_OPERATION` (`reason: unsupported-device-backend`), as is a click on an element with no accessibility action. Use the default XCTest backend for those. `screenshot` captures only the app's front window, even when other windows cover it. +- With `AGENT_DEVICE_MACOS_APP_BACKEND=native`, macOS app sessions run without XCTest: no Automation Mode overlay, the app can stay behind other windows, and the real pointer stays with you while a drawn pointer shows each action. `click`, `press`, and `fill` use accessibility actions on the element at their target point; `type` inserts at the focused control and falls back to key events sent to the app; `scroll` moves the accessibility scroll bar under the front window's center. Chromium-based apps expose their full tree. The daemon then never starts the XCTest runner on macOS: `record`, `prepare`, `back`, `longpress` and `--hold-ms`, `--double-tap`, `--button secondary`, and gestures are refused with `UNSUPPORTED_OPERATION` (`reason: unsupported-device-backend`), as is a click on an element with no accessibility action. Use the default XCTest backend for those. `screenshot` captures only the app's front window, even when other windows cover it. - Mobile-only helpers remain unsupported on macOS: `boot`, `shutdown`, `home`, `orientation`, `app-switcher`, `action-button`, `fold`, `install`, `reinstall`, `install-from-source`, and `push`. Recommended loops: diff --git a/website/docs/docs/configuration.md b/website/docs/docs/configuration.md index bb90c911b6..9e524a5da8 100644 --- a/website/docs/docs/configuration.md +++ b/website/docs/docs/configuration.md @@ -132,7 +132,7 @@ These env vars are the supported user-facing configuration surface. Other `AGENT | App hooks and logs | `AGENT_DEVICE_APP_EVENT_URL_TEMPLATE`, `AGENT_DEVICE_IOS_APP_EVENT_URL_TEMPLATE`, `AGENT_DEVICE_MACOS_APP_EVENT_URL_TEMPLATE`, `AGENT_DEVICE_ANDROID_APP_EVENT_URL_TEMPLATE`, `AGENT_DEVICE_APP_LOG_MAX_BYTES`, `AGENT_DEVICE_APP_LOG_MAX_FILES`, `AGENT_DEVICE_APP_LOG_REDACT_PATTERNS`, `AGENT_DEVICE_EVENT_LOG_MAX_BYTES` | Public. Byte caps take whole integers (`5242880`), not `5MB`. | | Apple runner setup | `AGENT_DEVICE_IOS_TEAM_ID`, `AGENT_DEVICE_IOS_SIGNING_IDENTITY`, `AGENT_DEVICE_IOS_PROVISIONING_PROFILE`, `AGENT_DEVICE_IOS_BUNDLE_ID`, `AGENT_DEVICE_IOS_RUNNER_DERIVED_PATH`, `AGENT_DEVICE_IOS_CLEAN_DERIVED` | Public operator controls. Cleanup is only automatic for override paths under project `.tmp/`. | | Install/update and platform helpers | `AGENT_DEVICE_NO_UPDATE_NOTIFIER`, `AGENT_DEVICE_MACOS_HELPER_BIN`, `AGENT_DEVICE_ANDROID_SNAPSHOT_HELPER_SESSION` | Public operator controls | -| macOS app backend | `AGENT_DEVICE_MACOS_APP_BACKEND` | Public operator control, read by the daemon. `native` drives macOS app sessions through the macOS helper instead of XCTest; see [Commands](/docs/commands). Unset or `xctest` keeps the runner. Restart the daemon after changing it. | +| macOS app backend | `AGENT_DEVICE_MACOS_APP_BACKEND`, `AGENT_DEVICE_MACOS_GHOST_CURSOR` | Public operator controls, read by the daemon. `native` drives macOS app sessions through the macOS helper instead of XCTest; see [Commands](/docs/commands). Unset or `xctest` keeps the runner. The drawn agent pointer adds about 0.3 s to each native click, fill, type, and scroll; `AGENT_DEVICE_MACOS_GHOST_CURSOR=0` turns it off. Restart the daemon after changing either value. | ## Command-specific defaults