diff --git a/packages/platform-android/src/__tests__/input-actions.test.ts b/packages/platform-android/src/__tests__/input-actions.test.ts index a6f55b214a..12b0ca8126 100644 --- a/packages/platform-android/src/__tests__/input-actions.test.ts +++ b/packages/platform-android/src/__tests__/input-actions.test.ts @@ -1,5 +1,7 @@ import { test, vi } from 'vitest'; import assert from 'node:assert/strict'; +import { GESTURE_SAMPLE_INTERVAL_MS } from '@agent-device/contracts/gesture-plan'; +import { GESTURE_DURATION_MAX_MS } from '@agent-device/contracts/gesture-plan-types'; import { backAndroid, homeAndroid, @@ -79,7 +81,14 @@ test('scrollAndroid accepts sub-frame public durations at the Android planner mi async () => { const outputs: Record[] = []; for (const durationMs of [0, 15]) { - outputs.push(await scrollAndroid(ANDROID_EMULATOR, 'down', { durationMs })); + // 'inertial' keeps the injected plan's durationMs equal to the honored move time, so the + // flooring this test targets is not conflated with the 'controlled' release tail below. + outputs.push( + await scrollAndroid(ANDROID_EMULATOR, 'down', { + durationMs, + releaseBehavior: 'inertial', + }), + ); } return outputs; }, @@ -95,6 +104,199 @@ test('scrollAndroid accepts sub-frame public durations at the Android planner mi ); }); +test('scrollAndroid defaults to a controlled release: a quivering tail past the pan endpoint', async () => { + const touchCalls: Parameters[0][] = []; + await withAndroidAdbProvider( + { + exec: async () => { + throw new Error('adb must not run'); + }, + gestureViewport: async () => ({ x: 10, y: 20, width: 1080, height: 1920 }), + touch: async (request) => { + touchCalls.push(request); + return { injected: true }; + }, + }, + { serial: ANDROID_EMULATOR.id }, + async () => await scrollAndroid(ANDROID_EMULATOR, 'down', { pixels: 240, durationMs: 120 }), + ); + + assert.equal(touchCalls.length, 1); + const [touch] = touchCalls; + const samples = touch!.pointers[0]!.samples; + const endpoint = samples.find((sample) => sample.offsetMs === 120)!; + const tail = samples.filter((sample) => sample.offsetMs > 120); + + // The plan carries a >=100ms tail past the honored 120ms move; the CLI-facing `durationMs` in + // the command result (asserted below) stays at the honored move time. + assert.equal(touch!.durationMs, 280); + assert.ok(tail.length >= 100 / GESTURE_SAMPLE_INTERVAL_MS); + for (const sample of tail) assert.equal(sample.point.y, endpoint.point.y); + const allPastEndpoint = [endpoint, ...tail]; + for (let index = 1; index < allPastEndpoint.length; index += 1) { + assert.notEqual(allPastEndpoint[index]!.point.x, allPastEndpoint[index - 1]!.point.x); + } +}); + +test('scrollAndroid composes the duration floor with the default controlled-release tail', async () => { + const touchCalls: Parameters[0][] = []; + await withAndroidAdbProvider( + { + exec: async () => { + throw new Error('adb must not run'); + }, + gestureViewport: async () => ({ x: 0, y: 0, width: 1080, height: 1920 }), + touch: async (request) => { + touchCalls.push(request); + }, + }, + { serial: ANDROID_EMULATOR.id }, + async () => await scrollAndroid(ANDROID_EMULATOR, 'down', { durationMs: 0 }), + ); + + // The move floors to the Android planner minimum (16ms) before the tail is appended, not after. + assert.equal(touchCalls[0]!.durationMs, GESTURE_SAMPLE_INTERVAL_MS + 160); +}); + +test('scrollAndroid runs the full controlled-release tail at the largest duration that still leaves it room', async () => { + const touchCalls: Parameters[0][] = []; + const maxControlledMoveMs = GESTURE_DURATION_MAX_MS - 160; + const result = await withAndroidAdbProvider( + { + exec: async () => { + throw new Error('adb must not run'); + }, + gestureViewport: async () => ({ x: 0, y: 0, width: 1080, height: 1920 }), + touch: async (request) => { + touchCalls.push(request); + }, + }, + { serial: ANDROID_EMULATOR.id }, + async () => + await scrollAndroid(ANDROID_EMULATOR, 'down', { + pixels: 1800, + durationMs: maxControlledMoveMs, + }), + ); + + // The requested move is honored in full, and the tail always runs at its full length — the + // dispatched plan lands exactly at GESTURE_DURATION_MAX_MS, never past it. + assert.equal(result.durationMs, maxControlledMoveMs); + const [touch] = touchCalls; + assert.equal(touch!.durationMs, GESTURE_DURATION_MAX_MS); + assert.equal(touch!.pointers[0]!.samples.at(-1)!.offsetMs, GESTURE_DURATION_MAX_MS); +}); + +test('scrollAndroid rejects a controlled-release durationMs that would leave the release tail no room, without shortening the move', async () => { + await withAndroidAdbProvider( + { + exec: async () => { + throw new Error('adb must not run'); + }, + gestureViewport: async () => ({ x: 0, y: 0, width: 1080, height: 1920 }), + touch: async () => { + throw new Error('touch must not run for a rejected request'); + }, + }, + { serial: ANDROID_EMULATOR.id }, + async () => { + await assert.rejects( + scrollAndroid(ANDROID_EMULATOR, 'down', { + pixels: 1800, + durationMs: GESTURE_DURATION_MAX_MS - 159, + }), + /scroll durationMs must be at most 9840 for a controlled release/, + ); + }, + ); +}); + +test("scrollAndroid accepts the full GESTURE_DURATION_MAX_MS for an 'inertial' release, which needs no tail", async () => { + const touchCalls: Parameters[0][] = []; + await withAndroidAdbProvider( + { + exec: async () => { + throw new Error('adb must not run'); + }, + gestureViewport: async () => ({ x: 0, y: 0, width: 1080, height: 1920 }), + touch: async (request) => { + touchCalls.push(request); + }, + }, + { serial: ANDROID_EMULATOR.id }, + async () => + await scrollAndroid(ANDROID_EMULATOR, 'down', { + pixels: 1800, + durationMs: GESTURE_DURATION_MAX_MS, + releaseBehavior: 'inertial', + }), + ); + + const [touch] = touchCalls; + assert.equal(touch!.durationMs, GESTURE_DURATION_MAX_MS); +}); + +test('scrollAndroid jitters the axis orthogonal to a horizontal scroll, holding the scroll axis fixed', async () => { + const touchCalls: Parameters[0][] = []; + await withAndroidAdbProvider( + { + exec: async () => { + throw new Error('adb must not run'); + }, + gestureViewport: async () => ({ x: 10, y: 20, width: 1080, height: 1920 }), + touch: async (request) => { + touchCalls.push(request); + return { injected: true }; + }, + }, + { serial: ANDROID_EMULATOR.id }, + async () => await scrollAndroid(ANDROID_EMULATOR, 'left', { pixels: 240, durationMs: 120 }), + ); + + assert.equal(touchCalls.length, 1); + const [touch] = touchCalls; + const samples = touch!.pointers[0]!.samples; + const endpoint = samples.find((sample) => sample.offsetMs === 120)!; + const tail = samples.filter((sample) => sample.offsetMs > 120); + + assert.ok(tail.length >= 100 / GESTURE_SAMPLE_INTERVAL_MS); + // The scroll axis (x, for a horizontal scroll) stays exactly at the endpoint — zero velocity + // there by construction; only the orthogonal axis (y) jitters to dodge the resampling quirk. + for (const sample of tail) assert.equal(sample.point.x, endpoint.point.x); + const allPastEndpoint = [endpoint, ...tail]; + for (let index = 1; index < allPastEndpoint.length; index += 1) { + assert.notEqual(allPastEndpoint[index]!.point.y, allPastEndpoint[index - 1]!.point.y); + } +}); + +test('scrollAndroid honors an inertial release (scroll top/bottom): lifts at the pan endpoint', async () => { + const touchCalls: Parameters[0][] = []; + await withAndroidAdbProvider( + { + exec: async () => { + throw new Error('adb must not run'); + }, + gestureViewport: async () => ({ x: 10, y: 20, width: 1080, height: 1920 }), + touch: async (request) => { + touchCalls.push(request); + return { injected: true }; + }, + }, + { serial: ANDROID_EMULATOR.id }, + async () => + await scrollAndroid(ANDROID_EMULATOR, 'down', { + pixels: 240, + durationMs: 120, + releaseBehavior: 'inertial', + }), + ); + + assert.equal(touchCalls.length, 1); + const [touch] = touchCalls; + assert.equal(touch!.durationMs, 120); + assert.equal(touch!.pointers[0]!.samples.at(-1)!.offsetMs, 120); +}); + test('longPressAndroid sends a stationary semantic touch plan', async () => { const touchCalls: Parameters[0][] = []; const result = await withAndroidAdbProvider( diff --git a/packages/platform-android/src/input-actions.ts b/packages/platform-android/src/input-actions.ts index 607122ed4a..7e94fbc940 100644 --- a/packages/platform-android/src/input-actions.ts +++ b/packages/platform-android/src/input-actions.ts @@ -3,9 +3,20 @@ * IME, and the adb-shell writer — is `text-input.ts`. */ import { DEVICE_ROTATION_SURFACE_INDEX, type DeviceRotation } from '@agent-device/contracts/device'; -import { buildGesturePlan } from '@agent-device/contracts/gesture-plan'; -import { GESTURE_DURATION_MIN_MS } from '@agent-device/contracts/gesture-plan-types'; -import { DEFAULT_MOBILE_SCROLL_DURATION_MS } from '@agent-device/contracts/scroll-command'; +import { GESTURE_SAMPLE_INTERVAL_MS, buildGesturePlan } from '@agent-device/contracts/gesture-plan'; +import { + GESTURE_DURATION_MAX_MS, + GESTURE_DURATION_MIN_MS, +} from '@agent-device/contracts/gesture-plan-types'; +import type { + GesturePlan, + PointerTrajectorySample, + SinglePointerTrajectory, +} from '@agent-device/contracts/gesture-plan-types'; +import { + DEFAULT_MOBILE_SCROLL_DURATION_MS, + type ScrollReleaseBehavior, +} from '@agent-device/contracts/scroll-command'; import { type ScrollDirection, buildScrollGesturePlan, @@ -13,6 +24,7 @@ import { import { type TvRemoteButton, toAndroidTvRemoteKeyevent } from '@agent-device/contracts/tv-remote'; import type { DeviceInfo } from '@agent-device/kernel/device'; import { AppError } from '@agent-device/kernel/errors'; +import type { Rect } from '@agent-device/kernel/snapshot'; import { sleep } from '@agent-device/host-kit/retry'; import { runAndroidAdb } from './adb.ts'; import { executeAndroidTouchPlan, readAndroidGestureViewport } from './touch-executor.ts'; @@ -163,7 +175,12 @@ export async function focusAndroid(device: DeviceInfo, x: number, y: number): Pr export async function scrollAndroid( device: DeviceInfo, direction: ScrollDirection, - options?: { amount?: number; pixels?: number; durationMs?: number } & AndroidHelperSessionOptions, + options?: { + amount?: number; + pixels?: number; + durationMs?: number; + releaseBehavior?: ScrollReleaseBehavior; + } & AndroidHelperSessionOptions, ): Promise> { // The viewport read and the gesture are two helper calls one command apart: giving the read the // command's session scope keeps both on the same instrumentation. @@ -192,21 +209,26 @@ export async function scrollAndroid( options?.durationMs ?? DEFAULT_MOBILE_SCROLL_DURATION_MS, GESTURE_DURATION_MIN_MS, ); + const releaseBehavior = options?.releaseBehavior ?? 'controlled'; + if (releaseBehavior === 'controlled') assertRoomForControlledReleaseTail(durationMs); + const gesturePlan = buildGesturePlan( + { + intent: 'pan', + origin: { x: scrollPlan.x1, y: scrollPlan.y1 }, + delta: { + x: scrollPlan.x2 - scrollPlan.x1, + y: scrollPlan.y2 - scrollPlan.y1, + }, + durationMs, + }, + viewport, + 'android', + ); const backend = await executeAndroidTouchPlan( device, - buildGesturePlan( - { - intent: 'pan', - origin: { x: scrollPlan.x1, y: scrollPlan.y1 }, - delta: { - x: scrollPlan.x2 - scrollPlan.x1, - y: scrollPlan.y2 - scrollPlan.y1, - }, - durationMs, - }, - viewport, - 'android', - ), + releaseBehavior === 'controlled' + ? withControlledReleaseTail(gesturePlan, viewport, direction) + : gesturePlan, ); return { @@ -216,6 +238,81 @@ export async function scrollAndroid( }; } +// Kept an even multiple of GESTURE_SAMPLE_INTERVAL_MS so the tail's last sample lands back on the +// pan's exact endpoint (an odd multiple would still avoid the fling — every consecutive sample +// still differs — but would leave the release 1px off the requested endpoint). +const CONTROLLED_RELEASE_TAIL_MS = 160; + +// The dispatched plan (move + tail) must never exceed GESTURE_DURATION_MAX_MS, the same ceiling +// every gesture plan is built under. Rather than silently dropping the tail for a move that +// leaves it no room, a controlled scroll's own accepted range stops short of the shared ceiling +// by the tail's length — the full tail runs for every accepted controlled scroll, and a request +// past this narrower range is rejected with the reason, not truncated. +const CONTROLLED_RELEASE_MAX_MOVE_MS = GESTURE_DURATION_MAX_MS - CONTROLLED_RELEASE_TAIL_MS; + +function assertRoomForControlledReleaseTail(durationMs: number): void { + if (durationMs <= CONTROLLED_RELEASE_MAX_MOVE_MS) return; + throw new AppError( + 'INVALID_ARGS', + `scroll durationMs must be at most ${CONTROLLED_RELEASE_MAX_MOVE_MS} for a controlled release ` + + `(leaves room for the ${CONTROLLED_RELEASE_TAIL_MS}ms release tail within the ` + + `${GESTURE_DURATION_MAX_MS}ms gesture ceiling)`, + { + hint: "Pass a shorter durationMs, or releaseBehavior 'inertial' if the fling is acceptable.", + }, + ); +} + +/** + * A short, quivering tail appended after a 'controlled' scroll's endpoint, adding + * `CONTROLLED_RELEASE_TAIL_MS` of real time to the gesture. AOSP's `InputConsumer::rewriteMessage` + * collapses a MOVE that repeats the previous coordinates into a "resampled" sample, and + * `VelocityTracker` skips resampled samples — so a truly stationary tail never reaches the + * tracker, and `ScrollView.onTouchEvent` (which computes release velocity before applying UP) + * still flings at the pan's velocity. Nudging the axis orthogonal to the scroll by 1px every frame + * (holding the scroll axis exactly at the endpoint — zero velocity there by construction) keeps + * every sample distinct without adding net travel along either axis. Measured fling-free for + * vertical scrolls on a `RecyclerView` and an RN `ScrollView` (issue #2371); not independently + * verified against every OEM skin or a Compose `LazyColumn`. An 'inertial' release (the + * `scroll top`/`scroll bottom` edge passes) lifts at the pan's endpoint unchanged. + * + * Callers must have already checked `assertRoomForControlledReleaseTail` on the move duration — + * this always appends the full tail. + */ +function withControlledReleaseTail( + plan: GesturePlan, + viewport: Rect, + direction: ScrollDirection, +): GesturePlan { + if (plan.topology !== 'single') return plan; + const steps = CONTROLLED_RELEASE_TAIL_MS / GESTURE_SAMPLE_INTERVAL_MS; + const [pointer] = plan.pointers; + const end = pointer.samples.at(-1)!; + const horizontal = direction === 'left' || direction === 'right'; + const jitterBase = horizontal ? end.point.y : end.point.x; + const jitterMin = (horizontal ? viewport.y : viewport.x) + 1; + const jitterMax = (horizontal ? viewport.y + viewport.height : viewport.x + viewport.width) - 1; + const nudged = jitterBase + 1 <= jitterMax ? jitterBase + 1 : Math.max(jitterMin, jitterBase - 1); + const tail: PointerTrajectorySample[] = Array.from({ length: steps }, (_, index) => { + const jitter = index % 2 === 0 ? nudged : jitterBase; + return { + offsetMs: plan.durationMs + (index + 1) * GESTURE_SAMPLE_INTERVAL_MS, + point: horizontal ? { x: end.point.x, y: jitter } : { x: jitter, y: end.point.y }, + }; + }); + const samples: SinglePointerTrajectory['samples'] = [ + pointer.samples[0], + pointer.samples[1], + ...pointer.samples.slice(2), + ...tail, + ]; + return { + ...plan, + durationMs: plan.durationMs + CONTROLLED_RELEASE_TAIL_MS, + pointers: [{ ...pointer, samples }], + }; +} + function resolveAndroidUserRotation(orientation: DeviceRotation): string { const index = DEVICE_ROTATION_SURFACE_INDEX[orientation]; if (index === undefined) { diff --git a/src/commands/interaction/metadata.ts b/src/commands/interaction/metadata.ts index 5c004007a4..f24bbc6f2e 100644 --- a/src/commands/interaction/metadata.ts +++ b/src/commands/interaction/metadata.ts @@ -68,7 +68,7 @@ const interactionCommandDescriptions = { 'Move input focus to explicit screen coordinates without entering text. Prefer semantic interactions when a snapshot ref or selector is available; use type or fill after focus.', type: 'Append text to the currently focused input. Use fill when the existing field value should be replaced, and focus first when no input is active.', scroll: - 'Scroll in a direction, or toward the top/bottom edge of scrollable content. The optional amount is the finger-path fraction of the viewport axis; app scroll physics determine the final content offset.', + 'Scroll in a direction, or toward the top/bottom edge of scrollable content. The optional amount is the finger-path fraction of the viewport axis; the gesture releases with reduced momentum rather than stopping at an exact offset, so app scroll physics determine where content actually lands.', get: 'Read text or accessibility attributes from a snapshot ref or selector without changing the app. Use format text for visible content or attrs for the element attribute map.', is: 'Check whether a selector satisfies a UI predicate such as visible, hidden, exists, absent, editable, selected, focused, or text. `absent` passes only when one readable, complete, unscoped, full-depth accessibility capture has zero matches. Use wait when the condition may appear asynchronously.', find: 'Find by text/label/value/role/id and run action', diff --git a/test/integration/provider-scenarios/android-lifecycle.test.ts b/test/integration/provider-scenarios/android-lifecycle.test.ts index a66d35e013..b62fa06345 100644 --- a/test/integration/provider-scenarios/android-lifecycle.test.ts +++ b/test/integration/provider-scenarios/android-lifecycle.test.ts @@ -226,7 +226,7 @@ test(ANDROID_TOUCH_CONTRACT_EVIDENCE.testName, async () => { })); assert.deepEqual(touchCalls, [ { topology: 'single', intent: 'longPress', pointerCount: 1, durationMs: 750 }, - { topology: 'single', intent: 'pan', pointerCount: 1, durationMs: 350 }, + { topology: 'single', intent: 'pan', pointerCount: 1, durationMs: 510 }, // 'controlled': 350ms move + 160ms tail (#2371) { topology: 'single', intent: 'pan', pointerCount: 1, durationMs: 300 }, { topology: 'single', intent: 'pan', pointerCount: 1, durationMs: 500 }, { topology: 'two', intent: 'pan', pointerCount: 2, durationMs: 500 }, diff --git a/website/docs/docs/commands.md b/website/docs/docs/commands.md index 7c93a218f4..560f244557 100644 --- a/website/docs/docs/commands.md +++ b/website/docs/docs/commands.md @@ -485,7 +485,8 @@ Target-authored drag is supported on Android touch devices and iOS/iPadOS. Backe `gesture transform` accepts `x y dx dy scale degrees [durationMs]` for one combined two-finger pan/zoom/rotate gesture on Android and iOS simulators. Pinch, rotate, two-finger pan, and transform use the same viewport-aware pointer planning; impossible paths fail before injection instead of clamping or distorting the requested motion. On iOS simulators it uses private XCTest synthesis for a continuous two-finger pan/scale/rotation path, so verify app-level metrics instead of assuming the requested values map exactly to recognizer output. On Android, `gesture transform` injects a geometric two-finger path. App recognizers may report non-exact pan, scale, and rotation values, so verify qualitative state such as `pan changed yes`, `pinch changed yes`, and `rotate changed yes` unless the app explicitly promises exact centroid metrics. If exact app-state values matter, prefer isolated `gesture pan`, `gesture pinch`, or `gesture rotate` commands. -`scroll` accepts either a relative amount (`0.5` means a finger path spanning half of the viewport on that axis) or `--pixels ` for a fixed-distance gesture. The final content offset can differ because apps apply pan-recognition thresholds, collapsing headers, bounds, and their own scroll physics. Large distances are clamped to the usable drag band so the gesture stays reliable across Android, iOS, and macOS. +`scroll` accepts either a relative amount (`0.5` means a finger path spanning half of the viewport on that axis) or `--pixels ` for a fixed-distance gesture. It releases with reduced momentum toward the requested distance, not an exact stop there — the final content offset can still differ because apps apply pan-recognition thresholds, collapsing headers, bounds, and their own scroll physics. Large distances are clamped to the usable drag band so the gesture stays reliable across Android, iOS, and macOS. +On Android, a plain `scroll ` releases in this reduced-momentum ("controlled") mode by default — it appends a short braking tail after the drag so the list does not fling past the requested distance — while `scroll top`/`scroll bottom` release inertially instead, coasting to the edge. That braking tail needs headroom within Android's 10000ms gesture ceiling, so a controlled scroll's own `--duration-ms` accepts at most 9840ms; a longer request is rejected with `INVALID_ARGS` rather than silently shortened or losing the braking tail. Default snapshot text output is visible-first, so off-screen interactive content is summarized instead of shown as tappable refs. When a target only appears in an off-screen summary, use `scroll --settle`: the response waits for the UI to go quiet and returns the diff against the tree you last observed, with fresh refs on the added lines, so no follow-up `snapshot -i` is needed. `back --settle` does the same for navigation. Both are best-effort and never fail the action. For repeated checks without settle, a small shell loop is enough: