Skip to content

fix(android): ease controlled scrolls within the requested duration #2393

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Merged
thymikee merged 1 commit into from
Sep 10, 2026
Merged

fix(android): ease controlled scrolls within the requested duration #2393

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
1 commit
Select commit
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
7 changes: 5 additions & 2 deletions docs/adr/0013-unified-gesture-plans.md
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -74,8 +74,11 @@ Platform adapters consume the canonical plan:
- Android's `executeAndroidTouchPlan` adapter seam sends planned touch, including gesture plans plus
the physical movement for scroll and long-press, to provider-native touch injection when
available, otherwise to the bundled instrumentation helper. One-contact endpoint plans lower in
`packages/platform-android/src/touch-plan.ts` to 16 ms linear transport samples before either injection
path; two-contact plans retain their exact planned samples. Transport samples are typed as
`packages/platform-android/src/touch-plan-lowering.ts` to approximately 16 ms transport samples before
either injection path. Controlled directional scrolls accelerate for one frame, then decelerate
through release within the requested duration, without an appended tail. Inertial scrolls and
general one-contact plans retain linear interpolation; two-contact plans retain their exact
planned samples. Easing reduces release momentum but does not guarantee an exact content offset. Transport samples are typed as
strictly denser than the canonical endpoint pair, so skipping that lowering is a type error at
the injection seams instead of a silently sparse gesture. A stationary long-press needs no
viewport on the helper path; the executor adds the paired provider-owned viewport only for
Expand Down
254 changes: 55 additions & 199 deletions packages/platform-android/src/__tests__/input-actions.test.ts
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -1,7 +1,5 @@
import { test, vi } from 'vitest';
import assert from 'node:assert/strict';
import { GESTURE_SAMPLE_INTERVAL_MS } from '@agent-device/contracts/gesture-plan';
import { GESTURE_DURATION_MAX_MS } from '@agent-device/contracts/gesture-plan-types';
import {
backAndroid,
homeAndroid,
Expand Down Expand Up @@ -81,14 +79,7 @@ test('scrollAndroid accepts sub-frame public durations at the Android planner mi
async () => {
const outputs: Record<string, unknown>[] = [];
for (const durationMs of [0, 15]) {
// 'inertial' keeps the injected plan's durationMs equal to the honored move time, so the
// flooring this test targets is not conflated with the 'controlled' release tail below.
outputs.push(
await scrollAndroid(ANDROID_EMULATOR, 'down', {
durationMs,
releaseBehavior: 'inertial',
}),
);
outputs.push(await scrollAndroid(ANDROID_EMULATOR, 'down', { durationMs }));
}
return outputs;
},
Expand All @@ -104,198 +95,63 @@ test('scrollAndroid accepts sub-frame public durations at the Android planner mi
);
});

test('scrollAndroid defaults to a controlled release: a quivering tail past the pan endpoint', async () => {
const touchCalls: Parameters<AndroidTouchInjector>[0][] = [];
await withAndroidAdbProvider(
{
exec: async () => {
throw new Error('adb must not run');
},
gestureViewport: async () => ({ x: 10, y: 20, width: 1080, height: 1920 }),
touch: async (request) => {
touchCalls.push(request);
return { injected: true };
},
},
{ serial: ANDROID_EMULATOR.id },
async () => await scrollAndroid(ANDROID_EMULATOR, 'down', { pixels: 240, durationMs: 120 }),
);

assert.equal(touchCalls.length, 1);
const [touch] = touchCalls;
const samples = touch!.pointers[0]!.samples;
const endpoint = samples.find((sample) => sample.offsetMs === 120)!;
const tail = samples.filter((sample) => sample.offsetMs > 120);

// The plan carries a >=100ms tail past the honored 120ms move; the CLI-facing `durationMs` in
// the command result (asserted below) stays at the honored move time.
assert.equal(touch!.durationMs, 280);
assert.ok(tail.length >= 100 / GESTURE_SAMPLE_INTERVAL_MS);
for (const sample of tail) assert.equal(sample.point.y, endpoint.point.y);
const allPastEndpoint = [endpoint, ...tail];
for (let index = 1; index < allPastEndpoint.length; index += 1) {
assert.notEqual(allPastEndpoint[index]!.point.x, allPastEndpoint[index - 1]!.point.x);
}
});

test('scrollAndroid composes the duration floor with the default controlled-release tail', async () => {
const touchCalls: Parameters<AndroidTouchInjector>[0][] = [];
await withAndroidAdbProvider(
{
exec: async () => {
throw new Error('adb must not run');
},
gestureViewport: async () => ({ x: 0, y: 0, width: 1080, height: 1920 }),
touch: async (request) => {
touchCalls.push(request);
},
},
{ serial: ANDROID_EMULATOR.id },
async () => await scrollAndroid(ANDROID_EMULATOR, 'down', { durationMs: 0 }),
);

// The move floors to the Android planner minimum (16ms) before the tail is appended, not after.
assert.equal(touchCalls[0]!.durationMs, GESTURE_SAMPLE_INTERVAL_MS + 160);
});

test('scrollAndroid runs the full controlled-release tail at the largest duration that still leaves it room', async () => {
const touchCalls: Parameters<AndroidTouchInjector>[0][] = [];
const maxControlledMoveMs = GESTURE_DURATION_MAX_MS - 160;
const result = await withAndroidAdbProvider(
{
exec: async () => {
throw new Error('adb must not run');
},
gestureViewport: async () => ({ x: 0, y: 0, width: 1080, height: 1920 }),
touch: async (request) => {
touchCalls.push(request);
},
},
{ serial: ANDROID_EMULATOR.id },
async () =>
await scrollAndroid(ANDROID_EMULATOR, 'down', {
pixels: 1800,
durationMs: maxControlledMoveMs,
}),
);

// The requested move is honored in full, and the tail always runs at its full length — the
// dispatched plan lands exactly at GESTURE_DURATION_MAX_MS, never past it.
assert.equal(result.durationMs, maxControlledMoveMs);
const [touch] = touchCalls;
assert.equal(touch!.durationMs, GESTURE_DURATION_MAX_MS);
assert.equal(touch!.pointers[0]!.samples.at(-1)!.offsetMs, GESTURE_DURATION_MAX_MS);
});

test('scrollAndroid rejects a controlled-release durationMs that would leave the release tail no room, without shortening the move', async () => {
await withAndroidAdbProvider(
{
exec: async () => {
throw new Error('adb must not run');
test.each([undefined, 'inertial'] as const)(
'scrollAndroid preserves path and duration with %s release',
async (releaseBehavior) => {
const touchCalls: Parameters<AndroidTouchInjector>[0][] = [];
await withAndroidAdbProvider(
{
exec: async () => {
throw new Error('adb must not run');
},
gestureViewport: async () => ({ x: 10, y: 20, width: 1080, height: 1920 }),
touch: async (request) => {
touchCalls.push(request);
},
},
gestureViewport: async () => ({ x: 0, y: 0, width: 1080, height: 1920 }),
touch: async () => {
throw new Error('touch must not run for a rejected request');
{ serial: ANDROID_EMULATOR.id },
async () => {
for (const direction of ['up', 'down', 'left', 'right'] as const) {
for (const durationMs of [16, 120, 300, 9841, 10000]) {
await scrollAndroid(ANDROID_EMULATOR, direction, {
pixels: 240,
durationMs,
releaseBehavior,
});
}
}
},
},
{ serial: ANDROID_EMULATOR.id },
async () => {
await assert.rejects(
scrollAndroid(ANDROID_EMULATOR, 'down', {
pixels: 1800,
durationMs: GESTURE_DURATION_MAX_MS - 159,
}),
/scroll durationMs must be at most 9840 for a controlled release/,
);
for (const touch of touchCalls) {
const samples = touch.pointers[0]!.samples;
const start = samples[0]!;
const end = samples.at(-1)!;
assert.equal(start.offsetMs, 0);
assert.equal(end.offsetMs, touch.durationMs);
const distance = (a: typeof start, b: typeof start) =>
Math.hypot(b.point.x - a.point.x, b.point.y - a.point.y);
assert.equal(distance(start, end), 240);
const velocities = samples
.slice(1)
.map(
(sample, index) =>
distance(samples[index]!, sample) / (sample.offsetMs - samples[index]!.offsetMs),
);
if (releaseBehavior === 'inertial') {
for (const velocity of velocities) assert.ok(Math.abs(velocity - velocities[0]!) < 1e-8);
continue;
}
const firstMove = samples[1]!;
assert.ok(
distance(start, firstMove) <= ((240 * firstMove.offsetMs) / touch.durationMs) * 1.1,
);
},
);
});

test("scrollAndroid accepts the full GESTURE_DURATION_MAX_MS for an 'inertial' release, which needs no tail", async () => {
const touchCalls: Parameters<AndroidTouchInjector>[0][] = [];
await withAndroidAdbProvider(
{
exec: async () => {
throw new Error('adb must not run');
},
gestureViewport: async () => ({ x: 0, y: 0, width: 1080, height: 1920 }),
touch: async (request) => {
touchCalls.push(request);
},
},
{ serial: ANDROID_EMULATOR.id },
async () =>
await scrollAndroid(ANDROID_EMULATOR, 'down', {
pixels: 1800,
durationMs: GESTURE_DURATION_MAX_MS,
releaseBehavior: 'inertial',
}),
);

const [touch] = touchCalls;
assert.equal(touch!.durationMs, GESTURE_DURATION_MAX_MS);
});

test('scrollAndroid jitters the axis orthogonal to a horizontal scroll, holding the scroll axis fixed', async () => {
const touchCalls: Parameters<AndroidTouchInjector>[0][] = [];
await withAndroidAdbProvider(
{
exec: async () => {
throw new Error('adb must not run');
},
gestureViewport: async () => ({ x: 10, y: 20, width: 1080, height: 1920 }),
touch: async (request) => {
touchCalls.push(request);
return { injected: true };
},
},
{ serial: ANDROID_EMULATOR.id },
async () => await scrollAndroid(ANDROID_EMULATOR, 'left', { pixels: 240, durationMs: 120 }),
);

assert.equal(touchCalls.length, 1);
const [touch] = touchCalls;
const samples = touch!.pointers[0]!.samples;
const endpoint = samples.find((sample) => sample.offsetMs === 120)!;
const tail = samples.filter((sample) => sample.offsetMs > 120);

assert.ok(tail.length >= 100 / GESTURE_SAMPLE_INTERVAL_MS);
// The scroll axis (x, for a horizontal scroll) stays exactly at the endpoint — zero velocity
// there by construction; only the orthogonal axis (y) jitters to dodge the resampling quirk.
for (const sample of tail) assert.equal(sample.point.x, endpoint.point.x);
const allPastEndpoint = [endpoint, ...tail];
for (let index = 1; index < allPastEndpoint.length; index += 1) {
assert.notEqual(allPastEndpoint[index]!.point.y, allPastEndpoint[index - 1]!.point.y);
}
});

test('scrollAndroid honors an inertial release (scroll top/bottom): lifts at the pan endpoint', async () => {
const touchCalls: Parameters<AndroidTouchInjector>[0][] = [];
await withAndroidAdbProvider(
{
exec: async () => {
throw new Error('adb must not run');
},
gestureViewport: async () => ({ x: 10, y: 20, width: 1080, height: 1920 }),
touch: async (request) => {
touchCalls.push(request);
return { injected: true };
},
},
{ serial: ANDROID_EMULATOR.id },
async () =>
await scrollAndroid(ANDROID_EMULATOR, 'down', {
pixels: 240,
durationMs: 120,
releaseBehavior: 'inertial',
}),
);

assert.equal(touchCalls.length, 1);
const [touch] = touchCalls;
assert.equal(touch!.durationMs, 120);
assert.equal(touch!.pointers[0]!.samples.at(-1)!.offsetMs, 120);
});
assert.ok(velocities.at(-1)! < Math.max(...velocities) / 2);
for (let i = Math.ceil(velocities.length / 2); i < velocities.length; i += 1) {
assert.ok(velocities[i]! <= velocities[i - 1]! + 1e-8);
}
}
},
);

test('longPressAndroid sends a stationary semantic touch plan', async () => {
const touchCalls: Parameters<AndroidTouchInjector>[0][] = [];
Expand Down
22 changes: 22 additions & 0 deletions packages/platform-android/src/__tests__/touch-plan-lowering.fixtures.ts
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -0,0 +1,22 @@
import fc from 'fast-check';
import type { Rect } from '@agent-device/kernel/snapshot';
import { SCROLL_DIRECTIONS } from '@agent-device/contracts/scroll-gesture';

export const PROPERTY_RUNS_SMALL = 40;

const viewportRectArb: fc.Arbitrary<Rect> = fc.oneof(
fc.constantFrom({ x: 0, y: 0, width: 320, height: 568 }, { x: 0, y: 0, width: 375, height: 667 }),
fc.record({
x: fc.integer({ min: 0, max: 200 }),
y: fc.integer({ min: 0, max: 200 }),
width: fc.integer({ min: 1, max: 2400 }),
height: fc.integer({ min: 1, max: 2400 }),
}),
);

export const scrollInViewportArb = fc.record({
viewport: viewportRectArb.filter(({ width, height }) => width >= 32 && height >= 32),
direction: fc.constantFrom(...SCROLL_DIRECTIONS),
durationMs: fc.integer({ min: 16, max: 10000 }),
pixels: fc.integer({ min: 1, max: 2000 }),
});
52 changes: 52 additions & 0 deletions packages/platform-android/src/__tests__/touch-plan-lowering.test.ts
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -1,3 +1,6 @@
import fc from 'fast-check';
import { buildScrollGesturePlan } from '@agent-device/contracts/scroll-gesture';
import { PROPERTY_RUNS_SMALL, scrollInViewportArb } from './touch-plan-lowering.fixtures.ts';
import assert from 'node:assert/strict';
import { expectTypeOf, test } from 'vitest';
import { buildDragGesturePlan, buildGesturePlan } from '@agent-device/contracts/gesture-plan';
Expand Down Expand Up @@ -128,3 +131,52 @@ test('a canonical endpoint plan cannot reach the transport unlowered', () => {
expectTypeOf<SinglePointerGesturePlan>().not.toExtend<AndroidLoweredTouchPlan>();
expectTypeOf<ReturnType<typeof lowerAndroidTouchPlan>>().toExtend<AndroidLoweredTouchPlan>();
});

test('Android controlled scroll sampling preserves the inertial path and viewport bounds', () => {
fc.assert(
fc.property(scrollInViewportArb, ({ viewport, direction, durationMs, pixels }) => {
const scroll = buildScrollGesturePlan({
direction,
pixels,
referenceWidth: viewport.width,
referenceHeight: viewport.height,
});
const plan = buildGesturePlan(
{
intent: 'pan',
origin: { x: viewport.x + scroll.x1, y: viewport.y + scroll.y1 },
delta: { x: scroll.x2 - scroll.x1, y: scroll.y2 - scroll.y1 },
durationMs,
},
viewport,
'android',
);
const controlled = lowerAndroidTouchPlan({ ...plan, releaseBehavior: 'controlled' });
const inertial = lowerAndroidTouchPlan({ ...plan, releaseBehavior: 'inertial' });
assert.equal(controlled.durationMs, durationMs);
assert.equal(inertial.durationMs, durationMs);
const samples = controlled.pointers[0].samples;
const linear = inertial.pointers[0].samples;
assert.deepEqual(
samples.map(({ offsetMs }) => offsetMs),
linear.map(({ offsetMs }) => offsetMs),
);
assert.deepEqual(samples[0], linear[0]);
assert.deepEqual(samples.at(-1), linear.at(-1));
for (const axis of ['x', 'y'] as const) {
const from = samples[0]!.point[axis];
const to = samples.at(-1)!.point[axis];
for (let i = 1; i < samples.length; i += 1) {
const sample = samples[i]!;
assert.ok(
sample.point[axis] >= Math.min(from, to) && sample.point[axis] <= Math.max(from, to),
);
assert.ok((sample.point[axis] - samples[i - 1]!.point[axis]) * (to - from) >= 0);
const expectedLinear = from + ((to - from) * sample.offsetMs) / durationMs;
assert.ok(Math.abs(linear[i]!.point[axis] - expectedLinear) < 1e-8);
}
}
}),
{ numRuns: PROPERTY_RUNS_SMALL },
);
});
Loading
Loading