From 7d914dfdf5a22a5a01b466567695799f34ef34a6 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Micha=C5=82=20Pierzcha=C5=82a?= Date: Mon, 13 Jul 2026 14:29:49 +0200 Subject: [PATCH 01/16] fix: remove Android gesture swipe fallback --- docs/adr/0013-unified-gesture-plans.md | 7 +-- .../android/__tests__/input-actions.test.ts | 61 +------------------ .../multitouch-helper-ownership.test.ts | 57 +++++++++++++++++ src/platforms/android/input-actions.ts | 20 ------ src/platforms/android/multitouch-helper.ts | 19 +----- .../android-lifecycle.test.ts | 8 +-- .../android-test-suite.test.ts | 13 +++- .../provider-scenarios/android-world.ts | 19 ++++++ 8 files changed, 96 insertions(+), 108 deletions(-) diff --git a/docs/adr/0013-unified-gesture-plans.md b/docs/adr/0013-unified-gesture-plans.md index ee35b36fa..13ebb7b94 100644 --- a/docs/adr/0013-unified-gesture-plans.md +++ b/docs/adr/0013-unified-gesture-plans.md @@ -55,10 +55,9 @@ no established automation use case justifies a public tuning surface. Platform adapters consume the canonical plan: - Android sends the plan to provider-native touch injection when available, otherwise to the - bundled instrumentation helper. The helper injects the exact planned pointer samples. A - two-contact plan never falls back to `adb input swipe`; issue #690 separately owns removal of the - existing one-contact fallback. The snapshot helper is stopped before local gesture - instrumentation because Android permits only one instrumentation owner of `UiAutomation`. + bundled instrumentation helper. The helper injects the exact planned pointer samples. Gesture + execution never falls back to `adb input swipe`; the snapshot helper is stopped before local + gesture instrumentation because Android permits only one instrumentation owner of `UiAutomation`. - iOS converts every planned point to native orientation and feeds the exact arrays to the existing private XCTest event bridge. macOS lowers a one-contact plan to its drag executor and tvOS lowers it to remote direction. Core admission and the Apple adapter both consume the same shared diff --git a/src/platforms/android/__tests__/input-actions.test.ts b/src/platforms/android/__tests__/input-actions.test.ts index d9067b94c..55d05c362 100644 --- a/src/platforms/android/__tests__/input-actions.test.ts +++ b/src/platforms/android/__tests__/input-actions.test.ts @@ -1,16 +1,7 @@ import { test } from 'vitest'; import assert from 'node:assert/strict'; import { promises as fs } from 'node:fs'; -import os from 'node:os'; -import path from 'node:path'; -import { - fillAndroid, - rotateAndroid, - scrollAndroid, - swipeAndroid, - typeAndroid, -} from '../input-actions.ts'; -import type { DeviceInfo } from '../../../kernel/device.ts'; +import { fillAndroid, rotateAndroid, scrollAndroid, typeAndroid } from '../input-actions.ts'; import { AppError } from '../../../kernel/errors.ts'; import { withScriptedAdb } from '../../../__tests__/test-utils/mocked-binaries.ts'; @@ -59,56 +50,6 @@ test('rotateAndroid locks auto-rotate and sets user rotation', async () => { ); }); -test('swipeAndroid invokes adb input swipe with duration', async () => { - const tmpDir = await fs.mkdtemp(path.join(os.tmpdir(), 'agent-device-swipe-test-')); - const adbPath = path.join(tmpDir, 'adb'); - const argsLogPath = path.join(tmpDir, 'args.log'); - await fs.writeFile( - adbPath, - '#!/bin/sh\nprintf "%s\\n" "$@" > "$AGENT_DEVICE_TEST_ARGS_FILE"\nexit 0\n', - 'utf8', - ); - await fs.chmod(adbPath, 0o755); - - const previousPath = process.env.PATH; - const previousArgsFile = process.env.AGENT_DEVICE_TEST_ARGS_FILE; - process.env.PATH = `${tmpDir}${path.delimiter}${previousPath ?? ''}`; - process.env.AGENT_DEVICE_TEST_ARGS_FILE = argsLogPath; - - const device: DeviceInfo = { - platform: 'android', - id: 'emulator-5554', - name: 'Pixel', - kind: 'emulator', - booted: true, - }; - - try { - await swipeAndroid(device, 10, 20, 30, 40, 250); - const args = (await fs.readFile(argsLogPath, 'utf8')).trim().split('\n').filter(Boolean); - assert.deepEqual(args, [ - '-s', - 'emulator-5554', - 'shell', - 'input', - 'swipe', - '10', - '20', - '30', - '40', - '250', - ]); - } finally { - process.env.PATH = previousPath; - if (previousArgsFile === undefined) { - delete process.env.AGENT_DEVICE_TEST_ARGS_FILE; - } else { - process.env.AGENT_DEVICE_TEST_ARGS_FILE = previousArgsFile; - } - await fs.rm(tmpDir, { recursive: true, force: true }); - } -}); - test('typeAndroid chunks ASCII input text for shell fallback', async () => { await withScriptedAdb( 'agent-device-android-type-ascii-chunked-', diff --git a/src/platforms/android/__tests__/multitouch-helper-ownership.test.ts b/src/platforms/android/__tests__/multitouch-helper-ownership.test.ts index 1a08c77b8..f0b2db814 100644 --- a/src/platforms/android/__tests__/multitouch-helper-ownership.test.ts +++ b/src/platforms/android/__tests__/multitouch-helper-ownership.test.ts @@ -92,3 +92,60 @@ test('helper gesture releases persistent snapshot instrumentation before touch i assert.equal(result?.backend, 'android-multitouch-helper'); assert.deepEqual(events, ['snapshot-stop', 'touch-instrumentation']); }); + +test('helper gesture failures do not fall back to adb input swipe', async () => { + const adbCalls: string[][] = []; + await withAndroidAdbProvider( + { + exec: async (args) => { + adbCalls.push(args); + if (args.includes('--show-versioncode')) { + return { + exitCode: 0, + stdout: `package:${ANDROID_MULTITOUCH_HELPER_MANIFEST.packageName} versionCode:999999`, + stderr: '', + }; + } + if (args.includes('instrument')) { + return { + exitCode: 1, + stdout: [ + androidMultiTouchResultRecord({ + ok: 'false', + errorType: 'java.lang.IllegalStateException', + message: 'injectInputEvent returned false', + }), + 'INSTRUMENTATION_CODE: 1', + ].join('\n'), + stderr: '', + }; + } + return { exitCode: 0, stdout: '', stderr: '' }; + }, + }, + { serial: ANDROID_EMULATOR.id }, + async () => { + await assert.rejects( + () => + performGestureAndroid( + ANDROID_EMULATOR, + buildGesturePlan( + { + intent: 'fling', + from: { x: 340, y: 400 }, + to: { x: 60, y: 400 }, + }, + { x: 0, y: 0, width: 400, height: 800 }, + 'android', + ), + ), + { code: 'COMMAND_FAILED', message: 'injectInputEvent returned false' }, + ); + }, + ); + + assert.equal( + adbCalls.some((args) => args.includes('input') && args.includes('swipe')), + false, + ); +}); diff --git a/src/platforms/android/input-actions.ts b/src/platforms/android/input-actions.ts index fe80fd0c1..9287ad17b 100644 --- a/src/platforms/android/input-actions.ts +++ b/src/platforms/android/input-actions.ts @@ -40,26 +40,6 @@ export async function pressAndroidTvRemote( await runAndroidAdb(device, ['shell', 'input', ...keyeventArgs, keyevent]); } -export async function swipeAndroid( - device: DeviceInfo, - x1: number, - y1: number, - x2: number, - y2: number, - durationMs = 250, -): Promise { - await runAndroidAdb(device, [ - 'shell', - 'input', - 'swipe', - String(x1), - String(y1), - String(x2), - String(y2), - String(durationMs), - ]); -} - export async function backAndroid(device: DeviceInfo): Promise { await runAndroidAdb(device, ['shell', 'input', 'keyevent', '4']); } diff --git a/src/platforms/android/multitouch-helper.ts b/src/platforms/android/multitouch-helper.ts index 4479a92c5..a64495119 100644 --- a/src/platforms/android/multitouch-helper.ts +++ b/src/platforms/android/multitouch-helper.ts @@ -1,7 +1,7 @@ import type { GesturePlan, PointerTrajectory } from '../../contracts/gesture-plan.ts'; import type { DeviceInfo } from '../../kernel/device.ts'; import type { Rect } from '../../kernel/snapshot.ts'; -import { AppError, normalizeError } from '../../kernel/errors.ts'; +import { AppError } from '../../kernel/errors.ts'; import { execFailureDetails } from '../../utils/exec.ts'; import { emitDiagnostic, withDiagnosticTimer } from '../../utils/diagnostics.ts'; import { @@ -24,7 +24,6 @@ import { readInstrumentationResultNumber, } from './instrumentation-helper.ts'; import { stopAndroidSnapshotHelperSessionForDevice } from './snapshot-helper.ts'; -import { swipeAndroid } from './input-actions.ts'; const HELPER_NAME = 'android-multitouch-helper'; const HELPER_PACKAGE = 'com.callstack.agentdevice.multitouchhelper'; @@ -73,21 +72,7 @@ export async function performGestureAndroid( const result = (await providerTouch(plan)) ?? {}; return { backend: 'provider-native-touch', ...result }; } - try { - return await runAndroidMultiTouchHelperGestureForDevice(device, plan); - } catch (error) { - if (plan.topology === 'two') throw error; - emitDiagnostic({ - level: 'warn', - phase: 'android_swipe_helper_fallback', - data: { error: normalizeError(error).message }, - }); - const first = plan.pointers[0].samples[0]?.point; - const last = plan.pointers[0].samples.at(-1)?.point; - if (!first || !last) throw error; - await swipeAndroid(device, first.x, first.y, last.x, last.y, plan.durationMs); - return { backend: 'adb-input-swipe-fallback' }; - } + return await runAndroidMultiTouchHelperGestureForDevice(device, plan); } async function runAndroidMultiTouchHelperGestureForDevice( diff --git a/test/integration/provider-scenarios/android-lifecycle.test.ts b/test/integration/provider-scenarios/android-lifecycle.test.ts index 63202ac17..e4ebbffbf 100644 --- a/test/integration/provider-scenarios/android-lifecycle.test.ts +++ b/test/integration/provider-scenarios/android-lifecycle.test.ts @@ -1460,6 +1460,10 @@ function assertAndroidInteractionContract(world: AndroidSettingsWorld): void { ), JSON.stringify(adbCalls), ); + const helperGestureCalls = adbCalls.filter((call) => + call.includes('com.callstack.agentdevice.multitouchhelper/.MultiTouchInstrumentation'), + ); + assert.equal(helperGestureCalls.length, 4, JSON.stringify(adbCalls)); assertCommandCall(adbCalls, ['shell', 'input', 'tap', '88', '151']); assertCommandCall(adbCalls, [ 'shell', @@ -1478,10 +1482,6 @@ function assertAndroidInteractionContract(world: AndroidSettingsWorld): void { assertCommandCall(adbCalls, ['shell', 'input', 'tap', '50', '60']); assertCommandCall(adbCalls, ['shell', 'input', 'swipe', '30', '40', '30', '40', '5']); assertCommandCall(adbCalls, ['shell', 'input', 'swipe', '31', '40', '31', '40', '5']); - assertCommandCall(adbCalls, ['shell', 'input', 'swipe', '20', '200', '20', '100', '100']); - assertCommandCall(adbCalls, ['shell', 'input', 'swipe', '20', '100', '20', '200', '100']); - assertCommandCall(adbCalls, ['shell', 'input', 'swipe', '100', '200', '150', '180', '400']); - assertCommandCall(adbCalls, ['shell', 'input', 'swipe', '100', '200', '280', '200', '100']); assert.equal( adbCalls.filter((call) => arrayEqual(call, ['shell', 'input', 'tap', '88', '151'])).length, 5, diff --git a/test/integration/provider-scenarios/android-test-suite.test.ts b/test/integration/provider-scenarios/android-test-suite.test.ts index 02e38f0de..08a7ce4ef 100644 --- a/test/integration/provider-scenarios/android-test-suite.test.ts +++ b/test/integration/provider-scenarios/android-test-suite.test.ts @@ -118,9 +118,16 @@ test('Provider-backed integration Android Maestro replay uses fresh snapshots an world.adbCalls.find((call) => call.slice(0, 3).join(' ') === 'shell input tap'), ['shell', 'input', 'tap', '180', '330'], ); - assert.deepEqual( - world.adbCalls.find((call) => call.slice(0, 3).join(' ') === 'shell input swipe'), - ['shell', 'input', 'swipe', '351', '300', '39', '300', '300'], + assert.ok( + world.adbCalls.some((call) => + call.includes('com.callstack.agentdevice.multitouchhelper/.MultiTouchInstrumentation'), + ), + JSON.stringify(world.adbCalls), + ); + assert.equal( + world.adbCalls.some((call) => call.slice(0, 3).join(' ') === 'shell input swipe'), + false, + JSON.stringify(world.adbCalls), ); // Percentage resolution snapshots remain fresh; gesture planning uses the provider viewport. assertSnapshotCountInRange(snapshots, 3, 5); diff --git a/test/integration/provider-scenarios/android-world.ts b/test/integration/provider-scenarios/android-world.ts index 31cde866e..92ca916a0 100644 --- a/test/integration/provider-scenarios/android-world.ts +++ b/test/integration/provider-scenarios/android-world.ts @@ -441,6 +441,13 @@ function androidCaptureAdbResult( searchText: string, snapshotXml?: () => string, ): AndroidAdbResult | undefined { + if (key.includes('com.callstack.agentdevice.multitouchhelper/.MultiTouchInstrumentation')) { + return { + stdout: androidMultiTouchHelperOutput(), + stderr: '', + exitCode: 0, + }; + } if (key.startsWith('shell am instrument ')) { return { stdout: androidSnapshotHelperOutput(snapshotXml?.() ?? androidSettingsXml(searchText)), @@ -477,6 +484,18 @@ export function androidSnapshotHelperOutput(xml: string): string { ].join('\n'); } +function androidMultiTouchHelperOutput(): string { + return [ + 'INSTRUMENTATION_RESULT: agentDeviceProtocol=android-multitouch-helper-v1', + 'INSTRUMENTATION_RESULT: helperApiVersion=1', + 'INSTRUMENTATION_RESULT: kind=swipe', + 'INSTRUMENTATION_RESULT: ok=true', + 'INSTRUMENTATION_RESULT: injectedEvents=2', + 'INSTRUMENTATION_RESULT: elapsedMs=100', + 'INSTRUMENTATION_CODE: 0', + ].join('\n'); +} + export function androidSettingsXml( searchText: string, options: { duplicateAppsRow?: boolean } = {}, From 2b6434fd6473c45f1061d23bba9588f1a84d09be Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Micha=C5=82=20Pierzcha=C5=82a?= Date: Mon, 13 Jul 2026 14:40:49 +0200 Subject: [PATCH 02/16] fix: tighten Android gesture review follow-up --- docs/adr/0013-unified-gesture-plans.md | 7 ++++--- .../provider-scenarios/android-world.ts | 16 ++++++++++++---- 2 files changed, 16 insertions(+), 7 deletions(-) diff --git a/docs/adr/0013-unified-gesture-plans.md b/docs/adr/0013-unified-gesture-plans.md index 13ebb7b94..a49543cbc 100644 --- a/docs/adr/0013-unified-gesture-plans.md +++ b/docs/adr/0013-unified-gesture-plans.md @@ -55,9 +55,10 @@ no established automation use case justifies a public tuning surface. Platform adapters consume the canonical plan: - Android sends the plan to provider-native touch injection when available, otherwise to the - bundled instrumentation helper. The helper injects the exact planned pointer samples. Gesture - execution never falls back to `adb input swipe`; the snapshot helper is stopped before local - gesture instrumentation because Android permits only one instrumentation owner of `UiAutomation`. + bundled instrumentation helper. The helper injects the exact planned pointer samples. Canonical + gesture-plan execution never falls back to `adb input swipe`; the snapshot helper is stopped + before local gesture instrumentation because Android permits only one instrumentation owner of + `UiAutomation`. - iOS converts every planned point to native orientation and feeds the exact arrays to the existing private XCTest event bridge. macOS lowers a one-contact plan to its drag executor and tvOS lowers it to remote direction. Core admission and the Apple adapter both consume the same shared diff --git a/test/integration/provider-scenarios/android-world.ts b/test/integration/provider-scenarios/android-world.ts index 92ca916a0..f265d08de 100644 --- a/test/integration/provider-scenarios/android-world.ts +++ b/test/integration/provider-scenarios/android-world.ts @@ -235,7 +235,7 @@ function androidAdbResult( androidDeviceStateAdbResult(key, args, clipboardText, options.pidof) ?? androidMetricsAdbResult(key) ?? androidPackageAdbResult(key, args, options.dumpsysWindow) ?? - androidCaptureAdbResult(key, searchText, options.snapshotXml) ?? { + androidCaptureAdbResult(args, key, searchText, options.snapshotXml) ?? { stdout: '', stderr: '', exitCode: 0, @@ -437,13 +437,21 @@ function androidPackageAdbResult( } function androidCaptureAdbResult( + args: string[], key: string, searchText: string, snapshotXml?: () => string, ): AndroidAdbResult | undefined { if (key.includes('com.callstack.agentdevice.multitouchhelper/.MultiTouchInstrumentation')) { + const payloadIndex = args.indexOf('payloadBase64'); + const encodedPayload = payloadIndex >= 0 ? args[payloadIndex + 1] : undefined; + const request = encodedPayload + ? (JSON.parse(Buffer.from(encodedPayload, 'base64').toString('utf8')) as { + kind?: string; + }) + : undefined; return { - stdout: androidMultiTouchHelperOutput(), + stdout: androidMultiTouchHelperOutput(request?.kind === 'transform' ? 'transform' : 'swipe'), stderr: '', exitCode: 0, }; @@ -484,11 +492,11 @@ export function androidSnapshotHelperOutput(xml: string): string { ].join('\n'); } -function androidMultiTouchHelperOutput(): string { +function androidMultiTouchHelperOutput(kind: 'swipe' | 'transform'): string { return [ 'INSTRUMENTATION_RESULT: agentDeviceProtocol=android-multitouch-helper-v1', 'INSTRUMENTATION_RESULT: helperApiVersion=1', - 'INSTRUMENTATION_RESULT: kind=swipe', + `INSTRUMENTATION_RESULT: kind=${kind}`, 'INSTRUMENTATION_RESULT: ok=true', 'INSTRUMENTATION_RESULT: injectedEvents=2', 'INSTRUMENTATION_RESULT: elapsedMs=100', From bdb0ee4806861cd014124968035ef677c3aa5829 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Micha=C5=82=20Pierzcha=C5=82a?= Date: Mon, 13 Jul 2026 15:01:23 +0200 Subject: [PATCH 03/16] fix: route Android touch actions through gesture helper --- .../MultiTouchInstrumentation.java | 9 +- docs/adr/0013-unified-gesture-plans.md | 15 +-- .../android/__tests__/input-actions.test.ts | 97 +++++++++++++------ .../__tests__/multitouch-helper.test.ts | 12 +++ src/platforms/android/adb-executor.ts | 4 +- src/platforms/android/input-actions.ts | 58 +++++------ src/platforms/android/multitouch-helper.ts | 21 +++- src/platforms/android/touch-plan.ts | 54 +++++++++++ .../android-lifecycle.test.ts | 27 +++++- 9 files changed, 224 insertions(+), 73 deletions(-) create mode 100644 src/platforms/android/touch-plan.ts diff --git a/android-multitouch-helper/src/main/java/com/callstack/agentdevice/multitouchhelper/MultiTouchInstrumentation.java b/android-multitouch-helper/src/main/java/com/callstack/agentdevice/multitouchhelper/MultiTouchInstrumentation.java index 234f6c3aa..96be8ef13 100644 --- a/android-multitouch-helper/src/main/java/com/callstack/agentdevice/multitouchhelper/MultiTouchInstrumentation.java +++ b/android-multitouch-helper/src/main/java/com/callstack/agentdevice/multitouchhelper/MultiTouchInstrumentation.java @@ -19,8 +19,8 @@ public final class MultiTouchInstrumentation extends Instrumentation { private static final String PROTOCOL = "android-multitouch-helper-v1"; private static final String HELPER_API_VERSION = "1"; - private static final int MIN_DURATION_MS = 16; - private static final int MAX_DURATION_MS = 10_000; + private static final int MIN_DURATION_MS = 0; + private static final int MAX_DURATION_MS = 120_000; private Bundle arguments; @Override @@ -204,7 +204,8 @@ private static PointerPath[] readPointers(JSONArray pointers, String kind, int d throw new IllegalArgumentException("Planned sample coordinates must be finite"); } long offsetMs = (long) rawOffsetMs; - if (offsetMs <= previousOffsetMs) { + if (offsetMs < previousOffsetMs + || (offsetMs == previousOffsetMs && durationMs != 0)) { throw new IllegalArgumentException("Planned sample offsets must be strictly increasing"); } parsed[sampleIndex] = new PointerSample(offsetMs, (float) x, (float) y); @@ -307,7 +308,7 @@ private static MotionEvent motionEvent( private static int requireDuration(int value) { if (value < MIN_DURATION_MS || value > MAX_DURATION_MS) { - throw new IllegalArgumentException("durationMs must be between 16 and 10000"); + throw new IllegalArgumentException("durationMs must be between 0 and 120000"); } return value; } diff --git a/docs/adr/0013-unified-gesture-plans.md b/docs/adr/0013-unified-gesture-plans.md index a49543cbc..ff23b22e5 100644 --- a/docs/adr/0013-unified-gesture-plans.md +++ b/docs/adr/0013-unified-gesture-plans.md @@ -15,8 +15,9 @@ Two-contact geometry also differed by platform. Android used a fixed radius whil radius from the app frame. Neither path validated every planned point against the active interaction viewport before injection. -`scroll` remains separate because it owns viewport/edge traversal, content-state verification, and -runner fallback policy rather than a single physical gesture. +`scroll` remains a separate command because it owns viewport/edge traversal and content-state +verification rather than a single physical gesture. Its Android physical movement still lowers to +the shared planned-touch executor. ## Decision @@ -54,9 +55,10 @@ no established automation use case justifies a public tuning surface. Platform adapters consume the canonical plan: -- Android sends the plan to provider-native touch injection when available, otherwise to the - bundled instrumentation helper. The helper injects the exact planned pointer samples. Canonical - gesture-plan execution never falls back to `adb input swipe`; the snapshot helper is stopped +- Android sends planned touch, including gesture plans plus the physical movement for scroll and + long-press, to provider-native touch injection when available, otherwise to the bundled + instrumentation helper. The helper injects the exact planned pointer samples. Planned Android + touch execution never falls back to `adb input swipe`; the snapshot helper is stopped before local gesture instrumentation because Android permits only one instrumentation owner of `UiAutomation`. - iOS converts every planned point to native orientation and feeds the exact arrays to the existing @@ -106,4 +108,5 @@ selectors or refs and therefore cannot claim element-targeting guarantees. fling/swipe as a quick directional throw and pan as deliberate timed translation. - Add `two-finger-pan`: rejected because pointer count is topology, not a new motion intent. - Expose span/angle controls: rejected until a concrete automation use case needs them. -- Consolidate scroll: rejected because edge/content verification and fallback policy are distinct. +- Consolidate scroll command semantics: rejected because edge/content verification is distinct; + only its Android physical touch execution is shared. diff --git a/src/platforms/android/__tests__/input-actions.test.ts b/src/platforms/android/__tests__/input-actions.test.ts index 55d05c362..5485c20c9 100644 --- a/src/platforms/android/__tests__/input-actions.test.ts +++ b/src/platforms/android/__tests__/input-actions.test.ts @@ -1,39 +1,82 @@ import { test } from 'vitest'; import assert from 'node:assert/strict'; import { promises as fs } from 'node:fs'; -import { fillAndroid, rotateAndroid, scrollAndroid, typeAndroid } from '../input-actions.ts'; +import { + fillAndroid, + longPressAndroid, + rotateAndroid, + scrollAndroid, + typeAndroid, +} from '../input-actions.ts'; import { AppError } from '../../../kernel/errors.ts'; import { withScriptedAdb } from '../../../__tests__/test-utils/mocked-binaries.ts'; +import { ANDROID_EMULATOR } from '../../../__tests__/test-utils/index.ts'; +import { withAndroidAdbProvider, type AndroidTouchInjector } from '../adb-executor.ts'; -test('scrollAndroid supports explicit pixel travel distance', async () => { - await withScriptedAdb( - 'agent-device-android-scroll-pixels-', - [ - '#!/bin/sh', - 'printf "%s\\n" "$@" >> "$AGENT_DEVICE_TEST_ARGS_FILE"', - 'if [ "$1" = "-s" ]; then', - ' shift', - ' shift', - 'fi', - 'if [ "$1" = "shell" ] && [ "$2" = "wm" ] && [ "$3" = "size" ]; then', - ' echo "Physical size: 1080x1920"', - ' exit 0', - 'fi', - 'exit 0', - '', - ].join('\n'), - async ({ argsLogPath, device }) => { - const result = await scrollAndroid(device, 'down', { pixels: 240, durationMs: 120 }); - const args = await fs.readFile(argsLogPath, 'utf8'); +test('scrollAndroid plans explicit pixel travel through semantic touch injection', async () => { + const touchCalls: Parameters[0][] = []; + const result = await withAndroidAdbProvider( + { + exec: async () => { + throw new Error('adb must not run'); + }, + gestureViewport: async () => ({ x: 10, y: 20, width: 1080, height: 1920 }), + touch: async (request) => { + touchCalls.push(request); + return { injected: true }; + }, + }, + { serial: ANDROID_EMULATOR.id }, + async () => await scrollAndroid(ANDROID_EMULATOR, 'down', { pixels: 240, durationMs: 120 }), + ); - assert.match(args, /shell\ninput\nswipe\n540\n1080\n540\n840\n120\n/); - assert.doesNotMatch(args, /uiautomator|dump/); - assert.equal(result.pixels, 240); - assert.equal(result.durationMs, 120); - assert.equal(result.referenceWidth, 1080); - assert.equal(result.referenceHeight, 1920); + assert.equal(touchCalls.length, 1); + const touch = touchCalls[0]!; + assert.equal(touch.intent, 'pan'); + assert.deepEqual(touch.pointers[0]?.samples[0]?.point, { x: 550, y: 1100 }); + assert.deepEqual(touch.pointers[0]?.samples.at(-1)?.point, { x: 550, y: 860 }); + assert.equal(result.pixels, 240); + assert.equal(result.durationMs, 120); + assert.equal(result.referenceWidth, 1080); + assert.equal(result.referenceHeight, 1920); + assert.equal(result.backend, 'provider-native-touch'); + assert.equal(result.injected, true); +}); + +test('longPressAndroid sends a stationary semantic touch plan', async () => { + const touchCalls: Parameters[0][] = []; + const result = await withAndroidAdbProvider( + { + exec: async () => { + throw new Error('adb must not run'); + }, + gestureViewport: async () => ({ x: 10, y: 20, width: 300, height: 500 }), + touch: async (request) => { + touchCalls.push(request); + }, }, + { serial: ANDROID_EMULATOR.id }, + async () => await longPressAndroid(ANDROID_EMULATOR, 30, 40, 750), ); + + assert.deepEqual(touchCalls, [ + { + topology: 'single', + intent: 'longPress', + durationMs: 750, + viewport: { x: 10, y: 20, width: 300, height: 500 }, + pointers: [ + { + pointerId: 0, + samples: [ + { offsetMs: 0, point: { x: 30, y: 40 } }, + { offsetMs: 750, point: { x: 30, y: 40 } }, + ], + }, + ], + }, + ]); + assert.equal(result.backend, 'provider-native-touch'); }); test('rotateAndroid locks auto-rotate and sets user rotation', async () => { diff --git a/src/platforms/android/__tests__/multitouch-helper.test.ts b/src/platforms/android/__tests__/multitouch-helper.test.ts index 2389a933c..81ed94b37 100644 --- a/src/platforms/android/__tests__/multitouch-helper.test.ts +++ b/src/platforms/android/__tests__/multitouch-helper.test.ts @@ -13,6 +13,7 @@ import { } from '../multitouch-helper.ts'; import { createAndroidInteractor } from '../../../core/interactors/android.ts'; import { withAndroidAdbProvider } from '../adb-executor.ts'; +import { buildAndroidLongPressTouchPlan } from '../touch-plan.ts'; import { ANDROID_MULTITOUCH_HELPER_MANIFEST as manifest, androidMultiTouchResultRecord as resultRecord, @@ -88,6 +89,17 @@ test('single-pointer plans use the same exact helper protocol', () => { assert.equal(request.durationMs, 100); }); +test('Android long press lowers to a stationary single-pointer helper request', () => { + const hold = buildAndroidLongPressTouchPlan({ x: 20, y: 30 }, 120_000, viewport); + const request = normalizeAndroidMultiTouchHelperGestureRequest(hold); + assert.equal(request.kind, 'swipe'); + assert.equal(request.durationMs, 120_000); + assert.deepEqual(request.pointers[0]?.samples, [ + { offsetMs: 0, x: 20, y: 30 }, + { offsetMs: 120_000, x: 20, y: 30 }, + ]); +}); + test('transform sends the planner-owned choreography without regenerating geometry', async () => { const plan = buildGesturePlan( { diff --git a/src/platforms/android/adb-executor.ts b/src/platforms/android/adb-executor.ts index 852a24586..7a9ea268d 100644 --- a/src/platforms/android/adb-executor.ts +++ b/src/platforms/android/adb-executor.ts @@ -2,7 +2,7 @@ import { AsyncLocalStorage } from 'node:async_hooks'; import type { Readable, Writable } from 'node:stream'; import type { DeviceInfo } from '../../kernel/device.ts'; import type { Rect } from '../../kernel/snapshot.ts'; -import type { GesturePlan } from '../../contracts/gesture-plan.ts'; +import type { AndroidTouchPlan } from './touch-plan.ts'; import { coerceExecResult, execFailureDetails, @@ -126,7 +126,7 @@ export type AndroidTextInjectionRequest = { export type AndroidTextInjector = (request: AndroidTextInjectionRequest) => Promise; export type AndroidTouchInjector = ( - request: GesturePlan, + request: AndroidTouchPlan, ) => Promise | void>; export type AndroidGestureViewportProvider = () => Promise; diff --git a/src/platforms/android/input-actions.ts b/src/platforms/android/input-actions.ts index 9287ad17b..97a8e202f 100644 --- a/src/platforms/android/input-actions.ts +++ b/src/platforms/android/input-actions.ts @@ -2,6 +2,7 @@ import { AppError } from '../../kernel/errors.ts'; import type { DeviceInfo } from '../../kernel/device.ts'; import { emitDiagnostic } from '../../utils/diagnostics.ts'; import type { DeviceRotation } from '../../contracts/device-rotation.ts'; +import { buildGesturePlan } from '../../contracts/gesture-plan.ts'; import { buildScrollGesturePlan, type ScrollDirection } from '../../contracts/scroll-gesture.ts'; import { toAndroidTvRemoteKeyevent, type TvRemoteButton } from '../../contracts/tv-remote.ts'; import { runAndroidAdb, sleep } from './adb.ts'; @@ -18,6 +19,12 @@ import { type AndroidFillVerification, } from './fill-verification.ts'; import { isAndroidTestImeActive } from './ime-lifecycle.ts'; +import { + performAndroidTouchPlan, + performGestureAndroid, + readAndroidGestureViewport, +} from './multitouch-helper.ts'; +import { buildAndroidLongPressTouchPlan } from './touch-plan.ts'; import { clearAndroidImeHelperText, resolveAndroidImeHelperArtifact, @@ -84,17 +91,12 @@ export async function longPressAndroid( x: number, y: number, durationMs = 800, -): Promise { - await runAndroidAdb(device, [ - 'shell', - 'input', - 'swipe', - String(x), - String(y), - String(x), - String(y), - String(durationMs), - ]); +): Promise> { + const viewport = await readAndroidGestureViewport(device); + return await performAndroidTouchPlan( + device, + buildAndroidLongPressTouchPlan({ x, y }, durationMs, viewport), + ); } export async function typeAndroid(device: DeviceInfo, text: string, delayMs = 0): Promise { @@ -263,30 +265,32 @@ export async function scrollAndroid( direction: ScrollDirection, options?: { amount?: number; pixels?: number; durationMs?: number }, ): Promise> { - const size = await getAndroidScreenSize(device); - const plan = buildScrollGesturePlan({ + const viewport = await readAndroidGestureViewport(device); + const scrollPlan = buildScrollGesturePlan({ direction, amount: options?.amount, pixels: options?.pixels, - referenceWidth: size.width, - referenceHeight: size.height, + referenceWidth: viewport.width, + referenceHeight: viewport.height, }); const durationMs = options?.durationMs ?? 300; - - await runAndroidAdb(device, [ - 'shell', - 'input', - 'swipe', - String(plan.x1), - String(plan.y1), - String(plan.x2), - String(plan.y2), - String(durationMs), - ]); + const origin = { x: viewport.x + scrollPlan.x1, y: viewport.y + scrollPlan.y1 }; + const gesturePlan = buildGesturePlan( + { + intent: 'pan', + origin, + delta: { x: scrollPlan.x2 - scrollPlan.x1, y: scrollPlan.y2 - scrollPlan.y1 }, + durationMs, + }, + viewport, + 'android', + ); + const backend = await performGestureAndroid(device, gesturePlan); return { - ...plan, + ...scrollPlan, ...(options?.durationMs !== undefined ? { durationMs: options.durationMs } : {}), + ...backend, }; } diff --git a/src/platforms/android/multitouch-helper.ts b/src/platforms/android/multitouch-helper.ts index a64495119..a4f1f9974 100644 --- a/src/platforms/android/multitouch-helper.ts +++ b/src/platforms/android/multitouch-helper.ts @@ -24,6 +24,7 @@ import { readInstrumentationResultNumber, } from './instrumentation-helper.ts'; import { stopAndroidSnapshotHelperSessionForDevice } from './snapshot-helper.ts'; +import type { AndroidTouchPlan } from './touch-plan.ts'; const HELPER_NAME = 'android-multitouch-helper'; const HELPER_PACKAGE = 'com.callstack.agentdevice.multitouchhelper'; @@ -31,6 +32,7 @@ const HELPER_RUNNER = 'com.callstack.agentdevice.multitouchhelper/.MultiTouchIns const HELPER_PROTOCOL = 'android-multitouch-helper-v1'; const HELPER_INSTALL_TIMEOUT_MS = 30_000; const HELPER_GESTURE_TIMEOUT_MS = 45_000; +const HELPER_GESTURE_TIMEOUT_OVERHEAD_MS = 15_000; const HELPER_NO_FINAL_RESULT = 'ANDROID_MULTITOUCH_HELPER_NO_FINAL_RESULT'; const HELPER_REPORTED_FAILURE = 'ANDROID_MULTITOUCH_HELPER_REPORTED_FAILURE'; const HELPER_LABEL = 'Android multi-touch helper'; @@ -66,6 +68,13 @@ type AndroidMultiTouchHelperGestureRequest = { export async function performGestureAndroid( device: DeviceInfo, plan: GesturePlan, +): Promise> { + return await performAndroidTouchPlan(device, plan); +} + +export async function performAndroidTouchPlan( + device: DeviceInfo, + plan: AndroidTouchPlan, ): Promise> { const providerTouch = resolveAndroidTouchInjector(device); if (providerTouch) { @@ -77,7 +86,7 @@ export async function performGestureAndroid( async function runAndroidMultiTouchHelperGestureForDevice( device: DeviceInfo, - plan: GesturePlan, + plan: AndroidTouchPlan, ): Promise> { const { adb, artifact, install } = await prepareAndroidMultiTouchHelper(device); const output = await withDiagnosticTimer( @@ -178,7 +187,7 @@ function validateAndroidGestureViewport(viewport: Rect): Rect { } export function normalizeAndroidMultiTouchHelperGestureRequest( - plan: GesturePlan, + plan: AndroidTouchPlan, ): AndroidMultiTouchHelperGestureRequest { return { kind: plan.topology === 'single' ? 'swipe' : 'transform', @@ -220,7 +229,13 @@ export async function runAndroidMultiTouchHelperGesture(options: { payloadBase64, options.instrumentationRunner, ], - { allowFailure: true, timeoutMs: HELPER_GESTURE_TIMEOUT_MS }, + { + allowFailure: true, + timeoutMs: Math.max( + HELPER_GESTURE_TIMEOUT_MS, + options.request.durationMs + HELPER_GESTURE_TIMEOUT_OVERHEAD_MS, + ), + }, ); let output: Record; try { diff --git a/src/platforms/android/touch-plan.ts b/src/platforms/android/touch-plan.ts new file mode 100644 index 000000000..63b9fa0e2 --- /dev/null +++ b/src/platforms/android/touch-plan.ts @@ -0,0 +1,54 @@ +import type { GesturePlan, PointerTrajectory } from '../../contracts/gesture-plan.ts'; +import { AppError } from '../../kernel/errors.ts'; +import type { Point, Rect } from '../../kernel/snapshot.ts'; + +export type AndroidLongPressTouchPlan = { + topology: 'single'; + intent: 'longPress'; + durationMs: number; + viewport: Rect; + pointers: readonly [PointerTrajectory]; +}; + +export type AndroidTouchPlan = GesturePlan | AndroidLongPressTouchPlan; + +export function buildAndroidLongPressTouchPlan( + point: Point, + durationMs: number, + viewport: Rect, +): AndroidLongPressTouchPlan { + assertPointInViewport(point, viewport); + if (!Number.isInteger(durationMs) || durationMs < 0 || durationMs > 120_000) { + throw new AppError('INVALID_ARGS', 'longPress durationMs must be between 0 and 120000'); + } + return { + topology: 'single', + intent: 'longPress', + durationMs, + viewport, + pointers: [ + { + pointerId: 0, + samples: [ + { offsetMs: 0, point }, + { offsetMs: durationMs, point }, + ], + }, + ], + }; +} + +function assertPointInViewport(point: Point, viewport: Rect): void { + const finite = Number.isFinite(point.x) && Number.isFinite(point.y); + const contained = + point.x >= viewport.x && + point.y >= viewport.y && + point.x < viewport.x + viewport.width && + point.y < viewport.y + viewport.height; + if (!finite || !contained) { + throw new AppError('GESTURE_TRAJECTORY_OUT_OF_BOUNDS', 'longPress point is outside viewport', { + point, + viewport, + }); + } +} diff --git a/test/integration/provider-scenarios/android-lifecycle.test.ts b/test/integration/provider-scenarios/android-lifecycle.test.ts index e4ebbffbf..8df1c79ac 100644 --- a/test/integration/provider-scenarios/android-lifecycle.test.ts +++ b/test/integration/provider-scenarios/android-lifecycle.test.ts @@ -85,6 +85,20 @@ test('Provider-backed integration Android touch provider handles multi-touch ges const client = world.daemon.client(); await client.apps.open({ app: 'settings', ...world.selection }); + await client.interactions.longPress({ + x: 195, + y: 320, + durationMs: 750, + ...world.selection, + }); + + await client.interactions.scroll({ + direction: 'down', + pixels: 120, + durationMs: 350, + ...world.selection, + }); + await client.interactions.swipe({ from: { x: 340, y: 400 }, to: { x: 60, y: 400 }, @@ -157,6 +171,8 @@ test('Provider-backed integration Android touch provider handles multi-touch ges durationMs: plan.durationMs, })); assert.deepEqual(touchCalls, [ + { topology: 'single', intent: 'longPress', pointerCount: 1, durationMs: 750 }, + { topology: 'single', intent: 'pan', pointerCount: 1, durationMs: 350 }, { topology: 'single', intent: 'pan', pointerCount: 1, durationMs: 300 }, { topology: 'single', intent: 'pan', pointerCount: 1, durationMs: 500 }, { topology: 'two', intent: 'pan', pointerCount: 2, durationMs: 500 }, @@ -164,7 +180,7 @@ test('Provider-backed integration Android touch provider handles multi-touch ges { topology: 'two', intent: 'rotate', pointerCount: 2, durationMs: 784 }, { topology: 'two', intent: 'transform', pointerCount: 2, durationMs: 700 }, ]); - assert.equal(world.gestureViewportCalls, 6); + assert.equal(world.gestureViewportCalls, 8); assert.equal( world.adbCalls.some( (call) => @@ -1463,7 +1479,7 @@ function assertAndroidInteractionContract(world: AndroidSettingsWorld): void { const helperGestureCalls = adbCalls.filter((call) => call.includes('com.callstack.agentdevice.multitouchhelper/.MultiTouchInstrumentation'), ); - assert.equal(helperGestureCalls.length, 4, JSON.stringify(adbCalls)); + assert.equal(helperGestureCalls.length, 6, JSON.stringify(adbCalls)); assertCommandCall(adbCalls, ['shell', 'input', 'tap', '88', '151']); assertCommandCall(adbCalls, [ 'shell', @@ -1480,8 +1496,11 @@ function assertAndroidInteractionContract(world: AndroidSettingsWorld): void { 2, ); assertCommandCall(adbCalls, ['shell', 'input', 'tap', '50', '60']); - assertCommandCall(adbCalls, ['shell', 'input', 'swipe', '30', '40', '30', '40', '5']); - assertCommandCall(adbCalls, ['shell', 'input', 'swipe', '31', '40', '31', '40', '5']); + assert.equal( + adbCalls.some((call) => call.slice(0, 3).join(' ') === 'shell input swipe'), + false, + JSON.stringify(adbCalls), + ); assert.equal( adbCalls.filter((call) => arrayEqual(call, ['shell', 'input', 'tap', '88', '151'])).length, 5, From 3b56e4d17fcd3f351c3712b5e418ce88dc8980b4 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Micha=C5=82=20Pierzcha=C5=82a?= Date: Mon, 13 Jul 2026 15:03:35 +0200 Subject: [PATCH 04/16] test: isolate Android touch provider fixture --- .../provider-scenarios/android-world.ts | 38 +++++++++++-------- 1 file changed, 22 insertions(+), 16 deletions(-) diff --git a/test/integration/provider-scenarios/android-world.ts b/test/integration/provider-scenarios/android-world.ts index f265d08de..cb86ec8f7 100644 --- a/test/integration/provider-scenarios/android-world.ts +++ b/test/integration/provider-scenarios/android-world.ts @@ -235,7 +235,8 @@ function androidAdbResult( androidDeviceStateAdbResult(key, args, clipboardText, options.pidof) ?? androidMetricsAdbResult(key) ?? androidPackageAdbResult(key, args, options.dumpsysWindow) ?? - androidCaptureAdbResult(args, key, searchText, options.snapshotXml) ?? { + androidMultiTouchHelperAdbResult(args, key) ?? + androidCaptureAdbResult(key, searchText, options.snapshotXml) ?? { stdout: '', stderr: '', exitCode: 0, @@ -436,26 +437,31 @@ function androidPackageAdbResult( return undefined; } -function androidCaptureAdbResult( +function androidMultiTouchHelperAdbResult( args: string[], key: string, +): AndroidAdbResult | undefined { + if (!key.includes('com.callstack.agentdevice.multitouchhelper/.MultiTouchInstrumentation')) + return undefined; + const payloadIndex = args.indexOf('payloadBase64'); + const encodedPayload = payloadIndex >= 0 ? args[payloadIndex + 1] : undefined; + const request = encodedPayload + ? (JSON.parse(Buffer.from(encodedPayload, 'base64').toString('utf8')) as { + kind?: string; + }) + : undefined; + return { + stdout: androidMultiTouchHelperOutput(request?.kind === 'transform' ? 'transform' : 'swipe'), + stderr: '', + exitCode: 0, + }; +} + +function androidCaptureAdbResult( + key: string, searchText: string, snapshotXml?: () => string, ): AndroidAdbResult | undefined { - if (key.includes('com.callstack.agentdevice.multitouchhelper/.MultiTouchInstrumentation')) { - const payloadIndex = args.indexOf('payloadBase64'); - const encodedPayload = payloadIndex >= 0 ? args[payloadIndex + 1] : undefined; - const request = encodedPayload - ? (JSON.parse(Buffer.from(encodedPayload, 'base64').toString('utf8')) as { - kind?: string; - }) - : undefined; - return { - stdout: androidMultiTouchHelperOutput(request?.kind === 'transform' ? 'transform' : 'swipe'), - stderr: '', - exitCode: 0, - }; - } if (key.startsWith('shell am instrument ')) { return { stdout: androidSnapshotHelperOutput(snapshotXml?.() ?? androidSettingsXml(searchText)), From 1a4860deb2f742886098e35918ddb4192b56fc23 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Micha=C5=82=20Pierzcha=C5=82a?= Date: Mon, 13 Jul 2026 15:09:23 +0200 Subject: [PATCH 05/16] test: drop Android swipe fallback assertions --- .../multitouch-helper-ownership.test.ts | 57 ------------------- .../__tests__/multitouch-helper.test.ts | 27 --------- .../android-lifecycle.test.ts | 5 -- .../android-test-suite.test.ts | 5 -- 4 files changed, 94 deletions(-) diff --git a/src/platforms/android/__tests__/multitouch-helper-ownership.test.ts b/src/platforms/android/__tests__/multitouch-helper-ownership.test.ts index f0b2db814..1a08c77b8 100644 --- a/src/platforms/android/__tests__/multitouch-helper-ownership.test.ts +++ b/src/platforms/android/__tests__/multitouch-helper-ownership.test.ts @@ -92,60 +92,3 @@ test('helper gesture releases persistent snapshot instrumentation before touch i assert.equal(result?.backend, 'android-multitouch-helper'); assert.deepEqual(events, ['snapshot-stop', 'touch-instrumentation']); }); - -test('helper gesture failures do not fall back to adb input swipe', async () => { - const adbCalls: string[][] = []; - await withAndroidAdbProvider( - { - exec: async (args) => { - adbCalls.push(args); - if (args.includes('--show-versioncode')) { - return { - exitCode: 0, - stdout: `package:${ANDROID_MULTITOUCH_HELPER_MANIFEST.packageName} versionCode:999999`, - stderr: '', - }; - } - if (args.includes('instrument')) { - return { - exitCode: 1, - stdout: [ - androidMultiTouchResultRecord({ - ok: 'false', - errorType: 'java.lang.IllegalStateException', - message: 'injectInputEvent returned false', - }), - 'INSTRUMENTATION_CODE: 1', - ].join('\n'), - stderr: '', - }; - } - return { exitCode: 0, stdout: '', stderr: '' }; - }, - }, - { serial: ANDROID_EMULATOR.id }, - async () => { - await assert.rejects( - () => - performGestureAndroid( - ANDROID_EMULATOR, - buildGesturePlan( - { - intent: 'fling', - from: { x: 340, y: 400 }, - to: { x: 60, y: 400 }, - }, - { x: 0, y: 0, width: 400, height: 800 }, - 'android', - ), - ), - { code: 'COMMAND_FAILED', message: 'injectInputEvent returned false' }, - ); - }, - ); - - assert.equal( - adbCalls.some((args) => args.includes('input') && args.includes('swipe')), - false, - ); -}); diff --git a/src/platforms/android/__tests__/multitouch-helper.test.ts b/src/platforms/android/__tests__/multitouch-helper.test.ts index 81ed94b37..474c9e851 100644 --- a/src/platforms/android/__tests__/multitouch-helper.test.ts +++ b/src/platforms/android/__tests__/multitouch-helper.test.ts @@ -168,33 +168,6 @@ test('provider-native touch receives the plan as its only source of truth', asyn assert.deepEqual(result, { backend: 'provider-native-touch', injected: true }); }); -test('provider-native failures propagate without adb single-pointer fallback', async () => { - const plan = buildGesturePlan( - { intent: 'fling', from: { x: 300, y: 400 }, to: { x: 100, y: 400 } }, - viewport, - ); - const adbCalls: string[][] = []; - await withAndroidAdbProvider( - { - exec: async (args) => { - adbCalls.push(args); - return { exitCode: 0, stdout: '', stderr: '' }; - }, - touch: async () => { - throw new Error('native touch failed'); - }, - }, - { serial: ANDROID_EMULATOR.id }, - async () => { - await assert.rejects( - () => performGestureAndroid(ANDROID_EMULATOR, plan), - /native touch failed/, - ); - }, - ); - assert.deepEqual(adbCalls, []); -}); - test('helper failures remain structured and actionable', async () => { const request = normalizeAndroidMultiTouchHelperGestureRequest( buildGesturePlan({ intent: 'pinch', scale: 1.5 }, viewport), diff --git a/test/integration/provider-scenarios/android-lifecycle.test.ts b/test/integration/provider-scenarios/android-lifecycle.test.ts index 8df1c79ac..86b0c34d8 100644 --- a/test/integration/provider-scenarios/android-lifecycle.test.ts +++ b/test/integration/provider-scenarios/android-lifecycle.test.ts @@ -1496,11 +1496,6 @@ function assertAndroidInteractionContract(world: AndroidSettingsWorld): void { 2, ); assertCommandCall(adbCalls, ['shell', 'input', 'tap', '50', '60']); - assert.equal( - adbCalls.some((call) => call.slice(0, 3).join(' ') === 'shell input swipe'), - false, - JSON.stringify(adbCalls), - ); assert.equal( adbCalls.filter((call) => arrayEqual(call, ['shell', 'input', 'tap', '88', '151'])).length, 5, diff --git a/test/integration/provider-scenarios/android-test-suite.test.ts b/test/integration/provider-scenarios/android-test-suite.test.ts index 08a7ce4ef..b25adf60a 100644 --- a/test/integration/provider-scenarios/android-test-suite.test.ts +++ b/test/integration/provider-scenarios/android-test-suite.test.ts @@ -124,11 +124,6 @@ test('Provider-backed integration Android Maestro replay uses fresh snapshots an ), JSON.stringify(world.adbCalls), ); - assert.equal( - world.adbCalls.some((call) => call.slice(0, 3).join(' ') === 'shell input swipe'), - false, - JSON.stringify(world.adbCalls), - ); // Percentage resolution snapshots remain fresh; gesture planning uses the provider viewport. assertSnapshotCountInRange(snapshots, 3, 5); }, From 54e68c4add0e234ab5cc1a59ceffb4222e38dfce Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Micha=C5=82=20Pierzcha=C5=82a?= Date: Mon, 13 Jul 2026 15:45:34 +0200 Subject: [PATCH 06/16] test: provide semantic Android touch in provider scenarios --- .../android-lifecycle.test.ts | 29 ++++++------ .../android-test-suite.test.ts | 12 ++--- .../provider-scenarios/android-world.ts | 44 ++----------------- 3 files changed, 24 insertions(+), 61 deletions(-) diff --git a/test/integration/provider-scenarios/android-lifecycle.test.ts b/test/integration/provider-scenarios/android-lifecycle.test.ts index 86b0c34d8..4b4a95e96 100644 --- a/test/integration/provider-scenarios/android-lifecycle.test.ts +++ b/test/integration/provider-scenarios/android-lifecycle.test.ts @@ -80,7 +80,7 @@ test('Provider-backed integration Android text provider handles Unicode without test('Provider-backed integration Android touch provider handles multi-touch gestures', async () => { await withProviderScenarioResource( - async () => await createAndroidSettingsWorld({ nativeTouchInjection: true }), + async () => await createAndroidSettingsWorld(), async (world) => { const client = world.daemon.client(); await client.apps.open({ app: 'settings', ...world.selection }); @@ -181,17 +181,6 @@ test('Provider-backed integration Android touch provider handles multi-touch ges { topology: 'two', intent: 'transform', pointerCount: 2, durationMs: 700 }, ]); assert.equal(world.gestureViewportCalls, 8); - assert.equal( - world.adbCalls.some( - (call) => - call[0] === 'shell' && - call[1] === 'am' && - call[2] === 'instrument' && - call.includes('com.callstack.agentdevice.multitouchhelper/.MultiTouchInstrumentation'), - ), - false, - JSON.stringify(world.adbCalls), - ); }, ); }); @@ -1476,10 +1465,20 @@ function assertAndroidInteractionContract(world: AndroidSettingsWorld): void { ), JSON.stringify(adbCalls), ); - const helperGestureCalls = adbCalls.filter((call) => - call.includes('com.callstack.agentdevice.multitouchhelper/.MultiTouchInstrumentation'), + assert.deepEqual( + world.touchInjectionCalls.map((plan) => ({ + intent: plan.intent, + durationMs: plan.durationMs, + })), + [ + { intent: 'longPress', durationMs: 5 }, + { intent: 'longPress', durationMs: 5 }, + { intent: 'fling', durationMs: 100 }, + { intent: 'fling', durationMs: 100 }, + { intent: 'pan', durationMs: 400 }, + { intent: 'fling', durationMs: 100 }, + ], ); - assert.equal(helperGestureCalls.length, 6, JSON.stringify(adbCalls)); assertCommandCall(adbCalls, ['shell', 'input', 'tap', '88', '151']); assertCommandCall(adbCalls, [ 'shell', diff --git a/test/integration/provider-scenarios/android-test-suite.test.ts b/test/integration/provider-scenarios/android-test-suite.test.ts index b25adf60a..f12fa9cfa 100644 --- a/test/integration/provider-scenarios/android-test-suite.test.ts +++ b/test/integration/provider-scenarios/android-test-suite.test.ts @@ -118,12 +118,12 @@ test('Provider-backed integration Android Maestro replay uses fresh snapshots an world.adbCalls.find((call) => call.slice(0, 3).join(' ') === 'shell input tap'), ['shell', 'input', 'tap', '180', '330'], ); - assert.ok( - world.adbCalls.some((call) => - call.includes('com.callstack.agentdevice.multitouchhelper/.MultiTouchInstrumentation'), - ), - JSON.stringify(world.adbCalls), - ); + assert.equal(world.touchInjectionCalls.length, 1); + const swipePlan = world.touchInjectionCalls[0]!; + assert.equal(swipePlan.intent, 'pan'); + assert.equal(swipePlan.durationMs, 300); + assert.deepEqual(swipePlan.pointers[0]?.samples[0]?.point, { x: 351, y: 300 }); + assert.deepEqual(swipePlan.pointers[0]?.samples.at(-1)?.point, { x: 39, y: 300 }); // Percentage resolution snapshots remain fresh; gesture planning uses the provider viewport. assertSnapshotCountInRange(snapshots, 3, 5); }, diff --git a/test/integration/provider-scenarios/android-world.ts b/test/integration/provider-scenarios/android-world.ts index cb86ec8f7..d605da3c0 100644 --- a/test/integration/provider-scenarios/android-world.ts +++ b/test/integration/provider-scenarios/android-world.ts @@ -48,7 +48,6 @@ type AndroidSettingsWorld = { export async function createAndroidSettingsWorld(options?: { nativeTextInjection?: boolean; - nativeTouchInjection?: boolean; snapshotXml?: () => string; dumpsysWindow?: () => string; onAdbExec?: (args: string[]) => void; @@ -98,6 +97,10 @@ export async function createAndroidSettingsWorld(options?: { pidof: (packageName) => androidPidofResult(appState, packageName), }); }, + touch: async (request) => { + touchInjectionCalls.push({ ...request }); + return { backend: 'provider-native-touch' }; + }, install: async (apk, options) => { apkInstallCalls.push({ apkPath: apk, replace: options?.replace }); return { stdout: '', stderr: '', exitCode: 0 }; @@ -142,12 +145,6 @@ export async function createAndroidSettingsWorld(options?: { shellState.searchText = request.text; }; } - if (options?.nativeTouchInjection) { - adbProvider.touch = async (request) => { - touchInjectionCalls.push({ ...request }); - return { backend: 'provider-native-touch' }; - }; - } const daemon = await createProviderScenarioHarness({ androidAdbProvider: () => adbProvider, deviceInventoryProvider: async (request) => { @@ -235,7 +232,6 @@ function androidAdbResult( androidDeviceStateAdbResult(key, args, clipboardText, options.pidof) ?? androidMetricsAdbResult(key) ?? androidPackageAdbResult(key, args, options.dumpsysWindow) ?? - androidMultiTouchHelperAdbResult(args, key) ?? androidCaptureAdbResult(key, searchText, options.snapshotXml) ?? { stdout: '', stderr: '', @@ -437,26 +433,6 @@ function androidPackageAdbResult( return undefined; } -function androidMultiTouchHelperAdbResult( - args: string[], - key: string, -): AndroidAdbResult | undefined { - if (!key.includes('com.callstack.agentdevice.multitouchhelper/.MultiTouchInstrumentation')) - return undefined; - const payloadIndex = args.indexOf('payloadBase64'); - const encodedPayload = payloadIndex >= 0 ? args[payloadIndex + 1] : undefined; - const request = encodedPayload - ? (JSON.parse(Buffer.from(encodedPayload, 'base64').toString('utf8')) as { - kind?: string; - }) - : undefined; - return { - stdout: androidMultiTouchHelperOutput(request?.kind === 'transform' ? 'transform' : 'swipe'), - stderr: '', - exitCode: 0, - }; -} - function androidCaptureAdbResult( key: string, searchText: string, @@ -498,18 +474,6 @@ export function androidSnapshotHelperOutput(xml: string): string { ].join('\n'); } -function androidMultiTouchHelperOutput(kind: 'swipe' | 'transform'): string { - return [ - 'INSTRUMENTATION_RESULT: agentDeviceProtocol=android-multitouch-helper-v1', - 'INSTRUMENTATION_RESULT: helperApiVersion=1', - `INSTRUMENTATION_RESULT: kind=${kind}`, - 'INSTRUMENTATION_RESULT: ok=true', - 'INSTRUMENTATION_RESULT: injectedEvents=2', - 'INSTRUMENTATION_RESULT: elapsedMs=100', - 'INSTRUMENTATION_CODE: 0', - ].join('\n'); -} - export function androidSettingsXml( searchText: string, options: { duplicateAppsRow?: boolean } = {}, From e3ef6d9ae03e3a8d3bdd91bf354f4690c59f3df0 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Micha=C5=82=20Pierzcha=C5=82a?= Date: Mon, 13 Jul 2026 16:13:07 +0200 Subject: [PATCH 07/16] refactor: drop redundant Android touch planning code --- .../__tests__/multitouch-helper.test.ts | 19 ++++++-- src/platforms/android/input-actions.ts | 27 +++++++----- src/platforms/android/multitouch-helper.ts | 9 +--- src/platforms/android/touch-plan.ts | 44 +------------------ 4 files changed, 35 insertions(+), 64 deletions(-) diff --git a/src/platforms/android/__tests__/multitouch-helper.test.ts b/src/platforms/android/__tests__/multitouch-helper.test.ts index 474c9e851..b9a796d3f 100644 --- a/src/platforms/android/__tests__/multitouch-helper.test.ts +++ b/src/platforms/android/__tests__/multitouch-helper.test.ts @@ -13,7 +13,6 @@ import { } from '../multitouch-helper.ts'; import { createAndroidInteractor } from '../../../core/interactors/android.ts'; import { withAndroidAdbProvider } from '../adb-executor.ts'; -import { buildAndroidLongPressTouchPlan } from '../touch-plan.ts'; import { ANDROID_MULTITOUCH_HELPER_MANIFEST as manifest, androidMultiTouchResultRecord as resultRecord, @@ -90,8 +89,22 @@ test('single-pointer plans use the same exact helper protocol', () => { }); test('Android long press lowers to a stationary single-pointer helper request', () => { - const hold = buildAndroidLongPressTouchPlan({ x: 20, y: 30 }, 120_000, viewport); - const request = normalizeAndroidMultiTouchHelperGestureRequest(hold); + const point = { x: 20, y: 30 }; + const request = normalizeAndroidMultiTouchHelperGestureRequest({ + topology: 'single', + intent: 'longPress', + durationMs: 120_000, + viewport, + pointers: [ + { + pointerId: 0, + samples: [ + { offsetMs: 0, point }, + { offsetMs: 120_000, point }, + ], + }, + ], + }); assert.equal(request.kind, 'swipe'); assert.equal(request.durationMs, 120_000); assert.deepEqual(request.pointers[0]?.samples, [ diff --git a/src/platforms/android/input-actions.ts b/src/platforms/android/input-actions.ts index 97a8e202f..359b85d3a 100644 --- a/src/platforms/android/input-actions.ts +++ b/src/platforms/android/input-actions.ts @@ -19,12 +19,7 @@ import { type AndroidFillVerification, } from './fill-verification.ts'; import { isAndroidTestImeActive } from './ime-lifecycle.ts'; -import { - performAndroidTouchPlan, - performGestureAndroid, - readAndroidGestureViewport, -} from './multitouch-helper.ts'; -import { buildAndroidLongPressTouchPlan } from './touch-plan.ts'; +import { performGestureAndroid, readAndroidGestureViewport } from './multitouch-helper.ts'; import { clearAndroidImeHelperText, resolveAndroidImeHelperArtifact, @@ -93,10 +88,22 @@ export async function longPressAndroid( durationMs = 800, ): Promise> { const viewport = await readAndroidGestureViewport(device); - return await performAndroidTouchPlan( - device, - buildAndroidLongPressTouchPlan({ x, y }, durationMs, viewport), - ); + const point = { x, y }; + return await performGestureAndroid(device, { + topology: 'single', + intent: 'longPress', + durationMs, + viewport, + pointers: [ + { + pointerId: 0, + samples: [ + { offsetMs: 0, point }, + { offsetMs: durationMs, point }, + ], + }, + ], + }); } export async function typeAndroid(device: DeviceInfo, text: string, delayMs = 0): Promise { diff --git a/src/platforms/android/multitouch-helper.ts b/src/platforms/android/multitouch-helper.ts index a4f1f9974..2b7e5dfc8 100644 --- a/src/platforms/android/multitouch-helper.ts +++ b/src/platforms/android/multitouch-helper.ts @@ -1,4 +1,4 @@ -import type { GesturePlan, PointerTrajectory } from '../../contracts/gesture-plan.ts'; +import type { PointerTrajectory } from '../../contracts/gesture-plan.ts'; import type { DeviceInfo } from '../../kernel/device.ts'; import type { Rect } from '../../kernel/snapshot.ts'; import { AppError } from '../../kernel/errors.ts'; @@ -66,13 +66,6 @@ type AndroidMultiTouchHelperGestureRequest = { }; export async function performGestureAndroid( - device: DeviceInfo, - plan: GesturePlan, -): Promise> { - return await performAndroidTouchPlan(device, plan); -} - -export async function performAndroidTouchPlan( device: DeviceInfo, plan: AndroidTouchPlan, ): Promise> { diff --git a/src/platforms/android/touch-plan.ts b/src/platforms/android/touch-plan.ts index 63b9fa0e2..7e3d9fd34 100644 --- a/src/platforms/android/touch-plan.ts +++ b/src/platforms/android/touch-plan.ts @@ -1,6 +1,5 @@ import type { GesturePlan, PointerTrajectory } from '../../contracts/gesture-plan.ts'; -import { AppError } from '../../kernel/errors.ts'; -import type { Point, Rect } from '../../kernel/snapshot.ts'; +import type { Rect } from '../../kernel/snapshot.ts'; export type AndroidLongPressTouchPlan = { topology: 'single'; @@ -11,44 +10,3 @@ export type AndroidLongPressTouchPlan = { }; export type AndroidTouchPlan = GesturePlan | AndroidLongPressTouchPlan; - -export function buildAndroidLongPressTouchPlan( - point: Point, - durationMs: number, - viewport: Rect, -): AndroidLongPressTouchPlan { - assertPointInViewport(point, viewport); - if (!Number.isInteger(durationMs) || durationMs < 0 || durationMs > 120_000) { - throw new AppError('INVALID_ARGS', 'longPress durationMs must be between 0 and 120000'); - } - return { - topology: 'single', - intent: 'longPress', - durationMs, - viewport, - pointers: [ - { - pointerId: 0, - samples: [ - { offsetMs: 0, point }, - { offsetMs: durationMs, point }, - ], - }, - ], - }; -} - -function assertPointInViewport(point: Point, viewport: Rect): void { - const finite = Number.isFinite(point.x) && Number.isFinite(point.y); - const contained = - point.x >= viewport.x && - point.y >= viewport.y && - point.x < viewport.x + viewport.width && - point.y < viewport.y + viewport.height; - if (!finite || !contained) { - throw new AppError('GESTURE_TRAJECTORY_OUT_OF_BOUNDS', 'longPress point is outside viewport', { - point, - viewport, - }); - } -} From 8b398dc16f4904ca77adf20c782ae9233fa46635 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Micha=C5=82=20Pierzcha=C5=82a?= Date: Mon, 13 Jul 2026 16:15:09 +0200 Subject: [PATCH 08/16] fix: require viewport for Android touch providers --- .../android/__tests__/multitouch-helper.test.ts | 1 + src/platforms/android/adb-executor.ts | 16 +++++++++++++--- 2 files changed, 14 insertions(+), 3 deletions(-) diff --git a/src/platforms/android/__tests__/multitouch-helper.test.ts b/src/platforms/android/__tests__/multitouch-helper.test.ts index b9a796d3f..37d354e9b 100644 --- a/src/platforms/android/__tests__/multitouch-helper.test.ts +++ b/src/platforms/android/__tests__/multitouch-helper.test.ts @@ -169,6 +169,7 @@ test('provider-native touch receives the plan as its only source of truth', asyn exec: async () => { throw new Error('adb must not run'); }, + gestureViewport: async () => viewport, touch: async (request) => { calls.push(request); return { injected: true }; diff --git a/src/platforms/android/adb-executor.ts b/src/platforms/android/adb-executor.ts index 7a9ea268d..78c16a5ec 100644 --- a/src/platforms/android/adb-executor.ts +++ b/src/platforms/android/adb-executor.ts @@ -131,7 +131,7 @@ export type AndroidTouchInjector = ( export type AndroidGestureViewportProvider = () => Promise; -export type AndroidAdbProvider = { +type AndroidAdbProviderBase = { /** * Fallback executor for device-scoped adb arguments. Providers may omit explicit * methods to keep the legacy exec-shaped pull/install fallback. @@ -143,10 +143,20 @@ export type AndroidAdbProvider = { install?: AndroidAdbInstaller; installBundle?: AndroidBundleInstaller; text?: AndroidTextInjector; - touch?: AndroidTouchInjector; - gestureViewport?: AndroidGestureViewportProvider; }; +type AndroidTouchCapabilities = + | { + touch?: never; + gestureViewport?: AndroidGestureViewportProvider; + } + | { + touch: AndroidTouchInjector; + gestureViewport: AndroidGestureViewportProvider; + }; + +export type AndroidAdbProvider = AndroidAdbProviderBase & AndroidTouchCapabilities; + export type AndroidAdbProviderScopeOptions = { serial: string; }; From 265434e730233e1067b7693b33518504f8369c74 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Micha=C5=82=20Pierzcha=C5=82a?= Date: Mon, 13 Jul 2026 17:02:00 +0200 Subject: [PATCH 09/16] refactor: extract Android touch executor --- src/core/interactors/android.ts | 6 +- .../multitouch-helper-ownership.test.ts | 8 +-- .../__tests__/multitouch-helper.test.ts | 57 ------------------- .../android/__tests__/touch-executor.test.ts | 55 ++++++++++++++++++ src/platforms/android/gesture-viewport.ts | 15 +++++ src/platforms/android/input-actions.ts | 6 +- src/platforms/android/multitouch-helper.ts | 34 +---------- src/platforms/android/touch-executor.ts | 30 ++++++++++ 8 files changed, 112 insertions(+), 99 deletions(-) create mode 100644 src/platforms/android/__tests__/touch-executor.test.ts create mode 100644 src/platforms/android/gesture-viewport.ts create mode 100644 src/platforms/android/touch-executor.ts diff --git a/src/core/interactors/android.ts b/src/core/interactors/android.ts index 26918ecaa..0a6ece3ac 100644 --- a/src/core/interactors/android.ts +++ b/src/core/interactors/android.ts @@ -17,9 +17,9 @@ import { typeAndroid, } from '../../platforms/android/input-actions.ts'; import { - performGestureAndroid, + executeAndroidTouchPlan, readAndroidGestureViewport, -} from '../../platforms/android/multitouch-helper.ts'; +} from '../../platforms/android/touch-executor.ts'; import { readAndroidClipboardText, writeAndroidClipboardText, @@ -53,7 +53,7 @@ export function createAndroidInteractor(device: DeviceInfo): Interactor { type: (text, delayMs) => typeAndroid(device, text, delayMs), fill: (x, y, text, delayMs) => fillAndroid(device, x, y, text, delayMs), scroll: (direction, options) => scrollAndroid(device, direction, options), - performGesture: (plan) => performGestureAndroid(device, plan), + performGesture: (plan) => executeAndroidTouchPlan(device, plan), gestureViewport: () => readAndroidGestureViewport(device), screenshot: (outPath, options) => screenshotAndroid(device, outPath, options), snapshot: async (options) => { diff --git a/src/platforms/android/__tests__/multitouch-helper-ownership.test.ts b/src/platforms/android/__tests__/multitouch-helper-ownership.test.ts index 1a08c77b8..d9e6dcfb4 100644 --- a/src/platforms/android/__tests__/multitouch-helper-ownership.test.ts +++ b/src/platforms/android/__tests__/multitouch-helper-ownership.test.ts @@ -3,11 +3,9 @@ import { beforeEach, test, vi } from 'vitest'; import { ANDROID_EMULATOR } from '../../../__tests__/test-utils/index.ts'; import { buildGesturePlan } from '../../../contracts/gesture-plan.ts'; import { withAndroidAdbProvider } from '../adb-executor.ts'; -import { - performGestureAndroid, - resetAndroidMultiTouchHelperInstallCache, -} from '../multitouch-helper.ts'; +import { resetAndroidMultiTouchHelperInstallCache } from '../multitouch-helper.ts'; import { stopAndroidSnapshotHelperSessionForDevice } from '../snapshot-helper.ts'; +import { executeAndroidTouchPlan } from '../touch-executor.ts'; import { ANDROID_MULTITOUCH_HELPER_MANIFEST, androidMultiTouchResultRecord, @@ -73,7 +71,7 @@ test('helper gesture releases persistent snapshot instrumentation before touch i }, { serial: ANDROID_EMULATOR.id }, async () => - await performGestureAndroid( + await executeAndroidTouchPlan( ANDROID_EMULATOR, buildGesturePlan( { diff --git a/src/platforms/android/__tests__/multitouch-helper.test.ts b/src/platforms/android/__tests__/multitouch-helper.test.ts index 37d354e9b..92304f351 100644 --- a/src/platforms/android/__tests__/multitouch-helper.test.ts +++ b/src/platforms/android/__tests__/multitouch-helper.test.ts @@ -1,18 +1,13 @@ import assert from 'node:assert/strict'; import { beforeEach, test } from 'vitest'; -import { ANDROID_EMULATOR } from '../../../__tests__/test-utils/index.ts'; import { buildGesturePlan } from '../../../contracts/gesture-plan.ts'; import { normalizeAndroidMultiTouchHelperGestureRequest, parseAndroidMultiTouchHelperOutput, - performGestureAndroid, - readAndroidGestureViewport, resetAndroidMultiTouchHelperInstallCache, runAndroidMultiTouchHelperGesture, parseAndroidGestureViewportResult, } from '../multitouch-helper.ts'; -import { createAndroidInteractor } from '../../../core/interactors/android.ts'; -import { withAndroidAdbProvider } from '../adb-executor.ts'; import { ANDROID_MULTITOUCH_HELPER_MANIFEST as manifest, androidMultiTouchResultRecord as resultRecord, @@ -158,30 +153,6 @@ test('transform sends the planner-owned choreography without regenerating geomet assert.deepEqual(payload, { protocol: 'android-multitouch-helper-v1', ...request }); }); -test('provider-native touch receives the plan as its only source of truth', async () => { - const plan = buildGesturePlan( - { intent: 'pinch', origin: { x: 200, y: 300 }, scale: 1.5 }, - viewport, - ); - const calls: unknown[] = []; - const result = await withAndroidAdbProvider( - { - exec: async () => { - throw new Error('adb must not run'); - }, - gestureViewport: async () => viewport, - touch: async (request) => { - calls.push(request); - return { injected: true }; - }, - }, - { serial: ANDROID_EMULATOR.id }, - async () => await performGestureAndroid(ANDROID_EMULATOR, plan), - ); - assert.deepEqual(calls, [plan]); - assert.deepEqual(result, { backend: 'provider-native-touch', injected: true }); -}); - test('helper failures remain structured and actionable', async () => { const request = normalizeAndroidMultiTouchHelperGestureRequest( buildGesturePlan({ intent: 'pinch', scale: 1.5 }, viewport), @@ -239,31 +210,3 @@ test('gesture viewport result is typed and rejects invalid bounds', () => { ); assert.throws(() => parseAndroidGestureViewportResult([]), { code: 'COMMAND_FAILED' }); }); - -test('provider gesture viewport bypasses local helper transport and is validated', async () => { - let calls = 0; - await withAndroidAdbProvider( - { - exec: async () => { - throw new Error('adb must not run'); - }, - gestureViewport: async () => { - calls += 1; - return calls === 1 ? viewport : { ...viewport, width: 0 }; - }, - }, - { serial: ANDROID_EMULATOR.id }, - async () => { - assert.deepEqual(await readAndroidGestureViewport(ANDROID_EMULATOR), viewport); - await assert.rejects(readAndroidGestureViewport(ANDROID_EMULATOR), { - code: 'COMMAND_FAILED', - }); - }, - ); - assert.equal(calls, 2); -}); - -test('production Android interactor exposes the native gesture viewport seam', () => { - const interactor = createAndroidInteractor(ANDROID_EMULATOR); - assert.equal(typeof interactor.gestureViewport, 'function'); -}); diff --git a/src/platforms/android/__tests__/touch-executor.test.ts b/src/platforms/android/__tests__/touch-executor.test.ts new file mode 100644 index 000000000..51593c329 --- /dev/null +++ b/src/platforms/android/__tests__/touch-executor.test.ts @@ -0,0 +1,55 @@ +import assert from 'node:assert/strict'; +import { test } from 'vitest'; +import { ANDROID_EMULATOR } from '../../../__tests__/test-utils/index.ts'; +import { buildGesturePlan } from '../../../contracts/gesture-plan.ts'; +import { withAndroidAdbProvider } from '../adb-executor.ts'; +import { executeAndroidTouchPlan, readAndroidGestureViewport } from '../touch-executor.ts'; + +const viewport = { x: 0, y: 0, width: 400, height: 800 }; + +test('provider-native touch receives the plan as its only source of truth', async () => { + const plan = buildGesturePlan( + { intent: 'pinch', origin: { x: 200, y: 300 }, scale: 1.5 }, + viewport, + ); + const calls: unknown[] = []; + const result = await withAndroidAdbProvider( + { + exec: async () => { + throw new Error('adb must not run'); + }, + gestureViewport: async () => viewport, + touch: async (request) => { + calls.push(request); + return { injected: true }; + }, + }, + { serial: ANDROID_EMULATOR.id }, + async () => await executeAndroidTouchPlan(ANDROID_EMULATOR, plan), + ); + assert.deepEqual(calls, [plan]); + assert.deepEqual(result, { backend: 'provider-native-touch', injected: true }); +}); + +test('provider touch viewport bypasses local helper transport and is validated', async () => { + let calls = 0; + await withAndroidAdbProvider( + { + exec: async () => { + throw new Error('adb must not run'); + }, + gestureViewport: async () => { + calls += 1; + return calls === 1 ? viewport : { ...viewport, width: 0 }; + }, + }, + { serial: ANDROID_EMULATOR.id }, + async () => { + assert.deepEqual(await readAndroidGestureViewport(ANDROID_EMULATOR), viewport); + await assert.rejects(readAndroidGestureViewport(ANDROID_EMULATOR), { + code: 'COMMAND_FAILED', + }); + }, + ); + assert.equal(calls, 2); +}); diff --git a/src/platforms/android/gesture-viewport.ts b/src/platforms/android/gesture-viewport.ts new file mode 100644 index 000000000..5f8b10b13 --- /dev/null +++ b/src/platforms/android/gesture-viewport.ts @@ -0,0 +1,15 @@ +import { AppError } from '../../kernel/errors.ts'; +import type { Rect } from '../../kernel/snapshot.ts'; + +export function validateAndroidGestureViewport(viewport: Rect): Rect { + if ( + !Number.isFinite(viewport.x) || + !Number.isFinite(viewport.y) || + !Number.isFinite(viewport.width) || + !Number.isFinite(viewport.height) || + viewport.width <= 0 || + viewport.height <= 0 + ) + throw new AppError('COMMAND_FAILED', 'Android helper returned an invalid gesture viewport'); + return viewport; +} diff --git a/src/platforms/android/input-actions.ts b/src/platforms/android/input-actions.ts index 359b85d3a..39faaf382 100644 --- a/src/platforms/android/input-actions.ts +++ b/src/platforms/android/input-actions.ts @@ -19,7 +19,7 @@ import { type AndroidFillVerification, } from './fill-verification.ts'; import { isAndroidTestImeActive } from './ime-lifecycle.ts'; -import { performGestureAndroid, readAndroidGestureViewport } from './multitouch-helper.ts'; +import { executeAndroidTouchPlan, readAndroidGestureViewport } from './touch-executor.ts'; import { clearAndroidImeHelperText, resolveAndroidImeHelperArtifact, @@ -89,7 +89,7 @@ export async function longPressAndroid( ): Promise> { const viewport = await readAndroidGestureViewport(device); const point = { x, y }; - return await performGestureAndroid(device, { + return await executeAndroidTouchPlan(device, { topology: 'single', intent: 'longPress', durationMs, @@ -292,7 +292,7 @@ export async function scrollAndroid( viewport, 'android', ); - const backend = await performGestureAndroid(device, gesturePlan); + const backend = await executeAndroidTouchPlan(device, gesturePlan); return { ...scrollPlan, diff --git a/src/platforms/android/multitouch-helper.ts b/src/platforms/android/multitouch-helper.ts index 2b7e5dfc8..ef42182da 100644 --- a/src/platforms/android/multitouch-helper.ts +++ b/src/platforms/android/multitouch-helper.ts @@ -7,8 +7,6 @@ import { emitDiagnostic, withDiagnosticTimer } from '../../utils/diagnostics.ts' import { resolveAndroidAdbExecutor, resolveAndroidAdbProvider, - resolveAndroidGestureViewportProvider, - resolveAndroidTouchInjector, type AndroidAdbExecutor, } from './adb-executor.ts'; import { @@ -24,6 +22,7 @@ import { readInstrumentationResultNumber, } from './instrumentation-helper.ts'; import { stopAndroidSnapshotHelperSessionForDevice } from './snapshot-helper.ts'; +import { validateAndroidGestureViewport } from './gesture-viewport.ts'; import type { AndroidTouchPlan } from './touch-plan.ts'; const HELPER_NAME = 'android-multitouch-helper'; @@ -65,19 +64,7 @@ type AndroidMultiTouchHelperGestureRequest = { pointers: AndroidPlannedPointerTrajectory[]; }; -export async function performGestureAndroid( - device: DeviceInfo, - plan: AndroidTouchPlan, -): Promise> { - const providerTouch = resolveAndroidTouchInjector(device); - if (providerTouch) { - const result = (await providerTouch(plan)) ?? {}; - return { backend: 'provider-native-touch', ...result }; - } - return await runAndroidMultiTouchHelperGestureForDevice(device, plan); -} - -async function runAndroidMultiTouchHelperGestureForDevice( +export async function executeAndroidMultiTouchHelperPlan( device: DeviceInfo, plan: AndroidTouchPlan, ): Promise> { @@ -126,9 +113,7 @@ async function prepareAndroidMultiTouchHelper(device: DeviceInfo) { return { adb, artifact, install }; } -export async function readAndroidGestureViewport(device: DeviceInfo): Promise { - const providerViewport = resolveAndroidGestureViewportProvider(device); - if (providerViewport) return validateAndroidGestureViewport(await providerViewport()); +export async function readAndroidMultiTouchHelperViewport(device: DeviceInfo): Promise { const { adb, artifact } = await prepareAndroidMultiTouchHelper(device); const result = await adb( [ @@ -166,19 +151,6 @@ export function parseAndroidGestureViewportResult(results: Array> { + const providerTouch = resolveAndroidTouchInjector(device); + if (providerTouch) { + const result = (await providerTouch(plan)) ?? {}; + return { backend: 'provider-native-touch', ...result }; + } + return await executeAndroidMultiTouchHelperPlan(device, plan); +} + +export async function readAndroidGestureViewport(device: DeviceInfo): Promise { + const providerViewport = resolveAndroidGestureViewportProvider(device); + if (providerViewport) return validateAndroidGestureViewport(await providerViewport()); + return await readAndroidMultiTouchHelperViewport(device); +} From b7a94292d1c5f88b04d5d52353e387df4ed6eaac Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Micha=C5=82=20Pierzcha=C5=82a?= Date: Mon, 13 Jul 2026 17:13:01 +0200 Subject: [PATCH 10/16] docs: clarify Android planned touch seam --- CONTEXT.md | 5 +++++ docs/adr/0013-unified-gesture-plans.md | 11 ++++++----- 2 files changed, 11 insertions(+), 5 deletions(-) diff --git a/CONTEXT.md b/CONTEXT.md index db4c60154..dca23d452 100644 --- a/CONTEXT.md +++ b/CONTEXT.md @@ -48,6 +48,11 @@ - Coordinate-first resolved element activation: iOS/macOS runner interaction pattern where a selector or text query resolves the semantic `XCUIElement`, then activation uses the element's resolved center coordinate when a frame is available. This keeps target selection semantic while avoiding `XCUIElement.tap()` post-action element re-resolution after normal navigation. tvOS remains focus/remote-driven. - Interaction dispatch path: one concrete route an interaction command takes to the device (runtime selector/ref resolution, direct iOS selector, native ref via web clickRef, coordinate, maestro non-hittable fallback). Every path classifies every guarantee in the ADR 0011 registry. - Gesture plan: typed, platform-neutral normalization of one- or two-contact gesture intent into bounded pointer trajectories. Contact topology is separate from motion; two-contact intent remains pan/pinch/rotate/transform even when native injection shares one executor. See ADR 0013. +- Android planned-touch executor: Android-local adapter seam that accepts `AndroidTouchPlan`—the + platform-neutral `GesturePlan` plus Android's stationary long-press plan—and selects the paired + provider-native touch/viewport adapter or bundled instrumentation-helper adapter. Scroll and + long-press retain their command semantics and only share physical touch execution through this + seam. See ADR 0013. - Multi-touch geometry: the internal initial span and angle plus centroid translation, scale, and rotation used to build both contact trajectories. Geometry is viewport-aware and fails early when the requested motion cannot fit; it is not a public tuning surface. - Guarantee cell: one (dispatch path, guarantee) entry in `src/contracts/interaction-guarantees.ts`, classified as runtime/runner/delegated/inapplicable/waived. Completeness is a compile error; honesty is gate-tested. - Owned waiver: a `gap:`-prefixed waived cell carrying a `trackingIssue` URL. Waivers are diffable debt with an owner, never folklore. diff --git a/docs/adr/0013-unified-gesture-plans.md b/docs/adr/0013-unified-gesture-plans.md index ff23b22e5..986cac42c 100644 --- a/docs/adr/0013-unified-gesture-plans.md +++ b/docs/adr/0013-unified-gesture-plans.md @@ -17,7 +17,7 @@ interaction viewport before injection. `scroll` remains a separate command because it owns viewport/edge traversal and content-state verification rather than a single physical gesture. Its Android physical movement still lowers to -the shared planned-touch executor. +the Android planned-touch executor. ## Decision @@ -55,10 +55,11 @@ no established automation use case justifies a public tuning surface. Platform adapters consume the canonical plan: -- Android sends planned touch, including gesture plans plus the physical movement for scroll and - long-press, to provider-native touch injection when available, otherwise to the bundled - instrumentation helper. The helper injects the exact planned pointer samples. Planned Android - touch execution never falls back to `adb input swipe`; the snapshot helper is stopped +- Android's `executeAndroidTouchPlan` adapter seam sends planned touch, including gesture plans plus + the physical movement for scroll and long-press, to provider-native touch injection when + available, otherwise to the bundled instrumentation helper. The helper injects the exact planned + pointer samples. Android touch execution never falls back to `adb input swipe`; the snapshot + helper is stopped before local gesture instrumentation because Android permits only one instrumentation owner of `UiAutomation`. - iOS converts every planned point to native orientation and feeds the exact arrays to the existing From 787cfaf2d3e7bd99b102ced26d4c51ea9ae2ecdc Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Micha=C5=82=20Pierzcha=C5=82a?= Date: Mon, 13 Jul 2026 17:43:34 +0200 Subject: [PATCH 11/16] fix: complete Android gesture failure handling --- .../__tests__/timeout-policy.test.ts | 1 + src/core/command-descriptor/registry.ts | 8 +- src/core/command-descriptor/timeout-policy.ts | 5 + .../multitouch-helper-ownership.test.ts | 92 --------- .../__tests__/touch-executor-helper.test.ts | 191 ++++++++++++++++++ src/platforms/android/multitouch-helper.ts | 23 ++- src/utils/__tests__/daemon-client.test.ts | 16 +- 7 files changed, 237 insertions(+), 99 deletions(-) delete mode 100644 src/platforms/android/__tests__/multitouch-helper-ownership.test.ts create mode 100644 src/platforms/android/__tests__/touch-executor-helper.test.ts diff --git a/src/core/command-descriptor/__tests__/timeout-policy.test.ts b/src/core/command-descriptor/__tests__/timeout-policy.test.ts index f0bddb799..c79d5852f 100644 --- a/src/core/command-descriptor/__tests__/timeout-policy.test.ts +++ b/src/core/command-descriptor/__tests__/timeout-policy.test.ts @@ -127,6 +127,7 @@ test('request envelopes deviating from the default are bounded, reviewed sets', install: 180_000, reinstall: 180_000, install_source: 180_000, + longpress: 150_000, test: 'unbounded', }; for (const descriptor of commandDescriptors) { diff --git a/src/core/command-descriptor/registry.ts b/src/core/command-descriptor/registry.ts index ab33a90c9..f05773048 100644 --- a/src/core/command-descriptor/registry.ts +++ b/src/core/command-descriptor/registry.ts @@ -4,6 +4,7 @@ import { resolveWaitBudgetMs } from '../wait-positionals.ts'; import { DEFAULT_TIMEOUT_POLICY, INSTALL_REQUEST_TIMEOUT_MS, + LONG_PRESS_REQUEST_TIMEOUT_MS, PREPARE_REQUEST_TIMEOUT_MS, } from './timeout-policy.ts'; import { resolvePostActionObservationSupport } from './post-action-observation.ts'; @@ -162,6 +163,11 @@ const SETTLE_FLAG_PRESERVE_DAEMON_TIMEOUT_POLICY: CommandTimeoutPolicy = { onTimeout: 'preserve-daemon', }; +const LONG_PRESS_TIMEOUT_POLICY: CommandTimeoutPolicy = { + ...SETTLE_FLAG_PRESERVE_DAEMON_TIMEOUT_POLICY, + envelopeMs: LONG_PRESS_REQUEST_TIMEOUT_MS, +}; + const TOUCH_INTERACTION_RESPONSE_DATA_TRANSFORM = { fields: { count: { defaultValue: 1, omitDefault: true }, @@ -703,7 +709,7 @@ export const RAW_COMMAND_DESCRIPTORS = [ daemon: { route: 'interaction', replayScopedAction: true, androidBlockingDialogGuard: true }, dispatch: {}, capability: { apple: APPLE_SIM_AND_DEVICE, android: ANDROID_ALL, linux: LINUX_DEVICE }, - timeoutPolicy: interactionTimeoutPolicy('longpress'), + timeoutPolicy: LONG_PRESS_TIMEOUT_POLICY, postActionObservation: postActionObservation('longpress'), batchable: true, }, diff --git a/src/core/command-descriptor/timeout-policy.ts b/src/core/command-descriptor/timeout-policy.ts index ed9b96984..cff073cea 100644 --- a/src/core/command-descriptor/timeout-policy.ts +++ b/src/core/command-descriptor/timeout-policy.ts @@ -7,6 +7,11 @@ import type { CommandTimeoutPolicy } from './types.ts'; const DAEMON_REQUEST_TIMEOUT_MS = 90_000; export const PREPARE_REQUEST_TIMEOUT_MS = 240_000; +// A valid longpress may hold for 120 seconds. Keep the outer request alive +// beyond both that command budget and the helper's 15-second completion +// overhead so the platform result, rather than the client envelope, wins. +export const LONG_PRESS_REQUEST_TIMEOUT_MS = 150_000; + // Keep this above the longest platform install subprocess timeout so the client // envelope does not abort a still-progressing device install first. export const INSTALL_REQUEST_TIMEOUT_MS = 180_000; diff --git a/src/platforms/android/__tests__/multitouch-helper-ownership.test.ts b/src/platforms/android/__tests__/multitouch-helper-ownership.test.ts deleted file mode 100644 index d9e6dcfb4..000000000 --- a/src/platforms/android/__tests__/multitouch-helper-ownership.test.ts +++ /dev/null @@ -1,92 +0,0 @@ -import assert from 'node:assert/strict'; -import { beforeEach, test, vi } from 'vitest'; -import { ANDROID_EMULATOR } from '../../../__tests__/test-utils/index.ts'; -import { buildGesturePlan } from '../../../contracts/gesture-plan.ts'; -import { withAndroidAdbProvider } from '../adb-executor.ts'; -import { resetAndroidMultiTouchHelperInstallCache } from '../multitouch-helper.ts'; -import { stopAndroidSnapshotHelperSessionForDevice } from '../snapshot-helper.ts'; -import { executeAndroidTouchPlan } from '../touch-executor.ts'; -import { - ANDROID_MULTITOUCH_HELPER_MANIFEST, - androidMultiTouchResultRecord, -} from './multitouch-helper.fixtures.ts'; - -vi.mock('../snapshot-helper.ts', async (importOriginal) => ({ - ...(await importOriginal()), - stopAndroidSnapshotHelperSessionForDevice: vi.fn(async () => {}), -})); - -vi.mock('../helper-package-install.ts', async (importOriginal) => { - const actual = await importOriginal(); - const fixture = await import('./android-helper-artifact.fixtures.ts'); - const multitouchFixture = await import('./multitouch-helper.fixtures.ts'); - return { - ...actual, - resolveAndroidHelperArtifact: vi.fn(async () => ({ - apkPath: fixture.ANDROID_HELPER_FIXTURE_APK_PATH, - manifest: { - ...multitouchFixture.ANDROID_MULTITOUCH_HELPER_MANIFEST, - sha256: fixture.ANDROID_HELPER_FIXTURE_APK_SHA256, - }, - })), - }; -}); - -const mockStopSnapshotSession = vi.mocked(stopAndroidSnapshotHelperSessionForDevice); - -beforeEach(() => { - resetAndroidMultiTouchHelperInstallCache(); - mockStopSnapshotSession.mockReset(); -}); - -test('helper gesture releases persistent snapshot instrumentation before touch instrumentation', async () => { - const events: string[] = []; - mockStopSnapshotSession.mockImplementation(async () => { - events.push('snapshot-stop'); - }); - - const result = await withAndroidAdbProvider( - { - exec: async (args) => { - if (args.includes('--show-versioncode')) { - return { - exitCode: 0, - stdout: `package:${ANDROID_MULTITOUCH_HELPER_MANIFEST.packageName} versionCode:999999`, - stderr: '', - }; - } - if (args.includes('instrument')) { - events.push('touch-instrumentation'); - return { - exitCode: 0, - stdout: [ - androidMultiTouchResultRecord({ ok: 'true', kind: 'swipe' }), - 'INSTRUMENTATION_CODE: 0', - ].join('\n'), - stderr: '', - }; - } - throw new Error(`unexpected adb call: ${args.join(' ')}`); - }, - }, - { serial: ANDROID_EMULATOR.id }, - async () => - await executeAndroidTouchPlan( - ANDROID_EMULATOR, - buildGesturePlan( - { - intent: 'pan', - pointerCount: 1, - origin: { x: 340, y: 400 }, - delta: { x: -280, y: 0 }, - durationMs: 300, - }, - { x: 0, y: 0, width: 400, height: 800 }, - 'android', - ), - ), - ); - - assert.equal(result?.backend, 'android-multitouch-helper'); - assert.deepEqual(events, ['snapshot-stop', 'touch-instrumentation']); -}); diff --git a/src/platforms/android/__tests__/touch-executor-helper.test.ts b/src/platforms/android/__tests__/touch-executor-helper.test.ts new file mode 100644 index 000000000..119f08ab9 --- /dev/null +++ b/src/platforms/android/__tests__/touch-executor-helper.test.ts @@ -0,0 +1,191 @@ +import assert from 'node:assert/strict'; +import { beforeEach, test, vi } from 'vitest'; +import { ANDROID_EMULATOR } from '../../../__tests__/test-utils/index.ts'; +import { buildGesturePlan } from '../../../contracts/gesture-plan.ts'; +import { AppError } from '../../../kernel/errors.ts'; +import { withAndroidAdbProvider } from '../adb-executor.ts'; +import { resetAndroidMultiTouchHelperInstallCache } from '../multitouch-helper.ts'; +import { stopAndroidSnapshotHelperSessionForDevice } from '../snapshot-helper.ts'; +import { executeAndroidTouchPlan, readAndroidGestureViewport } from '../touch-executor.ts'; +import { + ANDROID_MULTITOUCH_HELPER_MANIFEST, + androidMultiTouchResultRecord, +} from './multitouch-helper.fixtures.ts'; + +vi.mock('../snapshot-helper.ts', async (importOriginal) => ({ + ...(await importOriginal()), + stopAndroidSnapshotHelperSessionForDevice: vi.fn(async () => {}), +})); + +vi.mock('../helper-package-install.ts', async (importOriginal) => { + const actual = await importOriginal(); + const fixture = await import('./android-helper-artifact.fixtures.ts'); + const multitouchFixture = await import('./multitouch-helper.fixtures.ts'); + return { + ...actual, + resolveAndroidHelperArtifact: vi.fn(async () => ({ + apkPath: fixture.ANDROID_HELPER_FIXTURE_APK_PATH, + manifest: { + ...multitouchFixture.ANDROID_MULTITOUCH_HELPER_MANIFEST, + sha256: fixture.ANDROID_HELPER_FIXTURE_APK_SHA256, + }, + })), + }; +}); + +const mockStopSnapshotSession = vi.mocked(stopAndroidSnapshotHelperSessionForDevice); + +beforeEach(() => { + resetAndroidMultiTouchHelperInstallCache(); + mockStopSnapshotSession.mockReset(); +}); + +test('helper gesture releases persistent snapshot instrumentation before touch instrumentation', async () => { + const events: string[] = []; + mockStopSnapshotSession.mockImplementation(async () => { + events.push('snapshot-stop'); + }); + + const result = await withAndroidAdbProvider( + { + exec: async (args) => { + if (args.includes('--show-versioncode')) { + return { + exitCode: 0, + stdout: `package:${ANDROID_MULTITOUCH_HELPER_MANIFEST.packageName} versionCode:999999`, + stderr: '', + }; + } + if (args.includes('instrument')) { + events.push('touch-instrumentation'); + return { + exitCode: 0, + stdout: [ + androidMultiTouchResultRecord({ ok: 'true', kind: 'swipe' }), + 'INSTRUMENTATION_CODE: 0', + ].join('\n'), + stderr: '', + }; + } + throw new Error(`unexpected adb call: ${args.join(' ')}`); + }, + }, + { serial: ANDROID_EMULATOR.id }, + async () => + await executeAndroidTouchPlan( + ANDROID_EMULATOR, + buildGesturePlan( + { + intent: 'pan', + pointerCount: 1, + origin: { x: 340, y: 400 }, + delta: { x: -280, y: 0 }, + durationMs: 300, + }, + { x: 0, y: 0, width: 400, height: 800 }, + 'android', + ), + ), + ); + + assert.equal(result?.backend, 'android-multitouch-helper'); + assert.deepEqual(events, ['snapshot-stop', 'touch-instrumentation']); +}); + +test('single-pointer helper failure propagates through the touch executor', async () => { + await assert.rejects( + withAndroidAdbProvider( + { + exec: async (args) => { + if (args.includes('--show-versioncode')) { + return { + exitCode: 0, + stdout: `package:${ANDROID_MULTITOUCH_HELPER_MANIFEST.packageName} versionCode:999999`, + stderr: '', + }; + } + if (args.includes('instrument')) { + return { + exitCode: 1, + stdout: [ + androidMultiTouchResultRecord({ + ok: 'false', + errorType: 'java.lang.IllegalStateException', + message: 'injectInputEvent returned false', + }), + 'INSTRUMENTATION_CODE: 1', + ].join('\n'), + stderr: '', + }; + } + throw new Error(`unexpected adb call: ${args.join(' ')}`); + }, + }, + { serial: ANDROID_EMULATOR.id }, + async () => + await executeAndroidTouchPlan( + ANDROID_EMULATOR, + buildGesturePlan( + { + intent: 'pan', + pointerCount: 1, + origin: { x: 340, y: 400 }, + delta: { x: -280, y: 0 }, + durationMs: 300, + }, + { x: 0, y: 0, width: 400, height: 800 }, + 'android', + ), + ), + ), + (error: unknown) => { + assert.ok(error instanceof AppError); + assert.equal(error.code, 'COMMAND_FAILED'); + assert.equal(error.message, 'injectInputEvent returned false'); + assert.equal(error.details?.errorType, 'java.lang.IllegalStateException'); + return true; + }, + ); +}); + +test('helper viewport failure preserves its structured message and error type', async () => { + await assert.rejects( + withAndroidAdbProvider( + { + exec: async (args) => { + if (args.includes('--show-versioncode')) { + return { + exitCode: 0, + stdout: `package:${ANDROID_MULTITOUCH_HELPER_MANIFEST.packageName} versionCode:999999`, + stderr: '', + }; + } + if (args.includes('instrument')) { + return { + exitCode: 1, + stdout: [ + androidMultiTouchResultRecord({ + ok: 'false', + errorType: 'java.lang.SecurityException', + message: 'UiAutomation is unavailable', + }), + 'INSTRUMENTATION_CODE: 1', + ].join('\n'), + stderr: 'instrumentation failed', + }; + } + throw new Error(`unexpected adb call: ${args.join(' ')}`); + }, + }, + { serial: ANDROID_EMULATOR.id }, + async () => await readAndroidGestureViewport(ANDROID_EMULATOR), + ), + (error: unknown) => { + assert.ok(error instanceof AppError); + assert.equal(error.code, 'COMMAND_FAILED'); + assert.equal(error.message, 'UiAutomation is unavailable'); + assert.equal(error.details?.errorType, 'java.lang.SecurityException'); + return true; + }, + ); +}); diff --git a/src/platforms/android/multitouch-helper.ts b/src/platforms/android/multitouch-helper.ts index ef42182da..1e4787e5f 100644 --- a/src/platforms/android/multitouch-helper.ts +++ b/src/platforms/android/multitouch-helper.ts @@ -129,9 +129,25 @@ export async function readAndroidMultiTouchHelperViewport(device: DeviceInfo): P { allowFailure: true, timeoutMs: HELPER_GESTURE_TIMEOUT_MS }, ); const records = parseInstrumentationRecords(`${result.stdout}\n${result.stderr}`); - if (result.exitCode !== 0) - throw new AppError('COMMAND_FAILED', 'Android gesture viewport is unavailable'); - return parseAndroidGestureViewportResult(records.results); + const helperResult = records.results.find( + (record) => record.agentDeviceProtocol === HELPER_PROTOCOL, + ); + if (result.exitCode !== 0 && !helperResult) { + throw new AppError( + 'COMMAND_FAILED', + 'Android gesture viewport is unavailable', + execFailureDetails(result), + ); + } + const viewport = parseAndroidGestureViewportResult(records.results); + if (result.exitCode !== 0) { + throw new AppError( + 'COMMAND_FAILED', + 'Android gesture viewport is unavailable', + execFailureDetails(result, { helper: helperResult }), + ); + } + return viewport; } export function parseAndroidGestureViewportResult(results: Array>): Rect { @@ -140,6 +156,7 @@ export function parseAndroidGestureViewportResult(results: Array Date: Mon, 13 Jul 2026 17:48:18 +0200 Subject: [PATCH 12/16] refactor: tighten Android gesture review fixes --- src/core/command-descriptor/registry.ts | 13 ++++++------- src/core/command-descriptor/timeout-policy.ts | 5 ----- src/platforms/android/multitouch-helper.ts | 10 +++++----- 3 files changed, 11 insertions(+), 17 deletions(-) diff --git a/src/core/command-descriptor/registry.ts b/src/core/command-descriptor/registry.ts index f05773048..d61c48dbd 100644 --- a/src/core/command-descriptor/registry.ts +++ b/src/core/command-descriptor/registry.ts @@ -4,7 +4,6 @@ import { resolveWaitBudgetMs } from '../wait-positionals.ts'; import { DEFAULT_TIMEOUT_POLICY, INSTALL_REQUEST_TIMEOUT_MS, - LONG_PRESS_REQUEST_TIMEOUT_MS, PREPARE_REQUEST_TIMEOUT_MS, } from './timeout-policy.ts'; import { resolvePostActionObservationSupport } from './post-action-observation.ts'; @@ -163,11 +162,6 @@ const SETTLE_FLAG_PRESERVE_DAEMON_TIMEOUT_POLICY: CommandTimeoutPolicy = { onTimeout: 'preserve-daemon', }; -const LONG_PRESS_TIMEOUT_POLICY: CommandTimeoutPolicy = { - ...SETTLE_FLAG_PRESERVE_DAEMON_TIMEOUT_POLICY, - envelopeMs: LONG_PRESS_REQUEST_TIMEOUT_MS, -}; - const TOUCH_INTERACTION_RESPONSE_DATA_TRANSFORM = { fields: { count: { defaultValue: 1, omitDefault: true }, @@ -709,7 +703,12 @@ export const RAW_COMMAND_DESCRIPTORS = [ daemon: { route: 'interaction', replayScopedAction: true, androidBlockingDialogGuard: true }, dispatch: {}, capability: { apple: APPLE_SIM_AND_DEVICE, android: ANDROID_ALL, linux: LINUX_DEVICE }, - timeoutPolicy: LONG_PRESS_TIMEOUT_POLICY, + timeoutPolicy: { + ...SETTLE_FLAG_PRESERVE_DAEMON_TIMEOUT_POLICY, + // A valid hold may last 120 seconds; leave the helper's 15-second + // completion overhead inside the outer request envelope too. + envelopeMs: 150_000, + }, postActionObservation: postActionObservation('longpress'), batchable: true, }, diff --git a/src/core/command-descriptor/timeout-policy.ts b/src/core/command-descriptor/timeout-policy.ts index cff073cea..ed9b96984 100644 --- a/src/core/command-descriptor/timeout-policy.ts +++ b/src/core/command-descriptor/timeout-policy.ts @@ -7,11 +7,6 @@ import type { CommandTimeoutPolicy } from './types.ts'; const DAEMON_REQUEST_TIMEOUT_MS = 90_000; export const PREPARE_REQUEST_TIMEOUT_MS = 240_000; -// A valid longpress may hold for 120 seconds. Keep the outer request alive -// beyond both that command budget and the helper's 15-second completion -// overhead so the platform result, rather than the client envelope, wins. -export const LONG_PRESS_REQUEST_TIMEOUT_MS = 150_000; - // Keep this above the longest platform install subprocess timeout so the client // envelope does not abort a still-progressing device install first. export const INSTALL_REQUEST_TIMEOUT_MS = 180_000; diff --git a/src/platforms/android/multitouch-helper.ts b/src/platforms/android/multitouch-helper.ts index 1e4787e5f..64ab710a6 100644 --- a/src/platforms/android/multitouch-helper.ts +++ b/src/platforms/android/multitouch-helper.ts @@ -129,10 +129,10 @@ export async function readAndroidMultiTouchHelperViewport(device: DeviceInfo): P { allowFailure: true, timeoutMs: HELPER_GESTURE_TIMEOUT_MS }, ); const records = parseInstrumentationRecords(`${result.stdout}\n${result.stderr}`); - const helperResult = records.results.find( - (record) => record.agentDeviceProtocol === HELPER_PROTOCOL, - ); - if (result.exitCode !== 0 && !helperResult) { + if ( + result.exitCode !== 0 && + !records.results.some((record) => record.agentDeviceProtocol === HELPER_PROTOCOL) + ) { throw new AppError( 'COMMAND_FAILED', 'Android gesture viewport is unavailable', @@ -144,7 +144,7 @@ export async function readAndroidMultiTouchHelperViewport(device: DeviceInfo): P throw new AppError( 'COMMAND_FAILED', 'Android gesture viewport is unavailable', - execFailureDetails(result, { helper: helperResult }), + execFailureDetails(result), ); } return viewport; From ef98df88f6469e204a745a17a15a7f2f2aa4a686 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Micha=C5=82=20Pierzcha=C5=82a?= Date: Mon, 13 Jul 2026 18:18:09 +0200 Subject: [PATCH 13/16] perf: avoid unnecessary Android viewport probes --- CONTEXT.md | 3 +- docs/adr/0013-unified-gesture-plans.md | 8 +- .../android/__tests__/input-actions.test.ts | 4 + .../multitouch-helper-install.test.ts | 2 +- .../__tests__/multitouch-helper.test.ts | 3 +- .../__tests__/touch-executor-helper.test.ts | 39 +++++- .../android/__tests__/touch-executor.test.ts | 1 + src/platforms/android/adb-executor.ts | 21 ++- src/platforms/android/input-actions.ts | 37 +++-- .../android/multitouch-helper-install.ts | 99 ++++++++++++++ src/platforms/android/multitouch-helper.ts | 127 ++---------------- src/platforms/android/touch-executor.ts | 22 +-- src/platforms/android/touch-plan.ts | 5 +- 13 files changed, 215 insertions(+), 156 deletions(-) create mode 100644 src/platforms/android/multitouch-helper-install.ts diff --git a/CONTEXT.md b/CONTEXT.md index dca23d452..edaeac265 100644 --- a/CONTEXT.md +++ b/CONTEXT.md @@ -52,7 +52,8 @@ platform-neutral `GesturePlan` plus Android's stationary long-press plan—and selects the paired provider-native touch/viewport adapter or bundled instrumentation-helper adapter. Scroll and long-press retain their command semantics and only share physical touch execution through this - seam. See ADR 0013. + seam. Helper long-press executes its absolute stationary path without a viewport probe; provider + long-press receives its paired provider-owned viewport. See ADR 0013. - Multi-touch geometry: the internal initial span and angle plus centroid translation, scale, and rotation used to build both contact trajectories. Geometry is viewport-aware and fails early when the requested motion cannot fit; it is not a public tuning surface. - Guarantee cell: one (dispatch path, guarantee) entry in `src/contracts/interaction-guarantees.ts`, classified as runtime/runner/delegated/inapplicable/waived. Completeness is a compile error; honesty is gate-tested. - Owned waiver: a `gap:`-prefixed waived cell carrying a `trackingIssue` URL. Waivers are diffable debt with an owner, never folklore. diff --git a/docs/adr/0013-unified-gesture-plans.md b/docs/adr/0013-unified-gesture-plans.md index 986cac42c..669aa33bb 100644 --- a/docs/adr/0013-unified-gesture-plans.md +++ b/docs/adr/0013-unified-gesture-plans.md @@ -58,8 +58,9 @@ Platform adapters consume the canonical plan: - Android's `executeAndroidTouchPlan` adapter seam sends planned touch, including gesture plans plus the physical movement for scroll and long-press, to provider-native touch injection when available, otherwise to the bundled instrumentation helper. The helper injects the exact planned - pointer samples. Android touch execution never falls back to `adb input swipe`; the snapshot - helper is stopped + pointer samples. A stationary long-press needs no viewport on the helper path; the executor adds + the paired provider-owned viewport only for provider-native touch. Android touch execution never + falls back to `adb input swipe`; the snapshot helper is stopped before local gesture instrumentation because Android permits only one instrumentation owner of `UiAutomation`. - iOS converts every planned point to native orientation and feeds the exact arrays to the existing @@ -96,6 +97,9 @@ selectors or refs and therefore cannot claim element-targeting guarantees. - One-finger pan remains the default and explicit two-finger pan retains pan intent. - The active viewport is resolved for each gesture, so rotation, keyboard, and window changes do not use stale geometry. +- On bare ADB, Android scroll and long-press require the bundled touch helper and `UiAutomation`; + helper installation or runtime failure is surfaced directly rather than degrading to an + approximate `adb input swipe`. - Pointer plans are larger than scalar requests but bounded by duration and the 16 ms sample cadence; deleting duplicate scalar executors offsets the package cost. diff --git a/src/platforms/android/__tests__/input-actions.test.ts b/src/platforms/android/__tests__/input-actions.test.ts index 5485c20c9..4120b210f 100644 --- a/src/platforms/android/__tests__/input-actions.test.ts +++ b/src/platforms/android/__tests__/input-actions.test.ts @@ -39,6 +39,10 @@ test('scrollAndroid plans explicit pixel travel through semantic touch injection assert.equal(result.durationMs, 120); assert.equal(result.referenceWidth, 1080); assert.equal(result.referenceHeight, 1920); + assert.equal(result.x1, 550); + assert.equal(result.y1, 1100); + assert.equal(result.x2, 550); + assert.equal(result.y2, 860); assert.equal(result.backend, 'provider-native-touch'); assert.equal(result.injected, true); }); diff --git a/src/platforms/android/__tests__/multitouch-helper-install.test.ts b/src/platforms/android/__tests__/multitouch-helper-install.test.ts index 5f270b1e3..8cb3402fc 100644 --- a/src/platforms/android/__tests__/multitouch-helper-install.test.ts +++ b/src/platforms/android/__tests__/multitouch-helper-install.test.ts @@ -7,7 +7,7 @@ import { beforeEach, test } from 'vitest'; import { ensureAndroidMultiTouchHelper, resetAndroidMultiTouchHelperInstallCache, -} from '../multitouch-helper.ts'; +} from '../multitouch-helper-install.ts'; import type { AndroidAdbExecutor, AndroidAdbProvider } from '../adb-executor.ts'; import { ANDROID_MULTITOUCH_HELPER_MANIFEST } from './multitouch-helper.fixtures.ts'; diff --git a/src/platforms/android/__tests__/multitouch-helper.test.ts b/src/platforms/android/__tests__/multitouch-helper.test.ts index 92304f351..b6050e3b5 100644 --- a/src/platforms/android/__tests__/multitouch-helper.test.ts +++ b/src/platforms/android/__tests__/multitouch-helper.test.ts @@ -4,10 +4,10 @@ import { buildGesturePlan } from '../../../contracts/gesture-plan.ts'; import { normalizeAndroidMultiTouchHelperGestureRequest, parseAndroidMultiTouchHelperOutput, - resetAndroidMultiTouchHelperInstallCache, runAndroidMultiTouchHelperGesture, parseAndroidGestureViewportResult, } from '../multitouch-helper.ts'; +import { resetAndroidMultiTouchHelperInstallCache } from '../multitouch-helper-install.ts'; import { ANDROID_MULTITOUCH_HELPER_MANIFEST as manifest, androidMultiTouchResultRecord as resultRecord, @@ -89,7 +89,6 @@ test('Android long press lowers to a stationary single-pointer helper request', topology: 'single', intent: 'longPress', durationMs: 120_000, - viewport, pointers: [ { pointerId: 0, diff --git a/src/platforms/android/__tests__/touch-executor-helper.test.ts b/src/platforms/android/__tests__/touch-executor-helper.test.ts index 119f08ab9..680e50655 100644 --- a/src/platforms/android/__tests__/touch-executor-helper.test.ts +++ b/src/platforms/android/__tests__/touch-executor-helper.test.ts @@ -4,7 +4,8 @@ import { ANDROID_EMULATOR } from '../../../__tests__/test-utils/index.ts'; import { buildGesturePlan } from '../../../contracts/gesture-plan.ts'; import { AppError } from '../../../kernel/errors.ts'; import { withAndroidAdbProvider } from '../adb-executor.ts'; -import { resetAndroidMultiTouchHelperInstallCache } from '../multitouch-helper.ts'; +import { longPressAndroid } from '../input-actions.ts'; +import { resetAndroidMultiTouchHelperInstallCache } from '../multitouch-helper-install.ts'; import { stopAndroidSnapshotHelperSessionForDevice } from '../snapshot-helper.ts'; import { executeAndroidTouchPlan, readAndroidGestureViewport } from '../touch-executor.ts'; import { @@ -92,6 +93,42 @@ test('helper gesture releases persistent snapshot instrumentation before touch i assert.deepEqual(events, ['snapshot-stop', 'touch-instrumentation']); }); +test('bare-adb long press executes one helper gesture without a viewport probe', async () => { + const instrumentCalls: string[][] = []; + const result = await withAndroidAdbProvider( + { + exec: async (args) => { + if (args.includes('--show-versioncode')) { + return { + exitCode: 0, + stdout: `package:${ANDROID_MULTITOUCH_HELPER_MANIFEST.packageName} versionCode:999999`, + stderr: '', + }; + } + if (args.includes('instrument')) { + instrumentCalls.push(args); + return { + exitCode: 0, + stdout: [ + androidMultiTouchResultRecord({ ok: 'true', kind: 'swipe' }), + 'INSTRUMENTATION_CODE: 0', + ].join('\n'), + stderr: '', + }; + } + throw new Error(`unexpected adb call: ${args.join(' ')}`); + }, + }, + { serial: ANDROID_EMULATOR.id }, + async () => await longPressAndroid(ANDROID_EMULATOR, 30, 40, 750), + ); + + assert.equal(result.backend, 'android-multitouch-helper'); + assert.equal(instrumentCalls.length, 1); + assert.equal(instrumentCalls[0]?.includes('viewport'), false); + assert.equal(mockStopSnapshotSession.mock.calls.length, 1); +}); + test('single-pointer helper failure propagates through the touch executor', async () => { await assert.rejects( withAndroidAdbProvider( diff --git a/src/platforms/android/__tests__/touch-executor.test.ts b/src/platforms/android/__tests__/touch-executor.test.ts index 51593c329..db398d70d 100644 --- a/src/platforms/android/__tests__/touch-executor.test.ts +++ b/src/platforms/android/__tests__/touch-executor.test.ts @@ -42,6 +42,7 @@ test('provider touch viewport bypasses local helper transport and is validated', calls += 1; return calls === 1 ? viewport : { ...viewport, width: 0 }; }, + touch: async () => {}, }, { serial: ANDROID_EMULATOR.id }, async () => { diff --git a/src/platforms/android/adb-executor.ts b/src/platforms/android/adb-executor.ts index 78c16a5ec..0531f14a6 100644 --- a/src/platforms/android/adb-executor.ts +++ b/src/platforms/android/adb-executor.ts @@ -2,7 +2,7 @@ import { AsyncLocalStorage } from 'node:async_hooks'; import type { Readable, Writable } from 'node:stream'; import type { DeviceInfo } from '../../kernel/device.ts'; import type { Rect } from '../../kernel/snapshot.ts'; -import type { AndroidTouchPlan } from './touch-plan.ts'; +import type { AndroidProviderTouchPlan } from './touch-plan.ts'; import { coerceExecResult, execFailureDetails, @@ -126,7 +126,7 @@ export type AndroidTextInjectionRequest = { export type AndroidTextInjector = (request: AndroidTextInjectionRequest) => Promise; export type AndroidTouchInjector = ( - request: AndroidTouchPlan, + request: AndroidProviderTouchPlan, ) => Promise | void>; export type AndroidGestureViewportProvider = () => Promise; @@ -148,13 +148,17 @@ type AndroidAdbProviderBase = { type AndroidTouchCapabilities = | { touch?: never; - gestureViewport?: AndroidGestureViewportProvider; + gestureViewport?: never; } | { touch: AndroidTouchInjector; gestureViewport: AndroidGestureViewportProvider; }; +export type AndroidTouchProvider = Required< + Pick +>; + export type AndroidAdbProvider = AndroidAdbProviderBase & AndroidTouchCapabilities; export type AndroidAdbProviderScopeOptions = { @@ -483,16 +487,9 @@ export function resolveAndroidTextInjector(device: DeviceInfo): AndroidTextInjec return scoped?.serial === device.id ? scoped.provider.text : undefined; } -export function resolveAndroidTouchInjector(device: DeviceInfo): AndroidTouchInjector | undefined { - const scoped = androidAdbProviderScope.getStore(); - return scoped?.serial === device.id ? scoped.provider.touch : undefined; -} - -export function resolveAndroidGestureViewportProvider( - device: DeviceInfo, -): AndroidGestureViewportProvider | undefined { +export function resolveAndroidTouchProvider(device: DeviceInfo): AndroidTouchProvider | undefined { const scoped = androidAdbProviderScope.getStore(); - return scoped?.serial === device.id ? scoped.provider.gestureViewport : undefined; + return scoped?.serial === device.id && scoped.provider.touch ? scoped.provider : undefined; } export function createAndroidPortReverseManager( diff --git a/src/platforms/android/input-actions.ts b/src/platforms/android/input-actions.ts index 39faaf382..5aec06f39 100644 --- a/src/platforms/android/input-actions.ts +++ b/src/platforms/android/input-actions.ts @@ -87,13 +87,11 @@ export async function longPressAndroid( y: number, durationMs = 800, ): Promise> { - const viewport = await readAndroidGestureViewport(device); const point = { x, y }; return await executeAndroidTouchPlan(device, { topology: 'single', intent: 'longPress', durationMs, - viewport, pointers: [ { pointerId: 0, @@ -273,26 +271,37 @@ export async function scrollAndroid( options?: { amount?: number; pixels?: number; durationMs?: number }, ): Promise> { const viewport = await readAndroidGestureViewport(device); - const scrollPlan = buildScrollGesturePlan({ + const relativePlan = buildScrollGesturePlan({ direction, amount: options?.amount, pixels: options?.pixels, referenceWidth: viewport.width, referenceHeight: viewport.height, }); + const scrollPlan = { + ...relativePlan, + x1: viewport.x + relativePlan.x1, + y1: viewport.y + relativePlan.y1, + x2: viewport.x + relativePlan.x2, + y2: viewport.y + relativePlan.y2, + }; const durationMs = options?.durationMs ?? 300; - const origin = { x: viewport.x + scrollPlan.x1, y: viewport.y + scrollPlan.y1 }; - const gesturePlan = buildGesturePlan( - { - intent: 'pan', - origin, - delta: { x: scrollPlan.x2 - scrollPlan.x1, y: scrollPlan.y2 - scrollPlan.y1 }, - durationMs, - }, - viewport, - 'android', + const backend = await executeAndroidTouchPlan( + device, + buildGesturePlan( + { + intent: 'pan', + origin: { x: scrollPlan.x1, y: scrollPlan.y1 }, + delta: { + x: scrollPlan.x2 - scrollPlan.x1, + y: scrollPlan.y2 - scrollPlan.y1, + }, + durationMs, + }, + viewport, + 'android', + ), ); - const backend = await executeAndroidTouchPlan(device, gesturePlan); return { ...scrollPlan, diff --git a/src/platforms/android/multitouch-helper-install.ts b/src/platforms/android/multitouch-helper-install.ts new file mode 100644 index 000000000..59b7e8f55 --- /dev/null +++ b/src/platforms/android/multitouch-helper-install.ts @@ -0,0 +1,99 @@ +import { AppError } from '../../kernel/errors.ts'; +import { + makeEnsureAndroidHelperInstalled, + resolveAndroidHelperArtifact, +} from './helper-package-install.ts'; +import { + readAndroidHelperManifestInteger, + readAndroidHelperManifestLiteral, + readAndroidHelperManifestSha256, + readAndroidHelperManifestString, +} from './instrumentation-helper.ts'; + +const HELPER_NAME = 'android-multitouch-helper'; +const HELPER_PACKAGE = 'com.callstack.agentdevice.multitouchhelper'; +const HELPER_RUNNER = 'com.callstack.agentdevice.multitouchhelper/.MultiTouchInstrumentation'; +export const ANDROID_MULTITOUCH_HELPER_PROTOCOL = 'android-multitouch-helper-v1'; +const HELPER_INSTALL_TIMEOUT_MS = 30_000; +const HELPER_LABEL = 'Android multi-touch helper'; +const MANIFEST_HELPER_LABEL = 'multi-touch helper'; + +type AndroidMultiTouchHelperManifest = { + name: 'android-multitouch-helper'; + version: string; + assetName: string; + sha256: string; + packageName: string; + versionCode: number; + instrumentationRunner: string; + statusProtocol: 'android-multitouch-helper-v1'; +}; + +export type AndroidMultiTouchHelperArtifact = { + apkPath: string; + manifest: AndroidMultiTouchHelperManifest; +}; + +export async function resolveAndroidMultiTouchHelperArtifact(): Promise { + return await resolveAndroidHelperArtifact({ + helperDirName: 'android-multitouch-helper', + manifestFileName: (version) => + `agent-device-android-multitouch-helper-${version}.manifest.json`, + parseManifest: parseAndroidMultiTouchHelperManifest, + unavailableMessage: + 'Android touch gestures require the bundled Android touch helper artifact, but it was not found or could not be read', + }); +} + +function parseAndroidMultiTouchHelperManifest(value: unknown): AndroidMultiTouchHelperManifest { + if (!value || typeof value !== 'object' || Array.isArray(value)) { + throw new AppError('INVALID_ARGS', 'Android multi-touch helper manifest must be an object.'); + } + const record = value as Record; + return { + name: readAndroidHelperManifestLiteral(record.name, 'name', HELPER_NAME, MANIFEST_HELPER_LABEL), + version: readAndroidHelperManifestString(record.version, 'version', MANIFEST_HELPER_LABEL), + assetName: readAndroidHelperManifestString( + record.assetName, + 'assetName', + MANIFEST_HELPER_LABEL, + ), + sha256: readAndroidHelperManifestSha256(record.sha256, MANIFEST_HELPER_LABEL), + packageName: readAndroidHelperManifestLiteral( + record.packageName, + 'packageName', + HELPER_PACKAGE, + MANIFEST_HELPER_LABEL, + ), + versionCode: readAndroidHelperManifestInteger( + record.versionCode, + 'versionCode', + MANIFEST_HELPER_LABEL, + ), + instrumentationRunner: readAndroidHelperManifestLiteral( + record.instrumentationRunner, + 'instrumentationRunner', + HELPER_RUNNER, + MANIFEST_HELPER_LABEL, + ), + statusProtocol: readAndroidHelperManifestLiteral( + record.statusProtocol, + 'statusProtocol', + ANDROID_MULTITOUCH_HELPER_PROTOCOL, + MANIFEST_HELPER_LABEL, + ), + }; +} + +const installedMultiTouchHelpers = new Set(); + +export const ensureAndroidMultiTouchHelper = + makeEnsureAndroidHelperInstalled({ + cache: installedMultiTouchHelpers, + installTimeoutMs: HELPER_INSTALL_TIMEOUT_MS, + helperLabel: HELPER_LABEL, + }); + +export function resetAndroidMultiTouchHelperInstallCache(): void { + installedMultiTouchHelpers.clear(); +} diff --git a/src/platforms/android/multitouch-helper.ts b/src/platforms/android/multitouch-helper.ts index 64ab710a6..5904cd626 100644 --- a/src/platforms/android/multitouch-helper.ts +++ b/src/platforms/android/multitouch-helper.ts @@ -9,49 +9,24 @@ import { resolveAndroidAdbProvider, type AndroidAdbExecutor, } from './adb-executor.ts'; -import { - makeEnsureAndroidHelperInstalled, - resolveAndroidHelperArtifact, -} from './helper-package-install.ts'; import { parseInstrumentationRecords, - readAndroidHelperManifestInteger, - readAndroidHelperManifestLiteral, - readAndroidHelperManifestSha256, - readAndroidHelperManifestString, readInstrumentationResultNumber, } from './instrumentation-helper.ts'; import { stopAndroidSnapshotHelperSessionForDevice } from './snapshot-helper.ts'; import { validateAndroidGestureViewport } from './gesture-viewport.ts'; import type { AndroidTouchPlan } from './touch-plan.ts'; +import { + ANDROID_MULTITOUCH_HELPER_PROTOCOL, + ensureAndroidMultiTouchHelper, + resolveAndroidMultiTouchHelperArtifact, +} from './multitouch-helper-install.ts'; -const HELPER_NAME = 'android-multitouch-helper'; -const HELPER_PACKAGE = 'com.callstack.agentdevice.multitouchhelper'; -const HELPER_RUNNER = 'com.callstack.agentdevice.multitouchhelper/.MultiTouchInstrumentation'; -const HELPER_PROTOCOL = 'android-multitouch-helper-v1'; -const HELPER_INSTALL_TIMEOUT_MS = 30_000; +const HELPER_PROTOCOL = ANDROID_MULTITOUCH_HELPER_PROTOCOL; const HELPER_GESTURE_TIMEOUT_MS = 45_000; const HELPER_GESTURE_TIMEOUT_OVERHEAD_MS = 15_000; const HELPER_NO_FINAL_RESULT = 'ANDROID_MULTITOUCH_HELPER_NO_FINAL_RESULT'; const HELPER_REPORTED_FAILURE = 'ANDROID_MULTITOUCH_HELPER_REPORTED_FAILURE'; -const HELPER_LABEL = 'Android multi-touch helper'; -const MANIFEST_HELPER_LABEL = 'multi-touch helper'; - -type AndroidMultiTouchHelperManifest = { - name: 'android-multitouch-helper'; - version: string; - assetName: string; - sha256: string; - packageName: string; - versionCode: number; - instrumentationRunner: string; - statusProtocol: 'android-multitouch-helper-v1'; -}; - -type AndroidMultiTouchHelperArtifact = { - apkPath: string; - manifest: AndroidMultiTouchHelperManifest; -}; type AndroidPlannedPointerTrajectory = { pointerId: 0 | 1; @@ -129,25 +104,15 @@ export async function readAndroidMultiTouchHelperViewport(device: DeviceInfo): P { allowFailure: true, timeoutMs: HELPER_GESTURE_TIMEOUT_MS }, ); const records = parseInstrumentationRecords(`${result.stdout}\n${result.stderr}`); - if ( - result.exitCode !== 0 && - !records.results.some((record) => record.agentDeviceProtocol === HELPER_PROTOCOL) - ) { - throw new AppError( - 'COMMAND_FAILED', - 'Android gesture viewport is unavailable', - execFailureDetails(result), - ); - } - const viewport = parseAndroidGestureViewportResult(records.results); - if (result.exitCode !== 0) { - throw new AppError( - 'COMMAND_FAILED', - 'Android gesture viewport is unavailable', - execFailureDetails(result), - ); + if (result.exitCode === 0) return parseAndroidGestureViewportResult(records.results); + if (records.results.some((record) => record.agentDeviceProtocol === HELPER_PROTOCOL)) { + parseAndroidGestureViewportResult(records.results); } - return viewport; + throw new AppError( + 'COMMAND_FAILED', + 'Android gesture viewport is unavailable', + execFailureDetails(result), + ); } export function parseAndroidGestureViewportResult(results: Array>): Rect { @@ -274,67 +239,3 @@ export function parseAndroidMultiTouchHelperOutput(output: string): Record { - return await resolveAndroidHelperArtifact({ - helperDirName: 'android-multitouch-helper', - manifestFileName: (version) => - `agent-device-android-multitouch-helper-${version}.manifest.json`, - parseManifest: parseAndroidMultiTouchHelperManifest, - unavailableMessage: - 'Android touch gestures require the bundled Android touch helper artifact, but it was not found or could not be read', - }); -} - -function parseAndroidMultiTouchHelperManifest(value: unknown): AndroidMultiTouchHelperManifest { - if (!value || typeof value !== 'object' || Array.isArray(value)) { - throw new AppError('INVALID_ARGS', 'Android multi-touch helper manifest must be an object.'); - } - const record = value as Record; - return { - name: readAndroidHelperManifestLiteral(record.name, 'name', HELPER_NAME, MANIFEST_HELPER_LABEL), - version: readAndroidHelperManifestString(record.version, 'version', MANIFEST_HELPER_LABEL), - assetName: readAndroidHelperManifestString( - record.assetName, - 'assetName', - MANIFEST_HELPER_LABEL, - ), - sha256: readAndroidHelperManifestSha256(record.sha256, MANIFEST_HELPER_LABEL), - packageName: readAndroidHelperManifestLiteral( - record.packageName, - 'packageName', - HELPER_PACKAGE, - MANIFEST_HELPER_LABEL, - ), - versionCode: readAndroidHelperManifestInteger( - record.versionCode, - 'versionCode', - MANIFEST_HELPER_LABEL, - ), - instrumentationRunner: readAndroidHelperManifestLiteral( - record.instrumentationRunner, - 'instrumentationRunner', - HELPER_RUNNER, - MANIFEST_HELPER_LABEL, - ), - statusProtocol: readAndroidHelperManifestLiteral( - record.statusProtocol, - 'statusProtocol', - HELPER_PROTOCOL, - MANIFEST_HELPER_LABEL, - ), - }; -} - -const installedMultiTouchHelpers = new Set(); - -export const ensureAndroidMultiTouchHelper = - makeEnsureAndroidHelperInstalled({ - cache: installedMultiTouchHelpers, - installTimeoutMs: HELPER_INSTALL_TIMEOUT_MS, - helperLabel: HELPER_LABEL, - }); - -export function resetAndroidMultiTouchHelperInstallCache(): void { - installedMultiTouchHelpers.clear(); -} diff --git a/src/platforms/android/touch-executor.ts b/src/platforms/android/touch-executor.ts index 862e7bee5..09bed327e 100644 --- a/src/platforms/android/touch-executor.ts +++ b/src/platforms/android/touch-executor.ts @@ -1,9 +1,6 @@ import type { DeviceInfo } from '../../kernel/device.ts'; import type { Rect } from '../../kernel/snapshot.ts'; -import { - resolveAndroidGestureViewportProvider, - resolveAndroidTouchInjector, -} from './adb-executor.ts'; +import { resolveAndroidTouchProvider } from './adb-executor.ts'; import { executeAndroidMultiTouchHelperPlan, readAndroidMultiTouchHelperViewport, @@ -15,16 +12,23 @@ export async function executeAndroidTouchPlan( device: DeviceInfo, plan: AndroidTouchPlan, ): Promise> { - const providerTouch = resolveAndroidTouchInjector(device); - if (providerTouch) { - const result = (await providerTouch(plan)) ?? {}; + const provider = resolveAndroidTouchProvider(device); + if (provider) { + const providerPlan = + plan.intent === 'longPress' + ? { + ...plan, + viewport: validateAndroidGestureViewport(await provider.gestureViewport()), + } + : plan; + const result = (await provider.touch(providerPlan)) ?? {}; return { backend: 'provider-native-touch', ...result }; } return await executeAndroidMultiTouchHelperPlan(device, plan); } export async function readAndroidGestureViewport(device: DeviceInfo): Promise { - const providerViewport = resolveAndroidGestureViewportProvider(device); - if (providerViewport) return validateAndroidGestureViewport(await providerViewport()); + const provider = resolveAndroidTouchProvider(device); + if (provider) return validateAndroidGestureViewport(await provider.gestureViewport()); return await readAndroidMultiTouchHelperViewport(device); } diff --git a/src/platforms/android/touch-plan.ts b/src/platforms/android/touch-plan.ts index 7e3d9fd34..cc87e86a3 100644 --- a/src/platforms/android/touch-plan.ts +++ b/src/platforms/android/touch-plan.ts @@ -5,8 +5,11 @@ export type AndroidLongPressTouchPlan = { topology: 'single'; intent: 'longPress'; durationMs: number; - viewport: Rect; pointers: readonly [PointerTrajectory]; }; export type AndroidTouchPlan = GesturePlan | AndroidLongPressTouchPlan; + +export type AndroidProviderTouchPlan = + | GesturePlan + | (AndroidLongPressTouchPlan & { viewport: Rect }); From 547de1cda7b8871515454d26d4ce2ea11d1749e4 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Micha=C5=82=20Pierzcha=C5=82a?= Date: Mon, 13 Jul 2026 19:13:38 +0200 Subject: [PATCH 14/16] fix: remove unused Android helper cache export --- .../multitouch-helper-install.test.ts | 29 +++++++------------ .../__tests__/multitouch-helper.test.ts | 5 +--- .../__tests__/touch-executor-helper.test.ts | 28 +++++++++++------- .../android/multitouch-helper-install.ts | 4 --- 4 files changed, 30 insertions(+), 36 deletions(-) diff --git a/src/platforms/android/__tests__/multitouch-helper-install.test.ts b/src/platforms/android/__tests__/multitouch-helper-install.test.ts index 8cb3402fc..a96f8a39e 100644 --- a/src/platforms/android/__tests__/multitouch-helper-install.test.ts +++ b/src/platforms/android/__tests__/multitouch-helper-install.test.ts @@ -3,18 +3,11 @@ import crypto from 'node:crypto'; import { promises as fs } from 'node:fs'; import os from 'node:os'; import path from 'node:path'; -import { beforeEach, test } from 'vitest'; -import { - ensureAndroidMultiTouchHelper, - resetAndroidMultiTouchHelperInstallCache, -} from '../multitouch-helper-install.ts'; +import { test } from 'vitest'; +import { ensureAndroidMultiTouchHelper } from '../multitouch-helper-install.ts'; import type { AndroidAdbExecutor, AndroidAdbProvider } from '../adb-executor.ts'; import { ANDROID_MULTITOUCH_HELPER_MANIFEST } from './multitouch-helper.fixtures.ts'; -beforeEach(() => { - resetAndroidMultiTouchHelperInstallCache(); -}); - test('helper install uses replace and test-package semantics', async () => { const fixture = await makeInstallFixture('helper-apk'); const installCalls: unknown[] = []; @@ -36,7 +29,7 @@ test('helper install uses replace and test-package semantics', async () => { adb, adbProvider: provider, artifact: fixture.artifact, - deviceKey: 'android:emulator-5554', + deviceKey: 'android:install-semantics', }); assert.equal(result.reason, 'missing'); @@ -64,7 +57,7 @@ test('same-version helper is current only when installed APK bytes match', async adb, adbProvider: provider, artifact: fixture.artifact, - deviceKey: 'android:emulator-5554', + deviceKey: 'android:matching-bytes', }); assert.equal(result.reason, 'current'); @@ -85,7 +78,7 @@ test('same-version helper is replaced when installed APK bytes differ', async () adb, adbProvider: provider, artifact: fixture.artifact, - deviceKey: 'android:emulator-5554', + deviceKey: 'android:mismatched-bytes', }); assert.equal(result.reason, 'mismatched'); @@ -104,7 +97,7 @@ test('same-version helper is replaced when installed APK identity is unavailable adb, adbProvider: provider, artifact: fixture.artifact, - deviceKey: 'android:emulator-5554', + deviceKey: 'android:unverifiable-identity', }); assert.equal(result.reason, 'unverifiable'); @@ -122,7 +115,7 @@ test('newer installed helper remains current without an unsafe downgrade', async adb, adbProvider: provider, artifact: fixture.artifact, - deviceKey: 'android:emulator-5554', + deviceKey: 'android:newer-helper', }); assert.equal(result.reason, 'current'); @@ -143,14 +136,14 @@ test('install memo includes artifact identity, not only version code', async () adb: device.adb, adbProvider: device.provider, artifact: first.artifact, - deviceKey: 'android:emulator-5554', + deviceKey: 'android:artifact-identity', }); device.setInstalledApk('first-helper'); const result = await ensureAndroidMultiTouchHelper({ adb: device.adb, adbProvider: device.provider, artifact: second.artifact, - deviceKey: 'android:emulator-5554', + deviceKey: 'android:artifact-identity', }); assert.equal(result.reason, 'mismatched'); @@ -168,7 +161,7 @@ test('install memo skips repeated APK reads for the same artifact identity', asy adb: device.adb, adbProvider: device.provider, artifact: fixture.artifact, - deviceKey: 'android:emulator-5554', + deviceKey: 'android:install-memo', }); await fs.rm(fixture.artifact.apkPath); const result = await ensureAndroidMultiTouchHelper({ @@ -181,7 +174,7 @@ test('install memo skips repeated APK reads for the same artifact identity', asy }, }, artifact: fixture.artifact, - deviceKey: 'android:emulator-5554', + deviceKey: 'android:install-memo', }); assert.equal(result.reason, 'current'); diff --git a/src/platforms/android/__tests__/multitouch-helper.test.ts b/src/platforms/android/__tests__/multitouch-helper.test.ts index b6050e3b5..404096d70 100644 --- a/src/platforms/android/__tests__/multitouch-helper.test.ts +++ b/src/platforms/android/__tests__/multitouch-helper.test.ts @@ -1,5 +1,5 @@ import assert from 'node:assert/strict'; -import { beforeEach, test } from 'vitest'; +import { test } from 'vitest'; import { buildGesturePlan } from '../../../contracts/gesture-plan.ts'; import { normalizeAndroidMultiTouchHelperGestureRequest, @@ -7,7 +7,6 @@ import { runAndroidMultiTouchHelperGesture, parseAndroidGestureViewportResult, } from '../multitouch-helper.ts'; -import { resetAndroidMultiTouchHelperInstallCache } from '../multitouch-helper-install.ts'; import { ANDROID_MULTITOUCH_HELPER_MANIFEST as manifest, androidMultiTouchResultRecord as resultRecord, @@ -15,8 +14,6 @@ import { const viewport = { x: 0, y: 0, width: 400, height: 800 }; -beforeEach(resetAndroidMultiTouchHelperInstallCache); - test('helper response parsing returns instrumentation evidence', () => { assert.deepEqual( parseAndroidMultiTouchHelperOutput( diff --git a/src/platforms/android/__tests__/touch-executor-helper.test.ts b/src/platforms/android/__tests__/touch-executor-helper.test.ts index 680e50655..83cb1cceb 100644 --- a/src/platforms/android/__tests__/touch-executor-helper.test.ts +++ b/src/platforms/android/__tests__/touch-executor-helper.test.ts @@ -5,7 +5,6 @@ import { buildGesturePlan } from '../../../contracts/gesture-plan.ts'; import { AppError } from '../../../kernel/errors.ts'; import { withAndroidAdbProvider } from '../adb-executor.ts'; import { longPressAndroid } from '../input-actions.ts'; -import { resetAndroidMultiTouchHelperInstallCache } from '../multitouch-helper-install.ts'; import { stopAndroidSnapshotHelperSessionForDevice } from '../snapshot-helper.ts'; import { executeAndroidTouchPlan, readAndroidGestureViewport } from '../touch-executor.ts'; import { @@ -35,13 +34,19 @@ vi.mock('../helper-package-install.ts', async (importOriginal) => { }); const mockStopSnapshotSession = vi.mocked(stopAndroidSnapshotHelperSessionForDevice); +let deviceSequence = 0; beforeEach(() => { - resetAndroidMultiTouchHelperInstallCache(); mockStopSnapshotSession.mockReset(); }); +function makeIsolatedDevice() { + deviceSequence += 1; + return { ...ANDROID_EMULATOR, id: `emulator-touch-${deviceSequence}` }; +} + test('helper gesture releases persistent snapshot instrumentation before touch instrumentation', async () => { + const device = makeIsolatedDevice(); const events: string[] = []; mockStopSnapshotSession.mockImplementation(async () => { events.push('snapshot-stop'); @@ -71,10 +76,10 @@ test('helper gesture releases persistent snapshot instrumentation before touch i throw new Error(`unexpected adb call: ${args.join(' ')}`); }, }, - { serial: ANDROID_EMULATOR.id }, + { serial: device.id }, async () => await executeAndroidTouchPlan( - ANDROID_EMULATOR, + device, buildGesturePlan( { intent: 'pan', @@ -94,6 +99,7 @@ test('helper gesture releases persistent snapshot instrumentation before touch i }); test('bare-adb long press executes one helper gesture without a viewport probe', async () => { + const device = makeIsolatedDevice(); const instrumentCalls: string[][] = []; const result = await withAndroidAdbProvider( { @@ -119,8 +125,8 @@ test('bare-adb long press executes one helper gesture without a viewport probe', throw new Error(`unexpected adb call: ${args.join(' ')}`); }, }, - { serial: ANDROID_EMULATOR.id }, - async () => await longPressAndroid(ANDROID_EMULATOR, 30, 40, 750), + { serial: device.id }, + async () => await longPressAndroid(device, 30, 40, 750), ); assert.equal(result.backend, 'android-multitouch-helper'); @@ -130,6 +136,7 @@ test('bare-adb long press executes one helper gesture without a viewport probe', }); test('single-pointer helper failure propagates through the touch executor', async () => { + const device = makeIsolatedDevice(); await assert.rejects( withAndroidAdbProvider( { @@ -158,10 +165,10 @@ test('single-pointer helper failure propagates through the touch executor', asyn throw new Error(`unexpected adb call: ${args.join(' ')}`); }, }, - { serial: ANDROID_EMULATOR.id }, + { serial: device.id }, async () => await executeAndroidTouchPlan( - ANDROID_EMULATOR, + device, buildGesturePlan( { intent: 'pan', @@ -186,6 +193,7 @@ test('single-pointer helper failure propagates through the touch executor', asyn }); test('helper viewport failure preserves its structured message and error type', async () => { + const device = makeIsolatedDevice(); await assert.rejects( withAndroidAdbProvider( { @@ -214,8 +222,8 @@ test('helper viewport failure preserves its structured message and error type', throw new Error(`unexpected adb call: ${args.join(' ')}`); }, }, - { serial: ANDROID_EMULATOR.id }, - async () => await readAndroidGestureViewport(ANDROID_EMULATOR), + { serial: device.id }, + async () => await readAndroidGestureViewport(device), ), (error: unknown) => { assert.ok(error instanceof AppError); diff --git a/src/platforms/android/multitouch-helper-install.ts b/src/platforms/android/multitouch-helper-install.ts index 59b7e8f55..786778e98 100644 --- a/src/platforms/android/multitouch-helper-install.ts +++ b/src/platforms/android/multitouch-helper-install.ts @@ -93,7 +93,3 @@ export const ensureAndroidMultiTouchHelper = installTimeoutMs: HELPER_INSTALL_TIMEOUT_MS, helperLabel: HELPER_LABEL, }); - -export function resetAndroidMultiTouchHelperInstallCache(): void { - installedMultiTouchHelpers.clear(); -} From cab53f64f2138de8a7e619ea6f24d49cd70bd432 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Micha=C5=82=20Pierzcha=C5=82a?= Date: Mon, 13 Jul 2026 19:34:45 +0200 Subject: [PATCH 15/16] fix: close Android gesture contract gaps --- docs/adr/0013-unified-gesture-plans.md | 4 ++- .../__tests__/timeout-policy.test.ts | 2 +- src/core/command-descriptor/registry.ts | 7 ++-- .../__tests__/recording-gestures.test.ts | 32 +++++++++++------ .../android/__tests__/input-actions.test.ts | 36 +++++++++++++++++-- src/platforms/android/input-actions.ts | 10 ++++-- src/utils/__tests__/daemon-client.test.ts | 7 ++-- 7 files changed, 74 insertions(+), 24 deletions(-) diff --git a/docs/adr/0013-unified-gesture-plans.md b/docs/adr/0013-unified-gesture-plans.md index 669aa33bb..eda0a470d 100644 --- a/docs/adr/0013-unified-gesture-plans.md +++ b/docs/adr/0013-unified-gesture-plans.md @@ -60,7 +60,9 @@ Platform adapters consume the canonical plan: available, otherwise to the bundled instrumentation helper. The helper injects the exact planned pointer samples. A stationary long-press needs no viewport on the helper path; the executor adds the paired provider-owned viewport only for provider-native touch. Android touch execution never - falls back to `adb input swipe`; the snapshot helper is stopped + falls back to `adb input swipe`. Public scroll durations below one 16 ms planner frame normalize + to that physical minimum and report the executed duration. Scroll evidence reports absolute + injected coordinates against zero-origin extents that include the viewport offset. The snapshot helper is stopped before local gesture instrumentation because Android permits only one instrumentation owner of `UiAutomation`. - iOS converts every planned point to native orientation and feeds the exact arrays to the existing diff --git a/src/core/command-descriptor/__tests__/timeout-policy.test.ts b/src/core/command-descriptor/__tests__/timeout-policy.test.ts index c79d5852f..3d9bba776 100644 --- a/src/core/command-descriptor/__tests__/timeout-policy.test.ts +++ b/src/core/command-descriptor/__tests__/timeout-policy.test.ts @@ -127,7 +127,7 @@ test('request envelopes deviating from the default are bounded, reviewed sets', install: 180_000, reinstall: 180_000, install_source: 180_000, - longpress: 150_000, + longpress: 210_000, test: 'unbounded', }; for (const descriptor of commandDescriptors) { diff --git a/src/core/command-descriptor/registry.ts b/src/core/command-descriptor/registry.ts index d61c48dbd..9d5bc04c1 100644 --- a/src/core/command-descriptor/registry.ts +++ b/src/core/command-descriptor/registry.ts @@ -705,9 +705,10 @@ export const RAW_COMMAND_DESCRIPTORS = [ capability: { apple: APPLE_SIM_AND_DEVICE, android: ANDROID_ALL, linux: LINUX_DEVICE }, timeoutPolicy: { ...SETTLE_FLAG_PRESERVE_DAEMON_TIMEOUT_POLICY, - // A valid hold may last 120 seconds; leave the helper's 15-second - // completion overhead inside the outer request envelope too. - envelopeMs: 150_000, + // Android's cold path may inspect/install the helper, hand off a running + // snapshot helper, hold for 120 seconds, then use 15 seconds of helper + // completion overhead. Keep that complete route inside the envelope. + envelopeMs: 210_000, }, postActionObservation: postActionObservation('longpress'), batchable: true, diff --git a/src/daemon/__tests__/recording-gestures.test.ts b/src/daemon/__tests__/recording-gestures.test.ts index a636e14c5..81c220cbe 100644 --- a/src/daemon/__tests__/recording-gestures.test.ts +++ b/src/daemon/__tests__/recording-gestures.test.ts @@ -108,26 +108,36 @@ test('scroll augmentation preserves explicit reference frame from platform resul assert.equal(augmented.y2, 240); }); -test('scroll augmentation preserves explicit pixel travel coordinates', () => { +test('scroll visualization preserves absolute travel in its zero-origin reference frame', () => { const session = makeSession(); session.snapshot = undefined; const augmented = augmentScrollVisualizationResult(session, 'scroll', ['down'], { direction: 'down', pixels: 240, - x1: 201, - y1: 557, - x2: 201, - y2: 317, - referenceWidth: 402, - referenceHeight: 874, + x1: 211, + y1: 577, + x2: 211, + y2: 337, + referenceWidth: 412, + referenceHeight: 894, }) as Record; - assert.equal(augmented.x1, 201); - assert.equal(augmented.y1, 557); - assert.equal(augmented.x2, 201); - assert.equal(augmented.y2, 317); + recordTouchVisualizationEvent(session, 'scroll', ['down'], augmented, {}, 1_500); + + assert.equal(augmented.x1, 211); + assert.equal(augmented.y1, 577); + assert.equal(augmented.x2, 211); + assert.equal(augmented.y2, 337); assert.equal(augmented.pixels, 240); + const event = session.recording?.gestureEvents[0]; + assert.equal(event?.kind, 'scroll'); + assert.equal(event?.referenceWidth, 412); + assert.equal(event?.referenceHeight, 894); + assert.equal(event?.x, 211); + assert.equal(event?.y, 577); + assert.equal(event?.x2, 211); + assert.equal(event?.y2, 337); }); test('gesture recording prefers native runner timing when available', () => { diff --git a/src/platforms/android/__tests__/input-actions.test.ts b/src/platforms/android/__tests__/input-actions.test.ts index 4120b210f..8220f4e95 100644 --- a/src/platforms/android/__tests__/input-actions.test.ts +++ b/src/platforms/android/__tests__/input-actions.test.ts @@ -37,8 +37,8 @@ test('scrollAndroid plans explicit pixel travel through semantic touch injection assert.deepEqual(touch.pointers[0]?.samples.at(-1)?.point, { x: 550, y: 860 }); assert.equal(result.pixels, 240); assert.equal(result.durationMs, 120); - assert.equal(result.referenceWidth, 1080); - assert.equal(result.referenceHeight, 1920); + assert.equal(result.referenceWidth, 1090); + assert.equal(result.referenceHeight, 1940); assert.equal(result.x1, 550); assert.equal(result.y1, 1100); assert.equal(result.x2, 550); @@ -47,6 +47,38 @@ test('scrollAndroid plans explicit pixel travel through semantic touch injection assert.equal(result.injected, true); }); +test('scrollAndroid accepts sub-frame public durations at the Android planner minimum', async () => { + const touchCalls: Parameters[0][] = []; + const results = await withAndroidAdbProvider( + { + exec: async () => { + throw new Error('adb must not run'); + }, + gestureViewport: async () => ({ x: 0, y: 0, width: 1080, height: 1920 }), + touch: async (request) => { + touchCalls.push(request); + }, + }, + { serial: ANDROID_EMULATOR.id }, + async () => { + const outputs: Record[] = []; + for (const durationMs of [0, 15]) { + outputs.push(await scrollAndroid(ANDROID_EMULATOR, 'down', { durationMs })); + } + return outputs; + }, + ); + + assert.deepEqual( + touchCalls.map((call) => call.durationMs), + [16, 16], + ); + assert.deepEqual( + results.map((result) => result.durationMs), + [16, 16], + ); +}); + test('longPressAndroid sends a stationary semantic touch plan', async () => { const touchCalls: Parameters[0][] = []; const result = await withAndroidAdbProvider( diff --git a/src/platforms/android/input-actions.ts b/src/platforms/android/input-actions.ts index 5aec06f39..f725e9fe3 100644 --- a/src/platforms/android/input-actions.ts +++ b/src/platforms/android/input-actions.ts @@ -2,7 +2,7 @@ import { AppError } from '../../kernel/errors.ts'; import type { DeviceInfo } from '../../kernel/device.ts'; import { emitDiagnostic } from '../../utils/diagnostics.ts'; import type { DeviceRotation } from '../../contracts/device-rotation.ts'; -import { buildGesturePlan } from '../../contracts/gesture-plan.ts'; +import { buildGesturePlan, GESTURE_DURATION_MIN_MS } from '../../contracts/gesture-plan.ts'; import { buildScrollGesturePlan, type ScrollDirection } from '../../contracts/scroll-gesture.ts'; import { toAndroidTvRemoteKeyevent, type TvRemoteButton } from '../../contracts/tv-remote.ts'; import { runAndroidAdb, sleep } from './adb.ts'; @@ -280,12 +280,16 @@ export async function scrollAndroid( }); const scrollPlan = { ...relativePlan, + // Injected coordinates are absolute, so their zero-origin reference frame + // must include the viewport offset as well as its dimensions. + referenceWidth: viewport.x + viewport.width, + referenceHeight: viewport.y + viewport.height, x1: viewport.x + relativePlan.x1, y1: viewport.y + relativePlan.y1, x2: viewport.x + relativePlan.x2, y2: viewport.y + relativePlan.y2, }; - const durationMs = options?.durationMs ?? 300; + const durationMs = Math.max(options?.durationMs ?? 300, GESTURE_DURATION_MIN_MS); const backend = await executeAndroidTouchPlan( device, buildGesturePlan( @@ -305,7 +309,7 @@ export async function scrollAndroid( return { ...scrollPlan, - ...(options?.durationMs !== undefined ? { durationMs: options.durationMs } : {}), + ...(options?.durationMs !== undefined ? { durationMs } : {}), ...backend, }; } diff --git a/src/utils/__tests__/daemon-client.test.ts b/src/utils/__tests__/daemon-client.test.ts index 9bb15aa40..e2a0fba2e 100644 --- a/src/utils/__tests__/daemon-client.test.ts +++ b/src/utils/__tests__/daemon-client.test.ts @@ -295,7 +295,8 @@ test('interaction --settle budgets add post-action settle time on top of the nor }), 125_000, ); - // Longpress keeps its 120-second maximum hold inside the outer envelope. + // Longpress keeps the cold Android helper route and its 120-second maximum + // hold inside the outer envelope. assert.equal( resolveDaemonRequestTimeoutMs({ ...base, @@ -303,7 +304,7 @@ test('interaction --settle budgets add post-action settle time on top of the nor positionals: ['300', '500', '120000'], flags: {}, }), - 150_000, + 210_000, ); // Bare --settle adds its default budget after the longpress-specific base, // so a maximum hold still leaves room for post-action observation. @@ -313,7 +314,7 @@ test('interaction --settle budgets add post-action settle time on top of the nor command: 'longpress', flags: { settle: true }, }), - 190_000, + 250_000, ); // Bare timeoutMs without --settle remains wire-compatible with older touch // command clients: it is ignored instead of opting into settle semantics. From c971422776caeaf9d318797264d5ca7ee86f2584 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Micha=C5=82=20Pierzcha=C5=82a?= Date: Mon, 13 Jul 2026 20:01:26 +0200 Subject: [PATCH 16/16] test: cover max Android helper gesture timeout --- .../__tests__/multitouch-helper.test.ts | 50 +++++++++++++------ 1 file changed, 35 insertions(+), 15 deletions(-) diff --git a/src/platforms/android/__tests__/multitouch-helper.test.ts b/src/platforms/android/__tests__/multitouch-helper.test.ts index 404096d70..c073ce5d8 100644 --- a/src/platforms/android/__tests__/multitouch-helper.test.ts +++ b/src/platforms/android/__tests__/multitouch-helper.test.ts @@ -14,6 +14,24 @@ import { const viewport = { x: 0, y: 0, width: 400, height: 800 }; +function longPressRequest(durationMs = 120_000) { + const point = { x: 20, y: 30 }; + return normalizeAndroidMultiTouchHelperGestureRequest({ + topology: 'single', + intent: 'longPress', + durationMs, + pointers: [ + { + pointerId: 0, + samples: [ + { offsetMs: 0, point }, + { offsetMs: durationMs, point }, + ], + }, + ], + }); +} + test('helper response parsing returns instrumentation evidence', () => { assert.deepEqual( parseAndroidMultiTouchHelperOutput( @@ -81,21 +99,7 @@ test('single-pointer plans use the same exact helper protocol', () => { }); test('Android long press lowers to a stationary single-pointer helper request', () => { - const point = { x: 20, y: 30 }; - const request = normalizeAndroidMultiTouchHelperGestureRequest({ - topology: 'single', - intent: 'longPress', - durationMs: 120_000, - pointers: [ - { - pointerId: 0, - samples: [ - { offsetMs: 0, point }, - { offsetMs: 120_000, point }, - ], - }, - ], - }); + const request = longPressRequest(); assert.equal(request.kind, 'swipe'); assert.equal(request.durationMs, 120_000); assert.deepEqual(request.pointers[0]?.samples, [ @@ -104,6 +108,22 @@ test('Android long press lowers to a stationary single-pointer helper request', ]); }); +test('max-duration long press extends the helper process timeout', async () => { + await runAndroidMultiTouchHelperGesture({ + adb: async (_args, options) => { + assert.equal(options?.timeoutMs, 135_000); + return { + exitCode: 0, + stdout: [resultRecord({ ok: 'true', kind: 'swipe' }), 'INSTRUMENTATION_CODE: 0'].join('\n'), + stderr: '', + }; + }, + request: longPressRequest(), + packageName: manifest.packageName, + instrumentationRunner: manifest.instrumentationRunner, + }); +}); + test('transform sends the planner-owned choreography without regenerating geometry', async () => { const plan = buildGesturePlan( {