diff --git a/CONTEXT.md b/CONTEXT.md index db4c60154..edaeac265 100644 --- a/CONTEXT.md +++ b/CONTEXT.md @@ -48,6 +48,12 @@ - Coordinate-first resolved element activation: iOS/macOS runner interaction pattern where a selector or text query resolves the semantic `XCUIElement`, then activation uses the element's resolved center coordinate when a frame is available. This keeps target selection semantic while avoiding `XCUIElement.tap()` post-action element re-resolution after normal navigation. tvOS remains focus/remote-driven. - Interaction dispatch path: one concrete route an interaction command takes to the device (runtime selector/ref resolution, direct iOS selector, native ref via web clickRef, coordinate, maestro non-hittable fallback). Every path classifies every guarantee in the ADR 0011 registry. - Gesture plan: typed, platform-neutral normalization of one- or two-contact gesture intent into bounded pointer trajectories. Contact topology is separate from motion; two-contact intent remains pan/pinch/rotate/transform even when native injection shares one executor. See ADR 0013. +- Android planned-touch executor: Android-local adapter seam that accepts `AndroidTouchPlan`—the + platform-neutral `GesturePlan` plus Android's stationary long-press plan—and selects the paired + provider-native touch/viewport adapter or bundled instrumentation-helper adapter. Scroll and + long-press retain their command semantics and only share physical touch execution through this + seam. Helper long-press executes its absolute stationary path without a viewport probe; provider + long-press receives its paired provider-owned viewport. See ADR 0013. - Multi-touch geometry: the internal initial span and angle plus centroid translation, scale, and rotation used to build both contact trajectories. Geometry is viewport-aware and fails early when the requested motion cannot fit; it is not a public tuning surface. - Guarantee cell: one (dispatch path, guarantee) entry in `src/contracts/interaction-guarantees.ts`, classified as runtime/runner/delegated/inapplicable/waived. Completeness is a compile error; honesty is gate-tested. - Owned waiver: a `gap:`-prefixed waived cell carrying a `trackingIssue` URL. Waivers are diffable debt with an owner, never folklore. diff --git a/android-multitouch-helper/src/main/java/com/callstack/agentdevice/multitouchhelper/MultiTouchInstrumentation.java b/android-multitouch-helper/src/main/java/com/callstack/agentdevice/multitouchhelper/MultiTouchInstrumentation.java index 234f6c3aa..96be8ef13 100644 --- a/android-multitouch-helper/src/main/java/com/callstack/agentdevice/multitouchhelper/MultiTouchInstrumentation.java +++ b/android-multitouch-helper/src/main/java/com/callstack/agentdevice/multitouchhelper/MultiTouchInstrumentation.java @@ -19,8 +19,8 @@ public final class MultiTouchInstrumentation extends Instrumentation { private static final String PROTOCOL = "android-multitouch-helper-v1"; private static final String HELPER_API_VERSION = "1"; - private static final int MIN_DURATION_MS = 16; - private static final int MAX_DURATION_MS = 10_000; + private static final int MIN_DURATION_MS = 0; + private static final int MAX_DURATION_MS = 120_000; private Bundle arguments; @Override @@ -204,7 +204,8 @@ private static PointerPath[] readPointers(JSONArray pointers, String kind, int d throw new IllegalArgumentException("Planned sample coordinates must be finite"); } long offsetMs = (long) rawOffsetMs; - if (offsetMs <= previousOffsetMs) { + if (offsetMs < previousOffsetMs + || (offsetMs == previousOffsetMs && durationMs != 0)) { throw new IllegalArgumentException("Planned sample offsets must be strictly increasing"); } parsed[sampleIndex] = new PointerSample(offsetMs, (float) x, (float) y); @@ -307,7 +308,7 @@ private static MotionEvent motionEvent( private static int requireDuration(int value) { if (value < MIN_DURATION_MS || value > MAX_DURATION_MS) { - throw new IllegalArgumentException("durationMs must be between 16 and 10000"); + throw new IllegalArgumentException("durationMs must be between 0 and 120000"); } return value; } diff --git a/docs/adr/0013-unified-gesture-plans.md b/docs/adr/0013-unified-gesture-plans.md index ee35b36fa..eda0a470d 100644 --- a/docs/adr/0013-unified-gesture-plans.md +++ b/docs/adr/0013-unified-gesture-plans.md @@ -15,8 +15,9 @@ Two-contact geometry also differed by platform. Android used a fixed radius whil radius from the app frame. Neither path validated every planned point against the active interaction viewport before injection. -`scroll` remains separate because it owns viewport/edge traversal, content-state verification, and -runner fallback policy rather than a single physical gesture. +`scroll` remains a separate command because it owns viewport/edge traversal and content-state +verification rather than a single physical gesture. Its Android physical movement still lowers to +the Android planned-touch executor. ## Decision @@ -54,11 +55,16 @@ no established automation use case justifies a public tuning surface. Platform adapters consume the canonical plan: -- Android sends the plan to provider-native touch injection when available, otherwise to the - bundled instrumentation helper. The helper injects the exact planned pointer samples. A - two-contact plan never falls back to `adb input swipe`; issue #690 separately owns removal of the - existing one-contact fallback. The snapshot helper is stopped before local gesture - instrumentation because Android permits only one instrumentation owner of `UiAutomation`. +- Android's `executeAndroidTouchPlan` adapter seam sends planned touch, including gesture plans plus + the physical movement for scroll and long-press, to provider-native touch injection when + available, otherwise to the bundled instrumentation helper. The helper injects the exact planned + pointer samples. A stationary long-press needs no viewport on the helper path; the executor adds + the paired provider-owned viewport only for provider-native touch. Android touch execution never + falls back to `adb input swipe`. Public scroll durations below one 16 ms planner frame normalize + to that physical minimum and report the executed duration. Scroll evidence reports absolute + injected coordinates against zero-origin extents that include the viewport offset. The snapshot helper is stopped + before local gesture instrumentation because Android permits only one instrumentation owner of + `UiAutomation`. - iOS converts every planned point to native orientation and feeds the exact arrays to the existing private XCTest event bridge. macOS lowers a one-contact plan to its drag executor and tvOS lowers it to remote direction. Core admission and the Apple adapter both consume the same shared @@ -93,6 +99,9 @@ selectors or refs and therefore cannot claim element-targeting guarantees. - One-finger pan remains the default and explicit two-finger pan retains pan intent. - The active viewport is resolved for each gesture, so rotation, keyboard, and window changes do not use stale geometry. +- On bare ADB, Android scroll and long-press require the bundled touch helper and `UiAutomation`; + helper installation or runtime failure is surfaced directly rather than degrading to an + approximate `adb input swipe`. - Pointer plans are larger than scalar requests but bounded by duration and the 16 ms sample cadence; deleting duplicate scalar executors offsets the package cost. @@ -106,4 +115,5 @@ selectors or refs and therefore cannot claim element-targeting guarantees. fling/swipe as a quick directional throw and pan as deliberate timed translation. - Add `two-finger-pan`: rejected because pointer count is topology, not a new motion intent. - Expose span/angle controls: rejected until a concrete automation use case needs them. -- Consolidate scroll: rejected because edge/content verification and fallback policy are distinct. +- Consolidate scroll command semantics: rejected because edge/content verification is distinct; + only its Android physical touch execution is shared. diff --git a/src/core/command-descriptor/__tests__/timeout-policy.test.ts b/src/core/command-descriptor/__tests__/timeout-policy.test.ts index f0bddb799..3d9bba776 100644 --- a/src/core/command-descriptor/__tests__/timeout-policy.test.ts +++ b/src/core/command-descriptor/__tests__/timeout-policy.test.ts @@ -127,6 +127,7 @@ test('request envelopes deviating from the default are bounded, reviewed sets', install: 180_000, reinstall: 180_000, install_source: 180_000, + longpress: 210_000, test: 'unbounded', }; for (const descriptor of commandDescriptors) { diff --git a/src/core/command-descriptor/registry.ts b/src/core/command-descriptor/registry.ts index ab33a90c9..9d5bc04c1 100644 --- a/src/core/command-descriptor/registry.ts +++ b/src/core/command-descriptor/registry.ts @@ -703,7 +703,13 @@ export const RAW_COMMAND_DESCRIPTORS = [ daemon: { route: 'interaction', replayScopedAction: true, androidBlockingDialogGuard: true }, dispatch: {}, capability: { apple: APPLE_SIM_AND_DEVICE, android: ANDROID_ALL, linux: LINUX_DEVICE }, - timeoutPolicy: interactionTimeoutPolicy('longpress'), + timeoutPolicy: { + ...SETTLE_FLAG_PRESERVE_DAEMON_TIMEOUT_POLICY, + // Android's cold path may inspect/install the helper, hand off a running + // snapshot helper, hold for 120 seconds, then use 15 seconds of helper + // completion overhead. Keep that complete route inside the envelope. + envelopeMs: 210_000, + }, postActionObservation: postActionObservation('longpress'), batchable: true, }, diff --git a/src/core/interactors/android.ts b/src/core/interactors/android.ts index 26918ecaa..0a6ece3ac 100644 --- a/src/core/interactors/android.ts +++ b/src/core/interactors/android.ts @@ -17,9 +17,9 @@ import { typeAndroid, } from '../../platforms/android/input-actions.ts'; import { - performGestureAndroid, + executeAndroidTouchPlan, readAndroidGestureViewport, -} from '../../platforms/android/multitouch-helper.ts'; +} from '../../platforms/android/touch-executor.ts'; import { readAndroidClipboardText, writeAndroidClipboardText, @@ -53,7 +53,7 @@ export function createAndroidInteractor(device: DeviceInfo): Interactor { type: (text, delayMs) => typeAndroid(device, text, delayMs), fill: (x, y, text, delayMs) => fillAndroid(device, x, y, text, delayMs), scroll: (direction, options) => scrollAndroid(device, direction, options), - performGesture: (plan) => performGestureAndroid(device, plan), + performGesture: (plan) => executeAndroidTouchPlan(device, plan), gestureViewport: () => readAndroidGestureViewport(device), screenshot: (outPath, options) => screenshotAndroid(device, outPath, options), snapshot: async (options) => { diff --git a/src/daemon/__tests__/recording-gestures.test.ts b/src/daemon/__tests__/recording-gestures.test.ts index a636e14c5..81c220cbe 100644 --- a/src/daemon/__tests__/recording-gestures.test.ts +++ b/src/daemon/__tests__/recording-gestures.test.ts @@ -108,26 +108,36 @@ test('scroll augmentation preserves explicit reference frame from platform resul assert.equal(augmented.y2, 240); }); -test('scroll augmentation preserves explicit pixel travel coordinates', () => { +test('scroll visualization preserves absolute travel in its zero-origin reference frame', () => { const session = makeSession(); session.snapshot = undefined; const augmented = augmentScrollVisualizationResult(session, 'scroll', ['down'], { direction: 'down', pixels: 240, - x1: 201, - y1: 557, - x2: 201, - y2: 317, - referenceWidth: 402, - referenceHeight: 874, + x1: 211, + y1: 577, + x2: 211, + y2: 337, + referenceWidth: 412, + referenceHeight: 894, }) as Record; - assert.equal(augmented.x1, 201); - assert.equal(augmented.y1, 557); - assert.equal(augmented.x2, 201); - assert.equal(augmented.y2, 317); + recordTouchVisualizationEvent(session, 'scroll', ['down'], augmented, {}, 1_500); + + assert.equal(augmented.x1, 211); + assert.equal(augmented.y1, 577); + assert.equal(augmented.x2, 211); + assert.equal(augmented.y2, 337); assert.equal(augmented.pixels, 240); + const event = session.recording?.gestureEvents[0]; + assert.equal(event?.kind, 'scroll'); + assert.equal(event?.referenceWidth, 412); + assert.equal(event?.referenceHeight, 894); + assert.equal(event?.x, 211); + assert.equal(event?.y, 577); + assert.equal(event?.x2, 211); + assert.equal(event?.y2, 337); }); test('gesture recording prefers native runner timing when available', () => { diff --git a/src/platforms/android/__tests__/input-actions.test.ts b/src/platforms/android/__tests__/input-actions.test.ts index d9067b94c..8220f4e95 100644 --- a/src/platforms/android/__tests__/input-actions.test.ts +++ b/src/platforms/android/__tests__/input-actions.test.ts @@ -1,50 +1,120 @@ import { test } from 'vitest'; import assert from 'node:assert/strict'; import { promises as fs } from 'node:fs'; -import os from 'node:os'; -import path from 'node:path'; import { fillAndroid, + longPressAndroid, rotateAndroid, scrollAndroid, - swipeAndroid, typeAndroid, } from '../input-actions.ts'; -import type { DeviceInfo } from '../../../kernel/device.ts'; import { AppError } from '../../../kernel/errors.ts'; import { withScriptedAdb } from '../../../__tests__/test-utils/mocked-binaries.ts'; +import { ANDROID_EMULATOR } from '../../../__tests__/test-utils/index.ts'; +import { withAndroidAdbProvider, type AndroidTouchInjector } from '../adb-executor.ts'; -test('scrollAndroid supports explicit pixel travel distance', async () => { - await withScriptedAdb( - 'agent-device-android-scroll-pixels-', - [ - '#!/bin/sh', - 'printf "%s\\n" "$@" >> "$AGENT_DEVICE_TEST_ARGS_FILE"', - 'if [ "$1" = "-s" ]; then', - ' shift', - ' shift', - 'fi', - 'if [ "$1" = "shell" ] && [ "$2" = "wm" ] && [ "$3" = "size" ]; then', - ' echo "Physical size: 1080x1920"', - ' exit 0', - 'fi', - 'exit 0', - '', - ].join('\n'), - async ({ argsLogPath, device }) => { - const result = await scrollAndroid(device, 'down', { pixels: 240, durationMs: 120 }); - const args = await fs.readFile(argsLogPath, 'utf8'); +test('scrollAndroid plans explicit pixel travel through semantic touch injection', async () => { + const touchCalls: Parameters[0][] = []; + const result = await withAndroidAdbProvider( + { + exec: async () => { + throw new Error('adb must not run'); + }, + gestureViewport: async () => ({ x: 10, y: 20, width: 1080, height: 1920 }), + touch: async (request) => { + touchCalls.push(request); + return { injected: true }; + }, + }, + { serial: ANDROID_EMULATOR.id }, + async () => await scrollAndroid(ANDROID_EMULATOR, 'down', { pixels: 240, durationMs: 120 }), + ); + + assert.equal(touchCalls.length, 1); + const touch = touchCalls[0]!; + assert.equal(touch.intent, 'pan'); + assert.deepEqual(touch.pointers[0]?.samples[0]?.point, { x: 550, y: 1100 }); + assert.deepEqual(touch.pointers[0]?.samples.at(-1)?.point, { x: 550, y: 860 }); + assert.equal(result.pixels, 240); + assert.equal(result.durationMs, 120); + assert.equal(result.referenceWidth, 1090); + assert.equal(result.referenceHeight, 1940); + assert.equal(result.x1, 550); + assert.equal(result.y1, 1100); + assert.equal(result.x2, 550); + assert.equal(result.y2, 860); + assert.equal(result.backend, 'provider-native-touch'); + assert.equal(result.injected, true); +}); - assert.match(args, /shell\ninput\nswipe\n540\n1080\n540\n840\n120\n/); - assert.doesNotMatch(args, /uiautomator|dump/); - assert.equal(result.pixels, 240); - assert.equal(result.durationMs, 120); - assert.equal(result.referenceWidth, 1080); - assert.equal(result.referenceHeight, 1920); +test('scrollAndroid accepts sub-frame public durations at the Android planner minimum', async () => { + const touchCalls: Parameters[0][] = []; + const results = await withAndroidAdbProvider( + { + exec: async () => { + throw new Error('adb must not run'); + }, + gestureViewport: async () => ({ x: 0, y: 0, width: 1080, height: 1920 }), + touch: async (request) => { + touchCalls.push(request); + }, }, + { serial: ANDROID_EMULATOR.id }, + async () => { + const outputs: Record[] = []; + for (const durationMs of [0, 15]) { + outputs.push(await scrollAndroid(ANDROID_EMULATOR, 'down', { durationMs })); + } + return outputs; + }, + ); + + assert.deepEqual( + touchCalls.map((call) => call.durationMs), + [16, 16], + ); + assert.deepEqual( + results.map((result) => result.durationMs), + [16, 16], ); }); +test('longPressAndroid sends a stationary semantic touch plan', async () => { + const touchCalls: Parameters[0][] = []; + const result = await withAndroidAdbProvider( + { + exec: async () => { + throw new Error('adb must not run'); + }, + gestureViewport: async () => ({ x: 10, y: 20, width: 300, height: 500 }), + touch: async (request) => { + touchCalls.push(request); + }, + }, + { serial: ANDROID_EMULATOR.id }, + async () => await longPressAndroid(ANDROID_EMULATOR, 30, 40, 750), + ); + + assert.deepEqual(touchCalls, [ + { + topology: 'single', + intent: 'longPress', + durationMs: 750, + viewport: { x: 10, y: 20, width: 300, height: 500 }, + pointers: [ + { + pointerId: 0, + samples: [ + { offsetMs: 0, point: { x: 30, y: 40 } }, + { offsetMs: 750, point: { x: 30, y: 40 } }, + ], + }, + ], + }, + ]); + assert.equal(result.backend, 'provider-native-touch'); +}); + test('rotateAndroid locks auto-rotate and sets user rotation', async () => { await withScriptedAdb( 'agent-device-android-rotate-landscape-left-', @@ -59,56 +129,6 @@ test('rotateAndroid locks auto-rotate and sets user rotation', async () => { ); }); -test('swipeAndroid invokes adb input swipe with duration', async () => { - const tmpDir = await fs.mkdtemp(path.join(os.tmpdir(), 'agent-device-swipe-test-')); - const adbPath = path.join(tmpDir, 'adb'); - const argsLogPath = path.join(tmpDir, 'args.log'); - await fs.writeFile( - adbPath, - '#!/bin/sh\nprintf "%s\\n" "$@" > "$AGENT_DEVICE_TEST_ARGS_FILE"\nexit 0\n', - 'utf8', - ); - await fs.chmod(adbPath, 0o755); - - const previousPath = process.env.PATH; - const previousArgsFile = process.env.AGENT_DEVICE_TEST_ARGS_FILE; - process.env.PATH = `${tmpDir}${path.delimiter}${previousPath ?? ''}`; - process.env.AGENT_DEVICE_TEST_ARGS_FILE = argsLogPath; - - const device: DeviceInfo = { - platform: 'android', - id: 'emulator-5554', - name: 'Pixel', - kind: 'emulator', - booted: true, - }; - - try { - await swipeAndroid(device, 10, 20, 30, 40, 250); - const args = (await fs.readFile(argsLogPath, 'utf8')).trim().split('\n').filter(Boolean); - assert.deepEqual(args, [ - '-s', - 'emulator-5554', - 'shell', - 'input', - 'swipe', - '10', - '20', - '30', - '40', - '250', - ]); - } finally { - process.env.PATH = previousPath; - if (previousArgsFile === undefined) { - delete process.env.AGENT_DEVICE_TEST_ARGS_FILE; - } else { - process.env.AGENT_DEVICE_TEST_ARGS_FILE = previousArgsFile; - } - await fs.rm(tmpDir, { recursive: true, force: true }); - } -}); - test('typeAndroid chunks ASCII input text for shell fallback', async () => { await withScriptedAdb( 'agent-device-android-type-ascii-chunked-', diff --git a/src/platforms/android/__tests__/multitouch-helper-install.test.ts b/src/platforms/android/__tests__/multitouch-helper-install.test.ts index 5f270b1e3..a96f8a39e 100644 --- a/src/platforms/android/__tests__/multitouch-helper-install.test.ts +++ b/src/platforms/android/__tests__/multitouch-helper-install.test.ts @@ -3,18 +3,11 @@ import crypto from 'node:crypto'; import { promises as fs } from 'node:fs'; import os from 'node:os'; import path from 'node:path'; -import { beforeEach, test } from 'vitest'; -import { - ensureAndroidMultiTouchHelper, - resetAndroidMultiTouchHelperInstallCache, -} from '../multitouch-helper.ts'; +import { test } from 'vitest'; +import { ensureAndroidMultiTouchHelper } from '../multitouch-helper-install.ts'; import type { AndroidAdbExecutor, AndroidAdbProvider } from '../adb-executor.ts'; import { ANDROID_MULTITOUCH_HELPER_MANIFEST } from './multitouch-helper.fixtures.ts'; -beforeEach(() => { - resetAndroidMultiTouchHelperInstallCache(); -}); - test('helper install uses replace and test-package semantics', async () => { const fixture = await makeInstallFixture('helper-apk'); const installCalls: unknown[] = []; @@ -36,7 +29,7 @@ test('helper install uses replace and test-package semantics', async () => { adb, adbProvider: provider, artifact: fixture.artifact, - deviceKey: 'android:emulator-5554', + deviceKey: 'android:install-semantics', }); assert.equal(result.reason, 'missing'); @@ -64,7 +57,7 @@ test('same-version helper is current only when installed APK bytes match', async adb, adbProvider: provider, artifact: fixture.artifact, - deviceKey: 'android:emulator-5554', + deviceKey: 'android:matching-bytes', }); assert.equal(result.reason, 'current'); @@ -85,7 +78,7 @@ test('same-version helper is replaced when installed APK bytes differ', async () adb, adbProvider: provider, artifact: fixture.artifact, - deviceKey: 'android:emulator-5554', + deviceKey: 'android:mismatched-bytes', }); assert.equal(result.reason, 'mismatched'); @@ -104,7 +97,7 @@ test('same-version helper is replaced when installed APK identity is unavailable adb, adbProvider: provider, artifact: fixture.artifact, - deviceKey: 'android:emulator-5554', + deviceKey: 'android:unverifiable-identity', }); assert.equal(result.reason, 'unverifiable'); @@ -122,7 +115,7 @@ test('newer installed helper remains current without an unsafe downgrade', async adb, adbProvider: provider, artifact: fixture.artifact, - deviceKey: 'android:emulator-5554', + deviceKey: 'android:newer-helper', }); assert.equal(result.reason, 'current'); @@ -143,14 +136,14 @@ test('install memo includes artifact identity, not only version code', async () adb: device.adb, adbProvider: device.provider, artifact: first.artifact, - deviceKey: 'android:emulator-5554', + deviceKey: 'android:artifact-identity', }); device.setInstalledApk('first-helper'); const result = await ensureAndroidMultiTouchHelper({ adb: device.adb, adbProvider: device.provider, artifact: second.artifact, - deviceKey: 'android:emulator-5554', + deviceKey: 'android:artifact-identity', }); assert.equal(result.reason, 'mismatched'); @@ -168,7 +161,7 @@ test('install memo skips repeated APK reads for the same artifact identity', asy adb: device.adb, adbProvider: device.provider, artifact: fixture.artifact, - deviceKey: 'android:emulator-5554', + deviceKey: 'android:install-memo', }); await fs.rm(fixture.artifact.apkPath); const result = await ensureAndroidMultiTouchHelper({ @@ -181,7 +174,7 @@ test('install memo skips repeated APK reads for the same artifact identity', asy }, }, artifact: fixture.artifact, - deviceKey: 'android:emulator-5554', + deviceKey: 'android:install-memo', }); assert.equal(result.reason, 'current'); diff --git a/src/platforms/android/__tests__/multitouch-helper-ownership.test.ts b/src/platforms/android/__tests__/multitouch-helper-ownership.test.ts deleted file mode 100644 index 1a08c77b8..000000000 --- a/src/platforms/android/__tests__/multitouch-helper-ownership.test.ts +++ /dev/null @@ -1,94 +0,0 @@ -import assert from 'node:assert/strict'; -import { beforeEach, test, vi } from 'vitest'; -import { ANDROID_EMULATOR } from '../../../__tests__/test-utils/index.ts'; -import { buildGesturePlan } from '../../../contracts/gesture-plan.ts'; -import { withAndroidAdbProvider } from '../adb-executor.ts'; -import { - performGestureAndroid, - resetAndroidMultiTouchHelperInstallCache, -} from '../multitouch-helper.ts'; -import { stopAndroidSnapshotHelperSessionForDevice } from '../snapshot-helper.ts'; -import { - ANDROID_MULTITOUCH_HELPER_MANIFEST, - androidMultiTouchResultRecord, -} from './multitouch-helper.fixtures.ts'; - -vi.mock('../snapshot-helper.ts', async (importOriginal) => ({ - ...(await importOriginal()), - stopAndroidSnapshotHelperSessionForDevice: vi.fn(async () => {}), -})); - -vi.mock('../helper-package-install.ts', async (importOriginal) => { - const actual = await importOriginal(); - const fixture = await import('./android-helper-artifact.fixtures.ts'); - const multitouchFixture = await import('./multitouch-helper.fixtures.ts'); - return { - ...actual, - resolveAndroidHelperArtifact: vi.fn(async () => ({ - apkPath: fixture.ANDROID_HELPER_FIXTURE_APK_PATH, - manifest: { - ...multitouchFixture.ANDROID_MULTITOUCH_HELPER_MANIFEST, - sha256: fixture.ANDROID_HELPER_FIXTURE_APK_SHA256, - }, - })), - }; -}); - -const mockStopSnapshotSession = vi.mocked(stopAndroidSnapshotHelperSessionForDevice); - -beforeEach(() => { - resetAndroidMultiTouchHelperInstallCache(); - mockStopSnapshotSession.mockReset(); -}); - -test('helper gesture releases persistent snapshot instrumentation before touch instrumentation', async () => { - const events: string[] = []; - mockStopSnapshotSession.mockImplementation(async () => { - events.push('snapshot-stop'); - }); - - const result = await withAndroidAdbProvider( - { - exec: async (args) => { - if (args.includes('--show-versioncode')) { - return { - exitCode: 0, - stdout: `package:${ANDROID_MULTITOUCH_HELPER_MANIFEST.packageName} versionCode:999999`, - stderr: '', - }; - } - if (args.includes('instrument')) { - events.push('touch-instrumentation'); - return { - exitCode: 0, - stdout: [ - androidMultiTouchResultRecord({ ok: 'true', kind: 'swipe' }), - 'INSTRUMENTATION_CODE: 0', - ].join('\n'), - stderr: '', - }; - } - throw new Error(`unexpected adb call: ${args.join(' ')}`); - }, - }, - { serial: ANDROID_EMULATOR.id }, - async () => - await performGestureAndroid( - ANDROID_EMULATOR, - buildGesturePlan( - { - intent: 'pan', - pointerCount: 1, - origin: { x: 340, y: 400 }, - delta: { x: -280, y: 0 }, - durationMs: 300, - }, - { x: 0, y: 0, width: 400, height: 800 }, - 'android', - ), - ), - ); - - assert.equal(result?.backend, 'android-multitouch-helper'); - assert.deepEqual(events, ['snapshot-stop', 'touch-instrumentation']); -}); diff --git a/src/platforms/android/__tests__/multitouch-helper.test.ts b/src/platforms/android/__tests__/multitouch-helper.test.ts index 2389a933c..c073ce5d8 100644 --- a/src/platforms/android/__tests__/multitouch-helper.test.ts +++ b/src/platforms/android/__tests__/multitouch-helper.test.ts @@ -1,18 +1,12 @@ import assert from 'node:assert/strict'; -import { beforeEach, test } from 'vitest'; -import { ANDROID_EMULATOR } from '../../../__tests__/test-utils/index.ts'; +import { test } from 'vitest'; import { buildGesturePlan } from '../../../contracts/gesture-plan.ts'; import { normalizeAndroidMultiTouchHelperGestureRequest, parseAndroidMultiTouchHelperOutput, - performGestureAndroid, - readAndroidGestureViewport, - resetAndroidMultiTouchHelperInstallCache, runAndroidMultiTouchHelperGesture, parseAndroidGestureViewportResult, } from '../multitouch-helper.ts'; -import { createAndroidInteractor } from '../../../core/interactors/android.ts'; -import { withAndroidAdbProvider } from '../adb-executor.ts'; import { ANDROID_MULTITOUCH_HELPER_MANIFEST as manifest, androidMultiTouchResultRecord as resultRecord, @@ -20,7 +14,23 @@ import { const viewport = { x: 0, y: 0, width: 400, height: 800 }; -beforeEach(resetAndroidMultiTouchHelperInstallCache); +function longPressRequest(durationMs = 120_000) { + const point = { x: 20, y: 30 }; + return normalizeAndroidMultiTouchHelperGestureRequest({ + topology: 'single', + intent: 'longPress', + durationMs, + pointers: [ + { + pointerId: 0, + samples: [ + { offsetMs: 0, point }, + { offsetMs: durationMs, point }, + ], + }, + ], + }); +} test('helper response parsing returns instrumentation evidence', () => { assert.deepEqual( @@ -88,6 +98,32 @@ test('single-pointer plans use the same exact helper protocol', () => { assert.equal(request.durationMs, 100); }); +test('Android long press lowers to a stationary single-pointer helper request', () => { + const request = longPressRequest(); + assert.equal(request.kind, 'swipe'); + assert.equal(request.durationMs, 120_000); + assert.deepEqual(request.pointers[0]?.samples, [ + { offsetMs: 0, x: 20, y: 30 }, + { offsetMs: 120_000, x: 20, y: 30 }, + ]); +}); + +test('max-duration long press extends the helper process timeout', async () => { + await runAndroidMultiTouchHelperGesture({ + adb: async (_args, options) => { + assert.equal(options?.timeoutMs, 135_000); + return { + exitCode: 0, + stdout: [resultRecord({ ok: 'true', kind: 'swipe' }), 'INSTRUMENTATION_CODE: 0'].join('\n'), + stderr: '', + }; + }, + request: longPressRequest(), + packageName: manifest.packageName, + instrumentationRunner: manifest.instrumentationRunner, + }); +}); + test('transform sends the planner-owned choreography without regenerating geometry', async () => { const plan = buildGesturePlan( { @@ -133,56 +169,6 @@ test('transform sends the planner-owned choreography without regenerating geomet assert.deepEqual(payload, { protocol: 'android-multitouch-helper-v1', ...request }); }); -test('provider-native touch receives the plan as its only source of truth', async () => { - const plan = buildGesturePlan( - { intent: 'pinch', origin: { x: 200, y: 300 }, scale: 1.5 }, - viewport, - ); - const calls: unknown[] = []; - const result = await withAndroidAdbProvider( - { - exec: async () => { - throw new Error('adb must not run'); - }, - touch: async (request) => { - calls.push(request); - return { injected: true }; - }, - }, - { serial: ANDROID_EMULATOR.id }, - async () => await performGestureAndroid(ANDROID_EMULATOR, plan), - ); - assert.deepEqual(calls, [plan]); - assert.deepEqual(result, { backend: 'provider-native-touch', injected: true }); -}); - -test('provider-native failures propagate without adb single-pointer fallback', async () => { - const plan = buildGesturePlan( - { intent: 'fling', from: { x: 300, y: 400 }, to: { x: 100, y: 400 } }, - viewport, - ); - const adbCalls: string[][] = []; - await withAndroidAdbProvider( - { - exec: async (args) => { - adbCalls.push(args); - return { exitCode: 0, stdout: '', stderr: '' }; - }, - touch: async () => { - throw new Error('native touch failed'); - }, - }, - { serial: ANDROID_EMULATOR.id }, - async () => { - await assert.rejects( - () => performGestureAndroid(ANDROID_EMULATOR, plan), - /native touch failed/, - ); - }, - ); - assert.deepEqual(adbCalls, []); -}); - test('helper failures remain structured and actionable', async () => { const request = normalizeAndroidMultiTouchHelperGestureRequest( buildGesturePlan({ intent: 'pinch', scale: 1.5 }, viewport), @@ -240,31 +226,3 @@ test('gesture viewport result is typed and rejects invalid bounds', () => { ); assert.throws(() => parseAndroidGestureViewportResult([]), { code: 'COMMAND_FAILED' }); }); - -test('provider gesture viewport bypasses local helper transport and is validated', async () => { - let calls = 0; - await withAndroidAdbProvider( - { - exec: async () => { - throw new Error('adb must not run'); - }, - gestureViewport: async () => { - calls += 1; - return calls === 1 ? viewport : { ...viewport, width: 0 }; - }, - }, - { serial: ANDROID_EMULATOR.id }, - async () => { - assert.deepEqual(await readAndroidGestureViewport(ANDROID_EMULATOR), viewport); - await assert.rejects(readAndroidGestureViewport(ANDROID_EMULATOR), { - code: 'COMMAND_FAILED', - }); - }, - ); - assert.equal(calls, 2); -}); - -test('production Android interactor exposes the native gesture viewport seam', () => { - const interactor = createAndroidInteractor(ANDROID_EMULATOR); - assert.equal(typeof interactor.gestureViewport, 'function'); -}); diff --git a/src/platforms/android/__tests__/touch-executor-helper.test.ts b/src/platforms/android/__tests__/touch-executor-helper.test.ts new file mode 100644 index 000000000..83cb1cceb --- /dev/null +++ b/src/platforms/android/__tests__/touch-executor-helper.test.ts @@ -0,0 +1,236 @@ +import assert from 'node:assert/strict'; +import { beforeEach, test, vi } from 'vitest'; +import { ANDROID_EMULATOR } from '../../../__tests__/test-utils/index.ts'; +import { buildGesturePlan } from '../../../contracts/gesture-plan.ts'; +import { AppError } from '../../../kernel/errors.ts'; +import { withAndroidAdbProvider } from '../adb-executor.ts'; +import { longPressAndroid } from '../input-actions.ts'; +import { stopAndroidSnapshotHelperSessionForDevice } from '../snapshot-helper.ts'; +import { executeAndroidTouchPlan, readAndroidGestureViewport } from '../touch-executor.ts'; +import { + ANDROID_MULTITOUCH_HELPER_MANIFEST, + androidMultiTouchResultRecord, +} from './multitouch-helper.fixtures.ts'; + +vi.mock('../snapshot-helper.ts', async (importOriginal) => ({ + ...(await importOriginal()), + stopAndroidSnapshotHelperSessionForDevice: vi.fn(async () => {}), +})); + +vi.mock('../helper-package-install.ts', async (importOriginal) => { + const actual = await importOriginal(); + const fixture = await import('./android-helper-artifact.fixtures.ts'); + const multitouchFixture = await import('./multitouch-helper.fixtures.ts'); + return { + ...actual, + resolveAndroidHelperArtifact: vi.fn(async () => ({ + apkPath: fixture.ANDROID_HELPER_FIXTURE_APK_PATH, + manifest: { + ...multitouchFixture.ANDROID_MULTITOUCH_HELPER_MANIFEST, + sha256: fixture.ANDROID_HELPER_FIXTURE_APK_SHA256, + }, + })), + }; +}); + +const mockStopSnapshotSession = vi.mocked(stopAndroidSnapshotHelperSessionForDevice); +let deviceSequence = 0; + +beforeEach(() => { + mockStopSnapshotSession.mockReset(); +}); + +function makeIsolatedDevice() { + deviceSequence += 1; + return { ...ANDROID_EMULATOR, id: `emulator-touch-${deviceSequence}` }; +} + +test('helper gesture releases persistent snapshot instrumentation before touch instrumentation', async () => { + const device = makeIsolatedDevice(); + const events: string[] = []; + mockStopSnapshotSession.mockImplementation(async () => { + events.push('snapshot-stop'); + }); + + const result = await withAndroidAdbProvider( + { + exec: async (args) => { + if (args.includes('--show-versioncode')) { + return { + exitCode: 0, + stdout: `package:${ANDROID_MULTITOUCH_HELPER_MANIFEST.packageName} versionCode:999999`, + stderr: '', + }; + } + if (args.includes('instrument')) { + events.push('touch-instrumentation'); + return { + exitCode: 0, + stdout: [ + androidMultiTouchResultRecord({ ok: 'true', kind: 'swipe' }), + 'INSTRUMENTATION_CODE: 0', + ].join('\n'), + stderr: '', + }; + } + throw new Error(`unexpected adb call: ${args.join(' ')}`); + }, + }, + { serial: device.id }, + async () => + await executeAndroidTouchPlan( + device, + buildGesturePlan( + { + intent: 'pan', + pointerCount: 1, + origin: { x: 340, y: 400 }, + delta: { x: -280, y: 0 }, + durationMs: 300, + }, + { x: 0, y: 0, width: 400, height: 800 }, + 'android', + ), + ), + ); + + assert.equal(result?.backend, 'android-multitouch-helper'); + assert.deepEqual(events, ['snapshot-stop', 'touch-instrumentation']); +}); + +test('bare-adb long press executes one helper gesture without a viewport probe', async () => { + const device = makeIsolatedDevice(); + const instrumentCalls: string[][] = []; + const result = await withAndroidAdbProvider( + { + exec: async (args) => { + if (args.includes('--show-versioncode')) { + return { + exitCode: 0, + stdout: `package:${ANDROID_MULTITOUCH_HELPER_MANIFEST.packageName} versionCode:999999`, + stderr: '', + }; + } + if (args.includes('instrument')) { + instrumentCalls.push(args); + return { + exitCode: 0, + stdout: [ + androidMultiTouchResultRecord({ ok: 'true', kind: 'swipe' }), + 'INSTRUMENTATION_CODE: 0', + ].join('\n'), + stderr: '', + }; + } + throw new Error(`unexpected adb call: ${args.join(' ')}`); + }, + }, + { serial: device.id }, + async () => await longPressAndroid(device, 30, 40, 750), + ); + + assert.equal(result.backend, 'android-multitouch-helper'); + assert.equal(instrumentCalls.length, 1); + assert.equal(instrumentCalls[0]?.includes('viewport'), false); + assert.equal(mockStopSnapshotSession.mock.calls.length, 1); +}); + +test('single-pointer helper failure propagates through the touch executor', async () => { + const device = makeIsolatedDevice(); + await assert.rejects( + withAndroidAdbProvider( + { + exec: async (args) => { + if (args.includes('--show-versioncode')) { + return { + exitCode: 0, + stdout: `package:${ANDROID_MULTITOUCH_HELPER_MANIFEST.packageName} versionCode:999999`, + stderr: '', + }; + } + if (args.includes('instrument')) { + return { + exitCode: 1, + stdout: [ + androidMultiTouchResultRecord({ + ok: 'false', + errorType: 'java.lang.IllegalStateException', + message: 'injectInputEvent returned false', + }), + 'INSTRUMENTATION_CODE: 1', + ].join('\n'), + stderr: '', + }; + } + throw new Error(`unexpected adb call: ${args.join(' ')}`); + }, + }, + { serial: device.id }, + async () => + await executeAndroidTouchPlan( + device, + buildGesturePlan( + { + intent: 'pan', + pointerCount: 1, + origin: { x: 340, y: 400 }, + delta: { x: -280, y: 0 }, + durationMs: 300, + }, + { x: 0, y: 0, width: 400, height: 800 }, + 'android', + ), + ), + ), + (error: unknown) => { + assert.ok(error instanceof AppError); + assert.equal(error.code, 'COMMAND_FAILED'); + assert.equal(error.message, 'injectInputEvent returned false'); + assert.equal(error.details?.errorType, 'java.lang.IllegalStateException'); + return true; + }, + ); +}); + +test('helper viewport failure preserves its structured message and error type', async () => { + const device = makeIsolatedDevice(); + await assert.rejects( + withAndroidAdbProvider( + { + exec: async (args) => { + if (args.includes('--show-versioncode')) { + return { + exitCode: 0, + stdout: `package:${ANDROID_MULTITOUCH_HELPER_MANIFEST.packageName} versionCode:999999`, + stderr: '', + }; + } + if (args.includes('instrument')) { + return { + exitCode: 1, + stdout: [ + androidMultiTouchResultRecord({ + ok: 'false', + errorType: 'java.lang.SecurityException', + message: 'UiAutomation is unavailable', + }), + 'INSTRUMENTATION_CODE: 1', + ].join('\n'), + stderr: 'instrumentation failed', + }; + } + throw new Error(`unexpected adb call: ${args.join(' ')}`); + }, + }, + { serial: device.id }, + async () => await readAndroidGestureViewport(device), + ), + (error: unknown) => { + assert.ok(error instanceof AppError); + assert.equal(error.code, 'COMMAND_FAILED'); + assert.equal(error.message, 'UiAutomation is unavailable'); + assert.equal(error.details?.errorType, 'java.lang.SecurityException'); + return true; + }, + ); +}); diff --git a/src/platforms/android/__tests__/touch-executor.test.ts b/src/platforms/android/__tests__/touch-executor.test.ts new file mode 100644 index 000000000..db398d70d --- /dev/null +++ b/src/platforms/android/__tests__/touch-executor.test.ts @@ -0,0 +1,56 @@ +import assert from 'node:assert/strict'; +import { test } from 'vitest'; +import { ANDROID_EMULATOR } from '../../../__tests__/test-utils/index.ts'; +import { buildGesturePlan } from '../../../contracts/gesture-plan.ts'; +import { withAndroidAdbProvider } from '../adb-executor.ts'; +import { executeAndroidTouchPlan, readAndroidGestureViewport } from '../touch-executor.ts'; + +const viewport = { x: 0, y: 0, width: 400, height: 800 }; + +test('provider-native touch receives the plan as its only source of truth', async () => { + const plan = buildGesturePlan( + { intent: 'pinch', origin: { x: 200, y: 300 }, scale: 1.5 }, + viewport, + ); + const calls: unknown[] = []; + const result = await withAndroidAdbProvider( + { + exec: async () => { + throw new Error('adb must not run'); + }, + gestureViewport: async () => viewport, + touch: async (request) => { + calls.push(request); + return { injected: true }; + }, + }, + { serial: ANDROID_EMULATOR.id }, + async () => await executeAndroidTouchPlan(ANDROID_EMULATOR, plan), + ); + assert.deepEqual(calls, [plan]); + assert.deepEqual(result, { backend: 'provider-native-touch', injected: true }); +}); + +test('provider touch viewport bypasses local helper transport and is validated', async () => { + let calls = 0; + await withAndroidAdbProvider( + { + exec: async () => { + throw new Error('adb must not run'); + }, + gestureViewport: async () => { + calls += 1; + return calls === 1 ? viewport : { ...viewport, width: 0 }; + }, + touch: async () => {}, + }, + { serial: ANDROID_EMULATOR.id }, + async () => { + assert.deepEqual(await readAndroidGestureViewport(ANDROID_EMULATOR), viewport); + await assert.rejects(readAndroidGestureViewport(ANDROID_EMULATOR), { + code: 'COMMAND_FAILED', + }); + }, + ); + assert.equal(calls, 2); +}); diff --git a/src/platforms/android/adb-executor.ts b/src/platforms/android/adb-executor.ts index 852a24586..0531f14a6 100644 --- a/src/platforms/android/adb-executor.ts +++ b/src/platforms/android/adb-executor.ts @@ -2,7 +2,7 @@ import { AsyncLocalStorage } from 'node:async_hooks'; import type { Readable, Writable } from 'node:stream'; import type { DeviceInfo } from '../../kernel/device.ts'; import type { Rect } from '../../kernel/snapshot.ts'; -import type { GesturePlan } from '../../contracts/gesture-plan.ts'; +import type { AndroidProviderTouchPlan } from './touch-plan.ts'; import { coerceExecResult, execFailureDetails, @@ -126,12 +126,12 @@ export type AndroidTextInjectionRequest = { export type AndroidTextInjector = (request: AndroidTextInjectionRequest) => Promise; export type AndroidTouchInjector = ( - request: GesturePlan, + request: AndroidProviderTouchPlan, ) => Promise | void>; export type AndroidGestureViewportProvider = () => Promise; -export type AndroidAdbProvider = { +type AndroidAdbProviderBase = { /** * Fallback executor for device-scoped adb arguments. Providers may omit explicit * methods to keep the legacy exec-shaped pull/install fallback. @@ -143,10 +143,24 @@ export type AndroidAdbProvider = { install?: AndroidAdbInstaller; installBundle?: AndroidBundleInstaller; text?: AndroidTextInjector; - touch?: AndroidTouchInjector; - gestureViewport?: AndroidGestureViewportProvider; }; +type AndroidTouchCapabilities = + | { + touch?: never; + gestureViewport?: never; + } + | { + touch: AndroidTouchInjector; + gestureViewport: AndroidGestureViewportProvider; + }; + +export type AndroidTouchProvider = Required< + Pick +>; + +export type AndroidAdbProvider = AndroidAdbProviderBase & AndroidTouchCapabilities; + export type AndroidAdbProviderScopeOptions = { serial: string; }; @@ -473,16 +487,9 @@ export function resolveAndroidTextInjector(device: DeviceInfo): AndroidTextInjec return scoped?.serial === device.id ? scoped.provider.text : undefined; } -export function resolveAndroidTouchInjector(device: DeviceInfo): AndroidTouchInjector | undefined { - const scoped = androidAdbProviderScope.getStore(); - return scoped?.serial === device.id ? scoped.provider.touch : undefined; -} - -export function resolveAndroidGestureViewportProvider( - device: DeviceInfo, -): AndroidGestureViewportProvider | undefined { +export function resolveAndroidTouchProvider(device: DeviceInfo): AndroidTouchProvider | undefined { const scoped = androidAdbProviderScope.getStore(); - return scoped?.serial === device.id ? scoped.provider.gestureViewport : undefined; + return scoped?.serial === device.id && scoped.provider.touch ? scoped.provider : undefined; } export function createAndroidPortReverseManager( diff --git a/src/platforms/android/gesture-viewport.ts b/src/platforms/android/gesture-viewport.ts new file mode 100644 index 000000000..5f8b10b13 --- /dev/null +++ b/src/platforms/android/gesture-viewport.ts @@ -0,0 +1,15 @@ +import { AppError } from '../../kernel/errors.ts'; +import type { Rect } from '../../kernel/snapshot.ts'; + +export function validateAndroidGestureViewport(viewport: Rect): Rect { + if ( + !Number.isFinite(viewport.x) || + !Number.isFinite(viewport.y) || + !Number.isFinite(viewport.width) || + !Number.isFinite(viewport.height) || + viewport.width <= 0 || + viewport.height <= 0 + ) + throw new AppError('COMMAND_FAILED', 'Android helper returned an invalid gesture viewport'); + return viewport; +} diff --git a/src/platforms/android/input-actions.ts b/src/platforms/android/input-actions.ts index fe80fd0c1..f725e9fe3 100644 --- a/src/platforms/android/input-actions.ts +++ b/src/platforms/android/input-actions.ts @@ -2,6 +2,7 @@ import { AppError } from '../../kernel/errors.ts'; import type { DeviceInfo } from '../../kernel/device.ts'; import { emitDiagnostic } from '../../utils/diagnostics.ts'; import type { DeviceRotation } from '../../contracts/device-rotation.ts'; +import { buildGesturePlan, GESTURE_DURATION_MIN_MS } from '../../contracts/gesture-plan.ts'; import { buildScrollGesturePlan, type ScrollDirection } from '../../contracts/scroll-gesture.ts'; import { toAndroidTvRemoteKeyevent, type TvRemoteButton } from '../../contracts/tv-remote.ts'; import { runAndroidAdb, sleep } from './adb.ts'; @@ -18,6 +19,7 @@ import { type AndroidFillVerification, } from './fill-verification.ts'; import { isAndroidTestImeActive } from './ime-lifecycle.ts'; +import { executeAndroidTouchPlan, readAndroidGestureViewport } from './touch-executor.ts'; import { clearAndroidImeHelperText, resolveAndroidImeHelperArtifact, @@ -40,26 +42,6 @@ export async function pressAndroidTvRemote( await runAndroidAdb(device, ['shell', 'input', ...keyeventArgs, keyevent]); } -export async function swipeAndroid( - device: DeviceInfo, - x1: number, - y1: number, - x2: number, - y2: number, - durationMs = 250, -): Promise { - await runAndroidAdb(device, [ - 'shell', - 'input', - 'swipe', - String(x1), - String(y1), - String(x2), - String(y2), - String(durationMs), - ]); -} - export async function backAndroid(device: DeviceInfo): Promise { await runAndroidAdb(device, ['shell', 'input', 'keyevent', '4']); } @@ -104,17 +86,22 @@ export async function longPressAndroid( x: number, y: number, durationMs = 800, -): Promise { - await runAndroidAdb(device, [ - 'shell', - 'input', - 'swipe', - String(x), - String(y), - String(x), - String(y), - String(durationMs), - ]); +): Promise> { + const point = { x, y }; + return await executeAndroidTouchPlan(device, { + topology: 'single', + intent: 'longPress', + durationMs, + pointers: [ + { + pointerId: 0, + samples: [ + { offsetMs: 0, point }, + { offsetMs: durationMs, point }, + ], + }, + ], + }); } export async function typeAndroid(device: DeviceInfo, text: string, delayMs = 0): Promise { @@ -283,30 +270,47 @@ export async function scrollAndroid( direction: ScrollDirection, options?: { amount?: number; pixels?: number; durationMs?: number }, ): Promise> { - const size = await getAndroidScreenSize(device); - const plan = buildScrollGesturePlan({ + const viewport = await readAndroidGestureViewport(device); + const relativePlan = buildScrollGesturePlan({ direction, amount: options?.amount, pixels: options?.pixels, - referenceWidth: size.width, - referenceHeight: size.height, + referenceWidth: viewport.width, + referenceHeight: viewport.height, }); - const durationMs = options?.durationMs ?? 300; - - await runAndroidAdb(device, [ - 'shell', - 'input', - 'swipe', - String(plan.x1), - String(plan.y1), - String(plan.x2), - String(plan.y2), - String(durationMs), - ]); + const scrollPlan = { + ...relativePlan, + // Injected coordinates are absolute, so their zero-origin reference frame + // must include the viewport offset as well as its dimensions. + referenceWidth: viewport.x + viewport.width, + referenceHeight: viewport.y + viewport.height, + x1: viewport.x + relativePlan.x1, + y1: viewport.y + relativePlan.y1, + x2: viewport.x + relativePlan.x2, + y2: viewport.y + relativePlan.y2, + }; + const durationMs = Math.max(options?.durationMs ?? 300, GESTURE_DURATION_MIN_MS); + const backend = await executeAndroidTouchPlan( + device, + buildGesturePlan( + { + intent: 'pan', + origin: { x: scrollPlan.x1, y: scrollPlan.y1 }, + delta: { + x: scrollPlan.x2 - scrollPlan.x1, + y: scrollPlan.y2 - scrollPlan.y1, + }, + durationMs, + }, + viewport, + 'android', + ), + ); return { - ...plan, - ...(options?.durationMs !== undefined ? { durationMs: options.durationMs } : {}), + ...scrollPlan, + ...(options?.durationMs !== undefined ? { durationMs } : {}), + ...backend, }; } diff --git a/src/platforms/android/multitouch-helper-install.ts b/src/platforms/android/multitouch-helper-install.ts new file mode 100644 index 000000000..786778e98 --- /dev/null +++ b/src/platforms/android/multitouch-helper-install.ts @@ -0,0 +1,95 @@ +import { AppError } from '../../kernel/errors.ts'; +import { + makeEnsureAndroidHelperInstalled, + resolveAndroidHelperArtifact, +} from './helper-package-install.ts'; +import { + readAndroidHelperManifestInteger, + readAndroidHelperManifestLiteral, + readAndroidHelperManifestSha256, + readAndroidHelperManifestString, +} from './instrumentation-helper.ts'; + +const HELPER_NAME = 'android-multitouch-helper'; +const HELPER_PACKAGE = 'com.callstack.agentdevice.multitouchhelper'; +const HELPER_RUNNER = 'com.callstack.agentdevice.multitouchhelper/.MultiTouchInstrumentation'; +export const ANDROID_MULTITOUCH_HELPER_PROTOCOL = 'android-multitouch-helper-v1'; +const HELPER_INSTALL_TIMEOUT_MS = 30_000; +const HELPER_LABEL = 'Android multi-touch helper'; +const MANIFEST_HELPER_LABEL = 'multi-touch helper'; + +type AndroidMultiTouchHelperManifest = { + name: 'android-multitouch-helper'; + version: string; + assetName: string; + sha256: string; + packageName: string; + versionCode: number; + instrumentationRunner: string; + statusProtocol: 'android-multitouch-helper-v1'; +}; + +export type AndroidMultiTouchHelperArtifact = { + apkPath: string; + manifest: AndroidMultiTouchHelperManifest; +}; + +export async function resolveAndroidMultiTouchHelperArtifact(): Promise { + return await resolveAndroidHelperArtifact({ + helperDirName: 'android-multitouch-helper', + manifestFileName: (version) => + `agent-device-android-multitouch-helper-${version}.manifest.json`, + parseManifest: parseAndroidMultiTouchHelperManifest, + unavailableMessage: + 'Android touch gestures require the bundled Android touch helper artifact, but it was not found or could not be read', + }); +} + +function parseAndroidMultiTouchHelperManifest(value: unknown): AndroidMultiTouchHelperManifest { + if (!value || typeof value !== 'object' || Array.isArray(value)) { + throw new AppError('INVALID_ARGS', 'Android multi-touch helper manifest must be an object.'); + } + const record = value as Record; + return { + name: readAndroidHelperManifestLiteral(record.name, 'name', HELPER_NAME, MANIFEST_HELPER_LABEL), + version: readAndroidHelperManifestString(record.version, 'version', MANIFEST_HELPER_LABEL), + assetName: readAndroidHelperManifestString( + record.assetName, + 'assetName', + MANIFEST_HELPER_LABEL, + ), + sha256: readAndroidHelperManifestSha256(record.sha256, MANIFEST_HELPER_LABEL), + packageName: readAndroidHelperManifestLiteral( + record.packageName, + 'packageName', + HELPER_PACKAGE, + MANIFEST_HELPER_LABEL, + ), + versionCode: readAndroidHelperManifestInteger( + record.versionCode, + 'versionCode', + MANIFEST_HELPER_LABEL, + ), + instrumentationRunner: readAndroidHelperManifestLiteral( + record.instrumentationRunner, + 'instrumentationRunner', + HELPER_RUNNER, + MANIFEST_HELPER_LABEL, + ), + statusProtocol: readAndroidHelperManifestLiteral( + record.statusProtocol, + 'statusProtocol', + ANDROID_MULTITOUCH_HELPER_PROTOCOL, + MANIFEST_HELPER_LABEL, + ), + }; +} + +const installedMultiTouchHelpers = new Set(); + +export const ensureAndroidMultiTouchHelper = + makeEnsureAndroidHelperInstalled({ + cache: installedMultiTouchHelpers, + installTimeoutMs: HELPER_INSTALL_TIMEOUT_MS, + helperLabel: HELPER_LABEL, + }); diff --git a/src/platforms/android/multitouch-helper.ts b/src/platforms/android/multitouch-helper.ts index 4479a92c5..5904cd626 100644 --- a/src/platforms/android/multitouch-helper.ts +++ b/src/platforms/android/multitouch-helper.ts @@ -1,57 +1,32 @@ -import type { GesturePlan, PointerTrajectory } from '../../contracts/gesture-plan.ts'; +import type { PointerTrajectory } from '../../contracts/gesture-plan.ts'; import type { DeviceInfo } from '../../kernel/device.ts'; import type { Rect } from '../../kernel/snapshot.ts'; -import { AppError, normalizeError } from '../../kernel/errors.ts'; +import { AppError } from '../../kernel/errors.ts'; import { execFailureDetails } from '../../utils/exec.ts'; import { emitDiagnostic, withDiagnosticTimer } from '../../utils/diagnostics.ts'; import { resolveAndroidAdbExecutor, resolveAndroidAdbProvider, - resolveAndroidGestureViewportProvider, - resolveAndroidTouchInjector, type AndroidAdbExecutor, } from './adb-executor.ts'; -import { - makeEnsureAndroidHelperInstalled, - resolveAndroidHelperArtifact, -} from './helper-package-install.ts'; import { parseInstrumentationRecords, - readAndroidHelperManifestInteger, - readAndroidHelperManifestLiteral, - readAndroidHelperManifestSha256, - readAndroidHelperManifestString, readInstrumentationResultNumber, } from './instrumentation-helper.ts'; import { stopAndroidSnapshotHelperSessionForDevice } from './snapshot-helper.ts'; -import { swipeAndroid } from './input-actions.ts'; +import { validateAndroidGestureViewport } from './gesture-viewport.ts'; +import type { AndroidTouchPlan } from './touch-plan.ts'; +import { + ANDROID_MULTITOUCH_HELPER_PROTOCOL, + ensureAndroidMultiTouchHelper, + resolveAndroidMultiTouchHelperArtifact, +} from './multitouch-helper-install.ts'; -const HELPER_NAME = 'android-multitouch-helper'; -const HELPER_PACKAGE = 'com.callstack.agentdevice.multitouchhelper'; -const HELPER_RUNNER = 'com.callstack.agentdevice.multitouchhelper/.MultiTouchInstrumentation'; -const HELPER_PROTOCOL = 'android-multitouch-helper-v1'; -const HELPER_INSTALL_TIMEOUT_MS = 30_000; +const HELPER_PROTOCOL = ANDROID_MULTITOUCH_HELPER_PROTOCOL; const HELPER_GESTURE_TIMEOUT_MS = 45_000; +const HELPER_GESTURE_TIMEOUT_OVERHEAD_MS = 15_000; const HELPER_NO_FINAL_RESULT = 'ANDROID_MULTITOUCH_HELPER_NO_FINAL_RESULT'; const HELPER_REPORTED_FAILURE = 'ANDROID_MULTITOUCH_HELPER_REPORTED_FAILURE'; -const HELPER_LABEL = 'Android multi-touch helper'; -const MANIFEST_HELPER_LABEL = 'multi-touch helper'; - -type AndroidMultiTouchHelperManifest = { - name: 'android-multitouch-helper'; - version: string; - assetName: string; - sha256: string; - packageName: string; - versionCode: number; - instrumentationRunner: string; - statusProtocol: 'android-multitouch-helper-v1'; -}; - -type AndroidMultiTouchHelperArtifact = { - apkPath: string; - manifest: AndroidMultiTouchHelperManifest; -}; type AndroidPlannedPointerTrajectory = { pointerId: 0 | 1; @@ -64,35 +39,9 @@ type AndroidMultiTouchHelperGestureRequest = { pointers: AndroidPlannedPointerTrajectory[]; }; -export async function performGestureAndroid( +export async function executeAndroidMultiTouchHelperPlan( device: DeviceInfo, - plan: GesturePlan, -): Promise> { - const providerTouch = resolveAndroidTouchInjector(device); - if (providerTouch) { - const result = (await providerTouch(plan)) ?? {}; - return { backend: 'provider-native-touch', ...result }; - } - try { - return await runAndroidMultiTouchHelperGestureForDevice(device, plan); - } catch (error) { - if (plan.topology === 'two') throw error; - emitDiagnostic({ - level: 'warn', - phase: 'android_swipe_helper_fallback', - data: { error: normalizeError(error).message }, - }); - const first = plan.pointers[0].samples[0]?.point; - const last = plan.pointers[0].samples.at(-1)?.point; - if (!first || !last) throw error; - await swipeAndroid(device, first.x, first.y, last.x, last.y, plan.durationMs); - return { backend: 'adb-input-swipe-fallback' }; - } -} - -async function runAndroidMultiTouchHelperGestureForDevice( - device: DeviceInfo, - plan: GesturePlan, + plan: AndroidTouchPlan, ): Promise> { const { adb, artifact, install } = await prepareAndroidMultiTouchHelper(device); const output = await withDiagnosticTimer( @@ -139,9 +88,7 @@ async function prepareAndroidMultiTouchHelper(device: DeviceInfo) { return { adb, artifact, install }; } -export async function readAndroidGestureViewport(device: DeviceInfo): Promise { - const providerViewport = resolveAndroidGestureViewportProvider(device); - if (providerViewport) return validateAndroidGestureViewport(await providerViewport()); +export async function readAndroidMultiTouchHelperViewport(device: DeviceInfo): Promise { const { adb, artifact } = await prepareAndroidMultiTouchHelper(device); const result = await adb( [ @@ -157,9 +104,15 @@ export async function readAndroidGestureViewport(device: DeviceInfo): Promise record.agentDeviceProtocol === HELPER_PROTOCOL)) { + parseAndroidGestureViewportResult(records.results); + } + throw new AppError( + 'COMMAND_FAILED', + 'Android gesture viewport is unavailable', + execFailureDetails(result), + ); } export function parseAndroidGestureViewportResult(results: Array>): Rect { @@ -168,6 +121,7 @@ export function parseAndroidGestureViewportResult(results: Array; try { @@ -292,67 +239,3 @@ export function parseAndroidMultiTouchHelperOutput(output: string): Record { - return await resolveAndroidHelperArtifact({ - helperDirName: 'android-multitouch-helper', - manifestFileName: (version) => - `agent-device-android-multitouch-helper-${version}.manifest.json`, - parseManifest: parseAndroidMultiTouchHelperManifest, - unavailableMessage: - 'Android touch gestures require the bundled Android touch helper artifact, but it was not found or could not be read', - }); -} - -function parseAndroidMultiTouchHelperManifest(value: unknown): AndroidMultiTouchHelperManifest { - if (!value || typeof value !== 'object' || Array.isArray(value)) { - throw new AppError('INVALID_ARGS', 'Android multi-touch helper manifest must be an object.'); - } - const record = value as Record; - return { - name: readAndroidHelperManifestLiteral(record.name, 'name', HELPER_NAME, MANIFEST_HELPER_LABEL), - version: readAndroidHelperManifestString(record.version, 'version', MANIFEST_HELPER_LABEL), - assetName: readAndroidHelperManifestString( - record.assetName, - 'assetName', - MANIFEST_HELPER_LABEL, - ), - sha256: readAndroidHelperManifestSha256(record.sha256, MANIFEST_HELPER_LABEL), - packageName: readAndroidHelperManifestLiteral( - record.packageName, - 'packageName', - HELPER_PACKAGE, - MANIFEST_HELPER_LABEL, - ), - versionCode: readAndroidHelperManifestInteger( - record.versionCode, - 'versionCode', - MANIFEST_HELPER_LABEL, - ), - instrumentationRunner: readAndroidHelperManifestLiteral( - record.instrumentationRunner, - 'instrumentationRunner', - HELPER_RUNNER, - MANIFEST_HELPER_LABEL, - ), - statusProtocol: readAndroidHelperManifestLiteral( - record.statusProtocol, - 'statusProtocol', - HELPER_PROTOCOL, - MANIFEST_HELPER_LABEL, - ), - }; -} - -const installedMultiTouchHelpers = new Set(); - -export const ensureAndroidMultiTouchHelper = - makeEnsureAndroidHelperInstalled({ - cache: installedMultiTouchHelpers, - installTimeoutMs: HELPER_INSTALL_TIMEOUT_MS, - helperLabel: HELPER_LABEL, - }); - -export function resetAndroidMultiTouchHelperInstallCache(): void { - installedMultiTouchHelpers.clear(); -} diff --git a/src/platforms/android/touch-executor.ts b/src/platforms/android/touch-executor.ts new file mode 100644 index 000000000..09bed327e --- /dev/null +++ b/src/platforms/android/touch-executor.ts @@ -0,0 +1,34 @@ +import type { DeviceInfo } from '../../kernel/device.ts'; +import type { Rect } from '../../kernel/snapshot.ts'; +import { resolveAndroidTouchProvider } from './adb-executor.ts'; +import { + executeAndroidMultiTouchHelperPlan, + readAndroidMultiTouchHelperViewport, +} from './multitouch-helper.ts'; +import { validateAndroidGestureViewport } from './gesture-viewport.ts'; +import type { AndroidTouchPlan } from './touch-plan.ts'; + +export async function executeAndroidTouchPlan( + device: DeviceInfo, + plan: AndroidTouchPlan, +): Promise> { + const provider = resolveAndroidTouchProvider(device); + if (provider) { + const providerPlan = + plan.intent === 'longPress' + ? { + ...plan, + viewport: validateAndroidGestureViewport(await provider.gestureViewport()), + } + : plan; + const result = (await provider.touch(providerPlan)) ?? {}; + return { backend: 'provider-native-touch', ...result }; + } + return await executeAndroidMultiTouchHelperPlan(device, plan); +} + +export async function readAndroidGestureViewport(device: DeviceInfo): Promise { + const provider = resolveAndroidTouchProvider(device); + if (provider) return validateAndroidGestureViewport(await provider.gestureViewport()); + return await readAndroidMultiTouchHelperViewport(device); +} diff --git a/src/platforms/android/touch-plan.ts b/src/platforms/android/touch-plan.ts new file mode 100644 index 000000000..cc87e86a3 --- /dev/null +++ b/src/platforms/android/touch-plan.ts @@ -0,0 +1,15 @@ +import type { GesturePlan, PointerTrajectory } from '../../contracts/gesture-plan.ts'; +import type { Rect } from '../../kernel/snapshot.ts'; + +export type AndroidLongPressTouchPlan = { + topology: 'single'; + intent: 'longPress'; + durationMs: number; + pointers: readonly [PointerTrajectory]; +}; + +export type AndroidTouchPlan = GesturePlan | AndroidLongPressTouchPlan; + +export type AndroidProviderTouchPlan = + | GesturePlan + | (AndroidLongPressTouchPlan & { viewport: Rect }); diff --git a/src/utils/__tests__/daemon-client.test.ts b/src/utils/__tests__/daemon-client.test.ts index 266c0862d..e2a0fba2e 100644 --- a/src/utils/__tests__/daemon-client.test.ts +++ b/src/utils/__tests__/daemon-client.test.ts @@ -295,15 +295,26 @@ test('interaction --settle budgets add post-action settle time on top of the nor }), 125_000, ); - // Bare --settle uses the settle loop's default budget, so a slow pre-action - // capture still leaves room for the post-action observation to report. + // Longpress keeps the cold Android helper route and its 120-second maximum + // hold inside the outer envelope. + assert.equal( + resolveDaemonRequestTimeoutMs({ + ...base, + command: 'longpress', + positionals: ['300', '500', '120000'], + flags: {}, + }), + 210_000, + ); + // Bare --settle adds its default budget after the longpress-specific base, + // so a maximum hold still leaves room for post-action observation. assert.equal( resolveDaemonRequestTimeoutMs({ ...base, command: 'longpress', flags: { settle: true }, }), - 130_000, + 250_000, ); // Bare timeoutMs without --settle remains wire-compatible with older touch // command clients: it is ignored instead of opting into settle semantics. diff --git a/test/integration/provider-scenarios/android-lifecycle.test.ts b/test/integration/provider-scenarios/android-lifecycle.test.ts index 63202ac17..4b4a95e96 100644 --- a/test/integration/provider-scenarios/android-lifecycle.test.ts +++ b/test/integration/provider-scenarios/android-lifecycle.test.ts @@ -80,11 +80,25 @@ test('Provider-backed integration Android text provider handles Unicode without test('Provider-backed integration Android touch provider handles multi-touch gestures', async () => { await withProviderScenarioResource( - async () => await createAndroidSettingsWorld({ nativeTouchInjection: true }), + async () => await createAndroidSettingsWorld(), async (world) => { const client = world.daemon.client(); await client.apps.open({ app: 'settings', ...world.selection }); + await client.interactions.longPress({ + x: 195, + y: 320, + durationMs: 750, + ...world.selection, + }); + + await client.interactions.scroll({ + direction: 'down', + pixels: 120, + durationMs: 350, + ...world.selection, + }); + await client.interactions.swipe({ from: { x: 340, y: 400 }, to: { x: 60, y: 400 }, @@ -157,6 +171,8 @@ test('Provider-backed integration Android touch provider handles multi-touch ges durationMs: plan.durationMs, })); assert.deepEqual(touchCalls, [ + { topology: 'single', intent: 'longPress', pointerCount: 1, durationMs: 750 }, + { topology: 'single', intent: 'pan', pointerCount: 1, durationMs: 350 }, { topology: 'single', intent: 'pan', pointerCount: 1, durationMs: 300 }, { topology: 'single', intent: 'pan', pointerCount: 1, durationMs: 500 }, { topology: 'two', intent: 'pan', pointerCount: 2, durationMs: 500 }, @@ -164,18 +180,7 @@ test('Provider-backed integration Android touch provider handles multi-touch ges { topology: 'two', intent: 'rotate', pointerCount: 2, durationMs: 784 }, { topology: 'two', intent: 'transform', pointerCount: 2, durationMs: 700 }, ]); - assert.equal(world.gestureViewportCalls, 6); - assert.equal( - world.adbCalls.some( - (call) => - call[0] === 'shell' && - call[1] === 'am' && - call[2] === 'instrument' && - call.includes('com.callstack.agentdevice.multitouchhelper/.MultiTouchInstrumentation'), - ), - false, - JSON.stringify(world.adbCalls), - ); + assert.equal(world.gestureViewportCalls, 8); }, ); }); @@ -1460,6 +1465,20 @@ function assertAndroidInteractionContract(world: AndroidSettingsWorld): void { ), JSON.stringify(adbCalls), ); + assert.deepEqual( + world.touchInjectionCalls.map((plan) => ({ + intent: plan.intent, + durationMs: plan.durationMs, + })), + [ + { intent: 'longPress', durationMs: 5 }, + { intent: 'longPress', durationMs: 5 }, + { intent: 'fling', durationMs: 100 }, + { intent: 'fling', durationMs: 100 }, + { intent: 'pan', durationMs: 400 }, + { intent: 'fling', durationMs: 100 }, + ], + ); assertCommandCall(adbCalls, ['shell', 'input', 'tap', '88', '151']); assertCommandCall(adbCalls, [ 'shell', @@ -1476,12 +1495,6 @@ function assertAndroidInteractionContract(world: AndroidSettingsWorld): void { 2, ); assertCommandCall(adbCalls, ['shell', 'input', 'tap', '50', '60']); - assertCommandCall(adbCalls, ['shell', 'input', 'swipe', '30', '40', '30', '40', '5']); - assertCommandCall(adbCalls, ['shell', 'input', 'swipe', '31', '40', '31', '40', '5']); - assertCommandCall(adbCalls, ['shell', 'input', 'swipe', '20', '200', '20', '100', '100']); - assertCommandCall(adbCalls, ['shell', 'input', 'swipe', '20', '100', '20', '200', '100']); - assertCommandCall(adbCalls, ['shell', 'input', 'swipe', '100', '200', '150', '180', '400']); - assertCommandCall(adbCalls, ['shell', 'input', 'swipe', '100', '200', '280', '200', '100']); assert.equal( adbCalls.filter((call) => arrayEqual(call, ['shell', 'input', 'tap', '88', '151'])).length, 5, diff --git a/test/integration/provider-scenarios/android-test-suite.test.ts b/test/integration/provider-scenarios/android-test-suite.test.ts index 02e38f0de..f12fa9cfa 100644 --- a/test/integration/provider-scenarios/android-test-suite.test.ts +++ b/test/integration/provider-scenarios/android-test-suite.test.ts @@ -118,10 +118,12 @@ test('Provider-backed integration Android Maestro replay uses fresh snapshots an world.adbCalls.find((call) => call.slice(0, 3).join(' ') === 'shell input tap'), ['shell', 'input', 'tap', '180', '330'], ); - assert.deepEqual( - world.adbCalls.find((call) => call.slice(0, 3).join(' ') === 'shell input swipe'), - ['shell', 'input', 'swipe', '351', '300', '39', '300', '300'], - ); + assert.equal(world.touchInjectionCalls.length, 1); + const swipePlan = world.touchInjectionCalls[0]!; + assert.equal(swipePlan.intent, 'pan'); + assert.equal(swipePlan.durationMs, 300); + assert.deepEqual(swipePlan.pointers[0]?.samples[0]?.point, { x: 351, y: 300 }); + assert.deepEqual(swipePlan.pointers[0]?.samples.at(-1)?.point, { x: 39, y: 300 }); // Percentage resolution snapshots remain fresh; gesture planning uses the provider viewport. assertSnapshotCountInRange(snapshots, 3, 5); }, diff --git a/test/integration/provider-scenarios/android-world.ts b/test/integration/provider-scenarios/android-world.ts index 31cde866e..d605da3c0 100644 --- a/test/integration/provider-scenarios/android-world.ts +++ b/test/integration/provider-scenarios/android-world.ts @@ -48,7 +48,6 @@ type AndroidSettingsWorld = { export async function createAndroidSettingsWorld(options?: { nativeTextInjection?: boolean; - nativeTouchInjection?: boolean; snapshotXml?: () => string; dumpsysWindow?: () => string; onAdbExec?: (args: string[]) => void; @@ -98,6 +97,10 @@ export async function createAndroidSettingsWorld(options?: { pidof: (packageName) => androidPidofResult(appState, packageName), }); }, + touch: async (request) => { + touchInjectionCalls.push({ ...request }); + return { backend: 'provider-native-touch' }; + }, install: async (apk, options) => { apkInstallCalls.push({ apkPath: apk, replace: options?.replace }); return { stdout: '', stderr: '', exitCode: 0 }; @@ -142,12 +145,6 @@ export async function createAndroidSettingsWorld(options?: { shellState.searchText = request.text; }; } - if (options?.nativeTouchInjection) { - adbProvider.touch = async (request) => { - touchInjectionCalls.push({ ...request }); - return { backend: 'provider-native-touch' }; - }; - } const daemon = await createProviderScenarioHarness({ androidAdbProvider: () => adbProvider, deviceInventoryProvider: async (request) => {