diff --git a/.github/workflows/ios.yml b/.github/workflows/ios.yml index 25cbf5644..851fd81b6 100644 --- a/.github/workflows/ios.yml +++ b/.github/workflows/ios.yml @@ -31,6 +31,7 @@ jobs: AGENT_DEVICE_STATE_DIR: ${{ github.workspace }}/.tmp/agent-device-state AGENT_DEVICE_IOS_RUNNER_DERIVED_PATH: ${{ github.workspace }}/.tmp/ios-runner-derived AGENT_DEVICE_IOS_PREPARE_TIMEOUT_MS: '420000' + AGENT_DEVICE_XCUITEST_INCLUDE_UNIT_TESTS: '1' steps: - name: Checkout uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2 @@ -56,6 +57,17 @@ jobs: runtime-version: ${{ env.IOS_RUNTIME_VERSION }} preferred-device-name: iPhone 17 Pro + - name: Run iOS runner gesture and lifecycle regressions + run: | + XCTESTRUN_PATH="$(find "$AGENT_DEVICE_IOS_RUNNER_DERIVED_PATH/Build/Products" -maxdepth 1 -name '*.xctestrun' -print -quit)" + test -n "$XCTESTRUN_PATH" + xcodebuild test-without-building \ + -xctestrun "$XCTESTRUN_PATH" \ + -destination "platform=iOS Simulator,id=${{ steps.ios-simulator.outputs.simulator-udid }}" \ + -only-testing:AgentDeviceRunnerUITests/RunnerTests/testSinglePointerFlingFallsBackToXCTestCoordinateDragWhenPrivateSynthesisFails \ + -only-testing:AgentDeviceRunnerUITests/RunnerTests/testMissingBundleCommandInvalidatesCompleteCachedTargetState \ + -only-testing:AgentDeviceRunnerUITests/RunnerTests/testCachedTargetInvalidationClearsProcessBoundState + - name: Prepare iOS runner run: | pnpm clean:daemon diff --git a/AGENTS.md b/AGENTS.md index 9ad1a517a..c568cd260 100644 --- a/AGENTS.md +++ b/AGENTS.md @@ -61,6 +61,7 @@ Single-context repo. Read `CONTEXT.md` for domain language and testing/architect - prefer deep modules over mechanical splits: extract when it improves locality for a concept callers already need, not just to reduce line count. - Before finalizing a code change, do one tightening pass over touched and directly adjacent areas: drop obsolete code, redundant tests, stale helpers/fixtures, and needless duplication made unnecessary by the change. - Prefer existing helpers. Add a helper only when it reduces real repetition or clarifies domain behavior. +- Prefer composition at platform boundaries: public aliases normalize into shared primitives, and providers contribute transport/device bindings instead of cloning interaction runtimes. - When adding new guidance, examples, schemas, or command metadata, decide whether it belongs in the command surface, CLI grammar, CLI help, MCP projection, or daemon runtime before editing. - Prefer updating existing domain vocabulary in `CONTEXT.md` when naming a new durable module concept. Do not coin parallel names in docs, tests, and code. diff --git a/CONTEXT.md b/CONTEXT.md index a06cf80d8..b6a47f6e2 100644 --- a/CONTEXT.md +++ b/CONTEXT.md @@ -47,6 +47,8 @@ - Runner command traits: per-command-type classification for iOS/macOS runner lifecycle behavior, distinct from the public command surface and daemon command registry. The Swift runner traits classify interaction, read-only, and runner-lifecycle axes for XCTest execution; Swift resolves the alert command as read-only only for its `get` action. The TypeScript runner command traits classify daemon-side runner send/recovery policy such as read-only retry routing, readiness probes, and recent-healthy-mutation preflight skips; the TypeScript table is command-type keyed and currently classifies alert as read-only for daemon retry policy. Each side keeps one source of truth keyed by runner command type. - Coordinate-first resolved element activation: iOS/macOS runner interaction pattern where a selector or text query resolves the semantic `XCUIElement`, then activation uses the element's resolved center coordinate when a frame is available. This keeps target selection semantic while avoiding `XCUIElement.tap()` post-action element re-resolution after normal navigation. tvOS remains focus/remote-driven. - Interaction dispatch path: one concrete route an interaction command takes to the device (runtime selector/ref resolution, direct iOS selector, native ref via web clickRef, coordinate, maestro non-hittable fallback). Every path classifies every guarantee in the ADR 0011 registry. +- Gesture plan: typed, platform-neutral normalization of one- or two-contact gesture intent into bounded pointer trajectories. Contact topology is separate from motion; two-contact intent remains pan/pinch/rotate/transform even when native injection shares one executor. See ADR 0013. +- Multi-touch geometry: the internal initial span and angle plus centroid translation, scale, and rotation used to build both contact trajectories. Geometry is viewport-aware and fails early when the requested motion cannot fit; it is not a public tuning surface. - Guarantee cell: one (dispatch path, guarantee) entry in `src/contracts/interaction-guarantees.ts`, classified as runtime/runner/delegated/inapplicable/waived. Completeness is a compile error; honesty is gate-tested. - Owned waiver: a `gap:`-prefixed waived cell carrying a `trackingIssue` URL. Waivers are diffable debt with an owner, never folklore. - Parity table: golden JSON fixture under `contracts/fixtures/` consumed by both vitest and the runner's gated Swift tests, so a cross-language rule (e.g. tap-point policy) cannot drift silently. Change the rule only via the table. diff --git a/android-multitouch-helper/README.md b/android-multitouch-helper/README.md index 320c52ad7..e5266fcca 100644 --- a/android-multitouch-helper/README.md +++ b/android-multitouch-helper/README.md @@ -18,7 +18,16 @@ sh ./scripts/build-android-multitouch-helper.sh "$VERSION" .tmp/android-multitou ## Run ```sh -PAYLOAD="$(printf '%s' '{"kind":"transform","x":672,"y":1500,"dx":80,"dy":-40,"scale":1.8,"degrees":35,"durationMs":700}' | base64)" +PAYLOAD_JSON='{ + "protocol":"android-multitouch-helper-v1", + "kind":"transform", + "durationMs":32, + "pointers":[ + {"pointerId":0,"samples":[{"offsetMs":0,"x":120,"y":180},{"offsetMs":16,"x":130,"y":175},{"offsetMs":32,"x":140,"y":170}]}, + {"pointerId":1,"samples":[{"offsetMs":0,"x":120,"y":260},{"offsetMs":16,"x":135,"y":270},{"offsetMs":32,"x":150,"y":280}]} + ] +}' +PAYLOAD="$(printf '%s' "$PAYLOAD_JSON" | base64)" adb install -r -t ".tmp/android-multitouch-helper/agent-device-android-multitouch-helper-$VERSION.apk" adb shell am instrument -w \ -e payloadBase64 "$PAYLOAD" \ @@ -30,12 +39,19 @@ adb shell am instrument -w \ The APK emits instrumentation result records using `agentDeviceProtocol=android-multitouch-helper-v1`. +Before planning a gesture, the runtime invokes the same instrumentation with +`-e mode viewport`. That read-only mode waits for `UiAutomation` window state and returns the +active application window bounds, falling back to the active accessibility root when Android does +not expose interactive-window metadata. It does not use or cache the physical display size. + Successful results include: - `ok=true` - `helperApiVersion=1` -- `kind` (`swipe`, `pinch`, `rotate`, or `transform`) +- `kind` (`swipe` for one planned pointer path or `transform` for two) - `injectedEvents` - `elapsedMs` +Viewport results use `kind=viewport` plus `x`, `y`, `width`, and `height`. + Failures return `ok=false`, `errorType`, and `message`. diff --git a/android-multitouch-helper/src/main/java/com/callstack/agentdevice/multitouchhelper/MultiTouchInstrumentation.java b/android-multitouch-helper/src/main/java/com/callstack/agentdevice/multitouchhelper/MultiTouchInstrumentation.java index f6952d318..234f6c3aa 100644 --- a/android-multitouch-helper/src/main/java/com/callstack/agentdevice/multitouchhelper/MultiTouchInstrumentation.java +++ b/android-multitouch-helper/src/main/java/com/callstack/agentdevice/multitouchhelper/MultiTouchInstrumentation.java @@ -7,18 +7,20 @@ import android.util.Base64; import android.view.InputDevice; import android.view.MotionEvent; +import android.view.accessibility.AccessibilityNodeInfo; +import android.view.accessibility.AccessibilityWindowInfo; +import android.graphics.Rect; import java.nio.charset.StandardCharsets; +import java.util.List; +import java.util.concurrent.TimeoutException; +import org.json.JSONArray; import org.json.JSONObject; public final class MultiTouchInstrumentation extends Instrumentation { private static final String PROTOCOL = "android-multitouch-helper-v1"; private static final String HELPER_API_VERSION = "1"; - private static final int DEFAULT_RADIUS = 160; - private static final int MIN_RADIUS = 24; - private static final int MAX_RADIUS = 1200; private static final int MIN_DURATION_MS = 16; private static final int MAX_DURATION_MS = 10_000; - private static final int MOVE_FRAME_INTERVAL_MS = 16; private Bundle arguments; @Override @@ -36,10 +38,21 @@ public void onStart() { result.putString("helperApiVersion", HELPER_API_VERSION); try { long startedAtMs = System.currentTimeMillis(); - GestureSpec spec = readSpec(arguments); - int injectedEvents = injectGesture(spec); + if ("viewport".equals(arguments.getString("mode", "gesture"))) { + Rect viewport = readActiveApplicationViewport(); + result.putString("ok", "true"); + result.putString("kind", "viewport"); + result.putString("x", Integer.toString(viewport.left)); + result.putString("y", Integer.toString(viewport.top)); + result.putString("width", Integer.toString(viewport.width())); + result.putString("height", Integer.toString(viewport.height())); + finish(0, result); + return; + } + GesturePlan plan = readPlan(arguments); + int injectedEvents = injectPlan(plan); result.putString("ok", "true"); - result.putString("kind", spec.kind); + result.putString("kind", plan.kind); result.putString("injectedEvents", Integer.toString(injectedEvents)); result.putString("elapsedMs", Long.toString(System.currentTimeMillis() - startedAtMs)); finish(0, result); @@ -53,11 +66,46 @@ public void onStart() { } } - private GestureSpec readSpec(Bundle arguments) throws Exception { - String payloadBase64 = arguments.getString("payloadBase64", ""); - if (payloadBase64.isEmpty()) { - throw new IllegalArgumentException("Missing payloadBase64"); + @SuppressWarnings("deprecation") + private Rect readActiveApplicationViewport() { + UiAutomation automation = getUiAutomation(); + try { + automation.waitForIdle(100, 2_000); + } catch (TimeoutException ignored) { + // Window/root state can still be usable when the app is animating continuously. + } + List windows = automation.getWindows(); + AccessibilityWindowInfo fallback = null; + for (AccessibilityWindowInfo window : windows) { + if (window.getType() != AccessibilityWindowInfo.TYPE_APPLICATION) continue; + if (window.isActive() || window.isFocused()) { + Rect bounds = new Rect(); + window.getBoundsInScreen(bounds); + if (!bounds.isEmpty()) return bounds; + } + if (fallback == null) fallback = window; + } + AccessibilityNodeInfo activeRoot = automation.getRootInActiveWindow(); + if (activeRoot != null) { + try { + Rect bounds = new Rect(); + activeRoot.getBoundsInScreen(bounds); + if (!bounds.isEmpty()) return bounds; + } finally { + activeRoot.recycle(); + } + } + if (fallback != null) { + Rect bounds = new Rect(); + fallback.getBoundsInScreen(bounds); + if (!bounds.isEmpty()) return bounds; } + throw new IllegalStateException("Active application interaction viewport is unavailable"); + } + + private GesturePlan readPlan(Bundle arguments) throws Exception { + String payloadBase64 = arguments.getString("payloadBase64", ""); + if (payloadBase64.isEmpty()) throw new IllegalArgumentException("Missing payloadBase64"); String json = new String(Base64.decode(payloadBase64, Base64.DEFAULT), StandardCharsets.UTF_8); JSONObject payload = new JSONObject(json); @@ -66,148 +114,136 @@ private GestureSpec readSpec(Bundle arguments) throws Exception { throw new IllegalArgumentException("Unsupported protocol: " + protocol); } String kind = payload.getString("kind"); - if (!"swipe".equals(kind) - && !"pinch".equals(kind) - && !"rotate".equals(kind) - && !"transform".equals(kind)) { + if (!"swipe".equals(kind) && !"transform".equals(kind)) { throw new IllegalArgumentException("Unsupported kind: " + kind); } - int x = "swipe".equals(kind) ? payload.getInt("x1") : payload.getInt("x"); - int y = "swipe".equals(kind) ? payload.getInt("y1") : payload.getInt("y"); - int dx = payload.optInt("dx", 0); - int dy = payload.optInt("dy", 0); - int x2 = payload.optInt("x2", x + dx); - int y2 = payload.optInt("y2", y + dy); - int durationMs = clamp(payload.optInt("durationMs", 300), MIN_DURATION_MS, MAX_DURATION_MS); - int radius = clamp(payload.optInt("radius", DEFAULT_RADIUS), MIN_RADIUS, MAX_RADIUS); - double scale = payload.optDouble("scale", 1.0d); - double degrees = payload.optDouble("degrees", 0.0d); - if (("pinch".equals(kind) || "transform".equals(kind)) && (!isFinite(scale) || scale <= 0)) { - throw new IllegalArgumentException("Scale must be > 0"); - } - if (("rotate".equals(kind) || "transform".equals(kind)) && !isFinite(degrees)) { - throw new IllegalArgumentException("Degrees must be finite"); - } - return new GestureSpec(kind, x, y, dx, dy, x2, y2, durationMs, scale, degrees, radius); + int durationMs = requireDuration(payload.getInt("durationMs")); + PointerPath[] pointers = readPointers(payload.getJSONArray("pointers"), kind, durationMs); + return new GesturePlan(kind, durationMs, pointers); } - private int injectGesture(GestureSpec spec) { - if ("swipe".equals(spec.kind)) { - return injectSinglePointerGesture(spec); - } + private int injectPlan(GesturePlan plan) { UiAutomation automation = getUiAutomation(); long downTime = SystemClock.uptimeMillis(); long eventTime = downTime; - PointerPair start = pointerPairAt(spec, 0); - PointerPair end = pointerPairAt(spec, 1); - PointerPair activePointers = start.firstOnly(); + PointerState active = pointerStateAt(plan, 0).firstOnly(); int count = 0; - try { - injectScheduledEvent( - automation, - motionEvent(downTime, eventTime, MotionEvent.ACTION_DOWN, activePointers), - true); - count += 1; - eventTime += 8; - injectScheduledEvent( - automation, - motionEvent( - downTime, - eventTime, - MotionEvent.ACTION_POINTER_DOWN | (1 << MotionEvent.ACTION_POINTER_INDEX_SHIFT), - start), - true); - count += 1; - activePointers = start; - - int frameCount = - Math.max(3, Math.round(spec.durationMs / (float) MOVE_FRAME_INTERVAL_MS)); - for (int index = 1; index < frameCount; index += 1) { - double t = (double) index / (double) frameCount; - PointerPair frame = pointerPairAt(spec, t); - eventTime = downTime + Math.round(spec.durationMs * t); - injectScheduledEvent( + for (PointerEventSchedule.Step step : + PointerEventSchedule.create(plan.pointers.length, sampleOffsets(plan))) { + active = pointerStateAt(plan, step.sampleIndex); + if (step.pointerCount == 1) active = active.firstOnly(); + eventTime = downTime + step.offsetMs; + inject( automation, - motionEvent(downTime, eventTime, MotionEvent.ACTION_MOVE, frame), - false); + motionEvent(downTime, eventTime, motionEventAction(step.action), active), + step.action != PointerEventSchedule.Action.MOVE); count += 1; - activePointers = frame; } - - eventTime = downTime + spec.durationMs; - injectScheduledEvent( - automation, - motionEvent( - downTime, - eventTime, - MotionEvent.ACTION_POINTER_UP | (1 << MotionEvent.ACTION_POINTER_INDEX_SHIFT), - end), - true); - count += 1; - activePointers = end.firstOnly(); - injectScheduledEvent( - automation, - motionEvent(downTime, eventTime + 8, MotionEvent.ACTION_UP, activePointers), - true); - count += 1; return count; } catch (RuntimeException error) { - if (count > 0) { - injectCancel(automation, downTime, eventTime + 16, activePointers); - } + if (count > 0) injectCancel(automation, downTime, eventTime + 16, active); throw error; } } - private int injectSinglePointerGesture(GestureSpec spec) { - UiAutomation automation = getUiAutomation(); - long downTime = SystemClock.uptimeMillis(); - long eventTime = downTime; - PointerPair activePointer = pointerPairAt(spec, 0); - int count = 0; + private static long[] sampleOffsets(GesturePlan plan) { + PointerSample[] samples = plan.pointers[0].samples; + long[] offsets = new long[samples.length]; + for (int index = 0; index < samples.length; index += 1) { + offsets[index] = samples[index].offsetMs; + } + return offsets; + } - try { - injectScheduledEvent( - automation, - motionEvent(downTime, eventTime, MotionEvent.ACTION_DOWN, activePointer), - true); - count += 1; + private static int motionEventAction(PointerEventSchedule.Action action) { + switch (action) { + case DOWN: + return MotionEvent.ACTION_DOWN; + case POINTER_DOWN: + return MotionEvent.ACTION_POINTER_DOWN | (1 << MotionEvent.ACTION_POINTER_INDEX_SHIFT); + case MOVE: + return MotionEvent.ACTION_MOVE; + case POINTER_UP: + return MotionEvent.ACTION_POINTER_UP | (1 << MotionEvent.ACTION_POINTER_INDEX_SHIFT); + case UP: + return MotionEvent.ACTION_UP; + default: + throw new IllegalArgumentException("Unsupported pointer event action: " + action); + } + } - int frameCount = - Math.max(3, Math.round(spec.durationMs / (float) MOVE_FRAME_INTERVAL_MS)); - for (int index = 1; index < frameCount; index += 1) { - double t = (double) index / (double) frameCount; - PointerPair frame = pointerPairAt(spec, t); - eventTime = downTime + Math.round(spec.durationMs * t); - injectScheduledEvent( - automation, - motionEvent(downTime, eventTime, MotionEvent.ACTION_MOVE, frame), - false); - count += 1; - activePointer = frame; + private static PointerPath[] readPointers(JSONArray pointers, String kind, int durationMs) + throws Exception { + int expectedCount = "swipe".equals(kind) ? 1 : 2; + if (pointers.length() != expectedCount) { + throw new IllegalArgumentException( + "Planned " + kind + " gesture requires exactly " + expectedCount + " pointer paths"); + } + PointerPath[] result = new PointerPath[expectedCount]; + for (int pointerIndex = 0; pointerIndex < expectedCount; pointerIndex += 1) { + JSONObject pointer = pointers.getJSONObject(pointerIndex); + if (pointer.getInt("pointerId") != pointerIndex) { + throw new IllegalArgumentException("Planned pointer ids must be ordered from 0"); } + JSONArray samples = pointer.getJSONArray("samples"); + if (samples.length() < 2) { + throw new IllegalArgumentException("Planned pointer path requires at least two samples"); + } + PointerSample[] parsed = new PointerSample[samples.length()]; + long previousOffsetMs = -1; + for (int sampleIndex = 0; sampleIndex < samples.length(); sampleIndex += 1) { + JSONObject sample = samples.getJSONObject(sampleIndex); + double rawOffsetMs = sample.getDouble("offsetMs"); + double x = sample.getDouble("x"); + double y = sample.getDouble("y"); + if (!finite(rawOffsetMs) || rawOffsetMs != Math.rint(rawOffsetMs)) { + throw new IllegalArgumentException("Planned sample offsetMs must be a finite integer"); + } + if (!finite(x) || !finite(y)) { + throw new IllegalArgumentException("Planned sample coordinates must be finite"); + } + long offsetMs = (long) rawOffsetMs; + if (offsetMs <= previousOffsetMs) { + throw new IllegalArgumentException("Planned sample offsets must be strictly increasing"); + } + parsed[sampleIndex] = new PointerSample(offsetMs, (float) x, (float) y); + previousOffsetMs = offsetMs; + } + if (parsed[0].offsetMs != 0 || parsed[parsed.length - 1].offsetMs != durationMs) { + throw new IllegalArgumentException("Pointer path must start at 0 and end at durationMs"); + } + result[pointerIndex] = new PointerPath(parsed); + } + if (expectedCount == 2) assertMatchingSamples(result[0], result[1]); + return result; + } - eventTime = downTime + spec.durationMs; - activePointer = pointerPairAt(spec, 1); - injectScheduledEvent( - automation, - motionEvent(downTime, eventTime, MotionEvent.ACTION_UP, activePointer), - true); - count += 1; - return count; - } catch (RuntimeException error) { - if (count > 0) { - injectCancel(automation, downTime, eventTime + 16, activePointer); + private static void assertMatchingSamples(PointerPath first, PointerPath second) { + if (first.samples.length != second.samples.length) { + throw new IllegalArgumentException("Planned pointer paths must have matching samples"); + } + for (int index = 0; index < first.samples.length; index += 1) { + if (first.samples[index].offsetMs != second.samples[index].offsetMs) { + throw new IllegalArgumentException("Planned pointer sample offsets must match"); } - throw error; } } - private static void injectScheduledEvent( - UiAutomation automation, MotionEvent event, boolean waitForDispatch) { + private static PointerState pointerStateAt(GesturePlan plan, int sampleIndex) { + int pointerCount = plan.pointers.length; + float[] x = new float[pointerCount]; + float[] y = new float[pointerCount]; + for (int index = 0; index < pointerCount; index += 1) { + PointerSample sample = plan.pointers[index].samples[sampleIndex]; + x[index] = sample.x; + y[index] = sample.y; + } + return new PointerState(x, y); + } + + private static void inject(UiAutomation automation, MotionEvent event, boolean waitForDispatch) { try { - // Event timestamps alone do not pace UiAutomation injection; recognizers need real time. sleepUntil(event.getEventTime()); if (!automation.injectInputEvent(event, waitForDispatch)) { throw new IllegalStateException("injectInputEvent returned false"); @@ -219,34 +255,33 @@ private static void injectScheduledEvent( private static void sleepUntil(long targetUptimeMs) { long delayMs = targetUptimeMs - SystemClock.uptimeMillis(); - if (delayMs > 0) { - SystemClock.sleep(delayMs); - } + if (delayMs > 0) SystemClock.sleep(delayMs); } private static void injectCancel( - UiAutomation automation, long downTime, long eventTime, PointerPair pair) { + UiAutomation automation, long downTime, long eventTime, PointerState pointers) { try { - injectScheduledEvent( + inject( automation, - motionEvent(downTime, eventTime, MotionEvent.ACTION_CANCEL, pair), + motionEvent(downTime, eventTime, MotionEvent.ACTION_CANCEL, pointers), true); } catch (RuntimeException ignored) { - // Best-effort cleanup; preserve the original injection failure. + // Preserve the original injection failure. } } - private static MotionEvent motionEvent(long downTime, long eventTime, int action, PointerPair pair) { + private static MotionEvent motionEvent( + long downTime, long eventTime, int action, PointerState pointers) { MotionEvent.PointerProperties[] properties = - new MotionEvent.PointerProperties[pair.pointerCount]; - MotionEvent.PointerCoords[] coords = new MotionEvent.PointerCoords[pair.pointerCount]; - for (int index = 0; index < pair.pointerCount; index += 1) { + new MotionEvent.PointerProperties[pointers.count]; + MotionEvent.PointerCoords[] coords = new MotionEvent.PointerCoords[pointers.count]; + for (int index = 0; index < pointers.count; index += 1) { properties[index] = new MotionEvent.PointerProperties(); properties[index].id = index; properties[index].toolType = MotionEvent.TOOL_TYPE_FINGER; coords[index] = new MotionEvent.PointerCoords(); - coords[index].x = pair.x[index]; - coords[index].y = pair.y[index]; + coords[index].x = pointers.x[index]; + coords[index].y = pointers.y[index]; coords[index].pressure = 1.0f; coords[index].size = 1.0f; } @@ -255,7 +290,7 @@ private static MotionEvent motionEvent(long downTime, long eventTime, int action downTime, eventTime, action, - pair.pointerCount, + pointers.count, properties, coords, 0, @@ -270,112 +305,62 @@ private static MotionEvent motionEvent(long downTime, long eventTime, int action return event; } - private static PointerPair pointerPairAt(GestureSpec spec, double t) { - if ("swipe".equals(spec.kind)) { - return new PointerPair( - new float[] {(float) (spec.x + (spec.x2 - spec.x) * t)}, - new float[] {(float) (spec.y + (spec.y2 - spec.y) * t)}); + private static int requireDuration(int value) { + if (value < MIN_DURATION_MS || value > MAX_DURATION_MS) { + throw new IllegalArgumentException("durationMs must be between 16 and 10000"); } - if ("pinch".equals(spec.kind)) { - double startRadius = spec.radius / Math.max(spec.scale, 1.0d); - double endRadius = spec.radius; - if (spec.scale < 1.0d) { - startRadius = spec.radius; - endRadius = spec.radius * spec.scale; - } - double radius = startRadius + (endRadius - startRadius) * t; - return new PointerPair( - new float[] {(float) (spec.x - radius), (float) (spec.x + radius)}, - new float[] {(float) spec.y, (float) spec.y}); - } - double centerX = spec.x; - double centerY = spec.y; - double radius = spec.radius; - if ("transform".equals(spec.kind)) { - centerX = spec.x + spec.dx * t; - centerY = spec.y + spec.dy * t; - double startRadius = spec.radius / Math.max(spec.scale, 1.0d); - double endRadius = spec.radius; - if (spec.scale < 1.0d) { - startRadius = spec.radius; - endRadius = spec.radius * spec.scale; - } - radius = startRadius + (endRadius - startRadius) * t; - } - double angle = Math.toRadians(-90 + spec.degrees * t); - return new PointerPair( - new float[] { - (float) (centerX + Math.cos(angle) * radius), - (float) (centerX - Math.cos(angle) * radius) - }, - new float[] { - (float) (centerY + Math.sin(angle) * radius), - (float) (centerY - Math.sin(angle) * radius) - }); - } - - private static int clamp(int value, int min, int max) { - return Math.min(Math.max(value, min), max); + return value; } - private static boolean isFinite(double value) { + private static boolean finite(double value) { return !Double.isNaN(value) && !Double.isInfinite(value); } - private static final class GestureSpec { + private static final class GesturePlan { final String kind; - final int x; - final int y; - final int dx; - final int dy; - final int x2; - final int y2; final int durationMs; - final double scale; - final double degrees; - final int radius; + final PointerPath[] pointers; - GestureSpec( - String kind, - int x, - int y, - int dx, - int dy, - int x2, - int y2, - int durationMs, - double scale, - double degrees, - int radius) { + GesturePlan(String kind, int durationMs, PointerPath[] pointers) { this.kind = kind; + this.durationMs = durationMs; + this.pointers = pointers; + } + } + + private static final class PointerPath { + final PointerSample[] samples; + + PointerPath(PointerSample[] samples) { + this.samples = samples; + } + } + + private static final class PointerSample { + final long offsetMs; + final float x; + final float y; + + PointerSample(long offsetMs, float x, float y) { + this.offsetMs = offsetMs; this.x = x; this.y = y; - this.dx = dx; - this.dy = dy; - this.x2 = x2; - this.y2 = y2; - this.durationMs = durationMs; - this.scale = scale; - this.degrees = degrees; - this.radius = radius; } } - private static final class PointerPair { - final int pointerCount; + private static final class PointerState { + final int count; final float[] x; final float[] y; - PointerPair(float[] x, float[] y) { - this.pointerCount = x.length; + PointerState(float[] x, float[] y) { + this.count = x.length; this.x = x; this.y = y; } - PointerPair firstOnly() { - return new PointerPair( - new float[] {x[0]}, - new float[] {y[0]}); + PointerState firstOnly() { + return new PointerState(new float[] {x[0]}, new float[] {y[0]}); } } } diff --git a/android-multitouch-helper/src/main/java/com/callstack/agentdevice/multitouchhelper/PointerEventSchedule.java b/android-multitouch-helper/src/main/java/com/callstack/agentdevice/multitouchhelper/PointerEventSchedule.java new file mode 100644 index 000000000..04ea478f7 --- /dev/null +++ b/android-multitouch-helper/src/main/java/com/callstack/agentdevice/multitouchhelper/PointerEventSchedule.java @@ -0,0 +1,62 @@ +package com.callstack.agentdevice.multitouchhelper; + +import java.util.ArrayList; +import java.util.List; + +/** Defines the platform-independent event order consumed by Android touch injection. */ +final class PointerEventSchedule { + private static final long POINTER_LIFT_DELAY_MS = 8; + + enum Action { + DOWN, + POINTER_DOWN, + MOVE, + POINTER_UP, + UP + } + + static final class Step { + final Action action; + final int sampleIndex; + final int pointerCount; + final long offsetMs; + + Step(Action action, int sampleIndex, int pointerCount, long offsetMs) { + this.action = action; + this.sampleIndex = sampleIndex; + this.pointerCount = pointerCount; + this.offsetMs = offsetMs; + } + } + + private PointerEventSchedule() {} + + static List create(int pointerCount, long[] sampleOffsetsMs) { + if (pointerCount != 1 && pointerCount != 2) { + throw new IllegalArgumentException("Pointer event schedule requires one or two pointers"); + } + if (sampleOffsetsMs.length < 2) { + throw new IllegalArgumentException("Pointer event schedule requires at least two samples"); + } + + int lastIndex = sampleOffsetsMs.length - 1; + List steps = new ArrayList<>(sampleOffsetsMs.length + (pointerCount == 1 ? 1 : 3)); + steps.add(new Step(Action.DOWN, 0, 1, 0)); + if (pointerCount == 2) { + long pointerDownOffset = Math.max(1, Math.min(8, sampleOffsetsMs[1] - 1)); + steps.add(new Step(Action.POINTER_DOWN, 0, 2, pointerDownOffset)); + } + for (int sampleIndex = 1; sampleIndex <= lastIndex; sampleIndex += 1) { + steps.add( + new Step(Action.MOVE, sampleIndex, pointerCount, sampleOffsetsMs[sampleIndex])); + } + if (pointerCount == 2) { + steps.add( + new Step(Action.POINTER_UP, lastIndex, 2, sampleOffsetsMs[lastIndex])); + } + long finalUpOffset = + sampleOffsetsMs[lastIndex] + (pointerCount == 2 ? POINTER_LIFT_DELAY_MS : 0); + steps.add(new Step(Action.UP, lastIndex, 1, finalUpOffset)); + return steps; + } +} diff --git a/android-multitouch-helper/src/test/java/com/callstack/agentdevice/multitouchhelper/PointerEventScheduleTest.java b/android-multitouch-helper/src/test/java/com/callstack/agentdevice/multitouchhelper/PointerEventScheduleTest.java new file mode 100644 index 000000000..e4bb1f90e --- /dev/null +++ b/android-multitouch-helper/src/test/java/com/callstack/agentdevice/multitouchhelper/PointerEventScheduleTest.java @@ -0,0 +1,45 @@ +package com.callstack.agentdevice.multitouchhelper; + +import java.util.List; + +public final class PointerEventScheduleTest { + private PointerEventScheduleTest() {} + + public static void main(String[] args) { + assertSteps( + PointerEventSchedule.create(1, new long[] {0, 16, 32}), + "DOWN:0:1:0", + "MOVE:1:1:16", + "MOVE:2:1:32", + "UP:2:1:32"); + assertSteps( + PointerEventSchedule.create(2, new long[] {0, 16, 32}), + "DOWN:0:1:0", + "POINTER_DOWN:0:2:8", + "MOVE:1:2:16", + "MOVE:2:2:32", + "POINTER_UP:2:2:32", + "UP:2:1:40"); + } + + private static void assertSteps(List actual, String... expected) { + if (actual.size() != expected.length) { + throw new AssertionError("Expected " + expected.length + " steps, got " + actual.size()); + } + for (int index = 0; index < expected.length; index += 1) { + PointerEventSchedule.Step step = actual.get(index); + String description = + step.action + + ":" + + step.sampleIndex + + ":" + + step.pointerCount + + ":" + + step.offsetMs; + if (!expected[index].equals(description)) { + throw new AssertionError( + "Step " + index + ": expected " + expected[index] + ", got " + description); + } + } + } +} diff --git a/apple-runner/AgentDeviceRunner/AgentDeviceRunnerUITests/RunnerAXSnapshotBridge.h b/apple-runner/AgentDeviceRunner/AgentDeviceRunnerUITests/RunnerAXSnapshotBridge.h index 24d95de74..9f4ae95e0 100644 --- a/apple-runner/AgentDeviceRunner/AgentDeviceRunnerUITests/RunnerAXSnapshotBridge.h +++ b/apple-runner/AgentDeviceRunner/AgentDeviceRunnerUITests/RunnerAXSnapshotBridge.h @@ -9,6 +9,8 @@ NS_ASSUME_NONNULL_BEGIN maxDepth:(NSInteger)maxDepth maxNodes:(NSInteger)maxNodes; ++ (NSInteger)processIdentifierForApplication:(XCUIApplication *)application; + @end NS_ASSUME_NONNULL_END diff --git a/apple-runner/AgentDeviceRunner/AgentDeviceRunnerUITests/RunnerAXSnapshotBridge.m b/apple-runner/AgentDeviceRunner/AgentDeviceRunnerUITests/RunnerAXSnapshotBridge.m index b6a34db49..8fe3f6bd3 100644 --- a/apple-runner/AgentDeviceRunner/AgentDeviceRunnerUITests/RunnerAXSnapshotBridge.m +++ b/apple-runner/AgentDeviceRunner/AgentDeviceRunnerUITests/RunnerAXSnapshotBridge.m @@ -164,6 +164,11 @@ + (NSInteger)integerFrom:(id)target selectorName:(NSString *)selectorName return send(target, selector); } ++ (NSInteger)processIdentifierForApplication:(XCUIApplication *)application +{ + return [self integerFrom:application selectorName:@"processID"]; +} + + (id)accessibilityApplicationForApplication:(XCUIApplication *)application axClient:(id)axClient { NSInteger targetProcessID = [self integerFrom:application selectorName:@"processID"]; diff --git a/apple-runner/AgentDeviceRunner/AgentDeviceRunnerUITests/RunnerSynthesizedGesture.h b/apple-runner/AgentDeviceRunner/AgentDeviceRunnerUITests/RunnerSynthesizedGesture.h index fc2ca9a8a..8eb12ed7e 100644 --- a/apple-runner/AgentDeviceRunner/AgentDeviceRunnerUITests/RunnerSynthesizedGesture.h +++ b/apple-runner/AgentDeviceRunner/AgentDeviceRunnerUITests/RunnerSynthesizedGesture.h @@ -4,16 +4,6 @@ NS_ASSUME_NONNULL_BEGIN @interface RunnerSynthesizedGesture : NSObject -+ (NSString * _Nullable)synthesizeTransformWithApplication:(id)application - x:(double)x - y:(double)y - dx:(double)dx - dy:(double)dy - scale:(double)scale - degrees:(double)degrees - radius:(double)radius - durationMs:(double)durationMs; - + (NSString * _Nullable)synthesizeSwipeWithApplication:(id)application x:(double)x y:(double)y @@ -32,6 +22,11 @@ NS_ASSUME_NONNULL_BEGIN x:(double)x y:(double)y; +// Each pointer is an ordered array of { x, y, offsetMs } samples. The first sample +// starts contact; subsequent samples move it; all pointers lift at their final offset. ++ (NSString * _Nullable)synthesizeGestureWithApplication:(id)application + pointerSamples:(NSArray *> *> *)pointerSamples; + // UIInterfaceOrientation of the app (1 portrait, 2 upsideDown, 3 landscapeRight, // 4 landscapeLeft), or 0 if unreadable. + (NSInteger)interfaceOrientationForApplication:(id)application; diff --git a/apple-runner/AgentDeviceRunner/AgentDeviceRunnerUITests/RunnerSynthesizedGesture.m b/apple-runner/AgentDeviceRunner/AgentDeviceRunnerUITests/RunnerSynthesizedGesture.m index 2b7056fdb..dd87a0c0e 100644 --- a/apple-runner/AgentDeviceRunner/AgentDeviceRunnerUITests/RunnerSynthesizedGesture.m +++ b/apple-runner/AgentDeviceRunner/AgentDeviceRunnerUITests/RunnerSynthesizedGesture.m @@ -1,7 +1,6 @@ #import "RunnerSynthesizedGesture.h" #import -#import #import typedef NSInteger (*RunnerMsgSendInteger)(id, SEL); @@ -41,19 +40,6 @@ typedef id (*RunnerDragPointerPathFactory)( static NSString * _Nullable RunnerRequireClass(Class cls, NSString *className); static NSString * _Nullable RunnerRequireSelector(Class cls, SEL selector, NSString *selectorName); static NSString * _Nullable RunnerRequireApplicationSelector(id application, SEL selector, NSString *selectorName); -static id RunnerPointerPath( - const RunnerXCTestEventBridge *bridge, - CGPoint start, - double x, - double y, - double dx, - double dy, - double scale, - double degrees, - double radius, - double durationMs, - double side -); static id RunnerSwipePointerPath( const RunnerXCTestEventBridge *bridge, CGPoint start, @@ -74,54 +60,16 @@ static id RunnerContinuousDragPointerPath( NSString *recordName, RunnerDragPointerPathFactory pathFactory ); -// XCTest's synthesized swipe convention moves to the endpoint in 100 ms, then holds there for -// the requested duration. The fast movement is what distinguishes a swipe from a slow pan for -// nested UIKit gesture recognizers. +// XCTest's proven swipe profile reaches the endpoint in 100 ms, then holds for the planned +// fling duration. Fast movement is what lets UIKit distinguish a fling from a timed pan. static const NSTimeInterval RunnerSwipeMovementDurationSeconds = 0.1; static id RunnerTapPointerPath( const RunnerXCTestEventBridge *bridge, CGPoint point ); -static CGPoint RunnerPointerPointAt( - double x, - double y, - double dx, - double dy, - double scale, - double degrees, - double baseRadius, - double t, - double side -); @implementation RunnerSynthesizedGesture -+ (NSString * _Nullable)synthesizeTransformWithApplication:(id)application - x:(double)x - y:(double)y - dx:(double)dx - dy:(double)dy - scale:(double)scale - degrees:(double)degrees - radius:(double)radius - durationMs:(double)durationMs { - @try { - return [self trySynthesizeTransformWithApplication:application - x:x - y:y - dx:dx - dy:dy - scale:scale - degrees:degrees - radius:radius - durationMs:durationMs]; - } @catch (NSException *exception) { - NSString *name = exception.name ?: @"NSException"; - NSString *reason = exception.reason ?: @"private XCTest event synthesis failed"; - return [NSString stringWithFormat:@"%@: %@", name, reason]; - } -} - + (NSString * _Nullable)synthesizeSwipeWithApplication:(id)application x:(double)x y:(double)y @@ -178,76 +126,84 @@ + (NSString * _Nullable)synthesizeTapWithApplication:(id)application } } -+ (NSInteger)interfaceOrientationForApplication:(id)application { - SEL selector = NSSelectorFromString(@"interfaceOrientation"); - if (![application respondsToSelector:selector]) { - return 0; // UIInterfaceOrientationUnknown - } - return ((RunnerMsgSendInteger)objc_msgSend)(application, selector); -} - -+ (NSString * _Nullable)trySynthesizeTransformWithApplication:(id)application - x:(double)x - y:(double)y - dx:(double)dx - dy:(double)dy - scale:(double)scale - degrees:(double)degrees - radius:(double)radius - durationMs:(double)durationMs { - RunnerXCTestEventBridge bridge; - NSString *missing = RunnerResolveXCTestEventBridge(application, &bridge); - if (missing != nil) { - return missing; - } - - NSInteger interfaceOrientation = - ((RunnerMsgSendInteger)objc_msgSend)(application, bridge.interfaceOrientationSelector); - NSInteger targetProcessID = ((RunnerMsgSendInteger)objc_msgSend)(application, bridge.processIDSelector); - if (targetProcessID <= 0) { - return @"private XCTest event synthesis unavailable: could not resolve target process ID"; - } - - id record = ((RunnerMsgSendInitRecord)objc_msgSend)( - [bridge.recordClass alloc], - bridge.initRecordSelector, - @"agent-device-transform", - interfaceOrientation - ); - if (record == nil) { - return @"private XCTest event synthesis failed: could not create event record"; - } - ((RunnerMsgSendSetInteger)objc_msgSend)(record, bridge.setTargetProcessIDSelector, targetProcessID); ++ (NSString * _Nullable)synthesizeGestureWithApplication:(id)application + pointerSamples:(NSArray *> *> *)pointerSamples { + @try { + RunnerXCTestEventBridge bridge; + NSString *missing = RunnerResolveXCTestEventBridge(application, &bridge); + if (missing != nil) return missing; + + NSInteger interfaceOrientation = + ((RunnerMsgSendInteger)objc_msgSend)(application, bridge.interfaceOrientationSelector); + NSInteger targetProcessID = + ((RunnerMsgSendInteger)objc_msgSend)(application, bridge.processIDSelector); + if (targetProcessID <= 0) { + return @"private XCTest event synthesis unavailable: could not resolve target process ID"; + } - double sides[] = {1.0, -1.0}; - for (int index = 0; index < 2; index += 1) { - double side = sides[index]; - id path = RunnerPointerPath( - &bridge, - RunnerPointerPointAt(x, y, dx, dy, scale, degrees, radius, 0.0, side), - x, - y, - dx, - dy, - scale, - degrees, - radius, - durationMs, - side + id record = ((RunnerMsgSendInitRecord)objc_msgSend)( + [bridge.recordClass alloc], + bridge.initRecordSelector, + @"agent-device-gesture-plan", + interfaceOrientation ); - if (path == nil) { - return @"private XCTest event synthesis failed: could not create pointer path"; + if (record == nil) { + return @"private XCTest event synthesis failed: could not create event record"; + } + ((RunnerMsgSendSetInteger)objc_msgSend)(record, bridge.setTargetProcessIDSelector, targetProcessID); + + for (NSArray *> *samples in pointerSamples) { + NSDictionary *first = samples.firstObject; + if (first == nil) return @"private XCTest event synthesis failed: empty pointer path"; + CGPoint start = CGPointMake(first[@"x"].doubleValue, first[@"y"].doubleValue); + id path = ((RunnerMsgSendInitPath)objc_msgSend)( + [bridge.pathClass alloc], + bridge.initPathSelector, + start, + first[@"offsetMs"].doubleValue / 1000.0 + ); + if (path == nil) { + return @"private XCTest event synthesis failed: could not create pointer path"; + } + for (NSUInteger index = 1; index < samples.count; index += 1) { + NSDictionary *sample = samples[index]; + CGPoint point = CGPointMake(sample[@"x"].doubleValue, sample[@"y"].doubleValue); + ((RunnerMsgSendPathMove)objc_msgSend)( + path, + bridge.moveSelector, + point, + sample[@"offsetMs"].doubleValue / 1000.0 + ); + } + NSDictionary *last = samples.lastObject; + ((RunnerMsgSendPathOffset)objc_msgSend)( + path, + bridge.liftSelector, + last[@"offsetMs"].doubleValue / 1000.0 + ); + ((RunnerMsgSendAddPath)objc_msgSend)(record, bridge.addPathSelector, path); + } + + NSError *error = nil; + BOOL ok = ((RunnerMsgSendSynthesize)objc_msgSend)(record, bridge.synthesizeSelector, &error); + if (!ok) { + NSString *detail = error.localizedDescription ?: @"synthesizeWithError returned false"; + return [NSString stringWithFormat:@"private XCTest event synthesis failed: %@", detail]; } - ((RunnerMsgSendAddPath)objc_msgSend)(record, bridge.addPathSelector, path); + return nil; + } @catch (NSException *exception) { + NSString *name = exception.name ?: @"NSException"; + NSString *reason = exception.reason ?: @"private XCTest event synthesis failed"; + return [NSString stringWithFormat:@"%@: %@", name, reason]; } +} - NSError *error = nil; - BOOL ok = ((RunnerMsgSendSynthesize)objc_msgSend)(record, bridge.synthesizeSelector, &error); - if (!ok) { - NSString *detail = error.localizedDescription ?: @"synthesizeWithError returned false"; - return [NSString stringWithFormat:@"private XCTest event synthesis failed: %@", detail]; ++ (NSInteger)interfaceOrientationForApplication:(id)application { + SEL selector = NSSelectorFromString(@"interfaceOrientation"); + if (![application respondsToSelector:selector]) { + return 0; // UIInterfaceOrientationUnknown } - return nil; + return ((RunnerMsgSendInteger)objc_msgSend)(application, selector); } static NSString * _Nullable RunnerTrySynthesizeDrag( @@ -426,37 +382,6 @@ + (NSString * _Nullable)trySynthesizeTapWithApplication:(id)application return nil; } -static id RunnerPointerPath( - const RunnerXCTestEventBridge *bridge, - CGPoint start, - double x, - double y, - double dx, - double dy, - double scale, - double degrees, - double radius, - double durationMs, - double side -) { - id path = - ((RunnerMsgSendInitPath)objc_msgSend)([bridge->pathClass alloc], bridge->initPathSelector, start, 0.0); - if (path == nil) { - return nil; - } - - int frameCount = MAX(3, (int)(durationMs / 16.0)); - NSTimeInterval durationSeconds = durationMs / 1000.0; - for (int index = 1; index <= frameCount; index += 1) { - double t = (double)index / (double)frameCount; - CGPoint point = RunnerPointerPointAt(x, y, dx, dy, scale, degrees, radius, t, side); - NSTimeInterval offset = durationSeconds * t; - ((RunnerMsgSendPathMove)objc_msgSend)(path, bridge->moveSelector, point, offset); - } - - ((RunnerMsgSendPathOffset)objc_msgSend)(path, bridge->liftSelector, durationSeconds); - return path; -} static id RunnerSwipePointerPath( const RunnerXCTestEventBridge *bridge, @@ -477,7 +402,6 @@ static id RunnerSwipePointerPath( end, RunnerSwipeMovementDurationSeconds ); - ((RunnerMsgSendPathOffset)objc_msgSend)( path, bridge->liftSelector, @@ -527,28 +451,4 @@ static id RunnerTapPointerPath( return path; } -static CGPoint RunnerPointerPointAt( - double x, - double y, - double dx, - double dy, - double scale, - double degrees, - double baseRadius, - double t, - double side -) { - double centerX = x + dx * t; - double centerY = y + dy * t; - double startRadius = baseRadius / MAX(scale, 1.0); - double endRadius = baseRadius; - if (scale < 1.0) { - startRadius = baseRadius; - endRadius = baseRadius * scale; - } - double radius = startRadius + (endRadius - startRadius) * t; - double angle = (-M_PI_2) + (degrees * M_PI / 180.0) * t; - return CGPointMake(centerX + cos(angle) * radius * side, centerY + sin(angle) * radius * side); -} - @end diff --git a/apple-runner/AgentDeviceRunner/AgentDeviceRunnerUITests/RunnerTests+CommandExecution.swift b/apple-runner/AgentDeviceRunner/AgentDeviceRunnerUITests/RunnerTests+CommandExecution.swift index 6bf820edd..c9852205d 100644 --- a/apple-runner/AgentDeviceRunner/AgentDeviceRunnerUITests/RunnerTests+CommandExecution.swift +++ b/apple-runner/AgentDeviceRunner/AgentDeviceRunnerUITests/RunnerTests+CommandExecution.swift @@ -1,5 +1,23 @@ import XCTest +#if AGENT_DEVICE_RUNNER_UNIT_TESTS && os(iOS) +import ObjectiveC.runtime + +private final class RunnerSynthesizedSwipeFailureStub: NSObject { + @objc(synthesizeSwipeWithApplication:x:y:x2:y2:durationMs:) + class func synthesizeSwipe( + application: XCUIApplication, + x: Double, + y: Double, + x2: Double, + y2: Double, + durationMs: Double + ) -> String? { + "forced private synthesis failure" + } +} +#endif + extension RunnerTests { // MARK: - Main Thread Dispatch @@ -58,7 +76,7 @@ extension RunnerTests { /// Runs a gesture action with uniform timing capture. Touch gestures pass `idleTimeout: true` /// (the default) to run inside the scroll idle-timeout + quiescence-skip wrapper; synthesis - /// gestures (pinch/rotate/transform) pass `false` because RunnerSynthesizedGesture governs its + /// pointer-plan gestures pass `false` because RunnerSynthesizedGesture governs their /// own timing. Returns the captured timing and the action's outcome. /// /// NOTE: a new SYNTHESIS gesture must pass `idleTimeout: false` — the default `true` would wrap @@ -133,6 +151,24 @@ extension RunnerTests { return Response(ok: true, data: data) } + /// Gesture plans already return canonical centroid endpoints from the portable runtime. + /// Keep runner timing/fallback diagnostics, but do not leak the coordinate-drag adapter's + /// visualization frame into only the fast-fling response shape. + private func canonicalPlannedGestureResponse(_ response: Response) -> Response { + guard response.ok, let data = response.data else { return response } + return Response( + ok: true, + data: DataPayload( + message: data.message, + gestureStartUptimeMs: data.gestureStartUptimeMs, + gestureEndUptimeMs: data.gestureEndUptimeMs, + gestureFallback: data.gestureFallback, + gestureFallbackMessage: data.gestureFallbackMessage, + gestureFallbackHint: data.gestureFallbackHint + ) + ) + } + #if AGENT_DEVICE_RUNNER_UNIT_TESTS func testGestureResponseIncludesSynthesizedTapFallbackDiagnostics() { let response = gestureResponse( @@ -165,6 +201,89 @@ extension RunnerTests { XCTAssertEqual(response.data?.maestroNonHittableCoordinateFallbackUsed, true) } + func testCanonicalPlannedGestureResponseOmitsDragFrameAndPreservesDiagnostics() { + let response = gestureResponse( + message: "fling", + timing: (gestureStartUptimeMs: 1, gestureEndUptimeMs: 2), + frame: .drag( + DragVisualizationFrame( + x: 160, + y: 150, + x2: 40, + y2: 150, + referenceWidth: 200, + referenceHeight: 300 + ) + ), + fallback: GestureFallback( + strategy: "xctest-coordinate-drag", + message: "Private synthesis unavailable", + hint: "Using XCTest coordinate fallback." + ) + ) + + let canonical = canonicalPlannedGestureResponse(response) + + XCTAssertEqual(canonical.data?.gestureStartUptimeMs, 1) + XCTAssertEqual(canonical.data?.gestureEndUptimeMs, 2) + XCTAssertEqual(canonical.data?.gestureFallback, "xctest-coordinate-drag") + XCTAssertEqual(canonical.data?.gestureFallbackMessage, "Private synthesis unavailable") + XCTAssertEqual(canonical.data?.gestureFallbackHint, "Using XCTest coordinate fallback.") + XCTAssertNil(canonical.data?.x) + XCTAssertNil(canonical.data?.y) + XCTAssertNil(canonical.data?.x2) + XCTAssertNil(canonical.data?.y2) + XCTAssertNil(canonical.data?.referenceWidth) + XCTAssertNil(canonical.data?.referenceHeight) + } + +#if os(iOS) + func testSinglePointerFlingFallsBackToXCTestCoordinateDragWhenPrivateSynthesisFails() throws { + let selector = NSSelectorFromString( + "synthesizeSwipeWithApplication:x:y:x2:y2:durationMs:" + ) + guard + let synthesizedSwipeMethod = class_getClassMethod(RunnerSynthesizedGesture.self, selector), + let failureStubMethod = class_getClassMethod(RunnerSynthesizedSwipeFailureStub.self, selector) + else { + XCTFail("unable to install synthesized swipe failure stub") + return + } + let originalImplementation = method_getImplementation(synthesizedSwipeMethod) + method_setImplementation( + synthesizedSwipeMethod, + method_getImplementation(failureStubMethod) + ) + app.launch() + runnerAccessibilityHealth = .healthy + defer { + method_setImplementation(synthesizedSwipeMethod, originalImplementation) + invalidateCachedTarget(reason: "unit_test_cleanup") + app.terminate() + } + let command = try runnerCommandFixture( + """ + {"command":"gesture","commandId":"gesture-fling-fallback","gesturePlan":{"topology":"single","intent":"fling","durationMs":100,"viewport":{"x":0,"y":0,"width":200,"height":300},"pointers":[{"pointerId":0,"samples":[{"offsetMs":0,"point":{"x":160,"y":150}},{"offsetMs":100,"point":{"x":40,"y":150}}]}]}} + """ + ) + + let response = try executeOnMainPrepared(command: command, activeApp: app) + + XCTAssertTrue(response.ok) + XCTAssertEqual(response.data?.message, "fling") + XCTAssertEqual(response.data?.gestureFallback, "xctest-coordinate-drag") + XCTAssertEqual(response.data?.gestureFallbackMessage, "forced private synthesis failure") + XCTAssertEqual( + response.data?.gestureFallbackHint, + "Private XCTest event synthesis is required for AX-free coordinate drag on iOS; update Xcode if this persists." + ) + XCTAssertNil(response.data?.x) + XCTAssertNil(response.data?.y) + XCTAssertNil(response.data?.x2) + XCTAssertNil(response.data?.y2) + } +#endif + func testXCTestRecordedFailureResponseFailsMutatingSuccesses() throws { let command = try runnerCommandFixture(#"{"command":"tap","commandId":"tap-1"}"#) let response = Response(ok: true, data: DataPayload(message: "tapped")) @@ -196,6 +315,28 @@ extension RunnerTests { XCTAssertNil(xctestRecordedFailureResponse(command: tapCommand, response: runnerFatalResponse)) } + func testMissingBundleCommandInvalidatesCompleteCachedTargetState() throws { + app.launch() + currentApp = app + currentBundleId = "com.example.stale-target" + currentAppProcessIdentifier = 42 + snapshotXCTestPenaltyWarmupExemptionPending = true + defer { + invalidateCachedTarget(reason: "unit_test_cleanup") + app.terminate() + } + let command = try runnerCommandFixture( + #"{"command":"snapshot","commandId":"snapshot-without-bundle"}"# + ) + + _ = prepareActiveCommandContext(command: command) + + XCTAssertNil(currentApp) + XCTAssertNil(currentBundleId) + XCTAssertNil(currentAppProcessIdentifier) + XCTAssertFalse(snapshotXCTestPenaltyWarmupExemptionPending) + } + func testSkipAppActivationPreflightOnlyIncludesCoordinateOnlySynthesizedTaps() throws { currentApp = app currentBundleId = nil @@ -264,7 +405,7 @@ extension RunnerTests { """ {"command":"sequence","commandId":"seq-1","steps":[ {"kind":"tap","x":10,"y":20,"synthesized":true}, - {"kind":"drag","x":10,"y":200,"x2":10,"y2":100,"synthesized":true} + {"kind":"longPress","x":10,"y":200,"durationMs":300} ]} """ ) @@ -945,11 +1086,12 @@ extension RunnerTests { if let bundleId = requestedBundleId { if currentBundleId != bundleId || currentApp == nil { _ = activateTarget(bundleId: bundleId, reason: "bundle_changed") + } else { + refreshCachedTargetIfProcessChanged(bundleId: bundleId) } } else { // Do not reuse stale bundle targets when the caller does not explicitly request one. - currentApp = nil - currentBundleId = nil + invalidateCachedTarget(reason: "missing_app_bundle") } activeApp = currentApp ?? app @@ -1150,7 +1292,6 @@ extension RunnerTests { durationMs: command.durationMs, synthesized: command.synthesized == true, message: "dragged", - dragSemantics: command.dragSemantics, synthesizedPolicyKind: .synthesizedDrag ) case .scroll: @@ -1470,79 +1611,53 @@ extension RunnerTests { return Response(ok: false, error: ErrorPayload(message: "alert not found")) } return handleAlert(alert, action: action) - case .pinch: - guard let scale = command.scale, scale > 0 else { - return Response(ok: false, error: ErrorPayload(message: "pinch requires scale > 0")) - } - let (timing, outcome) = performGesture(activeApp, idleTimeout: false) { - pinch(app: activeApp, scale: scale, x: command.x, y: command.y) - } - if let response = unsupportedResponse(for: outcome) { - return response - } - return gestureResponse(message: "pinched", timing: timing) - case .sequence: - return executeSequence(command: command, activeApp: activeApp) - case .rotateGesture: - guard let degrees = command.degrees, degrees.isFinite else { - return Response(ok: false, error: ErrorPayload(message: "rotateGesture requires degrees")) - } - let velocity = command.velocity ?? (degrees >= 0 ? 1.0 : -1.0) - guard velocity.isFinite && velocity != 0 else { - return Response(ok: false, error: ErrorPayload(message: "rotateGesture velocity must be non-zero")) - } - let (timing, outcome) = performGesture(activeApp, idleTimeout: false) { - rotateGesture( - app: activeApp, - degrees: degrees, - x: command.x, - y: command.y, - velocity: velocity + case .gesture: + guard let plan = command.gesturePlan else { + return Response( + ok: false, + error: ErrorPayload(code: "INVALID_ARGS", message: "gesture requires gesturePlan") ) } - if let response = unsupportedResponse(for: outcome) { - return response - } - return gestureResponse(message: "rotatedGesture", timing: timing) - case .transformGesture: - guard - let x = command.x, - let y = command.y, - let dx = command.dx, - let dy = command.dy, - x.isFinite, - y.isFinite, - dx.isFinite, - dy.isFinite - else { - return Response(ok: false, error: ErrorPayload(message: "transformGesture requires finite x y dx dy")) - } - guard let scale = command.scale, scale.isFinite, scale > 0 else { - return Response(ok: false, error: ErrorPayload(message: "transformGesture requires scale > 0")) - } - guard let degrees = command.degrees, degrees.isFinite else { - return Response(ok: false, error: ErrorPayload(message: "transformGesture requires finite degrees")) + if let validationError = plannedGestureValidationError(plan) { + return Response( + ok: false, + error: ErrorPayload(code: "INVALID_ARGS", message: validationError) + ) } - let durationMs = command.durationMs ?? 300 - guard durationMs.isFinite && durationMs >= 16 else { - return Response(ok: false, error: ErrorPayload(message: "transformGesture durationMs must be >= 16")) + if plannedGestureExecution(for: plan) == .fastSwipe { + // Validation above guarantees a non-empty, single-pointer path for this execution kind. + let first = plan.pointers[0].samples.first!.point + let last = plan.pointers[0].samples.last!.point + return canonicalPlannedGestureResponse( + executeDragGesture( + activeApp: activeApp, + x: first.x, + y: first.y, + x2: last.x, + y2: last.y, + durationMs: plan.durationMs, + synthesized: true, + message: plan.intent, + synthesizedPolicyKind: .synthesizedDrag, + synthesizedProfile: .fastSwipe + ) + ) } let (timing, outcome) = performGesture(activeApp, idleTimeout: false) { - transformGesture( - app: activeApp, - x: x, - y: y, - dx: dx, - dy: dy, - scale: scale, - degrees: degrees, - durationMs: durationMs - ) + sampledPlannedGesture(app: activeApp, plan: plan) } if let response = unsupportedResponse(for: outcome) { return response } - return gestureResponse(message: "transformedGesture", timing: timing) + return gestureResponse(message: plan.intent, timing: timing) + case .gestureViewport: + let frame = resolvedTouchReferenceFrame(app: activeApp, appFrame: activeApp.frame) + guard !frame.isNull, !frame.isInfinite, !frame.isEmpty else { + return Response(ok: false, error: ErrorPayload(code: "COMMAND_FAILED", message: "Active app interaction viewport is unavailable")) + } + return Response(ok: true, data: DataPayload(message: "gestureViewport", x: frame.minX, y: frame.minY, x2: frame.width, y2: frame.height)) + case .sequence: + return executeSequence(command: command, activeApp: activeApp) } } @@ -1558,9 +1673,9 @@ extension RunnerTests { durationMs: Double?, synthesized: Bool, message: String, - dragSemantics: SynthesizedDragSemantics? = nil, synthesizedContext: SynthesizedCoordinateContext? = nil, - synthesizedPolicyKind: SynthesizedGesturePolicyKind + synthesizedPolicyKind: SynthesizedGesturePolicyKind, + synthesizedProfile: SynthesizedDragProfile = .continuous ) -> Response { let commandName = dragCommandName(message: message) guard x.isFinite, y.isFinite, x2.isFinite, y2.isFinite else { @@ -1577,9 +1692,9 @@ extension RunnerTests { y2: y2, durationMs: durationMs, message: message, - dragSemantics: dragSemantics, context: synthesizedContext, - policyKind: synthesizedPolicyKind + policyKind: synthesizedPolicyKind, + profile: synthesizedProfile ) { return synthesizedResponse } @@ -1603,7 +1718,7 @@ extension RunnerTests { x2: dragPoints.x2, y2: dragPoints.y2, durationMs: durationMs, - semantics: dragSemantics, + profile: synthesizedProfile, context: context ) } @@ -1644,9 +1759,9 @@ extension RunnerTests { y2: Double, durationMs: Double?, message: String, - dragSemantics: SynthesizedDragSemantics?, context: SynthesizedCoordinateContext?, - policyKind: SynthesizedGesturePolicyKind + policyKind: SynthesizedGesturePolicyKind, + profile: SynthesizedDragProfile ) -> Response? { #if os(iOS) let policy = synthesizedGesturePolicy(policyKind) @@ -1698,7 +1813,7 @@ extension RunnerTests { x2: plan.points.x2, y2: plan.points.y2, durationMs: durationMs, - semantics: dragSemantics, + profile: profile, context: plan.context ) } diff --git a/apple-runner/AgentDeviceRunner/AgentDeviceRunnerUITests/RunnerTests+CommandJournal.swift b/apple-runner/AgentDeviceRunner/AgentDeviceRunnerUITests/RunnerTests+CommandJournal.swift index 1132e0254..5168192f9 100644 --- a/apple-runner/AgentDeviceRunner/AgentDeviceRunnerUITests/RunnerTests+CommandJournal.swift +++ b/apple-runner/AgentDeviceRunner/AgentDeviceRunnerUITests/RunnerTests+CommandJournal.swift @@ -146,7 +146,7 @@ final class RunnerCommandJournal { case .tap, .mouseClick, .longPress, .drag, .remotePress, .type, .swipe, .scroll, .desktopScroll, .findText, .querySelector, .readText, .back, .backInApp, .backSystem, .home, .rotate, .appSwitcher, .keyboardDismiss, .keyboardReturn, - .alert, .pinch, .sequence, .rotateGesture, .transformGesture, .recordStart, .recordStop, + .alert, .sequence, .gesture, .gestureViewport, .recordStart, .recordStop, .status, .uptime, .shutdown: return true } @@ -383,7 +383,7 @@ extension RunnerTests { gestureStartUptimeMs: 100, gestureEndUptimeMs: 120), SequenceStepResult(ok: true, kind: "tap", errorCode: nil, errorMessage: nil, gestureStartUptimeMs: 130, gestureEndUptimeMs: 150), - SequenceStepResult(ok: false, kind: "drag", errorCode: "UNSUPPORTED_OPERATION", + SequenceStepResult(ok: false, kind: "longPress", errorCode: "UNSUPPORTED_OPERATION", errorMessage: longError, gestureStartUptimeMs: 160, gestureEndUptimeMs: 180), ] diff --git a/apple-runner/AgentDeviceRunner/AgentDeviceRunnerUITests/RunnerTests+Interaction.swift b/apple-runner/AgentDeviceRunner/AgentDeviceRunnerUITests/RunnerTests+Interaction.swift index c41a95f6a..14c85a40b 100644 --- a/apple-runner/AgentDeviceRunner/AgentDeviceRunnerUITests/RunnerTests+Interaction.swift +++ b/apple-runner/AgentDeviceRunner/AgentDeviceRunnerUITests/RunnerTests+Interaction.swift @@ -13,6 +13,16 @@ private enum RunnerInterfaceOrientation { } extension RunnerTests { + enum PlannedGestureExecution: Equatable { + case fastSwipe + case sampled + } + + enum SynthesizedDragProfile { + case continuous + case fastSwipe + } + struct TouchVisualizationFrame { let x: Double let y: Double @@ -775,7 +785,7 @@ extension RunnerTests { x2: Double, y2: Double, durationMs: Double, - semantics: SynthesizedDragSemantics?, + profile: SynthesizedDragProfile = .continuous, context: SynthesizedCoordinateContext? = nil ) -> RunnerInteractionOutcome { #if os(iOS) @@ -795,8 +805,9 @@ extension RunnerTests { let frame = context.referenceFrame let start = nativeSynthesizedPoint(orientedX: x, orientedY: y, in: frame, interfaceOrientation: orientation) let end = nativeSynthesizedPoint(orientedX: x2, orientedY: y2, in: frame, interfaceOrientation: orientation) - let message = if semantics == .swipe { - RunnerSynthesizedGesture.synthesizeSwipe( + let message = switch profile { + case .continuous: + RunnerSynthesizedGesture.synthesizeContinuousDrag( withApplication: app, x: Double(start.x), y: Double(start.y), @@ -804,8 +815,8 @@ extension RunnerTests { y2: Double(end.y), durationMs: durationMs ) - } else { - RunnerSynthesizedGesture.synthesizeContinuousDrag( + case .fastSwipe: + RunnerSynthesizedGesture.synthesizeSwipe( withApplication: app, x: Double(start.x), y: Double(start.y), @@ -1166,125 +1177,134 @@ extension RunnerTests { } } - func pinch(app: XCUIApplication, scale: Double, x: Double?, y: Double?) -> RunnerInteractionOutcome { -#if os(iOS) - // A coordinate tap+drag is a single-finger gesture: React Native reads it as a pan - // and the pinch scale never changes (#629). Drive the two-finger XCTest synthesis - // path (the same one transformGesture uses) with zero translation/rotation so RN's - // pinch recognizer actually fires. - let frame = interactionRoot(app: app).frame - let centerX = x ?? Double(frame.midX) - let centerY = y ?? Double(frame.midY) - return transformGesture( - app: app, - x: centerX, - y: centerY, - dx: 0, - dy: 0, - scale: scale, - degrees: 0, - durationMs: 300 - ) -#elseif os(tvOS) - return .unsupported( - message: "pinch is not supported on tvOS", - hint: "tvOS has no touch input; pinch requires a touchscreen (run on iOS)." - ) -#else - return .unsupported( - message: "pinch is not supported on macOS", - hint: "macOS automation has no multi-touch input; pinch requires a touchscreen (run on iOS)." - ) -#endif + func plannedGestureValidationError(_ plan: RunnerGesturePlan) -> String? { + guard plan.topology == "single" || plan.topology == "two" else { + return "planned gesture topology must be single or two" + } + let supportedIntent = plan.topology == "single" + ? plan.intent == "fling" || plan.intent == "pan" + : plan.intent == "pan" || plan.intent == "pinch" || plan.intent == "rotate" + || plan.intent == "transform" + guard supportedIntent else { return "planned gesture has unsupported intent for its topology" } + guard plan.durationMs.isFinite, plan.durationMs >= 16, plan.durationMs <= 10_000 else { + return "planned gesture durationMs must be between 16 and 10000" + } + let viewport = plan.viewport + guard viewport.x.isFinite, viewport.y.isFinite, viewport.width.isFinite, + viewport.height.isFinite, viewport.width > 0, viewport.height > 0 + else { + return "planned gesture viewport must be finite and positive" + } + let expectedPointerCount = plan.topology == "single" ? 1 : 2 + guard plan.pointers.count == expectedPointerCount else { + return "planned gesture pointer count does not match topology" + } + for (index, pointer) in plan.pointers.enumerated() where pointer.pointerId != index { + return "planned gesture requires ordered pointer ids" + } + let firstSamples = plan.pointers[0].samples + guard firstSamples.count >= 2 else { return "planned pointer paths require at least two samples" } + for pointer in plan.pointers { + guard pointer.samples.count == firstSamples.count else { + return "planned pointer paths require matching samples" + } + var previousOffset = -1.0 + for (index, sample) in pointer.samples.enumerated() { + guard sample.offsetMs.isFinite, + sample.offsetMs == firstSamples[index].offsetMs, + sample.offsetMs > previousOffset + else { + return "planned pointer sample offsets must match and strictly increase" + } + let point = sample.point + guard point.x.isFinite, point.y.isFinite, + point.x >= viewport.x, + point.x <= viewport.x + viewport.width, + point.y >= viewport.y, + point.y <= viewport.y + viewport.height + else { + return "planned pointer sample lies outside the viewport" + } + previousOffset = sample.offsetMs + } + guard pointer.samples.first?.offsetMs == 0, + pointer.samples.last?.offsetMs == plan.durationMs + else { + return "planned pointer paths must start at 0 and end at durationMs" + } + } + if plan.topology == "two" { + guard let firstStart = firstSamples.first?.point, + let secondStart = plan.pointers[1].samples.first?.point, + hypot(firstStart.x - secondStart.x, firstStart.y - secondStart.y) > 0 + else { + return "planned pointer paths require a positive initial span" + } + } + return nil } - func rotateGesture(app: XCUIApplication, degrees: Double, x: Double?, y: Double?, velocity: Double) -> RunnerInteractionOutcome { -#if os(iOS) - // Drive the two-finger XCTest synthesis path (the same one pinch/transformGesture use, #634) - // with zero translation/scale so React Native's rotation recognizer actually fires. The native - // XCUIElement.rotate(withVelocity:) injects a single synthetic rotation that RN's gesture - // handler does not read reliably — the same class of problem #629/#634 fixed for pinch. - // velocity is unused on iOS (synthesis speed is governed by durationMs); the wire contract - // keeps it for compatibility and direction is carried entirely by the sign of `degrees`. - let frame = interactionRoot(app: app).frame - let centerX = x ?? Double(frame.midX) - let centerY = y ?? Double(frame.midY) - return transformGesture( - app: app, - x: centerX, - y: centerY, - dx: 0, - dy: 0, - scale: 1, - degrees: degrees, - durationMs: 300 - ) -#elseif os(tvOS) - return .unsupported( - message: "rotate-gesture is not supported on tvOS", - hint: "tvOS has no touch input; rotation gestures require a touchscreen (run on iOS)." - ) -#else - return .unsupported( - message: "rotate-gesture is not supported on macOS", - hint: "macOS automation has no multi-touch input; rotation gestures require a touchscreen (run on iOS)." - ) -#endif + func plannedGestureExecution(for plan: RunnerGesturePlan) -> PlannedGestureExecution { + plan.topology == "single" && plan.intent == "fling" ? .fastSwipe : .sampled } - func transformGesture( + func sampledPlannedGesture( app: XCUIApplication, - x: Double, - y: Double, - dx: Double, - dy: Double, - scale: Double, - degrees: Double, - durationMs: Double + plan: RunnerGesturePlan ) -> RunnerInteractionOutcome { #if os(iOS) - let target = interactionRoot(app: app) let orientation = Int(RunnerSynthesizedGesture.interfaceOrientation(forApplication: app)) - let point = nativeSynthesizedPoint(orientedX: x, orientedY: y, in: app.frame, interfaceOrientation: orientation) - let vector = nativeSynthesizedVector(orientedDx: dx, orientedDy: dy, interfaceOrientation: orientation) - if let message = RunnerSynthesizedGesture.synthesizeTransform( + // The portable planner and validation use this exact viewport. Using app.frame here can + // diverge when XCTest unions transformed/off-screen descendants into the application frame. + let frame = CGRect( + x: plan.viewport.x, + y: plan.viewport.y, + width: plan.viewport.width, + height: plan.viewport.height + ) + let pointerSamples: [[[String: NSNumber]]] = plan.pointers.map { pointer in + pointer.samples.map { sample in + let point = nativeSynthesizedPoint( + orientedX: sample.point.x, + orientedY: sample.point.y, + in: frame, + interfaceOrientation: orientation + ) + return [ + "x": NSNumber(value: Double(point.x)), + "y": NSNumber(value: Double(point.y)), + "offsetMs": NSNumber(value: sample.offsetMs), + ] + } + } + if let message = RunnerSynthesizedGesture.synthesizeGesture( withApplication: app, - x: Double(point.x), - y: Double(point.y), - dx: Double(vector.dx), - dy: Double(vector.dy), - scale: scale, - degrees: degrees, - radius: transformGestureRadius(frame: target.frame, scale: scale), - durationMs: durationMs + pointerSamples: pointerSamples ) { return .unsupported( message: message, - hint: "This gesture uses private XCTest event-synthesis APIs; rebuild the runner with a supported Xcode (these APIs can change across Xcode versions)." + hint: "This gesture uses private XCTest event-synthesis APIs; rebuild the runner with a supported Xcode if this persists." ) } return .performed #elseif os(tvOS) return .unsupported( - message: "transformGesture is not supported on tvOS", - hint: "tvOS has no touch input; transform gestures require a touchscreen (run on iOS)." + message: "two-finger gestures are not supported on tvOS", + hint: "tvOS has no touch input; use remote-driven navigation." + ) +#elseif os(visionOS) + return .unsupported( + message: "two-finger touch gestures are not supported on visionOS", + hint: "The current XCTest synthesizer supports iOS and iPadOS touch simulators only." ) #else return .unsupported( - message: "transformGesture is not supported on macOS", - hint: "macOS automation has no multi-touch input; transform gestures require a touchscreen (run on iOS)." + message: "two-finger gestures are not supported on macOS", + hint: "macOS automation has no multi-touch input; run on an iOS simulator." ) #endif } - private func transformGestureRadius(frame: CGRect, scale: Double) -> Double { - let shorterSide = Double(min(frame.width, frame.height)) - let frameRadius = shorterSide * 0.20 - let minimumEndRadius = shorterSide * 0.08 - let scaleAdjustedRadius = scale < 1.0 ? max(frameRadius, minimumEndRadius / scale) : frameRadius - return min(max(scaleAdjustedRadius, 48.0), shorterSide * 0.35) - } - private func interactionRoot(app: XCUIApplication) -> XCUIElement { let windows = app.windows.allElementsBoundByIndex if let window = windows.first(where: { $0.exists && !$0.frame.isEmpty }) { @@ -1430,6 +1450,53 @@ extension RunnerTests { ) } + func testPlannedMultiTouchGestureAcceptsMatchingInBoundsTrajectories() throws { + let plan = try JSONDecoder().decode( + RunnerGesturePlan.self, + from: Data( + #"{"topology":"two","intent":"pan","durationMs":32,"viewport":{"x":0,"y":0,"width":200,"height":300},"pointers":[{"pointerId":0,"samples":[{"offsetMs":0,"point":{"x":80,"y":80}},{"offsetMs":16,"point":{"x":90,"y":85}},{"offsetMs":32,"point":{"x":100,"y":90}}]},{"pointerId":1,"samples":[{"offsetMs":0,"point":{"x":80,"y":120}},{"offsetMs":16,"point":{"x":90,"y":125}},{"offsetMs":32,"point":{"x":100,"y":130}}]}]}"#.utf8 + ) + ) + + XCTAssertNil(plannedGestureValidationError(plan)) + } + + func testPlannedMultiTouchGestureRejectsMismatchedOffsets() throws { + let plan = try JSONDecoder().decode( + RunnerGesturePlan.self, + from: Data( + #"{"topology":"two","intent":"transform","durationMs":32,"viewport":{"x":0,"y":0,"width":200,"height":300},"pointers":[{"pointerId":0,"samples":[{"offsetMs":0,"point":{"x":80,"y":80}},{"offsetMs":32,"point":{"x":100,"y":90}}]},{"pointerId":1,"samples":[{"offsetMs":0,"point":{"x":80,"y":120}},{"offsetMs":31,"point":{"x":100,"y":130}}]}]}"#.utf8 + ) + ) + + XCTAssertEqual( + plannedGestureValidationError(plan), + "planned pointer sample offsets must match and strictly increase" + ) + } + + func testSinglePointerFlingUsesFastSwipeExecution() throws { + let plan = try JSONDecoder().decode( + RunnerGesturePlan.self, + from: Data( + #"{"topology":"single","intent":"fling","durationMs":100,"viewport":{"x":0,"y":0,"width":200,"height":300},"pointers":[{"pointerId":0,"samples":[{"offsetMs":0,"point":{"x":160,"y":150}},{"offsetMs":100,"point":{"x":40,"y":150}}]}]}"#.utf8 + ) + ) + + XCTAssertEqual(plannedGestureExecution(for: plan), .fastSwipe) + } + + func testSinglePointerTimedPanUsesSampledExecution() throws { + let plan = try JSONDecoder().decode( + RunnerGesturePlan.self, + from: Data( + #"{"topology":"single","intent":"pan","durationMs":500,"viewport":{"x":0,"y":0,"width":200,"height":300},"pointers":[{"pointerId":0,"samples":[{"offsetMs":0,"point":{"x":160,"y":150}},{"offsetMs":250,"point":{"x":100,"y":150}},{"offsetMs":500,"point":{"x":40,"y":150}}]}]}"#.utf8 + ) + ) + + XCTAssertEqual(plannedGestureExecution(for: plan), .sampled) + } + func testDesktopScrollWheelDeltasMapDirections() throws { XCTAssertEqual(try XCTUnwrap(desktopScrollWheelDeltas(direction: "up", pixels: 120)).vertical, 120) XCTAssertEqual(try XCTUnwrap(desktopScrollWheelDeltas(direction: "down", pixels: 120)).vertical, -120) diff --git a/apple-runner/AgentDeviceRunner/AgentDeviceRunnerUITests/RunnerTests+Lifecycle.swift b/apple-runner/AgentDeviceRunner/AgentDeviceRunnerUITests/RunnerTests+Lifecycle.swift index 8c73d1c11..09721e8f9 100644 --- a/apple-runner/AgentDeviceRunner/AgentDeviceRunnerUITests/RunnerTests+Lifecycle.swift +++ b/apple-runner/AgentDeviceRunner/AgentDeviceRunnerUITests/RunnerTests+Lifecycle.swift @@ -85,6 +85,8 @@ extension RunnerTests { } currentApp = app currentBundleId = nil + currentAppProcessIdentifier = nil + snapshotXCTestPenaltyWarmupExemptionPending = false } func invalidateCachedTarget(reason: String) { @@ -93,6 +95,42 @@ extension RunnerTests { } currentApp = nil currentBundleId = nil + currentAppProcessIdentifier = nil + snapshotXCTestPenaltyWarmupExemptionPending = false + } + + func refreshCachedTargetIfProcessChanged(bundleId: String) { + guard currentBundleId == bundleId, currentApp != nil else { return } + let candidate = XCUIApplication(bundleIdentifier: bundleId) + let observedProcessIdentifier = Self.processIdentifier(of: candidate) + guard Self.shouldRefreshCachedTarget( + cachedProcessIdentifier: currentAppProcessIdentifier, + observedProcessIdentifier: observedProcessIdentifier + ) else { return } + NSLog( + "AGENT_DEVICE_RUNNER_TARGET_CACHE_REFRESH bundle=%@ previousPid=%d currentPid=%d", + bundleId, + currentAppProcessIdentifier ?? 0, + observedProcessIdentifier ?? 0 + ) + currentApp = candidate + currentAppProcessIdentifier = observedProcessIdentifier + clearSnapshotXCTestChannelPenalty(reason: "target_process_changed") + snapshotXCTestPenaltyWarmupExemptionPending = true + needsFirstInteractionDelay = true + } + + static func processIdentifier(of target: XCUIApplication) -> Int? { + let value = RunnerAXSnapshotBridge.processIdentifier(for: target) + return value > 0 ? value : nil + } + + static func shouldRefreshCachedTarget( + cachedProcessIdentifier: Int?, + observedProcessIdentifier: Int? + ) -> Bool { + guard let cachedProcessIdentifier, let observedProcessIdentifier else { return false } + return cachedProcessIdentifier != observedProcessIdentifier } func targetNeedsActivation(_ target: XCUIApplication) -> Bool { @@ -141,6 +179,8 @@ extension RunnerTests { target.activate() currentApp = target currentBundleId = bundleId + currentAppProcessIdentifier = Self.processIdentifier(of: target) + snapshotXCTestPenaltyWarmupExemptionPending = false needsFirstInteractionDelay = true return target } diff --git a/apple-runner/AgentDeviceRunner/AgentDeviceRunnerUITests/RunnerTests+LifecycleCacheTests.swift b/apple-runner/AgentDeviceRunner/AgentDeviceRunnerUITests/RunnerTests+LifecycleCacheTests.swift new file mode 100644 index 000000000..34898554f --- /dev/null +++ b/apple-runner/AgentDeviceRunner/AgentDeviceRunnerUITests/RunnerTests+LifecycleCacheTests.swift @@ -0,0 +1,54 @@ +import XCTest + +extension RunnerTests { + func testCachedTargetRefreshRequiresChangedPositiveProcessIdentity() { + XCTAssertFalse( + Self.shouldRefreshCachedTarget( + cachedProcessIdentifier: nil, + observedProcessIdentifier: 42 + ) + ) + XCTAssertFalse( + Self.shouldRefreshCachedTarget( + cachedProcessIdentifier: 42, + observedProcessIdentifier: 42 + ) + ) + XCTAssertTrue( + Self.shouldRefreshCachedTarget( + cachedProcessIdentifier: 41, + observedProcessIdentifier: 42 + ) + ) + } + + func testSnapshotPenaltyWarmupExemptionIsConsumedOnce() { + snapshotXCTestPenaltyWarmupExemptionPending = true + + XCTAssertTrue(consumeSnapshotXCTestPenaltyWarmupExemption()) + XCTAssertFalse(consumeSnapshotXCTestPenaltyWarmupExemption()) + } + + func testSnapshotPenaltyCanBeClearedAcrossTargetProcessReplacement() { + penalizeSnapshotXCTestChannel(bundleId: "com.example.app", reason: "test") + XCTAssertTrue(isSnapshotXCTestChannelPenalized(bundleId: "com.example.app")) + + clearSnapshotXCTestChannelPenalty(reason: "target_process_changed") + + XCTAssertFalse(isSnapshotXCTestChannelPenalized(bundleId: "com.example.app")) + } + + func testCachedTargetInvalidationClearsProcessBoundState() { + currentApp = app + currentBundleId = "com.example.app" + currentAppProcessIdentifier = 42 + snapshotXCTestPenaltyWarmupExemptionPending = true + + invalidateCachedTarget(reason: "unit_test") + + XCTAssertNil(currentApp) + XCTAssertNil(currentBundleId) + XCTAssertNil(currentAppProcessIdentifier) + XCTAssertFalse(snapshotXCTestPenaltyWarmupExemptionPending) + } +} diff --git a/apple-runner/AgentDeviceRunner/AgentDeviceRunnerUITests/RunnerTests+Models.swift b/apple-runner/AgentDeviceRunner/AgentDeviceRunnerUITests/RunnerTests+Models.swift index 1a3839042..912118085 100644 --- a/apple-runner/AgentDeviceRunner/AgentDeviceRunnerUITests/RunnerTests+Models.swift +++ b/apple-runner/AgentDeviceRunner/AgentDeviceRunnerUITests/RunnerTests+Models.swift @@ -24,10 +24,9 @@ enum CommandType: String, Codable { case keyboardDismiss case keyboardReturn case alert - case pinch case sequence - case rotateGesture - case transformGesture + case gesture + case gestureViewport case recordStart case recordStop case status @@ -35,12 +34,6 @@ enum CommandType: String, Codable { case shutdown } -enum SynthesizedDragSemantics: String, Codable { - case swipe - case pan - case fling -} - /// Runner command traits — see CONTEXT.md ("Runner command traits"). /// /// Single source of truth for how the runner classifies a command across three @@ -80,11 +73,11 @@ extension CommandType { // .sequence is the fused multi-step gesture batch. case .tap, .longPress, .drag, .remotePress, .type, .swipe, .scroll, .desktopScroll, .back, .backInApp, .backSystem, .rotate, .appSwitcher, - .keyboardDismiss, .keyboardReturn, .pinch, .sequence, .rotateGesture, .transformGesture: + .keyboardDismiss, .keyboardReturn, .sequence, .gesture: return CommandTraits(isInteraction: true, readOnly: .never, isLifecycle: false) // Read-only reads: eligible for the session-invalidating retry. - case .findText, .readText, .snapshot: + case .findText, .readText, .snapshot, .gestureViewport: return CommandTraits(isInteraction: false, readOnly: .always, isLifecycle: false) // Screenshot is both a read and a runner-lifecycle command (skips app-activation preflight). @@ -132,16 +125,12 @@ struct Command: Codable { let remoteButton: String? let x2: Double? let y2: Double? - let dx: Double? - let dy: Double? let durationMs: Double? let direction: String? let amount: Double? let pixels: Double? let orientation: String? - let scale: Double? - let degrees: Double? - let velocity: Double? + let gesturePlan: RunnerGesturePlan? let outPath: String? let fps: Int? let maxSize: Int? @@ -151,11 +140,40 @@ struct Command: Codable { let raw: Bool? let fullscreen: Bool? let synthesized: Bool? - /// Preserves the public gesture's timing semantics after it is lowered to runner `drag`. - let dragSemantics: SynthesizedDragSemantics? let steps: [SequenceStep]? } +/// Canonical one- or two-pointer plan produced by the portable TypeScript planner. +struct RunnerGesturePlan: Codable { + let topology: String + let intent: String + let durationMs: Double + let viewport: RunnerGestureViewport + let pointers: [RunnerGesturePointer] +} + +struct RunnerGestureViewport: Codable { + let x: Double + let y: Double + let width: Double + let height: Double +} + +struct RunnerGesturePointer: Codable { + let pointerId: Int + let samples: [RunnerGestureSample] +} + +struct RunnerGestureSample: Codable { + let offsetMs: Double + let point: RunnerGesturePoint +} + +struct RunnerGesturePoint: Codable { + let x: Double + let y: Double +} + /// One allowlisted coordinate gesture step inside a fused `sequence` command. /// `kind` is decoded as a raw String (not an enum) so the runner can return a clear /// INVALID_ARGS for an unknown kind instead of a generic decode failure. @@ -163,15 +181,11 @@ struct SequenceStep: Codable { let kind: String let x: Double? let y: Double? - let x2: Double? - let y2: Double? let durationMs: Double? let pauseMs: Double? - /// For `tap`/`drag` steps on iOS non-tv: use the synthesized HID fast path instead of the + /// For `tap` steps on iOS non-tv: use the synthesized HID fast path instead of the /// drag-based XCUICoordinate path, matching the individual command behavior. let synthesized: Bool? - /// Preserves one-shot swipe timing when repeated swipes are fused into a sequence. - let dragSemantics: SynthesizedDragSemantics? } /// Per-step result for a `sequence` response. `ok:false` carries the failing step's diff --git a/apple-runner/AgentDeviceRunner/AgentDeviceRunnerUITests/RunnerTests+SequenceExecution.swift b/apple-runner/AgentDeviceRunner/AgentDeviceRunnerUITests/RunnerTests+SequenceExecution.swift index 53c64480d..fc20b2ad7 100644 --- a/apple-runner/AgentDeviceRunner/AgentDeviceRunnerUITests/RunnerTests+SequenceExecution.swift +++ b/apple-runner/AgentDeviceRunner/AgentDeviceRunnerUITests/RunnerTests+SequenceExecution.swift @@ -9,7 +9,7 @@ extension RunnerTests { /// Allowlisted step kinds. Validated on both sides so an unsupported kind is rejected with a /// clear INVALID_ARGS naming the step index, executing nothing. - private var sequenceableStepKinds: Set { ["tap", "doubleTap", "longPress", "drag"] } + private var sequenceableStepKinds: Set { ["tap", "doubleTap", "longPress"] } /// Per-step outcome carried by `assembleSequenceExecution`. The timing is captured by the /// executor closure (via performGesture) so ordering/stop-on-failure stay device-free testable. @@ -127,17 +127,12 @@ extension RunnerTests { private func validateSequenceStep(_ step: SequenceStep, index: Int) -> Response? { guard sequenceableStepKinds.contains(step.kind) else { return sequenceInvalidArgs( - "sequence step \(index) has unsupported kind \"\(step.kind)\"; allowed: tap, doubleTap, longPress, drag" + "sequence step \(index) has unsupported kind \"\(step.kind)\"; allowed: tap, doubleTap, longPress" ) } guard let x = step.x, let y = step.y, x.isFinite, y.isFinite else { return sequenceInvalidArgs("sequence step \(index) (\(step.kind)) requires finite x and y") } - if step.kind == "drag" { - guard let x2 = step.x2, let y2 = step.y2, x2.isFinite, y2.isFinite else { - return sequenceInvalidArgs("sequence step \(index) (drag) requires finite x2 and y2") - } - } return nil } @@ -194,92 +189,6 @@ extension RunnerTests { #endif // Synthesis unsupported (e.g. macOS) — fall through to the drag-based tapAt below. } - if step.kind == "drag", step.synthesized == true { - let policyKind = SynthesizedGesturePolicyKind.synthesizedDrag - let dragPoints: DragPoints - let dragContext: SynthesizedCoordinateContext? -#if os(iOS) - guard let dragPlan = axFreeSynthesizedDragPlan( - app: activeApp, - x: x, - y: y, - x2: step.x2 ?? x, - y2: step.y2 ?? y, - context: synthesizedContext - ) else { - let nowMs = ProcessInfo.processInfo.systemUptime * 1000 - logSynthesizedGesturePolicyDecision(kind: policyKind, context: synthesizedContext, fallbackAttempted: false) - return SequenceStepOutcome( - outcome: .unsupported( - message: "synthesized coordinate drag could not resolve a finite coordinate frame", - hint: "Retry after the app is foregrounded, or use a plain screenshot to choose coordinates." - ), - gestureStartUptimeMs: nowMs, - gestureEndUptimeMs: nowMs - ) - } - dragPoints = dragPlan.points - dragContext = dragPlan.context -#else - dragPoints = keyboardAvoidingDragPoints( - app: activeApp, x: x, y: y, x2: step.x2 ?? x, y2: step.y2 ?? y) - dragContext = nil -#endif - let durationMs = min(max(step.durationMs ?? 250, 16), 10000) - let (timing, outcome) = performGesture(activeApp, idleTimeout: false) { - synthesizedDragAt( - app: activeApp, - x: dragPoints.x, - y: dragPoints.y, - x2: dragPoints.x2, - y2: dragPoints.y2, - durationMs: durationMs, - semantics: step.dragSemantics, - context: dragContext - ) - } - if case .performed = outcome { - logSynthesizedGesturePolicyDecision(kind: policyKind, context: dragContext, fallbackAttempted: false) - if let pauseMs = step.pauseMs, pauseMs > 0 { - sleepFor(min(max(pauseMs, 0), 10000) / 1000.0) - } - return SequenceStepOutcome( - outcome: outcome, - gestureStartUptimeMs: timing.gestureStartUptimeMs, - gestureEndUptimeMs: timing.gestureEndUptimeMs - ) - } -#if os(iOS) - guard dragContext?.allowsXCTestCoordinateFallback == true else { - logSynthesizedGesturePolicyDecision(kind: policyKind, context: dragContext, fallbackAttempted: false) - return SequenceStepOutcome( - outcome: outcome, - gestureStartUptimeMs: timing.gestureStartUptimeMs, - gestureEndUptimeMs: timing.gestureEndUptimeMs - ) - } -#endif - logSynthesizedGesturePolicyDecision(kind: policyKind, context: dragContext, fallbackAttempted: true) - let fallbackHoldDuration = synthesizedSwipeFallbackHoldDuration(durationMs: step.durationMs ?? 250) - let (fallbackTiming, fallbackOutcome) = performGesture(activeApp) { - dragAt( - app: activeApp, - x: dragPoints.x, - y: dragPoints.y, - x2: dragPoints.x2, - y2: dragPoints.y2, - holdDuration: fallbackHoldDuration - ) - } - if case .performed = fallbackOutcome, let pauseMs = step.pauseMs, pauseMs > 0 { - sleepFor(min(max(pauseMs, 0), 10000) / 1000.0) - } - return SequenceStepOutcome( - outcome: fallbackOutcome, - gestureStartUptimeMs: fallbackTiming.gestureStartUptimeMs, - gestureEndUptimeMs: fallbackTiming.gestureEndUptimeMs - ) - } let (timing, outcome) = performGesture(activeApp) { switch step.kind { case "doubleTap": @@ -288,20 +197,6 @@ extension RunnerTests { case "longPress": let duration = min(max(step.durationMs ?? 800, 16), 10000) / 1000.0 return longPressAt(app: activeApp, x: x, y: y, duration: duration) - case "drag": - // Route through keyboardAvoidingDragPoints for parity with the individual `.drag` command. - // The non-synthesized coordinate-drag path ignores durationMs, matching that command's - // non-synthesized branch. - let dragPoints = keyboardAvoidingDragPoints( - app: activeApp, x: x, y: y, x2: step.x2 ?? x, y2: step.y2 ?? y) - return dragAt( - app: activeApp, - x: dragPoints.x, - y: dragPoints.y, - x2: dragPoints.x2, - y2: dragPoints.y2, - holdDuration: coordinateDragHoldDuration() - ) default: return tapAt(app: activeApp, x: x, y: y) } @@ -354,17 +249,15 @@ extension RunnerTests { {"command":"sequence","commandId":"seq-1","steps":[ {"kind":"tap","x":100,"y":200}, {"kind":"doubleTap","x":101,"y":200}, - {"kind":"longPress","x":102,"y":200,"durationMs":300}, - {"kind":"drag","x":10,"y":600,"x2":10,"y2":200,"durationMs":250,"pauseMs":50} + {"kind":"longPress","x":102,"y":200,"durationMs":300,"pauseMs":50} ]} """ let command = try JSONDecoder().decode(Command.self, from: Data(json.utf8)) XCTAssertEqual(command.command, .sequence) - XCTAssertEqual(command.steps?.count, 4) + XCTAssertEqual(command.steps?.count, 3) XCTAssertEqual(command.steps?[0].kind, "tap") XCTAssertEqual(command.steps?[1].kind, "doubleTap") - XCTAssertEqual(command.steps?[3].x2, 10) - XCTAssertEqual(command.steps?[3].pauseMs, 50) + XCTAssertEqual(command.steps?[2].pauseMs, 50) } func testSequenceAcceptsDoubleTapKind() { @@ -404,21 +297,10 @@ extension RunnerTests { XCTAssertTrue(response.error?.message.contains("at most 20") ?? false) } - func testSequenceRejectsDragMissingSecondPoint() { - let response = executeSequenceForTest(steps: [ - sequenceStep(kind: "tap", x: 1, y: 2), - sequenceStep(kind: "drag", x: 3, y: 4), - ]) - XCTAssertEqual(response.ok, false) - XCTAssertEqual(response.error?.code, "INVALID_ARGS") - XCTAssertTrue(response.error?.message.contains("step 1") ?? false) - } - func testSequenceHasSynthesizedCoordinateStep() { XCTAssertTrue( sequenceHasSynthesizedCoordinateStep([ sequenceStep(kind: "tap", x: 1, y: 2, synthesized: true), - sequenceStep(kind: "drag", x: 1, y: 2, x2: 3, y2: 4, synthesized: true), ]) ) XCTAssertFalse( @@ -455,7 +337,7 @@ extension RunnerTests { func testAssembleSequenceStopsAtFirstFailure() { let steps = [ sequenceStep(kind: "tap", x: 1, y: 1), - sequenceStep(kind: "drag", x: 2, y: 2), + sequenceStep(kind: "longPress", x: 2, y: 2), sequenceStep(kind: "tap", x: 3, y: 3), ] var calls: [Int] = [] @@ -463,7 +345,7 @@ extension RunnerTests { calls.append(index) if index == 1 { return SequenceStepOutcome( - outcome: .unsupported(message: "drag unsupported", hint: nil), + outcome: .unsupported(message: "long press unsupported", hint: nil), gestureStartUptimeMs: 10, gestureEndUptimeMs: 15 ) @@ -478,7 +360,7 @@ extension RunnerTests { XCTAssertEqual(execution.results.count, 2) XCTAssertEqual(execution.results[1].ok, false) XCTAssertEqual(execution.results[1].errorCode, "UNSUPPORTED_OPERATION") - XCTAssertEqual(execution.results[1].errorMessage, "drag unsupported") + XCTAssertEqual(execution.results[1].errorMessage, "long press unsupported") } func testSequenceWorstCaseResponseStaysUnderJournalCap() throws { @@ -486,7 +368,7 @@ extension RunnerTests { let results = (0..<20).map { index in SequenceStepResult( ok: index < 19, - kind: "drag", + kind: "longPress", errorCode: index < 19 ? nil : "UNSUPPORTED_OPERATION", errorMessage: index < 19 ? nil : longMessage, gestureStartUptimeMs: 123456.789, @@ -510,21 +392,15 @@ extension RunnerTests { kind: String, x: Double?, y: Double? = nil, - x2: Double? = nil, - y2: Double? = nil, - synthesized: Bool? = nil, - dragSemantics: SynthesizedDragSemantics? = nil + synthesized: Bool? = nil ) -> SequenceStep { SequenceStep( kind: kind, x: x, y: y, - x2: x2, - y2: y2, durationMs: nil, pauseMs: nil, - synthesized: synthesized, - dragSemantics: dragSemantics + synthesized: synthesized ) } diff --git a/apple-runner/AgentDeviceRunner/AgentDeviceRunnerUITests/RunnerTests+SnapshotCapturePlan.swift b/apple-runner/AgentDeviceRunner/AgentDeviceRunnerUITests/RunnerTests+SnapshotCapturePlan.swift index a217496e8..d0d168441 100644 --- a/apple-runner/AgentDeviceRunner/AgentDeviceRunnerUITests/RunnerTests+SnapshotCapturePlan.swift +++ b/apple-runner/AgentDeviceRunner/AgentDeviceRunnerUITests/RunnerTests+SnapshotCapturePlan.swift @@ -118,6 +118,17 @@ extension RunnerTests { ) } + func clearSnapshotXCTestChannelPenalty(reason: String) { + snapshotXCTestChannelPenaltyLock.lock() + let hadActivePenalty = Date() < snapshotXCTestChannelPenaltyUntil + snapshotXCTestChannelPenaltyBundleId = nil + snapshotXCTestChannelPenaltyUntil = Date.distantPast + snapshotXCTestChannelPenaltyLock.unlock() + if hadActivePenalty { + NSLog("AGENT_DEVICE_RUNNER_SNAPSHOT_XCTEST_CHANNEL_PENALTY_CLEARED reason=%@", reason) + } + } + func isSnapshotXCTestChannelPenalized(bundleId: String?) -> Bool { snapshotXCTestChannelPenaltyLock.lock() defer { snapshotXCTestChannelPenaltyLock.unlock() } @@ -127,6 +138,12 @@ extension RunnerTests { return penalized == bundleId } + func consumeSnapshotXCTestPenaltyWarmupExemption() -> Bool { + let pending = snapshotXCTestPenaltyWarmupExemptionPending + snapshotXCTestPenaltyWarmupExemptionPending = false + return pending + } + /// Pure plan-reorder rule: a penalized XCTest accessibility channel uses independent backends /// when the platform has one, otherwise it keeps XCTest work on a short probe. The raw /// diagnostic plan keeps tree-first errors, and unknown plans are left untouched. @@ -174,6 +191,7 @@ extension RunnerTests { var firstFailure: (reason: String, code: String)? var axFailure: SnapshotCaptureFailure? let deadline = Date().addingTimeInterval(Self.snapshotPlanBudget) + let suppressXCTestPenalty = consumeSnapshotXCTestPenaltyWarmupExemption() // Reorder is iOS-only because hostile screens can make XCTest tree/query work grind while // the app remains visually responsive. Simulators can avoid that channel through private AX; @@ -237,14 +255,30 @@ extension RunnerTests { treeCaptureSliceBudgetOverride: effective.treeCaptureSliceBudgetOverride ) else { - recordSlowXCTestSnapshotBackendIfNeeded(kind, startedAt: backendStartedAt) + recordSlowXCTestSnapshotBackendIfNeeded( + kind, + startedAt: backendStartedAt, + penaltySuppressed: suppressXCTestPenalty + ) continue } capture = result - recordSlowXCTestSnapshotBackendIfNeeded(kind, startedAt: backendStartedAt) + recordSlowXCTestSnapshotBackendIfNeeded( + kind, + startedAt: backendStartedAt, + penaltySuppressed: suppressXCTestPenalty + ) } catch let failure as SnapshotCaptureFailure { - recordXCTestSnapshotBackendFailureIfNeeded(kind, failure: failure) - recordSlowXCTestSnapshotBackendIfNeeded(kind, startedAt: backendStartedAt) + recordXCTestSnapshotBackendFailureIfNeeded( + kind, + failure: failure, + penaltySuppressed: suppressXCTestPenalty + ) + recordSlowXCTestSnapshotBackendIfNeeded( + kind, + startedAt: backendStartedAt, + penaltySuppressed: suppressXCTestPenalty + ) if Self.isAxSnapshotFailure(failure) { axFailure = failure } if firstFailure == nil { firstFailure = (failure.message, Self.isAxSnapshotFailure(failure) ? "ax-rejected" : "capture-failed") @@ -313,7 +347,12 @@ extension RunnerTests { /// Marks XCTest-backed snapshot tiers as penalized when one attempt ground past the slow-capture /// threshold — even a successful one: the next capture of this screen must not re-grind. - private func recordSlowXCTestSnapshotBackendIfNeeded(_ kind: SnapshotBackendKind, startedAt: Date) { + private func recordSlowXCTestSnapshotBackendIfNeeded( + _ kind: SnapshotBackendKind, + startedAt: Date, + penaltySuppressed: Bool + ) { + guard !penaltySuppressed else { return } guard kind.usesXCTestAccessibilityChannel else { return } let elapsed = Date().timeIntervalSince(startedAt) guard elapsed > snapshotXCTestSlowCaptureThreshold else { return } @@ -325,8 +364,10 @@ extension RunnerTests { private func recordXCTestSnapshotBackendFailureIfNeeded( _ kind: SnapshotBackendKind, - failure: SnapshotCaptureFailure + failure: SnapshotCaptureFailure, + penaltySuppressed: Bool ) { + guard !penaltySuppressed else { return } guard kind.usesXCTestAccessibilityChannel, failure.code == Self.xCTestSnapshotTimeoutCode else { return } penalizeSnapshotXCTestChannel( bundleId: currentBundleId, diff --git a/apple-runner/AgentDeviceRunner/AgentDeviceRunnerUITests/RunnerTests+SynthesizedGesturePolicy.swift b/apple-runner/AgentDeviceRunner/AgentDeviceRunnerUITests/RunnerTests+SynthesizedGesturePolicy.swift index 878e693cc..0978b7252 100644 --- a/apple-runner/AgentDeviceRunner/AgentDeviceRunnerUITests/RunnerTests+SynthesizedGesturePolicy.swift +++ b/apple-runner/AgentDeviceRunner/AgentDeviceRunnerUITests/RunnerTests+SynthesizedGesturePolicy.swift @@ -103,7 +103,7 @@ func synthesizedGesturePolicy(_ kind: SynthesizedGesturePolicyKind) -> Synthesiz func sequenceHasSynthesizedCoordinateStep(_ steps: [SequenceStep]) -> Bool { steps.contains { step in - step.synthesized == true && (step.kind == "tap" || step.kind == "drag") + step.synthesized == true && step.kind == "tap" } } diff --git a/apple-runner/AgentDeviceRunner/AgentDeviceRunnerUITests/RunnerTests.swift b/apple-runner/AgentDeviceRunner/AgentDeviceRunnerUITests/RunnerTests.swift index 768683eb2..d1f6752ac 100644 --- a/apple-runner/AgentDeviceRunner/AgentDeviceRunnerUITests/RunnerTests.swift +++ b/apple-runner/AgentDeviceRunner/AgentDeviceRunnerUITests/RunnerTests.swift @@ -38,6 +38,7 @@ final class RunnerTests: XCTestCase { lazy var springboard = XCUIApplication(bundleIdentifier: Self.springboardBundleId) var currentApp: XCUIApplication? var currentBundleId: String? + var currentAppProcessIdentifier: Int? let maxRequestBytes = 2 * 1024 * 1024 let mainThreadExecutionTimeout: TimeInterval = 30 let appExistenceTimeout: TimeInterval = 30 @@ -76,6 +77,7 @@ final class RunnerTests: XCTestCase { var snapshotXCTestChannelPenaltyBundleId: String? var snapshotXCTestChannelPenaltyUntil = Date.distantPast let snapshotXCTestChannelPenaltyDuration: TimeInterval = 120 + var snapshotXCTestPenaltyWarmupExemptionPending = false // Bluesky-class screens can grind ~4-8s before an XCTest-backed snapshot tier fails; anything // past this threshold marks the screen hostile so the next capture uses non-XCTest recovery. let snapshotXCTestSlowCaptureThreshold: TimeInterval = 3 diff --git a/docs/adr/0008-command-descriptor-registry.md b/docs/adr/0008-command-descriptor-registry.md index 793150ae9..4564f4b97 100644 --- a/docs/adr/0008-command-descriptor-registry.md +++ b/docs/adr/0008-command-descriptor-registry.md @@ -77,7 +77,7 @@ As of 2026-07, the descriptor registry is live for command identity (`PUBLIC_COM `INTERNAL_COMMANDS`, and local CLI names), daemon registry traits, capability matrix, structured-batch allowlist, daemon-client timeout policy, MCP exposure list, capability-checked CLI command list, post-action observation traits, and the platform dispatch -command set (including dispatch-only aliases such as `read` and `swipe-preset`). Command-family +command set (including the remaining dispatch-only alias `read`). Command-family surface metadata still lives under `src/commands/**`, where the CLI grammar and client-backed executors already live, but it is coherence-guarded against the descriptor CLI catalog so new surface names cannot drift from the descriptor root. diff --git a/docs/adr/0011-interaction-guarantee-contract.md b/docs/adr/0011-interaction-guarantee-contract.md index 1e649795d..80c3fbed8 100644 --- a/docs/adr/0011-interaction-guarantee-contract.md +++ b/docs/adr/0011-interaction-guarantee-contract.md @@ -266,6 +266,11 @@ dimensions as their only frame source; cheaper frame sources such as intentionally excluded because they can diverge from full-screen screenshot coordinates on affected simulators. +Two-contact pan/pinch/rotate/transform planning is now owned by the typed +gesture-plan contract in [ADR 0013](0013-unified-gesture-plans.md). It remains a +sibling of this element-targeting matrix: coordinate gestures do not acquire +selector/ref guarantees merely because their native executor uses XCTest. + ## Migration plan Each step lands green and independently useful: diff --git a/docs/adr/0013-unified-gesture-plans.md b/docs/adr/0013-unified-gesture-plans.md new file mode 100644 index 000000000..ee35b36fa --- /dev/null +++ b/docs/adr/0013-unified-gesture-plans.md @@ -0,0 +1,109 @@ +# ADR 0013: Unified Gesture Plans + +## Status + +Accepted + +## Context + +Gesture intent was previously interpreted at several private boundaries: command aliases produced +positional strings, daemon dispatch reparsed them, the `Interactor` exposed parallel semantic +methods, and each platform derived its own two-contact geometry. That duplicated validation, +responses, and behavior behind the three public surfaces: CLI, Node.js, and MCP. + +Two-contact geometry also differed by platform. Android used a fixed radius while Apple derived a +radius from the app frame. Neither path validated every planned point against the active +interaction viewport before injection. + +`scroll` remains separate because it owns viewport/edge traversal, content-state verification, and +runner fallback policy rather than a single physical gesture. + +## Decision + +Public gesture inputs normalize once in `src/contracts/gesture-normalization.ts`. This is the +explicit compatibility boundary: convenience APIs and deprecated arguments become canonical +semantic intent before entering the runtime. The private daemon wire is free to evolve with that +model; compatibility is owed at CLI, Node.js, and MCP. + +The runtime plans canonical intent in `src/contracts/gesture-plan.ts`. Contact topology is separate +from motion: + +- one contact: pan or fling with a complete pointer trajectory; +- two contacts: pan, pinch, rotate, or transform with two complete, synchronized trajectories. + +`swipe` is public sugar for a fixed-duration fling. Its historical optional duration remains a +thin compatibility alias to pan and reports a deprecation. The same rule applies to the historical +fling duration. Pinch fixes translation and rotation at zero; rotate fixes translation at zero and +scale at one; two-finger pan fixes scale at one and rotation at zero; transform can apply all three +components atomically. Intent remains on the plan even when aliases share an executor. + +The planner owns deterministic multi-touch geometry. Contacts start at -90 degrees, except Android +pinch starts horizontally because a vertical pinch is captured by common vertical app scroll +containers before the pinch recognizer activates. The same explicit planning profile preserves the +proven frame-count convention: Android rounds while Apple truncates the duration/16 ms frame count. +These are planner inputs, not adapter-generated trajectories. The larger of pinch's initial and final spans is 40% of the +viewport's shorter side, preserving the proven Apple pinch geometry; other two-contact intents use +25% to keep translation and rotation trajectories compact. The other span follows from the requested scale, and both must +satisfy a 48-point reliability floor. Combined transforms progress translation, scale, and rotation +together inside one uninterrupted two-contact sequence so recognizers observe every intent without an +adapter regenerating geometry. The planner does not clamp points, cache the viewport, or distort +the requested components. Every injected sample must fit the freshly resolved active-app +interaction viewport; otherwise the request fails before injection with +`GESTURE_TRAJECTORY_OUT_OF_BOUNDS` and actionable details. Span and angle remain internal because +no established automation use case justifies a public tuning surface. + +Platform adapters consume the canonical plan: + +- Android sends the plan to provider-native touch injection when available, otherwise to the + bundled instrumentation helper. The helper injects the exact planned pointer samples. A + two-contact plan never falls back to `adb input swipe`; issue #690 separately owns removal of the + existing one-contact fallback. The snapshot helper is stopped before local gesture + instrumentation because Android permits only one instrumentation owner of `UiAutomation`. +- iOS converts every planned point to native orientation and feeds the exact arrays to the existing + private XCTest event bridge. macOS lowers a one-contact plan to its drag executor and tvOS lowers + it to remote direction. Core admission and the Apple adapter both consume the same shared + multi-touch support policy; multi-touch remains capability-gated to iOS simulators. +- WebDriver lowers a supported plan to synchronized W3C pointer action sources. Multi-touch remains + capability-gated until a provider proves it. + +The `Interactor` and backend expose one compositional `performGesture(plan)` primitive instead of a +method per semantic alias. The old scalar Apple and Android multi-touch executors and the +public-command alias-to-positionals-to-reparse route are deleted. `.ad` keeps its established +positional syntax through one named replay compatibility codec; CLI, Node.js, and MCP send +structured input. Providers should compose transport/device bindings with the shared platform +adapter rather than reimplement the interaction runtime. + +Repeated coordinate swipes are bounded at the public command contract and daemon trust boundary. +Individual count and pause limits prevent pathological fields, while the combined planned gesture +and pause schedule must fit within 60 seconds so valid fields cannot compose into an unbounded +session lock. + +Public two-finger pan is additive: `pointerCount?: 1 | 2` on pan and CLI +`--pointer-count 2`; omission remains one contact. Responses share the canonical +`kind`, `durationMs`, `pointerCount`, `from`, and `to` fields, followed by backend evidence. +Recording/replay keeps its existing public command identity and session semantics. + +ADR 0011's element dispatch-path matrix remains unchanged: coordinate gestures do not resolve +selectors or refs and therefore cannot claim element-targeting guarantees. + +## Consequences + +- CLI, Node.js, MCP, runtime, and platform adapters share one normalization and planning model. +- Adding an ergonomic gesture alias does not add a platform implementation. +- One-finger pan remains the default and explicit two-finger pan retains pan intent. +- The active viewport is resolved for each gesture, so rotation, keyboard, and window changes do + not use stale geometry. +- Pointer plans are larger than scalar requests but bounded by duration and the 16 ms sample + cadence; deleting duplicate scalar executors offsets the package cost. + +## Alternatives Considered + +- Keep positional aliases and share only geometry math: rejected because validation, response, and + routing would still have two implementations. +- Make platform gesture APIs the source of truth: rejected because their timing, geometry, and + recognizer behavior differ and cannot provide cross-platform semantics. +- Make swipe a public synonym for pan: rejected because battle-tested gesture vocabulary treats a + fling/swipe as a quick directional throw and pan as deliberate timed translation. +- Add `two-finger-pan`: rejected because pointer count is topology, not a new motion intent. +- Expose span/angle controls: rejected until a concrete automation use case needs them. +- Consolidate scroll: rejected because edge/content verification and fallback policy are distinct. diff --git a/docs/adr/README.md b/docs/adr/README.md index a15149ffd..79caf7955 100644 --- a/docs/adr/README.md +++ b/docs/adr/README.md @@ -14,6 +14,7 @@ | [0010 Error system conventions](0010-error-system.md) | error codes, hints, normalizeError, typed error signals | | [0011 Interaction Guarantee Contract](0011-interaction-guarantee-contract.md) | interaction dispatch paths, fast paths, guards, the guarantee matrix, parity tables | | [0012 Interactive Replay](0012-interactive-replay.md) | replay healing/`--update`, diagnostic resolution disclosure, bounded `.ad` target-binding evidence, bounded divergence wire/error handling, plan-bound replay-only `--from` semantics, and agent-supervised re-record repair ("heal-by-doing") | +| [0013 Unified Gesture Plans](0013-unified-gesture-plans.md) | gesture API/routing, contact topology, multi-touch geometry, native pointer injection, two-finger pan | ADRs record *why*; the registries and gates they describe are the living source of truth — when prose and a registry disagree, the registry wins and the ADR needs a follow-up. diff --git a/examples/test-app/README.md b/examples/test-app/README.md index 583f879e6..57e787619 100644 --- a/examples/test-app/README.md +++ b/examples/test-app/README.md @@ -178,11 +178,16 @@ pnpm test-app:replay:android These run the `.ad` replay suite in `examples/test-app/replays`. -`gesture-lab.ad` verifies `gesture pan`, `gesture fling`, `gesture pinch`, and -`gesture rotate` against the gesture metrics rendered by the Home screen on iOS -and Android. Android and iOS simulator sessions also support `gesture transform` -for a combined pan/zoom/rotate gesture. On Android, treat combined transform -assertions as qualitative because recognizers can report non-exact centroid, +The iOS `gesture-lab.ad` and Android `gesture-lab-android.ad` replays verify +`gesture pan`, `gesture fling`, `gesture pinch`, and `gesture rotate` against the +gesture metrics rendered by the Home screen. They also prove that the default pan +does not activate an exactly-two-pointer recognizer, while +`gesture pan ... --pointer-count 2` does without changing pinch or rotation state. + +Each gesture replay relaunches the app before its combined `gesture transform` +canary, verifies the clean pan/pinch/rotate state, then checks that one atomic +two-pointer gesture changes all three semantic states. On Android, these checks +are intentionally qualitative because recognizers can report non-exact centroid, scale, and rotation values for one simultaneous two-finger gesture. To target a specific iOS simulator or an installed Expo development build, run the diff --git a/examples/test-app/app.config.js b/examples/test-app/app.config.js index 1113b78e7..b6c2e4111 100644 --- a/examples/test-app/app.config.js +++ b/examples/test-app/app.config.js @@ -5,6 +5,7 @@ module.exports = { expo: { name: 'Agent Device Tester', slug: 'agent-device-test-app', + scheme: 'agent-device-test-app', version: '1.0.0', orientation: 'default', userInterfaceStyle: 'automatic', diff --git a/examples/test-app/package.json b/examples/test-app/package.json index f430675d0..74db8a433 100644 --- a/examples/test-app/package.json +++ b/examples/test-app/package.json @@ -17,6 +17,7 @@ "expo-constants": "56.0.18", "expo-dev-client": "~56.0.20", "expo-linking": "56.0.14", + "expo-modules-core": "56.0.17", "expo-router": "~56.2.11", "expo-status-bar": "~56.0.4", "react": "19.2.3", diff --git a/examples/test-app/pnpm-lock.yaml b/examples/test-app/pnpm-lock.yaml index 62c7e868f..77b0b2d11 100644 --- a/examples/test-app/pnpm-lock.yaml +++ b/examples/test-app/pnpm-lock.yaml @@ -37,6 +37,9 @@ importers: expo-linking: specifier: 56.0.14 version: 56.0.14(expo@56.0.12)(react-native@0.85.3(@babel/core@7.29.0)(@react-native/metro-config@0.86.0(@babel/core@7.29.0))(@types/react@19.2.14)(react@19.2.3))(react@19.2.3) + expo-modules-core: + specifier: 56.0.17 + version: 56.0.17(react-native-worklets@0.10.0(@babel/core@7.29.0)(@react-native/metro-config@0.86.0(@babel/core@7.29.0))(react-native@0.85.3(@babel/core@7.29.0)(@react-native/metro-config@0.86.0(@babel/core@7.29.0))(@types/react@19.2.14)(react@19.2.3))(react@19.2.3))(react-native@0.85.3(@babel/core@7.29.0)(@react-native/metro-config@0.86.0(@babel/core@7.29.0))(@types/react@19.2.14)(react@19.2.3))(react@19.2.3) expo-router: specifier: ~56.2.11 version: 56.2.11(4e54386c19a86f18d3fced032d4d30ae) diff --git a/examples/test-app/replays/gesture-lab-android.ad b/examples/test-app/replays/gesture-lab-android.ad new file mode 100644 index 000000000..c3000c853 --- /dev/null +++ b/examples/test-app/replays/gesture-lab-android.ad @@ -0,0 +1,48 @@ +context platform=android kind=emulator timeout=60000 + +env APP_TARGET="com.callstack.agentdevicelab" +env APP_URL="" + +open "${APP_TARGET}" --relaunch --launch-url "${APP_URL}" +react-native dismiss-overlay +wait "Gesture lab" 30000 +wait "gesture canary ready" 5000 +wait "two-pointer pan activations 0" 5000 + +gesture pan 350 1100 180 0 500 +wait "two-pointer pan activations 0" 5000 + +gesture pan 350 1100 180 0 500 --pointer-count 2 +wait "two-pointer pan activations 1" 5000 + +gesture fling left 850 1100 300 +wait "fling 1" 5000 + +gesture fling right 750 1100 300 +wait "fling 2" 5000 + +open "${APP_TARGET}" --relaunch --launch-url "${APP_URL}" +react-native dismiss-overlay +wait "gesture canary ready" 5000 +wait "pan changed no, pinch changed no, rotate changed no" 30000 + +gesture pinch 1.5 750 1100 +wait "pan changed no, pinch changed yes, rotate changed no" 5000 + +open "${APP_TARGET}" --relaunch --launch-url "${APP_URL}" +react-native dismiss-overlay +wait "gesture canary ready" 5000 +wait "pan changed no, pinch changed no, rotate changed no" 30000 + +gesture rotate 35 750 1100 +wait "pan changed no, pinch changed no, rotate changed yes" 5000 + +open "${APP_TARGET}" --relaunch --launch-url "${APP_URL}" +react-native dismiss-overlay +wait "gesture canary ready" 5000 +wait "pan changed no, pinch changed no, rotate changed no" 30000 + +gesture transform 750 1100 60 0 1.75 30 600 +wait "pan changed yes, pinch changed yes, rotate changed yes" 5000 + +close diff --git a/examples/test-app/replays/gesture-lab.ad b/examples/test-app/replays/gesture-lab.ad index 2d6b3aee5..ac3946e33 100644 --- a/examples/test-app/replays/gesture-lab.ad +++ b/examples/test-app/replays/gesture-lab.ad @@ -1,40 +1,47 @@ -context platform=ios timeout=60000 +context platform=ios kind=simulator timeout=180000 env APP_TARGET="Agent Device Tester" env APP_URL="" open "${APP_TARGET}" --relaunch --launch-url "${APP_URL}" -react-native dismiss-overlay wait "Gesture lab" 30000 +wait "gesture canary ready" 5000 +wait "two-pointer pan activations 0" 5000 -gesture fling left 195 443 180 -wait "fling 1" 5000 +gesture pan 110 443 48 0 500 +wait "two-pointer pan activations 0" 5000 -gesture fling right 195 443 180 -wait "fling 2" 5000 +gesture pan 110 443 48 0 500 --pointer-count 2 +wait "two-pointer pan activations 1" 5000 -gesture fling up 195 443 80 80 -wait "fling 3" 5000 +gesture fling left 280 443 120 +wait "fling 1" 5000 -gesture fling down 195 443 80 80 -wait "fling 4" 5000 +gesture fling right 280 443 120 +wait "fling 2" 5000 -gesture pan 195 443 -80 0 -wait "x -" 5000 +open "${APP_TARGET}" --relaunch --launch-url "${APP_URL}" +wait "Gesture lab" 30000 +wait "gesture canary ready" 5000 +wait "pan changed no, pinch changed no, rotate changed no" 30000 -gesture pan 195 443 160 0 -wait "x 72" 5000 +gesture pinch 1.5 280 443 +wait "pan changed no, pinch changed yes, rotate changed no" 5000 -gesture pan 195 443 0 -80 -wait "y -" 5000 +open "${APP_TARGET}" --relaunch --launch-url "${APP_URL}" +wait "Gesture lab" 30000 +wait "gesture canary ready" 5000 +wait "pan changed no, pinch changed no, rotate changed no" 30000 -gesture pan 195 443 0 160 -wait "y 56" 5000 +gesture rotate 35 280 443 +wait "pan changed no, pinch changed no, rotate changed yes" 5000 -gesture pinch 1.25 195 443 -wait "pinch changed yes" 5000 +open "${APP_TARGET}" --relaunch --launch-url "${APP_URL}" +wait "Gesture lab" 30000 +wait "gesture canary ready" 5000 +wait "pan changed no, pinch changed no, rotate changed no" 30000 -gesture rotate 35 195 443 -wait "rotate changed yes" 5000 +gesture transform 280 443 24 -24 1.35 30 300 +wait "pan changed yes, pinch changed yes, rotate changed yes" 5000 close diff --git a/examples/test-app/src/screens/GestureLab.tsx b/examples/test-app/src/screens/GestureLab.tsx index 64ba952a3..7554616fe 100644 --- a/examples/test-app/src/screens/GestureLab.tsx +++ b/examples/test-app/src/screens/GestureLab.tsx @@ -1,8 +1,10 @@ -import { useRef, useState } from 'react'; -import { Image, StyleSheet, Text, View } from 'react-native'; +import { useEffect, useRef, useState } from 'react'; +import { Image, Platform, Text, View, type GestureResponderEvent } from 'react-native'; import { Directions, FlingGestureHandler, + Gesture, + GestureDetector, PanGestureHandler, PinchGestureHandler, RotationGestureHandler, @@ -17,7 +19,9 @@ import { } from 'react-native-gesture-handler'; import { SectionCard } from '../components'; -import { useAppColors, type AppColors } from '../theme'; +import { useAppColors } from '../theme'; +import { createGestureLabStyles } from './gesture-lab-styles'; +import { disableDevelopmentGestureInterceptors } from './gesture-lab-dev-menu'; const gestureImageUri = 'https://reactnative.dev/img/logo-share.png'; @@ -30,6 +34,14 @@ type TransformState = { type GestureCounts = { fling: number; + twoPointerPan: number; +}; + +type AndroidTouchStart = TransformState & { + angle: number; + centroidX: number; + centroidY: number; + span: number; }; const initialTransform: TransformState = { @@ -41,27 +53,35 @@ const initialTransform: TransformState = { const initialCounts: GestureCounts = { fling: 0, + twoPointerPan: 0, }; const minScale = 0.5; const maxScale = 2; export function GestureLab() { const colors = useAppColors(); - const styles = createStyles(colors); + const styles = createGestureLabStyles(colors); + const [canaryReady, setCanaryReady] = useState(false); const [transform, setTransform] = useState(initialTransform); const [counts, setCounts] = useState(initialCounts); const transformRef = useRef(initialTransform); const gestureStartRef = useRef(initialTransform); - const flingDownRef = useRef(null); - const flingLeftRef = useRef(null); - const flingRightRef = useRef(null); - const flingUpRef = useRef(null); - const panRef = useRef(null); - const pinchRef = useRef(null); - const rotationRef = useRef(null); - const flingRefs = [flingLeftRef, flingRightRef, flingUpRef, flingDownRef]; + const androidTouchStartRef = useRef(undefined); + const legacyFlingDownRef = useRef(null); + const legacyFlingLeftRef = useRef(null); + const legacyFlingRightRef = useRef(null); + const legacyFlingUpRef = useRef(null); + const legacyPanRef = useRef(null); + const legacyPinchRef = useRef(null); + const legacyRotationRef = useRef(null); const activeTransformHandlerTagsRef = useRef(new Set()); + useEffect(() => { + void disableDevelopmentGestureInterceptors() + .catch(() => undefined) + .finally(() => setCanaryReady(true)); + }, []); + function updateTransform(nextTransform: TransformState) { transformRef.current = nextTransform; setTransform(nextTransform); @@ -79,32 +99,43 @@ export function GestureLab() { activeTransformHandlerTagsRef.current.delete(handlerTag); } - function handlePan(event: PanGestureHandlerGestureEvent) { + function handlePan(translationX: number, translationY: number) { const start = gestureStartRef.current; updateTransform({ ...transformRef.current, - offsetX: clamp(start.offsetX + event.nativeEvent.translationX, -72, 72), - offsetY: clamp(start.offsetY + event.nativeEvent.translationY, -56, 56), + offsetX: clamp(start.offsetX + translationX, -72, 72), + offsetY: clamp(start.offsetY + translationY, -56, 56), }); } - function handlePinch(event: PinchGestureHandlerGestureEvent) { + function handlePinch(scale: number) { const start = gestureStartRef.current; updateTransform({ ...transformRef.current, - scale: clamp(start.scale * event.nativeEvent.scale, minScale, maxScale), + scale: clamp(start.scale * scale, minScale, maxScale), }); } - function handleRotation(event: RotationGestureHandlerGestureEvent) { + function handleRotation(rotation: number) { const start = gestureStartRef.current; updateTransform({ ...transformRef.current, - rotation: start.rotation + event.nativeEvent.rotation, + rotation: start.rotation + rotation, }); } - function handleTransformStateChange( + function handleFling() { + setCounts((current) => ({ ...current, fling: current.fling + 1 })); + } + + function handleTwoPointerPan() { + setCounts((current) => ({ + ...current, + twoPointerPan: current.twoPointerPan + 1, + })); + } + + function handleLegacyTransformStateChange( event: | PanGestureHandlerStateChangeEvent | PinchGestureHandlerStateChangeEvent @@ -112,10 +143,7 @@ export function GestureLab() { ) { if (event.nativeEvent.state === State.BEGAN) { beginTransformGesture(event.nativeEvent.handlerTag); - return; - } - - if ( + } else if ( event.nativeEvent.state === State.END || event.nativeEvent.state === State.FAILED || event.nativeEvent.state === State.CANCELLED @@ -124,17 +152,124 @@ export function GestureLab() { } } - function handleFling(event: FlingGestureHandlerStateChangeEvent) { - if (event.nativeEvent.state === State.ACTIVE) { - setCounts((current) => ({ ...current, fling: current.fling + 1 })); - } + function handleLegacyFling(event: FlingGestureHandlerStateChangeEvent) { + if (event.nativeEvent.state === State.ACTIVE) handleFling(); + } + + function handleAndroidTouchStart(event: GestureResponderEvent) { + const [first, second] = event.nativeEvent.touches; + if (!first || !second) return; + androidTouchStartRef.current = { + ...transformRef.current, + angle: Math.atan2(second.pageY - first.pageY, second.pageX - first.pageX), + centroidX: (first.pageX + second.pageX) / 2, + centroidY: (first.pageY + second.pageY) / 2, + span: Math.hypot(second.pageX - first.pageX, second.pageY - first.pageY), + }; + } + + function handleAndroidTouchMove(event: GestureResponderEvent) { + const start = androidTouchStartRef.current; + const [first, second] = event.nativeEvent.touches; + if (!start || !first || !second) return; + const centroidX = (first.pageX + second.pageX) / 2; + const centroidY = (first.pageY + second.pageY) / 2; + const span = Math.hypot(second.pageX - first.pageX, second.pageY - first.pageY); + const angle = Math.atan2(second.pageY - first.pageY, second.pageX - first.pageX); + updateTransform({ + offsetX: clamp(start.offsetX + centroidX - start.centroidX, -72, 72), + offsetY: clamp(start.offsetY + centroidY - start.centroidY, -56, 56), + rotation: start.rotation + normalizedRadians(angle - start.angle), + scale: clamp(start.scale * (span / start.span), minScale, maxScale), + }); } + const twoPointerPanGesture = Gesture.Pan() + .averageTouches(true) + .minPointers(2) + .maxPointers(2) + .minDistance(4) + .runOnJS(true) + .onStart(handleTwoPointerPan); + const legacyFlingRefs = [ + legacyFlingLeftRef, + legacyFlingRightRef, + legacyFlingUpRef, + legacyFlingDownRef, + ]; + + const androidTransformTarget = ( + + + + + + handleRotation(event.nativeEvent.rotation) + } + onHandlerStateChange={handleLegacyTransformStateChange} + ref={legacyRotationRef} + simultaneousHandlers={[legacyPanRef, legacyPinchRef, ...legacyFlingRefs]} + > + + handlePinch(event.nativeEvent.scale) + } + onHandlerStateChange={handleLegacyTransformStateChange} + ref={legacyPinchRef} + simultaneousHandlers={[legacyPanRef, legacyRotationRef, ...legacyFlingRefs]} + > + + handlePan(event.nativeEvent.translationX, event.nativeEvent.translationY) + } + onHandlerStateChange={handleLegacyTransformStateChange} + ref={legacyPanRef} + simultaneousHandlers={[legacyPinchRef, legacyRotationRef, ...legacyFlingRefs]} + > + + + + + + + + + ); + const rotationDegrees = Math.round((transform.rotation * 180) / Math.PI); const statusLabel = `x ${Math.round(transform.offsetX)}, y ${Math.round( transform.offsetY, )}, scale ${transform.scale.toFixed(2)}, rotate ${rotationDegrees}`; - const panChanged = Math.abs(transform.offsetX) > 0 || Math.abs(transform.offsetY) > 0; + const panChanged = Math.abs(transform.offsetX) > 1 || Math.abs(transform.offsetY) > 1; const pinchChanged = Math.abs(transform.scale - 1) > 0.01; const rotateChanged = rotationDegrees !== 0; const changeStatusLabel = `pan changed ${panChanged ? 'yes' : 'no'}, pinch changed ${ @@ -147,89 +282,57 @@ export function GestureLab() { title="Gesture lab" testID="gesture-lab-card" > - (androidTouchStartRef.current = undefined) : undefined + } + onTouchMove={Platform.OS === 'android' ? handleAndroidTouchMove : undefined} + onTouchStart={Platform.OS === 'android' ? handleAndroidTouchStart : undefined} + style={styles.target} + testID="gesture-target" > - - - - - - - - - - - - - - - - + + {androidTransformTarget} + + + + + + gesture canary {canaryReady ? 'ready' : 'preparing'} + {statusLabel} fling {counts.fling} + + two-pointer pan activations {counts.twoPointerPan} + {changeStatusLabel} @@ -242,31 +345,8 @@ function clamp(value: number, min: number, max: number): number { return Math.min(Math.max(value, min), max); } -function createStyles(colors: AppColors) { - return StyleSheet.create({ - target: { - alignItems: 'center', - backgroundColor: colors.cardStrong, - borderColor: colors.line, - borderRadius: 4, - borderWidth: StyleSheet.hairlineWidth, - height: 220, - justifyContent: 'center', - overflow: 'hidden', - }, - image: { - borderRadius: 4, - height: 160, - width: 240, - }, - metrics: { - gap: 6, - }, - metric: { - color: colors.textSoft, - fontSize: 13, - fontWeight: '600', - lineHeight: 18, - }, - }); +function normalizedRadians(value: number): number { + if (value > Math.PI) return value - Math.PI * 2; + if (value < -Math.PI) return value + Math.PI * 2; + return value; } diff --git a/examples/test-app/src/screens/gesture-lab-dev-menu.ts b/examples/test-app/src/screens/gesture-lab-dev-menu.ts new file mode 100644 index 000000000..4c50d6184 --- /dev/null +++ b/examples/test-app/src/screens/gesture-lab-dev-menu.ts @@ -0,0 +1,21 @@ +import { requireNativeModule } from 'expo-modules-core'; + +type DevMenuPreferencesModule = { + setPreferencesAsync(settings: { + motionGestureEnabled: boolean; + touchGestureEnabled: boolean; + }): Promise; +}; + +/** Keep development-shell shortcuts from intercepting the gesture canary's pointer streams. */ +export async function disableDevelopmentGestureInterceptors(): Promise { + if (!__DEV__) { + return; + } + + const preferences = requireNativeModule('DevMenuPreferences'); + await preferences.setPreferencesAsync({ + motionGestureEnabled: false, + touchGestureEnabled: false, + }); +} diff --git a/examples/test-app/src/screens/gesture-lab-styles.ts b/examples/test-app/src/screens/gesture-lab-styles.ts new file mode 100644 index 000000000..121a2649c --- /dev/null +++ b/examples/test-app/src/screens/gesture-lab-styles.ts @@ -0,0 +1,48 @@ +import { StyleSheet } from 'react-native'; + +import type { AppColors } from '../theme'; + +export function createGestureLabStyles(colors: AppColors) { + return StyleSheet.create({ + target: { + alignItems: 'center', + backgroundColor: colors.cardStrong, + borderColor: colors.line, + borderRadius: 4, + borderWidth: StyleSheet.hairlineWidth, + height: 220, + justifyContent: 'center', + overflow: 'hidden', + }, + image: { + borderRadius: 4, + height: 160, + width: 240, + }, + androidTransformTarget: { + bottom: 0, + left: 0, + position: 'absolute', + right: 0, + top: 0, + zIndex: 1, + }, + twoPointerTarget: { + bottom: 0, + left: 0, + position: 'absolute', + top: 0, + width: '50%', + zIndex: 1, + }, + metrics: { + gap: 6, + }, + metric: { + color: colors.textSoft, + fontSize: 13, + fontWeight: '600', + lineHeight: 18, + }, + }); +} diff --git a/scripts/build-android-multitouch-helper.sh b/scripts/build-android-multitouch-helper.sh index eafb24ae8..ca494ed9d 100644 --- a/scripts/build-android-multitouch-helper.sh +++ b/scripts/build-android-multitouch-helper.sh @@ -47,6 +47,7 @@ VERSION_CODE="$( BUILD_DIR="$HELPER_DIR/build" CLASSES_DIR="$BUILD_DIR/classes" +TEST_CLASSES_DIR="$BUILD_DIR/test-classes" DEX_DIR="$BUILD_DIR/dex" KEYSTORE="$PROJECT_DIR/android-snapshot-helper/debug.keystore" UNSIGNED_APK="$BUILD_DIR/helper-unsigned.apk" @@ -54,7 +55,7 @@ ALIGNED_APK="$BUILD_DIR/helper-aligned.apk" APK_PATH="$OUTPUT_DIR/$APK_BASENAME" rm -rf "$BUILD_DIR" -mkdir -p "$CLASSES_DIR" "$DEX_DIR" "$OUTPUT_DIR" +mkdir -p "$CLASSES_DIR" "$TEST_CLASSES_DIR" "$DEX_DIR" "$OUTPUT_DIR" javac \ --release 11 \ @@ -62,6 +63,16 @@ javac \ -d "$CLASSES_DIR" \ $(find "$HELPER_DIR/src/main/java" -name '*.java' | sort) +javac \ + --release 11 \ + -classpath "$ANDROID_JAR:$CLASSES_DIR" \ + -d "$TEST_CLASSES_DIR" \ + $(find "$HELPER_DIR/src/test/java" -name '*.java' | sort) + +java \ + -classpath "$ANDROID_JAR:$CLASSES_DIR:$TEST_CLASSES_DIR" \ + com.callstack.agentdevice.multitouchhelper.PointerEventScheduleTest + "$BUILD_TOOLS_DIR/d8" \ --min-api "$MIN_SDK" \ --classpath "$ANDROID_JAR" \ diff --git a/scripts/integration-progress-model.ts b/scripts/integration-progress-model.ts index 49d79cfd1..49a63731a 100644 --- a/scripts/integration-progress-model.ts +++ b/scripts/integration-progress-model.ts @@ -147,6 +147,7 @@ function summarizeProviderScenarioFlagCoverage(files) { ['retainPaths', 'retained install-source materialization'], ['retentionMs', 'install-source materialization TTL'], ['count', 'repeated press/click/swipe input'], + ['pointerCount', 'one- vs two-pointer pan gesture topology'], ['fps', 'recording frame-rate request'], ['quality', 'recording quality scaling'], ['hideTouches', 'recording without touch overlays'], diff --git a/src/__tests__/client.test.ts b/src/__tests__/client.test.ts index c65cb6062..4cea2261b 100644 --- a/src/__tests__/client.test.ts +++ b/src/__tests__/client.test.ts @@ -614,6 +614,47 @@ test('interactions.rotateGesture rejects partial centers on the client side', as assert.equal(setup.calls.length, 0); }); +test('interactions.pan projects one- and two-finger requests through typed gesture input', async () => { + const setup = createTransport(async () => ({ ok: true, data: { message: 'Panned' } })); + const client = createAgentDeviceClient(setup.config, { transport: setup.transport }); + + await client.interactions.pan({ x: 100, y: 200, dx: 40, dy: -20 }); + await client.interactions.pan({ + x: 100, + y: 200, + dx: 40, + dy: -20, + pointerCount: 2, + durationMs: 600, + }); + + assert.deepEqual( + setup.calls.map(({ command, positionals, input }) => ({ command, positionals, input })), + [ + { + command: 'gesture', + positionals: [], + input: { + kind: 'pan', + origin: { x: 100, y: 200 }, + delta: { x: 40, y: -20 }, + }, + }, + { + command: 'gesture', + positionals: [], + input: { + kind: 'pan', + origin: { x: 100, y: 200 }, + delta: { x: 40, y: -20 }, + pointerCount: 2, + durationMs: 600, + }, + }, + ], + ); +}); + // fallow-ignore-next-line complexity test('replay.run serializes client-collected AD_VAR shell env into daemon request', async () => { const previousAppId = process.env.AD_VAR_APP_ID; diff --git a/src/__tests__/runtime-public.test.ts b/src/__tests__/runtime-public.test.ts index 6aace159d..34a2b7a22 100644 --- a/src/__tests__/runtime-public.test.ts +++ b/src/__tests__/runtime-public.test.ts @@ -254,9 +254,8 @@ test('internal backend, commands, and io modules are usable', () => { assert.equal(typeof commands.interactions.typeText, 'function'); assert.equal(typeof commands.interactions.focus, 'function'); assert.equal(typeof commands.interactions.longPress, 'function'); - assert.equal(typeof commands.interactions.swipe, 'function'); assert.equal(typeof commands.interactions.scroll, 'function'); - assert.equal(typeof commands.interactions.pinch, 'function'); + assert.equal(typeof commands.interactions.gesture, 'function'); assert.equal(typeof commands.system.back, 'function'); assert.equal(typeof commands.system.home, 'function'); assert.equal(typeof commands.system.rotate, 'function'); diff --git a/src/agent-device-client.ts b/src/agent-device-client.ts index c5ed72a75..e7243209d 100644 --- a/src/agent-device-client.ts +++ b/src/agent-device-client.ts @@ -53,6 +53,12 @@ import type { MaterializationReleaseOptions, MetroPrepareOptions, MetroPrepareResult, + PanOptions, + FlingOptions, + SwipeGestureOptions, + PinchOptions, + RotateGestureOptions, + TransformGestureOptions, } from './client/client-types.ts'; import type { CommandResult } from './core/command-descriptor/command-result.ts'; import { @@ -65,6 +71,7 @@ import { readSnapshotDiagnosticsSummary } from './snapshot-diagnostics.ts'; import type { CommandFlags } from './core/dispatch-context.ts'; import type { AgentArtifactsResult } from './cloud-artifacts.ts'; import type { ProjectedNavigationCommandClient } from './commands/system/navigation-projection.ts'; +import { AppError } from './kernel/errors.ts'; type ProjectedSystemCommandClient = ProjectedNavigationCommandClient & Pick; @@ -86,12 +93,14 @@ export function createAgentDeviceClient( positionals: string[] = [], options: InternalRequestOptions = {}, metadataFlags?: Partial, + input?: Record, ): Promise> => { const merged = mergeClientOptions(config, options); const response = await transport({ session: resolveSessionName(merged.session), command, positionals, + ...(input ? { input } : {}), flags: buildRequestFlags(merged, metadataFlags), runtime: merged.runtime, meta: buildMeta(merged), @@ -118,6 +127,7 @@ export function createAgentDeviceClient( request.positionals, request.options, request.metadataFlags, + request.input, )) as T; }; @@ -352,16 +362,19 @@ export function createAgentDeviceClient( press: async (options) => await executeCommand('press', options), longPress: async (options) => await executeCommand('longpress', options), swipe: async (options) => await executeCommand('swipe', options), - pan: async (options) => await executeCommand('gesture-pan', options), - fling: async (options) => await executeCommand('gesture-fling', options), - swipeGesture: async (options) => await executeCommand('gesture-swipe', options), + pan: async (options) => await executeCommand('gesture', panGestureInput(options)), + fling: async (options) => await executeCommand('gesture', flingGestureInput(options)), + swipeGesture: async (options) => + await executeCommand('gesture', swipePresetGestureInput(options)), focus: async (options) => await executeCommand('focus', options), type: async (options) => await executeCommand('type', options), fill: async (options) => await executeCommand('fill', options), scroll: async (options) => await executeCommand('scroll', options), - pinch: async (options) => await executeCommand('gesture-pinch', options), - rotateGesture: async (options) => await executeCommand('gesture-rotate', options), - transformGesture: async (options) => await executeCommand('gesture-transform', options), + pinch: async (options) => await executeCommand('gesture', pinchGestureInput(options)), + rotateGesture: async (options) => + await executeCommand('gesture', rotateGestureInput(options)), + transformGesture: async (options) => + await executeCommand('gesture', transformGestureInput(options)), get: async (options) => await executeCommand('get', options), is: async (options) => await executeCommand('is', options), find: async (options) => await executeCommand('find', options), @@ -397,6 +410,65 @@ export function createAgentDeviceClient( }; } +function panGestureInput(options: PanOptions): InternalRequestOptions & Record { + const { x, y, dx, dy, ...common } = options; + return { ...common, kind: 'pan', origin: { x, y }, delta: { x: dx, y: dy } }; +} + +function flingGestureInput( + options: FlingOptions, +): InternalRequestOptions & Record { + const { x, y, ...common } = options; + return { ...common, kind: 'fling', origin: { x, y } }; +} + +function swipePresetGestureInput( + options: SwipeGestureOptions, +): InternalRequestOptions & Record { + return { ...options, kind: 'swipe' }; +} + +function pinchGestureInput( + options: PinchOptions, +): InternalRequestOptions & Record { + const { x, y, ...common } = options; + assertCompleteGestureCenter(x, y, 'pinch'); + return { + ...common, + kind: 'pinch', + ...(x === undefined && y === undefined ? {} : { origin: { x, y } }), + }; +} + +function rotateGestureInput( + options: RotateGestureOptions, +): InternalRequestOptions & Record { + const { x, y, ...common } = options; + assertCompleteGestureCenter(x, y, 'rotate'); + return { + ...common, + kind: 'rotate', + ...(x === undefined && y === undefined ? {} : { origin: { x, y } }), + }; +} + +function transformGestureInput( + options: TransformGestureOptions, +): InternalRequestOptions & Record { + const { x, y, dx, dy, ...common } = options; + return { ...common, kind: 'transform', origin: { x, y }, delta: { x: dx, y: dy } }; +} + +function assertCompleteGestureCenter( + x: number | undefined, + y: number | undefined, + gesture: 'pinch' | 'rotate', +): void { + if ((x === undefined) !== (y === undefined)) { + throw new AppError('INVALID_ARGS', `gesture ${gesture} center requires both x and y`); + } +} + function normalizeSnapshotResult( data: Record, session: string | undefined, diff --git a/src/backend.ts b/src/backend.ts index 6d9314fe3..e6f5e7743 100644 --- a/src/backend.ts +++ b/src/backend.ts @@ -1,7 +1,13 @@ import type { AlertAction, AlertInfo } from './alert-contract.ts'; import type { AppsFilter } from './contracts/app-inventory.ts'; import type { JsonObject } from './contracts/json.ts'; -import type { Point, SnapshotNode, SnapshotOptions, SnapshotState } from './kernel/snapshot.ts'; +import type { + Point, + Rect, + SnapshotNode, + SnapshotOptions, + SnapshotState, +} from './kernel/snapshot.ts'; import type { NetworkIncludeMode } from './kernel/contracts.ts'; import type { DeviceTarget, Platform, PlatformSelector, PublicPlatform } from './kernel/device.ts'; import type { BackMode } from './contracts/back-mode.ts'; @@ -11,6 +17,7 @@ import type { DeviceRotation } from './contracts/device-rotation.ts'; import type { ScrollDirection } from './contracts/scroll-gesture.ts'; import type { SessionSurface } from './contracts/session-surface.ts'; import type { TvRemoteButton } from './contracts/tv-remote.ts'; +import type { GesturePlan } from './contracts/gesture-plan-types.ts'; import type { RecordingExportQuality } from './core/recording-export-quality.ts'; import type { SnapshotDiagnosticsSummary } from './snapshot-diagnostics.ts'; import type { @@ -155,10 +162,6 @@ export type BackendLongPressOptions = { durationMs?: number; }; -export type BackendSwipeOptions = { - durationMs?: number; -}; - export type BackendScrollTarget = | { kind: 'viewport'; @@ -175,11 +178,6 @@ export type BackendScrollOptions = { durationMs?: number; }; -export type BackendPinchOptions = { - scale: number; - center?: Point; -}; - export type BackendOpenTarget = { /** * Generic app identifier accepted by the backend. Hosted adapters should @@ -434,6 +432,7 @@ export type AgentDeviceBackend = { context: BackendCommandContext, options?: BackendSnapshotOptions, ): Promise; + resolveGestureViewport?(context: BackendCommandContext): Promise; captureScreenshot?( context: BackendCommandContext, outPath: string, @@ -474,21 +473,12 @@ export type AgentDeviceBackend = { point: Point, options?: BackendLongPressOptions, ): Promise; - swipe?( - context: BackendCommandContext, - from: Point, - to: Point, - options?: BackendSwipeOptions, - ): Promise; scroll?( context: BackendCommandContext, target: BackendScrollTarget, options: BackendScrollOptions, ): Promise; - pinch?( - context: BackendCommandContext, - options: BackendPinchOptions, - ): Promise; + performGesture?(context: BackendCommandContext, plan: GesturePlan): Promise; pressKey?( context: BackendCommandContext, key: string, diff --git a/src/batch-policy.ts b/src/batch-policy.ts index e9398f9c6..60a056b93 100644 --- a/src/batch-policy.ts +++ b/src/batch-policy.ts @@ -32,7 +32,13 @@ export const STRUCTURED_BATCH_COMMAND_NAMES: readonly StructuredBatchCommandName const BATCH_BLOCKED_COMMANDS: ReadonlySet = new Set(['batch', 'replay']); -export const BATCH_DAEMON_STEP_KEYS = ['command', 'positionals', 'flags', 'runtime'] as const; +export const BATCH_DAEMON_STEP_KEYS = [ + 'command', + 'positionals', + 'input', + 'flags', + 'runtime', +] as const; export const INHERITED_PARENT_FLAG_KEYS = [ 'platform', diff --git a/src/cli-schema/command-schema-guards.test.ts b/src/cli-schema/command-schema-guards.test.ts index 6ba092fbf..6df5e368d 100644 --- a/src/cli-schema/command-schema-guards.test.ts +++ b/src/cli-schema/command-schema-guards.test.ts @@ -17,7 +17,6 @@ import { getCliCommandSchema } from './command-schema.ts'; test('every public capability command has a parser schema entry', () => { const schemaCommands = new Set(listCliCommandNames()); for (const command of listCapabilityCommands()) { - if (INTERNAL_GESTURE_CAPABILITY_COMMANDS.has(command)) continue; assert.equal(schemaCommands.has(command), true, `Missing schema for command: ${command}`); } }); @@ -58,22 +57,9 @@ test('cli.ts command dispatch checks are recognized by parser-level unknown-comm }); test('schema capability mappings match capability source-of-truth', () => { - assert.deepEqual( - listCapabilityCheckedCommandNames(), - listCapabilityCommands().filter( - (command) => !INTERNAL_GESTURE_CAPABILITY_COMMANDS.has(command), - ), - ); + assert.deepEqual(listCapabilityCheckedCommandNames(), listCapabilityCommands()); }); -const INTERNAL_GESTURE_CAPABILITY_COMMANDS = new Set([ - 'pan', - 'fling', - 'pinch', - 'rotate-gesture', - 'transform-gesture', -]); - function collectCliDispatchCommandLiterals(): Set { const cliPath = fileURLToPath(new URL('../cli.ts', import.meta.url)); const sourceText = fs.readFileSync(cliPath, 'utf8'); diff --git a/src/cli/parser/__tests__/args-parse-interaction.test.ts b/src/cli/parser/__tests__/args-parse-interaction.test.ts index c691ba8d1..483bb5e17 100644 --- a/src/cli/parser/__tests__/args-parse-interaction.test.ts +++ b/src/cli/parser/__tests__/args-parse-interaction.test.ts @@ -145,11 +145,15 @@ test('parseArgs recognizes swipe positional + pattern flags', () => { }); test('parseArgs recognizes gesture subcommand positionals', () => { - const pan = parseArgs(['gesture', 'pan', '200', '420', '0', '-80', '500'], { - strictFlags: true, - }); + const pan = parseArgs( + ['gesture', 'pan', '200', '420', '0', '-80', '500', '--pointer-count', '2'], + { + strictFlags: true, + }, + ); assert.equal(pan.command, 'gesture'); assert.deepEqual(pan.positionals, ['pan', '200', '420', '0', '-80', '500']); + assert.equal(pan.flags.pointerCount, 2); const fling = parseArgs(['gesture', 'fling', 'right', '200', '420', '180'], { strictFlags: true, diff --git a/src/cli/parser/__tests__/cli-help-command-usage.test.ts b/src/cli/parser/__tests__/cli-help-command-usage.test.ts index b490e1d1a..d0bb47504 100644 --- a/src/cli/parser/__tests__/cli-help-command-usage.test.ts +++ b/src/cli/parser/__tests__/cli-help-command-usage.test.ts @@ -261,7 +261,8 @@ test('network command usage documents include flag', async () => { test('command usage shows command flags without global flags', async () => { const help = await usageForCommand('swipe'); if (help === null) throw new Error('Expected command help text'); - assert.match(help, /Swipe coordinates with optional repeat pattern/); + assert.match(help, /Quick coordinate fling with optional repeat pattern/); + assert.match(help, /duration positional is accepted as a deprecated alias to pan/); assert.match(help, /Command flags:/); assert.match(help, /--pattern one-way\|ping-pong/); assert.doesNotMatch(help, /Global flags:/); diff --git a/src/cli/parser/__tests__/cli-help-topics.test.ts b/src/cli/parser/__tests__/cli-help-topics.test.ts index 9f3a3c687..23194d1b0 100644 --- a/src/cli/parser/__tests__/cli-help-topics.test.ts +++ b/src/cli/parser/__tests__/cli-help-topics.test.ts @@ -225,7 +225,11 @@ test('usageForCommand resolves workflow help topic', async () => { assert.match(help, /Android transform injects a geometric two-finger path/); assert.match(help, /verify semantic app state or coarse per-component effects/); assert.match(help, /instead of exact numeric deltas/); - assert.match(help, /prefer isolated gesture pan, gesture pinch, or gesture rotate/); + assert.match(help, /prefer isolated gesture pan --pointer-count 2, gesture pinch/); + assert.match(help, /gesture pan is one finger by default/); + assert.match(help, /--pointer-count 2 for a parallel two-finger pan/); + assert.match(help, /falls back to the visible snapshot union/); + assert.match(help, /tvOS coordinate pan and fling preserve only the dominant direction/); assert.match(help, /longpress accepts coordinates, @refs, or selectors/); assert.match(help, /use help react-native for Metro\/Re\.Pack Fast Refresh/); assert.match(help, /iOS Allow Paste prompt cannot be exercised under XCUITest/); diff --git a/src/cli/parser/cli-help.ts b/src/cli/parser/cli-help.ts index 424cf8e0c..d3cf56058 100644 --- a/src/cli/parser/cli-help.ts +++ b/src/cli/parser/cli-help.ts @@ -193,7 +193,7 @@ Command shape: Snapshot refs look like @e12. After snapshot -i, use the exact @eN ref from that output. If the exact ref is not known yet, first output snapshot -i, then use a concrete example shape like press @e12 in the next command; do not write @, @ref, @Label_Name, or @eN placeholders. Close means agent-device close. App-owned back means back; system back means back --system. - Taps are press or click; tap is an alias for press. On Android TV and tvOS, read help tv and use tv-remote press up|down|left|right|select to move D-pad/remote focus before activating controls; use tv-remote longpress