From 569868bab8d3668e769ad817c964b019993152a2 Mon Sep 17 00:00:00 2001 From: cryptotavares Date: Thu, 13 Aug 2026 23:36:20 +0100 Subject: [PATCH] feat!: flat mobile a11y tree without layout-only nodes Ticket P1-14 (supersedes the Android-only first version of #44). Mobile a11y nodes carried `path`, the list of every ancestor's role. On mobile those roles are widget class names (`android.widget.FrameLayout`, `RCTViewComponentView`), with no targeting value; on Android wallet home they were 21,658 entries in a ~1 MB describe-screen. Most nodes were also unnamed layout views. - `A11yNodeTrimmed.path` is optional. Mobile nodes have no `path` (iOS and Android). Browser nodes keep it (ancestor dialog/heading names, used to collapse identical rows). The observation diff and row collapse treat two absent paths as equal. - Mobile: layout-only elements (no label, value or testId, not an actionable control, not a progress indicator) are dropped and replaced by their kept descendants, in order. Containers with a testId, named elements, controls and progress indicators stay, and so do unlabelled React Native leaf views of at least 24x24 (icon buttons without a label, such as the wallet action buttons), listed with bounds. - Refs exist for nodes with a testId, label or value (as before); listed nodes without one (unlabelled tap targets, progress indicators) have bounds for tap-coordinates. Ambiguity marking covers only listed nodes. - `bounds` keys are always in x, y, width, height order (idb returns them in varying order, which made unchanged screens look changed). - One busy-type regex shared by the settle logic and the filter. BREAKING CHANGE: `A11yNodeTrimmed.path` is optional and absent on mobile nodes; mobile trees no longer include unnamed layout nodes, so mobile ref numbers change. --- README.md | 2 + SKILL.md | 4 + .../ios-wallet-home-preorder.json | 1268 +++++++++++++++++ src/platform/mobile-hierarchy.test.ts | 285 ++++ src/platform/mobile-hierarchy.ts | 183 +++ src/platform/mobile-platform-driver.test.ts | 494 ++++++- src/platform/mobile-platform-driver.ts | 45 +- src/server/create-server.test.ts | 3 +- src/server/observation-compaction.test.ts | 54 + src/server/observation-compaction.ts | 21 +- src/server/screen-settle.ts | 22 +- src/tools/types/discovery.ts | 6 +- src/tools/utils/constants.ts | 27 + src/tools/utils/discovery.test.ts | 25 +- src/tools/utils/discovery.ts | 14 +- 15 files changed, 2400 insertions(+), 53 deletions(-) create mode 100644 src/platform/__fixtures__/ios-wallet-home-preorder.json diff --git a/README.md b/README.md index 3a6a040..a8c622e 100644 --- a/README.md +++ b/README.md @@ -453,6 +453,8 @@ On mobile, after a mutating or batch call, the daemon may take the snapshot agai On mobile, each observation, `describe_screen`, `list_testids` and `accessibility_snapshot` takes one device snapshot (plus the settle retries above). The viewport for `visible` comes from the snapshot's root frames; the device window size is read at most once per session (and not refreshed after a rotation), only when a snapshot has no root frame. Elements that are not part of the app are removed from the tree, the refs, the test IDs and `generate_locators`: on Android, the system status bar and navigation bar windows (a full-width or full-height bar window whose ids are all `com.android.systemui:id/*`; other System UI windows such as the biometric prompt stay); on both platforms, the Expo DevLauncher "Tools" button of development builds (iOS `gearshape.fill` and its "Tools" text). Pass `{ excludeDevOverlay: false }` as the third `MobilePlatformDriver` argument to keep the DevLauncher button. Taps and waits inside `@metamask/device-mcp` still search the whole snapshot. +The mobile accessibility tree (`describe_screen`, `accessibility_snapshot`, observations) is a flat list in pre-order with no `path` (on the browser, `path` holds the `role:name` of the ancestor dialogs and headings; on mobile the ancestors are native view classes). Layout-only elements are left out (`pruneLayoutOnly` in `src/platform/mobile-hierarchy.ts`): an element with no label, no value, no identifier, and a type that is not a control or a progress indicator. Its kept descendants take its place, in order. Controls are iOS `Button`, `Link`, `TextField`, `SecureTextField`, `SearchField`, `TextView`, `TextArea`, `Switch`, `Toggle`, `Slider`, `Stepper`, `Picker`, `Cell`, `Tab`, `MenuItem` and similar AX types (exact match, without the `XCUIElementType` prefix), and Android classes ending in `Button`, `EditText`, `CheckBox`, `CheckedTextView`, `Switch`, `SeekBar`, `Spinner`, `RatingBar` and similar. The device snapshot has no `clickable` flag, so a layout-only plain React Native view (`RCTViewComponentView` on iOS, `android.view.ViewGroup` on Android) that has no kept children and is at least 24×24 (iOS points, Android pixels) stays as an unlabelled tap target, for example the iOS wallet home action buttons. Node `bounds` are `{ x, y, width, height }`, in that key order. Node refs number the kept nodes only, and ambiguity is computed over them. Only nodes with a testId, label or value have a refMap entry, so only their refs can be targeted; unlabelled tap targets, progress indicators and unnamed controls are listed with `bounds` (use `tap_coordinates`) but their refs do not resolve. On real MetaMask screens: iOS wallet home 177 → 67 nodes (183 kB with `path` → 10 kB), Android wallet home 625 → 242 nodes (649 kB → 35 kB), Android login 47 → 15 nodes. `list_testids` and `generate_locators` are not affected, and `describe_screen` still returns the full `testIds` list next to the tree. + ### Session Metadata Consumers can persist launch metadata with `writeSessionMetadata(artifactsDir, metadata)`. It writes `//session.json` (for example `test-artifacts//session.json` with `artifactsDir: 'test-artifacts'`) and returns the file path. diff --git a/SKILL.md b/SKILL.md index 8ad09cf..a5a7659 100644 --- a/SKILL.md +++ b/SKILL.md @@ -232,8 +232,12 @@ Each node looks like: The `testId` and `textContent` fields appear only on nodes with short or generic names — they provide extra context from the DOM to help identify ambiguous elements. Nodes with clear names omit these fields. +On the browser, `path` lists the `role:name` of the dialogs and headings that contain the node. Mobile nodes have no `path`. + When 3+ consecutive identical nodes appear (same role, name, and path), they are collapsed into a summary like `… 3 more "maskicon" (refs e2–e4)` to reduce token waste. Individual refs still work for targeting. +On mobile (iOS, Android), the tree is a flat list in screen order. A node is a native element: `role` is its type (`Button`, `StaticText`, `android.widget.Button`, `android.view.ViewGroup`), and `bounds` is its frame (`{ x, y, width, height }`). Layout-only elements are left out: elements with no label or text, no testId, and a type that is not a control (button, link, text field, switch, slider, checkbox and similar). Their children stay in the list. Containers with a testId stay. Plain unlabelled views of at least 24×24 with nothing kept inside also stay: they are often icon buttons (for example the iOS wallet home action buttons). Only a node with a testId, label or text has a ref you can click or type into. Nodes without one (unlabelled views, spinners, unnamed controls) are listed with their `bounds` but their ref cannot be targeted: use `mm tap-coordinates ` with the center of `bounds`. + Use the `ref` value (`e3`) for click/type/get-text/wait-for commands. On the browser, each ref resolves to exactly one element, or the action fails (strict mode) instead of acting on another element: refs use the exact, case-sensitive name, and elements that share a role and name get a unique DOM selector, so two "Send" buttons get two refs that each click their own button. If a click fails with a strict mode violation, target the element by `--testid` or `--selector` instead. #### `mm list-testids` diff --git a/src/platform/__fixtures__/ios-wallet-home-preorder.json b/src/platform/__fixtures__/ios-wallet-home-preorder.json new file mode 100644 index 0000000..bc84480 --- /dev/null +++ b/src/platform/__fixtures__/ios-wallet-home-preorder.json @@ -0,0 +1,1268 @@ +[ + { + "depth": 0, + "type": "Other", + "label": "MetaMask", + "frame": { "x": 0, "y": 0, "width": 402, "height": 874 }, + "enabled": true + }, + { + "depth": 1, + "type": "Application", + "frame": { "x": 0, "y": 0, "width": 402, "height": 874 }, + "enabled": true + }, + { + "depth": 2, + "type": "UITransitionView", + "frame": { "x": 0, "y": 0, "width": 402, "height": 874 }, + "enabled": true + }, + { + "depth": 3, + "type": "UIDropShadowView", + "frame": { "x": 0, "y": 0, "width": 402, "height": 874 }, + "enabled": true + }, + { + "depth": 4, + "type": "DevLauncherWrapperView", + "frame": { "x": 0, "y": 0, "width": 402, "height": 874 }, + "enabled": true + }, + { + "depth": 5, + "type": "RCTSurfaceHostingProxyRootView", + "frame": { "x": 0, "y": 0, "width": 402, "height": 874 }, + "enabled": true + }, + { + "depth": 6, + "type": "RCTSurfaceView", + "frame": { "x": 0, "y": 0, "width": 402, "height": 874 }, + "enabled": true + }, + { + "depth": 7, + "type": "RCTRootComponentView", + "frame": { "x": 0, "y": 0, "width": 402, "height": 874 }, + "enabled": true + }, + { + "depth": 8, + "type": "RCTViewComponentView", + "frame": { "x": 0, "y": 0, "width": 402, "height": 874 }, + "enabled": true + }, + { + "depth": 9, + "type": "RNCSafeAreaProviderComponentView", + "frame": { "x": 0, "y": 0, "width": 402, "height": 874 }, + "enabled": true + }, + { + "depth": 10, + "type": "KeyboardControllerView", + "frame": { "x": 0, "y": 0, "width": 402, "height": 874 }, + "enabled": true + }, + { + "depth": 11, + "type": "RCTViewComponentView", + "frame": { "x": 0, "y": 0, "width": 402, "height": 874 }, + "enabled": true + }, + { + "depth": 11, + "type": "RNSScreenStackView", + "frame": { "x": 0, "y": 0, "width": 402, "height": 874 }, + "enabled": true + }, + { + "depth": 12, + "type": "UILayoutContainerView", + "frame": { "x": 0, "y": 0, "width": 402, "height": 874 }, + "enabled": true + }, + { + "depth": 13, + "type": "UINavigationTransitionView", + "frame": { "x": 0, "y": 0, "width": 402, "height": 874 }, + "enabled": true + }, + { + "depth": 14, + "type": "UIViewControllerWrapperView", + "frame": { "x": 0, "y": 0, "width": 402, "height": 874 }, + "enabled": true + }, + { + "depth": 15, + "type": "RNSScreenView", + "frame": { "x": 0, "y": 0, "width": 402, "height": 874 }, + "enabled": true + }, + { + "depth": 16, + "type": "RNSScreenContentWrapper", + "frame": { "x": 0, "y": 0, "width": 402, "height": 874 }, + "enabled": true + }, + { + "depth": 17, + "type": "RNSSafeAreaViewComponentView", + "frame": { "x": 0, "y": 0, "width": 402, "height": 874 }, + "enabled": true + }, + { + "depth": 18, + "type": "RNSScreenStackView", + "frame": { "x": 0, "y": 0, "width": 402, "height": 874 }, + "enabled": true + }, + { + "depth": 19, + "type": "UILayoutContainerView", + "frame": { "x": 0, "y": 0, "width": 402, "height": 874 }, + "enabled": true + }, + { + "depth": 20, + "type": "UINavigationTransitionView", + "frame": { "x": 0, "y": 0, "width": 402, "height": 874 }, + "enabled": true + }, + { + "depth": 21, + "type": "UIViewControllerWrapperView", + "frame": { "x": 0, "y": 0, "width": 402, "height": 874 }, + "enabled": true + }, + { + "depth": 22, + "type": "RNSScreenView", + "frame": { "x": 0, "y": 0, "width": 402, "height": 874 }, + "enabled": true + }, + { + "depth": 23, + "type": "RNSScreenContentWrapper", + "frame": { "x": 0, "y": 0, "width": 402, "height": 874 }, + "enabled": true + }, + { + "depth": 24, + "type": "RNSSafeAreaViewComponentView", + "frame": { "x": 0, "y": 0, "width": 402, "height": 874 }, + "enabled": true + }, + { + "depth": 25, + "type": "RNSScreenStackView", + "frame": { "x": 0, "y": 0, "width": 402, "height": 874 }, + "enabled": true + }, + { + "depth": 26, + "type": "UILayoutContainerView", + "frame": { "x": 0, "y": 0, "width": 402, "height": 874 }, + "enabled": true + }, + { + "depth": 27, + "type": "UINavigationTransitionView", + "frame": { "x": 0, "y": 0, "width": 402, "height": 874 }, + "enabled": true + }, + { + "depth": 28, + "type": "UIViewControllerWrapperView", + "frame": { "x": 0, "y": 0, "width": 402, "height": 874 }, + "enabled": true + }, + { + "depth": 29, + "type": "RNSScreenView", + "frame": { "x": 0, "y": 0, "width": 402, "height": 874 }, + "enabled": true + }, + { + "depth": 30, + "type": "RNSScreenContentWrapper", + "frame": { "x": 0, "y": 0, "width": 402, "height": 874 }, + "enabled": true + }, + { + "depth": 31, + "type": "RNSSafeAreaViewComponentView", + "frame": { "x": 0, "y": 0, "width": 402, "height": 874 }, + "enabled": true + }, + { + "depth": 32, + "type": "RNSTabsHostComponentView", + "frame": { "x": 0, "y": 0, "width": 402, "height": 874 }, + "enabled": true + }, + { + "depth": 33, + "type": "UILayoutContainerView", + "frame": { "x": 0, "y": 0, "width": 402, "height": 874 }, + "enabled": true + }, + { + "depth": 34, + "type": "UITransitionView", + "frame": { "x": 0, "y": 0, "width": 402, "height": 874 }, + "enabled": true + }, + { + "depth": 35, + "type": "UIViewControllerWrapperView", + "frame": { "x": 0, "y": 0, "width": 402, "height": 874 }, + "enabled": true + }, + { + "depth": 36, + "type": "RNSTabsScreenComponentView", + "frame": { "x": 0, "y": 0, "width": 402, "height": 874 }, + "enabled": true + }, + { + "depth": 37, + "type": "RCTViewComponentView", + "frame": { "x": 0, "y": 0, "width": 402, "height": 874 }, + "enabled": true + }, + { + "depth": 38, + "type": "RNSScreenStackView", + "frame": { "x": 0, "y": 0, "width": 402, "height": 874 }, + "enabled": true + }, + { + "depth": 39, + "type": "UILayoutContainerView", + "frame": { "x": 0, "y": 0, "width": 402, "height": 874 }, + "enabled": true + }, + { + "depth": 40, + "type": "UINavigationTransitionView", + "frame": { "x": 0, "y": 0, "width": 402, "height": 874 }, + "enabled": true + }, + { + "depth": 41, + "type": "UIViewControllerWrapperView", + "frame": { "x": 0, "y": 0, "width": 402, "height": 874 }, + "enabled": true + }, + { + "depth": 42, + "type": "RNSScreenView", + "frame": { "x": 0, "y": 0, "width": 402, "height": 874 }, + "enabled": true + }, + { + "depth": 43, + "type": "RNSScreenContentWrapper", + "frame": { "x": 0, "y": 0, "width": 402, "height": 874 }, + "enabled": true + }, + { + "depth": 44, + "type": "RNSSafeAreaViewComponentView", + "frame": { "x": 0, "y": 0, "width": 402, "height": 874 }, + "enabled": true + }, + { + "depth": 45, + "type": "RNCSafeAreaViewComponentView", + "identifier": "wallet-safe-area", + "frame": { "x": 0, "y": 0, "width": 402, "height": 874 }, + "enabled": true + }, + { + "depth": 46, + "type": "RCTViewComponentView", + "identifier": "wallet-floating-header", + "frame": { "x": 0, "y": 0, "width": 402, "height": 118 }, + "enabled": true + }, + { + "depth": 47, + "type": "RCTViewComponentView", + "frame": { "x": 0, "y": 0, "width": 402, "height": 142 }, + "enabled": true + }, + { + "depth": 48, + "type": "RCTLegacyViewManagerInteropComponentView", + "frame": { "x": 0, "y": 0, "width": 402, "height": 127.66666412353516 }, + "enabled": true + }, + { + "depth": 49, + "type": "RNCMaskedView", + "frame": { "x": 0, "y": 0, "width": 402, "height": 127.66666412353516 }, + "enabled": true + }, + { + "depth": 50, + "type": "RCTViewComponentView", + "frame": { "x": 0, "y": 0, "width": 402, "height": 127.66666412353516 }, + "enabled": true + }, + { + "depth": 47, + "type": "RCTViewComponentView", + "identifier": "wallet-header-root", + "frame": { "x": 0, "y": 62, "width": 402, "height": 56 }, + "enabled": true + }, + { + "depth": 47, + "type": "RCTViewComponentView", + "frame": { "x": 12, "y": 70, "width": 40, "height": 40 }, + "enabled": true + }, + { + "depth": 48, + "type": "ExpoGlassEffect.GlassView", + "frame": { "x": 12, "y": 70, "width": 40, "height": 40 }, + "enabled": true + }, + { + "depth": 49, + "type": "UIVisualEffectView", + "frame": { "x": 12, "y": 70, "width": 40, "height": 40 }, + "enabled": true + }, + { + "depth": 50, + "type": "Button", + "label": "Open my profile", + "identifier": "profile-button", + "frame": { "x": 12, "y": 70, "width": 40, "height": 40 }, + "enabled": true + }, + { + "depth": 47, + "type": "RCTViewComponentView", + "frame": { "x": 60, "y": 70, "width": 234, "height": 40 }, + "enabled": true + }, + { + "depth": 48, + "type": "ExpoGlassEffect.GlassView", + "identifier": "wallet-header-account-picker-capsule", + "frame": { "x": 60, "y": 70, "width": 162.6666717529297, "height": 40 }, + "enabled": true + }, + { + "depth": 49, + "type": "UIVisualEffectView", + "frame": { "x": 60, "y": 70, "width": 162.66666666666666, "height": 40 }, + "enabled": true + }, + { + "depth": 50, + "type": "Button", + "label": "Agent Account", + "identifier": "account-picker", + "frame": { "x": 60, "y": 78, "width": 162.6666717529297, "height": 24 }, + "enabled": true + }, + { + "depth": 47, + "type": "RCTViewComponentView", + "frame": { "x": 302, "y": 70, "width": 88, "height": 40 }, + "enabled": true + }, + { + "depth": 48, + "type": "ExpoGlassEffect.GlassView", + "identifier": "wallet-header-actions-capsule", + "frame": { "x": 302, "y": 70, "width": 88, "height": 40 }, + "enabled": true + }, + { + "depth": 49, + "type": "UIVisualEffectView", + "frame": { "x": 302, "y": 70, "width": 88, "height": 40 }, + "enabled": true + }, + { + "depth": 50, + "type": "RCTViewComponentView", + "frame": { "x": 310, "y": 74, "width": 72, "height": 32 }, + "enabled": true + }, + { + "depth": 46, + "type": "RCTViewComponentView", + "identifier": "wallet-screen", + "frame": { "x": 0, "y": 0, "width": 402, "height": 874 }, + "enabled": true + }, + { + "depth": 46, + "type": "RCTScrollViewComponentView", + "identifier": "wallet-scroll-view", + "frame": { "x": 0, "y": 0, "width": 402, "height": 874 }, + "enabled": true + }, + { + "depth": 47, + "type": "ScrollView", + "frame": { "x": 0, "y": 0, "width": 402, "height": 874 }, + "enabled": true + }, + { + "depth": 48, + "type": "_UITouchPassthroughView", + "frame": { "x": 0, "y": 0, "width": 402, "height": 874 }, + "enabled": true + }, + { + "depth": 49, + "type": "BackdropView", + "frame": { "x": 0, "y": 751, "width": 402, "height": 123 }, + "enabled": true + }, + { + "depth": 48, + "type": "_UITouchPassthroughView", + "frame": { "x": 0, "y": 0, "width": 402, "height": 874 }, + "enabled": true + }, + { + "depth": 49, + "type": "UIKit.ScrollEdgeEffectView", + "frame": { "x": 0, "y": 751, "width": 402, "height": 123 }, + "enabled": true + }, + { + "depth": 48, + "type": "UIView", + "frame": { "x": 0, "y": 0, "width": 402, "height": 2938 }, + "enabled": true + }, + { + "depth": 49, + "type": "RCTViewComponentView", + "frame": { "x": 0, "y": 0, "width": 402, "height": 2938 }, + "enabled": true + }, + { + "depth": 50, + "type": "RCTViewComponentView", + "identifier": "homepage-balance-breakdown", + "frame": { "x": 0, "y": 134, "width": 402, "height": 434.66668701171875 }, + "enabled": true + }, + { + "depth": 50, + "type": "RCTViewComponentView", + "identifier": "homepage-balance-breakdown-section", + "frame": { "x": 0, "y": 134, "width": 402, "height": 438.66668701171875 }, + "enabled": true + }, + { + "depth": 50, + "type": "RCTViewComponentView", + "identifier": "homepage-container", + "frame": { "x": 0, "y": 134, "width": 402, "height": 2688 }, + "enabled": true + }, + { + "depth": 50, + "type": "Button", + "label": "$, $, (%), Today", + "identifier": "homepage-balance-breakdown-hero", + "frame": { "x": 16, "y": 134, "width": 370, "height": 84 }, + "enabled": true + }, + { + "depth": 50, + "type": "RCTViewComponentView", + "identifier": "wallet-action-buttons", + "frame": { "x": 0, "y": 242, "width": 402, "height": 73.33334350585938 }, + "enabled": true + }, + { + "depth": 50, + "type": "RCTViewComponentView", + "frame": { + "x": 16, + "y": 242, + "width": 86.66666412353516, + "height": 73.33334350585938 + }, + "enabled": true + }, + { + "depth": 50, + "type": "RCTViewComponentView", + "frame": { + "x": 110.66666412353516, + "y": 242, + "width": 86.33333587646484, + "height": 73.33334350585938 + }, + "enabled": true + }, + { + "depth": 50, + "type": "RCTViewComponentView", + "frame": { + "x": 205, + "y": 242, + "width": 86.66665649414062, + "height": 73.33334350585938 + }, + "enabled": true + }, + { + "depth": 50, + "type": "RCTViewComponentView", + "frame": { + "x": 299.6666564941406, + "y": 242, + "width": 86.33334350585938, + "height": 73.33334350585938 + }, + "enabled": true + }, + { + "depth": 50, + "type": "RCTViewComponentView", + "identifier": "braze-banner-container", + "frame": { + "x": 16, + "y": 331.33333587646484, + "width": 370, + "height": 97.33331298828125 + }, + "enabled": true + }, + { + "depth": 50, + "type": "RCTViewComponentView", + "label": "Test Banner 9, Testing banners from Braze. Tap to try Rewards.", + "identifier": "braze-banner-pressable", + "frame": { + "x": 16, + "y": 331.33333587646484, + "width": 370, + "height": 97.33331298828125 + }, + "enabled": true + }, + { + "depth": 50, + "type": "RCTViewComponentView", + "identifier": "homepage-balance-breakdown-rows", + "frame": { + "x": 0, + "y": 440.6666564941406, + "width": 402, + "height": 128.00003051757812 + }, + "enabled": true + }, + { + "depth": 50, + "type": "Button", + "label": "Money, $, %, % APY", + "identifier": "homepage-balance-breakdown-row-money", + "frame": { "x": 0, "y": 448.6666564941406, "width": 402, "height": 40 }, + "enabled": true + }, + { + "depth": 50, + "type": "Button", + "label": "Tokens, $, %", + "identifier": "homepage-balance-breakdown-row-tokens", + "frame": { + "x": 0, + "y": 488.6666564941406, + "width": 402, + "height": 40.000030517578125 + }, + "enabled": true + }, + { + "depth": 50, + "type": "Button", + "label": "Perps, $, %", + "identifier": "homepage-balance-breakdown-row-perps", + "frame": { "x": 0, "y": 528.6666564941406, "width": 402, "height": 40 }, + "enabled": true + }, + { + "depth": 50, + "type": "RCTViewComponentView", + "frame": { "x": 0, "y": 592.6666564941406, "width": 402, "height": 1 }, + "enabled": true + }, + { + "depth": 50, + "type": "Button", + "label": "Tokens", + "identifier": "homepage-section-title-tokens", + "frame": { "x": 0, "y": 613.6666564941406, "width": 402, "height": 44 }, + "enabled": true + }, + { + "depth": 50, + "type": "RCTViewComponentView", + "label": "Ethereum, Stake %, $, $ • , %, ETH", + "identifier": "asset-ETH", + "frame": { "x": 16, "y": 657.6666564941406, "width": 370, "height": 64 }, + "enabled": true + }, + { + "depth": 50, + "type": "RCTViewComponentView", + "label": "USD Coin, $, $ • , Get % APY, USDC", + "identifier": "asset-USDC", + "frame": { "x": 16, "y": 721.6666564941406, "width": 370, "height": 64 }, + "enabled": true + }, + { + "depth": 50, + "type": "RCTViewComponentView", + "label": "Ethereum, $, $ • , %, ETH", + "identifier": "asset-ETH", + "frame": { "x": 16, "y": 785.6666564941406, "width": 370, "height": 64 }, + "enabled": true + }, + { + "depth": 50, + "type": "RCTViewComponentView", + "label": "USD Coin (Native), $, $ • , Get % APY, USDC", + "identifier": "asset-USDC", + "frame": { "x": 16, "y": 849.6666564941406, "width": 370, "height": 64 }, + "enabled": true + }, + { + "depth": 50, + "type": "RCTViewComponentView", + "label": "MetaMask USD, $, $ • , Get % APY, mUSD", + "identifier": "asset-mUSD", + "frame": { "x": 16, "y": 913.6666564941406, "width": 370, "height": 64 }, + "enabled": true + }, + { + "depth": 50, + "type": "RCTViewComponentView", + "frame": { "x": 0, "y": 997.6666870117188, "width": 402, "height": 1 }, + "enabled": true + }, + { + "depth": 50, + "type": "Button", + "label": "Perps", + "identifier": "homepage-section-title-perps", + "frame": { "x": 0, "y": 1018.6666870117188, "width": 402, "height": 44 }, + "enabled": true + }, + { + "depth": 50, + "type": "RCTScrollViewComponentView", + "identifier": "homepage-trending-perps-carousel", + "frame": { + "x": 0, + "y": 1074.6666870117188, + "width": 402, + "height": 179.99993896484375 + }, + "enabled": true + }, + { + "depth": 50, + "type": "RCTViewComponentView", + "frame": { "x": 0, "y": 1286.6666259765625, "width": 402, "height": 1 }, + "enabled": true + }, + { + "depth": 50, + "type": "Button", + "label": "Predictions", + "identifier": "homepage-section-title-predictions", + "frame": { "x": 0, "y": 1307.6666259765625, "width": 402, "height": 44 }, + "enabled": true + }, + { + "depth": 50, + "type": "RCTViewComponentView", + "label": "Pro Football: 2027 Champion, % chance on San Francisco 49ers", + "identifier": "homepage-predict-discovery-market-slot-1", + "frame": { "x": 16, "y": 1351.6666259765625, "width": 370, "height": 64 }, + "enabled": true + }, + { + "depth": 50, + "type": "Button", + "label": "BTC Price: —, Price to beat: —, Live, --:--", + "identifier": "homepage-predict-discovery-btc-row", + "frame": { "x": 16, "y": 1415.6666259765625, "width": 370, "height": 64 }, + "enabled": true + }, + { + "depth": 50, + "type": "RCTViewComponentView", + "label": "EPL: 2027 Champion, % chance on Arsenal", + "identifier": "homepage-predict-discovery-market-slot-3", + "frame": { "x": 16, "y": 1479.6666259765625, "width": 370, "height": 64 }, + "enabled": true + }, + { + "depth": 50, + "type": "RCTViewComponentView", + "frame": { "x": 0, "y": 1575.6666259765625, "width": 402, "height": 1 }, + "enabled": true + }, + { + "depth": 50, + "type": "Button", + "label": "Watchlist", + "identifier": "homepage-section-title-watchlist", + "frame": { "x": 0, "y": 1596.6666259765625, "width": 402, "height": 44 }, + "enabled": true + }, + { + "depth": 50, + "type": "RCTViewComponentView", + "frame": { "x": 16, "y": 1640.5819848651226, "width": 370, "height": 320 }, + "enabled": true + }, + { + "depth": 50, + "type": "RCTViewComponentView", + "identifier": "watchlist-suggested-section", + "frame": { "x": 16, "y": 1640.6666259765625, "width": 370, "height": 320 }, + "enabled": true + }, + { + "depth": 50, + "type": "RCTViewComponentView", + "identifier": "homepage-top-traders-section-root", + "frame": { + "x": 0, + "y": 1960.6666259765625, + "width": 402, + "height": 229.0001220703125 + }, + "enabled": true + }, + { + "depth": 50, + "type": "RCTViewComponentView", + "frame": { "x": 0, "y": 1980.6666259765625, "width": 402, "height": 1 }, + "enabled": true + }, + { + "depth": 50, + "type": "Button", + "label": "Top traders", + "identifier": "homepage-section-title-top-traders", + "frame": { "x": 0, "y": 2001.6666259765625, "width": 402, "height": 44 }, + "enabled": true + }, + { + "depth": 50, + "type": "RCTScrollViewComponentView", + "identifier": "homepage-top-traders-carousel", + "frame": { "x": 0, "y": 2057.6666259765625, "width": 402, "height": 120 }, + "enabled": true + }, + { + "depth": 50, + "type": "RCTViewComponentView", + "frame": { "x": 0, "y": 2209.666748046875, "width": 402, "height": 1 }, + "enabled": true + }, + { + "depth": 50, + "type": "Button", + "label": "DeFi", + "identifier": "homepage-section-title-defi", + "frame": { "x": 0, "y": 2230.666748046875, "width": 402, "height": 44 }, + "enabled": true + }, + { + "depth": 50, + "type": "RCTLegacyViewManagerInteropComponentView", + "frame": { "x": 16, "y": 2274.666748046875, "width": 370, "height": 56 }, + "enabled": true + }, + { + "depth": 50, + "type": "RCTViewComponentView", + "frame": { "x": 0, "y": 2350.666748046875, "width": 402, "height": 1 }, + "enabled": true + }, + { + "depth": 50, + "type": "RCTViewComponentView", + "identifier": "homepage-section-title-nfts", + "frame": { + "x": 0, + "y": 2371.666748046875, + "width": 402, + "height": 43.9998779296875 + }, + "enabled": true + }, + { + "depth": 50, + "type": "StaticText", + "label": "NFTs", + "frame": { + "x": 16, + "y": 2383.666748046875, + "width": 50.666664123535156, + "height": 24 + }, + "enabled": true + }, + { + "depth": 50, + "type": "RCTLegacyViewManagerInteropComponentView", + "frame": { + "x": 16, + "y": 2415.666748046875, + "width": 115.33332824707031, + "height": 153.33349609375 + }, + "enabled": true + }, + { + "depth": 50, + "type": "RCTLegacyViewManagerInteropComponentView", + "frame": { + "x": 143.33333587646484, + "y": 2415.666748046875, + "width": 115.33332824707031, + "height": 153.33349609375 + }, + "enabled": true + }, + { + "depth": 50, + "type": "RCTLegacyViewManagerInteropComponentView", + "frame": { + "x": 270.6666717529297, + "y": 2415.666748046875, + "width": 115.33334350585938, + "height": 153.33349609375 + }, + "enabled": true + }, + { + "depth": 50, + "type": "RCTViewComponentView", + "identifier": "homepage-more-section", + "frame": { "x": 0, "y": 2593, "width": 402, "height": 229 }, + "enabled": true + }, + { + "depth": 50, + "type": "RCTViewComponentView", + "frame": { "x": 0, "y": 2613, "width": 402, "height": 1 }, + "enabled": true + }, + { + "depth": 50, + "type": "StaticText", + "label": "More", + "frame": { "x": 16, "y": 2646, "width": 50, "height": 24 }, + "enabled": true + }, + { + "depth": 50, + "type": "Button", + "label": "Import a token", + "identifier": "import-token-button", + "frame": { "x": 0, "y": 2678, "width": 402, "height": 48 }, + "enabled": true + }, + { + "depth": 50, + "type": "Button", + "label": "Import an NFT", + "identifier": "import-collectible-button", + "frame": { "x": 0, "y": 2726, "width": 402, "height": 48 }, + "enabled": true + }, + { + "depth": 50, + "type": "Button", + "label": "Contact support", + "identifier": "homepage-more-contact-support-button", + "frame": { "x": 0, "y": 2774, "width": 402, "height": 48.00006103515625 }, + "enabled": true + }, + { + "depth": 48, + "type": "_UIScrollViewScrollIndicator", + "label": "Horizontal scroll bar, 1 page", + "value": "%", + "frame": { "x": 62, "y": 758, "width": 278, "height": 30 }, + "enabled": true + }, + { + "depth": 49, + "type": "UIView", + "frame": { "x": 65, "y": 785, "width": 272, "height": 3 }, + "enabled": true + }, + { + "depth": 40, + "type": "Toolbar", + "label": "Toolbar", + "identifier": "Toolbar", + "frame": { "x": 0, "y": 0, "width": 402, "height": 874 }, + "enabled": true + }, + { + "depth": 41, + "type": "FloatingBarHostingView", + "frame": { "x": 0, "y": 0, "width": 402, "height": 874 }, + "enabled": true + }, + { + "depth": 34, + "type": "UIKit._UITabBarContainerWrapperView", + "frame": { "x": 0, "y": 791, "width": 402, "height": 83 }, + "enabled": true + }, + { + "depth": 35, + "type": "UIKit._UITabBarContainerView", + "frame": { "x": 0, "y": 0, "width": 402, "height": 874 }, + "enabled": true + }, + { + "depth": 36, + "type": "TabBar", + "label": "Tab Bar", + "frame": { "x": 0, "y": 791, "width": 402, "height": 83 }, + "enabled": true + }, + { + "depth": 37, + "type": "UIKit._UITabBarItemPlatterView", + "frame": { "x": 21, "y": 791, "width": 290, "height": 62 }, + "enabled": true + }, + { + "depth": 38, + "type": "DestOutView", + "frame": { "x": 21, "y": 791, "width": 76, "height": 54 }, + "enabled": true + }, + { + "depth": 38, + "type": "ContentView", + "frame": { "x": 21, "y": 791, "width": 290, "height": 62 }, + "enabled": true + }, + { + "depth": 39, + "type": "Button", + "label": "Home", + "frame": { "x": 25, "y": 795, "width": 76, "height": 54 }, + "enabled": true + }, + { + "depth": 39, + "type": "Button", + "label": "Explore", + "frame": { + "x": 93.66666666666667, + "y": 795, + "width": 76.00000000000001, + "height": 54 + }, + "enabled": true + }, + { + "depth": 39, + "type": "Button", + "label": "Money", + "frame": { "x": 162.33333333333334, "y": 795, "width": 76, "height": 54 }, + "enabled": true + }, + { + "depth": 39, + "type": "Button", + "label": "Social", + "frame": { "x": 231, "y": 795, "width": 76, "height": 54 }, + "enabled": true + }, + { + "depth": 38, + "type": "BadgeContainerView", + "frame": { "x": 21, "y": 791, "width": 290, "height": 62 }, + "enabled": true + }, + { + "depth": 38, + "type": "_UILiquidLensView", + "frame": { "x": 25, "y": 795, "width": 76, "height": 54 }, + "enabled": true + }, + { + "depth": 39, + "type": "UIView", + "frame": { "x": 25, "y": 795, "width": 76, "height": 54 }, + "enabled": true + }, + { + "depth": 40, + "type": "_UITabSelectionView", + "frame": { "x": 25, "y": 795, "width": 76, "height": 54 }, + "enabled": true + }, + { + "depth": 40, + "type": "ClearGlassView", + "frame": { "x": 25, "y": 795, "width": 76, "height": 54 }, + "enabled": true + }, + { + "depth": 41, + "type": "UIView", + "frame": { "x": 25, "y": 795, "width": 76, "height": 54 }, + "enabled": true + }, + { + "depth": 42, + "type": "SDFView", + "frame": { "x": 25, "y": 795, "width": 76, "height": 54 }, + "enabled": true + }, + { + "depth": 43, + "type": "SDFElementView", + "frame": { "x": 25, "y": 795, "width": 76, "height": 54 }, + "enabled": true + }, + { + "depth": 42, + "type": "_UIPortalView", + "frame": { "x": 25, "y": 795, "width": 76, "height": 54 }, + "enabled": true + }, + { + "depth": 41, + "type": "UIView", + "frame": { "x": 25, "y": 795, "width": 76, "height": 54 }, + "enabled": true + }, + { + "depth": 37, + "type": "UIKit._UITabBarAuxiliaryView", + "frame": { "x": 319, "y": 791, "width": 62, "height": 62 }, + "enabled": true + }, + { + "depth": 38, + "type": "UIKit._UITabBarItemPlatterView", + "frame": { "x": 319, "y": 791, "width": 62, "height": 62 }, + "enabled": true + }, + { + "depth": 39, + "type": "ContentView", + "frame": { "x": 319, "y": 791, "width": 62, "height": 62 }, + "enabled": true + }, + { + "depth": 40, + "type": "Button", + "label": "Trade", + "frame": { "x": 319, "y": 791, "width": 62, "height": 62 }, + "enabled": true + }, + { + "depth": 39, + "type": "BadgeContainerView", + "frame": { "x": 319, "y": 791, "width": 62, "height": 62 }, + "enabled": true + }, + { + "depth": 35, + "type": "UIKit._GlassGroupView", + "frame": { "x": 0, "y": 791, "width": 402, "height": 83 }, + "enabled": true + }, + { + "depth": 36, + "type": "GlassGroupLayerView", + "frame": { "x": 0, "y": 791, "width": 402, "height": 83 }, + "enabled": true + }, + { + "depth": 37, + "type": "UIKit._UIMaterialDefinitionView", + "frame": { "x": 0, "y": 791, "width": 402, "height": 83 }, + "enabled": true + }, + { + "depth": 38, + "type": "UIKit.UISDFBackdropView", + "frame": { "x": 0, "y": 791, "width": 402, "height": 83 }, + "enabled": true + }, + { + "depth": 39, + "type": "UIKit.UISDFView", + "frame": { "x": 0, "y": 791, "width": 402, "height": 83 }, + "enabled": true + }, + { + "depth": 40, + "type": "UIView", + "frame": { "x": 0, "y": 791, "width": 402, "height": 83 }, + "enabled": true + }, + { + "depth": 38, + "type": "UIKit.UISDFView", + "frame": { "x": 0, "y": 791, "width": 402, "height": 83 }, + "enabled": true + }, + { + "depth": 39, + "type": "_UIPortalView", + "frame": { "x": 0, "y": 791, "width": 402, "height": 83 }, + "enabled": true + }, + { + "depth": 27, + "type": "Toolbar", + "label": "Toolbar", + "identifier": "Toolbar", + "frame": { "x": 0, "y": 0, "width": 402, "height": 874 }, + "enabled": true + }, + { + "depth": 28, + "type": "FloatingBarHostingView", + "frame": { "x": 0, "y": 0, "width": 402, "height": 874 }, + "enabled": true + }, + { + "depth": 20, + "type": "Toolbar", + "label": "Toolbar", + "identifier": "Toolbar", + "frame": { "x": 0, "y": 0, "width": 402, "height": 874 }, + "enabled": true + }, + { + "depth": 21, + "type": "FloatingBarHostingView", + "frame": { "x": 0, "y": 0, "width": 402, "height": 874 }, + "enabled": true + }, + { + "depth": 13, + "type": "Toolbar", + "label": "Toolbar", + "identifier": "Toolbar", + "frame": { "x": 0, "y": 0, "width": 402, "height": 874 }, + "enabled": true + }, + { + "depth": 14, + "type": "FloatingBarHostingView", + "frame": { "x": 0, "y": 0, "width": 402, "height": 874 }, + "enabled": true + }, + { + "depth": 11, + "type": "RCTViewComponentView", + "frame": { "x": 129, "y": 365, "width": 144, "height": 144 }, + "enabled": true + }, + { + "depth": 8, + "type": "RCTViewComponentView", + "frame": { "x": 0, "y": 0, "width": 402, "height": 874 }, + "enabled": true + }, + { + "depth": 9, + "type": "RCTDebuggingOverlayComponentView", + "frame": { "x": 0, "y": 0, "width": 402, "height": 874 }, + "enabled": true + }, + { + "depth": 10, + "type": "RCTDebuggingOverlay", + "frame": { "x": 0, "y": 0, "width": 402, "height": 874 }, + "enabled": true + }, + { + "depth": 1, + "type": "Application", + "frame": { "x": 0, "y": 0, "width": 402, "height": 874 }, + "enabled": true + }, + { + "depth": 2, + "type": "UITrackingWindowView", + "frame": { "x": 0, "y": 0, "width": 402, "height": 874 }, + "enabled": true + }, + { + "depth": 3, + "type": "UIEditingOverlayGestureView", + "frame": { "x": 0, "y": 0, "width": 402, "height": 874 }, + "enabled": true + }, + { + "depth": 1, + "type": "Application", + "frame": { "x": 0, "y": 0, "width": 402, "height": 874 }, + "enabled": true + }, + { + "depth": 2, + "type": "UITransitionView", + "frame": { "x": 0, "y": 0, "width": 402, "height": 874 }, + "enabled": true + }, + { + "depth": 3, + "type": "UIDropShadowView", + "frame": { "x": 0, "y": 0, "width": 402, "height": 874 }, + "enabled": true + }, + { + "depth": 4, + "type": "_UIHostingView", + "frame": { "x": 0, "y": 0, "width": 402, "height": 874 }, + "enabled": true + }, + { + "depth": 5, + "type": "SwiftUI.AccessibilityNode", + "frame": { "x": 0, "y": 0, "width": 402, "height": 874 }, + "enabled": true + }, + { + "depth": 6, + "type": "Image", + "label": "Gear Shape, Filled", + "identifier": "gearshape.fill", + "frame": { "x": 348, "y": 96, "width": 26, "height": 26 }, + "enabled": true + } +] diff --git a/src/platform/mobile-hierarchy.test.ts b/src/platform/mobile-hierarchy.test.ts index 57ffdc9..d2951c8 100644 --- a/src/platform/mobile-hierarchy.test.ts +++ b/src/platform/mobile-hierarchy.test.ts @@ -4,9 +4,14 @@ import { describe, it, expect } from 'vitest'; import androidLoginSnapshot from './__fixtures__/android-login-snapshot.json'; import { filterAppHierarchy, + isActionableElement, + isLayoutOnly, + isUnlabelledTapTarget, + pruneLayoutOnly, resourceIdPackage, viewportFromHierarchy, } from './mobile-hierarchy.js'; +import { isBusyNode } from '../server/screen-settle.js'; function el( type: string, @@ -512,6 +517,286 @@ describe('filterAppHierarchy', () => { }); }); +describe('isActionableElement', () => { + it.each([ + 'Button', + 'Link', + 'TextField', + 'SecureTextField', + 'SearchField', + 'TextView', + 'Switch', + 'Slider', + 'Cell', + 'Tab', + ])('treats iOS %s as actionable', (type) => { + expect(isActionableElement(el(type), 'ios')).toBe(true); + }); + + it.each([ + 'UIView', + 'RCTViewComponentView', + 'UIKit._UITabBarItemPlatterView', + 'TabBar', + 'Toolbar', + 'StaticText', + 'Image', + 'Other', + // Assumes the backend strips the XCUITest prefix (device-mcp Appium + // backend does): a prefixed type is not matched. + 'XCUIElementTypeButton', + 'XCUIElementTypeTextField', + ])('does not treat iOS %s as actionable', (type) => { + expect(isActionableElement(el(type), 'ios')).toBe(false); + }); + + it.each([ + 'android.widget.Button', + 'android.widget.ImageButton', + 'android.widget.RadioButton', + 'android.widget.ToggleButton', + 'com.google.android.material.button.MaterialButton', + 'android.widget.EditText', + 'androidx.appcompat.widget.AppCompatEditText', + 'android.widget.CheckBox', + 'android.widget.Switch', + 'androidx.appcompat.widget.SwitchCompat', + 'android.widget.SeekBar', + 'android.widget.Spinner', + 'android.widget.CheckedTextView', + 'android.widget.RatingBar', + ])('treats Android %s as actionable', (type) => { + expect(isActionableElement(el(type), 'android')).toBe(true); + }); + + it.each([ + 'android.view.ViewGroup', + 'android.widget.FrameLayout', + 'android.widget.TextView', + 'android.widget.ImageView', + 'com.horcrux.svg.SvgView', + ])('does not treat Android %s as actionable', (type) => { + expect(isActionableElement(el(type), 'android')).toBe(false); + }); +}); + +describe('isLayoutOnly', () => { + it('is true for an element with no name, testId or control type', () => { + expect(isLayoutOnly(el('android.view.ViewGroup'), 'android')).toBe(true); + expect(isLayoutOnly(el('RNSScreenView'), 'ios')).toBe(true); + }); + + it.each([ + ['a label', el('UIView', { label: 'Balance' })], + ['a value', el('UIView', { value: '0.5 ETH' })], + ['a testId', el('UIView', { identifier: 'wallet-screen' })], + ['a control type', el('SecureTextField')], + ['a progress indicator type', el('UIActivityIndicatorView')], + ])('is false for an element with %s', (_case, element) => { + expect(isLayoutOnly(element, 'ios')).toBe(false); + }); + + it('is false for every progress indicator the screen settle detects', () => { + // Same MOBILE_BUSY_TYPE as isBusyNode (server/screen-settle.ts). + for (const type of [ + 'android.widget.ProgressBar', + 'androidx.core.widget.ContentLoadingProgressBar', + ]) { + expect(isLayoutOnly(el(type), 'android')).toBe(false); + expect(isBusyNode({ ref: 'e1', role: type, name: '' })).toBe(true); + } + for (const type of [ + 'ActivityIndicator', + 'ProgressIndicator', + 'UIActivityIndicatorView', + ]) { + expect(isLayoutOnly(el(type), 'ios')).toBe(false); + expect(isBusyNode({ ref: 'e1', role: type, name: '' })).toBe(true); + } + }); +}); + +describe('isUnlabelledTapTarget', () => { + const frame = (width: number, height: number) => ({ + frame: { x: 0, y: 0, width, height }, + }); + + it.each([ + ['ios', 'RCTViewComponentView'], + ['android', 'android.view.ViewGroup'], + ] as const)('accepts a %s %s of at least 24x24', (platform, type) => { + expect(isUnlabelledTapTarget(el(type, frame(24, 24)), platform)).toBe(true); + expect(isUnlabelledTapTarget(el(type, frame(87, 73)), platform)).toBe(true); + }); + + it.each([ + [23, 48], + [48, 23], + [0, 0], + ])('rejects a %sx%s frame', (width, height) => { + expect( + isUnlabelledTapTarget( + el('RCTViewComponentView', frame(width, height)), + 'ios', + ), + ).toBe(false); + }); + + it.each([ + ['ios', 'UIView'], + ['ios', 'android.view.ViewGroup'], + ['android', 'android.widget.FrameLayout'], + ['android', 'RCTViewComponentView'], + ] as const)('rejects a %s %s', (platform, type) => { + expect(isUnlabelledTapTarget(el(type, frame(100, 100)), platform)).toBe( + false, + ); + }); +}); + +describe('pruneLayoutOnly', () => { + it('keeps an unlabelled RN view that is a leaf after pruning, without its children', () => { + const icon = el( + 'RCTViewComponentView', + { frame: { x: 16, y: 242, width: 87, height: 73 } }, + [el('RNSVGGroup', {}, [el('RNSVGPath')])], + ); + + const pruned = pruneLayoutOnly([el('UIView', {}, [icon])], 'ios'); + + expect(pruned).toStrictEqual([ + el('RCTViewComponentView', { + frame: { x: 16, y: 242, width: 87, height: 73 }, + }), + ]); + }); + + it('hoists the kept children of an unlabelled RN view instead of keeping it', () => { + const text = el('android.widget.TextView', { value: 'Buy' }); + const pruned = pruneLayoutOnly( + [ + el( + 'android.view.ViewGroup', + { frame: { x: 0, y: 0, width: 200, height: 200 } }, + [text], + ), + ], + 'android', + ); + + expect(pruned).toStrictEqual([text]); + }); + + it('hoists the kept descendants of a layout-only element, in order', () => { + const pruned = pruneLayoutOnly( + [ + el('UIView', {}, [ + el('StaticText', { label: 'A' }), + el('UIView', {}, [el('StaticText', { label: 'B' })]), + el('StaticText', { label: 'C' }), + ]), + el('StaticText', { label: 'D' }), + ], + 'ios', + ); + + expect(pruned.map((node) => node.label)).toStrictEqual([ + 'A', + 'B', + 'C', + 'D', + ]); + }); + + it('drops a layout-only subtree with no kept element', () => { + expect( + pruneLayoutOnly( + [el('UIView', {}, [el('UIView', {}, [el('UIView')])])], + 'ios', + ), + ).toStrictEqual([]); + }); + + it('keeps a container with a testId and nests its kept descendants', () => { + const button = el('android.widget.Button', { label: 'Send' }); + const pruned = pruneLayoutOnly( + [ + el('android.view.ViewGroup', { identifier: 'actions' }, [ + el('android.view.ViewGroup', {}, [button]), + el('android.view.ViewGroup'), + ]), + ], + 'android', + ); + + expect(pruned).toStrictEqual([ + el('android.view.ViewGroup', { identifier: 'actions' }, [button]), + ]); + expect(pruned[0]?.children?.[0]).toBe(button); + }); + + it('removes the children key of a kept element whose children are all dropped', () => { + const pruned = pruneLayoutOnly( + [el('Button', { label: 'OK' }, [el('UIView')])], + 'ios', + ); + + expect(pruned).toStrictEqual([el('Button', { label: 'OK' })]); + expect(pruned[0]).not.toHaveProperty('children'); + }); + + it('keeps an unnamed control and a disabled element', () => { + const pruned = pruneLayoutOnly( + [ + el('UIView', {}, [ + el('TextField'), + el('Button', { label: 'Next', enabled: false }), + ]), + ], + 'ios', + ); + + expect(pruned).toStrictEqual([ + el('TextField'), + el('Button', { label: 'Next', enabled: false }), + ]); + }); + + it('returns unchanged subtrees by reference and does not modify the input', () => { + const row = el('android.view.ViewGroup', { identifier: 'row' }, [ + el('android.widget.TextView', { value: 'Hi' }), + ]); + const snapshot = [el('android.widget.FrameLayout', {}, [row])]; + const before = JSON.stringify(snapshot); + + const pruned = pruneLayoutOnly(snapshot, 'android'); + + expect(JSON.stringify(snapshot)).toBe(before); + expect(pruned).toStrictEqual([row]); + expect(pruned[0]).toBe(row); + }); + + it('keeps the named, identified, actionable and tap-target elements of a real Android login snapshot', () => { + const app = filterAppHierarchy(androidLoginSnapshot as UIElement[], { + platform: 'android', + }); + const count = (elements: UIElement[]): number => + elements.reduce( + (total, node) => total + 1 + count(node.children ?? []), + 0, + ); + + const pruned = pruneLayoutOnly(app, 'android'); + + expect(count(app)).toBe(47); + // 12 named, identified or actionable elements, plus 3 unlabelled + // ViewGroup leaves kept as tap targets. + expect(count(pruned)).toBe(15); + expect(identifiers(pruned)).toStrictEqual(identifiers(app)); + expect(labels(pruned)).toStrictEqual(labels(app)); + }); +}); + describe('viewportFromHierarchy', () => { it('reads the iOS viewport from the first element', () => { expect(viewportFromHierarchy(iosSnapshot(), 'ios')).toStrictEqual({ diff --git a/src/platform/mobile-hierarchy.ts b/src/platform/mobile-hierarchy.ts index d73d9b2..a0342a8 100644 --- a/src/platform/mobile-hierarchy.ts +++ b/src/platform/mobile-hierarchy.ts @@ -1,5 +1,10 @@ import type { UIElement } from '@metamask/device-mcp'; +import { + MOBILE_BUSY_TYPE, + MOBILE_TAP_TARGET_MIN_SIZE, +} from '../tools/utils/constants.js'; + /** Options of `filterAppHierarchy`. */ export type AppHierarchyFilterOptions = { /** Device platform of the snapshot. */ @@ -151,6 +156,184 @@ export function filterAppHierarchy( return walk(roots, false); } +/** + * iOS element types (idb `describe-all` AX types, also used by Appium + * XCUITest) of controls a user can tap or type into. Matched exactly: the + * nested idb format reports UIKit class names + * (`UIKit._UITabBarItemPlatterView`, `RNSTabsHostComponentView`) for the + * views that are not accessibility elements, and a substring match would + * keep those. Assumes the backend strips the `XCUIElementType` prefix (the + * device-mcp Appium backend does): `XCUIElementTypeButton` is not matched. + */ +const IOS_ACTIONABLE_TYPES = new Set([ + 'Button', + 'Cell', + 'CheckBox', + 'ComboBox', + 'DatePicker', + 'Incrementor', + 'Key', + 'Link', + 'MenuItem', + 'Picker', + 'PickerWheel', + 'PopUpButton', + 'RadioButton', + 'SearchField', + 'SecureTextField', + 'SegmentedControl', + 'Slider', + 'Stepper', + 'Switch', + 'Tab', + 'TextArea', + 'TextField', + 'TextView', + 'Toggle', +]); + +/** + * Android widget classes of controls a user can tap or type into, matched as + * a suffix of the class name so subclasses match too (`Button` also matches + * `ImageButton`, `RadioButton`, `ToggleButton`, `MaterialButton`; + * `EditText` matches `AppCompatEditText`). The device snapshot has no + * `clickable` attribute, so a tappable `ViewGroup` (React Native + * `Pressable`) is kept only through its testId or label. + */ +const ANDROID_ACTIONABLE_CLASS = + /(?:Button|EditText|AutoCompleteTextView|CheckBox|CheckedTextView|Switch|SwitchCompat|SwitchMaterial|SeekBar|Spinner|RatingBar)$/u; + +/** + * Plain React Native view classes: `View` and `Pressable` render as these. + * An unlabelled one with no children is often a tap target (an icon button + * whose label is on a sibling, or an image), so it is kept when large enough + * (see `isUnlabelledTapTarget`). + */ +const RN_VIEW_TYPES: Record<'ios' | 'android', ReadonlySet> = { + ios: new Set(['RCTViewComponentView']), + android: new Set(['android.view.ViewGroup']), +}; + +/** + * @param el - A snapshot element. + * @param platform - Device platform of the snapshot. + * @returns True when the element type is a control a user can tap or type + * into (see `IOS_ACTIONABLE_TYPES`, `ANDROID_ACTIONABLE_CLASS`). + */ +export function isActionableElement( + el: UIElement, + platform: 'ios' | 'android', +): boolean { + return platform === 'ios' + ? IOS_ACTIONABLE_TYPES.has(el.type) + : ANDROID_ACTIONABLE_CLASS.test(el.type); +} + +/** + * @param el - A snapshot element. + * @param platform - Device platform of the snapshot. + * @returns True when the element only lays out other elements: it has no + * name (label or value), no identifier (testId), is not actionable and is + * not a progress indicator. + */ +export function isLayoutOnly( + el: UIElement, + platform: 'ios' | 'android', +): boolean { + return ( + !el.label && + !el.value && + !el.identifier && + !isActionableElement(el, platform) && + !MOBILE_BUSY_TYPE.test(el.type) + ); +} + +/** + * @param el - A layout-only element (see `isLayoutOnly`) whose kept + * children were checked. + * @param platform - Device platform of the snapshot. + * @returns True when the element is a plain React Native view + * (`RCTViewComponentView`, `android.view.ViewGroup`) at least + * `MOBILE_TAP_TARGET_MIN_SIZE` wide and high. Call it only for an element + * with no kept children (a leaf after pruning). Such views are often + * unlabelled buttons (the wallet home action buttons on iOS), usable with + * `tap-coordinates`. + */ +export function isUnlabelledTapTarget( + el: UIElement, + platform: 'ios' | 'android', +): boolean { + return ( + RN_VIEW_TYPES[platform].has(el.type) && + el.frame.width >= MOBILE_TAP_TARGET_MIN_SIZE && + el.frame.height >= MOBILE_TAP_TARGET_MIN_SIZE + ); +} + +/** + * Removes the layout-only elements (see `isLayoutOnly`) from a snapshot + * hierarchy, for the accessibility tree. A removed element is replaced by + * its kept descendants, in order, so the pre-order of the kept elements does + * not change. Containers with a testId, named elements (text included), + * actionable controls and progress indicators stay. A layout-only element + * that is left with no kept children also stays when it is an unlabelled tap + * target (see `isUnlabelledTapTarget`), without its children. The input is + * not modified; unchanged subtrees are returned by reference. + * + * On real MetaMask snapshots (after `filterAppHierarchy`) this keeps 15 of + * 47 elements on the Android login screen and 67 of 177 on iOS wallet home + * (idb nested format). + * + * @param hierarchy - The snapshot hierarchy (after `filterAppHierarchy`). + * @param platform - Device platform of the snapshot. + * @returns The hierarchy without layout-only elements. + */ +export function pruneLayoutOnly( + hierarchy: UIElement[], + platform: 'ios' | 'android', +): UIElement[] { + const kept: UIElement[] = []; + for (const el of hierarchy) { + const children = el.children?.length + ? pruneLayoutOnly(el.children, platform) + : undefined; + if (isLayoutOnly(el, platform)) { + if (children?.length) { + kept.push(...children); + } else if (isUnlabelledTapTarget(el, platform)) { + kept.push(withoutChildren(el)); + } + continue; + } + const unchanged = + children === undefined || + (children.length === el.children?.length && + children.every((child, at) => child === el.children?.[at])); + if (unchanged) { + kept.push(el); + } else if (children.length) { + kept.push({ ...el, children }); + } else { + kept.push(withoutChildren(el)); + } + } + return kept; +} + +/** + * @param el - A snapshot element. + * @returns The element itself when it has no `children` key, else a copy + * without it. + */ +function withoutChildren(el: UIElement): UIElement { + if (!('children' in el)) { + return el; + } + const { children: _children, ...leaf } = el; + return leaf; +} + /** * Android: removes the status bar and navigation bar (see * `filterAppHierarchy`). diff --git a/src/platform/mobile-platform-driver.test.ts b/src/platform/mobile-platform-driver.test.ts index 34061a7..7219e44 100644 --- a/src/platform/mobile-platform-driver.test.ts +++ b/src/platform/mobile-platform-driver.test.ts @@ -2,6 +2,7 @@ import type { DeviceBackend, UIElement } from '@metamask/device-mcp'; import { describe, it, expect, vi, afterEach } from 'vitest'; import androidLoginSnapshot from './__fixtures__/android-login-snapshot.json'; +import iosWalletHomePreorder from './__fixtures__/ios-wallet-home-preorder.json'; import { MobilePlatformDriver } from './mobile-platform-driver.js'; function makeElement(overrides: Partial = {}): UIElement { @@ -67,6 +68,35 @@ function createMockBackend( }; } +/** + * The real iOS wallet home snapshot (idb nested format, device-mcp P1-26, + * proof/p1-26/ios/home-describe-screen.txt, amounts sanitized). The fixture + * stores the elements in pre-order with their depth, to keep the file small; + * this rebuilds the nested hierarchy. It is the raw (intentionally + * unpruned) snapshot: the DevLauncher window and all layout-only elements + * are in it, so the tests exercise `filterAppHierarchy` and + * `pruneLayoutOnly` end to end. + * + * @returns The hierarchy. + */ +function iosWalletHomeSnapshot(): UIElement[] { + const roots: UIElement[] = []; + const stack: UIElement[] = []; + for (const { depth, ...element } of iosWalletHomePreorder as (UIElement & { + depth: number; + })[]) { + stack.length = depth; + const parent = stack[depth - 1]; + if (parent) { + parent.children = [...(parent.children ?? []), element]; + } else { + roots.push(element); + } + stack.push(element); + } + return roots; +} + describe('MobilePlatformDriver', () => { afterEach(() => { vi.restoreAllMocks(); @@ -271,8 +301,10 @@ describe('MobilePlatformDriver', () => { expect( screen.nodes.flatMap((node) => (node.testId ? [node.testId] : [])), ).toStrictEqual(testIds.map((item) => item.testId)); + // The unnamed app window is layout-only (P1-14): the first node is + // its first child with a testId. expect(screen.nodes[0]?.ref).toBe('e1'); - expect(screen.nodes[0]?.role).toBe('android.widget.FrameLayout'); + expect(screen.nodes[0]?.role).toBe('android.widget.LinearLayout'); }); it('drops the iOS DevLauncher overlay by default', async () => { @@ -803,7 +835,6 @@ describe('MobilePlatformDriver', () => { ref: 'e1', role: 'Button', name: 'Submit', - path: [], testId: 'submit-btn', bounds: { x: 0, y: 0, width: 100, height: 44 }, }); @@ -811,7 +842,6 @@ describe('MobilePlatformDriver', () => { ref: 'e2', role: 'TextField', name: 'Email', - path: [], disabled: true, textContent: 'user@test.com', bounds: { x: 0, y: 0, width: 100, height: 44 }, @@ -842,7 +872,7 @@ describe('MobilePlatformDriver', () => { expect(nodes[0].bounds).toBeUndefined(); }); - it('assigns sequential refs to nested children', async () => { + it('assigns sequential refs to nested children, with no path', async () => { const backend = createMockBackend({ snapshot: vi.fn().mockResolvedValue({ platform: 'ios', @@ -864,13 +894,401 @@ describe('MobilePlatformDriver', () => { const { nodes } = await driver.getAccessibilityTree(); - expect(nodes).toHaveLength(3); - expect(nodes[0].ref).toBe('e1'); - expect(nodes[0].path).toStrictEqual([]); - expect(nodes[1].ref).toBe('e2'); - expect(nodes[1].path).toStrictEqual(['Window']); - expect(nodes[2].ref).toBe('e3'); - expect(nodes[2].path).toStrictEqual(['Window']); + expect(nodes.map((node) => node.ref)).toStrictEqual(['e1', 'e2', 'e3']); + for (const node of nodes) { + expect(node).not.toHaveProperty('path'); + } + }); + + it.each(['ios', 'android'] as const)( + 'emits no path on %s', + async (platform) => { + const button = platform === 'ios' ? 'Button' : 'android.widget.Button'; + const backend = createMockBackend({ + platform, + snapshot: vi.fn().mockResolvedValue({ + platform, + hierarchy: [ + makeElement({ + type: 'Container', + identifier: 'root', + children: [ + makeElement({ + type: 'Container', + identifier: 'row', + children: [ + makeElement({ type: button, label: 'OK' }), + makeElement({ type: button, label: 'Cancel' }), + ], + }), + ], + }), + ], + raw: '[]', + timestamp: Date.now(), + }), + }); + const driver = new MobilePlatformDriver(backend); + + const screen = await driver.captureScreen(); + const tree = await driver.getAccessibilityTree(); + + expect(tree.nodes).toHaveLength(4); + for (const node of [...tree.nodes, ...screen.nodes]) { + expect(node).not.toHaveProperty('path'); + } + }, + ); + + describe('layout-only nodes (P1-14)', () => { + const snapshotWith = ( + platform: 'ios' | 'android', + hierarchy: UIElement[], + ): DeviceBackend => + createMockBackend({ + platform, + snapshot: vi.fn().mockResolvedValue({ + platform, + hierarchy, + raw: '[]', + timestamp: Date.now(), + }), + }); + + it('drops unnamed containers and hoists their children in order', async () => { + const driver = new MobilePlatformDriver( + snapshotWith('android', [ + makeElement({ + type: 'android.widget.FrameLayout', + children: [ + makeElement({ + type: 'android.view.ViewGroup', + children: [ + makeElement({ + type: 'android.widget.TextView', + value: 'A', + }), + makeElement({ + type: 'android.view.ViewGroup', + children: [ + makeElement({ + type: 'android.widget.TextView', + value: 'B', + }), + ], + }), + ], + }), + makeElement({ type: 'android.widget.TextView', value: 'C' }), + // Unnamed, no testId, no kept descendant, and too small + // to be a tap target (a 1 px divider). + makeElement({ + type: 'android.view.ViewGroup', + frame: { x: 0, y: 0, width: 1080, height: 1 }, + children: [makeElement({ type: 'android.view.View' })], + }), + ], + }), + ]), + ); + + const { nodes, refMap } = await driver.getAccessibilityTree(); + + expect(nodes.map(({ ref, name }) => [ref, name])).toStrictEqual([ + ['e1', 'A'], + ['e2', 'B'], + ['e3', 'C'], + ]); + expect([...refMap]).toStrictEqual([ + ['e1', 'value:A|type:android.widget.TextView'], + ['e2', 'value:B|type:android.widget.TextView'], + ['e3', 'value:C|type:android.widget.TextView'], + ]); + }); + + it('keeps containers with a testId, actionable controls, text and disabled state', async () => { + const driver = new MobilePlatformDriver( + snapshotWith('ios', [ + makeElement({ + type: 'UIView', + children: [ + makeElement({ + type: 'RCTViewComponentView', + identifier: 'wallet-screen', + children: [ + makeElement({ type: 'StaticText', label: 'Balance' }), + makeElement({ type: 'SecureTextField', enabled: false }), + makeElement({ type: 'UIActivityIndicatorView' }), + ], + }), + ], + }), + ]), + ); + + const { nodes, refMap } = await driver.getAccessibilityTree(); + + expect(nodes).toStrictEqual([ + expect.objectContaining({ + ref: 'e1', + role: 'RCTViewComponentView', + testId: 'wallet-screen', + }), + expect.objectContaining({ ref: 'e2', name: 'Balance' }), + expect.objectContaining({ + ref: 'e3', + role: 'SecureTextField', + name: '', + disabled: true, + }), + expect.objectContaining({ + ref: 'e4', + role: 'UIActivityIndicatorView', + }), + ]); + // Only nodes with a testId, label or value can be resolved. + expect([...refMap.keys()]).toStrictEqual(['e1', 'e2']); + }); + + it('gives a dropped node no ref, so every ref resolves', async () => { + const backend = snapshotWith('android', [ + makeElement({ + type: 'android.view.ViewGroup', + children: [ + makeElement({ + type: 'android.widget.Button', + identifier: 'send', + label: 'Send', + }), + ], + }), + ]); + const driver = new MobilePlatformDriver(backend); + + const { nodes, refMap } = await driver.getAccessibilityTree(); + + expect(nodes.map((node) => node.ref)).toStrictEqual(['e1']); + expect([...refMap]).toStrictEqual([['e1', 'identifier:send']]); + await driver.click('a11yRef', 'e1', refMap, 1000); + expect(backend.tapElement).toHaveBeenCalledWith({ identifier: 'send' }); + await expect( + driver.click('a11yRef', 'e2', refMap, 1000), + ).rejects.toThrowError('Unknown a11yRef: e2'); + }); + + it('marks ambiguity over the kept nodes only', async () => { + const driver = new MobilePlatformDriver( + snapshotWith('ios', [ + makeElement({ + type: 'UIView', + children: [ + makeElement({ type: 'Button', label: 'OK' }), + makeElement({ + type: 'UIView', + children: [makeElement({ type: 'Button', label: 'OK' })], + }), + makeElement({ type: 'Button', label: 'Cancel' }), + ], + }), + ]), + ); + + const { nodes } = await driver.getAccessibilityTree(); + + expect( + nodes.map(({ ref, name, ambiguous }) => [ref, name, ambiguous]), + ).toStrictEqual([ + ['e1', 'OK', true], + ['e2', 'OK', true], + ['e3', 'Cancel', undefined], + ]); + }); + + it('lists an unlabelled tap target with bounds but no ref', async () => { + const driver = new MobilePlatformDriver( + snapshotWith('ios', [ + makeElement({ + type: 'UIView', + children: [ + // A Pressable icon: an unlabelled RN view, 48x48. + makeElement({ + type: 'RCTViewComponentView', + frame: { x: 10, y: 20, width: 48, height: 48 }, + children: [makeElement({ type: 'RNSVGPath' })], + }), + // Too small: a spacer or divider. + makeElement({ + type: 'RCTViewComponentView', + frame: { x: 0, y: 80, width: 300, height: 1 }, + }), + // Not a plain RN view. + makeElement({ + type: 'UIView', + frame: { x: 0, y: 100, width: 300, height: 300 }, + }), + makeElement({ type: 'Button', label: 'Send' }), + ], + }), + ]), + ); + + const { nodes, refMap } = await driver.getAccessibilityTree(); + + expect(nodes).toStrictEqual([ + { + ref: 'e1', + role: 'RCTViewComponentView', + name: '', + bounds: { x: 10, y: 20, width: 48, height: 48 }, + }, + expect.objectContaining({ ref: 'e2', name: 'Send' }), + ]); + expect([...refMap]).toStrictEqual([['e2', 'label:Send|type:Button']]); + }); + + it('reduces the real Android login snapshot (47 -> 15 nodes)', async () => { + const driver = new MobilePlatformDriver( + snapshotWith('android', androidLoginSnapshot as UIElement[]), + ); + + const { nodes, refMap } = await driver.getAccessibilityTree(); + + // After filterAppHierarchy: 47 elements. 32 are layout-only + // (unnamed FrameLayout/View/ScrollView/WebView/small ViewGroups...); + // 3 unlabelled ViewGroup leaves are kept as tap targets (no ref). + expect( + nodes.map((node) => node.testId ?? (node.name || node.role)), + ).toStrictEqual([ + 'io.metamask:id/action_bar_root', + 'android:id/content', + 'android.view.ViewGroup', + 'npm:@metamask/bitcoin-wallet-snap', + 'android.view.ViewGroup', + 'login', + 'login-download-logs-button', + 'metamask-wordmark', + 'android.view.ViewGroup', + 'login-password-input', + 'log-in-button', + 'Unlock', + 'reset-wallet-button', + 'Forgot password?', + 'fox-animation', + ]); + expect(nodes.map((node) => node.ref)).toStrictEqual( + nodes.map((_node, index) => `e${index + 1}`), + ); + expect([...refMap.keys()]).toStrictEqual( + nodes + .filter((node) => node.testId ?? node.name) + .map((node) => node.ref), + ); + expect( + nodes + .filter((node) => !refMap.has(node.ref)) + .map((node) => node.bounds), + ).toStrictEqual([ + { x: 0, y: 0, width: 1080, height: 2400 }, + { x: 0, y: 0, width: 1080, height: 2400 }, + { x: 42, y: 1232, width: 996, height: 126 }, + ]); + // Before P1-14 (with path): 24154 bytes. Without path and layout-only + // nodes: about 2 kB. + expect(JSON.stringify(nodes).length).toBeLessThan(2500); + }); + + it('keeps the buttons, tabs and unlabelled action buttons of the real iOS wallet home (183 -> 67 nodes)', async () => { + const driver = new MobilePlatformDriver( + snapshotWith('ios', iosWalletHomeSnapshot()), + ); + + const { nodes, refMap } = await driver.getAccessibilityTree(); + + // 183 elements; 6 are the DevLauncher window (filterAppHierarchy). + // 57 have a testId or a name; 10 are unlabelled tap targets. + expect(nodes).toHaveLength(67); + expect( + nodes.filter((node) => node.role === 'Button').map((n) => n.name), + ).toStrictEqual([ + 'Open my profile', + 'Agent Account', + '$, $, (%), Today', + 'Money, $, %, % APY', + 'Tokens, $, %', + 'Perps, $, %', + 'Tokens', + 'Perps', + 'Predictions', + 'BTC Price: \u2014, Price to beat: \u2014, Live, --:--', + 'Watchlist', + 'Top traders', + 'DeFi', + 'Import a token', + 'Import an NFT', + 'Contact support', + 'Home', + 'Explore', + 'Money', + 'Social', + 'Trade', + ]); + // The 4 unlabelled action buttons (Buy, Swap, Send, Receive) right + // after `wallet-action-buttons`: listed with bounds, no ref. + const actions = nodes.findIndex( + (node) => node.testId === 'wallet-action-buttons', + ); + const actionButtons = nodes.slice(actions + 1, actions + 5); + expect( + actionButtons.map(({ role, name, bounds }) => ({ + role, + name, + x: Math.round(bounds?.x ?? -1), + y: bounds?.y, + })), + ).toStrictEqual([ + { role: 'RCTViewComponentView', name: '', x: 16, y: 242 }, + { role: 'RCTViewComponentView', name: '', x: 111, y: 242 }, + { role: 'RCTViewComponentView', name: '', x: 205, y: 242 }, + { role: 'RCTViewComponentView', name: '', x: 300, y: 242 }, + ]); + for (const button of actionButtons) { + expect(refMap.has(button.ref)).toBe(false); + } + // Every node with a testId or a name has a ref that resolves. + expect([...refMap.keys()]).toStrictEqual( + nodes + .filter((node) => node.testId ?? node.name) + .map((node) => node.ref), + ); + expect(refMap.size).toBe(57); + // P1-14 AC1: at most 15 kB (was 183 kB with path, 55 kB flat). + expect(JSON.stringify(nodes).length).toBeLessThan(11_000); + }); + + it('writes bounds with a fixed key order', async () => { + const driver = new MobilePlatformDriver( + snapshotWith('ios', [ + makeElement({ + type: 'Button', + label: 'OK', + // idb returns frame keys in varying order. + frame: { height: 44, width: 100, y: 20, x: 10 }, + }), + ]), + ); + + const { nodes } = await driver.getAccessibilityTree(); + + expect(Object.keys(nodes[0]?.bounds ?? {})).toStrictEqual([ + 'x', + 'y', + 'width', + 'height', + ]); + expect(JSON.stringify(nodes[0]?.bounds)).toBe( + '{"x":10,"y":20,"width":100,"height":44}', + ); + }); }); it('builds refMap with value fallback', async () => { @@ -1592,3 +2010,57 @@ describe('MobilePlatformDriver', () => { }); }); }); + +describe('MobilePlatformDriver refs across layout changes', () => { + it('keeps mobile refs when only layout-only containers change (P1-14)', async () => { + const element = ( + type: string, + extra: Partial = {}, + children?: UIElement[], + ): UIElement => ({ + type, + frame: { x: 0, y: 0, width: 10, height: 10 }, + enabled: true, + ...extra, + ...(children ? { children } : {}), + }); + const send = element('android.widget.Button', { + identifier: 'send', + label: 'Send', + }); + const swap = element('android.widget.Button', { label: 'Swap' }); + const readTree = async (hierarchy: UIElement[]) => + new MobilePlatformDriver({ + platform: 'android', + snapshot: async () => ({ + platform: 'android', + hierarchy, + raw: '', + timestamp: 0, + }), + } as unknown as DeviceBackend).getAccessibilityTree(); + + const previous = await readTree([ + element('android.view.ViewGroup', {}, [send, swap]), + ]); + // Two more unnamed wrappers, and an unnamed zero-height spacer. + const next = await readTree([ + element('android.widget.FrameLayout', {}, [ + element('android.view.ViewGroup', {}, [ + element('android.view.ViewGroup', {}, [send]), + element('android.view.View', { + frame: { x: 0, y: 0, width: 10, height: 0 }, + }), + swap, + ]), + ]), + ]); + + expect(next.nodes).toStrictEqual(previous.nodes); + expect([...next.refMap]).toStrictEqual([...previous.refMap]); + expect(previous.nodes.map((node) => node.path)).toStrictEqual([ + undefined, + undefined, + ]); + }); +}); diff --git a/src/platform/mobile-platform-driver.ts b/src/platform/mobile-platform-driver.ts index 1933ccc..0dfc37a 100644 --- a/src/platform/mobile-platform-driver.ts +++ b/src/platform/mobile-platform-driver.ts @@ -16,6 +16,7 @@ import type { import { filterAppHierarchy, + pruneLayoutOnly, viewportFromHierarchy, } from './mobile-hierarchy.js'; import type { Viewport } from './mobile-hierarchy.js'; @@ -226,7 +227,7 @@ export class MobilePlatformDriver implements IPlatformDriver { _rootSelector?: string, ): Promise<{ nodes: A11yNodeTrimmed[]; refMap: Map }> { const { hierarchy } = await this.#snapshotApp(); - return normalizeSnapshot(hierarchy); + return normalizeSnapshot(hierarchy, this.#backend.platform); } /** @@ -249,7 +250,7 @@ export class MobilePlatformDriver implements IPlatformDriver { async captureScreen(): Promise { const { hierarchy, viewport } = await this.#snapshotApp(); return { - ...normalizeSnapshot(hierarchy), + ...normalizeSnapshot(hierarchy, this.#backend.platform), getTestIds: async (limit?: number) => this.#testIdsOf(hierarchy, viewport, limit), }; @@ -846,12 +847,32 @@ function parseStableIdentifier(stableId: string): ElementQuery { } /** - * Normalizes a UIElement hierarchy into A11yNodeTrimmed nodes with sequential refs. + * Normalizes a UIElement hierarchy into a flat, pre-order list of + * A11yNodeTrimmed nodes with sequential refs. * - * @param hierarchy - The raw UIElement tree from a device snapshot. + * Layout-only elements (no name, no testId, not actionable, see + * `pruneLayoutOnly`) are removed first, so they get no node, and node refs + * number only the kept nodes. Nodes have no `path`: on mobile the ancestors + * are native view classes (`android.widget.FrameLayout`, `RNSScreenView`) + * with no targeting value, and the ancestor list was most of the payload. + * Targeting is by `ref` or `testId`. + * + * Only a node with a testId, label or value gets a refMap entry, so only + * its ref can be targeted. Kept nodes with none of them are listed (with + * `bounds`, for `tap-coordinates`) but their ref does not resolve: + * unlabelled tap targets (see `isUnlabelledTapTarget`), progress indicators + * (kept for the screen settle) and unnamed controls. Ambiguity is computed + * over the kept nodes. + * + * @param hierarchy - The UIElement tree from a device snapshot (after + * `filterAppHierarchy`). + * @param platform - The device platform. * @returns Trimmed nodes and the ref-to-stable-identifier map. */ -function normalizeSnapshot(hierarchy: UIElement[]): { +function normalizeSnapshot( + hierarchy: UIElement[], + platform: 'ios' | 'android', +): { nodes: A11yNodeTrimmed[]; refMap: Map; } { @@ -862,16 +883,15 @@ function normalizeSnapshot(hierarchy: UIElement[]): { /** * @param elements - UIElement nodes to walk. - * @param path - Accumulated ancestor roles. */ - function walk(elements: UIElement[], path: string[]): void { + function walk(elements: UIElement[]): void { for (const el of elements) { refCounter += 1; const ref = `e${refCounter}`; const role = el.type || 'element'; const name = el.label ?? el.value ?? ''; - const node: A11yNodeTrimmed = { ref, role, name, path }; + const node: A11yNodeTrimmed = { ref, role, name }; if (!el.enabled) { node.disabled = true; } @@ -882,7 +902,10 @@ function normalizeSnapshot(hierarchy: UIElement[]): { node.textContent = el.value; } if (el.frame.width > 0 || el.frame.height > 0) { - node.bounds = { ...el.frame }; + // Fixed key order: idb returns frame keys in varying order, and + // screenSignature / diffs compare the serialized nodes. + const { x, y, width, height } = el.frame; + node.bounds = { x, y, width, height }; } nodes.push(node); @@ -903,12 +926,12 @@ function normalizeSnapshot(hierarchy: UIElement[]): { } if (el.children?.length) { - walk(el.children, [...path, role]); + walk(el.children); } } } - walk(hierarchy, []); + walk(pruneLayoutOnly(hierarchy, platform)); for (const refs of stableIdCount.values()) { if (refs.length > 1) { diff --git a/src/server/create-server.test.ts b/src/server/create-server.test.ts index 7db8f0a..45aed5b 100644 --- a/src/server/create-server.test.ts +++ b/src/server/create-server.test.ts @@ -2793,10 +2793,11 @@ describe('createServer abort, ref generations and observations', () => { // removed, so the P1-09 settle takes its one unchanged retry. expect(backend.snapshot).toHaveBeenCalledTimes(3); expect(body.observations?.a11y.nodes).toStrictEqual([]); + // The button only: the unnamed app window is a layout-only node (P1-14). expect(body.observations?.a11y.diff).toStrictEqual({ added: [], removed: [], - unchanged: 2, + unchanged: 1, }); }); diff --git a/src/server/observation-compaction.test.ts b/src/server/observation-compaction.test.ts index c88e857..6123fc5 100644 --- a/src/server/observation-compaction.test.ts +++ b/src/server/observation-compaction.test.ts @@ -67,6 +67,21 @@ describe('collapseOptionSubtrees', () => { }); }); + it('builds a summary without a path when the options have none (mobile)', () => { + const { path: _comboPath, ...combobox } = createNode('e1', 'combobox'); + const options = createOptionRun(5, 2).map( + ({ path: _path, ...option }) => option, + ); + + const result = collapseOptionSubtrees([combobox, ...options]); + + expect(result[1]).toStrictEqual({ + ref: 'e2\u2013e6', + role: 'option', + name: '5 options (refs e2\u2013e6)', + }); + }); + it('does not collapse runs below the threshold', () => { const combobox = createNode('e1', 'combobox'); const optionOne = createNode('e2', 'option'); @@ -343,6 +358,25 @@ describe('nodeChanged', () => { expect(nodeChanged(current, previous)).toBe(false); }); + it('treats two absent paths as equal (mobile nodes have no path)', () => { + const { path: _previousPath, ...previous } = createNode('e1', 'button', { + name: 'Send', + }); + const { path: _currentPath, ...current } = createNode('e1', 'button', { + name: 'Send', + }); + + expect(nodeChanged(current, previous)).toBe(false); + }); + + it('returns true when a path appears or disappears', () => { + const withPath = createNode('e1', 'button', { name: 'Send', path: [] }); + const { path: _path, ...withoutPath } = withPath; + + expect(nodeChanged(withPath, withoutPath)).toBe(true); + expect(nodeChanged(withoutPath, withPath)).toBe(true); + }); + it('does not compare refs', () => { const previous = createNode('e1', 'button', { name: 'Continue', @@ -358,6 +392,26 @@ describe('nodeChanged', () => { }); describe('diffObservation', () => { + it('diffs mobile nodes that have no path', () => { + const strip = ({ path: _path, ...node }: A11yNodeTrimmed) => node; + const send = strip(createNode('e1', 'button', { name: 'Send' })); + const balance = strip(createNode('e2', 'text', { name: '1 ETH' })); + const previous = createObservation([send, balance]); + const current = createObservation([ + { ...send }, + { ...balance, name: '2 ETH' }, + ]); + + const result = diffObservation(current, previous); + + expect(result.a11y.nodes).toStrictEqual([{ ...balance, name: '2 ETH' }]); + expect(result.a11y.diff).toStrictEqual({ + added: [], + removed: [], + unchanged: 1, + }); + }); + it('tracks added nodes and omits unchanged nodes from the diff payload', () => { const stable = createNode('e1', 'button', { name: 'Continue' }); const added = createNode('e2', 'button', { name: 'Cancel' }); diff --git a/src/server/observation-compaction.ts b/src/server/observation-compaction.ts index 7121811..6c3547f 100644 --- a/src/server/observation-compaction.ts +++ b/src/server/observation-compaction.ts @@ -55,12 +55,15 @@ function buildOptionSummary(nodes: A11yNodeTrimmed[]): A11yNodeTrimmed { ); const refRange = `${firstRange.firstRef}\u2013${lastRange.lastRef}`; - return { + const summary: A11yNodeTrimmed = { ref: refRange, role: 'option', name: `${optionCount} options (refs ${refRange})`, - path: nodes[0].path, }; + if (nodes[0].path) { + summary.path = nodes[0].path; + } + return summary; } export const observationCompactionDeps = { @@ -168,13 +171,21 @@ export function compactObservation( } /** - * Checks if two string arrays are equal. + * Checks if two optional string arrays are equal. Two absent arrays are + * equal; an absent array never equals a present one (even an empty one). * * @param left - First array to compare. * @param right - Second array to compare. - * @returns True if arrays have equal length and identical elements. + * @returns True if both are absent, or both have equal length and identical + * elements. */ -function arraysEqual(left: string[], right: string[]): boolean { +function arraysEqual( + left: string[] | undefined, + right: string[] | undefined, +): boolean { + if (!left || !right) { + return left === right; + } return ( left.length === right.length && left.every((val, idx) => val === right[idx]) ); diff --git a/src/server/screen-settle.ts b/src/server/screen-settle.ts index d7c8182..e39d636 100644 --- a/src/server/screen-settle.ts +++ b/src/server/screen-settle.ts @@ -1,4 +1,5 @@ import type { A11yNodeTrimmed } from '../tools/types/discovery.js'; +import { MOBILE_BUSY_TYPE } from '../tools/utils/constants.js'; /** * React Native adds ", busy" to the accessibility label of an element with @@ -7,23 +8,6 @@ import type { A11yNodeTrimmed } from '../tools/types/discovery.js'; */ const BUSY_LABEL = /,\s*busy$/iu; -/** - * Roles of progress indicators, matched as a suffix of the node role (the - * mobile driver uses the backend element type as the role, see - * `normalizeSnapshot`): - * - * - Android (adb/Appium `class`): `android.widget.ProgressBar` and - * subclasses whose class name ends in `ProgressBar`. - * - iOS via Appium XCUITest: `XCUIElementTypeActivityIndicator` and - * `XCUIElementTypeProgressIndicator`, reported without the - * `XCUIElementType` prefix (`ActivityIndicator`, `ProgressIndicator`). - * - iOS via idb `describe-all`: the element `type`, either the AX role - * without `AX` (`ProgressIndicator`) or the UIKit class name - * (`UIActivityIndicatorView`, used by React Native `ActivityIndicator`). - */ -const BUSY_ROLE = - /(?:ProgressBar|ActivityIndicator(?:View)?|ProgressIndicator)$/u; - /** * Builds a ref-independent signature of a screen, used to tell whether the * screen changed. @@ -38,13 +22,13 @@ export function screenSignature(nodes: A11yNodeTrimmed[]): string { /** * Checks whether a node shows that work is in progress: its label ends in * ", busy" (React Native busy state), or it is a progress indicator (see - * `BUSY_ROLE`). + * `MOBILE_BUSY_TYPE`). * * @param node - The accessibility node. * @returns True when the node is busy. */ export function isBusyNode(node: A11yNodeTrimmed): boolean { - return BUSY_LABEL.test(node.name) || BUSY_ROLE.test(node.role); + return BUSY_LABEL.test(node.name) || MOBILE_BUSY_TYPE.test(node.role); } /** diff --git a/src/tools/types/discovery.ts b/src/tools/types/discovery.ts index 85ea6b6..160e161 100644 --- a/src/tools/types/discovery.ts +++ b/src/tools/types/discovery.ts @@ -57,7 +57,11 @@ export type A11yNodeTrimmed = { disabled?: boolean; checked?: boolean; expanded?: boolean; - path: string[]; + /** + * Browser only: the `role:name` of the ancestor dialogs and headings. Not + * set on mobile, where the ancestors are native view classes. + */ + path?: string[]; testId?: string; textContent?: string; ambiguous?: boolean; diff --git a/src/tools/utils/constants.ts b/src/tools/utils/constants.ts index 825b207..2d6b087 100644 --- a/src/tools/utils/constants.ts +++ b/src/tools/utils/constants.ts @@ -93,3 +93,30 @@ export const TEXT_PREVIEW_MAX_LENGTH = 100; /** Minimum number of option nodes under a combobox/listbox to trigger collapsing */ export const OPTION_COLLAPSE_MIN_COUNT = 3; + +// ============================================================================ +// Mobile accessibility tree +// ============================================================================ + +/** + * Types of progress indicators, matched as a suffix of the element type + * (the mobile node `role`). Used by the screen settle to detect a busy + * screen, and by the mobile tree filter to keep these elements: + * + * - Android (adb/Appium `class`): `android.widget.ProgressBar` and + * subclasses whose class name ends in `ProgressBar`. + * - iOS via Appium XCUITest: `XCUIElementTypeActivityIndicator` and + * `XCUIElementTypeProgressIndicator`, reported without the + * `XCUIElementType` prefix (`ActivityIndicator`, `ProgressIndicator`). + * - iOS via idb `describe-all`: the element `type`, either the AX role + * without `AX` (`ProgressIndicator`) or the UIKit class name + * (`UIActivityIndicatorView`, used by React Native `ActivityIndicator`). + */ +export const MOBILE_BUSY_TYPE = + /(?:ProgressBar|ActivityIndicator(?:View)?|ProgressIndicator)$/u; + +/** + * Minimum frame size (width and height, in snapshot units: iOS points, + * Android pixels) of an unlabelled tap target kept in the mobile tree. + */ +export const MOBILE_TAP_TARGET_MIN_SIZE = 24; diff --git a/src/tools/utils/discovery.test.ts b/src/tools/utils/discovery.test.ts index 6430fc8..7feb8cf 100644 --- a/src/tools/utils/discovery.test.ts +++ b/src/tools/utils/discovery.test.ts @@ -951,6 +951,27 @@ describe('collectTrimmedA11ySnapshot', () => { expect(result.refMap.get('e5')).toBe('role=button[name="Submit" s]'); }); + it('keeps the dialog and heading ancestry in path on the browser', async () => { + const a11yTree = [ + '- main:', + ' - heading "Send" [level=2]', + ' - dialog "Transaction":', + ' - button "Confirm"', + ' - button "Confirm"', + ' - button "Confirm"', + ].join('\n'); + + const page = createMockPage({ a11ySnapshot: a11yTree }); + const result = await collectTrimmedA11ySnapshot(page); + + expect(result.nodes.map((node) => [node.name, node.path])).toStrictEqual([ + ['Send', ['heading:Send']], + ['Transaction', ['dialog:Transaction']], + ['Confirm', ['dialog:Transaction']], + ['\u2026 2 more "Confirm" (refs e3\u2013e5)', ['dialog:Transaction']], + ]); + }); + it('does not collapse nodes with same role and name but different paths', async () => { const a11yTree = [ '- main:', @@ -968,10 +989,10 @@ describe('collectTrimmedA11ySnapshot', () => { const result = await collectTrimmedA11ySnapshot(page); const dialogAButtons = result.nodes.filter( - (n) => n.role === 'button' && n.path.some((p) => p.includes('dialog:A')), + (n) => n.role === 'button' && n.path?.some((p) => p.includes('dialog:A')), ); const dialogBButtons = result.nodes.filter( - (n) => n.role === 'button' && n.path.some((p) => p.includes('dialog:B')), + (n) => n.role === 'button' && n.path?.some((p) => p.includes('dialog:B')), ); expect(dialogAButtons.length).toBeGreaterThanOrEqual(1); expect(dialogBButtons.length).toBeGreaterThanOrEqual(1); diff --git a/src/tools/utils/discovery.ts b/src/tools/utils/discovery.ts index 08f5312..2ac5b33 100644 --- a/src/tools/utils/discovery.ts +++ b/src/tools/utils/discovery.ts @@ -1109,13 +1109,21 @@ async function enrichNodesWithDOMContext( const COLLAPSE_THRESHOLD = 3; /** - * Checks whether two string arrays contain identical elements in order. + * Checks whether two optional string arrays (an a11y node `path`) contain + * identical elements in order. Two absent arrays are equal; an absent array + * never equals a present one, even an empty one. * * @param left - First array to compare. * @param right - Second array to compare. - * @returns True if both arrays are equal. + * @returns True if both arrays are absent or equal. */ -function arraysEqual(left: string[], right: string[]): boolean { +function arraysEqual( + left: string[] | undefined, + right: string[] | undefined, +): boolean { + if (!left || !right) { + return left === right; + } return ( left.length === right.length && left.every((val, idx) => val === right[idx]) );