diff --git a/src/__tests__/test-utils/android-ui-hierarchy-fixtures.ts b/src/__tests__/test-utils/android-ui-hierarchy-fixtures.ts new file mode 100644 index 000000000..dcf883527 --- /dev/null +++ b/src/__tests__/test-utils/android-ui-hierarchy-fixtures.ts @@ -0,0 +1,927 @@ +import { + buildUiHierarchySnapshot, + type AndroidUiHierarchy, +} from '../../platforms/android/ui-hierarchy.ts'; +import type { RawSnapshotNode } from '../../kernel/snapshot.ts'; + +/** + * Reconstructs the `AndroidUiHierarchy` tree `buildUiHierarchySnapshot` expects + * (see `platforms/android/ui-hierarchy.ts`) from a flat `RawSnapshotNode[]` + * fixture that carries `index`/`parentIndex` (the shape Android capture + * fixtures already use throughout the test suite). Only the fields the walker + * actually reads for its inclusion decisions + * (`shouldIncludeAndroidNode`/`shouldIncludeStructuralAndroidNode`) are + * threaded through: `type`, `identifier`, `label`, `value`, `hittable`, + * `visibleToUser`, `rect`, plus `bundleId` -> `packageName` (echoed back onto + * the walked output as `bundleId`, which the chrome/divergence classifiers key + * on). `depth`/`parentIndex` on the tree node itself are vestigial (the walker + * tracks depth and parent via its own recursion, never `node.depth`), so they + * are not populated here. + */ +export function rawFixtureToAndroidTree(rawNodes: RawSnapshotNode[]): AndroidUiHierarchy { + const root: AndroidUiHierarchy = { + type: null, + label: null, + value: null, + identifier: null, + packageName: null, + depth: -1, + children: [], + }; + const byIndex = new Map(); + for (const raw of rawNodes) { + byIndex.set(raw.index, { + type: raw.type ?? null, + label: raw.label ?? null, + value: raw.value ?? null, + identifier: raw.identifier ?? null, + packageName: raw.bundleId ?? null, + rect: raw.rect, + hittable: raw.hittable, + visibleToUser: raw.visibleToUser, + depth: 0, + children: [], + }); + } + for (const raw of rawNodes) { + const node = byIndex.get(raw.index); + if (!node) continue; + const parent = raw.parentIndex !== undefined ? byIndex.get(raw.parentIndex) : undefined; + (parent ?? root).children.push(node); + } + return root; +} + +/** + * Runs a `RawSnapshotNode[]` fixture through the REAL non-raw Android walk + * (`buildUiHierarchySnapshot(..., { raw: false })`), the same drop+reparent + * pass a device `snapshot` capture applies before nodes ever reach + * `collectSettleChromeRefs` / the replay divergence screen builder. Unlabeled, + * non-hittable, generic-id structural nodes (`shouldIncludeStructuralAndroidNode`) + * are dropped and their children re-parented upward — so fixtures built this + * way reproduce production's inclusion decisions exactly, instead of hand- + * simulating only the `status_bar*`/`navigation_bar*` wrapper removal. + */ +export function walkNonRawAndroidFixture(rawNodes: RawSnapshotNode[]): RawSnapshotNode[] { + const tree = rawFixtureToAndroidTree(rawNodes); + return buildUiHierarchySnapshot(tree, undefined, { raw: false }).nodes; +} + +/** + * Real device capture (checkout-form fixture app, Gboard open, status bar + * visible) archived at `~/.agent-device-bench/replay-runs/android-ime/raw-ime2.json` + * (#1251). This is the `--raw` tree: `--raw` keeps every structural wrapper + * node (`status_bar_container`, `status_bar_contents`, ...), unlike a default + * capture. Shared across the chrome-classification tests + * (`core/__tests__/snapshot-chrome-android-statusbar.test.ts`) and the replay + * divergence route test (`daemon/handlers/__tests__/session-replay-divergence.test.ts`) + * so both exercise the exact same real screen through `walkNonRawAndroidFixture`. + */ +export const ANDROID_IME_CAPTURE_RAW_NODES: RawSnapshotNode[] = [ + { index: 0, type: 'android.widget.FrameLayout', bundleId: 'com.android.systemui' }, + { + index: 1, + parentIndex: 0, + type: 'android.widget.FrameLayout', + bundleId: 'com.android.systemui', + identifier: 'com.android.systemui:id/status_bar_launch_animation_container', + }, + { + index: 2, + parentIndex: 0, + type: 'android.widget.FrameLayout', + bundleId: 'com.android.systemui', + identifier: 'com.android.systemui:id/status_bar_container', + }, + { + index: 3, + parentIndex: 2, + type: 'androidx.compose.ui.platform.ComposeView', + bundleId: 'com.android.systemui', + }, + { index: 4, parentIndex: 3, type: 'android.view.View', bundleId: 'com.android.systemui' }, + { + index: 5, + parentIndex: 4, + type: 'androidx.compose.ui.viewinterop.ViewFactoryHolder', + bundleId: 'com.android.systemui', + }, + { + index: 6, + parentIndex: 5, + type: 'android.widget.FrameLayout', + bundleId: 'com.android.systemui', + identifier: 'com.android.systemui:id/status_bar', + }, + { + index: 7, + parentIndex: 6, + type: 'android.widget.LinearLayout', + bundleId: 'com.android.systemui', + identifier: 'com.android.systemui:id/status_bar_contents', + }, + { + index: 8, + parentIndex: 7, + type: 'android.widget.FrameLayout', + bundleId: 'com.android.systemui', + identifier: 'com.android.systemui:id/status_bar_start_side_container', + }, + { + index: 9, + parentIndex: 8, + type: 'android.widget.FrameLayout', + bundleId: 'com.android.systemui', + identifier: 'com.android.systemui:id/status_bar_start_side_content', + }, + { + index: 10, + parentIndex: 9, + type: 'android.widget.LinearLayout', + bundleId: 'com.android.systemui', + identifier: 'com.android.systemui:id/status_bar_start_side_except_heads_up', + }, + { + index: 11, + parentIndex: 10, + type: 'android.widget.TextView', + bundleId: 'com.android.systemui', + identifier: 'com.android.systemui:id/clock', + label: '7:03', + }, + { + index: 12, + parentIndex: 10, + type: 'android.widget.FrameLayout', + bundleId: 'com.android.systemui', + identifier: 'com.android.systemui:id/start_side_notif_and_chip_container', + }, + { + index: 13, + parentIndex: 12, + type: 'android.widget.FrameLayout', + bundleId: 'com.android.systemui', + identifier: 'com.android.systemui:id/notification_icon_area', + }, + { + index: 14, + parentIndex: 13, + type: 'android.view.ViewGroup', + bundleId: 'com.android.systemui', + identifier: 'com.android.systemui:id/notificationIcons', + }, + { + index: 15, + parentIndex: 14, + type: 'android.widget.ImageView', + bundleId: 'com.android.systemui', + label: 'Security & privacy notification: ', + }, + { + index: 16, + parentIndex: 7, + type: 'android.view.View', + bundleId: 'com.android.systemui', + identifier: 'com.android.systemui:id/cutout_space_view', + }, + { + index: 17, + parentIndex: 7, + type: 'android.widget.FrameLayout', + bundleId: 'com.android.systemui', + identifier: 'com.android.systemui:id/status_bar_end_side_container', + }, + { + index: 18, + parentIndex: 17, + type: 'android.widget.LinearLayout', + bundleId: 'com.android.systemui', + identifier: 'com.android.systemui:id/status_bar_end_side_content', + }, + { + index: 19, + parentIndex: 18, + type: 'android.widget.LinearLayout', + bundleId: 'com.android.systemui', + identifier: 'com.android.systemui:id/system_icons', + }, + { + index: 20, + parentIndex: 19, + type: 'android.widget.LinearLayout', + bundleId: 'com.android.systemui', + identifier: 'com.android.systemui:id/statusIcons', + }, + { + index: 21, + parentIndex: 20, + type: 'android.widget.FrameLayout', + bundleId: 'com.android.systemui', + identifier: 'com.android.systemui:id/mobile_combo', + label: 'T-Mobile, signal full.', + }, + { + index: 22, + parentIndex: 21, + type: 'android.widget.LinearLayout', + bundleId: 'com.android.systemui', + identifier: 'com.android.systemui:id/mobile_group', + }, + { + index: 23, + parentIndex: 22, + type: 'android.widget.FrameLayout', + bundleId: 'com.android.systemui', + }, + { + index: 24, + parentIndex: 23, + type: 'android.widget.ImageView', + bundleId: 'com.android.systemui', + identifier: 'com.android.systemui:id/mobile_signal', + }, + { + index: 25, + parentIndex: 20, + type: 'android.widget.FrameLayout', + bundleId: 'com.android.systemui', + identifier: 'com.android.systemui:id/wifi_combo', + }, + { + index: 26, + parentIndex: 25, + type: 'android.widget.LinearLayout', + bundleId: 'com.android.systemui', + identifier: 'com.android.systemui:id/wifi_group', + }, + { + index: 27, + parentIndex: 26, + type: 'android.widget.FrameLayout', + bundleId: 'com.android.systemui', + identifier: 'com.android.systemui:id/wifi_combo', + }, + { + index: 28, + parentIndex: 27, + type: 'android.widget.ImageView', + bundleId: 'com.android.systemui', + identifier: 'com.android.systemui:id/wifi_signal', + label: 'Wifi signal full.', + }, + { + index: 29, + parentIndex: 19, + type: 'androidx.compose.ui.platform.ComposeView', + bundleId: 'com.android.systemui', + }, + { index: 30, parentIndex: 29, type: 'android.view.View', bundleId: 'com.android.systemui' }, + { + index: 31, + parentIndex: 30, + type: 'android.view.View', + bundleId: 'com.android.systemui', + identifier: 'com.android.systemui:id/battery', + }, + { + index: 32, + parentIndex: 31, + type: 'android.view.View', + bundleId: 'com.android.systemui', + label: 'Battery 100 percent.', + }, + { + index: 33, + type: 'android.widget.FrameLayout', + bundleId: 'com.google.android.inputmethod.latin', + }, + { + index: 34, + parentIndex: 33, + type: 'android.widget.LinearLayout', + bundleId: 'com.google.android.inputmethod.latin', + }, + { + index: 35, + parentIndex: 34, + type: 'android.widget.FrameLayout', + bundleId: 'com.google.android.inputmethod.latin', + identifier: 'android:id/content', + }, + { + index: 36, + parentIndex: 35, + type: 'android.widget.LinearLayout', + bundleId: 'com.google.android.inputmethod.latin', + identifier: 'android:id/parentPanel', + }, + { + index: 37, + parentIndex: 36, + type: 'android.widget.FrameLayout', + bundleId: 'com.google.android.inputmethod.latin', + identifier: 'android:id/inputArea', + }, + { + index: 38, + parentIndex: 37, + type: 'android.widget.FrameLayout', + bundleId: 'com.google.android.inputmethod.latin', + identifier: 'com.google.android.inputmethod.latin:id/0_resource_name_obfuscated', + }, + { + index: 39, + parentIndex: 38, + type: 'android.widget.FrameLayout', + bundleId: 'com.google.android.inputmethod.latin', + identifier: 'com.google.android.inputmethod.latin:id/0_resource_name_obfuscated', + }, + { + index: 40, + parentIndex: 39, + type: 'android.widget.LinearLayout', + bundleId: 'com.google.android.inputmethod.latin', + }, + { + index: 41, + parentIndex: 40, + type: 'android.widget.LinearLayout', + bundleId: 'com.google.android.inputmethod.latin', + identifier: 'com.google.android.inputmethod.latin:id/keyboard_holder', + }, + { + index: 42, + parentIndex: 41, + type: 'android.widget.FrameLayout', + bundleId: 'com.google.android.inputmethod.latin', + identifier: 'com.google.android.inputmethod.latin:id/keyboard_header_view_holder', + }, + { + index: 43, + parentIndex: 42, + type: 'android.widget.FrameLayout', + bundleId: 'com.google.android.inputmethod.latin', + }, + { + index: 44, + parentIndex: 43, + type: 'android.view.ViewGroup', + bundleId: 'com.google.android.inputmethod.latin', + identifier: 'com.google.android.inputmethod.latin:id/0_resource_name_obfuscated', + }, + { + index: 45, + parentIndex: 44, + type: 'android.view.View', + bundleId: 'com.google.android.inputmethod.latin', + identifier: 'com.google.android.inputmethod.latin:id/0_resource_name_obfuscated', + }, + { + index: 46, + parentIndex: 44, + type: 'android.widget.FrameLayout', + bundleId: 'com.google.android.inputmethod.latin', + identifier: 'com.google.android.inputmethod.latin:id/key_pos_header_access_points_menu', + label: 'Open features menu', + }, + { + index: 47, + parentIndex: 46, + type: 'android.widget.FrameLayout', + bundleId: 'com.google.android.inputmethod.latin', + }, + { + index: 48, + parentIndex: 47, + type: 'android.widget.ImageView', + bundleId: 'com.google.android.inputmethod.latin', + identifier: 'com.google.android.inputmethod.latin:id/0_resource_name_obfuscated', + }, + { + index: 49, + parentIndex: 44, + type: 'android.widget.LinearLayout', + bundleId: 'com.google.android.inputmethod.latin', + identifier: 'com.google.android.inputmethod.latin:id/0_resource_name_obfuscated', + }, + { + index: 50, + parentIndex: 49, + type: 'android.widget.FrameLayout', + bundleId: 'com.google.android.inputmethod.latin', + identifier: 'com.google.android.inputmethod.latin:id/0_resource_name_obfuscated', + }, + { + index: 51, + parentIndex: 50, + type: 'android.widget.LinearLayout', + bundleId: 'com.google.android.inputmethod.latin', + identifier: 'com.google.android.inputmethod.latin:id/softkey_holder_fixed_candidates', + }, + { + index: 52, + parentIndex: 51, + type: 'android.widget.FrameLayout', + bundleId: 'com.google.android.inputmethod.latin', + label: 'name', + }, + { + index: 53, + parentIndex: 52, + type: 'android.widget.FrameLayout', + bundleId: 'com.google.android.inputmethod.latin', + identifier: 'com.google.android.inputmethod.latin:id/0_resource_name_obfuscated', + }, + { + index: 54, + parentIndex: 53, + type: 'android.widget.LinearLayout', + bundleId: 'com.google.android.inputmethod.latin', + }, + { + index: 55, + parentIndex: 54, + type: 'android.widget.TextView', + bundleId: 'com.google.android.inputmethod.latin', + identifier: 'com.google.android.inputmethod.latin:id/0_resource_name_obfuscated', + label: 'name', + }, + { + index: 56, + parentIndex: 53, + type: 'android.widget.ImageView', + bundleId: 'com.google.android.inputmethod.latin', + identifier: 'com.google.android.inputmethod.latin:id/0_resource_name_obfuscated', + }, + { + index: 57, + parentIndex: 51, + type: 'android.widget.FrameLayout', + bundleId: 'com.google.android.inputmethod.latin', + label: 'names', + }, + { + index: 58, + parentIndex: 57, + type: 'android.widget.FrameLayout', + bundleId: 'com.google.android.inputmethod.latin', + identifier: 'com.google.android.inputmethod.latin:id/0_resource_name_obfuscated', + }, + { + index: 59, + parentIndex: 58, + type: 'android.widget.LinearLayout', + bundleId: 'com.google.android.inputmethod.latin', + }, + { + index: 60, + parentIndex: 59, + type: 'android.widget.TextView', + bundleId: 'com.google.android.inputmethod.latin', + identifier: 'com.google.android.inputmethod.latin:id/0_resource_name_obfuscated', + label: 'names', + }, + { + index: 61, + parentIndex: 58, + type: 'android.widget.ImageView', + bundleId: 'com.google.android.inputmethod.latin', + identifier: 'com.google.android.inputmethod.latin:id/0_resource_name_obfuscated', + }, + { + index: 62, + parentIndex: 51, + type: 'android.widget.LinearLayout', + bundleId: 'com.google.android.inputmethod.latin', + identifier: 'com.google.android.inputmethod.latin:id/0_resource_name_obfuscated', + }, + { + index: 63, + parentIndex: 62, + type: 'android.widget.FrameLayout', + bundleId: 'com.google.android.inputmethod.latin', + label: '📛 emoji', + }, + { + index: 64, + parentIndex: 63, + type: 'android.widget.FrameLayout', + bundleId: 'com.google.android.inputmethod.latin', + identifier: 'com.google.android.inputmethod.latin:id/0_resource_name_obfuscated', + }, + { + index: 65, + parentIndex: 64, + type: 'android.widget.FrameLayout', + bundleId: 'com.google.android.inputmethod.latin', + }, + { + index: 66, + parentIndex: 65, + type: 'android.view.View', + bundleId: 'com.google.android.inputmethod.latin', + identifier: 'com.google.android.inputmethod.latin:id/0_resource_name_obfuscated', + label: '📛 emoji', + }, + { + index: 67, + parentIndex: 44, + type: 'android.speech.SpeechRecognizer.VoiceDictationButton', + bundleId: 'com.google.android.inputmethod.latin', + identifier: 'com.google.android.inputmethod.latin:id/key_pos_header_power_key', + label: 'Use voice typing', + }, + { + index: 68, + parentIndex: 67, + type: 'android.widget.FrameLayout', + bundleId: 'com.google.android.inputmethod.latin', + identifier: 'com.google.android.inputmethod.latin:id/host', + }, + { + index: 69, + parentIndex: 68, + type: 'android.widget.FrameLayout', + bundleId: 'com.google.android.inputmethod.latin', + }, + { + index: 70, + parentIndex: 69, + type: 'android.widget.FrameLayout', + bundleId: 'com.google.android.inputmethod.latin', + identifier: 'com.google.android.inputmethod.latin:id/0_resource_name_obfuscated', + }, + { + index: 71, + parentIndex: 69, + type: 'android.widget.FrameLayout', + bundleId: 'com.google.android.inputmethod.latin', + identifier: 'com.google.android.inputmethod.latin:id/0_resource_name_obfuscated', + }, + { + index: 72, + parentIndex: 69, + type: 'android.view.View', + bundleId: 'com.google.android.inputmethod.latin', + }, + { + index: 73, + parentIndex: 69, + type: 'android.widget.ImageView', + bundleId: 'com.google.android.inputmethod.latin', + identifier: 'com.google.android.inputmethod.latin:id/0_resource_name_obfuscated', + }, + { + index: 74, + parentIndex: 41, + type: 'android.widget.FrameLayout', + bundleId: 'com.google.android.inputmethod.latin', + identifier: 'com.google.android.inputmethod.latin:id/0_resource_name_obfuscated', + }, + { + index: 75, + parentIndex: 74, + type: 'android.widget.FrameLayout', + bundleId: 'com.google.android.inputmethod.latin', + }, + { + index: 76, + parentIndex: 75, + type: 'android.widget.LinearLayout', + bundleId: 'com.google.android.inputmethod.latin', + }, + { + index: 77, + parentIndex: 76, + type: 'android.widget.LinearLayout', + bundleId: 'com.google.android.inputmethod.latin', + identifier: 'com.google.android.inputmethod.latin:id/input_area', + }, + { + index: 78, + parentIndex: 77, + type: 'android.widget.LinearLayout', + bundleId: 'com.google.android.inputmethod.latin', + }, + { + index: 79, + parentIndex: 78, + type: 'android.widget.FrameLayout', + bundleId: 'com.google.android.inputmethod.latin', + identifier: 'com.google.android.inputmethod.latin:id/D01', + label: 'q', + }, + { + index: 80, + parentIndex: 79, + type: 'android.widget.TextView', + bundleId: 'com.google.android.inputmethod.latin', + identifier: 'com.google.android.inputmethod.latin:id/0_resource_name_obfuscated', + label: 'q', + }, + { + index: 81, + parentIndex: 79, + type: 'android.widget.TextView', + bundleId: 'com.google.android.inputmethod.latin', + identifier: 'com.google.android.inputmethod.latin:id/0_resource_name_obfuscated', + label: '1', + }, + { + index: 82, + parentIndex: 78, + type: 'android.widget.FrameLayout', + bundleId: 'com.google.android.inputmethod.latin', + identifier: 'com.google.android.inputmethod.latin:id/D02', + label: 'w', + }, + { + index: 109, + parentIndex: 77, + type: 'android.widget.LinearLayout', + bundleId: 'com.google.android.inputmethod.latin', + }, + { + index: 110, + parentIndex: 109, + type: 'android.widget.FrameLayout', + bundleId: 'com.google.android.inputmethod.latin', + identifier: 'com.google.android.inputmethod.latin:id/C01', + label: 'a', + }, + { + index: 132, + parentIndex: 77, + type: 'android.widget.LinearLayout', + bundleId: 'com.google.android.inputmethod.latin', + }, + { + index: 133, + parentIndex: 132, + type: 'android.widget.FrameLayout', + bundleId: 'com.google.android.inputmethod.latin', + identifier: 'com.google.android.inputmethod.latin:id/key_pos_shift', + label: 'Shift', + }, + { + index: 149, + parentIndex: 132, + type: 'android.widget.FrameLayout', + bundleId: 'com.google.android.inputmethod.latin', + identifier: 'com.google.android.inputmethod.latin:id/key_pos_del', + label: 'Delete', + }, + { + index: 151, + parentIndex: 76, + type: 'android.widget.LinearLayout', + bundleId: 'com.google.android.inputmethod.latin', + identifier: 'com.google.android.inputmethod.latin:id/0_resource_name_obfuscated', + }, + { + index: 158, + parentIndex: 151, + type: 'android.widget.FrameLayout', + bundleId: 'com.google.android.inputmethod.latin', + identifier: 'com.google.android.inputmethod.latin:id/key_pos_space', + label: 'Space', + }, + { + index: 163, + parentIndex: 151, + type: 'android.widget.FrameLayout', + bundleId: 'com.google.android.inputmethod.latin', + identifier: 'com.google.android.inputmethod.latin:id/key_pos_ime_action', + label: 'Done', + }, + { + index: 169, + parentIndex: 33, + type: 'android.widget.FrameLayout', + bundleId: 'com.google.android.inputmethod.latin', + }, + { + index: 170, + parentIndex: 169, + type: 'android.widget.FrameLayout', + bundleId: 'com.google.android.inputmethod.latin', + identifier: 'android:id/input_method_navigation_bar_view', + }, + { + index: 176, + parentIndex: 170, + type: 'android.widget.ImageView', + bundleId: 'com.google.android.inputmethod.latin', + identifier: 'android:id/input_method_nav_back', + label: 'Back', + }, + { + index: 178, + parentIndex: 170, + type: 'android.widget.ImageView', + bundleId: 'com.google.android.inputmethod.latin', + identifier: 'android:id/input_method_nav_ime_switcher', + label: 'Switch input method', + }, + { index: 180, type: 'android.widget.FrameLayout', bundleId: 'com.callstack.agentdevicelab' }, + { + index: 181, + parentIndex: 180, + type: 'android.widget.LinearLayout', + bundleId: 'com.callstack.agentdevicelab', + }, + { + index: 182, + parentIndex: 181, + type: 'android.widget.FrameLayout', + bundleId: 'com.callstack.agentdevicelab', + }, + { + index: 183, + parentIndex: 182, + type: 'android.widget.LinearLayout', + bundleId: 'com.callstack.agentdevicelab', + identifier: 'com.callstack.agentdevicelab:id/action_bar_root', + }, + { + index: 184, + parentIndex: 183, + type: 'android.widget.FrameLayout', + bundleId: 'com.callstack.agentdevicelab', + identifier: 'android:id/content', + }, + { + index: 185, + parentIndex: 184, + type: 'android.widget.FrameLayout', + bundleId: 'com.callstack.agentdevicelab', + }, + { + index: 193, + parentIndex: 185, + type: 'android.widget.FrameLayout', + bundleId: 'com.callstack.agentdevicelab', + }, + { + index: 194, + parentIndex: 193, + type: 'android.view.ViewGroup', + bundleId: 'com.callstack.agentdevicelab', + }, + { + index: 196, + parentIndex: 193, + type: 'android.view.ViewGroup', + bundleId: 'com.callstack.agentdevicelab', + }, + { + index: 197, + parentIndex: 196, + type: 'android.view.ViewGroup', + bundleId: 'com.callstack.agentdevicelab', + }, + { + index: 198, + parentIndex: 197, + type: 'android.view.ViewGroup', + bundleId: 'com.callstack.agentdevicelab', + }, + { + index: 199, + parentIndex: 198, + type: 'android.view.ViewGroup', + bundleId: 'com.callstack.agentdevicelab', + }, + { + index: 200, + parentIndex: 199, + type: 'android.widget.ScrollView', + bundleId: 'com.callstack.agentdevicelab', + }, + { + index: 201, + parentIndex: 200, + type: 'android.view.ViewGroup', + bundleId: 'com.callstack.agentdevicelab', + }, + { + index: 202, + parentIndex: 201, + type: 'android.view.ViewGroup', + bundleId: 'com.callstack.agentdevicelab', + }, + { + index: 203, + parentIndex: 202, + type: 'android.widget.FrameLayout', + bundleId: 'com.callstack.agentdevicelab', + }, + { + index: 204, + parentIndex: 203, + type: 'android.widget.FrameLayout', + bundleId: 'com.callstack.agentdevicelab', + }, + { + index: 205, + parentIndex: 204, + type: 'android.widget.FrameLayout', + bundleId: 'com.callstack.agentdevicelab', + }, + { + index: 206, + parentIndex: 205, + type: 'android.view.ViewGroup', + bundleId: 'com.callstack.agentdevicelab', + }, + { + index: 207, + parentIndex: 206, + type: 'android.view.ViewGroup', + bundleId: 'com.callstack.agentdevicelab', + }, + { + index: 208, + parentIndex: 207, + type: 'android.view.ViewGroup', + bundleId: 'com.callstack.agentdevicelab', + }, + { + index: 210, + parentIndex: 208, + type: 'android.widget.ScrollView', + bundleId: 'com.callstack.agentdevicelab', + }, + { + index: 211, + parentIndex: 210, + type: 'android.view.ViewGroup', + bundleId: 'com.callstack.agentdevicelab', + }, + { + index: 212, + parentIndex: 211, + type: 'android.view.ViewGroup', + bundleId: 'com.callstack.agentdevicelab', + identifier: 'form-title', + }, + { + index: 213, + parentIndex: 211, + type: 'android.widget.TextView', + bundleId: 'com.callstack.agentdevicelab', + label: 'Checkout form', + }, + { + index: 220, + parentIndex: 211, + type: 'android.widget.TextView', + bundleId: 'com.callstack.agentdevicelab', + label: 'Full name', + }, + { + index: 221, + parentIndex: 211, + type: 'android.widget.EditText', + bundleId: 'com.callstack.agentdevicelab', + identifier: 'field-name', + label: 'review name', + }, + { + index: 222, + parentIndex: 211, + type: 'android.widget.TextView', + bundleId: 'com.callstack.agentdevicelab', + label: 'Email', + }, + { + index: 223, + parentIndex: 211, + type: 'android.widget.EditText', + bundleId: 'com.callstack.agentdevicelab', + identifier: 'field-email', + label: 'ada@example.com', + }, + { + index: 224, + parentIndex: 211, + type: 'android.widget.TextView', + bundleId: 'com.callstack.agentdevicelab', + label: 'Phone', + }, + { + index: 225, + parentIndex: 211, + type: 'android.widget.EditText', + bundleId: 'com.callstack.agentdevicelab', + identifier: 'field-phone', + label: '+48 555 010 010', + }, + { + index: 226, + parentIndex: 211, + type: 'android.view.ViewGroup', + bundleId: 'com.callstack.agentdevicelab', + identifier: 'android-ime-capture-fixture', + }, +]; diff --git a/src/__tests__/test-utils/index.ts b/src/__tests__/test-utils/index.ts index 56fd18592..e1bad8706 100644 --- a/src/__tests__/test-utils/index.ts +++ b/src/__tests__/test-utils/index.ts @@ -20,6 +20,11 @@ export { export { makeSnapshotState } from './snapshot-builders.ts'; +export { + rawFixtureToAndroidTree, + walkNonRawAndroidFixture, +} from './android-ui-hierarchy-fixtures.ts'; + export { makeSessionStore } from './store-factory.ts'; export { withNoColor } from './color.ts'; diff --git a/src/core/__tests__/snapshot-chrome-android-statusbar.test.ts b/src/core/__tests__/snapshot-chrome-android-statusbar.test.ts new file mode 100644 index 000000000..9b07479a7 --- /dev/null +++ b/src/core/__tests__/snapshot-chrome-android-statusbar.test.ts @@ -0,0 +1,232 @@ +import { test } from 'vitest'; +import assert from 'node:assert/strict'; +import { attachRefs, type RawSnapshotNode, type SnapshotNode } from '../../kernel/snapshot.ts'; +import { collectSettleChromeRefs } from '../snapshot-chrome.ts'; +import { + ANDROID_IME_CAPTURE_RAW_NODES, + walkNonRawAndroidFixture, +} from '../../__tests__/test-utils/android-ui-hierarchy-fixtures.ts'; + +/** + * `ANDROID_IME_CAPTURE_RAW_NODES` (shared test util, see its own doc comment) + * is a real device `--raw` capture: `--raw` keeps every structural wrapper + * node (`status_bar_container`, `status_bar_contents`, ...) so the OLD + * prefix-only marker check finds them and drops the whole run. + * + * The default (non-raw) walk drops unlabeled/unidentified structural nodes + * (`shouldIncludeStructuralAndroidNode` in `platforms/android/ui-hierarchy.ts`), + * re-parenting their children upward — which silently removes every one of + * those marker-bearing wrappers AND several other anonymous structural nodes + * that have neither a hittable descendant nor meaningful text/id (a real + * `android.view.View` example is `com.android.systemui:id/home_handle`, see + * the nav-bar test below). `walkNonRawAndroidFixture` (shared test util) runs + * the fixture through the REAL `buildUiHierarchySnapshot({ raw: false })` + * walk instead of hand-simulating a subset of its drops, so every inclusion + * decision below is production's, not a guess at which nodes matter. + */ + +function refForIdentifier(nodes: SnapshotNode[], identifier: string): string { + const node = nodes.find((candidate) => candidate.identifier === identifier); + assert.ok(node?.ref, `expected a node identified "${identifier}" with a ref`); + return node.ref; +} + +/** + * Status-bar leaf identifiers that SURVIVE the real non-raw walk for this + * fixture (verified against `buildUiHierarchySnapshot({ raw: false })` + * directly). Several ids the naive prefix-only simulation used to assert + * (`notification_icon_area`, `notificationIcons`, `cutout_space_view`, + * `system_icons`, `statusIcons`, `mobile_group`, `wifi_combo`, `wifi_group`, + * `start_side_notif_and_chip_container`, `battery`) are unlabeled structural + * wrappers with a generic resource id and no hittable descendant: production + * `shouldIncludeStructuralAndroidNode` drops every one of them too, same as + * the `status_bar*`/`navigation_bar*` wrappers. Their content isn't lost — + * `battery`'s labeled child ("Battery 100 percent.") re-parents upward and is + * covered by the "every systemui-owned node is chrome" assertion below — but + * the WRAPPER identifier itself is gone from the walked tree, so it can't be + * looked up by id. + */ +const STATUS_BAR_LEAF_IDENTIFIERS = [ + 'com.android.systemui:id/clock', + 'com.android.systemui:id/mobile_combo', + 'com.android.systemui:id/mobile_signal', + 'com.android.systemui:id/wifi_signal', +]; + +test('Android non-raw capture: status-bar leaves are recognized as chrome once their status_bar*/navigation_bar* marker wrapper is dropped by the walk (#1251)', () => { + const walkedNodes = walkNonRawAndroidFixture(ANDROID_IME_CAPTURE_RAW_NODES); + const nodes = attachRefs(walkedNodes); + const chromeRefs = collectSettleChromeRefs(nodes, 'com.callstack.agentdevicelab'); + + // None of the status_bar*/navigation_bar* WRAPPER identifiers survive the + // real walk at all — confirms the walk, not this test, is doing the drop. + for (const node of nodes) { + const identifier = node.identifier ?? ''; + assert.equal( + identifier.startsWith('com.android.systemui:id/status_bar') || + identifier.startsWith('com.android.systemui:id/navigation_bar'), + false, + `expected the walk to have dropped wrapper "${identifier}"`, + ); + } + + for (const identifier of STATUS_BAR_LEAF_IDENTIFIERS) { + assert.equal( + chromeRefs.has(refForIdentifier(nodes, identifier)), + true, + `expected ${identifier} to be classified as systemui chrome`, + ); + } + + // The whole systemui run drops together, including the anonymous Compose + // plumbing nodes above/around the leaves that carry no identifier at all + // (and the re-parented, now-anonymous battery leaf). + const systemUiRefs = nodes + .filter((node) => node.bundleId === 'com.android.systemui') + .map((node) => node.ref); + assert.equal(systemUiRefs.length > 0, true); + for (const ref of systemUiRefs) { + assert.equal(chromeRefs.has(ref), true, 'expected every systemui-owned node to be chrome'); + } + + // App fields and the IME keyboard, both real nodes from the same capture, + // are handled exactly as before: the checkout form stays fully visible and + // the IME keyboard is still classified as chrome in full. + assert.equal(chromeRefs.has(refForIdentifier(nodes, 'field-name')), false); + assert.equal(chromeRefs.has(refForIdentifier(nodes, 'field-email')), false); + assert.equal(chromeRefs.has(refForIdentifier(nodes, 'field-phone')), false); + const imeRefs = nodes + .filter((node) => node.bundleId === 'com.google.android.inputmethod.latin') + .map((node) => node.ref); + assert.equal(imeRefs.length > 0, true); + for (const ref of imeRefs) { + assert.equal(chromeRefs.has(ref), true, 'expected every IME-owned node to be chrome'); + } +}); + +test('Android actionable systemui overlay (volume dialog) still survives with the status-bar leak fix (#1251)', () => { + // Extend the RAW capture with a disjoint systemui run whose leaf ids look + // like an actionable overlay (volume dialog): no real capture of that + // surface was available, but the point of this test is exactly that the + // new leaf-id set must NOT broaden to "any systemui id". Appended to the + // RAW array (not the already-walked output) so the whole tree — status bar + // AND volume dialog — goes through the real walk together, exactly as a + // single device capture would. + const rawWithVolumeDialog: RawSnapshotNode[] = [ + ...ANDROID_IME_CAPTURE_RAW_NODES, + { + index: 9000, + type: 'android.widget.FrameLayout', + bundleId: 'com.android.systemui', + identifier: 'com.android.systemui:id/volume_dialog_container', + }, + { + index: 9001, + parentIndex: 9000, + type: 'android.widget.ImageButton', + bundleId: 'com.android.systemui', + identifier: 'com.android.systemui:id/volume_dialog_slider', + label: 'Media volume', + hittable: true, + }, + ]; + const nodes = attachRefs(walkNonRawAndroidFixture(rawWithVolumeDialog)); + const chromeRefs = collectSettleChromeRefs(nodes, 'com.callstack.agentdevicelab'); + + // The container is non-hittable with a generic id, but its hittable + // `volume_dialog_slider` child keeps it in the walked tree (descendantHittable). + assert.equal( + chromeRefs.has(refForIdentifier(nodes, 'com.android.systemui:id/volume_dialog_container')), + false, + ); + assert.equal( + chromeRefs.has(refForIdentifier(nodes, 'com.android.systemui:id/volume_dialog_slider')), + false, + ); + // The status-bar leak fix stays active in the very same tree. + assert.equal(chromeRefs.has(refForIdentifier(nodes, 'com.android.systemui:id/clock')), true); +}); + +/** + * Synthetic: the real capture (#1251) has no nav bar (gesture-nav device). + * `home_handle` is an `android.view.View` with a generic resource id, no + * label, and (unlike `volume_dialog_slider` above) is neither hittable nor + * home to a hittable descendant — production's `shouldIncludeStructuralAndroidNode` + * drops it before `collectSettleChromeRefs` ever runs, so it is simply ABSENT + * from the walked tree, not present-and-classified-as-chrome (the reviewer's + * exact test-validity finding on #1256: the old hand-simulation only stripped + * `status_bar*`/`navigation_bar*` wrappers and kept `home_handle` around to + * assert it as chrome, which the real walk never does). + */ +test('Android nav-bar leaves are recognized as chrome once their navigation_bar* marker wrapper is dropped (synthetic: the real capture has no nav bar, gesture-nav device) (#1251)', () => { + const rawNavBarNodes: RawSnapshotNode[] = [ + { index: 0, type: 'android.widget.FrameLayout', bundleId: 'com.android.systemui' }, + { + index: 1, + parentIndex: 0, + type: 'android.widget.FrameLayout', + bundleId: 'com.android.systemui', + identifier: 'com.android.systemui:id/navigation_bar_frame', + }, + { + index: 2, + parentIndex: 1, + type: 'android.widget.LinearLayout', + bundleId: 'com.android.systemui', + identifier: 'com.android.systemui:id/nav_buttons', + }, + { + index: 3, + parentIndex: 2, + type: 'android.widget.ImageView', + bundleId: 'com.android.systemui', + identifier: 'com.android.systemui:id/back', + label: 'Back', + }, + { + index: 4, + parentIndex: 2, + type: 'android.widget.ImageView', + bundleId: 'com.android.systemui', + identifier: 'com.android.systemui:id/home', + label: 'Home', + }, + { + index: 5, + parentIndex: 2, + type: 'android.widget.ImageView', + bundleId: 'com.android.systemui', + identifier: 'com.android.systemui:id/recent_apps', + label: 'Overview', + }, + { + // Unlabeled, non-hittable, generic-id `android.view.View`: the real + // walk drops this (no meaningful text/id, no hittable descendant), + // unlike `back`/`home`/`recent_apps` which carry a label. + index: 6, + parentIndex: 1, + type: 'android.view.View', + bundleId: 'com.android.systemui', + identifier: 'com.android.systemui:id/home_handle', + }, + ]; + const walkedNodes = walkNonRawAndroidFixture(rawNavBarNodes); + const nodes = attachRefs(walkedNodes); + const chromeRefs = collectSettleChromeRefs(nodes, 'com.example.app'); + + for (const identifier of [ + 'com.android.systemui:id/back', + 'com.android.systemui:id/home', + 'com.android.systemui:id/recent_apps', + ]) { + assert.equal(chromeRefs.has(refForIdentifier(nodes, identifier)), true, identifier); + } + + // home_handle is DROPPED by the real walk (unlabeled, non-hittable, no + // hittable descendant, generic id) — it must be absent, not chrome-classified. + assert.equal( + walkedNodes.some((node) => node.identifier === 'com.android.systemui:id/home_handle'), + false, + 'expected home_handle to be dropped by the real non-raw walk, not retained', + ); +}); diff --git a/src/core/snapshot-chrome.ts b/src/core/snapshot-chrome.ts index eb401a720..bd36af76f 100644 --- a/src/core/snapshot-chrome.ts +++ b/src/core/snapshot-chrome.ts @@ -212,8 +212,57 @@ const ANDROID_SYSTEM_CHROME_MARKER_PREFIXES = [ 'com.android.systemui:id/navigation_bar', ]; +/** + * Surviving status-bar/nav-bar LEAF ids (#1251). The non-raw Android walk + * (`walkUiHierarchyNode` in `platforms/android/ui-hierarchy.ts`) drops + * unlabeled/unidentified structural nodes via `shouldIncludeStructuralAndroidNode`, + * re-parenting their children upward — and that silently swallows every + * `status_bar*`/`navigation_bar*` WRAPPER node, i.e. the only nodes the prefix + * check above matches. A non-raw capture is left with just their labeled/ + * identified LEAVES (clock, battery, wifi/mobile icons, nav buttons), whose + * own resource-ids carry no `status_bar`/`navigation_bar` prefix, so the run + * loses its marker and `collectAndroidSystemChromeRunIndexes` stops dropping + * it (verified against a real `--raw` vs. default capture pair of the same + * screen). Recognize those leaves directly, by EXACT id — not prefix, to stay + * tight: nothing here should ever swallow an actionable systemui overlay like + * the volume dialog or a media/output picker, which live under unrelated ids. + * `--raw` keeps the wrapper markers, so the prefix check above stays + * load-bearing there. + * + * A more robust fix would thread the AOSP window-type constants + * (`TYPE_STATUS_BAR` = 2000, `TYPE_NAVIGATION_BAR` = 2019) already parsed in + * `readNodeAttributes` (ui-hierarchy.ts) through to the output `SnapshotNode` + * and key off that instead of resource-ids — deferred until the id-based + * approach here proves insufficient. + */ +const ANDROID_SYSTEM_CHROME_MARKER_LEAF_IDS = new Set( + [ + // status bar + 'clock', + 'battery', + 'statusIcons', + 'notificationIcons', + 'notification_icon_area', + 'system_icons', + 'cutout_space_view', + 'mobile_signal', + 'mobile_combo', + 'mobile_group', + 'wifi_signal', + 'wifi_combo', + 'wifi_group', + 'start_side_notif_and_chip_container', + // nav bar + 'back', + 'home', + 'recent_apps', + 'home_handle', + ].map((leaf) => `${ANDROID_SYSTEM_CHROME_PACKAGE}:id/${leaf}`), +); + function hasAndroidSystemChromeMarker(node: SnapshotNode): boolean { const identifier = node.identifier ?? ''; + if (ANDROID_SYSTEM_CHROME_MARKER_LEAF_IDS.has(identifier)) return true; return ANDROID_SYSTEM_CHROME_MARKER_PREFIXES.some((prefix) => identifier.startsWith(prefix)); } diff --git a/src/daemon/handlers/__tests__/session-replay-divergence.test.ts b/src/daemon/handlers/__tests__/session-replay-divergence.test.ts index 29e080475..9da29bb5b 100644 --- a/src/daemon/handlers/__tests__/session-replay-divergence.test.ts +++ b/src/daemon/handlers/__tests__/session-replay-divergence.test.ts @@ -9,7 +9,14 @@ vi.mock('../../../core/dispatch.ts', async (importOriginal) => { }); import { dispatchCommand } from '../../../core/dispatch.ts'; -import { makeIosSession } from '../../../__tests__/test-utils/session-factories.ts'; +import { + makeAndroidSession, + makeIosSession, +} from '../../../__tests__/test-utils/session-factories.ts'; +import { + ANDROID_IME_CAPTURE_RAW_NODES, + walkNonRawAndroidFixture, +} from '../../../__tests__/test-utils/android-ui-hierarchy-fixtures.ts'; import { SessionStore } from '../../session-store.ts'; import { buildReplayFailureDivergence } from '../session-replay-divergence.ts'; @@ -174,6 +181,124 @@ test('buildReplayFailureDivergence excludes keyboard chrome from screen.refs and expect(screen.truncated).toBeUndefined(); }); +// 25 unlabeled, non-interactive structural containers — the RN ViewGroup/ +// FrameLayout wrappers a full (non-interactive) divergence capture pulls in. +// They carry refs but no identity and aren't tappable, and they sort ahead of +// the actionable controls in document order, so without the meaningful-target +// filter they consume the whole SCREEN_REF_CAPTURE_LIMIT and truncate the real +// controls out (the same shape as the keyboard swamp, minus the chrome). +function structuralNoiseNodes() { + const noise = Array.from({ length: 25 }, (_, i) => ({ + index: 1 + i, + depth: 1, + parentIndex: 0, + type: 'Other', + rect: { x: 0, y: 10 * i, width: 402, height: 8 }, + })); + const base = 1 + noise.length; + return [ + { + index: 0, + depth: 0, + type: 'Application', + label: 'Checkout form', + rect: { x: 0, y: 0, width: 402, height: 874 }, + hittable: true, + }, + ...noise, + { + index: base, + depth: 1, + parentIndex: 0, + type: 'StaticText', + label: 'Full name', + rect: { x: 20, y: 300, width: 160, height: 20 }, + }, + { + index: base + 1, + depth: 1, + parentIndex: 0, + type: 'TextField', + value: 'ada@example.com', + rect: { x: 20, y: 330, width: 300, height: 44 }, + hittable: true, + }, + { + index: base + 2, + depth: 1, + parentIndex: 0, + type: 'Button', + label: 'Submit order', + identifier: 'submit-order', + rect: { x: 20, y: 400, width: 160, height: 44 }, + hittable: true, + }, + // Unlabeled but interactive — a bare icon button. `hittable` alone makes it a + // legitimate target, so it must survive even without any label/identifier. + { + index: base + 3, + depth: 1, + parentIndex: 0, + type: 'Button', + rect: { x: 340, y: 20, width: 44, height: 44 }, + hittable: true, + }, + ]; +} + +test('buildReplayFailureDivergence drops unlabeled non-interactive structural nodes so actionable controls surface within the cap', async () => { + const root = fs.mkdtempSync(path.join(os.tmpdir(), 'agent-device-replay-divergence-structural-')); + const sessionStore = new SessionStore(path.join(root, 'sessions')); + const sessionName = 'default'; + sessionStore.set(sessionName, makeIosSession(sessionName, { appBundleId: 'com.example.app' })); + + mockDispatchCommand.mockResolvedValue({ + nodes: structuralNoiseNodes(), + truncated: false, + backend: 'xctest', + }); + + const action = { + ts: 0, + command: 'get', + positionals: ['text', 'label="Submit order RENAMED"'], + flags: {}, + result: { selectorChain: ['label="Submit order RENAMED"'] }, + }; + const divergence = await buildReplayFailureDivergence({ + error: { code: 'COMMAND_FAILED', message: 'target not found' }, + action, + index: 0, + sourcePath: path.join(root, 'flow.ad'), + sourceLine: 1, + session: sessionStore.get(sessionName), + sessionName, + sessionStore, + logPath: path.join(root, 'daemon.log'), + responseLevel: 'default', + planActions: [action], + planDigest: 'test-plan-digest', + }); + + expect(divergence.screen.state).toBe('available'); + const screen = divergence.screen as Extract; + // The 25 structural wrappers no longer consume the cap, so the actionable + // controls — which sorted after them and were truncated out before — surface. + expect(screen.refs.some((ref) => ref.label === 'Submit order')).toBe(true); + expect(screen.refs.some((ref) => ref.label === 'Full name')).toBe(true); + expect(screen.refs.some((ref) => ref.label === 'ada@example.com')).toBe(true); + // An unlabeled-but-hittable control is still a valid target. + expect(screen.refs.some((ref) => !ref.label && ref.role.toLowerCase().includes('button'))).toBe( + true, + ); + // No unlabeled, non-interactive structural container leaks into the ref list. + expect( + screen.refs.every( + (ref) => ref.label !== undefined || ref.role.toLowerCase().includes('button'), + ), + ).toBe(true); +}); + // Same live-shape keyboard window, but the app hosts an inputAccessoryView // toolbar (a "Send" button) as a bar ABOVE the keys inside the keyboard // window. The keys are still chrome; the app-owned accessory control must NOT @@ -276,3 +401,73 @@ test('buildReplayFailureDivergence keeps an app inputAccessoryView control in sc // The app's accessory "Send" button survives and is available to heal against. expect(screen.refs.some((ref) => ref.label === 'Send')).toBe(true); }); + +// Android target-binding divergence route (#1256 test-validity fix): every +// other test in this file uses an iOS session with hand-picked mock nodes, +// which never exercises Android's status-bar/IME chrome classification. This +// test uses an ANDROID session instead, and — since `dispatchCommand` here is +// mocked (this handler's own capture step, `captureDivergenceObservation`, +// calls it downstream of the on-device Android walk, which cannot run inside +// a unit test) — feeds the mock the output of the REAL non-raw Android walk +// (`walkNonRawAndroidFixture`, `buildUiHierarchySnapshot({ raw: false })`) +// over the same real-device capture used in +// `core/__tests__/snapshot-chrome-android-statusbar.test.ts`. That means the +// walk's own inclusion/drop decisions are exercised for real; only the +// on-device transport that produces the pre-walk raw tree is stubbed. +test('buildReplayFailureDivergence excludes Android status-bar/IME chrome from screen.refs on a real (walked) non-raw capture', async () => { + const root = fs.mkdtempSync(path.join(os.tmpdir(), 'agent-device-replay-divergence-android-')); + const sessionStore = new SessionStore(path.join(root, 'sessions')); + const sessionName = 'default'; + sessionStore.set( + sessionName, + makeAndroidSession(sessionName, { appBundleId: 'com.callstack.agentdevicelab' }), + ); + + mockDispatchCommand.mockResolvedValue({ + nodes: walkNonRawAndroidFixture(ANDROID_IME_CAPTURE_RAW_NODES), + truncated: false, + backend: 'android', + }); + + const action = { + ts: 0, + command: 'get', + positionals: ['text', 'label="Full name RENAMED"'], + flags: {}, + result: { selectorChain: ['label="Full name RENAMED"'] }, + }; + const divergence = await buildReplayFailureDivergence({ + error: { code: 'COMMAND_FAILED', message: 'target not found' }, + action, + index: 0, + sourcePath: path.join(root, 'flow.ad'), + sourceLine: 1, + session: sessionStore.get(sessionName), + sessionName, + sessionStore, + logPath: path.join(root, 'daemon.log'), + responseLevel: 'default', + planActions: [action], + planDigest: 'test-plan-digest', + }); + + expect(divergence.screen.state).toBe('available'); + const screen = divergence.screen as Extract; + + // No status-bar/mobile/wifi systemui chrome, nor IME keys, leak into the ref + // budget — the exact same classification as the chrome-unit test, now + // exercised through the real divergence route. + expect(screen.refs.some((ref) => ref.label === '7:03')).toBe(false); // clock + expect(screen.refs.some((ref) => ref.label === 'T-Mobile, signal full.')).toBe(false); // mobile_combo + expect(screen.refs.some((ref) => ref.label === 'Wifi signal full.')).toBe(false); // wifi_signal + expect(screen.refs.some((ref) => ref.label === 'Shift')).toBe(false); // IME key + expect(screen.refs.some((ref) => ref.label === 'Space')).toBe(false); // IME key + expect(screen.refs.some((ref) => ref.label === 'Use voice typing')).toBe(false); // IME key + + // The app's own controls, real nodes from the same capture, stay visible. + expect(screen.refs.some((ref) => ref.label === 'Checkout form')).toBe(true); + expect(screen.refs.some((ref) => ref.label === 'review name')).toBe(true); // field-name + expect(screen.refs.some((ref) => ref.label === 'ada@example.com')).toBe(true); // field-email + expect(screen.refs.some((ref) => ref.label === '+48 555 010 010')).toBe(true); // field-phone + expect(screen.truncated).toBeUndefined(); +}); diff --git a/src/daemon/handlers/session-replay-divergence.ts b/src/daemon/handlers/session-replay-divergence.ts index c9c076d5d..7278a3ce0 100644 --- a/src/daemon/handlers/session-replay-divergence.ts +++ b/src/daemon/handlers/session-replay-divergence.ts @@ -274,6 +274,21 @@ export function buildDivergenceScreen( // by boundReplayDivergence/applyReplayDivergenceLevelCaps. const SCREEN_REF_CAPTURE_LIMIT = 20; +/** + * A divergence `screen.ref` is only useful if an agent could actually re-target + * it: it must be identifiable (a display label / value / non-generic identifier) + * or interactive (`hittable`). The `get`/`is`/`wait` divergence uses a full + * (non-interactive) capture so static-text targets survive, but that also pulls + * in unlabeled structural containers — ViewGroups / ComposeViews that carry a + * ref yet no identity and aren't tappable. Those are never valid repair targets, + * and on deeply-nested RN trees they would otherwise consume the + * `SCREEN_REF_CAPTURE_LIMIT` budget ahead of the actionable controls (and the + * app content the excluded status/nav chrome just freed room for). + */ +function isMeaningfulDivergenceTarget(node: SnapshotNode): boolean { + return Boolean(displayLabel(node, formatRole(node.type ?? 'Element'))) || node.hittable === true; +} + function buildReplayDivergenceScreenRefs( nodes: SnapshotNode[], sanitize: DivergenceFieldSanitizer, @@ -287,7 +302,11 @@ function buildReplayDivergenceScreenRefs( // rather than a second keyboard/IME node-type list. const chromeRefs = collectSettleChromeRefs(nodes, appBundleId); const candidates = nodes.filter( - (node) => node.ref && node.interactionBlocked !== 'covered' && !chromeRefs.has(node.ref), + (node) => + node.ref && + node.interactionBlocked !== 'covered' && + !chromeRefs.has(node.ref) && + isMeaningfulDivergenceTarget(node), ); const refs = candidates.slice(0, SCREEN_REF_CAPTURE_LIMIT).map((node) => { const role = formatRole(node.type ?? 'Element');