diff --git a/src/commands/interaction/runtime/settle.ts b/src/commands/interaction/runtime/settle.ts index b29eff794..8c48ef767 100644 --- a/src/commands/interaction/runtime/settle.ts +++ b/src/commands/interaction/runtime/settle.ts @@ -7,8 +7,7 @@ import type { import { isSparseSnapshotQualityVerdict } from '../../../snapshot/snapshot-quality.ts'; import { buildSnapshotDiff } from '../../../snapshot/snapshot-diff.ts'; import { displayLabel, formatRole } from '../../../snapshot/snapshot-lines.ts'; -import { isAndroidInputMethodSnapshotNode } from '../../../snapshot/android-input-method-overlays.ts'; -import { normalizeType } from '../../../utils/text-surface.ts'; +import { collectSettleChromeRefs, withoutSettleChrome } from '../../../core/snapshot-chrome.ts'; import { summarizeAxEvidence } from '../../../utils/ax-digest.ts'; import type { InteractionEvidence, @@ -279,285 +278,6 @@ export function buildSettleTailEntries( }; } -// Editable text roles (normalized `node.type` vocabulary) used by the -// keyboard-window guard below. -const EDITABLE_TEXT_TYPES = new Set([ - 'textfield', - 'securetextfield', - 'textview', - 'textarea', - 'searchfield', - 'edittext', -]); - -/** - * Keyboard chrome classification, live-verified against the iPhone 17 Pro - * simulator (iOS 26, July 2026): the software keyboard renders in its OWN - * dedicated window (UIRemoteKeyboardWindow). That window contains the - * `[Keyboard]` container (keys plus real XCUIElementTypeButton chrome like - * shift/Emoji/return) AND a SIBLING subtree holding the "Next keyboard" and - * "Dictate" buttons — siblings of the container, so a container-descendant - * walk alone provably misses them. The rule is therefore: every node inside - * a window that has a `[Keyboard]` descendant is keyboard chrome. - * - * Detection stays structural (`parentIndex` chains), never label-based: - * keyboard chrome text is locale-dependent (the verified capture had Polish - * key labels) and a label list would silently stop matching under a - * different input language. - * - * Conservative guard: a window that also hosts an editable text node OUTSIDE - * the keyboard container is never window-classified — iOS hosts - * inputAccessoryView content (e.g. a messaging composer) in the keyboard - * window, and hiding the field the user is typing into would be worse than - * leaking chrome. Such windows fall back to the container-descendant walk. - */ -type KeyboardChrome = { - /** Chrome node indexes EXCLUDING the containers: stripped from the diff. */ - strippedIndexes: ReadonlySet; - /** Refs of ALL chrome incl. containers/window: trigger + tail exclusion. */ - refs: ReadonlySet; -}; - -const EMPTY_KEYBOARD_CHROME: KeyboardChrome = { strippedIndexes: new Set(), refs: new Set() }; - -function collectKeyboardChrome(nodes: SnapshotNode[]): KeyboardChrome { - const containerIndexes = new Set( - nodes.filter((node) => normalizeType(node.type ?? '') === 'keyboard').map((node) => node.index), - ); - if (containerIndexes.size === 0) return EMPTY_KEYBOARD_CHROME; - const byIndex = new Map(nodes.map((node) => [node.index, node])); - const chromeIndexes = collectSubtreeIndexes(nodes, byIndex, containerIndexes); - const windowIndexes = resolveKeyboardWindowIndexes(nodes, byIndex, chromeIndexes); - for (const index of collectSubtreeIndexes(nodes, byIndex, windowIndexes)) { - chromeIndexes.add(index); - } - const refs = new Set( - nodes.filter((node) => node.ref && chromeIndexes.has(node.index)).map((node) => node.ref), - ); - // The [Keyboard] container line itself survives the diff so "keyboard - // appeared/left" stays one visible signal line; everything else collapses. - const strippedIndexes = new Set( - [...chromeIndexes].filter((index) => !containerIndexes.has(index)), - ); - return { strippedIndexes, refs }; -} - -/** The root indexes plus every node whose parent chain passes through one. */ -function collectSubtreeIndexes( - nodes: SnapshotNode[], - byIndex: Map, - rootIndexes: ReadonlySet, -): Set { - const indexes = new Set(rootIndexes); - if (rootIndexes.size === 0) return indexes; - for (const node of nodes) { - if (hasAncestorIn(node, byIndex, rootIndexes)) indexes.add(node.index); - } - return indexes; -} - -// SystemUI hosts BOTH persistent chrome and actionable overlays (volume -// panel, media/output pickers), so chrome is never a package-level fact. -// Within `com.android.systemui`, only window-runs carrying a status-bar or -// navigation-bar marker resource-id drop; every other systemui surface is -// kept. Marker set live-verified on the emulator: the status-bar window -// carries `status_bar*` ids throughout while the VolumeDialog window carries -// only `volume_dialog*` ids (`input_method_nav*` bars are IME-owned and -// handled by the IME tier). -const ANDROID_SYSTEM_CHROME_PACKAGE = 'com.android.systemui'; -const ANDROID_SYSTEM_CHROME_MARKER_PREFIXES = [ - 'com.android.systemui:id/status_bar', - 'com.android.systemui:id/navigation_bar', -]; - -function hasAndroidSystemChromeMarker(node: SnapshotNode): boolean { - const identifier = node.identifier ?? ''; - return ANDROID_SYSTEM_CHROME_MARKER_PREFIXES.some((prefix) => identifier.startsWith(prefix)); -} - -/** - * Android settle chrome (#1198): IME-owned nodes collapse to one surviving - * line per contiguous run; systemui status/nav-bar window-runs drop from both - * diff sides; every other foreign node — system dialogs (package `android`), - * permission prompts, AND actionable systemui overlays like the volume panel - * — is kept in full. Constraint: package membership is strictly per-node by - * the node's own `bundleId` — parentIndex chains can cross windows on Android - * (enforced by the settle.test.ts cross-window regression test); run grouping - * only ever walks parent chains BETWEEN same-package nodes, so it cannot - * swallow another package's node. Inert on iOS/macOS: those nodes never set - * `bundleId`. - */ -function collectAndroidSettleChrome( - nodes: SnapshotNode[], - appBundleId: string | undefined, -): KeyboardChrome { - const byIndex = new Map(nodes.map((node) => [node.index, node])); - const imeIndexes = new Set( - nodes.filter((node) => isAndroidInputMethodSnapshotNode(node)).map((node) => node.index), - ); - const imeContainerIndexes = new Set( - [...imeIndexes].filter((index) => { - const parentIndex = byIndex.get(index)?.parentIndex; - const parent = parentIndex !== undefined ? byIndex.get(parentIndex) : undefined; - return !parent || !imeIndexes.has(parent.index); - }), - ); - // appBundleId is the session's pre-action value (not refreshed inside the - // settle loop); it is only a never-drop-the-app-under-test guard here, so - // staleness cannot hide a foreign dialog. - const systemChromeIndexes = - appBundleId === ANDROID_SYSTEM_CHROME_PACKAGE - ? new Set() - : collectAndroidSystemChromeRunIndexes(nodes, byIndex, imeIndexes); - // The one surviving container line per IME run; the rest of the run and all - // status/nav-bar chrome never spend diff/tail budget. - const strippedIndexes = new Set( - [...imeIndexes].filter((index) => !imeContainerIndexes.has(index)), - ); - for (const index of systemChromeIndexes) strippedIndexes.add(index); - const refs = new Set( - nodes - .filter( - (node) => node.ref && (imeIndexes.has(node.index) || systemChromeIndexes.has(node.index)), - ) - .map((node) => node.ref), - ); - if (strippedIndexes.size === 0 && refs.size === 0) return EMPTY_KEYBOARD_CHROME; - return { strippedIndexes, refs }; -} - -/** - * Systemui window-runs (contiguous same-package parent chains) that contain a - * status/nav-bar marker anywhere in the run. The whole marked run drops — - * unmarked wrappers above `status_bar_container` churn with the bar itself — - * while unmarked runs (volume panel, media pickers) are kept whole. - */ -function collectAndroidSystemChromeRunIndexes( - nodes: SnapshotNode[], - byIndex: Map, - imeIndexes: ReadonlySet, -): Set { - const systemUiIndexes = new Set( - nodes - .filter( - (node) => node.bundleId === ANDROID_SYSTEM_CHROME_PACKAGE && !imeIndexes.has(node.index), - ) - .map((node) => node.index), - ); - if (systemUiIndexes.size === 0) return new Set(); - // Union-find-lite: each systemui node resolves to its run root (the nearest - // ancestor chain member whose parent is absent or not systemui). - const runRootByIndex = new Map(); - const resolveRunRoot = (index: number): number => { - const cached = runRootByIndex.get(index); - if (cached !== undefined) return cached; - const parentIndex = byIndex.get(index)?.parentIndex; - const root = - parentIndex !== undefined && systemUiIndexes.has(parentIndex) - ? resolveRunRoot(parentIndex) - : index; - runRootByIndex.set(index, root); - return root; - }; - const markedRunRoots = new Set( - [...systemUiIndexes] - .filter((index) => { - const node = byIndex.get(index); - return node !== undefined && hasAndroidSystemChromeMarker(node); - }) - .map((index) => resolveRunRoot(index)), - ); - return new Set([...systemUiIndexes].filter((index) => markedRunRoots.has(resolveRunRoot(index)))); -} - -/** iOS keyboard-window chrome unioned with Android IME/system chrome. */ -function collectSettleChrome( - nodes: SnapshotNode[], - appBundleId: string | undefined, -): KeyboardChrome { - const keyboard = collectKeyboardChrome(nodes); - const android = collectAndroidSettleChrome(nodes, appBundleId); - if (keyboard.strippedIndexes.size === 0 && keyboard.refs.size === 0) return android; - if (android.strippedIndexes.size === 0 && android.refs.size === 0) return keyboard; - return { - strippedIndexes: new Set([...keyboard.strippedIndexes, ...android.strippedIndexes]), - refs: new Set([...keyboard.refs, ...android.refs]), - }; -} - -function withoutSettleChrome( - nodes: SnapshotNode[], - appBundleId: string | undefined, -): SnapshotNode[] { - const { strippedIndexes } = collectSettleChrome(nodes, appBundleId); - if (strippedIndexes.size === 0) return nodes; - return nodes.filter((node) => !strippedIndexes.has(node.index)); -} - -function collectSettleChromeRefs( - nodes: SnapshotNode[], - appBundleId: string | undefined, -): ReadonlySet { - return collectSettleChrome(nodes, appBundleId).refs; -} - -/** - * Windows eligible for whole-window chrome classification: nearest `[window]` - * ancestor of each `[Keyboard]` container, minus windows hosting editable - * text outside the container subtree (the inputAccessoryView guard above). - */ -function resolveKeyboardWindowIndexes( - nodes: SnapshotNode[], - byIndex: Map, - containerChromeIndexes: ReadonlySet, -): Set { - const windowIndexes = new Set(); - for (const index of containerChromeIndexes) { - const node = byIndex.get(index); - if (!node || normalizeType(node.type ?? '') !== 'keyboard') continue; - const windowAncestor = findNearestWindowAncestor(node, byIndex); - if (windowAncestor !== undefined) windowIndexes.add(windowAncestor); - } - for (const windowIndex of windowIndexes) { - const windowSet = new Set([windowIndex]); - const hostsEditableText = nodes.some( - (node) => - EDITABLE_TEXT_TYPES.has(normalizeType(node.type ?? '')) && - !containerChromeIndexes.has(node.index) && - hasAncestorIn(node, byIndex, windowSet), - ); - if (hostsEditableText) windowIndexes.delete(windowIndex); - } - return windowIndexes; -} - -function findNearestWindowAncestor( - node: SnapshotNode, - byIndex: Map, -): number | undefined { - let current = typeof node.parentIndex === 'number' ? byIndex.get(node.parentIndex) : undefined; - while (current) { - if (normalizeType(current.type ?? '') === 'window') return current.index; - current = - typeof current.parentIndex === 'number' ? byIndex.get(current.parentIndex) : undefined; - } - return undefined; -} - -function hasAncestorIn( - node: SnapshotNode, - byIndex: Map, - ancestorIndexes: ReadonlySet, -): boolean { - let current = typeof node.parentIndex === 'number' ? byIndex.get(node.parentIndex) : undefined; - while (current) { - if (ancestorIndexes.has(current.index)) return true; - current = - typeof current.parentIndex === 'number' ? byIndex.get(current.parentIndex) : undefined; - } - return false; -} - /** * Truncation policy: added lines win. They carry the settled tree's fresh * refs — the actionable half of the diff — while removals only describe what diff --git a/src/core/__tests__/snapshot-chrome.test.ts b/src/core/__tests__/snapshot-chrome.test.ts new file mode 100644 index 000000000..ce626c2c8 --- /dev/null +++ b/src/core/__tests__/snapshot-chrome.test.ts @@ -0,0 +1,202 @@ +import { test } from 'vitest'; +import assert from 'node:assert/strict'; +import { attachRefs, type RawSnapshotNode, type SnapshotNode } from '../../kernel/snapshot.ts'; +import { collectSettleChromeRefs } from '../snapshot-chrome.ts'; + +function refFor(nodes: SnapshotNode[], label: string): string { + const node = nodes.find((candidate) => candidate.label === label); + assert.ok(node?.ref, `expected a node labelled "${label}" with a ref`); + return node.ref; +} + +// iOS keyboard window (iPhone 17 Pro shape): the [Keyboard] container holds the +// keys, and the "Next keyboard"/"Dictate" assistant buttons render inside the +// keyboard's own frame at the bottom of the screen. An inputAccessoryView +// toolbar the app hosts in the same window renders as a bar ABOVE the keys. +function iosKeyboardWindowNodes(options: { withAccessory: boolean }): RawSnapshotNode[] { + const keyboardTop = 360; + const keys = Array.from({ length: 26 }, (_, key) => ({ + index: 100 + key, + depth: 4, + parentIndex: 5, + type: 'Key', + label: String.fromCharCode(97 + key), + rect: { + x: (key % 10) * 40, + y: keyboardTop + 20 + Math.floor(key / 10) * 54, + width: 39, + height: 54, + }, + })); + return [ + { + index: 0, + depth: 0, + type: 'Application', + label: 'Example', + rect: { x: 0, y: 0, width: 402, height: 874 }, + hittable: true, + }, + { + index: 1, + depth: 1, + parentIndex: 0, + type: 'Window', + label: 'Keyboard Window', + rect: { x: 0, y: 300, width: 402, height: 574 }, + }, + ...(options.withAccessory + ? [ + { + index: 2, + depth: 2, + parentIndex: 1, + type: 'Other', + label: 'Accessory Bar', + rect: { x: 0, y: 300, width: 402, height: 44 }, + }, + { + index: 3, + depth: 3, + parentIndex: 2, + type: 'Button', + label: 'Send', + identifier: 'composer-send', + rect: { x: 300, y: 306, width: 90, height: 36 }, + hittable: true, + }, + ] + : []), + { + index: 4, + depth: 2, + parentIndex: 1, + type: 'Other', + label: 'Keyboard Host', + rect: { x: 0, y: keyboardTop, width: 402, height: 514 }, + }, + { + index: 5, + depth: 3, + parentIndex: 4, + type: 'Keyboard', + label: 'Padding-Left', + rect: { x: 0, y: keyboardTop, width: 402, height: 514 }, + }, + ...keys, + { + index: 200, + depth: 3, + parentIndex: 4, + type: 'Button', + label: 'Next keyboard', + rect: { x: 8, y: 806, width: 68, height: 44 }, + hittable: true, + }, + { + index: 201, + depth: 3, + parentIndex: 4, + type: 'Button', + label: 'Dictate', + rect: { x: 325, y: 805, width: 68, height: 44 }, + }, + ]; +} + +test('iOS app-owned inputAccessoryView controls above the keyboard survive chrome classification', () => { + const nodes = attachRefs(iosKeyboardWindowNodes({ withAccessory: true })); + const chromeRefs = collectSettleChromeRefs(nodes, undefined); + + // Keyboard keys and the system assistant buttons ARE chrome. + assert.equal(chromeRefs.has(refFor(nodes, 'q')), true); + assert.equal(chromeRefs.has(refFor(nodes, 'Next keyboard')), true); + assert.equal(chromeRefs.has(refFor(nodes, 'Dictate')), true); + // The app's accessory "Send" button (rendered above the keys) is NOT chrome. + assert.equal(chromeRefs.has(refFor(nodes, 'Send')), false); +}); + +test('iOS genuine key-only keyboard window still classifies every keyboard control as chrome', () => { + const nodes = attachRefs(iosKeyboardWindowNodes({ withAccessory: false })); + const chromeRefs = collectSettleChromeRefs(nodes, undefined); + + assert.equal(chromeRefs.has(refFor(nodes, 'q')), true); + assert.equal(chromeRefs.has(refFor(nodes, 'Next keyboard')), true); + assert.equal(chromeRefs.has(refFor(nodes, 'Dictate')), true); + // The keyboard window node itself is chrome (structural spine, never an app target). + assert.equal(chromeRefs.has(refFor(nodes, 'Keyboard Window')), true); +}); + +const LATIN_IME = 'com.google.android.inputmethod.latin'; + +test('Android classifier filters IME keys but keeps app dialog and unmarked SystemUI controls', () => { + const nodes = attachRefs([ + { + index: 0, + depth: 0, + type: 'android.widget.FrameLayout', + bundleId: 'com.example.app', + rect: { x: 0, y: 0, width: 1080, height: 2400 }, + }, + { + index: 1, + depth: 1, + parentIndex: 0, + type: 'android.widget.Button', + label: 'Submit', + bundleId: 'com.example.app', + rect: { x: 40, y: 200, width: 1000, height: 140 }, + hittable: true, + }, + // System dialog button (package `android`) — a legitimate actionable control. + { + index: 2, + depth: 1, + parentIndex: 0, + type: 'android.widget.Button', + label: 'Allow', + bundleId: 'android', + identifier: 'android:id/button1', + rect: { x: 600, y: 1200, width: 400, height: 140 }, + hittable: true, + }, + // Unmarked SystemUI overlay control (volume panel), NOT status/nav-bar chrome. + { + index: 3, + depth: 1, + parentIndex: 0, + type: 'android.widget.ImageButton', + label: 'Volume', + bundleId: 'com.android.systemui', + identifier: 'com.android.systemui:id/volume_dialog_button', + rect: { x: 980, y: 900, width: 80, height: 80 }, + hittable: true, + }, + // IME container + a key: both IME-owned, both chrome. + { + index: 4, + depth: 1, + parentIndex: 0, + type: 'android.inputmethodservice.SoftInputWindow', + bundleId: LATIN_IME, + rect: { x: 0, y: 1600, width: 1080, height: 800 }, + }, + { + index: 5, + depth: 2, + parentIndex: 4, + type: 'android.inputmethodservice.Keyboard$Key', + label: 'q', + bundleId: LATIN_IME, + rect: { x: 0, y: 1700, width: 108, height: 160 }, + }, + ]); + const chromeRefs = collectSettleChromeRefs(nodes, 'com.example.app'); + + // IME key IS filtered. + assert.equal(chromeRefs.has(refFor(nodes, 'q')), true); + // Legitimate app / system-dialog / unmarked-SystemUI controls are NOT filtered. + assert.equal(chromeRefs.has(refFor(nodes, 'Submit')), false); + assert.equal(chromeRefs.has(refFor(nodes, 'Allow')), false); + assert.equal(chromeRefs.has(refFor(nodes, 'Volume')), false); +}); diff --git a/src/core/snapshot-chrome.ts b/src/core/snapshot-chrome.ts new file mode 100644 index 000000000..eb401a720 --- /dev/null +++ b/src/core/snapshot-chrome.ts @@ -0,0 +1,404 @@ +import type { SnapshotNode } from '../kernel/snapshot.ts'; +import { isAndroidInputMethodSnapshotNode } from '../snapshot/android-input-method-overlays.ts'; +import { normalizeType } from '../utils/text-surface.ts'; + +/** + * Structural keyboard / IME / persistent-system chrome classifier over a + * captured `SnapshotNode[]`. Lives in `core/` (below both `commands/` and + * `daemon/`) so every ref-selection budget — `--settle`'s content-first diff + * and unchanged-interactive tail (#1101/#1167/#1198/#1200) AND the replay + * divergence `screen.refs` cap (ADR 0012 decision 6) — reuses the SAME + * classification instead of duplicating a keyboard/IME node-type list. + * + * Detection is deliberately structural (element type + `parentIndex` chains + + * Android `bundleId`/resource-id markers), never label-based: keyboard chrome + * text is locale-dependent. + */ + +// Editable text roles (normalized `node.type` vocabulary) used by the +// keyboard-window guard below. +const EDITABLE_TEXT_TYPES = new Set([ + 'textfield', + 'securetextfield', + 'textview', + 'textarea', + 'searchfield', + 'edittext', +]); + +/** + * Keyboard chrome classification, live-verified against the iPhone 17 Pro + * simulator (iOS 26, July 2026): the software keyboard renders in its OWN + * dedicated window (UIRemoteKeyboardWindow). That window contains the + * `[Keyboard]` container (keys plus real XCUIElementTypeButton chrome like + * shift/Emoji/return) AND a SIBLING subtree holding the "Next keyboard" and + * "Dictate" buttons — siblings of the container, so a container-descendant + * walk alone provably misses them. The rule is therefore: every node inside + * a window that has a `[Keyboard]` descendant is keyboard chrome. + * + * Detection stays structural (`parentIndex` chains), never label-based: + * keyboard chrome text is locale-dependent (the verified capture had Polish + * key labels) and a label list would silently stop matching under a + * different input language. + * + * Conservative guard: a window that also hosts an editable text node OUTSIDE + * the keyboard container is never window-classified — iOS hosts + * inputAccessoryView content (e.g. a messaging composer) in the keyboard + * window, and hiding the field the user is typing into would be worse than + * leaking chrome. Such windows fall back to the container-descendant walk. + * + * App-owned accessory guard: even when whole-window classification IS applied + * (a button-only accessory has no editable text to trip the guard above), an + * `inputAccessoryView` / formatting / send toolbar the app hosts in the + * keyboard window is a legitimate control the agent must still see. All of the + * keyboard's OWN chrome — keys, shift/Emoji/return, and the "Next keyboard" / + * "Dictate" assistant buttons — renders within the keyboard container's own + * frame (bottom of the screen), while an inputAccessoryView renders as a bar + * ABOVE the keys. So any non-keyboard node in the keyboard window whose center + * sits above the keyboard container's top edge is app-owned and survives + * classification (`collectKeyboardAccessoryIndexes`). Structural spine nodes + * (the window and wrappers that CONTAIN the keyboard) are never exempted, so a + * genuine key-only keyboard window is classified exactly as before. + */ +type SettleChrome = { + /** Chrome node indexes EXCLUDING the containers: stripped from the diff. */ + strippedIndexes: ReadonlySet; + /** Refs of ALL chrome incl. containers/window: trigger + tail exclusion. */ + refs: ReadonlySet; +}; + +const EMPTY_SETTLE_CHROME: SettleChrome = { strippedIndexes: new Set(), refs: new Set() }; + +function collectKeyboardChrome(nodes: SnapshotNode[]): SettleChrome { + const containerIndexes = new Set( + nodes.filter((node) => normalizeType(node.type ?? '') === 'keyboard').map((node) => node.index), + ); + if (containerIndexes.size === 0) return EMPTY_SETTLE_CHROME; + const byIndex = new Map(nodes.map((node) => [node.index, node])); + const containerSubtreeIndexes = collectSubtreeIndexes(nodes, byIndex, containerIndexes); + const chromeIndexes = new Set(containerSubtreeIndexes); + const windowIndexes = resolveKeyboardWindowIndexes(nodes, byIndex, containerSubtreeIndexes); + for (const index of collectSubtreeIndexes(nodes, byIndex, windowIndexes)) { + chromeIndexes.add(index); + } + // Preserve app-owned accessory controls hosted above the keyboard: they were + // swept in by the whole-window pass but are not keyboard chrome. + for (const index of collectKeyboardAccessoryIndexes( + nodes, + byIndex, + windowIndexes, + containerIndexes, + containerSubtreeIndexes, + )) { + chromeIndexes.delete(index); + } + const refs = new Set( + nodes.filter((node) => node.ref && chromeIndexes.has(node.index)).map((node) => node.ref), + ); + // The [Keyboard] container line itself survives the diff so "keyboard + // appeared/left" stays one visible signal line; everything else collapses. + const strippedIndexes = new Set( + [...chromeIndexes].filter((index) => !containerIndexes.has(index)), + ); + return { strippedIndexes, refs }; +} + +/** + * Indexes of app-owned accessory content in a keyboard window: non-keyboard + * nodes whose center sits ABOVE the keyboard container's top edge (an + * `inputAccessoryView` toolbar renders above the keys). Structural spine nodes + * that CONTAIN the keyboard (the window and its wrappers) are excluded — their + * frames span the keys, and exempting them would leak the keyboard itself. + */ +function collectKeyboardAccessoryIndexes( + nodes: SnapshotNode[], + byIndex: Map, + windowIndexes: ReadonlySet, + containerIndexes: ReadonlySet, + containerSubtreeIndexes: ReadonlySet, +): Set { + const exempt = new Set(); + if (windowIndexes.size === 0) return exempt; + const keyboardTopByWindow = collectKeyboardTopByWindow(byIndex, windowIndexes, containerIndexes); + if (keyboardTopByWindow.size === 0) return exempt; + const spineIndexes = collectContainerAncestorIndexes(byIndex, containerIndexes); + for (const node of nodes) { + const eligible = + !containerSubtreeIndexes.has(node.index) && !spineIndexes.has(node.index) && node.rect; + if (eligible && isAboveKeyboard(node, byIndex, keyboardTopByWindow)) exempt.add(node.index); + } + return exempt; +} + +/** True when the node renders above its keyboard window's key area (an accessory bar). */ +function isAboveKeyboard( + node: SnapshotNode, + byIndex: Map, + keyboardTopByWindow: ReadonlyMap, +): boolean { + if (!node.rect) return false; + const window = findNearestWindowAncestor(node, byIndex); + if (window === undefined) return false; + const keyboardTop = keyboardTopByWindow.get(window); + return keyboardTop !== undefined && node.rect.y + node.rect.height / 2 < keyboardTop; +} + +/** Per keyboard window, the top edge (min `rect.y`) of its keyboard container(s). */ +function collectKeyboardTopByWindow( + byIndex: Map, + windowIndexes: ReadonlySet, + containerIndexes: ReadonlySet, +): Map { + const keyboardTopByWindow = new Map(); + for (const containerIndex of containerIndexes) { + const container = byIndex.get(containerIndex); + if (!container?.rect) continue; + const window = findNearestWindowAncestor(container, byIndex); + if (window === undefined || !windowIndexes.has(window)) continue; + const currentTop = keyboardTopByWindow.get(window); + if (currentTop === undefined || container.rect.y < currentTop) { + keyboardTopByWindow.set(window, container.rect.y); + } + } + return keyboardTopByWindow; +} + +/** Every ancestor of every keyboard container (the structural spine that hosts it). */ +function collectContainerAncestorIndexes( + byIndex: Map, + containerIndexes: ReadonlySet, +): Set { + const ancestors = new Set(); + for (const containerIndex of containerIndexes) { + const container = byIndex.get(containerIndex); + let current = + container && typeof container.parentIndex === 'number' + ? byIndex.get(container.parentIndex) + : undefined; + while (current) { + ancestors.add(current.index); + current = + typeof current.parentIndex === 'number' ? byIndex.get(current.parentIndex) : undefined; + } + } + return ancestors; +} + +/** The root indexes plus every node whose parent chain passes through one. */ +function collectSubtreeIndexes( + nodes: SnapshotNode[], + byIndex: Map, + rootIndexes: ReadonlySet, +): Set { + const indexes = new Set(rootIndexes); + if (rootIndexes.size === 0) return indexes; + for (const node of nodes) { + if (hasAncestorIn(node, byIndex, rootIndexes)) indexes.add(node.index); + } + return indexes; +} + +// SystemUI hosts BOTH persistent chrome and actionable overlays (volume +// panel, media/output pickers), so chrome is never a package-level fact. +// Within `com.android.systemui`, only window-runs carrying a status-bar or +// navigation-bar marker resource-id drop; every other systemui surface is +// kept. Marker set live-verified on the emulator: the status-bar window +// carries `status_bar*` ids throughout while the VolumeDialog window carries +// only `volume_dialog*` ids (`input_method_nav*` bars are IME-owned and +// handled by the IME tier). +const ANDROID_SYSTEM_CHROME_PACKAGE = 'com.android.systemui'; +const ANDROID_SYSTEM_CHROME_MARKER_PREFIXES = [ + 'com.android.systemui:id/status_bar', + 'com.android.systemui:id/navigation_bar', +]; + +function hasAndroidSystemChromeMarker(node: SnapshotNode): boolean { + const identifier = node.identifier ?? ''; + return ANDROID_SYSTEM_CHROME_MARKER_PREFIXES.some((prefix) => identifier.startsWith(prefix)); +} + +/** + * Android settle chrome (#1198): IME-owned nodes collapse to one surviving + * line per contiguous run; systemui status/nav-bar window-runs drop from both + * diff sides; every other foreign node — system dialogs (package `android`), + * permission prompts, AND actionable systemui overlays like the volume panel + * — is kept in full. Constraint: package membership is strictly per-node by + * the node's own `bundleId` — parentIndex chains can cross windows on Android + * (enforced by the settle.test.ts cross-window regression test); run grouping + * only ever walks parent chains BETWEEN same-package nodes, so it cannot + * swallow another package's node. Inert on iOS/macOS: those nodes never set + * `bundleId`. + */ +function collectAndroidSettleChrome( + nodes: SnapshotNode[], + appBundleId: string | undefined, +): SettleChrome { + const byIndex = new Map(nodes.map((node) => [node.index, node])); + const imeIndexes = new Set( + nodes.filter((node) => isAndroidInputMethodSnapshotNode(node)).map((node) => node.index), + ); + const imeContainerIndexes = new Set( + [...imeIndexes].filter((index) => { + const parentIndex = byIndex.get(index)?.parentIndex; + const parent = parentIndex !== undefined ? byIndex.get(parentIndex) : undefined; + return !parent || !imeIndexes.has(parent.index); + }), + ); + // appBundleId is the session's pre-action value (not refreshed inside the + // settle loop); it is only a never-drop-the-app-under-test guard here, so + // staleness cannot hide a foreign dialog. + const systemChromeIndexes = + appBundleId === ANDROID_SYSTEM_CHROME_PACKAGE + ? new Set() + : collectAndroidSystemChromeRunIndexes(nodes, byIndex, imeIndexes); + // The one surviving container line per IME run; the rest of the run and all + // status/nav-bar chrome never spend diff/tail budget. + const strippedIndexes = new Set( + [...imeIndexes].filter((index) => !imeContainerIndexes.has(index)), + ); + for (const index of systemChromeIndexes) strippedIndexes.add(index); + const refs = new Set( + nodes + .filter( + (node) => node.ref && (imeIndexes.has(node.index) || systemChromeIndexes.has(node.index)), + ) + .map((node) => node.ref), + ); + if (strippedIndexes.size === 0 && refs.size === 0) return EMPTY_SETTLE_CHROME; + return { strippedIndexes, refs }; +} + +/** + * Systemui window-runs (contiguous same-package parent chains) that contain a + * status/nav-bar marker anywhere in the run. The whole marked run drops — + * unmarked wrappers above `status_bar_container` churn with the bar itself — + * while unmarked runs (volume panel, media pickers) are kept whole. + */ +function collectAndroidSystemChromeRunIndexes( + nodes: SnapshotNode[], + byIndex: Map, + imeIndexes: ReadonlySet, +): Set { + const systemUiIndexes = new Set( + nodes + .filter( + (node) => node.bundleId === ANDROID_SYSTEM_CHROME_PACKAGE && !imeIndexes.has(node.index), + ) + .map((node) => node.index), + ); + if (systemUiIndexes.size === 0) return new Set(); + // Union-find-lite: each systemui node resolves to its run root (the nearest + // ancestor chain member whose parent is absent or not systemui). + const runRootByIndex = new Map(); + const resolveRunRoot = (index: number): number => { + const cached = runRootByIndex.get(index); + if (cached !== undefined) return cached; + const parentIndex = byIndex.get(index)?.parentIndex; + const root = + parentIndex !== undefined && systemUiIndexes.has(parentIndex) + ? resolveRunRoot(parentIndex) + : index; + runRootByIndex.set(index, root); + return root; + }; + const markedRunRoots = new Set( + [...systemUiIndexes] + .filter((index) => { + const node = byIndex.get(index); + return node !== undefined && hasAndroidSystemChromeMarker(node); + }) + .map((index) => resolveRunRoot(index)), + ); + return new Set([...systemUiIndexes].filter((index) => markedRunRoots.has(resolveRunRoot(index)))); +} + +/** iOS keyboard-window chrome unioned with Android IME/system chrome. */ +function collectSettleChrome(nodes: SnapshotNode[], appBundleId: string | undefined): SettleChrome { + const keyboard = collectKeyboardChrome(nodes); + const android = collectAndroidSettleChrome(nodes, appBundleId); + if (keyboard.strippedIndexes.size === 0 && keyboard.refs.size === 0) return android; + if (android.strippedIndexes.size === 0 && android.refs.size === 0) return keyboard; + return { + strippedIndexes: new Set([...keyboard.strippedIndexes, ...android.strippedIndexes]), + refs: new Set([...keyboard.refs, ...android.refs]), + }; +} + +export function withoutSettleChrome( + nodes: SnapshotNode[], + appBundleId: string | undefined, +): SnapshotNode[] { + const { strippedIndexes } = collectSettleChrome(nodes, appBundleId); + if (strippedIndexes.size === 0) return nodes; + return nodes.filter((node) => !strippedIndexes.has(node.index)); +} + +/** + * Refs of iOS keyboard-window chrome unioned with Android IME/system chrome + * (see `collectSettleChrome`). Used by any ref-selection budget — the settle + * tail AND the replay divergence `screen.refs` cap (ADR 0012 decision 6) — to + * exclude keyboard/IME chrome without duplicating the structural classification. + */ +export function collectSettleChromeRefs( + nodes: SnapshotNode[], + appBundleId: string | undefined, +): ReadonlySet { + return collectSettleChrome(nodes, appBundleId).refs; +} + +/** + * Windows eligible for whole-window chrome classification: nearest `[window]` + * ancestor of each `[Keyboard]` container, minus windows hosting editable + * text outside the container subtree (the inputAccessoryView guard above). + */ +function resolveKeyboardWindowIndexes( + nodes: SnapshotNode[], + byIndex: Map, + containerChromeIndexes: ReadonlySet, +): Set { + const windowIndexes = new Set(); + for (const index of containerChromeIndexes) { + const node = byIndex.get(index); + if (!node || normalizeType(node.type ?? '') !== 'keyboard') continue; + const windowAncestor = findNearestWindowAncestor(node, byIndex); + if (windowAncestor !== undefined) windowIndexes.add(windowAncestor); + } + for (const windowIndex of windowIndexes) { + const windowSet = new Set([windowIndex]); + const hostsEditableText = nodes.some( + (node) => + EDITABLE_TEXT_TYPES.has(normalizeType(node.type ?? '')) && + !containerChromeIndexes.has(node.index) && + hasAncestorIn(node, byIndex, windowSet), + ); + if (hostsEditableText) windowIndexes.delete(windowIndex); + } + return windowIndexes; +} + +function findNearestWindowAncestor( + node: SnapshotNode, + byIndex: Map, +): number | undefined { + let current = typeof node.parentIndex === 'number' ? byIndex.get(node.parentIndex) : undefined; + while (current) { + if (normalizeType(current.type ?? '') === 'window') return current.index; + current = + typeof current.parentIndex === 'number' ? byIndex.get(current.parentIndex) : undefined; + } + return undefined; +} + +function hasAncestorIn( + node: SnapshotNode, + byIndex: Map, + ancestorIndexes: ReadonlySet, +): boolean { + let current = typeof node.parentIndex === 'number' ? byIndex.get(node.parentIndex) : undefined; + while (current) { + if (ancestorIndexes.has(current.index)) return true; + current = + typeof current.parentIndex === 'number' ? byIndex.get(current.parentIndex) : undefined; + } + return false; +} diff --git a/src/daemon/handlers/__tests__/session-replay-divergence.test.ts b/src/daemon/handlers/__tests__/session-replay-divergence.test.ts index acf535811..29e080475 100644 --- a/src/daemon/handlers/__tests__/session-replay-divergence.test.ts +++ b/src/daemon/handlers/__tests__/session-replay-divergence.test.ts @@ -69,3 +69,210 @@ test('buildReplayFailureDivergence dedupes suggestions using the strongest basis expect(divergence.suggestions[0]?.ref).toBe('e1'); expect(divergence.suggestions[0]?.basis).toBe('id'); }); + +// Live-shape fixture (iPhone 17 Pro sim, iOS 26): the software keyboard +// renders in its own window ([Window] > [Keyboard] > 26 [Key] children). +// Onscreen, that subtree alone blows past SCREEN_REF_CAPTURE_LIMIT (20), +// which is exactly the bug: the real actionable target (a sibling of the +// keyboard window, not inside it) sorts after the keys in document order and +// gets truncated out unless keyboard/IME chrome is excluded from the budget +// first. +function keyboardSwampedNodes(actionableLabel: string) { + const keyboardWindowIndex = 1; + const keyboardContainerIndex = 2; + const keys = Array.from({ length: 26 }, (_, key) => ({ + index: 3 + key, + depth: 3, + parentIndex: keyboardContainerIndex, + type: 'Key', + label: String.fromCharCode(97 + key), + rect: { x: (key % 10) * 40, y: 600 + Math.floor(key / 10) * 54, width: 39, height: 54 }, + hittable: true, + })); + return [ + { + index: 0, + depth: 0, + type: 'Application', + label: 'Example', + rect: { x: 0, y: 0, width: 402, height: 874 }, + hittable: true, + }, + { + index: keyboardWindowIndex, + depth: 1, + parentIndex: 0, + type: 'Window', + label: 'Keyboard Window', + rect: { x: 0, y: 583, width: 402, height: 291 }, + hittable: true, + }, + { + index: keyboardContainerIndex, + depth: 2, + parentIndex: keyboardWindowIndex, + type: 'Keyboard', + label: 'Padding-Left', + rect: { x: 0, y: 583, width: 402, height: 291 }, + }, + ...keys, + { + index: 3 + keys.length, + depth: 1, + parentIndex: 0, + type: 'Button', + label: actionableLabel, + identifier: 'push-article', + rect: { x: 20, y: 100, width: 160, height: 44 }, + hittable: true, + }, + ]; +} + +test('buildReplayFailureDivergence excludes keyboard chrome from screen.refs and surfaces the actionable target within the cap', async () => { + const root = fs.mkdtempSync(path.join(os.tmpdir(), 'agent-device-replay-divergence-keyboard-')); + const sessionStore = new SessionStore(path.join(root, 'sessions')); + const sessionName = 'default'; + sessionStore.set(sessionName, makeIosSession(sessionName, { appBundleId: 'com.example.app' })); + + mockDispatchCommand.mockResolvedValue({ + nodes: keyboardSwampedNodes('Push Article'), + truncated: false, + backend: 'xctest', + }); + + const action = { + ts: 0, + command: 'press', + positionals: ['label="Push Article"'], + flags: {}, + result: { selectorChain: ['label="Push Article"'] }, + }; + const divergence = await buildReplayFailureDivergence({ + error: { code: 'COMMAND_FAILED', message: 'target not found' }, + action, + index: 0, + sourcePath: path.join(root, 'flow.ad'), + sourceLine: 1, + session: sessionStore.get(sessionName), + sessionName, + sessionStore, + logPath: path.join(root, 'daemon.log'), + responseLevel: 'default', + planActions: [action], + planDigest: 'test-plan-digest', + }); + + expect(divergence.screen.state).toBe('available'); + const screen = divergence.screen as Extract; + // No keyboard/IME chrome (container or individual keys) reached the ref list. + expect(screen.refs.some((ref) => ref.role.toLowerCase() === 'key')).toBe(false); + expect(screen.refs.some((ref) => ref.role.toLowerCase() === 'keyboard')).toBe(false); + // The real actionable target — pushed past the cap by 28 keyboard nodes + // before this fix — is now surfaced. + expect(screen.refs.some((ref) => ref.label === 'Push Article')).toBe(true); + expect(screen.truncated).toBeUndefined(); +}); + +// Same live-shape keyboard window, but the app hosts an inputAccessoryView +// toolbar (a "Send" button) as a bar ABOVE the keys inside the keyboard +// window. The keys are still chrome; the app-owned accessory control must NOT +// be filtered — otherwise the agent can't see/heal a control that lives there. +function keyboardWithAccessoryNodes() { + const keyboardTop = 583; + const keys = Array.from({ length: 26 }, (_, key) => ({ + index: 10 + key, + depth: 3, + parentIndex: 2, + type: 'Key', + label: String.fromCharCode(97 + key), + rect: { + x: (key % 10) * 40, + y: keyboardTop + 10 + Math.floor(key / 10) * 54, + width: 39, + height: 54, + }, + hittable: true, + })); + return [ + { + index: 0, + depth: 0, + type: 'Application', + label: 'Example', + rect: { x: 0, y: 0, width: 402, height: 874 }, + hittable: true, + }, + { + index: 1, + depth: 1, + parentIndex: 0, + type: 'Window', + label: 'Keyboard Window', + rect: { x: 0, y: 539, width: 402, height: 335 }, + hittable: true, + }, + // App inputAccessoryView toolbar, rendered above the keys. + { + index: 2, + depth: 2, + parentIndex: 1, + type: 'Keyboard', + label: 'Padding-Left', + rect: { x: 0, y: keyboardTop, width: 402, height: 291 }, + }, + ...keys, + { + index: 40, + depth: 2, + parentIndex: 1, + type: 'Button', + label: 'Send', + identifier: 'composer-send', + rect: { x: 320, y: 545, width: 74, height: 40 }, + hittable: true, + }, + ]; +} + +test('buildReplayFailureDivergence keeps an app inputAccessoryView control in screen.refs while filtering keyboard keys', async () => { + const root = fs.mkdtempSync(path.join(os.tmpdir(), 'agent-device-replay-divergence-accessory-')); + const sessionStore = new SessionStore(path.join(root, 'sessions')); + const sessionName = 'default'; + sessionStore.set(sessionName, makeIosSession(sessionName, { appBundleId: 'com.example.app' })); + + mockDispatchCommand.mockResolvedValue({ + nodes: keyboardWithAccessoryNodes(), + truncated: false, + backend: 'xctest', + }); + + const action = { + ts: 0, + command: 'press', + positionals: ['label="Send"'], + flags: {}, + result: { selectorChain: ['label="Send"'] }, + }; + const divergence = await buildReplayFailureDivergence({ + error: { code: 'COMMAND_FAILED', message: 'target not found' }, + action, + index: 0, + sourcePath: path.join(root, 'flow.ad'), + sourceLine: 1, + session: sessionStore.get(sessionName), + sessionName, + sessionStore, + logPath: path.join(root, 'daemon.log'), + responseLevel: 'default', + planActions: [action], + planDigest: 'test-plan-digest', + }); + + expect(divergence.screen.state).toBe('available'); + const screen = divergence.screen as Extract; + // Keyboard keys are still filtered. + expect(screen.refs.some((ref) => ref.role.toLowerCase() === 'key')).toBe(false); + // The app's accessory "Send" button survives and is available to heal against. + expect(screen.refs.some((ref) => ref.label === 'Send')).toBe(true); +}); diff --git a/src/daemon/handlers/session-replay-divergence.ts b/src/daemon/handlers/session-replay-divergence.ts index e818ffbf9..c9c076d5d 100644 --- a/src/daemon/handlers/session-replay-divergence.ts +++ b/src/daemon/handlers/session-replay-divergence.ts @@ -16,6 +16,7 @@ import { type Selector, } from '../../selectors/index.ts'; import { collectReplaySelectorCandidates } from './session-replay-heal.ts'; +import { collectSettleChromeRefs } from '../../core/snapshot-chrome.ts'; import { buildReplayDivergenceResume } from './session-replay-resume.ts'; import { formatDivergenceActionLabel, isTouchTargetCommand } from '../../replay/script-utils.ts'; import { @@ -157,7 +158,13 @@ export function boundReplayDivergenceForSession(params: { } export type DivergenceObservation = - | { state: 'available'; nodes: SnapshotNode[]; refsGeneration: number } + | { + state: 'available'; + nodes: SnapshotNode[]; + refsGeneration: number; + /** Session's app bundle id at capture time; threaded to `buildDivergenceScreen`'s chrome filter (Android IME-scope guard — inert on iOS). */ + appBundleId: string | undefined; + } | { state: 'unavailable'; reason: string; hint: string }; /** Adapts a capture observation to the `repairHint` container-presence test's input shape. */ @@ -216,6 +223,7 @@ export async function captureDivergenceObservation(params: { state: 'available', nodes: snapshot.nodes, refsGeneration: session.snapshotGeneration ?? 0, + appBundleId: session.appBundleId, }; } catch (error) { return { @@ -249,7 +257,11 @@ export function buildDivergenceScreen( hint: sanitize(observation.hint), }; } - const { refs, truncated } = buildReplayDivergenceScreenRefs(observation.nodes, sanitize); + const { refs, truncated } = buildReplayDivergenceScreenRefs( + observation.nodes, + sanitize, + observation.appBundleId, + ); return { state: 'available', refsGeneration: observation.refsGeneration, @@ -265,11 +277,18 @@ const SCREEN_REF_CAPTURE_LIMIT = 20; function buildReplayDivergenceScreenRefs( nodes: SnapshotNode[], sanitize: DivergenceFieldSanitizer, + appBundleId: string | undefined, ): { refs: ReplayDivergenceScreenRef[]; truncated: boolean; } { - const candidates = nodes.filter((node) => node.ref && node.interactionBlocked !== 'covered'); + // Keyboard/IME chrome must not consume the ref budget: it reuses the exact + // structural classifier `--settle`'s tail already relies on (#1198/#1200) + // rather than a second keyboard/IME node-type list. + const chromeRefs = collectSettleChromeRefs(nodes, appBundleId); + const candidates = nodes.filter( + (node) => node.ref && node.interactionBlocked !== 'covered' && !chromeRefs.has(node.ref), + ); const refs = candidates.slice(0, SCREEN_REF_CAPTURE_LIMIT).map((node) => { const role = formatRole(node.type ?? 'Element'); const label = displayLabel(node, role);