Skip to content

fix: exclude keyboard/IME chrome from replay divergence screen.refs #1233

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Merged
thymikee merged 3 commits into from
Jul 13, 2026
+836 −284
Merged

fix: exclude keyboard/IME chrome from replay divergence screen.refs #1233

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
3 commits
Select commit Hold shift + click to select a range
a80a3b7
fix: exclude keyboard/IME chrome from replay divergence screen.refs
thymikee Jul 13, 2026
6029951
refactor: move settle chrome classifier to core/ for cross-layer reuse
thymikee Jul 13, 2026
dfdeaba
fix: preserve app-owned keyboard accessory controls in chrome filter
thymikee Jul 13, 2026
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
282 changes: 1 addition & 281 deletions src/commands/interaction/runtime/settle.ts
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -7,8 +7,7 @@ import type {
import { isSparseSnapshotQualityVerdict } from '../../../snapshot/snapshot-quality.ts';
import { buildSnapshotDiff } from '../../../snapshot/snapshot-diff.ts';
import { displayLabel, formatRole } from '../../../snapshot/snapshot-lines.ts';
import { isAndroidInputMethodSnapshotNode } from '../../../snapshot/android-input-method-overlays.ts';
import { normalizeType } from '../../../utils/text-surface.ts';
import { collectSettleChromeRefs, withoutSettleChrome } from '../../../core/snapshot-chrome.ts';
import { summarizeAxEvidence } from '../../../utils/ax-digest.ts';
import type {
InteractionEvidence,
Expand Down Expand Up @@ -279,285 +278,6 @@ export function buildSettleTailEntries(
};
}

// Editable text roles (normalized `node.type` vocabulary) used by the
// keyboard-window guard below.
const EDITABLE_TEXT_TYPES = new Set([
'textfield',
'securetextfield',
'textview',
'textarea',
'searchfield',
'edittext',
]);

/**
* Keyboard chrome classification, live-verified against the iPhone 17 Pro
* simulator (iOS 26, July 2026): the software keyboard renders in its OWN
* dedicated window (UIRemoteKeyboardWindow). That window contains the
* `[Keyboard]` container (keys plus real XCUIElementTypeButton chrome like
* shift/Emoji/return) AND a SIBLING subtree holding the "Next keyboard" and
* "Dictate" buttons — siblings of the container, so a container-descendant
* walk alone provably misses them. The rule is therefore: every node inside
* a window that has a `[Keyboard]` descendant is keyboard chrome.
*
* Detection stays structural (`parentIndex` chains), never label-based:
* keyboard chrome text is locale-dependent (the verified capture had Polish
* key labels) and a label list would silently stop matching under a
* different input language.
*
* Conservative guard: a window that also hosts an editable text node OUTSIDE
* the keyboard container is never window-classified — iOS hosts
* inputAccessoryView content (e.g. a messaging composer) in the keyboard
* window, and hiding the field the user is typing into would be worse than
* leaking chrome. Such windows fall back to the container-descendant walk.
*/
type KeyboardChrome = {
/** Chrome node indexes EXCLUDING the containers: stripped from the diff. */
strippedIndexes: ReadonlySet<number>;
/** Refs of ALL chrome incl. containers/window: trigger + tail exclusion. */
refs: ReadonlySet<string>;
};

const EMPTY_KEYBOARD_CHROME: KeyboardChrome = { strippedIndexes: new Set(), refs: new Set() };

function collectKeyboardChrome(nodes: SnapshotNode[]): KeyboardChrome {
const containerIndexes = new Set(
nodes.filter((node) => normalizeType(node.type ?? '') === 'keyboard').map((node) => node.index),
);
if (containerIndexes.size === 0) return EMPTY_KEYBOARD_CHROME;
const byIndex = new Map(nodes.map((node) => [node.index, node]));
const chromeIndexes = collectSubtreeIndexes(nodes, byIndex, containerIndexes);
const windowIndexes = resolveKeyboardWindowIndexes(nodes, byIndex, chromeIndexes);
for (const index of collectSubtreeIndexes(nodes, byIndex, windowIndexes)) {
chromeIndexes.add(index);
}
const refs = new Set(
nodes.filter((node) => node.ref && chromeIndexes.has(node.index)).map((node) => node.ref),
);
// The [Keyboard] container line itself survives the diff so "keyboard
// appeared/left" stays one visible signal line; everything else collapses.
const strippedIndexes = new Set(
[...chromeIndexes].filter((index) => !containerIndexes.has(index)),
);
return { strippedIndexes, refs };
}

/** The root indexes plus every node whose parent chain passes through one. */
function collectSubtreeIndexes(
nodes: SnapshotNode[],
byIndex: Map<number, SnapshotNode>,
rootIndexes: ReadonlySet<number>,
): Set<number> {
const indexes = new Set(rootIndexes);
if (rootIndexes.size === 0) return indexes;
for (const node of nodes) {
if (hasAncestorIn(node, byIndex, rootIndexes)) indexes.add(node.index);
}
return indexes;
}

// SystemUI hosts BOTH persistent chrome and actionable overlays (volume
// panel, media/output pickers), so chrome is never a package-level fact.
// Within `com.android.systemui`, only window-runs carrying a status-bar or
// navigation-bar marker resource-id drop; every other systemui surface is
// kept. Marker set live-verified on the emulator: the status-bar window
// carries `status_bar*` ids throughout while the VolumeDialog window carries
// only `volume_dialog*` ids (`input_method_nav*` bars are IME-owned and
// handled by the IME tier).
const ANDROID_SYSTEM_CHROME_PACKAGE = 'com.android.systemui';
const ANDROID_SYSTEM_CHROME_MARKER_PREFIXES = [
'com.android.systemui:id/status_bar',
'com.android.systemui:id/navigation_bar',
];

function hasAndroidSystemChromeMarker(node: SnapshotNode): boolean {
const identifier = node.identifier ?? '';
return ANDROID_SYSTEM_CHROME_MARKER_PREFIXES.some((prefix) => identifier.startsWith(prefix));
}

/**
* Android settle chrome (#1198): IME-owned nodes collapse to one surviving
* line per contiguous run; systemui status/nav-bar window-runs drop from both
* diff sides; every other foreign node — system dialogs (package `android`),
* permission prompts, AND actionable systemui overlays like the volume panel
* — is kept in full. Constraint: package membership is strictly per-node by
* the node's own `bundleId` — parentIndex chains can cross windows on Android
* (enforced by the settle.test.ts cross-window regression test); run grouping
* only ever walks parent chains BETWEEN same-package nodes, so it cannot
* swallow another package's node. Inert on iOS/macOS: those nodes never set
* `bundleId`.
*/
function collectAndroidSettleChrome(
nodes: SnapshotNode[],
appBundleId: string | undefined,
): KeyboardChrome {
const byIndex = new Map(nodes.map((node) => [node.index, node]));
const imeIndexes = new Set(
nodes.filter((node) => isAndroidInputMethodSnapshotNode(node)).map((node) => node.index),
);
const imeContainerIndexes = new Set(
[...imeIndexes].filter((index) => {
const parentIndex = byIndex.get(index)?.parentIndex;
const parent = parentIndex !== undefined ? byIndex.get(parentIndex) : undefined;
return !parent || !imeIndexes.has(parent.index);
}),
);
// appBundleId is the session's pre-action value (not refreshed inside the
// settle loop); it is only a never-drop-the-app-under-test guard here, so
// staleness cannot hide a foreign dialog.
const systemChromeIndexes =
appBundleId === ANDROID_SYSTEM_CHROME_PACKAGE
? new Set<number>()
: collectAndroidSystemChromeRunIndexes(nodes, byIndex, imeIndexes);
// The one surviving container line per IME run; the rest of the run and all
// status/nav-bar chrome never spend diff/tail budget.
const strippedIndexes = new Set(
[...imeIndexes].filter((index) => !imeContainerIndexes.has(index)),
);
for (const index of systemChromeIndexes) strippedIndexes.add(index);
const refs = new Set(
nodes
.filter(
(node) => node.ref && (imeIndexes.has(node.index) || systemChromeIndexes.has(node.index)),
)
.map((node) => node.ref),
);
if (strippedIndexes.size === 0 && refs.size === 0) return EMPTY_KEYBOARD_CHROME;
return { strippedIndexes, refs };
}

/**
* Systemui window-runs (contiguous same-package parent chains) that contain a
* status/nav-bar marker anywhere in the run. The whole marked run drops —
* unmarked wrappers above `status_bar_container` churn with the bar itself —
* while unmarked runs (volume panel, media pickers) are kept whole.
*/
function collectAndroidSystemChromeRunIndexes(
nodes: SnapshotNode[],
byIndex: Map<number, SnapshotNode>,
imeIndexes: ReadonlySet<number>,
): Set<number> {
const systemUiIndexes = new Set(
nodes
.filter(
(node) => node.bundleId === ANDROID_SYSTEM_CHROME_PACKAGE && !imeIndexes.has(node.index),
)
.map((node) => node.index),
);
if (systemUiIndexes.size === 0) return new Set();
// Union-find-lite: each systemui node resolves to its run root (the nearest
// ancestor chain member whose parent is absent or not systemui).
const runRootByIndex = new Map<number, number>();
const resolveRunRoot = (index: number): number => {
const cached = runRootByIndex.get(index);
if (cached !== undefined) return cached;
const parentIndex = byIndex.get(index)?.parentIndex;
const root =
parentIndex !== undefined && systemUiIndexes.has(parentIndex)
? resolveRunRoot(parentIndex)
: index;
runRootByIndex.set(index, root);
return root;
};
const markedRunRoots = new Set(
[...systemUiIndexes]
.filter((index) => {
const node = byIndex.get(index);
return node !== undefined && hasAndroidSystemChromeMarker(node);
})
.map((index) => resolveRunRoot(index)),
);
return new Set([...systemUiIndexes].filter((index) => markedRunRoots.has(resolveRunRoot(index))));
}

/** iOS keyboard-window chrome unioned with Android IME/system chrome. */
function collectSettleChrome(
nodes: SnapshotNode[],
appBundleId: string | undefined,
): KeyboardChrome {
const keyboard = collectKeyboardChrome(nodes);
const android = collectAndroidSettleChrome(nodes, appBundleId);
if (keyboard.strippedIndexes.size === 0 && keyboard.refs.size === 0) return android;
if (android.strippedIndexes.size === 0 && android.refs.size === 0) return keyboard;
return {
strippedIndexes: new Set([...keyboard.strippedIndexes, ...android.strippedIndexes]),
refs: new Set([...keyboard.refs, ...android.refs]),
};
}

function withoutSettleChrome(
nodes: SnapshotNode[],
appBundleId: string | undefined,
): SnapshotNode[] {
const { strippedIndexes } = collectSettleChrome(nodes, appBundleId);
if (strippedIndexes.size === 0) return nodes;
return nodes.filter((node) => !strippedIndexes.has(node.index));
}

function collectSettleChromeRefs(
nodes: SnapshotNode[],
appBundleId: string | undefined,
): ReadonlySet<string> {
return collectSettleChrome(nodes, appBundleId).refs;
}

/**
* Windows eligible for whole-window chrome classification: nearest `[window]`
* ancestor of each `[Keyboard]` container, minus windows hosting editable
* text outside the container subtree (the inputAccessoryView guard above).
*/
function resolveKeyboardWindowIndexes(
nodes: SnapshotNode[],
byIndex: Map<number, SnapshotNode>,
containerChromeIndexes: ReadonlySet<number>,
): Set<number> {
const windowIndexes = new Set<number>();
for (const index of containerChromeIndexes) {
const node = byIndex.get(index);
if (!node || normalizeType(node.type ?? '') !== 'keyboard') continue;
const windowAncestor = findNearestWindowAncestor(node, byIndex);
if (windowAncestor !== undefined) windowIndexes.add(windowAncestor);
}
for (const windowIndex of windowIndexes) {
const windowSet = new Set([windowIndex]);
const hostsEditableText = nodes.some(
(node) =>
EDITABLE_TEXT_TYPES.has(normalizeType(node.type ?? '')) &&
!containerChromeIndexes.has(node.index) &&
hasAncestorIn(node, byIndex, windowSet),
);
if (hostsEditableText) windowIndexes.delete(windowIndex);
}
return windowIndexes;
}

function findNearestWindowAncestor(
node: SnapshotNode,
byIndex: Map<number, SnapshotNode>,
): number | undefined {
let current = typeof node.parentIndex === 'number' ? byIndex.get(node.parentIndex) : undefined;
while (current) {
if (normalizeType(current.type ?? '') === 'window') return current.index;
current =
typeof current.parentIndex === 'number' ? byIndex.get(current.parentIndex) : undefined;
}
return undefined;
}

function hasAncestorIn(
node: SnapshotNode,
byIndex: Map<number, SnapshotNode>,
ancestorIndexes: ReadonlySet<number>,
): boolean {
let current = typeof node.parentIndex === 'number' ? byIndex.get(node.parentIndex) : undefined;
while (current) {
if (ancestorIndexes.has(current.index)) return true;
current =
typeof current.parentIndex === 'number' ? byIndex.get(current.parentIndex) : undefined;
}
return false;
}

/**
* Truncation policy: added lines win. They carry the settled tree's fresh
* refs — the actionable half of the diff — while removals only describe what
Expand Down
Loading
Loading