diff --git a/packages/capture-kit/package.json b/packages/capture-kit/package.json index 71c3f6af49..39049c52c0 100644 --- a/packages/capture-kit/package.json +++ b/packages/capture-kit/package.json @@ -18,6 +18,10 @@ "types": "./src/ios-snapshot-planning.ts", "default": "./src/ios-snapshot-planning.ts" }, + "./ios-snapshot-engine": { + "types": "./src/ios-snapshot-engine/index.ts", + "default": "./src/ios-snapshot-engine/index.ts" + }, "./mobile-snapshot-semantics": { "types": "./src/mobile-snapshot-semantics.ts", "default": "./src/mobile-snapshot-semantics.ts" diff --git a/src/snapshot/snapshot-presentation/ios/action-shelf.ts b/packages/capture-kit/src/ios-snapshot-engine/action-shelf.ts similarity index 99% rename from src/snapshot/snapshot-presentation/ios/action-shelf.ts rename to packages/capture-kit/src/ios-snapshot-engine/action-shelf.ts index 1c62d3baec..f054637b2d 100644 --- a/src/snapshot/snapshot-presentation/ios/action-shelf.ts +++ b/packages/capture-kit/src/ios-snapshot-engine/action-shelf.ts @@ -8,7 +8,7 @@ import { isSemanticActionNode, isScrollableSnapshotType, type SnapshotTreeRuleContext, -} from '../tree.ts'; +} from './tree.ts'; const ACTION_SHELF_MINIMUM_BUTTONS = 3; const ACTION_SHELF_EDGE_TOLERANCE = 2; diff --git a/src/snapshot/snapshot-presentation/ios/actions.ts b/packages/capture-kit/src/ios-snapshot-engine/actions.ts similarity index 98% rename from src/snapshot/snapshot-presentation/ios/actions.ts rename to packages/capture-kit/src/ios-snapshot-engine/actions.ts index a1679e322d..3b66ceafd8 100644 --- a/src/snapshot/snapshot-presentation/ios/actions.ts +++ b/packages/capture-kit/src/ios-snapshot-engine/actions.ts @@ -6,7 +6,7 @@ import { isMostlyViewportSizedRect, mergeReplacement, type SnapshotTreeRuleContext, -} from '../tree.ts'; +} from './tree.ts'; export function collectIosImplicitScrollableActions( nodes: RawSnapshotNode[], diff --git a/packages/capture-kit/src/ios-snapshot-engine/engine.test.ts b/packages/capture-kit/src/ios-snapshot-engine/engine.test.ts new file mode 100644 index 0000000000..dfc4c12da5 --- /dev/null +++ b/packages/capture-kit/src/ios-snapshot-engine/engine.test.ts @@ -0,0 +1,303 @@ +import assert from 'node:assert/strict'; +import { test } from 'vitest'; +import type { + IosSnapshotAcquisition, + IosSnapshotInput, + IosSnapshotRequest, + IosSnapshotValidationFacts, +} from '@agent-device/contracts/ios-snapshot'; +import { + buildIosSnapshotPresentationKey, + createIosSnapshotRequest, + deriveIosCaptureHint, +} from '@agent-device/capture-kit/ios-snapshot-planning'; +import { + compactIosInteractiveSnapshot, + createIosSnapshotEngine, + IosSnapshotEngineError, + presentIosSnapshot, + publishIosSnapshot, +} from './index.ts'; +import type { Rect, RawSnapshotNode } from '@agent-device/kernel/snapshot'; + +const viewport: Rect = { x: 0, y: 0, width: 320, height: 240 }; + +test('regular presentation folds nested clips and keeps effective actionability', () => { + const request = createIosSnapshotRequest(); + const result = publishIosSnapshot(acquiredInput(request, nestedNodes()), request); + + assert.deepEqual( + result.payload.nodes.map((node) => [node.label, node.rect, node.parentIndex]), + [ + ['App', viewport, undefined], + ['Outer', { x: 16, y: 20, width: 180, height: 180 }, 0], + ['Inner', { x: 120, y: 40, width: 76, height: 160 }, 1], + ['Partially visible', { x: 150, y: 80, width: 46, height: 40 }, 2], + ], + ); + assert.equal(result.payload.nodes[3]?.hittable, true); + assert.equal(result.payload.nodes[3]?.ref, 'e4'); + assert.equal(result.comparisonIdentity.lineage.targetId, 'simulator-1'); +}); + +test('raw projection preserves reported geometry while regular projection clips it', () => { + const regularRequest = createIosSnapshotRequest(); + const rawRequest = createIosSnapshotRequest({ raw: true, interactiveOnly: true }); + const nodes = nestedNodes(); + const regular = publishIosSnapshot(acquiredInput(regularRequest, nodes), regularRequest); + const raw = publishIosSnapshot(acquiredInput(rawRequest, nodes), rawRequest); + + assert.equal( + regular.payload.nodes.some((node) => node.label === 'Escaped child'), + false, + ); + assert.equal(raw.payload.nodes.find((node) => node.label === 'Escaped child')?.rect?.x, 210); + assert.equal(raw.payload.nodes.find((node) => node.label === 'Escaped child')?.hittable, true); +}); + +test('cursor projection keeps geometryless nodes neutral while plain viewport keeps child visibility independent', () => { + const request = createIosSnapshotRequest(); + const nodes = [ + node(0, 'Application', 'App', viewport), + { ...node(1, 'Other', 'No frame', viewport, 0, 1), rect: undefined }, + node(2, 'Button', 'Child', { x: 20, y: 20, width: 40, height: 40 }, 1, 2), + ]; + const cursor = publishIosSnapshot(acquiredInput(request, nodes), request); + const plain = publishIosSnapshot(acquiredInput(request, nodes), request, { + foldPolicy: 'plain-viewport', + }); + + assert.deepEqual( + cursor.payload.nodes.map((entry) => entry.label), + ['App', 'No frame', 'Child'], + ); + assert.deepEqual( + plain.payload.nodes.map((entry) => entry.label), + ['App', 'Child'], + ); +}); + +test('scope reparents wrappers and regular depth counts presented nodes', () => { + const request = createIosSnapshotRequest({ scope: 'Target', depth: 1 }); + const result = publishIosSnapshot(acquiredInput(request, scopedNodes()), request); + + assert.deepEqual( + result.payload.nodes.map((node) => [node.type, node.label, node.depth, node.parentIndex]), + [ + ['Other', 'Target', 0, undefined], + ['Button', 'Target', 1, 0], + ], + ); +}); + +test('plain viewport policy does not inherit cursor clipping', () => { + const request = createIosSnapshotRequest(); + const result = publishIosSnapshot( + acquiredInput(request, [ + node(0, 'Application', 'App', viewport), + node(1, 'ScrollView', 'Scroll', { x: 0, y: 100, width: 320, height: 40 }, 0), + node(2, 'StaticText', 'Outside scroll clip', { x: 10, y: 200, width: 80, height: 20 }, 1), + ]), + request, + { foldPolicy: 'plain-viewport' }, + ); + + assert.deepEqual( + result.payload.nodes.map((entry) => entry.label), + ['App', 'Scroll', 'Outside scroll clip'], + ); +}); + +test('regular presentation fails typed when the viewport is missing or the graph is malformed', () => { + const request = createIosSnapshotRequest(); + const missingViewport: IosSnapshotAcquisition = { + ...acquisition(request, nestedNodes()), + viewport: { kind: 'missing', reason: 'not-provided' }, + }; + assert.throws( + () => publishIosSnapshot({ stage: 'acquired', acquisition: missingViewport }, request), + (error: unknown) => + error instanceof IosSnapshotEngineError && error.reason === 'missing-viewport', + ); + + const malformed = acquisition(request, [ + node(0, 'Application', 'App', viewport), + { ...node(1, 'Button', 'Broken', { x: 1, y: 1, width: 10, height: 10 }, 0), parentIndex: 99 }, + ]); + assert.throws( + () => publishIosSnapshot({ stage: 'acquired', acquisition: malformed }, request), + (error: unknown) => + error instanceof IosSnapshotEngineError && error.reason === 'malformed-graph', + ); +}); + +test('presented runner payloads and optional quality payloads cross the host invariant', () => { + const request = createIosSnapshotRequest(); + const presentedCapture = publishIosSnapshot(acquiredInput(request, nestedNodes()), request); + const input: IosSnapshotInput = { + stage: 'presented', + presentation: { + producer: 'apple-runner', + intent: 'full', + payload: { nodes: presentedCapture.payload.nodes, truncated: false }, + qualityPayload: { + nodes: presentedCapture.payload.nodes, + truncated: false, + scope: null, + }, + }, + validation: validationFacts(request), + }; + const result = publishIosSnapshot(input, request); + + assert.equal(result.payload.nodes.length, 4); + assert.equal(result.payload.nodes[3]?.ref, 'e4'); + assert.equal(result.residue.length, 1); +}); + +test('scoped raw presentation retains an unscoped quality view', () => { + const request = createIosSnapshotRequest({ raw: true, scope: 'Target' }); + const result = presentIosSnapshot(acquiredInput(request, scopedNodes()), request); + + assert.deepEqual( + result.nodes.map((entry) => entry.label), + ['Target', 'Target'], + ); + assert.deepEqual( + result.qualityNodes?.map((entry) => entry.label), + ['App', 'Wrapper', 'Target', 'Target'], + ); +}); + +test('unavailable hittability never becomes regular actionability', () => { + const request = createIosSnapshotRequest(); + const unavailable = { + ...acquisition(request, nestedNodes()), + residue: [{ kind: 'unavailable-fact' as const, fact: 'hittability' as const }], + } satisfies IosSnapshotAcquisition; + const acquired = publishIosSnapshot({ stage: 'acquired', acquisition: unavailable }, request); + assert.equal( + acquired.payload.nodes.find((node) => node.label === 'Partially visible')?.hittable, + false, + ); + + const available = publishIosSnapshot(acquiredInput(request, nestedNodes()), request); + const presented: IosSnapshotInput = { + stage: 'presented', + presentation: { + producer: 'apple-runner', + intent: 'full', + payload: { nodes: available.payload.nodes, truncated: false }, + }, + validation: { + ...validationFacts(request), + hittability: { kind: 'unavailable', reason: 'not-provided' }, + }, + }; + assert.throws( + () => publishIosSnapshot(presented, request), + (error: unknown) => + error instanceof IosSnapshotEngineError && + error.code === 'IOS_SNAPSHOT_ENGINE_FAILED' && + error.reason === 'invalid-presented-payload', + ); +}); + +test('interactive compaction stays available through the engine boundary', () => { + const rowRect = { x: 16, y: 80, width: 288, height: 52 }; + const compacted = compactIosInteractiveSnapshot([ + node(0, 'Application', 'App', viewport), + node(1, 'Table', 'Settings', { x: 0, y: 40, width: 320, height: 200 }, 0), + node(2, 'Cell', 'General', rowRect, 1, 2), + node(3, 'Button', 'General', rowRect, 2, 3), + node(4, 'StaticText', 'General', rowRect, 3, 4), + ]); + + assert.deepEqual( + compacted.map((entry) => entry.type), + ['Application', 'Table', 'Cell'], + ); +}); + +test('the configured engine keeps its fold policy and exposes the contract operations', () => { + const engine = createIosSnapshotEngine({ foldPolicy: 'plain-viewport' }); + const request = createIosSnapshotRequest(); + const presented = presentIosSnapshot(acquiredInput(request, nestedNodes()), request, { + foldPolicy: 'plain-viewport', + }); + + assert.equal(typeof engine.plan, 'function'); + assert.equal(typeof engine.publish, 'function'); + assert.equal(presented.stats.sourceNodeCount, nestedNodes().length); +}); + +function acquisition( + request: IosSnapshotRequest, + nodes: RawSnapshotNode[], +): IosSnapshotAcquisition { + const hint = deriveIosCaptureHint(request); + assert.equal(hint.acquisitionIntent, 'full'); + return { + producer: 'simulator-ax-bridge', + intent: 'full', + hint: { ...hint, acquisitionIntent: 'full' }, + nodes, + truncated: false, + viewport: { kind: 'reported', rect: viewport }, + lineage: { targetId: 'simulator-1', generation: 'generation-1' }, + residue: [{ kind: 'truncated', dimension: 'payload', limit: 2000 }], + }; +} + +function acquiredInput(request: IosSnapshotRequest, nodes: RawSnapshotNode[]): IosSnapshotInput { + return { stage: 'acquired', acquisition: acquisition(request, nodes) }; +} + +function validationFacts(request: IosSnapshotRequest): IosSnapshotValidationFacts { + return { + presentationKey: buildIosSnapshotPresentationKey(request), + viewport: { kind: 'reported', rect: viewport }, + hittability: { kind: 'available' }, + lineage: { targetId: 'simulator-1', generation: 'generation-1' }, + residue: [{ kind: 'truncated', dimension: 'payload', limit: 2000 }], + }; +} + +function nestedNodes(): RawSnapshotNode[] { + return [ + node(0, 'Application', 'App', viewport), + node(1, 'ScrollView', 'Outer', { x: 16, y: 20, width: 180, height: 180 }, 0, 1), + node(2, 'ScrollView', 'Inner', { x: 120, y: 40, width: 180, height: 160 }, 1, 2), + node(3, 'Button', 'Partially visible', { x: 150, y: 80, width: 100, height: 40 }, 2, 3), + node(4, 'Button', 'Escaped child', { x: 210, y: 80, width: 100, height: 40 }, 2, 3), + ]; +} + +function scopedNodes(): RawSnapshotNode[] { + return [ + node(0, 'Application', 'App', viewport), + node(1, 'Other', 'Wrapper', { x: 10, y: 10, width: 200, height: 100 }, 0, 1), + node(2, 'Other', 'Target', { x: 10, y: 10, width: 200, height: 100 }, 1, 2), + node(3, 'Button', 'Target', { x: 20, y: 20, width: 80, height: 40 }, 2, 3), + ]; +} + +function node( + index: number, + type: string, + label: string, + rect: Rect, + parentIndex?: number, + depth?: number, +): RawSnapshotNode { + return { + index, + type, + label, + rect, + parentIndex, + depth: depth ?? (parentIndex === undefined ? 0 : 1), + enabled: true, + hittable: type === 'Button', + }; +} diff --git a/packages/capture-kit/src/ios-snapshot-engine/engine.ts b/packages/capture-kit/src/ios-snapshot-engine/engine.ts new file mode 100644 index 0000000000..590a392872 --- /dev/null +++ b/packages/capture-kit/src/ios-snapshot-engine/engine.ts @@ -0,0 +1,196 @@ +import { + buildIosSnapshotComparisonIdentity, + buildIosSnapshotPresentationKey, + deriveIosCaptureHint, + IOS_SNAPSHOT_PRODUCER_CAPABILITIES, + planIosSnapshot, +} from '../ios-snapshot-planning.ts'; +import type { + IosSnapshotAcquisition, + IosSnapshotEngine, + IosSnapshotInput, + IosSnapshotPublication, + IosSnapshotRequest, +} from '@agent-device/contracts/ios-snapshot'; +import { attachRefs, type RawSnapshotNode } from '@agent-device/kernel/snapshot'; +import { buildIosInteractiveSnapshotPresentation } from './semantic-index.ts'; +import { validateIosSnapshotGraph } from './graph.ts'; +import { foldIosSnapshot } from './geometry.ts'; +import { resolveIosViewport, validateIosPayload } from './invariants.ts'; +import { projectIosQualitySnapshot, projectIosSnapshot } from './projection.ts'; +import { presentIosRunnerSnapshot } from './runner-presentation.ts'; +import type { + IosSnapshotEngineOptions, + IosSnapshotEnginePresentation, + IosSnapshotFoldPolicy, +} from './types.ts'; +import { IosSnapshotEngineError } from './types.ts'; + +const DEFAULT_FOLD_POLICY: IosSnapshotFoldPolicy = 'cursor-projected'; + +export function createIosSnapshotEngine(options: IosSnapshotEngineOptions = {}): IosSnapshotEngine { + const foldPolicy = options.foldPolicy ?? DEFAULT_FOLD_POLICY; + return Object.freeze({ + plan: planIosSnapshot, + publish: (input, request) => publishIosSnapshot(input, request, { foldPolicy }), + }); +} + +export function publishIosSnapshot( + input: IosSnapshotInput, + request: IosSnapshotRequest, + options: IosSnapshotEngineOptions = {}, +): IosSnapshotPublication { + const presentation = presentIosSnapshot(input, request, options); + const presentationKey = + input.stage === 'presented' + ? input.validation.presentationKey + : buildIosSnapshotPresentationKey(request); + return { + payload: { + nodes: attachRefs(presentation.nodes), + truncated: + input.stage === 'acquired' + ? input.acquisition.truncated + : input.presentation.payload.truncated, + }, + presentationKey, + comparisonIdentity: buildIosSnapshotComparisonIdentity(input, request), + residue: + input.stage === 'acquired' ? [...input.acquisition.residue] : [...input.validation.residue], + }; +} + +export function presentIosSnapshot( + input: IosSnapshotInput, + request: IosSnapshotRequest, + options: IosSnapshotEngineOptions = {}, +): IosSnapshotEnginePresentation { + const foldPolicy = options.foldPolicy ?? DEFAULT_FOLD_POLICY; + if (input.stage === 'acquired') { + return presentAcquiredSnapshot(input.acquisition, request, foldPolicy); + } + return presentIosRunnerSnapshot(input, request, foldPolicy); +} + +export function compactIosInteractiveSnapshot(nodes: RawSnapshotNode[]): RawSnapshotNode[] { + return buildIosInteractiveSnapshotPresentation(nodes).nodes; +} + +function presentAcquiredSnapshot( + acquisition: IosSnapshotAcquisition, + request: IosSnapshotRequest, + foldPolicy: IosSnapshotFoldPolicy, +): IosSnapshotEnginePresentation { + const expectedHint = deriveIosCaptureHint(request); + assertCaptureHintMatches(acquisition, expectedHint); + + if (request.projection === 'raw') { + validateIosSnapshotGraph(acquisition.nodes); + const projected = projectIosSnapshot({ + nodes: acquisition.nodes.map((raw) => ({ raw })), + projection: 'raw', + scope: request.scope, + depth: request.depth, + foldPolicy, + }); + const qualityNodes = + request.scope === null + ? undefined + : projectIosQualitySnapshot({ + nodes: acquisition.nodes.map((raw) => ({ raw })), + projection: 'raw', + depth: null, + foldPolicy, + }).nodes; + return { + nodes: projected.nodes, + ...(qualityNodes ? { qualityNodes } : {}), + presentedIndexesBySourceIndex: identityMapping(projected.nodes), + stats: { + presentedNodeCount: projected.nodes.length, + sourceNodeCount: acquisition.nodes.length, + parentClipLookups: 0, + }, + }; + } + + const viewport = resolveIosViewport(acquisition); + const hittabilityAvailable = + IOS_SNAPSHOT_PRODUCER_CAPABILITIES[acquisition.producer].hittabilityEvidence === 'available' && + !hasUnavailableHittability(acquisition.residue); + const folded = foldIosSnapshot(acquisition.nodes, viewport, request.interactiveOnly, foldPolicy, { + hittabilityAvailable, + }); + const foldedInput = { + nodes: folded.nodes, + projection: 'regular' as const, + scope: request.scope, + depth: request.depth, + foldPolicy, + }; + const projected = projectIosSnapshot(foldedInput); + const compacted = request.interactiveOnly + ? buildIosInteractiveSnapshotPresentation(projected.nodes) + : { + nodes: projected.nodes, + presentedIndexesBySourceIndex: identityMapping(projected.nodes), + }; + const validation = validateIosPayload( + compacted.nodes, + 'regular', + viewport, + foldPolicy, + hittabilityAvailable, + ); + const qualityNodes = + request.scope === null + ? undefined + : projectIosQualitySnapshot({ + nodes: folded.nodes, + projection: 'regular', + depth: null, + foldPolicy, + }).nodes; + return { + nodes: compacted.nodes, + ...(qualityNodes ? { qualityNodes } : {}), + presentedIndexesBySourceIndex: compacted.presentedIndexesBySourceIndex, + stats: { + presentedNodeCount: compacted.nodes.length, + sourceNodeCount: acquisition.nodes.length, + parentClipLookups: folded.stats.parentClipLookups + validation.parentClipLookups, + }, + }; +} + +function hasUnavailableHittability(residue: IosSnapshotAcquisition['residue']): boolean { + return residue.some((entry) => entry.kind === 'unavailable-fact' && entry.fact === 'hittability'); +} + +function assertCaptureHintMatches( + acquisition: IosSnapshotAcquisition, + expected: ReturnType, +): void { + const actual = acquisition.hint; + if ( + actual.projection !== expected.projection || + actual.rawTraversalDepth !== expected.rawTraversalDepth || + actual.regularPresentedDepth !== expected.regularPresentedDepth || + actual.interactiveOnly !== expected.interactiveOnly || + actual.customActions !== expected.customActions || + actual.acquisitionIntent !== expected.acquisitionIntent + ) { + throw new IosSnapshotEngineError( + 'projection-mismatch', + 'acquired iOS snapshot does not match the requested capture hint', + { projection: actual.projection }, + ); + } +} + +function identityMapping( + nodes: readonly RawSnapshotNode[], +): ReadonlyMap { + return new Map(nodes.map((node) => [node.index, [node.index]])); +} diff --git a/packages/capture-kit/src/ios-snapshot-engine/geometry-policy.ts b/packages/capture-kit/src/ios-snapshot-engine/geometry-policy.ts new file mode 100644 index 0000000000..0dde0c6f3c --- /dev/null +++ b/packages/capture-kit/src/ios-snapshot-engine/geometry-policy.ts @@ -0,0 +1,237 @@ +import type { Rect, RawSnapshotNode } from '@agent-device/kernel/snapshot'; +import { containsPoint, isPositiveFiniteRect } from '@agent-device/kernel/rect'; +import { normalizeType } from '@agent-device/contracts/snapshot'; +import type { IosSnapshotFoldPolicy } from './types.ts'; + +const SCROLL_CONTAINER_TYPES = new Set(['collectionview', 'scrollview', 'table']); +const VISIBILITY_CARRIER_TYPES = new Set(['application', 'window']); +const NEGLIGIBLE_DECORATION_TOLERANCE = 1; + +export type TraversalState = Readonly<{ + projectedOut: boolean; + ancestorClip?: Rect; +}>; + +export type BranchState = Readonly<{ + traversal: TraversalState; + anchor?: { index: number; rect: Rect }; + keptIndex?: number; + keptDepth: number; +}>; + +export type GeometryDecision = Readonly<{ + isIncluded: boolean; + descendants: TraversalState; + effectiveRect?: Rect; + hiddenContentFrame?: Rect; + establishesScrollAnchor: boolean; +}>; + +export function isGeometricallyActionable( + enabled: boolean, + rect: Rect | undefined, + viewport: Rect, +): boolean { + return Boolean( + enabled && + isPositiveFiniteRect(rect) && + containsPoint(viewport, rect.x + rect.width / 2, rect.y + rect.height / 2), + ); +} + +export function rootTraversal(): TraversalState { + return { projectedOut: false }; +} + +export function traversalDecision( + node: RawSnapshotNode, + parentTraversal: TraversalState, + viewport: Rect, + interactiveOnly: boolean, + hasChildren: boolean, + policy: IosSnapshotFoldPolicy, +): GeometryDecision { + const ancestorClip = policy === 'cursor-projected' ? parentTraversal.ancestorClip : undefined; + const effectiveRect = effectiveSnapshotRect(node.rect, viewport, ancestorClip); + const hasFrame = isPositiveFiniteRect(normalizeRect(node.rect)); + const intersectsClip = isPositiveFiniteRect(effectiveRect); + const type = normalizeType(node.type ?? ''); + const projectedOut = descendantsProjectedOut( + policy, + parentTraversal.projectedOut, + hasFrame, + intersectsClip, + type, + hasChildren, + ); + const visible = + presentationVisible(policy, parentTraversal.projectedOut, hasFrame, intersectsClip) && + !isNegligibleDecoration(node, hasFrame, policy); + const isIncluded = shouldInclude(node, visible, interactiveOnly, policy); + const establishesScrollAnchor = ownsScrollAnchor(type, isIncluded, intersectsClip, hasChildren); + const hiddenFrame = hiddenContentFrame( + policy, + parentTraversal.projectedOut, + hasFrame, + intersectsClip, + node, + ); + return { + isIncluded, + descendants: descendantTraversal( + policy, + projectedOut, + ancestorClip, + effectiveRect, + establishesScrollAnchor, + ), + effectiveRect, + ...(hiddenFrame ? { hiddenContentFrame: hiddenFrame } : {}), + establishesScrollAnchor, + }; +} + +function effectiveSnapshotRect( + reportedRect: Rect | undefined, + viewport: Rect, + ancestorClip?: Rect, +): Rect | undefined { + const normalized = normalizeRect(reportedRect); + if (!normalized) return undefined; + let effective = intersectRect(normalized, viewport); + if (ancestorClip) effective = intersectRect(effective, ancestorClip); + return effective; +} + +function descendantsProjectedOut( + policy: IosSnapshotFoldPolicy, + parentProjectedOut: boolean, + hasFrame: boolean, + intersectsClip: boolean, + type: string, + hasChildren: boolean, +): boolean { + if (policy === 'plain-viewport') return !intersectsClip; + return parentProjectedOut || (hasFrame && !intersectsClip && ownsDescendants(type, hasChildren)); +} + +function presentationVisible( + policy: IosSnapshotFoldPolicy, + parentProjectedOut: boolean, + hasFrame: boolean, + intersectsClip: boolean, +): boolean { + if (policy === 'plain-viewport') return intersectsClip; + return !parentProjectedOut && (!hasFrame || intersectsClip); +} + +function isNegligibleDecoration( + node: RawSnapshotNode, + hasFrame: boolean, + policy: IosSnapshotFoldPolicy, +): boolean { + if ( + policy !== 'cursor-projected' || + node.parentIndex === undefined || + hasSemanticContent(node) || + !hasFrame + ) { + return false; + } + return ( + normalizedRectWidth(node.rect) <= NEGLIGIBLE_DECORATION_TOLERANCE || + normalizedRectHeight(node.rect) <= NEGLIGIBLE_DECORATION_TOLERANCE + ); +} + +function descendantTraversal( + policy: IosSnapshotFoldPolicy, + projectedOut: boolean, + ancestorClip: Rect | undefined, + effectiveRect: Rect | undefined, + establishesScrollAnchor: boolean, +): TraversalState { + return { + projectedOut, + ...(policy === 'cursor-projected' && establishesScrollAnchor + ? { ancestorClip: effectiveRect } + : { ancestorClip }), + }; +} + +function hiddenContentFrame( + policy: IosSnapshotFoldPolicy, + parentProjectedOut: boolean, + hasFrame: boolean, + intersectsClip: boolean, + node: RawSnapshotNode, +): Rect | undefined { + if (policy !== 'cursor-projected' || parentProjectedOut || !hasFrame || intersectsClip) { + return undefined; + } + return normalizeRect(node.rect); +} + +function ownsDescendants(type: string, hasChildren: boolean): boolean { + return hasChildren && (type === 'cell' || SCROLL_CONTAINER_TYPES.has(type)); +} + +function ownsScrollAnchor( + type: string, + isIncluded: boolean, + intersectsClip: boolean, + hasChildren: boolean, +): boolean { + return isIncluded && intersectsClip && hasChildren && SCROLL_CONTAINER_TYPES.has(type); +} + +function shouldInclude( + node: RawSnapshotNode, + visible: boolean, + interactiveOnly: boolean, + policy: IosSnapshotFoldPolicy, +): boolean { + if (node.parentIndex === undefined) return true; + const type = normalizeType(node.type ?? ''); + if (policy === 'plain-viewport' && interactiveOnly && !visible && type !== 'application') { + return false; + } + return VISIBILITY_CARRIER_TYPES.has(type) || visible; +} + +function hasSemanticContent(node: RawSnapshotNode): boolean { + return [node.label, node.identifier, node.value].some( + (value) => typeof value === 'string' && value.trim().length > 0, + ); +} + +function normalizeRect(rect: Rect | undefined): Rect | undefined { + if (!rect) return undefined; + if (![rect.x, rect.y, rect.width, rect.height].every(Number.isFinite)) return undefined; + if (rect.width < 0 || rect.height < 0) return undefined; + return { ...rect, width: Math.max(0, rect.width), height: Math.max(0, rect.height) }; +} + +function normalizedRectWidth(rect: Rect | undefined): number { + return normalizeRect(rect)?.width ?? 0; +} + +function normalizedRectHeight(rect: Rect | undefined): number { + return normalizeRect(rect)?.height ?? 0; +} + +function intersectRect(left: Rect, right: Rect): Rect { + const x = Math.max(left.x, right.x); + const y = Math.max(left.y, right.y); + const rightEdge = Math.min(left.x + left.width, right.x + right.width); + const bottomEdge = Math.min(left.y + left.height, right.y + right.height); + if (rightEdge <= x || bottomEdge <= y) { + return { x: left.x, y: left.y, width: 0, height: 0 }; + } + return { + x: rightEdge > x ? x : left.x, + y: bottomEdge > y ? y : left.y, + width: rightEdge - x, + height: bottomEdge - y, + }; +} diff --git a/packages/capture-kit/src/ios-snapshot-engine/geometry.ts b/packages/capture-kit/src/ios-snapshot-engine/geometry.ts new file mode 100644 index 0000000000..313abe853a --- /dev/null +++ b/packages/capture-kit/src/ios-snapshot-engine/geometry.ts @@ -0,0 +1,167 @@ +import type { Rect, RawSnapshotNode } from '@agent-device/kernel/snapshot'; +import { validateIosSnapshotGraph } from './graph.ts'; +import { + isGeometricallyActionable, + rootTraversal, + traversalDecision, + type BranchState, + type GeometryDecision, +} from './geometry-policy.ts'; +import type { + IosSnapshotFoldOptions, + IosSnapshotFoldPolicy, + IosSnapshotPresentationNode, + IosSnapshotPresentationStats, +} from './types.ts'; + +export function foldIosSnapshot( + nodes: readonly RawSnapshotNode[], + viewport: Rect, + interactiveOnly: boolean, + policy: IosSnapshotFoldPolicy, + options: IosSnapshotFoldOptions = {}, +): { nodes: IosSnapshotPresentationNode[]; stats: IosSnapshotPresentationStats } { + validateIosSnapshotGraph(nodes); + const hasChildren = buildChildPresence(nodes); + const states = new Map(); + const kept: IosSnapshotPresentationNode[] = []; + const hints = new Map(); + let parentClipLookups = 0; + + for (const node of nodes) { + const parentState = readParentState(node, states, () => { + parentClipLookups += 1; + }); + const parentTraversal = parentState?.traversal ?? rootTraversal(); + const parentAnchor = policy === 'cursor-projected' ? parentState?.anchor : undefined; + const decision = traversalDecision( + node, + parentTraversal, + viewport, + interactiveOnly, + hasChildren.has(node.index), + policy, + ); + + recordHiddenContentHint(decision, parentAnchor, hints); + const foldedNode = appendFoldedNode(node, decision, parentState, kept, viewport, options); + const anchor = nextScrollAnchor(decision, parentAnchor, foldedNode.keptIndex); + states.set(node.index, { + traversal: decision.descendants, + anchor, + keptIndex: foldedNode.keptIndex, + keptDepth: foldedNode.keptDepth, + }); + } + + const presented = applyHiddenContentHints(hints, kept); + return { + nodes: presented, + stats: { + presentedNodeCount: presented.length, + sourceNodeCount: nodes.length, + parentClipLookups, + }, + }; +} + +function buildChildPresence(nodes: readonly RawSnapshotNode[]): Set { + const parents = new Set(); + for (const node of nodes) { + if (node.parentIndex !== undefined) parents.add(node.parentIndex); + } + return parents; +} + +function readParentState( + node: RawSnapshotNode, + states: ReadonlyMap, + onLookup: () => void, +): BranchState | undefined { + if (node.parentIndex === undefined) return undefined; + onLookup(); + return states.get(node.parentIndex); +} + +function recordHiddenContentHint( + decision: GeometryDecision, + parentAnchor: BranchState['anchor'], + hints: Map, +): void { + if (decision.hiddenContentFrame && parentAnchor) { + rememberHiddenContentHint(decision.hiddenContentFrame, parentAnchor, hints); + } +} + +function appendFoldedNode( + node: RawSnapshotNode, + decision: GeometryDecision, + parentState: BranchState | undefined, + kept: IosSnapshotPresentationNode[], + viewport: Rect, + options: IosSnapshotFoldOptions, +): { keptIndex?: number; keptDepth: number } { + let keptIndex = parentState?.keptIndex; + let keptDepth = parentState?.keptDepth ?? -1; + if (!decision.isIncluded) return { keptIndex, keptDepth }; + + const index = kept.length; + keptDepth += 1; + kept.push({ + raw: { + ...node, + index, + depth: keptDepth, + parentIndex: keptIndex, + hittable: + node.parentIndex !== undefined && + options.hittabilityAvailable !== false && + node.hittable === true && + isGeometricallyActionable(node.enabled !== false, decision.effectiveRect, viewport), + }, + ...(decision.effectiveRect ? { effectiveRect: decision.effectiveRect } : {}), + }); + keptIndex = index; + return { keptIndex, keptDepth }; +} + +function nextScrollAnchor( + decision: GeometryDecision, + parentAnchor: BranchState['anchor'], + keptIndex: number | undefined, +): BranchState['anchor'] { + if (decision.establishesScrollAnchor && keptIndex !== undefined && decision.effectiveRect) { + return { index: keptIndex, rect: decision.effectiveRect }; + } + return parentAnchor; +} + +function rememberHiddenContentHint( + frame: Rect, + anchor: { index: number; rect: Rect }, + hints: Map, +): void { + const hint = hints.get(anchor.index) ?? { above: false, below: false }; + if (frame.y + frame.height <= anchor.rect.y) hint.above = true; + else if (frame.y >= anchor.rect.y + anchor.rect.height) hint.below = true; + hints.set(anchor.index, hint); +} + +function applyHiddenContentHints( + hints: ReadonlyMap, + nodes: IosSnapshotPresentationNode[], +): IosSnapshotPresentationNode[] { + return nodes.map((presentation) => { + const hint = hints.get(presentation.raw.index); + if (!hint) return presentation; + const node = presentation.raw; + return { + ...presentation, + raw: { + ...node, + hiddenContentAbove: node.hiddenContentAbove === true || hint.above ? true : undefined, + hiddenContentBelow: node.hiddenContentBelow === true || hint.below ? true : undefined, + }, + }; + }); +} diff --git a/packages/capture-kit/src/ios-snapshot-engine/graph.ts b/packages/capture-kit/src/ios-snapshot-engine/graph.ts new file mode 100644 index 0000000000..8b0575683f --- /dev/null +++ b/packages/capture-kit/src/ios-snapshot-engine/graph.ts @@ -0,0 +1,35 @@ +import type { RawSnapshotNode } from '@agent-device/kernel/snapshot'; +import { IosSnapshotEngineError } from './types.ts'; + +export function validateIosSnapshotGraph(nodes: readonly RawSnapshotNode[]): void { + const positions = new Map(); + for (const [position, node] of nodes.entries()) { + if (!Number.isInteger(node.index) || node.index < 0 || positions.has(node.index)) { + throw new IosSnapshotEngineError( + 'malformed-graph', + 'iOS snapshot graph contains a duplicate or invalid node index', + { index: node.index }, + ); + } + positions.set(node.index, position); + if (node.depth !== undefined && (!Number.isInteger(node.depth) || node.depth < 0)) { + throw new IosSnapshotEngineError( + 'malformed-graph', + 'iOS snapshot graph contains an invalid node depth', + { index: node.index, field: 'depth' }, + ); + } + } + + for (const [position, node] of nodes.entries()) { + if (node.parentIndex === undefined) continue; + const parentPosition = positions.get(node.parentIndex); + if (parentPosition === undefined || parentPosition >= position) { + throw new IosSnapshotEngineError( + 'malformed-graph', + 'iOS snapshot graph parent must precede its child', + { index: node.index, parentIndex: node.parentIndex }, + ); + } + } +} diff --git a/packages/capture-kit/src/ios-snapshot-engine/index.ts b/packages/capture-kit/src/ios-snapshot-engine/index.ts new file mode 100644 index 0000000000..7a55916eb5 --- /dev/null +++ b/packages/capture-kit/src/ios-snapshot-engine/index.ts @@ -0,0 +1,14 @@ +export { + compactIosInteractiveSnapshot, + createIosSnapshotEngine, + presentIosSnapshot, + publishIosSnapshot, +} from './engine.ts'; +export { + buildIosInteractiveSnapshotPresentation, + presentIosInteractiveSnapshot, +} from './semantic-index.ts'; +export { collectIosStructuralIdentifierSuppression } from './noise-structural.ts'; +export { findNearestScrollableContainer, mergeReplacement, updateReplacement } from './tree.ts'; +export { IosSnapshotEngineError } from './types.ts'; +export type { SnapshotTreeRuleContext } from './tree.ts'; diff --git a/packages/capture-kit/src/ios-snapshot-engine/invariants.ts b/packages/capture-kit/src/ios-snapshot-engine/invariants.ts new file mode 100644 index 0000000000..c289485a7d --- /dev/null +++ b/packages/capture-kit/src/ios-snapshot-engine/invariants.ts @@ -0,0 +1,189 @@ +import type { + IosSnapshotAcquisition, + IosViewportEvidence, +} from '@agent-device/contracts/ios-snapshot'; +import type { Rect, RawSnapshotNode } from '@agent-device/kernel/snapshot'; +import { containsPoint, rectContains, isPositiveFiniteRect } from '@agent-device/kernel/rect'; +import { normalizeType } from '@agent-device/contracts/snapshot'; +import { validateIosSnapshotGraph } from './graph.ts'; +import { IosSnapshotEngineError } from './types.ts'; +import type { IosSnapshotFoldPolicy } from './types.ts'; + +const SCROLL_CONTAINER_TYPES = new Set(['collectionview', 'scrollview', 'table']); + +export function resolveIosViewport(acquisition: IosSnapshotAcquisition): Rect { + return resolveViewportEvidence(acquisition.viewport); +} + +export function resolveViewportEvidence(evidence: IosViewportEvidence): Rect { + if (evidence.kind === 'missing') { + throw new IosSnapshotEngineError( + evidence.reason === 'invalid' ? 'invalid-viewport' : 'missing-viewport', + 'regular iOS snapshot presentation requires a valid viewport', + { field: 'viewport' }, + ); + } + if (!isPositiveFiniteRect(evidence.rect)) { + throw new IosSnapshotEngineError( + 'invalid-viewport', + 'regular iOS snapshot presentation requires a positive finite viewport', + { field: 'viewport' }, + ); + } + return evidence.rect; +} + +function validateIosRegularInvariant( + nodes: readonly RawSnapshotNode[], + viewport: Rect, + policy: IosSnapshotFoldPolicy, + hittabilityAvailable = true, +): { parentClipLookups: number } { + validateIosSnapshotGraph(nodes); + const clipByIndex = new Map(); + let parentClipLookups = 0; + + for (const node of nodes) { + const ancestorClip = resolveAncestorClip(node, clipByIndex, viewport, () => { + parentClipLookups += 1; + }); + + const frame = normalizeRect(node.rect); + validateHittabilityEvidence(node, hittabilityAvailable); + if (!frame || !isPositiveFiniteRect(frame)) { + validateDegenerateActionability(node, frame); + clipByIndex.set(node.index, ancestorClip); + continue; + } + + validateContainedFrame(node, frame, ancestorClip); + validateActionability(node, frame, viewport); + + clipByIndex.set(node.index, clipForNode(node, frame, ancestorClip, policy)); + } + return { parentClipLookups }; +} + +function resolveAncestorClip( + node: RawSnapshotNode, + clipByIndex: ReadonlyMap, + viewport: Rect, + onParentLookup: () => void, +): Rect { + if (node.parentIndex === undefined) return viewport; + onParentLookup(); + const parentClip = clipByIndex.get(node.parentIndex); + if (!parentClip) { + throw new IosSnapshotEngineError( + 'invalid-presented-payload', + 'regular iOS snapshot payload refers to a parent outside the payload', + { index: node.index, parentIndex: node.parentIndex }, + ); + } + return parentClip; +} + +function validateHittabilityEvidence(node: RawSnapshotNode, available: boolean): void { + if (node.hittable !== true || available) return; + throw new IosSnapshotEngineError( + 'invalid-presented-payload', + 'iOS snapshot payload marked a node actionable without hittability evidence', + { index: node.index }, + ); +} + +function validateDegenerateActionability(node: RawSnapshotNode, frame: Rect | undefined): void { + if (node.hittable !== true) return; + throw new IosSnapshotEngineError( + 'regular-degenerate-actionable-node', + 'regular iOS snapshot node with a missing or degenerate frame is actionable', + { index: node.index, frame: frame ?? node.rect }, + ); +} + +function validateContainedFrame(node: RawSnapshotNode, frame: Rect, clip: Rect): void { + if (containsWithTolerance(frame, clip)) return; + throw new IosSnapshotEngineError( + 'regular-node-outside-cumulative-clip', + 'regular iOS snapshot node escaped its cumulative clip', + { index: node.index, frame, clip }, + ); +} + +function validateActionability(node: RawSnapshotNode, frame: Rect, viewport: Rect): void { + if ( + node.hittable !== true || + (node.enabled !== false && + containsPoint(viewport, frame.x + frame.width / 2, frame.y + frame.height / 2)) + ) { + return; + } + throw new IosSnapshotEngineError( + 'invalid-presented-payload', + 'regular iOS snapshot payload marked a disabled or off-viewport node actionable', + { index: node.index, frame }, + ); +} + +function clipForNode( + node: RawSnapshotNode, + frame: Rect, + ancestorClip: Rect, + policy: IosSnapshotFoldPolicy, +): Rect { + return policy === 'cursor-projected' && SCROLL_CONTAINER_TYPES.has(normalizeType(node.type ?? '')) + ? frame + : ancestorClip; +} + +export function validateIosPayload( + nodes: readonly RawSnapshotNode[], + projection: 'regular' | 'raw', + viewport: Rect | undefined, + policy: IosSnapshotFoldPolicy, + hittabilityAvailable = true, +): { parentClipLookups: number } { + if (projection === 'raw') { + validateIosSnapshotGraph(nodes); + return { parentClipLookups: 0 }; + } + if (!viewport) { + throw new IosSnapshotEngineError( + 'missing-viewport', + 'regular iOS snapshot payload cannot be validated without a viewport', + { field: 'viewport' }, + ); + } + try { + return validateIosRegularInvariant(nodes, viewport, policy, hittabilityAvailable); + } catch (error) { + if (error instanceof IosSnapshotEngineError && error.code === 'IOS_SNAPSHOT_ENGINE_FAILED') { + if (error.reason === 'malformed-graph') { + throw new IosSnapshotEngineError('invalid-presented-payload', error.message, error.details); + } + throw error; + } + throw error; + } +} + +function normalizeRect(rect: Rect | undefined): Rect | undefined { + if (!rect || ![rect.x, rect.y, rect.width, rect.height].every(Number.isFinite)) { + return undefined; + } + if (rect.width < 0 || rect.height < 0) return undefined; + return { ...rect, width: Math.max(0, rect.width), height: Math.max(0, rect.height) }; +} + +function containsWithTolerance(frame: Rect, clip: Rect): boolean { + const tolerance = 0.0001; + return rectContains( + { + x: clip.x - tolerance, + y: clip.y - tolerance, + width: clip.width + tolerance * 2, + height: clip.height + tolerance * 2, + }, + frame, + ); +} diff --git a/packages/capture-kit/src/ios-snapshot-engine/noise-helpers.ts b/packages/capture-kit/src/ios-snapshot-engine/noise-helpers.ts new file mode 100644 index 0000000000..f96e919b51 --- /dev/null +++ b/packages/capture-kit/src/ios-snapshot-engine/noise-helpers.ts @@ -0,0 +1,15 @@ +import type { RawSnapshotNode } from '@agent-device/kernel/snapshot'; +import { normalizeType } from '@agent-device/contracts/snapshot'; + +export function forEachOtherNodeWithLabel( + nodes: RawSnapshotNode[], + visitor: (node: RawSnapshotNode, label: string, position: number) => void, +): void { + for (let position = 0; position < nodes.length; position += 1) { + const node = nodes[position]; + const label = node?.label?.trim(); + if (node && label && normalizeType(node.type ?? '') === 'other') { + visitor(node, label, position); + } + } +} diff --git a/packages/capture-kit/src/ios-snapshot-engine/noise-overlay.ts b/packages/capture-kit/src/ios-snapshot-engine/noise-overlay.ts new file mode 100644 index 0000000000..ec27682de7 --- /dev/null +++ b/packages/capture-kit/src/ios-snapshot-engine/noise-overlay.ts @@ -0,0 +1,111 @@ +import type { RawSnapshotNode } from '@agent-device/kernel/snapshot'; +import { rectArea, rectContains } from '@agent-device/kernel/rect'; +import { + isReactNativeCollapsedWarningWrapperCandidate, + isReactNativeCollapsedWarningWrapperWithVisibleBanner, + isReactNativeOverlayDismissLabel, + isReactNativeOverlayMinimizeLabel, +} from '@agent-device/contracts/react-native-overlay'; +import { + areRectsApproximatelyEqual, + findDescendant, + forEachDescendant, + mergeReplacement, + type SnapshotTreeRuleContext, +} from './tree.ts'; +import { forEachOtherNodeWithLabel } from './noise-helpers.ts'; + +export function collectIosReactNativeOverlayActionPresentation( + nodes: RawSnapshotNode[], + replacements: Map, +): void { + forEachOtherNodeWithLabel(nodes, (node, nodeLabel, position) => { + if (!isReactNativeOverlayDismissLabel(nodeLabel) || !node.rect) return; + const minimize = findDescendant( + nodes, + position, + (descendant) => + Boolean(descendant.rect) && + isReactNativeOverlayMinimizeLabel(descendant.label?.trim() ?? ''), + ); + if (!minimize?.rect) return; + const dismissRect = remainingHorizontalPartition(node.rect, minimize.rect); + if (!dismissRect) return; + const representativeRect = smallestContainedDismissRect(nodes, position, dismissRect); + mergeReplacement(replacements, node, { rect: representativeRect }); + forEachDescendant(nodes, position, (descendant) => { + if (isReactNativeOverlayDismissLabel(descendant.label?.trim() ?? '')) { + mergeReplacement(replacements, descendant, { rect: representativeRect }); + } + }); + }); +} + +function smallestContainedDismissRect( + nodes: RawSnapshotNode[], + position: number, + partition: NonNullable, +): NonNullable { + let representative = partition; + forEachDescendant(nodes, position, (descendant) => { + const label = descendant.label?.trim() ?? ''; + if (!descendant.rect || !isReactNativeOverlayDismissLabel(label)) return; + if (!rectContains(partition, descendant.rect)) return; + if (rectArea(descendant.rect) < rectArea(representative)) { + representative = descendant.rect; + } + }); + return representative; +} + +function remainingHorizontalPartition( + wrapper: NonNullable, + occupied: NonNullable, +): NonNullable | undefined { + const wrapperRight = wrapper.x + wrapper.width; + const occupiedRight = occupied.x + occupied.width; + const expectedRightPartition = { + x: occupied.x, + y: wrapper.y, + width: wrapperRight - occupied.x, + height: wrapper.height, + }; + if (occupied.x > wrapper.x && areRectsApproximatelyEqual(occupied, expectedRightPartition)) { + return { ...wrapper, width: occupied.x - wrapper.x }; + } + const expectedLeftPartition = { + x: wrapper.x, + y: wrapper.y, + width: occupiedRight - wrapper.x, + height: wrapper.height, + }; + if (occupiedRight < wrapperRight && areRectsApproximatelyEqual(occupied, expectedLeftPartition)) { + return { ...wrapper, x: occupiedRight, width: wrapperRight - occupiedRight }; + } + return undefined; +} + +export function collectIosReactNativeOverlayWrapperSuppression( + nodes: RawSnapshotNode[], + context: SnapshotTreeRuleContext, +): void { + forEachOtherNodeWithLabel(nodes, (node, _nodeLabel, position) => { + if (!isReactNativeCollapsedWarningWrapperCandidate(node)) return; + if ( + isReactNativeCollapsedWarningWrapperWithVisibleBanner( + node, + collectDescendantNodes(nodes, position), + ) + ) { + context.suppressNode(node, collectDescendantNodes(nodes, position)); + } + }); +} + +function collectDescendantNodes(nodes: RawSnapshotNode[], position: number): RawSnapshotNode[] { + const descendants: RawSnapshotNode[] = []; + forEachDescendant(nodes, position, (descendant) => { + descendants.push(descendant); + }); + return descendants; +} diff --git a/packages/capture-kit/src/ios-snapshot-engine/noise-redundancy.ts b/packages/capture-kit/src/ios-snapshot-engine/noise-redundancy.ts new file mode 100644 index 0000000000..6b196fa901 --- /dev/null +++ b/packages/capture-kit/src/ios-snapshot-engine/noise-redundancy.ts @@ -0,0 +1,137 @@ +import type { RawSnapshotNode } from '@agent-device/kernel/snapshot'; +import { normalizeType } from '@agent-device/contracts/snapshot'; +import { + areRectsApproximatelyEqual, + findDescendant, + forEachDescendant, + isRepeatedStaticNode, + isScrollableSnapshotType, + isSemanticActionNode, + type SnapshotTreeRuleContext, +} from './tree.ts'; +import { forEachOtherNodeWithLabel } from './noise-helpers.ts'; + +export function collectIosRepeatedStaticSuppression( + nodes: RawSnapshotNode[], + context: SnapshotTreeRuleContext, +): void { + for (let position = 0; position < nodes.length; position += 1) { + const node = nodes[position]; + const nodeLabel = node?.label?.trim(); + if (!node || context.isSuppressed(node) || !nodeLabel) { + continue; + } + + collectRepeatedStaticSuppressionForNode(nodes, position, node, nodeLabel, context); + } +} + +function collectRepeatedStaticSuppressionForNode( + nodes: RawSnapshotNode[], + position: number, + node: RawSnapshotNode, + nodeLabel: string, + context: SnapshotTreeRuleContext, +): void { + const type = normalizeType(node.type ?? ''); + if (type === 'statictext' || type === 'link') { + suppressRepeatedStaticDescendants(nodes, position, nodeLabel, node, context); + return; + } + if (type !== 'other') { + return; + } + const semanticDescendant = findEquivalentSemanticDescendant(nodes, position, nodeLabel); + if (semanticDescendant) { + context.suppressNode(node, [semanticDescendant]); + return; + } + suppressRepeatedStaticDescendants(nodes, position, nodeLabel, node, context); +} + +function findEquivalentSemanticDescendant( + nodes: RawSnapshotNode[], + position: number, + nodeLabel: string, +): RawSnapshotNode | undefined { + return findDescendant(nodes, position, (descendant) => { + const type = normalizeType(descendant.type ?? ''); + return ( + (type === 'link' || type === 'searchfield' || isScrollableSnapshotType(descendant.type)) && + descendant.label?.trim() === nodeLabel + ); + }); +} + +function suppressRepeatedStaticDescendants( + nodes: RawSnapshotNode[], + position: number, + label: string, + representative: RawSnapshotNode, + context: SnapshotTreeRuleContext, +): void { + forEachDescendant(nodes, position, (descendant) => { + if ( + !context.semanticRepresentativeIndexes.has(descendant.index) && + isRepeatedStaticNode(descendant, label) + ) { + context.suppressNode(descendant, [representative]); + } + }); +} + +export function collectIosActionWrapperSuppression( + nodes: RawSnapshotNode[], + context: SnapshotTreeRuleContext, +): void { + forEachOtherNodeWithLabel(nodes, (node, nodeLabel, position) => { + const semanticDescendant = findDescendant(nodes, position, (descendant) => { + return ( + isSemanticActionNode(descendant) && + descendant.label?.trim() === nodeLabel && + (areRectsApproximatelyEqual(descendant.rect, node.rect) || + isIosBackdropDismissWrapper(node, descendant)) + ); + }); + if (semanticDescendant) { + context.suppressNode(node, [semanticDescendant]); + } + }); +} + +function isIosBackdropDismissWrapper(node: RawSnapshotNode, descendant: RawSnapshotNode): boolean { + if (descendant.label?.trim() !== node.label?.trim()) { + return false; + } + const descendantType = normalizeType(descendant.type ?? ''); + return ( + isNamedButtonBackdrop(node, descendantType) || + descendantType === 'textfield' || + isFullscreenActionLabelWrapper(node, descendantType, descendant) + ); +} + +function isNamedButtonBackdrop(node: RawSnapshotNode, descendantType: string): boolean { + const label = node.label?.trim(); + return descendantType === 'button' && (label === 'Dismiss' || label === 'Back'); +} + +function isFullscreenActionLabelWrapper( + node: RawSnapshotNode, + descendantType: string, + descendant: RawSnapshotNode, +): boolean { + if (descendantType !== 'button') { + return false; + } + if (!node.rect || !descendant.rect) { + return false; + } + return ( + node.rect.x === 0 && + node.rect.y === 0 && + node.rect.width >= 300 && + node.rect.height >= 600 && + descendant.rect.width < node.rect.width + ); +} diff --git a/packages/capture-kit/src/ios-snapshot-engine/noise-search.ts b/packages/capture-kit/src/ios-snapshot-engine/noise-search.ts new file mode 100644 index 0000000000..0589b5d985 --- /dev/null +++ b/packages/capture-kit/src/ios-snapshot-engine/noise-search.ts @@ -0,0 +1,84 @@ +import type { RawSnapshotNode } from '@agent-device/kernel/snapshot'; +import { normalizeType } from '@agent-device/contracts/snapshot'; +import { findDescendant, forEachDescendant, type SnapshotTreeRuleContext } from './tree.ts'; + +export function collectIosSearchToolbarSuppression( + nodes: RawSnapshotNode[], + context: SnapshotTreeRuleContext, +): void { + for (let position = 0; position < nodes.length; position += 1) { + const node = nodes[position]; + if (!node) continue; + if (isExposedSearchField(node)) { + suppressSearchToolbarDescendants(nodes, position, node, context); + continue; + } + if (!isSearchToolbar(node)) continue; + + const innerSearch = findDescendant( + nodes, + position, + (candidate) => + normalizeType(candidate.type ?? '') === 'searchfield' && candidate.label === 'Search', + ); + if (!innerSearch) { + continue; + } + + context.suppressNode(node, [innerSearch]); + suppressToolbarAncestors(node, innerSearch, context); + suppressSearchToolbarDescendants(nodes, position, innerSearch, context); + } +} + +function isExposedSearchField(node: RawSnapshotNode): boolean { + return normalizeType(node.type ?? '') === 'searchfield' && node.label === 'Search'; +} + +function isSearchToolbar(node: RawSnapshotNode): boolean { + const type = normalizeType(node.type ?? ''); + return node.label === 'Toolbar' && (type === 'toolbar' || type === 'searchfield'); +} + +function suppressSearchToolbarDescendants( + nodes: RawSnapshotNode[], + position: number, + keptSearch: RawSnapshotNode, + context: SnapshotTreeRuleContext, +): void { + forEachDescendant(nodes, position, (descendant) => { + if (descendant.index === keptSearch.index) { + return; + } + if (shouldSuppressIosSearchToolbarDescendant(descendant)) { + context.suppressNode(descendant, [keptSearch]); + } + }); +} + +function suppressToolbarAncestors( + node: RawSnapshotNode, + representative: RawSnapshotNode, + context: SnapshotTreeRuleContext, +): void { + let current = node; + while (typeof current.parentIndex === 'number') { + const parent = context.sourceNodesByIndex.get(current.parentIndex); + if (!parent || parent.label !== 'Toolbar') { + return; + } + context.suppressNode(parent, [representative]); + current = parent; + } +} + +function shouldSuppressIosSearchToolbarDescendant(node: RawSnapshotNode): boolean { + const type = normalizeType(node.type ?? ''); + if (type === 'button') { + return false; + } + if (type === 'image') { + return true; + } + return node.label === 'Search'; +} diff --git a/packages/capture-kit/src/ios-snapshot-engine/noise-structural.ts b/packages/capture-kit/src/ios-snapshot-engine/noise-structural.ts new file mode 100644 index 0000000000..a7ae117622 --- /dev/null +++ b/packages/capture-kit/src/ios-snapshot-engine/noise-structural.ts @@ -0,0 +1,40 @@ +import type { RawSnapshotNode } from '@agent-device/kernel/snapshot'; +import { normalizeType } from '@agent-device/contracts/snapshot'; +import { collectChildrenByParent, type SnapshotTreeRuleContext } from './tree.ts'; + +export function collectIosStructuralIdentifierSuppression( + nodes: RawSnapshotNode[], + context: SnapshotTreeRuleContext, +): void { + const childrenByParent = collectChildrenByParent(nodes); + for (const node of nodes) { + if (normalizeType(node.type ?? '') !== 'other') { + continue; + } + if (node.hittable === true || node.label?.trim() || node.value?.trim()) { + continue; + } + if (!node.identifier?.trim()) { + continue; + } + context.suppressNode(node, collectSubtreeByParentLinks(node, childrenByParent)); + } +} + +function collectSubtreeByParentLinks( + root: RawSnapshotNode, + childrenByParent: ReadonlyMap, +): RawSnapshotNode[] { + const descendants: RawSnapshotNode[] = []; + const visited = new Set([root.index]); + const pending = [...(childrenByParent.get(root.index) ?? [])]; + while (pending.length > 0) { + const current = pending.pop(); + if (!current || visited.has(current.index)) continue; + visited.add(current.index); + descendants.push(current); + const children = childrenByParent.get(current.index); + if (children) pending.push(...children); + } + return descendants; +} diff --git a/packages/capture-kit/src/ios-snapshot-engine/noise-viewport.ts b/packages/capture-kit/src/ios-snapshot-engine/noise-viewport.ts new file mode 100644 index 0000000000..e50409c187 --- /dev/null +++ b/packages/capture-kit/src/ios-snapshot-engine/noise-viewport.ts @@ -0,0 +1,54 @@ +import type { RawSnapshotNode } from '@agent-device/kernel/snapshot'; +import { normalizeType } from '@agent-device/contracts/snapshot'; +import { + findLargestViewportRect, + forEachDescendant, + type SnapshotTreeRuleContext, +} from './tree.ts'; + +export function collectIosOffscreenKeyboardSuppression( + nodes: RawSnapshotNode[], + context: SnapshotTreeRuleContext, +): void { + const viewport = findLargestViewportRect(nodes); + const screenBottom = viewport ? viewport.y + viewport.height : null; + if (screenBottom === null) { + return; + } + for (let position = 0; position < nodes.length; position += 1) { + const node = nodes[position]; + if (!node || !isOffscreenKeyboardNode(node, screenBottom)) { + continue; + } + context.suppressNode(node, []); + suppressOffscreenKeyboardAncestors(node, context, screenBottom); + forEachDescendant(nodes, position, (descendant) => { + context.suppressNode(descendant, []); + }); + } +} + +function isOffscreenKeyboardNode(node: RawSnapshotNode, screenBottom: number): boolean { + if (!node.rect || normalizeType(node.type ?? '') !== 'keyboard') { + return false; + } + return node.rect.y >= screenBottom; +} + +function suppressOffscreenKeyboardAncestors( + node: RawSnapshotNode, + context: SnapshotTreeRuleContext, + screenBottom: number, +): void { + let current = + typeof node.parentIndex === 'number' + ? context.sourceNodesByIndex.get(node.parentIndex) + : undefined; + while (current?.rect && current.rect.y >= screenBottom) { + context.suppressNode(current, []); + current = + typeof current.parentIndex === 'number' + ? context.sourceNodesByIndex.get(current.parentIndex) + : undefined; + } +} diff --git a/packages/capture-kit/src/ios-snapshot-engine/noise.ts b/packages/capture-kit/src/ios-snapshot-engine/noise.ts new file mode 100644 index 0000000000..433cf90fcf --- /dev/null +++ b/packages/capture-kit/src/ios-snapshot-engine/noise.ts @@ -0,0 +1,28 @@ +import type { RawSnapshotNode } from '@agent-device/kernel/snapshot'; +import { collectIosScrollIndicatorPresentation } from './scroll.ts'; +import { + collectIosReactNativeOverlayActionPresentation, + collectIosReactNativeOverlayWrapperSuppression, +} from './noise-overlay.ts'; +import { collectIosOffscreenKeyboardSuppression } from './noise-viewport.ts'; +import { collectIosSearchToolbarSuppression } from './noise-search.ts'; +import { + collectIosActionWrapperSuppression, + collectIosRepeatedStaticSuppression, +} from './noise-redundancy.ts'; +import { collectIosStructuralIdentifierSuppression } from './noise-structural.ts'; +import type { SnapshotTreeRuleContext } from './tree.ts'; + +export function collectIosPresentationNoiseSuppression( + nodes: RawSnapshotNode[], + context: SnapshotTreeRuleContext, +): void { + collectIosOffscreenKeyboardSuppression(nodes, context); + collectIosStructuralIdentifierSuppression(nodes, context); + collectIosScrollIndicatorPresentation(nodes, context); + collectIosSearchToolbarSuppression(nodes, context); + collectIosActionWrapperSuppression(nodes, context); + collectIosReactNativeOverlayActionPresentation(nodes, context.replacements); + collectIosReactNativeOverlayWrapperSuppression(nodes, context); + collectIosRepeatedStaticSuppression(nodes, context); +} diff --git a/packages/capture-kit/src/ios-snapshot-engine/projection.ts b/packages/capture-kit/src/ios-snapshot-engine/projection.ts new file mode 100644 index 0000000000..ecb33c8c3e --- /dev/null +++ b/packages/capture-kit/src/ios-snapshot-engine/projection.ts @@ -0,0 +1,203 @@ +import type { RawSnapshotNode } from '@agent-device/kernel/snapshot'; +import { normalizeType } from '@agent-device/contracts/snapshot'; +import type { IosSnapshotFoldPolicy, IosSnapshotPresentationNode } from './types.ts'; + +const REGULAR_ELIGIBLE_TYPES = new Set([ + 'button', + 'cell', + 'checkbox', + 'collectionview', + 'link', + 'menuitem', + 'picker', + 'searchfield', + 'securetextfield', + 'segmentedcontrol', + 'slider', + 'scrollview', + 'stepper', + 'switch', + 'tabbar', + 'table', + 'textfield', + 'textview', + 'webview', +]); + +type ProjectionInput = Readonly<{ + nodes: readonly IosSnapshotPresentationNode[]; + projection: 'regular' | 'raw'; + scope: string | null; + depth: number | null; + foldPolicy: IosSnapshotFoldPolicy; +}>; + +export type IosSnapshotProjectionResult = Readonly<{ + nodes: RawSnapshotNode[]; + sourceIndexes: readonly number[]; +}>; + +export function projectIosSnapshot(input: ProjectionInput): IosSnapshotProjectionResult { + const scoped = scopeIosSnapshotNodes(input); + return input.projection === 'raw' + ? projectRawNodes(scoped, input.depth) + : projectRegularNodes(scoped, input.depth); +} + +export function projectIosQualitySnapshot( + input: Omit, +): IosSnapshotProjectionResult { + return input.projection === 'raw' + ? projectRawNodes(input.nodes, input.depth) + : projectRegularNodes(input.nodes, input.depth); +} + +function isEligibleForIosRegularPresentation(node: RawSnapshotNode): boolean { + if (node.parentIndex === undefined) return true; + return REGULAR_ELIGIBLE_TYPES.has(normalizeType(node.type ?? '')) || hasSemanticContent(node); +} + +function scopeIosSnapshotNodes(input: ProjectionInput): IosSnapshotPresentationNode[] { + const query = input.scope?.trim().toLowerCase(); + if (!query) return [...input.nodes]; + + for (let start = 0; start < input.nodes.length; start += 1) { + const candidate = input.nodes[start]; + if (!candidate || !matchesScope(candidate.raw, query)) continue; + const range = subtreeRange(input.nodes, start); + const contributes = + input.projection === 'raw' || + range.some((position) => isEligibleForIosRegularPresentation(input.nodes[position]!.raw)); + if (!contributes) continue; + + const scoped = range.map((position) => input.nodes[position]!); + const limited = + input.projection === 'raw' && input.depth !== null + ? scoped.filter((node) => rawDepth(node) - rawDepth(candidate) <= input.depth!) + : scoped; + return reindexScopedNodes(limited, rawDepth(candidate)); + } + return []; +} + +function projectRawNodes( + nodes: readonly IosSnapshotPresentationNode[], + maximumDepth: number | null, +): IosSnapshotProjectionResult { + const selected = + maximumDepth === null ? [...nodes] : nodes.filter((node) => rawDepth(node) <= maximumDepth); + return { + nodes: selected.map((node) => ({ ...node.raw, rect: node.raw.rect })), + sourceIndexes: selected.map((node) => node.raw.index), + }; +} + +function projectRegularNodes( + sourceNodes: readonly IosSnapshotPresentationNode[], + maximumDepth: number | null, +): IosSnapshotProjectionResult { + const presented: RawSnapshotNode[] = []; + const sourceIndexes: number[] = []; + const nearestPresented = new Map(); + + for (const node of sourceNodes) { + const parent = readNearestPresentedParent(node, nearestPresented); + const projected = createRegularProjectedNode(node, parent, maximumDepth, presented.length); + if (!projected) { + rememberNearestPresented(node, parent, nearestPresented); + continue; + } + presented.push(projected); + sourceIndexes.push(node.raw.index); + nearestPresented.set(node.raw.index, projected); + } + return { nodes: presented, sourceIndexes }; +} + +function readNearestPresentedParent( + node: IosSnapshotPresentationNode, + nearestPresented: ReadonlyMap, +): RawSnapshotNode | undefined { + return node.raw.parentIndex === undefined + ? undefined + : nearestPresented.get(node.raw.parentIndex); +} + +function createRegularProjectedNode( + node: IosSnapshotPresentationNode, + parent: RawSnapshotNode | undefined, + maximumDepth: number | null, + index: number, +): RawSnapshotNode | undefined { + if (!isEligibleForIosRegularPresentation(node.raw)) return undefined; + const depth = parent ? (parent.depth ?? 0) + 1 : 0; + if (maximumDepth !== null && depth > maximumDepth) return undefined; + return { + ...node.raw, + index, + depth, + parentIndex: parent?.index, + ...(node.effectiveRect ? { rect: node.effectiveRect } : { rect: undefined }), + hittable: isProjectedNodeHittable(node), + }; +} + +function isProjectedNodeHittable(node: IosSnapshotPresentationNode): boolean { + return Boolean( + node.raw.hittable === true && + node.effectiveRect && + node.effectiveRect.width > 0 && + node.effectiveRect.height > 0, + ); +} + +function rememberNearestPresented( + node: IosSnapshotPresentationNode, + parent: RawSnapshotNode | undefined, + nearestPresented: Map, +): void { + if (parent) nearestPresented.set(node.raw.index, parent); +} + +function reindexScopedNodes( + nodes: readonly IosSnapshotPresentationNode[], + depthOffset: number, +): IosSnapshotPresentationNode[] { + const indexMap = new Map(nodes.map((node, index) => [node.raw.index, index])); + return nodes.map((node, index) => ({ + ...node, + raw: { + ...node.raw, + index, + depth: Math.max(0, rawDepth(node) - depthOffset), + parentIndex: + node.raw.parentIndex === undefined ? undefined : indexMap.get(node.raw.parentIndex), + }, + })); +} + +function matchesScope(node: RawSnapshotNode, query: string): boolean { + return [node.label, node.identifier, node.value].some( + (value) => typeof value === 'string' && value.toLowerCase().includes(query), + ); +} + +function subtreeRange(nodes: readonly IosSnapshotPresentationNode[], start: number): number[] { + const rootDepth = rawDepth(nodes[start]!); + const positions: number[] = []; + for (let position = start; position < nodes.length; position += 1) { + if (position > start && rawDepth(nodes[position]!) <= rootDepth) break; + positions.push(position); + } + return positions; +} + +function rawDepth(node: IosSnapshotPresentationNode): number { + return Math.max(0, node.raw.depth ?? 0); +} + +function hasSemanticContent(node: RawSnapshotNode): boolean { + return [node.label, node.identifier, node.value].some( + (value) => typeof value === 'string' && value.trim().length > 0, + ); +} diff --git a/src/snapshot/snapshot-presentation/ios/rows.ts b/packages/capture-kit/src/ios-snapshot-engine/rows.ts similarity index 99% rename from src/snapshot/snapshot-presentation/ios/rows.ts rename to packages/capture-kit/src/ios-snapshot-engine/rows.ts index 06613ba7a0..541adf29da 100644 --- a/src/snapshot/snapshot-presentation/ios/rows.ts +++ b/packages/capture-kit/src/ios-snapshot-engine/rows.ts @@ -8,7 +8,7 @@ import { mergeReplacement, shouldSuppressRepeatedTextDescendant, type SnapshotTreeRuleContext, -} from '../tree.ts'; +} from './tree.ts'; export function collectIosRowPresentation( nodes: RawSnapshotNode[], diff --git a/packages/capture-kit/src/ios-snapshot-engine/runner-presentation.ts b/packages/capture-kit/src/ios-snapshot-engine/runner-presentation.ts new file mode 100644 index 0000000000..ce86199bf0 --- /dev/null +++ b/packages/capture-kit/src/ios-snapshot-engine/runner-presentation.ts @@ -0,0 +1,139 @@ +import { buildIosSnapshotPresentationKey } from '../ios-snapshot-planning.ts'; +import type { IosSnapshotInput, IosSnapshotRequest } from '@agent-device/contracts/ios-snapshot'; +import type { Rect, RawSnapshotNode } from '@agent-device/kernel/snapshot'; +import { buildIosInteractiveSnapshotPresentation } from './semantic-index.ts'; +import { resolveViewportEvidence, validateIosPayload } from './invariants.ts'; +import type { IosSnapshotEnginePresentation, IosSnapshotFoldPolicy } from './types.ts'; +import { IosSnapshotEngineError } from './types.ts'; + +export function presentIosRunnerSnapshot( + input: Extract, + request: IosSnapshotRequest, + foldPolicy: IosSnapshotFoldPolicy, +): IosSnapshotEnginePresentation { + validateRunnerRequest(input, request); + const projection = input.validation.presentationKey.projection; + const viewport = + projection === 'regular' ? resolveViewportEvidence(input.validation.viewport) : undefined; + const hittabilityAvailable = input.validation.hittability.kind === 'available'; + const payloadValidation = validateRunnerPayloads( + input, + projection, + viewport, + foldPolicy, + hittabilityAvailable, + ); + const compacted = compactRunnerPayload(input.presentation.payload.nodes, projection, request); + const validationStats = validateRunnerOutput( + compacted.nodes, + projection, + viewport, + foldPolicy, + hittabilityAvailable, + payloadValidation, + ); + return { + nodes: compacted.nodes, + ...(input.presentation.qualityPayload + ? { qualityNodes: [...input.presentation.qualityPayload.nodes] } + : {}), + presentedIndexesBySourceIndex: compacted.presentedIndexesBySourceIndex, + stats: { + presentedNodeCount: compacted.nodes.length, + sourceNodeCount: input.presentation.payload.nodes.length, + parentClipLookups: validationStats.parentClipLookups, + }, + }; +} + +function validateRunnerRequest( + input: Extract, + request: IosSnapshotRequest, +): void { + const expectedKey = buildIosSnapshotPresentationKey(request); + if (!presentationKeysEqual(expectedKey, input.validation.presentationKey)) { + throw new IosSnapshotEngineError( + 'projection-mismatch', + 'presented iOS snapshot does not match the requested presentation key', + { projection: input.validation.presentationKey.projection }, + ); + } + if (input.presentation.intent !== request.acquisitionIntent) { + throw new IosSnapshotEngineError( + 'projection-mismatch', + 'presented iOS snapshot does not match the requested acquisition intent', + { field: 'acquisitionIntent' }, + ); + } +} + +function validateRunnerPayloads( + input: Extract, + projection: 'regular' | 'raw', + viewport: Rect | undefined, + foldPolicy: IosSnapshotFoldPolicy, + hittabilityAvailable: boolean, +): { parentClipLookups: number } { + const payloadValidation = validateIosPayload( + input.presentation.payload.nodes, + projection, + viewport, + foldPolicy, + hittabilityAvailable, + ); + if (input.presentation.qualityPayload) { + validateIosPayload( + input.presentation.qualityPayload.nodes, + projection, + viewport, + foldPolicy, + hittabilityAvailable, + ); + } + return payloadValidation; +} + +function compactRunnerPayload( + nodes: readonly RawSnapshotNode[], + projection: 'regular' | 'raw', + request: IosSnapshotRequest, +): ReturnType { + if (projection === 'regular' && request.interactiveOnly) { + return buildIosInteractiveSnapshotPresentation([...nodes]); + } + return { + nodes: [...nodes], + presentedIndexesBySourceIndex: identityMapping(nodes), + }; +} + +function validateRunnerOutput( + nodes: readonly RawSnapshotNode[], + projection: 'regular' | 'raw', + viewport: Rect | undefined, + foldPolicy: IosSnapshotFoldPolicy, + hittabilityAvailable: boolean, + payloadValidation: { parentClipLookups: number }, +): { parentClipLookups: number } { + if (projection !== 'regular' || !viewport) return payloadValidation; + return validateIosPayload(nodes, projection, viewport, foldPolicy, hittabilityAvailable); +} + +function presentationKeysEqual( + left: ReturnType, + right: ReturnType, +): boolean { + return ( + left.projection === right.projection && + left.interactiveOnly === right.interactiveOnly && + left.depth === right.depth && + left.scope === right.scope && + left.customActions === right.customActions + ); +} + +function identityMapping( + nodes: readonly RawSnapshotNode[], +): ReadonlyMap { + return new Map(nodes.map((node) => [node.index, [node.index]])); +} diff --git a/src/snapshot/snapshot-presentation/ios/scroll.ts b/packages/capture-kit/src/ios-snapshot-engine/scroll.ts similarity index 99% rename from src/snapshot/snapshot-presentation/ios/scroll.ts rename to packages/capture-kit/src/ios-snapshot-engine/scroll.ts index 09ede96247..537a2d5942 100644 --- a/src/snapshot/snapshot-presentation/ios/scroll.ts +++ b/packages/capture-kit/src/ios-snapshot-engine/scroll.ts @@ -8,7 +8,7 @@ import { isScrollableSnapshotType, updateReplacement, type SnapshotTreeRuleContext, -} from '../tree.ts'; +} from './tree.ts'; export function collectIosScrollIndicatorPresentation( nodes: RawSnapshotNode[], diff --git a/src/snapshot/snapshot-presentation/ios/index.ts b/packages/capture-kit/src/ios-snapshot-engine/semantic-index.ts similarity index 97% rename from src/snapshot/snapshot-presentation/ios/index.ts rename to packages/capture-kit/src/ios-snapshot-engine/semantic-index.ts index 124cb552f0..615428b153 100644 --- a/src/snapshot/snapshot-presentation/ios/index.ts +++ b/packages/capture-kit/src/ios-snapshot-engine/semantic-index.ts @@ -4,10 +4,7 @@ import { collectIosPresentationNoiseSuppression } from './noise.ts'; import { collectIosRowPresentation } from './rows.ts'; import { collectIosTransitionPresentation } from './transitions.ts'; import { collectIosWebSemanticPresentation } from './web.ts'; -import { - reindexSnapshotNodesWithSuppressedParents, - type SnapshotTreeRuleContext, -} from '../tree.ts'; +import { reindexSnapshotNodesWithSuppressedParents, type SnapshotTreeRuleContext } from './tree.ts'; const IOS_PRESENTATION_RULES: Array< (nodes: RawSnapshotNode[], context: SnapshotTreeRuleContext) => void diff --git a/src/snapshot/snapshot-presentation/ios/transitions.ts b/packages/capture-kit/src/ios-snapshot-engine/transitions.ts similarity index 97% rename from src/snapshot/snapshot-presentation/ios/transitions.ts rename to packages/capture-kit/src/ios-snapshot-engine/transitions.ts index 92e01933be..5439c6eda9 100644 --- a/src/snapshot/snapshot-presentation/ios/transitions.ts +++ b/packages/capture-kit/src/ios-snapshot-engine/transitions.ts @@ -1,11 +1,7 @@ import type { RawSnapshotNode, Rect } from '@agent-device/kernel/snapshot'; import { rectContains } from '@agent-device/kernel/rect'; import { extractNodeText, normalizeType } from '@agent-device/contracts/snapshot'; -import { - collectChildrenByParent, - mergeReplacement, - type SnapshotTreeRuleContext, -} from '../tree.ts'; +import { collectChildrenByParent, mergeReplacement, type SnapshotTreeRuleContext } from './tree.ts'; import { collectIosReplacedActionShelves } from './action-shelf.ts'; const TITLE_PIECE_GAP_TOLERANCE = 12; diff --git a/src/snapshot/snapshot-presentation/tree.ts b/packages/capture-kit/src/ios-snapshot-engine/tree.ts similarity index 100% rename from src/snapshot/snapshot-presentation/tree.ts rename to packages/capture-kit/src/ios-snapshot-engine/tree.ts diff --git a/packages/capture-kit/src/ios-snapshot-engine/types.ts b/packages/capture-kit/src/ios-snapshot-engine/types.ts new file mode 100644 index 0000000000..88b4970788 --- /dev/null +++ b/packages/capture-kit/src/ios-snapshot-engine/types.ts @@ -0,0 +1,65 @@ +import type { Rect, RawSnapshotNode } from '@agent-device/kernel/snapshot'; + +export type IosSnapshotFoldPolicy = 'cursor-projected' | 'plain-viewport'; + +export type IosSnapshotEngineOptions = Readonly<{ + foldPolicy?: IosSnapshotFoldPolicy; +}>; + +export type IosSnapshotFoldOptions = Readonly<{ + hittabilityAvailable?: boolean; +}>; + +export type IosSnapshotPresentationNode = Readonly<{ + raw: RawSnapshotNode; + effectiveRect?: Rect; +}>; + +export type IosSnapshotPresentationStats = Readonly<{ + presentedNodeCount: number; + sourceNodeCount: number; + parentClipLookups: number; +}>; + +export type IosSnapshotEnginePresentation = Readonly<{ + nodes: RawSnapshotNode[]; + qualityNodes?: RawSnapshotNode[]; + presentedIndexesBySourceIndex: ReadonlyMap; + stats: IosSnapshotPresentationStats; +}>; + +export type IosSnapshotEngineFailureReason = + | 'projection-mismatch' + | 'missing-viewport' + | 'invalid-viewport' + | 'malformed-graph' + | 'regular-node-outside-cumulative-clip' + | 'regular-degenerate-actionable-node' + | 'invalid-presented-payload' + | 'invalid-quality-payload'; + +export type IosSnapshotEngineFailureDetails = Readonly<{ + index?: number; + parentIndex?: number; + frame?: Rect; + clip?: Rect; + projection?: string; + field?: string; +}>; + +export class IosSnapshotEngineError extends Error { + readonly code = 'IOS_SNAPSHOT_ENGINE_FAILED'; + readonly reason: IosSnapshotEngineFailureReason; + readonly details: IosSnapshotEngineFailureDetails; + + constructor( + reason: IosSnapshotEngineFailureReason, + message: string, + details: IosSnapshotEngineFailureDetails = {}, + ) { + super(message); + this.name = 'IosSnapshotEngineError'; + this.reason = reason; + this.details = details; + } +} diff --git a/src/snapshot/snapshot-presentation/ios/web.ts b/packages/capture-kit/src/ios-snapshot-engine/web.ts similarity index 99% rename from src/snapshot/snapshot-presentation/ios/web.ts rename to packages/capture-kit/src/ios-snapshot-engine/web.ts index 7e7c83fbc8..a5d1b699e6 100644 --- a/src/snapshot/snapshot-presentation/ios/web.ts +++ b/packages/capture-kit/src/ios-snapshot-engine/web.ts @@ -5,7 +5,7 @@ import { findNearestAncestor, mergeReplacement, type SnapshotTreeRuleContext, -} from '../tree.ts'; +} from './tree.ts'; /** * WebKit exposes HTML text through an `Other -> StaticText` wrapper pair on iOS. diff --git a/scripts/check-affected/device-lanes.test.ts b/scripts/check-affected/device-lanes.test.ts index 4bd07f1f31..72c44b9fca 100644 --- a/scripts/check-affected/device-lanes.test.ts +++ b/scripts/check-affected/device-lanes.test.ts @@ -32,7 +32,7 @@ test('a TypeScript-only Apple change selects the iOS and macOS lanes without a S ), ); } - assert.deepEqual(lanes('src/snapshot/snapshot-presentation/ios/action-shelf.ts'), [...IOS]); + assert.deepEqual(lanes('packages/capture-kit/src/ios-snapshot-engine/action-shelf.ts'), [...IOS]); assert.deepEqual(lanes('test/integration/replays/macos/01-desktop.ad'), ['replay-macos']); }); diff --git a/scripts/check-affected/device-lanes.ts b/scripts/check-affected/device-lanes.ts index 051f2a3922..7ba9503c6d 100644 --- a/scripts/check-affected/device-lanes.ts +++ b/scripts/check-affected/device-lanes.ts @@ -14,7 +14,7 @@ // can: `CANONICAL_PLATFORM_FAMILIES` names the families, layering R13 pins each family's runtime // to `packages/platform-/`, and the remaining family-owned // trees are named by a family or Apple-leaf directory segment (`android/`, `linux/`, -// `test/integration/replays//`, `src/snapshot/snapshot-presentation/ios/`) or, under +// `test/integration/replays//`, `packages/capture-kit/src/ios-snapshot-engine/`) or, under // `test/integration/`, by the lane prefix of the smoke file. A path tagged with exactly one // family owns that family's lanes; a path tagged with none — or with two — is shared runtime // surface and owns every lane. Unit tests (`*.test.ts`, `__tests__/`) under `src/` and @@ -55,6 +55,7 @@ const LEAF_LANES: Readonly> = { // `xcuitest`, whose only lanes today are the Apple family's) owns every Apple lane. const APPLE_LEAF_TAGS: Readonly> = { ios: 'ios', + 'ios-snapshot-engine': 'ios', macos: 'macos', apple: 'apple', tvos: 'apple', diff --git a/scripts/layering/package-boundaries.test.ts b/scripts/layering/package-boundaries.test.ts index aad7777b3e..104f1a77ab 100644 --- a/scripts/layering/package-boundaries.test.ts +++ b/scripts/layering/package-boundaries.test.ts @@ -458,6 +458,7 @@ test('the real tree parses, declares, and passes R11', () => { ); assert.deepEqual([...captureKitPackage.exportTargets.keys()].sort(), [ '@agent-device/capture-kit', + '@agent-device/capture-kit/ios-snapshot-engine', '@agent-device/capture-kit/ios-snapshot-planning', '@agent-device/capture-kit/mobile-snapshot-semantics', '@agent-device/capture-kit/png', diff --git a/src/__tests__/eager-closure-budgets.ts b/src/__tests__/eager-closure-budgets.ts index 172bd39efe..9925dd7d6f 100644 --- a/src/__tests__/eager-closure-budgets.ts +++ b/src/__tests__/eager-closure-budgets.ts @@ -122,6 +122,8 @@ export const FACADE_BUDGETS: Readonly> = Object.freeze({ 'packages/capture-kit/src/index.ts': 32, // #2190 keeps iOS snapshot planning behind its dedicated subpath instead of the broad root. 'packages/capture-kit/src/ios-snapshot-planning.ts': 1, + // #2191 keeps the iOS snapshot engine behind its dedicated subpath instead of the broad root. + 'packages/capture-kit/src/ios-snapshot-engine/index.ts': 36, 'packages/capture-kit/src/png-resize.ts': 18, 'packages/capture-kit/src/png-rgb-difference.ts': 1, 'packages/capture-kit/src/png-size.ts': 3, diff --git a/src/core/snapshot-state.ts b/src/core/snapshot-state.ts index 5e9a0b83de..b914c80c7a 100644 --- a/src/core/snapshot-state.ts +++ b/src/core/snapshot-state.ts @@ -21,7 +21,7 @@ import { import { coveredAndroidReplacementNodeIndexes } from '../snapshot/android-replacement-surface-occlusion.ts'; import { scopeSnapshotNodes } from '@agent-device/capture-kit/snapshot-desktop-projection'; import { normalizeSnapshotTree, pruneGroupNodes } from '../core/snapshot-tree-ingestion.ts'; -import { presentIosInteractiveSnapshot } from '../snapshot/snapshot-presentation/ios/index.ts'; +import { presentIosInteractiveSnapshot } from '@agent-device/capture-kit/ios-snapshot-engine'; /** * The ONE daemon presentation of a captured tree (ADR 0004 / #1797 "compaction layer"): normalize, diff --git a/src/daemon/__tests__/snapshot-presentation-transitions.test.ts b/src/daemon/__tests__/snapshot-presentation-transitions.test.ts index 7926fb6955..f9123598dc 100644 --- a/src/daemon/__tests__/snapshot-presentation-transitions.test.ts +++ b/src/daemon/__tests__/snapshot-presentation-transitions.test.ts @@ -2,7 +2,7 @@ import { expect, test } from 'vitest'; import { makeSnapshotState } from '../../__tests__/test-utils/snapshot-builders.ts'; import { createInteractionDevice } from '../../commands/interaction/runtime/__tests__/test-utils/index.ts'; import { buildSnapshotState } from '../../core/snapshot-state.ts'; -import { presentIosInteractiveSnapshot } from '../../snapshot/snapshot-presentation/ios/index.ts'; +import { presentIosInteractiveSnapshot } from '@agent-device/capture-kit/ios-snapshot-engine'; import { navigationTitleWithAppProvidedDetailsAffordanceNodes } from '../../snapshot/snapshot-presentation/ios/transitions.fixtures.ts'; test('iOS daemon presentation applies transitions without reapplying runner-owned scope', () => { diff --git a/src/daemon/adapters/maestro/daemon-runtime-port-observation.ts b/src/daemon/adapters/maestro/daemon-runtime-port-observation.ts index a5056036f6..2275e2e7f6 100644 --- a/src/daemon/adapters/maestro/daemon-runtime-port-observation.ts +++ b/src/daemon/adapters/maestro/daemon-runtime-port-observation.ts @@ -24,7 +24,7 @@ import { } from '../../touch-reference-frame.ts'; import { isPositiveFiniteRect, rectContains } from '@agent-device/kernel/rect'; import type { Rect, SnapshotState } from '@agent-device/kernel/snapshot'; -import { buildIosInteractiveSnapshotPresentation } from '../../../snapshot/snapshot-presentation/ios/index.ts'; +import { buildIosInteractiveSnapshotPresentation } from '@agent-device/capture-kit/ios-snapshot-engine'; export const MAESTRO_OBSERVATION_POLL_MS = MAESTRO_RUNTIME_ADAPTER_POLICY.observationPollMs; export type DaemonMaestroRuntimeDependencies = { diff --git a/src/daemon/session-target-evidence.ts b/src/daemon/session-target-evidence.ts index 7a3173de4c..0a60907d01 100644 --- a/src/daemon/session-target-evidence.ts +++ b/src/daemon/session-target-evidence.ts @@ -22,7 +22,7 @@ import type { SnapshotNode } from '@agent-device/kernel/snapshot'; import { resolveRectCenter } from '@agent-device/kernel/rect-center'; -import { findNearestScrollableContainer } from '../snapshot/snapshot-presentation/tree.ts'; +import { findNearestScrollableContainer } from '@agent-device/capture-kit/ios-snapshot-engine'; import { buildAncestryChain, buildIndexMap, diff --git a/src/snapshot/snapshot-presentation/ios/action-shelf.test.ts b/src/snapshot/snapshot-presentation/ios/action-shelf.test.ts index da751573ab..7625505c14 100644 --- a/src/snapshot/snapshot-presentation/ios/action-shelf.test.ts +++ b/src/snapshot/snapshot-presentation/ios/action-shelf.test.ts @@ -1,5 +1,5 @@ import { expect, test } from 'vitest'; -import { presentIosInteractiveSnapshot } from './index.ts'; +import { presentIosInteractiveSnapshot } from '@agent-device/capture-kit/ios-snapshot-engine'; import { closedComposerWithRetainedActionShelfNodes, closedComposerWithRetainedRegularTreeActionNodes, diff --git a/src/snapshot/snapshot-presentation/ios/mapping.test.ts b/src/snapshot/snapshot-presentation/ios/mapping.test.ts index f4dd23dc4f..885f3071c8 100644 --- a/src/snapshot/snapshot-presentation/ios/mapping.test.ts +++ b/src/snapshot/snapshot-presentation/ios/mapping.test.ts @@ -1,6 +1,6 @@ import { expect, test } from 'vitest'; import type { RawSnapshotNode } from '@agent-device/kernel/snapshot'; -import { buildIosInteractiveSnapshotPresentation } from './index.ts'; +import { buildIosInteractiveSnapshotPresentation } from '@agent-device/capture-kit/ios-snapshot-engine'; test('publishes an exact representative for every semantic source index', () => { const nodes: RawSnapshotNode[] = [ diff --git a/src/snapshot/snapshot-presentation/ios/noise.test.ts b/src/snapshot/snapshot-presentation/ios/noise.test.ts index f0df0c7870..fa6a3b277f 100644 --- a/src/snapshot/snapshot-presentation/ios/noise.test.ts +++ b/src/snapshot/snapshot-presentation/ios/noise.test.ts @@ -2,8 +2,8 @@ import { describe, expect, test } from 'vitest'; import type { RawSnapshotNode } from '@agent-device/kernel/snapshot'; -import { collectIosStructuralIdentifierSuppression } from './noise.ts'; -import type { SnapshotTreeRuleContext } from '../tree.ts'; +import { collectIosStructuralIdentifierSuppression } from '@agent-device/capture-kit/ios-snapshot-engine'; +import type { SnapshotTreeRuleContext } from '@agent-device/capture-kit/ios-snapshot-engine'; type ReadCounter = { reads: number }; diff --git a/src/snapshot/snapshot-presentation/ios/noise.ts b/src/snapshot/snapshot-presentation/ios/noise.ts deleted file mode 100644 index 6b0b0d97e5..0000000000 --- a/src/snapshot/snapshot-presentation/ios/noise.ts +++ /dev/null @@ -1,434 +0,0 @@ -import type { RawSnapshotNode } from '@agent-device/kernel/snapshot'; -import { rectArea, rectContains } from '@agent-device/kernel/rect'; -import { - isReactNativeCollapsedWarningWrapperCandidate, - isReactNativeCollapsedWarningWrapperWithVisibleBanner, - isReactNativeOverlayDismissLabel, - isReactNativeOverlayMinimizeLabel, -} from '@agent-device/contracts/react-native-overlay'; -import { normalizeType } from '@agent-device/contracts/snapshot'; -import { collectIosScrollIndicatorPresentation } from './scroll.ts'; -import { - areRectsApproximatelyEqual, - collectChildrenByParent, - findDescendant, - findLargestViewportRect, - forEachDescendant, - isRepeatedStaticNode, - isScrollableSnapshotType, - isSemanticActionNode, - mergeReplacement, - type SnapshotTreeRuleContext, -} from '../tree.ts'; - -export function collectIosPresentationNoiseSuppression( - nodes: RawSnapshotNode[], - context: SnapshotTreeRuleContext, -): void { - collectIosOffscreenKeyboardSuppression(nodes, context); - collectIosStructuralIdentifierSuppression(nodes, context); - collectIosScrollIndicatorPresentation(nodes, context); - collectIosSearchToolbarSuppression(nodes, context); - collectIosActionWrapperSuppression(nodes, context); - collectIosReactNativeOverlayActionPresentation(nodes, context.replacements); - collectIosReactNativeOverlayWrapperSuppression(nodes, context); - collectIosRepeatedStaticSuppression(nodes, context); -} - -function collectIosReactNativeOverlayActionPresentation( - nodes: RawSnapshotNode[], - replacements: Map, -): void { - forEachOtherNodeWithLabel(nodes, (node, nodeLabel, position) => { - if (!isReactNativeOverlayDismissLabel(nodeLabel) || !node.rect) return; - const minimize = findDescendant( - nodes, - position, - (descendant) => - Boolean(descendant.rect) && - isReactNativeOverlayMinimizeLabel(descendant.label?.trim() ?? ''), - ); - if (!minimize?.rect) return; - const dismissRect = remainingHorizontalPartition(node.rect, minimize.rect); - if (!dismissRect) return; - const representativeRect = smallestContainedDismissRect(nodes, position, dismissRect); - mergeReplacement(replacements, node, { rect: representativeRect }); - forEachDescendant(nodes, position, (descendant) => { - if (isReactNativeOverlayDismissLabel(descendant.label?.trim() ?? '')) { - mergeReplacement(replacements, descendant, { rect: representativeRect }); - } - }); - }); -} - -function smallestContainedDismissRect( - nodes: RawSnapshotNode[], - position: number, - partition: NonNullable, -): NonNullable { - let representative = partition; - forEachDescendant(nodes, position, (descendant) => { - const label = descendant.label?.trim() ?? ''; - if (!descendant.rect || !isReactNativeOverlayDismissLabel(label)) return; - if (!rectContains(partition, descendant.rect)) return; - if (rectArea(descendant.rect) < rectArea(representative)) { - representative = descendant.rect; - } - }); - return representative; -} - -function remainingHorizontalPartition( - wrapper: NonNullable, - occupied: NonNullable, -): NonNullable | undefined { - const wrapperRight = wrapper.x + wrapper.width; - const occupiedRight = occupied.x + occupied.width; - const expectedRightPartition = { - x: occupied.x, - y: wrapper.y, - width: wrapperRight - occupied.x, - height: wrapper.height, - }; - if (occupied.x > wrapper.x && areRectsApproximatelyEqual(occupied, expectedRightPartition)) { - return { ...wrapper, width: occupied.x - wrapper.x }; - } - const expectedLeftPartition = { - x: wrapper.x, - y: wrapper.y, - width: occupiedRight - wrapper.x, - height: wrapper.height, - }; - if (occupiedRight < wrapperRight && areRectsApproximatelyEqual(occupied, expectedLeftPartition)) { - return { ...wrapper, x: occupiedRight, width: wrapperRight - occupiedRight }; - } - return undefined; -} - -function collectIosReactNativeOverlayWrapperSuppression( - nodes: RawSnapshotNode[], - context: SnapshotTreeRuleContext, -): void { - forEachOtherNodeWithLabel(nodes, (node, _nodeLabel, position) => { - if (!isReactNativeCollapsedWarningWrapperCandidate(node)) return; - if ( - isReactNativeCollapsedWarningWrapperWithVisibleBanner( - node, - collectDescendantNodes(nodes, position), - ) - ) { - context.suppressNode(node, collectDescendantNodes(nodes, position)); - } - }); -} - -function collectDescendantNodes(nodes: RawSnapshotNode[], position: number): RawSnapshotNode[] { - const descendants: RawSnapshotNode[] = []; - forEachDescendant(nodes, position, (descendant) => { - descendants.push(descendant); - }); - return descendants; -} - -function collectIosRepeatedStaticSuppression( - nodes: RawSnapshotNode[], - context: SnapshotTreeRuleContext, -): void { - for (let position = 0; position < nodes.length; position += 1) { - const node = nodes[position]; - const nodeLabel = node?.label?.trim(); - if (!node || context.isSuppressed(node) || !nodeLabel) { - continue; - } - - collectRepeatedStaticSuppressionForNode(nodes, position, node, nodeLabel, context); - } -} - -function collectRepeatedStaticSuppressionForNode( - nodes: RawSnapshotNode[], - position: number, - node: RawSnapshotNode, - nodeLabel: string, - context: SnapshotTreeRuleContext, -): void { - const type = normalizeType(node.type ?? ''); - if (type === 'statictext' || type === 'link') { - suppressRepeatedStaticDescendants(nodes, position, nodeLabel, node, context); - return; - } - if (type !== 'other') { - return; - } - const semanticDescendant = findEquivalentSemanticDescendant(nodes, position, nodeLabel); - if (semanticDescendant) { - context.suppressNode(node, [semanticDescendant]); - return; - } - suppressRepeatedStaticDescendants(nodes, position, nodeLabel, node, context); -} - -function findEquivalentSemanticDescendant( - nodes: RawSnapshotNode[], - position: number, - nodeLabel: string, -): RawSnapshotNode | undefined { - return findDescendant(nodes, position, (descendant) => { - const type = normalizeType(descendant.type ?? ''); - return ( - (type === 'link' || type === 'searchfield' || isScrollableSnapshotType(descendant.type)) && - descendant.label?.trim() === nodeLabel - ); - }); -} - -function suppressRepeatedStaticDescendants( - nodes: RawSnapshotNode[], - position: number, - label: string, - representative: RawSnapshotNode, - context: SnapshotTreeRuleContext, -): void { - forEachDescendant(nodes, position, (descendant) => { - if ( - !context.semanticRepresentativeIndexes.has(descendant.index) && - isRepeatedStaticNode(descendant, label) - ) { - context.suppressNode(descendant, [representative]); - } - }); -} - -function collectIosActionWrapperSuppression( - nodes: RawSnapshotNode[], - context: SnapshotTreeRuleContext, -): void { - forEachOtherNodeWithLabel(nodes, (node, nodeLabel, position) => { - const semanticDescendant = findDescendant(nodes, position, (descendant) => { - return ( - isSemanticActionNode(descendant) && - descendant.label?.trim() === nodeLabel && - (areRectsApproximatelyEqual(descendant.rect, node.rect) || - isIosBackdropDismissWrapper(node, descendant)) - ); - }); - if (semanticDescendant) { - context.suppressNode(node, [semanticDescendant]); - } - }); -} - -function isIosBackdropDismissWrapper(node: RawSnapshotNode, descendant: RawSnapshotNode): boolean { - if (descendant.label?.trim() !== node.label?.trim()) { - return false; - } - const descendantType = normalizeType(descendant.type ?? ''); - return ( - isNamedButtonBackdrop(node, descendantType) || - descendantType === 'textfield' || - isFullscreenActionLabelWrapper(node, descendantType, descendant) - ); -} - -function isNamedButtonBackdrop(node: RawSnapshotNode, descendantType: string): boolean { - const label = node.label?.trim(); - return descendantType === 'button' && (label === 'Dismiss' || label === 'Back'); -} - -function isFullscreenActionLabelWrapper( - node: RawSnapshotNode, - descendantType: string, - descendant: RawSnapshotNode, -): boolean { - if (descendantType !== 'button') { - return false; - } - if (!node.rect || !descendant.rect) { - return false; - } - return ( - node.rect.x === 0 && - node.rect.y === 0 && - node.rect.width >= 300 && - node.rect.height >= 600 && - descendant.rect.width < node.rect.width - ); -} - -function collectIosOffscreenKeyboardSuppression( - nodes: RawSnapshotNode[], - context: SnapshotTreeRuleContext, -): void { - const viewport = findLargestViewportRect(nodes); - const screenBottom = viewport ? viewport.y + viewport.height : null; - if (screenBottom === null) { - return; - } - for (let position = 0; position < nodes.length; position += 1) { - const node = nodes[position]; - if (!node || !isOffscreenKeyboardNode(node, screenBottom)) { - continue; - } - context.suppressNode(node, []); - suppressOffscreenKeyboardAncestors(node, context, screenBottom); - forEachDescendant(nodes, position, (descendant) => { - context.suppressNode(descendant, []); - }); - } -} - -function isOffscreenKeyboardNode(node: RawSnapshotNode, screenBottom: number): boolean { - if (!node.rect || normalizeType(node.type ?? '') !== 'keyboard') { - return false; - } - return node.rect.y >= screenBottom; -} - -function suppressOffscreenKeyboardAncestors( - node: RawSnapshotNode, - context: SnapshotTreeRuleContext, - screenBottom: number, -): void { - let current = - typeof node.parentIndex === 'number' - ? context.sourceNodesByIndex.get(node.parentIndex) - : undefined; - while (current?.rect && current.rect.y >= screenBottom) { - context.suppressNode(current, []); - current = - typeof current.parentIndex === 'number' - ? context.sourceNodesByIndex.get(current.parentIndex) - : undefined; - } -} - -export function collectIosStructuralIdentifierSuppression( - nodes: RawSnapshotNode[], - context: SnapshotTreeRuleContext, -): void { - const childrenByParent = collectChildrenByParent(nodes); - for (const node of nodes) { - if (normalizeType(node.type ?? '') !== 'other') { - continue; - } - if (node.hittable === true || node.label?.trim() || node.value?.trim()) { - continue; - } - if (!node.identifier?.trim()) { - continue; - } - context.suppressNode(node, collectSubtreeByParentLinks(node, childrenByParent)); - } -} - -function collectSubtreeByParentLinks( - root: RawSnapshotNode, - childrenByParent: ReadonlyMap, -): RawSnapshotNode[] { - const descendants: RawSnapshotNode[] = []; - const visited = new Set([root.index]); - const pending = [...(childrenByParent.get(root.index) ?? [])]; - while (pending.length > 0) { - const current = pending.pop(); - if (!current || visited.has(current.index)) continue; - visited.add(current.index); - descendants.push(current); - const children = childrenByParent.get(current.index); - if (children) pending.push(...children); - } - return descendants; -} - -function collectIosSearchToolbarSuppression( - nodes: RawSnapshotNode[], - context: SnapshotTreeRuleContext, -): void { - for (let position = 0; position < nodes.length; position += 1) { - const node = nodes[position]; - if (!node) continue; - if (isExposedSearchField(node)) { - suppressSearchToolbarDescendants(nodes, position, node, context); - continue; - } - if (!isSearchToolbar(node)) continue; - - const innerSearch = findDescendant( - nodes, - position, - (candidate) => - normalizeType(candidate.type ?? '') === 'searchfield' && candidate.label === 'Search', - ); - if (!innerSearch) { - continue; - } - - context.suppressNode(node, [innerSearch]); - suppressToolbarAncestors(node, innerSearch, context); - suppressSearchToolbarDescendants(nodes, position, innerSearch, context); - } -} - -function isExposedSearchField(node: RawSnapshotNode): boolean { - return normalizeType(node.type ?? '') === 'searchfield' && node.label === 'Search'; -} - -function isSearchToolbar(node: RawSnapshotNode): boolean { - const type = normalizeType(node.type ?? ''); - return node.label === 'Toolbar' && (type === 'toolbar' || type === 'searchfield'); -} - -function suppressSearchToolbarDescendants( - nodes: RawSnapshotNode[], - position: number, - keptSearch: RawSnapshotNode, - context: SnapshotTreeRuleContext, -): void { - forEachDescendant(nodes, position, (descendant) => { - if (descendant.index === keptSearch.index) { - return; - } - if (shouldSuppressIosSearchToolbarDescendant(descendant)) { - context.suppressNode(descendant, [keptSearch]); - } - }); -} - -function suppressToolbarAncestors( - node: RawSnapshotNode, - representative: RawSnapshotNode, - context: SnapshotTreeRuleContext, -): void { - let current = node; - while (typeof current.parentIndex === 'number') { - const parent = context.sourceNodesByIndex.get(current.parentIndex); - if (!parent || parent.label !== 'Toolbar') { - return; - } - context.suppressNode(parent, [representative]); - current = parent; - } -} - -function shouldSuppressIosSearchToolbarDescendant(node: RawSnapshotNode): boolean { - const type = normalizeType(node.type ?? ''); - if (type === 'button') { - return false; - } - if (type === 'image') { - return true; - } - return node.label === 'Search'; -} - -function forEachOtherNodeWithLabel( - nodes: RawSnapshotNode[], - visitor: (node: RawSnapshotNode, label: string, position: number) => void, -): void { - for (let position = 0; position < nodes.length; position += 1) { - const node = nodes[position]; - const label = node?.label?.trim(); - if (node && label && normalizeType(node.type ?? '') === 'other') { - visitor(node, label, position); - } - } -} diff --git a/src/snapshot/snapshot-presentation/ios/presentation-visibility.test.ts b/src/snapshot/snapshot-presentation/ios/presentation-visibility.test.ts index dc963d6bb4..abcda11a20 100644 --- a/src/snapshot/snapshot-presentation/ios/presentation-visibility.test.ts +++ b/src/snapshot/snapshot-presentation/ios/presentation-visibility.test.ts @@ -1,7 +1,7 @@ import { expect, test } from 'vitest'; import { attachRefs, type RawSnapshotNode } from '@agent-device/kernel/snapshot'; import { buildSnapshotVisibility } from '../../snapshot-visibility.ts'; -import { presentIosInteractiveSnapshot } from './index.ts'; +import { presentIosInteractiveSnapshot } from '@agent-device/capture-kit/ios-snapshot-engine'; function buildSnapshotState(data: { nodes?: RawSnapshotNode[]; backend?: 'xctest' }) { return { diff --git a/src/snapshot/snapshot-presentation/ios/presentation.test.ts b/src/snapshot/snapshot-presentation/ios/presentation.test.ts index e8c2ae66b4..ac6adfe851 100644 --- a/src/snapshot/snapshot-presentation/ios/presentation.test.ts +++ b/src/snapshot/snapshot-presentation/ios/presentation.test.ts @@ -1,6 +1,6 @@ import { expect, test } from 'vitest'; import { attachRefs, type RawSnapshotNode } from '@agent-device/kernel/snapshot'; -import { presentIosInteractiveSnapshot } from './index.ts'; +import { presentIosInteractiveSnapshot } from '@agent-device/capture-kit/ios-snapshot-engine'; function buildSnapshotState(data: { nodes?: RawSnapshotNode[]; backend?: 'xctest' }) { return { diff --git a/src/snapshot/snapshot-presentation/ios/rows.test.ts b/src/snapshot/snapshot-presentation/ios/rows.test.ts index fdba04e41e..5700da810f 100644 --- a/src/snapshot/snapshot-presentation/ios/rows.test.ts +++ b/src/snapshot/snapshot-presentation/ios/rows.test.ts @@ -1,6 +1,6 @@ import { expect, test } from 'vitest'; import { elementClassicRoomListNodes, legitimatelyLabeledCellNodes } from './rows.fixtures.ts'; -import { presentIosInteractiveSnapshot } from './index.ts'; +import { presentIosInteractiveSnapshot } from '@agent-device/capture-kit/ios-snapshot-engine'; test('iOS row presentation associates generic room cells with their descendant titles', () => { const nodes = presentIosInteractiveSnapshot(elementClassicRoomListNodes); diff --git a/src/snapshot/snapshot-presentation/ios/web.test.ts b/src/snapshot/snapshot-presentation/ios/web.test.ts index 8798727812..f742a4044d 100644 --- a/src/snapshot/snapshot-presentation/ios/web.test.ts +++ b/src/snapshot/snapshot-presentation/ios/web.test.ts @@ -1,6 +1,6 @@ import { expect, test } from 'vitest'; import type { RawSnapshotNode } from '@agent-device/kernel/snapshot'; -import { presentIosInteractiveSnapshot } from './index.ts'; +import { presentIosInteractiveSnapshot } from '@agent-device/capture-kit/ios-snapshot-engine'; test('projects iOS WebKit heading and text wrappers to semantic roles', () => { const nodes: RawSnapshotNode[] = [ diff --git a/src/snapshot/snapshot-presentation/tree.test.ts b/src/snapshot/snapshot-presentation/tree.test.ts index 12076b44fd..ce388f57c1 100644 --- a/src/snapshot/snapshot-presentation/tree.test.ts +++ b/src/snapshot/snapshot-presentation/tree.test.ts @@ -1,6 +1,6 @@ import { expect, test } from 'vitest'; import type { RawSnapshotNode } from '@agent-device/kernel/snapshot'; -import { mergeReplacement, updateReplacement } from './tree.ts'; +import { mergeReplacement, updateReplacement } from '@agent-device/capture-kit/ios-snapshot-engine'; test('replacement updates derive patches from the composed node', () => { const node: RawSnapshotNode = { index: 1, type: 'Table', hiddenContentBelow: true };