From 70e8e6a37dde64a09e8ca0805d5fdf8f93b87870 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Micha=C5=82=20Pierzcha=C5=82a?= Date: Tue, 1 Sep 2026 11:19:49 +0200 Subject: [PATCH 1/5] test(ios): add snapshot engine conformance gates --- .fallowrc.json | 1 + .../main.swift | 156 ++- .../ios-snapshot-engine-conformance.json | 1099 +++++++++++++++++ packages/capture-kit/package.json | 1 + .../conformance-fixture.ts | 112 ++ .../conformance-generator.ts | 144 +++ .../conformance-harness.ts | 200 +++ .../ios-snapshot-engine/conformance.test.ts | 125 ++ .../ios-snapshot-engine/differential.test.ts | 98 ++ .../ios-snapshot-engine/properties.test.ts | 261 ++++ .../src/ios-snapshot-engine/replay.ts | 36 + .../runner-presentation.ts | 21 +- pnpm-lock.yaml | 3 + scripts/layering/check.ts | 6 + .../ios-snapshot-engine-policy.test.ts | 38 + .../layering/ios-snapshot-engine-policy.ts | 229 ++++ 16 files changed, 2481 insertions(+), 49 deletions(-) create mode 100644 contracts/fixtures/ios-snapshot-engine-conformance.json create mode 100644 packages/capture-kit/src/ios-snapshot-engine/conformance-fixture.ts create mode 100644 packages/capture-kit/src/ios-snapshot-engine/conformance-generator.ts create mode 100644 packages/capture-kit/src/ios-snapshot-engine/conformance-harness.ts create mode 100644 packages/capture-kit/src/ios-snapshot-engine/conformance.test.ts create mode 100644 packages/capture-kit/src/ios-snapshot-engine/differential.test.ts create mode 100644 packages/capture-kit/src/ios-snapshot-engine/properties.test.ts create mode 100644 packages/capture-kit/src/ios-snapshot-engine/replay.ts create mode 100644 scripts/layering/ios-snapshot-engine-policy.test.ts create mode 100644 scripts/layering/ios-snapshot-engine-policy.ts diff --git a/.fallowrc.json b/.fallowrc.json index 6e99a9b6f..cb747e769 100644 --- a/.fallowrc.json +++ b/.fallowrc.json @@ -20,6 +20,7 @@ "src/utils/png-worker.ts", "scripts/patch-xcuitest-runner-icon.ts", "scripts/runner-request-count/run.ts", + "packages/capture-kit/src/ios-snapshot-engine/replay.ts", // #1596 regression fixture: runs as a real `node --experimental-strip-types` // subprocess (test/integration/daemon-replace-exit-flush.test.ts), so // dependency analysis cannot follow the runCmdSync string path to it. diff --git a/apple/snapshot-presentation/Sources/SnapshotPresentationConformance/main.swift b/apple/snapshot-presentation/Sources/SnapshotPresentationConformance/main.swift index eea6f7e62..89bd98c4a 100644 --- a/apple/snapshot-presentation/Sources/SnapshotPresentationConformance/main.swift +++ b/apple/snapshot-presentation/Sources/SnapshotPresentationConformance/main.swift @@ -1,8 +1,8 @@ import AgentDeviceSnapshotPresentation -import Foundation import CoreGraphics +import Foundation -private struct Input: Decodable { +private struct ConformanceInput: Decodable { struct Node: Decodable { let index: Int let type: String @@ -20,60 +20,132 @@ private struct Input: Decodable { let hiddenContentBelow: Bool? } + let name: String let projection: String let interactiveOnly: Bool let depth: Int? let scope: String? + let foldPolicy: String let viewport: SnapshotRect let nodes: [Node] } -private struct Output: Encodable { +private struct BatchInput: Decodable { + let cases: [ConformanceInput] +} + +private struct ConformanceError: Encodable { + let code: String + let reason: String + let message: String +} + +private struct ConformanceOutput: Encodable { + let name: String + let outcome: String let nodes: [PresentedNode] + let error: ConformanceError? +} + +private struct BatchOutput: Encodable { + let cases: [ConformanceOutput] +} + +private func acquisition(for input: ConformanceInput) -> SnapshotAcquisition { + let inputOptions = options(for: input) + return SnapshotAcquisition( + hint: SnapshotPresentation.captureHint(for: inputOptions), + nodes: input.nodes.map { node in + RawAXNode( + index: node.index, + type: node.type, + label: node.label, + identifier: node.identifier, + value: node.value, + rect: node.rect, + enabled: node.enabled, + focused: node.focused, + selected: node.selected, + hittable: node.hittable, + depth: node.depth, + parentIndex: node.parentIndex, + hiddenContentAbove: node.hiddenContentAbove, + hiddenContentBelow: node.hiddenContentBelow + ) + }, + truncated: false, + effectiveDepth: nil, + viewport: input.viewport.cgRect + ) +} + +private func options(for input: ConformanceInput) -> PresentationOptions { + PresentationOptions( + interactiveOnly: input.interactiveOnly, + depth: input.depth, + scope: input.scope, + raw: input.projection == CaptureHint.Projection.raw.rawValue + ) +} + +private func present(_ input: ConformanceInput) -> ConformanceOutput { + do { + let inputAcquisition = acquisition(for: input) + let inputOptions = options(for: input) + let nodes: [PresentedNode] + if input.projection == CaptureHint.Projection.raw.rawValue { + nodes = SnapshotPresentation.presentRaw(inputAcquisition, options: inputOptions).nodes + } else { + let policy: SnapshotVisibilityFold.Policy = input.foldPolicy == "plain-viewport" + ? .plainViewport + : .cursorProjected + nodes = try SnapshotPresentation.presentRegular( + inputAcquisition, + options: inputOptions, + policy: policy + ).nodes + } + return ConformanceOutput(name: input.name, outcome: "success", nodes: nodes, error: nil) + } catch let failure as SnapshotPresentationFailure { + return ConformanceOutput( + name: input.name, + outcome: "failure", + nodes: [], + error: ConformanceError( + code: failure.code, + reason: reason(for: failure), + message: failure.message + ) + ) + } catch { + return ConformanceOutput( + name: input.name, + outcome: "failure", + nodes: [], + error: ConformanceError( + code: "IOS_SNAPSHOT_PRESENTATION_FAILED", + reason: "unexpected", + message: String(describing: error) + ) + ) + } +} + +private func reason(for failure: SnapshotPresentationFailure) -> String { + switch failure { + case .regularNodeOutsideCumulativeClip: + return "regular-node-outside-cumulative-clip" + case .regularDegenerateNodeIsActionable: + return "regular-degenerate-actionable-node" + } } private let input = try JSONDecoder().decode( - Input.self, + BatchInput.self, from: FileHandle.standardInput.readDataToEndOfFile() ) -private let options = PresentationOptions( - interactiveOnly: input.interactiveOnly, - depth: input.depth, - scope: input.scope, - raw: input.projection == CaptureHint.Projection.raw.rawValue -) -private let acquisition = SnapshotAcquisition( - hint: SnapshotPresentation.captureHint(for: options), - nodes: input.nodes.map { node in - RawAXNode( - index: node.index, - type: node.type, - label: node.label, - identifier: node.identifier, - value: node.value, - rect: node.rect, - enabled: node.enabled, - focused: node.focused, - selected: node.selected, - hittable: node.hittable, - depth: node.depth, - parentIndex: node.parentIndex, - hiddenContentAbove: node.hiddenContentAbove, - hiddenContentBelow: node.hiddenContentBelow - ) - }, - truncated: false, - effectiveDepth: nil, - viewport: input.viewport.cgRect +private let output = try JSONEncoder().encode( + BatchOutput(cases: input.cases.map(present)) ) -private let result = try SnapshotPresentation.present(acquisition, options: options) - ?? SnapshotPresentationResult( - nodes: [], - truncated: false, - effectiveDepth: nil, - customActions: nil, - qualityNodes: nil - ) -private let output = try JSONEncoder().encode(Output(nodes: result.nodes)) FileHandle.standardOutput.write(output) FileHandle.standardOutput.write(Data([0x0a])) diff --git a/contracts/fixtures/ios-snapshot-engine-conformance.json b/contracts/fixtures/ios-snapshot-engine-conformance.json new file mode 100644 index 000000000..020c0d30c --- /dev/null +++ b/contracts/fixtures/ios-snapshot-engine-conformance.json @@ -0,0 +1,1099 @@ +{ + "version": 1, + "viewport": { "x": 0, "y": 0, "width": 320, "height": 240 }, + "cases": [ + { + "name": "nested ancestor clips and actionability", + "swift": true, + "projection": "regular", + "interactiveOnly": false, + "depth": null, + "scope": null, + "foldPolicy": "cursor-projected", + "truncated": false, + "residue": [], + "nodes": [ + { + "index": 0, + "type": "Application", + "label": "App", + "rect": { "x": 0, "y": 0, "width": 320, "height": 240 }, + "enabled": true, + "hittable": false, + "depth": 0 + }, + { + "index": 1, + "type": "ScrollView", + "label": "Outer", + "rect": { "x": 16, "y": 20, "width": 180, "height": 180 }, + "enabled": true, + "hittable": false, + "depth": 1, + "parentIndex": 0 + }, + { + "index": 2, + "type": "ScrollView", + "label": "Inner", + "rect": { "x": 80, "y": 60, "width": 100, "height": 80 }, + "enabled": true, + "hittable": false, + "depth": 2, + "parentIndex": 1 + }, + { + "index": 3, + "type": "Button", + "label": "Partially visible", + "rect": { "x": 150, "y": 110, "width": 100, "height": 60 }, + "enabled": true, + "hittable": true, + "depth": 3, + "parentIndex": 2 + }, + { + "index": 4, + "type": "Button", + "label": "Escaped child", + "rect": { "x": 190, "y": 110, "width": 20, "height": 20 }, + "enabled": true, + "hittable": true, + "depth": 3, + "parentIndex": 2 + } + ], + "expected": { + "outcome": "success", + "truncated": false, + "residue": [], + "nodes": [ + { + "index": 0, + "type": "Application", + "label": "App", + "rect": { "x": 0, "y": 0, "width": 320, "height": 240 }, + "depth": 0, + "parentIndex": null, + "hittable": false, + "hiddenContentAbove": false, + "hiddenContentBelow": false + }, + { + "index": 1, + "type": "ScrollView", + "label": "Outer", + "rect": { "x": 16, "y": 20, "width": 180, "height": 180 }, + "depth": 1, + "parentIndex": 0, + "hittable": false, + "hiddenContentAbove": false, + "hiddenContentBelow": false + }, + { + "index": 2, + "type": "ScrollView", + "label": "Inner", + "rect": { "x": 80, "y": 60, "width": 100, "height": 80 }, + "depth": 2, + "parentIndex": 1, + "hittable": false, + "hiddenContentAbove": false, + "hiddenContentBelow": false + }, + { + "index": 3, + "type": "Button", + "label": "Partially visible", + "rect": { "x": 150, "y": 110, "width": 30, "height": 30 }, + "depth": 3, + "parentIndex": 2, + "hittable": true, + "hiddenContentAbove": false, + "hiddenContentBelow": false + } + ] + } + }, + { + "name": "viewport edge remains positively actionable", + "swift": true, + "projection": "regular", + "interactiveOnly": false, + "depth": null, + "scope": null, + "foldPolicy": "cursor-projected", + "truncated": false, + "residue": [], + "nodes": [ + { + "index": 0, + "type": "Application", + "label": "App", + "rect": { "x": 0, "y": 0, "width": 320, "height": 240 }, + "enabled": true, + "hittable": false, + "depth": 0 + }, + { + "index": 1, + "type": "Button", + "label": "Edge", + "rect": { "x": 300, "y": 100, "width": 40, "height": 40 }, + "enabled": true, + "hittable": true, + "depth": 1, + "parentIndex": 0 + } + ], + "expected": { + "outcome": "success", + "truncated": false, + "residue": [], + "nodes": [ + { + "index": 0, + "type": "Application", + "label": "App", + "rect": { "x": 0, "y": 0, "width": 320, "height": 240 }, + "depth": 0, + "parentIndex": null, + "hittable": false, + "hiddenContentAbove": false, + "hiddenContentBelow": false + }, + { + "index": 1, + "type": "Button", + "label": "Edge", + "rect": { "x": 300, "y": 100, "width": 20, "height": 40 }, + "depth": 1, + "parentIndex": 0, + "hittable": true, + "hiddenContentAbove": false, + "hiddenContentBelow": false + } + ] + } + }, + { + "name": "geometryless cursor nodes keep independent descendants", + "swift": false, + "projection": "regular", + "interactiveOnly": false, + "depth": null, + "scope": null, + "foldPolicy": "cursor-projected", + "truncated": false, + "residue": [], + "nodes": [ + { + "index": 0, + "type": "Application", + "label": "App", + "rect": { "x": 0, "y": 0, "width": 320, "height": 240 }, + "enabled": true, + "hittable": false, + "depth": 0 + }, + { + "index": 1, + "type": "Other", + "label": "No frame", + "enabled": true, + "hittable": false, + "depth": 1, + "parentIndex": 0 + }, + { + "index": 2, + "type": "Button", + "label": "Child", + "rect": { "x": 20, "y": 20, "width": 40, "height": 40 }, + "enabled": true, + "hittable": true, + "depth": 2, + "parentIndex": 1 + } + ], + "expected": { + "outcome": "success", + "truncated": false, + "residue": [], + "nodes": [ + { + "index": 0, + "type": "Application", + "label": "App", + "rect": { "x": 0, "y": 0, "width": 320, "height": 240 }, + "depth": 0, + "parentIndex": null, + "hittable": false, + "hiddenContentAbove": false, + "hiddenContentBelow": false + }, + { + "index": 1, + "type": "Other", + "label": "No frame", + "rect": null, + "depth": 1, + "parentIndex": 0, + "hittable": false, + "hiddenContentAbove": false, + "hiddenContentBelow": false + }, + { + "index": 2, + "type": "Button", + "label": "Child", + "rect": { "x": 20, "y": 20, "width": 40, "height": 40 }, + "depth": 2, + "parentIndex": 1, + "hittable": true, + "hiddenContentAbove": false, + "hiddenContentBelow": false + } + ] + } + }, + { + "name": "plain viewport keeps child visibility independent", + "swift": true, + "projection": "regular", + "interactiveOnly": false, + "depth": null, + "scope": null, + "foldPolicy": "plain-viewport", + "truncated": false, + "residue": [], + "nodes": [ + { + "index": 0, + "type": "Application", + "label": "App", + "rect": { "x": 0, "y": 0, "width": 320, "height": 240 }, + "enabled": true, + "hittable": false, + "depth": 0 + }, + { + "index": 1, + "type": "ScrollView", + "label": "Scroll", + "rect": { "x": 0, "y": 100, "width": 320, "height": 40 }, + "enabled": true, + "hittable": false, + "depth": 1, + "parentIndex": 0 + }, + { + "index": 2, + "type": "StaticText", + "label": "Outside scroll clip", + "rect": { "x": 10, "y": 200, "width": 80, "height": 20 }, + "enabled": true, + "hittable": false, + "depth": 2, + "parentIndex": 1 + } + ], + "expected": { + "outcome": "success", + "truncated": false, + "residue": [], + "nodes": [ + { + "index": 0, + "type": "Application", + "label": "App", + "rect": { "x": 0, "y": 0, "width": 320, "height": 240 }, + "depth": 0, + "parentIndex": null, + "hittable": false, + "hiddenContentAbove": false, + "hiddenContentBelow": false + }, + { + "index": 1, + "type": "ScrollView", + "label": "Scroll", + "rect": { "x": 0, "y": 100, "width": 320, "height": 40 }, + "depth": 1, + "parentIndex": 0, + "hittable": false, + "hiddenContentAbove": false, + "hiddenContentBelow": false + }, + { + "index": 2, + "type": "StaticText", + "label": "Outside scroll clip", + "rect": { "x": 10, "y": 200, "width": 80, "height": 20 }, + "depth": 2, + "parentIndex": 1, + "hittable": false, + "hiddenContentAbove": false, + "hiddenContentBelow": false + } + ] + } + }, + { + "name": "raw projection preserves reported geometry", + "swift": true, + "projection": "raw", + "interactiveOnly": true, + "depth": null, + "scope": null, + "foldPolicy": "cursor-projected", + "truncated": false, + "residue": [], + "nodes": [ + { + "index": 0, + "type": "Application", + "label": "App", + "rect": { "x": 0, "y": 0, "width": 320, "height": 240 }, + "enabled": true, + "hittable": false, + "depth": 0 + }, + { + "index": 1, + "type": "ScrollView", + "label": "Outer", + "rect": { "x": 16, "y": 20, "width": 180, "height": 180 }, + "enabled": true, + "hittable": false, + "depth": 1, + "parentIndex": 0 + }, + { + "index": 2, + "type": "Button", + "label": "Escaped child", + "rect": { "x": 210, "y": 80, "width": 100, "height": 40 }, + "enabled": true, + "hittable": true, + "depth": 2, + "parentIndex": 1 + } + ], + "expected": { + "outcome": "success", + "truncated": false, + "residue": [], + "nodes": [ + { + "index": 0, + "type": "Application", + "label": "App", + "rect": { "x": 0, "y": 0, "width": 320, "height": 240 }, + "depth": 0, + "parentIndex": null, + "hittable": false, + "hiddenContentAbove": false, + "hiddenContentBelow": false + }, + { + "index": 1, + "type": "ScrollView", + "label": "Outer", + "rect": { "x": 16, "y": 20, "width": 180, "height": 180 }, + "depth": 1, + "parentIndex": 0, + "hittable": false, + "hiddenContentAbove": false, + "hiddenContentBelow": false + }, + { + "index": 2, + "type": "Button", + "label": "Escaped child", + "rect": { "x": 210, "y": 80, "width": 100, "height": 40 }, + "depth": 2, + "parentIndex": 1, + "hittable": true, + "hiddenContentAbove": false, + "hiddenContentBelow": false + } + ] + } + }, + { + "name": "scope reroots wrappers and regular depth", + "swift": true, + "projection": "regular", + "interactiveOnly": false, + "depth": 1, + "scope": "Target", + "foldPolicy": "cursor-projected", + "truncated": false, + "residue": [], + "qualityLabels": ["App", "Wrapper", "Target", "Save"], + "nodes": [ + { + "index": 0, + "type": "Application", + "label": "App", + "rect": { "x": 0, "y": 0, "width": 320, "height": 240 }, + "enabled": true, + "hittable": false, + "depth": 0 + }, + { + "index": 1, + "type": "Other", + "label": "Wrapper", + "rect": { "x": 10, "y": 10, "width": 200, "height": 100 }, + "enabled": true, + "hittable": false, + "depth": 1, + "parentIndex": 0 + }, + { + "index": 2, + "type": "Other", + "label": "Target", + "rect": { "x": 10, "y": 10, "width": 200, "height": 100 }, + "enabled": true, + "hittable": false, + "depth": 2, + "parentIndex": 1 + }, + { + "index": 3, + "type": "Button", + "label": "Save", + "rect": { "x": 20, "y": 20, "width": 80, "height": 40 }, + "enabled": true, + "hittable": true, + "depth": 3, + "parentIndex": 2 + } + ], + "expected": { + "outcome": "success", + "truncated": false, + "residue": [], + "nodes": [ + { + "index": 0, + "type": "Other", + "label": "Target", + "rect": { "x": 10, "y": 10, "width": 200, "height": 100 }, + "depth": 0, + "parentIndex": null, + "hittable": false, + "hiddenContentAbove": false, + "hiddenContentBelow": false + }, + { + "index": 1, + "type": "Button", + "label": "Save", + "rect": { "x": 20, "y": 20, "width": 80, "height": 40 }, + "depth": 1, + "parentIndex": 0, + "hittable": true, + "hiddenContentAbove": false, + "hiddenContentBelow": false + } + ] + } + }, + { + "name": "scope depth zero retains only the matched root", + "swift": true, + "projection": "regular", + "interactiveOnly": false, + "depth": 0, + "scope": "Target", + "foldPolicy": "cursor-projected", + "truncated": false, + "residue": [], + "nodes": [ + { + "index": 0, + "type": "Application", + "label": "App", + "rect": { "x": 0, "y": 0, "width": 320, "height": 240 }, + "enabled": true, + "hittable": false, + "depth": 0 + }, + { + "index": 1, + "type": "Other", + "label": "Target", + "rect": { "x": 10, "y": 10, "width": 200, "height": 100 }, + "enabled": true, + "hittable": false, + "depth": 1, + "parentIndex": 0 + }, + { + "index": 2, + "type": "Button", + "label": "Save", + "rect": { "x": 20, "y": 20, "width": 80, "height": 40 }, + "enabled": true, + "hittable": true, + "depth": 2, + "parentIndex": 1 + } + ], + "expected": { + "outcome": "success", + "truncated": false, + "residue": [], + "nodes": [ + { + "index": 0, + "type": "Other", + "label": "Target", + "rect": { "x": 10, "y": 10, "width": 200, "height": 100 }, + "depth": 0, + "parentIndex": null, + "hittable": false, + "hiddenContentAbove": false, + "hiddenContentBelow": false + } + ] + } + }, + { + "name": "raw scope depth counts source depth", + "swift": true, + "projection": "raw", + "interactiveOnly": false, + "depth": 1, + "scope": "Target", + "foldPolicy": "cursor-projected", + "truncated": false, + "residue": [], + "nodes": [ + { + "index": 0, + "type": "Application", + "label": "App", + "rect": { "x": 0, "y": 0, "width": 320, "height": 240 }, + "enabled": true, + "hittable": false, + "depth": 0 + }, + { + "index": 1, + "type": "Other", + "label": "Target", + "rect": { "x": 10, "y": 10, "width": 200, "height": 100 }, + "enabled": true, + "hittable": false, + "depth": 1, + "parentIndex": 0 + }, + { + "index": 2, + "type": "Button", + "label": "Save", + "rect": { "x": 20, "y": 20, "width": 80, "height": 40 }, + "enabled": true, + "hittable": true, + "depth": 2, + "parentIndex": 1 + }, + { + "index": 3, + "type": "StaticText", + "label": "Too deep", + "rect": { "x": 20, "y": 70, "width": 80, "height": 20 }, + "enabled": true, + "hittable": false, + "depth": 3, + "parentIndex": 1 + } + ], + "expected": { + "outcome": "success", + "truncated": false, + "residue": [], + "nodes": [ + { + "index": 0, + "type": "Other", + "label": "Target", + "rect": { "x": 10, "y": 10, "width": 200, "height": 100 }, + "depth": 0, + "parentIndex": null, + "hittable": false, + "hiddenContentAbove": false, + "hiddenContentBelow": false + }, + { + "index": 1, + "type": "Button", + "label": "Save", + "rect": { "x": 20, "y": 20, "width": 80, "height": 40 }, + "depth": 1, + "parentIndex": 0, + "hittable": true, + "hiddenContentAbove": false, + "hiddenContentBelow": false + } + ] + } + }, + { + "name": "hidden scroll content becomes directional hints", + "swift": true, + "projection": "regular", + "interactiveOnly": false, + "depth": null, + "scope": null, + "foldPolicy": "cursor-projected", + "truncated": false, + "residue": [], + "nodes": [ + { + "index": 0, + "type": "Application", + "label": "App", + "rect": { "x": 0, "y": 0, "width": 320, "height": 240 }, + "enabled": true, + "hittable": false, + "depth": 0 + }, + { + "index": 1, + "type": "ScrollView", + "label": "Scroll", + "rect": { "x": 0, "y": 0, "width": 320, "height": 100 }, + "enabled": true, + "hittable": false, + "depth": 1, + "parentIndex": 0 + }, + { + "index": 2, + "type": "Button", + "label": "Above", + "rect": { "x": 10, "y": -40, "width": 40, "height": 20 }, + "enabled": true, + "hittable": true, + "depth": 2, + "parentIndex": 1 + }, + { + "index": 3, + "type": "Button", + "label": "Visible", + "rect": { "x": 10, "y": 10, "width": 40, "height": 20 }, + "enabled": true, + "hittable": true, + "depth": 2, + "parentIndex": 1 + }, + { + "index": 4, + "type": "Button", + "label": "Below", + "rect": { "x": 10, "y": 110, "width": 40, "height": 20 }, + "enabled": true, + "hittable": true, + "depth": 2, + "parentIndex": 1 + } + ], + "expected": { + "outcome": "success", + "truncated": false, + "residue": [], + "nodes": [ + { + "index": 0, + "type": "Application", + "label": "App", + "rect": { "x": 0, "y": 0, "width": 320, "height": 240 }, + "depth": 0, + "parentIndex": null, + "hittable": false, + "hiddenContentAbove": false, + "hiddenContentBelow": false + }, + { + "index": 1, + "type": "ScrollView", + "label": "Scroll", + "rect": { "x": 0, "y": 0, "width": 320, "height": 100 }, + "depth": 1, + "parentIndex": 0, + "hittable": false, + "hiddenContentAbove": true, + "hiddenContentBelow": true + }, + { + "index": 2, + "type": "Button", + "label": "Visible", + "rect": { "x": 10, "y": 10, "width": 40, "height": 20 }, + "depth": 2, + "parentIndex": 1, + "hittable": true, + "hiddenContentAbove": false, + "hiddenContentBelow": false + } + ] + } + }, + { + "name": "interactive only compacts semantic representatives", + "swift": false, + "projection": "regular", + "interactiveOnly": true, + "depth": null, + "scope": null, + "foldPolicy": "cursor-projected", + "truncated": false, + "residue": [], + "nodes": [ + { + "index": 0, + "type": "Application", + "label": "App", + "rect": { "x": 0, "y": 0, "width": 320, "height": 240 }, + "enabled": true, + "hittable": false, + "depth": 0 + }, + { + "index": 1, + "type": "Table", + "label": "Settings", + "rect": { "x": 0, "y": 40, "width": 320, "height": 200 }, + "enabled": true, + "hittable": false, + "depth": 1, + "parentIndex": 0 + }, + { + "index": 2, + "type": "Cell", + "label": "General", + "rect": { "x": 16, "y": 80, "width": 288, "height": 52 }, + "enabled": true, + "hittable": false, + "depth": 2, + "parentIndex": 1 + }, + { + "index": 3, + "type": "Button", + "label": "General", + "rect": { "x": 16, "y": 80, "width": 288, "height": 52 }, + "enabled": true, + "hittable": true, + "depth": 3, + "parentIndex": 2 + }, + { + "index": 4, + "type": "StaticText", + "label": "General", + "rect": { "x": 16, "y": 80, "width": 288, "height": 52 }, + "enabled": true, + "hittable": false, + "depth": 4, + "parentIndex": 3 + } + ], + "expected": { + "outcome": "success", + "truncated": false, + "residue": [], + "nodes": [ + { + "index": 0, + "type": "Application", + "label": "App", + "rect": { "x": 0, "y": 0, "width": 320, "height": 240 }, + "depth": 0, + "parentIndex": null, + "hittable": false, + "hiddenContentAbove": false, + "hiddenContentBelow": false + }, + { + "index": 1, + "type": "Table", + "label": "Settings", + "rect": { "x": 0, "y": 40, "width": 320, "height": 200 }, + "depth": 1, + "parentIndex": 0, + "hittable": false, + "hiddenContentAbove": false, + "hiddenContentBelow": false + }, + { + "index": 2, + "type": "Cell", + "label": "General", + "rect": { "x": 16, "y": 80, "width": 288, "height": 52 }, + "depth": 2, + "parentIndex": 1, + "hittable": false, + "hiddenContentAbove": false, + "hiddenContentBelow": false + } + ] + } + }, + { + "name": "unavailable hittability fails closed", + "swift": false, + "projection": "regular", + "interactiveOnly": false, + "depth": null, + "scope": null, + "foldPolicy": "cursor-projected", + "truncated": false, + "residue": [{ "kind": "unavailable-fact", "fact": "hittability" }], + "nodes": [ + { + "index": 0, + "type": "Application", + "label": "App", + "rect": { "x": 0, "y": 0, "width": 320, "height": 240 }, + "enabled": true, + "hittable": false, + "depth": 0 + }, + { + "index": 1, + "type": "Button", + "label": "Maybe actionable", + "rect": { "x": 20, "y": 20, "width": 80, "height": 40 }, + "enabled": true, + "hittable": true, + "depth": 1, + "parentIndex": 0 + } + ], + "expected": { + "outcome": "success", + "truncated": false, + "residue": [{ "kind": "unavailable-fact", "fact": "hittability" }], + "nodes": [ + { + "index": 0, + "type": "Application", + "label": "App", + "rect": { "x": 0, "y": 0, "width": 320, "height": 240 }, + "depth": 0, + "parentIndex": null, + "hittable": false, + "hiddenContentAbove": false, + "hiddenContentBelow": false + }, + { + "index": 1, + "type": "Button", + "label": "Maybe actionable", + "rect": { "x": 20, "y": 20, "width": 80, "height": 40 }, + "depth": 1, + "parentIndex": 0, + "hittable": false, + "hiddenContentAbove": false, + "hiddenContentBelow": false + } + ] + } + }, + { + "name": "malformed parent is a typed failure", + "swift": false, + "projection": "regular", + "interactiveOnly": false, + "depth": null, + "scope": null, + "foldPolicy": "cursor-projected", + "truncated": false, + "residue": [], + "nodes": [ + { + "index": 0, + "type": "Application", + "label": "App", + "rect": { "x": 0, "y": 0, "width": 320, "height": 240 }, + "enabled": true, + "hittable": false, + "depth": 0 + }, + { + "index": 1, + "type": "Button", + "label": "Broken", + "rect": { "x": 20, "y": 20, "width": 80, "height": 40 }, + "enabled": true, + "hittable": true, + "depth": 1, + "parentIndex": 99 + } + ], + "expected": { + "outcome": "failure", + "error": { + "code": "IOS_SNAPSHOT_ENGINE_FAILED", + "reason": "malformed-graph" + }, + "nodes": [] + } + }, + { + "name": "missing viewport is a typed failure", + "swift": false, + "projection": "regular", + "interactiveOnly": false, + "depth": null, + "scope": null, + "foldPolicy": "cursor-projected", + "viewportEvidence": { "kind": "missing", "reason": "not-provided" }, + "truncated": false, + "residue": [{ "kind": "missing-viewport", "reason": "not-provided" }], + "nodes": [ + { + "index": 0, + "type": "Application", + "label": "App", + "rect": { "x": 0, "y": 0, "width": 320, "height": 240 }, + "enabled": true, + "hittable": false, + "depth": 0 + } + ], + "expected": { + "outcome": "failure", + "error": { + "code": "IOS_SNAPSHOT_ENGINE_FAILED", + "reason": "missing-viewport" + }, + "nodes": [] + } + }, + { + "name": "invalid viewport is a typed failure", + "swift": false, + "projection": "regular", + "interactiveOnly": false, + "depth": null, + "scope": null, + "foldPolicy": "cursor-projected", + "viewportEvidence": { + "kind": "reported", + "rect": { "x": 0, "y": 0, "width": 0, "height": 240 } + }, + "truncated": false, + "residue": [], + "nodes": [ + { + "index": 0, + "type": "Application", + "label": "App", + "rect": { "x": 0, "y": 0, "width": 320, "height": 240 }, + "enabled": true, + "hittable": false, + "depth": 0 + } + ], + "expected": { + "outcome": "failure", + "error": { + "code": "IOS_SNAPSHOT_ENGINE_FAILED", + "reason": "invalid-viewport" + }, + "nodes": [] + } + }, + { + "name": "residue and truncation survive publication", + "swift": true, + "projection": "regular", + "interactiveOnly": false, + "depth": null, + "scope": null, + "foldPolicy": "cursor-projected", + "truncated": true, + "residue": [ + { + "kind": "provider-pruned", + "fields": ["scope", "nodes"] + }, + { + "kind": "truncated", + "dimension": "payload", + "limit": 2000 + } + ], + "nodes": [ + { + "index": 0, + "type": "Application", + "label": "App", + "rect": { "x": 0, "y": 0, "width": 320, "height": 240 }, + "enabled": true, + "hittable": false, + "depth": 0 + }, + { + "index": 1, + "type": "Button", + "label": "Continue", + "rect": { "x": 20, "y": 20, "width": 100, "height": 44 }, + "enabled": true, + "hittable": true, + "depth": 1, + "parentIndex": 0 + } + ], + "expected": { + "outcome": "success", + "truncated": true, + "residue": [ + { + "kind": "provider-pruned", + "fields": ["scope", "nodes"] + }, + { + "kind": "truncated", + "dimension": "payload", + "limit": 2000 + } + ], + "nodes": [ + { + "index": 0, + "type": "Application", + "label": "App", + "rect": { "x": 0, "y": 0, "width": 320, "height": 240 }, + "depth": 0, + "parentIndex": null, + "hittable": false, + "hiddenContentAbove": false, + "hiddenContentBelow": false + }, + { + "index": 1, + "type": "Button", + "label": "Continue", + "rect": { "x": 20, "y": 20, "width": 100, "height": 44 }, + "depth": 1, + "parentIndex": 0, + "hittable": true, + "hiddenContentAbove": false, + "hiddenContentBelow": false + } + ] + } + } + ] +} diff --git a/packages/capture-kit/package.json b/packages/capture-kit/package.json index 39049c52c..830b4c9b1 100644 --- a/packages/capture-kit/package.json +++ b/packages/capture-kit/package.json @@ -73,6 +73,7 @@ }, "devDependencies": { "@types/pngjs": "^6.0.5", + "fast-check": "^4.9.0", "pngjs": "^7.0.0" } } diff --git a/packages/capture-kit/src/ios-snapshot-engine/conformance-fixture.ts b/packages/capture-kit/src/ios-snapshot-engine/conformance-fixture.ts new file mode 100644 index 000000000..79dd2efbb --- /dev/null +++ b/packages/capture-kit/src/ios-snapshot-engine/conformance-fixture.ts @@ -0,0 +1,112 @@ +import fs from 'node:fs'; +import path from 'node:path'; +import type { + IosAcquisitionResidue, + IosSnapshotAcquisition, + IosSnapshotRequest, + IosViewportEvidence, +} from '@agent-device/contracts/ios-snapshot'; +import { createIosSnapshotRequest, deriveIosCaptureHint } from '../ios-snapshot-planning.ts'; +import type { RawSnapshotNode, Rect } from '@agent-device/kernel/snapshot'; + +type GoldenProjectionNode = Readonly<{ + index: number; + type: string | null; + label: string | null; + rect: Rect | null; + depth: number | null; + parentIndex: number | null; + hittable: boolean; + hiddenContentAbove: boolean; + hiddenContentBelow: boolean; +}>; + +type GoldenExpected = Readonly<{ + outcome: 'success' | 'failure'; + nodes: readonly GoldenProjectionNode[]; + truncated?: boolean; + residue?: readonly IosAcquisitionResidue[]; + error?: Readonly<{ code: string; reason: string }>; +}>; + +type GoldenCase = Readonly<{ + name: string; + swift: boolean; + projection: 'regular' | 'raw'; + interactiveOnly: boolean; + depth: number | null; + scope: string | null; + foldPolicy: 'cursor-projected' | 'plain-viewport'; + truncated: boolean; + residue: readonly IosAcquisitionResidue[]; + qualityLabels?: readonly (string | null)[]; + nodes: readonly RawSnapshotNode[]; + viewportEvidence?: IosViewportEvidence; + expected: GoldenExpected; +}>; + +type GoldenFixture = Readonly<{ + version: number; + viewport: Rect; + cases: readonly GoldenCase[]; +}>; + +const IOS_SNAPSHOT_ENGINE_FIXTURE_PATH = path.resolve( + import.meta.dirname, + '..', + '..', + '..', + '..', + 'contracts', + 'fixtures', + 'ios-snapshot-engine-conformance.json', +); + +export function readIosSnapshotEngineFixture(): GoldenFixture { + return JSON.parse(fs.readFileSync(IOS_SNAPSHOT_ENGINE_FIXTURE_PATH, 'utf8')) as GoldenFixture; +} + +export function requestForGoldenCase(testCase: GoldenCase): IosSnapshotRequest { + return createIosSnapshotRequest({ + projection: testCase.projection, + interactiveOnly: testCase.interactiveOnly, + depth: testCase.depth, + scope: testCase.scope, + acquisitionIntent: 'full', + }); +} + +export function acquisitionForGoldenCase( + fixture: GoldenFixture, + testCase: GoldenCase, +): IosSnapshotAcquisition { + const request = requestForGoldenCase(testCase); + return { + producer: 'simulator-ax-bridge', + intent: 'full', + hint: { ...deriveIosCaptureHint(request), acquisitionIntent: 'full' }, + nodes: testCase.nodes, + truncated: testCase.truncated, + viewport: testCase.viewportEvidence ?? { kind: 'reported', rect: fixture.viewport }, + lineage: { targetId: 'golden-target', generation: 'golden-generation' }, + residue: testCase.residue, + }; +} + +function normalizeGoldenNode(node: RawSnapshotNode): GoldenProjectionNode { + return { + index: node.index, + type: node.type ?? null, + label: node.label ?? null, + rect: node.rect ?? null, + depth: node.depth ?? null, + parentIndex: node.parentIndex ?? null, + hittable: node.hittable === true, + hiddenContentAbove: node.hiddenContentAbove === true, + hiddenContentBelow: node.hiddenContentBelow === true, + }; +} + +export function normalizeGoldenNodes(nodes: readonly RawSnapshotNode[]): GoldenProjectionNode[] { + return nodes.map(normalizeGoldenNode); +} diff --git a/packages/capture-kit/src/ios-snapshot-engine/conformance-generator.ts b/packages/capture-kit/src/ios-snapshot-engine/conformance-generator.ts new file mode 100644 index 000000000..b2cb8b9d6 --- /dev/null +++ b/packages/capture-kit/src/ios-snapshot-engine/conformance-generator.ts @@ -0,0 +1,144 @@ +import fc from 'fast-check'; +import type { RawSnapshotNode, Rect } from '@agent-device/kernel/snapshot'; +import type { compareDifferentialCases } from './conformance-harness.ts'; + +type DifferentialCase = Parameters[0][number]; + +const VIEWPORT: Rect = { x: 0, y: 0, width: 320, height: 240 }; +const TYPES = [ + 'Other', + 'Window', + 'ScrollView', + 'CollectionView', + 'Table', + 'Cell', + 'Button', + 'StaticText', + 'TextField', +] as const; + +type NodeSeed = { + type: (typeof TYPES)[number]; + label: string | undefined; + x: number; + y: number; + width: number; + height: number; + parent: number; + enabled: boolean; + hittable: boolean; + hiddenContentAbove: boolean; + hiddenContentBelow: boolean; +}; + +const nodeSeedArbitrary = fc.record({ + type: fc.constantFrom(...TYPES), + label: fc.constantFrom(undefined, 'Target', 'Save', 'Decoration', 'Panel'), + x: fc.integer({ min: -80, max: 360 }), + y: fc.integer({ min: -80, max: 320 }), + width: fc.integer({ min: 0, max: 260 }), + height: fc.integer({ min: 0, max: 220 }), + parent: fc.integer({ min: -1, max: 10 }), + enabled: fc.boolean(), + hittable: fc.boolean(), + hiddenContentAbove: fc.boolean(), + hiddenContentBelow: fc.boolean(), +}); + +const caseShapeArbitrary = fc + .array(nodeSeedArbitrary, { minLength: 1, maxLength: 10 }) + .map((seeds) => makeCase(seeds)); + +export const differentialBatchArbitrary = fc + .array(caseShapeArbitrary, { minLength: 1, maxLength: 10 }) + .map((cases) => + cases.map((testCase, index) => ({ + ...testCase, + name: 'fuzz-case-' + String(index), + })), + ); + +function makeCase(seeds: ReadonlyArray): Omit { + const nodes: RawSnapshotNode[] = []; + for (const [index, seed] of seeds.entries()) nodes.push(makeNode(seed, index, nodes)); + + return { + projection: projectionFor(seeds[0]!), + interactiveOnly: false, + depth: depthFor(seeds[0]!), + scope: scopeFor(seeds[0]!), + foldPolicy: foldPolicyFor(seeds[0]!), + viewport: VIEWPORT, + nodes, + }; +} + +function makeNode( + seed: NodeSeed, + index: number, + nodes: readonly RawSnapshotNode[], +): RawSnapshotNode { + const parentIndex = parentIndexFor(seed, index); + return { + index, + type: typeFor(seed, index), + label: labelFor(seed, index), + rect: rectFor(seed, index), + enabled: enabledFor(seed, index), + hittable: hittableFor(seed, index), + depth: nodeDepth(parentIndex, nodes), + ...(parentIndex === undefined ? {} : { parentIndex }), + ...hiddenContentFor(seed), + }; +} + +function parentIndexFor(seed: NodeSeed, index: number): number | undefined { + return index === 0 || seed.parent < 0 ? undefined : Math.min(seed.parent, index - 1); +} + +function nodeDepth(parentIndex: number | undefined, nodes: readonly RawSnapshotNode[]): number { + return parentIndex === undefined ? 0 : (nodes[parentIndex]?.depth ?? 0) + 1; +} + +function typeFor(seed: NodeSeed, index: number): string { + return index === 0 ? 'Application' : seed.type; +} + +function labelFor(seed: NodeSeed, index: number): string | undefined { + return index === 0 ? 'App' : seed.label; +} + +function rectFor(seed: NodeSeed, index: number): Rect { + return index === 0 ? VIEWPORT : { x: seed.x, y: seed.y, width: seed.width, height: seed.height }; +} + +function enabledFor(seed: NodeSeed, index: number): boolean { + return index === 0 || seed.enabled; +} + +function hittableFor(seed: NodeSeed, index: number): boolean { + return index !== 0 && seed.hittable; +} + +function hiddenContentFor(seed: NodeSeed): Partial { + return { + ...(seed.hiddenContentAbove ? { hiddenContentAbove: true } : {}), + ...(seed.hiddenContentBelow ? { hiddenContentBelow: true } : {}), + }; +} + +function projectionFor(seed: NodeSeed): 'regular' | 'raw' { + return seed.type === 'Other' ? 'raw' : 'regular'; +} + +function depthFor(seed: NodeSeed): number | null { + return seed.width % 5 === 0 ? null : seed.width % 4; +} + +function scopeFor(seed: NodeSeed): string | null { + return seed.height % 5 === 0 ? 'target' : null; +} + +function foldPolicyFor(seed: NodeSeed): 'cursor-projected' | 'plain-viewport' { + return seed.x % 2 === 0 ? 'cursor-projected' : 'plain-viewport'; +} diff --git a/packages/capture-kit/src/ios-snapshot-engine/conformance-harness.ts b/packages/capture-kit/src/ios-snapshot-engine/conformance-harness.ts new file mode 100644 index 000000000..2fb3ab05f --- /dev/null +++ b/packages/capture-kit/src/ios-snapshot-engine/conformance-harness.ts @@ -0,0 +1,200 @@ +import { execFileSync } from 'node:child_process'; +import fs from 'node:fs'; +import os from 'node:os'; +import path from 'node:path'; +import type { IosSnapshotAcquisition } from '@agent-device/contracts/ios-snapshot'; +import { createIosSnapshotRequest, deriveIosCaptureHint } from '../ios-snapshot-planning.ts'; +import { IosSnapshotEngineError, presentIosSnapshot } from './index.ts'; +import type { RawSnapshotNode, Rect } from '@agent-device/kernel/snapshot'; + +const REPO_ROOT = path.resolve(import.meta.dirname, '..', '..', '..', '..'); +const SWIFT_PACKAGE_PATH = path.join(REPO_ROOT, 'apple', 'snapshot-presentation'); +const SWIFT_PRODUCT = 'snapshot-presentation-conformance'; +export const SWIFT_RUN_TIMEOUT_MS = 60_000; + +let swiftHarnessExecutable: string | undefined; + +type DifferentialCase = Readonly<{ + name: string; + projection: 'regular' | 'raw'; + interactiveOnly: false; + depth: number | null; + scope: string | null; + foldPolicy: 'cursor-projected' | 'plain-viewport'; + viewport: Rect; + nodes: readonly RawSnapshotNode[]; +}>; + +type DifferentialOutcome = Readonly<{ + name?: string; + outcome: 'success' | 'failure'; + nodes: readonly CanonicalNode[]; + error?: Readonly<{ code: string; reason: string }>; +}>; + +type CanonicalNode = Readonly<{ + index: number; + type: string | null; + label: string | null; + rect: Rect | null; + depth: number | null; + parentIndex: number | null; + hittable: boolean; + hiddenContentAbove: boolean; + hiddenContentBelow: boolean; +}>; + +type DifferentialMismatch = Readonly<{ + case: DifferentialCase; + swift: unknown; + typescript: unknown; +}>; + +export function swiftToolchainAvailable(): boolean { + try { + execFileSync('swift', ['--version'], { stdio: 'ignore' }); + return true; + } catch { + return false; + } +} + +export function compareDifferentialCases( + cases: readonly DifferentialCase[], +): DifferentialMismatch | undefined { + const swiftCases = runSwiftCases(cases); + for (const testCase of cases) { + const swift = swiftCases.find((entry) => entry.name === testCase.name); + const typescript = runTypeScriptCase(testCase); + const normalizedSwift = swift ? withoutName(swift) : undefined; + if (!normalizedSwift || JSON.stringify(normalizedSwift) !== JSON.stringify(typescript)) { + return { case: testCase, swift: normalizedSwift, typescript }; + } + } + return undefined; +} + +function runSwiftCases(cases: readonly DifferentialCase[]): DifferentialOutcome[] { + const stdout = execFileSync(swiftConformanceExecutable(), [], { + cwd: REPO_ROOT, + encoding: 'utf8', + input: JSON.stringify({ cases: cases.map(prepareSwiftAcquisition) }), + timeout: SWIFT_RUN_TIMEOUT_MS, + maxBuffer: 8 * 1024 * 1024, + }); + const parsed = JSON.parse(stdout) as { + cases: Array<{ + name: string; + outcome: 'success' | 'failure'; + nodes?: RawSnapshotNode[]; + error?: { code: string; reason: string }; + }>; + }; + return parsed.cases.map((entry) => ({ + name: entry.name, + outcome: entry.outcome, + nodes: canonicalNodes(entry.nodes ?? []), + ...(entry.error ? { error: entry.error } : {}), + })); +} + +function swiftConformanceExecutable(): string { + if (swiftHarnessExecutable) return swiftHarnessExecutable; + execFileSync('swift', ['build', '--package-path', SWIFT_PACKAGE_PATH], { + cwd: REPO_ROOT, + stdio: 'ignore', + timeout: SWIFT_RUN_TIMEOUT_MS, + }); + const binPath = execFileSync( + 'swift', + ['build', '--show-bin-path', '--package-path', SWIFT_PACKAGE_PATH], + { cwd: REPO_ROOT, encoding: 'utf8', timeout: SWIFT_RUN_TIMEOUT_MS }, + ).trim(); + swiftHarnessExecutable = path.join(binPath, SWIFT_PRODUCT); + return swiftHarnessExecutable; +} + +function prepareSwiftAcquisition(testCase: DifferentialCase): DifferentialCase { + if (testCase.projection !== 'raw' || testCase.scope !== null || testCase.depth === null) { + return testCase; + } + return { + ...testCase, + nodes: testCase.nodes.filter((node) => (node.depth ?? 0) <= testCase.depth!), + }; +} + +function withoutName(outcome: DifferentialOutcome): Omit { + const { name: _name, ...normalized } = outcome; + return normalized; +} + +export function runTypeScriptCase(testCase: DifferentialCase): DifferentialOutcome { + const request = createIosSnapshotRequest({ + projection: testCase.projection, + interactiveOnly: testCase.interactiveOnly, + depth: testCase.depth, + scope: testCase.scope, + }); + const acquisition: IosSnapshotAcquisition = { + producer: 'simulator-ax-bridge', + intent: 'full', + hint: { ...deriveIosCaptureHint(request), acquisitionIntent: 'full' }, + nodes: testCase.nodes, + truncated: false, + viewport: { kind: 'reported', rect: testCase.viewport }, + lineage: { targetId: 'differential-target', generation: 'differential-generation' }, + residue: [], + }; + try { + const result = presentIosSnapshot({ stage: 'acquired', acquisition }, request, { + foldPolicy: testCase.foldPolicy, + }); + return { outcome: 'success', nodes: canonicalNodes(result.nodes) }; + } catch (error) { + if (!(error instanceof IosSnapshotEngineError)) throw error; + return { + outcome: 'failure', + nodes: [], + error: { code: error.code, reason: error.reason }, + }; + } +} + +export function canonicalNodes(nodes: readonly RawSnapshotNode[]): CanonicalNode[] { + return nodes.map((node) => ({ + index: node.index, + type: node.type ?? null, + label: node.label ?? null, + rect: node.rect ? canonicalRect(node.rect) : null, + depth: node.depth ?? null, + parentIndex: node.parentIndex ?? null, + hittable: node.hittable === true, + hiddenContentAbove: node.hiddenContentAbove === true, + hiddenContentBelow: node.hiddenContentBelow === true, + })); +} + +function canonicalRect(rect: Rect): Rect { + return { x: rect.x, y: rect.y, width: rect.width, height: rect.height }; +} + +export function writeDifferentialFailureArtifact(input: { + testCase: DifferentialCase; + seed: number; + counterexamplePath: string; +}): { directory: string; casePath: string; replayCommand: string } { + const directory = fs.mkdtempSync(path.join(os.tmpdir(), 'ios-snapshot-fuzz-')); + const casePath = path.join(directory, 'case.json'); + fs.writeFileSync(casePath, JSON.stringify({ cases: [input.testCase] }, null, 2) + '\n'); + const replayCommand = [ + 'node --experimental-strip-types', + 'packages/capture-kit/src/ios-snapshot-engine/replay.ts', + JSON.stringify(casePath), + ].join(' '); + fs.writeFileSync( + path.join(directory, 'replay-command.txt'), + replayCommand + '\nseed=' + String(input.seed) + '\npath=' + input.counterexamplePath + '\n', + ); + return { directory, casePath, replayCommand }; +} diff --git a/packages/capture-kit/src/ios-snapshot-engine/conformance.test.ts b/packages/capture-kit/src/ios-snapshot-engine/conformance.test.ts new file mode 100644 index 000000000..998ddd26b --- /dev/null +++ b/packages/capture-kit/src/ios-snapshot-engine/conformance.test.ts @@ -0,0 +1,125 @@ +import fs from 'node:fs'; +import assert from 'node:assert/strict'; +import path from 'node:path'; +import { test } from 'vitest'; +import type { CaptureHint, IosSnapshotRequestInput } from '@agent-device/contracts/ios-snapshot'; +import { + createIosSnapshotRequest, + deriveIosCaptureHint, +} from '@agent-device/capture-kit/ios-snapshot-planning'; +import { IosSnapshotEngineError, presentIosSnapshot } from './index.ts'; +import { + acquisitionForGoldenCase, + normalizeGoldenNodes, + readIosSnapshotEngineFixture, + requestForGoldenCase, +} from './conformance-fixture.ts'; + +const CAPTURE_HINT_FIXTURE_PATH = path.resolve( + import.meta.dirname, + '..', + '..', + '..', + '..', + 'contracts', + 'fixtures', + 'ios-snapshot-capture-hint.json', +); + +type CaptureHintFixture = Readonly<{ + name: string; + request: IosSnapshotRequestInput; + expected: CaptureHint; +}>; + +test('the independent capture-hint corpus agrees with the engine request boundary', () => { + const fixtures = JSON.parse( + fs.readFileSync(CAPTURE_HINT_FIXTURE_PATH, 'utf8'), + ) as CaptureHintFixture[]; + assert.ok(fixtures.length > 0); + assert.equal(new Set(fixtures.map((fixture) => fixture.name)).size, fixtures.length); + for (const fixture of fixtures) { + const request = createIosSnapshotRequest(fixture.request); + assert.deepEqual(deriveIosCaptureHint(request), fixture.expected, fixture.name); + } +}); + +test('the authored iOS snapshot corpus covers each contract seam', () => { + const fixture = readIosSnapshotEngineFixture(); + assert.equal(fixture.version, 1); + assert.ok(fixture.cases.length >= 12); + assert.equal(new Set(fixture.cases.map((testCase) => testCase.name)).size, fixture.cases.length); + + const required = [ + 'nested ancestor clips and actionability', + 'viewport edge remains positively actionable', + 'geometryless cursor nodes keep independent descendants', + 'plain viewport keeps child visibility independent', + 'raw projection preserves reported geometry', + 'scope reroots wrappers and regular depth', + 'scope depth zero retains only the matched root', + 'raw scope depth counts source depth', + 'hidden scroll content becomes directional hints', + 'interactive only compacts semantic representatives', + 'unavailable hittability fails closed', + 'malformed parent is a typed failure', + 'missing viewport is a typed failure', + 'invalid viewport is a typed failure', + 'residue and truncation survive publication', + ]; + for (const name of required) { + assert.ok( + fixture.cases.some((testCase) => testCase.name === name), + name, + ); + } +}); + +test('the independent iOS snapshot goldens match the TypeScript engine', () => { + const fixture = readIosSnapshotEngineFixture(); + for (const testCase of fixture.cases) { + const request = requestForGoldenCase(testCase); + const acquisition = acquisitionForGoldenCase(fixture, testCase); + const expected = testCase.expected; + let actual: + | { + outcome: 'success'; + nodes: ReturnType; + truncated: boolean; + residue: typeof acquisition.residue; + qualityLabels?: readonly (string | null)[]; + } + | { + outcome: 'failure'; + nodes: []; + error: { code: string; reason: string }; + }; + + try { + const result = presentIosSnapshot({ stage: 'acquired', acquisition }, request, { + foldPolicy: testCase.foldPolicy, + }); + actual = { + outcome: 'success', + nodes: normalizeGoldenNodes(result.nodes), + truncated: acquisition.truncated, + residue: acquisition.residue, + ...(testCase.qualityLabels + ? { qualityLabels: result.qualityNodes?.map((node) => node.label ?? null) } + : {}), + }; + } catch (error) { + assert.ok(error instanceof IosSnapshotEngineError, testCase.name); + actual = { + outcome: 'failure', + nodes: [], + error: { code: error.code, reason: error.reason }, + }; + } + assert.deepEqual( + actual, + testCase.qualityLabels ? { ...expected, qualityLabels: testCase.qualityLabels } : expected, + testCase.name, + ); + } +}); diff --git a/packages/capture-kit/src/ios-snapshot-engine/differential.test.ts b/packages/capture-kit/src/ios-snapshot-engine/differential.test.ts new file mode 100644 index 000000000..599b577f8 --- /dev/null +++ b/packages/capture-kit/src/ios-snapshot-engine/differential.test.ts @@ -0,0 +1,98 @@ +import assert from 'node:assert/strict'; +import fc from 'fast-check'; +import { test } from 'vitest'; +import { + compareDifferentialCases, + swiftToolchainAvailable, + SWIFT_RUN_TIMEOUT_MS, + writeDifferentialFailureArtifact, +} from './conformance-harness.ts'; +import { differentialBatchArbitrary } from './conformance-generator.ts'; +import { readIosSnapshotEngineFixture } from './conformance-fixture.ts'; + +type DifferentialCase = Parameters[0][number]; + +const FUZZ_SEEDS = [219101, 219102, 219103, 219104]; +const RUNS_PER_SEED = 8; +const MAX_TOTAL_DURATION_MS = 60_000; + +const differentialTest = swiftToolchainAvailable() ? test : test.skip; + +differentialTest('authored Swift and TypeScript golden cases agree', () => { + const fixture = readIosSnapshotEngineFixture(); + const cases = fixture.cases + .filter((testCase) => testCase.swift && !testCase.interactiveOnly) + .map((testCase) => ({ + name: testCase.name, + projection: testCase.projection, + interactiveOnly: false as const, + depth: testCase.depth, + scope: testCase.scope, + foldPolicy: testCase.foldPolicy, + viewport: fixture.viewport, + nodes: testCase.nodes, + })); + const mismatch = compareDifferentialCases(cases); + assert.equal(mismatch, undefined, mismatch ? JSON.stringify(mismatch, null, 2) : ''); +}); + +differentialTest('deterministic Swift/TypeScript differential fuzz stays under 60000ms', () => { + assertDifferentialFuzz(); +}); + +function assertDifferentialFuzz(): void { + const startedAt = performance.now(); + for (const seed of FUZZ_SEEDS) { + assertDifferentialSeed(seed); + assertWithinKillCriterion(startedAt, seed); + } + assert.ok(true); +} + +function assertDifferentialSeed(seed: number): void { + const result = fc.check( + fc.property( + differentialBatchArbitrary, + (cases) => compareDifferentialCases(cases) === undefined, + ), + { + seed, + numRuns: RUNS_PER_SEED, + endOnFailure: true, + interruptAfterTimeLimit: SWIFT_RUN_TIMEOUT_MS, + }, + ); + if (!result.failed) return; + + const counterexample = Array.isArray(result.counterexample?.[0]) + ? (result.counterexample[0] as DifferentialCase[]) + : []; + const mismatch = compareDifferentialCases(counterexample); + const testCase = mismatch?.case ?? counterexample[0]; + if (!testCase) { + throw new Error('differential fuzz failed without a reproducible case: ' + String(result)); + } + const artifact = writeDifferentialFailureArtifact({ + testCase, + seed, + counterexamplePath: result.counterexamplePath ?? 'unknown', + }); + throw new Error( + 'Swift/TypeScript differential mismatch for ' + + testCase.name + + '; minimal case: ' + + artifact.casePath + + '; replay: ' + + artifact.replayCommand, + ); +} + +function assertWithinKillCriterion(startedAt: number, seed: number): void { + if (performance.now() - startedAt <= MAX_TOTAL_DURATION_MS) return; + throw new Error( + 'differential fuzz exceeded its ' + + String(MAX_TOTAL_DURATION_MS) + + 'ms kill criterion after seed ' + + String(seed), + ); +} diff --git a/packages/capture-kit/src/ios-snapshot-engine/properties.test.ts b/packages/capture-kit/src/ios-snapshot-engine/properties.test.ts new file mode 100644 index 000000000..db1113008 --- /dev/null +++ b/packages/capture-kit/src/ios-snapshot-engine/properties.test.ts @@ -0,0 +1,261 @@ +import assert from 'node:assert/strict'; +import fc from 'fast-check'; +import { test } from 'vitest'; +import type { + IosSnapshotAcquisition, + IosSnapshotInput, + IosSnapshotRequest, + IosSnapshotValidationFacts, +} from '@agent-device/contracts/ios-snapshot'; +import type { RawSnapshotNode } from '@agent-device/kernel/snapshot'; +import { + buildIosSnapshotPresentationKey, + createIosSnapshotRequest, + deriveIosCaptureHint, +} from '@agent-device/capture-kit/ios-snapshot-planning'; +import { canonicalNodes, runTypeScriptCase } from './conformance-harness.ts'; +import { differentialBatchArbitrary } from './conformance-generator.ts'; +import { acquisitionForGoldenCase, readIosSnapshotEngineFixture } from './conformance-fixture.ts'; +import { IosSnapshotEngineError, presentIosSnapshot, publishIosSnapshot } from './index.ts'; + +test('property: regular effective geometry stays within the viewport and actionability fails closed', () => { + fc.assert(fc.property(differentialBatchArbitrary, assertRegularCases), { + seed: 219105, + numRuns: 100, + }); +}); + +test('property: an unscoped, unlimited raw presentation preserves every reported frame', () => { + fc.assert(fc.property(differentialBatchArbitrary, assertRawCases), { + seed: 219106, + numRuns: 100, + }); +}); + +test('property: interactive is a subset of regular, and regular is a subset of raw', () => { + fc.assert(fc.property(differentialBatchArbitrary, assertProjectionSubsets), { + seed: 219107, + numRuns: 100, + }); +}); + +test('property: unavailable hittability fails closed without changing comparison identity', () => { + const fixture = readIosSnapshotEngineFixture(); + const testCase = fixture.cases.find( + (entry) => entry.name === 'unavailable hittability fails closed', + ); + assert.ok(testCase); + const request = createIosSnapshotRequest({ projection: 'regular' }); + const acquisition = acquisitionForRequest(acquisitionForGoldenCase(fixture, testCase), request); + const input: IosSnapshotInput = { stage: 'acquired', acquisition }; + const presentation = presentIosSnapshot(input, request, { foldPolicy: testCase.foldPolicy }); + assert.ok(presentation.nodes.every((node) => node.hittable !== true)); + + const publication = publishIosSnapshot(input, request, { foldPolicy: testCase.foldPolicy }); + assert.deepEqual(publication.comparisonIdentity.lineage, acquisition.lineage); + assert.deepEqual(publication.comparisonIdentity.residue, acquisition.residue); + assert.deepEqual(publication.residue, acquisition.residue); +}); + +test('property: runner validation distinguishes an invalid quality payload', () => { + const fixture = readIosSnapshotEngineFixture(); + const testCase = fixture.cases.find( + (entry) => entry.name === 'nested ancestor clips and actionability', + ); + assert.ok(testCase); + const request = createIosSnapshotRequest({ projection: 'regular' }); + const acquisition = acquisitionForRequest(acquisitionForGoldenCase(fixture, testCase), request); + const acquired = publishIosSnapshot({ stage: 'acquired', acquisition }, request, { + foldPolicy: testCase.foldPolicy, + }); + const validation: IosSnapshotValidationFacts = { + presentationKey: buildIosSnapshotPresentationKey(request), + viewport: acquisition.viewport, + hittability: { kind: 'available' }, + lineage: acquisition.lineage, + residue: acquisition.residue, + }; + const input: IosSnapshotInput = { + stage: 'presented', + presentation: { + producer: 'apple-runner', + intent: 'full', + payload: { nodes: acquired.payload.nodes, truncated: false }, + qualityPayload: { + nodes: [ + { + ...acquired.payload.nodes[0]!, + rect: { x: 0, y: 0, width: 321, height: 240 }, + }, + ], + truncated: false, + scope: null, + }, + }, + validation, + }; + assert.throws( + () => presentIosSnapshot(input, request, { foldPolicy: testCase.foldPolicy }), + (error: unknown) => + error instanceof IosSnapshotEngineError && error.reason === 'invalid-quality-payload', + ); +}); + +type DifferentialCase = Parameters[0]; +type DifferentialNode = ReturnType['nodes'][number]; + +function assertProjectionSubsets(cases: readonly DifferentialCase[]): void { + for (const testCase of cases) { + const regularRequest = createIosSnapshotRequest({ projection: 'regular' }); + const interactiveRequest = createIosSnapshotRequest({ + projection: 'regular', + interactiveOnly: true, + }); + const rawRequest = createIosSnapshotRequest({ projection: 'raw' }); + const acquisition = acquisitionForDifferentialCase(testCase, regularRequest); + const regular = presentIosSnapshot({ stage: 'acquired', acquisition }, regularRequest, { + foldPolicy: testCase.foldPolicy, + }); + const interactive = presentIosSnapshot( + { + stage: 'acquired', + acquisition: acquisitionForRequest(acquisition, interactiveRequest), + }, + interactiveRequest, + { foldPolicy: testCase.foldPolicy }, + ); + const raw = presentIosSnapshot( + { stage: 'acquired', acquisition: acquisitionForRequest(acquisition, rawRequest) }, + rawRequest, + { foldPolicy: testCase.foldPolicy }, + ); + assertMultisetSubset(nodeIdentities(interactive.nodes), nodeIdentities(regular.nodes)); + assertMultisetSubset(nodeIdentities(regular.nodes), nodeIdentities(raw.nodes)); + } +} + +function assertMultisetSubset(subset: readonly string[], superset: readonly string[]): void { + const remaining = new Map(); + for (const identity of superset) remaining.set(identity, (remaining.get(identity) ?? 0) + 1); + for (const identity of subset) { + const count = remaining.get(identity) ?? 0; + assert.ok(count > 0, identity); + remaining.set(identity, count - 1); + } +} + +function nodeIdentities(nodes: readonly RawSnapshotNode[]): string[] { + return nodes.map((node) => + JSON.stringify([node.label ?? null, node.identifier ?? null, node.value ?? null]), + ); +} + +function acquisitionForDifferentialCase( + testCase: DifferentialCase, + request: IosSnapshotRequest, +): IosSnapshotAcquisition { + const fullRequest = requireFullRequest(request); + return { + producer: 'simulator-ax-bridge', + intent: fullRequest.acquisitionIntent, + hint: { + ...deriveIosCaptureHint(fullRequest), + acquisitionIntent: fullRequest.acquisitionIntent, + }, + nodes: testCase.nodes, + truncated: false, + viewport: { kind: 'reported', rect: testCase.viewport }, + lineage: { targetId: 'property-target', generation: 'property-generation' }, + residue: [], + }; +} + +function acquisitionForRequest( + acquisition: IosSnapshotAcquisition, + request: IosSnapshotRequest, +): IosSnapshotAcquisition { + const fullRequest = requireFullRequest(request); + return { + producer: acquisition.producer, + intent: fullRequest.acquisitionIntent, + hint: { + ...deriveIosCaptureHint(fullRequest), + acquisitionIntent: fullRequest.acquisitionIntent, + }, + nodes: acquisition.nodes, + truncated: acquisition.truncated, + viewport: acquisition.viewport, + lineage: acquisition.lineage, + residue: acquisition.residue, + }; +} + +type FullSnapshotRequest = IosSnapshotRequest & { acquisitionIntent: 'full' }; + +function requireFullRequest(request: IosSnapshotRequest): FullSnapshotRequest { + if (request.acquisitionIntent !== 'full') { + throw new Error('property fixtures require full acquisition'); + } + return request as FullSnapshotRequest; +} + +function assertRegularCases(cases: readonly DifferentialCase[]): void { + for (const testCase of cases) assertRegularCase(testCase); +} + +function assertRegularCase(testCase: DifferentialCase): void { + const result = runTypeScriptCase(testCase); + assert.equal(result.outcome, 'success'); + if (testCase.projection === 'raw') return; + for (const node of result.nodes) assertRegularNode(node, testCase.viewport); +} + +function assertRegularNode(node: DifferentialNode, viewport: DifferentialCase['viewport']): void { + assertRect(node, viewport); + assertActionability(node, viewport); + assertParentOrder(node); +} + +function assertRect(node: DifferentialNode, viewport: DifferentialCase['viewport']): void { + if (!node.rect) return; + assert.ok(node.rect.width >= 0 && node.rect.height >= 0); + if (node.rect.width > 0 && node.rect.height > 0) assertRectWithinViewport(node.rect, viewport); +} + +function assertRectWithinViewport( + rect: NonNullable, + viewport: DifferentialCase['viewport'], +): void { + assert.ok(rect.x >= viewport.x - 0.0001); + assert.ok(rect.y >= viewport.y - 0.0001); + assert.ok(rect.x + rect.width <= viewport.x + viewport.width + 0.0001); + assert.ok(rect.y + rect.height <= viewport.y + viewport.height + 0.0001); +} + +function assertActionability(node: DifferentialNode, viewport: DifferentialCase['viewport']): void { + if (!node.hittable) return; + assert.ok(node.rect && node.rect.width > 0 && node.rect.height > 0); + const centerX = node.rect.x + node.rect.width / 2; + const centerY = node.rect.y + node.rect.height / 2; + assert.ok(centerX >= viewport.x && centerX <= viewport.x + viewport.width); + assert.ok(centerY >= viewport.y && centerY <= viewport.y + viewport.height); +} + +function assertParentOrder(node: DifferentialNode): void { + if (node.parentIndex !== null) assert.ok(node.parentIndex < node.index); +} + +function assertRawCases(cases: readonly DifferentialCase[]): void { + for (const testCase of cases) { + const rawCase = { + ...testCase, + name: testCase.name + '-raw', + projection: 'raw' as const, + depth: null, + scope: null, + }; + const result = runTypeScriptCase(rawCase); + assert.equal(result.outcome, 'success'); + assert.deepEqual(result.nodes, canonicalNodes(testCase.nodes)); + } +} diff --git a/packages/capture-kit/src/ios-snapshot-engine/replay.ts b/packages/capture-kit/src/ios-snapshot-engine/replay.ts new file mode 100644 index 000000000..fb841637d --- /dev/null +++ b/packages/capture-kit/src/ios-snapshot-engine/replay.ts @@ -0,0 +1,36 @@ +import fs from 'node:fs'; +import path from 'node:path'; +import { compareDifferentialCases } from './conformance-harness.ts'; +import type { RawSnapshotNode, Rect } from '@agent-device/kernel/snapshot'; + +type DifferentialCase = Readonly<{ + name: string; + projection: 'regular' | 'raw'; + interactiveOnly: false; + depth: number | null; + scope: string | null; + foldPolicy: 'cursor-projected' | 'plain-viewport'; + viewport: Rect; + nodes: readonly RawSnapshotNode[]; +}>; + +const casePath = process.argv[2]; +if (!casePath) { + throw new Error('usage: replay.ts '); +} + +const input = JSON.parse(fs.readFileSync(path.resolve(casePath), 'utf8')) as { + cases?: DifferentialCase[]; +}; +const cases = input.cases ?? []; +if (cases.length !== 1) { + throw new Error('case.json must contain exactly one differential case'); +} + +const mismatch = compareDifferentialCases(cases); +if (mismatch) { + console.error(JSON.stringify(mismatch, null, 2)); + process.exitCode = 1; +} else { + console.log('replayed ' + cases[0]!.name + ': Swift and TypeScript agree'); +} diff --git a/packages/capture-kit/src/ios-snapshot-engine/runner-presentation.ts b/packages/capture-kit/src/ios-snapshot-engine/runner-presentation.ts index ce86199bf..ee46511dc 100644 --- a/packages/capture-kit/src/ios-snapshot-engine/runner-presentation.ts +++ b/packages/capture-kit/src/ios-snapshot-engine/runner-presentation.ts @@ -82,13 +82,20 @@ function validateRunnerPayloads( hittabilityAvailable, ); if (input.presentation.qualityPayload) { - validateIosPayload( - input.presentation.qualityPayload.nodes, - projection, - viewport, - foldPolicy, - hittabilityAvailable, - ); + try { + validateIosPayload( + input.presentation.qualityPayload.nodes, + projection, + viewport, + foldPolicy, + hittabilityAvailable, + ); + } catch (error) { + if (error instanceof IosSnapshotEngineError) { + throw new IosSnapshotEngineError('invalid-quality-payload', error.message, error.details); + } + throw error; + } } return payloadValidation; } diff --git a/pnpm-lock.yaml b/pnpm-lock.yaml index ae15c6892..fc05d47a7 100644 --- a/pnpm-lock.yaml +++ b/pnpm-lock.yaml @@ -211,6 +211,9 @@ importers: '@types/pngjs': specifier: ^6.0.5 version: 6.0.5 + fast-check: + specifier: ^4.9.0 + version: 4.9.0 pngjs: specifier: ^7.0.0 version: 7.0.0 diff --git a/scripts/layering/check.ts b/scripts/layering/check.ts index c87f85aae..bd19c8a46 100644 --- a/scripts/layering/check.ts +++ b/scripts/layering/check.ts @@ -110,6 +110,7 @@ import { sourceExecutionCompatibilityViolations } from './source-execution-polic import { sessionResourceOwnershipViolations } from './session-resource-ownership.ts'; import { replayOwnershipViolations } from './replay-ownership.ts'; import { applicationLifecycleOwnershipViolations } from './application-lifecycle-policy.ts'; +import { iosSnapshotEngineOwnershipViolations } from './ios-snapshot-engine-policy.ts'; const repoRoot = execFileSync('git', ['rev-parse', '--show-toplevel'], { encoding: 'utf8', @@ -557,6 +558,7 @@ export const LAYERING_RULE_IDS = [ 'platform-package-policy', 'retired-platforms-zone', 'replay-ownership', + 'ios-snapshot-engine-ownership', ] as const; export type LayeringRuleId = (typeof LAYERING_RULE_IDS)[number]; @@ -598,6 +600,10 @@ export const LAYERING_RULES: Readonly> = { ), 'retired-platforms-zone': () => checkRetiredPlatformsZone(listTrackedPlatformZoneFiles(repoRoot)), 'replay-ownership': (context) => replayOwnershipViolations(context.sourceFiles), + 'ios-snapshot-engine-ownership': (context) => + iosSnapshotEngineOwnershipViolations( + [...context.sources].map(([path, source]) => ({ path, source })), + ), }; export function main(): number { diff --git a/scripts/layering/ios-snapshot-engine-policy.test.ts b/scripts/layering/ios-snapshot-engine-policy.test.ts new file mode 100644 index 000000000..f874841ed --- /dev/null +++ b/scripts/layering/ios-snapshot-engine-policy.test.ts @@ -0,0 +1,38 @@ +import assert from 'node:assert/strict'; +import fs from 'node:fs'; +import path from 'node:path'; +import { test } from 'node:test'; +import { + IOS_SNAPSHOT_ENGINE_FILE, + IOS_SNAPSHOT_RUNNER_FILE, + iosSnapshotEngineOwnershipViolations, +} from './ios-snapshot-engine-policy.ts'; + +const repoRoot = path.resolve(import.meta.dirname, '../..'); + +function sources(overrides: ReadonlyMap = new Map()) { + return [IOS_SNAPSHOT_ENGINE_FILE, IOS_SNAPSHOT_RUNNER_FILE].map((file) => ({ + path: file, + source: overrides.get(file) ?? fs.readFileSync(path.join(repoRoot, file), 'utf8'), + })); +} + +test('the iOS snapshot engine owns each presentation boundary exactly once', () => { + assert.deepEqual(iosSnapshotEngineOwnershipViolations(sources()), []); +}); + +test('the structural gate rejects a planted duplicate acquired presentation', () => { + const engine = fs.readFileSync(path.join(repoRoot, IOS_SNAPSHOT_ENGINE_FILE), 'utf8'); + const planted = engine.replace( + 'return presentAcquiredSnapshot(input.acquisition, request, foldPolicy);', + 'return presentAcquiredSnapshot(input.acquisition, request, foldPolicy);\n presentAcquiredSnapshot(input.acquisition, request, foldPolicy);', + ); + assert.notEqual(planted, engine); + const violations = iosSnapshotEngineOwnershipViolations( + sources(new Map([[IOS_SNAPSHOT_ENGINE_FILE, planted]])), + ); + assert.ok( + violations.some((violation) => violation.message.includes('presentAcquiredSnapshot exactly 1')), + JSON.stringify(violations), + ); +}); diff --git a/scripts/layering/ios-snapshot-engine-policy.ts b/scripts/layering/ios-snapshot-engine-policy.ts new file mode 100644 index 000000000..d801ee6b2 --- /dev/null +++ b/scripts/layering/ios-snapshot-engine-policy.ts @@ -0,0 +1,229 @@ +import { parseSync } from 'oxc-parser'; +import type { LayeringViolation } from './model.ts'; +import { memberPath, visitAst } from './layering-ast.ts'; + +export const IOS_SNAPSHOT_ENGINE_OWNERSHIP_RULE = 'R72 ios-snapshot-engine-ownership'; +export const IOS_SNAPSHOT_ENGINE_FILE = 'packages/capture-kit/src/ios-snapshot-engine/engine.ts'; +export const IOS_SNAPSHOT_RUNNER_FILE = + 'packages/capture-kit/src/ios-snapshot-engine/runner-presentation.ts'; + +type SourceFile = Readonly<{ path: string; source: string }>; +type AstNode = Record; +type CallSite = Readonly<{ line: number; arguments: readonly AstNode[] }>; + +export function iosSnapshotEngineOwnershipViolations( + sources: readonly SourceFile[], +): LayeringViolation[] { + const byPath = new Map(sources.map((file) => [file.path, file.source])); + const engineSource = byPath.get(IOS_SNAPSHOT_ENGINE_FILE); + const runnerSource = byPath.get(IOS_SNAPSHOT_RUNNER_FILE); + const violations: LayeringViolation[] = []; + if (!engineSource) violations.push(missingFile(IOS_SNAPSHOT_ENGINE_FILE)); + if (!runnerSource) violations.push(missingFile(IOS_SNAPSHOT_RUNNER_FILE)); + if (!engineSource || !runnerSource) return violations; + + const engine = parseSource(IOS_SNAPSHOT_ENGINE_FILE, engineSource); + const runner = parseSource(IOS_SNAPSHOT_RUNNER_FILE, runnerSource); + const present = functionBody(engine, 'presentIosSnapshot'); + const publish = functionBody(engine, 'publishIosSnapshot'); + const acquired = functionBody(engine, 'presentAcquiredSnapshot'); + const presented = functionBody(runner, 'presentIosRunnerSnapshot'); + const runnerPayloads = functionBody(runner, 'validateRunnerPayloads'); + const runnerCompaction = functionBody(runner, 'compactRunnerPayload'); + + requireCallCount( + violations, + IOS_SNAPSHOT_ENGINE_FILE, + 'presentIosSnapshot', + present, + 'presentAcquiredSnapshot', + 1, + ); + requireCallCount( + violations, + IOS_SNAPSHOT_ENGINE_FILE, + 'presentIosSnapshot', + present, + 'presentIosRunnerSnapshot', + 1, + ); + requireCallCount( + violations, + IOS_SNAPSHOT_ENGINE_FILE, + 'publishIosSnapshot', + publish, + 'presentIosSnapshot', + 1, + ); + requireCallCount( + violations, + IOS_SNAPSHOT_ENGINE_FILE, + 'presentAcquiredSnapshot', + acquired, + 'buildIosInteractiveSnapshotPresentation', + 1, + ); + requireCallCount( + violations, + IOS_SNAPSHOT_RUNNER_FILE, + 'presentIosRunnerSnapshot', + presented, + 'validateRunnerPayloads', + 1, + ); + requireCallCount( + violations, + IOS_SNAPSHOT_RUNNER_FILE, + 'presentIosRunnerSnapshot', + presented, + 'compactRunnerPayload', + 1, + ); + requireCallCount( + violations, + IOS_SNAPSHOT_RUNNER_FILE, + 'presentIosRunnerSnapshot', + presented, + 'foldIosSnapshot', + 0, + ); + requireCallCount( + violations, + IOS_SNAPSHOT_RUNNER_FILE, + 'presentIosRunnerSnapshot', + presented, + 'validateIosPayload', + 0, + ); + requireCallCount( + violations, + IOS_SNAPSHOT_RUNNER_FILE, + 'presentIosRunnerSnapshot', + presented, + 'buildIosInteractiveSnapshotPresentation', + 0, + ); + requireCallCount( + violations, + IOS_SNAPSHOT_RUNNER_FILE, + 'validateRunnerPayloads', + runnerPayloads, + 'validateIosPayload', + 2, + ); + requireArgumentPath( + violations, + IOS_SNAPSHOT_RUNNER_FILE, + 'validateRunnerPayloads', + runnerPayloads, + 'validateIosPayload', + ['input', 'presentation', 'payload', 'nodes'], + ); + requireArgumentPath( + violations, + IOS_SNAPSHOT_RUNNER_FILE, + 'validateRunnerPayloads', + runnerPayloads, + 'validateIosPayload', + ['input', 'presentation', 'qualityPayload', 'nodes'], + ); + requireCallCount( + violations, + IOS_SNAPSHOT_RUNNER_FILE, + 'compactRunnerPayload', + runnerCompaction, + 'buildIosInteractiveSnapshotPresentation', + 1, + ); + return violations; +} + +function missingFile(file: string): LayeringViolation { + return { + rule: IOS_SNAPSHOT_ENGINE_OWNERSHIP_RULE, + file, + line: 1, + message: `${file} is missing, so the iOS snapshot engine ownership paths cannot be checked`, + }; +} + +function parseSource(file: string, source: string): AstNode { + return parseSync(file, source).program as unknown as AstNode; +} + +function functionBody(program: AstNode, name: string): AstNode | undefined { + let body: AstNode | undefined; + visitAst(program, (node) => { + if (body || node.type !== 'FunctionDeclaration') return; + const id = node.id as AstNode | undefined; + if (id?.type !== 'Identifier' || id.name !== name) return; + body = node.body as AstNode | undefined; + }); + return body; +} + +function callSites(body: AstNode | undefined, name: string): CallSite[] { + if (!body) return []; + const sites: CallSite[] = []; + visitAst(body, (node) => { + if (node.type !== 'CallExpression' || identifierName(node.callee) !== name) return; + sites.push({ + line: 1, + arguments: (node.arguments as AstNode[] | undefined) ?? [], + }); + }); + return sites; +} + +function identifierName(node: unknown): string | undefined { + if (!node || typeof node !== 'object') return undefined; + const record = node as AstNode; + return record.type === 'Identifier' && typeof record.name === 'string' ? record.name : undefined; +} + +function requireCallCount( + violations: LayeringViolation[], + file: string, + functionName: string, + body: AstNode | undefined, + callName: string, + expected: number, +): void { + const sites = callSites(body, callName); + if (sites.length === expected) return; + violations.push({ + rule: IOS_SNAPSHOT_ENGINE_OWNERSHIP_RULE, + file, + line: sites[0]?.line ?? 1, + message: + `${functionName} must call ${callName} exactly ${String(expected)} time(s); found ` + + String(sites.length), + }); +} + +function requireArgumentPath( + violations: LayeringViolation[], + file: string, + functionName: string, + body: AstNode | undefined, + callName: string, + expectedPath: readonly string[], +): void { + const sites = callSites(body, callName); + if (sites.some((site) => site.arguments.some((argument) => samePath(argument, expectedPath)))) { + return; + } + violations.push({ + rule: IOS_SNAPSHOT_ENGINE_OWNERSHIP_RULE, + file, + line: sites[0]?.line ?? 1, + message: `${functionName} must validate ${expectedPath.join('.')}`, + }); +} + +function samePath(node: AstNode, expected: readonly string[]): boolean { + const actual = memberPath(node); + return ( + actual?.length === expected.length && actual.every((part, index) => part === expected[index]) + ); +} From 623a13661b961d5417b8fac57c80faa37e94d391 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Micha=C5=82=20Pierzcha=C5=82a?= Date: Tue, 1 Sep 2026 11:34:44 +0200 Subject: [PATCH 2/5] test(ios): align differential acquisition inputs --- .../ios-snapshot-engine-conformance.json | 71 +++++++++++++++++++ .../conformance-harness.ts | 7 +- .../ios-snapshot-engine/differential.test.ts | 23 ++++++ 3 files changed, 98 insertions(+), 3 deletions(-) diff --git a/contracts/fixtures/ios-snapshot-engine-conformance.json b/contracts/fixtures/ios-snapshot-engine-conformance.json index 020c0d30c..40be79491 100644 --- a/contracts/fixtures/ios-snapshot-engine-conformance.json +++ b/contracts/fixtures/ios-snapshot-engine-conformance.json @@ -421,6 +421,77 @@ ] } }, + { + "name": "raw unscoped depth uses the acquisition frontier", + "swift": true, + "projection": "raw", + "interactiveOnly": false, + "depth": 1, + "scope": null, + "foldPolicy": "cursor-projected", + "truncated": false, + "residue": [], + "nodes": [ + { + "index": 0, + "type": "Application", + "label": "App", + "rect": { "x": 0, "y": 0, "width": 320, "height": 240 }, + "enabled": true, + "hittable": false, + "depth": 0 + }, + { + "index": 1, + "type": "ScrollView", + "label": "Outer", + "rect": { "x": 16, "y": 20, "width": 180, "height": 180 }, + "enabled": true, + "hittable": false, + "depth": 1, + "parentIndex": 0 + }, + { + "index": 2, + "type": "Button", + "label": "Deep child", + "rect": { "x": 20, "y": 40, "width": 80, "height": 40 }, + "enabled": true, + "hittable": true, + "depth": 2, + "parentIndex": 1 + } + ], + "expected": { + "outcome": "success", + "truncated": false, + "residue": [], + "nodes": [ + { + "index": 0, + "type": "Application", + "label": "App", + "rect": { "x": 0, "y": 0, "width": 320, "height": 240 }, + "depth": 0, + "parentIndex": null, + "hittable": false, + "hiddenContentAbove": false, + "hiddenContentBelow": false + }, + { + "index": 1, + "type": "ScrollView", + "label": "Outer", + "rect": { "x": 16, "y": 20, "width": 180, "height": 180 }, + "depth": 1, + "parentIndex": 0, + "hittable": false, + "hiddenContentAbove": false, + "hiddenContentBelow": false + } + ] + } + }, { "name": "scope reroots wrappers and regular depth", "swift": true, diff --git a/packages/capture-kit/src/ios-snapshot-engine/conformance-harness.ts b/packages/capture-kit/src/ios-snapshot-engine/conformance-harness.ts index 2fb3ab05f..47c8c2130 100644 --- a/packages/capture-kit/src/ios-snapshot-engine/conformance-harness.ts +++ b/packages/capture-kit/src/ios-snapshot-engine/conformance-harness.ts @@ -78,7 +78,7 @@ function runSwiftCases(cases: readonly DifferentialCase[]): DifferentialOutcome[ const stdout = execFileSync(swiftConformanceExecutable(), [], { cwd: REPO_ROOT, encoding: 'utf8', - input: JSON.stringify({ cases: cases.map(prepareSwiftAcquisition) }), + input: JSON.stringify({ cases: cases.map(prepareDifferentialAcquisition) }), timeout: SWIFT_RUN_TIMEOUT_MS, maxBuffer: 8 * 1024 * 1024, }); @@ -114,7 +114,7 @@ function swiftConformanceExecutable(): string { return swiftHarnessExecutable; } -function prepareSwiftAcquisition(testCase: DifferentialCase): DifferentialCase { +function prepareDifferentialAcquisition(testCase: DifferentialCase): DifferentialCase { if (testCase.projection !== 'raw' || testCase.scope !== null || testCase.depth === null) { return testCase; } @@ -130,6 +130,7 @@ function withoutName(outcome: DifferentialOutcome): Omit { assert.equal(mismatch, undefined, mismatch ? JSON.stringify(mismatch, null, 2) : ''); }); +differentialTest('raw unscoped depth compares the same acquisition frontier', () => { + const fixture = readIosSnapshotEngineFixture(); + const depthCase = fixture.cases.find( + (testCase) => testCase.name === 'raw unscoped depth uses the acquisition frontier', + ); + assert.ok(depthCase); + const deepNode = depthCase.nodes.at(-1); + assert.ok(deepNode); + const mismatch = compareDifferentialCases([ + { + name: 'raw-depth-frontier-with-malformed-tail', + projection: 'raw', + interactiveOnly: false, + depth: 1, + scope: null, + foldPolicy: 'cursor-projected', + viewport: fixture.viewport, + nodes: [...depthCase.nodes, { ...deepNode, index: 1, parentIndex: 1, depth: 2 }], + }, + ]); + assert.equal(mismatch, undefined, mismatch ? JSON.stringify(mismatch, null, 2) : ''); +}); + differentialTest('deterministic Swift/TypeScript differential fuzz stays under 60000ms', () => { assertDifferentialFuzz(); }); From e2c90ad53bbfe05f4415868470eff957efe4b4b1 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Micha=C5=82=20Pierzcha=C5=82a?= Date: Tue, 1 Sep 2026 13:39:35 +0200 Subject: [PATCH 3/5] fix(ios): gate Swift differential on macOS --- .../capture-kit/src/ios-snapshot-engine/conformance-harness.ts | 1 + 1 file changed, 1 insertion(+) diff --git a/packages/capture-kit/src/ios-snapshot-engine/conformance-harness.ts b/packages/capture-kit/src/ios-snapshot-engine/conformance-harness.ts index 47c8c2130..4f35d6ed3 100644 --- a/packages/capture-kit/src/ios-snapshot-engine/conformance-harness.ts +++ b/packages/capture-kit/src/ios-snapshot-engine/conformance-harness.ts @@ -51,6 +51,7 @@ type DifferentialMismatch = Readonly<{ }>; export function swiftToolchainAvailable(): boolean { + if (process.platform !== 'darwin') return false; try { execFileSync('swift', ['--version'], { stdio: 'ignore' }); return true; From 5cd3ce8eb37c589764b63a248647b55cae099bcb Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Micha=C5=82=20Pierzcha=C5=82a?= Date: Tue, 1 Sep 2026 13:53:22 +0200 Subject: [PATCH 4/5] test(ios): keep differential coverage host-aware --- .../conformance-harness.ts | 10 ++++ .../ios-snapshot-engine/conformance.test.ts | 48 +++++++++++++++++++ .../src/ios-snapshot-engine/replay.ts | 1 + .../runner-presentation.ts | 1 + 4 files changed, 60 insertions(+) diff --git a/packages/capture-kit/src/ios-snapshot-engine/conformance-harness.ts b/packages/capture-kit/src/ios-snapshot-engine/conformance-harness.ts index 4f35d6ed3..708319335 100644 --- a/packages/capture-kit/src/ios-snapshot-engine/conformance-harness.ts +++ b/packages/capture-kit/src/ios-snapshot-engine/conformance-harness.ts @@ -52,14 +52,17 @@ type DifferentialMismatch = Readonly<{ export function swiftToolchainAvailable(): boolean { if (process.platform !== 'darwin') return false; + /* c8 ignore start */ try { execFileSync('swift', ['--version'], { stdio: 'ignore' }); return true; } catch { return false; } + /* c8 ignore stop */ } +/* c8 ignore start */ export function compareDifferentialCases( cases: readonly DifferentialCase[], ): DifferentialMismatch | undefined { @@ -74,7 +77,9 @@ export function compareDifferentialCases( } return undefined; } +/* c8 ignore stop */ +/* c8 ignore start */ function runSwiftCases(cases: readonly DifferentialCase[]): DifferentialOutcome[] { const stdout = execFileSync(swiftConformanceExecutable(), [], { cwd: REPO_ROOT, @@ -98,7 +103,9 @@ function runSwiftCases(cases: readonly DifferentialCase[]): DifferentialOutcome[ ...(entry.error ? { error: entry.error } : {}), })); } +/* c8 ignore stop */ +/* c8 ignore start */ function swiftConformanceExecutable(): string { if (swiftHarnessExecutable) return swiftHarnessExecutable; execFileSync('swift', ['build', '--package-path', SWIFT_PACKAGE_PATH], { @@ -114,6 +121,7 @@ function swiftConformanceExecutable(): string { swiftHarnessExecutable = path.join(binPath, SWIFT_PRODUCT); return swiftHarnessExecutable; } +/* c8 ignore stop */ function prepareDifferentialAcquisition(testCase: DifferentialCase): DifferentialCase { if (testCase.projection !== 'raw' || testCase.scope !== null || testCase.depth === null) { @@ -125,10 +133,12 @@ function prepareDifferentialAcquisition(testCase: DifferentialCase): Differentia }; } +/* c8 ignore start */ function withoutName(outcome: DifferentialOutcome): Omit { const { name: _name, ...normalized } = outcome; return normalized; } +/* c8 ignore stop */ export function runTypeScriptCase(testCase: DifferentialCase): DifferentialOutcome { const acquisitionInput = prepareDifferentialAcquisition(testCase); diff --git a/packages/capture-kit/src/ios-snapshot-engine/conformance.test.ts b/packages/capture-kit/src/ios-snapshot-engine/conformance.test.ts index 998ddd26b..8afcd9fee 100644 --- a/packages/capture-kit/src/ios-snapshot-engine/conformance.test.ts +++ b/packages/capture-kit/src/ios-snapshot-engine/conformance.test.ts @@ -8,6 +8,7 @@ import { deriveIosCaptureHint, } from '@agent-device/capture-kit/ios-snapshot-planning'; import { IosSnapshotEngineError, presentIosSnapshot } from './index.ts'; +import { runTypeScriptCase, writeDifferentialFailureArtifact } from './conformance-harness.ts'; import { acquisitionForGoldenCase, normalizeGoldenNodes, @@ -123,3 +124,50 @@ test('the independent iOS snapshot goldens match the TypeScript engine', () => { ); } }); + +test('the differential TypeScript runner preserves typed failures', () => { + const fixture = readIosSnapshotEngineFixture(); + const source = fixture.cases.find( + (testCase) => testCase.name === 'malformed parent is a typed failure', + ); + assert.ok(source); + const result = runTypeScriptCase({ + name: source.name, + projection: source.projection, + interactiveOnly: false, + depth: source.depth, + scope: source.scope, + foldPolicy: source.foldPolicy, + viewport: fixture.viewport, + nodes: source.nodes, + }); + assert.equal(result.outcome, 'failure'); + assert.ok(result.error?.code); +}); + +test('differential failure artifacts preserve replay metadata', () => { + const fixture = readIosSnapshotEngineFixture(); + const source = fixture.cases[0]!; + const testCase = { + name: source.name, + projection: source.projection, + interactiveOnly: false as const, + depth: source.depth, + scope: source.scope, + foldPolicy: source.foldPolicy, + viewport: fixture.viewport, + nodes: source.nodes, + }; + const artifact = writeDifferentialFailureArtifact({ + testCase, + seed: 219101, + counterexamplePath: '0:0', + }); + const stored = JSON.parse(fs.readFileSync(artifact.casePath, 'utf8')) as { + cases: readonly unknown[]; + }; + const metadata = fs.readFileSync(path.join(artifact.directory, 'replay-command.txt'), 'utf8'); + assert.equal(stored.cases.length, 1); + assert.match(metadata, /seed=219101/); + assert.match(metadata, /path=0:0/); +}); diff --git a/packages/capture-kit/src/ios-snapshot-engine/replay.ts b/packages/capture-kit/src/ios-snapshot-engine/replay.ts index fb841637d..e75d183c2 100644 --- a/packages/capture-kit/src/ios-snapshot-engine/replay.ts +++ b/packages/capture-kit/src/ios-snapshot-engine/replay.ts @@ -1,3 +1,4 @@ +/* c8 ignore file */ import fs from 'node:fs'; import path from 'node:path'; import { compareDifferentialCases } from './conformance-harness.ts'; diff --git a/packages/capture-kit/src/ios-snapshot-engine/runner-presentation.ts b/packages/capture-kit/src/ios-snapshot-engine/runner-presentation.ts index ee46511dc..361f7bbcd 100644 --- a/packages/capture-kit/src/ios-snapshot-engine/runner-presentation.ts +++ b/packages/capture-kit/src/ios-snapshot-engine/runner-presentation.ts @@ -94,6 +94,7 @@ function validateRunnerPayloads( if (error instanceof IosSnapshotEngineError) { throw new IosSnapshotEngineError('invalid-quality-payload', error.message, error.details); } + /* c8 ignore next */ throw error; } } From cbb3dae97a7bea963a32c0ff300e20c24313bc09 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Micha=C5=82=20Pierzcha=C5=82a?= Date: Tue, 1 Sep 2026 14:31:19 +0200 Subject: [PATCH 5/5] test(ios): own snapshot differential on macOS --- .github/workflows/macos.yml | 4 + package.json | 1 + scripts/check-affected/checks.ts | 6 ++ scripts/check-affected/model.ts | 11 +++ .../ios-snapshot-differential.test.ts | 85 ++++++++++--------- 5 files changed, 67 insertions(+), 40 deletions(-) rename packages/capture-kit/src/ios-snapshot-engine/differential.test.ts => scripts/ios-snapshot-differential.test.ts (57%) diff --git a/.github/workflows/macos.yml b/.github/workflows/macos.yml index 64e59ce5f..8cf7cd8bb 100644 --- a/.github/workflows/macos.yml +++ b/.github/workflows/macos.yml @@ -70,6 +70,10 @@ jobs: uses: ./.github/actions/run-gate with: { gate: macos-coverage } + - name: Run iOS snapshot Swift/TypeScript differential + uses: ./.github/actions/run-gate + with: { gate: ios-snapshot-differential } + - name: Restore and build macOS XCTest runner uses: ./.github/actions/setup-apple-runner-build with: diff --git a/package.json b/package.json index b3411919c..ca0faab80 100644 --- a/package.json +++ b/package.json @@ -117,6 +117,7 @@ "maestro:conformance": "node --experimental-strip-types scripts/node-test-tmpdir.ts --experimental-strip-types --test scripts/maestro-conformance/format-generated-json.test.mjs packages/maestro/test/conformance/verify.test.ts packages/maestro/test/conformance/differential/engine-process.test.ts packages/maestro/test/conformance/differential/report-output.test.ts packages/maestro/test/conformance/differential/run.test.ts packages/maestro/test/conformance/differential/invariants.test.ts", "maestro:conformance:regenerate": "node --experimental-strip-types scripts/maestro-conformance/regenerate.mjs", "maestro:conformance:differential": "node --experimental-strip-types packages/maestro/test/conformance/differential/run.ts", + "test:ios-snapshot-differential": "node --experimental-strip-types scripts/node-test-tmpdir.ts --experimental-strip-types --test scripts/ios-snapshot-differential.test.ts", "size": "node scripts/size-report.mjs", "perf": "node --experimental-strip-types scripts/perf/run.ts", "mutation:run": "node --experimental-strip-types scripts/mutation/run.ts", diff --git a/scripts/check-affected/checks.ts b/scripts/check-affected/checks.ts index 6fd590076..b31004f58 100644 --- a/scripts/check-affected/checks.ts +++ b/scripts/check-affected/checks.ts @@ -46,6 +46,12 @@ export const CHECK_CATALOG: readonly CheckSpec[] = [ gate('package', 'Published package (publint, attw, clean-install resolution)', 'check:package'), gate('integration-node', 'Node integration smoke', 'test:integration:node'), gate('macos-coverage', 'macOS command coverage manifest', 'test:integration:macos-coverage'), + gate( + 'ios-snapshot-differential', + 'iOS snapshot Swift/TypeScript differential', + 'test:ios-snapshot-differential', + false, + ), { id: 'vitest-related', label: 'Tests related by Vitest module graph', diff --git a/scripts/check-affected/model.ts b/scripts/check-affected/model.ts index 6c6dc130a..65028f6f2 100644 --- a/scripts/check-affected/model.ts +++ b/scripts/check-affected/model.ts @@ -46,6 +46,7 @@ export type CheckId = | 'provider-integration' | 'integration-node' | 'macos-coverage' + | 'ios-snapshot-differential' | 'integration-progress' | 'swift-runner-ios' | 'swift-runner-macos' @@ -105,6 +106,7 @@ export const ALL_CHECKS: readonly CheckId[] = [ // run before the related-project workload heats the host. 'integration-node', 'macos-coverage', + 'ios-snapshot-differential', 'vitest-related', 'unit', 'unit-ci', @@ -435,6 +437,15 @@ const BUILD_OWNERSHIP: ReadonlyArray<{ detail: string; owns: (file: string) => boolean; }> = [ + { + check: 'ios-snapshot-differential', + rule: 'own:ios-snapshot-differential', + detail: 'the required macOS lane runs the Swift/TypeScript snapshot differential', + owns: (file) => + file.startsWith('packages/capture-kit/src/ios-snapshot-engine/') || + file.startsWith('apple/snapshot-presentation/') || + file === 'contracts/fixtures/ios-snapshot-engine-conformance.json', + }, // Both platform builds compile the same runner sources, and each is a separate // gate in a separate lane, so a Swift change owns both. { diff --git a/packages/capture-kit/src/ios-snapshot-engine/differential.test.ts b/scripts/ios-snapshot-differential.test.ts similarity index 57% rename from packages/capture-kit/src/ios-snapshot-engine/differential.test.ts rename to scripts/ios-snapshot-differential.test.ts index 0cd83748f..a8c8d3041 100644 --- a/packages/capture-kit/src/ios-snapshot-engine/differential.test.ts +++ b/scripts/ios-snapshot-differential.test.ts @@ -1,14 +1,14 @@ import assert from 'node:assert/strict'; +import { test } from 'node:test'; import fc from 'fast-check'; -import { test } from 'vitest'; import { compareDifferentialCases, swiftToolchainAvailable, SWIFT_RUN_TIMEOUT_MS, writeDifferentialFailureArtifact, -} from './conformance-harness.ts'; -import { differentialBatchArbitrary } from './conformance-generator.ts'; -import { readIosSnapshotEngineFixture } from './conformance-fixture.ts'; +} from '../packages/capture-kit/src/ios-snapshot-engine/conformance-harness.ts'; +import { differentialBatchArbitrary } from '../packages/capture-kit/src/ios-snapshot-engine/conformance-generator.ts'; +import { readIosSnapshotEngineFixture } from '../packages/capture-kit/src/ios-snapshot-engine/conformance-fixture.ts'; type DifferentialCase = Parameters[0][number]; @@ -16,9 +16,11 @@ const FUZZ_SEEDS = [219101, 219102, 219103, 219104]; const RUNS_PER_SEED = 8; const MAX_TOTAL_DURATION_MS = 60_000; -const differentialTest = swiftToolchainAvailable() ? test : test.skip; +if (!swiftToolchainAvailable()) { + throw new Error('iOS snapshot differential requires the macOS Swift toolchain'); +} -differentialTest('authored Swift and TypeScript golden cases agree', () => { +test('authored Swift and TypeScript golden cases agree', { timeout: SWIFT_RUN_TIMEOUT_MS }, () => { const fixture = readIosSnapshotEngineFixture(); const cases = fixture.cases .filter((testCase) => testCase.swift && !testCase.interactiveOnly) @@ -36,41 +38,44 @@ differentialTest('authored Swift and TypeScript golden cases agree', () => { assert.equal(mismatch, undefined, mismatch ? JSON.stringify(mismatch, null, 2) : ''); }); -differentialTest('raw unscoped depth compares the same acquisition frontier', () => { - const fixture = readIosSnapshotEngineFixture(); - const depthCase = fixture.cases.find( - (testCase) => testCase.name === 'raw unscoped depth uses the acquisition frontier', - ); - assert.ok(depthCase); - const deepNode = depthCase.nodes.at(-1); - assert.ok(deepNode); - const mismatch = compareDifferentialCases([ - { - name: 'raw-depth-frontier-with-malformed-tail', - projection: 'raw', - interactiveOnly: false, - depth: 1, - scope: null, - foldPolicy: 'cursor-projected', - viewport: fixture.viewport, - nodes: [...depthCase.nodes, { ...deepNode, index: 1, parentIndex: 1, depth: 2 }], - }, - ]); - assert.equal(mismatch, undefined, mismatch ? JSON.stringify(mismatch, null, 2) : ''); -}); - -differentialTest('deterministic Swift/TypeScript differential fuzz stays under 60000ms', () => { - assertDifferentialFuzz(); -}); +test( + 'raw unscoped depth compares the same acquisition frontier', + { timeout: SWIFT_RUN_TIMEOUT_MS }, + () => { + const fixture = readIosSnapshotEngineFixture(); + const depthCase = fixture.cases.find( + (testCase) => testCase.name === 'raw unscoped depth uses the acquisition frontier', + ); + assert.ok(depthCase); + const deepNode = depthCase.nodes.at(-1); + assert.ok(deepNode); + const mismatch = compareDifferentialCases([ + { + name: 'raw-depth-frontier-with-malformed-tail', + projection: 'raw', + interactiveOnly: false, + depth: 1, + scope: null, + foldPolicy: 'cursor-projected', + viewport: fixture.viewport, + nodes: [...depthCase.nodes, { ...deepNode, index: 1, parentIndex: 1, depth: 2 }], + }, + ]); + assert.equal(mismatch, undefined, mismatch ? JSON.stringify(mismatch, null, 2) : ''); + }, +); -function assertDifferentialFuzz(): void { - const startedAt = performance.now(); - for (const seed of FUZZ_SEEDS) { - assertDifferentialSeed(seed); - assertWithinKillCriterion(startedAt, seed); - } - assert.ok(true); -} +test( + 'deterministic Swift/TypeScript differential fuzz stays under 60000ms', + { timeout: SWIFT_RUN_TIMEOUT_MS * FUZZ_SEEDS.length }, + () => { + const startedAt = performance.now(); + for (const seed of FUZZ_SEEDS) { + assertDifferentialSeed(seed); + assertWithinKillCriterion(startedAt, seed); + } + }, +); function assertDifferentialSeed(seed: number): void { const result = fc.check(