fix(ios): read system dialogs from a throwing snapshot so the location prompt cannot kill the runner

maestro-ios-xctest-runner/MaestroDriverLib/Sources/MaestroDriverLib/Helpers/SystemDialogMatcher.swift

@@ -0,0 +1,226 @@
+import Foundation
+
+/// How to recognise one iOS system permission dialog and which button answers it.
+public struct SystemPermissionDialog: Equatable {
+    /// Key in the permissions map sent by the CLI (e.g. "location").
+    public let key: String
+    /// Case-insensitive substring of the dialog title (or, failing that, its static texts).
+    public let labelPattern: String
+    /// Number of action buttons at the bottom of the dialog. Buttons in the dialog body
+    /// (e.g. the location prompt's "Precise: On" map toggle, which can appear more than once)
+    /// come before them, so action buttons are the last `actionButtonCount` buttons.
+    public let actionButtonCount: Int
+    /// PermissionValue → index among the action buttons. No entry means "leave it alone".
+    public let buttonMap: [PermissionValue: Int]
+    /// Optional exact (case-insensitive) labels tried before the index, per value.
+    public let preferredLabels: [PermissionValue: [String]]
+
+    public init(
+        key: String,
+        labelPattern: String,
+        actionButtonCount: Int = 2,
+        buttonMap: [PermissionValue: Int],
+        preferredLabels: [PermissionValue: [String]] = [:]
+    ) {
+        self.key = key
+        self.labelPattern = labelPattern
+        self.actionButtonCount = actionButtonCount
+        self.buttonMap = buttonMap
+        self.preferredLabels = preferredLabels
+    }
+}
+
+/// What the runner should tap, decided purely from an `AXElement` snapshot.
+public struct SystemDialogTap: Equatable {
+    public enum Reason: Equatable {
+        case accountAlert
+        case permission(key: String, value: PermissionValue)
+        case customPermission(key: String, value: PermissionValue)
+    }
+
+    public let reason: Reason
+    public let buttonLabel: String
+    public let frame: AXFrame
+}
+
+/// Pure logic for finding and answering SpringBoard system dialogs (permission prompts and
+/// Apple Account password alerts) in an `AXElement` tree taken with one throwing `snapshot()`.
+///
+/// This replaces per-element XCUIElement queries (`.exists`, `.label`,
+/// `allElementsBoundByIndex`, `element(boundBy:)`). Those resolve every element again,
+/// and when one cannot be resolved — e.g. the remote map view in the location prompt — they
+/// record an XCTest failure instead of throwing, which ends `testHttpServer` and the runner.
+public enum SystemDialogMatcher {
+
+    /// All known iOS system permission dialogs.
+    public static let permissionDialogs: [SystemPermissionDialog] = [
+        // [Don't Allow (0), Allow (1)]
+        SystemPermissionDialog(key: "notifications", labelPattern: "Would Like to Send You Notifications",
+                               buttonMap: [.allow: 1, .deny: 0]),
+        // [Allow Once (0), Allow While Using App (1), Don't Allow (2)], preceded by the
+        // "Precise: On" map button(s) in the body.
+        SystemPermissionDialog(key: "location", labelPattern: "to use your location",
+                               actionButtonCount: 3,
+                               buttonMap: [.allow: 1, .always: 1, .inuse: 1, .deny: 2, .never: 2],
+                               preferredLabels: [
+                                   .allow: ["allow while using app"],
+                                   .always: ["allow while using app"],
+                                   .inuse: ["allow while using app"],
+                               ]),
+        // "Would Like to Use Bluetooth" or "to find Bluetooth devices"
+        SystemPermissionDialog(key: "bluetooth", labelPattern: "Bluetooth", buttonMap: [.allow: 1, .deny: 0]),
+        SystemPermissionDialog(key: "camera", labelPattern: "Would Like to Access the Camera", buttonMap: [.allow: 1, .deny: 0]),
+        SystemPermissionDialog(key: "microphone", labelPattern: "Would Like to Access the Microphone", buttonMap: [.allow: 1, .deny: 0]),
+        // [Don't Allow (0), Allow Limited Access (1), Allow Full Access (2)]
+        SystemPermissionDialog(key: "photos", labelPattern: "Would Like to Access Your Photos",
+                               actionButtonCount: 3,
+                               buttonMap: [.allow: 2, .limited: 1, .deny: 0]),
+        SystemPermissionDialog(key: "contacts", labelPattern: "Would Like to Access Your Contacts", buttonMap: [.allow: 1, .deny: 0]),
+        SystemPermissionDialog(key: "calendar", labelPattern: "Would Like to Access Your Calendar", buttonMap: [.allow: 1, .deny: 0]),
+        SystemPermissionDialog(key: "reminders", labelPattern: "Would Like to Access Your Reminders", buttonMap: [.allow: 1, .deny: 0]),
+        SystemPermissionDialog(key: "speech", labelPattern: "Would Like to Access Speech Recognition", buttonMap: [.allow: 1, .deny: 0]),
+        SystemPermissionDialog(key: "motion", labelPattern: "Would Like to Access Your Motion", buttonMap: [.allow: 1, .deny: 0]),
+        SystemPermissionDialog(key: "homekit", labelPattern: "Would Like to Access Your Home Data", buttonMap: [.allow: 1, .deny: 0]),
+        SystemPermissionDialog(key: "medialibrary", labelPattern: "Would Like to Access Apple Music", buttonMap: [.allow: 1, .deny: 0]),
+        SystemPermissionDialog(key: "siri", labelPattern: "Would Like to Access Siri", buttonMap: [.allow: 1, .deny: 0]),
+        // [Ask App Not to Track (0), Allow (1)]
+        SystemPermissionDialog(key: "usertracking", labelPattern: "Would Like Permission to Track", buttonMap: [.allow: 1, .deny: 0]),
+        // "Allow X to find devices on local networks?"
+        SystemPermissionDialog(key: "localnetwork", labelPattern: "local network", buttonMap: [.allow: 1, .deny: 0]),
+    ]
+
+    /// Permissions answered with "Allow" unless configured otherwise: these dialogs block the
+    /// whole UI and are rarely denied on purpose.
+    public static let autoAllowPermissions = ["bluetooth", "localnetwork"]
+
+    // MARK: - Tree helpers
+
+    /// Alerts and sheets in the tree, outermost first, in depth-first pre-order.
+    public static func dialogs(in root: AXElement) -> [AXElement] {
+        var result = [AXElement]()
+        func visit(_ element: AXElement) {
+            if element.elementType == ElementType.alert || element.elementType == ElementType.sheet {
+                result.append(element)
+                return
+            }
+            element.children?.forEach(visit)
+        }
+        visit(root)
+        return result
+    }
+
+    /// The dialog's own label followed by its static text labels (non-empty, in order).
+    public static func texts(of dialog: AXElement) -> [String] {
+        ([dialog.label] + dialog.descendants(ofType: ElementType.staticText).map(\.label))
+            .filter { !$0.isEmpty }
+    }
+
+    /// Buttons in the dialog, in depth-first pre-order (the order XCUIElement queries use).
+    public static func buttons(of dialog: AXElement) -> [AXElement] {
+        dialog.descendants(ofType: ElementType.button)
+    }
+
+    static func normalize(_ string: String) -> String {
+        string
+            .replacingOccurrences(of: "\u{2019}", with: "'")
+            .trimmingCharacters(in: .whitespacesAndNewlines)
+            .lowercased()
+    }
+
+    // MARK: - Apple Account alerts
+
+    /// The "Not Now"/"Cancel" button of an Apple Account password alert anywhere in `root`,
+    /// or nil when there is no such alert or it has no safe dismiss button.
+    public static func accountAlertDismissal(in root: AXElement) -> SystemDialogTap? {
+        for dialog in dialogs(in: root) where dialog.elementType == ElementType.alert {
+            guard AccountAlertMatcher.isAccountAlert(texts: texts(of: dialog)) else { continue }
+            let buttons = buttons(of: dialog)
+            guard let index = AccountAlertMatcher.dismissButtonIndex(buttonLabels: buttons.map(\.label)) else {
+                return nil
+            }
+            return SystemDialogTap(reason: .accountAlert, buttonLabel: buttons[index].label, frame: buttons[index].frame)
+        }
+        return nil
+    }
+
+    // MARK: - Permission dialogs
+
+    /// The permission dialog that matches `dialog`, by title first and static texts second.
+    /// Titles are tried first so a purpose string that mentions e.g. "Bluetooth" cannot
+    /// steer a location prompt to the bluetooth answer.
+    public static func matchingPermissionDialog(for dialog: AXElement) -> SystemPermissionDialog? {
+        let title = dialog.label.lowercased()
+        if let match = permissionDialogs.first(where: { title.contains($0.labelPattern.lowercased()) }) {
+            return match
+        }
+        let allText = texts(of: dialog).joined(separator: " ").lowercased()
+        return permissionDialogs.first { allText.contains($0.labelPattern.lowercased()) }
+    }
+
+    /// The button to tap for `value` in a dialog already known to be `definition`.
+    public static func button(for value: PermissionValue, definition: SystemPermissionDialog, in dialog: AXElement) -> AXElement? {
+        let buttons = buttons(of: dialog)
+        guard !buttons.isEmpty else { return nil }
+
+        if let wanted = definition.preferredLabels[value]?.map(normalize),
+           let byLabel = buttons.first(where: { wanted.contains(normalize($0.label)) }) {
+            return byLabel
+        }
+
+        guard let index = definition.buttonMap[value] else { return nil }
+        let actionButtons = buttons.count >= definition.actionButtonCount
+            ? Array(buttons.suffix(definition.actionButtonCount))
+            : buttons
+        return actionButtons.indices.contains(index) ? actionButtons[index] : nil
+    }
+
+    /// The button to tap to answer a permission dialog in `root` according to `permissions`,
+    /// or nil when nothing should be tapped.
+    public static func permissionTap(in root: AXElement, permissions: [String: PermissionValue]) -> SystemDialogTap? {
+        let dialogs = dialogs(in: root)
+        for dialog in dialogs {
+            guard let definition = matchingPermissionDialog(for: dialog),
+                  let value = permissions[definition.key],
+                  value != .unset, value != .unknown,
+                  let button = button(for: value, definition: definition, in: dialog) else {
+                continue
+            }
+            return SystemDialogTap(reason: .permission(key: definition.key, value: value),
+                                   buttonLabel: button.label, frame: button.frame)
+        }
+        // An alert/sheet we chose not to answer (e.g. location unset) must not be answered
+        // through the custom path because its purpose string mentions "Bluetooth".
+        guard dialogs.isEmpty else { return nil }
+        return customPermissionTap(in: root, permissions: permissions)
+    }
+
+    /// Bluetooth / Local Network prompts that are neither alerts nor sheets: a static text
+    /// mentioning the pattern plus an "Allow"/"Don't Allow" button somewhere in the tree.
+    /// Only the auto-allow permissions are handled this way, and only when the tree has no
+    /// alert or sheet.
+    static func customPermissionTap(in root: AXElement, permissions: [String: PermissionValue]) -> SystemDialogTap? {
+        let staticTexts = root.descendants(ofType: ElementType.staticText).map { $0.label.lowercased() }
+        guard !staticTexts.isEmpty else { return nil }
+        let buttons = root.descendants(ofType: ElementType.button)
+
+        for key in autoAllowPermissions {
+            guard let definition = permissionDialogs.first(where: { $0.key == key }),
+                  let value = permissions[key],
+                  value != .unset, value != .unknown else {
+                continue
+            }
+            let pattern = definition.labelPattern.lowercased()
+            guard staticTexts.contains(where: { $0.contains(pattern) }) else { continue }
+
+            let target: String
+            switch value {
+            case .deny, .never: target = "don't allow"
+            default: target = "allow"
+            }
+            guard let button = buttons.first(where: { normalize($0.label) == target }) else { continue }
+            return SystemDialogTap(reason: .customPermission(key: key, value: value),
+                                   buttonLabel: button.label, frame: button.frame)
+        }
+        return nil
+    }
+}

maestro-ios-xctest-runner/MaestroDriverLib/Sources/MaestroDriverLib/Models/ElementType.swift

@@ -2,6 +2,12 @@ import Foundation
 
 /// Element type constants matching XCUIElement.ElementType raw values
 public enum ElementType {
+    /// XCUIElement.ElementType.sheet.rawValue
+    public static let sheet: Int = 5
+    /// XCUIElement.ElementType.alert.rawValue
+    public static let alert: Int = 7
     /// XCUIElement.ElementType.button.rawValue
     public static let button: Int = 9
-}
+    /// XCUIElement.ElementType.staticText.rawValue
+    public static let staticText: Int = 48
+}

maestro-ios-xctest-runner/MaestroDriverLib/Tests/MaestroDriverLibTests/SystemDialogMatcherTests.swift

@@ -0,0 +1,140 @@
+import XCTest
+@testable import MaestroDriverLib
+
+final class SystemDialogMatcherTests: XCTestCase {
+
+    // MARK: - Fixtures
+
+    private func frame(_ y: Double) -> AXFrame {
+        ["X": 20, "Y": y, "Width": 300, "Height": 44]
+    }
+
+    private func button(_ label: String, y: Double, children: [AXElement]? = nil) -> AXElement {
+        AXElement(frame: frame(y), label: label, elementType: ElementType.button, children: children)
+    }
+
+    private func text(_ label: String) -> AXElement {
+        AXElement(label: label, elementType: ElementType.staticText)
+    }
+
+    /// The map / "Precise: On" part of the iOS location prompt. It is a remote view hosted by
+    /// CoreLocationVanillaWhenInUseAuthPromptPlugin and shows up twice in SpringBoard's snapshot.
+    private func mapSubtree(y: Double) -> [AXElement] {
+        [
+            AXElement(label: "Map", elementType: 43),
+            button("Precise: On", y: y, children: [text("Precise: On")]),
+        ]
+    }
+
+    /// SpringBoard tree rebuilt from the TestingBot session f3ffd402 hierarchy JSON.
+    private func locationPrompt(title: String = "Allow “FR8BOT” to use your location?") -> AXElement {
+        let body = AXElement(elementType: 46, children: [
+            AXElement(label: "Location@3x.png", elementType: 43),
+            text(title),
+            text("We use your location to track deliveries and Bluetooth beacons."),
+        ] + mapSubtree(y: 300) + mapSubtree(y: 300))
+        let actions = AXElement(elementType: 46, children: [
+            button("Allow Once", y: 500),
+            button("Allow While Using App", y: 550),
+            button("Don’t Allow", y: 600),
+        ])
+        let alert = AXElement(label: title, elementType: ElementType.alert, children: [body, actions])
+        let window = AXElement(elementType: 4, children: [alert])
+        return AXElement(label: " ", elementType: 2, children: [window])
+    }
+
+    private func twoButtonAlert(title: String, buttons labels: [String]) -> AXElement {
+        let alert = AXElement(label: title, elementType: ElementType.alert, children: [
+            text(title),
+        ] + labels.enumerated().map { button($0.element, y: 500 + Double($0.offset) * 50) })
+        return AXElement(elementType: 2, children: [AXElement(elementType: 4, children: [alert])])
+    }
+
+    // MARK: - Location prompt with map
+
+    func testLocationPromptAllowTapsAllowWhileUsingApp() {
+        let tap = SystemDialogMatcher.permissionTap(in: locationPrompt(), permissions: ["location": .allow])
+        XCTAssertEqual(tap?.buttonLabel, "Allow While Using App")
+        XCTAssertEqual(tap?.frame, frame(550))
+        XCTAssertEqual(tap?.reason, .permission(key: "location", value: .allow))
+    }
+
+    func testLocationPromptDenyTapsDontAllowDespiteMapButtons() {
+        let tap = SystemDialogMatcher.permissionTap(in: locationPrompt(), permissions: ["location": .deny])
+        XCTAssertEqual(tap?.buttonLabel, "Don’t Allow")
+    }
+
+    func testLocationPromptIndexFallbackSkipsDuplicatedPreciseButtons() {
+        // No preferred label for .inuse would still land on index 1 of the last 3 buttons.
+        let definition = SystemPermissionDialog(key: "location", labelPattern: "to use your location",
+                                                actionButtonCount: 3, buttonMap: [.inuse: 1])
+        let alert = SystemDialogMatcher.dialogs(in: locationPrompt()).first!
+        XCTAssertEqual(SystemDialogMatcher.buttons(of: alert).count, 5)
+        XCTAssertEqual(SystemDialogMatcher.button(for: .inuse, definition: definition, in: alert)?.label,
+                       "Allow While Using App")
+    }
+
+    func testLocationPromptUnsetIsLeftAlone() {
+        XCTAssertNil(SystemDialogMatcher.permissionTap(in: locationPrompt(), permissions: ["location": .unset]))
+        XCTAssertNil(SystemDialogMatcher.permissionTap(in: locationPrompt(), permissions: [:]))
+    }
+
+    func testPurposeStringMentioningBluetoothDoesNotSteerLocationPrompt() {
+        // Location unset but bluetooth auto-allowed: the purpose string says "Bluetooth", but the
+        // title is a location prompt, so nothing must be tapped.
+        let tap = SystemDialogMatcher.permissionTap(in: locationPrompt(), permissions: ["bluetooth": .allow])
+        XCTAssertNil(tap)
+    }
+
+    func testLocationPromptIsNotAnAccountAlert() {
+        XCTAssertNil(SystemDialogMatcher.accountAlertDismissal(in: locationPrompt()))
+    }
+
+    // MARK: - Other dialogs
+
+    func testNotificationsAllowAndDeny() {
+        let tree = twoButtonAlert(title: "“App” Would Like to Send You Notifications",
+                                  buttons: ["Don’t Allow", "Allow"])
+        XCTAssertEqual(SystemDialogMatcher.permissionTap(in: tree, permissions: ["notifications": .allow])?.buttonLabel, "Allow")
+        XCTAssertEqual(SystemDialogMatcher.permissionTap(in: tree, permissions: ["notifications": .deny])?.buttonLabel, "Don’t Allow")
+    }
+
+    func testPhotosLimited() {
+        let tree = twoButtonAlert(title: "“App” Would Like to Access Your Photos",
+                                  buttons: ["Don’t Allow", "Allow Limited Access", "Allow Full Access"])
+        XCTAssertEqual(SystemDialogMatcher.permissionTap(in: tree, permissions: ["photos": .limited])?.buttonLabel, "Allow Limited Access")
+        XCTAssertEqual(SystemDialogMatcher.permissionTap(in: tree, permissions: ["photos": .allow])?.buttonLabel, "Allow Full Access")
+    }
+
+    func testCustomLocalNetworkDialogWithoutAlert() {
+        let tree = AXElement(elementType: 2, children: [
+            text("“App” would like to find and connect to devices on your local network."),
+            button("Don’t Allow", y: 500),
+            button("Allow", y: 550),
+        ])
+        let tap = SystemDialogMatcher.permissionTap(in: tree, permissions: ["localnetwork": .allow])
+        XCTAssertEqual(tap?.buttonLabel, "Allow")
+        XCTAssertEqual(tap?.reason, .customPermission(key: "localnetwork", value: .allow))
+    }
+
+    func testNoDialogNoTap() {
+        let tree = AXElement(elementType: 2, children: [text("Home"), button("Settings", y: 10)])
+        XCTAssertNil(SystemDialogMatcher.permissionTap(in: tree, permissions: ["location": .allow, "bluetooth": .allow]))
+        XCTAssertNil(SystemDialogMatcher.accountAlertDismissal(in: tree))
+    }
+
+    // MARK: - Apple Account alert
+
+    func testAccountAlertDismissedWithNotNow() {
+        let tree = twoButtonAlert(title: "Apple Account Verification", buttons: ["Settings", "Not Now"])
+        let tap = SystemDialogMatcher.accountAlertDismissal(in: tree)
+        XCTAssertEqual(tap?.buttonLabel, "Not Now")
+        XCTAssertEqual(tap?.frame, frame(550))
+        XCTAssertEqual(tap?.reason, .accountAlert)
+    }
+
+    func testAccountAlertWithoutDismissButtonIsLeftAlone() {
+        let tree = twoButtonAlert(title: "Apple Account Verification", buttons: ["Settings"])
+        XCTAssertNil(SystemDialogMatcher.accountAlertDismissal(in: tree))
+    }
+}