CI: GitHub-hosted fallback while the mini is offline

.github/workflows/ci.yml

@@ -17,27 +17,38 @@ concurrency:
   group: ci-${{ github.event.pull_request.number || github.ref }}
   cancel-in-progress: true
 
+# Hosted fallback for while the mini's runners are unavailable: set the repo
+# variable KEYPATH_CI_RUNNER=hosted and both jobs run on GitHub's macOS 27 /
+# Xcode 27 image (`xcode-27`) instead. Delete the variable to return to the
+# mini. Job names, and so the required checks, are the same either way.
+#   gh variable set KEYPATH_CI_RUNNER --repo malpern/KeyPath --body hosted
+#   gh variable delete KEYPATH_CI_RUNNER --repo malpern/KeyPath
+# Only ci.yml has the fallback; coverage, cache-warm and runner-health are
+# scheduled jobs that stay on the mini and don't gate PRs.
+
 jobs:
   build-and-test:
     # macOS 27 AppSSO trapped SwiftPM binary-artifact requests from a headless
     # LaunchDaemon (docs: 2026-07-16-headless-swiftpm-appsso-trap.md). Since
     # 2026-10-19 the swiftpm-safe runner is a daemon too, because clawd has no GUI
     # session; the persistent scratch below means Sparkle is fetched only on a
     # cold scratch, not on every job.
-    runs-on: [self-hosted, macOS, keypath, swiftpm-safe]
+    runs-on: ${{ fromJSON(vars.KEYPATH_CI_RUNNER == 'hosted' && '"xcode-27"' || '["self-hosted", "macOS", "keypath", "swiftpm-safe"]') }}
     # Warm runs normally finish in 2-3 minutes. A measured cold run spent about
     # 3 minutes on setup/Kanata and was still compiling Swift tests after another
     # 12 minutes, before test execution. Keep a finite bound with real headroom.
-    timeout-minutes: 25
+    # Every hosted run is cold, on a smaller machine.
+    timeout-minutes: ${{ vars.KEYPATH_CI_RUNNER == 'hosted' && 60 || 25 }}
     # Separate from the workflow-level concurrency group above (which cancels
-    # stale runs of the *same* PR): this queues jobs from *different* PRs onto
-    # the single physical runner instead of letting them execute in parallel.
+    # stale runs of the *same* PR): on the mini this queues jobs from *different*
+    # PRs onto the single physical runner instead of letting them execute in
+    # parallel. Hosted jobs each get their own VM, so their group is unique.
     # Concurrent swift test runs share unsandboxed local state (e.g.
     # RuleCollectionsManager's config file), which caused spurious
     # CLIPackCRUDTests "could not enable associated rule collection" failures
     # when two PRs' CI happened to overlap.
     concurrency:
-      group: keypath-self-hosted-runner
+      group: ${{ vars.KEYPATH_CI_RUNNER == 'hosted' && format('hosted-{0}-{1}-{2}', github.workflow, github.run_id, github.job) || 'keypath-self-hosted-runner' }}
       cancel-in-progress: false
 
     steps:
@@ -52,9 +63,29 @@ jobs:
         submodules: true
         persist-credentials: false
 
+    - name: Select Xcode 27.0 (hosted runner)
+      # GitHub's image defaults to a newer Xcode; its Xcode 27.0 is build 27A266a,
+      # the one the Macs use. The stable-Xcode path keeps the scripts' version
+      # check, and DEVELOPER_DIR covers tools that ask xcode-select.
+      if: vars.KEYPATH_CI_RUNNER == 'hosted'
+      run: |
+        xcode=/Applications/Xcode_27.app/Contents/Developer
+        version=$("$xcode/usr/bin/xcodebuild" -version | sed -n 's/^Xcode //p' | head -1)
+        if [[ "$version" != "27.0" ]]; then
+          echo "::error::$xcode is Xcode '$version', not 27.0; the image changed"
+          exit 1
+        fi
+        echo "KEYPATH_STABLE_XCODE_DEVELOPER_DIR=$xcode" >> "$GITHUB_ENV"
+        echo "DEVELOPER_DIR=$xcode" >> "$GITHUB_ENV"
+        # cargo reaches the compiler through cc, which needs the SDK named.
+        echo "SDKROOT=$xcode/Platforms/MacOSX.platform/Developer/SDKs/MacOSX.sdk" >> "$GITHUB_ENV"
+
     - name: Verify runner disk reserve
       env:
-        KEYPATH_MIN_FREE_DISK_GIB: "40"
+        # A hosted VM has far less disk than the mini. 12 GiB is a floor, not a
+        # measurement of what a cold run uses; raise it once a hosted run has
+        # shown the real number.
+        KEYPATH_MIN_FREE_DISK_GIB: ${{ vars.KEYPATH_CI_RUNNER == 'hosted' && '12' || '40' }}
       run: ./Scripts/lab/host-disk-reserve
 
     - name: Ensure Metal toolchain
@@ -111,17 +142,13 @@ jobs:
         kanata --version
         kanata-simulator --version
 
-    - name: Lint WizardAutoFixer for forbidden subprocess usage
-      run: |
-        chmod +x ./Scripts/lint-no-subprocess-in-autofixer.sh
-        ./Scripts/lint-no-subprocess-in-autofixer.sh
-
     - name: Prepare persistent SwiftPM scratch
-      # Persistent per runner: runner.workspace (_work/KeyPath) survives between
-      # jobs, while runner.temp is wiped after every job and made each build cold.
-      # One runner runs one job at a time, so jobs never share this directory
-      # concurrently. It is cleared when the toolchain changes or it grows past
-      # the cap; the 40 GiB host disk reserve above still applies.
+      # Persistent per runner on the mini: runner.workspace (_work/KeyPath)
+      # survives between jobs, while runner.temp is wiped after every job and
+      # made each build cold. One runner runs one job at a time, so jobs never
+      # share this directory concurrently. It is cleared when the toolchain
+      # changes or it grows past the cap. A hosted VM starts empty every time,
+      # so there it is always cold.
       env:
         KEYPATH_SCRATCH_CAP_GIB: "30"
       run: |
@@ -153,7 +180,8 @@ jobs:
         KEYPATH_BUNDLED_SIMULATOR_OVERRIDE: /opt/homebrew/bin/kanata-simulator
         # SwiftPM can wrap XCTest post-run SIGABRT as exit 1 on the self-hosted runner;
         # run-tests-safe.sh still fails if it parses any test failures and emits a warning when this path is used.
-        KEYPATH_ALLOW_TEST_RUNNER_CRASH_SUCCESS: "1"
+        # Not on hosted runners, where that quirk hasn't been seen.
+        KEYPATH_ALLOW_TEST_RUNNER_CRASH_SUCCESS: ${{ vars.KEYPATH_CI_RUNNER == 'hosted' && '0' || '1' }}
         KEYPATH_TEST_ENFORCE_CLEAN_SUMMARY: "1"
         KEYPATH_TEST_RESET_MODULE_CACHE: "0"
         KEYPATH_SWIFT_BUILD_JOBS: "4"
@@ -233,13 +261,14 @@ jobs:
         echo "- Narrow coverage enforcement moved to the scheduled/manual \`KeyPath Coverage\` workflow." >> $GITHUB_STEP_SUMMARY
 
   code-quality:
-    runs-on: [self-hosted, macOS, keypath]
-    timeout-minutes: 15
+    runs-on: ${{ fromJSON(vars.KEYPATH_CI_RUNNER == 'hosted' && '"xcode-27"' || '["self-hosted", "macOS", "keypath"]') }}
+    # Hosted runs also install mise and SwiftLint on a smaller machine.
+    timeout-minutes: ${{ vars.KEYPATH_CI_RUNNER == 'hosted' && 30 || 15 }}
     # This job does not share test state, but it does share the only physical
     # runner. Queue it with builds so SwiftFormat gets enough CPU to finish
     # before its fixed per-rule safety timeout.
     concurrency:
-      group: keypath-self-hosted-runner
+      group: ${{ vars.KEYPATH_CI_RUNNER == 'hosted' && format('hosted-{0}-{1}-{2}', github.workflow, github.run_id, github.job) || 'keypath-self-hosted-runner' }}
       cancel-in-progress: false
 
     steps:
@@ -257,6 +286,31 @@ jobs:
     - name: Add Homebrew to PATH
       run: echo "/opt/homebrew/bin" >> $GITHUB_PATH
 
+    - name: Select Xcode 27.0 (hosted runner)
+      # GitHub's image defaults to a newer Xcode; its Xcode 27.0 is build 27A266a,
+      # the one the Macs use. The stable-Xcode path keeps the scripts' version
+      # check, and DEVELOPER_DIR covers tools that ask xcode-select.
+      if: vars.KEYPATH_CI_RUNNER == 'hosted'
+      run: |
+        xcode=/Applications/Xcode_27.app/Contents/Developer
+        version=$("$xcode/usr/bin/xcodebuild" -version | sed -n 's/^Xcode //p' | head -1)
+        if [[ "$version" != "27.0" ]]; then
+          echo "::error::$xcode is Xcode '$version', not 27.0; the image changed"
+          exit 1
+        fi
+        echo "KEYPATH_STABLE_XCODE_DEVELOPER_DIR=$xcode" >> "$GITHUB_ENV"
+        echo "DEVELOPER_DIR=$xcode" >> "$GITHUB_ENV"
+        # cargo reaches the compiler through cc, which needs the SDK named.
+        echo "SDKROOT=$xcode/Platforms/MacOSX.platform/Developer/SDKs/MacOSX.sdk" >> "$GITHUB_ENV"
+
+    - name: Install mise and SwiftLint (hosted runner)
+      # The mini has both; GitHub's image has neither. mise ignores a repo config
+      # it hasn't been told to trust, which would skip the SwiftFormat pin.
+      if: vars.KEYPATH_CI_RUNNER == 'hosted'
+      run: |
+        brew install mise swiftlint ripgrep
+        mise trust mise.toml
+
     - name: Install pinned SwiftFormat
       run: mise install swiftformat
 

AGENTS.md

@@ -130,6 +130,11 @@ Notes:
 
 ### PR Workflow & Git Safety
 
+- CI normally runs on the mini's self-hosted runners. While they are
+  unavailable, the repo variable `KEYPATH_CI_RUNNER=hosted` moves both required
+  jobs to GitHub's `xcode-27` image (see the header of `.github/workflows/ci.yml`).
+  Delete the variable when the mini is back.
+
 - Follow the invariants in `docs/process/agent-pr-invariants.md`. Step-by-step
   reference: `docs/process/agent-pr-workflow.md`.
 - After merging a PR, always pull `master` and deploy from `master` before

Scripts/lint-no-sleep.sh

@@ -1,19 +1,36 @@
 #!/usr/bin/env bash
 set -euo pipefail
 
+# A missing rg exits 127, which would otherwise read as "no hits".
+command -v rg >/dev/null || { echo "❌ ripgrep (rg) is required" >&2; exit 1; }
+
 ROOT="$(cd -- "$(dirname -- "$0")/.." && pwd)"
+HITS="$(mktemp)"
+trap 'rm -f "$HITS"' EXIT
+
+# rg exits 0 on a match, 1 on none, and 2 on an error such as a missing path.
+# Only 1 is a pass.
+search() {
+  local rc=0
+  rg "$@" >"$HITS" || rc=$?
+  if (( rc > 1 )); then
+    echo "❌ rg failed (exit $rc)" >&2
+    exit 1
+  fi
+  return "$rc"
+}
 
 # Disallow Task.sleep in wizard code and Swift tests.
-if rg "Task\\.sleep" "$ROOT/Sources/KeyPathAppKit/InstallationWizard" "$ROOT/Sources/KeyPathWizardCore" >/tmp/lint_no_sleep_hits 2>/dev/null; then
+if search "Task\\.sleep" "$ROOT/Sources/KeyPathAppKit/InstallationWizard" "$ROOT/Sources/KeyPathWizardCore"; then
   echo "❌ Task.sleep found in wizard sources:"
-  cat /tmp/lint_no_sleep_hits
+  cat "$HITS"
   echo "Replace with readiness polling / health checks."
   exit 1
 fi
 
-if rg "Task\\.sleep|Thread\\.sleep|usleep" "$ROOT/Tests" -g '*.swift' >/tmp/lint_no_sleep_hits 2>/dev/null; then
+if search "Task\\.sleep|Thread\\.sleep|usleep" "$ROOT/Tests" -g '*.swift'; then
   echo "❌ Real sleeps found in Swift tests:"
-  cat /tmp/lint_no_sleep_hits
+  cat "$HITS"
   echo "Use injected clocks, expectations, Task.yield(), or DEBUG-only drains instead."
   exit 1
 fi

Scripts/test-ci-locally.sh

@@ -3,10 +3,6 @@ set -euo pipefail
 
 # Local runner to approximate CI steps
 
-echo "🔍 Lint: WizardAutoFixer subprocess guard"
-chmod +x ./Scripts/lint-no-subprocess-in-autofixer.sh
-./Scripts/lint-no-subprocess-in-autofixer.sh
-
 echo "🔍 Lint: Wizard sleep guard"
 chmod +x ./Scripts/lint-no-sleep.sh
 ./Scripts/lint-no-sleep.sh

Tests/KeyPathTests/Lint/DeploymentScriptContractTests.swift

@@ -22,7 +22,7 @@
             "Scripts/run-tests-safe.sh",
             "Scripts/quick-deploy.sh",
             "Scripts/build-and-sign.sh",
             "Scripts/release-doctor.sh",
         ]
 
         XCTAssertTrue(xcodeContract.contains("Xcode-27.app/Contents/Developer"))
@@ -250,11 +250,18 @@
         let installer = try contents(of: root.appendingPathComponent("Scripts/install-ci-runner-service.sh"))
         let safeRunner = "runs-on: [self-hosted, macOS, keypath, swiftpm-safe]"
 
+        // ci.yml selects its runner with an expression so a repo variable can move
+        // it to GitHub-hosted runners; the self-hosted branch keeps the safe label.
+        let safeRunnerChoice = #"'["self-hosted", "macOS", "keypath", "swiftpm-safe"]'"#
         XCTAssertEqual(
-            ci.components(separatedBy: safeRunner).count - 1,
+            ci.components(separatedBy: safeRunnerChoice).count - 1,
             1,
             "Only build-and-test should require the SwiftPM-safe interactive runner in ci.yml."
         )
+        XCTAssertEqual(ci.components(separatedBy: #"vars.KEYPATH_CI_RUNNER == 'hosted' && '"xcode-27"'"#).count - 1, 2,
+                       "Both required jobs move to the hosted image together.")
+        XCTAssertFalse(ci.contains(safeRunner),
+                       "A literal safe-runner job in ci.yml would bypass the single-job check above.")
         XCTAssertTrue(cacheWarm.contains(safeRunner))
         XCTAssertEqual(
             coverage.components(separatedBy: safeRunner).count - 1,

Tests/KeyPathTests/Services/KarabinerConflictServiceTests.swift

@@ -10,8 +10,8 @@ struct KarabinerConflictServiceTests {
     @Test("Karabiner grabber detection uses injected SystemStateProvider")
     func karabinerGrabberDetectionUsesInjectedSystemStateProvider() async {
         TestSingletonReset.resetAll()
-        let runner = SubprocessRunnerFake.shared
-        await runner.reset()
+        // Its own fake: a parallel suite resetting `shared` would clear the command log.
+        let runner = SubprocessRunnerFake()
         await runner.configurePgrepResult { pattern in
             pattern == "karabiner_grabber" ? [1111] : []
         }
@@ -32,8 +32,8 @@ struct KarabinerConflictServiceTests {
     @Test("Karabiner 16 installed without grabber is not a conflict")
     func karabiner16WithoutGrabberIsNotAConflict() async {
         TestSingletonReset.resetAll()
-        let runner = SubprocessRunnerFake.shared
-        await runner.reset()
+        // Its own fake: a parallel suite resetting `shared` would clear the command log.
+        let runner = SubprocessRunnerFake()
         await runner.configurePgrepResult { pattern in
             // Modern Karabiner releases may run harmless supporting agents,
             // but only the input grabber competes with Kanata for exclusive
@@ -63,8 +63,8 @@ struct KarabinerConflictServiceTests {
             Date().addingTimeInterval(-60),
             serviceID: "com.keypath.karabiner-vhiddaemon"
         )
-        let runner = SubprocessRunnerFake.shared
-        await runner.reset()
+        // Its own fake: a parallel suite resetting `shared` would clear the command log.
+        let runner = SubprocessRunnerFake()
         await runner.configurePgrepResult { pattern in
             pattern == "VirtualHIDDevice-Daemon" ? [2222] : []
         }
@@ -85,8 +85,8 @@ struct KarabinerConflictServiceTests {
     @Test("Stopped-process verification uses injected SystemStateProvider")
     func stoppedProcessVerificationUsesInjectedSystemStateProvider() async {
         TestSingletonReset.resetAll()
-        let runner = SubprocessRunnerFake.shared
-        await runner.reset()
+        // Its own fake: a parallel suite resetting `shared` would clear the command log.
+        let runner = SubprocessRunnerFake()
         await runner.configurePgrepResult { pattern in
             pattern == "Karabiner-DriverKit-VirtualHIDDevice" ? [3333] : []
         }

Tests/KeyPathTests/TestHelpers/SubprocessRunnerFake.swift

@@ -49,7 +49,10 @@ actor SubprocessRunnerFake: SubprocessRunning {
 
     // MARK: - Initialization
 
-    private init() {}
+    /// Tests that inject the fake (through `systemProbeClient()`) should create
+    /// their own: suites run in parallel, and another suite's `reset()` on
+    /// `shared` clears `executedCommands` mid-test.
+    init() {}
 
     // MARK: - Reset