diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 07b91e6059..d6b5dde70e 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -44,6 +44,12 @@ on: - ".github/workflows/enforce-pr-target.yml" - ".github/workflows/stale-needs-info.yml" workflow_dispatch: + inputs: + lane: + description: "all (default) or macos-control" + type: choice + default: all + options: [all, macos-control] permissions: contents: read @@ -442,22 +448,109 @@ jobs: - name: CLI help smoke run: bun run src/cli/index.ts help - # macOS runs on every pull request, and runs the WHOLE suite unsharded. - # - # That is the point of it. The four Linux shards each cover a quarter of the - # files, which quietly assumes no test depends on a sibling file having run in - # the same process pool. This leg is the control that would notice if that - # assumption ever broke. It is also the cheapest leg on the board — 5m23s on - # the baseline run, faster than the ubuntu leg it sits beside — so there was - # never a latency argument for touching it. - # - # It does not repeat the gates: typecheck, privacy, lint, and build are - # platform-independent and already ran once above. platform-macos: - name: macos + name: macos ${{ matrix.shard }}/2 needs: changes if: github.event_name != 'pull_request' || needs.changes.outputs.ci == 'true' runs-on: macos-latest + # Two shards. Unsharded, this job was the critical path on every green dev + # push (mean 14.9 min against a 4.7 min Linux maximum; devlog + # 260905_test_modularization_and_windows/003). Two halves finish in ~7.7 and + # cost 0.6 extra macOS minutes of setup per run. The whole-pool control that + # the single job used to provide lives in macos-control below, on dispatch. + timeout-minutes: 20 + strategy: + fail-fast: false + matrix: + shard: [1, 2] + steps: + - name: Checkout + uses: actions/checkout@9c091bb21b7c1c1d1991bb908d89e4e9dddfe3e0 # v7 + with: + # No job here pushes, and the self-hosted box keeps its checkout + # between jobs, so leaving a usable token in .git/config is avoidable + # residue. Matches the convention already used by the other workflows. + persist-credentials: false + # tests/release-version-line.test.ts compares package.json against the + # newest release tag. actions/checkout fetches no tags by default, so + # without this the check reads an empty tag set and passes on anything - + # the exact regression it exists to catch would ride through CI green. + # + # Tags only, not full history: `fetch-depth: 0` would clone every commit to + # answer a question about refs. A shallow fetch still brings each tag and its + # target commit, which is all the check reads - the tag list, and whether the + # newest tag names HEAD. That second read only happens on a release commit, + # where the tag points at HEAD and the commit is present by definition. + fetch-tags: true + + - name: Setup project Bun + uses: ./.github/actions/setup-project-bun + + - name: Install dependencies + run: | + bun install --frozen-lockfile + cd gui + bun install --frozen-lockfile + + # Same reason as the shards: the suite serves gui/dist and reads it back. + - name: Build GUI + run: | + cd gui + bun run build + + # Bun 1.3.14 segfaults while reclaiming a Worker at an `--isolate` file + # boundary: the header shows BALANCED `workers_spawned(N) + # workers_terminated(N)` and the process dies with exit 133 after the last + # assertion in the file already passed. It landed on + # `storage-worker-lifecycle` and `server-background-lifecycle` — the two + # files that tear down a still-busy policy Worker — and accounted for most + # of this leg's red runs on `dev` while every failing SHA passed on rerun. + # + # This is a runtime crash, not a test result, and the Linux shards already + # retry exactly this class through `scripts/ci/run-bun-test-batches.sh` + # (`is_bun_runtime_crash`). The unsharded macOS control had no equivalent, + # so the same crash that Linux absorbs failed the whole promotion here. + # + # Keep the signature list in sync with `is_bun_runtime_crash` in + # scripts/ci/run-bun-test-batches.sh. An assertion failure still fails on + # the first attempt — only the crash signature is retried, exactly once. + - name: Test + run: | + # GitHub Actions starts bash `run:` blocks with `-e`. Disable + # errexit so a Bun crash reaches PIPESTATUS and the bounded retry. + set +e + set -uo pipefail + suite_log="$(mktemp -t ocx-macos-suite.XXXXXX)" + for attempt in 1 2; do + # --timeout: Bun's default 5s per-test ceiling is the recurring flake + # class on this loaded shared runner (real retry windows + server + # round-trips exceed 5s under contention; a 10s-floor in-test + # watchdog fired at 10.16s there). 60s keeps hangs bounded (the 30m + # job timeout is the outer backstop) while removing the timing + # flakes — assertions are untouched. Pairs with the 30s CI floor in + # tests/helpers/ci-watchdog.ts. + bun test --isolate --timeout 60000 tests --shard=${{ matrix.shard }}/2 2>&1 | tee "$suite_log" + suite_status="${PIPESTATUS[0]}" + if [ "$suite_status" -eq 0 ]; then + exit 0 + fi + if ! grep -Eqi 'oh no: Bun has crashed|Internal assertion failure|Segmentation fault at address|Illegal instruction|Bus error|Aborted \(core dumped\)' "$suite_log"; then + echo "::error::macOS suite failed on attempt ${attempt} (exit ${suite_status}); assertion failures are not retried." + exit "$suite_status" + fi + echo "::warning::Bun runtime crash in the macOS suite (exit ${suite_status}, attempt ${attempt})." + done + echo "::error::Bun runtime crash repeated on the macOS suite; failing after one retry." + exit 1 + + - name: CLI help smoke + run: bun run src/cli/index.ts help + + macos-control: + name: macos control + needs: changes + if: github.event_name == 'workflow_dispatch' + runs-on: macos-latest # The unsharded control for the sharded Linux lane: the only place the whole # suite runs in one pool, so it is the place that catches what sharding # hides. The flakes it keeps surfacing are timing, not logic, and the fix @@ -566,7 +659,7 @@ jobs: name: windows ${{ matrix.shard }}/4 needs: select-windows-runner if: >- - github.event_name == 'workflow_dispatch' + github.event_name == 'workflow_dispatch' && (github.event.inputs.lane == '' || github.event.inputs.lane == 'all') runs-on: ${{ fromJSON(needs.select-windows-runner.outputs.runner) }} # Sharded like the Linux legs. The single-leg run reached 30 minutes on a # green suite and was killed in cleanup; four shards put each leg inside the @@ -811,7 +904,7 @@ jobs: # direct dependencies only, so a failing `select-windows-runner` would # otherwise reach this gate as nothing at all while its dependents report # `skipped` — which the gate is required to read as a deliberate skip. - needs: [changes, select-windows-runner, test, storage-policy, api-usage, gates, platform-macos, platform-windows, keyring-smoke, npm-global-smoke] + needs: [changes, select-windows-runner, test, storage-policy, api-usage, gates, platform-macos, macos-control, platform-windows, keyring-smoke, npm-global-smoke] runs-on: ubuntu-latest timeout-minutes: 5 steps: diff --git a/tests/ci-workflows.test.ts b/tests/ci-workflows.test.ts index bbe5a5f8a0..cf85e49189 100644 --- a/tests/ci-workflows.test.ts +++ b/tests/ci-workflows.test.ts @@ -90,6 +90,18 @@ describe("GitHub Actions hardening", () => { const ci = Bun.YAML.parse(workflow) as { permissions?: Record; jobs?: Record; + on?: { + workflow_dispatch?: { + inputs?: { + lane?: { + description?: string; + type?: string; + default?: string; + options?: string[]; + }; + }; + }; + }; }; // Job-scoped: a global count still passes if values are swapped between jobs. @@ -100,7 +112,8 @@ describe("GitHub Actions hardening", () => { expect(ci.jobs?.["select-windows-runner"]?.["timeout-minutes"]).toBe(2); expect(ci.jobs?.test?.["timeout-minutes"]).toBe(15); expect(ci.jobs?.gates?.["timeout-minutes"]).toBe(15); - expect(ci.jobs?.["platform-macos"]?.["timeout-minutes"]).toBe(30); + expect(ci.jobs?.["platform-macos"]?.["timeout-minutes"]).toBe(20); + expect(ci.jobs?.["macos-control"]?.["timeout-minutes"]).toBe(30); // Higher than the Linux shards on purpose: at 15 the Windows leg cancelled a // shard mid-suite, which reports as neither pass nor fail (#2152). expect(ci.jobs?.["platform-windows"]?.["timeout-minutes"]).toBe(25); @@ -166,7 +179,7 @@ describe("GitHub Actions hardening", () => { // how the first cut of that test shipped, so pin the flag rather than trusting a // comment. Asserted per job so a future edit cannot drop it from one leg while // the other still carries it. - for (const jobName of ["test", "platform-macos", "platform-windows"]) { + for (const jobName of ["test", "platform-macos", "macos-control", "platform-windows"]) { const steps = (ci.jobs?.[jobName] as { steps?: Array<{ uses?: string; with?: Record }> })?.steps ?? []; const checkout = steps.find(step => typeof step.uses === "string" && step.uses.includes("actions/checkout")); expect(`${jobName}:${String(checkout?.with?.["fetch-tags"])}`).toBe(`${jobName}:true`); @@ -219,8 +232,12 @@ describe("GitHub Actions hardening", () => { const macosSteps = (ci.jobs?.["platform-macos"] as { steps?: { run?: string }[] })?.steps ?? []; // The 60s per-test ceiling is part of the pinned shape: dropping it silently // restores the timing-flake class this lane kept surfacing. - expect(macosSteps.some(step => step.run?.includes("bun test --isolate --timeout 60000 tests"))).toBe(true); - expect(macosSteps.some(step => step.run?.includes("--shard"))).toBe(false); + expect(macosSteps.some(step => step.run?.includes("bun test --isolate --timeout 60000 tests --shard=${{ matrix.shard }}/2"))).toBe(true); + const macosShards = (ci.jobs?.["platform-macos"] as { + strategy?: { "fail-fast"?: boolean; matrix?: { shard?: number[] } }; + })?.strategy; + expect(macosShards?.["fail-fast"]).toBe(false); + expect(macosShards?.matrix?.shard).toEqual([1, 2]); // The macOS leg retries ONLY a Bun runtime crash, and only once. Bun 1.3.14 // segfaults reclaiming a Worker at an `--isolate` file boundary with @@ -229,7 +246,7 @@ describe("GitHub Actions hardening", () => { // `scripts/ci/run-bun-test-batches.sh`. Two ways to break this silently: // drop the crash-signature guard so an assertion failure gets retried into // green, or let the retry loop swallow a repeated crash. Pin both. - const macosTestRun = macosSteps.find(step => step.run?.includes("bun test --isolate --timeout 60000 tests"))?.run ?? ""; + const macosTestRun = macosSteps.find(step => step.run?.includes("bun test --isolate --timeout 60000 tests --shard=${{ matrix.shard }}/2"))?.run ?? ""; // Actions invokes multiline `run:` blocks with `bash -e`. The retry loop // must disable errexit before the crash-prone command or exit 133 aborts // the step before PIPESTATUS can be inspected and the retry can run. @@ -245,6 +262,33 @@ describe("GitHub Actions hardening", () => { expect((ci.jobs?.["platform-macos"] as { if?: string })?.if) .toBe("github.event_name != 'pull_request' || needs.changes.outputs.ci == 'true'"); + // Whole-pool control lives on dispatch so every push does not pay the + // unsharded macOS critical path. Keep the unsharded bun test line and the + // 30-minute budget; do not sneak a shard divisor into this job. + const macosControlJob = ci.jobs?.["macos-control"] as { + name?: string; + needs?: string; + if?: string; + "runs-on"?: string; + "timeout-minutes"?: number; + strategy?: unknown; + steps?: { run?: string }[]; + } | undefined; + expect(macosControlJob?.name).toBe("macos control"); + expect(macosControlJob?.needs).toBe("changes"); + expect(macosControlJob?.if).toBe("github.event_name == 'workflow_dispatch'"); + expect(macosControlJob?.["runs-on"]).toBe("macos-latest"); + expect(macosControlJob?.strategy).toBeUndefined(); + const macosControlSteps = macosControlJob?.steps ?? []; + expect(macosControlSteps.some(step => step.run?.includes("bun test --isolate --timeout 60000 tests"))).toBe(true); + expect(macosControlSteps.some(step => step.run?.includes("--shard"))).toBe(false); + const macosControlTestRun = macosControlSteps.find(step => step.run?.includes("bun test --isolate --timeout 60000 tests"))?.run ?? ""; + expect(hasExactShellCommand(macosControlTestRun, "set +e")).toBe(true); + expect(macosControlTestRun).toContain("for attempt in 1 2"); + expect(macosControlTestRun).not.toContain("while true"); + expect(macosControlTestRun).toContain("assertion failures are not retried"); + expect(macosControlTestRun).toContain("failing after one retry"); + // Windows is dispatch-only: it gates nothing, not even the shipping // boundary. The sharded promotion run surfaced ~207 Windows-only failures // that pre-date every released version, so the leg became a measurement @@ -252,12 +296,24 @@ describe("GitHub Actions hardening", () => { // condition and the absence of every automatic trigger — a stray // `|| github.ref == ...` would restore a red leg to the release path. const windowsIf = String((ci.jobs?.["platform-windows"] as { if?: string })?.if ?? ""); - expect(windowsIf).toContain("github.event_name == 'workflow_dispatch'"); + expect(windowsIf).toBe( + "github.event_name == 'workflow_dispatch' && (github.event.inputs.lane == '' || github.event.inputs.lane == 'all')", + ); expect(windowsIf).not.toContain("refs/heads/main"); expect(windowsIf).not.toContain("refs/heads/preview"); expect(windowsIf).not.toContain("refs/heads/dev"); expect(windowsIf).not.toContain("pull_request"); + // A lane=macos-control dispatch must skip Windows so a red Windows burn-down + // cannot fail the unsharded macOS control run. A plain dispatch still runs + // everything, including Windows, which is what empty-or-all encodes. + expect(ci.on?.workflow_dispatch?.inputs?.lane).toEqual({ + description: "all (default) or macos-control", + type: "choice", + default: "all", + options: ["all", "macos-control"], + }); + // Windows runs the same suite, sharded like the Linux legs, and keeps the // self-hosted workspace wipe. Without the wipe a deleted file survives on // the runner's disk and the suite passes against a tree that no longer @@ -299,11 +355,13 @@ describe("GitHub Actions hardening", () => { const batchScript = await readText("scripts/ci/run-bun-test-batches.sh"); for (const signature of crashSignatures) { expect(`macos:${signature}:${macosTestRun.includes(signature)}`).toBe(`macos:${signature}:true`); + expect(`macos-control:${signature}:${macosControlTestRun.includes(signature)}`).toBe(`macos-control:${signature}:true`); expect(`windows:${signature}:${windowsTestRun.includes(signature)}`).toBe(`windows:${signature}:true`); expect(`script:${signature}:${batchScript.includes(signature)}`).toBe(`script:${signature}:true`); } // The thread-numbered form must not be the anchor anywhere. expect(macosTestRun).not.toContain("panic\\(thread"); + expect(macosControlTestRun).not.toContain("panic\\(thread"); expect(windowsTestRun).not.toContain("panic\\(thread"); expect(batchScript).not.toContain("panic\\(thread"); @@ -321,7 +379,7 @@ describe("GitHub Actions hardening", () => { // accident, because the same job also ran the GUI build — splitting the suite // away from the gates removed that coincidence, and the shards went red on a // pull request before this pin existed. - for (const jobName of ["test", "platform-macos", "platform-windows"]) { + for (const jobName of ["test", "platform-macos", "macos-control", "platform-windows"]) { const steps = (ci.jobs?.[jobName] as { steps?: { if?: string; run?: string }[] })?.steps ?? []; const build = steps.find(step => step.run?.includes("bun run build")); expect(`${jobName}:${build === undefined}`).toBe(`${jobName}:false`); @@ -493,6 +551,9 @@ describe("GitHub Actions hardening", () => { expect(`${jobName}:${job?.needs}`).toBe(`${jobName}:changes`); expect(`${jobName}:${job?.if}`).toBe(`${jobName}:${scopedCondition}`); } + const macosControlIf = ci.jobs?.["macos-control"] as { needs?: string; if?: string } | undefined; + expect(macosControlIf?.needs).toBe("changes"); + expect(macosControlIf?.if).toBe("github.event_name == 'workflow_dispatch'"); }); test("cross-platform CI keeps the GUI lint and build gates", async () => {