From af2c6836ab2051b1e233663c7ed997099197f222 Mon Sep 17 00:00:00 2001 From: Anton Wiklund Date: Tue, 20 Oct 2026 14:17:28 +0200 Subject: [PATCH 1/4] ci(interop): skip a release commit, without canceling its parent's run A release commit changes only the version's files, but it matches Interop's paths (package.json, package-lock.json), so Interop ran the advisory servers' suites again on code it had just run them on, and, sharing one concurrency group on main, canceled that run if it was still going. Its job is now skipped, and its run has a group of its own, so the run on the code it releases always finishes. Refs: #199 --- .github/workflows/interop.yml | 16 ++++++++++++++-- 1 file changed, 14 insertions(+), 2 deletions(-) diff --git a/.github/workflows/interop.yml b/.github/workflows/interop.yml index d5429b52..d8b5ce9d 100644 --- a/.github/workflows/interop.yml +++ b/.github/workflows/interop.yml @@ -9,7 +9,11 @@ name: Interop # job per server, as CI runs the blocking ones. A job fails only on a test # failing that its servers//expected-failures.json does not list, or # when no test ran; the job's summary says what passed, what failed as -# expected and why, and what passes though listed. +# expected and why, and what passes though listed. A release commit, +# whose push its paths match (package.json, package-lock.json), runs +# nothing: it changes only the version's files, so its code is the +# commit's before it, which Interop already ran on +# (docs/releasing.md). on: push: @@ -31,13 +35,21 @@ on: permissions: contents: read +# A newer push's run cancels the one still going. A release commit's run, +# which runs nothing, has a group of its own, so it never cancels the run +# on the code it releases. concurrency: - group: interop-${{ github.ref }} + group: >- + interop-${{ github.ref }}${{ + startsWith(github.event.head_commit.message, 'chore(release): ') && + format('-{0}', github.sha) || '' }} cancel-in-progress: true jobs: pod: name: pod (${{ matrix.server }}) + if: >- + !startsWith(github.event.head_commit.message, 'chore(release): ') runs-on: ubuntu-latest timeout-minutes: 15 strategy: From 4d842f8732a89b22dea5b14da802a06fcbb760af Mon Sep 17 00:00:00 2001 From: Anton Wiklund Date: Tue, 20 Oct 2026 14:17:28 +0200 Subject: [PATCH 2/4] ci: run timing-dependent jobs only where a tree is first tested A change met CI up to four times on its way to the site, and every run had the same say: 4.1.1 stayed released but undeployed until the engines' journeys, flaky on the release commit, were re-run by hand three times, although its code had passed in the merge queue and on main already. Each of those runs also spent its slowest jobs, the timing-dependent ones, on code they had already passed. A new gate job (npm run ci:gate) decides what each run runs. A pull request, the merge queue, a run by hand and a push to main that no merge_group run passed (an advisory's merge through the ruleset bypass, say) run every job, and every job blocks. A push the queue passed, and a release commit whose parent CI passed on main, skip the timing-dependent jobs (pod, contract, the journeys in Firefox, WebKit and on a phone) instead of running them again; ci-ok lets them be skipped then and only then. The gate fails a release commit that changes anything but the files release.config.mjs commits. The report the site publishes at /journeys/ keeps every browser's journeys, as the release commit's run takes the engines' from the run that ran them on its code, which the gate names, and the queue's engines keep their traces and blob for it. When that blob has expired, the gate runs every job instead. The caches the skipped jobs saved on main, each server's images and the engines' browsers, are looked up by a warm job and only the missing ones made and saved. Each job in ci.yml is labeled deterministic or timing-dependent, as in docs/testing.md, which also says to count requests rather than time them; docs/releasing.md says what each stage runs. Closes: #199 --- .github/workflows/ci.yml | 291 ++++++++++++++++++++++++++++--- docs/deployment.md | 8 +- docs/glossary.md | 3 +- docs/releasing.md | 83 ++++++++- docs/standard-for-public-code.md | 4 +- docs/testing.md | 88 +++++++--- e2e/pod/servers.ts | 10 +- package.json | 1 + scripts/ciGate.test.ts | 289 ++++++++++++++++++++++++++++++ scripts/ciGate.ts | 225 ++++++++++++++++++++++++ 10 files changed, 940 insertions(+), 62 deletions(-) create mode 100644 scripts/ciGate.test.ts create mode 100644 scripts/ciGate.ts diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index d64178c6..57a03951 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -6,7 +6,8 @@ name: CI # Every change is checked before it is merged (docs/testing.md#continuous-integration): # a pull request, from this repository or a fork, runs every job here, -# and so does each push to main. A pull request is merged through the +# and a push to main every job or the deterministic ones, as the gate +# decides (below). A pull request is merged through the # merge queue (merge_group), which runs them again on the pull request # on top of main and of those ahead of it in the queue, and merges # exactly the commit that passed. A branch's pushes alone run nothing, @@ -16,6 +17,17 @@ name: CI # ours. The one exception, caches, runs on main alone and runs no code # of the repository's. Only runs on main save Turbo's cache and the # end-to-end images; a pull request and the merge queue read main's. +# +# Each job is labeled deterministic, the same answer on any runner, or +# timing-dependent, its answer hanging on how busy the runner or a real +# server is (docs/testing.md#continuous-integration). The gate job +# decides whether the timing-dependent ones run +# (docs/releasing.md#what-each-stage-checks): in a pull request, in the +# merge queue, by hand, and on a push to main that skipped the queue, +# but not on a push the queue passed, nor on a release commit, whose +# tree the queue (or its parent's full run) already tested. A job that +# runs blocks. A new job is given its label, and so its place in the +# gates, when it is added. on: pull_request: @@ -50,6 +62,54 @@ concurrency: cancel-in-progress: ${{ github.ref != 'refs/heads/main' }} jobs: + # What this run runs (npm run ci:gate): every job (jobs=all), or the + # deterministic ones alone (jobs=deterministic), with the run that ran + # the others on this tree (tested-by). On a release commit, it also + # fails unless the commit changes nothing but the files a release + # writes. Only on a push to main does it ask GitHub's API: whether the + # merge queue passed the commit, or CI on main a release commit's + # parent. It also gives journeys its matrix, the engines left out when + # the timing-dependent jobs are. Deterministic. + gate: + runs-on: ubuntu-latest + timeout-minutes: 5 + permissions: + contents: read + # To look up CI's runs on the commit and on its parent. + actions: read + outputs: + jobs: ${{ steps.gate.outputs.jobs }} + tested-by: ${{ steps.gate.outputs.tested-by }} + journeys: ${{ steps.journeys.outputs.matrix }} + steps: + - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 + with: + fetch-depth: 2 + persist-credentials: false + - uses: actions/setup-node@949feb2413d6458794dcd2491c4babbbce0c15c1 # v7.1.0 + with: + node-version-file: .nvmrc + # The script imports no package, so nothing is installed. + - id: gate + env: + GH_TOKEN: ${{ github.token }} + run: npm run ci:gate + # The build draws the docs' diagrams in Chromium's headless shell, + # which the engines' job installs beside their browsers. + - id: journeys + env: + JOBS: ${{ steps.gate.outputs.jobs }} + MATRIX: >- + [{"name": "chromium", "projects": "--project=chromium", + "browsers": "chromium", "timing": false}, + {"name": "engines", "projects": "--project=firefox --project=webkit --project=phone", + "browsers": "chromium-headless-shell firefox webkit", "timing": true}] + run: | + echo "matrix=$(jq -c --arg jobs "$JOBS" 'map(select($jobs == "all" or (.timing | not)))' <<< "$MATRIX")" >> "$GITHUB_OUTPUT" + + # Deterministic. Its growth tests time CPU work, but as the ratio of two + # runs on the same runner, which a busy runner leaves about the same + # (packages/domain/src/testing/growth.ts). build: runs-on: ubuntu-latest timeout-minutes: 20 @@ -108,7 +168,7 @@ jobs: # repository's 10 GB until GitHub's 7 days are up. A job of its own, so # the only token that can delete caches is in a job that runs none of # the repository's code; and one that may fail without failing the run, - # which release.yml and deploy.yml wait for. + # which release.yml and deploy.yml wait for. Deterministic. caches: if: github.ref == 'refs/heads/main' needs: build @@ -136,7 +196,7 @@ jobs: # API (docs/releasing.md#how-ci-holds-to-it): no line of api/ gone # unless a commit says the change breaks, and none of api/frozen/ at # all. It needs the whole history, which build does not, so it is a - # job of its own. + # job of its own. Deterministic. library: runs-on: ubuntu-latest timeout-minutes: 10 @@ -194,6 +254,7 @@ jobs: # included. It reads only files in the repository (no npm install, no # generated code, no build), so it runs beside build rather than after # it, and the deploy waits for it like every other job here. + # Deterministic. crosscheck: runs-on: ubuntu-latest timeout-minutes: 15 @@ -219,8 +280,12 @@ jobs: # in Docker on this runner and runs the whole suite (docs/testing.md). One server failing does not stop the others; # what the server printed is kept when its job fails. Only a run on main # saves the images; elsewhere, new images are pulled through the mirror. + # Timing-dependent: real servers, started on a shared runner. Where it is + # skipped on main, warm saves the images it would have. pod: name: pod (${{ matrix.server }}) + needs: gate + if: needs.gate.outputs.jobs == 'all' runs-on: ubuntu-latest timeout-minutes: 15 strategy: @@ -267,8 +332,11 @@ jobs: # The advisory servers (docs/testing.md) only report, in the Interop # workflow, but each must still start and meet the harness's contract: # a change that breaks one fails here, not unnoticed until Interop runs. + # Timing-dependent: real servers, which must start in time. contract: name: contract (${{ matrix.server }}) + needs: gate + if: needs.gate.outputs.jobs == 'all' runs-on: ubuntu-latest timeout-minutes: 15 strategy: @@ -319,24 +387,29 @@ jobs: # step it failed in, and its artifact holds the trace to replay it step # by step (`npx playwright show-trace`), the report, a video, and the # server's log. On main the traces are kept always, and each job keeps - # its run as a blob, for journeys-report to merge. + # its run as a blob, for journeys-report to merge; so do the engines in + # the merge queue, as main's report takes theirs from the run that + # tested its tree when it skips them (gate's tested-by). + # + # Chromium's journeys run like a deterministic job: they are every + # journey, in the browser most people use, and their waits are fixed + # where they flake, not retried (retries: 0, playwright.config.ts). The + # engines' are timing-dependent: slower browsers on a shared runner. + # The matrix is gate's (name, projects, browsers, timing), without the + # engines when the timing-dependent jobs are skipped. journeys: name: journeys (${{ matrix.name }}) + needs: gate runs-on: ubuntu-latest timeout-minutes: 25 strategy: fail-fast: false matrix: - include: - - name: chromium - projects: --project=chromium - browsers: chromium - # The build draws the docs' diagrams in Chromium's headless shell. - - name: engines - projects: --project=firefox --project=webkit --project=phone - browsers: chromium-headless-shell firefox webkit + include: ${{ fromJSON(needs.gate.outputs.journeys) }} env: - JOURNEY_TRACE: ${{ (inputs.trace || github.ref == 'refs/heads/main') && 'on' || '' }} + JOURNEY_TRACE: >- + ${{ (inputs.trace || github.ref == 'refs/heads/main' || + (github.event_name == 'merge_group' && matrix.timing)) && 'on' || '' }} # The blob's name, so the two jobs' blobs sit side by side when merged. PLAYWRIGHT_BLOB_OUTPUT_NAME: report-${{ matrix.name }}.zip steps: @@ -391,7 +464,7 @@ jobs: # A job run again replaces its blob from the attempt before; kept as # long as the report, so that re-running one job still finds the # other's. - - if: always() && github.ref == 'refs/heads/main' + - if: always() && (github.ref == 'refs/heads/main' || (github.event_name == 'merge_group' && matrix.timing)) uses: actions/upload-artifact@cf430e030ddbb5b0abf93d22962f4752f3646cd9 # v7.0.2 with: name: journeys-blob-${{ matrix.name }} @@ -401,16 +474,22 @@ jobs: # On main, the two journeys jobs' runs merged into one report, every # browser's journeys and their traces in it, even when a journey failed: - # the deploy publishes it at /journeys/ (docs/deployment.md). When both - # jobs passed, both blobs must be there, so a run the deploy may publish - # never has a report missing a browser. ci-ok doesn't need it, as it - # runs on main alone; its failure still fails the run, and so holds - # back release and deploy. + # the deploy publishes it at /journeys/ (docs/deployment.md). When this + # run skipped the engines, theirs come from the run gate names, which + # ran them on this tree: the merge queue's, or a release commit's + # parent's. When the journeys passed, both blobs must be there, so a run + # the deploy may publish never has a report missing a browser. ci-ok + # doesn't need it, as it runs on main alone; its failure still fails the + # run, and so holds back release and deploy. Deterministic. journeys-report: if: ${{ !cancelled() && github.ref == 'refs/heads/main' }} - needs: journeys + needs: [gate, journeys] runs-on: ubuntu-latest timeout-minutes: 10 + permissions: + contents: read + # To download the engines' blob from the run that tested this tree. + actions: read steps: - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 with: @@ -425,6 +504,13 @@ jobs: pattern: journeys-blob-* merge-multiple: true path: e2e/journeys/blob-reports/ + - if: needs.gate.outputs.jobs == 'deterministic' + uses: actions/download-artifact@9000827ccba6bdab643e8b6fd33ac0654aef8333 # v8.0.2 + with: + name: journeys-blob-engines + run-id: ${{ needs.gate.outputs.tested-by }} + github-token: ${{ github.token }} + path: e2e/journeys/blob-reports/ - working-directory: e2e/journeys env: JOURNEYS: ${{ needs.journeys.result }} @@ -444,6 +530,133 @@ jobs: retention-days: 7 overwrite: true + # On main, when the timing-dependent jobs are skipped, the caches they + # would have saved there, which every pull request and queue run reads: + # each server's images (e2e-images; the journeys' are saved by Chromium's + # job, which still runs) and the engines' browsers. This job only looks + # them up, by the keys those jobs use; warm-images and warm-browsers + # make and save what is missing, and nothing else, so a run with every + # cache in place spends one short job. Like caches, it may fail without + # failing the run. Deterministic. + warm: + if: github.ref == 'refs/heads/main' && needs.gate.outputs.jobs == 'deterministic' + needs: gate + runs-on: ubuntu-latest + timeout-minutes: 5 + continue-on-error: true + permissions: + contents: read + # To list main's caches. + actions: read + outputs: + servers: ${{ steps.missing.outputs.servers }} + browsers: ${{ steps.missing.outputs.browsers }} + steps: + - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 + with: + persist-credentials: false + - uses: actions/setup-node@949feb2413d6458794dcd2491c4babbbce0c15c1 # v7.1.0 + with: + node-version-file: .nvmrc + cache: npm + - run: npm ci + # shell: bash runs with pipefail, so a listing that fails fails the + # step instead of finding every cache missing. + - name: What main's cache lacks + id: missing + shell: bash + env: + GH_TOKEN: ${{ github.token }} + GH_REPO: ${{ github.repository }} + run: | + # Listed once each, outside any condition, where a failure + # fails the step: inside one, bash would read it as a miss. + listed=$(gh cache list --ref refs/heads/main --key e2e-images- --limit 1000 --json key) + listed+=$(gh cache list --ref refs/heads/main --key playwright- --limit 1000 --json key) + cached() { + jq -se --arg key "$1" 'add | any(.[]; .key == $key)' <<< "$listed" > /dev/null + } + keys=$(node e2e/pod/servers.ts keys all) + servers=() + while read -r id key; do + cached "e2e-images-$RUNNER_OS-$RUNNER_ARCH-pod-$id-$key" || servers+=("$id") + done <<< "$keys" + echo "Images missing: ${servers[*]:-none}" + echo "servers=$(jq -cn '$ARGS.positional' --args "${servers[@]}")" >> "$GITHUB_OUTPUT" + version=$(node -p "require('@playwright/test/package.json').version") + if cached "playwright-$RUNNER_OS-$version-engines"; then browsers=false; else browsers=true; fi + echo "Engines' browsers missing: $browsers" + echo "browsers=$browsers" >> "$GITHUB_OUTPUT" + + # A missing server's images, pulled or built as pod and contract would, + # and saved under their key. Deterministic. + warm-images: + name: warm-images (${{ matrix.server }}) + if: needs.warm.outputs.servers != '' && needs.warm.outputs.servers != '[]' + needs: warm + runs-on: ubuntu-latest + timeout-minutes: 15 + continue-on-error: true + strategy: + fail-fast: false + matrix: + server: ${{ fromJSON(needs.warm.outputs.servers) }} + # The servers come from an output, so they reach the shell as a + # variable, never written into a script. + env: + SERVER: ${{ matrix.server }} + steps: + - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 + with: + persist-credentials: false + - uses: actions/setup-node@949feb2413d6458794dcd2491c4babbbce0c15c1 # v7.1.0 + with: + node-version-file: .nvmrc + cache: npm + - run: npm ci + - id: key + run: echo "key=$(node e2e/pod/servers.ts key "$SERVER")" >> "$GITHUB_OUTPUT" + - id: images + uses: ./.github/actions/e2e-images + with: + name: pod-${{ matrix.server }} + key: ${{ steps.key.outputs.key }} + - run: npm run servers -w @solid-memo/e2e-pod -- "$SERVER" + - if: steps.images.outputs.hit != 'true' + env: + FILE: ${{ steps.images.outputs.file }} + run: node e2e/pod/images.ts save "$FILE" "e2e/pod/servers/$SERVER/compose.yml" + - if: steps.images.outputs.hit != 'true' + uses: actions/cache/save@55cc8345863c7cc4c66a329aec7e433d2d1c52a9 # v6.1.0 + with: + path: ${{ steps.images.outputs.file }} + key: ${{ steps.images.outputs.key }} + + # The engines' browsers, downloaded as journeys would, and saved under + # the key it restores them by. Deterministic. + warm-browsers: + if: needs.warm.outputs.browsers == 'true' + needs: warm + runs-on: ubuntu-latest + timeout-minutes: 10 + continue-on-error: true + steps: + - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 + with: + persist-credentials: false + - uses: actions/setup-node@949feb2413d6458794dcd2491c4babbbce0c15c1 # v7.1.0 + with: + node-version-file: .nvmrc + cache: npm + - run: npm ci + - id: playwright + run: echo "version=$(node -p "require('@playwright/test/package.json').version")" >> "$GITHUB_OUTPUT" + - run: npx playwright install chromium-headless-shell firefox webkit + - uses: actions/cache/save@55cc8345863c7cc4c66a329aec7e433d2d1c52a9 # v6.1.0 + with: + path: ~/.cache/ms-playwright + key: playwright-${{ runner.os }}-${{ steps.playwright.outputs.version }}-engines + # The docs (docs/testing.md#documentation): every link to a file or a # heading in the repository resolves (lychee, offline: the web's # addresses are checked weekly, by links.yml, so a site that is down @@ -454,7 +667,7 @@ jobs: # the job installs no npm packages: reuse comes from PyPI, pinned in # scripts/requirements-lint.txt, not as fsfe/reuse-action's Docker # image, which Docker Hub's limit on pulls without an account, shared - # by the runners, could refuse, and which names its image by a tag. + # by the runners, could refuse, and which names its image by a tag. Deterministic. docs: runs-on: ubuntu-latest timeout-minutes: 10 @@ -485,7 +698,7 @@ jobs: # shellcheck), zizmor their security, such as an expression written # into a script or a job given more rights than it uses. Offline: # zizmor's audits that ask GitHub's API are left out. Both are Python - # packages, pinned in scripts/requirements-lint.txt. + # packages, pinned in scripts/requirements-lint.txt. Deterministic. workflows: runs-on: ubuntu-latest timeout-minutes: 10 @@ -522,6 +735,7 @@ jobs: # from the API when the step runs, not from the event, so editing it # and running the job again is enough. Renovate's pull requests have # no issue, and are left out, as commitlint leaves out their commits. + # Deterministic. commits: runs-on: ubuntu-latest timeout-minutes: 10 @@ -574,20 +788,39 @@ jobs: npm run check:closes -- --base "$BASE" --head "$HEAD" --body-file "$body" # The one check a merge to main waits for (the ruleset on main is to - # require it): it passes only when every job above passed. A matrix - # job's checks are named per server, and jobs come and go; this name - # stays. + # require it): it passes only when every job it needs passed (not + # journeys-report, caches or the warm jobs), but for the + # timing-dependent jobs gate skipped (jobs=deterministic), which may be + # skipped then and only then. Every job it needs is in one of its two + # lists, or it fails. A matrix job's checks are named per server, and + # jobs come and go; this name stays. ci-ok: if: always() - needs: [build, library, crosscheck, pod, contract, journeys, docs, workflows, commits] + needs: [gate, build, library, crosscheck, pod, contract, journeys, docs, workflows, commits] runs-on: ubuntu-latest timeout-minutes: 5 steps: - - name: Every job passed + - name: Every job passed, or gate skipped it env: RESULTS: ${{ join(needs.*.result, ' ') }} + DETERMINISTIC: >- + ${{ needs.gate.result }} ${{ needs.build.result }} ${{ needs.library.result }} + ${{ needs.crosscheck.result }} ${{ needs.journeys.result }} ${{ needs.docs.result }} + ${{ needs.workflows.result }} ${{ needs.commits.result }} + TIMING: ${{ needs.pod.result }} ${{ needs.contract.result }} + JOBS: ${{ needs.gate.outputs.jobs }} run: | - echo "The jobs' results: $RESULTS" - for result in $RESULTS; do + echo "The jobs' results: $RESULTS (gate: ${JOBS:-none})" + read -ra all <<< "$RESULTS" + read -ra deterministic <<< "$DETERMINISTIC" + read -ra timing <<< "$TIMING" + [ "${#all[@]}" -eq $(( ${#deterministic[@]} + ${#timing[@]} )) ] || { + echo "::error::ci-ok needs a job neither of its lists names: add it to one." + exit 1 + } + for result in "${deterministic[@]}"; do [ "$result" = success ] || exit 1 done + for result in "${timing[@]}"; do + [ "$result" = success ] || { [ "$result" = skipped ] && [ "$JOBS" = deterministic ]; } || exit 1 + done diff --git a/docs/deployment.md b/docs/deployment.md index e8ef4a24..1e140af7 100644 --- a/docs/deployment.md +++ b/docs/deployment.md @@ -18,7 +18,7 @@ is made: flowchart LR accTitle: When a version is deployed accDescr: A push to main, or a run by hand, deploys only the newest version, from its release commit and with its GitHub release, after the build, each release's docs and the guard. - push["push to main"] --> ci["CI: npm run check,
build, pySHACL, pods,
journeys, docs"] -->|success| tag{"deploy.yml:
a version tag
names the commit?"} + push["push to main"] --> ci["CI: npm run check,
build, pySHACL,
journeys, docs"] -->|success| tag{"deploy.yml:
a version tag
names the commit?"} tag -->|no| skip["not deployed"] tag -->|yes| release{"the newest version,
its release commit,
its GitHub release?"} release -->|no| skip @@ -34,8 +34,10 @@ repository: - **After CI on a release.** A release ([releasing.md](releasing.md)) follows a green CI run on `main`, in a workflow of its own. It pushes a commit to `main` that sets the new version, - `chore(release): X.Y.Z`, and tags it `vX.Y.Z`. CI runs on that commit - as on any push, and once it passes, the deploy builds it, so the + `chore(release): X.Y.Z`, and tags it `vX.Y.Z`. CI runs on that commit, + its deterministic jobs only, as the code is its parent's + ([releasing.md](releasing.md#what-each-stage-checks)), and once it + passes, the deploy builds it, so the site's footer shows the new version. The deploy looks for a version tag that names the commit CI checked, and deploys nothing when there is none: any other push to `main`, a merged pull request or a diff --git a/docs/glossary.md b/docs/glossary.md index d8c608cd..25cc10ca 100644 --- a/docs/glossary.md +++ b/docs/glossary.md @@ -221,7 +221,8 @@ it most. The specifications' versions and status are in every pull request and every push to `main`. A change is merged only when they pass. [testing.md](testing.md#continuous-integration). - **`ci-ok`**: the one CI check a merge to `main` waits for, which - passes only when every other job passed. + passes only when every other job passed, or was skipped as CI's gate + decides ([releasing.md](releasing.md#what-each-stage-checks)). [testing.md](testing.md#continuous-integration). - **commitlint**: the tool that checks that a commit message follows Conventional Commits. CI runs it on a pull request's commits. diff --git a/docs/releasing.md b/docs/releasing.md index 65663306..4a0c3448 100644 --- a/docs/releasing.md +++ b/docs/releasing.md @@ -206,6 +206,78 @@ checks it). At the end of the file, a section written by hand sums up what the first version, 1.0.0, contains, which its generated entry lists only in part. +## What each stage checks + +A change to `main` meets CI up to four times on its way to the site, +and the stages do not all have the same say. Each job of +[ci.yml](../.github/workflows/ci.yml) is deterministic or +timing-dependent ([testing.md](testing.md#deterministic-and-timing-dependent-jobs)), +and CI's `gate` job (`npm run ci:gate`, +[ciGate.ts](../scripts/ciGate.ts)) decides, for each run, whether the +timing-dependent jobs run. Every job that runs blocks; the +timing-dependent ones are skipped, not run to no purpose, where the same +code already passed them, as they are the slowest jobs and the ones a +busy runner fails. + +| Stage | What runs, and blocks | +|---|---| +| A pull request | every job | +| The merge queue (`merge_group`) | every job: this is where a tree is tested at full strength before it lands, and where a flaky check is fixed, not retried | +| A push to `main` the merge queue passed | the deterministic jobs; the same commit passed every job in the queue | +| A push to `main` that skipped the queue | every job, as in the queue | +| A release commit | the deterministic jobs, and that the commit changes nothing but the files a release writes, when CI passed its parent on `main`; every job when it did not | +| A run by hand | every job | + +So a change's timing-dependent jobs run in its pull request, once for +each push to it, and once more in the merge queue, and never again for +the same code. Interop ([testing.md](testing.md)), which runs the +advisory servers' suites after a push to `main`, likewise skips a +release commit. + +**Test a tree once, at full strength, before it lands.** The merge +queue builds exactly the commit `main` moves to +([testing.md](testing.md#the-merge-queue)), so the push run on `main` +that follows checks a tree CI already passed in full. The gate asks +GitHub's API whether a `merge_group` run passed that commit; when one +did, only the deterministic jobs run. + +**Know when a change skipped the queue.** A change can reach `main` +without the queue: a merge through a ruleset bypass, such as one from a +security advisory's private fork, where Actions do not run. Then the +push run is the first CI the change gets, no `merge_group` run passed +its commit, and every job runs. The release follows that run only +once it passes ([below](#what-a-release-does)), so a release built on +such a commit waits for the full run. + +**Deploy on the tested tree.** A release commit changes only the +version's files: the five that `release.config.mjs` lists (`assets`): +`CHANGELOG.md`, `CITATION.cff`, `package.json`, `package-lock.json` +and `publiccode.yml`. Its code is its parent's, which CI passed on +`main` before the release was made. So its run runs the deterministic +jobs, the build and the journeys in Chromium among them, and the gate +fails it if the commit changes any other file, or has more than one +parent. The timing-dependent jobs are skipped, and cannot hold up the +deploy. Should CI not have passed its parent on `main` (a release +commit pushed by hand), every job runs. + +The report of the journeys the site publishes at `/journeys/` +([deployment.md](deployment.md)) still has every browser's: the release +commit's run takes the engines' journeys from the run that ran them on +its code, which the gate names (`tested-by`). That is its parent's +merge queue run, which keeps the engines' traces and report for this, +or, for a parent that skipped the queue, the parent's own push run. +GitHub keeps that run's report of them for 7 days; once it has expired, +as for a release made late by hand, the gate runs every job instead. + +What a skipped job would have saved on `main`, the caches every pull +request reads, CI's `warm` jobs save instead: `warm` looks up each +server's images and the engines' browsers, and `warm-images` and +`warm-browsers` pull, build or download only what is missing, and save +it ([testing.md](testing.md#continuous-integration)). + +A `workflow_dispatch` run of the deploy runs `npm run check` itself, +as before, and needs no CI run. + ## What a release does [release.yml](../.github/workflows/release.yml) runs when CI has passed @@ -421,9 +493,14 @@ next push to `main` that passes CI also tries again. It deploys nothing, and fails, if the tag names any commit but `chore(release): X.Y.Z`, or if the GitHub release is missing. -- **CI failed on the release commit** (a flaky test, say): the version - is released but not deployed. Re-run CI's failed jobs on that commit; - once it passes, the deploy follows as usual. Or run the deploy by +- **CI failed on the release commit**: the version is released but not + deployed. No timing-dependent job runs there + ([What each stage checks](#what-each-stage-checks)), so the failure is + a deterministic job's, or the `gate`'s, which refuses a release commit + that changes more than the version's files. A deterministic job that + failed on code its parent passed with is flaky: re-run CI's failed + jobs on that commit, and fix the flake in a pull request of its own; + once the run passes, the deploy follows as usual. Or run the deploy by hand, which checks the version itself. Either deploys the version only while it is the newest: once a newer one is released, the deploy after a re-run deploys nothing, and says so, and the site stays on diff --git a/docs/standard-for-public-code.md b/docs/standard-for-public-code.md index 75c6f1fa..4b88b6f2 100644 --- a/docs/standard-for-public-code.md +++ b/docs/standard-for-public-code.md @@ -651,7 +651,9 @@ What holds today, and what does not, criterion by criterion. tests, which CI runs before the cross-check ([testing.md](testing.md)). - MUST, met — contributions pass all tests before they are merged. CI runs on every pull request, in the merge queue and on every push to - `main`, and `ci-ok` passes only when all its jobs did: `build`, + `main`, and `ci-ok` passes only when all its jobs did (on `main`, + the timing-dependent ones that the merge queue already passed are + skipped, [releasing.md](releasing.md#what-each-stage-checks)): `build`, `library` (the deck library and the public API compared with the change's base), `crosscheck`, `pod`, `contract`, `journeys` (every journey in Chromium, and a set of them in Firefox, WebKit and on a diff --git a/docs/testing.md b/docs/testing.md index 80559a53..715a4862 100644 --- a/docs/testing.md +++ b/docs/testing.md @@ -132,6 +132,10 @@ limit on pulls without an account the runners share: each job restores its server's images ([e2e-images](../.github/actions/e2e-images/action.yml)), and only when the cache has none under the job's key does it pull or build them, then, on `main`, save them ([e2e/pod/images.ts](../e2e/pod/images.ts)). +Where `main`'s run skips the jobs that would save them +([releasing.md](releasing.md#what-each-stage-checks)), CI's `warm` jobs +save the images of every server, blocking and advisory, that the cache +lacks (`node e2e/pod/servers.ts keys all` names their keys). A pull request and the merge queue read `main`'s images but save none: GitHub keeps what a run saves under its ref, so their copies would never be read again, and they would count toward the repository's 10 GB of @@ -303,8 +307,10 @@ for). It keeps its pods in files, not in memory, because the journeys PATCH Swedish text (see above). The journeys need Docker and the browsers (`npm run browsers -w @solid-memo/e2e-journeys` installs Chromium, Firefox and WebKit, with the system libraries they need), and CI -runs them on every pull request and every push to `main`. They block, -as the blocking servers do. +runs them on every pull request and in the merge queue, and on `main` +those in Chromium always, the others only where the code has not passed +them yet ([releasing.md](releasing.md#what-each-stage-checks)). They +block, as the blocking servers do. Further journeys go through: @@ -438,11 +444,15 @@ error, and the browser it failed in. The job's published with each deploy: the report of the CI run on the release commit the site runs (none after a deploy by hand), every browser's journeys in one, as the `journeys-report` job merges them from both -journeys jobs (Playwright's blob reports). It shows every +journeys jobs (Playwright's blob reports); the engines', when the +release commit's run skips them, from the run that ran them on its code, +usually the merge queue's +([releasing.md](releasing.md#what-each-stage-checks)). It shows every journey's steps and screenshots, and its **Trace** link opens the step-by-step timeline in the browser (the trace viewer is part of the report and -needs it served, as it is there, not opened as a file). On `main` the -journeys always keep their traces for this. The report is public and +needs it served, as it is there, not opened as a file). On `main`, and +for the engines in the merge queue, the journeys always keep their +traces for this. The report is public and holds the throwaway accounts' passwords and tokens, which die with the run's server, and it is kept out of search engines (`robots.txt`). Since only a green run on a release commit deploys, a failure on @@ -541,29 +551,32 @@ they show. fork's included, in the merge queue ([below](#the-merge-queue)), on every push to `main`, and by hand. A branch's pushes alone run nothing, so a pull request's commits are not checked twice for one push. Its -jobs, side by side: - -| Job | What it checks | -|---|---| -| `build` | `npm run check:ci`, which is `npm run check` without the deck library (the `library` job checks it), then `npm run build`, with Chromium's headless shell from GitHub's cache or installed ([.github/actions/chromium](../.github/actions/chromium/action.yml)) for the docs' diagrams; the coverage of every package goes to the run's summary (`npm run coverage:summary`). Turbo's cache is kept between runs under a key of this job's own, so a task whose inputs have not changed is not run again; only a run on `main` saves it, and a pull request and the merge queue start from `main`'s | -| `library` | the change against its base: the deck library, no published version edited or removed, and every new version naming the library's contact ([deck-library.md](deck-library.md#checks)); the public API, no line of `api/` gone unless a commit says the change breaks, and none of `api/frozen/` at all (`npm run check:breaking`, [releasing.md](releasing.md#how-ci-holds-to-it)) | -| `crosscheck` | the Python scripts' tests (the cross-check's and the taxonomy deck generator's), then the vocabulary, the deck library and the pod catalog documents, by pySHACL ([validation.md](validation.md#the-ci-cross-check)) | -| `pod ()` | the end-to-end tests, one job per blocking server (above) | -| `contract ()` | that each advisory server starts and meets the contract | -| `journeys ()` | the user journeys: every one in Chromium, and those that differ between engines in Firefox, WebKit and on a phone ([Browsers](#browsers)) | -| `journeys-report` | nothing: on `main`, after both `journeys` jobs, even when one failed, it merges their runs into one report, the `journeys-report` artifact the deploy publishes at `/journeys/` ([User journeys](#user-journeys)). When both passed, it fails without both their reports; its failure fails the run, so it holds back the release and the deploy, but not `ci-ok` | -| `docs` | the docs' links, every file's license, and `publiccode.yml` ([Documentation](#documentation)); their style and spelling are `build`'s | -| `workflows` | the workflows and local actions, by actionlint and zizmor (`npm run lint:workflows`, [The workflows](#the-workflows)) | -| `commits` | on a pull request, that each commit it adds follows Conventional Commits, by commitlint ([CONTRIBUTING.md](../CONTRIBUTING.md#commit-messages)), and that its body or a commit closes its issue, or the body says it is one step of it (`npm run check:closes`; not for Renovate's); in the merge queue or on a push to `main`, nothing | -| `caches` | nothing: on `main`, after `build`, it deletes `main`'s older Turbo caches, the newest holding what they do. The one job whose token can write (`actions: write`, to delete caches); it runs none of the repository's code, and its failure fails neither the run nor `ci-ok` | -| `ci-ok` | that every job above passed, `journeys-report` and `caches` aside | +jobs, side by side, each [deterministic or timing-dependent](#deterministic-and-timing-dependent-jobs): + +| Job | Kind | What it checks | +|---|---|---| +| `gate` | deterministic | what the run runs: every job, or the deterministic ones alone (`npm run ci:gate`, [releasing.md](releasing.md#what-each-stage-checks)); on a release commit, that it changes nothing but the files a release writes; and the journeys' matrix, without the engines when they are skipped | +| `build` | deterministic | `npm run check:ci`, which is `npm run check` without the deck library (the `library` job checks it), then `npm run build`, with Chromium's headless shell from GitHub's cache or installed ([.github/actions/chromium](../.github/actions/chromium/action.yml)) for the docs' diagrams; the coverage of every package goes to the run's summary (`npm run coverage:summary`). Turbo's cache is kept between runs under a key of this job's own, so a task whose inputs have not changed is not run again; only a run on `main` saves it, and a pull request and the merge queue start from `main`'s | +| `library` | deterministic | the change against its base: the deck library, no published version edited or removed, and every new version naming the library's contact ([deck-library.md](deck-library.md#checks)); the public API, no line of `api/` gone unless a commit says the change breaks, and none of `api/frozen/` at all (`npm run check:breaking`, [releasing.md](releasing.md#how-ci-holds-to-it)) | +| `crosscheck` | deterministic | the Python scripts' tests (the cross-check's and the taxonomy deck generator's), then the vocabulary, the deck library and the pod catalog documents, by pySHACL ([validation.md](validation.md#the-ci-cross-check)) | +| `pod ()` | timing-dependent | the end-to-end tests, one job per blocking server (above) | +| `contract ()` | timing-dependent | that each advisory server starts and meets the contract | +| `journeys ()` | deterministic in Chromium, timing-dependent in the other engines | the user journeys: every one in Chromium, and those that differ between engines in Firefox, WebKit and on a phone ([Browsers](#browsers)) | +| `journeys-report` | deterministic | nothing: on `main`, after both `journeys` jobs, even when one failed, it merges their runs into one report, the `journeys-report` artifact the deploy publishes at `/journeys/` ([User journeys](#user-journeys)); when this run skipped the engines, their run comes from the run that tested its code (the gate's `tested-by`). When the journeys passed, it fails without both their reports; its failure fails the run, so it holds back the release and the deploy, but not `ci-ok` | +| `docs` | deterministic | the docs' links, every file's license, and `publiccode.yml` ([Documentation](#documentation)); their style and spelling are `build`'s | +| `workflows` | deterministic | the workflows and local actions, by actionlint and zizmor (`npm run lint:workflows`, [The workflows](#the-workflows)) | +| `commits` | deterministic | on a pull request, that each commit it adds follows Conventional Commits, by commitlint ([CONTRIBUTING.md](../CONTRIBUTING.md#commit-messages)), and that its body or a commit closes its issue, or the body says it is one step of it (`npm run check:closes`; not for Renovate's); in the merge queue or on a push to `main`, nothing | +| `caches` | deterministic | nothing: on `main`, after `build`, it deletes `main`'s older Turbo caches, the newest holding what they do. The one job whose token can write (`actions: write`, to delete caches); it runs none of the repository's code, and its failure fails neither the run nor `ci-ok` | +| `warm`, `warm-images`, `warm-browsers` | deterministic | nothing: on `main`, when the timing-dependent jobs are skipped, `warm` looks up the caches they would have saved, each server's images and the engines' browsers, and the other two pull, build or download only what is missing, and save it. Like `caches`, they fail neither the run nor `ci-ok` | +| `ci-ok` | deterministic | that every job above passed, `journeys-report`, `caches` and the `warm` jobs aside; `pod` and `contract` may be skipped only when the gate skipped them | **`ci-ok` is the one check a merge to `main` waits for**: the ruleset on `main` (a setting of the repository, Settings › Rules) is to require it and nothing else, and Renovate merges its updates through a pull request once it passes (`automergeType: 'pr'` in `renovate.json5`), as anyone's change is merged. It runs even when a job -before it failed, and fails unless every one passed. A matrix job's +before it failed, and fails unless every one passed, or, for `pod` and +`contract`, was skipped by the gate. A matrix job's checks are named per server, and jobs come and go, so the ruleset names this job alone. (Another workflow's job named `build`, deploy.yml's, would also answer to a required check of that name.) Deploys follow a @@ -572,6 +585,37 @@ green run on a release commit on `main` `ns/` and `decks/` runs here, the comparison with the base included, so `ci-ok` gates them all and nothing is checked twice. +### Deterministic and timing-dependent jobs + +A **deterministic** job gives the same answer on any runner: types, +lint, unit tests, request counts, the docs. A **timing-dependent** one +can fail on sound code because the runner was busy or a real server +was slow: the end-to-end tests against real servers (`pod`, +`contract`) and the journeys in Firefox, WebKit and on a phone. Each +job in [ci.yml](../.github/workflows/ci.yml) says which it is, in its +comment and in the table above, and a new job is given its kind when it +is added, which decides where it runs +([releasing.md](releasing.md#what-each-stage-checks)): every job runs in +a pull request and the merge queue, and blocks there, but a +timing-dependent one is skipped on a push to `main` the queue passed and +on a release commit, whose code it already ran on. A job that runs +always blocks. + +The journeys in Chromium are end to end too, yet they run everywhere: +they are every journey, in the browser most people use, so a wait that +flakes there is fixed, not retried (`retries: 0` in +[playwright.config.ts](../e2e/journeys/playwright.config.ts)). + +**Count, don't time, where the answer can be counted.** A regression +in network use, round trips growing from linear to quadratic, is caught +by counting the requests a use case makes against a fake pod, as the +Solid adapters' tests do; a count is deterministic, so it can block at +every stage. Time is measured only for CPU work, and then as a ratio of +two runs on the same machine (the growth tests, +[growth.ts](../packages/domain/src/testing/growth.ts)), not against a +budget in milliseconds. The pod tests that time a server only print +what they measure, and fail on none of it. + ### The merge queue A pull request is merged through GitHub's merge queue, never straight diff --git a/e2e/pod/servers.ts b/e2e/pod/servers.ts index e9499621..333ddd50 100644 --- a/e2e/pod/servers.ts +++ b/e2e/pod/servers.ts @@ -580,11 +580,13 @@ async function untilUp(name: string, url: string, timeoutMs: number, log: string * `node servers.ts prepare [id...]` pulls or builds the servers' images * (every one when none is named), `contract [id...]` starts each, checks * the contract and stops it, `key ` prints what names its images - * (imageKey), `clean` takes down what interrupted runs left. + * (imageKey), `keys [id...]` each server's id and key, one a line (the + * blocking servers when none is named), `clean` takes down what + * interrupted runs left. */ if (import.meta.main) { const [command, ...ids] = process.argv.slice(2); - const named = serversNamed(ids.length > 0 ? ids.join(",") : "all"); + const named = serversNamed(ids.length > 0 ? ids.join(",") : command === "keys" ? "" : "all"); if (command === "prepare") { for (const id of named) { console.log(`Preparing ${id}`); @@ -599,10 +601,12 @@ if (import.meta.main) { } } else if (command === "key" && named.length === 1) { console.log(imageKey(named[0]!)); + } else if (command === "keys") { + for (const id of named) console.log(`${id} ${imageKey(id)}`); } else if (command === "clean") { const taken = await clean(); console.log(taken.length > 0 ? `Took down ${taken.join(", ")}.` : "Nothing to take down."); } else { - throw new Error(`Usage: node servers.ts prepare|contract|clean [id...] | key `); + throw new Error(`Usage: node servers.ts prepare|contract|keys|clean [id...] | key `); } } diff --git a/package.json b/package.json index ed97de51..24c4d553 100644 --- a/package.json +++ b/package.json @@ -52,6 +52,7 @@ "check:dependencies": "node -e \"import('./scripts/checkDependencies.ts').then((m) => m.run(process))\"", "check:reuse": "node -e \"import('./scripts/checkReuse.ts').then((m) => m.run(process))\"", "check:actions": "node -e \"import('./scripts/checkActions.ts').then((m) => m.run(process))\"", + "ci:gate": "node -e \"import('./scripts/ciGate.ts').then((m) => m.run(process))\"", "check:providers": "node -e \"import('./scripts/checkProviders.ts').then((m) => m.run(process))\"", "coverage:summary": "node -e \"import('./scripts/coverageSummary.ts').then((m) => m.run(process))\"", "release:version": "node -e \"import('./scripts/releaseVersion.ts').then((m) => m.run(process))\" --", diff --git a/scripts/ciGate.test.ts b/scripts/ciGate.test.ts new file mode 100644 index 00000000..6250c053 --- /dev/null +++ b/scripts/ciGate.test.ts @@ -0,0 +1,289 @@ +// SPDX-FileCopyrightText: 2026 Anton Wiklund and Solid Memo contributors +// +// SPDX-License-Identifier: MIT + +import { execFileSync } from "node:child_process"; +import { mkdtempSync, readFileSync, rmSync, writeFileSync } from "node:fs"; +import { tmpdir } from "node:os"; +import { join } from "node:path"; +import { afterEach, beforeEach, describe, expect, it, vi } from "vitest"; +import { + decide, + defaultIo, + ENGINES_BLOB, + type GateIo, + isRelease, + main, + run, + strayFiles, +} from "./ciGate.ts"; + +const SHA = "a".repeat(40); +const PARENT = "b".repeat(40); + +/** A push to main of an ordinary commit, which no run has passed. */ +function fakeIo(overrides: Partial = {}): GateIo & { outputs: string[]; logs: string[] } { + const outputs: string[] = []; + const logs: string[] = []; + return { + event: "push", + ref: "refs/heads/main", + head: () => ({ sha: SHA, subject: "feat(app): something", parents: [PARENT] }), + changed: () => ["CHANGELOG.md", "package.json"], + passed: async () => undefined, + kept: async () => true, + output: (name, value) => outputs.push(`${name}=${value}`), + log: (message) => logs.push(message), + outputs, + logs, + ...overrides, + }; +} + +describe("isRelease", () => { + it("takes semantic-release's commits, and nothing else", () => { + expect(isRelease("chore(release): 4.1.1")).toBe(true); + expect(isRelease("chore(deps): update zizmor")).toBe(false); + expect(isRelease("fix(release): chore(release): x")).toBe(false); + }); +}); + +describe("strayFiles", () => { + it("names the files a release does not write", () => { + expect( + strayFiles([ + "CHANGELOG.md", + "CITATION.cff", + "package.json", + "package-lock.json", + "publiccode.yml", + ]), + ).toEqual([]); + expect(strayFiles(["package.json", "apps/web/src/main.ts"])).toEqual(["apps/web/src/main.ts"]); + }); +}); + +/** CI's passed runs, by ":", and their ids. */ +const runs = + (passed: Record) => + async (sha: string, event: string): Promise => + passed[`${sha}:${event}`]; + +describe("decide", () => { + it("runs every job in a pull request, the merge queue and a run by hand", async () => { + const passed = vi.fn(async () => 1); + for (const [event, ref] of [ + ["pull_request", "refs/pull/1/merge"], + ["merge_group", "refs/heads/gh-readonly-queue/main/pr-1-abc"], + ["workflow_dispatch", "refs/heads/main"], + ["push", "refs/heads/other"], + ] as const) { + expect((await decide(fakeIo({ event, ref, passed }))).jobs).toBe("all"); + } + expect(passed).not.toHaveBeenCalled(); + }); + + it("runs the deterministic jobs on main when the merge queue passed the commit", async () => { + const passed = runs({ [`${SHA}:merge_group`]: 9 }); + expect(await decide(fakeIo({ passed }))).toEqual({ + jobs: "deterministic", + testedBy: 9, + why: "aaaaaaaa passed the merge queue in run 9: the deterministic jobs run.", + }); + }); + + it("runs every job on main when the queue run's report of the engines has expired", async () => { + const kept = vi.fn(async () => false); + const io = fakeIo({ passed: runs({ [`${SHA}:merge_group`]: 9 }), kept }); + expect(await decide(io)).toEqual({ + jobs: "all", + why: "aaaaaaaa passed the merge queue in run 9, but its journeys-blob-engines has expired: every job runs, so the journeys' report has the engines'.", + }); + expect(kept).toHaveBeenCalledWith(9, ENGINES_BLOB); + }); + + it("runs every job on main when the commit skipped the merge queue", async () => { + expect(await decide(fakeIo())).toEqual({ + jobs: "all", + why: "aaaaaaaa reached main without passing the merge queue: every job runs, as in the queue.", + }); + }); + + describe("on a release commit", () => { + const release = () => ({ sha: SHA, subject: "chore(release): 4.1.1", parents: [PARENT] }); + + it("runs the deterministic jobs, tested by its parent's queue run, when CI passed the parent on main", async () => { + const passed = runs({ [`${PARENT}:push`]: 7, [`${PARENT}:merge_group`]: 9 }); + expect(await decide(fakeIo({ head: release, passed }))).toEqual({ + jobs: "deterministic", + testedBy: 9, + why: "A release commit, changing only the version's files, on bbbbbbbb, which CI passed on main; run 9 ran every job on its code: the deterministic jobs run.", + }); + }); + + it("is tested by its parent's push run when the parent skipped the queue", async () => { + const passed = runs({ [`${PARENT}:push`]: 7 }); + expect(await decide(fakeIo({ head: release, passed }))).toMatchObject({ + jobs: "deterministic", + testedBy: 7, + }); + }); + + it("runs every job when CI has not passed its parent on main", async () => { + const passed = runs({ [`${PARENT}:merge_group`]: 9 }); + expect((await decide(fakeIo({ head: release, passed }))).jobs).toBe("all"); + }); + + it("refuses one that changes a file a release never writes", async () => { + const io = fakeIo({ head: release, changed: () => ["package.json", "apps/web/x.ts"] }); + await expect(decide(io)).rejects.toThrow(/changes apps\/web\/x\.ts, which a release never/); + }); + + it("refuses a merge", async () => { + const io = fakeIo({ head: () => ({ ...release(), parents: [PARENT, SHA] }) }); + await expect(decide(io)).rejects.toThrow(/has 2 parents, not one/); + }); + }); +}); + +describe("main", () => { + it("writes what runs and says why", async () => { + const io = fakeIo(); + expect(await main(io)).toBe(0); + expect(io.outputs).toEqual(["jobs=all"]); + expect(io.logs).toEqual([expect.stringContaining("without passing the merge queue")]); + }); + + it("writes which run tested the code when only the deterministic jobs run", async () => { + const io = fakeIo({ passed: runs({ [`${SHA}:merge_group`]: 9 }) }); + expect(await main(io)).toBe(0); + expect(io.outputs).toEqual(["jobs=deterministic", "tested-by=9"]); + }); + + it("fails, writing nothing, on a release commit that is not one", async () => { + const io = fakeIo({ + head: () => ({ sha: SHA, subject: "chore(release): 4.1.1", parents: [] }), + }); + expect(await main(io)).toBe(1); + expect(io.outputs).toEqual([]); + expect(io.logs).toEqual([expect.stringContaining("has 0 parents")]); + }); +}); + +describe("defaultIo", () => { + let repo: string; + const git = (...args: string[]) => + execFileSync("git", ["-C", repo, ...args], { encoding: "utf8" }).trim(); + + beforeEach(() => { + repo = mkdtempSync(join(tmpdir(), "ci-gate-")); + git("init", "-q", "-b", "main"); + git("config", "user.email", "test@example.org"); + git("config", "user.name", "Test"); + git("config", "commit.gpgsign", "false"); + writeFileSync(join(repo, "package.json"), "{}\n"); + git("add", "."); + git("commit", "-q", "-m", "feat: start"); + writeFileSync(join(repo, "package.json"), '{"version":"1.0.0"}\n'); + writeFileSync(join(repo, "CHANGELOG.md"), "# Changelog\n"); + git("add", "."); + git("commit", "-q", "-m", "chore(release): 1.0.0", "-m", "The notes."); + }); + + afterEach(() => rmSync(repo, { recursive: true, force: true })); + + it("reads the commit checked and the files it changes", () => { + const io = defaultIo(repo, { GITHUB_EVENT_NAME: "push", GITHUB_REF: "refs/heads/main" }, fetch); + const head = io.head(); + expect(io.event).toBe("push"); + expect(io.ref).toBe("refs/heads/main"); + expect(head).toEqual({ + sha: git("rev-parse", "HEAD"), + subject: "chore(release): 1.0.0", + parents: [git("rev-parse", "HEAD^")], + }); + expect(io.changed(head.sha)).toEqual(["CHANGELOG.md", "package.json"]); + }); + + it("reads a root commit's parents as none, and no event as none", () => { + const io = defaultIo(repo, {}, fetch); + git("checkout", "-q", "HEAD^"); + expect(io.head().parents).toEqual([]); + expect(io.event).toBe(""); + expect(io.ref).toBe(""); + }); + + it("asks GitHub for a CI run that passed on a commit, and gives its id", async () => { + const fetch = vi.fn(async () => Response.json({ total_count: 1, workflow_runs: [{ id: 42 }] })); + const io = defaultIo( + repo, + { GITHUB_REPOSITORY: "owner/repo", GITHUB_API_URL: "https://api.example", GH_TOKEN: "t" }, + fetch, + ); + expect(await io.passed(SHA, "merge_group")).toBe(42); + const [url, init] = fetch.mock.calls[0] as unknown as [URL, RequestInit]; + expect(String(url)).toBe( + `https://api.example/repos/owner/repo/actions/workflows/ci.yml/runs?head_sha=${SHA}&event=merge_group&status=success&per_page=1`, + ); + expect(init.headers).toMatchObject({ authorization: "Bearer t" }); + }); + + it("asks GitHub whether a run keeps an artifact, expired ones aside", async () => { + const answers = [ + { artifacts: [{ expired: true }, { expired: false }] }, + { artifacts: [{ expired: true }] }, + { artifacts: [] }, + ]; + const fetch = vi.fn(async () => Response.json(answers.shift())); + const io = defaultIo(repo, { GITHUB_REPOSITORY: "owner/repo" }, fetch); + expect(await io.kept(9, "journeys-blob-engines")).toBe(true); + expect(await io.kept(9, "journeys-blob-engines")).toBe(false); + expect(await io.kept(9, "journeys-blob-engines")).toBe(false); + expect(String((fetch.mock.calls[0] as unknown as [URL])[0])).toBe( + "https://api.github.com/repos/owner/repo/actions/runs/9/artifacts?name=journeys-blob-engines", + ); + }); + + it("finds none passed when GitHub lists none, on github.com by default", async () => { + const fetch = vi.fn(async () => Response.json({ total_count: 0, workflow_runs: [] })); + const io = defaultIo(repo, { GITHUB_REPOSITORY: "owner/repo" }, fetch); + expect(await io.passed(SHA, "push")).toBeUndefined(); + expect(String((fetch.mock.calls[0] as unknown as [URL])[0])).toMatch( + /^https:\/\/api\.github\.com\/repos\/owner\/repo\//, + ); + }); + + it("fails when GitHub refuses", async () => { + const fetch = vi.fn(async () => new Response("", { status: 403 })); + const io = defaultIo(repo, { GITHUB_REPOSITORY: "owner/repo" }, fetch); + await expect(io.passed(SHA, "push")).rejects.toThrow(/GitHub answered 403/); + }); + + it("writes its outputs to GITHUB_OUTPUT, and nowhere without it", () => { + const file = join(repo, "output"); + writeFileSync(file, ""); + defaultIo(repo, { GITHUB_OUTPUT: file }, fetch).output("jobs", "all"); + defaultIo(repo, {}, fetch).output("jobs", "deterministic"); + expect(readFileSync(file, "utf8")).toBe("jobs=all\n"); + }); + + it("logs to the console", () => { + const log = vi.spyOn(console, "log").mockImplementation(() => {}); + defaultIo(repo, {}, fetch).log("hello"); + expect(log).toHaveBeenCalledWith("hello"); + log.mockRestore(); + }); +}); + +describe("run", () => { + it("sets the exit code from the repository's own gate", async () => { + const log = vi.spyOn(console, "log").mockImplementation(() => {}); + const process: { env: Record; exitCode?: number } = { + env: { GITHUB_EVENT_NAME: "pull_request", GITHUB_REF: "refs/pull/1/merge" }, + }; + await run(process); + expect(process.exitCode).toBe(0); + expect(log).toHaveBeenCalledWith("A pull_request run on refs/pull/1/merge: every job runs."); + log.mockRestore(); + }); +}); diff --git a/scripts/ciGate.ts b/scripts/ciGate.ts new file mode 100644 index 00000000..0e7f52c3 --- /dev/null +++ b/scripts/ciGate.ts @@ -0,0 +1,225 @@ +// SPDX-FileCopyrightText: 2026 Anton Wiklund and Solid Memo contributors +// +// SPDX-License-Identifier: MIT + +/** + * `npm run ci:gate`: which of CI's jobs this run runs + * (docs/releasing.md#what-each-stage-checks): every job, or only the + * deterministic ones, the timing-dependent ones (real servers, the + * journeys in Firefox, WebKit and on a phone) skipped where the same + * tree already passed them. Every job that runs blocks. + * + * - A pull request, the merge queue and a run by hand: every job. The + * merge queue is where a tree is tested at full strength before it + * lands, so a flaky check is fixed there, not retried later. + * - A push to main whose commit a merge_group run passed: the + * deterministic jobs. The same tree passed every job already. + * - A push to main that skipped the queue (a merge through a ruleset + * bypass, such as an advisory's): every job, as in the queue, so a + * release built on it waits for the full run. + * - A release commit, whose parent passed CI on main: the deterministic + * jobs, and the commit must change nothing but the files a release + * writes (release.config.mjs), or the gate fails. Its code is its + * parent's, already tested. + * + * It writes `jobs=all` or `jobs=deterministic` to GITHUB_OUTPUT, and with + * the latter `tested-by=`: the CI run that ran the timing-dependent + * jobs on this tree, whose engines' journeys the report on main takes in. + * When that run's report of them has expired (GitHub keeps it 7 days, as + * after a release made late by hand), every job runs instead, so the + * report is never short of them. + */ +import { execFileSync } from "node:child_process"; +import { appendFileSync } from "node:fs"; +import { resolve } from "node:path"; + +/** The files a release commits (release.config.mjs), each naming the version. */ +const { assets } = (await import(new URL("../release.config.mjs", import.meta.url).href)) as { + assets: string[]; +}; + +/** + * What a run runs, every job or the deterministic ones alone, and, when + * those alone, the run that ran the others on this code. + */ +export type Gate = + | { jobs: "all"; why: string } + | { jobs: "deterministic"; testedBy: number; why: string }; + +const MAIN = "refs/heads/main"; +/** The artifact of the engines' journeys that main's report takes from the run that ran them. */ +export const ENGINES_BLOB = "journeys-blob-engines"; +const RELEASE = "chore(release): "; + +/** Whether a commit is a release commit, as ci.yml's concurrency group tells them apart. */ +export function isRelease(subject: string): boolean { + return subject.startsWith(RELEASE); +} + +/** The files a release commit changes that a release never writes. */ +export function strayFiles(changed: readonly string[]): string[] { + return changed.filter((file) => !assets.includes(file)); +} + +/** What the gate reads. */ +export interface GateIo { + /** The run's event (GITHUB_EVENT_NAME) and ref (GITHUB_REF). */ + event: string; + ref: string; + /** The commit checked: its hash, subject and parents. */ + head(): { sha: string; subject: string; parents: string[] }; + /** The files a commit changes against its parent. */ + changed(sha: string): string[]; + /** The id of a CI run for `event` that passed on the commit, if there is one. */ + passed(sha: string, event: "merge_group" | "push"): Promise; + /** Whether a run still has an artifact of that name, not expired. */ + kept(run: number, artifact: string): Promise; + output(name: string, value: string): void; + log(message: string): void; +} + +/** The deterministic jobs, the others having run in `testedBy`, while its engines' report is kept. */ +async function testedIn(io: GateIo, testedBy: number, why: string): Promise { + if (await io.kept(testedBy, ENGINES_BLOB)) { + return { jobs: "deterministic", testedBy, why: `${why}: the deterministic jobs run.` }; + } + return { + jobs: "all", + why: `${why}, but its ${ENGINES_BLOB} has expired: every job runs, so the journeys' report has the engines'.`, + }; +} + +/** What this run runs, and why; throws when a release commit is not one. */ +export async function decide(io: GateIo): Promise { + if (io.event !== "push" || io.ref !== MAIN) { + return { jobs: "all", why: `A ${io.event} run on ${io.ref}: every job runs.` }; + } + const { sha, subject, parents } = io.head(); + if (isRelease(subject)) { + if (parents.length !== 1) { + throw new Error(`The release commit ${sha} has ${parents.length} parents, not one.`); + } + const stray = strayFiles(io.changed(sha)); + if (stray.length > 0) { + throw new Error( + `The release commit ${sha} changes ${stray.join(", ")}, which a release never writes (release.config.mjs): only ${assets.join(", ")}.`, + ); + } + const parent = parents[0]!; + const onMain = await io.passed(parent, "push"); + if (onMain === undefined) { + return { + jobs: "all", + why: `A release commit on ${parent.slice(0, 8)}, which CI has not passed on main: every job runs.`, + }; + } + // The parent's push run ran every job only when the queue had not. + const testedBy = (await io.passed(parent, "merge_group")) ?? onMain; + return testedIn( + io, + testedBy, + `A release commit, changing only the version's files, on ${parent.slice(0, 8)}, which CI passed on main; run ${testedBy} ran every job on its code`, + ); + } + const queued = await io.passed(sha, "merge_group"); + if (queued !== undefined) { + return testedIn(io, queued, `${sha.slice(0, 8)} passed the merge queue in run ${queued}`); + } + return { + jobs: "all", + why: `${sha.slice(0, 8)} reached main without passing the merge queue: every job runs, as in the queue.`, + }; +} + +/** Exit code: 0 once the gate is written, 1 when a release commit is not one. */ +export async function main(io: GateIo): Promise { + try { + const gate = await decide(io); + io.log(gate.why); + io.output("jobs", gate.jobs); + if (gate.jobs === "deterministic") io.output("tested-by", String(gate.testedBy)); + return 0; + } catch (error) { + io.log((error as Error).message); + return 1; + } +} + +/** The environment a GitHub Actions step gives the gate. */ +export type GateEnv = Partial< + Record< + | "GITHUB_EVENT_NAME" + | "GITHUB_REF" + | "GITHUB_REPOSITORY" + | "GITHUB_API_URL" + | "GITHUB_OUTPUT" + | "GH_TOKEN", + string + > +>; + +/** The io of the repository at `repo`, in a run with environment `env`. */ +export function defaultIo(repo: string, env: GateEnv, fetch: typeof globalThis.fetch): GateIo { + const git = (...args: string[]) => + execFileSync("git", ["-C", repo, ...args], { encoding: "utf8" }).trim(); + /** A GET of the repository's REST API, its JSON answer. */ + const api = async (path: string, query: Record): Promise => { + const url = new URL( + `repos/${env.GITHUB_REPOSITORY}/${path}`, + `${env.GITHUB_API_URL ?? "https://api.github.com"}/`, + ); + url.search = new URLSearchParams(query).toString(); + const response = await fetch(url, { + headers: { + accept: "application/vnd.github+json", + authorization: `Bearer ${env.GH_TOKEN}`, + "x-github-api-version": "2022-11-28", + }, + }); + if (!response.ok) throw new Error(`GitHub answered ${response.status} for ${url}.`); + return response.json(); + }; + return { + event: env.GITHUB_EVENT_NAME ?? "", + ref: env.GITHUB_REF ?? "", + head: () => { + const [sha, parents, subject] = git("log", "-1", "--format=%H%x1f%P%x1f%s", "HEAD").split( + "\x1f", + ); + return { sha: sha!, subject: subject!, parents: parents!.split(" ").filter(Boolean) }; + }, + changed: (sha) => + git("diff", "--name-only", `${sha}^`, sha) + .split("\n") + .filter((file) => file !== ""), + passed: async (sha, event) => + ( + (await api(`actions/workflows/ci.yml/runs`, { + head_sha: sha, + event, + status: "success", + per_page: "1", + })) as { workflow_runs: { id: number }[] } + ).workflow_runs[0]?.id, + kept: async (run, artifact) => + ( + (await api(`actions/runs/${run}/artifacts`, { name: artifact })) as { + artifacts: { expired: boolean }[]; + } + ).artifacts.some(({ expired }) => !expired), + output: (name, value) => { + if (env.GITHUB_OUTPUT !== undefined) appendFileSync(env.GITHUB_OUTPUT, `${name}=${value}\n`); + }, + log: (message) => console.log(message), + }; +} + +/** The script entry: `node -e "import('./scripts/ciGate.ts').then((m) => m.run(process))"`. */ +export async function run(process: { + env: GateEnv; + exitCode?: number | string | null | undefined; +}): Promise { + process.exitCode = await main( + defaultIo(resolve(import.meta.dirname, ".."), process.env, globalThis.fetch), + ); +} From 6e429a494ddffea5096845c2f19e2916eb7ab8ec Mon Sep 17 00:00:00 2001 From: Anton Wiklund Date: Tue, 20 Oct 2026 14:48:38 +0200 Subject: [PATCH 3/4] ci: skip the Chromium journeys too where the queue ran them The Chromium journeys were kept on every push to main and on the release commit, as the deterministic jobs were, although they wait on a real server behind TLS as the other engines' do, and although that tree had just passed them in the merge queue. They are the run's longest job, about 8 minutes, spent twice more on main for every merged pull request. Both journeys jobs are now timing-dependent, and skipped where the gate skips the others. The merge queue keeps both jobs' traces and blobs, and main's report takes both from the run the gate names, which must still keep them. The warm jobs also refill the journeys' server image and Chromium's browsers, which the Chromium job saved on main. A skipped warm-images job no longer shows its name's unexpanded matrix. Refs: #199 --- .github/workflows/ci.yml | 169 +++++++++++++++++++++++---------------- docs/deployment.md | 2 +- docs/releasing.md | 22 ++--- docs/testing.md | 40 ++++----- scripts/ciGate.test.ts | 17 ++-- scripts/ciGate.ts | 20 ++--- 6 files changed, 153 insertions(+), 117 deletions(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 57a03951..90bfc81a 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -68,8 +68,7 @@ jobs: # fails unless the commit changes nothing but the files a release # writes. Only on a push to main does it ask GitHub's API: whether the # merge queue passed the commit, or CI on main a release commit's - # parent. It also gives journeys its matrix, the engines left out when - # the timing-dependent jobs are. Deterministic. + # parent. Deterministic. gate: runs-on: ubuntu-latest timeout-minutes: 5 @@ -80,7 +79,6 @@ jobs: outputs: jobs: ${{ steps.gate.outputs.jobs }} tested-by: ${{ steps.gate.outputs.tested-by }} - journeys: ${{ steps.journeys.outputs.matrix }} steps: - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 with: @@ -94,18 +92,6 @@ jobs: env: GH_TOKEN: ${{ github.token }} run: npm run ci:gate - # The build draws the docs' diagrams in Chromium's headless shell, - # which the engines' job installs beside their browsers. - - id: journeys - env: - JOBS: ${{ steps.gate.outputs.jobs }} - MATRIX: >- - [{"name": "chromium", "projects": "--project=chromium", - "browsers": "chromium", "timing": false}, - {"name": "engines", "projects": "--project=firefox --project=webkit --project=phone", - "browsers": "chromium-headless-shell firefox webkit", "timing": true}] - run: | - echo "matrix=$(jq -c --arg jobs "$JOBS" 'map(select($jobs == "all" or (.timing | not)))' <<< "$MATRIX")" >> "$GITHUB_OUTPUT" # Deterministic. Its growth tests time CPU work, but as the ratio of two # runs on the same runner, which a busy runner leaves about the same @@ -386,30 +372,34 @@ jobs: # (docs/testing.md#browsers). When one fails, the job's summary names the # step it failed in, and its artifact holds the trace to replay it step # by step (`npx playwright show-trace`), the report, a video, and the - # server's log. On main the traces are kept always, and each job keeps - # its run as a blob, for journeys-report to merge; so do the engines in - # the merge queue, as main's report takes theirs from the run that - # tested its tree when it skips them (gate's tested-by). - # - # Chromium's journeys run like a deterministic job: they are every - # journey, in the browser most people use, and their waits are fixed - # where they flake, not retried (retries: 0, playwright.config.ts). The - # engines' are timing-dependent: slower browsers on a shared runner. - # The matrix is gate's (name, projects, browsers, timing), without the - # engines when the timing-dependent jobs are skipped. + # server's log. On main and in the merge queue the traces are kept + # always, and each job keeps its run as a blob, for journeys-report to + # merge: main's report takes them from the run that tested its tree + # when it skips them (gate's tested-by), the queue's as a rule. + # Timing-dependent: end to end, waiting on a real server behind TLS, + # on a shared runner. Their waits are fixed where they flake, not + # retried (retries: 0, playwright.config.ts). journeys: name: journeys (${{ matrix.name }}) needs: gate + if: needs.gate.outputs.jobs == 'all' runs-on: ubuntu-latest timeout-minutes: 25 strategy: fail-fast: false matrix: - include: ${{ fromJSON(needs.gate.outputs.journeys) }} + include: + - name: chromium + projects: --project=chromium + browsers: chromium + # The build draws the docs' diagrams in Chromium's headless shell. + - name: engines + projects: --project=firefox --project=webkit --project=phone + browsers: chromium-headless-shell firefox webkit env: JOURNEY_TRACE: >- ${{ (inputs.trace || github.ref == 'refs/heads/main' || - (github.event_name == 'merge_group' && matrix.timing)) && 'on' || '' }} + github.event_name == 'merge_group') && 'on' || '' }} # The blob's name, so the two jobs' blobs sit side by side when merged. PLAYWRIGHT_BLOB_OUTPUT_NAME: report-${{ matrix.name }}.zip steps: @@ -452,7 +442,8 @@ jobs: - run: npm run journeys -- ${{ matrix.projects }} - if: always() run: node e2e/journeys/harness/summary.ts journeys-${{ matrix.name }}-${{ github.run_attempt }} - - if: failure() || env.JOURNEY_TRACE == 'on' + # In the queue, the blob below holds the traces; this, only on a failure. + - if: failure() || (env.JOURNEY_TRACE == 'on' && github.event_name != 'merge_group') uses: actions/upload-artifact@cf430e030ddbb5b0abf93d22962f4752f3646cd9 # v7.0.2 with: name: journeys-${{ matrix.name }}-${{ github.run_attempt }} @@ -464,7 +455,7 @@ jobs: # A job run again replaces its blob from the attempt before; kept as # long as the report, so that re-running one job still finds the # other's. - - if: always() && (github.ref == 'refs/heads/main' || (github.event_name == 'merge_group' && matrix.timing)) + - if: always() && (github.ref == 'refs/heads/main' || github.event_name == 'merge_group') uses: actions/upload-artifact@cf430e030ddbb5b0abf93d22962f4752f3646cd9 # v7.0.2 with: name: journeys-blob-${{ matrix.name }} @@ -475,10 +466,11 @@ jobs: # On main, the two journeys jobs' runs merged into one report, every # browser's journeys and their traces in it, even when a journey failed: # the deploy publishes it at /journeys/ (docs/deployment.md). When this - # run skipped the engines, theirs come from the run gate names, which - # ran them on this tree: the merge queue's, or a release commit's - # parent's. When the journeys passed, both blobs must be there, so a run - # the deploy may publish never has a report missing a browser. ci-ok + # run skipped the journeys, their blobs come from the run gate names, + # which ran them on this tree: the merge queue's, or a release commit's + # parent's. When the journeys passed, or ran there, both blobs must be + # there, so a run the deploy may publish never has a report missing a + # browser. ci-ok # doesn't need it, as it runs on main alone; its failure still fails the # run, and so holds back release and deploy. Deterministic. journeys-report: @@ -488,7 +480,7 @@ jobs: timeout-minutes: 10 permissions: contents: read - # To download the engines' blob from the run that tested this tree. + # To download the journeys' blobs from the run that tested this tree. actions: read steps: - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 @@ -499,7 +491,8 @@ jobs: node-version-file: .nvmrc cache: npm - run: npm ci - - uses: actions/download-artifact@9000827ccba6bdab643e8b6fd33ac0654aef8333 # v8.0.2 + - if: needs.gate.outputs.jobs == 'all' + uses: actions/download-artifact@9000827ccba6bdab643e8b6fd33ac0654aef8333 # v8.0.2 with: pattern: journeys-blob-* merge-multiple: true @@ -507,15 +500,17 @@ jobs: - if: needs.gate.outputs.jobs == 'deterministic' uses: actions/download-artifact@9000827ccba6bdab643e8b6fd33ac0654aef8333 # v8.0.2 with: - name: journeys-blob-engines + pattern: journeys-blob-* + merge-multiple: true run-id: ${{ needs.gate.outputs.tested-by }} github-token: ${{ github.token }} path: e2e/journeys/blob-reports/ - working-directory: e2e/journeys env: JOURNEYS: ${{ needs.journeys.result }} + JOBS: ${{ needs.gate.outputs.jobs }} run: | - if [ "$JOURNEYS" = success ]; then + if [ "$JOURNEYS" = success ] || [ "$JOBS" = deterministic ]; then test -f blob-reports/report-chromium.zip test -f blob-reports/report-engines.zip fi @@ -532,12 +527,12 @@ jobs: # On main, when the timing-dependent jobs are skipped, the caches they # would have saved there, which every pull request and queue run reads: - # each server's images (e2e-images; the journeys' are saved by Chromium's - # job, which still runs) and the engines' browsers. This job only looks - # them up, by the keys those jobs use; warm-images and warm-browsers - # make and save what is missing, and nothing else, so a run with every - # cache in place spends one short job. Like caches, it may fail without - # failing the run. Deterministic. + # each server's images and the journeys' (e2e-images), and the + # journeys' browsers. This job only looks them up, by the keys those + # jobs use; warm-images and warm-browsers make and save what is + # missing, and nothing else, so a run with every cache in place spends + # one short job. Like caches, they may fail without failing the run. + # Deterministic. warm: if: github.ref == 'refs/heads/main' && needs.gate.outputs.jobs == 'deterministic' needs: gate @@ -549,7 +544,7 @@ jobs: # To list main's caches. actions: read outputs: - servers: ${{ steps.missing.outputs.servers }} + images: ${{ steps.missing.outputs.images }} browsers: ${{ steps.missing.outputs.browsers }} steps: - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 @@ -568,6 +563,7 @@ jobs: env: GH_TOKEN: ${{ github.token }} GH_REPO: ${{ github.repository }} + JOURNEYS_KEY: ${{ hashFiles('e2e/journeys/css/compose.yml') }} run: | # Listed once each, outside any condition, where a failure # fails the step: inside one, bash would read it as a miss. @@ -576,23 +572,27 @@ jobs: cached() { jq -se --arg key "$1" 'add | any(.[]; .key == $key)' <<< "$listed" > /dev/null } - keys=$(node e2e/pod/servers.ts keys all) - servers=() - while read -r id key; do - cached "e2e-images-$RUNNER_OS-$RUNNER_ARCH-pod-$id-$key" || servers+=("$id") + # Named as e2e-images names them: pod-, and journeys. + keys=$(node e2e/pod/servers.ts keys all | sed 's/^/pod-/') + keys+=$'\n'"journeys $JOURNEYS_KEY" + images=() + while read -r name key; do + cached "e2e-images-$RUNNER_OS-$RUNNER_ARCH-$name-$key" || images+=("$name") done <<< "$keys" - echo "Images missing: ${servers[*]:-none}" - echo "servers=$(jq -cn '$ARGS.positional' --args "${servers[@]}")" >> "$GITHUB_OUTPUT" + echo "Images missing: ${images[*]:-none}" + echo "images=$(jq -cn '$ARGS.positional' --args "${images[@]}")" >> "$GITHUB_OUTPUT" version=$(node -p "require('@playwright/test/package.json').version") - if cached "playwright-$RUNNER_OS-$version-engines"; then browsers=false; else browsers=true; fi - echo "Engines' browsers missing: $browsers" - echo "browsers=$browsers" >> "$GITHUB_OUTPUT" + browsers=() + for name in chromium engines; do + cached "playwright-$RUNNER_OS-$version-$name" || browsers+=("$name") + done + echo "Browsers missing: ${browsers[*]:-none}" + echo "browsers=$(jq -cn '$ARGS.positional' --args "${browsers[@]}")" >> "$GITHUB_OUTPUT" - # A missing server's images, pulled or built as pod and contract would, + # Missing images, pulled or built as pod, contract and journeys would, # and saved under their key. Deterministic. warm-images: - name: warm-images (${{ matrix.server }}) - if: needs.warm.outputs.servers != '' && needs.warm.outputs.servers != '[]' + if: needs.warm.outputs.images != '' && needs.warm.outputs.images != '[]' needs: warm runs-on: ubuntu-latest timeout-minutes: 15 @@ -600,11 +600,11 @@ jobs: strategy: fail-fast: false matrix: - server: ${{ fromJSON(needs.warm.outputs.servers) }} - # The servers come from an output, so they reach the shell as a + images: ${{ fromJSON(needs.warm.outputs.images) }} + # The names come from an output, so they reach the shell as a # variable, never written into a script. env: - SERVER: ${{ matrix.server }} + IMAGES: ${{ matrix.images }} steps: - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 with: @@ -615,31 +615,53 @@ jobs: cache: npm - run: npm ci - id: key - run: echo "key=$(node e2e/pod/servers.ts key "$SERVER")" >> "$GITHUB_OUTPUT" + env: + JOURNEYS_KEY: ${{ hashFiles('e2e/journeys/css/compose.yml') }} + run: | + if [ "$IMAGES" = journeys ]; then + echo "key=$JOURNEYS_KEY" >> "$GITHUB_OUTPUT" + echo "compose=e2e/journeys/css/compose.yml" >> "$GITHUB_OUTPUT" + else + echo "key=$(node e2e/pod/servers.ts key "${IMAGES#pod-}")" >> "$GITHUB_OUTPUT" + echo "compose=e2e/pod/servers/${IMAGES#pod-}/compose.yml" >> "$GITHUB_OUTPUT" + fi - id: images uses: ./.github/actions/e2e-images with: - name: pod-${{ matrix.server }} + name: ${{ matrix.images }} key: ${{ steps.key.outputs.key }} - - run: npm run servers -w @solid-memo/e2e-pod -- "$SERVER" + - run: | + if [ "$IMAGES" = journeys ]; then + npm run css:prepare -w @solid-memo/e2e-journeys + else + npm run servers -w @solid-memo/e2e-pod -- "${IMAGES#pod-}" + fi - if: steps.images.outputs.hit != 'true' env: FILE: ${{ steps.images.outputs.file }} - run: node e2e/pod/images.ts save "$FILE" "e2e/pod/servers/$SERVER/compose.yml" + COMPOSE: ${{ steps.key.outputs.compose }} + run: node e2e/pod/images.ts save "$FILE" "$COMPOSE" - if: steps.images.outputs.hit != 'true' uses: actions/cache/save@55cc8345863c7cc4c66a329aec7e433d2d1c52a9 # v6.1.0 with: path: ${{ steps.images.outputs.file }} key: ${{ steps.images.outputs.key }} - # The engines' browsers, downloaded as journeys would, and saved under - # the key it restores them by. Deterministic. + # A journeys job's missing browsers, downloaded as it would (its + # matrix's browsers), and saved under the key it restores them by. + # Deterministic. warm-browsers: - if: needs.warm.outputs.browsers == 'true' + if: needs.warm.outputs.browsers != '' && needs.warm.outputs.browsers != '[]' needs: warm runs-on: ubuntu-latest timeout-minutes: 10 continue-on-error: true + strategy: + fail-fast: false + matrix: + browsers: ${{ fromJSON(needs.warm.outputs.browsers) }} + env: + BROWSERS: ${{ matrix.browsers }} steps: - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 with: @@ -651,11 +673,16 @@ jobs: - run: npm ci - id: playwright run: echo "version=$(node -p "require('@playwright/test/package.json').version")" >> "$GITHUB_OUTPUT" - - run: npx playwright install chromium-headless-shell firefox webkit + - run: | + if [ "$BROWSERS" = chromium ]; then + npx playwright install chromium + else + npx playwright install chromium-headless-shell firefox webkit + fi - uses: actions/cache/save@55cc8345863c7cc4c66a329aec7e433d2d1c52a9 # v6.1.0 with: path: ~/.cache/ms-playwright - key: playwright-${{ runner.os }}-${{ steps.playwright.outputs.version }}-engines + key: playwright-${{ runner.os }}-${{ steps.playwright.outputs.version }}-${{ matrix.browsers }} # The docs (docs/testing.md#documentation): every link to a file or a # heading in the repository resolves (lychee, offline: the web's @@ -805,9 +832,9 @@ jobs: RESULTS: ${{ join(needs.*.result, ' ') }} DETERMINISTIC: >- ${{ needs.gate.result }} ${{ needs.build.result }} ${{ needs.library.result }} - ${{ needs.crosscheck.result }} ${{ needs.journeys.result }} ${{ needs.docs.result }} + ${{ needs.crosscheck.result }} ${{ needs.docs.result }} ${{ needs.workflows.result }} ${{ needs.commits.result }} - TIMING: ${{ needs.pod.result }} ${{ needs.contract.result }} + TIMING: ${{ needs.pod.result }} ${{ needs.contract.result }} ${{ needs.journeys.result }} JOBS: ${{ needs.gate.outputs.jobs }} run: | echo "The jobs' results: $RESULTS (gate: ${JOBS:-none})" diff --git a/docs/deployment.md b/docs/deployment.md index 1e140af7..91751b1e 100644 --- a/docs/deployment.md +++ b/docs/deployment.md @@ -18,7 +18,7 @@ is made: flowchart LR accTitle: When a version is deployed accDescr: A push to main, or a run by hand, deploys only the newest version, from its release commit and with its GitHub release, after the build, each release's docs and the guard. - push["push to main"] --> ci["CI: npm run check,
build, pySHACL,
journeys, docs"] -->|success| tag{"deploy.yml:
a version tag
names the commit?"} + push["push to main"] --> ci["CI: npm run check,
build, pySHACL, docs"] -->|success| tag{"deploy.yml:
a version tag
names the commit?"} tag -->|no| skip["not deployed"] tag -->|yes| release{"the newest version,
its release commit,
its GitHub release?"} release -->|no| skip diff --git a/docs/releasing.md b/docs/releasing.md index 4a0c3448..dba2a052 100644 --- a/docs/releasing.md +++ b/docs/releasing.md @@ -254,24 +254,24 @@ version's files: the five that `release.config.mjs` lists (`assets`): `CHANGELOG.md`, `CITATION.cff`, `package.json`, `package-lock.json` and `publiccode.yml`. Its code is its parent's, which CI passed on `main` before the release was made. So its run runs the deterministic -jobs, the build and the journeys in Chromium among them, and the gate -fails it if the commit changes any other file, or has more than one -parent. The timing-dependent jobs are skipped, and cannot hold up the -deploy. Should CI not have passed its parent on `main` (a release +jobs, the build among them, and the gate fails it if the commit changes +any other file, or has more than one parent. The timing-dependent jobs, +the user journeys in every browser among them, are skipped, and cannot +hold up the deploy. Should CI not have passed its parent on `main` (a release commit pushed by hand), every job runs. The report of the journeys the site publishes at `/journeys/` ([deployment.md](deployment.md)) still has every browser's: the release -commit's run takes the engines' journeys from the run that ran them on -its code, which the gate names (`tested-by`). That is its parent's -merge queue run, which keeps the engines' traces and report for this, -or, for a parent that skipped the queue, the parent's own push run. -GitHub keeps that run's report of them for 7 days; once it has expired, -as for a release made late by hand, the gate runs every job instead. +commit's run takes the journeys from the run that ran them on its code, +which the gate names (`tested-by`). That is its parent's merge queue +run, which keeps the journeys' traces and reports for this, or, for a +parent that skipped the queue, the parent's own push run. GitHub keeps +that run's reports for 7 days; once one has expired, as for a release +made late by hand, the gate runs every job instead. What a skipped job would have saved on `main`, the caches every pull request reads, CI's `warm` jobs save instead: `warm` looks up each -server's images and the engines' browsers, and `warm-images` and +server's images, the journeys' server's, and the journeys' browsers, and `warm-images` and `warm-browsers` pull, build or download only what is missing, and save it ([testing.md](testing.md#continuous-integration)). diff --git a/docs/testing.md b/docs/testing.md index 715a4862..694aaee9 100644 --- a/docs/testing.md +++ b/docs/testing.md @@ -308,9 +308,9 @@ PATCH Swedish text (see above). The journeys need Docker and the browsers (`npm run browsers -w @solid-memo/e2e-journeys` installs Chromium, Firefox and WebKit, with the system libraries they need), and CI runs them on every pull request and in the merge queue, and on `main` -those in Chromium always, the others only where the code has not passed -them yet ([releasing.md](releasing.md#what-each-stage-checks)). They -block, as the blocking servers do. +only where the code has not passed them yet +([releasing.md](releasing.md#what-each-stage-checks)). They block, as +the blocking servers do. Further journeys go through: @@ -444,15 +444,15 @@ error, and the browser it failed in. The job's published with each deploy: the report of the CI run on the release commit the site runs (none after a deploy by hand), every browser's journeys in one, as the `journeys-report` job merges them from both -journeys jobs (Playwright's blob reports); the engines', when the -release commit's run skips them, from the run that ran them on its code, -usually the merge queue's +journeys jobs (Playwright's blob reports). As the release commit's run +skips the journeys, it takes their reports from the run that ran them +on its code, usually the merge queue's ([releasing.md](releasing.md#what-each-stage-checks)). It shows every journey's steps and screenshots, and its **Trace** link opens the step-by-step timeline in the browser (the trace viewer is part of the report and -needs it served, as it is there, not opened as a file). On `main`, and -for the engines in the merge queue, the journeys always keep their -traces for this. The report is public and +needs it served, as it is there, not opened as a file). In the merge +queue, and on `main` when its run runs them, the journeys always keep +their traces for this. The report is public and holds the throwaway accounts' passwords and tokens, which die with the run's server, and it is kept out of search engines (`robots.txt`). Since only a green run on a release commit deploys, a failure on @@ -555,28 +555,28 @@ jobs, side by side, each [deterministic or timing-dependent](#deterministic-and- | Job | Kind | What it checks | |---|---|---| -| `gate` | deterministic | what the run runs: every job, or the deterministic ones alone (`npm run ci:gate`, [releasing.md](releasing.md#what-each-stage-checks)); on a release commit, that it changes nothing but the files a release writes; and the journeys' matrix, without the engines when they are skipped | +| `gate` | deterministic | what the run runs: every job, or the deterministic ones alone (`npm run ci:gate`, [releasing.md](releasing.md#what-each-stage-checks)); on a release commit, that it changes nothing but the files a release writes | | `build` | deterministic | `npm run check:ci`, which is `npm run check` without the deck library (the `library` job checks it), then `npm run build`, with Chromium's headless shell from GitHub's cache or installed ([.github/actions/chromium](../.github/actions/chromium/action.yml)) for the docs' diagrams; the coverage of every package goes to the run's summary (`npm run coverage:summary`). Turbo's cache is kept between runs under a key of this job's own, so a task whose inputs have not changed is not run again; only a run on `main` saves it, and a pull request and the merge queue start from `main`'s | | `library` | deterministic | the change against its base: the deck library, no published version edited or removed, and every new version naming the library's contact ([deck-library.md](deck-library.md#checks)); the public API, no line of `api/` gone unless a commit says the change breaks, and none of `api/frozen/` at all (`npm run check:breaking`, [releasing.md](releasing.md#how-ci-holds-to-it)) | | `crosscheck` | deterministic | the Python scripts' tests (the cross-check's and the taxonomy deck generator's), then the vocabulary, the deck library and the pod catalog documents, by pySHACL ([validation.md](validation.md#the-ci-cross-check)) | | `pod ()` | timing-dependent | the end-to-end tests, one job per blocking server (above) | | `contract ()` | timing-dependent | that each advisory server starts and meets the contract | -| `journeys ()` | deterministic in Chromium, timing-dependent in the other engines | the user journeys: every one in Chromium, and those that differ between engines in Firefox, WebKit and on a phone ([Browsers](#browsers)) | -| `journeys-report` | deterministic | nothing: on `main`, after both `journeys` jobs, even when one failed, it merges their runs into one report, the `journeys-report` artifact the deploy publishes at `/journeys/` ([User journeys](#user-journeys)); when this run skipped the engines, their run comes from the run that tested its code (the gate's `tested-by`). When the journeys passed, it fails without both their reports; its failure fails the run, so it holds back the release and the deploy, but not `ci-ok` | +| `journeys ()` | timing-dependent | the user journeys: every one in Chromium, and those that differ between engines in Firefox, WebKit and on a phone ([Browsers](#browsers)) | +| `journeys-report` | deterministic | nothing: on `main`, after both `journeys` jobs, even when one failed, it merges their runs into one report, the `journeys-report` artifact the deploy publishes at `/journeys/` ([User journeys](#user-journeys)); when this run skipped them, their runs come from the run that tested its code (the gate's `tested-by`). When the journeys passed, or were skipped, it fails without both their reports; its failure fails the run, so it holds back the release and the deploy, but not `ci-ok` | | `docs` | deterministic | the docs' links, every file's license, and `publiccode.yml` ([Documentation](#documentation)); their style and spelling are `build`'s | | `workflows` | deterministic | the workflows and local actions, by actionlint and zizmor (`npm run lint:workflows`, [The workflows](#the-workflows)) | | `commits` | deterministic | on a pull request, that each commit it adds follows Conventional Commits, by commitlint ([CONTRIBUTING.md](../CONTRIBUTING.md#commit-messages)), and that its body or a commit closes its issue, or the body says it is one step of it (`npm run check:closes`; not for Renovate's); in the merge queue or on a push to `main`, nothing | | `caches` | deterministic | nothing: on `main`, after `build`, it deletes `main`'s older Turbo caches, the newest holding what they do. The one job whose token can write (`actions: write`, to delete caches); it runs none of the repository's code, and its failure fails neither the run nor `ci-ok` | -| `warm`, `warm-images`, `warm-browsers` | deterministic | nothing: on `main`, when the timing-dependent jobs are skipped, `warm` looks up the caches they would have saved, each server's images and the engines' browsers, and the other two pull, build or download only what is missing, and save it. Like `caches`, they fail neither the run nor `ci-ok` | -| `ci-ok` | deterministic | that every job above passed, `journeys-report`, `caches` and the `warm` jobs aside; `pod` and `contract` may be skipped only when the gate skipped them | +| `warm`, `warm-images`, `warm-browsers` | deterministic | nothing: on `main`, when the timing-dependent jobs are skipped, `warm` looks up the caches they would have saved, each server's images, the journeys' server's and the journeys' browsers, and the other two pull, build or download only what is missing, and save it. Like `caches`, they fail neither the run nor `ci-ok` | +| `ci-ok` | deterministic | that every job above passed, `journeys-report`, `caches` and the `warm` jobs aside; `pod`, `contract` and `journeys` may be skipped only when the gate skipped them | **`ci-ok` is the one check a merge to `main` waits for**: the ruleset on `main` (a setting of the repository, Settings › Rules) is to require it and nothing else, and Renovate merges its updates through a pull request once it passes (`automergeType: 'pr'` in `renovate.json5`), as anyone's change is merged. It runs even when a job -before it failed, and fails unless every one passed, or, for `pod` and -`contract`, was skipped by the gate. A matrix job's +before it failed, and fails unless every one passed, or, for `pod`, +`contract` and `journeys`, was skipped by the gate. A matrix job's checks are named per server, and jobs come and go, so the ruleset names this job alone. (Another workflow's job named `build`, deploy.yml's, would also answer to a required check of that name.) Deploys follow a @@ -591,7 +591,7 @@ A **deterministic** job gives the same answer on any runner: types, lint, unit tests, request counts, the docs. A **timing-dependent** one can fail on sound code because the runner was busy or a real server was slow: the end-to-end tests against real servers (`pod`, -`contract`) and the journeys in Firefox, WebKit and on a phone. Each +`contract`) and the user journeys in every browser. Each job in [ci.yml](../.github/workflows/ci.yml) says which it is, in its comment and in the table above, and a new job is given its kind when it is added, which decides where it runs @@ -601,9 +601,9 @@ timing-dependent one is skipped on a push to `main` the queue passed and on a release commit, whose code it already ran on. A job that runs always blocks. -The journeys in Chromium are end to end too, yet they run everywhere: -they are every journey, in the browser most people use, so a wait that -flakes there is fixed, not retried (`retries: 0` in +A timing-dependent job that fails where it runs is fixed, not retried: +the journeys' waits, for one, are made to hold, with no second try +(`retries: 0` in [playwright.config.ts](../e2e/journeys/playwright.config.ts)). **Count, don't time, where the answer can be counted.** A regression diff --git a/scripts/ciGate.test.ts b/scripts/ciGate.test.ts index 6250c053..6c9620de 100644 --- a/scripts/ciGate.test.ts +++ b/scripts/ciGate.test.ts @@ -10,7 +10,7 @@ import { afterEach, beforeEach, describe, expect, it, vi } from "vitest"; import { decide, defaultIo, - ENGINES_BLOB, + JOURNEYS_BLOBS, type GateIo, isRelease, main, @@ -92,14 +92,21 @@ describe("decide", () => { }); }); - it("runs every job on main when the queue run's report of the engines has expired", async () => { - const kept = vi.fn(async () => false); + it("runs every job on main when the queue run's journeys' blobs have expired", async () => { + const kept = vi.fn(async (_run: number, blob: string) => blob === "journeys-blob-chromium"); const io = fakeIo({ passed: runs({ [`${SHA}:merge_group`]: 9 }), kept }); expect(await decide(io)).toEqual({ jobs: "all", - why: "aaaaaaaa passed the merge queue in run 9, but its journeys-blob-engines has expired: every job runs, so the journeys' report has the engines'.", + why: "aaaaaaaa passed the merge queue in run 9, but its journeys-blob-engines expired: every job runs, so the journeys' report is whole.", }); - expect(kept).toHaveBeenCalledWith(9, ENGINES_BLOB); + expect(kept.mock.calls).toEqual(JOURNEYS_BLOBS.map((blob) => [9, blob])); + }); + + it("names every blob that expired", async () => { + const io = fakeIo({ passed: runs({ [`${SHA}:merge_group`]: 9 }), kept: async () => false }); + expect((await decide(io)).why).toContain( + "journeys-blob-chromium and journeys-blob-engines expired", + ); }); it("runs every job on main when the commit skipped the merge queue", async () => { diff --git a/scripts/ciGate.ts b/scripts/ciGate.ts index 0e7f52c3..0afbb515 100644 --- a/scripts/ciGate.ts +++ b/scripts/ciGate.ts @@ -6,8 +6,8 @@ * `npm run ci:gate`: which of CI's jobs this run runs * (docs/releasing.md#what-each-stage-checks): every job, or only the * deterministic ones, the timing-dependent ones (real servers, the - * journeys in Firefox, WebKit and on a phone) skipped where the same - * tree already passed them. Every job that runs blocks. + * user journeys in every browser) skipped where the same tree already + * passed them. Every job that runs blocks. * * - A pull request, the merge queue and a run by hand: every job. The * merge queue is where a tree is tested at full strength before it @@ -24,8 +24,8 @@ * * It writes `jobs=all` or `jobs=deterministic` to GITHUB_OUTPUT, and with * the latter `tested-by=`: the CI run that ran the timing-dependent - * jobs on this tree, whose engines' journeys the report on main takes in. - * When that run's report of them has expired (GitHub keeps it 7 days, as + * jobs on this tree, whose journeys the report on main takes in. When + * that run's blobs of them have expired (GitHub keeps them 7 days, as * after a release made late by hand), every job runs instead, so the * report is never short of them. */ @@ -47,8 +47,8 @@ export type Gate = | { jobs: "deterministic"; testedBy: number; why: string }; const MAIN = "refs/heads/main"; -/** The artifact of the engines' journeys that main's report takes from the run that ran them. */ -export const ENGINES_BLOB = "journeys-blob-engines"; +/** The journeys jobs' blobs, which main's report takes from the run that ran them. */ +export const JOURNEYS_BLOBS = ["journeys-blob-chromium", "journeys-blob-engines"]; const RELEASE = "chore(release): "; /** Whether a commit is a release commit, as ci.yml's concurrency group tells them apart. */ @@ -78,14 +78,16 @@ export interface GateIo { log(message: string): void; } -/** The deterministic jobs, the others having run in `testedBy`, while its engines' report is kept. */ +/** The deterministic jobs, the others having run in `testedBy`, while it keeps its journeys' blobs. */ async function testedIn(io: GateIo, testedBy: number, why: string): Promise { - if (await io.kept(testedBy, ENGINES_BLOB)) { + const gone = []; + for (const blob of JOURNEYS_BLOBS) if (!(await io.kept(testedBy, blob))) gone.push(blob); + if (gone.length === 0) { return { jobs: "deterministic", testedBy, why: `${why}: the deterministic jobs run.` }; } return { jobs: "all", - why: `${why}, but its ${ENGINES_BLOB} has expired: every job runs, so the journeys' report has the engines'.`, + why: `${why}, but its ${gone.join(" and ")} expired: every job runs, so the journeys' report is whole.`, }; } From b28aa1d6a5db2a13a8ca37dfe9e43b2b5c5f16df Mon Sep 17 00:00:00 2001 From: Anton Wiklund Date: Tue, 20 Oct 2026 14:51:39 +0200 Subject: [PATCH 4/4] ci(journeys): split each browser's journeys in two shards The journeys are the run's longest jobs, about 8.5 minutes each, and a pull request and the merge queue wait for them. Each browser's journeys now run in two shards side by side (npx playwright test --shard), so the run waits for about half as long. Each shard pays the job's setup again, about a minute, for the shorter wait. Each shard keeps a blob of its own, which journeys-report merges and requires, and the gate checks that the run it takes them from keeps its journeys-blob-* artifacts, whatever the number of shards. Refs: #199 --- .github/workflows/ci.yml | 51 +++++++++++++++++++++++++--------------- docs/testing.md | 17 ++++++++------ scripts/ciGate.test.ts | 36 ++++++++++++++-------------- scripts/ciGate.ts | 30 ++++++++++++----------- 4 files changed, 76 insertions(+), 58 deletions(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 90bfc81a..85c2d8ee 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -367,10 +367,13 @@ jobs: # The user journeys (e2e/journeys, docs/testing.md): the built app in # Chromium, logging in to a Community Solid Server 7 behind TLS that the # harness starts in Docker, each journey with an account and pod of its - # own. The other engines and the phone run a few of them, in a job of - # their own so neither job takes longer than one did - # (docs/testing.md#browsers). When one fails, the job's summary names the - # step it failed in, and its artifact holds the trace to replay it step + # own. The other engines and the phone run a few of them + # (docs/testing.md#browsers). Each browser's journeys are split in + # SHARDS jobs (npx playwright --shard), side by side, so the run waits + # for one shard's share of them, not for all: each shard pays the + # setup again (about a minute), for a shorter wall time. When one + # fails, the job's summary names the step it failed in, and its + # artifact holds the trace to replay it step # by step (`npx playwright show-trace`), the report, a video, and the # server's log. On main and in the merge queue the traces are kept # always, and each job keeps its run as a blob, for journeys-report to @@ -380,7 +383,7 @@ jobs: # on a shared runner. Their waits are fixed where they flake, not # retried (retries: 0, playwright.config.ts). journeys: - name: journeys (${{ matrix.name }}) + name: journeys (${{ matrix.name }}, ${{ matrix.shard }}/2) needs: gate if: needs.gate.outputs.jobs == 'all' runs-on: ubuntu-latest @@ -388,6 +391,10 @@ jobs: strategy: fail-fast: false matrix: + name: [chromium, engines] + # 1 to SHARDS, which journeys-report's env and the name above + # repeat. + shard: [1, 2] include: - name: chromium projects: --project=chromium @@ -400,8 +407,9 @@ jobs: JOURNEY_TRACE: >- ${{ (inputs.trace || github.ref == 'refs/heads/main' || github.event_name == 'merge_group') && 'on' || '' }} - # The blob's name, so the two jobs' blobs sit side by side when merged. - PLAYWRIGHT_BLOB_OUTPUT_NAME: report-${{ matrix.name }}.zip + SHARDS: 2 + # The blob's name, so the jobs' blobs sit side by side when merged. + PLAYWRIGHT_BLOB_OUTPUT_NAME: report-${{ matrix.name }}-${{ matrix.shard }}.zip steps: - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 with: @@ -439,14 +447,14 @@ jobs: with: path: ${{ steps.images.outputs.file }} key: ${{ steps.images.outputs.key }} - - run: npm run journeys -- ${{ matrix.projects }} + - run: npm run journeys -- ${{ matrix.projects }} --shard="${{ matrix.shard }}/$SHARDS" - if: always() - run: node e2e/journeys/harness/summary.ts journeys-${{ matrix.name }}-${{ github.run_attempt }} + run: node e2e/journeys/harness/summary.ts journeys-${{ matrix.name }}-${{ matrix.shard }}-${{ github.run_attempt }} # In the queue, the blob below holds the traces; this, only on a failure. - if: failure() || (env.JOURNEY_TRACE == 'on' && github.event_name != 'merge_group') uses: actions/upload-artifact@cf430e030ddbb5b0abf93d22962f4752f3646cd9 # v7.0.2 with: - name: journeys-${{ matrix.name }}-${{ github.run_attempt }} + name: journeys-${{ matrix.name }}-${{ matrix.shard }}-${{ github.run_attempt }} path: | e2e/journeys/playwright-report/ e2e/journeys/test-results/ @@ -458,21 +466,21 @@ jobs: - if: always() && (github.ref == 'refs/heads/main' || github.event_name == 'merge_group') uses: actions/upload-artifact@cf430e030ddbb5b0abf93d22962f4752f3646cd9 # v7.0.2 with: - name: journeys-blob-${{ matrix.name }} + name: journeys-blob-${{ matrix.name }}-${{ matrix.shard }} path: e2e/journeys/blob-report/ retention-days: 7 overwrite: true - # On main, the two journeys jobs' runs merged into one report, every + # On main, the journeys jobs' runs merged into one report, every # browser's journeys and their traces in it, even when a journey failed: # the deploy publishes it at /journeys/ (docs/deployment.md). When this # run skipped the journeys, their blobs come from the run gate names, # which ran them on this tree: the merge queue's, or a release commit's - # parent's. When the journeys passed, or ran there, both blobs must be - # there, so a run the deploy may publish never has a report missing a - # browser. ci-ok - # doesn't need it, as it runs on main alone; its failure still fails the - # run, and so holds back release and deploy. Deterministic. + # parent's. When the journeys passed, or ran there, every shard's blob + # must be there, so a run the deploy may publish never has a report + # missing a browser. ci-ok doesn't need it, as it runs on main alone; + # its failure still fails the run, and so holds back release and + # deploy. Deterministic. journeys-report: if: ${{ !cancelled() && github.ref == 'refs/heads/main' }} needs: [gate, journeys] @@ -509,10 +517,15 @@ jobs: env: JOURNEYS: ${{ needs.journeys.result }} JOBS: ${{ needs.gate.outputs.jobs }} + # As journeys' matrix has them. + SHARDS: 2 run: | if [ "$JOURNEYS" = success ] || [ "$JOBS" = deterministic ]; then - test -f blob-reports/report-chromium.zip - test -f blob-reports/report-engines.zip + for name in chromium engines; do + for shard in $(seq "$SHARDS"); do + test -f "blob-reports/report-$name-$shard.zip" + done + done fi if compgen -G 'blob-reports/*.zip' > /dev/null; then npx playwright merge-reports --reporter html blob-reports diff --git a/docs/testing.md b/docs/testing.md index 694aaee9..32616908 100644 --- a/docs/testing.md +++ b/docs/testing.md @@ -370,9 +370,12 @@ journeys test the app's logic, which is the same in every engine, so running them all in each would about triple the time for little gain. A journey tagged with one of the three runs in every browser; tag another only when what it tests differs between engines. CI runs Chromium's -journeys in one job and the other three projects' in another -(`journeys (chromium)`, `journeys (engines)`), so neither takes much -longer than one job did. The [user guide](user-guide.md#supported-browsers) +journeys apart from the other three projects', each split in two shards +(`npx playwright test --shard`), four jobs side by side: +`journeys (chromium, 1/2)` to `journeys (engines, 2/2)`. The run waits +for the slowest shard, about half of what one job took, and each shard +pays the job's setup again, about a minute. Playwright shards Chromium's +journeys by file, so a file's journeys stay in one shard. The [user guide](user-guide.md#supported-browsers) says which browsers Solid Memo supports. The engines differ in three ways the journeys allow for: @@ -443,8 +446,8 @@ error, and the browser it failed in. The job's **The latest report is online** at , published with each deploy: the report of the CI run on the release commit the site runs (none after a deploy by hand), every browser's -journeys in one, as the `journeys-report` job merges them from both -journeys jobs (Playwright's blob reports). As the release commit's run +journeys in one, as the `journeys-report` job merges them from every +journeys job (Playwright's blob reports, one a shard). As the release commit's run skips the journeys, it takes their reports from the run that ran them on its code, usually the merge queue's ([releasing.md](releasing.md#what-each-stage-checks)). It shows every @@ -561,8 +564,8 @@ jobs, side by side, each [deterministic or timing-dependent](#deterministic-and- | `crosscheck` | deterministic | the Python scripts' tests (the cross-check's and the taxonomy deck generator's), then the vocabulary, the deck library and the pod catalog documents, by pySHACL ([validation.md](validation.md#the-ci-cross-check)) | | `pod ()` | timing-dependent | the end-to-end tests, one job per blocking server (above) | | `contract ()` | timing-dependent | that each advisory server starts and meets the contract | -| `journeys ()` | timing-dependent | the user journeys: every one in Chromium, and those that differ between engines in Firefox, WebKit and on a phone ([Browsers](#browsers)) | -| `journeys-report` | deterministic | nothing: on `main`, after both `journeys` jobs, even when one failed, it merges their runs into one report, the `journeys-report` artifact the deploy publishes at `/journeys/` ([User journeys](#user-journeys)); when this run skipped them, their runs come from the run that tested its code (the gate's `tested-by`). When the journeys passed, or were skipped, it fails without both their reports; its failure fails the run, so it holds back the release and the deploy, but not `ci-ok` | +| `journeys (, )` | timing-dependent | the user journeys: every one in Chromium, and those that differ between engines in Firefox, WebKit and on a phone, each browser's in two shards ([Browsers](#browsers)) | +| `journeys-report` | deterministic | nothing: on `main`, after the `journeys` jobs, even when one failed, it merges their runs into one report, the `journeys-report` artifact the deploy publishes at `/journeys/` ([User journeys](#user-journeys)); when this run skipped them, their runs come from the run that tested its code (the gate's `tested-by`). When the journeys passed, or were skipped, it fails without every shard's report; its failure fails the run, so it holds back the release and the deploy, but not `ci-ok` | | `docs` | deterministic | the docs' links, every file's license, and `publiccode.yml` ([Documentation](#documentation)); their style and spelling are `build`'s | | `workflows` | deterministic | the workflows and local actions, by actionlint and zizmor (`npm run lint:workflows`, [The workflows](#the-workflows)) | | `commits` | deterministic | on a pull request, that each commit it adds follows Conventional Commits, by commitlint ([CONTRIBUTING.md](../CONTRIBUTING.md#commit-messages)), and that its body or a commit closes its issue, or the body says it is one step of it (`npm run check:closes`; not for Renovate's); in the merge queue or on a push to `main`, nothing | diff --git a/scripts/ciGate.test.ts b/scripts/ciGate.test.ts index 6c9620de..0b21dc31 100644 --- a/scripts/ciGate.test.ts +++ b/scripts/ciGate.test.ts @@ -93,20 +93,13 @@ describe("decide", () => { }); it("runs every job on main when the queue run's journeys' blobs have expired", async () => { - const kept = vi.fn(async (_run: number, blob: string) => blob === "journeys-blob-chromium"); + const kept = vi.fn(async () => false); const io = fakeIo({ passed: runs({ [`${SHA}:merge_group`]: 9 }), kept }); expect(await decide(io)).toEqual({ jobs: "all", - why: "aaaaaaaa passed the merge queue in run 9, but its journeys-blob-engines expired: every job runs, so the journeys' report is whole.", + why: "aaaaaaaa passed the merge queue in run 9, but it keeps no journeys-blob-* blobs, or they expired: every job runs, so the journeys' report is whole.", }); - expect(kept.mock.calls).toEqual(JOURNEYS_BLOBS.map((blob) => [9, blob])); - }); - - it("names every blob that expired", async () => { - const io = fakeIo({ passed: runs({ [`${SHA}:merge_group`]: 9 }), kept: async () => false }); - expect((await decide(io)).why).toContain( - "journeys-blob-chromium and journeys-blob-engines expired", - ); + expect(kept).toHaveBeenCalledWith(9, JOURNEYS_BLOBS); }); it("runs every job on main when the commit skipped the merge queue", async () => { @@ -235,19 +228,26 @@ describe("defaultIo", () => { expect(init.headers).toMatchObject({ authorization: "Bearer t" }); }); - it("asks GitHub whether a run keeps an artifact, expired ones aside", async () => { + it("asks GitHub whether a run keeps its artifacts of a prefix, none expired", async () => { + const blob = (name: string, expired = false) => ({ name, expired }); const answers = [ - { artifacts: [{ expired: true }, { expired: false }] }, - { artifacts: [{ expired: true }] }, - { artifacts: [] }, + { + artifacts: [ + blob("journeys-blob-chromium-1"), + blob("journeys-blob-engines-2"), + blob("x", true), + ], + }, + { artifacts: [blob("journeys-blob-chromium-1"), blob("journeys-blob-engines-2", true)] }, + { artifacts: [blob("journeys-report")] }, ]; const fetch = vi.fn(async () => Response.json(answers.shift())); const io = defaultIo(repo, { GITHUB_REPOSITORY: "owner/repo" }, fetch); - expect(await io.kept(9, "journeys-blob-engines")).toBe(true); - expect(await io.kept(9, "journeys-blob-engines")).toBe(false); - expect(await io.kept(9, "journeys-blob-engines")).toBe(false); + expect(await io.kept(9, "journeys-blob-")).toBe(true); + expect(await io.kept(9, "journeys-blob-")).toBe(false); + expect(await io.kept(9, "journeys-blob-")).toBe(false); expect(String((fetch.mock.calls[0] as unknown as [URL])[0])).toBe( - "https://api.github.com/repos/owner/repo/actions/runs/9/artifacts?name=journeys-blob-engines", + "https://api.github.com/repos/owner/repo/actions/runs/9/artifacts?per_page=100", ); }); diff --git a/scripts/ciGate.ts b/scripts/ciGate.ts index 0afbb515..197733fe 100644 --- a/scripts/ciGate.ts +++ b/scripts/ciGate.ts @@ -47,8 +47,11 @@ export type Gate = | { jobs: "deterministic"; testedBy: number; why: string }; const MAIN = "refs/heads/main"; -/** The journeys jobs' blobs, which main's report takes from the run that ran them. */ -export const JOURNEYS_BLOBS = ["journeys-blob-chromium", "journeys-blob-engines"]; +/** + * What the journeys jobs' blobs are named by (journeys-blob--), + * which main's report takes from the run that ran them. + */ +export const JOURNEYS_BLOBS = "journeys-blob-"; const RELEASE = "chore(release): "; /** Whether a commit is a release commit, as ci.yml's concurrency group tells them apart. */ @@ -72,22 +75,20 @@ export interface GateIo { changed(sha: string): string[]; /** The id of a CI run for `event` that passed on the commit, if there is one. */ passed(sha: string, event: "merge_group" | "push"): Promise; - /** Whether a run still has an artifact of that name, not expired. */ - kept(run: number, artifact: string): Promise; + /** Whether a run has artifacts named with that prefix, and none of them expired. */ + kept(run: number, prefix: string): Promise; output(name: string, value: string): void; log(message: string): void; } /** The deterministic jobs, the others having run in `testedBy`, while it keeps its journeys' blobs. */ async function testedIn(io: GateIo, testedBy: number, why: string): Promise { - const gone = []; - for (const blob of JOURNEYS_BLOBS) if (!(await io.kept(testedBy, blob))) gone.push(blob); - if (gone.length === 0) { + if (await io.kept(testedBy, JOURNEYS_BLOBS)) { return { jobs: "deterministic", testedBy, why: `${why}: the deterministic jobs run.` }; } return { jobs: "all", - why: `${why}, but its ${gone.join(" and ")} expired: every job runs, so the journeys' report is whole.`, + why: `${why}, but it keeps no ${JOURNEYS_BLOBS}* blobs, or they expired: every job runs, so the journeys' report is whole.`, }; } @@ -203,12 +204,13 @@ export function defaultIo(repo: string, env: GateEnv, fetch: typeof globalThis.f per_page: "1", })) as { workflow_runs: { id: number }[] } ).workflow_runs[0]?.id, - kept: async (run, artifact) => - ( - (await api(`actions/runs/${run}/artifacts`, { name: artifact })) as { - artifacts: { expired: boolean }[]; - } - ).artifacts.some(({ expired }) => !expired), + kept: async (run, prefix) => { + const { artifacts } = (await api(`actions/runs/${run}/artifacts`, { per_page: "100" })) as { + artifacts: { name: string; expired: boolean }[]; + }; + const named = artifacts.filter(({ name }) => name.startsWith(prefix)); + return named.length > 0 && named.every(({ expired }) => !expired); + }, output: (name, value) => { if (env.GITHUB_OUTPUT !== undefined) appendFileSync(env.GITHUB_OUTPUT, `${name}=${value}\n`); },