test(journeys): spend the journeys' CI time only on what each run checks

.github/workflows/ci.yml

@@ -368,10 +368,12 @@ jobs:
   # Chromium, logging in to a Community Solid Server 7 behind TLS that the
   # harness starts in Docker, each journey with an account and pod of its
   # own. The other engines and the phone run a few of them
-  # (docs/testing.md#browsers). Each browser's journeys are split in
-  # SHARDS jobs (npx playwright --shard), side by side, so the run waits
-  # for one shard's share of them, not for all: each shard pays the
-  # setup again (about a minute), for a shorter wall time. When one
+  # (docs/testing.md#browsers). Five jobs run them side by side, so the
+  # run waits for the slowest, not for all: Chromium's journeys in two
+  # shards (npx playwright --shard), and each other engine's in a job of
+  # its own, so that WebKit's and the phone's, the slowest, never land
+  # together. Each job pays the setup again (about a minute), for a
+  # shorter wall time. When one
   # fails, the job's summary names the step it failed in, and its
   # artifact holds the trace to replay it step
   # by step (`npx playwright show-trace`), the report, a video, and the
@@ -383,33 +385,45 @@ jobs:
   # on a shared runner. Their waits are fixed where they flake, not
   # retried (retries: 0, playwright.config.ts).
   journeys:
-    name: journeys (${{ matrix.name }}, ${{ matrix.shard }}/2)
+    name: journeys (${{ matrix.name }})
     needs: gate
     if: needs.gate.outputs.jobs == 'all'
     runs-on: ubuntu-latest
     timeout-minutes: 25
     strategy:
       fail-fast: false
+      # name: the job's, and its blob's (journeys-report lists them);
+      # projects: what it runs; browsers: what it installs, the build
+      # drawing the docs' diagrams in Chromium's headless shell; cache: the
+      # key its browsers are kept under, the phone's being WebKit's.
       matrix:
-        name: [chromium, engines]
-        # 1 to SHARDS, which journeys-report's env and the name above
-        # repeat.
-        shard: [1, 2]
         include:
-          - name: chromium
-            projects: --project=chromium
+          - name: chromium-1
+            projects: --project=chromium --shard=1/2
             browsers: chromium
-          # The build draws the docs' diagrams in Chromium's headless shell.
-          - name: engines
-            projects: --project=firefox --project=webkit --project=phone
-            browsers: chromium-headless-shell firefox webkit
+            cache: chromium
+          - name: chromium-2
+            projects: --project=chromium --shard=2/2
+            browsers: chromium
+            cache: chromium
+          - name: firefox
+            projects: --project=firefox
+            browsers: chromium-headless-shell firefox
+            cache: firefox
+          - name: webkit
+            projects: --project=webkit
+            browsers: chromium-headless-shell webkit
+            cache: webkit
+          - name: phone
+            projects: --project=phone
+            browsers: chromium-headless-shell webkit
+            cache: webkit
     env:
       JOURNEY_TRACE: >-
         ${{ (inputs.trace || github.ref == 'refs/heads/main' ||
         github.event_name == 'merge_group') && 'on' || '' }}
-      SHARDS: 2
       # The blob's name, so the jobs' blobs sit side by side when merged.
-      PLAYWRIGHT_BLOB_OUTPUT_NAME: report-${{ matrix.name }}-${{ matrix.shard }}.zip
+      PLAYWRIGHT_BLOB_OUTPUT_NAME: report-${{ matrix.name }}.zip
     steps:
       - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1
         with:
@@ -425,7 +439,7 @@ jobs:
         id: browsers
         with:
           path: ~/.cache/ms-playwright
-          key: playwright-${{ runner.os }}-${{ steps.playwright.outputs.version }}-${{ matrix.name }}
+          key: playwright-${{ runner.os }}-${{ steps.playwright.outputs.version }}-${{ matrix.cache }}
       - if: steps.browsers.outputs.cache-hit != 'true'
         run: npx playwright install --with-deps ${{ matrix.browsers }}
       - if: steps.browsers.outputs.cache-hit == 'true'
@@ -447,14 +461,14 @@ jobs:
         with:
           path: ${{ steps.images.outputs.file }}
           key: ${{ steps.images.outputs.key }}
-      - run: npm run journeys -- ${{ matrix.projects }} --shard="${{ matrix.shard }}/$SHARDS"
+      - run: npm run journeys -- ${{ matrix.projects }}
       - if: always()
-        run: node e2e/journeys/harness/summary.ts journeys-${{ matrix.name }}-${{ matrix.shard }}-${{ github.run_attempt }}
+        run: node e2e/journeys/harness/summary.ts journeys-${{ matrix.name }}-${{ github.run_attempt }}
       # In the queue, the blob below holds the traces; this, only on a failure.
       - if: failure() || (env.JOURNEY_TRACE == 'on' && github.event_name != 'merge_group')
         uses: actions/upload-artifact@cf430e030ddbb5b0abf93d22962f4752f3646cd9 # v7.0.2
         with:
-          name: journeys-${{ matrix.name }}-${{ matrix.shard }}-${{ github.run_attempt }}
+          name: journeys-${{ matrix.name }}-${{ github.run_attempt }}
           path: |
             e2e/journeys/playwright-report/
             e2e/journeys/test-results/
@@ -466,7 +480,7 @@ jobs:
       - if: always() && (github.ref == 'refs/heads/main' || github.event_name == 'merge_group')
         uses: actions/upload-artifact@cf430e030ddbb5b0abf93d22962f4752f3646cd9 # v7.0.2
         with:
-          name: journeys-blob-${{ matrix.name }}-${{ matrix.shard }}
+          name: journeys-blob-${{ matrix.name }}
           path: e2e/journeys/blob-report/
           retention-days: 7
           overwrite: true
@@ -476,7 +490,7 @@ jobs:
   # the deploy publishes it at /journeys/ (docs/deployment.md). When this
   # run skipped the journeys, their blobs come from the run gate names,
   # which ran them on this tree: the merge queue's, or a release commit's
-  # parent's. When the journeys passed, or ran there, every shard's blob
+  # parent's. When the journeys passed, or ran there, every job's blob
   # must be there, so a run the deploy may publish never has a report
   # missing a browser. ci-ok doesn't need it, as it runs on main alone;
   # its failure still fails the run, and so holds back release and
@@ -517,14 +531,12 @@ jobs:
         env:
           JOURNEYS: ${{ needs.journeys.result }}
           JOBS: ${{ needs.gate.outputs.jobs }}
-          # As journeys' matrix has them.
-          SHARDS: 2
+          # The journeys jobs' names, as their matrix has them.
+          REPORTS: chromium-1 chromium-2 firefox webkit phone
         run: |
           if [ "$JOURNEYS" = success ] || [ "$JOBS" = deterministic ]; then
-            for name in chromium engines; do
-              for shard in $(seq "$SHARDS"); do
-                test -f "blob-reports/report-$name-$shard.zip"
-              done
+            for name in $REPORTS; do
+              test -f "blob-reports/report-$name.zip"
             done
           fi
           if compgen -G 'blob-reports/*.zip' > /dev/null; then
@@ -596,7 +608,8 @@ jobs:
           echo "images=$(jq -cn '$ARGS.positional' --args "${images[@]}")" >> "$GITHUB_OUTPUT"
           version=$(node -p "require('@playwright/test/package.json').version")
           browsers=()
-          for name in chromium engines; do
+          # The journeys matrix's caches.
+          for name in chromium firefox webkit; do
             cached "playwright-$RUNNER_OS-$version-$name" || browsers+=("$name")
           done
           echo "Browsers missing: ${browsers[*]:-none}"
@@ -687,10 +700,11 @@ jobs:
       - id: playwright
         run: echo "version=$(node -p "require('@playwright/test/package.json').version")" >> "$GITHUB_OUTPUT"
       - run: |
+          # As the journeys matrix installs them for this cache.
           if [ "$BROWSERS" = chromium ]; then
             npx playwright install chromium
           else
-            npx playwright install chromium-headless-shell firefox webkit
+            npx playwright install chromium-headless-shell "$BROWSERS"
           fi
       - uses: actions/cache/save@55cc8345863c7cc4c66a329aec7e433d2d1c52a9 # v6.1.0
         with:

docs/accessibility.md

@@ -46,8 +46,8 @@ but the journeys:
 - **Every screen a journey visits**: the
   [user journeys](testing.md#user-journeys) run
   [axe](https://github.com/dequelabs/axe-core) on the screen each step
-  ends on, in the light and the dark theme, in each browser a journey
-  runs in (on a phone too), with its rules for WCAG 2.0,
+  ends on, in the light and the dark theme, in Chromium and on the
+  phone (WebKit), with its rules for WCAG 2.0,
   2.1 and 2.2 at levels A and AA. A violation fails the journey, and the
   report says which rule, on which element, with what axe found
   ([`harness/accessibility.ts`](../e2e/journeys/harness/accessibility.ts)).
@@ -100,7 +100,10 @@ and an issue for one is welcome ([below](#report-a-barrier)).
 - **Browsers**: every journey runs in Chromium, but only those that
   differ between engines run in Firefox, WebKit and on a phone
   ([testing.md](testing.md#browsers)), so axe sees the other screens
-  in Chromium alone.
+  in Chromium alone. Desktop Firefox and WebKit run no axe at all: the
+  check takes about 40% of a journey's time, and the phone already
+  checks WebKit's rendering. What only Firefox computes, such as the
+  contrast of a color it mixes (`color-mix`), goes unchecked.
 - **The graph**: the Studio's graph of an instance is drawn as one
   image, its nodes reached one by one with the arrow keys, by where
   they are drawn, not along their links; the edges to vocabulary

docs/statistics.md

@@ -436,8 +436,8 @@ rest of their study when they log in.
   and not as recall, and the chapter just completed under "Your
   courses", done, with its graded answers; and step 10 that the
   course's deck names the wrong options chosen in its final review. In
-  [studio.journey.ts](../e2e/journeys/journeys/studio.journey.ts),
-  steps 30 and 32 check that trying a course in the Studio's sandbox
+  [studio-course.journey.ts](../e2e/journeys/journeys/studio-course.journey.ts),
+  steps 07 and 09 check that trying a course in the Studio's sandbox
   leaves the statistics as they were. The page object is
   [Statistics.ts](../e2e/journeys/pages/Statistics.ts). The latest
   run's screenshots are online at <https://solid-memo.com/journeys/>.

docs/testing.md

@@ -359,7 +359,7 @@ Further journeys go through:
 
 Every journey runs in desktop Chromium, the `chromium` project in
 [playwright.config.ts](../e2e/journeys/playwright.config.ts). Three
-more projects run only the journeys tagged `@smoke`, `@guest` and
+more projects run only the journeys tagged `@guest` and
 `@reload`: `firefox` (desktop Firefox), `webkit` (desktop Safari's
 engine) and `phone` (an iPhone 14's screen and touch, on WebKit). Those
 journeys go through what differs most between engines: the login's
@@ -368,14 +368,24 @@ browser's storage through them, the silent restore of a session after a
 reload, and a page narrow enough to lay out for a phone. The other
 journeys test the app's logic, which is the same in every engine, so
 running them all in each would about triple the time for little gain. A
-journey tagged with one of the three runs in every browser; tag another
-only when what it tests differs between engines. CI runs Chromium's
-journeys apart from the other three projects', each split in two shards
-(`npx playwright test --shard`), four jobs side by side:
-`journeys (chromium, 1/2)` to `journeys (engines, 2/2)`. The run waits
-for the slowest shard, about half of what one job took, and each shard
-pays the job's setup again, about a minute. Playwright shards Chromium's
-journeys by file, so a file's journeys stay in one shard. The [user guide](user-guide.md#supported-browsers)
+journey tagged with one of the two runs in every browser; tag another
+only when what it tests differs between engines. A step of such a
+journey that tests the app's logic again names the projects it runs in
+(`app.step(title, body, { in: ["chromium"] })`), and the report shows
+it skipped in the others, as the reload journey's statistics and the
+shared link's import are. CI runs them in five
+jobs side by side: Chromium's journeys in two shards
+(`npx playwright test --shard`), `journeys (chromium-1)` and
+`journeys (chromium-2)`, and each other project's in a job of its own,
+`journeys (firefox)`, `journeys (webkit)` and `journeys (phone)`. The
+run waits for the slowest job, and each job pays the setup again, about
+a minute. Chromium's journeys run fully parallel
+(`fullyParallel: true`), each a unit the shards and workers share out,
+so a file of several journeys, such as `pod-safety`, is spread over
+both shards; a file whose journeys depend on each other runs them in
+turn (`test.describe.configure({ mode: "serial" })`). A job a project makes its own, rather than a shard of all
+three, keeps WebKit's and the phone's journeys, the slowest, from
+landing in one job. The [user guide](user-guide.md#supported-browsers)
 says which browsers Solid Memo supports.
 
 The engines differ in three ways the journeys allow for:
@@ -414,8 +424,11 @@ are.
 **Accessibility.** Each step's end checks the screen it leaves the app
 on with [axe](https://github.com/dequelabs/axe-core), in the light and
 the dark theme, against WCAG 2.0, 2.1 and 2.2 at levels A and AA
-(`App.step`, [harness/accessibility.ts](../e2e/journeys/harness/accessibility.ts)).
-A violation fails the journey, which goes on to its end so that every
+(`App.step`, [harness/accessibility.ts](../e2e/journeys/harness/accessibility.ts)),
+in a step of its own, "Accessibility", within the journey's step, so the
+report shows how long the check took. Chromium and the phone run it;
+desktop Firefox and WebKit leave it out, as their projects' metadata
+says (`accessibility: false`, [accessibility.md](accessibility.md)). A violation fails the journey, which goes on to its end so that every
 screen is checked. A violation let pass is listed in that file's
 `KNOWN_EXCEPTIONS`, with its reason. The Solid server's own pages, its
 login, are not checked. What the app aims for, and what else is
@@ -447,7 +460,7 @@ error, and the browser it failed in. The job's
 published with each deploy: the report of the CI run on the release
 commit the site runs (none after a deploy by hand), every browser's
 journeys in one, as the `journeys-report` job merges them from every
-journeys job (Playwright's blob reports, one a shard). As the release commit's run
+journeys job (Playwright's blob reports, one a job). As the release commit's run
 skips the journeys, it takes their reports from the run that ran them
 on its code, usually the merge queue's
 ([releasing.md](releasing.md#what-each-stage-checks)). It shows every
@@ -517,8 +530,9 @@ test("a learner's first deck @smoke", async ({ app, account, runId }) => {
   ([test/dialogs.ts](../apps/studio/src/test/dialogs.ts)).
 - **One after another**: a journey that needs what another made is in
   its file, after it, and the file runs in turn (`mode: "serial"`): the
-  course the Studio journey publishes is added from its link by the next
-  journey in [studio.journey.ts](../e2e/journeys/journeys/studio.journey.ts),
+  course the Studio's course journey publishes is added from its link by
+  the next journey in
+  [studio-course.journey.ts](../e2e/journeys/journeys/studio-course.journey.ts),
   which is skipped when the first fails.
 
 The harness's own code (`harness/`) has unit tests (`npm test`, so `npm
@@ -564,8 +578,8 @@ jobs, side by side, each [deterministic or timing-dependent](#deterministic-and-
 | `crosscheck` | deterministic | the Python scripts' tests (the cross-check's and the taxonomy deck generator's), then the vocabulary, the deck library and the pod catalog documents, by pySHACL ([validation.md](validation.md#the-ci-cross-check)) |
 | `pod (<server>)` | timing-dependent | the end-to-end tests, one job per blocking server (above) |
 | `contract (<server>)` | timing-dependent | that each advisory server starts and meets the contract |
-| `journeys (<browsers>, <shard>)` | timing-dependent | the user journeys: every one in Chromium, and those that differ between engines in Firefox, WebKit and on a phone, each browser's in two shards ([Browsers](#browsers)) |
-| `journeys-report` | deterministic | nothing: on `main`, after the `journeys` jobs, even when one failed, it merges their runs into one report, the `journeys-report` artifact the deploy publishes at `/journeys/` ([User journeys](#user-journeys)); when this run skipped them, their runs come from the run that tested its code (the gate's `tested-by`). When the journeys passed, or were skipped, it fails without every shard's report; its failure fails the run, so it holds back the release and the deploy, but not `ci-ok` |
+| `journeys (<project>)` | timing-dependent | the user journeys: every one in Chromium, in two shards, and those that differ between engines in Firefox, WebKit and on a phone, a job each ([Browsers](#browsers)) |
+| `journeys-report` | deterministic | nothing: on `main`, after the `journeys` jobs, even when one failed, it merges their runs into one report, the `journeys-report` artifact the deploy publishes at `/journeys/` ([User journeys](#user-journeys)); when this run skipped them, their runs come from the run that tested its code (the gate's `tested-by`). When the journeys passed, or were skipped, it fails without every job's report; its failure fails the run, so it holds back the release and the deploy, but not `ci-ok` |
 | `docs` | deterministic | the docs' links, every file's license, and `publiccode.yml` ([Documentation](#documentation)); their style and spelling are `build`'s |
 | `workflows` | deterministic | the workflows and local actions, by actionlint and zizmor (`npm run lint:workflows`, [The workflows](#the-workflows)) |
 | `commits` | deterministic | on a pull request, that each commit it adds follows Conventional Commits, by commitlint ([CONTRIBUTING.md](../CONTRIBUTING.md#commit-messages)), and that its body or a commit closes its issue, or the body says it is one step of it (`npm run check:closes`; not for Renovate's); in the merge queue or on a push to `main`, nothing |