From eaf73fd42b26ce975ed21e830287be6c95aa5fea Mon Sep 17 00:00:00 2001 From: Anton Wiklund Date: Tue, 20 Oct 2026 15:22:52 +0200 Subject: [PATCH 1/7] test(journeys): time each step's accessibility check apart Every journey step ends with axe, twice, once per theme, and the same steps take 5 to 8 seconds longer on WebKit and the phone than in Chromium. Nothing said how much of that was axe, which decides whether leaving it out of some browsers is worth what it checks. The check now runs in a step of its own, "Accessibility", so the report times it. Refs: #201 --- docs/testing.md | 5 +++-- e2e/journeys/pages/App.ts | 5 +++-- 2 files changed, 6 insertions(+), 4 deletions(-) diff --git a/docs/testing.md b/docs/testing.md index 32616908..b8192a9d 100644 --- a/docs/testing.md +++ b/docs/testing.md @@ -414,8 +414,9 @@ are. **Accessibility.** Each step's end checks the screen it leaves the app on with [axe](https://github.com/dequelabs/axe-core), in the light and the dark theme, against WCAG 2.0, 2.1 and 2.2 at levels A and AA -(`App.step`, [harness/accessibility.ts](../e2e/journeys/harness/accessibility.ts)). -A violation fails the journey, which goes on to its end so that every +(`App.step`, [harness/accessibility.ts](../e2e/journeys/harness/accessibility.ts)), +in a step of its own, "Accessibility", within the journey's step, so the +report shows how long the check took. A violation fails the journey, which goes on to its end so that every screen is checked. A violation let pass is listed in that file's `KNOWN_EXCEPTIONS`, with its reason. The Solid server's own pages, its login, are not checked. What the app aims for, and what else is diff --git a/e2e/journeys/pages/App.ts b/e2e/journeys/pages/App.ts index f145867b..242a65cd 100644 --- a/e2e/journeys/pages/App.ts +++ b/e2e/journeys/pages/App.ts @@ -133,7 +133,8 @@ export class App { * One step of a journey, as the journey's spec names it ("03 · Set new * cards per day to 1"): a group in the report and the trace, with a * screenshot of where it left the app, and that screen checked for - * accessibility. + * accessibility, in a step of its own, "Accessibility", so the report + * says how long the check took apart from the step's own work. */ async step(title: string, body: () => Promise): Promise { await test.step(title, async () => { @@ -144,7 +145,7 @@ export class App { body: await this.page.screenshot(), contentType: "image/png", }); - await this.checkAccessibility(title); + await test.step("Accessibility", () => this.checkAccessibility(title)); }); } From 6acc3e0726926cb3a8aa12d5a09f2b5fb2122e60 Mon Sep 17 00:00:00 2001 From: Anton Wiklund Date: Tue, 20 Oct 2026 15:24:42 +0200 Subject: [PATCH 2/7] ci(journeys): run each engine's journeys in a job of its own Playwright shards the engine projects by test count, in project order, so one shard got every WebKit and phone journey, the slowest, about 505 worker-seconds against 276, and every run waited for it. Firefox, WebKit and the phone now each have a job of their own, beside Chromium's two shards, so the slowest journeys never land together. Five jobs, one more than before, each paying about a minute of setup. WebKit's and the phone's browsers share a cache, as they install the same ones. Refs: #201 --- .github/workflows/ci.yml | 76 ++++++++++++++++++++++++---------------- docs/testing.md | 22 ++++++------ 2 files changed, 57 insertions(+), 41 deletions(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 85c2d8ee..4f42caac 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -368,10 +368,12 @@ jobs: # Chromium, logging in to a Community Solid Server 7 behind TLS that the # harness starts in Docker, each journey with an account and pod of its # own. The other engines and the phone run a few of them - # (docs/testing.md#browsers). Each browser's journeys are split in - # SHARDS jobs (npx playwright --shard), side by side, so the run waits - # for one shard's share of them, not for all: each shard pays the - # setup again (about a minute), for a shorter wall time. When one + # (docs/testing.md#browsers). Five jobs run them side by side, so the + # run waits for the slowest, not for all: Chromium's journeys in two + # shards (npx playwright --shard), and each other engine's in a job of + # its own, so that WebKit's and the phone's, the slowest, never land + # together. Each job pays the setup again (about a minute), for a + # shorter wall time. When one # fails, the job's summary names the step it failed in, and its # artifact holds the trace to replay it step # by step (`npx playwright show-trace`), the report, a video, and the @@ -383,33 +385,45 @@ jobs: # on a shared runner. Their waits are fixed where they flake, not # retried (retries: 0, playwright.config.ts). journeys: - name: journeys (${{ matrix.name }}, ${{ matrix.shard }}/2) + name: journeys (${{ matrix.name }}) needs: gate if: needs.gate.outputs.jobs == 'all' runs-on: ubuntu-latest timeout-minutes: 25 strategy: fail-fast: false + # name: the job's, and its blob's (journeys-report lists them); + # projects: what it runs; browsers: what it installs, the build + # drawing the docs' diagrams in Chromium's headless shell; cache: the + # key its browsers are kept under, the phone's being WebKit's. matrix: - name: [chromium, engines] - # 1 to SHARDS, which journeys-report's env and the name above - # repeat. - shard: [1, 2] include: - - name: chromium - projects: --project=chromium + - name: chromium-1 + projects: --project=chromium --shard=1/2 browsers: chromium - # The build draws the docs' diagrams in Chromium's headless shell. - - name: engines - projects: --project=firefox --project=webkit --project=phone - browsers: chromium-headless-shell firefox webkit + cache: chromium + - name: chromium-2 + projects: --project=chromium --shard=2/2 + browsers: chromium + cache: chromium + - name: firefox + projects: --project=firefox + browsers: chromium-headless-shell firefox + cache: firefox + - name: webkit + projects: --project=webkit + browsers: chromium-headless-shell webkit + cache: webkit + - name: phone + projects: --project=phone + browsers: chromium-headless-shell webkit + cache: webkit env: JOURNEY_TRACE: >- ${{ (inputs.trace || github.ref == 'refs/heads/main' || github.event_name == 'merge_group') && 'on' || '' }} - SHARDS: 2 # The blob's name, so the jobs' blobs sit side by side when merged. - PLAYWRIGHT_BLOB_OUTPUT_NAME: report-${{ matrix.name }}-${{ matrix.shard }}.zip + PLAYWRIGHT_BLOB_OUTPUT_NAME: report-${{ matrix.name }}.zip steps: - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 with: @@ -425,7 +439,7 @@ jobs: id: browsers with: path: ~/.cache/ms-playwright - key: playwright-${{ runner.os }}-${{ steps.playwright.outputs.version }}-${{ matrix.name }} + key: playwright-${{ runner.os }}-${{ steps.playwright.outputs.version }}-${{ matrix.cache }} - if: steps.browsers.outputs.cache-hit != 'true' run: npx playwright install --with-deps ${{ matrix.browsers }} - if: steps.browsers.outputs.cache-hit == 'true' @@ -447,14 +461,14 @@ jobs: with: path: ${{ steps.images.outputs.file }} key: ${{ steps.images.outputs.key }} - - run: npm run journeys -- ${{ matrix.projects }} --shard="${{ matrix.shard }}/$SHARDS" + - run: npm run journeys -- ${{ matrix.projects }} - if: always() - run: node e2e/journeys/harness/summary.ts journeys-${{ matrix.name }}-${{ matrix.shard }}-${{ github.run_attempt }} + run: node e2e/journeys/harness/summary.ts journeys-${{ matrix.name }}-${{ github.run_attempt }} # In the queue, the blob below holds the traces; this, only on a failure. - if: failure() || (env.JOURNEY_TRACE == 'on' && github.event_name != 'merge_group') uses: actions/upload-artifact@cf430e030ddbb5b0abf93d22962f4752f3646cd9 # v7.0.2 with: - name: journeys-${{ matrix.name }}-${{ matrix.shard }}-${{ github.run_attempt }} + name: journeys-${{ matrix.name }}-${{ github.run_attempt }} path: | e2e/journeys/playwright-report/ e2e/journeys/test-results/ @@ -466,7 +480,7 @@ jobs: - if: always() && (github.ref == 'refs/heads/main' || github.event_name == 'merge_group') uses: actions/upload-artifact@cf430e030ddbb5b0abf93d22962f4752f3646cd9 # v7.0.2 with: - name: journeys-blob-${{ matrix.name }}-${{ matrix.shard }} + name: journeys-blob-${{ matrix.name }} path: e2e/journeys/blob-report/ retention-days: 7 overwrite: true @@ -476,7 +490,7 @@ jobs: # the deploy publishes it at /journeys/ (docs/deployment.md). When this # run skipped the journeys, their blobs come from the run gate names, # which ran them on this tree: the merge queue's, or a release commit's - # parent's. When the journeys passed, or ran there, every shard's blob + # parent's. When the journeys passed, or ran there, every job's blob # must be there, so a run the deploy may publish never has a report # missing a browser. ci-ok doesn't need it, as it runs on main alone; # its failure still fails the run, and so holds back release and @@ -517,14 +531,12 @@ jobs: env: JOURNEYS: ${{ needs.journeys.result }} JOBS: ${{ needs.gate.outputs.jobs }} - # As journeys' matrix has them. - SHARDS: 2 + # The journeys jobs' names, as their matrix has them. + REPORTS: chromium-1 chromium-2 firefox webkit phone run: | if [ "$JOURNEYS" = success ] || [ "$JOBS" = deterministic ]; then - for name in chromium engines; do - for shard in $(seq "$SHARDS"); do - test -f "blob-reports/report-$name-$shard.zip" - done + for name in $REPORTS; do + test -f "blob-reports/report-$name.zip" done fi if compgen -G 'blob-reports/*.zip' > /dev/null; then @@ -596,7 +608,8 @@ jobs: echo "images=$(jq -cn '$ARGS.positional' --args "${images[@]}")" >> "$GITHUB_OUTPUT" version=$(node -p "require('@playwright/test/package.json').version") browsers=() - for name in chromium engines; do + # The journeys matrix's caches. + for name in chromium firefox webkit; do cached "playwright-$RUNNER_OS-$version-$name" || browsers+=("$name") done echo "Browsers missing: ${browsers[*]:-none}" @@ -687,10 +700,11 @@ jobs: - id: playwright run: echo "version=$(node -p "require('@playwright/test/package.json').version")" >> "$GITHUB_OUTPUT" - run: | + # As the journeys matrix installs them for this cache. if [ "$BROWSERS" = chromium ]; then npx playwright install chromium else - npx playwright install chromium-headless-shell firefox webkit + npx playwright install chromium-headless-shell "$BROWSERS" fi - uses: actions/cache/save@55cc8345863c7cc4c66a329aec7e433d2d1c52a9 # v6.1.0 with: diff --git a/docs/testing.md b/docs/testing.md index b8192a9d..cd03ea78 100644 --- a/docs/testing.md +++ b/docs/testing.md @@ -369,13 +369,15 @@ reload, and a page narrow enough to lay out for a phone. The other journeys test the app's logic, which is the same in every engine, so running them all in each would about triple the time for little gain. A journey tagged with one of the three runs in every browser; tag another -only when what it tests differs between engines. CI runs Chromium's -journeys apart from the other three projects', each split in two shards -(`npx playwright test --shard`), four jobs side by side: -`journeys (chromium, 1/2)` to `journeys (engines, 2/2)`. The run waits -for the slowest shard, about half of what one job took, and each shard -pays the job's setup again, about a minute. Playwright shards Chromium's -journeys by file, so a file's journeys stay in one shard. The [user guide](user-guide.md#supported-browsers) +only when what it tests differs between engines. CI runs them in five +jobs side by side: Chromium's journeys in two shards +(`npx playwright test --shard`), `journeys (chromium-1)` and +`journeys (chromium-2)`, and each other project's in a job of its own, +`journeys (firefox)`, `journeys (webkit)` and `journeys (phone)`. The +run waits for the slowest job, and each job pays the setup again, about +a minute. A job a project makes its own, rather than a shard of all +three, keeps WebKit's and the phone's journeys, the slowest, from +landing in one job. The [user guide](user-guide.md#supported-browsers) says which browsers Solid Memo supports. The engines differ in three ways the journeys allow for: @@ -448,7 +450,7 @@ error, and the browser it failed in. The job's published with each deploy: the report of the CI run on the release commit the site runs (none after a deploy by hand), every browser's journeys in one, as the `journeys-report` job merges them from every -journeys job (Playwright's blob reports, one a shard). As the release commit's run +journeys job (Playwright's blob reports, one a job). As the release commit's run skips the journeys, it takes their reports from the run that ran them on its code, usually the merge queue's ([releasing.md](releasing.md#what-each-stage-checks)). It shows every @@ -565,8 +567,8 @@ jobs, side by side, each [deterministic or timing-dependent](#deterministic-and- | `crosscheck` | deterministic | the Python scripts' tests (the cross-check's and the taxonomy deck generator's), then the vocabulary, the deck library and the pod catalog documents, by pySHACL ([validation.md](validation.md#the-ci-cross-check)) | | `pod ()` | timing-dependent | the end-to-end tests, one job per blocking server (above) | | `contract ()` | timing-dependent | that each advisory server starts and meets the contract | -| `journeys (, )` | timing-dependent | the user journeys: every one in Chromium, and those that differ between engines in Firefox, WebKit and on a phone, each browser's in two shards ([Browsers](#browsers)) | -| `journeys-report` | deterministic | nothing: on `main`, after the `journeys` jobs, even when one failed, it merges their runs into one report, the `journeys-report` artifact the deploy publishes at `/journeys/` ([User journeys](#user-journeys)); when this run skipped them, their runs come from the run that tested its code (the gate's `tested-by`). When the journeys passed, or were skipped, it fails without every shard's report; its failure fails the run, so it holds back the release and the deploy, but not `ci-ok` | +| `journeys ()` | timing-dependent | the user journeys: every one in Chromium, in two shards, and those that differ between engines in Firefox, WebKit and on a phone, a job each ([Browsers](#browsers)) | +| `journeys-report` | deterministic | nothing: on `main`, after the `journeys` jobs, even when one failed, it merges their runs into one report, the `journeys-report` artifact the deploy publishes at `/journeys/` ([User journeys](#user-journeys)); when this run skipped them, their runs come from the run that tested its code (the gate's `tested-by`). When the journeys passed, or were skipped, it fails without every job's report; its failure fails the run, so it holds back the release and the deploy, but not `ci-ok` | | `docs` | deterministic | the docs' links, every file's license, and `publiccode.yml` ([Documentation](#documentation)); their style and spelling are `build`'s | | `workflows` | deterministic | the workflows and local actions, by actionlint and zizmor (`npm run lint:workflows`, [The workflows](#the-workflows)) | | `commits` | deterministic | on a pull request, that each commit it adds follows Conventional Commits, by commitlint ([CONTRIBUTING.md](../CONTRIBUTING.md#commit-messages)), and that its body or a commit closes its issue, or the body says it is one step of it (`npm run check:closes`; not for Renovate's); in the merge queue or on a push to `main`, nothing | From d7ca3c89fa025e7356817e2a2c5bb6535890c202 Mon Sep 17 00:00:00 2001 From: Anton Wiklund Date: Tue, 20 Oct 2026 15:25:44 +0200 Subject: [PATCH 3/7] test(journeys): check accessibility in Chromium and on the phone only Timed apart, axe takes 35 to 47% of a journey's time: 9 to 15 seconds of the 25 to 33 that the guest and reload journeys take, in Chromium and Firefox. Desktop Firefox and WebKit ran it on the same few screens that Chromium checks on every journey, and the phone checks WebKit's rendering. Their projects now leave it out (metadata accessibility: false), and docs/accessibility.md says what that leaves unchecked: what only Firefox computes, such as a mixed color's contrast. Refs: #201 --- docs/accessibility.md | 9 ++++++--- docs/testing.md | 4 +++- e2e/journeys/pages/App.ts | 8 ++++++-- e2e/journeys/playwright.config.ts | 18 ++++++++++++++++-- 4 files changed, 31 insertions(+), 8 deletions(-) diff --git a/docs/accessibility.md b/docs/accessibility.md index 785eefb1..f2f6c6fd 100644 --- a/docs/accessibility.md +++ b/docs/accessibility.md @@ -46,8 +46,8 @@ but the journeys: - **Every screen a journey visits**: the [user journeys](testing.md#user-journeys) run [axe](https://github.com/dequelabs/axe-core) on the screen each step - ends on, in the light and the dark theme, in each browser a journey - runs in (on a phone too), with its rules for WCAG 2.0, + ends on, in the light and the dark theme, in Chromium and on the + phone (WebKit), with its rules for WCAG 2.0, 2.1 and 2.2 at levels A and AA. A violation fails the journey, and the report says which rule, on which element, with what axe found ([`harness/accessibility.ts`](../e2e/journeys/harness/accessibility.ts)). @@ -100,7 +100,10 @@ and an issue for one is welcome ([below](#report-a-barrier)). - **Browsers**: every journey runs in Chromium, but only those that differ between engines run in Firefox, WebKit and on a phone ([testing.md](testing.md#browsers)), so axe sees the other screens - in Chromium alone. + in Chromium alone. Desktop Firefox and WebKit run no axe at all: the + check takes about 40% of a journey's time, and the phone already + checks WebKit's rendering. What only Firefox computes, such as the + contrast of a color it mixes (`color-mix`), goes unchecked. - **The graph**: the Studio's graph of an instance is drawn as one image, its nodes reached one by one with the arrow keys, by where they are drawn, not along their links; the edges to vocabulary diff --git a/docs/testing.md b/docs/testing.md index cd03ea78..6f75db78 100644 --- a/docs/testing.md +++ b/docs/testing.md @@ -418,7 +418,9 @@ on with [axe](https://github.com/dequelabs/axe-core), in the light and the dark theme, against WCAG 2.0, 2.1 and 2.2 at levels A and AA (`App.step`, [harness/accessibility.ts](../e2e/journeys/harness/accessibility.ts)), in a step of its own, "Accessibility", within the journey's step, so the -report shows how long the check took. A violation fails the journey, which goes on to its end so that every +report shows how long the check took. Chromium and the phone run it; +desktop Firefox and WebKit leave it out, as their projects' metadata +says (`accessibility: false`, [accessibility.md](accessibility.md)). A violation fails the journey, which goes on to its end so that every screen is checked. A violation let pass is listed in that file's `KNOWN_EXCEPTIONS`, with its reason. The Solid server's own pages, its login, are not checked. What the app aims for, and what else is diff --git a/e2e/journeys/pages/App.ts b/e2e/journeys/pages/App.ts index 242a65cd..691e0d35 100644 --- a/e2e/journeys/pages/App.ts +++ b/e2e/journeys/pages/App.ts @@ -134,7 +134,9 @@ export class App { * cards per day to 1"): a group in the report and the trace, with a * screenshot of where it left the app, and that screen checked for * accessibility, in a step of its own, "Accessibility", so the report - * says how long the check took apart from the step's own work. + * says how long the check took apart from the step's own work. A + * project whose metadata says `accessibility: false` leaves the check + * out (playwright.config.ts). */ async step(title: string, body: () => Promise): Promise { await test.step(title, async () => { @@ -145,7 +147,9 @@ export class App { body: await this.page.screenshot(), contentType: "image/png", }); - await test.step("Accessibility", () => this.checkAccessibility(title)); + if (this.testInfo.project.metadata.accessibility !== false) { + await test.step("Accessibility", () => this.checkAccessibility(title)); + } }); } diff --git a/e2e/journeys/playwright.config.ts b/e2e/journeys/playwright.config.ts index 43df2645..b74d40d0 100644 --- a/e2e/journeys/playwright.config.ts +++ b/e2e/journeys/playwright.config.ts @@ -64,10 +64,24 @@ export default defineConfig({ video: "retain-on-failure", screenshot: "only-on-failure", }, + // Axe checks each step's screen in Chromium and on the phone, which is + // WebKit too; desktop Firefox and WebKit leave it out + // (docs/accessibility.md): it takes about 40% of a journey's time, and + // the same screens are checked in Chromium. projects: [ { name: "chromium", use: { ...devices["Desktop Chrome"] } }, - { name: "firefox", use: { ...devices["Desktop Firefox"] }, grep: ENGINE_JOURNEYS }, - { name: "webkit", use: { ...devices["Desktop Safari"] }, grep: ENGINE_JOURNEYS }, + { + name: "firefox", + use: { ...devices["Desktop Firefox"] }, + grep: ENGINE_JOURNEYS, + metadata: { accessibility: false }, + }, + { + name: "webkit", + use: { ...devices["Desktop Safari"] }, + grep: ENGINE_JOURNEYS, + metadata: { accessibility: false }, + }, // Safari on an iPhone: a phone's viewport, touch, and WebKit. { name: "phone", use: { ...devices["iPhone 14"] }, grep: ENGINE_JOURNEYS }, ], From 17fc9eaa66cd9a7eb40d68d8509b66d52afe3371 Mon Sep 17 00:00:00 2001 From: Anton Wiklund Date: Tue, 20 Oct 2026 15:26:45 +0200 Subject: [PATCH 4/7] test(journeys): log out at the end of the reload journey, not in a smoke run per engine Firefox, WebKit and the phone each ran the smoke journey, which logs in and out, beside the reload journey, which logs in too. The reload journey now also logs out, so each engine still logs in and out once, and the engines no longer run @smoke: three journeys fewer a run, each with its own account, login and instance. Chromium still runs it. Refs: #201 --- docs/testing.md | 4 ++-- e2e/journeys/journeys/study-reload.journey.ts | 9 ++++++++- e2e/journeys/playwright.config.ts | 11 ++++++----- 3 files changed, 16 insertions(+), 8 deletions(-) diff --git a/docs/testing.md b/docs/testing.md index 6f75db78..22bc5434 100644 --- a/docs/testing.md +++ b/docs/testing.md @@ -359,7 +359,7 @@ Further journeys go through: Every journey runs in desktop Chromium, the `chromium` project in [playwright.config.ts](../e2e/journeys/playwright.config.ts). Three -more projects run only the journeys tagged `@smoke`, `@guest` and +more projects run only the journeys tagged `@guest` and `@reload`: `firefox` (desktop Firefox), `webkit` (desktop Safari's engine) and `phone` (an iPhone 14's screen and touch, on WebKit). Those journeys go through what differs most between engines: the login's @@ -368,7 +368,7 @@ browser's storage through them, the silent restore of a session after a reload, and a page narrow enough to lay out for a phone. The other journeys test the app's logic, which is the same in every engine, so running them all in each would about triple the time for little gain. A -journey tagged with one of the three runs in every browser; tag another +journey tagged with one of the two runs in every browser; tag another only when what it tests differs between engines. CI runs them in five jobs side by side: Chromium's journeys in two shards (`npx playwright test --shard`), `journeys (chromium-1)` and diff --git a/e2e/journeys/journeys/study-reload.journey.ts b/e2e/journeys/journeys/study-reload.journey.ts index f8efc103..8232aabe 100644 --- a/e2e/journeys/journeys/study-reload.journey.ts +++ b/e2e/journeys/journeys/study-reload.journey.ts @@ -12,7 +12,8 @@ import { logInAndCreateInstance } from "../flows/logIn.ts"; * deck list, the statistics and each card's history count every answer * once (docs/srs.md#study-sessions). Catches answers kept only in the * page, a session that starts over with the cards already answered, and - * a reload that loses the route. + * a reload that loses the route. Then the learner logs out: in the other + * engines, which run no smoke journey, this is the one that does. * * Four new cards, all within the five new cards a day: two answered * Good before the reload, the other two after it. Good keeps a card out @@ -113,4 +114,10 @@ test("reload in the middle of a study session @reload", async ({ app, account, r ]); } }); + + // The other engines run no smoke journey: this one logs out for it. + await app.step("11 · Log out", async () => { + await app.chrome.logOut(); + await app.onboarding.expectLoggedOut(); + }); }); diff --git a/e2e/journeys/playwright.config.ts b/e2e/journeys/playwright.config.ts index b74d40d0..e254c6bd 100644 --- a/e2e/journeys/playwright.config.ts +++ b/e2e/journeys/playwright.config.ts @@ -20,12 +20,13 @@ if (!(TRACE_MODES as readonly string[]).includes(trace)) throw new Error(`JOURNEY_TRACE is "${trace}"; it may be ${TRACE_MODES.join(", ")}.`); /** - * The journeys the other engines and the phone run: logging in, a guest's - * study kept in the browser through the login's redirects, and the silent - * restore of a session after a reload, which differ most between engines. - * The rest exercise the app's logic, which doesn't (docs/testing.md#browsers). + * The journeys the other engines and the phone run: a guest's study kept + * in the browser through the login's redirects, and the silent restore of + * a session after a reload, which differ most between engines; the + * reload's journey also logs in and out, as @smoke does in Chromium. The + * rest exercise the app's logic, which doesn't (docs/testing.md#browsers). */ -const ENGINE_JOURNEYS = /@smoke|@guest|@reload/; +const ENGINE_JOURNEYS = /@guest|@reload/; export default defineConfig({ testDir: "journeys", From c6794ded42649bd465345777be871e7e4ef4062c Mon Sep 17 00:00:00 2001 From: Anton Wiklund Date: Tue, 20 Oct 2026 15:27:51 +0200 Subject: [PATCH 5/7] test(journeys): check the app's logic in Chromium only, in the journeys every engine runs The reload journey checked the statistics and each card's history, and the shared link's journey imported and studied the deck and followed a link to no deck, in every engine. That is the app's logic, the same in each, and Chromium checks it; the guest journey imports and studies in every engine too. App.step now takes the projects a step runs in, and skips it in the others through test.step.skip, so the report shows it skipped. In Firefox, the reload journey went from 31.7 to 14.4 seconds with axe left out too, and the shared link's from 11 to 5. Refs: #201 --- docs/testing.md | 6 +- e2e/journeys/journeys/study-reload.journey.ts | 58 +++++++++++-------- e2e/journeys/journeys/try-link.journey.ts | 41 ++++++++----- e2e/journeys/pages/App.ts | 16 ++++- 4 files changed, 80 insertions(+), 41 deletions(-) diff --git a/docs/testing.md b/docs/testing.md index 22bc5434..5859a16f 100644 --- a/docs/testing.md +++ b/docs/testing.md @@ -369,7 +369,11 @@ reload, and a page narrow enough to lay out for a phone. The other journeys test the app's logic, which is the same in every engine, so running them all in each would about triple the time for little gain. A journey tagged with one of the two runs in every browser; tag another -only when what it tests differs between engines. CI runs them in five +only when what it tests differs between engines. A step of such a +journey that tests the app's logic again names the projects it runs in +(`app.step(title, body, { in: ["chromium"] })`), and the report shows +it skipped in the others, as the reload journey's statistics and the +shared link's import are. CI runs them in five jobs side by side: Chromium's journeys in two shards (`npx playwright test --shard`), `journeys (chromium-1)` and `journeys (chromium-2)`, and each other project's in a job of its own, diff --git a/e2e/journeys/journeys/study-reload.journey.ts b/e2e/journeys/journeys/study-reload.journey.ts index 8232aabe..98af0b5d 100644 --- a/e2e/journeys/journeys/study-reload.journey.ts +++ b/e2e/journeys/journeys/study-reload.journey.ts @@ -12,7 +12,8 @@ import { logInAndCreateInstance } from "../flows/logIn.ts"; * deck list, the statistics and each card's history count every answer * once (docs/srs.md#study-sessions). Catches answers kept only in the * page, a session that starts over with the cards already answered, and - * a reload that loses the route. Then the learner logs out: in the other + * a reload that loses the route. The statistics and each card's history + * are checked in Chromium. Then the learner logs out: in the other * engines, which run no smoke journey, this is the one that does. * * Four new cards, all within the five new cards a day: two answered @@ -90,30 +91,39 @@ test("reload in the middle of a study session @reload", async ({ app, account, r await app.decks.expectAnsweredToday(cards); }); - await app.step("09 · The statistics count each answer once", async () => { - await app.statistics.open(); - await app.statistics.expectStatistics({ - answers: cards, - studyDays: 1, - cards, - // Four new cards, each answered once, Good: no review, no repeat. - today: { answers: cards, introduced: cards, remembered: 0, lapses: 0, repeats: 0 }, - decks: [{ name: deck, answers: cards, introduced: cards, lapses: 0, lastStudied: "today" }], - maturity: { new: 0, relearning: 0, young: cards, mature: 0 }, - }); - }); + // The statistics and the history are the app's logic: Chromium checks them. + await app.step( + "09 · The statistics count each answer once", + async () => { + await app.statistics.open(); + await app.statistics.expectStatistics({ + answers: cards, + studyDays: 1, + cards, + // Four new cards, each answered once, Good: no review, no repeat. + today: { answers: cards, introduced: cards, remembered: 0, lapses: 0, repeats: 0 }, + decks: [{ name: deck, answers: cards, introduced: cards, lapses: 0, lastStudied: "today" }], + maturity: { new: 0, relearning: 0, young: cards, mature: 0 }, + }); + }, + { in: ["chromium"] }, + ); - await app.step("10 · Each card's history holds one answer", async () => { - for (const front of fronts) { - await app.chrome.breadcrumb("breadcrumbs.decks"); - await app.deckBrowser.openFor(deck); - await app.cardEditor.open(front); - // The history names an answer by the four buttons' words, whichever scale gave it. - expect(await app.cardEditor.history(), front).toEqual([ - { kind: app.t("card.historyFirst"), answer: app.t("study.minimal.good") }, - ]); - } - }); + await app.step( + "10 · Each card's history holds one answer", + async () => { + for (const front of fronts) { + await app.chrome.breadcrumb("breadcrumbs.decks"); + await app.deckBrowser.openFor(deck); + await app.cardEditor.open(front); + // The history names an answer by the four buttons' words, whichever scale gave it. + expect(await app.cardEditor.history(), front).toEqual([ + { kind: app.t("card.historyFirst"), answer: app.t("study.minimal.good") }, + ]); + } + }, + { in: ["chromium"] }, + ); // The other engines run no smoke journey: this one logs out for it. await app.step("11 · Log out", async () => { diff --git a/e2e/journeys/journeys/try-link.journey.ts b/e2e/journeys/journeys/try-link.journey.ts index 51c72dce..2bf6e61b 100644 --- a/e2e/journeys/journeys/try-link.journey.ts +++ b/e2e/journeys/journeys/try-link.journey.ts @@ -17,7 +17,8 @@ import { BRIGHTEST_STARS } from "../pages/Library.ts"; * no one else can open (a guest's exists only in their browser); the * landing page shown instead of a guest's study started; and the link * left in the history, or not followed on to the deck's page once the - * guest's instance is made. + * guest's instance is made. The other engines follow the link; importing, + * studying and the unknown link are checked in Chromium. */ test("a library deck shared by its link @guest", async ({ app, newDevice }) => { const stars = BRIGHTEST_STARS.en; @@ -41,19 +42,29 @@ test("a library deck shared by its link @guest", async ({ app, newDevice }) => { friend.libraryDeck.followLink(link, stars), ); - await friend.step("04 · Import it and study a card", async () => { - await friend.libraryDeck.import(stars); - await friend.decks.study(stars); - await friend.study.expectStudying(stars); - await friend.study.answer("good"); - await friend.study.endSession(); - await friend.decks.expectAnsweredToday(1); - }); + // Importing and studying are the app's logic, which the guest journey + // goes through in every engine: Chromium checks them here. + await friend.step( + "04 · Import it and study a card", + async () => { + await friend.libraryDeck.import(stars); + await friend.decks.study(stars); + await friend.study.expectStudying(stars); + await friend.study.answer("good"); + await friend.study.endSession(); + await friend.decks.expectAnsweredToday(1); + }, + { in: ["chromium"] }, + ); - await friend.step("05 · Follow a link to a deck the library does not have", async () => { - const url = new URL(link); - url.hash = `#/try?deck=${encodeURIComponent(new URL("decks/index.ttl#no-such-deck", url).href)}`; - await friend.page.goto(url.href); - await friend.library.expectFromUnknownLink(); - }); + await friend.step( + "05 · Follow a link to a deck the library does not have", + async () => { + const url = new URL(link); + url.hash = `#/try?deck=${encodeURIComponent(new URL("decks/index.ttl#no-such-deck", url).href)}`; + await friend.page.goto(url.href); + await friend.library.expectFromUnknownLink(); + }, + { in: ["chromium"] }, + ); }); diff --git a/e2e/journeys/pages/App.ts b/e2e/journeys/pages/App.ts index 691e0d35..4ccb2c9f 100644 --- a/e2e/journeys/pages/App.ts +++ b/e2e/journeys/pages/App.ts @@ -137,8 +137,22 @@ export class App { * says how long the check took apart from the step's own work. A * project whose metadata says `accessibility: false` leaves the check * out (playwright.config.ts). + * + * `in` names the projects the step runs in; elsewhere it is skipped, + * and the report says so. It is for a step that tests the app's logic, + * the same in every engine, in a journey the other engines run for + * what differs between them (docs/testing.md#browsers). */ - async step(title: string, body: () => Promise): Promise { + async step( + title: string, + body: () => Promise, + { in: projects }: { in?: readonly string[] } = {}, + ): Promise { + if (projects !== undefined && !projects.includes(this.testInfo.project.name)) { + await test.step + .skip(title, async () => {}); + return; + } await test.step(title, async () => { await body(); const unmet = this.expectedDialogs.splice(0).map(String); From d61712a93dd1b26a702ddd0c0b104d57b351b176 Mon Sep 17 00:00:00 2001 From: Anton Wiklund Date: Tue, 20 Oct 2026 15:29:11 +0200 Subject: [PATCH 6/7] test(journeys): let Chromium share out a file's journeys one by one With fullyParallel off, a shard and a worker took a journey file whole, so pod-safety's six journeys, about 100 seconds, ran one after another at the end of one Chromium shard. Chromium's project now runs its journeys fully in parallel, each a unit the shards and workers share out, and pod-safety's are spread over both shards. The studio file, whose second journey needs what the first publishes, stays serial, as it says. Refs: #201 --- docs/testing.md | 6 +++++- e2e/journeys/playwright.config.ts | 6 +++++- 2 files changed, 10 insertions(+), 2 deletions(-) diff --git a/docs/testing.md b/docs/testing.md index 5859a16f..5dbcae4e 100644 --- a/docs/testing.md +++ b/docs/testing.md @@ -379,7 +379,11 @@ jobs side by side: Chromium's journeys in two shards `journeys (chromium-2)`, and each other project's in a job of its own, `journeys (firefox)`, `journeys (webkit)` and `journeys (phone)`. The run waits for the slowest job, and each job pays the setup again, about -a minute. A job a project makes its own, rather than a shard of all +a minute. Chromium's journeys run fully parallel +(`fullyParallel: true`), each a unit the shards and workers share out, +so a file of several journeys, such as `pod-safety`, is spread over +both shards; a file whose journeys depend on each other runs them in +turn (`test.describe.configure({ mode: "serial" })`). A job a project makes its own, rather than a shard of all three, keeps WebKit's and the phone's journeys, the slowest, from landing in one job. The [user guide](user-guide.md#supported-browsers) says which browsers Solid Memo supports. diff --git a/e2e/journeys/playwright.config.ts b/e2e/journeys/playwright.config.ts index e254c6bd..e99edcfb 100644 --- a/e2e/journeys/playwright.config.ts +++ b/e2e/journeys/playwright.config.ts @@ -70,7 +70,11 @@ export default defineConfig({ // (docs/accessibility.md): it takes about 40% of a journey's time, and // the same screens are checked in Chromium. projects: [ - { name: "chromium", use: { ...devices["Desktop Chrome"] } }, + // Chromium runs every journey, in two shards in CI: a file's journeys + // too may run side by side, and in either shard, so pod-safety's six + // do not take one shard's last 100 seconds. A file whose journeys + // depend on each other says so (studio.journey.ts, serial). + { name: "chromium", use: { ...devices["Desktop Chrome"] }, fullyParallel: true }, { name: "firefox", use: { ...devices["Desktop Firefox"] }, From 488a3ab6fc8021d34d974c6bc602c5af07cd5fe6 Mon Sep 17 00:00:00 2001 From: Anton Wiklund Date: Tue, 20 Oct 2026 15:37:33 +0200 Subject: [PATCH 7/7] test(journeys): write, try and publish a course in a journey of its own The Studio journey took 2.4 minutes in CI, the longest test by far, and one test is the one thing a shard or a worker cannot split. Its course steps, writing a course, checking its release, test-playing it in the trial and publishing it, now make a journey of their own in studio-course.journey.ts, with its own user, instance and studied deck, followed by the journey that adds the published course from its link. The Studio journey keeps the decks, the workbench, the drafts of a deck and a library deck's next version, and the graph; locally it went from the whole to 64 seconds, and the course journey takes 28. Closes: #201 --- docs/statistics.md | 4 +- docs/testing.md | 5 +- docs/user-guide.md | 7 +- .../journeys/studio-course.journey.ts | 235 ++++++++++++++++++ e2e/journeys/journeys/studio.journey.ts | 221 +--------------- e2e/journeys/playwright.config.ts | 2 +- 6 files changed, 258 insertions(+), 216 deletions(-) create mode 100644 e2e/journeys/journeys/studio-course.journey.ts diff --git a/docs/statistics.md b/docs/statistics.md index 816fdae6..10be3158 100644 --- a/docs/statistics.md +++ b/docs/statistics.md @@ -436,8 +436,8 @@ rest of their study when they log in. and not as recall, and the chapter just completed under "Your courses", done, with its graded answers; and step 10 that the course's deck names the wrong options chosen in its final review. In - [studio.journey.ts](../e2e/journeys/journeys/studio.journey.ts), - steps 30 and 32 check that trying a course in the Studio's sandbox + [studio-course.journey.ts](../e2e/journeys/journeys/studio-course.journey.ts), + steps 07 and 09 check that trying a course in the Studio's sandbox leaves the statistics as they were. The page object is [Statistics.ts](../e2e/journeys/pages/Statistics.ts). The latest run's screenshots are online at . diff --git a/docs/testing.md b/docs/testing.md index 5dbcae4e..38dd9e73 100644 --- a/docs/testing.md +++ b/docs/testing.md @@ -530,8 +530,9 @@ test("a learner's first deck @smoke", async ({ app, account, runId }) => { ([test/dialogs.ts](../apps/studio/src/test/dialogs.ts)). - **One after another**: a journey that needs what another made is in its file, after it, and the file runs in turn (`mode: "serial"`): the - course the Studio journey publishes is added from its link by the next - journey in [studio.journey.ts](../e2e/journeys/journeys/studio.journey.ts), + course the Studio's course journey publishes is added from its link by + the next journey in + [studio-course.journey.ts](../e2e/journeys/journeys/studio-course.journey.ts), which is skipped when the first fails. The harness's own code (`harness/`) has unit tests (`npm test`, so `npm diff --git a/docs/user-guide.md b/docs/user-guide.md index fe58538d..3641f810 100644 --- a/docs/user-guide.md +++ b/docs/user-guide.md @@ -455,14 +455,15 @@ screenshot of each. | The course for newcomers, from an empty deck list | [newcomer-course.journey.ts](../e2e/journeys/journeys/newcomer-course.journey.ts) | 03–05 | | A course's first chapter and its final review | [course-chapter.journey.ts](../e2e/journeys/journeys/course-chapter.journey.ts) | 03–08 | | A course's answers and chapters in the statistics, and its most-confused options | course-chapter | 09–10 | -| Adding a course from a link and playing it | [studio.journey.ts](../e2e/journeys/journeys/studio.journey.ts), second journey | 03–05 | -| Solid Memo Studio | studio.journey.ts, first journey | 06–36 | +| Adding a course from a link and playing it | [studio-course.journey.ts](../e2e/journeys/journeys/studio-course.journey.ts), second journey | 03–05 | +| Solid Memo Studio | [studio.journey.ts](../e2e/journeys/journeys/studio.journey.ts) | 06–31 | +| Writing a course in the Studio, trying it and publishing it | studio-course.journey.ts, first journey | 05–11 | | Trying the app as a guest, and moving the study into a pod | [guest.journey.ts](../e2e/journeys/journeys/guest.journey.ts) | 02–11 | | Copying a library deck's link, and opening it in another browser as a guest | [try-link.journey.ts](../e2e/journeys/journeys/try-link.journey.ts) | 02–05 | | A screen that breaks: Something went wrong, its details and its report | [render-error.journey.ts](../e2e/journeys/journeys/render-error.journey.ts) | 02–06 | | The dark theme, kept in the Pod and found on a second device | [returning-learner.journey.ts](../e2e/journeys/journeys/returning-learner.journey.ts) | 03–04, 08–10 | | A second instance, and switching between the two | [second-instance.journey.ts](../e2e/journeys/journeys/second-instance.journey.ts) | 04–08 | -| Exporting a deck as a file and importing it again, with its progress | studio.journey.ts, first journey | 21–23 | +| Exporting a deck as a file and importing it again, with its progress | studio.journey.ts | 21–23 | | Switching the scheduler to FSRS-6 from the deck list's invitation, then reviewing with it | [later-day.journey.ts](../e2e/journeys/journeys/later-day.journey.ts) | 03, 06–09 | | Reviewing on a later study day, forgetting cards, and the hardest cards | later-day | 07–12 | | Updating a library deck to its new release, its progress kept | [library-upgrade.journey.ts](../e2e/journeys/journeys/library-upgrade.journey.ts) | 04–10 | diff --git a/e2e/journeys/journeys/studio-course.journey.ts b/e2e/journeys/journeys/studio-course.journey.ts new file mode 100644 index 00000000..bbeae22f --- /dev/null +++ b/e2e/journeys/journeys/studio-course.journey.ts @@ -0,0 +1,235 @@ +// SPDX-FileCopyrightText: 2026 Anton Wiklund and Solid Memo contributors +// +// SPDX-License-Identifier: MIT + +import { expect } from "@playwright/test"; +import { test } from "../fixtures.ts"; +import { logInAndCreateInstance } from "../flows/logIn.ts"; + +// The second journey adds what the first publishes: they run in turn, the second only after the first passed. +test.describe.configure({ mode: "serial" }); + +/** The course the first journey published, at its address: what the next journey adds from a link. */ +let published: { url: string; title: string } | null = null; + +/** The course the first journey writes: its one chapter, its step's theory, and its questions by their right answers. */ +const COURSE = { + chapter: "Pods", + theory: "A pod is where your data lives.", + answers: { "Where does your data live?": "In a pod", "Who chooses the app?": "You do" }, +}; + +/** + * Writing a course in Solid Memo Studio (docs/studio.md), from a blank + * draft to its release in the user's pod. A user logs in, creates an + * instance, makes a deck with a card and studies it, and opens the + * instance in the Studio. They write a course in a blank + * draft: a chapter, a step with its theory, and two questions, the first + * with two wrong options, saved as they go, trying one question as the + * course would ask it. Its release check lists what is still wrong: the + * second question has no wrong options and the release no description. + * They follow each problem to its field, fix it, and see it clear; the + * shapes find what a release still lacks, and the listing preview shows + * the course as the library will. They note what Solid Memo's statistics + * count, then test-play the course in the Studio's trial, to the end of + * its one chapter's final review. The instance, and Solid Memo's + * statistics, are as they were: the trial is played in a sandbox. They + * say how the course was made, then publish it to their pod, saying it + * will be public: someone with no login reads it, and the releases list + * it. Managing an instance's decks in the Studio is studio.journey.ts's. + */ +test("write a course in the Studio, try it, and publish it to the pod @studio", async ({ + app, + account, + runId, + playwright, +}) => { + const instance = `Courses ${runId}`; + const deck = `Studied ${runId}`; + const authored = `Authored ${runId}`; + + await app.step("01 · Visit Solid Memo", () => app.onboarding.visit()); + await app.step("02 · Log in and create an instance", () => + logInAndCreateInstance(app, account, instance), + ); + // Something studied, for the statistics the trial must leave as they were. + await app.step("03 · Create a deck with a card, and study it", async () => { + await app.decks.openDeckCreator(); + await app.deckCreator.create(deck); + await app.deckBrowser.openFor(deck); + await app.deckBrowser.openCardCreator(); + await app.cardCreator.addCards([["Sun", "Sol"]], { front: "en", back: "sv" }); + await app.cardCreator.back(); + await app.chrome.breadcrumb("breadcrumbs.decks"); + await app.decks.study(deck); + await app.study.studyAll({ grade: "good" }); + await app.study.endSession(); + }); + await app.step("04 · Open the instance in Solid Memo Studio", () => app.studio.openFromApp()); + await app.step( + "05 · Write a course: a chapter, a step, and two questions with two wrong options each", + async () => { + await app.studio.openDrafts(instance); + await app.studio.draftCourse(instance, authored); + await app.draftEditor.openDraft(authored); + await app.draftEditor.addChapter("Pods"); + await app.draftEditor.openChapter("Pods"); + await app.draftEditor.addStep(); + await app.draftEditor.writeTheory("A pod is where your data lives."); + await app.draftEditor.addQuestion("Where does your data live?", "In a pod"); + await app.draftEditor.addQuestion("Who chooses the app?", "You do"); + await app.draftEditor.openQuestion("Where does your data live?"); + await app.draftEditor.addWrongOption("In the app", "Apps only read it."); + await app.draftEditor.addWrongOption("On a server of the app's", "The pod is yours."); + await app.draftEditor.answerPreviewWrong("In the app", "Apps only read it."); + await app.draftEditor.expectOutline(authored, "Pods", "Step 1.1", [ + "Where does your data live?", + "Who chooses the app?", + ]); + await app.draftEditor.expectBadge("Who chooses the app?", 1); + }, + ); + + await app.step( + "06 · Check the release: follow its problems to their fields, fix them, and see them clear", + async () => { + const fewOptions = app.t("studio.problem.fewDistractors", { count: 0, least: 2 }); + const noDescription = app.t("studio.problem.required", { + field: app.t("studio.field.description"), + }); + const release = app.t("studio.check.release"); + await app.draftEditor.openCheck(); + await app.draftEditor.expectProblem("Who chooses the app?", fewOptions); + await app.draftEditor.expectProblem(release, noDescription); + await app.draftEditor.followProblem("Who chooses the app?", fewOptions); + await app.draftEditor.addWrongOption("The pod", "A pod holds data."); + await app.draftEditor.addWrongOption("Its maker", "The user chooses."); + await app.draftEditor.openCheck(); + await app.draftEditor.expectNoProblem("Who chooses the app?", fewOptions); + await app.draftEditor.followProblem(release, noDescription); + await app.draftEditor.writeDescription("What a pod is, and who holds the data in it."); + await app.draftEditor.openCheck(); + await app.draftEditor.expectNoProblem(release, noDescription); + await app.draftEditor.checkShapes(); + await app.draftEditor.openPreview(authored); + await app.chrome.breadcrumb("breadcrumbs.decks"); + }, + ); + + const counted: Record = {}; + const tiles = ["answers", "studyDays", "cards"] as const; + await app.step("07 · Note what Solid Memo's statistics count", async () => { + await app.studio.backToApp(); + await app.statistics.open(); + for (const tile of tiles) counted[tile] = await app.statistics.value(tile); + }); + + await app.step( + "08 · Test-play the course in the Studio's trial, to the end of its final review", + async () => { + await app.studio.openFromApp(); + await app.studio.openDrafts(instance); + await app.draftEditor.openDraft(authored); + await app.trial.open(authored); + await app.trial.playToTheEnd(COURSE.chapter, COURSE.theory, COURSE.answers); + }, + ); + + await app.step( + "09 · The instance's decks and Solid Memo's statistics are as they were", + async () => { + await app.chrome.breadcrumb("breadcrumbs.decks"); + await app.studio.expectDeck(instance, deck, { cards: 1, due: 0 }); + await app.studio.expectNoDeck(instance, authored); + await app.studio.backToApp(); + await app.statistics.open(); + for (const tile of tiles) expect(await app.statistics.value(tile), tile).toBe(counted[tile]); + }, + ); + + await app.step( + "10 · Say how the course was made: its author, an attribution, its license, a source and an AI check; follow the source's warning and fix it", + async () => { + const source = "https://example.org/pods-explained"; + const undescribed = app.t("studio.problem.sourceUndescribed", { + missing: app.t("studio.release.detail.creator"), + }); + await app.studio.openFromApp(); + await app.studio.openDrafts(instance); + await app.draftEditor.openDraft(authored); + await app.draftEditor.openRelease(); + await app.draftEditor.addAuthor("Journey Author"); + await app.draftEditor.attributeWithAi("Journey Author"); + await app.draftEditor.chooseReleaseLicense("CC0 1.0"); + await app.draftEditor.addSource(source, "Pods explained", "The page's footer: CC0."); + await app.draftEditor.recordAiCheck("Facts", "every question", "none wrong"); + await app.draftEditor.openCheck(); + await app.draftEditor.expectProblem(source, undescribed); + await app.draftEditor.followProblem(source, undescribed); + await app.draftEditor.describeSource(source, "Pods explained", "Example authors"); + await app.draftEditor.openCheck(); + await app.draftEditor.expectNoProblem(source, undescribed); + }, + ); + + await app.step( + "11 · Publish the course to the pod, public: anyone can read it, and the releases list it", + async () => { + await app.draftEditor.openRelease(); + await app.draftEditor.choosePublisher("Journey Author"); + const url = await app.draftEditor.publish(); + // Someone with no login reads it. + const anyone = await playwright.request.newContext(); + const read = await anyone.get(url, { headers: { accept: "text/turtle" } }); + expect(read.status()).toBe(200); + expect(await read.text()).toContain(`"${authored}"@en`); + await anyone.dispose(); + await app.studio.openReleases(instance); + await app.studio.expectRelease(instance, url, `authored-${runId}`, 1); + published = { url, title: authored }; + }, + ); +}); + +/** + * A course published in a pod, added from its link by someone else + * (docs/deck-library.md, From a link): another user, with an account and + * a pod of their own, opens "Add from a link" from their empty deck list + * and pastes the address of the course the journey before published. + * The release is read and shown, with the host it is published on, as a + * course. They start it, and play its chapter to the end of its final + * review, as a library course is played. Their deck list then says where + * the course came from. + */ +test("add a course published in a pod from its link, as another user, and play its chapter @studio", async ({ + app, + account, + runId, +}) => { + const course = published; + expect(course, "the journey just before publishes the course").not.toBeNull(); + const { url, title } = course!; + + await app.step("01 · Visit Solid Memo", () => app.onboarding.visit()); + await app.step("02 · Log in as another user and create an instance", () => + logInAndCreateInstance(app, account, `Learner ${runId}`), + ); + await app.step( + "03 · Paste the course's link: what it is, and where it is published", + async () => { + await app.importUrl.openFromDeckList(); + await app.importUrl.showCourse(url, title); + }, + ); + await app.step( + "04 · Start the course and play its chapter to the end of its final review", + async () => { + await app.importUrl.startCourse(title); + await app.course.playChapter(COURSE.chapter, COURSE.theory, COURSE.answers); + }, + ); + await app.step("05 · Back at Decks: the course says where it came from", async () => { + await app.chrome.breadcrumb("breadcrumbs.decks"); + await app.decks.expectDeckFrom(title, new URL(url).host); + }); +}); diff --git a/e2e/journeys/journeys/studio.journey.ts b/e2e/journeys/journeys/studio.journey.ts index 4f768fe4..3c5d1701 100644 --- a/e2e/journeys/journeys/studio.journey.ts +++ b/e2e/journeys/journeys/studio.journey.ts @@ -4,22 +4,8 @@ import { expect } from "@playwright/test"; import { test } from "../fixtures.ts"; -import { logInAndCreateInstance } from "../flows/logIn.ts"; import { BRIGHTEST_STARS } from "../pages/Library.ts"; -// The second journey adds what the first publishes: they run in turn, the second only after the first passed. -test.describe.configure({ mode: "serial" }); - -/** The course the Studio journey published, at its address: what the next journey adds from a link. */ -let published: { url: string; title: string } | null = null; - -/** The course the Studio journey writes: its one chapter, its step's theory, and its questions by their right answers. */ -const COURSE = { - chapter: "Pods", - theory: "A pod is where your data lives.", - answers: { "Where does your data live?": "In a pod", "Who chooses the app?": "You do" }, -}; - /** * Solid Memo Studio in Solid Memo's page (docs/studio.md): a visitor * opens the Studio's old address, /studio/, which lands on the Studio's @@ -53,26 +39,15 @@ const COURSE = { * export one of them as a Turtle file, with their progress, delete it, * once the user confirms, and import it again from the file, its card * due as before. They draft a release of a deck and a blank course, - * which the sidebar lists, and delete the course's draft, confirming. They - * write a course in a blank draft: a chapter, a step with its theory, - * and two questions, the first with two wrong options, saved as they - * go, trying one question as the course would ask it. Its release check - * lists what is still wrong: the second question has no wrong options - * and the release no description. They follow each problem to its field, - * fix it, and see it clear; the shapes find what a release still lacks, - * and the listing preview shows the course as the library will. They rename the instance + * which the sidebar lists, and delete the course's draft, confirming. + * They rename the instance * and describe its catalog, under a license, then pick it, by its new * name, from the picker. Among its library copies, the library deck is - * up to date. They note what Solid Memo's statistics count, then - * test-play the course they wrote in the Studio's trial, to the end of - * its one chapter's final review. The instance's decks, and Solid - * Memo's statistics, are as they were: the trial is played in a sandbox. + * up to date. * They draft the next version of the library deck, by its address, * retire one of its cards, and compare the draft with the release: it * keeps the series' rules, lists the card retired, and a learner's copy - * would retire it too, losing nothing. They say how the course was made, - * then publish it to their pod, saying it will be public: someone with no - * login reads it, and the releases list it. They move down the library + * would retire it too, losing nothing. They move down the library * deck's cards with the keyboard and back, one selected: the row with * the focus stays in sight between the header row and the bar of the * rows selected, which stick. They draw the instance as a graph, open @@ -80,13 +55,13 @@ const COURSE = { * draft of the next version follows: a leaf outside the pod, never * fetched, shown with the draft that links to it as a table, and let go * of it with a click on the drawing's background. They go back to Solid Memo, still - * logged in, which lists the deck they made that is left. + * logged in, which lists the deck they made that is left. Writing, + * checking, trying and publishing a course is studio-course.journey.ts's. */ test("open the Studio from Solid Memo and manage an instance's decks @studio", async ({ app, account, runId, - playwright, }) => { const instance = `Studio ${runId}`; const alpha = `Alpha ${runId}`; @@ -94,7 +69,6 @@ test("open the Studio from Solid Memo and manage an instance's decks @studio", a const group = `Pair ${runId}`; const renamed = `Renamed ${runId}`; const course = `Course ${runId}`; - const authored = `Authored ${runId}`; await app.step("01 · Visit the Studio's old address: the Studio's landing page", () => app.studio.visitOldAddress(), @@ -315,108 +289,25 @@ test("open the Studio from Solid Memo and manage an instance's decks @studio", a }, ); - await app.step( - "25 · Write a course: a chapter, a step, and two questions with two wrong options each", - async () => { - await app.studio.openDrafts(instance); - await app.studio.draftCourse(instance, authored); - await app.draftEditor.openDraft(authored); - await app.draftEditor.addChapter("Pods"); - await app.draftEditor.openChapter("Pods"); - await app.draftEditor.addStep(); - await app.draftEditor.writeTheory("A pod is where your data lives."); - await app.draftEditor.addQuestion("Where does your data live?", "In a pod"); - await app.draftEditor.addQuestion("Who chooses the app?", "You do"); - await app.draftEditor.openQuestion("Where does your data live?"); - await app.draftEditor.addWrongOption("In the app", "Apps only read it."); - await app.draftEditor.addWrongOption("On a server of the app's", "The pod is yours."); - await app.draftEditor.answerPreviewWrong("In the app", "Apps only read it."); - await app.draftEditor.expectOutline(authored, "Pods", "Step 1.1", [ - "Where does your data live?", - "Who chooses the app?", - ]); - await app.draftEditor.expectBadge("Who chooses the app?", 1); - }, - ); - - await app.step( - "26 · Check the release: follow its problems to their fields, fix them, and see them clear", - async () => { - const fewOptions = app.t("studio.problem.fewDistractors", { count: 0, least: 2 }); - const noDescription = app.t("studio.problem.required", { - field: app.t("studio.field.description"), - }); - const release = app.t("studio.check.release"); - await app.draftEditor.openCheck(); - await app.draftEditor.expectProblem("Who chooses the app?", fewOptions); - await app.draftEditor.expectProblem(release, noDescription); - await app.draftEditor.followProblem("Who chooses the app?", fewOptions); - await app.draftEditor.addWrongOption("The pod", "A pod holds data."); - await app.draftEditor.addWrongOption("Its maker", "The user chooses."); - await app.draftEditor.openCheck(); - await app.draftEditor.expectNoProblem("Who chooses the app?", fewOptions); - await app.draftEditor.followProblem(release, noDescription); - await app.draftEditor.writeDescription("What a pod is, and who holds the data in it."); - await app.draftEditor.openCheck(); - await app.draftEditor.expectNoProblem(release, noDescription); - await app.draftEditor.checkShapes(); - await app.draftEditor.openPreview(authored); - await app.chrome.breadcrumb("breadcrumbs.decks"); - }, - ); - - await app.step("27 · Rename the instance and describe its catalog", async () => { + await app.step("25 · Rename the instance and describe its catalog", async () => { await app.studio.openInstance(instance); await app.studio.renameInstance(renamed); await app.studio.describeCatalog("Decks for the Studio journey.", "CC0 1.0"); }); - await app.step("28 · Pick the instance, by its new name, from the instance picker", async () => { + await app.step("26 · Pick the instance, by its new name, from the instance picker", async () => { await app.studio.pickInstance(renamed); await app.studio.expectDeck(renamed, alpha, { cards: 1, due: 0 }); }); - await app.step("29 · See the library deck among the library copies, up to date", async () => { + await app.step("27 · See the library deck among the library copies, up to date", async () => { await app.studio.openLibraryCopies(renamed, BRIGHTEST_STARS.en); await app.chrome.breadcrumb("breadcrumbs.decks"); }); - const counted: Record = {}; - const tiles = ["answers", "studyDays", "cards"] as const; - await app.step("30 · Note what Solid Memo's statistics count", async () => { - await app.studio.backToApp(); - await app.statistics.open(); - for (const tile of tiles) counted[tile] = await app.statistics.value(tile); - }); - - await app.step( - "31 · Test-play the course in the Studio's trial, to the end of its final review", - async () => { - await app.studio.openFromApp(); - await app.studio.openDrafts(renamed); - await app.draftEditor.openDraft(authored); - await app.trial.open(authored); - await app.trial.playToTheEnd(COURSE.chapter, COURSE.theory, COURSE.answers); - }, - ); - - await app.step( - "32 · The instance's decks and Solid Memo's statistics are as they were", - async () => { - await app.chrome.breadcrumb("breadcrumbs.decks"); - await app.studio.expectDeck(renamed, alpha, { cards: 1, due: 0 }); - await app.studio.expectDeck(renamed, beta, { cards: 1, due: 1 }); - await app.studio.expectNoDeck(renamed, authored); - await app.studio.backToApp(); - await app.statistics.open(); - for (const tile of tiles) expect(await app.statistics.value(tile), tile).toBe(counted[tile]); - }, - ); - await app.step( - "33 · Draft the library deck's next version, retire a card, and compare it with the release: learners lose nothing", + "28 · Draft the library deck's next version, retire a card, and compare it with the release: learners lose nothing", async () => { - await app.studio.openFromApp(); await app.studio.openDrafts(renamed); // A library release by its address, which the site serves (docs/deck-library.md). const release = "https://solid-memo.com/decks/brightest-stars/v1.ttl"; @@ -435,50 +326,7 @@ test("open the Studio from Solid Memo and manage an instance's decks @studio", a ); await app.step( - "34 · Say how the course was made: its author, an attribution, its license, a source and an AI check; follow the source's warning and fix it", - async () => { - const source = "https://example.org/pods-explained"; - const undescribed = app.t("studio.problem.sourceUndescribed", { - missing: app.t("studio.release.detail.creator"), - }); - await app.chrome.breadcrumb("breadcrumbs.decks"); - await app.studio.openDrafts(renamed); - await app.draftEditor.openDraft(authored); - await app.draftEditor.openRelease(); - await app.draftEditor.addAuthor("Journey Author"); - await app.draftEditor.attributeWithAi("Journey Author"); - await app.draftEditor.chooseReleaseLicense("CC0 1.0"); - await app.draftEditor.addSource(source, "Pods explained", "The page's footer: CC0."); - await app.draftEditor.recordAiCheck("Facts", "every question", "none wrong"); - await app.draftEditor.openCheck(); - await app.draftEditor.expectProblem(source, undescribed); - await app.draftEditor.followProblem(source, undescribed); - await app.draftEditor.describeSource(source, "Pods explained", "Example authors"); - await app.draftEditor.openCheck(); - await app.draftEditor.expectNoProblem(source, undescribed); - }, - ); - - await app.step( - "35 · Publish the course to the pod, public: anyone can read it, and the releases list it", - async () => { - await app.draftEditor.openRelease(); - await app.draftEditor.choosePublisher("Journey Author"); - const url = await app.draftEditor.publish(); - // Someone with no login reads it. - const anyone = await playwright.request.newContext(); - const read = await anyone.get(url, { headers: { accept: "text/turtle" } }); - expect(read.status()).toBe(200); - expect(await read.text()).toContain(`"${authored}"@en`); - await anyone.dispose(); - await app.studio.openReleases(renamed); - await app.studio.expectRelease(renamed, url, `authored-${runId}`, 1); - published = { url, title: authored }; - }, - ); - - await app.step( - "36 · Move down the library deck's cards with the keyboard and back, one selected: each row in sight between the header row and the bar", + "29 · Move down the library deck's cards with the keyboard and back, one selected: each row in sight between the header row and the bar", async () => { await app.chrome.breadcrumb("breadcrumbs.decks"); await app.studio.moveThroughCardsInSight(renamed, BRIGHTEST_STARS.en, 30); @@ -486,7 +334,7 @@ test("open the Studio from Solid Memo and manage an instance's decks @studio", a ); await app.step( - "37 · Draw the instance as a graph: open a node by keyboard, find the library release the next version follows, a leaf never fetched, see it and its draft in the table, and let go of it", + "30 · Draw the instance as a graph: open a node by keyboard, find the library release the next version follows, a leaf never fetched, see it and its draft in the table, and let go of it", async () => { await app.chrome.breadcrumb("breadcrumbs.decks"); await app.studio.openGraph(renamed); @@ -503,52 +351,9 @@ test("open the Studio from Solid Memo and manage an instance's decks @studio", a }, ); - await app.step("38 · Go back to Solid Memo, still logged in", async () => { + await app.step("31 · Go back to Solid Memo, still logged in", async () => { await app.studio.backToApp(); await app.chrome.expectLoggedInAs(account.webId); await app.decks.expectDeck(alpha); }); }); - -/** - * A course published in a pod, added from its link by someone else - * (docs/deck-library.md, From a link): another user, with an account and - * a pod of their own, opens "Add from a link" from their empty deck list - * and pastes the address of the course the Studio journey published. - * The release is read and shown, with the host it is published on, as a - * course. They start it, and play its chapter to the end of its final - * review, as a library course is played. Their deck list then says where - * the course came from. - */ -test("add a course published in a pod from its link, as another user, and play its chapter @studio", async ({ - app, - account, - runId, -}) => { - const course = published; - expect(course, "the Studio journey, just before, publishes the course").not.toBeNull(); - const { url, title } = course!; - - await app.step("01 · Visit Solid Memo", () => app.onboarding.visit()); - await app.step("02 · Log in as another user and create an instance", () => - logInAndCreateInstance(app, account, `Learner ${runId}`), - ); - await app.step( - "03 · Paste the course's link: what it is, and where it is published", - async () => { - await app.importUrl.openFromDeckList(); - await app.importUrl.showCourse(url, title); - }, - ); - await app.step( - "04 · Start the course and play its chapter to the end of its final review", - async () => { - await app.importUrl.startCourse(title); - await app.course.playChapter(COURSE.chapter, COURSE.theory, COURSE.answers); - }, - ); - await app.step("05 · Back at Decks: the course says where it came from", async () => { - await app.chrome.breadcrumb("breadcrumbs.decks"); - await app.decks.expectDeckFrom(title, new URL(url).host); - }); -}); diff --git a/e2e/journeys/playwright.config.ts b/e2e/journeys/playwright.config.ts index e99edcfb..348c91f3 100644 --- a/e2e/journeys/playwright.config.ts +++ b/e2e/journeys/playwright.config.ts @@ -73,7 +73,7 @@ export default defineConfig({ // Chromium runs every journey, in two shards in CI: a file's journeys // too may run side by side, and in either shard, so pod-safety's six // do not take one shard's last 100 seconds. A file whose journeys - // depend on each other says so (studio.journey.ts, serial). + // depend on each other says so (studio-course.journey.ts, serial). { name: "chromium", use: { ...devices["Desktop Chrome"] }, fullyParallel: true }, { name: "firefox",