feat(decks): give 17 label decks Korean, German, Spanish and French card text from Wikidata

.github/workflows/ci.yml

@@ -351,7 +351,8 @@ jobs:
       - run: pip install -r scripts/requirements-ci.txt
       # The script's own tests (scripts/test_shacl_crosscheck.py), on
       # small documents of their own, before it checks the real ones, and
-      # the taxonomy deck generator's (scripts/test_taxonomy_deck.py).
+      # the taxonomy deck generator's (scripts/test_taxonomy_deck.py) and the
+      # Wikidata labels script's (scripts/test_wikidata_labels.py).
       - run: python -m unittest discover -s scripts
       - run: python scripts/shacl_crosscheck.py
 

.github/workflows/wikidata.yml

@@ -0,0 +1,45 @@
+# SPDX-FileCopyrightText: 2026 Anton Wiklund and Solid Memo contributors
+#
+# SPDX-License-Identifier: MIT
+
+name: Wikidata
+
+# Once a week, check that the decks whose cards take their Korean, German,
+# Spanish and French text from Wikidata come out the same again: for each
+# deck with a mapping in scripts/wikidata-labels/, the newest version
+# scripts/wikidata_labels.py made is made again from the Wikidata revisions
+# it records (docs/deck-library.md#languages-from-wikidata). It runs on a
+# schedule and by hand, never on a pull request: when Wikidata cannot be
+# reached, the script warns and the run passes.
+
+on:
+  schedule:
+    - cron: "41 5 * * 2"
+  workflow_dispatch:
+
+permissions:
+  contents: read
+
+concurrency:
+  group: wikidata
+  cancel-in-progress: false
+
+jobs:
+  verify:
+    runs-on: ubuntu-latest
+    timeout-minutes: 30
+    steps:
+      - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1
+        with:
+          persist-credentials: false
+      - uses: actions/setup-python@5fda3b95a4ea91299a34e894583c3862153e4b97 # v7.0.0
+        with:
+          python-version: "3.15"
+      # The labels of each revision read: a revision never changes, so they
+      # are kept, and Wikidata is asked again only for revisions not kept.
+      - uses: actions/cache@55cc8345863c7cc4c66a329aec7e433d2d1c52a9 # v6.1.0
+        with:
+          path: ~/.cache/wikidata
+          key: wikidata-${{ hashFiles('decks/**/*.ttl') }}
+          restore-keys: wikidata-
+      - run: python3 scripts/wikidata_labels.py --verify --cache-dir ~/.cache/wikidata

cspell.config.yaml

@@ -118,6 +118,7 @@ words:
   - foaf
   - FSRS
   - iiif
+  - maxlag
   - PKCE
   - RDAP
   - rdfs

docs/deck-library.md

@@ -261,7 +261,8 @@ in two decks. A card about several things names each.
 - **A deck compiled from linked data keeps its IDs** from its next
   version on: the taxonomy deck, whose script writes them, and the
   decks compiled from Wikidata, such as `famous-paintings` and
-  `capitals-of-the-world`, their QIDs.
+  `capitals-of-the-world`, their QIDs. The label decks get theirs with
+  their [languages from Wikidata](#languages-from-wikidata).
 
 ## Publishing a new version
 
@@ -434,6 +435,91 @@ failure issue links that run. To change the key, generate a new one in
 the App's settings, replace the environment's secret, then delete the
 old key there.
 
+## Languages from Wikidata
+
+Many decks are labels of well-known things, a country and its capital, an
+element, a star, and their cards were written in English and Swedish.
+Wikidata has a name for each thing in every language the app speaks,
+written by people and CC0, as the decks are.
+[scripts/wikidata_labels.py](../scripts/wikidata_labels.py) makes a
+deck's next version with its cards in Korean, German, Spanish and
+French as well, from those labels, with Python's standard library
+alone. Its docstring says how, in full.
+
+**The mapping** says what each card is about:
+`scripts/wikidata-labels/<deck>.json`, one per deck, written once and
+checked by a person, so nothing is guessed again on a later run. It
+names every card in use, by its fragment id, with what its front and
+back are made of: a template such as `"{Q224}"` or
+`"{Q199471} (AFN)"`, in which `{Q…}` is the item's label, or a frame
+written in each language, such as "Highest point of {0}", for text
+built from more than one part. A card it cannot make it skips, saying
+why. A card may refuse a label Wikidata gives that is wrong for it, such
+as a species' former scientific name or the name of another species:
+the card goes without that language until the label is mended on
+Wikidata. It also translates the deck's card labels ("Capital"), and gives
+the next version's description, which says which languages the cards
+are in.
+
+**What a card gets.** A card gets a language when every item its
+sides name has a label in it; else it goes without that language, and
+the report says which item lacks it. Its labels get the languages its
+sides get; its notes stay as they are. English and Swedish text never
+changes: some Swedish names follow the Swedish Language Council rather
+than Wikidata, and some English names are the author's own wording.
+Each card names its items as `dcterms:subject`
+([What a card is about](#what-a-card-is-about)). The report also lists
+where the card's English or Swedish text differs from what the mapping
+makes of Wikidata's labels in those languages: often on purpose, but it
+is how a wrong item shows.
+
+**Reproducible.** The version records how it was made, in
+`<#wikidata-labels>`: the script at the commit that last changed it,
+with its SHA-256 and the Python and Unicode versions, the mapping by its
+path and SHA-256, the command line, and the revision of every item it
+read
+(`https://www.wikidata.org/wiki/Special:EntityData/Q224.json?revision=N`).
+`--verify` makes the cards of the newest version the script made again,
+from the version before it, the mapping and those revisions, and fails
+unless they come out byte for byte the same. So the script refuses to
+run unless it is committed and unmodified (`--allow-uncommitted` writes
+a draft), and the mapping is committed with the version it made.
+
+**Polite to Wikidata.** One request at a time, 50 items a request, an
+honest User-Agent with a contact URL, gzip, and waiting as long as
+Wikidata asks when it is busy, as Wikimedia's
+[User-Agent policy](https://foundation.wikimedia.org/wiki/Policy:User-Agent_policy)
+and [API etiquette](https://www.mediawiki.org/wiki/API:Etiquette) ask.
+The requests only read, so they send no `maxlag`: on Wikidata it counts
+the query service's lag, which edits cause, to hold back bots that
+edit, and it can refuse every request for hours.
+`--cache-dir` keeps the labels of each revision read, since a revision
+never changes. `--verify` asks for an old revision's content only for an
+item edited since.
+
+To make a deck's next version, from the repository root on an
+up-to-date `main` with the mapping written:
+
+```sh
+python3 scripts/wikidata_labels.py --deck capitals-of-the-world --cache-dir ~/.cache/wikidata
+npm run format:turtle
+npm run library
+npm run library:check -- --base origin/main
+```
+
+Commit the new version with the index and the mapping as a
+`feat(decks)` commit, as for
+[any new version](#publishing-a-new-version). A version made by hand
+later keeps the cards' text; the next run of the script writes the four
+languages afresh from the labels, so a wrong label is mended on
+Wikidata, or in the mapping.
+
+It is no check of CI, which never reaches Wikidata. The
+[Wikidata workflow](../.github/workflows/wikidata.yml) runs `--verify`
+on every deck with a mapping each Tuesday, and by hand (Actions ›
+Wikidata › Run workflow). When Wikidata cannot be reached, it warns and
+passes; when a deck's cards come out otherwise, it fails.
+
 ## Authoring Markdown
 
 A card, a course step or a chapter may be written in Markdown

docs/testing.md

@@ -32,7 +32,7 @@ npm run test:watch # watch mode, every package's tests
 npm run test:unit -w @solid-memo/domain -- account.test.ts   # one package's (test:watch too)
 npm run test:roundtrip -w @solid-memo/solid   # every library release through a draft and back (below)
 npm run crosscheck # the pySHACL cross-check, as CI runs it (needs scripts/requirements-ci.txt)
-npm run test:crosscheck   # the Python scripts' own tests (the cross-check's, the taxonomy deck generator's), Python's unittest (needs scripts/requirements-ci.txt too)
+npm run test:crosscheck   # the Python scripts' own tests (the cross-check's, the taxonomy deck generator's, the Wikidata labels script's), Python's unittest (needs scripts/requirements-ci.txt too)
 npm run check:closes -- --base origin/main   # a commit since the branch left main closes an issue (add --body-file <file> for the pull request's body), as CI's commits job checks it (CONTRIBUTING.md)
 npm run check:breaking -- --base origin/main   # the public API against where the branch left main, as CI's library job checks it (docs/releasing.md)
 npm run test:pod  # the end-to-end tests, against the Solid servers they start in Docker
@@ -273,11 +273,18 @@ DCAT-AP adds, `rdfs:subClassOf`, a subject with no shape) and what it
 reports when a document fails. The same command runs the tests of the
 taxonomy deck's generator
 ([scripts/test_taxonomy_deck.py](../scripts/test_taxonomy_deck.py),
-[deck-library.md](deck-library.md#the-taxonomy-deck)). CI's
+[deck-library.md](deck-library.md#the-taxonomy-deck)), and of the
+script that gives label decks their languages from Wikidata
+([scripts/test_wikidata_labels.py](../scripts/test_wikidata_labels.py),
+[deck-library.md](deck-library.md#languages-from-wikidata)), each against
+a fake taxonomy or Wikidata of its own. CI's
 `crosscheck` job runs them all before the cross-check; `npm run check`
 does not, as it needs no Python. The generator itself runs weekly, and
 by hand, in the [Taxonomy workflow](../.github/workflows/taxonomy.yml),
-which proposes each new version of its deck as a pull request.
+which proposes each new version of its deck as a pull request. The Wikidata
+labels script checks its decks against the revisions they record weekly,
+and by hand, in the [Wikidata workflow](../.github/workflows/wikidata.yml),
+never on a pull request.
 
 ## User journeys
 
@@ -618,7 +625,7 @@ jobs, side by side, each [deterministic or timing-dependent](#deterministic-and-
 | `reuse` | deterministic | nothing: on `main`, when the run reuses the one that tested its tree, it takes over what that run made, its site, kept as this run's `site` artifact, and, but on a release commit, its Turbo cache, saved as `main`'s. It runs none of the repository's code |
 | `release-files` | deterministic | on a release commit, when the run reuses the one that tested its parent: the files a release writes, by Biome, markdownlint and cspell, as `npm run check` has them, and `publiccode.yml` by its parser |
 | `library` | deterministic | the change against its base: the deck library, no published version edited or removed, and every new version naming the library's contact ([deck-library.md](deck-library.md#checks)); the public API, no line of `api/` gone unless a commit says the change breaks, and none of `api/frozen/` at all (`npm run check:breaking`, [releasing.md](releasing.md#how-ci-holds-to-it)) |
-| `crosscheck` | deterministic | the Python scripts' tests (the cross-check's and the taxonomy deck generator's), then the vocabulary, the deck library and the pod catalog documents, by pySHACL ([validation.md](validation.md#the-ci-cross-check)) |
+| `crosscheck` | deterministic | the Python scripts' tests (the cross-check's, the taxonomy deck generator's and the Wikidata labels script's), then the vocabulary, the deck library and the pod catalog documents, by pySHACL ([validation.md](validation.md#the-ci-cross-check)) |
 | `pod (<server>)` | timing-dependent | the end-to-end tests, one job per blocking server (above) |
 | `contract (<server>)` | timing-dependent | that each advisory server starts and meets the contract |
 | `journeys (<project>)` | timing-dependent | the user journeys, on `site`'s build: every one in Chromium, in two shards, and those that differ between engines in Firefox, WebKit and on a phone, a job each ([Browsers](#browsers)) |

e2e/journeys/journeys/library-upgrade.journey.ts

@@ -9,7 +9,7 @@ import { BRIGHTEST_STARS } from "../pages/Library.ts";
 /**
  * A learner who holds an older release of a library deck, as the
  * journey's spec numbers it: add release 1 of the brightest stars from
- * its link, study it, see release 2 offered in Solid Memo and in the
+ * its link, study it, see the newest release offered in Solid Memo and in the
  * Studio and read what it changes, update the copy, reload, and find the
  * progress kept, Solid Memo and the Studio agreeing that the copy is up
  * to date. A card studied offers to report a mistake in it, the
@@ -25,25 +25,33 @@ import { BRIGHTEST_STARS } from "../pages/Library.ts";
  * release, of a series the index lists, so it is a library copy.
  * Release 2 changes no card: it adds the deck's name, description and
  * keywords in Korean, German, Spanish and French, which the German name
- * shows once the copy is updated.
+ * shows once the copy is updated. Release 3 adds those languages to every
+ * card, from Wikidata's labels (docs/deck-library.md#languages-from-wikidata),
+ * so updating to it changes every card, the progress kept all the same.
  */
 test("a learner updates a library copy @upgrade", async ({ app, account, runId }) => {
   const instance = `Journey ${runId}`;
   const deck = BRIGHTEST_STARS.en;
   const releaseOne = "https://solid-memo.com/decks/brightest-stars/v1.ttl";
-  // decks/brightest-stars/v2.ttl: its name in German, its notes, and the
-  // offer's words for what a change of name, description and keywords does.
+  const cards = 52;
+  // decks/brightest-stars/v2.ttl and v3.ttl: the name in German, the notes
+  // of each, and the offer's words for what they do together: every card
+  // changed, and the name, description and keywords.
   const german = "Die hellsten Sterne des Nachthimmels";
-  const notes =
-    "Title, description and keywords in Korean, German, Spanish and French as well, now that the app speaks them; no card was changed.";
+  const notes = [
+    "Title, description and keywords in Korean, German, Spanish and French as well, now that the app speaks them; no card was changed.",
+    "Card text in Korean, German, Spanish and French as well, from Wikidata's labels of what each card names: of the 52 cards in use, 52 in Korean, 52 in German, 52 in Spanish, 52 in French; English and Swedish text unchanged. Each card names the Wikidata items it is about (dcterms:subject). The deck names the project's issue tracker as its contact, and no email.",
+  ];
   const upgrade = {
     from: 1,
-    to: 2,
-    changes: app.t("libraryUpgradeNotice.describes"),
+    to: 3,
+    changes: app.t("libraryUpgradeNotice.list", {
+      items: app.t("libraryUpgradeNotice.changes", { count: cards }),
+      last: app.t("libraryUpgradeNotice.describes"),
+    }),
     kept: app.t("libraryUpgradeNotice.historyKept"),
-    notes: [notes],
+    notes,
   };
-  const cards = 52;
   const newPerDay = 3;
 
   await app.step("01 · Visit Solid Memo", () => app.onboarding.visit());
@@ -79,7 +87,7 @@ test("a learner updates a library copy @upgrade", async ({ app, account, runId }
     await app.decks.expectAnsweredToday(newPerDay);
   });
 
-  await app.step("06 · Solid Memo offers release 2 and says what it changes", async () => {
+  await app.step("06 · Solid Memo offers release 3 and says what it changes", async () => {
     await app.decks.expectNewRelease(deck);
     await app.decks.openDeck(deck);
     await app.deckDetail.expectUpgradeOffer({ deck, ...upgrade });
@@ -89,25 +97,25 @@ test("a learner updates a library copy @upgrade", async ({ app, account, runId }
     await app.chrome.switchLanguage("en");
   });
 
-  await app.step("07 · The Studio offers release 2 too, with the same changes", async () => {
+  await app.step("07 · The Studio offers release 3 too, with the same changes", async () => {
     await app.studio.openFromApp();
     await app.studio.followUpdateBadge(instance, deck, upgrade.to);
     await app.studio.readCopyUpgrade(instance, deck, upgrade);
     await app.studio.backToApp();
   });
 
-  await app.step("08 · Update the copy to release 2 from its page", async () => {
+  await app.step("08 · Update the copy to release 3 from its page", async () => {
     await app.decks.openDeck(deck);
     await app.deckDetail.upgrade(upgrade.to);
   });
 
-  await app.step("09 · Reload: the copy is release 2, its progress kept", async () => {
+  await app.step("09 · Reload: the copy is release 3, its progress kept", async () => {
     await app.chrome.reload();
     await app.deckDetail.expectDeck(deck);
     await app.deckDetail.expectNoUpgradeOffer();
     await app.deckDetail.expectCardCount(cards);
     await app.deckDetail.expectDoneForToday(newPerDay);
-    // What release 2 changes: the deck now has a German name.
+    // What release 2 changed: the deck now has a German name.
     await app.chrome.switchLanguage("de");
     await app.deckDetail.expectDeck(german);
     await app.chrome.switchLanguage("en");

e2e/journeys/pages/Studio.ts

@@ -1089,7 +1089,8 @@ export class Studio extends Screen {
       const aside = this.page.getByRole("region", { name: deck, exact: true });
       await expect(aside).toContainText(said);
       await expect(aside).toContainText(upgrade.kept);
-      await expect(aside.getByRole("listitem")).toHaveText(
+      // The releases' notes are its first list; the cards each change touches follow in lists of their own.
+      await expect(aside.getByRole("list").first().getByRole("listitem")).toHaveText(
         upgrade.notes.map((notes, at) =>
           this.t("libraryUpgradeNotice.releaseNote", { version: upgrade.from + 1 + at, notes }),
         ),

e2e/pod/src/releases.integration.test.ts

@@ -325,6 +325,8 @@ describe.each(SERVERS)("releases on $name", ({ url: server }) => {
       keptReleases: true,
       keptDocuments: [],
     });
-    expect((await fetch(target, { headers: { accept: "text/turtle" } })).status).toBe(200);
+    // Asked by its head: a body left unread past 64 KiB, as a release grows, fails Node 24's fetch
+    // when the server then closes the connection (nodejs/undici#5360).
+    expect((await fetch(target, { method: "HEAD" })).status).toBe(200);
   });
 });

packages/solid/src/libraryImport.test.ts

@@ -261,7 +261,7 @@ describe("a deck imported from the library", () => {
 });
 
 const FIRST = `${SITE}decks/capitals-of-the-world/v1.ttl`;
-const LATEST = `${SITE}decks/capitals-of-the-world/v3.ttl`;
+const LATEST = `${SITE}decks/capitals-of-the-world/v4.ttl`;
 
 /** The card ids a release of the library holds, as the site publishes it through `site`. */
 async function releaseCardIds(url: string, site: typeof fetch = siteFetch): Promise<string[]> {