Generate the glossary from the Allen Glossary repository

.gitattributes

@@ -0,0 +1,6 @@
+# databook/glossary.md is rendered from the Allen Glossary by
+# scripts/build-glossary-page.mjs and committed because the databook is built
+# from Markdown by a capsule that has no Node. Marking it generated collapses it
+# in pull request diffs, so a reviewer reads the generator rather than several
+# thousand lines of its output.
+databook/glossary.md linguist-generated=true

.github/workflows/publish-glossary.yml

@@ -0,0 +1,161 @@
+name: Publish glossary page
+
+# Publishes just databook/glossary.md to the live site, without Code Ocean.
+#
+# The rest of the databook is built in the Code Ocean capsule, because its pages
+# execute notebooks against attached data assets. The glossary page executes
+# nothing, so it can be built by Actions alone and dropped onto gh-pages as a
+# single file. The other ~120 pages keep whatever the last capsule run deployed.
+#
+# That means the glossary can be published ahead of a full rebuild and may sit
+# slightly out of step with the rest of the site — which is the accepted
+# trade-off for not needing a capsule run to ship a definition fix.
+#
+# Why a full `jb build` and not something cheaper: the page has to carry the
+# theme's chrome (header, sidebar nav, breadcrumbs) to look like part of the
+# book, and only Sphinx can produce that. Execution is switched off, so the
+# build is quick and the notebook cache is never touched. glossary.html comes
+# out byte-identical to what the capsule would produce, provided the toolchain
+# matches — see the asset check below, which is what enforces that.
+
+on:
+  push:
+    branches: [main]
+    paths:
+      - databook/glossary.md
+      - databook/_toc.yml
+      - databook/_config.yml
+      - scripts/build-glossary-page.mjs
+      - .github/workflows/publish-glossary.yml
+  workflow_dispatch: {}
+
+permissions:
+  contents: write
+
+concurrency:
+  group: publish-glossary
+  cancel-in-progress: false   # never interrupt a push to gh-pages
+
+jobs:
+  publish:
+    runs-on: ubuntu-latest
+    steps:
+      - name: Checkout main
+        uses: actions/checkout@v4
+        with:
+          path: src
+
+      - name: Checkout gh-pages
+        uses: actions/checkout@v4
+        with:
+          ref: gh-pages
+          path: site
+
+      - uses: actions/setup-python@v5
+        with:
+          python-version: "3.11"
+
+      # Must match the version the Code Ocean capsule builds the rest of the
+      # book with — it is pinned there too, in the capsule's environment file in
+      # the databook-build repository. This page links to the _static bundle
+      # already on gh-pages rather than shipping its own, so a mismatch still
+      # builds but renders against the wrong stylesheet: the theme's header
+      # dropdowns collapse into bare bullet lists.
+      #
+      # If the capsule is ever upgraded, change it there first, let a full
+      # capsule build deploy, then change it here. To check the two agree, the
+      # theme digests must be equal:
+      #
+      #   curl -s https://allenswdb.github.io/background/background.html \
+      #     | grep -o 'pydata-sphinx-theme.css?digest=[a-f0-9]*'
+      #
+      # 0.15.1 gives dfe6caa3a7d634c4db9b, matching the deployed site, and needs
+      # no assets the live site lacks.
+      - name: Install Jupyter Book
+        run: pip install "jupyter-book==0.15.1"
+
+      - name: Build the book with execution off
+        working-directory: src
+        run: |
+          python - <<'PY'
+          import re
+          p = "databook/_config.yml"
+          s = open(p).read()
+          # the capsule's cache path does not exist here, and nothing on the
+          # glossary page needs a kernel
+          s = re.sub(r"execute:\n(?:  .*\n)+", "execute:\n  execute_notebooks: 'off'\n", s)
+          open(p, "w").write(s)
+          PY
+          jb build -n --keep-going databook
+
+      # gh-pages is force-pushed wholesale by the capsule's `ghp-import -f`, so
+      # it can move under us at any moment. Everything below is therefore done
+      # against a freshly fetched gh-pages and retried on rejection, rather than
+      # against the checkout taken at the start of the job.
+      #
+      # Losing a race is harmless: the capsule builds from main, and main's
+      # glossary.md is kept current by sync-glossary.yml, so a capsule deploy
+      # that lands on top of us carries the same page anyway.
+      - name: Publish onto gh-pages
+        working-directory: site
+        run: |
+          build=../src/databook/_build/html
+          test -f "$build/glossary.html" || { echo "::error::glossary.html was not built"; exit 1; }
+
+          git config user.name  "github-actions[bot]"
+          git config user.email "41898282+github-actions[bot]@users.noreply.github.com"
+
+          # every _static path the page links to, digests stripped
+          grep -oE '_static/[^"?'"'"' ]+' "$build/glossary.html" | sed 's/?.*//' | sort -u > /tmp/need.txt
+
+          for attempt in 1 2 3; do
+            git fetch origin gh-pages
+            git reset --hard origin/gh-pages
+
+            missing=0
+            : > /tmp/add.txt
+            while read -r f; do
+              [ -f "$f" ] && continue
+              # a filename the live site has never seen: safe to add, because
+              # nothing already deployed links to it
+              if [ -f "$build/$f" ]; then
+                echo "$f" >> /tmp/add.txt
+              else
+                echo "::error::page needs $f but the build did not produce it"
+                missing=1
+              fi
+            done < /tmp/need.txt
+            [ "$missing" = "1" ] && exit 1
+
+            if [ -s /tmp/add.txt ]; then
+              echo "::warning::Toolchain here differs from the one that built the live site;"
+              echo "::warning::adding $(wc -l < /tmp/add.txt) new asset(s). Shared assets are left alone, so this"
+              echo "::warning::page may drift visually until the capsule and this workflow are pinned alike."
+              cat /tmp/add.txt
+            fi
+
+            cp "$build/glossary.html" glossary.html
+            mkdir -p _sources
+            cp "$build/_sources/glossary.md.txt" _sources/ 2>/dev/null || true
+            # additive only: never overwrite an asset the other pages depend on
+            while read -r f; do
+              mkdir -p "$(dirname "$f")"
+              cp "$build/$f" "$f"
+            done < /tmp/add.txt
+
+            if [ -z "$(git status --porcelain)" ]; then
+              echo "Nothing changed; the published page is already current."
+              exit 0
+            fi
+
+            git add -A
+            git commit -m "Publish glossary page from ${GITHUB_SHA::7}"
+            if git push origin gh-pages; then
+              echo "Published on attempt $attempt."
+              exit 0
+            fi
+            echo "::notice::gh-pages moved under us; refetching and retrying."
+          done
+
+          echo "::error::could not publish after 3 attempts"
+          exit 1

.github/workflows/sync-glossary.yml

@@ -0,0 +1,132 @@
+name: Sync glossary
+
+# Regenerates databook/glossary.md from the Allen Glossary repository, which is
+# the source of truth for the definitions. This repository only ever reads that
+# one; the renderer (scripts/build-glossary-page.mjs) lives here, so the
+# databook owns how it presents the data and the glossary stays a pure source.
+#
+# Why polling rather than being pushed to: the glossary is a personal repo
+# outside this org, so a push-based hook would need a token minted here and
+# stored there. Polling a public repo needs no secret anywhere, and a daily
+# check is enough to keep the page from drifting.
+#
+# If someone later wants the page to update within seconds of a merge rather
+# than within a day, add a workflow over in the glossary repo that fires a
+# repository_dispatch with type `glossary-updated` at this repo. The trigger is
+# already wired up below; it needs a fine-grained PAT with "contents: write" on
+# this repository, stored as a secret there. Nothing here changes.
+#
+# This never pushes to main. It opens a PR, which is also where you preview the
+# rendered page before it goes live in the next databook build.
+
+on:
+  # GitHub switches scheduled workflows off after 60 days without a commit, and
+  # this book is worked on in a burst once a year, so the schedule alone would
+  # be dormant for most of it. Syncing on push as well covers that: during the
+  # quiet months nothing needs syncing anyway, and the first push of the next
+  # season pulls the glossary current before anyone looks at it. The run is
+  # cheap and opens nothing when the glossary has not moved.
+  push:
+    branches: [main]
+  schedule:
+    - cron: "17 13 * * *" # ~06:17 Pacific, daily
+  workflow_dispatch: {}
+  repository_dispatch:
+    types: [glossary-updated]
+
+permissions:
+  contents: write
+  pull-requests: write
+
+concurrency:
+  group: sync-glossary
+  cancel-in-progress: true
+
+jobs:
+  sync:
+    runs-on: ubuntu-latest
+    steps:
+      - name: Checkout databook
+        uses: actions/checkout@v4
+        with:
+          path: databook-repo
+
+      # Defaults to the canonical glossary. A fork can point this at its own
+      # copy by setting a GLOSSARY_REPO repository variable, so the file stays
+      # correct as-is when a fork's branch is opened as a PR upstream.
+      - name: Checkout glossary source (read-only)
+        uses: actions/checkout@v4
+        with:
+          repository: ${{ vars.GLOSSARY_REPO || 'AllenInstitute/allen-connectomics-glossary' }}
+          path: glossary-src
+
+      - name: Set up Node
+        uses: actions/setup-node@v4
+        with:
+          node-version: 20
+
+      - name: Regenerate the glossary page
+        id: gen
+        run: |
+          sha="$(git -C glossary-src rev-parse HEAD)"
+          echo "sha=$sha" >> "$GITHUB_OUTPUT"
+          # Tee the generator's own report so the pull request can carry it.
+          # A stale alias does not stop the sync, it just quietly costs a
+          # cross-reference its link, so the report has to reach a reviewer.
+          node databook-repo/scripts/build-glossary-page.mjs \
+            --source glossary-src \
+            --repo "${{ vars.GLOSSARY_REPO || 'AllenInstitute/allen-connectomics-glossary' }}" \
+            --out databook-repo/databook/glossary.md \
+            --commit "$sha" 2>&1 | tee /tmp/gen.log
+
+          {
+            echo 'report<<GENEOF'
+            grep -E '^(::warning::|warning: |aliases:)' /tmp/gen.log \
+              | sed 's/^::warning:://; s/^warning: //' || true
+            echo GENEOF
+          } >> "$GITHUB_OUTPUT"
+
+      # The generator is deterministic, so an unchanged glossary leaves the
+      # working tree clean and no PR is opened.
+      - name: Check for changes
+        id: diff
+        working-directory: databook-repo
+        run: |
+          if git diff --quiet -- databook/glossary.md; then
+            echo "changed=false" >> "$GITHUB_OUTPUT"
+            echo "Glossary is already up to date."
+          else
+            echo "changed=true" >> "$GITHUB_OUTPUT"
+            git diff --stat -- databook/glossary.md
+          fi
+
+      - name: Open pull request
+        if: steps.diff.outputs.changed == 'true'
+        uses: peter-evans/create-pull-request@v6
+        with:
+          path: databook-repo
+          branch: bot/sync-glossary
+          delete-branch: true
+          title: "Sync glossary from allen-connectomics-glossary@${{ steps.gen.outputs.sha }}"
+          commit-message: |
+            Regenerate databook/glossary.md from AllenInstitute/allen-connectomics-glossary@${{ steps.gen.outputs.sha }}
+          body: |
+            Regenerated `databook/glossary.md` from
+            `${{ vars.GLOSSARY_REPO || 'AllenInstitute/allen-connectomics-glossary' }}`
+            at commit `${{ steps.gen.outputs.sha }}`.
+
+            This file is generated — review the change upstream rather than editing it here.
+            Note that this PR does not build the databook, so give the rendered page a look
+            before merging if the diff touches anything structural.
+
+            <details><summary>Alias map report</summary>
+
+            ```
+            ${{ steps.gen.outputs.report }}
+            ```
+
+            Stale entries are skipped rather than failing the sync, so anything flagged
+            here means a cross-reference has quietly lost its link. Fix in
+            `databook/glossary-aliases.json`.
+            </details>
+          labels: automated,glossary

databook/glossary-aliases.json

@@ -0,0 +1,52 @@
+{
+  "//": [
+    "Extra names under which a glossary term can be cross-referenced.",
+    "",
+    "The databook writes {term}`basket cell`; the glossary calls that entry",
+    "'Basket cell (BC)'. Sphinx matches glossary terms by their exact text, so",
+    "without a mapping the reference resolves to nothing, renders as plain text,",
+    "and warns under `jb build -n`. Each key below is added as an additional term",
+    "line on the entry it names, so both spellings reach the same definition.",
+    "",
+    "Keys are what the databook writes. Values are glossary term ids, taken from",
+    "the `id` field in the glossary repo's data/terms.js. Ids are used rather",
+    "than display names so that rewording a term upstream does not silently",
+    "break the mapping — build-glossary-page.mjs fails loudly if an id here no",
+    "longer exists, or if an alias collides with a real term name.",
+    "",
+    "Delete an entry once the glossary defines that spelling itself.",
+    "",
+    "Matching is case-insensitive, so one casing per name is enough — but the",
+    "casing chosen here is what the anchor is named after. Pages already built",
+    "and deployed link to the anchors the previous glossary produced, so these",
+    "keys follow that capitalisation (Spike, not spike) to keep those links",
+    "working until the whole book is rebuilt."
+  ],
+
+  "aliases": {
+    "Basket cell": "basket-cell",
+    "Bipolar cell": "bipolar-cell",
+    "CCF": "ccf",
+    "GECI": "geci",
+    "HVA": "higher-visual-area",
+    "ISI": "intrinsic-signal-imaging",
+    "LFP": "local-field-potential",
+    "Local field potential": "local-field-potential",
+    "Martinotti cell": "martinotti-cell",
+    "Minnie column": "column-microns",
+    "Parvalbumin-positive interneuron": "pv-neuron",
+    "Primary visual cortex": "visp-visal-visrl",
+    "retinotopic map": "retinotopy",
+    "ROI": "roi-mask",
+    "Somatostatin cell": "somatostatin-sst-cell",
+    "Spike": "action-potential",
+    "V1": "visp-visal-visrl",
+    "VISp": "visp-visal-visrl"
+  },
+
+  "unmapped": {
+    "//": "Referenced by the databook, absent from the glossary. Left unresolved on purpose rather than pointed at an approximate entry — the fix is to define them upstream, then delete them from here.",
+    "CSV": "A generic file format, not a glossary concept. Already unresolved before this page was generated.",
+    "GFP": "The nearest entry is [fluorophore], which defines the class of molecule rather than this particular one."
+  }
+}