From 87084983dff02a08aff627dc76ca56a0cf4096b5 Mon Sep 17 00:00:00 2001
From: Harmen Stoppels <me+bot@harmenstoppels.nl>
Date: Thu, 12 Nov 2026 23:37:15 +0100
Subject: [PATCH 01/20] linux-gnu: AlmaLinux 8 based CI image built from a
 Spack environment

Replaces the centos7 / ubi8 stage1+stage2 Makefile flow with a single
environment using spec groups (system gcc -> gcc@14 -> tools), built natively
on x86_64 and aarch64 GitHub runners, cached in an OCI build cache and turned
into an image with `spack buildcache push --base-image --tag`.
---
 .github/workflows/linux-gnu.yml | 161 ++++++++++++++++++++++++++++++++
 linux-gnu/base.dockerfile       |  20 ++++
 linux-gnu/build.sh              |  33 +++++++
 linux-gnu/builder.dockerfile    |  27 ++++++
 linux-gnu/check-glibc.sh        |  14 +++
 linux-gnu/spack.yaml            |  74 +++++++++++++++
 6 files changed, 329 insertions(+)
 create mode 100644 .github/workflows/linux-gnu.yml
 create mode 100644 linux-gnu/base.dockerfile
 create mode 100755 linux-gnu/build.sh
 create mode 100644 linux-gnu/builder.dockerfile
 create mode 100755 linux-gnu/check-glibc.sh
 create mode 100644 linux-gnu/spack.yaml

diff --git a/.github/workflows/linux-gnu.yml b/.github/workflows/linux-gnu.yml
new file mode 100644
index 0000000..ab996a3
--- /dev/null
+++ b/.github/workflows/linux-gnu.yml
@@ -0,0 +1,161 @@
+name: linux-gnu image
+
+# Builds linux-gnu/spack.yaml natively on x86_64 and aarch64 runners inside an
+# AlmaLinux 8 builder container. Every installed package is pushed to an OCI
+# build cache as soon as it is built (autopush), so a re-run of a timed-out or
+# failed job resumes where it left off. The image itself is assembled by
+# `spack buildcache push --base-image ... --tag ...`: the runtime base plus one
+# layer per Spack package, with PATH etc. set for the root specs.
+
+on:
+  workflow_dispatch:
+  push:
+    branches: [main]
+    paths:
+    - linux-gnu/**
+    - .github/workflows/linux-gnu.yml
+  pull_request:
+    paths:
+    - linux-gnu/**
+    - .github/workflows/linux-gnu.yml
+
+concurrency:
+  group: linux-gnu-${{ github.ref }}
+  cancel-in-progress: true
+
+env:
+  # Pinned Spack and package repository. Point these at forks to build with
+  # unmerged fixes.
+  SPACK_REPOSITORY: spack/spack
+  SPACK_COMMIT: e9a7922af3e28d24308f47ed7ce1ecd5eb9db817
+  PACKAGES_REPOSITORY: spack/spack-packages
+  PACKAGES_COMMIT: 8933fbb6925be105d493d2eff0c0b375a2ac1cc8
+  REGISTRY: ghcr.io/${{ github.repository_owner }}
+
+jobs:
+  base:
+    runs-on: ${{ matrix.runner }}
+    strategy:
+      matrix:
+        include:
+        - {runner: ubuntu-24.04, arch: amd64}
+        - {runner: ubuntu-24.04-arm, arch: arm64}
+    permissions:
+      contents: read
+      packages: write
+    outputs:
+      tag: ${{ steps.tag.outputs.tag }}
+    steps:
+    - uses: actions/checkout@v7
+    - name: Tag
+      id: tag
+      # Same tag for both arches: the base inputs, not the build date.
+      run: echo "tag=$(cat linux-gnu/base.dockerfile | sha256sum | cut -c1-12)" >> "$GITHUB_OUTPUT"
+    - uses: docker/login-action@v4
+      with:
+        registry: ghcr.io
+        username: ${{ github.actor }}
+        password: ${{ secrets.GITHUB_TOKEN }}
+    - name: Build and push runtime base and builder
+      run: |
+        base=$REGISTRY/linux-gnu-base:${{ steps.tag.outputs.tag }}-${{ matrix.arch }}
+        docker build -t $base -f linux-gnu/base.dockerfile linux-gnu
+        docker build -t linux-gnu-builder --build-arg BASE=$base -f linux-gnu/builder.dockerfile linux-gnu
+        docker push $base
+
+  base-manifest:
+    needs: base
+    runs-on: ubuntu-24.04
+    permissions:
+      packages: write
+    steps:
+    - uses: docker/login-action@v4
+      with:
+        registry: ghcr.io
+        username: ${{ github.actor }}
+        password: ${{ secrets.GITHUB_TOKEN }}
+    - run: |
+        b=$REGISTRY/linux-gnu-base:${{ needs.base.outputs.tag }}
+        docker buildx imagetools create -t $b $b-amd64 $b-arm64
+
+  build:
+    needs: [base, base-manifest]
+    runs-on: ${{ matrix.runner }}
+    timeout-minutes: 355
+    strategy:
+      fail-fast: false
+      matrix:
+        include:
+        - {runner: ubuntu-24.04, arch: amd64}
+        - {runner: ubuntu-24.04-arm, arch: arm64}
+    permissions:
+      contents: read
+      packages: write
+    env:
+      BASE: ghcr.io/${{ github.repository_owner }}/linux-gnu-base:${{ needs.base.outputs.tag }}
+    steps:
+    - uses: actions/checkout@v7
+
+    - name: Check out Spack and spack-packages
+      run: |
+        for r in "spack $SPACK_REPOSITORY $SPACK_COMMIT" "spack-packages $PACKAGES_REPOSITORY $PACKAGES_COMMIT"; do
+          set -- $r
+          git init -q $1
+          git -C $1 fetch -q --depth 1 https://github.com/$2.git $3
+          git -C $1 checkout -q FETCH_HEAD
+        done
+
+    - uses: docker/login-action@v4
+      with:
+        registry: ghcr.io
+        username: ${{ github.actor }}
+        password: ${{ secrets.GITHUB_TOKEN }}
+
+    - name: Builder image
+      run: |
+        docker pull $BASE-${{ matrix.arch }}
+        docker build -t linux-gnu-builder --build-arg BASE=$BASE-${{ matrix.arch }} \
+          -f linux-gnu/builder.dockerfile linux-gnu
+
+    - name: Concretize, install and push
+      env:
+        GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
+        TAG: ${{ github.event_name == 'pull_request' && format('pr{0}', github.event.number) || github.ref_name }}
+      run: |
+        mkdir -p env out
+        cp linux-gnu/spack.yaml env/
+        docker run --rm -e GITHUB_ACTOR -e GITHUB_TOKEN -e REGISTRY -e TAG \
+          -e ARCH=${{ matrix.arch }} -e BASE \
+          -e SPACK_COLOR=always -e PYTHONUNBUFFERED=1 \
+          -v $PWD/spack:/spack -v $PWD/spack-packages:/spack-packages \
+          -v $PWD/env:/env -v $PWD/out:/out -v $PWD/linux-gnu:/linux-gnu:ro \
+          linux-gnu-builder /linux-gnu/build.sh
+
+    - uses: actions/upload-artifact@v4
+      if: always()
+      with:
+        name: linux-gnu-${{ matrix.arch }}
+        path: |
+          env/spack.yaml
+          env/spack.lock
+          out/
+
+  image:
+    needs: [base, build]
+    runs-on: ubuntu-24.04
+    permissions:
+      packages: write
+    steps:
+    - uses: docker/login-action@v4
+      with:
+        registry: ghcr.io
+        username: ${{ github.actor }}
+        password: ${{ secrets.GITHUB_TOKEN }}
+    - name: Multi-arch image
+      env:
+        TAG: ${{ github.event_name == 'pull_request' && format('pr{0}', github.event.number) || github.ref_name }}
+      run: |
+        i=$REGISTRY/linux-gnu
+        day=$(date -u +%Y-%m-%d)
+        docker buildx imagetools create -t $i:$TAG -t $i:$day-${{ github.run_id }} $i:$TAG-amd64 $i:$TAG-arm64
+        docker buildx imagetools inspect $i:$TAG
diff --git a/linux-gnu/base.dockerfile b/linux-gnu/base.dockerfile
new file mode 100644
index 0000000..2181d0f
--- /dev/null
+++ b/linux-gnu/base.dockerfile
@@ -0,0 +1,20 @@
+# Runtime base of the linux-gnu CI image. Everything else in the image is
+# Spack-built and layered on top by `spack buildcache push --base-image`.
+#
+# AlmaLinux 8 fixes the glibc floor of whatever CI builds in this image at
+# 2.28; check-glibc.sh guards against RHEL-style symbol backports raising it.
+ARG BASE_IMAGE=docker.io/library/almalinux:8@sha256:2d0f2eb2a83cc09dbc2eb92fade4dbe8f709628149e81aa00aec70630406603b
+FROM ${BASE_IMAGE}
+
+# glibc-devel: headers, crt objects and linker scripts for the Spack-built gcc.
+# Do not upgrade glibc here: the floor must stay at the pinned base image's.
+RUN dnf install -y --setopt=install_weak_deps=False \
+      glibc-devel \
+      procps-ng \
+      which \
+ && dnf clean all \
+ && rm -rf /var/cache/dnf \
+ # Make packages link Spack's libxcrypt instead of the system libcrypt.
+ && rm -f /usr/lib64/libcrypt.so
+
+CMD ["/bin/bash"]
diff --git a/linux-gnu/build.sh b/linux-gnu/build.sh
new file mode 100755
index 0000000..4975a03
--- /dev/null
+++ b/linux-gnu/build.sh
@@ -0,0 +1,33 @@
+#!/bin/bash
+# Runs inside builder.dockerfile; see .github/workflows/linux-gnu.yml.
+#
+# Inputs (environment): REGISTRY, TAG, ARCH, BASE, GITHUB_ACTOR, GITHUB_TOKEN.
+# Mounts: /spack, /spack-packages, /env (spack.yaml, writable), /out.
+set -euo pipefail
+
+export PATH=/spack/bin:$PATH
+export SPACK_DISABLE_LOCAL_CONFIG=1
+export SPACK_USER_CACHE_PATH=/tmp/spack-cache
+
+cache="oci://$REGISTRY/linux-gnu-cache"
+
+# Use the pinned spack-packages checkout instead of cloning builtin.
+spack -e /env config add "repos:builtin:/spack-packages/repos/spack_repo/builtin"
+spack -e /env repo list
+spack -e /env mirror add --unsigned --autopush \
+  --oci-username-variable GITHUB_ACTOR --oci-password-variable GITHUB_TOKEN \
+  cache "$cache"
+
+spack -e /env concretize --fresh 2>&1 | tee /out/concretize.txt
+spack -e /env find -c --show-full-compiler -L > /out/specs.txt || true
+
+# Installs pull what is already in the cache and push what they build.
+spack -e /env install --fail-fast -j "$(nproc)"
+
+# Tag one image with every root and its runtime closure, on top of the base.
+spack -e /env buildcache push --unsigned --update-index --base-image "$BASE-$ARCH" \
+  --tag "$TAG-$ARCH" cache
+
+# Smoke test of the environment the image gets (PATH etc. of the roots).
+spack -e /env load --sh --only package \
+  gcc binutils python git curl gnupg awscli-v2 clingo-bootstrap > /out/load.sh
diff --git a/linux-gnu/builder.dockerfile b/linux-gnu/builder.dockerfile
new file mode 100644
index 0000000..9d4706a
--- /dev/null
+++ b/linux-gnu/builder.dockerfile
@@ -0,0 +1,27 @@
+# Build environment for linux-gnu/spack.yaml: the runtime base plus the system
+# toolchain that bootstraps the Spack-built gcc (spec group `compiler`).
+ARG BASE
+FROM ${BASE}
+
+RUN dnf install -y --setopt=install_weak_deps=False \
+      binutils \
+      bzip2 \
+      diffutils \
+      file \
+      findutils \
+      gcc \
+      gcc-c++ \
+      git \
+      gzip \
+      libstdc++-static \
+      make \
+      patch \
+      python3 \
+      tar \
+      unzip \
+      xz \
+ && dnf clean all \
+ && rm -rf /var/cache/dnf
+
+COPY check-glibc.sh /usr/local/bin/check-glibc.sh
+RUN /usr/local/bin/check-glibc.sh
diff --git a/linux-gnu/check-glibc.sh b/linux-gnu/check-glibc.sh
new file mode 100755
index 0000000..3e6545b
--- /dev/null
+++ b/linux-gnu/check-glibc.sh
@@ -0,0 +1,14 @@
+#!/bin/sh
+# Fail if the system libc defines a symbol version newer than its nominal
+# release. RHEL backports symbols under newer version nodes (e.g. GLIBC_2.35 in
+# RHEL 9.7's glibc 2.34, spack/spack#51888); binaries linked against those
+# would not run on other distros with the "same" glibc.
+set -eu
+
+expected="GLIBC_${EXPECTED_GLIBC:-2.28}"
+libc=$(ls /lib64/libc.so.6 /lib/*/libc.so.6 2>/dev/null | head -n1)
+
+newest=$(readelf -V "$libc" | grep -oE 'GLIBC_[0-9]+\.[0-9]+(\.[0-9]+)?' | sort -uV | tail -n1)
+
+echo "$libc: newest symbol version $newest (expected $expected)"
+test "$newest" = "$expected"
diff --git a/linux-gnu/spack.yaml b/linux-gnu/spack.yaml
new file mode 100644
index 0000000..a4b3a3d
--- /dev/null
+++ b/linux-gnu/spack.yaml
@@ -0,0 +1,74 @@
+# Tools for Spack's GitLab CI on linux, built inside builder.dockerfile and
+# pushed as an image on top of base.dockerfile (see README.md).
+spack:
+  view: false
+
+  specs:
+  # Built with the system gcc 8.5 of AlmaLinux 8.
+  - group: compiler
+    specs:
+    - gcc@14 +binutils +profiled build_type=Release languages=c,c++,fortran
+    - binutils +ld +gas +plugins ~libiberty
+    override:
+      packages:
+        c:
+          require: [gcc@8.5.0]
+        cxx:
+          require: [gcc@8.5.0]
+
+  # Everything else is built with the gcc from the compiler group.
+  - group: tools
+    needs: [compiler]
+    specs:
+    - awscli-v2
+    - bzip2
+    - clingo-bootstrap@spack +optimized
+    - curl
+    - file
+    - git
+    - gmake
+    - gnupg
+    - gzip
+    - libtree
+    - lz4
+    - patch
+    - patchelf
+    - py-boto3
+    - python +optimizations
+    - tar
+    - unzip
+    - vim
+    - xz
+    - zstd
+    override:
+      packages:
+        c:
+          require: [gcc@14]
+        cxx:
+          require: [gcc@14]
+        fortran:
+          require: [gcc@14]
+
+  concretizer:
+    unify: true
+    reuse: false
+
+  config:
+    install_tree:
+      root: /opt/spack
+
+  packages:
+    all:
+      require:
+      - spec: target=x86_64_v3
+        when: target=x86_64:
+      - spec: target=aarch64
+        when: target=aarch64:
+    gcc:
+      externals:
+      - spec: gcc@8.5.0 languages:='c,c++'
+        prefix: /usr
+        extra_attributes:
+          compilers:
+            c: /usr/bin/gcc
+            cxx: /usr/bin/g++

From 0d322823d0cd593430e541b909d77c8b3c2c4d1b Mon Sep 17 00:00:00 2001
From: Harmen Stoppels <me+bot@harmenstoppels.nl>
Date: Thu, 12 Nov 2026 23:37:52 +0100
Subject: [PATCH 02/20] linux-gnu: build on pushes to any branch

---
 .github/workflows/linux-gnu.yml | 1 -
 1 file changed, 1 deletion(-)

diff --git a/.github/workflows/linux-gnu.yml b/.github/workflows/linux-gnu.yml
index ab996a3..f403a9d 100644
--- a/.github/workflows/linux-gnu.yml
+++ b/.github/workflows/linux-gnu.yml
@@ -10,7 +10,6 @@ name: linux-gnu image
 on:
   workflow_dispatch:
   push:
-    branches: [main]
     paths:
     - linux-gnu/**
     - .github/workflows/linux-gnu.yml

From 69be65d8379a8e44fd8b230fbbc5980879801399 Mon Sep 17 00:00:00 2001
From: Harmen Stoppels <me+bot@harmenstoppels.nl>
Date: Thu, 12 Nov 2026 23:39:15 +0100
Subject: [PATCH 03/20] linux-gnu: enable powertools for libstdc++-static

---
 linux-gnu/builder.dockerfile | 3 ++-
 1 file changed, 2 insertions(+), 1 deletion(-)

diff --git a/linux-gnu/builder.dockerfile b/linux-gnu/builder.dockerfile
index 9d4706a..f539a4b 100644
--- a/linux-gnu/builder.dockerfile
+++ b/linux-gnu/builder.dockerfile
@@ -3,7 +3,8 @@
 ARG BASE
 FROM ${BASE}
 
-RUN dnf install -y --setopt=install_weak_deps=False \
+# libstdc++-static (for gcc's bootstrap) is in powertools.
+RUN dnf install -y --setopt=install_weak_deps=False --enablerepo=powertools \
       binutils \
       bzip2 \
       diffutils \

From d33d508bb157a433f1c35a6ce9955a9aef6eb1e2 Mon Sep 17 00:00:00 2001
From: Harmen Stoppels <me+bot@harmenstoppels.nl>
Date: Thu, 12 Nov 2026 23:43:19 +0100
Subject: [PATCH 04/20] linux-gnu: build everything with the system gcc; quote
 target ranges

---
 linux-gnu/spack.yaml | 81 +++++++++++++++++++-------------------------
 1 file changed, 35 insertions(+), 46 deletions(-)

diff --git a/linux-gnu/spack.yaml b/linux-gnu/spack.yaml
index a4b3a3d..2be514e 100644
--- a/linux-gnu/spack.yaml
+++ b/linux-gnu/spack.yaml
@@ -1,53 +1,38 @@
 # Tools for Spack's GitLab CI on linux, built inside builder.dockerfile and
 # pushed as an image on top of base.dockerfile (see README.md).
+#
+# Everything, including gcc@14, is built with AlmaLinux 8's gcc 8.5: the tools
+# only run Spack and its helpers, and CI builds with gcc@14 and its own
+# gcc-runtime. Not building the tools with gcc@14 keeps them off the critical
+# path of the (slow) gcc build and avoids rebuilding all of them on gcc bumps.
 spack:
   view: false
 
   specs:
-  # Built with the system gcc 8.5 of AlmaLinux 8.
-  - group: compiler
-    specs:
-    - gcc@14 +binutils +profiled build_type=Release languages=c,c++,fortran
-    - binutils +ld +gas +plugins ~libiberty
-    override:
-      packages:
-        c:
-          require: [gcc@8.5.0]
-        cxx:
-          require: [gcc@8.5.0]
-
-  # Everything else is built with the gcc from the compiler group.
-  - group: tools
-    needs: [compiler]
-    specs:
-    - awscli-v2
-    - bzip2
-    - clingo-bootstrap@spack +optimized
-    - curl
-    - file
-    - git
-    - gmake
-    - gnupg
-    - gzip
-    - libtree
-    - lz4
-    - patch
-    - patchelf
-    - py-boto3
-    - python +optimizations
-    - tar
-    - unzip
-    - vim
-    - xz
-    - zstd
-    override:
-      packages:
-        c:
-          require: [gcc@14]
-        cxx:
-          require: [gcc@14]
-        fortran:
-          require: [gcc@14]
+  # The compiler CI builds with.
+  - gcc@14 +binutils +profiled build_type=Release languages=c,c++,fortran
+  - binutils +ld +gas +plugins ~libiberty
+  # Spack's own dependencies and CI tooling.
+  - awscli-v2
+  - bzip2
+  - clingo-bootstrap@spack +optimized
+  - curl
+  - file
+  - git
+  - gmake
+  - gnupg
+  - gzip
+  - libtree
+  - lz4
+  - patch
+  - patchelf
+  - py-boto3
+  - python +optimizations
+  - tar
+  - unzip
+  - vim
+  - xz
+  - zstd
 
   concretizer:
     unify: true
@@ -61,9 +46,13 @@ spack:
     all:
       require:
       - spec: target=x86_64_v3
-        when: target=x86_64:
+        when: "target=x86_64:"
       - spec: target=aarch64
-        when: target=aarch64:
+        when: "target=aarch64:"
+    c:
+      require: [gcc@8.5.0]
+    cxx:
+      require: [gcc@8.5.0]
     gcc:
       externals:
       - spec: gcc@8.5.0 languages:='c,c++'

From ef3127a0e63a8fb101e8f12bf779afffa8358de3 Mon Sep 17 00:00:00 2001
From: Harmen Stoppels <me+bot@harmenstoppels.nl>
Date: Thu, 12 Nov 2026 23:48:11 +0100
Subject: [PATCH 05/20] linux-gnu: build with spack-packages version bumps from
 the fork

---
 .github/workflows/linux-gnu.yml | 5 +++--
 1 file changed, 3 insertions(+), 2 deletions(-)

diff --git a/.github/workflows/linux-gnu.yml b/.github/workflows/linux-gnu.yml
index f403a9d..4114782 100644
--- a/.github/workflows/linux-gnu.yml
+++ b/.github/workflows/linux-gnu.yml
@@ -27,8 +27,9 @@ env:
   # unmerged fixes.
   SPACK_REPOSITORY: spack/spack
   SPACK_COMMIT: e9a7922af3e28d24308f47ed7ce1ecd5eb9db817
-  PACKAGES_REPOSITORY: spack/spack-packages
-  PACKAGES_COMMIT: 8933fbb6925be105d493d2eff0c0b375a2ac1cc8
+  # haampie-llms/spack-packages linux-gnu-ci-image: develop 8933fbb + version bumps
+  PACKAGES_REPOSITORY: haampie-llms/spack-packages
+  PACKAGES_COMMIT: e0051ddce1d5ba25c74844143f20d63656c08158
   REGISTRY: ghcr.io/${{ github.repository_owner }}
 
 jobs:

From ca78bbc45a2c45dfd44365be9f3e670e6c69b69a Mon Sep 17 00:00:00 2001
From: Harmen Stoppels <me+bot@harmenstoppels.nl>
Date: Thu, 12 Nov 2026 23:49:25 +0100
Subject: [PATCH 06/20] linux-gnu: test the image like spack-packages CI uses
 it

---
 .github/workflows/linux-gnu.yml | 32 ++++++++++++++
 linux-gnu/test-ci.sh            | 78 +++++++++++++++++++++++++++++++++
 2 files changed, 110 insertions(+)
 create mode 100755 linux-gnu/test-ci.sh

diff --git a/.github/workflows/linux-gnu.yml b/.github/workflows/linux-gnu.yml
index 4114782..088bf1d 100644
--- a/.github/workflows/linux-gnu.yml
+++ b/.github/workflows/linux-gnu.yml
@@ -159,3 +159,35 @@ jobs:
         day=$(date -u +%Y-%m-%d)
         docker buildx imagetools create -t $i:$TAG -t $i:$day-${{ github.run_id }} $i:$TAG-amd64 $i:$TAG-arm64
         docker buildx imagetools inspect $i:$TAG
+
+  # Is the image usable for spack-packages CI? Concretize a developer-tools
+  # stack the way generate jobs do and build a sample of it like build jobs.
+  test:
+    needs: build
+    runs-on: ${{ matrix.runner }}
+    strategy:
+      fail-fast: false
+      matrix:
+        include:
+        - {runner: ubuntu-24.04, arch: amd64, target: x86_64_v3, stack: developer-tools-x86_64_v3-linux-gnu}
+        - {runner: ubuntu-24.04-arm, arch: arm64, target: aarch64, stack: developer-tools-aarch64-linux-gnu}
+    permissions:
+      contents: read
+      packages: read
+    steps:
+    - uses: actions/checkout@v7
+    - uses: docker/login-action@v4
+      with:
+        registry: ghcr.io
+        username: ${{ github.actor }}
+        password: ${{ secrets.GITHUB_TOKEN }}
+    - name: Test ${{ matrix.stack }}
+      env:
+        TAG: ${{ github.event_name == 'pull_request' && format('pr{0}', github.event.number) || github.ref_name }}
+      run: |
+        docker run --rm -e PACKAGES_REPOSITORY -e PACKAGES_COMMIT \
+          -e SPACK_CI_TARGET=${{ matrix.target }} -e STACK=${{ matrix.stack }} \
+          -e BUILD_SPECS="libtree tree jq ncdu tmux patchelf py-isort gmake" \
+          -e SPACK_COLOR=always -e PYTHONUNBUFFERED=1 \
+          -v $PWD/linux-gnu:/linux-gnu:ro \
+          $REGISTRY/linux-gnu:$TAG-${{ matrix.arch }} /linux-gnu/test-ci.sh
diff --git a/linux-gnu/test-ci.sh b/linux-gnu/test-ci.sh
new file mode 100755
index 0000000..c74dc52
--- /dev/null
+++ b/linux-gnu/test-ci.sh
@@ -0,0 +1,78 @@
+#!/bin/bash
+# Smoke test of the linux-gnu image as a spack-packages GitLab CI image. Runs
+# inside the image; mimics the generate and build jobs of the developer-tools
+# stacks (.ci/gitlab/.gitlab-ci.yml, .ci/gitlab/configs/ci.yaml).
+#
+# Inputs (environment): PACKAGES_REPOSITORY, PACKAGES_COMMIT, SPACK_CI_TARGET,
+# STACK, BUILD_SPECS (space separated roots of the stack to build).
+set -euo pipefail
+
+section() { echo; echo "### $*"; }
+
+section "Tools in the image"
+for exe in python3 gcc g++ gfortran ld ar git curl gpg aws make tar xz zstd unzip patch patchelf file; do
+  printf '%-10s %s\n' "$exe" "$(command -v "$exe")"
+done
+gcc --version | head -n1
+python3 -c 'import clingo, sys; print("clingo", clingo.__version__, clingo.__file__)'
+python3 -c 'import boto3, botocore; print("boto3", boto3.__version__, "botocore", botocore.__version__)'
+# spack buildcache update-index uses put_object(IfMatch=...): botocore >= 1.35.70
+python3 -c 'import botocore.session as s; m = s.get_session().get_service_model("s3"); assert "IfMatch" in m.operation_model("PutObject").input_shape.members'
+aws --version
+gpg --version | head -n1
+git --version
+curl --version | head -n1
+
+workdir=$(mktemp -d)
+cd "$workdir"
+
+section "Check out spack-packages and the Spack commit its CI uses"
+git init -q packages
+git -C packages fetch -q --depth 1 "https://github.com/$PACKAGES_REPOSITORY.git" "$PACKAGES_COMMIT"
+git -C packages checkout -q FETCH_HEAD
+set -a; . packages/.ci/env; set +a
+git init -q spack
+git -C spack fetch -q --depth 1 "https://github.com/$SPACK_CHECKOUT_REPO.git" "$SPACK_CHECKOUT_VERSION"
+git -C spack checkout -q FETCH_HEAD
+
+export SPACK_DISABLE_LOCAL_CONFIG=1
+export SPACK_USER_CACHE_PATH="$workdir/tmp/_user_cache/"
+export SPACK_CI_TARGET
+. spack/share/spack/setup-env.sh
+spack --version
+spack arch
+
+stack="$workdir/packages/stacks/$STACK"
+
+section "Generate: concretize $STACK"
+spack env activate --without-view "$stack"
+spack repo list
+spack compiler find
+spack compiler list
+spack audit configs
+start=$(date +%s)
+spack -v concretize -j 4 > "$workdir/concretize.log" 2>&1 || { tail -n 60 "$workdir/concretize.log"; exit 1; }
+echo "concretized in $(( $(date +%s) - start ))s"
+tail -n 3 "$workdir/concretize.log"
+if [ -d "$SPACK_USER_CACHE_PATH/bootstrap/store" ]; then
+  echo "ERROR: Spack bootstrapped (clingo?) instead of using the image's tools:"
+  find "$SPACK_USER_CACHE_PATH/bootstrap/store" -maxdepth 3 -mindepth 3
+  exit 1
+fi
+echo "OK: nothing bootstrapped"
+
+section "Build: ${BUILD_SPECS}"
+# Like build jobs: padded install tree plus the relocation test projection.
+for spec in $BUILD_SPECS; do
+  hash=$(spack find -c --format '{hash}' "$spec" | head -n1)
+  spack config add "config:install_tree:projections:$spec/$hash:'morepadding/{architecture.platform}-{architecture.target}/{name}-{version}-{hash}'"
+done
+spack install --fail-fast --no-check-signature $BUILD_SPECS
+spack find -lp $BUILD_SPECS
+spack env deactivate
+
+section "Outputs do not need a glibc newer than 2.28"
+newest=$(find /home/software/spack -type f \( -name '*.so*' -o -perm -u+x \) -exec sh -c 'file -b "$1" | grep -q ELF && objdump -T "$1" 2>/dev/null' _ {} \; \
+  | grep -oE 'GLIBC_[0-9]+\.[0-9]+(\.[0-9]+)?' | sort -uV | tail -n1)
+echo "newest GLIBC symbol version required: $newest"
+test "$(printf '%s\nGLIBC_2.28\n' "$newest" | sort -V | tail -n1)" = GLIBC_2.28

From 589cd2ba2dd87ce8520ece3ca1ef7f4c6822ef00 Mon Sep 17 00:00:00 2001
From: Harmen Stoppels <me+bot@harmenstoppels.nl>
Date: Thu, 12 Nov 2026 23:49:59 +0100
Subject: [PATCH 07/20] linux-gnu: update the cache index when the build exits

---
 linux-gnu/build.sh | 4 ++++
 1 file changed, 4 insertions(+)

diff --git a/linux-gnu/build.sh b/linux-gnu/build.sh
index 4975a03..fa4fab1 100755
--- a/linux-gnu/build.sh
+++ b/linux-gnu/build.sh
@@ -18,6 +18,10 @@ spack -e /env mirror add --unsigned --autopush \
   --oci-username-variable GITHUB_ACTOR --oci-password-variable GITHUB_TOKEN \
   cache "$cache"
 
+# Packages are autopushed as they are built; keep the index current even if
+# the build fails, so the cache can be browsed (installs do not need it).
+trap 'spack -e /env buildcache update-index cache || true' EXIT
+
 spack -e /env concretize --fresh 2>&1 | tee /out/concretize.txt
 spack -e /env find -c --show-full-compiler -L > /out/specs.txt || true
 

From 5821e927a9539981bf8d341a187d3ca9ab540863 Mon Sep 17 00:00:00 2001
From: Harmen Stoppels <me+bot@harmenstoppels.nl>
Date: Fri, 13 Nov 2026 00:34:37 +0100
Subject: [PATCH 08/20] linux-gnu: README

---
 linux-gnu/README.md | 31 +++++++++++++++++++++++++++++++
 1 file changed, 31 insertions(+)
 create mode 100644 linux-gnu/README.md

diff --git a/linux-gnu/README.md b/linux-gnu/README.md
new file mode 100644
index 0000000..e1c3150
--- /dev/null
+++ b/linux-gnu/README.md
@@ -0,0 +1,31 @@
+Build image for Spack's GitLab CI on linux (x86_64_v3 and aarch64), made entirely
+from a Spack environment on top of a minimal AlmaLinux 8 base.
+
+- [`base.dockerfile`](base.dockerfile): AlmaLinux 8 (pinned by digest) plus
+  `glibc-devel`. glibc 2.28 is the floor of everything CI builds in this image;
+  [`check-glibc.sh`](check-glibc.sh) fails the build if the system libc exports
+  symbol versions newer than that (RHEL backports, see spack/spack#51888).
+- [`builder.dockerfile`](builder.dockerfile): the base plus AlmaLinux's gcc 8.5
+  and the bare minimum to run Spack.
+- [`spack.yaml`](spack.yaml): gcc@14 (the compiler CI builds with) and the tools
+  CI needs: Python with Spack's optimized clingo, boto3, AWS CLI v2, git, curl,
+  gnupg, compression tools, ...
+- [`build.sh`](build.sh): concretizes and installs the environment in the
+  builder, then runs `spack buildcache push --base-image <base> --tag <tag>`,
+  which produces the image: the base plus one layer per package, with `PATH` and
+  friends set for the root specs.
+- [`test-ci.sh`](test-ci.sh): runs in the image like spack-packages CI does:
+  concretizes a developer-tools stack (generate job) and builds part of it under
+  CI's padded install tree (build jobs).
+
+[`.github/workflows/linux-gnu.yml`](../.github/workflows/linux-gnu.yml) does all
+of this natively on x86_64 and aarch64 GitHub runners and publishes:
+
+- `ghcr.io/<owner>/linux-gnu-base:<hash>`: the runtime base
+- `ghcr.io/<owner>/linux-gnu-cache`: the OCI build cache. Packages are pushed as
+  soon as they are built (`autopush`), so a re-run after a timeout resumes.
+- `ghcr.io/<owner>/linux-gnu:<branch>-{amd64,arm64}`, and multi-arch
+  `ghcr.io/<owner>/linux-gnu:<branch>` plus `:<date>-<run id>`.
+
+Spack and spack-packages are pinned by commit in the workflow. Point them at
+forks to build with unmerged fixes.

From 0a2f19985dd8423cbedd30afbd7227313c0dcb96 Mon Sep 17 00:00:00 2001
From: Harmen Stoppels <me+bot@harmenstoppels.nl>
Date: Fri, 13 Nov 2026 00:44:19 +0100
Subject: [PATCH 09/20] linux-gnu: require a generic target per arch

The conditional requirement (target=aarch64 when target=aarch64:) was not
enforced on the arm runner, which concretized to neoverse_n2; with gcc 8.5 that
also broke py-awscrt (-march with +sve, but no arm_sve.h before GCC 10).
---
 .github/workflows/linux-gnu.yml | 6 +++---
 linux-gnu/build.sh              | 7 ++++++-
 linux-gnu/spack.yaml            | 7 +------
 3 files changed, 10 insertions(+), 10 deletions(-)

diff --git a/.github/workflows/linux-gnu.yml b/.github/workflows/linux-gnu.yml
index 088bf1d..9cfde72 100644
--- a/.github/workflows/linux-gnu.yml
+++ b/.github/workflows/linux-gnu.yml
@@ -86,8 +86,8 @@ jobs:
       fail-fast: false
       matrix:
         include:
-        - {runner: ubuntu-24.04, arch: amd64}
-        - {runner: ubuntu-24.04-arm, arch: arm64}
+        - {runner: ubuntu-24.04, arch: amd64, target: x86_64_v3}
+        - {runner: ubuntu-24.04-arm, arch: arm64, target: aarch64}
     permissions:
       contents: read
       packages: write
@@ -125,7 +125,7 @@ jobs:
         mkdir -p env out
         cp linux-gnu/spack.yaml env/
         docker run --rm -e GITHUB_ACTOR -e GITHUB_TOKEN -e REGISTRY -e TAG \
-          -e ARCH=${{ matrix.arch }} -e BASE \
+          -e ARCH=${{ matrix.arch }} -e TARGET=${{ matrix.target }} -e BASE \
           -e SPACK_COLOR=always -e PYTHONUNBUFFERED=1 \
           -v $PWD/spack:/spack -v $PWD/spack-packages:/spack-packages \
           -v $PWD/env:/env -v $PWD/out:/out -v $PWD/linux-gnu:/linux-gnu:ro \
diff --git a/linux-gnu/build.sh b/linux-gnu/build.sh
index fa4fab1..fc4f204 100755
--- a/linux-gnu/build.sh
+++ b/linux-gnu/build.sh
@@ -1,7 +1,7 @@
 #!/bin/bash
 # Runs inside builder.dockerfile; see .github/workflows/linux-gnu.yml.
 #
-# Inputs (environment): REGISTRY, TAG, ARCH, BASE, GITHUB_ACTOR, GITHUB_TOKEN.
+# Inputs (environment): REGISTRY, TAG, ARCH, TARGET, BASE, GITHUB_ACTOR, GITHUB_TOKEN.
 # Mounts: /spack, /spack-packages, /env (spack.yaml, writable), /out.
 set -euo pipefail
 
@@ -14,6 +14,8 @@ cache="oci://$REGISTRY/linux-gnu-cache"
 # Use the pinned spack-packages checkout instead of cloning builtin.
 spack -e /env config add "repos:builtin:/spack-packages/repos/spack_repo/builtin"
 spack -e /env repo list
+# Generic microarchitecture, not the runner's (e.g. neoverse_n2 / zen4).
+spack -e /env config add "packages:all:require:[target=$TARGET]"
 spack -e /env mirror add --unsigned --autopush \
   --oci-username-variable GITHUB_ACTOR --oci-password-variable GITHUB_TOKEN \
   cache "$cache"
@@ -23,6 +25,9 @@ spack -e /env mirror add --unsigned --autopush \
 trap 'spack -e /env buildcache update-index cache || true' EXIT
 
 spack -e /env concretize --fresh 2>&1 | tee /out/concretize.txt
+targets=$(grep -oE 'target=[a-z0-9_]+' /out/concretize.txt | sort -u | tr '\n' ' ')
+echo "targets: $targets"
+test "$targets" = "target=$TARGET "
 spack -e /env find -c --show-full-compiler -L > /out/specs.txt || true
 
 # Installs pull what is already in the cache and push what they build.
diff --git a/linux-gnu/spack.yaml b/linux-gnu/spack.yaml
index 2be514e..686c4d1 100644
--- a/linux-gnu/spack.yaml
+++ b/linux-gnu/spack.yaml
@@ -42,13 +42,8 @@ spack:
     install_tree:
       root: /opt/spack
 
+  # build.sh adds `packages:all:require:[target=...]` (x86_64_v3 or aarch64).
   packages:
-    all:
-      require:
-      - spec: target=x86_64_v3
-        when: "target=x86_64:"
-      - spec: target=aarch64
-        when: "target=aarch64:"
     c:
       require: [gcc@8.5.0]
     cxx:

From 8660854eecae37f89708674de9c06c4275860c1e Mon Sep 17 00:00:00 2001
From: Harmen Stoppels <me+bot@harmenstoppels.nl>
Date: Fri, 13 Nov 2026 00:44:40 +0100
Subject: [PATCH 10/20] linux-gnu: check targets without color codes

---
 linux-gnu/build.sh | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/linux-gnu/build.sh b/linux-gnu/build.sh
index fc4f204..9ad2f51 100755
--- a/linux-gnu/build.sh
+++ b/linux-gnu/build.sh
@@ -25,9 +25,9 @@ spack -e /env mirror add --unsigned --autopush \
 trap 'spack -e /env buildcache update-index cache || true' EXIT
 
 spack -e /env concretize --fresh 2>&1 | tee /out/concretize.txt
-targets=$(grep -oE 'target=[a-z0-9_]+' /out/concretize.txt | sort -u | tr '\n' ' ')
+targets=$(spack --color=never -e /env find -c --format '{target}' | grep -v '^==>' | sort -u | xargs)
 echo "targets: $targets"
-test "$targets" = "target=$TARGET "
+test "$targets" = "$TARGET"
 spack -e /env find -c --show-full-compiler -L > /out/specs.txt || true
 
 # Installs pull what is already in the cache and push what they build.

From 16e96a842148e8b0043def08f1385941f87c7c9a Mon Sep 17 00:00:00 2001
From: Harmen Stoppels <me+bot@harmenstoppels.nl>
Date: Fri, 13 Nov 2026 00:44:58 +0100
Subject: [PATCH 11/20] linux-gnu: ignore install status in target check

---
 linux-gnu/build.sh | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/linux-gnu/build.sh b/linux-gnu/build.sh
index 9ad2f51..8456187 100755
--- a/linux-gnu/build.sh
+++ b/linux-gnu/build.sh
@@ -25,7 +25,7 @@ spack -e /env mirror add --unsigned --autopush \
 trap 'spack -e /env buildcache update-index cache || true' EXIT
 
 spack -e /env concretize --fresh 2>&1 | tee /out/concretize.txt
-targets=$(spack --color=never -e /env find -c --format '{target}' | grep -v '^==>' | sort -u | xargs)
+targets=$(spack --color=never -e /env find -c --format '{target}' | grep -v '^==>' | awk '{print $NF}' | sort -u | xargs)
 echo "targets: $targets"
 test "$targets" = "$TARGET"
 spack -e /env find -c --show-full-compiler -L > /out/specs.txt || true

From fcd4480136986d46e4e779aa7afdb5f904a6242c Mon Sep 17 00:00:00 2001
From: Harmen Stoppels <me+bot@harmenstoppels.nl>
Date: Fri, 13 Nov 2026 03:51:51 +0100
Subject: [PATCH 12/20] linux-gnu: use the image tags spack creates in the
 cache repository

Also drop the broken `spack load --only` smoke test; test-ci.sh covers it.
---
 .github/workflows/linux-gnu.yml | 6 ++++--
 linux-gnu/README.md             | 2 +-
 linux-gnu/build.sh              | 7 ++-----
 3 files changed, 7 insertions(+), 8 deletions(-)

diff --git a/.github/workflows/linux-gnu.yml b/.github/workflows/linux-gnu.yml
index 9cfde72..ea6f2d2 100644
--- a/.github/workflows/linux-gnu.yml
+++ b/.github/workflows/linux-gnu.yml
@@ -155,9 +155,11 @@ jobs:
       env:
         TAG: ${{ github.event_name == 'pull_request' && format('pr{0}', github.event.number) || github.ref_name }}
       run: |
+        # `spack buildcache push --tag` tags per-arch images in the cache repository
+        c=$REGISTRY/linux-gnu-cache
         i=$REGISTRY/linux-gnu
         day=$(date -u +%Y-%m-%d)
-        docker buildx imagetools create -t $i:$TAG -t $i:$day-${{ github.run_id }} $i:$TAG-amd64 $i:$TAG-arm64
+        docker buildx imagetools create -t $i:$TAG -t $i:$day-${{ github.run_id }} $c:$TAG-amd64 $c:$TAG-arm64
         docker buildx imagetools inspect $i:$TAG
 
   # Is the image usable for spack-packages CI? Concretize a developer-tools
@@ -190,4 +192,4 @@ jobs:
           -e BUILD_SPECS="libtree tree jq ncdu tmux patchelf py-isort gmake" \
           -e SPACK_COLOR=always -e PYTHONUNBUFFERED=1 \
           -v $PWD/linux-gnu:/linux-gnu:ro \
-          $REGISTRY/linux-gnu:$TAG-${{ matrix.arch }} /linux-gnu/test-ci.sh
+          $REGISTRY/linux-gnu-cache:$TAG-${{ matrix.arch }} /linux-gnu/test-ci.sh
diff --git a/linux-gnu/README.md b/linux-gnu/README.md
index e1c3150..01b22eb 100644
--- a/linux-gnu/README.md
+++ b/linux-gnu/README.md
@@ -24,7 +24,7 @@ of this natively on x86_64 and aarch64 GitHub runners and publishes:
 - `ghcr.io/<owner>/linux-gnu-base:<hash>`: the runtime base
 - `ghcr.io/<owner>/linux-gnu-cache`: the OCI build cache. Packages are pushed as
   soon as they are built (`autopush`), so a re-run after a timeout resumes.
-- `ghcr.io/<owner>/linux-gnu:<branch>-{amd64,arm64}`, and multi-arch
+- `ghcr.io/<owner>/linux-gnu-cache:<branch>-{amd64,arm64}` (`--tag` tags in the cache repository), and multi-arch
   `ghcr.io/<owner>/linux-gnu:<branch>` plus `:<date>-<run id>`.
 
 Spack and spack-packages are pinned by commit in the workflow. Point them at
diff --git a/linux-gnu/build.sh b/linux-gnu/build.sh
index 8456187..a4c6be8 100755
--- a/linux-gnu/build.sh
+++ b/linux-gnu/build.sh
@@ -33,10 +33,7 @@ spack -e /env find -c --show-full-compiler -L > /out/specs.txt || true
 # Installs pull what is already in the cache and push what they build.
 spack -e /env install --fail-fast -j "$(nproc)"
 
-# Tag one image with every root and its runtime closure, on top of the base.
+# Tag one image with every root and its runtime closure, on top of the base. The
+# tag is created in the cache's repository: $REGISTRY/linux-gnu-cache:$TAG-$ARCH.
 spack -e /env buildcache push --unsigned --update-index --base-image "$BASE-$ARCH" \
   --tag "$TAG-$ARCH" cache
-
-# Smoke test of the environment the image gets (PATH etc. of the roots).
-spack -e /env load --sh --only package \
-  gcc binutils python git curl gnupg awscli-v2 clingo-bootstrap > /out/load.sh

From d0eace0204449c08aa9452f258183fe6b5d9a09d Mon Sep 17 00:00:00 2001
From: Harmen Stoppels <me+bot@harmenstoppels.nl>
Date: Fri, 13 Nov 2026 04:18:20 +0100
Subject: [PATCH 13/20] linux-gnu: push only the runtime closure when tagging

Build-only dependencies are not installed when everything comes from the
cache (and autopush already pushed them when they were built).
---
 linux-gnu/build.sh | 3 ++-
 1 file changed, 2 insertions(+), 1 deletion(-)

diff --git a/linux-gnu/build.sh b/linux-gnu/build.sh
index a4c6be8..9d4e63a 100755
--- a/linux-gnu/build.sh
+++ b/linux-gnu/build.sh
@@ -35,5 +35,6 @@ spack -e /env install --fail-fast -j "$(nproc)"
 
 # Tag one image with every root and its runtime closure, on top of the base. The
 # tag is created in the cache's repository: $REGISTRY/linux-gnu-cache:$TAG-$ARCH.
-spack -e /env buildcache push --unsigned --update-index --base-image "$BASE-$ARCH" \
+spack -e /env buildcache push --unsigned --update-index --without-build-dependencies \
+  --base-image "$BASE-$ARCH" \
   --tag "$TAG-$ARCH" cache

From efb5c2b9ae702f33d54ee222cdbb314348e8bd87 Mon Sep 17 00:00:00 2001
From: Harmen Stoppels <me+bot@harmenstoppels.nl>
Date: Fri, 13 Nov 2026 04:26:23 +0100
Subject: [PATCH 14/20] linux-gnu: test-ci: strip install status from hashes

---
 linux-gnu/test-ci.sh | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/linux-gnu/test-ci.sh b/linux-gnu/test-ci.sh
index c74dc52..cdea8b1 100755
--- a/linux-gnu/test-ci.sh
+++ b/linux-gnu/test-ci.sh
@@ -64,7 +64,7 @@ echo "OK: nothing bootstrapped"
 section "Build: ${BUILD_SPECS}"
 # Like build jobs: padded install tree plus the relocation test projection.
 for spec in $BUILD_SPECS; do
-  hash=$(spack find -c --format '{hash}' "$spec" | head -n1)
+  hash=$(spack --color=never find -c --format '{hash}' "$spec" | awk '{print $NF}' | head -n1)
   spack config add "config:install_tree:projections:$spec/$hash:'morepadding/{architecture.platform}-{architecture.target}/{name}-{version}-{hash}'"
 done
 spack install --fail-fast --no-check-signature $BUILD_SPECS

From 07df91fbe6147be2ab2c3af9da55a9d2b0b5d0e5 Mon Sep 17 00:00:00 2001
From: Harmen Stoppels <me+bot@harmenstoppels.nl>
Date: Fri, 13 Nov 2026 04:56:47 +0100
Subject: [PATCH 15/20] linux-gnu: build with Spack fixes from the fork

---
 .github/workflows/linux-gnu.yml | 5 +++--
 1 file changed, 3 insertions(+), 2 deletions(-)

diff --git a/.github/workflows/linux-gnu.yml b/.github/workflows/linux-gnu.yml
index ea6f2d2..0792393 100644
--- a/.github/workflows/linux-gnu.yml
+++ b/.github/workflows/linux-gnu.yml
@@ -25,8 +25,9 @@ concurrency:
 env:
   # Pinned Spack and package repository. Point these at forks to build with
   # unmerged fixes.
-  SPACK_REPOSITORY: spack/spack
-  SPACK_COMMIT: e9a7922af3e28d24308f47ed7ce1ecd5eb9db817
+  # haampie-llms/spack yaml-keep-trailing-colon-in-values: develop e9a7922 + YAML fix
+  SPACK_REPOSITORY: haampie-llms/spack
+  SPACK_COMMIT: 1cf09a150e070a2423a85aee8846174dc15727fb
   # haampie-llms/spack-packages linux-gnu-ci-image: develop 8933fbb + version bumps
   PACKAGES_REPOSITORY: haampie-llms/spack-packages
   PACKAGES_COMMIT: e0051ddce1d5ba25c74844143f20d63656c08158

From a5870848792e3e9c08737796fa0bd59f12fed65d Mon Sep 17 00:00:00 2001
From: Harmen Stoppels <me+bot@harmenstoppels.nl>
Date: Fri, 13 Nov 2026 08:49:31 +0100
Subject: [PATCH 16/20] linux-gnu: pin Spack fix rebased on develop

---
 .github/workflows/linux-gnu.yml | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/.github/workflows/linux-gnu.yml b/.github/workflows/linux-gnu.yml
index 0792393..5d3568e 100644
--- a/.github/workflows/linux-gnu.yml
+++ b/.github/workflows/linux-gnu.yml
@@ -25,9 +25,9 @@ concurrency:
 env:
   # Pinned Spack and package repository. Point these at forks to build with
   # unmerged fixes.
-  # haampie-llms/spack yaml-keep-trailing-colon-in-values: develop e9a7922 + YAML fix
+  # haampie-llms/spack yaml-keep-trailing-colon-in-values: develop e05599c + YAML fix
   SPACK_REPOSITORY: haampie-llms/spack
-  SPACK_COMMIT: 1cf09a150e070a2423a85aee8846174dc15727fb
+  SPACK_COMMIT: 11b8cae1bab08a94c39d180d68cd469d5f3c08ac
   # haampie-llms/spack-packages linux-gnu-ci-image: develop 8933fbb + version bumps
   PACKAGES_REPOSITORY: haampie-llms/spack-packages
   PACKAGES_COMMIT: e0051ddce1d5ba25c74844143f20d63656c08158

From 69f7f3e4d94f094ad06f55e4e71e90e1417d200f Mon Sep 17 00:00:00 2001
From: Harmen Stoppels <me+bot@harmenstoppels.nl>
Date: Fri, 13 Nov 2026 10:27:32 +0100
Subject: [PATCH 17/20] linux-gnu: add llvm@23 (clang) built with gcc@14 in a
 second spec group

The ml-linux stacks require %c,cxx=llvm for py-jaxlib and py-tensorflow.
llvm writes --gcc-install-dir of the compiler that built it into clang.cfg,
and the image has no system gcc, so llvm is built with the Spack-built
gcc@14 in a group that needs the tools group.

test-ci.sh checks that clang++ compiles and runs C++ with that toolchain and
concretizes the ml-linux stacks.
---
 .github/workflows/linux-gnu.yml |  6 +--
 linux-gnu/README.md             |  6 ++-
 linux-gnu/spack.yaml            | 84 ++++++++++++++++++++-------------
 linux-gnu/test-ci.sh            | 22 ++++++++-
 4 files changed, 79 insertions(+), 39 deletions(-)

diff --git a/.github/workflows/linux-gnu.yml b/.github/workflows/linux-gnu.yml
index 5d3568e..e2ad042 100644
--- a/.github/workflows/linux-gnu.yml
+++ b/.github/workflows/linux-gnu.yml
@@ -172,8 +172,8 @@ jobs:
       fail-fast: false
       matrix:
         include:
-        - {runner: ubuntu-24.04, arch: amd64, target: x86_64_v3, stack: developer-tools-x86_64_v3-linux-gnu}
-        - {runner: ubuntu-24.04-arm, arch: arm64, target: aarch64, stack: developer-tools-aarch64-linux-gnu}
+        - {runner: ubuntu-24.04, arch: amd64, target: x86_64_v3, stack: developer-tools-x86_64_v3-linux-gnu, ml_stack: ml-linux-x86_64}
+        - {runner: ubuntu-24.04-arm, arch: arm64, target: aarch64, stack: developer-tools-aarch64-linux-gnu, ml_stack: ml-linux-aarch64}
     permissions:
       contents: read
       packages: read
@@ -189,7 +189,7 @@ jobs:
         TAG: ${{ github.event_name == 'pull_request' && format('pr{0}', github.event.number) || github.ref_name }}
       run: |
         docker run --rm -e PACKAGES_REPOSITORY -e PACKAGES_COMMIT \
-          -e SPACK_CI_TARGET=${{ matrix.target }} -e STACK=${{ matrix.stack }} \
+          -e SPACK_CI_TARGET=${{ matrix.target }} -e STACK=${{ matrix.stack }} -e ML_STACK=${{ matrix.ml_stack }} \
           -e BUILD_SPECS="libtree tree jq ncdu tmux patchelf py-isort gmake" \
           -e SPACK_COLOR=always -e PYTHONUNBUFFERED=1 \
           -v $PWD/linux-gnu:/linux-gnu:ro \
diff --git a/linux-gnu/README.md b/linux-gnu/README.md
index 01b22eb..4dfd139 100644
--- a/linux-gnu/README.md
+++ b/linux-gnu/README.md
@@ -9,12 +9,14 @@ from a Spack environment on top of a minimal AlmaLinux 8 base.
   and the bare minimum to run Spack.
 - [`spack.yaml`](spack.yaml): gcc@14 (the compiler CI builds with) and the tools
   CI needs: Python with Spack's optimized clingo, boto3, AWS CLI v2, git, curl,
-  gnupg, compression tools, ...
+  gnupg, compression tools, ... All built with AlmaLinux's gcc 8.5, except
+  llvm@23 (clang, for stacks that require `%c,cxx=llvm`), which a second spec
+  group builds with the Spack-built gcc@14 so that clang uses its libstdc++.
 - [`build.sh`](build.sh): concretizes and installs the environment in the
   builder, then runs `spack buildcache push --base-image <base> --tag <tag>`,
   which produces the image: the base plus one layer per package, with `PATH` and
   friends set for the root specs.
-- [`test-ci.sh`](test-ci.sh): runs in the image like spack-packages CI does:
+- [`test-ci.sh`](test-ci.sh): checks clang, then runs in the image like spack-packages CI does:
   concretizes a developer-tools stack (generate job) and builds part of it under
   CI's padded install tree (build jobs).
 
diff --git a/linux-gnu/spack.yaml b/linux-gnu/spack.yaml
index 686c4d1..043e898 100644
--- a/linux-gnu/spack.yaml
+++ b/linux-gnu/spack.yaml
@@ -1,38 +1,62 @@
 # Tools for Spack's GitLab CI on linux, built inside builder.dockerfile and
 # pushed as an image on top of base.dockerfile (see README.md).
 #
-# Everything, including gcc@14, is built with AlmaLinux 8's gcc 8.5: the tools
-# only run Spack and its helpers, and CI builds with gcc@14 and its own
-# gcc-runtime. Not building the tools with gcc@14 keeps them off the critical
-# path of the (slow) gcc build and avoids rebuilding all of them on gcc bumps.
+# Everything except llvm, including gcc@14, is built with AlmaLinux 8's gcc 8.5:
+# the tools only run Spack and its helpers, and CI builds with gcc@14 and its
+# own gcc-runtime. Not building the tools with gcc@14 keeps them off the
+# critical path of the (slow) gcc build and avoids rebuilding all of them on gcc
+# bumps.
+#
+# llvm is built with gcc@14 in a second group: llvm writes `--gcc-install-dir`
+# of the compiler that built it into clang.cfg, and the image has no system gcc,
+# only Spack's gcc@14. Spack only builds with external or concrete compilers,
+# hence the group that `needs` the first one.
 spack:
   view: false
 
   specs:
-  # The compiler CI builds with.
-  - gcc@14 +binutils +profiled build_type=Release languages=c,c++,fortran
-  - binutils +ld +gas +plugins ~libiberty
-  # Spack's own dependencies and CI tooling.
-  - awscli-v2
-  - bzip2
-  - clingo-bootstrap@spack +optimized
-  - curl
-  - file
-  - git
-  - gmake
-  - gnupg
-  - gzip
-  - libtree
-  - lz4
-  - patch
-  - patchelf
-  - py-boto3
-  - python +optimizations
-  - tar
-  - unzip
-  - vim
-  - xz
-  - zstd
+  - group: tools
+    specs:
+    # The compiler CI builds with.
+    - gcc@14 +binutils +profiled build_type=Release languages=c,c++,fortran
+    - binutils +ld +gas +plugins ~libiberty
+    # Spack's own dependencies and CI tooling.
+    - awscli-v2
+    - bzip2
+    - clingo-bootstrap@spack +optimized
+    - curl
+    - file
+    - git
+    - gmake
+    - gnupg
+    - gzip
+    - libtree
+    - lz4
+    - patch
+    - patchelf
+    - py-boto3
+    - python +optimizations
+    - tar
+    - unzip
+    - vim
+    - xz
+    - zstd
+    override:
+      packages:
+        c:
+          require: [gcc@8.5.0]
+        cxx:
+          require: [gcc@8.5.0]
+
+  # clang, for stacks that require `%c,cxx=llvm` (py-jaxlib, py-tensorflow).
+  - group: llvm
+    needs: [tools]
+    specs:
+    - llvm@23 +clang +lld ~lldb ~libomptarget ~offload ~python
+    override:
+      packages:
+        llvm:
+          require: ["%c,cxx=gcc@14"]
 
   concretizer:
     unify: true
@@ -44,10 +68,6 @@ spack:
 
   # build.sh adds `packages:all:require:[target=...]` (x86_64_v3 or aarch64).
   packages:
-    c:
-      require: [gcc@8.5.0]
-    cxx:
-      require: [gcc@8.5.0]
     gcc:
       externals:
       - spec: gcc@8.5.0 languages:='c,c++'
diff --git a/linux-gnu/test-ci.sh b/linux-gnu/test-ci.sh
index cdea8b1..12b92d0 100755
--- a/linux-gnu/test-ci.sh
+++ b/linux-gnu/test-ci.sh
@@ -4,13 +4,14 @@
 # stacks (.ci/gitlab/.gitlab-ci.yml, .ci/gitlab/configs/ci.yaml).
 #
 # Inputs (environment): PACKAGES_REPOSITORY, PACKAGES_COMMIT, SPACK_CI_TARGET,
-# STACK, BUILD_SPECS (space separated roots of the stack to build).
+# STACK, BUILD_SPECS (space separated roots of the stack to build), ML_STACK
+# (optional: a stack that needs clang, only concretized).
 set -euo pipefail
 
 section() { echo; echo "### $*"; }
 
 section "Tools in the image"
-for exe in python3 gcc g++ gfortran ld ar git curl gpg aws make tar xz zstd unzip patch patchelf file; do
+for exe in python3 gcc g++ gfortran clang clang++ ld.lld ld ar git curl gpg aws make tar xz zstd unzip patch patchelf file; do
   printf '%-10s %s\n' "$exe" "$(command -v "$exe")"
 done
 gcc --version | head -n1
@@ -26,6 +27,12 @@ curl --version | head -n1
 workdir=$(mktemp -d)
 cd "$workdir"
 
+section "clang uses Spack's gcc@14 toolchain (there is no system gcc)"
+clang --version | head -n1
+clang++ -v -fsyntax-only -x c++ /dev/null 2>&1 | grep 'Selected GCC installation'
+printf '#include <iostream>\nint main() { std::cout << "hello from clang++" << std::endl; }\n' > hello.cpp
+clang++ -O2 hello.cpp -o hello && ./hello
+
 section "Check out spack-packages and the Spack commit its CI uses"
 git init -q packages
 git -C packages fetch -q --depth 1 "https://github.com/$PACKAGES_REPOSITORY.git" "$PACKAGES_COMMIT"
@@ -60,6 +67,8 @@ if [ -d "$SPACK_USER_CACHE_PATH/bootstrap/store" ]; then
   exit 1
 fi
 echo "OK: nothing bootstrapped"
+echo "compilers used:"
+spack --color=never find -c --format '{compiler.name}@{compiler.version}' | grep -v '^==>' | sort | uniq -c
 
 section "Build: ${BUILD_SPECS}"
 # Like build jobs: padded install tree plus the relocation test projection.
@@ -71,6 +80,15 @@ spack install --fail-fast --no-check-signature $BUILD_SPECS
 spack find -lp $BUILD_SPECS
 spack env deactivate
 
+if [ -n "${ML_STACK:-}" ]; then
+  section "Generate: concretize $ML_STACK (needs %c,cxx=llvm)"
+  start=$(date +%s)
+  spack -e "$workdir/packages/stacks/$ML_STACK" concretize -j 4 > "$workdir/ml-concretize.log" 2>&1 \
+    || { tail -n 60 "$workdir/ml-concretize.log"; exit 1; }
+  echo "concretized in $(( $(date +%s) - start ))s"
+  spack --color=never -e "$workdir/packages/stacks/$ML_STACK" find -c --format '{name}@{version} %{compiler.name}@{compiler.version}' py-jaxlib py-tensorflow cuda | sort -u
+fi
+
 section "Outputs do not need a glibc newer than 2.28"
 newest=$(find /home/software/spack -type f \( -name '*.so*' -o -perm -u+x \) -exec sh -c 'file -b "$1" | grep -q ELF && objdump -T "$1" 2>/dev/null' _ {} \; \
   | grep -oE 'GLIBC_[0-9]+\.[0-9]+(\.[0-9]+)?' | sort -uV | tail -n1)

From f484325978afa2a15ca6f61538a7a725086b5dbe Mon Sep 17 00:00:00 2001
From: Harmen Stoppels <me+bot@harmenstoppels.nl>
Date: Fri, 13 Nov 2026 14:22:31 +0100
Subject: [PATCH 18/20] linux-gnu: test-ci: rpath clang++ test binary to
 gcc@14's libstdc++

---
 linux-gnu/test-ci.sh | 6 +++++-
 1 file changed, 5 insertions(+), 1 deletion(-)

diff --git a/linux-gnu/test-ci.sh b/linux-gnu/test-ci.sh
index 12b92d0..6684bd2 100755
--- a/linux-gnu/test-ci.sh
+++ b/linux-gnu/test-ci.sh
@@ -31,7 +31,11 @@ section "clang uses Spack's gcc@14 toolchain (there is no system gcc)"
 clang --version | head -n1
 clang++ -v -fsyntax-only -x c++ /dev/null 2>&1 | grep 'Selected GCC installation'
 printf '#include <iostream>\nint main() { std::cout << "hello from clang++" << std::endl; }\n' > hello.cpp
-clang++ -O2 hello.cpp -o hello && ./hello
+# RPATH to gcc@14's libstdc++ like Spack's compiler wrapper does; without it the
+# loader picks AlmaLinux's older /lib64/libstdc++.so.6 (same for plain g++).
+libstdcxx_dir=$(dirname "$(clang++ -print-file-name=libstdc++.so)")
+echo "libstdc++: $libstdcxx_dir"
+clang++ -O2 hello.cpp -o hello -Wl,-rpath,"$libstdcxx_dir" && ./hello
 
 section "Check out spack-packages and the Spack commit its CI uses"
 git init -q packages

From f0a0f72cade0fdb4cbc6c22f12fd01bcadf912c6 Mon Sep 17 00:00:00 2001
From: Harmen Stoppels <me+bot@harmenstoppels.nl>
Date: Fri, 13 Nov 2026 14:45:49 +0100
Subject: [PATCH 19/20] linux-gnu: llvm +link_llvm_dylib, with the clang.cfg
 rpath and libexec fixes

- spack/spack-packages#7671: clang.cfg rpaths gcc@14's runtime libraries,
  so test-ci.sh no longer passes -rpath and checks RUNPATH instead (for
  g++ via gcc's specs file, for clang++ via clang++.cfg).
- spack/spack-packages#7672: no copy of the build's bin in libexec/llvm.
- +link_llvm_dylib: tools link libLLVM.so and libclang-cpp.so instead of
  each statically linking LLVM; llvm was a 3.4 GB compressed layer.
---
 .github/workflows/linux-gnu.yml |  3 ++-
 linux-gnu/spack.yaml            |  4 +++-
 linux-gnu/test-ci.sh            | 13 ++++++++-----
 3 files changed, 13 insertions(+), 7 deletions(-)

diff --git a/.github/workflows/linux-gnu.yml b/.github/workflows/linux-gnu.yml
index e2ad042..f874611 100644
--- a/.github/workflows/linux-gnu.yml
+++ b/.github/workflows/linux-gnu.yml
@@ -29,8 +29,9 @@ env:
   SPACK_REPOSITORY: haampie-llms/spack
   SPACK_COMMIT: 11b8cae1bab08a94c39d180d68cd469d5f3c08ac
   # haampie-llms/spack-packages linux-gnu-ci-image: develop 8933fbb + version bumps
+  # + llvm: clang.cfg rpath fix, no libexec/llvm copy
   PACKAGES_REPOSITORY: haampie-llms/spack-packages
-  PACKAGES_COMMIT: e0051ddce1d5ba25c74844143f20d63656c08158
+  PACKAGES_COMMIT: 2995557483bcb25530dc419800f52333adb445c2
   REGISTRY: ghcr.io/${{ github.repository_owner }}
 
 jobs:
diff --git a/linux-gnu/spack.yaml b/linux-gnu/spack.yaml
index 043e898..4d4aeba 100644
--- a/linux-gnu/spack.yaml
+++ b/linux-gnu/spack.yaml
@@ -52,7 +52,9 @@ spack:
   - group: llvm
     needs: [tools]
     specs:
-    - llvm@23 +clang +lld ~lldb ~libomptarget ~offload ~python
+    # +link_llvm_dylib: tools link libLLVM.so/libclang-cpp.so instead of each
+    # statically linking LLVM (3.4 GB compressed layer otherwise).
+    - llvm@23 +clang +lld +link_llvm_dylib ~lldb ~libomptarget ~offload ~python
     override:
       packages:
         llvm:
diff --git a/linux-gnu/test-ci.sh b/linux-gnu/test-ci.sh
index 6684bd2..79eaa59 100755
--- a/linux-gnu/test-ci.sh
+++ b/linux-gnu/test-ci.sh
@@ -31,11 +31,14 @@ section "clang uses Spack's gcc@14 toolchain (there is no system gcc)"
 clang --version | head -n1
 clang++ -v -fsyntax-only -x c++ /dev/null 2>&1 | grep 'Selected GCC installation'
 printf '#include <iostream>\nint main() { std::cout << "hello from clang++" << std::endl; }\n' > hello.cpp
-# RPATH to gcc@14's libstdc++ like Spack's compiler wrapper does; without it the
-# loader picks AlmaLinux's older /lib64/libstdc++.so.6 (same for plain g++).
-libstdcxx_dir=$(dirname "$(clang++ -print-file-name=libstdc++.so)")
-echo "libstdc++: $libstdcxx_dir"
-clang++ -O2 hello.cpp -o hello -Wl,-rpath,"$libstdcxx_dir" && ./hello
+# Like gcc's specs file, clang++.cfg must rpath gcc@14's runtime libraries;
+# otherwise the loader picks AlmaLinux's older /lib64/libstdc++.so.6.
+cat "$(dirname "$(readlink -f "$(command -v clang++)")")/clang++.cfg"
+for cxx in g++ clang++; do
+  $cxx -O2 hello.cpp -o hello-$cxx
+  readelf -d hello-$cxx | grep -E 'RPATH|RUNPATH'
+  ./hello-$cxx
+done
 
 section "Check out spack-packages and the Spack commit its CI uses"
 git init -q packages

From d5b185a10bf5536bd5cb8706b75d22c3c2d5b79b Mon Sep 17 00:00:00 2001
From: Harmen Stoppels <me+bot@harmenstoppels.nl>
Date: Fri, 13 Nov 2026 17:09:16 +0100
Subject: [PATCH 20/20] linux-gnu: pin spack-packages with the py-libclang fix
 (spack/spack-packages#7676)

---
 .github/workflows/linux-gnu.yml | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/.github/workflows/linux-gnu.yml b/.github/workflows/linux-gnu.yml
index f874611..ecf4e4a 100644
--- a/.github/workflows/linux-gnu.yml
+++ b/.github/workflows/linux-gnu.yml
@@ -29,9 +29,9 @@ env:
   SPACK_REPOSITORY: haampie-llms/spack
   SPACK_COMMIT: 11b8cae1bab08a94c39d180d68cd469d5f3c08ac
   # haampie-llms/spack-packages linux-gnu-ci-image: develop 8933fbb + version bumps
-  # + llvm: clang.cfg rpath fix, no libexec/llvm copy
+  # + llvm: clang.cfg rpath, no libexec/llvm copy; py-libclang: allow newer llvm
   PACKAGES_REPOSITORY: haampie-llms/spack-packages
-  PACKAGES_COMMIT: 2995557483bcb25530dc419800f52333adb445c2
+  PACKAGES_COMMIT: 78fee03288bd14eff22a6d8dd10a2a9c3efdd00d
   REGISTRY: ghcr.io/${{ github.repository_owner }}
 
 jobs: