diff --git a/.github/workflows/linux-gnu.yml b/.github/workflows/linux-gnu.yml
new file mode 100644
index 0000000..ecf4e4a
--- /dev/null
+++ b/.github/workflows/linux-gnu.yml
@@ -0,0 +1,197 @@
+name: linux-gnu image
+
+# Builds linux-gnu/spack.yaml natively on x86_64 and aarch64 runners inside an
+# AlmaLinux 8 builder container. Every installed package is pushed to an OCI
+# build cache as soon as it is built (autopush), so a re-run of a timed-out or
+# failed job resumes where it left off. The image itself is assembled by
+# `spack buildcache push --base-image ... --tag ...`: the runtime base plus one
+# layer per Spack package, with PATH etc. set for the root specs.
+
+on:
+ workflow_dispatch:
+ push:
+ paths:
+ - linux-gnu/**
+ - .github/workflows/linux-gnu.yml
+ pull_request:
+ paths:
+ - linux-gnu/**
+ - .github/workflows/linux-gnu.yml
+
+concurrency:
+ group: linux-gnu-${{ github.ref }}
+ cancel-in-progress: true
+
+env:
+ # Pinned Spack and package repository. Point these at forks to build with
+ # unmerged fixes.
+ # haampie-llms/spack yaml-keep-trailing-colon-in-values: develop e05599c + YAML fix
+ SPACK_REPOSITORY: haampie-llms/spack
+ SPACK_COMMIT: 11b8cae1bab08a94c39d180d68cd469d5f3c08ac
+ # haampie-llms/spack-packages linux-gnu-ci-image: develop 8933fbb + version bumps
+ # + llvm: clang.cfg rpath, no libexec/llvm copy; py-libclang: allow newer llvm
+ PACKAGES_REPOSITORY: haampie-llms/spack-packages
+ PACKAGES_COMMIT: 78fee03288bd14eff22a6d8dd10a2a9c3efdd00d
+ REGISTRY: ghcr.io/${{ github.repository_owner }}
+
+jobs:
+ base:
+ runs-on: ${{ matrix.runner }}
+ strategy:
+ matrix:
+ include:
+ - {runner: ubuntu-24.04, arch: amd64}
+ - {runner: ubuntu-24.04-arm, arch: arm64}
+ permissions:
+ contents: read
+ packages: write
+ outputs:
+ tag: ${{ steps.tag.outputs.tag }}
+ steps:
+ - uses: actions/checkout@v7
+ - name: Tag
+ id: tag
+ # Same tag for both arches: the base inputs, not the build date.
+ run: echo "tag=$(cat linux-gnu/base.dockerfile | sha256sum | cut -c1-12)" >> "$GITHUB_OUTPUT"
+ - uses: docker/login-action@v4
+ with:
+ registry: ghcr.io
+ username: ${{ github.actor }}
+ password: ${{ secrets.GITHUB_TOKEN }}
+ - name: Build and push runtime base and builder
+ run: |
+ base=$REGISTRY/linux-gnu-base:${{ steps.tag.outputs.tag }}-${{ matrix.arch }}
+ docker build -t $base -f linux-gnu/base.dockerfile linux-gnu
+ docker build -t linux-gnu-builder --build-arg BASE=$base -f linux-gnu/builder.dockerfile linux-gnu
+ docker push $base
+
+ base-manifest:
+ needs: base
+ runs-on: ubuntu-24.04
+ permissions:
+ packages: write
+ steps:
+ - uses: docker/login-action@v4
+ with:
+ registry: ghcr.io
+ username: ${{ github.actor }}
+ password: ${{ secrets.GITHUB_TOKEN }}
+ - run: |
+ b=$REGISTRY/linux-gnu-base:${{ needs.base.outputs.tag }}
+ docker buildx imagetools create -t $b $b-amd64 $b-arm64
+
+ build:
+ needs: [base, base-manifest]
+ runs-on: ${{ matrix.runner }}
+ timeout-minutes: 355
+ strategy:
+ fail-fast: false
+ matrix:
+ include:
+ - {runner: ubuntu-24.04, arch: amd64, target: x86_64_v3}
+ - {runner: ubuntu-24.04-arm, arch: arm64, target: aarch64}
+ permissions:
+ contents: read
+ packages: write
+ env:
+ BASE: ghcr.io/${{ github.repository_owner }}/linux-gnu-base:${{ needs.base.outputs.tag }}
+ steps:
+ - uses: actions/checkout@v7
+
+ - name: Check out Spack and spack-packages
+ run: |
+ for r in "spack $SPACK_REPOSITORY $SPACK_COMMIT" "spack-packages $PACKAGES_REPOSITORY $PACKAGES_COMMIT"; do
+ set -- $r
+ git init -q $1
+ git -C $1 fetch -q --depth 1 https://github.com/$2.git $3
+ git -C $1 checkout -q FETCH_HEAD
+ done
+
+ - uses: docker/login-action@v4
+ with:
+ registry: ghcr.io
+ username: ${{ github.actor }}
+ password: ${{ secrets.GITHUB_TOKEN }}
+
+ - name: Builder image
+ run: |
+ docker pull $BASE-${{ matrix.arch }}
+ docker build -t linux-gnu-builder --build-arg BASE=$BASE-${{ matrix.arch }} \
+ -f linux-gnu/builder.dockerfile linux-gnu
+
+ - name: Concretize, install and push
+ env:
+ GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
+ TAG: ${{ github.event_name == 'pull_request' && format('pr{0}', github.event.number) || github.ref_name }}
+ run: |
+ mkdir -p env out
+ cp linux-gnu/spack.yaml env/
+ docker run --rm -e GITHUB_ACTOR -e GITHUB_TOKEN -e REGISTRY -e TAG \
+ -e ARCH=${{ matrix.arch }} -e TARGET=${{ matrix.target }} -e BASE \
+ -e SPACK_COLOR=always -e PYTHONUNBUFFERED=1 \
+ -v $PWD/spack:/spack -v $PWD/spack-packages:/spack-packages \
+ -v $PWD/env:/env -v $PWD/out:/out -v $PWD/linux-gnu:/linux-gnu:ro \
+ linux-gnu-builder /linux-gnu/build.sh
+
+ - uses: actions/upload-artifact@v4
+ if: always()
+ with:
+ name: linux-gnu-${{ matrix.arch }}
+ path: |
+ env/spack.yaml
+ env/spack.lock
+ out/
+
+ image:
+ needs: [base, build]
+ runs-on: ubuntu-24.04
+ permissions:
+ packages: write
+ steps:
+ - uses: docker/login-action@v4
+ with:
+ registry: ghcr.io
+ username: ${{ github.actor }}
+ password: ${{ secrets.GITHUB_TOKEN }}
+ - name: Multi-arch image
+ env:
+ TAG: ${{ github.event_name == 'pull_request' && format('pr{0}', github.event.number) || github.ref_name }}
+ run: |
+ # `spack buildcache push --tag` tags per-arch images in the cache repository
+ c=$REGISTRY/linux-gnu-cache
+ i=$REGISTRY/linux-gnu
+ day=$(date -u +%Y-%m-%d)
+ docker buildx imagetools create -t $i:$TAG -t $i:$day-${{ github.run_id }} $c:$TAG-amd64 $c:$TAG-arm64
+ docker buildx imagetools inspect $i:$TAG
+
+ # Is the image usable for spack-packages CI? Concretize a developer-tools
+ # stack the way generate jobs do and build a sample of it like build jobs.
+ test:
+ needs: build
+ runs-on: ${{ matrix.runner }}
+ strategy:
+ fail-fast: false
+ matrix:
+ include:
+ - {runner: ubuntu-24.04, arch: amd64, target: x86_64_v3, stack: developer-tools-x86_64_v3-linux-gnu, ml_stack: ml-linux-x86_64}
+ - {runner: ubuntu-24.04-arm, arch: arm64, target: aarch64, stack: developer-tools-aarch64-linux-gnu, ml_stack: ml-linux-aarch64}
+ permissions:
+ contents: read
+ packages: read
+ steps:
+ - uses: actions/checkout@v7
+ - uses: docker/login-action@v4
+ with:
+ registry: ghcr.io
+ username: ${{ github.actor }}
+ password: ${{ secrets.GITHUB_TOKEN }}
+ - name: Test ${{ matrix.stack }}
+ env:
+ TAG: ${{ github.event_name == 'pull_request' && format('pr{0}', github.event.number) || github.ref_name }}
+ run: |
+ docker run --rm -e PACKAGES_REPOSITORY -e PACKAGES_COMMIT \
+ -e SPACK_CI_TARGET=${{ matrix.target }} -e STACK=${{ matrix.stack }} -e ML_STACK=${{ matrix.ml_stack }} \
+ -e BUILD_SPECS="libtree tree jq ncdu tmux patchelf py-isort gmake" \
+ -e SPACK_COLOR=always -e PYTHONUNBUFFERED=1 \
+ -v $PWD/linux-gnu:/linux-gnu:ro \
+ $REGISTRY/linux-gnu-cache:$TAG-${{ matrix.arch }} /linux-gnu/test-ci.sh
diff --git a/linux-gnu/README.md b/linux-gnu/README.md
new file mode 100644
index 0000000..4dfd139
--- /dev/null
+++ b/linux-gnu/README.md
@@ -0,0 +1,33 @@
+Build image for Spack's GitLab CI on linux (x86_64_v3 and aarch64), made entirely
+from a Spack environment on top of a minimal AlmaLinux 8 base.
+
+- [`base.dockerfile`](base.dockerfile): AlmaLinux 8 (pinned by digest) plus
+ `glibc-devel`. glibc 2.28 is the floor of everything CI builds in this image;
+ [`check-glibc.sh`](check-glibc.sh) fails the build if the system libc exports
+ symbol versions newer than that (RHEL backports, see spack/spack#51888).
+- [`builder.dockerfile`](builder.dockerfile): the base plus AlmaLinux's gcc 8.5
+ and the bare minimum to run Spack.
+- [`spack.yaml`](spack.yaml): gcc@14 (the compiler CI builds with) and the tools
+ CI needs: Python with Spack's optimized clingo, boto3, AWS CLI v2, git, curl,
+ gnupg, compression tools, ... All built with AlmaLinux's gcc 8.5, except
+ llvm@23 (clang, for stacks that require `%c,cxx=llvm`), which a second spec
+ group builds with the Spack-built gcc@14 so that clang uses its libstdc++.
+- [`build.sh`](build.sh): concretizes and installs the environment in the
+ builder, then runs `spack buildcache push --base-image --tag `,
+ which produces the image: the base plus one layer per package, with `PATH` and
+ friends set for the root specs.
+- [`test-ci.sh`](test-ci.sh): checks clang, then runs in the image like spack-packages CI does:
+ concretizes a developer-tools stack (generate job) and builds part of it under
+ CI's padded install tree (build jobs).
+
+[`.github/workflows/linux-gnu.yml`](../.github/workflows/linux-gnu.yml) does all
+of this natively on x86_64 and aarch64 GitHub runners and publishes:
+
+- `ghcr.io//linux-gnu-base:`: the runtime base
+- `ghcr.io//linux-gnu-cache`: the OCI build cache. Packages are pushed as
+ soon as they are built (`autopush`), so a re-run after a timeout resumes.
+- `ghcr.io//linux-gnu-cache:-{amd64,arm64}` (`--tag` tags in the cache repository), and multi-arch
+ `ghcr.io//linux-gnu:` plus `:-`.
+
+Spack and spack-packages are pinned by commit in the workflow. Point them at
+forks to build with unmerged fixes.
diff --git a/linux-gnu/base.dockerfile b/linux-gnu/base.dockerfile
new file mode 100644
index 0000000..2181d0f
--- /dev/null
+++ b/linux-gnu/base.dockerfile
@@ -0,0 +1,20 @@
+# Runtime base of the linux-gnu CI image. Everything else in the image is
+# Spack-built and layered on top by `spack buildcache push --base-image`.
+#
+# AlmaLinux 8 fixes the glibc floor of whatever CI builds in this image at
+# 2.28; check-glibc.sh guards against RHEL-style symbol backports raising it.
+ARG BASE_IMAGE=docker.io/library/almalinux:8@sha256:2d0f2eb2a83cc09dbc2eb92fade4dbe8f709628149e81aa00aec70630406603b
+FROM ${BASE_IMAGE}
+
+# glibc-devel: headers, crt objects and linker scripts for the Spack-built gcc.
+# Do not upgrade glibc here: the floor must stay at the pinned base image's.
+RUN dnf install -y --setopt=install_weak_deps=False \
+ glibc-devel \
+ procps-ng \
+ which \
+ && dnf clean all \
+ && rm -rf /var/cache/dnf \
+ # Make packages link Spack's libxcrypt instead of the system libcrypt.
+ && rm -f /usr/lib64/libcrypt.so
+
+CMD ["/bin/bash"]
diff --git a/linux-gnu/build.sh b/linux-gnu/build.sh
new file mode 100755
index 0000000..9d4e63a
--- /dev/null
+++ b/linux-gnu/build.sh
@@ -0,0 +1,40 @@
+#!/bin/bash
+# Runs inside builder.dockerfile; see .github/workflows/linux-gnu.yml.
+#
+# Inputs (environment): REGISTRY, TAG, ARCH, TARGET, BASE, GITHUB_ACTOR, GITHUB_TOKEN.
+# Mounts: /spack, /spack-packages, /env (spack.yaml, writable), /out.
+set -euo pipefail
+
+export PATH=/spack/bin:$PATH
+export SPACK_DISABLE_LOCAL_CONFIG=1
+export SPACK_USER_CACHE_PATH=/tmp/spack-cache
+
+cache="oci://$REGISTRY/linux-gnu-cache"
+
+# Use the pinned spack-packages checkout instead of cloning builtin.
+spack -e /env config add "repos:builtin:/spack-packages/repos/spack_repo/builtin"
+spack -e /env repo list
+# Generic microarchitecture, not the runner's (e.g. neoverse_n2 / zen4).
+spack -e /env config add "packages:all:require:[target=$TARGET]"
+spack -e /env mirror add --unsigned --autopush \
+ --oci-username-variable GITHUB_ACTOR --oci-password-variable GITHUB_TOKEN \
+ cache "$cache"
+
+# Packages are autopushed as they are built; keep the index current even if
+# the build fails, so the cache can be browsed (installs do not need it).
+trap 'spack -e /env buildcache update-index cache || true' EXIT
+
+spack -e /env concretize --fresh 2>&1 | tee /out/concretize.txt
+targets=$(spack --color=never -e /env find -c --format '{target}' | grep -v '^==>' | awk '{print $NF}' | sort -u | xargs)
+echo "targets: $targets"
+test "$targets" = "$TARGET"
+spack -e /env find -c --show-full-compiler -L > /out/specs.txt || true
+
+# Installs pull what is already in the cache and push what they build.
+spack -e /env install --fail-fast -j "$(nproc)"
+
+# Tag one image with every root and its runtime closure, on top of the base. The
+# tag is created in the cache's repository: $REGISTRY/linux-gnu-cache:$TAG-$ARCH.
+spack -e /env buildcache push --unsigned --update-index --without-build-dependencies \
+ --base-image "$BASE-$ARCH" \
+ --tag "$TAG-$ARCH" cache
diff --git a/linux-gnu/builder.dockerfile b/linux-gnu/builder.dockerfile
new file mode 100644
index 0000000..f539a4b
--- /dev/null
+++ b/linux-gnu/builder.dockerfile
@@ -0,0 +1,28 @@
+# Build environment for linux-gnu/spack.yaml: the runtime base plus the system
+# toolchain that bootstraps the Spack-built gcc (spec group `compiler`).
+ARG BASE
+FROM ${BASE}
+
+# libstdc++-static (for gcc's bootstrap) is in powertools.
+RUN dnf install -y --setopt=install_weak_deps=False --enablerepo=powertools \
+ binutils \
+ bzip2 \
+ diffutils \
+ file \
+ findutils \
+ gcc \
+ gcc-c++ \
+ git \
+ gzip \
+ libstdc++-static \
+ make \
+ patch \
+ python3 \
+ tar \
+ unzip \
+ xz \
+ && dnf clean all \
+ && rm -rf /var/cache/dnf
+
+COPY check-glibc.sh /usr/local/bin/check-glibc.sh
+RUN /usr/local/bin/check-glibc.sh
diff --git a/linux-gnu/check-glibc.sh b/linux-gnu/check-glibc.sh
new file mode 100755
index 0000000..3e6545b
--- /dev/null
+++ b/linux-gnu/check-glibc.sh
@@ -0,0 +1,14 @@
+#!/bin/sh
+# Fail if the system libc defines a symbol version newer than its nominal
+# release. RHEL backports symbols under newer version nodes (e.g. GLIBC_2.35 in
+# RHEL 9.7's glibc 2.34, spack/spack#51888); binaries linked against those
+# would not run on other distros with the "same" glibc.
+set -eu
+
+expected="GLIBC_${EXPECTED_GLIBC:-2.28}"
+libc=$(ls /lib64/libc.so.6 /lib/*/libc.so.6 2>/dev/null | head -n1)
+
+newest=$(readelf -V "$libc" | grep -oE 'GLIBC_[0-9]+\.[0-9]+(\.[0-9]+)?' | sort -uV | tail -n1)
+
+echo "$libc: newest symbol version $newest (expected $expected)"
+test "$newest" = "$expected"
diff --git a/linux-gnu/spack.yaml b/linux-gnu/spack.yaml
new file mode 100644
index 0000000..4d4aeba
--- /dev/null
+++ b/linux-gnu/spack.yaml
@@ -0,0 +1,80 @@
+# Tools for Spack's GitLab CI on linux, built inside builder.dockerfile and
+# pushed as an image on top of base.dockerfile (see README.md).
+#
+# Everything except llvm, including gcc@14, is built with AlmaLinux 8's gcc 8.5:
+# the tools only run Spack and its helpers, and CI builds with gcc@14 and its
+# own gcc-runtime. Not building the tools with gcc@14 keeps them off the
+# critical path of the (slow) gcc build and avoids rebuilding all of them on gcc
+# bumps.
+#
+# llvm is built with gcc@14 in a second group: llvm writes `--gcc-install-dir`
+# of the compiler that built it into clang.cfg, and the image has no system gcc,
+# only Spack's gcc@14. Spack only builds with external or concrete compilers,
+# hence the group that `needs` the first one.
+spack:
+ view: false
+
+ specs:
+ - group: tools
+ specs:
+ # The compiler CI builds with.
+ - gcc@14 +binutils +profiled build_type=Release languages=c,c++,fortran
+ - binutils +ld +gas +plugins ~libiberty
+ # Spack's own dependencies and CI tooling.
+ - awscli-v2
+ - bzip2
+ - clingo-bootstrap@spack +optimized
+ - curl
+ - file
+ - git
+ - gmake
+ - gnupg
+ - gzip
+ - libtree
+ - lz4
+ - patch
+ - patchelf
+ - py-boto3
+ - python +optimizations
+ - tar
+ - unzip
+ - vim
+ - xz
+ - zstd
+ override:
+ packages:
+ c:
+ require: [gcc@8.5.0]
+ cxx:
+ require: [gcc@8.5.0]
+
+ # clang, for stacks that require `%c,cxx=llvm` (py-jaxlib, py-tensorflow).
+ - group: llvm
+ needs: [tools]
+ specs:
+ # +link_llvm_dylib: tools link libLLVM.so/libclang-cpp.so instead of each
+ # statically linking LLVM (3.4 GB compressed layer otherwise).
+ - llvm@23 +clang +lld +link_llvm_dylib ~lldb ~libomptarget ~offload ~python
+ override:
+ packages:
+ llvm:
+ require: ["%c,cxx=gcc@14"]
+
+ concretizer:
+ unify: true
+ reuse: false
+
+ config:
+ install_tree:
+ root: /opt/spack
+
+ # build.sh adds `packages:all:require:[target=...]` (x86_64_v3 or aarch64).
+ packages:
+ gcc:
+ externals:
+ - spec: gcc@8.5.0 languages:='c,c++'
+ prefix: /usr
+ extra_attributes:
+ compilers:
+ c: /usr/bin/gcc
+ cxx: /usr/bin/g++
diff --git a/linux-gnu/test-ci.sh b/linux-gnu/test-ci.sh
new file mode 100755
index 0000000..79eaa59
--- /dev/null
+++ b/linux-gnu/test-ci.sh
@@ -0,0 +1,103 @@
+#!/bin/bash
+# Smoke test of the linux-gnu image as a spack-packages GitLab CI image. Runs
+# inside the image; mimics the generate and build jobs of the developer-tools
+# stacks (.ci/gitlab/.gitlab-ci.yml, .ci/gitlab/configs/ci.yaml).
+#
+# Inputs (environment): PACKAGES_REPOSITORY, PACKAGES_COMMIT, SPACK_CI_TARGET,
+# STACK, BUILD_SPECS (space separated roots of the stack to build), ML_STACK
+# (optional: a stack that needs clang, only concretized).
+set -euo pipefail
+
+section() { echo; echo "### $*"; }
+
+section "Tools in the image"
+for exe in python3 gcc g++ gfortran clang clang++ ld.lld ld ar git curl gpg aws make tar xz zstd unzip patch patchelf file; do
+ printf '%-10s %s\n' "$exe" "$(command -v "$exe")"
+done
+gcc --version | head -n1
+python3 -c 'import clingo, sys; print("clingo", clingo.__version__, clingo.__file__)'
+python3 -c 'import boto3, botocore; print("boto3", boto3.__version__, "botocore", botocore.__version__)'
+# spack buildcache update-index uses put_object(IfMatch=...): botocore >= 1.35.70
+python3 -c 'import botocore.session as s; m = s.get_session().get_service_model("s3"); assert "IfMatch" in m.operation_model("PutObject").input_shape.members'
+aws --version
+gpg --version | head -n1
+git --version
+curl --version | head -n1
+
+workdir=$(mktemp -d)
+cd "$workdir"
+
+section "clang uses Spack's gcc@14 toolchain (there is no system gcc)"
+clang --version | head -n1
+clang++ -v -fsyntax-only -x c++ /dev/null 2>&1 | grep 'Selected GCC installation'
+printf '#include \nint main() { std::cout << "hello from clang++" << std::endl; }\n' > hello.cpp
+# Like gcc's specs file, clang++.cfg must rpath gcc@14's runtime libraries;
+# otherwise the loader picks AlmaLinux's older /lib64/libstdc++.so.6.
+cat "$(dirname "$(readlink -f "$(command -v clang++)")")/clang++.cfg"
+for cxx in g++ clang++; do
+ $cxx -O2 hello.cpp -o hello-$cxx
+ readelf -d hello-$cxx | grep -E 'RPATH|RUNPATH'
+ ./hello-$cxx
+done
+
+section "Check out spack-packages and the Spack commit its CI uses"
+git init -q packages
+git -C packages fetch -q --depth 1 "https://github.com/$PACKAGES_REPOSITORY.git" "$PACKAGES_COMMIT"
+git -C packages checkout -q FETCH_HEAD
+set -a; . packages/.ci/env; set +a
+git init -q spack
+git -C spack fetch -q --depth 1 "https://github.com/$SPACK_CHECKOUT_REPO.git" "$SPACK_CHECKOUT_VERSION"
+git -C spack checkout -q FETCH_HEAD
+
+export SPACK_DISABLE_LOCAL_CONFIG=1
+export SPACK_USER_CACHE_PATH="$workdir/tmp/_user_cache/"
+export SPACK_CI_TARGET
+. spack/share/spack/setup-env.sh
+spack --version
+spack arch
+
+stack="$workdir/packages/stacks/$STACK"
+
+section "Generate: concretize $STACK"
+spack env activate --without-view "$stack"
+spack repo list
+spack compiler find
+spack compiler list
+spack audit configs
+start=$(date +%s)
+spack -v concretize -j 4 > "$workdir/concretize.log" 2>&1 || { tail -n 60 "$workdir/concretize.log"; exit 1; }
+echo "concretized in $(( $(date +%s) - start ))s"
+tail -n 3 "$workdir/concretize.log"
+if [ -d "$SPACK_USER_CACHE_PATH/bootstrap/store" ]; then
+ echo "ERROR: Spack bootstrapped (clingo?) instead of using the image's tools:"
+ find "$SPACK_USER_CACHE_PATH/bootstrap/store" -maxdepth 3 -mindepth 3
+ exit 1
+fi
+echo "OK: nothing bootstrapped"
+echo "compilers used:"
+spack --color=never find -c --format '{compiler.name}@{compiler.version}' | grep -v '^==>' | sort | uniq -c
+
+section "Build: ${BUILD_SPECS}"
+# Like build jobs: padded install tree plus the relocation test projection.
+for spec in $BUILD_SPECS; do
+ hash=$(spack --color=never find -c --format '{hash}' "$spec" | awk '{print $NF}' | head -n1)
+ spack config add "config:install_tree:projections:$spec/$hash:'morepadding/{architecture.platform}-{architecture.target}/{name}-{version}-{hash}'"
+done
+spack install --fail-fast --no-check-signature $BUILD_SPECS
+spack find -lp $BUILD_SPECS
+spack env deactivate
+
+if [ -n "${ML_STACK:-}" ]; then
+ section "Generate: concretize $ML_STACK (needs %c,cxx=llvm)"
+ start=$(date +%s)
+ spack -e "$workdir/packages/stacks/$ML_STACK" concretize -j 4 > "$workdir/ml-concretize.log" 2>&1 \
+ || { tail -n 60 "$workdir/ml-concretize.log"; exit 1; }
+ echo "concretized in $(( $(date +%s) - start ))s"
+ spack --color=never -e "$workdir/packages/stacks/$ML_STACK" find -c --format '{name}@{version} %{compiler.name}@{compiler.version}' py-jaxlib py-tensorflow cuda | sort -u
+fi
+
+section "Outputs do not need a glibc newer than 2.28"
+newest=$(find /home/software/spack -type f \( -name '*.so*' -o -perm -u+x \) -exec sh -c 'file -b "$1" | grep -q ELF && objdump -T "$1" 2>/dev/null' _ {} \; \
+ | grep -oE 'GLIBC_[0-9]+\.[0-9]+(\.[0-9]+)?' | sort -uV | tail -n1)
+echo "newest GLIBC symbol version required: $newest"
+test "$(printf '%s\nGLIBC_2.28\n' "$newest" | sort -V | tail -n1)" = GLIBC_2.28