linux-gnu: CI image for x86_64_v3 and aarch64 built from a Spack environment

.github/workflows/linux-gnu.yml

@@ -0,0 +1,197 @@
+name: linux-gnu image
+
+# Builds linux-gnu/spack.yaml natively on x86_64 and aarch64 runners inside an
+# AlmaLinux 8 builder container. Every installed package is pushed to an OCI
+# build cache as soon as it is built (autopush), so a re-run of a timed-out or
+# failed job resumes where it left off. The image itself is assembled by
+# `spack buildcache push --base-image ... --tag ...`: the runtime base plus one
+# layer per Spack package, with PATH etc. set for the root specs.
+
+on:
+  workflow_dispatch:
+  push:
+    paths:
+    - linux-gnu/**
+    - .github/workflows/linux-gnu.yml
+  pull_request:
+    paths:
+    - linux-gnu/**
+    - .github/workflows/linux-gnu.yml
+
+concurrency:
+  group: linux-gnu-${{ github.ref }}
+  cancel-in-progress: true
+
+env:
+  # Pinned Spack and package repository. Point these at forks to build with
+  # unmerged fixes.
+  # haampie-llms/spack yaml-keep-trailing-colon-in-values: develop e05599c + YAML fix
+  SPACK_REPOSITORY: haampie-llms/spack
+  SPACK_COMMIT: 11b8cae1bab08a94c39d180d68cd469d5f3c08ac
+  # haampie-llms/spack-packages linux-gnu-ci-image: develop 8933fbb + version bumps
+  # + llvm: clang.cfg rpath, no libexec/llvm copy; py-libclang: allow newer llvm
+  PACKAGES_REPOSITORY: haampie-llms/spack-packages
+  PACKAGES_COMMIT: 78fee03288bd14eff22a6d8dd10a2a9c3efdd00d
+  REGISTRY: ghcr.io/${{ github.repository_owner }}
+
+jobs:
+  base:
+    runs-on: ${{ matrix.runner }}
+    strategy:
+      matrix:
+        include:
+        - {runner: ubuntu-24.04, arch: amd64}
+        - {runner: ubuntu-24.04-arm, arch: arm64}
+    permissions:
+      contents: read
+      packages: write
+    outputs:
+      tag: ${{ steps.tag.outputs.tag }}
+    steps:
+    - uses: actions/checkout@v7
+    - name: Tag
+      id: tag
+      # Same tag for both arches: the base inputs, not the build date.
+      run: echo "tag=$(cat linux-gnu/base.dockerfile | sha256sum | cut -c1-12)" >> "$GITHUB_OUTPUT"
+    - uses: docker/login-action@v4
+      with:
+        registry: ghcr.io
+        username: ${{ github.actor }}
+        password: ${{ secrets.GITHUB_TOKEN }}
+    - name: Build and push runtime base and builder
+      run: |
+        base=$REGISTRY/linux-gnu-base:${{ steps.tag.outputs.tag }}-${{ matrix.arch }}
+        docker build -t $base -f linux-gnu/base.dockerfile linux-gnu
+        docker build -t linux-gnu-builder --build-arg BASE=$base -f linux-gnu/builder.dockerfile linux-gnu
+        docker push $base
+
+  base-manifest:
+    needs: base
+    runs-on: ubuntu-24.04
+    permissions:
+      packages: write
+    steps:
+    - uses: docker/login-action@v4
+      with:
+        registry: ghcr.io
+        username: ${{ github.actor }}
+        password: ${{ secrets.GITHUB_TOKEN }}
+    - run: |
+        b=$REGISTRY/linux-gnu-base:${{ needs.base.outputs.tag }}
+        docker buildx imagetools create -t $b $b-amd64 $b-arm64
+
+  build:
+    needs: [base, base-manifest]
+    runs-on: ${{ matrix.runner }}
+    timeout-minutes: 355
+    strategy:
+      fail-fast: false
+      matrix:
+        include:
+        - {runner: ubuntu-24.04, arch: amd64, target: x86_64_v3}
+        - {runner: ubuntu-24.04-arm, arch: arm64, target: aarch64}
+    permissions:
+      contents: read
+      packages: write
+    env:
+      BASE: ghcr.io/${{ github.repository_owner }}/linux-gnu-base:${{ needs.base.outputs.tag }}
+    steps:
+    - uses: actions/checkout@v7
+
+    - name: Check out Spack and spack-packages
+      run: |
+        for r in "spack $SPACK_REPOSITORY $SPACK_COMMIT" "spack-packages $PACKAGES_REPOSITORY $PACKAGES_COMMIT"; do
+          set -- $r
+          git init -q $1
+          git -C $1 fetch -q --depth 1 https://github.com/$2.git $3
+          git -C $1 checkout -q FETCH_HEAD
+        done
+
+    - uses: docker/login-action@v4
+      with:
+        registry: ghcr.io
+        username: ${{ github.actor }}
+        password: ${{ secrets.GITHUB_TOKEN }}
+
+    - name: Builder image
+      run: |
+        docker pull $BASE-${{ matrix.arch }}
+        docker build -t linux-gnu-builder --build-arg BASE=$BASE-${{ matrix.arch }} \
+          -f linux-gnu/builder.dockerfile linux-gnu
+
+    - name: Concretize, install and push
+      env:
+        GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
+        TAG: ${{ github.event_name == 'pull_request' && format('pr{0}', github.event.number) || github.ref_name }}
+      run: |
+        mkdir -p env out
+        cp linux-gnu/spack.yaml env/
+        docker run --rm -e GITHUB_ACTOR -e GITHUB_TOKEN -e REGISTRY -e TAG \
+          -e ARCH=${{ matrix.arch }} -e TARGET=${{ matrix.target }} -e BASE \
+          -e SPACK_COLOR=always -e PYTHONUNBUFFERED=1 \
+          -v $PWD/spack:/spack -v $PWD/spack-packages:/spack-packages \
+          -v $PWD/env:/env -v $PWD/out:/out -v $PWD/linux-gnu:/linux-gnu:ro \
+          linux-gnu-builder /linux-gnu/build.sh
+
+    - uses: actions/upload-artifact@v4
+      if: always()
+      with:
+        name: linux-gnu-${{ matrix.arch }}
+        path: |
+          env/spack.yaml
+          env/spack.lock
+          out/
+
+  image:
+    needs: [base, build]
+    runs-on: ubuntu-24.04
+    permissions:
+      packages: write
+    steps:
+    - uses: docker/login-action@v4
+      with:
+        registry: ghcr.io
+        username: ${{ github.actor }}
+        password: ${{ secrets.GITHUB_TOKEN }}
+    - name: Multi-arch image
+      env:
+        TAG: ${{ github.event_name == 'pull_request' && format('pr{0}', github.event.number) || github.ref_name }}
+      run: |
+        # `spack buildcache push --tag` tags per-arch images in the cache repository
+        c=$REGISTRY/linux-gnu-cache
+        i=$REGISTRY/linux-gnu
+        day=$(date -u +%Y-%m-%d)
+        docker buildx imagetools create -t $i:$TAG -t $i:$day-${{ github.run_id }} $c:$TAG-amd64 $c:$TAG-arm64
+        docker buildx imagetools inspect $i:$TAG
+
+  # Is the image usable for spack-packages CI? Concretize a developer-tools
+  # stack the way generate jobs do and build a sample of it like build jobs.
+  test:
+    needs: build
+    runs-on: ${{ matrix.runner }}
+    strategy:
+      fail-fast: false
+      matrix:
+        include:
+        - {runner: ubuntu-24.04, arch: amd64, target: x86_64_v3, stack: developer-tools-x86_64_v3-linux-gnu, ml_stack: ml-linux-x86_64}
+        - {runner: ubuntu-24.04-arm, arch: arm64, target: aarch64, stack: developer-tools-aarch64-linux-gnu, ml_stack: ml-linux-aarch64}
+    permissions:
+      contents: read
+      packages: read
+    steps:
+    - uses: actions/checkout@v7
+    - uses: docker/login-action@v4
+      with:
+        registry: ghcr.io
+        username: ${{ github.actor }}
+        password: ${{ secrets.GITHUB_TOKEN }}
+    - name: Test ${{ matrix.stack }}
+      env:
+        TAG: ${{ github.event_name == 'pull_request' && format('pr{0}', github.event.number) || github.ref_name }}
+      run: |
+        docker run --rm -e PACKAGES_REPOSITORY -e PACKAGES_COMMIT \
+          -e SPACK_CI_TARGET=${{ matrix.target }} -e STACK=${{ matrix.stack }} -e ML_STACK=${{ matrix.ml_stack }} \
+          -e BUILD_SPECS="libtree tree jq ncdu tmux patchelf py-isort gmake" \
+          -e SPACK_COLOR=always -e PYTHONUNBUFFERED=1 \
+          -v $PWD/linux-gnu:/linux-gnu:ro \
+          $REGISTRY/linux-gnu-cache:$TAG-${{ matrix.arch }} /linux-gnu/test-ci.sh

linux-gnu/README.md

@@ -0,0 +1,33 @@
+Build image for Spack's GitLab CI on linux (x86_64_v3 and aarch64), made entirely
+from a Spack environment on top of a minimal AlmaLinux 8 base.
+
+- [`base.dockerfile`](base.dockerfile): AlmaLinux 8 (pinned by digest) plus
+  `glibc-devel`. glibc 2.28 is the floor of everything CI builds in this image;
+  [`check-glibc.sh`](check-glibc.sh) fails the build if the system libc exports
+  symbol versions newer than that (RHEL backports, see spack/spack#51888).
+- [`builder.dockerfile`](builder.dockerfile): the base plus AlmaLinux's gcc 8.5
+  and the bare minimum to run Spack.
+- [`spack.yaml`](spack.yaml): gcc@14 (the compiler CI builds with) and the tools
+  CI needs: Python with Spack's optimized clingo, boto3, AWS CLI v2, git, curl,
+  gnupg, compression tools, ... All built with AlmaLinux's gcc 8.5, except
+  llvm@23 (clang, for stacks that require `%c,cxx=llvm`), which a second spec
+  group builds with the Spack-built gcc@14 so that clang uses its libstdc++.
+- [`build.sh`](build.sh): concretizes and installs the environment in the
+  builder, then runs `spack buildcache push --base-image <base> --tag <tag>`,
+  which produces the image: the base plus one layer per package, with `PATH` and
+  friends set for the root specs.
+- [`test-ci.sh`](test-ci.sh): checks clang, then runs in the image like spack-packages CI does:
+  concretizes a developer-tools stack (generate job) and builds part of it under
+  CI's padded install tree (build jobs).
+
+[`.github/workflows/linux-gnu.yml`](../.github/workflows/linux-gnu.yml) does all
+of this natively on x86_64 and aarch64 GitHub runners and publishes:
+
+- `ghcr.io/<owner>/linux-gnu-base:<hash>`: the runtime base
+- `ghcr.io/<owner>/linux-gnu-cache`: the OCI build cache. Packages are pushed as
+  soon as they are built (`autopush`), so a re-run after a timeout resumes.
+- `ghcr.io/<owner>/linux-gnu-cache:<branch>-{amd64,arm64}` (`--tag` tags in the cache repository), and multi-arch
+  `ghcr.io/<owner>/linux-gnu:<branch>` plus `:<date>-<run id>`.
+
+Spack and spack-packages are pinned by commit in the workflow. Point them at
+forks to build with unmerged fixes.

linux-gnu/base.dockerfile

@@ -0,0 +1,20 @@
+# Runtime base of the linux-gnu CI image. Everything else in the image is
+# Spack-built and layered on top by `spack buildcache push --base-image`.
+#
+# AlmaLinux 8 fixes the glibc floor of whatever CI builds in this image at
+# 2.28; check-glibc.sh guards against RHEL-style symbol backports raising it.
+ARG BASE_IMAGE=docker.io/library/almalinux:8@sha256:2d0f2eb2a83cc09dbc2eb92fade4dbe8f709628149e81aa00aec70630406603b
+FROM ${BASE_IMAGE}
+
+# glibc-devel: headers, crt objects and linker scripts for the Spack-built gcc.
+# Do not upgrade glibc here: the floor must stay at the pinned base image's.
+RUN dnf install -y --setopt=install_weak_deps=False \
+      glibc-devel \
+      procps-ng \
+      which \
+ && dnf clean all \
+ && rm -rf /var/cache/dnf \
+ # Make packages link Spack's libxcrypt instead of the system libcrypt.
+ && rm -f /usr/lib64/libcrypt.so
+
+CMD ["/bin/bash"]

linux-gnu/build.sh

@@ -0,0 +1,40 @@
+#!/bin/bash
+# Runs inside builder.dockerfile; see .github/workflows/linux-gnu.yml.
+#
+# Inputs (environment): REGISTRY, TAG, ARCH, TARGET, BASE, GITHUB_ACTOR, GITHUB_TOKEN.
+# Mounts: /spack, /spack-packages, /env (spack.yaml, writable), /out.
+set -euo pipefail
+
+export PATH=/spack/bin:$PATH
+export SPACK_DISABLE_LOCAL_CONFIG=1
+export SPACK_USER_CACHE_PATH=/tmp/spack-cache
+
+cache="oci://$REGISTRY/linux-gnu-cache"
+
+# Use the pinned spack-packages checkout instead of cloning builtin.
+spack -e /env config add "repos:builtin:/spack-packages/repos/spack_repo/builtin"
+spack -e /env repo list
+# Generic microarchitecture, not the runner's (e.g. neoverse_n2 / zen4).
+spack -e /env config add "packages:all:require:[target=$TARGET]"
+spack -e /env mirror add --unsigned --autopush \
+  --oci-username-variable GITHUB_ACTOR --oci-password-variable GITHUB_TOKEN \
+  cache "$cache"
+
+# Packages are autopushed as they are built; keep the index current even if
+# the build fails, so the cache can be browsed (installs do not need it).
+trap 'spack -e /env buildcache update-index cache || true' EXIT
+
+spack -e /env concretize --fresh 2>&1 | tee /out/concretize.txt
+targets=$(spack --color=never -e /env find -c --format '{target}' | grep -v '^==>' | awk '{print $NF}' | sort -u | xargs)
+echo "targets: $targets"
+test "$targets" = "$TARGET"
+spack -e /env find -c --show-full-compiler -L > /out/specs.txt || true
+
+# Installs pull what is already in the cache and push what they build.
+spack -e /env install --fail-fast -j "$(nproc)"
+
+# Tag one image with every root and its runtime closure, on top of the base. The
+# tag is created in the cache's repository: $REGISTRY/linux-gnu-cache:$TAG-$ARCH.
+spack -e /env buildcache push --unsigned --update-index --without-build-dependencies \
+  --base-image "$BASE-$ARCH" \
+  --tag "$TAG-$ARCH" cache

linux-gnu/builder.dockerfile

@@ -0,0 +1,28 @@
+# Build environment for linux-gnu/spack.yaml: the runtime base plus the system
+# toolchain that bootstraps the Spack-built gcc (spec group `compiler`).
+ARG BASE
+FROM ${BASE}
+
+# libstdc++-static (for gcc's bootstrap) is in powertools.
+RUN dnf install -y --setopt=install_weak_deps=False --enablerepo=powertools \
+      binutils \
+      bzip2 \
+      diffutils \
+      file \
+      findutils \
+      gcc \
+      gcc-c++ \
+      git \
+      gzip \
+      libstdc++-static \
+      make \
+      patch \
+      python3 \
+      tar \
+      unzip \
+      xz \
+ && dnf clean all \
+ && rm -rf /var/cache/dnf
+
+COPY check-glibc.sh /usr/local/bin/check-glibc.sh
+RUN /usr/local/bin/check-glibc.sh

linux-gnu/check-glibc.sh

@@ -0,0 +1,14 @@
+#!/bin/sh
+# Fail if the system libc defines a symbol version newer than its nominal
+# release. RHEL backports symbols under newer version nodes (e.g. GLIBC_2.35 in
+# RHEL 9.7's glibc 2.34, spack/spack#51888); binaries linked against those
+# would not run on other distros with the "same" glibc.
+set -eu
+
+expected="GLIBC_${EXPECTED_GLIBC:-2.28}"
+libc=$(ls /lib64/libc.so.6 /lib/*/libc.so.6 2>/dev/null | head -n1)
+
+newest=$(readelf -V "$libc" | grep -oE 'GLIBC_[0-9]+\.[0-9]+(\.[0-9]+)?' | sort -uV | tail -n1)
+
+echo "$libc: newest symbol version $newest (expected $expected)"
+test "$newest" = "$expected"