diff --git a/.github/workflows/linux-gnu.yml b/.github/workflows/linux-gnu.yml new file mode 100644 index 0000000..ecf4e4a --- /dev/null +++ b/.github/workflows/linux-gnu.yml @@ -0,0 +1,197 @@ +name: linux-gnu image + +# Builds linux-gnu/spack.yaml natively on x86_64 and aarch64 runners inside an +# AlmaLinux 8 builder container. Every installed package is pushed to an OCI +# build cache as soon as it is built (autopush), so a re-run of a timed-out or +# failed job resumes where it left off. The image itself is assembled by +# `spack buildcache push --base-image ... --tag ...`: the runtime base plus one +# layer per Spack package, with PATH etc. set for the root specs. + +on: + workflow_dispatch: + push: + paths: + - linux-gnu/** + - .github/workflows/linux-gnu.yml + pull_request: + paths: + - linux-gnu/** + - .github/workflows/linux-gnu.yml + +concurrency: + group: linux-gnu-${{ github.ref }} + cancel-in-progress: true + +env: + # Pinned Spack and package repository. Point these at forks to build with + # unmerged fixes. + # haampie-llms/spack yaml-keep-trailing-colon-in-values: develop e05599c + YAML fix + SPACK_REPOSITORY: haampie-llms/spack + SPACK_COMMIT: 11b8cae1bab08a94c39d180d68cd469d5f3c08ac + # haampie-llms/spack-packages linux-gnu-ci-image: develop 8933fbb + version bumps + # + llvm: clang.cfg rpath, no libexec/llvm copy; py-libclang: allow newer llvm + PACKAGES_REPOSITORY: haampie-llms/spack-packages + PACKAGES_COMMIT: 78fee03288bd14eff22a6d8dd10a2a9c3efdd00d + REGISTRY: ghcr.io/${{ github.repository_owner }} + +jobs: + base: + runs-on: ${{ matrix.runner }} + strategy: + matrix: + include: + - {runner: ubuntu-24.04, arch: amd64} + - {runner: ubuntu-24.04-arm, arch: arm64} + permissions: + contents: read + packages: write + outputs: + tag: ${{ steps.tag.outputs.tag }} + steps: + - uses: actions/checkout@v7 + - name: Tag + id: tag + # Same tag for both arches: the base inputs, not the build date. + run: echo "tag=$(cat linux-gnu/base.dockerfile | sha256sum | cut -c1-12)" >> "$GITHUB_OUTPUT" + - uses: docker/login-action@v4 + with: + registry: ghcr.io + username: ${{ github.actor }} + password: ${{ secrets.GITHUB_TOKEN }} + - name: Build and push runtime base and builder + run: | + base=$REGISTRY/linux-gnu-base:${{ steps.tag.outputs.tag }}-${{ matrix.arch }} + docker build -t $base -f linux-gnu/base.dockerfile linux-gnu + docker build -t linux-gnu-builder --build-arg BASE=$base -f linux-gnu/builder.dockerfile linux-gnu + docker push $base + + base-manifest: + needs: base + runs-on: ubuntu-24.04 + permissions: + packages: write + steps: + - uses: docker/login-action@v4 + with: + registry: ghcr.io + username: ${{ github.actor }} + password: ${{ secrets.GITHUB_TOKEN }} + - run: | + b=$REGISTRY/linux-gnu-base:${{ needs.base.outputs.tag }} + docker buildx imagetools create -t $b $b-amd64 $b-arm64 + + build: + needs: [base, base-manifest] + runs-on: ${{ matrix.runner }} + timeout-minutes: 355 + strategy: + fail-fast: false + matrix: + include: + - {runner: ubuntu-24.04, arch: amd64, target: x86_64_v3} + - {runner: ubuntu-24.04-arm, arch: arm64, target: aarch64} + permissions: + contents: read + packages: write + env: + BASE: ghcr.io/${{ github.repository_owner }}/linux-gnu-base:${{ needs.base.outputs.tag }} + steps: + - uses: actions/checkout@v7 + + - name: Check out Spack and spack-packages + run: | + for r in "spack $SPACK_REPOSITORY $SPACK_COMMIT" "spack-packages $PACKAGES_REPOSITORY $PACKAGES_COMMIT"; do + set -- $r + git init -q $1 + git -C $1 fetch -q --depth 1 https://github.com/$2.git $3 + git -C $1 checkout -q FETCH_HEAD + done + + - uses: docker/login-action@v4 + with: + registry: ghcr.io + username: ${{ github.actor }} + password: ${{ secrets.GITHUB_TOKEN }} + + - name: Builder image + run: | + docker pull $BASE-${{ matrix.arch }} + docker build -t linux-gnu-builder --build-arg BASE=$BASE-${{ matrix.arch }} \ + -f linux-gnu/builder.dockerfile linux-gnu + + - name: Concretize, install and push + env: + GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} + TAG: ${{ github.event_name == 'pull_request' && format('pr{0}', github.event.number) || github.ref_name }} + run: | + mkdir -p env out + cp linux-gnu/spack.yaml env/ + docker run --rm -e GITHUB_ACTOR -e GITHUB_TOKEN -e REGISTRY -e TAG \ + -e ARCH=${{ matrix.arch }} -e TARGET=${{ matrix.target }} -e BASE \ + -e SPACK_COLOR=always -e PYTHONUNBUFFERED=1 \ + -v $PWD/spack:/spack -v $PWD/spack-packages:/spack-packages \ + -v $PWD/env:/env -v $PWD/out:/out -v $PWD/linux-gnu:/linux-gnu:ro \ + linux-gnu-builder /linux-gnu/build.sh + + - uses: actions/upload-artifact@v4 + if: always() + with: + name: linux-gnu-${{ matrix.arch }} + path: | + env/spack.yaml + env/spack.lock + out/ + + image: + needs: [base, build] + runs-on: ubuntu-24.04 + permissions: + packages: write + steps: + - uses: docker/login-action@v4 + with: + registry: ghcr.io + username: ${{ github.actor }} + password: ${{ secrets.GITHUB_TOKEN }} + - name: Multi-arch image + env: + TAG: ${{ github.event_name == 'pull_request' && format('pr{0}', github.event.number) || github.ref_name }} + run: | + # `spack buildcache push --tag` tags per-arch images in the cache repository + c=$REGISTRY/linux-gnu-cache + i=$REGISTRY/linux-gnu + day=$(date -u +%Y-%m-%d) + docker buildx imagetools create -t $i:$TAG -t $i:$day-${{ github.run_id }} $c:$TAG-amd64 $c:$TAG-arm64 + docker buildx imagetools inspect $i:$TAG + + # Is the image usable for spack-packages CI? Concretize a developer-tools + # stack the way generate jobs do and build a sample of it like build jobs. + test: + needs: build + runs-on: ${{ matrix.runner }} + strategy: + fail-fast: false + matrix: + include: + - {runner: ubuntu-24.04, arch: amd64, target: x86_64_v3, stack: developer-tools-x86_64_v3-linux-gnu, ml_stack: ml-linux-x86_64} + - {runner: ubuntu-24.04-arm, arch: arm64, target: aarch64, stack: developer-tools-aarch64-linux-gnu, ml_stack: ml-linux-aarch64} + permissions: + contents: read + packages: read + steps: + - uses: actions/checkout@v7 + - uses: docker/login-action@v4 + with: + registry: ghcr.io + username: ${{ github.actor }} + password: ${{ secrets.GITHUB_TOKEN }} + - name: Test ${{ matrix.stack }} + env: + TAG: ${{ github.event_name == 'pull_request' && format('pr{0}', github.event.number) || github.ref_name }} + run: | + docker run --rm -e PACKAGES_REPOSITORY -e PACKAGES_COMMIT \ + -e SPACK_CI_TARGET=${{ matrix.target }} -e STACK=${{ matrix.stack }} -e ML_STACK=${{ matrix.ml_stack }} \ + -e BUILD_SPECS="libtree tree jq ncdu tmux patchelf py-isort gmake" \ + -e SPACK_COLOR=always -e PYTHONUNBUFFERED=1 \ + -v $PWD/linux-gnu:/linux-gnu:ro \ + $REGISTRY/linux-gnu-cache:$TAG-${{ matrix.arch }} /linux-gnu/test-ci.sh diff --git a/linux-gnu/README.md b/linux-gnu/README.md new file mode 100644 index 0000000..4dfd139 --- /dev/null +++ b/linux-gnu/README.md @@ -0,0 +1,33 @@ +Build image for Spack's GitLab CI on linux (x86_64_v3 and aarch64), made entirely +from a Spack environment on top of a minimal AlmaLinux 8 base. + +- [`base.dockerfile`](base.dockerfile): AlmaLinux 8 (pinned by digest) plus + `glibc-devel`. glibc 2.28 is the floor of everything CI builds in this image; + [`check-glibc.sh`](check-glibc.sh) fails the build if the system libc exports + symbol versions newer than that (RHEL backports, see spack/spack#51888). +- [`builder.dockerfile`](builder.dockerfile): the base plus AlmaLinux's gcc 8.5 + and the bare minimum to run Spack. +- [`spack.yaml`](spack.yaml): gcc@14 (the compiler CI builds with) and the tools + CI needs: Python with Spack's optimized clingo, boto3, AWS CLI v2, git, curl, + gnupg, compression tools, ... All built with AlmaLinux's gcc 8.5, except + llvm@23 (clang, for stacks that require `%c,cxx=llvm`), which a second spec + group builds with the Spack-built gcc@14 so that clang uses its libstdc++. +- [`build.sh`](build.sh): concretizes and installs the environment in the + builder, then runs `spack buildcache push --base-image --tag `, + which produces the image: the base plus one layer per package, with `PATH` and + friends set for the root specs. +- [`test-ci.sh`](test-ci.sh): checks clang, then runs in the image like spack-packages CI does: + concretizes a developer-tools stack (generate job) and builds part of it under + CI's padded install tree (build jobs). + +[`.github/workflows/linux-gnu.yml`](../.github/workflows/linux-gnu.yml) does all +of this natively on x86_64 and aarch64 GitHub runners and publishes: + +- `ghcr.io//linux-gnu-base:`: the runtime base +- `ghcr.io//linux-gnu-cache`: the OCI build cache. Packages are pushed as + soon as they are built (`autopush`), so a re-run after a timeout resumes. +- `ghcr.io//linux-gnu-cache:-{amd64,arm64}` (`--tag` tags in the cache repository), and multi-arch + `ghcr.io//linux-gnu:` plus `:-`. + +Spack and spack-packages are pinned by commit in the workflow. Point them at +forks to build with unmerged fixes. diff --git a/linux-gnu/base.dockerfile b/linux-gnu/base.dockerfile new file mode 100644 index 0000000..2181d0f --- /dev/null +++ b/linux-gnu/base.dockerfile @@ -0,0 +1,20 @@ +# Runtime base of the linux-gnu CI image. Everything else in the image is +# Spack-built and layered on top by `spack buildcache push --base-image`. +# +# AlmaLinux 8 fixes the glibc floor of whatever CI builds in this image at +# 2.28; check-glibc.sh guards against RHEL-style symbol backports raising it. +ARG BASE_IMAGE=docker.io/library/almalinux:8@sha256:2d0f2eb2a83cc09dbc2eb92fade4dbe8f709628149e81aa00aec70630406603b +FROM ${BASE_IMAGE} + +# glibc-devel: headers, crt objects and linker scripts for the Spack-built gcc. +# Do not upgrade glibc here: the floor must stay at the pinned base image's. +RUN dnf install -y --setopt=install_weak_deps=False \ + glibc-devel \ + procps-ng \ + which \ + && dnf clean all \ + && rm -rf /var/cache/dnf \ + # Make packages link Spack's libxcrypt instead of the system libcrypt. + && rm -f /usr/lib64/libcrypt.so + +CMD ["/bin/bash"] diff --git a/linux-gnu/build.sh b/linux-gnu/build.sh new file mode 100755 index 0000000..9d4e63a --- /dev/null +++ b/linux-gnu/build.sh @@ -0,0 +1,40 @@ +#!/bin/bash +# Runs inside builder.dockerfile; see .github/workflows/linux-gnu.yml. +# +# Inputs (environment): REGISTRY, TAG, ARCH, TARGET, BASE, GITHUB_ACTOR, GITHUB_TOKEN. +# Mounts: /spack, /spack-packages, /env (spack.yaml, writable), /out. +set -euo pipefail + +export PATH=/spack/bin:$PATH +export SPACK_DISABLE_LOCAL_CONFIG=1 +export SPACK_USER_CACHE_PATH=/tmp/spack-cache + +cache="oci://$REGISTRY/linux-gnu-cache" + +# Use the pinned spack-packages checkout instead of cloning builtin. +spack -e /env config add "repos:builtin:/spack-packages/repos/spack_repo/builtin" +spack -e /env repo list +# Generic microarchitecture, not the runner's (e.g. neoverse_n2 / zen4). +spack -e /env config add "packages:all:require:[target=$TARGET]" +spack -e /env mirror add --unsigned --autopush \ + --oci-username-variable GITHUB_ACTOR --oci-password-variable GITHUB_TOKEN \ + cache "$cache" + +# Packages are autopushed as they are built; keep the index current even if +# the build fails, so the cache can be browsed (installs do not need it). +trap 'spack -e /env buildcache update-index cache || true' EXIT + +spack -e /env concretize --fresh 2>&1 | tee /out/concretize.txt +targets=$(spack --color=never -e /env find -c --format '{target}' | grep -v '^==>' | awk '{print $NF}' | sort -u | xargs) +echo "targets: $targets" +test "$targets" = "$TARGET" +spack -e /env find -c --show-full-compiler -L > /out/specs.txt || true + +# Installs pull what is already in the cache and push what they build. +spack -e /env install --fail-fast -j "$(nproc)" + +# Tag one image with every root and its runtime closure, on top of the base. The +# tag is created in the cache's repository: $REGISTRY/linux-gnu-cache:$TAG-$ARCH. +spack -e /env buildcache push --unsigned --update-index --without-build-dependencies \ + --base-image "$BASE-$ARCH" \ + --tag "$TAG-$ARCH" cache diff --git a/linux-gnu/builder.dockerfile b/linux-gnu/builder.dockerfile new file mode 100644 index 0000000..f539a4b --- /dev/null +++ b/linux-gnu/builder.dockerfile @@ -0,0 +1,28 @@ +# Build environment for linux-gnu/spack.yaml: the runtime base plus the system +# toolchain that bootstraps the Spack-built gcc (spec group `compiler`). +ARG BASE +FROM ${BASE} + +# libstdc++-static (for gcc's bootstrap) is in powertools. +RUN dnf install -y --setopt=install_weak_deps=False --enablerepo=powertools \ + binutils \ + bzip2 \ + diffutils \ + file \ + findutils \ + gcc \ + gcc-c++ \ + git \ + gzip \ + libstdc++-static \ + make \ + patch \ + python3 \ + tar \ + unzip \ + xz \ + && dnf clean all \ + && rm -rf /var/cache/dnf + +COPY check-glibc.sh /usr/local/bin/check-glibc.sh +RUN /usr/local/bin/check-glibc.sh diff --git a/linux-gnu/check-glibc.sh b/linux-gnu/check-glibc.sh new file mode 100755 index 0000000..3e6545b --- /dev/null +++ b/linux-gnu/check-glibc.sh @@ -0,0 +1,14 @@ +#!/bin/sh +# Fail if the system libc defines a symbol version newer than its nominal +# release. RHEL backports symbols under newer version nodes (e.g. GLIBC_2.35 in +# RHEL 9.7's glibc 2.34, spack/spack#51888); binaries linked against those +# would not run on other distros with the "same" glibc. +set -eu + +expected="GLIBC_${EXPECTED_GLIBC:-2.28}" +libc=$(ls /lib64/libc.so.6 /lib/*/libc.so.6 2>/dev/null | head -n1) + +newest=$(readelf -V "$libc" | grep -oE 'GLIBC_[0-9]+\.[0-9]+(\.[0-9]+)?' | sort -uV | tail -n1) + +echo "$libc: newest symbol version $newest (expected $expected)" +test "$newest" = "$expected" diff --git a/linux-gnu/spack.yaml b/linux-gnu/spack.yaml new file mode 100644 index 0000000..4d4aeba --- /dev/null +++ b/linux-gnu/spack.yaml @@ -0,0 +1,80 @@ +# Tools for Spack's GitLab CI on linux, built inside builder.dockerfile and +# pushed as an image on top of base.dockerfile (see README.md). +# +# Everything except llvm, including gcc@14, is built with AlmaLinux 8's gcc 8.5: +# the tools only run Spack and its helpers, and CI builds with gcc@14 and its +# own gcc-runtime. Not building the tools with gcc@14 keeps them off the +# critical path of the (slow) gcc build and avoids rebuilding all of them on gcc +# bumps. +# +# llvm is built with gcc@14 in a second group: llvm writes `--gcc-install-dir` +# of the compiler that built it into clang.cfg, and the image has no system gcc, +# only Spack's gcc@14. Spack only builds with external or concrete compilers, +# hence the group that `needs` the first one. +spack: + view: false + + specs: + - group: tools + specs: + # The compiler CI builds with. + - gcc@14 +binutils +profiled build_type=Release languages=c,c++,fortran + - binutils +ld +gas +plugins ~libiberty + # Spack's own dependencies and CI tooling. + - awscli-v2 + - bzip2 + - clingo-bootstrap@spack +optimized + - curl + - file + - git + - gmake + - gnupg + - gzip + - libtree + - lz4 + - patch + - patchelf + - py-boto3 + - python +optimizations + - tar + - unzip + - vim + - xz + - zstd + override: + packages: + c: + require: [gcc@8.5.0] + cxx: + require: [gcc@8.5.0] + + # clang, for stacks that require `%c,cxx=llvm` (py-jaxlib, py-tensorflow). + - group: llvm + needs: [tools] + specs: + # +link_llvm_dylib: tools link libLLVM.so/libclang-cpp.so instead of each + # statically linking LLVM (3.4 GB compressed layer otherwise). + - llvm@23 +clang +lld +link_llvm_dylib ~lldb ~libomptarget ~offload ~python + override: + packages: + llvm: + require: ["%c,cxx=gcc@14"] + + concretizer: + unify: true + reuse: false + + config: + install_tree: + root: /opt/spack + + # build.sh adds `packages:all:require:[target=...]` (x86_64_v3 or aarch64). + packages: + gcc: + externals: + - spec: gcc@8.5.0 languages:='c,c++' + prefix: /usr + extra_attributes: + compilers: + c: /usr/bin/gcc + cxx: /usr/bin/g++ diff --git a/linux-gnu/test-ci.sh b/linux-gnu/test-ci.sh new file mode 100755 index 0000000..79eaa59 --- /dev/null +++ b/linux-gnu/test-ci.sh @@ -0,0 +1,103 @@ +#!/bin/bash +# Smoke test of the linux-gnu image as a spack-packages GitLab CI image. Runs +# inside the image; mimics the generate and build jobs of the developer-tools +# stacks (.ci/gitlab/.gitlab-ci.yml, .ci/gitlab/configs/ci.yaml). +# +# Inputs (environment): PACKAGES_REPOSITORY, PACKAGES_COMMIT, SPACK_CI_TARGET, +# STACK, BUILD_SPECS (space separated roots of the stack to build), ML_STACK +# (optional: a stack that needs clang, only concretized). +set -euo pipefail + +section() { echo; echo "### $*"; } + +section "Tools in the image" +for exe in python3 gcc g++ gfortran clang clang++ ld.lld ld ar git curl gpg aws make tar xz zstd unzip patch patchelf file; do + printf '%-10s %s\n' "$exe" "$(command -v "$exe")" +done +gcc --version | head -n1 +python3 -c 'import clingo, sys; print("clingo", clingo.__version__, clingo.__file__)' +python3 -c 'import boto3, botocore; print("boto3", boto3.__version__, "botocore", botocore.__version__)' +# spack buildcache update-index uses put_object(IfMatch=...): botocore >= 1.35.70 +python3 -c 'import botocore.session as s; m = s.get_session().get_service_model("s3"); assert "IfMatch" in m.operation_model("PutObject").input_shape.members' +aws --version +gpg --version | head -n1 +git --version +curl --version | head -n1 + +workdir=$(mktemp -d) +cd "$workdir" + +section "clang uses Spack's gcc@14 toolchain (there is no system gcc)" +clang --version | head -n1 +clang++ -v -fsyntax-only -x c++ /dev/null 2>&1 | grep 'Selected GCC installation' +printf '#include \nint main() { std::cout << "hello from clang++" << std::endl; }\n' > hello.cpp +# Like gcc's specs file, clang++.cfg must rpath gcc@14's runtime libraries; +# otherwise the loader picks AlmaLinux's older /lib64/libstdc++.so.6. +cat "$(dirname "$(readlink -f "$(command -v clang++)")")/clang++.cfg" +for cxx in g++ clang++; do + $cxx -O2 hello.cpp -o hello-$cxx + readelf -d hello-$cxx | grep -E 'RPATH|RUNPATH' + ./hello-$cxx +done + +section "Check out spack-packages and the Spack commit its CI uses" +git init -q packages +git -C packages fetch -q --depth 1 "https://github.com/$PACKAGES_REPOSITORY.git" "$PACKAGES_COMMIT" +git -C packages checkout -q FETCH_HEAD +set -a; . packages/.ci/env; set +a +git init -q spack +git -C spack fetch -q --depth 1 "https://github.com/$SPACK_CHECKOUT_REPO.git" "$SPACK_CHECKOUT_VERSION" +git -C spack checkout -q FETCH_HEAD + +export SPACK_DISABLE_LOCAL_CONFIG=1 +export SPACK_USER_CACHE_PATH="$workdir/tmp/_user_cache/" +export SPACK_CI_TARGET +. spack/share/spack/setup-env.sh +spack --version +spack arch + +stack="$workdir/packages/stacks/$STACK" + +section "Generate: concretize $STACK" +spack env activate --without-view "$stack" +spack repo list +spack compiler find +spack compiler list +spack audit configs +start=$(date +%s) +spack -v concretize -j 4 > "$workdir/concretize.log" 2>&1 || { tail -n 60 "$workdir/concretize.log"; exit 1; } +echo "concretized in $(( $(date +%s) - start ))s" +tail -n 3 "$workdir/concretize.log" +if [ -d "$SPACK_USER_CACHE_PATH/bootstrap/store" ]; then + echo "ERROR: Spack bootstrapped (clingo?) instead of using the image's tools:" + find "$SPACK_USER_CACHE_PATH/bootstrap/store" -maxdepth 3 -mindepth 3 + exit 1 +fi +echo "OK: nothing bootstrapped" +echo "compilers used:" +spack --color=never find -c --format '{compiler.name}@{compiler.version}' | grep -v '^==>' | sort | uniq -c + +section "Build: ${BUILD_SPECS}" +# Like build jobs: padded install tree plus the relocation test projection. +for spec in $BUILD_SPECS; do + hash=$(spack --color=never find -c --format '{hash}' "$spec" | awk '{print $NF}' | head -n1) + spack config add "config:install_tree:projections:$spec/$hash:'morepadding/{architecture.platform}-{architecture.target}/{name}-{version}-{hash}'" +done +spack install --fail-fast --no-check-signature $BUILD_SPECS +spack find -lp $BUILD_SPECS +spack env deactivate + +if [ -n "${ML_STACK:-}" ]; then + section "Generate: concretize $ML_STACK (needs %c,cxx=llvm)" + start=$(date +%s) + spack -e "$workdir/packages/stacks/$ML_STACK" concretize -j 4 > "$workdir/ml-concretize.log" 2>&1 \ + || { tail -n 60 "$workdir/ml-concretize.log"; exit 1; } + echo "concretized in $(( $(date +%s) - start ))s" + spack --color=never -e "$workdir/packages/stacks/$ML_STACK" find -c --format '{name}@{version} %{compiler.name}@{compiler.version}' py-jaxlib py-tensorflow cuda | sort -u +fi + +section "Outputs do not need a glibc newer than 2.28" +newest=$(find /home/software/spack -type f \( -name '*.so*' -o -perm -u+x \) -exec sh -c 'file -b "$1" | grep -q ELF && objdump -T "$1" 2>/dev/null' _ {} \; \ + | grep -oE 'GLIBC_[0-9]+\.[0-9]+(\.[0-9]+)?' | sort -uV | tail -n1) +echo "newest GLIBC symbol version required: $newest" +test "$(printf '%s\nGLIBC_2.28\n' "$newest" | sort -V | tail -n1)" = GLIBC_2.28