CI: install apt packages from ghcr .deb bundles

.github/actions/install-apt-deps/action.yml

@@ -0,0 +1,181 @@
+name: 'Install apt dependencies'
+description: 'Install apt packages from a prebuilt ghcr .deb bundle, falling back to the apt mirror'
+# Ported from wolfSSL's .github/actions/install-apt-deps.
+inputs:
+  packages:
+    description: 'Space-separated list of apt packages to install'
+    required: true
+  ghcr-debs-tag:
+    description: >
+      Tag of a .deb bundle published to ghcr.io/wolfssl/wolfssh-ci-debs by
+      the ci-deps-image workflow (e.g. "ubuntu-24.04-tests"). The packages
+      are installed from that bundle with no network access. Any failure -
+      bundle missing, not public, or not covering every requested package -
+      falls back to the apt mirror, so it is always safe to set.
+    required: false
+    default: ''
+  retries:
+    description: 'Attempts at the apt mirror fallback'
+    required: false
+    default: '2'
+  budget-seconds:
+    description: >
+      Wall-clock for the whole mirror fallback, split across the attempts as
+      per-command deadlines so a wedged mirror is reported here instead of
+      the job being cancelled around it. It overshoots by retry-delay plus
+      10s of kill grace, and the per-command floors make values below
+      retries*80 inert. This plus pull-timeout has to fit the caller's
+      timeout-minutes.
+    required: false
+    default: '180'
+  pull-timeout:
+    description: 'Deadline in seconds for the ghcr bundle pull'
+    required: false
+    default: '60'
+  retry-delay:
+    description: 'Initial delay between retries (seconds, doubles each attempt)'
+    required: false
+    default: '5'
+  no-install-recommends:
+    description: 'Pass --no-install-recommends to apt-get install'
+    required: false
+    default: 'false'
+runs:
+  using: 'composite'
+  steps:
+    # Resolve against ONLY the bundle's own index, through a private
+    # sources.list and lists dir. The runner's lists are frozen into its image
+    # and lag the archive the bundle was built from, so resolving against them
+    # asks for versions the bundle does not carry. The system lists are left
+    # untouched for the fallback below.
+    - name: Install from the ghcr .deb bundle (offline)
+      id: ghcr
+      if: inputs.ghcr-debs-tag != ''
+      shell: bash
+      run: |
+        set -u
+        IMG="ghcr.io/wolfssl/wolfssh-ci-debs:${{ inputs.ghcr-debs-tag }}"
+
+        fallback() {
+          echo "::warning::$IMG: $1; falling back to the apt mirror"
+          echo "apt fallback: \`$IMG\`: $1" \
+            >> "${GITHUB_STEP_SUMMARY:-/dev/null}"
+          exit 0
+        }
+
+        # A bundle holds .debs for one Ubuntu release.
+        WANT=$(printf '%s' "${{ inputs.ghcr-debs-tag }}" \
+               | sed -n 's/^\(ubuntu-[0-9]*\.[0-9]*\).*/\1/p')
+        HAVE=""
+        if [ -r /etc/os-release ]; then
+          HAVE=$( . /etc/os-release
+                  printf '%s-%s' "${ID:-?}" "${VERSION_ID:-?}" )
+        fi
+        if [ -n "$WANT" ] && [ -n "$HAVE" ] && [ "$WANT" != "$HAVE" ]; then
+          fallback "holds .debs for $WANT, but this job runs on $HAVE"
+        fi
+        command -v docker >/dev/null 2>&1 || fallback "no docker CLI"
+
+        BUNDLE="$RUNNER_TEMP/ghcr-debs"
+        APTD="$RUNNER_TEMP/ghcr-apt"
+        rm -rf "$BUNDLE" "$APTD"
+        mkdir -p "$BUNDLE" "$APTD/etc" "$APTD/lists/partial"
+
+        # The image must be PUBLIC so an anonymous pull works from fork PRs.
+        timeout -k 10 ${{ inputs.pull-timeout }} docker pull -q "$IMG" \
+          >/dev/null 2>&1 || fallback "pull failed or timed out"
+        cid=$(docker create "$IMG" 2>/dev/null) || fallback "cannot open the image"
+        docker cp "$cid:/debs/." "$BUNDLE/" >/dev/null \
+          || fallback "unpacking failed"
+        docker rm "$cid" >/dev/null 2>&1 || true
+        ls "$BUNDLE"/*.deb >/dev/null 2>&1 || fallback "no .debs inside"
+
+        if [ -f "$BUNDLE/bundle-info" ]; then
+          echo "Bundle: $(tr '\n' ' ' < "$BUNDLE/bundle-info")"
+          echo "Runner: image ${ImageOS:-?} ${ImageVersion:-?}"
+        fi
+        [ -s "$BUNDLE/Packages" ] || fallback "no Packages index"
+
+        printf 'deb [trusted=yes] file:%s ./\n' "$BUNDLE" > "$APTD/etc/sources.list"
+        # The bundle is the only repository apt can see. The unroutable proxy
+        # is a tripwire: a non-file: URI fails at once instead of hanging on
+        # the mirror.
+        APT_LOCAL=(-o Dir::Etc::SourceList="$APTD/etc/sources.list"
+                   -o Dir::Etc::SourceParts=/dev/null
+                   -o Dir::State::Lists="$APTD/lists"
+                   -o Acquire::Languages=none
+                   -o Acquire::Retries=0
+                   -o APT::Sandbox::User=root
+                   -o Acquire::http::Proxy=http://127.0.0.1:9
+                   -o Acquire::https::Proxy=http://127.0.0.1:9)
+
+        sudo apt-get "${APT_LOCAL[@]}" update >/dev/null 2>&1 \
+          || fallback "apt could not read the bundle index"
+
+        NO_REC=""
+        if [ "${{ inputs.no-install-recommends }}" = "true" ]; then
+          NO_REC="--no-install-recommends"
+        fi
+        if sudo DEBIAN_FRONTEND=noninteractive apt-get "${APT_LOCAL[@]}" \
+             install -y $NO_REC ${{ inputs.packages }}; then
+          echo "satisfied=true" >> "$GITHUB_OUTPUT"
+          echo "Installed offline from $IMG"
+        else
+          fallback "does not cover ${{ inputs.packages }}"
+        fi
+
+    - name: Install packages
+      if: steps.ghcr.outputs.satisfied != 'true'
+      shell: bash
+      run: |
+        RETRIES=${{ inputs.retries }}
+        DELAY=${{ inputs.retry-delay }}
+        BUDGET=${{ inputs.budget-seconds }}
+        NO_REC=""
+        if [ "${{ inputs.no-install-recommends }}" = "true" ]; then
+          NO_REC="--no-install-recommends"
+        fi
+
+        # A wedged mirror hangs apt rather than failing it. apt drops a
+        # stalled connection after 30s and retries it, `timeout` kills an
+        # apt-get that wedged anyway, and the loop re-runs, re-reading
+        # apt-mirrors.txt so a retry can land on another mirror. apt resumes
+        # from archives/partial/, so a killed transfer is not restarted.
+
+        # No wolfSSH job installs from the runner's Google/Microsoft repos, and
+        # a bad index on either fails apt-get update. Drop them.
+        grep -rlE 'dl\.google\.com|packages\.microsoft\.com' \
+          /etc/apt/sources.list.d/ 2>/dev/null | xargs -r sudo rm -vf || true
+        APT_OPTS=(-o Acquire::Retries=3 -o Acquire::http::Timeout=30
+                  -o Acquire::https::Timeout=30)
+
+        DEADLINE=$(($(date +%s) + BUDGET))
+
+        # sudo resets the environment, so DEBIAN_FRONTEND rides along on each
+        # privileged command.
+        for i in $(seq 1 $RETRIES); do
+          # Split what is left over the attempts still to come. update gets
+          # half an attempt, capped at the 90s apt's own retries need.
+          PER=$(( (DEADLINE - $(date +%s)) / (RETRIES - i + 1) ))
+          UPD=$((PER / 2))
+          [ "$UPD" -le 90 ] || UPD=90
+          [ "$UPD" -ge 20 ] || UPD=20
+          INS=$((PER - UPD))
+          [ "$INS" -ge 40 ] || INS=40
+          # A previous attempt killed mid-unpack leaves dpkg needing this.
+          sudo dpkg --configure -a >/dev/null 2>&1 || true
+          if sudo DEBIAN_FRONTEND=noninteractive timeout -k 10 $UPD \
+                 apt-get "${APT_OPTS[@]}" update -q && \
+             sudo DEBIAN_FRONTEND=noninteractive timeout -k 10 $INS \
+                 apt-get "${APT_OPTS[@]}" install -y \
+                 $NO_REC ${{ inputs.packages }}; then
+            exit 0
+          fi
+          if [ "$i" -eq "$RETRIES" ] || [ "$(date +%s)" -ge "$DEADLINE" ]; then
+            echo "::error::apt-get failed after $i attempt(s) in ${BUDGET}s"
+            exit 1
+          fi
+          echo "::warning::apt-get failed (attempt $i/$RETRIES), retrying in ${DELAY}s..."
+          sleep $DELAY
+          DELAY=$((DELAY * 2))
+        done

.github/ci-deps/packages-ubuntu-24.04-compilers.txt

@@ -0,0 +1,15 @@
+# Compilers for multi-compiler, code-coverage and the mingw cross build,
+# bundled by ci-deps-image.yml. Kept apart from -tests for its size.
+autoconf
+automake
+clang-14
+clang-15
+clang-17
+clang-18
+gcc-11
+gcc-12
+gcc-13
+gcc-mingw-w64-x86-64
+libclang-rt-18-dev
+libtool
+llvm-18

.github/ci-deps/packages-ubuntu-24.04-tests.txt

@@ -0,0 +1,24 @@
+# Packages for the wolfSSH test workflows, bundled by ci-deps-image.yml.
+# A step's whole package list must be here or it falls back to the mirror.
+autoconf
+autoconf-archive
+automake
+build-essential
+cppcheck
+dosfstools
+expect
+git
+libcmocka-dev
+libglib2.0-dev
+libssl-dev
+libtool
+netcat-traditional
+openssh-client
+openssh-server
+pkg-config
+python3-pip
+sshpass
+tpm2-tools
+uthash-dev
+valgrind
+zlib1g-dev

.github/scripts/download-deb-closure.sh

@@ -0,0 +1,42 @@
+#!/usr/bin/env bash
+# Download the .deb closure for one package list into a directory, for
+# .github/workflows/ci-deps-image.yml to publish as a bundle. Ported from
+# wolfSSL's .github/scripts/download-deb-closure.sh.
+#
+# Runs as root on the runner. apt downloads only what is not already
+# installed, so the closure carries nothing the runner image preinstalls and
+# is only installable on that same runner image.
+#
+# usage: download-deb-closure.sh <package-list> <dest-dir>
+set -uo pipefail
+
+LIST=${1:?package list}
+DEST=${2:?destination directory}
+
+mapfile -t PKGS < <(grep -vE '^[[:space:]]*#|^[[:space:]]*$' "$LIST")
+echo "Packages (${#PKGS[@]}): ${PKGS[*]}"
+export DEBIAN_FRONTEND=noninteractive
+mkdir -p "$DEST" && rm -f "$DEST"/*.deb
+apt-get clean
+# No wolfSSH job installs from the runner's Google/Microsoft apt repos, and a
+# bad index on either fails apt-get update. Drop them.
+grep -rlE 'dl\.google\.com|packages\.microsoft\.com' \
+  /etc/apt/sources.list.d/ 2>/dev/null | xargs -r rm -vf || true
+# apt drops a stalled connection after 30s and retries it, `timeout` kills a
+# wedged apt-get, then retry() re-runs it.
+APT_OPTS=(-o Acquire::Retries=3 -o Acquire::http::Timeout=30
+          -o Acquire::https::Timeout=30)
+retry() { local i; for i in 1 2 3 4 5; do "$@" && return 0; sleep $((2**i)); done; "$@"; }
+retry timeout -k 10 120 apt-get "${APT_OPTS[@]}" update -q
+# One closure per package, so one unbundleable package cannot abort the rest;
+# install-apt-deps falls back to apt for anything missing.
+skipped=0
+for pkg in "${PKGS[@]}"; do
+  retry timeout -k 10 300 apt-get "${APT_OPTS[@]}" install -y --download-only "$pkg" \
+    || { echo "::warning::could not download $pkg"; skipped=$((skipped+1)); }
+done
+cp /var/cache/apt/archives/*.deb "$DEST/" 2>/dev/null || true
+# The index and image steps run unprivileged.
+chown --reference="$DEST" "$DEST"/*.deb 2>/dev/null || true
+echo "Bundled $(ls "$DEST"/*.deb 2>/dev/null | wc -l) .deb files ($(du -sh "$DEST" | cut -f1)); ${skipped} skipped"
+test -n "$(ls "$DEST"/*.deb 2>/dev/null)"  # fail if nothing was bundled

.github/workflows/ci-deps-image.yml

@@ -0,0 +1,88 @@
+name: CI deps image
+
+# Builds the apt .deb bundles that .github/actions/install-apt-deps installs
+# offline (input ghcr-debs-tag), so PR jobs stay off the apt mirror. Each
+# bundle holds the .debs for a package list in .github/ci-deps/ - every
+# package plus the dependencies not already on the runner image - and its own
+# apt index, published to ghcr.io/wolfssl/wolfssh-ci-debs:<tag>. Ported from
+# wolfSSL's ci-deps-image workflow.
+#
+# ONE-TIME SETUP: after the first successful run, make the package
+# `wolfssh-ci-debs` PUBLIC (org > Packages > Package settings > Change
+# visibility). Until then install-apt-deps falls back to apt.
+
+on:
+  schedule:
+    - cron: '0 2 * * *'
+  push:
+    branches: [ master ]
+    paths:
+      - '.github/ci-deps/**'
+      - '.github/scripts/download-deb-closure.sh'
+      - '.github/workflows/ci-deps-image.yml'
+  workflow_dispatch:
+
+concurrency:
+  group: ci-deps-image-${{ github.ref }}
+  cancel-in-progress: true
+
+permissions:
+  contents: read
+  packages: write
+
+jobs:
+  build:
+    name: build ${{ matrix.tag }}
+    if: github.repository_owner == 'wolfssl'
+    strategy:
+      fail-fast: false
+      matrix:
+        include:
+          # The .debs must be downloaded on the Ubuntu version that consumes
+          # them. -compilers is its own tag so the other jobs do not pull it.
+          - runner: ubuntu-24.04
+            tag: ubuntu-24.04-tests
+          - runner: ubuntu-24.04
+            tag: ubuntu-24.04-compilers
+    runs-on: ${{ matrix.runner }}
+    # Backstop only; the download script kills and retries stalled apt calls.
+    timeout-minutes: 60
+    steps:
+      - uses: actions/checkout@v6
+
+      - name: Resolve and download the .deb closure
+        run: |
+          # Created unprivileged so the later steps can write the index.
+          mkdir -p debs
+          sudo bash .github/scripts/download-deb-closure.sh \
+            ".github/ci-deps/packages-${{ matrix.tag }}.txt" debs
+
+      - name: Index the bundle
+        run: |
+          set -euo pipefail
+          # The consumer resolves against this index, not the runner's lists.
+          # dpkg-scanpackages is from dpkg-dev, preinstalled on the runners.
+          ( cd debs && dpkg-scanpackages --multiversion . /dev/null > Packages )
+          gzip -9kf debs/Packages
+          printf 'tag=%s\nrunner=%s %s\nbuilt=%s\ndebs=%s\n' \
+            "${{ matrix.tag }}" "${ImageOS:-?}" "${ImageVersion:-?}" \
+            "$(date -u +%Y-%m-%dT%H:%M:%SZ)" "$(ls debs/*.deb | wc -l)" \
+            > debs/bundle-info
+          cat debs/bundle-info
+
+      - name: Build bundle image
+        run: |
+          printf 'FROM busybox\nCOPY debs /debs\nLABEL org.opencontainers.image.source=https://github.com/wolfSSL/wolfssh\n' \
+            > Dockerfile.debs
+          docker build -f Dockerfile.debs -t bundle .
+
+      - name: Log in to ghcr
+        run: echo "${{ secrets.GITHUB_TOKEN }}" | docker login ghcr.io -u "${{ github.actor }}" --password-stdin
+
+      - name: Push to ghcr
+        run: |
+          set -euo pipefail
+          IMG="ghcr.io/wolfssl/wolfssh-ci-debs:${{ matrix.tag }}"
+          docker tag bundle "$IMG"
+          docker push "$IMG"
+          echo "Pushed $IMG"

.github/workflows/code-coverage.yml

@@ -56,9 +56,11 @@ jobs:
 
       # clang 18 is the min: -fcoverage-mcdc does not exist before it.
       - name: Install clang and LLVM coverage tools
-        run: |
-          sudo apt-get update
-          sudo apt-get install -y clang-18 llvm-18 libclang-rt-18-dev
+        uses: ./.github/actions/install-apt-deps
+        with:
+          packages: clang-18 llvm-18 libclang-rt-18-dev
+          ghcr-debs-tag: ubuntu-24.04-compilers
+          budget-seconds: 300
 
       - name: Download wolfSSL
         uses: actions/download-artifact@v8

.github/workflows/cppcheck.yml

@@ -16,7 +16,10 @@ jobs:
 
       - name: Install cppcheck
         if: always()
-        run: sudo apt-get install cppcheck
+        uses: ./.github/actions/install-apt-deps
+        with:
+          packages: cppcheck
+          ghcr-debs-tag: ubuntu-24.04-tests
 
       - name: Run CppCheck
         id: cpp_check_run