Fix ci

.github/workflows/build.yml

@@ -99,7 +99,7 @@ jobs:
           sudo apt-get update
           sudo apt-get install -y --no-install-recommends \
             build-essential bc bison flex libssl-dev libelf-dev libdw-dev dwarves \
-            gcc-aarch64-linux-gnu ccache kmod rsync cpio \
+            gcc-aarch64-linux-gnu libc6-dev-arm64-cross ccache kmod rsync cpio \
             debhelper dpkg-dev zstd
           # DEBUG_INFO_BTF needs pahole >= 1.21
           pahole --version

.github/workflows/ci.yml

@@ -57,5 +57,42 @@ jobs:
       - name: Check ${{ matrix.target }} configuration
         env:
           CONFIG_ONLY: "1"
+          # The branch tip, not a pinned version: these gates check that our
+          # scripts still work against current rpi-6.18.y, and it skips the
+          # history fetch a version lookup would need.
+          KERNEL_REF: rpi-6.18.y
           GH_TOKEN: ${{ github.token }}
         run: ./scripts/build.sh ${{ matrix.target }}
+
+  # Builds a tiny kernel through the *complete* bindeb-pkg path in minutes, so
+  # packaging breakage is caught here instead of after an hour of compiling in
+  # build.yml. Deliberately uses the same apt list as the real build.
+  packaging:
+    runs-on: ubuntu-24.04
+    timeout-minutes: 40
+    steps:
+      - uses: actions/checkout@v4
+
+      - name: Install build dependencies
+        run: |
+          sudo apt-get update
+          sudo apt-get install -y --no-install-recommends \
+            build-essential bc bison flex libssl-dev libelf-dev libdw-dev dwarves \
+            gcc-aarch64-linux-gnu libc6-dev-arm64-cross ccache kmod rsync cpio \
+            debhelper dpkg-dev zstd
+
+      - name: Packaging smoke test
+        env:
+          KERNEL_REF: rpi-6.18.y
+          GH_TOKEN: ${{ github.token }}
+        run: ./scripts/build.sh smoke
+
+      - name: Summarise
+        if: always()
+        run: |
+          {
+            echo "### packaging smoke"
+            echo '```'
+            ls -lh build/dist-smoke/ 2>/dev/null || echo "no packages produced"
+            echo '```'
+          } >> "$GITHUB_STEP_SUMMARY"

README.md

@@ -52,10 +52,21 @@ merges the upstream stable releases into a rolling `rpi-<maj>.<min>.y` branch.
 [`scripts/resolve-kernel-ref.sh`](scripts/resolve-kernel-ref.sh) therefore turns
 `v6.18.40` into the newest commit on `rpi-6.18.y` whose `Makefile` still reports
 6.18.40 — i.e. the stable release plus any Pi fixes on top of it, and never the
-first commit of 6.18.41. It uses the GitHub API only, no clone, and takes a few
-seconds. The workflow resolves once in the `prepare` job so both boards build
-the identical commit, and `build.sh` re-asserts the version of the checked-out
-tree before configuring, so a tag can never quietly build something else.
+first commit of 6.18.41.
+
+If that version is still at the branch tip the tip is used directly. Otherwise
+the search runs over the branch's **first-parent** history, which is the part
+that is easy to get wrong: the branch merges upstream stable, so a plain history
+walk also sees `Linux 6.18.41` and its ancestors, which report the right version
+while containing none of the Pi patches — no `bcm2711_defconfig`, no Pi drivers.
+Every first-parent ancestor is a Pi-branch commit by construction, and the
+result is checked for `arch/arm64/configs/bcm2711_defconfig` regardless. The
+history is fetched commits-only (`--filter=tree:0`), so it costs a few hundred
+MB once rather than a full clone.
+
+The workflow resolves once in the `prepare` job so both boards build the
+identical commit, and `build.sh` re-asserts the version of the checked-out tree
+before configuring, so a tag can never quietly build something else.
 
 Two consequences worth knowing:
 
@@ -74,10 +85,16 @@ Locally the same build runs with:
 Local requirements (Debian/Ubuntu):
 
 ```sh
-sudo apt install gcc-aarch64-linux-gnu bc bison flex libssl-dev libelf-dev \
-                 dwarves debhelper dpkg-dev kmod cpio rsync zstd ccache
+sudo apt install gcc-aarch64-linux-gnu libc6-dev-arm64-cross bc bison flex \
+                 libssl-dev libelf-dev dwarves debhelper dpkg-dev kmod cpio \
+                 rsync zstd ccache
 ```
 
+`libc6-dev-arm64-cross` is easy to miss: kernel code is freestanding and never
+includes libc headers, but the `linux-headers` package rebuilds `fixdep` and
+`modpost` **for the target** so out-of-tree modules can be built on the Pi, and
+that compiles userspace code. `build.sh` probes for it up front.
+
 A `CONFIG_ONLY=1` run needs only the first six plus `dwarves`; the packaging
 tools are not checked for. Either way `build.sh` verifies what it needs in its
 first second and names anything missing.
@@ -102,6 +119,21 @@ pushes go to `build.yml` instead) and takes a few minutes:
   workflow file.
 - **config** — `CONFIG_ONLY=1 ./scripts/build.sh` for both boards, so a Kconfig
   option that silently stops applying is caught without compiling anything.
+- **packaging** — `./scripts/build.sh smoke`: a `tinyconfig` kernel taken
+  through the *complete* `bindeb-pkg` path in a few minutes, so packaging
+  breakage no longer waits an hour to appear.
+
+`smoke` is not an RT build and proves nothing about `rt.fragment` — it exists to
+exercise the packaging work: modules get installed, stripped, compressed and
+`depmod`-ed, the `-dbg` package is generated, and `linux-headers` cross-builds
+`fixdep`/`modpost`. It then checks the results: the image package carries
+`vmlinuz` and at least one `.ko`, and the headers package's `fixdep` is an
+**AArch64** binary — if it were not, out-of-tree module builds on the Pi would
+fail, and nothing else in the pipeline would notice.
+
+```sh
+./scripts/build.sh smoke     # ~7 min cold, ~2 min incremental
+```
 
 ## Installing on the Pi
 

config/smoke.fragment

@@ -0,0 +1,28 @@
+# Fragment for `./scripts/build.sh smoke`: the smallest kernel that still
+# exercises every part of the packaging path we depend on.
+#
+# Applied on top of tinyconfig, so this is NOT an RT configuration and says
+# nothing about config/rt.fragment - scripts/verify-config.sh covers that. What
+# it has to reproduce is the packaging work:
+#
+#   MODULES        module install, strip, xz and depmod inside linux-image
+#   DEBUG_INFO     makes mkdebian emit the -dbg package, as a real build does
+#   BTF            runs pahole over vmlinux and the modules
+#   a real module  so lib/modules/<release>/kernel actually contains a .ko
+#
+# The linux-headers package is built regardless, and with it the cross-compiled
+# fixdep/modpost that broke the first real release build.
+
+CONFIG_MODULES=y
+CONFIG_MODULE_UNLOAD=y
+
+CONFIG_NET=y
+CONFIG_NETDEVICES=y
+CONFIG_DUMMY=m
+
+CONFIG_BPF_SYSCALL=y
+CONFIG_DEBUG_KERNEL=y
+# CONFIG_DEBUG_INFO_NONE is not set
+CONFIG_DEBUG_INFO_DWARF_TOOLCHAIN_DEFAULT=y
+CONFIG_DEBUG_INFO_BTF=y
+CONFIG_DEBUG_INFO_BTF_MODULES=y

scripts/build.sh

@@ -29,25 +29,42 @@ REPO_ROOT="$(cd "$(dirname "${BASH_SOURCE[0]}")/.." && pwd)"
 
 TARGET="${1:-}"
 case "$TARGET" in
-pi4) DEFAULT_DEFCONFIG=bcm2711_defconfig ;; # BCM2711, 4K pages, CONFIG_LOCALVERSION=-v8
-pi5) DEFAULT_DEFCONFIG=bcm2712_defconfig ;; # BCM2712, 16K pages, CONFIG_LOCALVERSION=-v8-16k
+pi4) # BCM2711, 4K pages, CONFIG_LOCALVERSION=-v8
+	DEFAULT_DEFCONFIG=bcm2711_defconfig
+	FRAGMENT="$REPO_ROOT/config/rt.fragment"
+	DEFAULT_LOCALVERSION=-rt
+	;;
+pi5) # BCM2712, 16K pages, CONFIG_LOCALVERSION=-v8-16k
+	DEFAULT_DEFCONFIG=bcm2712_defconfig
+	FRAGMENT="$REPO_ROOT/config/rt.fragment"
+	DEFAULT_LOCALVERSION=-rt
+	;;
+smoke) # tiny kernel, same packaging path - see config/smoke.fragment
+	DEFAULT_DEFCONFIG=tinyconfig
+	FRAGMENT="$REPO_ROOT/config/smoke.fragment"
+	DEFAULT_LOCALVERSION=-smoke
+	;;
 *)
-	echo "usage: $0 {pi4|pi5}" >&2
+	echo "usage: $0 {pi4|pi5|smoke}" >&2
 	exit 2
 	;;
 esac
 
 KERNEL_REPO="${KERNEL_REPO:-https://github.com/raspberrypi/linux.git}"
 KERNEL_REF="${KERNEL_REF:-v6.18.40}"
 DEFCONFIG="${DEFCONFIG:-$DEFAULT_DEFCONFIG}"
-LOCALVERSION="${LOCALVERSION:--rt}"
+LOCALVERSION="${LOCALVERSION:-$DEFAULT_LOCALVERSION}"
 # dpkg-buildpackage is invoked with -a arm64, so dpkg-checkbuilddeps insists on
 # arm64 builds of libssl-dev/libdw-dev, which cannot be installed on an amd64
 # runner. Everything is compiled here with our own cross toolchain before the
 # packaging step runs, so the check is skipped rather than satisfied.
 DPKG_FLAGS="${DPKG_FLAGS:--d}"
 WORK_DIR="${WORK_DIR:-$REPO_ROOT/build}"
-DIST_DIR="${DIST_DIR:-$REPO_ROOT/dist}"
+if [ "$TARGET" = smoke ]; then
+	DIST_DIR="${DIST_DIR:-$REPO_ROOT/build/dist-smoke}"
+else
+	DIST_DIR="${DIST_DIR:-$REPO_ROOT/dist}"
+fi
 JOBS="${JOBS:-$(nproc)}"
 
 SRC="$WORK_DIR/linux"
@@ -118,6 +135,27 @@ if [ "$(printf '%s\n1.21\n' "$pahole_version" | sort -V | head -1)" != "1.21" ];
 fi
 echo "  toolchain ok (pahole $pahole_version${CROSS_COMPILE:+, cross $CROSS_COMPILE})"
 
+# The linux-headers package rebuilds fixdep and modpost for the *target* with
+# CC=<target>-gcc, so they can run on the Pi (see install_kernel_headers in
+# scripts/package/builddeb). That is the only thing here that compiles
+# userspace code for the target, so it is the only thing that needs the target
+# libc headers - kernel code is freestanding and never includes them. Without
+# this probe the mistake only surfaces in binary-headers, an hour in.
+if [ "${CONFIG_ONLY:-0}" != "1" ] && [ -n "$CROSS_COMPILE" ]; then
+	probe="$(mktemp -d)"
+	printf '#include <sys/types.h>\nint main(void){return 0;}\n' >"$probe/probe.c"
+	if ! "${CROSS_COMPILE}gcc" "$probe/probe.c" -o "$probe/probe" 2>"$probe/err"; then
+		echo "error: ${CROSS_COMPILE}gcc cannot compile a target userspace program:" >&2
+		sed 's/^/  /' "$probe/err" >&2
+		echo "the linux-headers package needs this; install the target libc headers," \
+			"e.g. apt install libc6-dev-arm64-cross" >&2
+		rm -rf "$probe"
+		exit 1
+	fi
+	rm -rf "$probe"
+	echo "  cross toolchain can build target userspace (linux-headers needs it)"
+fi
+
 # --- source -----------------------------------------------------------------
 # A version number is turned into the exact commit on the matching
 # rpi-<maj>.<min>.y branch; branches, tags and SHAs pass through untouched.
@@ -160,13 +198,25 @@ fi
 step "Kernel version: $KERNEL_VERSION ($KERNEL_SHA), package revision $KDEB_REVISION"
 
 # --- configure --------------------------------------------------------------
-step "Configuring ($DEFCONFIG + config/rt.fragment)"
+step "Configuring ($DEFCONFIG + ${FRAGMENT#"$REPO_ROOT"/})"
 mkdir -p "$OBJ"
 kmake "$DEFCONFIG"
-"$SRC/scripts/kconfig/merge_config.sh" -m -O "$OBJ" \
-	"$OBJ/.config" "$REPO_ROOT/config/rt.fragment"
+"$SRC/scripts/kconfig/merge_config.sh" -m -O "$OBJ" "$OBJ/.config" "$FRAGMENT"
 kmake olddefconfig
-"$REPO_ROOT/scripts/verify-config.sh" "$OBJ/.config"
+if [ "$TARGET" = smoke ]; then
+	# The RT assertions do not apply to a tiny kernel; only the options that
+	# make the packaging path representative have to survive.
+	for sym in CONFIG_MODULES CONFIG_DEBUG_INFO_BTF CONFIG_DEBUG_INFO_BTF_MODULES; do
+		grep -q "^$sym=y" "$OBJ/.config" ||
+			{
+				echo "error: smoke config lost $sym" >&2
+				exit 1
+			}
+	done
+	echo "  smoke config ok (modules + BTF, so module install and -dbg run)"
+else
+	"$REPO_ROOT/scripts/verify-config.sh" "$OBJ/.config"
+fi
 
 KERNELRELEASE="$(kmake -s kernelrelease | tail -1)"
 step "Kernel release: $KERNELRELEASE"
@@ -195,7 +245,11 @@ fi
 # -j1 and only inherits parallelism through make's jobserver, so doing the
 # heavy lifting here keeps the packaging step short whatever happens there.
 step "Building (-j$JOBS)"
-kmake -j"$JOBS" Image modules dtbs
+if [ "$TARGET" = smoke ]; then
+	kmake -j"$JOBS" Image modules # a tiny kernel has no device trees
+else
+	kmake -j"$JOBS" Image modules dtbs
+fi
 
 step "Packaging (bindeb-pkg)"
 rm -f "$WORK_DIR"/*.deb "$WORK_DIR"/*.buildinfo "$WORK_DIR"/*.changes
@@ -226,5 +280,47 @@ EOF
 
 (cd "$DIST_DIR" && sha256sum ./*.deb >"sha256sums-$TARGET.txt")
 
+# --- verify the packages ----------------------------------------------------
+# Cheap assertions on what the .debs actually contain. The headers check is the
+# one that matters: those host tools have to be target binaries or out-of-tree
+# module builds on the Pi cannot work, and getting that wrong is invisible until
+# someone tries it.
+step "Verifying packages"
+image_deb="$(ls "$DIST_DIR"/linux-image-*_"${KDEB_PKGVERSION}"_arm64.deb)"
+headers_deb="$(ls "$DIST_DIR"/linux-headers-*_"${KDEB_PKGVERSION}"_arm64.deb)"
+
+# Listed once into a variable: piping dpkg-deb into `grep -q` makes grep exit
+# early, and the resulting SIGPIPE trips `set -o pipefail`.
+image_list="$(dpkg-deb -c "$image_deb")"
+grep -q "boot/vmlinuz-$KERNELRELEASE" <<<"$image_list" ||
+	{
+		echo "error: $image_deb has no boot/vmlinuz-$KERNELRELEASE" >&2
+		exit 1
+	}
+grep -q "lib/modules/$KERNELRELEASE/kernel/.*\.ko" <<<"$image_list" ||
+	{
+		echo "error: $image_deb ships no kernel modules" >&2
+		exit 1
+	}
+echo "  linux-image: kernel and modules present"
+
+unpack="$(mktemp -d)"
+trap 'rm -rf "$unpack"' EXIT
+dpkg-deb -x "$headers_deb" "$unpack"
+fixdep="$unpack/usr/src/linux-headers-$KERNELRELEASE/scripts/basic/fixdep"
+[ -x "$fixdep" ] || {
+	echo "error: $headers_deb has no scripts/basic/fixdep" >&2
+	exit 1
+}
+fixdep_machine="$(readelf -h "$fixdep" | sed -n 's/^ *Machine: *//p')"
+case "$fixdep_machine" in
+*AArch64*) echo "  linux-headers: fixdep is $fixdep_machine, runs on the target" ;;
+*)
+	echo "error: $headers_deb ships a $fixdep_machine fixdep;" \
+		"out-of-tree module builds on the Pi would fail" >&2
+	exit 1
+	;;
+esac
+
 step "Done"
 ls -lh "$DIST_DIR"