From ba6ceb427ff2a8eaa5fef172c8d69eb9f2964e5f Mon Sep 17 00:00:00 2001 From: Julian Goldstein Date: Fri, 19 Jun 2026 22:04:03 -0500 Subject: [PATCH] toolchain: sync vendored v0.10.0 Single non-merge commit (archive replace, not subtree pull) so the update stays rebase-mergeable. git-subtree-dir: toolchain git-subtree-split: 6a4c4597d1042c631beb181c76723701a19fde69 --- template/build/fetch-toolchain.sh | 1 + template/build/toolchain.lock | 16 +++++++--- template/build/toolchain.mk | 6 ++-- toolchain/.github/workflows/vendor.yml | 33 ++++++++++++-------- toolchain/README.md | 5 +-- toolchain/build/fetch-gh.sh | 42 ++++++++++++++++++++++++++ toolchain/build/versions.env | 16 +++++++--- toolchain/embed/fetch-toolchain.sh | 1 + toolchain/embed/toolchain.mk | 6 ++-- 9 files changed, 100 insertions(+), 26 deletions(-) create mode 100755 toolchain/build/fetch-gh.sh diff --git a/template/build/fetch-toolchain.sh b/template/build/fetch-toolchain.sh index 824b3f9..eff6a87 100755 --- a/template/build/fetch-toolchain.sh +++ b/template/build/fetch-toolchain.sh @@ -80,6 +80,7 @@ fetch_bin bpftool fetch_bin veristat fetch_bin esbuild fetch_bin git +fetch_bin gh # libbpf program headers: arch-independent, one copy per version, beside the # per-arch tool dirs ($key/include/bpf/*.h). diff --git a/template/build/toolchain.lock b/template/build/toolchain.lock index 07321f9..ac204f1 100644 --- a/template/build/toolchain.lock +++ b/template/build/toolchain.lock @@ -7,7 +7,7 @@ # sets it to the next semver tag it computes from the [bump:*] commit markers # (default minor; see the README). A release is cut only when build/ or the # vendor workflow changes. -TOOLCHAIN_VERSION=0.8.0 +TOOLCHAIN_VERSION=0.10.0 # Where a consumer fetches assets from: the version-tagged release. The fetch # appends `/v/-`. @@ -29,7 +29,7 @@ CLANG_SHA256_aarch64=105fb2d324095127a127bcba56702a86229ed8892ce6429802d30585fa2 BPFTOOL_VERSION=7.7.0 BPFTOOL_SHA256_x86_64=74bd16335aa1c40714fb50287a42766c6faa4958f969cce32fef89485ce4934c BPFTOOL_SHA256_aarch64=2b3fc4dd5e4e40bd8d670c5f1fa9693b3f879a122c0f6a3eb806dfc6735da6b7 -LIBBPF_HEADERS_SHA256=dba9a3583076fce1ab2739b6edba1460f64327accda449202892108e9d1a3c65 +LIBBPF_HEADERS_SHA256=d712858662168e4e04cdc35e4a962a6056c429ed57461b974ec59b819c60e3c3 # veristat — BPF verifier statistics tool, used to check that built `*.bpf.o` # programs load and to track verifier complexity. Official static binary from @@ -61,6 +61,14 @@ ESBUILD_VERSION=0.28.1 ESBUILD_SHA256_x86_64=0c6588b092a2c291a72bab90659f3c9e0e25e0fe59c9ac12b4dae4d945e5548c ESBUILD_SHA256_aarch64=51e829ba36f36be6d9aea6e329ddc4f9350302339b16aaca96a3cb97f64a8ebb +# gh — the GitHub CLI, for release/PR automation (CI and consumer tooling). +# Official static (Go) binary from cli/cli, re-hosted on our "toolchain" +# release like esbuild. CI records the per-arch binary checksum (consumers +# verify the binary, not the upstream tarball). +GH_VERSION=2.95.0 +GH_SHA256_x86_64=62c11fbaa08835168c3d1acf8a645ac6268a13a5682c73581388c9df0c622617 +GH_SHA256_aarch64=706b030846db08d0dac0392d6f74416e0bbf0d9ac5ecbd54c9046305c10eac59 + # qemu — NOT part of the build toolchain: a fully-static qemu-system for the # optional kernel-matrix test runner (boots lvh kernel images). Built per-arch # from source (see Dockerfile.qemu / build-qemu.sh), trimmed to the binary plus @@ -81,8 +89,8 @@ LVH_SHA256_x86_64=ca3b958ffc08a4b65e4c6c8d29a4dad077acd11f146bc84f75a30ecaa031e9 LVH_SHA256_aarch64=49b7cef6376c445c0ab23465ecc984a771e06b3b37c39b486a4fed7fb1ded476 # Per-arch checksum of the published qemu-.tar.gz (binary + minimal # share/qemu blobs). CI records these; the matrix runner verifies the tarball. -QEMU_SHA256_x86_64=0add524e0dd366ead34332aed0dafb4f11bc75f17884896aedb91faa28176dc2 -QEMU_SHA256_aarch64=be362492c2054c734bac78a776f17c0e0511f317d1dd8c924e82e00f8a7b2137 +QEMU_SHA256_x86_64=2e26722e965018c145bab180cee8feace50f91841bf51b350f14527c0d5da637 +QEMU_SHA256_aarch64=6fe353ef43918c14088c7a3741bfac54e6c17084ea462454a5b2a525ba0aa330 # Alpine base used for the musl-static clang build. ALPINE_TAG=alpine:3.21 diff --git a/template/build/toolchain.mk b/template/build/toolchain.mk index 52698b3..6cb5f18 100644 --- a/template/build/toolchain.mk +++ b/template/build/toolchain.mk @@ -1,4 +1,4 @@ -# Resolve the static build toolchain (clang, bpftool, veristat, esbuild, git) — included +# Resolve the static build toolchain (clang, bpftool, veristat, esbuild, git, gh) — included # by the project Makefile before build/bpf.mk, so the tools are set before any # rule uses them. A `make CLANG=…` CLI override beats this. # @@ -11,7 +11,7 @@ # Falls back to host tools on PATH when no lock is present. # # The vendored binaries are Linux musl-static, so the cache is only used on -# Linux. On any other host (macOS, BSD) we leave CLANG/BPFTOOL/ESBUILD/GIT +# Linux. On any other host (macOS, BSD) we leave CLANG/BPFTOOL/ESBUILD/GIT/GH # unset, falling through to host tools on PATH — macOS is an edit-here, # build-on-Linux host for BPF work. @@ -36,6 +36,7 @@ ifneq ($(TOOLCHAIN_LOCK),) VERISTAT ?= $(TOOLCHAIN_DIR)/veristat ESBUILD ?= $(TOOLCHAIN_DIR)/esbuild GIT ?= $(TOOLCHAIN_DIR)/git + GH ?= $(TOOLCHAIN_DIR)/gh # libbpf program headers are arch-independent: one copy per version key, # beside the per-arch tool dirs. BPF_SYSINCLUDE ?= $(YEET_CACHE_DIR)/toolchain/$(TOOLCHAIN_KEY)/include @@ -45,6 +46,7 @@ endif VERISTAT ?= veristat ESBUILD ?= esbuild GIT ?= git +GH ?= gh # Fill the cache for this arch, downloading any missing tool once. A no-op # when no lock is present (PATH case). diff --git a/toolchain/.github/workflows/vendor.yml b/toolchain/.github/workflows/vendor.yml index f7580c1..0d10912 100644 --- a/toolchain/.github/workflows/vendor.yml +++ b/toolchain/.github/workflows/vendor.yml @@ -3,7 +3,7 @@ name: vendor-toolchain # Build and publish the static toolchain for x86_64 and aarch64: clang, make # and git are compiled from source on NATIVE runners per arch (no QEMU — the # multi-hour x86_64 LLVM build runs here in normal CI time), while bpftool, -# esbuild and the libbpf headers are re-hosted from upstream. All are published +# esbuild, gh and the libbpf headers are re-hosted from upstream. All are published # as version-addressed assets on the `toolchain` release, with checksums # recorded back into build/versions.env. # @@ -84,7 +84,8 @@ jobs: tagpat="^v${mm}(\.[0-9]+)?$" ;; *) tagpat='^v[0-9]+\.[0-9]+(\.[0-9]+)?$' ;; esac - prev="$(git tag -l 'v[0-9]*' | grep -E "$tagpat" | sed 's/^v//' | sort -V | tail -1)" + # `|| true` so an empty tag set (no match) doesn't fail under pipefail. + prev="$(git tag -l 'v[0-9]*' | { grep -E "$tagpat" || true; } | sed 's/^v//' | sort -V | tail -1)" prevtag="" [ -n "$prev" ] && prevtag="v$prev" echo "prevtag=$prevtag" >> "$GITHUB_OUTPUT" @@ -226,7 +227,7 @@ jobs: strategy: fail-fast: false matrix: - tool: [bpftool, veristat, lvh, esbuild] + tool: [bpftool, veristat, lvh, esbuild, gh] arch: - { platform: amd64, name: x86_64 } - { platform: arm64, name: aarch64 } @@ -268,7 +269,7 @@ jobs: # - holding one file, so install them all the same way. # qemu is the exception (a bin/+share tree), packed separately below. for a in x86_64 aarch64; do - for t in clang make git bpftool veristat lvh esbuild; do + for t in clang make git bpftool veristat lvh esbuild gh; do install -Dm755 "artifacts/${t}-${a}/${t}" "${a}/${t}" done done @@ -327,7 +328,8 @@ jobs: esac # Clean tags on this line only; suffixed tags (vX.Y.Z-) are # unordered variants and never advance the line. - latest="$(git tag -l 'v[0-9]*' | grep -E "$tagpat" | sed 's/^v//' | sort -V | tail -1)" + # `|| true` so an empty tag set (no match) doesn't fail under pipefail. + latest="$(git tag -l 'v[0-9]*' | { grep -E "$tagpat" || true; } | sed 's/^v//' | sort -V | tail -1)" flavor="${FLAVOR:-}" if [ -n "$flavor" ]; then # A flavor build is an unordered VARIANT of an existing release: it @@ -380,7 +382,7 @@ jobs: # Record the checksum of every binary we publish, so a consumer # verifies the exact artifact it downloads from our release. for arch in x86_64 aarch64; do - for tool in clang make bpftool veristat lvh esbuild git; do + for tool in clang make bpftool veristat lvh esbuild git gh; do s="$(sha256sum "$arch/$tool" | awk '{print $1}')" key="$(echo "$tool" | tr a-z A-Z)_SHA256_${arch}" sed -i "s|^${key}=.*|${key}=${s}|" build/versions.env @@ -429,17 +431,24 @@ jobs: # Consumers pin this one tag. mkdir -p dist for a in x86_64 aarch64; do - for t in clang make bpftool veristat lvh esbuild git; do cp "$a/$t" "dist/${t}-${a}"; done + for t in clang make bpftool veristat lvh esbuild git gh; do cp "$a/$t" "dist/${t}-${a}"; done done cp libbpf-headers.tar.gz dist/libbpf-headers.tar.gz # qemu for the optional kernel-matrix runner (fetched on demand, not by `make`). for a in x86_64 aarch64; do cp "qemu-$a.tar.gz" "dist/qemu-$a.tar.gz"; done - # Flavored tags (vX.Y.Z-) are opt-in variants — mark them - # prerelease so GitHub never surfaces one as "Latest". - prerelease=""; case "$tag" in *-*) prerelease="--prerelease" ;; esac + # Pick the "Latest" flag explicitly (GitHub's default is recency-based, + # so a release/* back-port cut after a newer mainline release would + # otherwise steal the badge): + # * flavor (vX.Y.Z-) -> --prerelease (GitHub never marks it Latest) + # * mainline (master) -> --latest (newest mainline IS Latest) + # * release/* back-port -> --latest=false (can't steal it) + case "$tag" in + *-*) flags="--prerelease --latest=false" ;; + *) if [ "${GITHUB_REF_NAME}" = master ]; then flags="--latest"; else flags="--latest=false"; fi ;; + esac gh release view "$tag" >/dev/null 2>&1 \ - || gh release create "$tag" --title "$tag" $prerelease \ - --notes "Static build toolchain $tag — clang/make/git built from source; bpftool/esbuild/libbpf headers re-hosted; qemu for the kernel-matrix runner." + || gh release create "$tag" --title "$tag" $flags \ + --notes "Static build toolchain $tag — clang/make/git built from source; bpftool/veristat/esbuild/gh/libbpf headers re-hosted; qemu for the kernel-matrix runner." # --clobber so a same-version re-run refreshes assets. gh release upload "$tag" dist/* --clobber diff --git a/toolchain/README.md b/toolchain/README.md index c15ff39..7373d7d 100644 --- a/toolchain/README.md +++ b/toolchain/README.md @@ -40,6 +40,7 @@ before merge, so a typo can't silently mis-version a release. | `bpftool` | `vmlinux.h` (BTF dump) + link BPF objects | official static release, re-hosted | | `veristat`| check `*.bpf.o` load + BPF verifier statistics | official static release, re-hosted | | `esbuild` | bundle the JS entry | official static (Go) binary, re-hosted | +| `gh` | release/PR automation (CI + consumer tooling) | official static (Go) binary, re-hosted | | `bpf/*.h` | libbpf program headers (``, …) | libbpf bundled with bpftool | The table above is the **build** toolchain — what `make` resolves. The same @@ -81,7 +82,7 @@ from this repo's release, checksum-verified. Pull updates with Change a tool pin in [`build/versions.env`](build/versions.env) and push — the [`vendor-toolchain`](.github/workflows/vendor.yml) workflow rebuilds clang/make/ git (and the test-runner qemu) on native x86_64 and arm64 runners, re-hosts -bpftool/veristat/lvh/esbuild/headers, +bpftool/veristat/lvh/esbuild/gh/headers, **computes the next semver tag** (highest existing, bumped by the level the commit messages ask for — minor by default), publishes all assets to that immutable release, and records the version + checksums into `versions.env`. @@ -134,5 +135,5 @@ Build a single tool locally: ```sh build/build-clang.sh arm64 # or amd64; also build-make.sh / build-git.sh / build-qemu.sh -build/fetch-bpftool.sh # prebuilt; also fetch-veristat.sh / fetch-lvh.sh / fetch-esbuild.sh / fetch-libbpf-headers.sh +build/fetch-bpftool.sh # prebuilt; also fetch-veristat.sh / fetch-lvh.sh / fetch-esbuild.sh / fetch-gh.sh / fetch-libbpf-headers.sh ``` diff --git a/toolchain/build/fetch-gh.sh b/toolchain/build/fetch-gh.sh new file mode 100755 index 0000000..8123a27 --- /dev/null +++ b/toolchain/build/fetch-gh.sh @@ -0,0 +1,42 @@ +#!/bin/sh +# Producer step: pull the official static gh (GitHub CLI) binary from cli/cli +# into v//gh so CI can re-host it on our toolchain release. gh is a single +# fully-static Go binary, so we re-host it like esbuild/veristat rather than +# build it. The consumer-facing integrity check is the published binary's +# checksum (versions.env, verified by build/fetch-toolchain.sh), so this +# upstream fetch — run in CI from the official release over TLS — isn't pinned. +# +# build/fetch-gh.sh [amd64|arm64] (default: both) + +set -eu + +HERE="$(cd "$(dirname "$0")" && pwd)" +V="$(dirname "$HERE")" +. "$HERE/versions.env" + +fetch() { + plat="$1" + case "$plat" in + amd64) arch=x86_64 ;; + arm64) arch=aarch64 ;; + *) echo "error: unknown arch '$plat'" >&2; exit 1 ;; + esac + url="https://github.com/cli/cli/releases/download/v${GH_VERSION}/gh_${GH_VERSION}_linux_${plat}.tar.gz" + tmp="$(mktemp -d)" + echo ">> fetching gh ${GH_VERSION} for ${plat}" + curl -fSL -o "$tmp/gh.tar.gz" "$url" + mkdir -p "$V/$arch" + tar xzf "$tmp/gh.tar.gz" -C "$tmp" + # Tarball extracts to gh__linux_/bin/gh; glob the version dir. + cp "$tmp"/gh_*/bin/gh "$V/$arch/gh" + chmod +x "$V/$arch/gh" + rm -rf "$tmp" + echo ">> done: $V/$arch/gh" +} + +if [ "$#" -eq 0 ]; then + fetch amd64 + fetch arm64 +else + fetch "$1" +fi diff --git a/toolchain/build/versions.env b/toolchain/build/versions.env index 07321f9..ac204f1 100644 --- a/toolchain/build/versions.env +++ b/toolchain/build/versions.env @@ -7,7 +7,7 @@ # sets it to the next semver tag it computes from the [bump:*] commit markers # (default minor; see the README). A release is cut only when build/ or the # vendor workflow changes. -TOOLCHAIN_VERSION=0.8.0 +TOOLCHAIN_VERSION=0.10.0 # Where a consumer fetches assets from: the version-tagged release. The fetch # appends `/v/-`. @@ -29,7 +29,7 @@ CLANG_SHA256_aarch64=105fb2d324095127a127bcba56702a86229ed8892ce6429802d30585fa2 BPFTOOL_VERSION=7.7.0 BPFTOOL_SHA256_x86_64=74bd16335aa1c40714fb50287a42766c6faa4958f969cce32fef89485ce4934c BPFTOOL_SHA256_aarch64=2b3fc4dd5e4e40bd8d670c5f1fa9693b3f879a122c0f6a3eb806dfc6735da6b7 -LIBBPF_HEADERS_SHA256=dba9a3583076fce1ab2739b6edba1460f64327accda449202892108e9d1a3c65 +LIBBPF_HEADERS_SHA256=d712858662168e4e04cdc35e4a962a6056c429ed57461b974ec59b819c60e3c3 # veristat — BPF verifier statistics tool, used to check that built `*.bpf.o` # programs load and to track verifier complexity. Official static binary from @@ -61,6 +61,14 @@ ESBUILD_VERSION=0.28.1 ESBUILD_SHA256_x86_64=0c6588b092a2c291a72bab90659f3c9e0e25e0fe59c9ac12b4dae4d945e5548c ESBUILD_SHA256_aarch64=51e829ba36f36be6d9aea6e329ddc4f9350302339b16aaca96a3cb97f64a8ebb +# gh — the GitHub CLI, for release/PR automation (CI and consumer tooling). +# Official static (Go) binary from cli/cli, re-hosted on our "toolchain" +# release like esbuild. CI records the per-arch binary checksum (consumers +# verify the binary, not the upstream tarball). +GH_VERSION=2.95.0 +GH_SHA256_x86_64=62c11fbaa08835168c3d1acf8a645ac6268a13a5682c73581388c9df0c622617 +GH_SHA256_aarch64=706b030846db08d0dac0392d6f74416e0bbf0d9ac5ecbd54c9046305c10eac59 + # qemu — NOT part of the build toolchain: a fully-static qemu-system for the # optional kernel-matrix test runner (boots lvh kernel images). Built per-arch # from source (see Dockerfile.qemu / build-qemu.sh), trimmed to the binary plus @@ -81,8 +89,8 @@ LVH_SHA256_x86_64=ca3b958ffc08a4b65e4c6c8d29a4dad077acd11f146bc84f75a30ecaa031e9 LVH_SHA256_aarch64=49b7cef6376c445c0ab23465ecc984a771e06b3b37c39b486a4fed7fb1ded476 # Per-arch checksum of the published qemu-.tar.gz (binary + minimal # share/qemu blobs). CI records these; the matrix runner verifies the tarball. -QEMU_SHA256_x86_64=0add524e0dd366ead34332aed0dafb4f11bc75f17884896aedb91faa28176dc2 -QEMU_SHA256_aarch64=be362492c2054c734bac78a776f17c0e0511f317d1dd8c924e82e00f8a7b2137 +QEMU_SHA256_x86_64=2e26722e965018c145bab180cee8feace50f91841bf51b350f14527c0d5da637 +QEMU_SHA256_aarch64=6fe353ef43918c14088c7a3741bfac54e6c17084ea462454a5b2a525ba0aa330 # Alpine base used for the musl-static clang build. ALPINE_TAG=alpine:3.21 diff --git a/toolchain/embed/fetch-toolchain.sh b/toolchain/embed/fetch-toolchain.sh index 824b3f9..eff6a87 100755 --- a/toolchain/embed/fetch-toolchain.sh +++ b/toolchain/embed/fetch-toolchain.sh @@ -80,6 +80,7 @@ fetch_bin bpftool fetch_bin veristat fetch_bin esbuild fetch_bin git +fetch_bin gh # libbpf program headers: arch-independent, one copy per version, beside the # per-arch tool dirs ($key/include/bpf/*.h). diff --git a/toolchain/embed/toolchain.mk b/toolchain/embed/toolchain.mk index 52698b3..6cb5f18 100644 --- a/toolchain/embed/toolchain.mk +++ b/toolchain/embed/toolchain.mk @@ -1,4 +1,4 @@ -# Resolve the static build toolchain (clang, bpftool, veristat, esbuild, git) — included +# Resolve the static build toolchain (clang, bpftool, veristat, esbuild, git, gh) — included # by the project Makefile before build/bpf.mk, so the tools are set before any # rule uses them. A `make CLANG=…` CLI override beats this. # @@ -11,7 +11,7 @@ # Falls back to host tools on PATH when no lock is present. # # The vendored binaries are Linux musl-static, so the cache is only used on -# Linux. On any other host (macOS, BSD) we leave CLANG/BPFTOOL/ESBUILD/GIT +# Linux. On any other host (macOS, BSD) we leave CLANG/BPFTOOL/ESBUILD/GIT/GH # unset, falling through to host tools on PATH — macOS is an edit-here, # build-on-Linux host for BPF work. @@ -36,6 +36,7 @@ ifneq ($(TOOLCHAIN_LOCK),) VERISTAT ?= $(TOOLCHAIN_DIR)/veristat ESBUILD ?= $(TOOLCHAIN_DIR)/esbuild GIT ?= $(TOOLCHAIN_DIR)/git + GH ?= $(TOOLCHAIN_DIR)/gh # libbpf program headers are arch-independent: one copy per version key, # beside the per-arch tool dirs. BPF_SYSINCLUDE ?= $(YEET_CACHE_DIR)/toolchain/$(TOOLCHAIN_KEY)/include @@ -45,6 +46,7 @@ endif VERISTAT ?= veristat ESBUILD ?= esbuild GIT ?= git +GH ?= gh # Fill the cache for this arch, downloading any missing tool once. A no-op # when no lock is present (PATH case).