Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
1 change: 1 addition & 0 deletions template/build/fetch-toolchain.sh
Original file line number Diff line number Diff line change
Expand Up @@ -80,6 +80,7 @@ fetch_bin bpftool
fetch_bin veristat
fetch_bin esbuild
fetch_bin git
fetch_bin gh

# libbpf program headers: arch-independent, one copy per version, beside the
# per-arch tool dirs ($key/include/bpf/*.h).
Expand Down
16 changes: 12 additions & 4 deletions template/build/toolchain.lock
Original file line number Diff line number Diff line change
Expand Up @@ -7,7 +7,7 @@
# sets it to the next semver tag it computes from the [bump:*] commit markers
# (default minor; see the README). A release is cut only when build/ or the
# vendor workflow changes.
TOOLCHAIN_VERSION=0.8.0
TOOLCHAIN_VERSION=0.10.0

# Where a consumer fetches assets from: the version-tagged release. The fetch
# appends `/v<TOOLCHAIN_VERSION>/<tool>-<arch>`.
Expand All @@ -29,7 +29,7 @@ CLANG_SHA256_aarch64=105fb2d324095127a127bcba56702a86229ed8892ce6429802d30585fa2
BPFTOOL_VERSION=7.7.0
BPFTOOL_SHA256_x86_64=74bd16335aa1c40714fb50287a42766c6faa4958f969cce32fef89485ce4934c
BPFTOOL_SHA256_aarch64=2b3fc4dd5e4e40bd8d670c5f1fa9693b3f879a122c0f6a3eb806dfc6735da6b7
LIBBPF_HEADERS_SHA256=dba9a3583076fce1ab2739b6edba1460f64327accda449202892108e9d1a3c65
LIBBPF_HEADERS_SHA256=d712858662168e4e04cdc35e4a962a6056c429ed57461b974ec59b819c60e3c3

# veristat — BPF verifier statistics tool, used to check that built `*.bpf.o`
# programs load and to track verifier complexity. Official static binary from
Expand Down Expand Up @@ -61,6 +61,14 @@ ESBUILD_VERSION=0.28.1
ESBUILD_SHA256_x86_64=0c6588b092a2c291a72bab90659f3c9e0e25e0fe59c9ac12b4dae4d945e5548c
ESBUILD_SHA256_aarch64=51e829ba36f36be6d9aea6e329ddc4f9350302339b16aaca96a3cb97f64a8ebb

# gh — the GitHub CLI, for release/PR automation (CI and consumer tooling).
# Official static (Go) binary from cli/cli, re-hosted on our "toolchain"
# release like esbuild. CI records the per-arch binary checksum (consumers
# verify the binary, not the upstream tarball).
GH_VERSION=2.95.0
GH_SHA256_x86_64=62c11fbaa08835168c3d1acf8a645ac6268a13a5682c73581388c9df0c622617
GH_SHA256_aarch64=706b030846db08d0dac0392d6f74416e0bbf0d9ac5ecbd54c9046305c10eac59

# qemu — NOT part of the build toolchain: a fully-static qemu-system for the
# optional kernel-matrix test runner (boots lvh kernel images). Built per-arch
# from source (see Dockerfile.qemu / build-qemu.sh), trimmed to the binary plus
Expand All @@ -81,8 +89,8 @@ LVH_SHA256_x86_64=ca3b958ffc08a4b65e4c6c8d29a4dad077acd11f146bc84f75a30ecaa031e9
LVH_SHA256_aarch64=49b7cef6376c445c0ab23465ecc984a771e06b3b37c39b486a4fed7fb1ded476
# Per-arch checksum of the published qemu-<arch>.tar.gz (binary + minimal
# share/qemu blobs). CI records these; the matrix runner verifies the tarball.
QEMU_SHA256_x86_64=0add524e0dd366ead34332aed0dafb4f11bc75f17884896aedb91faa28176dc2
QEMU_SHA256_aarch64=be362492c2054c734bac78a776f17c0e0511f317d1dd8c924e82e00f8a7b2137
QEMU_SHA256_x86_64=2e26722e965018c145bab180cee8feace50f91841bf51b350f14527c0d5da637
QEMU_SHA256_aarch64=6fe353ef43918c14088c7a3741bfac54e6c17084ea462454a5b2a525ba0aa330

# Alpine base used for the musl-static clang build.
ALPINE_TAG=alpine:3.21
6 changes: 4 additions & 2 deletions template/build/toolchain.mk
Original file line number Diff line number Diff line change
@@ -1,4 +1,4 @@
# Resolve the static build toolchain (clang, bpftool, veristat, esbuild, git) — included
# Resolve the static build toolchain (clang, bpftool, veristat, esbuild, git, gh) — included
# by the project Makefile before build/bpf.mk, so the tools are set before any
# rule uses them. A `make CLANG=…` CLI override beats this.
#
Expand All @@ -11,7 +11,7 @@
# Falls back to host tools on PATH when no lock is present.
#
# The vendored binaries are Linux musl-static, so the cache is only used on
# Linux. On any other host (macOS, BSD) we leave CLANG/BPFTOOL/ESBUILD/GIT
# Linux. On any other host (macOS, BSD) we leave CLANG/BPFTOOL/ESBUILD/GIT/GH
# unset, falling through to host tools on PATH — macOS is an edit-here,
# build-on-Linux host for BPF work.

Expand All @@ -36,6 +36,7 @@ ifneq ($(TOOLCHAIN_LOCK),)
VERISTAT ?= $(TOOLCHAIN_DIR)/veristat
ESBUILD ?= $(TOOLCHAIN_DIR)/esbuild
GIT ?= $(TOOLCHAIN_DIR)/git
GH ?= $(TOOLCHAIN_DIR)/gh
# libbpf program headers are arch-independent: one copy per version key,
# beside the per-arch tool dirs.
BPF_SYSINCLUDE ?= $(YEET_CACHE_DIR)/toolchain/$(TOOLCHAIN_KEY)/include
Expand All @@ -45,6 +46,7 @@ endif
VERISTAT ?= veristat
ESBUILD ?= esbuild
GIT ?= git
GH ?= gh

# Fill the cache for this arch, downloading any missing tool once. A no-op
# when no lock is present (PATH case).
Expand Down
33 changes: 21 additions & 12 deletions toolchain/.github/workflows/vendor.yml
Original file line number Diff line number Diff line change
Expand Up @@ -3,7 +3,7 @@ name: vendor-toolchain
# Build and publish the static toolchain for x86_64 and aarch64: clang, make
# and git are compiled from source on NATIVE runners per arch (no QEMU — the
# multi-hour x86_64 LLVM build runs here in normal CI time), while bpftool,
# esbuild and the libbpf headers are re-hosted from upstream. All are published
# esbuild, gh and the libbpf headers are re-hosted from upstream. All are published
# as version-addressed assets on the `toolchain` release, with checksums
# recorded back into build/versions.env.
#
Expand Down Expand Up @@ -84,7 +84,8 @@ jobs:
tagpat="^v${mm}(\.[0-9]+)?$" ;;
*) tagpat='^v[0-9]+\.[0-9]+(\.[0-9]+)?$' ;;
esac
prev="$(git tag -l 'v[0-9]*' | grep -E "$tagpat" | sed 's/^v//' | sort -V | tail -1)"
# `|| true` so an empty tag set (no match) doesn't fail under pipefail.
prev="$(git tag -l 'v[0-9]*' | { grep -E "$tagpat" || true; } | sed 's/^v//' | sort -V | tail -1)"
prevtag=""
[ -n "$prev" ] && prevtag="v$prev"
echo "prevtag=$prevtag" >> "$GITHUB_OUTPUT"
Expand Down Expand Up @@ -226,7 +227,7 @@ jobs:
strategy:
fail-fast: false
matrix:
tool: [bpftool, veristat, lvh, esbuild]
tool: [bpftool, veristat, lvh, esbuild, gh]
arch:
- { platform: amd64, name: x86_64 }
- { platform: arm64, name: aarch64 }
Expand Down Expand Up @@ -268,7 +269,7 @@ jobs:
# <tool>-<arch> holding one file, so install them all the same way.
# qemu is the exception (a bin/+share tree), packed separately below.
for a in x86_64 aarch64; do
for t in clang make git bpftool veristat lvh esbuild; do
for t in clang make git bpftool veristat lvh esbuild gh; do
install -Dm755 "artifacts/${t}-${a}/${t}" "${a}/${t}"
done
done
Expand Down Expand Up @@ -327,7 +328,8 @@ jobs:
esac
# Clean tags on this line only; suffixed tags (vX.Y.Z-<flavor>) are
# unordered variants and never advance the line.
latest="$(git tag -l 'v[0-9]*' | grep -E "$tagpat" | sed 's/^v//' | sort -V | tail -1)"
# `|| true` so an empty tag set (no match) doesn't fail under pipefail.
latest="$(git tag -l 'v[0-9]*' | { grep -E "$tagpat" || true; } | sed 's/^v//' | sort -V | tail -1)"
flavor="${FLAVOR:-}"
if [ -n "$flavor" ]; then
# A flavor build is an unordered VARIANT of an existing release: it
Expand Down Expand Up @@ -380,7 +382,7 @@ jobs:
# Record the checksum of every binary we publish, so a consumer
# verifies the exact artifact it downloads from our release.
for arch in x86_64 aarch64; do
for tool in clang make bpftool veristat lvh esbuild git; do
for tool in clang make bpftool veristat lvh esbuild git gh; do
s="$(sha256sum "$arch/$tool" | awk '{print $1}')"
key="$(echo "$tool" | tr a-z A-Z)_SHA256_${arch}"
sed -i "s|^${key}=.*|${key}=${s}|" build/versions.env
Expand Down Expand Up @@ -429,17 +431,24 @@ jobs:
# Consumers pin this one tag.
mkdir -p dist
for a in x86_64 aarch64; do
for t in clang make bpftool veristat lvh esbuild git; do cp "$a/$t" "dist/${t}-${a}"; done
for t in clang make bpftool veristat lvh esbuild git gh; do cp "$a/$t" "dist/${t}-${a}"; done
done
cp libbpf-headers.tar.gz dist/libbpf-headers.tar.gz
# qemu for the optional kernel-matrix runner (fetched on demand, not by `make`).
for a in x86_64 aarch64; do cp "qemu-$a.tar.gz" "dist/qemu-$a.tar.gz"; done
# Flavored tags (vX.Y.Z-<flavor>) are opt-in variants — mark them
# prerelease so GitHub never surfaces one as "Latest".
prerelease=""; case "$tag" in *-*) prerelease="--prerelease" ;; esac
# Pick the "Latest" flag explicitly (GitHub's default is recency-based,
# so a release/* back-port cut after a newer mainline release would
# otherwise steal the badge):
# * flavor (vX.Y.Z-<flavor>) -> --prerelease (GitHub never marks it Latest)
# * mainline (master) -> --latest (newest mainline IS Latest)
# * release/* back-port -> --latest=false (can't steal it)
case "$tag" in
*-*) flags="--prerelease --latest=false" ;;
*) if [ "${GITHUB_REF_NAME}" = master ]; then flags="--latest"; else flags="--latest=false"; fi ;;
esac
gh release view "$tag" >/dev/null 2>&1 \
|| gh release create "$tag" --title "$tag" $prerelease \
--notes "Static build toolchain $tag — clang/make/git built from source; bpftool/esbuild/libbpf headers re-hosted; qemu for the kernel-matrix runner."
|| gh release create "$tag" --title "$tag" $flags \
--notes "Static build toolchain $tag — clang/make/git built from source; bpftool/veristat/esbuild/gh/libbpf headers re-hosted; qemu for the kernel-matrix runner."
# --clobber so a same-version re-run refreshes assets.
gh release upload "$tag" dist/* --clobber

Expand Down
5 changes: 3 additions & 2 deletions toolchain/README.md
Original file line number Diff line number Diff line change
Expand Up @@ -40,6 +40,7 @@ before merge, so a typo can't silently mis-version a release.
| `bpftool` | `vmlinux.h` (BTF dump) + link BPF objects | official static release, re-hosted |
| `veristat`| check `*.bpf.o` load + BPF verifier statistics | official static release, re-hosted |
| `esbuild` | bundle the JS entry | official static (Go) binary, re-hosted |
| `gh` | release/PR automation (CI + consumer tooling) | official static (Go) binary, re-hosted |
| `bpf/*.h` | libbpf program headers (`<bpf/bpf_helpers.h>`, …) | libbpf bundled with bpftool |

The table above is the **build** toolchain — what `make` resolves. The same
Expand Down Expand Up @@ -81,7 +82,7 @@ from this repo's release, checksum-verified. Pull updates with
Change a tool pin in [`build/versions.env`](build/versions.env) and push — the
[`vendor-toolchain`](.github/workflows/vendor.yml) workflow rebuilds clang/make/
git (and the test-runner qemu) on native x86_64 and arm64 runners, re-hosts
bpftool/veristat/lvh/esbuild/headers,
bpftool/veristat/lvh/esbuild/gh/headers,
**computes the next semver tag** (highest existing, bumped by the level the
commit messages ask for — minor by default), publishes all assets to
that immutable release, and records the version + checksums into `versions.env`.
Expand Down Expand Up @@ -134,5 +135,5 @@ Build a single tool locally:

```sh
build/build-clang.sh arm64 # or amd64; also build-make.sh / build-git.sh / build-qemu.sh
build/fetch-bpftool.sh # prebuilt; also fetch-veristat.sh / fetch-lvh.sh / fetch-esbuild.sh / fetch-libbpf-headers.sh
build/fetch-bpftool.sh # prebuilt; also fetch-veristat.sh / fetch-lvh.sh / fetch-esbuild.sh / fetch-gh.sh / fetch-libbpf-headers.sh
```
42 changes: 42 additions & 0 deletions toolchain/build/fetch-gh.sh
Original file line number Diff line number Diff line change
@@ -0,0 +1,42 @@
#!/bin/sh
# Producer step: pull the official static gh (GitHub CLI) binary from cli/cli
# into v/<arch>/gh so CI can re-host it on our toolchain release. gh is a single
# fully-static Go binary, so we re-host it like esbuild/veristat rather than
# build it. The consumer-facing integrity check is the published binary's
# checksum (versions.env, verified by build/fetch-toolchain.sh), so this
# upstream fetch — run in CI from the official release over TLS — isn't pinned.
#
# build/fetch-gh.sh [amd64|arm64] (default: both)

set -eu

HERE="$(cd "$(dirname "$0")" && pwd)"
V="$(dirname "$HERE")"
. "$HERE/versions.env"

fetch() {
plat="$1"
case "$plat" in
amd64) arch=x86_64 ;;
arm64) arch=aarch64 ;;
*) echo "error: unknown arch '$plat'" >&2; exit 1 ;;
esac
url="https://github.com/cli/cli/releases/download/v${GH_VERSION}/gh_${GH_VERSION}_linux_${plat}.tar.gz"
tmp="$(mktemp -d)"
echo ">> fetching gh ${GH_VERSION} for ${plat}"
curl -fSL -o "$tmp/gh.tar.gz" "$url"
mkdir -p "$V/$arch"
tar xzf "$tmp/gh.tar.gz" -C "$tmp"
# Tarball extracts to gh_<ver>_linux_<plat>/bin/gh; glob the version dir.
cp "$tmp"/gh_*/bin/gh "$V/$arch/gh"
chmod +x "$V/$arch/gh"
rm -rf "$tmp"
echo ">> done: $V/$arch/gh"
}

if [ "$#" -eq 0 ]; then
fetch amd64
fetch arm64
else
fetch "$1"
fi
16 changes: 12 additions & 4 deletions toolchain/build/versions.env
Original file line number Diff line number Diff line change
Expand Up @@ -7,7 +7,7 @@
# sets it to the next semver tag it computes from the [bump:*] commit markers
# (default minor; see the README). A release is cut only when build/ or the
# vendor workflow changes.
TOOLCHAIN_VERSION=0.8.0
TOOLCHAIN_VERSION=0.10.0

# Where a consumer fetches assets from: the version-tagged release. The fetch
# appends `/v<TOOLCHAIN_VERSION>/<tool>-<arch>`.
Expand All @@ -29,7 +29,7 @@ CLANG_SHA256_aarch64=105fb2d324095127a127bcba56702a86229ed8892ce6429802d30585fa2
BPFTOOL_VERSION=7.7.0
BPFTOOL_SHA256_x86_64=74bd16335aa1c40714fb50287a42766c6faa4958f969cce32fef89485ce4934c
BPFTOOL_SHA256_aarch64=2b3fc4dd5e4e40bd8d670c5f1fa9693b3f879a122c0f6a3eb806dfc6735da6b7
LIBBPF_HEADERS_SHA256=dba9a3583076fce1ab2739b6edba1460f64327accda449202892108e9d1a3c65
LIBBPF_HEADERS_SHA256=d712858662168e4e04cdc35e4a962a6056c429ed57461b974ec59b819c60e3c3

# veristat — BPF verifier statistics tool, used to check that built `*.bpf.o`
# programs load and to track verifier complexity. Official static binary from
Expand Down Expand Up @@ -61,6 +61,14 @@ ESBUILD_VERSION=0.28.1
ESBUILD_SHA256_x86_64=0c6588b092a2c291a72bab90659f3c9e0e25e0fe59c9ac12b4dae4d945e5548c
ESBUILD_SHA256_aarch64=51e829ba36f36be6d9aea6e329ddc4f9350302339b16aaca96a3cb97f64a8ebb

# gh — the GitHub CLI, for release/PR automation (CI and consumer tooling).
# Official static (Go) binary from cli/cli, re-hosted on our "toolchain"
# release like esbuild. CI records the per-arch binary checksum (consumers
# verify the binary, not the upstream tarball).
GH_VERSION=2.95.0
GH_SHA256_x86_64=62c11fbaa08835168c3d1acf8a645ac6268a13a5682c73581388c9df0c622617
GH_SHA256_aarch64=706b030846db08d0dac0392d6f74416e0bbf0d9ac5ecbd54c9046305c10eac59

# qemu — NOT part of the build toolchain: a fully-static qemu-system for the
# optional kernel-matrix test runner (boots lvh kernel images). Built per-arch
# from source (see Dockerfile.qemu / build-qemu.sh), trimmed to the binary plus
Expand All @@ -81,8 +89,8 @@ LVH_SHA256_x86_64=ca3b958ffc08a4b65e4c6c8d29a4dad077acd11f146bc84f75a30ecaa031e9
LVH_SHA256_aarch64=49b7cef6376c445c0ab23465ecc984a771e06b3b37c39b486a4fed7fb1ded476
# Per-arch checksum of the published qemu-<arch>.tar.gz (binary + minimal
# share/qemu blobs). CI records these; the matrix runner verifies the tarball.
QEMU_SHA256_x86_64=0add524e0dd366ead34332aed0dafb4f11bc75f17884896aedb91faa28176dc2
QEMU_SHA256_aarch64=be362492c2054c734bac78a776f17c0e0511f317d1dd8c924e82e00f8a7b2137
QEMU_SHA256_x86_64=2e26722e965018c145bab180cee8feace50f91841bf51b350f14527c0d5da637
QEMU_SHA256_aarch64=6fe353ef43918c14088c7a3741bfac54e6c17084ea462454a5b2a525ba0aa330

# Alpine base used for the musl-static clang build.
ALPINE_TAG=alpine:3.21
1 change: 1 addition & 0 deletions toolchain/embed/fetch-toolchain.sh
Original file line number Diff line number Diff line change
Expand Up @@ -80,6 +80,7 @@ fetch_bin bpftool
fetch_bin veristat
fetch_bin esbuild
fetch_bin git
fetch_bin gh

# libbpf program headers: arch-independent, one copy per version, beside the
# per-arch tool dirs ($key/include/bpf/*.h).
Expand Down
6 changes: 4 additions & 2 deletions toolchain/embed/toolchain.mk
Original file line number Diff line number Diff line change
@@ -1,4 +1,4 @@
# Resolve the static build toolchain (clang, bpftool, veristat, esbuild, git) — included
# Resolve the static build toolchain (clang, bpftool, veristat, esbuild, git, gh) — included
# by the project Makefile before build/bpf.mk, so the tools are set before any
# rule uses them. A `make CLANG=…` CLI override beats this.
#
Expand All @@ -11,7 +11,7 @@
# Falls back to host tools on PATH when no lock is present.
#
# The vendored binaries are Linux musl-static, so the cache is only used on
# Linux. On any other host (macOS, BSD) we leave CLANG/BPFTOOL/ESBUILD/GIT
# Linux. On any other host (macOS, BSD) we leave CLANG/BPFTOOL/ESBUILD/GIT/GH
# unset, falling through to host tools on PATH — macOS is an edit-here,
# build-on-Linux host for BPF work.

Expand All @@ -36,6 +36,7 @@ ifneq ($(TOOLCHAIN_LOCK),)
VERISTAT ?= $(TOOLCHAIN_DIR)/veristat
ESBUILD ?= $(TOOLCHAIN_DIR)/esbuild
GIT ?= $(TOOLCHAIN_DIR)/git
GH ?= $(TOOLCHAIN_DIR)/gh
# libbpf program headers are arch-independent: one copy per version key,
# beside the per-arch tool dirs.
BPF_SYSINCLUDE ?= $(YEET_CACHE_DIR)/toolchain/$(TOOLCHAIN_KEY)/include
Expand All @@ -45,6 +46,7 @@ endif
VERISTAT ?= veristat
ESBUILD ?= esbuild
GIT ?= git
GH ?= gh

# Fill the cache for this arch, downloading any missing tool once. A no-op
# when no lock is present (PATH case).
Expand Down
Loading