diff --git a/.github/workflows/release.yml b/.github/workflows/release.yml index 5c87856..6170f06 100644 --- a/.github/workflows/release.yml +++ b/.github/workflows/release.yml @@ -1,18 +1,15 @@ name: Release -# Package + publish on a version tag. The matrix lives here (the caller owns it); -# each leg calls the single-combo reusable helper. The publish feed is derived from -# distro-release/distro-channel. The tag must equal the avocado.yaml version. +# Build the agent binaries, attach them to the release, build the extension +# against that release, then package + publish the extension. The tag must equal +# the avocado.yaml version. # -# The matrix dimension is the FEED, not the target. An extension version is -# global within a (distro-release, distro-channel), so two rows differing only by -# target publish the same version twice into the same feed and the second fails -# with "that extension version is already taken". Connect fans a published -# extension out to the per-target feeds itself - avocado-ext-cli and -# avocado-ext-microclaw are arch-specific and reach both qemux86-64-ext and -# imx93-frdm-ext while publishing only a qemux86-64 row - so `target` here just -# names the target used to package, and adding rows for other targets buys -# nothing while breaking the release. +# Two matrices here answer different questions, and conflating them broke 0.1.0. +# `binaries` varies ARCHITECTURE, because a binary is architecture-specific. +# `publish` varies FEED, because an extension version is global within a distro +# release and channel - two publish rows differing only by target send the same +# version into the same feed and the second is refused as a duplicate. Connect +# distributes a published extension to the per-target feeds itself. on: push: tags: @@ -21,10 +18,189 @@ on: workflow_dispatch: jobs: + # Runs before anything is built or released. GITHUB_REF_NAME names the assets + # and the release, so a dispatch from a branch would create a public release + # and tag named after the branch, and a tag that differs from avocado.yaml + # would publish assets under a version no consumer resolves. The + # tag-equals-version guard inside extension-release.yml runs too late to stop + # either, and skips non-tag refs entirely. + guard: + name: Tag matches avocado.yaml + if: startsWith(github.ref, 'refs/tags/') + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v5 + + - name: Compare the tag with the manifest version + run: | + manifest="$(sed -nE "/^[[:space:]]+version:/{s/^[[:space:]]+version:[[:space:]]*['\"]?([^'\"[:space:]#]+)['\"]?.*/\1/p;q;}" avocado.yaml)" + if [ "$GITHUB_REF_NAME" != "$manifest" ]; then + echo "::error::tag '$GITHUB_REF_NAME' does not match avocado.yaml version '$manifest'" >&2 + exit 1 + fi + echo "tag and avocado.yaml agree on $manifest" + + binaries: + name: Build ${{ matrix.rust-target }} + needs: guard + runs-on: ubuntu-latest + strategy: + fail-fast: false + matrix: + include: + - { rust-target: x86_64-unknown-linux-musl } + - { rust-target: aarch64-unknown-linux-musl } + steps: + - uses: actions/checkout@v5 + + # The ring-only provider rule, checked against the triple that ships. This + # job is the only place the published binaries are built, so it is the + # enforcement point. + - name: Check the rustls crypto provider + working-directory: agent + run: TARGET=${{ matrix.rust-target }} scripts/check-provider.sh + + - name: Install cross + run: cargo install cross --locked --version 0.2.5 + + # musl, statically linked, so the artifact carries no libc version coupling + # and runs on any Avocado target of this architecture whatever the device's + # glibc is. A gnu build would pin the runner's glibc into the binary and + # fail on older devices at exec time rather than here. --locked fails the + # build when Cargo.lock does not match Cargo.toml, so the binary is built + # from the committed lockfile; agent/rust-toolchain.toml pins the compiler. + - name: Build + working-directory: agent + run: cross build --release --locked --target ${{ matrix.rust-target }} + + - name: Verify the artifact is static and the right architecture + working-directory: agent + run: | + bin="target/${{ matrix.rust-target }}/release/avocado-container-agent-dev" + desc="$(file -b "$bin")" + echo "$desc" + case "$desc" in + *"statically linked"*|*"static-pie linked"*) ;; + *) echo "::error::artifact is not static: $desc" >&2; exit 1 ;; + esac + case "${{ matrix.rust-target }}" in + x86_64-*) want="x86-64" ;; + aarch64-*) want="aarch64" ;; + esac + case "$desc" in + *"$want"*) ;; + *) echo "::error::artifact is not $want: $desc" >&2; exit 1 ;; + esac + + - name: Package + working-directory: agent + run: | + asset="avocado-container-agent-dev-${GITHUB_REF_NAME}-${{ matrix.rust-target }}.tar.gz" + tar -czf "../$asset" -C "target/${{ matrix.rust-target }}/release" \ + avocado-container-agent-dev + echo "asset=$asset" >> "$GITHUB_ENV" + + - uses: actions/upload-artifact@v4 + with: + name: ${{ matrix.rust-target }} + path: ${{ env.asset }} + if-no-files-found: error + + release-assets: + name: Attach binaries to the release + needs: binaries + runs-on: ubuntu-latest + permissions: + contents: write + steps: + - uses: actions/download-artifact@v4 + with: + path: dist + merge-multiple: true + + # cad-compile.sh refuses when SHA256SUMS does not list the asset it wants, + # so an architecture missing here fails the consumer's build loudly rather + # than leaving the agent silently absent from their image. + - name: Checksum + working-directory: dist + run: | + sha256sum *.tar.gz > SHA256SUMS + cat SHA256SUMS + + # A release is created once per tag. A consumer that already fetched an + # asset holds its checksum, so a rebuilt binary under the same tag must + # fail here rather than replace the files it verified. A draft left by a + # failed run also counts as existing and has to be deleted by hand. + - name: Refuse an existing release + env: + GH_TOKEN: ${{ github.token }} + run: | + if gh release view "$GITHUB_REF_NAME" --repo "$GITHUB_REPOSITORY" >/dev/null 2>&1; then + echo "::error::a release for tag '$GITHUB_REF_NAME' already exists; refusing to replace its assets" >&2 + exit 1 + fi + + # Draft first so the release never shows without its assets. + - name: Create the draft release + env: + GH_TOKEN: ${{ github.token }} + run: | + gh release create "$GITHUB_REF_NAME" --repo "$GITHUB_REPOSITORY" \ + --draft \ + --title "$GITHUB_REF_NAME" --notes "Agent binaries for $GITHUB_REF_NAME." + + - name: Upload the assets + working-directory: dist + env: + GH_TOKEN: ${{ github.token }} + run: | + gh release upload "$GITHUB_REF_NAME" --repo "$GITHUB_REPOSITORY" \ + *.tar.gz SHA256SUMS + + # Last step on purpose. `verify` and every consumer build fetch these + # assets anonymously through cad-compile.sh, and a draft release's assets + # are not downloadable that way, so the release has to be public before + # this job ends. + - name: Publish the release + env: + GH_TOKEN: ${{ github.token }} + run: gh release edit "$GITHUB_REF_NAME" --repo "$GITHUB_REPOSITORY" --draft=false + + # Builds the extension the way a consumer does, against the release that + # `release-assets` just made public, before anything reaches a feed. + # `publish` cannot catch a broken fetch: extension-release.yml runs only + # `avocado ext package` and the upload, never `avocado ext install` or + # `avocado ext build`, so cad-compile.sh does not run there. Without this + # job the first run of the fetch against a real release is a consumer's + # build, after the extension is already in both feeds. + # + # The rows are the publish feeds times one target per architecture the + # release ships. ext-version is the tag, which `guard` has already checked + # against avocado.yaml. + verify: + name: Verify ${{ matrix.target }} @ ${{ matrix.distro-release }}/${{ matrix.distro-channel }} + needs: release-assets + strategy: + fail-fast: false + matrix: + include: + - { target: qemux86-64, distro-release: "2026", distro-channel: next } + - { target: qemux86-64, distro-release: "2024", distro-channel: next } + - { target: imx93-frdm, distro-release: "2026", distro-channel: next } + - { target: imx93-frdm, distro-release: "2024", distro-channel: next } + uses: avocado-linux/actions/.github/workflows/extension-test.yml@v1 + with: + target: ${{ matrix.target }} + distro-release: ${{ matrix.distro-release }} + distro-channel: ${{ matrix.distro-channel }} + ext-version: ${{ github.ref_name }} + publish: + # Gated on `verify`, which builds against the published release. Packaging + # here runs no build step, so this job alone cannot tell whether a consumer + # can build what it publishes. + needs: verify strategy: - # Each feed reports its own outcome. Without this, one feed's failure - # cancels the other and a partial publish looks like a total one. fail-fast: false matrix: include: diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml index fcd54ef..d77d7f0 100644 --- a/.github/workflows/test.yml +++ b/.github/workflows/test.yml @@ -12,7 +12,82 @@ on: workflow_dispatch: jobs: + # The extension build below fetches the released binary for the avocado.yaml + # version, so it never compiles agent/. This job does, the same way the + # release `binaries` job does, so a Rust compile error or a provider-rule + # violation fails the PR instead of the next release. + agent: + name: Agent ${{ matrix.rust-target }} + runs-on: ubuntu-latest + strategy: + fail-fast: false + matrix: + include: + - { rust-target: x86_64-unknown-linux-musl } + - { rust-target: aarch64-unknown-linux-musl } + steps: + - uses: actions/checkout@v5 + + - name: Check the rustls crypto provider + working-directory: agent + run: TARGET=${{ matrix.rust-target }} scripts/check-provider.sh + + - name: Install cross + run: cargo install cross --locked --version 0.2.5 + + - name: Build + working-directory: agent + run: cross build --release --locked --target ${{ matrix.rust-target }} + + # `build` runs cad-compile.sh, which downloads the agent binary from the GitHub + # release named after the avocado.yaml version. On a version-bump PR that + # release does not exist until the tag is pushed, so `build` would fail on a + # 404 that says nothing about the PR. This job checks first, and `build` is + # skipped with the reason in this job's log instead of going red. + # + # A draft release counts as absent. The API hides drafts from a token without + # write access, which this one is, and cad-compile.sh cannot download a + # draft's assets anonymously either. + release-exists: + name: Release exists for the avocado.yaml version + runs-on: ubuntu-latest + permissions: + contents: read + outputs: + released: ${{ steps.check.outputs.released }} + steps: + - uses: actions/checkout@v5 + + - name: Look up the release + id: check + env: + GH_TOKEN: ${{ github.token }} + run: | + # Same parse as the `guard` job in release.yml, so a quoted version works. + version="$(sed -nE "/^[[:space:]]+version:/{s/^[[:space:]]+version:[[:space:]]*['\"]?([^'\"[:space:]#]+)['\"]?.*/\1/p;q;}" avocado.yaml)" + if [ -z "$version" ]; then + echo "::error::could not read the extension version from avocado.yaml" >&2 + exit 1 + fi + # Only a "not found" answer means unreleased. A network or auth failure + # fails the job instead of skipping. gh prints "not found" for a + # missing tag and for a repository it cannot see alike; the repository + # here is $GITHUB_REPOSITORY, the one this job runs in, so the second + # case cannot arise. + if err="$(gh release view "$version" --repo "$GITHUB_REPOSITORY" 2>&1 >/dev/null)"; then + echo "released=true" >> "$GITHUB_OUTPUT" + echo "Release $version exists; the extension build runs against it." + elif printf '%s' "$err" | grep -qi 'not found'; then + echo "released=false" >> "$GITHUB_OUTPUT" + echo "::notice::build skipped: no published release named $version yet. cad-compile.sh downloads the agent binary from that release, and the release workflow creates it when the $version tag is pushed. The agent job still compiles agent/." + else + echo "::error::could not look up release $version: $err" >&2 + exit 1 + fi + build: + needs: release-exists + if: needs.release-exists.outputs.released == 'true' strategy: fail-fast: false matrix: diff --git a/.gitignore b/.gitignore index 39f33ee..42469bc 100644 --- a/.gitignore +++ b/.gitignore @@ -1,10 +1,14 @@ # Cargo build output for the staged crate. -/agent/target/ +/agent/target*/ # Cross-compilation config written by cad-compile.sh at build time. It pins one # machine's SDK sysroot path, so a committed copy is wrong for everyone else. /agent/.cargo/ +# Prebuilt tarball and SHA256SUMS staged for cad-compile.sh to use instead of +# the release. Binaries, not source. +/dist/ + # avocado-cli build state. .avocado-state is a JSON file, not a directory, so it # needs a slash-free pattern - a trailing slash silently fails to match it. /.avocado-state diff --git a/CHANGELOG.md b/CHANGELOG.md index 6b36e97..fdfa76f 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -4,6 +4,43 @@ All notable changes to avocado-ext-container-agent-dev are documented in this fi The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.1.0/), and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0.html). +## [0.2.0] + +### Changed +- The extension ships a prebuilt agent binary instead of its source. `avocado + ext package` runs no build step, so the published RPM could only carry what + was in the repo; shipping the crate pushed the compile onto consumers, who + have no Rust cross toolchain in their SDK and no reason to. A clean project + following the documented setup failed inside `ring`'s build script with + `ToolNotFound: failed to find tool "x86_64-avocado-linux-gcc"`, after both + `avocado install` and the extension's own sysroot creation reported success. +- Binaries are statically linked against musl, so they carry no libc version + coupling and run on any Avocado target of the right architecture. +- `release.yml` builds x86_64 and aarch64 binaries, verifies each is static and + the right architecture, and attaches them with a `SHA256SUMS` to the release + before the extension publishes. A `verify` job then builds the extension + against that release for qemux86-64 and imx93-frdm on each publish feed, and + `publish` waits for it. +- The agent is built with `--locked` on a toolchain pinned in + `agent/rust-toolchain.toml`. +- `cad-compile.sh` caches `SHA256SUMS` per version, retries downloads, verifies + a download before keeping it, downloads a cached tarball that fails the + checksum again once, fetches a cached `SHA256SUMS` that does not list the + asset again once, and uses a tarball and `SHA256SUMS` staged in `dist/` + instead of the release when both are present. A `dist/` with only one of the + two files is ignored with a warning. +- The PR `build` check is skipped, with the reason logged, when no release + exists yet for the `avocado.yaml` version. +- `cad-install.sh` checks the staged binary's architecture against the target + before installing. The packaged extension is noarch and nothing downstream + re-checks, so a mismatch would otherwise install cleanly and fail at exec time + on the device. + +### Removed +- The Rust cross toolchain from the extension's SDK requirements, and `libstd-rs` + from the compile section. Nothing compiles at build time any more. +- The crate source from `package_files`. The RPM is no longer a source package. + ## [0.1.1] ### Fixed diff --git a/README.md b/README.md index cb71006..d9b63d3 100644 --- a/README.md +++ b/README.md @@ -15,10 +15,10 @@ on a production device. | Path | What it is | |---|---| -| `agent/` | the Rust crate, cross-compiled at package time | -| `cad-compile.sh` | resolves the target triple and builds the crate | -| `cad-install.sh` | stages the built binary into the extension sysroot | -| `cad-clean.sh` | clears the build directory | +| `agent/` | the Rust crate; the release workflow builds it into static musl binaries | +| `cad-compile.sh` | downloads the prebuilt binary for the target's architecture from the matching release and verifies it against `SHA256SUMS` | +| `cad-install.sh` | stages the binary into the extension sysroot | +| `cad-clean.sh` | removes the staged download and binary | | `overlay/` | the `container-agent-dev` path and service units | | `avocado.yaml` | extension manifest | @@ -33,8 +33,13 @@ preflight is what makes a missing engine fail loudly rather than silently. ## Building locally -The SDK image tag is release-scoped and comes from the environment, so export it -before building: +The build downloads the agent binary from the GitHub release named after the +`avocado.yaml` version, so it only works for a version that has been released. +An unreleased version fails the fetch with a 404. To build against binaries that +are not released, put the tarball and its `SHA256SUMS`, named as the release +workflow names them, in `dist/` next to `avocado.yaml`; `cad-compile.sh` verifies +and uses them instead of downloading. The SDK image tag is +release-scoped and comes from the environment, so export it before building: ``` AVOCADO_DISTRO_RELEASE=2024 avocado ext install -t qemux86-64 avocado-ext-container-agent-dev @@ -53,14 +58,40 @@ Two things that will otherwise cost you time: `avocado.yaml` lists them explicitly rather than using `'*'`. A target belongs there once it has produced an installable package and the produced binary has -been seen to exec on that architecture. The release matrix in -`.github/workflows/release.yml` should carry one row per declared target. +been seen to exec on that architecture. The release workflow does not need a +row per target: `binaries` builds one artifact per architecture, and `publish` +has one row per feed, as the header of `.github/workflows/release.yml` explains. ## Releasing -Tag the version in `avocado.yaml`. The release workflow guards that the tag and -the manifest version match, packages one leg per matrix row, and publishes each -to the feed derived from that row's distro release and channel. +Tag the version in `avocado.yaml`. The release workflow first checks that the +tag and the manifest version match, then builds the agent binaries, attaches +them and `SHA256SUMS` to the GitHub release of that tag, and publishes the +extension to the feed derived from each `publish` row's distro release and +channel. + +The `build` check in `test.yml` builds the extension against the release named +after the `avocado.yaml` version, because `cad-compile.sh` downloads the agent +binary from it. On a version-bump PR that release does not exist yet, so the +`release-exists` job finds that out and `build` is skipped, with the reason in +that job's log, instead of failing on a 404. On a PR that does not bump the +version, `build` exercises the last released binary, not the agent code in the +PR; the `agent` job compiles that code but does not package or install it. + +The order is: + +1. Merge the version-bump PR. +2. Push the version tag. +3. The release workflow builds both architectures, creates the release as a + draft, uploads the assets and publishes the release. +4. `verify` builds the extension against that release for qemux86-64 and + imx93-frdm on each publish feed. +5. The extension publish job runs. + +A release is created once per tag. Re-running the workflow for a tag that +already has a release refuses instead of replacing its files, so a rebuilt +binary needs a new version. The feed index is rebuilt separately, so the new +package can appear after the release workflow finishes. Bump `release:` whenever the packaged payload changes without a version change. An unchanged version-release identity means the package manager never delivers diff --git a/agent/README.md b/agent/README.md index c5637b7..5de31c2 100644 --- a/agent/README.md +++ b/agent/README.md @@ -32,12 +32,14 @@ The guard also backstops by asserting `ring` IS present: a guard that cannot see ### Enforced gate vs advisory run -- **Enforced gate (in-hook, with `TARGET`):** `cad-compile.sh` runs the guard as - `TARGET="$RUST_TARGET" scripts/check-provider.sh` immediately before - `cargo build`, resolving the same SDK triple, manifest, and `CARGO_HOME` the - build uses. Under `set -e`, a non-zero exit fails the compile. This is the - real enforcement point - this repository has no pre-commit hook, - and CI reaches the guard only through this compile. +- **Enforced gate (in CI, with `TARGET`):** the `binaries` job in + `.github/workflows/release.yml` runs the guard as + `TARGET= scripts/check-provider.sh` immediately before + `cross build`, resolving the same musl triple the shipped binary is built + for. A non-zero exit fails the job before anything is built or released. The + `agent` job in `.github/workflows/test.yml` runs the same step on every pull + request. This repository has no pre-commit hook, and `cad-compile.sh` no + longer builds the crate, so these two jobs are the only enforcement points. - **Advisory run (standalone, no `TARGET`):** running `scripts/check-provider.sh` directly (e.g. from the repo root) checks the **host** triple, not the shipped cross target. Treat it as an advisory diff --git a/agent/rust-toolchain.toml b/agent/rust-toolchain.toml new file mode 100644 index 0000000..f3a9251 --- /dev/null +++ b/agent/rust-toolchain.toml @@ -0,0 +1,7 @@ +# Pinned so a release built from a tag uses the same compiler whenever it runs, +# instead of whatever stable the runner has that day. cross reads this file from +# the crate directory. Bump it deliberately, in its own change. +[toolchain] +channel = "1.100.0" +profile = "minimal" +targets = ["x86_64-unknown-linux-musl", "aarch64-unknown-linux-musl"] diff --git a/agent/scripts/check-provider.sh b/agent/scripts/check-provider.sh index d19d9af..8e295d5 100755 --- a/agent/scripts/check-provider.sh +++ b/agent/scripts/check-provider.sh @@ -10,9 +10,11 @@ # already turned out to be wrong, so this comment states only what was measured. # # - "the agent cross-compiles against the musl SDK, which clears only ring." -# No build has targeted musl. Every triple built so far is a glibc one -# (x86_64-avocado-linux-gnu), and no published feed carries a 32-bit ARM or -# musl target to build against. +# Not the reason. The shipped binaries are now built for musl +# (x86_64-unknown-linux-musl and aarch64-unknown-linux-musl, by `cross` in +# the release workflow), but the rule predates that: every earlier build +# was a glibc one (x86_64-avocado-linux-gnu), and nobody has built the +# crate with aws-lc-rs for musl to see whether it fails. # - "ring is pure Rust, so unlike aws-lc-rs it needs no C toolchain." # False. Measured 2026-09-15: building this crate for # x86_64-avocado-linux-gnu without the cross toolchain on PATH fails inside @@ -42,9 +44,10 @@ # entirely", and the rcgen dev-dependency is pinned to ring for exactly that. # # TARGET: when set (env var or first positional arg) the guard resolves against -# that target triple, matching the build's cross target. In the build hook this -# is mandatory (TARGET="$RUST_TARGET"). Run standalone with no TARGET, the guard -# checks the host triple as an ADVISORY check. +# that target triple, matching the build's cross target. In CI this is +# mandatory (TARGET= in the release `binaries` job and the test +# `agent` job). Run standalone with no TARGET, the guard checks the host triple +# as an ADVISORY check. set -euo pipefail diff --git a/avocado.yaml b/avocado.yaml index 790c42c..c7537d1 100644 --- a/avocado.yaml +++ b/avocado.yaml @@ -12,7 +12,7 @@ supported_targets: extensions: avocado-ext-container-agent-dev: - version: 0.1.1 + version: 0.2.0 release: r0 summary: Container Dev Mode device agent (development only) description: | @@ -40,10 +40,12 @@ extensions: url: https://github.com/avocado-linux/ext-container-agent-dev vendor: Avocado Linux - # The crate is staged and cross-compiled at package time, so its sources are - # part of the payload rather than a prebuilt artifact. + # No `agent` here. The crate is no longer part of the payload: cad-compile.sh + # fetches a prebuilt static binary from this version's release instead of + # compiling. Shipping the source made the RPM a source package, which pushed + # the compile onto consumers who have no Rust cross toolchain in their SDK. + # avocado.yaml stays because cad-compile.sh reads the version out of it. package_files: - - agent - overlay - avocado.yaml - cad-*.sh @@ -72,15 +74,9 @@ extensions: compile: avocado-container-agent-dev install: cad-install.sh - sdk: - packages: - nativesdk-binutils: '*' - nativesdk-cargo: '*' - nativesdk-gcc: '*' - nativesdk-glibc-dev: '*' - nativesdk-libgcc-dev: '*' - nativesdk-rust: '*' - packagegroup-rust-cross-canadian-avocado-{{ avocado.target }}: '*' + # No SDK toolchain. Nothing is compiled at build time any more, so a consumer + # needs no Rust cross toolchain in their SDK - which is the point of the + # switch. # Release-scoped, not channel-scoped: this resolves to avocadolinux/sdk:2026. # The SDK image is per-release only - appending the channel points at a tag that @@ -93,8 +89,7 @@ sdk: avocado-container-agent-dev: compile: cad-compile.sh clean: cad-clean.sh - packages: - libstd-rs: '*' - libstd-rs-dev: '*' + # No libstd-rs. The hook fetches and unpacks a prebuilt binary; it runs no + # compiler, so the Rust runtime libraries it used to need are dead weight. packages: avocado-sdk-toolchain: '*' diff --git a/cad-clean.sh b/cad-clean.sh index 772828a..ef05bb5 100755 --- a/cad-clean.sh +++ b/cad-clean.sh @@ -1,10 +1,22 @@ #!/bin/bash -set -e +# Remove what cad-compile.sh staged: the downloaded tarball, SHA256SUMS and the +# unpacked binary. Nothing is compiled any more, so there is no cargo state to +# clean, and a consumer has no cargo to clean it with. +set -euo pipefail -echo "Cleaning avocado-container-agent-dev build artifacts" - -cd "$(dirname "$0")/agent" - -cargo clean --target-dir "$AVOCADO_BUILD_DIR" +# `avocado sdk clean` sets AVOCADO_BUILD_DIR, but `avocado ext clean` passes +# only AVOCADO_SDK_PREFIX. Fall back to the directory the compile step uses, +# $AVOCADO_SDK_PREFIX/build/, and refuse rather than remove a +# path built from an empty variable. +if [ -n "${AVOCADO_BUILD_DIR:-}" ]; then + BUILD_DIR="$AVOCADO_BUILD_DIR" +elif [ -n "${AVOCADO_SDK_PREFIX:-}" ]; then + BUILD_DIR="${AVOCADO_SDK_PREFIX}/build/avocado-container-agent-dev" +else + echo "Error: neither AVOCADO_BUILD_DIR nor AVOCADO_SDK_PREFIX is set; not removing anything" >&2 + exit 1 +fi +echo "Cleaning avocado-container-agent-dev staged files in ${BUILD_DIR}/agent" +rm -rf "${BUILD_DIR}/agent" echo "avocado-container-agent-dev cleaned successfully" diff --git a/cad-compile.sh b/cad-compile.sh index 795aa61..95d7baa 100755 --- a/cad-compile.sh +++ b/cad-compile.sh @@ -1,103 +1,159 @@ #!/bin/bash +# Stage the prebuilt agent binary for this target so cad-install.sh can place it. +# +# This fetches a release artifact rather than compiling from source, as +# ext-microclaw does, with one difference: mc-compile.sh pins a sha256 per +# architecture in the script, and this script reads its digests from the same +# release as the artifact. The reason for fetching is not laziness about +# cross-compiling: `avocado ext package` runs no build step, so the published RPM +# can only carry what is in the repo. Shipping the crate source instead pushed the +# compile onto the consumer, and a consumer has no reason to have a Rust cross +# toolchain in their SDK. Measured 2026-09-15: a clean project following the +# documented setup failed inside ring's build script with +# `ToolNotFound: failed to find tool "x86_64-avocado-linux-gcc"`, after both +# `avocado install` and the extension's own sysroot creation had reported success. +# +# The binaries are statically linked against musl, so they carry no libc version +# coupling and run on any Avocado target of the right architecture regardless of +# what the device's glibc is. set -euo pipefail -RUST_TARGET="" -match_count=0 -matches="" +VERSION="${AVOCADO_EXT_VERSION:-}" +if [ -z "$VERSION" ]; then + # The first indented `version:` is the extension's own. Accept it bare or + # quoted: `avocado ext package` rewrites it to `version: '0.2.0'` in the + # avocado.yaml a consumer receives, so the unquoted form only ever appears in + # this repo. Keep a prerelease suffix, since the release workflow tags + # `X.Y.Z-*` and those assets are named after the full tag. + VERSION=$(sed -nE "/^[[:space:]]+version:/{s/^[[:space:]]+version:[[:space:]]*['\"]?([0-9]+\.[0-9]+\.[0-9]+(-[0-9A-Za-z.-]+)?)['\"]?[[:space:]]*(#.*)?\$/\1/p;q;}" \ + "$(dirname "$0")/avocado.yaml") +fi +if [ -z "$VERSION" ]; then + echo "Error: could not determine the extension version" >&2 + exit 1 +fi -# Resolve the Rust target from RUST_TARGET_PATH. Enumerate every candidate -# instead of stopping at the first: an OECORE_TARGET_ARCH that prefix-matches -# two target JSONs used to take whichever the glob happened to return first and -# discard the rest without a word. That is the worst failure shape available -# here - a binary for the wrong architecture packages, publishes and installs -# successfully, and only fails when the device tries to exec it, several layers -# from the cause. Refusing to guess keeps the failure local and attributable. -# -# devtool-debt: this block is duplicated, still unfixed, in the three sibling -# extensions that have not yet moved to their own ext-* repo - cli, connect and -# tunnels - in both their compile and install scripts. jtop and microclaw carry -# a copy too, but theirs are superseded: both migrated to ext-* repos, and -# ext-microclaw resolves the architecture with a plain case statement that does -# not have this defect at all. -# Ceiling: those three are safe by GLOB ORDERING ALONE, not by construction. -# Every target whose architecture matches the SDK host architecture has two -# matching JSONs - the device triple and the SDK's own -avocadosdk-* - and -# first-match-and-break survives only because "avocado-" sorts ahead of -# "avocadosdk-" on the hyphen. Nothing enforces that ordering. This is not a -# theoretical ceiling: it is exactly what broke qemux86-64 here the first time -# the guard below ran. -# Upgrade trigger: a vendor string that reorders those two, a third triple -# sorting between them, or any of the three gaining a target whose architecture -# prefix-matches more than one device triple. -for json_file in "$RUST_TARGET_PATH"/*.json; do - if [ -f "$json_file" ]; then - json_name=$(basename "$json_file" .json) +# Map the SDK's architecture onto the published artifact. An unmapped +# architecture fails loudly here rather than installing nothing and leaving the +# agent silently absent from the image. +case "${OECORE_TARGET_ARCH:-}" in +x86_64) ASSET_ARCH="x86_64-unknown-linux-musl" ;; +aarch64) ASSET_ARCH="aarch64-unknown-linux-musl" ;; +"") + echo "Error: OECORE_TARGET_ARCH is unset; cannot choose an artifact" >&2 + exit 1 + ;; +*) + echo "Error: no prebuilt agent for OECORE_TARGET_ARCH=${OECORE_TARGET_ARCH}" >&2 + echo "Published artifacts cover x86_64 and aarch64 only. Add the architecture" >&2 + echo "to the binaries matrix in .github/workflows/release.yml and re-release." >&2 + exit 1 + ;; +esac - # Skip the SDK's own nativesdk triple. It shares the architecture prefix - # with the device triple whenever the SDK host arch matches the target - # arch, which on x86_64 means RUST_TARGET_PATH holds both - # x86_64-avocado-linux-gnu and x86_64-avocadosdk-linux-gnu. The - # avocadosdk vendor is host-side by construction - avocado-cli tags every - # nativesdk artifact _avocadosdk - so it is never a candidate for a - # device binary and must not count as a competing match. - case "$json_name" in - "${OECORE_TARGET_ARCH}-avocadosdk-"*) continue ;; - esac +ASSET="avocado-container-agent-dev-${VERSION}-${ASSET_ARCH}.tar.gz" +BASE="https://github.com/avocado-linux/ext-container-agent-dev/releases/download/${VERSION}" - if [[ "$json_name" == "${OECORE_TARGET_ARCH}-"* ]]; then - RUST_TARGET="$json_name" - match_count=$((match_count + 1)) - matches="$matches $json_name" - fi - fi -done +# A tarball and SHA256SUMS placed in dist/ next to this script are used instead +# of the release, with the same file names the release workflow produces. This +# lets a build run against binaries that are not released yet, and without +# network access. +PREBUILT_DIR="$(cd "$(dirname "$0")" && pwd)/dist" -if [ -z "$RUST_TARGET" ]; then - echo "Error: Could not find Rust target for $OECORE_TARGET_ARCH" >&2 - exit 1 -fi +STAGE_DIR="${AVOCADO_BUILD_DIR}/agent" +mkdir -p "$STAGE_DIR" +cd "$STAGE_DIR" -if [ "$match_count" -gt 1 ]; then - echo "Error: $OECORE_TARGET_ARCH matches $match_count Rust targets:$matches" >&2 - echo "Error: refusing to pick one - the wrong triple yields a binary that packages and installs but cannot exec on the device." >&2 - exit 1 -fi +CLEAN_HINT="Run 'avocado ext clean' (or 'avocado sdk clean') to discard the cached download in ${STAGE_DIR}, then build again." -# Stable marker: the resolution fixture test greps for this to tell "resolution -# succeeded and a later step failed" from "resolution itself failed", which exit -# code alone cannot distinguish when the test runs without a real SDK. Load- -# bearing test surface, not a debug echo. -echo "resolved-target: $RUST_TARGET" +# Print the digest SUMS_FILE records for ASSET, or nothing when it is not listed. +listed_digest() { + awk -v a="$ASSET" '$2 == a { print $1; exit }' "$1" +} -echo "Compiling avocado-container-agent-dev for target: $RUST_TARGET" +# True when FILE hashes to the digest SUMS_FILE records for ASSET. +matches_sums() { + local want got + want="$(listed_digest "$2")" + got="$(sha256sum "$1" | cut -d' ' -f1)" + [ -n "$want" ] && [ "$want" = "$got" ] +} -cd "$(dirname "$0")/agent" +# Download through a .part file and rename only on success, so an interrupted +# fetch never leaves a partial file behind under the final name. +fetch() { + echo "Fetching $1" + curl -fsSL --retry 3 --retry-delay 2 -o "$2.part" "$1" + mv "$2.part" "$2" +} -# Clear any rustflags that might cause conflicts with our .cargo/config.toml -unset RUSTFLAGS -unset CARGO_BUILD_RUSTFLAGS -for var in $(env | grep -o 'CARGO_TARGET_[A-Z0-9_]*_RUSTFLAGS'); do - unset "$var" -done +echo "resolved-artifact: ${ASSET}" -# Remove any existing config that might conflict -rm -rf .cargo +# SHA256SUMS comes from the same release as the artifact, so this verifies the +# transfer, not the provenance. Anyone who can replace the release assets can +# replace both files together and this check still passes. The digests are not +# pinned anywhere outside the release; dist/ is where a pinned copy would go. +if [ -f "${PREBUILT_DIR}/${ASSET}" ] && [ -f "${PREBUILT_DIR}/SHA256SUMS" ]; then + echo "Using ${PREBUILT_DIR}/${ASSET} instead of the ${VERSION} release" + SUMS="${PREBUILT_DIR}/SHA256SUMS" + if [ -z "$(listed_digest "$SUMS")" ]; then + echo "Error: ${SUMS} does not list ${ASSET}" >&2 + exit 1 + fi + if ! matches_sums "${PREBUILT_DIR}/${ASSET}" "$SUMS"; then + echo "Error: ${PREBUILT_DIR}/${ASSET} does not match ${SUMS}" >&2 + exit 1 + fi + TARBALL="${PREBUILT_DIR}/${ASSET}" +else + if [ -f "${PREBUILT_DIR}/${ASSET}" ] || [ -f "${PREBUILT_DIR}/SHA256SUMS" ]; then + echo "Warning: ${PREBUILT_DIR} holds only one of ${ASSET} and SHA256SUMS, so it is ignored and the ${VERSION} release is used" >&2 + fi + # Cached per version, so a rebuild needs no network once both files are + # here, and a version bump never reads the previous version's sums. + SUMS="SHA256SUMS-${VERSION}" + if [ ! -f "$SUMS" ]; then + fetch "${BASE}/SHA256SUMS" "$SUMS" + elif [ -z "$(listed_digest "$SUMS")" ]; then + # A cached file that does not list this asset may be a page a proxy + # served with a 200, or a copy fetched before the release was complete. + # Fetch it once more rather than failing on it for every later build. + echo "Cached ${SUMS} does not list ${ASSET}; fetching it again" + fetch "${BASE}/SHA256SUMS" "$SUMS" + fi + if [ -z "$(listed_digest "$SUMS")" ]; then + echo "Error: SHA256SUMS from the ${VERSION} release does not list ${ASSET}" >&2 + echo "The release is incomplete for this architecture; do not install it." >&2 + echo "${CLEAN_HINT}" >&2 + exit 1 + fi -# Create config.toml with cross-compilation settings -mkdir -p .cargo -cat > .cargo/config.toml << EOF -[target.$RUST_TARGET] -rustflags = ["--sysroot=$SDKTARGETSYSROOT/usr", "-C", "link-arg=--sysroot=$SDKTARGETSYSROOT"] -EOF + if [ -f "$ASSET" ] && ! matches_sums "$ASSET" "$SUMS"; then + echo "Cached ${ASSET} does not match ${SUMS}; downloading it again" + rm -f "$ASSET" + fi + if [ ! -f "$ASSET" ]; then + echo "Fetching ${BASE}/${ASSET}" + curl -fsSL --retry 3 --retry-delay 2 -o "${ASSET}.part" "${BASE}/${ASSET}" + # Verify before the rename, so a file this script writes under the final + # name has matched the sums. + if ! matches_sums "${ASSET}.part" "$SUMS"; then + rm -f "${ASSET}.part" + echo "Error: the downloaded ${ASSET} does not match ${SUMS}" >&2 + echo "$CLEAN_HINT" >&2 + exit 1 + fi + mv "${ASSET}.part" "$ASSET" + fi + TARBALL="$ASSET" +fi -# Use a persistent cargo registry cache to avoid re-downloading crates -export CARGO_HOME="${AVOCADO_BUILD_DIR}/.cargo-cache" +tar -xzf "$TARBALL" -# Enforce the ring-only rustls crypto provider before building. TARGET is -# mandatory here so the guard resolves the same SDK triple the build ships; -# without it the guard would check the host triple instead. Runs under set -e, -# so a non-zero exit (aws-lc-rs present, or a guard error) fails the compile. -TARGET="$RUST_TARGET" scripts/check-provider.sh -cargo build --release --target "$RUST_TARGET" --target-dir "$AVOCADO_BUILD_DIR" +if [ ! -f avocado-container-agent-dev ]; then + echo "Error: ${ASSET} did not contain avocado-container-agent-dev" >&2 + exit 1 +fi -echo "avocado-container-agent-dev compiled successfully" +chmod 0755 avocado-container-agent-dev +echo "avocado-container-agent-dev ${VERSION} staged for ${ASSET_ARCH}" diff --git a/cad-install.sh b/cad-install.sh index 856cfcf..17d3dbf 100755 --- a/cad-install.sh +++ b/cad-install.sh @@ -1,62 +1,44 @@ #!/bin/bash +# Place the binary cad-compile.sh staged into the extension sysroot. set -euo pipefail -RUST_TARGET="" -match_count=0 -matches="" +BINARY_PATH="${AVOCADO_BUILD_DIR}/agent/avocado-container-agent-dev" -# Resolution must stay byte-for-byte equivalent to cad-compile.sh's. The two run -# in separate invocations against the same SDK, so a divergence here silently -# installs a different triple's binary than the one that was compiled - the -# install would look successful and ship the wrong artifact. Enumerate every -# candidate rather than stopping at the first; see cad-compile.sh for why an -# ambiguous match must refuse rather than pick. -for json_file in "$RUST_TARGET_PATH"/*.json; do - if [ -f "$json_file" ]; then - json_name=$(basename "$json_file" .json) - - # Skip the SDK's own nativesdk triple - see cad-compile.sh for why. This - # exclusion must stay identical in both scripts: if only one of them - # skipped it, that script would refuse while the other resolved, and the - # extension would compile but fail to install (or worse, install a - # host-triple binary onto the device). - case "$json_name" in - "${OECORE_TARGET_ARCH}-avocadosdk-"*) continue ;; - esac - - if [[ "$json_name" == "${OECORE_TARGET_ARCH}-"* ]]; then - RUST_TARGET="$json_name" - match_count=$((match_count + 1)) - matches="$matches $json_name" - fi - fi -done - -# Same guard cad-compile.sh has. Without it an unmatched loop leaves RUST_TARGET -# empty, BINARY_PATH collapses to "$AVOCADO_BUILD_DIR//release/...", and the -f -# check below reports a missing binary instead of the actual fault. -if [ -z "$RUST_TARGET" ]; then - echo "Error: Could not find Rust target for $OECORE_TARGET_ARCH" >&2 - exit 1 -fi - -if [ "$match_count" -gt 1 ]; then - echo "Error: $OECORE_TARGET_ARCH matches $match_count Rust targets:$matches" >&2 - echo "Error: refusing to pick one - installing a different triple's binary than was compiled would ship a wrong-architecture artifact that looks installed." >&2 +if [ ! -f "$BINARY_PATH" ]; then + echo "Error: no staged binary at $BINARY_PATH" >&2 + echo "cad-compile.sh fetches it; this script only places it." >&2 exit 1 fi -# Stable marker, same contract as cad-compile.sh's: the resolution fixture test -# greps for it to separate a successful resolution from a later failure. -echo "resolved-target: $RUST_TARGET" - -BINARY_PATH="$AVOCADO_BUILD_DIR/$RUST_TARGET/release/avocado-container-agent-dev" - -if [ ! -f "$BINARY_PATH" ]; then - echo "Error: Binary not found at $BINARY_PATH" +# The staged artifact is architecture-specific and the packaged extension is +# noarch, so nothing downstream re-checks that the two agree. Verify here, where +# the failure is one line from its cause: a mismatch otherwise ships an +# unrunnable binary that installs cleanly and fails at exec time on the device, +# past every gate that could have caught it. +case "${OECORE_TARGET_ARCH:-}" in +x86_64) EXPECT="x86-64" ;; +aarch64) EXPECT="aarch64" ;; +*) + echo "Error: unexpected OECORE_TARGET_ARCH=${OECORE_TARGET_ARCH:-unset}" >&2 exit 1 + ;; +esac + +if command -v file >/dev/null 2>&1; then + DESC=$(file -b "$BINARY_PATH") + case "$DESC" in + *"$EXPECT"*) ;; + *) + echo "Error: staged binary is not ${EXPECT}: ${DESC}" >&2 + exit 1 + ;; + esac + echo "verified-arch: ${EXPECT}" +else + echo "Warning: 'file' unavailable; shipping the staged binary unverified" >&2 fi echo "Installing avocado-container-agent-dev into extension" -install -D -m 755 "$BINARY_PATH" "$AVOCADO_BUILD_EXT_SYSROOT/usr/bin/avocado-container-agent-dev" +install -D -m 755 "$BINARY_PATH" \ + "$AVOCADO_BUILD_EXT_SYSROOT/usr/bin/avocado-container-agent-dev" echo "avocado-container-agent-dev installed successfully"