diff --git a/.github/workflows/release.yml b/.github/workflows/release.yml index bb10286..893173d 100644 --- a/.github/workflows/release.yml +++ b/.github/workflows/release.yml @@ -3,6 +3,11 @@ name: Release # Package + publish on a version tag. The matrix lives here (the caller owns it); # each leg calls the single-combo reusable helper. The publish feed is derived from # distro-release/distro-channel. The tag must equal the avocado.yaml version. +# +# One row per target in avocado.yaml `supported_targets`. Keep the two lists in +# step: a target published here but not declared there ships something the +# extension does not claim to support, and the reverse claims support for +# something never built. on: push: tags: @@ -13,11 +18,16 @@ on: jobs: publish: strategy: + # Each target reports its own outcome. Without this one target's failure + # cancels the rest, and a partial publish looks like a total one. fail-fast: false matrix: include: - { target: qemux86-64, distro-release: "2024", distro-channel: next } - # Add once your target is published in the 2026 feed: + - { target: imx93-frdm, distro-release: "2024", distro-channel: next } + # Add once these targets are published in the 2026 feed. As of + # 2026-09-09 the 2026 feed carries jetson-agx-thor, qemuarm64 and + # qemux86-64 only, so an imx93-frdm row there would fail. # - { target: qemux86-64, distro-release: "2026", distro-channel: next } uses: avocado-linux/actions/.github/workflows/extension-release.yml@v1 with: diff --git a/.gitignore b/.gitignore new file mode 100644 index 0000000..39f33ee --- /dev/null +++ b/.gitignore @@ -0,0 +1,11 @@ +# Cargo build output for the staged crate. +/agent/target/ + +# Cross-compilation config written by cad-compile.sh at build time. It pins one +# machine's SDK sysroot path, so a committed copy is wrong for everyone else. +/agent/.cargo/ + +# avocado-cli build state. .avocado-state is a JSON file, not a directory, so it +# needs a slash-free pattern - a trailing slash silently fails to match it. +/.avocado-state +/avocado.lock diff --git a/README.md b/README.md index 83f39af..cb71006 100644 --- a/README.md +++ b/README.md @@ -1,64 +1,68 @@ -# ext-template - -GitHub template repo for new Avocado extensions. **Use this template** → then work -through the checklist below. - -## New-extension checklist - -1. Rename the repo `ext-` (or `bsp-`). -2. `avocado.yaml` — rename the extension key to `avocado-ext-`, fill in - `summary`/`description`, set `supported_targets` (and `default_target` for a BSP), - list your `packages`. Leave the `sdk.image` line alone. -3. `.github/workflows/test.yml` — set the matrix `target` to something the extension - actually supports (`qemux86-64` is fine for target-agnostic extensions). -4. `.github/workflows/release.yml` — same, plus add matrix rows for every feed you - publish into. -5. `CHANGELOG.md` — replace the placeholder `0.1.0` entry. -6. Delete this section from the README and describe the extension instead (see - "Using this extension" below — keep that part). -7. Repo secrets `AVOCADO_CONNECT_TOKEN` and `AVOCADO_CONNECT_ORG` must be set (org-level - secrets cover this if the repo is in `avocado-linux`). - -Release: tag the commit with the exact `avocado.yaml` version, e.g. `git tag 0.1.0 && git push --tags`. - -## Conventions - -- **SDK image is release-scoped, not channel-scoped.** - `docker.io/avocadolinux/sdk:{{ env.AVOCADO_DISTRO_RELEASE }}` → `avocadolinux/sdk:2026` - on a 2026 CI leg, `avocadolinux/sdk:2024` on a 2024 one. Do not append - `-{{ env.AVOCADO_DISTRO_CHANNEL }}` — there is no such tag on any release. The channel - selects the *package feed*; the image tag is per-release only. -- **CI comes from `avocado-linux/actions@v1`**, pinned. The matrix lives in the caller so - each repo owns its own target/feed combinations. -- **Feed today is `2024`/`edge-next`.** `avocado-linux/actions@v1` *defaults* to - `2026`/`edge`, but the 2026 feed currently publishes only `jetson-agx-thor`, so the - workflows here pass 2024 explicitly. Move a target to 2026 as it lands there. - -## Using this extension - -`ext-template` is an [Avocado](https://avocadolinux.org) extension — a reusable fragment of -build- and runtime-configuration that you compose into your own Avocado project. To use it, -declare it as a package-sourced extension in your `avocado.yaml` and add it to a runtime: - -```yaml -extensions: - avocado-ext-template: - source: - type: package - version: "*" # or pin an exact version - -runtimes: - my-runtime: - extensions: - - avocado-ext-template -``` +# ext-container-agent-dev + +`avocado-ext-container-agent-dev` - the device side of Container Dev Mode. + +The agent holds a control WebSocket open to the host CLI, proxies image pulls +from the host's registry over a bootstrap-delivered pinned CA, and hot-reloads +the target container when a sync lands. It is what makes `avocado container dev` +transfer only the changed layer instead of a whole image. + +**Development only.** This extension exists so it can be composed into a dev +runtime and left out of production. The `-dev` suffix is the gate. Do not ship it +on a production device. + +## Layout + +| Path | What it is | +|---|---| +| `agent/` | the Rust crate, cross-compiled at package time | +| `cad-compile.sh` | resolves the target triple and builds the crate | +| `cad-install.sh` | stages the built binary into the extension sysroot | +| `cad-clean.sh` | clears the build directory | +| `overlay/` | the `container-agent-dev` path and service units | +| `avocado.yaml` | extension manifest | + +## Requires a container engine + +The agent execs the engine CLI as its own child on every sync, so a runtime +without the `docker` extension gives it nothing to do. That dependency is stated +rather than declared: avocado-cli parses an inter-extension dependency and logs +it without installing anything, so declaring it would read as a guarantee and +deliver none. Compose both into the runtime yourself. The agent's startup +preflight is what makes a missing engine fail loudly rather than silently. -Then install and build: +## Building locally -```sh -avocado install # fetches + installs the SDK, extensions and runtime deps from your config -avocado build # builds the SDK compile steps, extensions and runtime images +The SDK image tag is release-scoped and comes from the environment, so export it +before building: + +``` +AVOCADO_DISTRO_RELEASE=2024 avocado ext install -t qemux86-64 avocado-ext-container-agent-dev +AVOCADO_DISTRO_RELEASE=2024 avocado ext build -t qemux86-64 avocado-ext-container-agent-dev ``` -`avocado install` pulls the extension from your target's package feed and merges its -config into your project; `avocado build` then produces the runtime. +Two things that will otherwise cost you time: + +- `avocado` runs its SDK container with a TTY, so a build launched without one + fails with `cannot attach stdin to a TTY-enabled container`. Wrap it in + `script -qefc '' /dev/null` when running from CI or a background shell. +- dnf prompts for confirmation and blocks; `--no-tui` does not suppress it. Pass + `--dnf-arg -y`. + +## Supported targets + +`avocado.yaml` lists them explicitly rather than using `'*'`. A target belongs +there once it has produced an installable package and the produced binary has +been seen to exec on that architecture. The release matrix in +`.github/workflows/release.yml` should carry one row per declared target. + +## Releasing + +Tag the version in `avocado.yaml`. The release workflow guards that the tag and +the manifest version match, packages one leg per matrix row, and publishes each +to the feed derived from that row's distro release and channel. + +Bump `release:` whenever the packaged payload changes without a version change. +An unchanged version-release identity means the package manager never delivers +the new payload to a device that already has the extension installed, and both +the build and the install report success anyway. diff --git a/agent/.gitignore b/agent/.gitignore new file mode 100644 index 0000000..50abe41 --- /dev/null +++ b/agent/.gitignore @@ -0,0 +1,3 @@ +/target +# Cross-compilation config written by cad-compile.sh at build time. +/.cargo/ diff --git a/agent/Cargo.lock b/agent/Cargo.lock new file mode 100644 index 0000000..cbf96e0 --- /dev/null +++ b/agent/Cargo.lock @@ -0,0 +1,1098 @@ +# This file is automatically @generated by Cargo. +# It is not intended for manual editing. +version = 4 + +[[package]] +name = "aho-corasick" +version = "1.1.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ddd31a130427c27518df266943a5308ed92d4b226cc639f5a8f1002816174301" +dependencies = [ + "memchr", +] + +[[package]] +name = "anyhow" +version = "1.0.104" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "330a5ed07fa54e4702c9d6c4174f74427fc0ef6e214bbd677ae50a5099946470" + +[[package]] +name = "atomic-waker" +version = "1.1.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1505bd5d3d116872e7271a6d4e16d81d0c8570876c8de68093a09ac269d8aac0" + +[[package]] +name = "avocado-container-agent-dev" +version = "0.1.0" +dependencies = [ + "anyhow", + "bytes", + "futures-util", + "http-body-util", + "hyper", + "hyper-util", + "rcgen", + "rustls", + "rustls-pemfile", + "serde", + "serde_json", + "tempfile", + "tokio", + "tokio-rustls", + "tokio-tungstenite", + "tracing", + "tracing-subscriber", +] + +[[package]] +name = "base64" +version = "0.22.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "72b3254f16251a8381aa12e40e3c4d2f0199f8c6508fbecb9d91f575e0fbb8c6" + +[[package]] +name = "bitflags" +version = "2.13.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b588b76d00fde79687d7646a9b5bdf3cc0f655e0bbd080335a95d7e96f3587da" + +[[package]] +name = "block-buffer" +version = "0.10.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3078c7629b62d3f0439517fa394996acacc5cbc91c5a20d8c658e77abd503a71" +dependencies = [ + "generic-array", +] + +[[package]] +name = "bytes" +version = "1.12.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "fc652a48c352aef3ea3aed32080501cf3ef6ed5da78602a020c991775b0aff04" + +[[package]] +name = "cc" +version = "1.3.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c89588d05638b5b4594a3348a2d6c20277e43a7f5c5202b05cc56888475a47b8" +dependencies = [ + "find-msvc-tools", + "shlex", +] + +[[package]] +name = "cfg-if" +version = "1.0.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9330f8b2ff13f34540b44e946ef35111825727b38d33286ef986142615121801" + +[[package]] +name = "cpufeatures" +version = "0.2.17" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "59ed5838eebb26a2bb2e58f6d5b5316989ae9d08bab10e0e6d103e656d1b0280" +dependencies = [ + "libc", +] + +[[package]] +name = "crypto-common" +version = "0.1.7" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "78c8292055d1c1df0cce5d180393dc8cce0abec0a7102adb6c7b1eef6016d60a" +dependencies = [ + "generic-array", + "typenum", +] + +[[package]] +name = "data-encoding" +version = "2.11.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a4ae5f15dda3c708c0ade84bfee31ccab44a3da4f88015ed22f63732abe300c8" + +[[package]] +name = "deranged" +version = "0.5.8" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7cd812cc2bc1d69d4764bd80df88b4317eaef9e773c75226407d9bc0876b211c" + +[[package]] +name = "digest" +version = "0.10.7" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9ed9a281f7bc9b7576e61468ba615a66a5c8cfdff42420a70aa82701a3b1e292" +dependencies = [ + "block-buffer", + "crypto-common", +] + +[[package]] +name = "errno" +version = "0.3.14" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "39cab71617ae0d63f51a36d69f866391735b51691dbda63cf6f96d042b63efeb" +dependencies = [ + "libc", + "windows-sys 0.61.2", +] + +[[package]] +name = "fastrand" +version = "2.5.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "da7c62ceae207dd37ea5b845da6a0696c799f85e97da1ab5b7910be3c1c80223" + +[[package]] +name = "find-msvc-tools" +version = "0.1.9" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5baebc0774151f905a1a2cc41989300b1e6fbb29aff0ceffa1064fdd3088d582" + +[[package]] +name = "futures-channel" +version = "0.3.33" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "262590f4fe6afeb0bc83be1daa64e52657fe185690a958af7f3ad0e92085c5ae" +dependencies = [ + "futures-core", +] + +[[package]] +name = "futures-core" +version = "0.3.33" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2cd50c473c80f6d7c3670a752354b8e569b1a7cbfdc0419ec88e5edad85e0dc7" + +[[package]] +name = "futures-macro" +version = "0.3.33" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2d6d3cde68c518367be28956066ddfef33813991b77a55005a69dae04bf3b10b" +dependencies = [ + "proc-macro2", + "quote", + "syn 2.0.119", +] + +[[package]] +name = "futures-sink" +version = "0.3.33" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e34418ac499d6305c2fb5ad0ed2f6ac998c5f8ca209b4510f7f94242c647e307" + +[[package]] +name = "futures-task" +version = "0.3.33" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b231ed28831efb4a61a08580c4bc233ec56bc009f4cd8f52da2c3cb97df0c109" + +[[package]] +name = "futures-util" +version = "0.3.33" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a77a90a256fce34da66415271e30f94ee91c57b04b8a2c042d9cf3220179deaa" +dependencies = [ + "futures-core", + "futures-macro", + "futures-sink", + "futures-task", + "pin-project-lite", + "slab", +] + +[[package]] +name = "generic-array" +version = "0.14.7" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "85649ca51fd72272d7821adaf274ad91c288277713d9c18820d8499a7ff69e9a" +dependencies = [ + "typenum", + "version_check", +] + +[[package]] +name = "getrandom" +version = "0.2.17" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ff2abc00be7fca6ebc474524697ae276ad847ad0a6b3faa4bcb027e9a4614ad0" +dependencies = [ + "cfg-if", + "libc", + "wasi", +] + +[[package]] +name = "getrandom" +version = "0.3.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "899def5c37c4fd7b2664648c28120ecec138e4d395b459e5ca34f9cce2dd77fd" +dependencies = [ + "cfg-if", + "libc", + "r-efi", + "wasip2", +] + +[[package]] +name = "http" +version = "1.4.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6970f50e31d6fc17d3fa27329444bfa74e196cf62e95052a3f6fee181dba6425" +dependencies = [ + "bytes", + "itoa", +] + +[[package]] +name = "http-body" +version = "1.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ca2a8f2913ee65f60facd6a5905613afaa448497a0230cc41ce022d93290bc2c" +dependencies = [ + "bytes", + "http", +] + +[[package]] +name = "http-body-util" +version = "0.1.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e9f41fd6a08e4d4ec69df65976da761afd5ad5e58a9d4acb46bd1c953a9e3ff2" +dependencies = [ + "bytes", + "futures-core", + "http", + "http-body", + "pin-project-lite", +] + +[[package]] +name = "httparse" +version = "1.10.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6dbf3de79e51f3d586ab4cb9d5c3e2c14aa28ed23d180cf89b4df0454a69cc87" + +[[package]] +name = "httpdate" +version = "1.0.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "df3b46402a9d5adb4c86a0cf463f42e19994e3ee891101b1841f30a545cb49a9" + +[[package]] +name = "hyper" +version = "1.11.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d22053281f852e11534f5198498373cbb59295120a20771d90f7ed1897490a72" +dependencies = [ + "atomic-waker", + "bytes", + "futures-channel", + "futures-core", + "http", + "http-body", + "httparse", + "httpdate", + "itoa", + "pin-project-lite", + "smallvec", + "tokio", + "want", +] + +[[package]] +name = "hyper-util" +version = "0.1.20" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "96547c2556ec9d12fb1578c4eaf448b04993e7fb79cbaad930a656880a6bdfa0" +dependencies = [ + "bytes", + "http", + "http-body", + "hyper", + "pin-project-lite", + "tokio", +] + +[[package]] +name = "itoa" +version = "1.0.18" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8f42a60cbdf9a97f5d2305f08a87dc4e09308d1276d28c869c684d7777685682" + +[[package]] +name = "lazy_static" +version = "1.5.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "bbd2bcb4c963f2ddae06a2efc7e9f3591312473c50c6685e1f298068316e66fe" + +[[package]] +name = "libc" +version = "0.2.189" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3eaf3ede3fee6db1a4c2ee091bf8a8b4dccdc6d17f656fb07896ee72867612f2" + +[[package]] +name = "linux-raw-sys" +version = "0.12.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "32a66949e030da00e8c7d4434b251670a91556f4144941d37452769c25d58a53" + +[[package]] +name = "log" +version = "0.4.33" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0ceec5bc11778974d1bcb055b18002eba7f4b3518b6a0081b3af5f21666da9ad" + +[[package]] +name = "matchers" +version = "0.2.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d1525a2a28c7f4fa0fc98bb91ae755d1e2d1505079e05539e35bc876b5d65ae9" +dependencies = [ + "regex-automata", +] + +[[package]] +name = "memchr" +version = "2.8.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "cf8baf1c55e62ffcace7a9f06f4bd9cd3f0c4beb022d3b367256b91b87513d98" + +[[package]] +name = "mio" +version = "1.2.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "30d65c71f1ce40ab09135ce117d742b9f8a19ff91a41a8b57ed50bc2de59c427" +dependencies = [ + "libc", + "wasi", + "windows-sys 0.61.2", +] + +[[package]] +name = "nu-ansi-term" +version = "0.50.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7957b9740744892f114936ab4a57b3f487491bbeafaf8083688b16841a4240e5" +dependencies = [ + "windows-sys 0.61.2", +] + +[[package]] +name = "num-conv" +version = "0.2.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "521739c6d2bac4aa25192232afe6841231376b2b26d4d9fae5ecf8ca5772e441" + +[[package]] +name = "once_cell" +version = "1.21.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9f7c3e4beb33f85d45ae3e3a1792185706c8e16d043238c593331cc7cd313b50" + +[[package]] +name = "pem" +version = "3.0.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1d30c53c26bc5b31a98cd02d20f25a7c8567146caf63ed593a9d87b2775291be" +dependencies = [ + "base64", + "serde_core", +] + +[[package]] +name = "pin-project-lite" +version = "0.2.17" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a89322df9ebe1c1578d689c92318e070967d1042b512afbe49518723f4e6d5cd" + +[[package]] +name = "powerfmt" +version = "0.2.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "439ee305def115ba05938db6eb1644ff94165c5ab5e9420d1c1bcedbba909391" + +[[package]] +name = "ppv-lite86" +version = "0.2.21" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "85eae3c4ed2f50dcfe72643da4befc30deadb458a9b590d720cde2f2b1e97da9" +dependencies = [ + "zerocopy", +] + +[[package]] +name = "proc-macro2" +version = "1.0.107" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "985e7ec9bb745e6ce6535b544d84d6cd6f7ad8bd711c398938ae983b91a766d9" +dependencies = [ + "unicode-ident", +] + +[[package]] +name = "quote" +version = "1.0.47" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1fbf4db142a473a8d80c26bbf18454ed458bf8d26c8219c331daecfdbd079001" +dependencies = [ + "proc-macro2", +] + +[[package]] +name = "r-efi" +version = "5.3.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "69cdb34c158ceb288df11e18b4bd39de994f6657d83847bdffdbd7f346754b0f" + +[[package]] +name = "rand" +version = "0.9.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b9ef1d0d795eb7d84685bca4f72f3649f064e6641543d3a8c415898726a57b41" +dependencies = [ + "rand_chacha", + "rand_core", +] + +[[package]] +name = "rand_chacha" +version = "0.9.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d3022b5f1df60f26e1ffddd6c66e8aa15de382ae63b3a0c1bfc0e4d3e3f325cb" +dependencies = [ + "ppv-lite86", + "rand_core", +] + +[[package]] +name = "rand_core" +version = "0.9.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "76afc826de14238e6e8c374ddcc1fa19e374fd8dd986b0d2af0d02377261d83c" +dependencies = [ + "getrandom 0.3.4", +] + +[[package]] +name = "rcgen" +version = "0.13.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "75e669e5202259b5314d1ea5397316ad400819437857b90861765f24c4cf80a2" +dependencies = [ + "pem", + "ring", + "rustls-pki-types", + "time", + "yasna", +] + +[[package]] +name = "regex-automata" +version = "0.4.16" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8fcfdb36bda0c880c5931cdc7a2bcdc8ba4556847b9d912bca70bc94708711ad" +dependencies = [ + "aho-corasick", + "memchr", + "regex-syntax", +] + +[[package]] +name = "regex-syntax" +version = "0.8.11" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d6f6ff9a378485b298a5286656da665ba74413d36db0979633275d2e708145d4" + +[[package]] +name = "ring" +version = "0.17.14" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a4689e6c2294d81e88dc6261c768b63bc4fcdb852be6d1352498b114f61383b7" +dependencies = [ + "cc", + "cfg-if", + "getrandom 0.2.17", + "libc", + "untrusted", + "windows-sys 0.52.0", +] + +[[package]] +name = "rustix" +version = "1.1.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b6fe4565b9518b83ef4f91bb47ce29620ca828bd32cb7e408f0062e9930ba190" +dependencies = [ + "bitflags", + "errno", + "libc", + "linux-raw-sys", + "windows-sys 0.61.2", +] + +[[package]] +name = "rustls" +version = "0.23.42" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3c54fcab019b409d04215d3a17cb438fd7fbf192ee61461f20f4fe18704bc138" +dependencies = [ + "log", + "once_cell", + "ring", + "rustls-pki-types", + "rustls-webpki", + "subtle", + "zeroize", +] + +[[package]] +name = "rustls-pemfile" +version = "2.2.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "dce314e5fee3f39953d46bb63bb8a46d40c2f8fb7cc5a3b6cab2bde9721d6e50" +dependencies = [ + "rustls-pki-types", +] + +[[package]] +name = "rustls-pki-types" +version = "1.15.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "764899a24af3980067ee14bc143654f297b22eaebfe3c7b6b211920a5a59b046" +dependencies = [ + "zeroize", +] + +[[package]] +name = "rustls-webpki" +version = "0.103.13" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "61c429a8649f110dddef65e2a5ad240f747e85f7758a6bccc7e5777bd33f756e" +dependencies = [ + "ring", + "rustls-pki-types", + "untrusted", +] + +[[package]] +name = "serde" +version = "1.0.229" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "4148590afebada386688f18773da617792bf2ef03ffc1e4cbd2b1d45b023e0ba" +dependencies = [ + "serde_core", + "serde_derive", +] + +[[package]] +name = "serde_core" +version = "1.0.229" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "67dca2c9c51e58a4791a4b1ed58308b39c64224d349a935ab5039aa360942a48" +dependencies = [ + "serde_derive", +] + +[[package]] +name = "serde_derive" +version = "1.0.229" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e7a5d71263a5a7d47b41f6b3f06ba276f10cc18b0931f1799f710578e2309348" +dependencies = [ + "proc-macro2", + "quote", + "syn 3.0.3", +] + +[[package]] +name = "serde_json" +version = "1.0.151" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c841b55ecdae098c80dcae9cf767f6f8a0c2cdb3416bbef72181df4d0fe73f14" +dependencies = [ + "itoa", + "memchr", + "serde", + "serde_core", + "zmij", +] + +[[package]] +name = "sha1" +version = "0.10.7" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a978451301f4db1d02937a4ab3ccce137717b81826e79b7d49ffe3244a13c3b8" +dependencies = [ + "cfg-if", + "cpufeatures", + "digest", +] + +[[package]] +name = "sharded-slab" +version = "0.1.7" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f40ca3c46823713e0d4209592e8d6e826aa57e928f09752619fc696c499637f6" +dependencies = [ + "lazy_static", +] + +[[package]] +name = "shlex" +version = "2.0.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f8fadd59c855ef2080decdef8ff161eb6661b86933c9d82e5ba29dc602a55aba" + +[[package]] +name = "signal-hook-registry" +version = "1.4.8" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c4db69cba1110affc0e9f7bcd48bbf87b3f4fc7c61fc9155afd4c469eb3d6c1b" +dependencies = [ + "errno", + "libc", +] + +[[package]] +name = "slab" +version = "0.4.12" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0c790de23124f9ab44544d7ac05d60440adc586479ce501c1d6d7da3cd8c9cf5" + +[[package]] +name = "smallvec" +version = "1.15.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8ed6a63f02c8539c91a8685a86f4099661ba3da017932f6ebbea6de3f0fa7c90" + +[[package]] +name = "socket2" +version = "0.6.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c3d1e2c7f27f8d4cb10542a02c49005dbd6e93095799d6f3be745fae9f8fedd4" +dependencies = [ + "libc", + "windows-sys 0.61.2", +] + +[[package]] +name = "subtle" +version = "2.6.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "13c2bddecc57b384dee18652358fb23172facb8a2c51ccc10d74c157bdea3292" + +[[package]] +name = "syn" +version = "2.0.119" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "872831b642d1a07999a962a351ed35b955ea2cfc8f3862091e2a240a84f17297" +dependencies = [ + "proc-macro2", + "quote", + "unicode-ident", +] + +[[package]] +name = "syn" +version = "3.0.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "53e9bae58849f64dfa4f5d5ae372c8341f7305f82a3868709269343628b659a3" +dependencies = [ + "proc-macro2", + "quote", + "unicode-ident", +] + +[[package]] +name = "tempfile" +version = "3.27.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "32497e9a4c7b38532efcdebeef879707aa9f794296a4f0244f6f69e9bc8574bd" +dependencies = [ + "fastrand", + "getrandom 0.3.4", + "once_cell", + "rustix", + "windows-sys 0.61.2", +] + +[[package]] +name = "thiserror" +version = "2.0.19" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "09a43598840e33d5b0331f38c5e30d13bb11c11210a4b58f0d9b18a5a5eefcd9" +dependencies = [ + "thiserror-impl", +] + +[[package]] +name = "thiserror-impl" +version = "2.0.19" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "43cbfe0cf76104d42a574802844187e84a305e531ed54455f11fbde0f10541cd" +dependencies = [ + "proc-macro2", + "quote", + "syn 3.0.3", +] + +[[package]] +name = "thread_local" +version = "1.1.10" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1ad99c4c6d32803332c548b1af0540b357b3f5fc0be8f6c6bfe8b2e6ae784070" +dependencies = [ + "cfg-if", +] + +[[package]] +name = "time" +version = "0.3.54" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3e1d5e639ff6bab73cb6885cc7e7b1de96c3f32c68ec55f3952614bec1092244" +dependencies = [ + "deranged", + "num-conv", + "powerfmt", + "serde_core", + "time-core", +] + +[[package]] +name = "time-core" +version = "0.1.9" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9e1c906769ad99c88eaa54e728060edef082f8e358ff32030cb7c7d315e81109" + +[[package]] +name = "tokio" +version = "1.53.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "202caea871b69668250d242070849eb495be178ed697a3e98aebce5bc81a0bed" +dependencies = [ + "bytes", + "libc", + "mio", + "pin-project-lite", + "signal-hook-registry", + "socket2", + "tokio-macros", + "windows-sys 0.61.2", +] + +[[package]] +name = "tokio-macros" +version = "2.7.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6328af13490e73a9b4694030fafd93f8c8c6a9dede33e821c3fc63eddf8042ba" +dependencies = [ + "proc-macro2", + "quote", + "syn 2.0.119", +] + +[[package]] +name = "tokio-rustls" +version = "0.26.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1729aa945f29d91ba541258c8df89027d5792d85a8841fb65e8bf0f4ede4ef61" +dependencies = [ + "rustls", + "tokio", +] + +[[package]] +name = "tokio-tungstenite" +version = "0.26.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7a9daff607c6d2bf6c16fd681ccb7eecc83e4e2cdc1ca067ffaadfca5de7f084" +dependencies = [ + "futures-util", + "log", + "tokio", + "tungstenite", +] + +[[package]] +name = "tracing" +version = "0.1.44" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "63e71662fa4b2a2c3a26f570f037eb95bb1f85397f3cd8076caed2f026a6d100" +dependencies = [ + "pin-project-lite", + "tracing-attributes", + "tracing-core", +] + +[[package]] +name = "tracing-attributes" +version = "0.1.31" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7490cfa5ec963746568740651ac6781f701c9c5ea257c58e057f3ba8cf69e8da" +dependencies = [ + "proc-macro2", + "quote", + "syn 2.0.119", +] + +[[package]] +name = "tracing-core" +version = "0.1.36" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "db97caf9d906fbde555dd62fa95ddba9eecfd14cb388e4f491a66d74cd5fb79a" +dependencies = [ + "once_cell", + "valuable", +] + +[[package]] +name = "tracing-log" +version = "0.2.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ee855f1f400bd0e5c02d150ae5de3840039a3f54b025156404e34c23c03f47c3" +dependencies = [ + "log", + "once_cell", + "tracing-core", +] + +[[package]] +name = "tracing-subscriber" +version = "0.3.23" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "cb7f578e5945fb242538965c2d0b04418d38ec25c79d160cd279bf0731c8d319" +dependencies = [ + "matchers", + "nu-ansi-term", + "once_cell", + "regex-automata", + "sharded-slab", + "smallvec", + "thread_local", + "tracing", + "tracing-core", + "tracing-log", +] + +[[package]] +name = "try-lock" +version = "0.2.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e421abadd41a4225275504ea4d6566923418b7f05506fbc9c0fe86ba7396114b" + +[[package]] +name = "tungstenite" +version = "0.26.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "4793cb5e56680ecbb1d843515b23b6de9a75eb04b66643e256a396d43be33c13" +dependencies = [ + "bytes", + "data-encoding", + "http", + "httparse", + "log", + "rand", + "sha1", + "thiserror", + "utf-8", +] + +[[package]] +name = "typenum" +version = "1.20.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b6f5e870be6c3b371b77fe0ee0bafb859fa4964b4404c27de1d380043c4dda20" + +[[package]] +name = "unicode-ident" +version = "1.0.24" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e6e4313cd5fcd3dad5cafa179702e2b244f760991f45397d14d4ebf38247da75" + +[[package]] +name = "untrusted" +version = "0.9.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8ecb6da28b8a351d773b68d5825ac39017e680750f980f3a1a85cd8dd28a47c1" + +[[package]] +name = "utf-8" +version = "0.7.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "09cc8ee72d2a9becf2f2febe0205bbed8fc6615b7cb429ad062dc7b7ddd036a9" + +[[package]] +name = "valuable" +version = "0.1.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ba73ea9cf16a25df0c8caa16c51acb937d5712a8429db78a3ee29d5dcacd3a65" + +[[package]] +name = "version_check" +version = "0.9.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0b928f33d975fc6ad9f86c8f283853ad26bdd5b10b7f1542aa2fa15e2289105a" + +[[package]] +name = "want" +version = "0.3.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "bfa7760aed19e106de2c7c0b581b509f2f25d3dacaf737cb82ac61bc6d760b0e" +dependencies = [ + "try-lock", +] + +[[package]] +name = "wasi" +version = "0.11.1+wasi-snapshot-preview1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ccf3ec651a847eb01de73ccad15eb7d99f80485de043efb2f370cd654f4ea44b" + +[[package]] +name = "wasip2" +version = "1.0.4+wasi-0.2.12" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b67efb37e106e55ce722a510d6b5f9c17f083e5fc79afc2badeb12cc313d9487" +dependencies = [ + "wit-bindgen", +] + +[[package]] +name = "windows-link" +version = "0.2.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f0805222e57f7521d6a62e36fa9163bc891acd422f971defe97d64e70d0a4fe5" + +[[package]] +name = "windows-sys" +version = "0.52.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "282be5f36a8ce781fad8c8ae18fa3f9beff57ec1b52cb3de0789201425d9a33d" +dependencies = [ + "windows-targets", +] + +[[package]] +name = "windows-sys" +version = "0.61.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ae137229bcbd6cdf0f7b80a31df61766145077ddf49416a728b02cb3921ff3fc" +dependencies = [ + "windows-link", +] + +[[package]] +name = "windows-targets" +version = "0.52.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9b724f72796e036ab90c1021d4780d4d3d648aca59e491e6b98e725b84e99973" +dependencies = [ + "windows_aarch64_gnullvm", + "windows_aarch64_msvc", + "windows_i686_gnu", + "windows_i686_gnullvm", + "windows_i686_msvc", + "windows_x86_64_gnu", + "windows_x86_64_gnullvm", + "windows_x86_64_msvc", +] + +[[package]] +name = "windows_aarch64_gnullvm" +version = "0.52.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "32a4622180e7a0ec044bb555404c800bc9fd9ec262ec147edd5989ccd0c02cd3" + +[[package]] +name = "windows_aarch64_msvc" +version = "0.52.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "09ec2a7bb152e2252b53fa7803150007879548bc709c039df7627cabbd05d469" + +[[package]] +name = "windows_i686_gnu" +version = "0.52.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8e9b5ad5ab802e97eb8e295ac6720e509ee4c243f69d781394014ebfe8bbfa0b" + +[[package]] +name = "windows_i686_gnullvm" +version = "0.52.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0eee52d38c090b3caa76c563b86c3a4bd71ef1a819287c19d586d7334ae8ed66" + +[[package]] +name = "windows_i686_msvc" +version = "0.52.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "240948bc05c5e7c6dabba28bf89d89ffce3e303022809e73deaefe4f6ec56c66" + +[[package]] +name = "windows_x86_64_gnu" +version = "0.52.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "147a5c80aabfbf0c7d901cb5895d1de30ef2907eb21fbbab29ca94c5b08b1a78" + +[[package]] +name = "windows_x86_64_gnullvm" +version = "0.52.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "24d5b23dc417412679681396f2b49f3de8c1473deb516bd34410872eff51ed0d" + +[[package]] +name = "windows_x86_64_msvc" +version = "0.52.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "589f6da84c646204747d1270a2a5661ea66ed1cced2631d546fdfb155959f9ec" + +[[package]] +name = "wit-bindgen" +version = "0.57.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1ebf944e87a7c253233ad6766e082e3cd714b5d03812acc24c318f549614536e" + +[[package]] +name = "yasna" +version = "0.5.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e17bb3549cc1321ae1296b9cdc2698e2b6cb1992adfa19a8c72e5b7a738f44cd" +dependencies = [ + "time", +] + +[[package]] +name = "zerocopy" +version = "0.8.55" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b5a105cd7b140f6eeec8acff2ea38135d3cab283ada58540f629fe51e46696eb" +dependencies = [ + "zerocopy-derive", +] + +[[package]] +name = "zerocopy-derive" +version = "0.8.55" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0fe976fb70c78cd64cccfe3a6fc142244e8a77b70959b30faf9d0ac37ee228eb" +dependencies = [ + "proc-macro2", + "quote", + "syn 2.0.119", +] + +[[package]] +name = "zeroize" +version = "1.9.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e13c156562582aa81c60cb29407084cdb54c4164760106ab78e6c5b0858cf64e" + +[[package]] +name = "zmij" +version = "1.0.23" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "29666d0abbfad1e3dc4dcf6144730dd3a3ab225bbbdac83319345b1b44ccfc1b" diff --git a/agent/Cargo.toml b/agent/Cargo.toml new file mode 100644 index 0000000..3578f29 --- /dev/null +++ b/agent/Cargo.toml @@ -0,0 +1,88 @@ +[package] +name = "avocado-container-agent-dev" +version = "0.1.0" +edition = "2024" + +[[bin]] +name = "avocado-container-agent-dev" +path = "src/main.rs" + +[dependencies] +# `sync` is used directly (`Notify` for the sync-worker wakeup and the +# re-bootstrap signal, `mpsc` for the outbound frame channel). It builds today +# only because hyper 1.x depends on tokio with `sync` enabled and cargo unifies +# features - `cargo tree -e features -i tokio` shows it sourced from hyper, not +# from here. A hyper bump that drops that dependency would break this crate with +# no local change, so the feature it actually uses is declared. +tokio = { version = "1", features = [ + "rt-multi-thread", + "macros", + "net", + "time", + "io-util", + "process", + "sync", +] } +# WS client only. We deliberately DO NOT enable tokio-tungstenite's +# `rustls-tls-*` features: those pull `tokio-rustls` with its default features, +# which selects the `aws_lc_rs` crypto provider (forbidden on the device — A9 +# clears the `ring` stack for the musl cross targets, same rule avocado-conn +# enforces). Instead we establish TLS ourselves with a ring-only +# `tokio-rustls::TlsConnector` and hand the stream to `client_async`, so +# `aws-lc-rs` never enters the tree. +tokio-tungstenite = { version = "0.26", default-features = false, features = [ + "handshake", +] } +tokio-rustls = { version = "0.26", default-features = false, features = [ + "ring", + "logging", + "tls12", +] } +# Direct rustls dep so we can build a `ClientConfig` pinned to the ring +# provider and install it as the process default (defense-in-depth), matching +# avocado-conn's explicit-vertex pattern. +rustls = { version = "0.23", default-features = false, features = [ + "ring", + "logging", + "std", + "tls12", +] } +rustls-pemfile = "2" +futures-util = "0.3" +serde = { version = "1", features = ["derive"] } +serde_json = "1" +anyhow = "1" +tracing = "0.1" +tracing-subscriber = { version = "0.3", features = ["env-filter"] } +# Loopback registry proxy (task 6.2). hyper is a pure-Rust HTTP/1 stack with no +# crypto of its own; the upstream leg's TLS is driven by the same ring-backed +# `tokio-rustls`/`rustls` above, so no `aws-lc-rs` enters the tree. We +# deliberately do NOT use `reqwest`, whose default rustls would pull `aws-lc-rs`. +hyper = { version = "1", default-features = false, features = [ + "server", + "client", + "http1", +] } +hyper-util = { version = "0.1", default-features = false, features = ["tokio"] } +http-body-util = "0.1" +bytes = "1" + +[dev-dependencies] +# Throwaway CA/leaf generation for the in-process TLS upstream in proxy tests. +# `ring` (not the crate default `aws_lc_rs`) so the pinned-CA test stack matches +# the agent's own ring provider and keeps `aws-lc-rs` out of the tree entirely. +rcgen = { version = "0.13", default-features = false, features = [ + "crypto", + "pem", + "ring", +] } +# Virtual time for the proxy's accept-retry tests: `start_paused` lets them assert +# the backoff elapsed without spending it, which both pins the sleep and keeps the +# suite fast. Dev-only - the edition-2024 resolver keeps it out of the shipped binary. +tokio = { version = "1", features = ["test-util"] } +# Temp dir standing in for the writable partition in the sync pointer tests. +tempfile = "3" + +[profile.release] +strip = true +lto = true diff --git a/agent/README.md b/agent/README.md new file mode 100644 index 0000000..3c40de6 --- /dev/null +++ b/agent/README.md @@ -0,0 +1,47 @@ +# avocado-container-agent-dev + +The device-side agent for Container Dev Mode. It stays **ring-only**: the +rustls crypto provider must be `ring`, and `aws-lc-rs` is forbidden. + +## Why ring-only (aws-lc-rs forbidden) + +The agent cross-compiles against the musl SDK (assumption A9), which clears only +the `ring` stack for the target - `aws-lc-rs` will not build there. This is the +same rule `avocado-conn` enforces. `aws-lc-rs` most commonly sneaks in through a +default rustls feature (e.g. `tokio-tungstenite`'s or `reqwest`'s default TLS), +so `Cargo.toml` disables default features on every TLS-touching dependency and +pins `ring` explicitly (including the `rcgen` dev-dependency, so aws-lc-rs stays +out of the tree entirely - dev-deps included). + +## Provider guard: `scripts/check-provider.sh` + +`scripts/check-provider.sh` asserts the crypto provider by inspecting the +dependency tree. It uses a three-way exit contract - a naive `! cargo tree` is +wrong, because `cargo tree -i ` exits 0 when the package is present and +non-zero (with `did not match any packages`) when absent, so a bare negation +would treat every network or manifest failure as a pass. + +| Exit | Meaning | +|------|---------| +| 0 | Clean: aws-lc-rs absent, ring present. | +| 1 | aws-lc-rs is in the tree (forbidden provider present). | +| 2 | Guard error: tree unresolvable (network, bad manifest, unresolvable target, or ring missing). Never treated as a pass. | + +The guard also backstops by asserting `ring` IS present: a guard that cannot see +`ring` cannot trust its aws-lc-rs result, so that case exits 2. + +### Enforced gate vs advisory run + +- **Enforced gate (in-hook, with `TARGET`):** `cad-compile.sh` runs the guard as + `TARGET="$RUST_TARGET" scripts/check-provider.sh` immediately before + `cargo build`, resolving the same SDK triple, manifest, and `CARGO_HOME` the + build uses. Under `set -e`, a non-zero exit fails the compile. This is the + real enforcement point - avocado-os has no CI or pre-commit hook. +- **Advisory run (standalone, no `TARGET`):** running + `scripts/check-provider.sh` directly (e.g. from the repo root) checks the + **host** triple, not the shipped cross target. Treat it as an advisory + smoke-check; it does not gate anything. + +The guard uses the default `cargo tree` edge set (dev-dependencies included) and +deliberately does not pass `--edges normal,build` or `--locked`, so it resolves +the same tree the build does within the shared `CARGO_HOME`. diff --git a/agent/scripts/check-provider.sh b/agent/scripts/check-provider.sh new file mode 100755 index 0000000..0f7ee05 --- /dev/null +++ b/agent/scripts/check-provider.sh @@ -0,0 +1,80 @@ +#!/bin/bash +# +# check-provider.sh - enforce the ring-only rustls crypto provider for the +# Container Dev Mode device agent. +# +# aws-lc-rs is forbidden in this crate (assumption A9: the agent cross-compiles +# against the musl SDK, which clears only the `ring` stack - the same rule +# avocado-conn enforces). This guard fails the build if aws-lc-rs ever resolves +# into the dependency tree, and also asserts that `ring` IS present so a guard +# that cannot see the tree at all can never masquerade as a pass. +# +# Exit contract (three-way, deliberately NOT a bare `! cargo tree`): +# 0 - clean ring-only tree (aws-lc-rs absent, ring present) +# 1 - aws-lc-rs PRESENT (forbidden provider in the tree) +# 2 - guard error: the dependency tree could not be resolved (network, bad +# manifest, unresolvable target, or ring itself missing). Never a pass. +# +# The check uses the default `cargo tree` edge set (includes dev-dependencies) +# so it is tree-wide: the agent Cargo.toml keeps aws-lc-rs "out of the tree +# entirely", and the rcgen dev-dependency is pinned to ring for exactly that. +# +# TARGET: when set (env var or first positional arg) the guard resolves against +# that target triple, matching the build's cross target. In the build hook this +# is mandatory (TARGET="$RUST_TARGET"). Run standalone with no TARGET, the guard +# checks the host triple as an ADVISORY check. + +set -euo pipefail + +script_dir="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" +manifest="$script_dir/../Cargo.toml" + +target="${TARGET:-${1:-}}" + +target_args=() +if [ -n "$target" ]; then + target_args+=(--target "$target") + scope="target $target" +else + scope="host triple (advisory)" +fi + +echo "check-provider: enforcing ring-only rustls provider ($scope)" + +# Inverse-tree query for the forbidden provider. Capture status AND output so a +# non-zero exit does not abort under `set -e`. On success (aws-lc-rs present) +# stdout carries the offending inverse-tree path; on failure the captured text +# carries cargo's error on stderr. +set +e +awslc_out="$(cargo tree -i aws-lc-rs --manifest-path "$manifest" "${target_args[@]}" 2>&1)" +awslc_status=$? +set -e + +if [ "$awslc_status" -eq 0 ]; then + echo "check-provider: FAIL - forbidden crypto provider aws-lc-rs is in the tree:" >&2 + echo "$awslc_out" >&2 + exit 1 +fi + +if ! printf '%s\n' "$awslc_out" | grep -q 'did not match any packages'; then + echo "check-provider: ERROR - guard could not resolve the dependency tree." >&2 + echo "check-provider: this is NOT a pass; refusing to certify the provider." >&2 + echo "$awslc_out" >&2 + exit 2 +fi + +# aws-lc-rs is provably absent. Backstop: `ring` MUST be present. A guard that +# cannot see ring cannot trust its aws-lc-rs result either. +set +e +cargo tree -i ring --manifest-path "$manifest" "${target_args[@]}" >/dev/null 2>&1 +ring_status=$? +set -e + +if [ "$ring_status" -ne 0 ]; then + echo "check-provider: ERROR - ring is not present in the tree; guard cannot" >&2 + echo "check-provider: trust the aws-lc-rs result. Treating as a guard error." >&2 + exit 2 +fi + +echo "check-provider: PASS - aws-lc-rs absent, ring-only provider confirmed." +exit 0 diff --git a/agent/src/main.rs b/agent/src/main.rs new file mode 100644 index 0000000..cec0ba5 --- /dev/null +++ b/agent/src/main.rs @@ -0,0 +1,1349 @@ +//! Device-side control agent for Container Dev Mode (design D9; task 6.1). +//! +//! The agent dials the host's control-only WebSocket, authenticates with the +//! read/control token (the only token a device ever holds — design D2), holds +//! the connection to receive [`HostFrame::Sync`] notifications, and +//! auto-reconnects with exponential backoff, re-sending its [`DeviceFrame::Hello`] +//! (carrying the current running digest) on every (re)connect. +//! +//! Scope of task 6.1 is the connect / hold / reconnect loop only. The actual +//! pull + container restart on a `Sync` frame is task 6.3; a clear seam is left +//! in [`handle_host_frame`]. +//! +//! TLS uses the **ring** rustls provider, never `aws-lc-rs`: the device +//! cross-compiles to musl and A9 clears only the ring stack (same rule +//! avocado-conn enforces). The control WS leaf is pinned to the bootstrap CA +//! alone — no native roots, no webpki-roots. + +use std::path::PathBuf; +use std::sync::Arc; +use std::sync::Mutex; +use std::sync::atomic::{AtomicBool, Ordering}; +use std::time::Duration; + +use anyhow::{Context, Result, bail, ensure}; +use futures_util::{SinkExt, StreamExt}; +use serde::{Deserialize, Serialize}; +use tokio::net::TcpStream; +use tokio::time::{Instant, sleep}; +use tokio_rustls::TlsConnector; +use tokio_rustls::rustls::pki_types::ServerName; +use tokio_tungstenite::client_async; +use tokio_tungstenite::tungstenite::Message; +use tokio_tungstenite::tungstenite::client::IntoClientRequest; +use tokio_tungstenite::tungstenite::http::HeaderValue; +use tokio_tungstenite::tungstenite::http::header::AUTHORIZATION; +use tracing::{info, warn}; + +mod proxy; +mod sync; + +/// Default on-device path the host writes the bootstrap file to (design D5 / A7). +const DEFAULT_BOOTSTRAP_PATH: &str = "/var/lib/avocado/container-dev/bootstrap.json"; +/// Env var that overrides the bootstrap path (used by tests and staging). +const BOOTSTRAP_PATH_ENV: &str = "AVOCADO_CONTAINER_DEV_BOOTSTRAP"; +/// Env var that supplies the device identity reported in `Hello`. +const DEVICE_ID_ENV: &str = "AVOCADO_DEVICE_ID"; +/// Default loopback port the device engine pulls from (task 6.2). Chosen off +/// 5000 to avoid the macOS AirPlay clash noted in the Phase 0 port survey +/// (design 1.6); the engine config (task 6.4) points at `127.0.0.1:`. +const DEFAULT_LOOPBACK_PORT: u16 = 15151; +/// Env var overriding the loopback registry port. +const LOOPBACK_PORT_ENV: &str = "AVOCADO_CONTAINER_DEV_LOOPBACK_PORT"; + +// --------------------------------------------------------------------------- +// Wire contract (MUST match avocado-cli/src/utils/container_dev/ws.rs). +// +// Frames are JSON, serde-tagged with `"type"`, `rename_all = "snake_case"`. +// These types are re-declared here (not imported) because avocado-cli is a +// separate repo with no shared workspace. The `frame_wire_shapes` test locks +// the JSON shape to the host's. +// --------------------------------------------------------------------------- + +/// A host -> device control frame. Carries only image coordinates plus a +/// content-digest reference — never blob bytes. +#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)] +#[serde(tag = "type", rename_all = "snake_case")] +pub enum HostFrame { + /// The `{image, tag, digest}` now available for the device to pull, and the + /// systemd unit that consumes it. + Sync { + image: String, + tag: String, + digest: String, + /// The device unit owning this image, from the host's own + /// `container_dev.images[].service`. Optional on the wire so a device + /// running against an older host still parses the frame - it then falls + /// back to `$AVOCADO_CONTAINER_DEV_SERVICE` and finally to restarting + /// the container, which is what every device did before this field + /// existed. See `sync::SERVICE_ENV` for why the container branch cannot + /// adopt a new image. + #[serde(default, skip_serializing_if = "Option::is_none")] + service: Option, + }, +} + +/// A device -> host control frame. +/// +/// `Hello` is an internally-tagged newtype variant: it serializes flat, as a +/// single object tagged `"hello"` carrying the `Hello` struct's fields — this +/// matches the host's `DeviceFrame::Hello(Hello)` wire form exactly. +#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)] +#[serde(tag = "type", rename_all = "snake_case")] +pub enum DeviceFrame { + Hello(Hello), + Progress(Progress), + Status(Status), +} + +/// The device's `hello`: who it is, its CPU arch, and the digest it runs now. +#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)] +pub struct Hello { + pub device_id: String, + pub arch: String, + pub running_digest: String, +} + +/// Progress of an in-flight device pull (informational). +#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)] +pub struct Progress { + pub image: String, + pub bytes_pulled: u64, +} + +/// A device state report (informational). +#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)] +pub struct Status { + pub device_id: String, + pub state: String, + #[serde(default, skip_serializing_if = "Option::is_none")] + pub detail: Option, +} + +// --------------------------------------------------------------------------- +// Device bootstrap. +// --------------------------------------------------------------------------- + +/// The device-side bootstrap, read from the writable partition. +/// +/// `bulk_endpoint`, `read_token`, and `ca_cert_pem` mirror the host's +/// `DeviceBootstrap` (avocado-cli/src/utils/container_dev/bootstrap.rs). +#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)] +pub struct Bootstrap { + /// The bulk read-listener endpoint (`host:port`) the device pulls from. + pub bulk_endpoint: String, + /// The Bearer read/control token — the only token a device holds. + pub read_token: String, + /// The per-project CA certificate (PEM) the device pins the host leaf to. + pub ca_cert_pem: String, + /// The `host:port` of the control WS to dial. + /// + /// FOLLOW-UP: the host side must be extended to emit this — avocado-cli's + /// `DeviceBootstrap` (bootstrap.rs) does not yet carry it, and the + /// control-WS bind in `dev.rs` must publish its address here. Tracked as a + /// devspec discovery event for this change. Defaulted so parsing a current + /// host payload (without the field) does not hard-fail; the agent errors + /// clearly at dial time if it is still empty. + #[serde(default)] + pub ws_endpoint: String, +} + +impl Bootstrap { + /// Whether this bootstrap and `other` carry the same session material. + /// + /// Only the fields a live session depends on: a token, endpoint or CA change + /// means the session in progress is finished, whatever else the file says. + fn same_session_as(&self, other: &Bootstrap) -> bool { + self.read_token == other.read_token + && self.bulk_endpoint == other.bulk_endpoint + && self.ws_endpoint == other.ws_endpoint + && self.ca_cert_pem == other.ca_cert_pem + } +} + +/// Resolve the bootstrap path: `$AVOCADO_CONTAINER_DEV_BOOTSTRAP` or the default. +fn bootstrap_path() -> PathBuf { + std::env::var_os(BOOTSTRAP_PATH_ENV) + .map(PathBuf::from) + .unwrap_or_else(|| PathBuf::from(DEFAULT_BOOTSTRAP_PATH)) +} + +/// Resolve the loopback registry port: `$AVOCADO_CONTAINER_DEV_LOOPBACK_PORT` +/// or the default. +fn loopback_port() -> u16 { + std::env::var(LOOPBACK_PORT_ENV) + .ok() + .and_then(|v| v.trim().parse().ok()) + .unwrap_or(DEFAULT_LOOPBACK_PORT) +} + +/// Load and parse the bootstrap from `path`. +fn load_bootstrap(path: &std::path::Path) -> Result { + let raw = std::fs::read_to_string(path) + .with_context(|| format!("reading bootstrap at {}", path.display()))?; + let bootstrap: Bootstrap = serde_json::from_str(&raw) + .with_context(|| format!("parsing bootstrap at {}", path.display()))?; + Ok(bootstrap) +} + +/// Resolve the device identity reported in `Hello`. +/// +/// Prefers `$AVOCADO_DEVICE_ID`; falls back to the machine hostname +/// (`/etc/hostname`); finally `"unknown"`. Kept dependency-free. +fn resolve_device_id() -> String { + if let Ok(id) = std::env::var(DEVICE_ID_ENV) { + let id = id.trim().to_string(); + if !id.is_empty() { + return id; + } + } + if let Ok(hostname) = std::fs::read_to_string("/etc/hostname") { + let hostname = hostname.trim().to_string(); + if !hostname.is_empty() { + return hostname; + } + } + "unknown".to_string() +} + +// --------------------------------------------------------------------------- +// Reconnect backoff. +// --------------------------------------------------------------------------- + +/// Exponential backoff with a cap for the reconnect loop. +/// +/// Doubles the delay on each `next_delay` up to `max`, and resets to `base` +/// after a successful connection so a long-lived link that briefly drops does +/// not inherit a stale, large delay. +#[derive(Debug, Clone)] +struct Backoff { + base: Duration, + max: Duration, + current: Duration, +} + +impl Backoff { + fn new(base: Duration, max: Duration) -> Self { + Self { + base, + max, + current: base, + } + } + + /// Return the delay to wait before the next attempt, then advance (doubling + /// up to `max`). + fn next_delay(&mut self) -> Duration { + let delay = self.current; + self.current = (self.current * 2).min(self.max); + delay + } + + /// Reset to the base delay (call after a successful connection). + fn reset(&mut self) { + self.current = self.base; + } +} + +impl Default for Backoff { + fn default() -> Self { + Backoff::new(Duration::from_secs(1), Duration::from_secs(30)) + } +} + +/// How long a session must last before it counts as evidence the link works. +/// +/// The reset used to key on the handshake alone, which a broken host completes +/// happily before dropping the connection - so the backoff never grew past its +/// 1s base. Ten seconds is far longer than the accept-panic-drop cycle that +/// motivated it and far shorter than any real session. +const MIN_SESSION_FOR_RESET: Duration = Duration::from_secs(10); + +/// Whether a finished session is evidence the link works, and so whether the +/// reconnect backoff may reset. +/// +/// A function rather than an inline condition so the duration floor is +/// assertable: `run()` needs a real host to exercise, and an inline test +/// re-stating the same expression would pass with the floor removed - which is +/// exactly how this shipped without one. +fn session_proves_the_link_works(handshook: bool, session: Duration) -> bool { + handshook && session >= MIN_SESSION_FOR_RESET +} + +// --------------------------------------------------------------------------- +// Agent state. +// --------------------------------------------------------------------------- + +/// Shared, mutable agent state carried across reconnects. +struct AgentState { + device_id: String, + /// The content digest the agent is currently running. Empty until a pull + /// lands (task 6.3 updates this); re-sent in `Hello` on every reconnect. + running_digest: Mutex, + /// Raised by the loopback proxy (task 6.2) when the bulk upstream rejects + /// the read/control token. Surfaced to the host as a `Status` frame on the + /// next (re)connect; a new token comes only from a host re-`up`. + rebootstrap: proxy::ReBootstrap, +} + +impl AgentState { + fn new(device_id: String) -> Self { + Self { + device_id, + running_digest: Mutex::new(String::new()), + rebootstrap: proxy::ReBootstrap::default(), + } + } + + fn hello(&self) -> DeviceFrame { + let running_digest = self + .running_digest + .lock() + .expect("running_digest mutex poisoned") + .clone(); + DeviceFrame::Hello(Hello { + device_id: self.device_id.clone(), + arch: std::env::consts::ARCH.to_string(), + running_digest, + }) + } +} + +// --------------------------------------------------------------------------- +// TLS. +// --------------------------------------------------------------------------- + +/// Build a rustls `ClientConfig` pinned to ONLY the bootstrap CA, using the +/// ring provider. No native roots, no webpki-roots. +pub(crate) fn build_tls_config(ca_cert_pem: &str) -> Result { + let mut root_store = rustls::RootCertStore::empty(); + let mut reader = std::io::Cursor::new(ca_cert_pem.as_bytes()); + let certs = rustls_pemfile::certs(&mut reader) + .collect::, _>>() + .context("parsing bootstrap ca_cert_pem")?; + ensure!( + !certs.is_empty(), + "bootstrap ca_cert_pem contained no certificates" + ); + for cert in certs { + root_store + .add(cert) + .context("adding bootstrap CA to root store")?; + } + + let provider = Arc::new(rustls::crypto::ring::default_provider()); + let config = rustls::ClientConfig::builder_with_provider(provider) + .with_safe_default_protocol_versions() + .context("selecting rustls protocol versions")? + .with_root_certificates(root_store) + .with_no_client_auth(); + Ok(config) +} + +/// Ceiling on each control-WS dial stage. +/// +/// The serve loop itself is deliberately unbounded - a session is meant to stay +/// open - but every stage of GETTING there should complete in milliseconds on a +/// working link, and hanging in one means the reconnect loop never runs again. +const WS_DIAL_TIMEOUT: Duration = Duration::from_secs(30); + +/// Run `fut`, failing with a named error if it does not finish in time. +async fn with_dial_timeout( + stage: &'static str, + fut: impl std::future::Future>, +) -> Result { + match tokio::time::timeout(WS_DIAL_TIMEOUT, fut).await { + Ok(result) => result, + Err(_) => anyhow::bail!( + "{stage} did not complete within {}s; the host is unreachable or the connection \ + is half-open", + WS_DIAL_TIMEOUT.as_secs() + ), + } +} + +/// Split a `host:port` endpoint into its host component for SNI. +pub(crate) fn endpoint_host(endpoint: &str) -> &str { + endpoint + .rsplit_once(':') + .map(|(h, _)| h) + .unwrap_or(endpoint) +} + +// --------------------------------------------------------------------------- +// Connect + serve. +// --------------------------------------------------------------------------- + +/// Handle one host frame. On a `Sync`, pull the new image through the loopback +/// proxy, rewrite the active-image pointer on the writable partition, restart +/// the container, and record the running digest (task 6.3, [`sync::on_sync`]). +/// The engine and config `handle_host_frame` runs a sync against. +/// +/// Taken as parameters rather than built inside, so a test can drive the +/// reporting path with a fake engine. Building them internally made the failure +/// branch unreachable from a test: the only coverage was a test that constructed +/// the `Status` frame itself and asserted serde round-tripped it, which stayed +/// green with the whole reporting path reverted to a bare `warn!`. +async fn handle_host_frame( + engine: &E, + cfg: &sync::SyncConfig, + frame: HostFrame, + state: &AgentState, +) -> Option { + match frame { + HostFrame::Sync { + image, + tag, + digest, + service, + } => { + info!(%image, %tag, %digest, ?service, "received sync; pulling + restarting"); + match sync::on_sync( + engine, + cfg, + &image, + &tag, + &digest, + service.as_deref(), + state, + ) + .await + { + Ok(()) => None, + Err(e) => { + // Report it, do not only log it. A swallowed failure left the + // host with no Status, no Progress, and no updated Hello - + // Hello is re-sent only on reconnect, which on a healthy link + // may be hours - so it went on showing the device synced at + // the old digest with no error surface anywhere. + warn!(error = %e, %image, %tag, %digest, "sync failed"); + Some(DeviceFrame::Status(Status { + device_id: state.device_id.clone(), + state: "sync_failed".to_string(), + detail: Some(format!("{image}:{tag} @ {digest}: {e:#}")), + })) + } + } + } + } +} + +/// The device-to-host frames one session may emit, and how they reach the sink. +/// +/// Syncs run on their own task rather than inline in the read loop, which is +/// what lets the stream keep being polled while a pull is in flight. Inline, +/// `handle_host_frame` awaited three unbounded subprocesses inside +/// `while let Some(msg) = stream.next().await`, so nothing was read for the +/// duration - and N saves during one slow pull queued N frames in the socket +/// buffer, each getting its own full pull/tag/restart afterwards: N container +/// restarts instead of one, N-1 onto an already-superseded digest. +/// Aborts the sync worker when the session ends, however it ends. +/// +/// The abort used to sit after the serve loop, reachable only by falling out of +/// it. Three `?` sites returned early instead - the two `send_frame` calls and +/// the stream read - and the read one is the COMMON path, because tungstenite +/// yields `Some(Err(..))` on a reset and only `None` on a clean close. The +/// worker then parked on `work.notified()` forever: one leaked task per errored +/// session, and against a host that accepts the upgrade then dies, one per +/// backoff cycle for the process lifetime. +/// +/// The leak was the smaller half. A worker that outlived its session kept +/// running its pull/tag/restart while the next session spawned another one, so +/// two sequences could race on `running_digest` and the pointer file with +/// nothing serializing them. +/// Holds an [`tokio::task::AbortHandle`] rather than the `JoinHandle`: the +/// session never joins the worker, it only needs to be able to cancel it, and +/// the narrower handle is what makes the guard's effect assertable - a test can +/// keep the `JoinHandle` and await the cancelled `JoinError`. +struct AbortOnDrop(tokio::task::AbortHandle); + +impl Drop for AbortOnDrop { + fn drop(&mut self) { + self.0.abort(); + } +} + +struct SessionChannels { + /// Latest-wins slot. A Sync that arrives while another is running replaces + /// any already-queued one, because only the newest digest is worth applying. + pending: Arc>>, + /// Wakes the sync worker when `pending` is refilled. + work: Arc, +} + +/// Dial the control WS, send `Hello`, and serve frames until the link closes. +/// +/// Sets `connected` to `true` once the WS handshake succeeds, so the caller can +/// reset the backoff on a link that actually established. +async fn connect_and_serve( + bootstrap: &Bootstrap, + state: &Arc, + connected: &AtomicBool, +) -> Result<()> { + ensure!( + !bootstrap.ws_endpoint.is_empty(), + "bootstrap ws_endpoint is empty; the host must emit it (tracked follow-up)" + ); + + let tls_config = build_tls_config(&bootstrap.ca_cert_pem)?; + let connector = TlsConnector::from(Arc::new(tls_config)); + + // Same reasoning as the proxy's stage timeouts: without one, a half-open + // connection to a sleeping host wedges the dial forever, and the reconnect + // loop that is supposed to recover never gets another turn. + let tcp = with_dial_timeout("the control WS TCP connect", async { + TcpStream::connect(&bootstrap.ws_endpoint) + .await + .with_context(|| format!("connecting TCP to {}", bootstrap.ws_endpoint)) + }) + .await?; + + let host = endpoint_host(&bootstrap.ws_endpoint).to_string(); + let server_name = ServerName::try_from(host).context("invalid TLS server name")?; + let tls = with_dial_timeout("the control WS TLS handshake", async { + connector + .connect(server_name, tcp) + .await + .context("TLS handshake to control WS") + }) + .await?; + + let mut request = format!("wss://{}/", bootstrap.ws_endpoint) + .into_client_request() + .context("building WS upgrade request")?; + request.headers_mut().insert( + AUTHORIZATION, + HeaderValue::from_str(&format!("Bearer {}", bootstrap.read_token)) + .context("building Authorization header")?, + ); + + let (ws, _response) = with_dial_timeout("the control WS upgrade", async { + client_async(request, tls) + .await + .context("WebSocket upgrade to control WS") + }) + .await?; + connected.store(true, Ordering::SeqCst); + info!(endpoint = %bootstrap.ws_endpoint, "control WS connected"); + + // The sink goes idle after the frames below and only the stream is polled. + // That does NOT strand the automatic Pong replies to host keepalive pings: + // tungstenite queues a Pong as `additional_send` and flushes it from inside + // `read()` itself (protocol/mod.rs), which is the same call `poll_next` + // drives on the read half — so a ping is answered without the sink being + // touched, and the connection is not churned by a keepalive timeout. + let (mut sink, mut stream) = ws.split(); + + let hello = state.hello(); + sink.send(Message::Text( + serde_json::to_string(&hello) + .context("serializing Hello")? + .into(), + )) + .await + .context("sending Hello")?; + + // The sync worker and the frames it produces. Everything the device sends + // after `Hello` arrives on `outbound` and is written by the select loop + // below, so the sink has exactly one writer. + let channels = SessionChannels { + pending: Arc::new(Mutex::new(None)), + work: Arc::new(tokio::sync::Notify::new()), + }; + let (outbound_tx, mut outbound) = tokio::sync::mpsc::channel::(8); + + let _worker = AbortOnDrop({ + let pending = Arc::clone(&channels.pending); + let work = Arc::clone(&channels.work); + let state = Arc::clone(state); + let outbound_tx = outbound_tx.clone(); + tokio::spawn(async move { + loop { + work.notified().await; + // Drain the slot, not a queue: whatever is there is the newest + // Sync, and any earlier one it replaced is already superseded. + let Some(frame) = pending.lock().expect("pending mutex poisoned").take() else { + continue; + }; + // The engine shell-out and env-resolved config are the thin, + // untested production glue; the ordering/pointer/digest core + // lives in the fully-tested `sync::on_sync`. + let engine = sync::CommandEngine::from_env(); + let cfg = sync::SyncConfig::from_env(); + if let Some(report) = handle_host_frame(&engine, &cfg, frame, &state).await + && outbound_tx.send(report).await.is_err() + { + // The session ended while the sync ran; the next Hello + // carries the running digest, so nothing is lost. + return; + } + } + }) + .abort_handle() + }); + // Dropping our own copy means `outbound` closes when the worker does. + drop(outbound_tx); + + // A re-bootstrap raised BEFORE this session started still needs announcing; + // one raised during it arrives through `rebootstrap.raised()` below. + report_rebootstrap(&mut sink, state).await?; + + loop { + tokio::select! { + // Biased so an outbound report is written before another host frame + // is accepted; without it a busy host could starve the reports the + // whole restructure exists to deliver. + biased; + + Some(frame) = outbound.recv() => { + send_frame(&mut sink, &frame).await?; + } + + // Raised by the loopback proxy on a bulk 401, which happens during a + // pull - inside this loop, not before it. Reading the flag once up + // front meant it was always set strictly after the only read that + // would have transmitted it. + () = state.rebootstrap.raised() => { + report_rebootstrap(&mut sink, state).await?; + } + + msg = stream.next() => { + let Some(msg) = msg else { break Ok(()) }; + let msg = msg.context("reading control WS frame")?; + match msg { + Message::Text(text) => match serde_json::from_str::(text.as_str()) { + Ok(frame) => { + // Latest wins. Replacing a queued Sync is the point: + // applying a superseded digest costs a container + // restart and achieves nothing. + let replaced = channels + .pending + .lock() + .expect("pending mutex poisoned") + .replace(frame) + .is_some(); + if replaced { + info!("coalesced a superseded sync still waiting to run"); + } + channels.work.notify_one(); + } + Err(e) => warn!(error = %e, raw = %text, "ignoring malformed host frame"), + }, + Message::Close(_) => { + info!("control WS closed by host"); + break Ok(()); + } + Message::Ping(_) | Message::Pong(_) => {} + Message::Binary(_) | Message::Frame(_) => { + warn!("ignoring unexpected non-text control frame"); + } + } + } + } + } + // The sync worker is aborted by `_worker`'s Drop on every exit from here, + // including the `?` early-returns above. +} + +/// Report a raised re-bootstrap to the host, if one is raised. +/// +/// Lowers the flag BEFORE sending. Clearing afterwards erased any raise that +/// landed during the send itself, which is a live window: the proxy raises from +/// its own task while this await is parked, so the 401 that arrived mid-send was +/// dropped with no report and no way to ask again. +/// +/// A failed send re-raises rather than swallowing, because the session is about +/// to end and the pre-loop report on the next one is the only thing that would +/// carry it. +async fn report_rebootstrap(sink: &mut S, state: &AgentState) -> Result<()> +where + S: futures_util::SinkExt + Unpin, + >::Error: std::error::Error + Send + Sync + 'static, +{ + if !state.rebootstrap.take() { + return Ok(()); + } + if let Err(e) = send_frame(sink, &rebootstrap_status(state)).await { + state.rebootstrap.raise(); + return Err(e); + } + Ok(()) +} + +/// The `needs_rebootstrap` status. There is no token-renewal endpoint; the +/// operator must re-run `avocado container dev up` to mint a fresh token. +fn rebootstrap_status(state: &AgentState) -> DeviceFrame { + DeviceFrame::Status(Status { + device_id: state.device_id.clone(), + state: "needs_rebootstrap".to_string(), + detail: Some( + "bulk pull rejected the read/control token; re-run `avocado container dev up`" + .to_string(), + ), + }) +} + +/// Serialize and write one device frame to the control WS sink. +async fn send_frame(sink: &mut S, frame: &DeviceFrame) -> Result<()> +where + S: futures_util::SinkExt + Unpin, + >::Error: std::error::Error + Send + Sync + 'static, +{ + sink.send(Message::Text( + serde_json::to_string(frame) + .context("serializing device frame")? + .into(), + )) + .await + .context("sending device frame")?; + Ok(()) +} + +/// The reconnect loop: connect, serve, and on any close/error back off and +/// redial, re-sending `Hello` with the current running digest. +/// Returns `Ok(())` when the bootstrap on disk stops matching the one this +/// process started with, so `main` exits and the supervisor rebuilds against the +/// new material. Reloading in place is not enough: the read token is also baked +/// into the loopback proxy's upstream (`HttpsUpstream::from_bootstrap`), and a +/// token the proxy still sends is exactly what 401s the bulk leg. Exiting +/// re-reads both from one code path. +/// +/// Without this the documented recovery could not work. Re-running `avocado +/// container dev up` mints a fresh token and overwrites bootstrap.json; the +/// agent kept the old one, so bulk pulls 401'd, the WS upgrade was rejected by +/// the same host-side auth check, `connect_and_serve` errored on every redial, +/// and this loop spun forever. Nothing exited, so `Restart=always` never fired, +/// and the only fix was a manual unit restart that no message mentioned. +async fn run(bootstrap: Bootstrap, state: Arc) -> Result<()> { + let path = bootstrap_path(); + let mut backoff = Backoff::default(); + loop { + let connected = AtomicBool::new(false); + let started = Instant::now(); + match connect_and_serve(&bootstrap, &state, &connected).await { + Ok(()) => info!("control WS session ended; reconnecting"), + Err(e) => warn!(error = %e, "control WS session failed; reconnecting"), + } + + // A session that completed its handshake and then ended immediately is + // not evidence the link works. `connected` flips the moment + // `client_async` returns, so without a duration floor EVERY + // handshake-completing session reset the backoff and the 30s cap was + // unreachable: a host whose `desired` mutex is poisoned accepts the + // upgrade, panics on the first Hello and drops the socket, which pinned + // an embedded device to a ~1 Hz pinned-CA TLS handshake loop against a + // host that would never work again. + if session_proves_the_link_works(connected.load(Ordering::SeqCst), started.elapsed()) { + backoff.reset(); + } + + // Adopt new session material by exiting, not by continuing. + match load_bootstrap(&path) { + Ok(current) if !current.same_session_as(&bootstrap) => { + info!( + bootstrap = %path.display(), + "bootstrap changed on disk; exiting so the supervisor rebuilds this \ + process against the new session material" + ); + return Ok(()); + } + Ok(_) => {} + // Unreadable or malformed: keep the session we have rather than + // exiting on a file caught mid-delivery. + Err(e) => { + warn!(error = %e, "could not re-read the bootstrap; keeping the current one") + } + } + + let delay = backoff.next_delay(); + info!(delay_secs = delay.as_secs(), "backing off before reconnect"); + sleep(delay).await; + } +} + +#[tokio::main] +async fn main() -> Result<()> { + tracing_subscriber::fmt() + .with_env_filter( + tracing_subscriber::EnvFilter::try_from_default_env() + .unwrap_or_else(|_| tracing_subscriber::EnvFilter::new("info")), + ) + .init(); + + // Defense-in-depth: pin ring as the process-wide default provider so any + // implicit rustls builder cannot fall back to a different provider. + let _ = rustls::crypto::ring::default_provider().install_default(); + + let path = bootstrap_path(); + let bootstrap = load_bootstrap(&path)?; + let device_id = resolve_device_id(); + info!(%device_id, bootstrap = %path.display(), "container dev agent starting"); + + let state = Arc::new(AgentState::new(device_id)); + + // Start the loopback registry proxy (task 6.2): the device engine pulls + // from this plain-HTTP loopback (no insecure flag, no per-engine trust + // file); each read is forwarded to the host bulk HTTPS listener over the + // pinned CA + read/control token. Bulk NEVER rides the control WS. + // Warn once, here, if the configured engine cannot work under the shipped + // systemd unit. This is the only place that runs exactly once per process: + // `CommandEngine::from_env` is constructed per `Sync` frame, so warning + // there logged nothing at `systemctl start` (when the operator's override is + // new information) and then repeated the warning before every pull failure. + let engine_binary = sync::engine_binary_from_env(); + sync::warn_if_unsupported_engine(&engine_binary); + // Refuse to start without the engine, rather than staying `active (running)` + // and no-oping every sync. See `ensure_engine_available`. + sync::ensure_engine_available(&engine_binary)?; + + // Failing to BUILD the upstream is as fatal as losing it later. + // + // This used to `warn!` once and carry on with `None`, three lines above a + // comment arguing at length that losing the proxy must end the process + // "because without it no pull can succeed". Two tests locked in "losing the + // proxy is fatal"; never having it was uncovered. A truncated `ca_cert_pem` + // from a partial SSH write, or a bulk endpoint host `ServerName::try_from` + // rejects, left the agent `active (running)` with port 15151 closed and + // every Sync failing connection-refused - the exact state that comment was + // written to prevent. + // + // The stated justification, that a re-bootstrap could fix it, was + // self-defeating twice over: `raise()` is only reached from `proxy_read`, so + // with no proxy the needs_rebootstrap frame can never be sent, and the + // bootstrap was never re-read anyway. + let upstream = proxy::HttpsUpstream::from_bootstrap(&bootstrap) + .context("building the loopback proxy's upstream; without it no pull can succeed")?; + let proxy_task = { + let port = loopback_port(); + let upstream = Arc::new(upstream); + let rebootstrap = state.rebootstrap.clone(); + tokio::spawn(async move { proxy::serve_loopback(port, upstream, rebootstrap).await }) + }; + + // The proxy's bounded accept-retry only reaches `Restart=always` if losing + // the proxy ends the PROCESS. Spawning it detached and dropping the handle + // meant `serve_loopback` returning Err killed only its own task: `run()` + // kept the control WS up, systemd saw a healthy unit, and every later Sync + // failed connection-refused against a proxy that was gone - while the agent + // went on reporting a stale running_digest. Nothing restarted it, because + // from systemd's view nothing had failed. + // + // Select the two so whichever ends first ends `main`. The proxy is not + // optional: without it no pull can succeed, so exiting and letting the + // supervisor rebuild the process is the honest response to losing it. + end_when_either_ends(proxy_task, run(bootstrap, state)).await +} + +/// Return as soon as EITHER the proxy task or the control-WS loop ends. +/// +/// Extracted from `main` so the seam is testable. It is the whole reason the +/// proxy's bounded accept-retry means anything: the retry makes `serve_loopback` +/// return, and only this makes that return end the process, which is what +/// `Restart=always` needs in order to fire. Spawned detached with its handle +/// dropped, a returning proxy killed only its own task - systemd saw a healthy +/// unit while every later pull failed connection-refused. +async fn end_when_either_ends( + proxy: tokio::task::JoinHandle>, + serving: impl std::future::Future>, +) -> Result<()> { + tokio::select! { + proxied = proxy => match proxied { + // The proxy is not optional - without it no pull can succeed - so + // even a clean stop is a reason to exit and be rebuilt. + Ok(Ok(())) => bail!("loopback registry proxy stopped serving"), + Ok(Err(e)) => Err(e).context("loopback registry proxy failed"), + Err(e) => Err(anyhow::Error::new(e)) + .context("loopback registry proxy task panicked"), + }, + served = serving => served, + } +} + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn host_sync_frame_deserializes_from_wire() { + // A host that predates the `service` field must still parse, or a CLI + // upgrade becomes mandatory for every device in the field. + let wire = r#"{"type":"sync","image":"my-app","tag":"dev","digest":"sha256:abc"}"#; + let frame: HostFrame = serde_json::from_str(wire).expect("sync frame parses"); + assert_eq!( + frame, + HostFrame::Sync { + image: "my-app".to_string(), + tag: "dev".to_string(), + digest: "sha256:abc".to_string(), + service: None, + } + ); + } + + #[test] + fn host_sync_frame_carries_the_owning_service() { + // The field that makes the sync actually take effect: without it the + // agent restarts the container, which stays bound to its create-time + // image id, so every shipped device no-ops. + let wire = r#"{"type":"sync","image":"my-app","tag":"dev","digest":"sha256:abc","service":"app.service"}"#; + let frame: HostFrame = serde_json::from_str(wire).expect("sync frame parses"); + assert_eq!( + frame, + HostFrame::Sync { + image: "my-app".to_string(), + tag: "dev".to_string(), + digest: "sha256:abc".to_string(), + service: Some("app.service".to_string()), + } + ); + } + + #[test] + fn hello_frame_serializes_flat_with_tag() { + let hello = DeviceFrame::Hello(Hello { + device_id: "dev-01".to_string(), + arch: "aarch64".to_string(), + running_digest: "sha256:running".to_string(), + }); + let value: serde_json::Value = + serde_json::to_value(&hello).expect("hello serializes to json"); + assert_eq!(value["type"], "hello"); + assert_eq!(value["device_id"], "dev-01"); + assert_eq!(value["arch"], "aarch64"); + assert_eq!(value["running_digest"], "sha256:running"); + // The tag object is flat: no nested wrapper key for the Hello struct. + assert!(value.get("Hello").is_none()); + } + + #[test] + fn progress_frame_serializes_flat_with_tag() { + let frame = DeviceFrame::Progress(Progress { + image: "my-app".to_string(), + bytes_pulled: 4096, + }); + let value: serde_json::Value = serde_json::to_value(&frame).unwrap(); + assert_eq!(value["type"], "progress"); + assert_eq!(value["image"], "my-app"); + assert_eq!(value["bytes_pulled"], 4096); + } + + #[test] + fn status_frame_serializes_flat_with_tag() { + let frame = DeviceFrame::Status(Status { + device_id: "dev-01".to_string(), + state: "running".to_string(), + detail: None, + }); + let value: serde_json::Value = serde_json::to_value(&frame).unwrap(); + assert_eq!(value["type"], "status"); + assert_eq!(value["device_id"], "dev-01"); + assert_eq!(value["state"], "running"); + // Optional detail is skipped when None. + assert!(value.get("detail").is_none()); + } + + #[test] + fn device_frame_round_trips() { + let frames = vec![ + DeviceFrame::Hello(Hello { + device_id: "d".to_string(), + arch: "x86_64".to_string(), + running_digest: String::new(), + }), + DeviceFrame::Progress(Progress { + image: "img".to_string(), + bytes_pulled: 1, + }), + DeviceFrame::Status(Status { + device_id: "d".to_string(), + state: "s".to_string(), + detail: Some("ok".to_string()), + }), + ]; + for frame in frames { + let json = serde_json::to_string(&frame).unwrap(); + let back: DeviceFrame = serde_json::from_str(&json).unwrap(); + assert_eq!(frame, back); + } + } + + #[test] + fn bootstrap_parses_without_ws_endpoint_and_defaults_it() { + // A current host payload carries only the three DeviceBootstrap fields. + let wire = r#"{ + "bulk_endpoint": "10.0.0.1:8443", + "read_token": "tok", + "ca_cert_pem": "-----BEGIN CERTIFICATE-----\nMII...\n-----END CERTIFICATE-----\n" + }"#; + let bootstrap: Bootstrap = serde_json::from_str(wire).expect("bootstrap parses"); + assert_eq!(bootstrap.bulk_endpoint, "10.0.0.1:8443"); + assert_eq!(bootstrap.read_token, "tok"); + assert_eq!(bootstrap.ws_endpoint, ""); + } + + #[test] + fn bootstrap_parses_with_ws_endpoint() { + let wire = r#"{ + "bulk_endpoint": "10.0.0.1:8443", + "read_token": "tok", + "ca_cert_pem": "pem", + "ws_endpoint": "10.0.0.1:9443" + }"#; + let bootstrap: Bootstrap = serde_json::from_str(wire).unwrap(); + assert_eq!(bootstrap.ws_endpoint, "10.0.0.1:9443"); + } + + #[test] + fn backoff_doubles_then_caps() { + let mut backoff = Backoff::new(Duration::from_secs(1), Duration::from_secs(30)); + assert_eq!(backoff.next_delay(), Duration::from_secs(1)); + assert_eq!(backoff.next_delay(), Duration::from_secs(2)); + assert_eq!(backoff.next_delay(), Duration::from_secs(4)); + assert_eq!(backoff.next_delay(), Duration::from_secs(8)); + assert_eq!(backoff.next_delay(), Duration::from_secs(16)); + // 32 would exceed the 30s cap. + assert_eq!(backoff.next_delay(), Duration::from_secs(30)); + assert_eq!(backoff.next_delay(), Duration::from_secs(30)); + } + + #[test] + fn backoff_resets_to_base_after_connect() { + let mut backoff = Backoff::new(Duration::from_secs(1), Duration::from_secs(30)); + backoff.next_delay(); + backoff.next_delay(); + backoff.next_delay(); + backoff.reset(); + assert_eq!(backoff.next_delay(), Duration::from_secs(1)); + assert_eq!(backoff.next_delay(), Duration::from_secs(2)); + } + + #[test] + fn a_handshake_that_immediately_drops_does_not_reset_the_backoff() { + // `connected` flips the moment `client_async` returns, so a host that + // accepts the upgrade and then panics on the first Hello - a poisoned + // `desired` mutex does exactly that - completed a "successful" + // connection every time. Without a duration floor the backoff reset on + // each one, the 30s cap was unreachable, and the device sat in a ~1 Hz + // pinned-CA TLS handshake loop indefinitely. + assert!( + !session_proves_the_link_works(true, Duration::from_millis(5)), + "a 5ms session must not count as a working link" + ); + assert!( + !session_proves_the_link_works(true, MIN_SESSION_FOR_RESET - Duration::from_millis(1)), + "just under the floor must not reset" + ); + assert!( + session_proves_the_link_works(true, MIN_SESSION_FOR_RESET), + "a session at the floor is evidence the link works" + ); + assert!( + !session_proves_the_link_works(false, Duration::from_secs(600)), + "a dial that never handshook must not reset however long it took" + ); + } + + #[test] + fn a_changed_bootstrap_ends_the_process_so_the_new_token_is_adopted() { + // Re-running `avocado container dev up` mints a fresh token and + // overwrites bootstrap.json. The agent kept the old one: bulk pulls + // 401'd, the WS upgrade was rejected by the same host-side auth check, + // and the reconnect loop spun forever without ever exiting - so + // Restart=always never fired and the documented recovery could not work. + let base = Bootstrap { + bulk_endpoint: "10.0.2.2:15150".to_string(), + read_token: "T1".to_string(), + ca_cert_pem: "PEM".to_string(), + ws_endpoint: "10.0.2.2:15152".to_string(), + }; + + assert!(base.same_session_as(&base.clone())); + + for changed in [ + Bootstrap { + read_token: "T2".to_string(), + ..base.clone() + }, + Bootstrap { + ca_cert_pem: "OTHER".to_string(), + ..base.clone() + }, + Bootstrap { + bulk_endpoint: "10.0.2.2:15999".to_string(), + ..base.clone() + }, + Bootstrap { + ws_endpoint: "10.0.2.2:15999".to_string(), + ..base.clone() + }, + ] { + assert!( + !base.same_session_as(&changed), + "a changed {changed:?} must end the session" + ); + } + } + + #[tokio::test] + async fn a_raised_rebootstrap_wakes_a_waiter_and_clears_once_reported() { + // The flag was read exactly once per session, BEFORE the frame loop - + // and the 401 that raises it happens during a pull, inside that loop. So + // it was always set strictly after the only read that would have sent + // it. Nothing could lower it either, so one transient 401 latched for + // the process lifetime. + let flag = proxy::ReBootstrap::default(); + assert!(!flag.is_raised(), "a fresh flag must start lowered"); + + let waiter = { + let flag = flag.clone(); + tokio::spawn(async move { flag.raised().await }) + }; + // Let the waiter reach `notified()` before raising. + tokio::task::yield_now().await; + flag.raise(); + + tokio::time::timeout(Duration::from_secs(5), waiter) + .await + .expect("raise must wake a mid-session waiter") + .expect("waiter task"); + + assert!(flag.take(), "take must report that the flag was raised"); + assert!( + !flag.take(), + "the flag must lower once reported, or every later session re-announces it" + ); + } + + /// An engine whose pull always fails, so `handle_host_frame` takes its + /// reporting branch without execing anything. + struct FailingEngine; + + impl sync::Engine for FailingEngine { + async fn pull(&self, _reference: &str) -> Result<()> { + anyhow::bail!("simulated pull failure") + } + async fn tag(&self, _source: &str, _target: &str) -> Result<()> { + Ok(()) + } + async fn restart(&self, _container: &str, _service: Option<&str>) -> Result<()> { + Ok(()) + } + async fn running_image_id(&self, _container: &str) -> Result> { + Ok(None) + } + async fn resolve_image_id(&self, _reference: &str) -> Result> { + Ok(None) + } + } + + #[tokio::test] + async fn the_session_guard_aborts_its_worker_on_drop() { + // The abort used to sit after the serve loop, so the three `?` sites + // inside it - the two send_frame calls and the stream read - returned + // without it. The read one is the COMMON session end (tungstenite yields + // Some(Err(..)) on a reset), leaking one parked worker per errored + // session, and letting the old worker's pull/tag/restart race the new + // session's on `running_digest` and the pointer file. + // + // The guard makes that unreachable by construction: every exit from + // `connect_and_serve`, including a `?`, drops it. + // A worker that never finishes on its own, so the only way this await + // resolves is the guard cancelling it. + let handle = tokio::spawn(std::future::pending::<()>()); + let guard = AbortOnDrop(handle.abort_handle()); + + drop(guard); + + // Bounded: without the abort this await never resolves, and a hung test + // reads as CI infrastructure trouble rather than the regression it is. + let err = tokio::time::timeout(Duration::from_secs(5), handle) + .await + .expect("dropping the guard must abort the worker, not leave it parked") + .expect_err("an aborted worker must not report success"); + assert!( + err.is_cancelled(), + "the worker must end cancelled rather than panicking: {err}" + ); + } + + #[tokio::test] + async fn a_failed_sync_produces_a_status_frame_for_the_host() { + // A swallowed sync failure left the host with no Status, no Progress and + // no updated Hello - Hello is re-sent only on reconnect - so it went on + // showing the device synced at the old digest with no error surface. + // + // This drives `handle_host_frame` itself. The previous version built a + // DeviceFrame::Status inside the test body and asserted serde_json + // emitted its fields, so it covered serde and nothing else: reverting + // the whole reporting path back to a bare `warn!` returning None left it + // green. + let state = AgentState::new("dev-01".to_string()); + let tmp = tempfile::tempdir().unwrap(); + let cfg = sync::SyncConfig { + loopback_port: 15151, + container: "avocado-dev".to_string(), + writable_root: tmp.path().to_path_buf(), + }; + let frame = HostFrame::Sync { + image: "my-app".to_string(), + tag: "dev".to_string(), + digest: "sha256:new".to_string(), + service: None, + }; + + let report = handle_host_frame(&FailingEngine, &cfg, frame, &state) + .await + .expect("a failed sync must report a Status frame to the host"); + + match report { + DeviceFrame::Status(status) => { + assert_eq!(status.device_id, "dev-01"); + assert_eq!(status.state, "sync_failed"); + let detail = status.detail.expect("the failure must carry a detail"); + assert!( + detail.contains("my-app:dev") && detail.contains("sha256:new"), + "the detail must name the image that failed: {detail}" + ); + } + other => panic!("expected a Status frame, got {other:?}"), + } + } + + #[tokio::test] + async fn a_successful_sync_reports_nothing() { + // The mirror: `handle_host_frame` must stay silent on success, or every + // sync would emit a spurious Status. Without this the test above passes + // for an implementation that reports unconditionally. + struct OkEngine; + impl sync::Engine for OkEngine { + async fn pull(&self, _reference: &str) -> Result<()> { + Ok(()) + } + async fn tag(&self, _source: &str, _target: &str) -> Result<()> { + Ok(()) + } + async fn restart(&self, _container: &str, _service: Option<&str>) -> Result<()> { + Ok(()) + } + async fn running_image_id(&self, _container: &str) -> Result> { + Ok(Some("sha256:same".to_string())) + } + async fn resolve_image_id(&self, _reference: &str) -> Result> { + Ok(Some("sha256:same".to_string())) + } + } + + let state = AgentState::new("dev-01".to_string()); + let tmp = tempfile::tempdir().unwrap(); + let cfg = sync::SyncConfig { + loopback_port: 15151, + container: "avocado-dev".to_string(), + writable_root: tmp.path().to_path_buf(), + }; + let frame = HostFrame::Sync { + image: "my-app".to_string(), + tag: "dev".to_string(), + digest: "sha256:new".to_string(), + service: None, + }; + + assert!( + handle_host_frame(&OkEngine, &cfg, frame, &state) + .await + .is_none(), + "a successful sync must not report a Status frame" + ); + assert_eq!(*state.running_digest.lock().unwrap(), "sha256:new"); + } + + #[test] + fn hello_carries_current_running_digest() { + let state = AgentState::new("dev-01".to_string()); + // Simulate a pull landing (task 6.3 behavior). + *state.running_digest.lock().unwrap() = "sha256:new".to_string(); + match state.hello() { + DeviceFrame::Hello(hello) => { + assert_eq!(hello.device_id, "dev-01"); + assert_eq!(hello.running_digest, "sha256:new"); + assert_eq!(hello.arch, std::env::consts::ARCH); + } + other => panic!("expected Hello, got {other:?}"), + } + } + + #[test] + fn endpoint_host_strips_port() { + assert_eq!(endpoint_host("10.0.0.1:9443"), "10.0.0.1"); + assert_eq!(endpoint_host("host.local:443"), "host.local"); + assert_eq!(endpoint_host("nohost"), "nohost"); + } + + #[test] + fn build_tls_config_rejects_empty_pem() { + let err = build_tls_config("not a cert").unwrap_err(); + assert!( + err.to_string().contains("no certificates"), + "unexpected error: {err}" + ); + } + + // The seam that makes the proxy's accept-retry mean anything. Losing the + // proxy must end `main`, because that is what `Restart=always` needs in + // order to fire; a returning proxy task whose handle was dropped left + // systemd looking at a healthy unit while every pull failed + // connection-refused. + #[tokio::test] + async fn a_failing_proxy_ends_the_process_even_while_the_ws_keeps_serving() { + let proxy = tokio::spawn(async { Err(anyhow::anyhow!("listener died")) }); + // A control-WS loop that would otherwise run forever, as `run()` does. + let serving = std::future::pending::>(); + + let err = tokio::time::timeout( + std::time::Duration::from_secs(5), + end_when_either_ends(proxy, serving), + ) + .await + .expect("losing the proxy must end this, not hang on the WS loop") + .expect_err("a failed proxy must surface an error"); + + assert!( + format!("{err:#}").contains("listener died"), + "the proxy's own error must survive to the exit: {err:#}" + ); + } + + // A proxy that stops cleanly is still fatal: without it no pull can + // succeed, so exiting to be rebuilt beats serving a control WS that can + // only ever report failures. + #[tokio::test] + async fn a_cleanly_stopped_proxy_also_ends_the_process() { + let proxy = tokio::spawn(async { Ok(()) }); + let serving = std::future::pending::>(); + + let err = tokio::time::timeout( + std::time::Duration::from_secs(5), + end_when_either_ends(proxy, serving), + ) + .await + .expect("a stopped proxy must end this") + .expect_err("a stopped proxy must be reported as an error"); + assert!(format!("{err:#}").contains("stopped serving"), "{err:#}"); + } + + // The other direction: the WS loop ending returns its own result rather + // than being masked by the still-running proxy. + #[tokio::test] + async fn a_failing_ws_loop_returns_its_own_error() { + let proxy = tokio::spawn(async { std::future::pending::>().await }); + let serving = async { Err(anyhow::anyhow!("ws gave up")) }; + + let err = tokio::time::timeout( + std::time::Duration::from_secs(5), + end_when_either_ends(proxy, serving), + ) + .await + .expect("the WS loop ending must end this") + .expect_err("a failed WS loop must surface an error"); + assert!(format!("{err:#}").contains("ws gave up"), "{err:#}"); + } +} diff --git a/agent/src/proxy.rs b/agent/src/proxy.rs new file mode 100644 index 0000000..bebd02c --- /dev/null +++ b/agent/src/proxy.rs @@ -0,0 +1,1284 @@ +//! Device loopback registry proxy (design D3/D9; task 6.2). +//! +//! The device container engine (docker/podman) pulls from a **plain-HTTP +//! loopback** registry this module serves on `127.0.0.1:`. Engines +//! auto-trust a loopback registry, so the device needs NO insecure-registry +//! flag and NO per-engine trust/cert file — that is how the two "NO ..." +//! guarantees are met: by serving loopback HTTP we need zero engine trust +//! config at all. +//! +//! Each engine read (`GET`/`HEAD` on `/v2/…/manifests/…` and +//! `/v2/…/blobs/…`, plus the `GET /v2/` ping) is forwarded to the host's +//! dedicated **bulk HTTPS listener** (D9) over a rustls client pinned to ONLY +//! the bootstrap CA (reusing [`crate::build_tls_config`]), carrying the +//! read/control token as `Authorization: Bearer …`. Blob bytes ride THIS +//! HTTPS path, never the control WebSocket (D9 splits bulk from control). +//! +//! Fail-closed pre-stream (M-A / H-2 / M-3): the UPSTREAM status is inspected +//! BEFORE any engine-facing `200` is opened. On an upstream `2xx` we open the +//! `2xx` and stream the body through. On an upstream `401`/`403` we do NOT open +//! a `200`: we return a clean gateway error that carries neither the upstream +//! status semantics nor its `WWW-Authenticate` header, and we raise a shared +//! "needs re-bootstrap" flag the agent surfaces (the agent does not loop +//! silently). There is NO token refresh/renewal endpoint — a new token comes +//! only from a host re-`up` re-bootstrap. The mid-stream 401 case is PREVENTED +//! by the host's drain-based grace-overlap (D5), not signalled: OCI/HTTP have +//! no "terminal, do not retry" wire signal, so if the drain ceiling is ever +//! exceeded the agent's mid-stream behavior is best-effort (the stream simply +//! cuts), explicitly NOT a load-bearing terminal signal. The upstream +//! `WWW-Authenticate` is never proxied back to the engine under any status. +//! +//! # Trust assumptions +//! +//! **The loopback listener performs no device-side authorization.** Any local +//! process that can reach `127.0.0.1:` gets its `GET`/`HEAD` forwarded +//! upstream under the agent's `Authorization: Bearer `, so the read +//! token is effectively shared with every process on the device. That is +//! deliberate, and it is the same trust boundary the "no engine trust config" +//! guarantee buys: the engine authenticates to this proxy by being local, and +//! adding a device-side gate would need a credential the engine would then have +//! to be configured with — reintroducing exactly the per-engine trust file the +//! design exists to avoid. What bounds the exposure is that the socket is +//! loopback-only (never `0.0.0.0`), the method allowlist is GET/HEAD so a local +//! process can only read, and the whole extension is dev-only and gated on a +//! bootstrap the host must deliver. A device running untrusted local workloads +//! is outside this threat model; the mitigation there is not to install the dev +//! extension. +//! +//! **Each upstream request opens a fresh TCP + TLS + HTTP/1 connection.** There +//! is no connection pool, so pulling a multi-layer image pays one full TLS +//! handshake per manifest and per blob. That is accepted rather than optimized: +//! a pool would need idle-connection eviction and reuse-after-error handling on +//! a dev-only path whose cost is a handful of handshakes per sync. Revisit it if +//! this proxy ever carries a production pull. + +use std::convert::Infallible; +use std::future::Future; +use std::net::{Ipv4Addr, SocketAddr}; +use std::sync::Arc; +use std::sync::atomic::{AtomicBool, Ordering}; +use std::time::Duration; + +use anyhow::{Context, Result}; +use bytes::Bytes; +use http_body_util::combinators::BoxBody; +use http_body_util::{BodyExt, Empty, Full}; +use hyper::body::Incoming; +use hyper::header::HeaderValue; +use hyper::header::{AUTHORIZATION, HOST, RANGE}; +use hyper::service::service_fn; +use hyper::{HeaderMap, Method, Request, Response, StatusCode}; +use hyper_util::rt::TokioIo; +use tokio::net::{TcpListener, TcpStream}; +use tokio::time::sleep; +use tokio_rustls::TlsConnector; +use tokio_rustls::rustls::pki_types::ServerName; +use tracing::{info, warn}; + +use crate::{Bootstrap, build_tls_config, endpoint_host}; + +/// The engine-facing response body: boxed so both streamed upstream bodies and +/// small static error bodies share one type. +type RespBody = BoxBody; + +/// Upstream headers copied through on a successful (`2xx`) read. Deliberately a +/// strict allowlist: `www-authenticate` (and every other header) is dropped +/// unless it is on this list, so an upstream auth header can never reach the +/// engine even on a `2xx`. +const PASSTHROUGH_HEADERS: &[&str] = &[ + "content-type", + "content-length", + "content-range", + "accept-ranges", + "docker-content-digest", + "etag", + "last-modified", +]; + +/// The plain-text body returned to the engine when the bulk upstream rejects +/// the read/control token. It carries no upstream status or header semantics. +const REBOOTSTRAP_MSG: &str = + "bulk upstream rejected the read/control token; re-run `avocado container dev up`"; + +/// A source of inbound connections. +/// +/// Exists so the accept-error retry policy can be tested; production always +/// uses the [`TcpListener`] impl below. +pub(crate) trait Accept: Send { + type Conn: tokio::io::AsyncRead + tokio::io::AsyncWrite + Send + Unpin + 'static; + + fn accept(&self) -> impl Future> + Send; +} + +impl Accept for TcpListener { + type Conn = tokio::net::TcpStream; + + async fn accept(&self) -> std::io::Result { + TcpListener::accept(self) + .await + .map(|(stream, _peer)| stream) + } +} + +/// Pause after a failed `accept()` before trying again, so a transient error +/// does not turn the serve loop into a busy spin. +const ACCEPT_ERROR_BACKOFF: Duration = Duration::from_millis(100); + +/// How many consecutive `accept()` failures end the serve loop. +/// +/// A transient failure should not kill the proxy, but a permanently bad +/// listener must not be retried forever: the task would never return, so the +/// process would never exit and `Restart=always` would have nothing to restart. +/// Returning after a run of failures hands recovery back to the supervisor. +const ACCEPT_ERROR_LIMIT: u32 = 32; + +// --------------------------------------------------------------------------- +// Re-bootstrap status flag. +// --------------------------------------------------------------------------- + +/// A shared, clonable "needs re-bootstrap" flag. +/// +/// Raised when the bulk upstream rejects the read/control token (a stale token +/// after a host re-`up`). The agent surfaces it (task 6.1's WS loop emits a +/// `Status` frame); it is never a silent retry loop, and there is no renewal +/// endpoint — the token is refreshed only by a host re-`up` re-bootstrap. +/// Carries a `Notify` as well as the flag, because a bare flag could not be +/// delivered. The WS loop read `is_raised()` exactly once per session, BEFORE +/// entering its frame loop - and the 401 that raises it happens during a pull, +/// which is inside that loop. So the flag was set strictly after the only read +/// that would have transmitted it, and with the sink otherwise idle the host was +/// never told. There was also no way to lower it, so one transient 401 latched +/// for the process lifetime. +#[derive(Clone, Default)] +pub(crate) struct ReBootstrap { + raised: Arc, + changed: Arc, +} + +impl ReBootstrap { + /// Raise the flag and wake anyone waiting to report it. Idempotent in the + /// flag; the wake is what makes it reach the host mid-session. + /// + /// `notify_one`, not `notify_waiters`: the reporting select arm builds a + /// fresh `notified()` future on every iteration, so it is only registered as + /// a waiter while parked in the select. A 401 raised while the loop body ran + /// another arm - mid-`send_frame`, say - found no waiter, and + /// `notify_waiters` stores no permit, so the wake was dropped. The flag + /// stayed set and surfaced at the next session start, which is exactly the + /// mid-session delivery this was supposed to fix. `notify_one` stores the + /// permit, so the next `notified()` completes immediately. + pub(crate) fn raise(&self) { + self.raised.store(true, Ordering::SeqCst); + self.changed.notify_one(); + } + + /// Whether a re-bootstrap has been requested. + /// + /// Test-only: the reporting path reads the flag exclusively through + /// [`Self::take`], because observing and lowering have to be one step for a + /// raise landing mid-send not to be erased. A non-consuming read is still + /// what the proxy's own tests want - "the 401 raised it", "the 200 did not" + /// - and those must not lower it as a side effect. + #[cfg(test)] + pub(crate) fn is_raised(&self) -> bool { + self.raised.load(Ordering::SeqCst) + } + + /// Lower the flag, reporting whether it had been raised. + /// + /// Atomic so the caller can lower it BEFORE reporting without racing a + /// concurrent `raise`. Lowering is what keeps a single transient 401 from + /// making every later session announce `needs_rebootstrap` long after the + /// token it referred to was replaced. + pub(crate) fn take(&self) -> bool { + self.raised.swap(false, Ordering::SeqCst) + } + + /// Resolve when the flag is next raised. + pub(crate) async fn raised(&self) { + self.changed.notified().await; + } +} + +// --------------------------------------------------------------------------- +// Upstream abstraction (injectable for tests). +// --------------------------------------------------------------------------- + +/// One upstream read response: the status is available BEFORE the body is +/// consumed, which is what lets the proxy fail closed pre-stream. +pub(crate) struct UpstreamResponse { + pub(crate) status: StatusCode, + pub(crate) headers: HeaderMap, + pub(crate) body: RespBody, +} + +/// The bulk-read upstream. Abstracted so the proxy logic is unit-testable with +/// an in-process fake, while [`HttpsUpstream`] carries the real pinned-CA TLS +/// leg used on the device. +pub(crate) trait Upstream: Send + Sync + 'static { + fn fetch( + &self, + method: Method, + path: String, + range: Option, + ) -> impl Future> + Send; +} + +/// The real upstream: a fresh TLS connection per request to the host bulk +/// listener, pinned to the bootstrap CA, carrying the Bearer read/control +/// token. TLS is driven by `tokio-rustls` (ring) directly — never a +/// higher-level client that could drag in a second crypto provider. +pub(crate) struct HttpsUpstream { + connector: TlsConnector, + server_name: ServerName<'static>, + /// `host:port` of the bulk listener — used for the TCP connect and the + /// `Host` header. + authority: String, + read_token: String, +} + +impl HttpsUpstream { + /// Build the upstream from the bootstrap: pins the CA and captures the bulk + /// endpoint + read/control token. Fails if the CA PEM is unusable or the + /// endpoint host is not a valid TLS server name. + pub(crate) fn from_bootstrap(bootstrap: &Bootstrap) -> Result { + let tls_config = build_tls_config(&bootstrap.ca_cert_pem)?; + let connector = TlsConnector::from(Arc::new(tls_config)); + let host = endpoint_host(&bootstrap.bulk_endpoint).to_string(); + let server_name = ServerName::try_from(host).context("invalid bulk TLS server name")?; + Ok(Self { + connector, + server_name, + authority: bootstrap.bulk_endpoint.clone(), + read_token: bootstrap.read_token.clone(), + }) + } +} + +/// Ceiling on each pre-body upstream stage: TCP connect, TLS handshake, HTTP/1 +/// handshake, and the request/response head. +/// +/// None of these had one. A laptop that sleeps or drops off Wi-Fi mid-pull leaves +/// a half-open connection with no RST; no SO_KEEPALIVE is set anywhere, so the +/// 7200s tcp_keepalive_time never applies, and after `send_request` the agent is +/// purely READING, so there are no outbound retransmits for tcp_retries2 to give +/// up on. The read hung forever: `docker pull` waited on a loopback response that +/// never arrived, `on_sync` never returned, and - before syncs moved off the read +/// loop - the WS was never polled again either. systemd reported +/// `active (running)` with the device completely dead and nothing logged. +/// +/// Generous on purpose: this bounds a stall, not a slow link. Body streaming is +/// deliberately NOT wrapped, because a multi-gigabyte layer over a slow link is +/// legitimately long; the stages here are the ones that should complete in +/// milliseconds on any working link. +const UPSTREAM_STAGE_TIMEOUT: Duration = Duration::from_secs(30); + +/// Run `fut`, failing with a named error if it does not finish in time. +async fn with_stage_timeout( + stage: &'static str, + fut: impl Future>, +) -> Result { + match tokio::time::timeout(UPSTREAM_STAGE_TIMEOUT, fut).await { + Ok(result) => result, + Err(_) => anyhow::bail!( + "{stage} did not complete within {}s; the host is unreachable or the connection \ + is half-open", + UPSTREAM_STAGE_TIMEOUT.as_secs() + ), + } +} + +impl Upstream for HttpsUpstream { + async fn fetch( + &self, + method: Method, + path: String, + range: Option, + ) -> Result { + let tcp = with_stage_timeout("connecting to the bulk listener", async { + TcpStream::connect(&self.authority) + .await + .with_context(|| format!("connecting to bulk listener {}", self.authority)) + }) + .await?; + let tls = with_stage_timeout("the bulk TLS handshake", async { + self.connector + .connect(self.server_name.clone(), tcp) + .await + .context("bulk TLS handshake (pinned CA)") + }) + .await?; + + let io = TokioIo::new(tls); + let (mut sender, conn) = with_stage_timeout("the bulk HTTP/1 handshake", async { + hyper::client::conn::http1::handshake(io) + .await + .context("bulk HTTP/1 handshake") + }) + .await?; + tokio::spawn(async move { + if let Err(e) = conn.await { + warn!(error = %e, "bulk upstream connection closed with error"); + } + }); + + let mut builder = Request::builder() + .method(method) + .uri(path) + .header(HOST, &self.authority) + .header(AUTHORIZATION, format!("Bearer {}", self.read_token)); + if let Some(range) = range { + builder = builder.header(RANGE, range); + } + let request = builder + .body(Empty::::new()) + .context("building upstream request")?; + + let response = with_stage_timeout("the upstream request", async { + sender + .send_request(request) + .await + .context("sending upstream request") + }) + .await?; + + let status = response.status(); + let headers = response.headers().clone(); + let body = response.into_body().map_err(std::io::Error::other).boxed(); + Ok(UpstreamResponse { + status, + headers, + body, + }) + } +} + +// --------------------------------------------------------------------------- +// Proxy logic. +// --------------------------------------------------------------------------- + +/// Ceiling on the gap between two body chunks from the bulk upstream. +/// +/// [`UPSTREAM_STAGE_TIMEOUT`] bounds every stage BEFORE the body, but the stall +/// those were written for - a laptop that sleeps or drops off Wi-Fi mid-pull, +/// leaving a half-open connection with no RST - lands overwhelmingly DURING body +/// transfer, which was the one stage still uncovered. The outcome had been +/// softened rather than removed: the read loop keeps the WS polled so the socket +/// no longer wedges, but the sync worker parked forever, later `Sync` frames +/// piled into the latest-wins slot and never ran, no `Status` was emitted +/// because `on_sync` never returned, and systemd still reported +/// `active (running)`. +/// +/// An IDLE deadline, not a total-transfer one: capping total transfer would kill +/// a legitimate multi-gigabyte layer over a slow link, while a link that is +/// moving bytes at all keeps resetting this. +#[cfg(not(test))] +const UPSTREAM_BODY_IDLE_TIMEOUT: Duration = Duration::from_secs(60); + +/// Short under test so the WIRING is assertable, not just the wrapper. +/// +/// Virtual time cannot cover it: with the wrapper removed there is no timer left +/// inside the body at all, and tokio's paused-clock auto-advance does not fire +/// for an outer `timeout` when the inner future parks without registering a +/// waker - the test hangs instead of failing, which is the one outcome a +/// regression must not produce. [`the_production_body_idle_timeout_is_generous`] +/// pins the shipped value so this override cannot hide a typo in it. +#[cfg(test)] +const UPSTREAM_BODY_IDLE_TIMEOUT: Duration = Duration::from_millis(50); + +/// The idle deadline the shipped binary uses, kept beside the override so both +/// values are visible together and the real one stays asserted. +#[cfg(test)] +const PRODUCTION_BODY_IDLE_TIMEOUT: Duration = Duration::from_secs(60); + +/// Wraps a body so a gap of [`UPSTREAM_BODY_IDLE_TIMEOUT`] between chunks ends +/// the stream with an error instead of hanging forever. +struct IdleTimeoutBody { + inner: B, + idle: std::pin::Pin>, +} + +impl IdleTimeoutBody { + fn new(inner: B) -> Self { + Self { + inner, + idle: Box::pin(sleep(UPSTREAM_BODY_IDLE_TIMEOUT)), + } + } +} + +impl hyper::body::Body for IdleTimeoutBody +where + B: hyper::body::Body + Unpin, +{ + type Data = Bytes; + type Error = std::io::Error; + + fn poll_frame( + mut self: std::pin::Pin<&mut Self>, + cx: &mut std::task::Context<'_>, + ) -> std::task::Poll, std::io::Error>>> + { + use std::task::Poll; + + let this = self.as_mut().get_mut(); + match std::pin::Pin::new(&mut this.inner).poll_frame(cx) { + // Any progress - a chunk, the end of the stream, or an error - + // restarts the clock from here. + Poll::Ready(frame) => { + this.idle + .as_mut() + .reset(tokio::time::Instant::now() + UPSTREAM_BODY_IDLE_TIMEOUT); + Poll::Ready(frame) + } + Poll::Pending => match this.idle.as_mut().poll(cx) { + Poll::Ready(()) => Poll::Ready(Some(Err(std::io::Error::new( + std::io::ErrorKind::TimedOut, + format!( + "the bulk upstream sent no body data for {}s; the host is unreachable \ + or the connection is half-open", + UPSTREAM_BODY_IDLE_TIMEOUT.as_secs() + ), + )))), + Poll::Pending => Poll::Pending, + }, + } + } + + fn size_hint(&self) -> hyper::body::SizeHint { + self.inner.size_hint() + } +} + +/// Build a small static-body engine response (used for every gateway error). +fn gateway_error(status: StatusCode, message: &'static str) -> Response { + let body = Full::new(Bytes::from_static(message.as_bytes())) + .map_err(|never| match never {}) + .boxed(); + Response::builder() + .status(status) + .header(hyper::header::CONTENT_TYPE, "text/plain; charset=utf-8") + .body(body) + .expect("static gateway error response is always valid") +} + +/// Core read proxy. Operates on already-extracted request parts so it is +/// unit-testable without constructing a hyper `Incoming` body. +/// +/// Fail-closed pre-stream: the upstream status is fully classified before any +/// engine-facing `2xx` (and its body) is opened. +pub(crate) async fn proxy_read( + method: Method, + path: String, + range: Option, + upstream: &U, + rebootstrap: &ReBootstrap, +) -> Response { + // The device only ever reads. Anything else never reaches the host. + if method != Method::GET && method != Method::HEAD { + return gateway_error( + StatusCode::METHOD_NOT_ALLOWED, + "only GET and HEAD are proxied", + ); + } + + match upstream.fetch(method, path, range).await { + Ok(upstream) if upstream.status.is_success() => { + // Only now, with a confirmed 2xx, do we open the engine response + // and stream the body through. Headers are a strict allowlist — + // `WWW-Authenticate` is never among them. + let mut builder = Response::builder().status(upstream.status); + for name in PASSTHROUGH_HEADERS { + if let Some(value) = upstream.headers.get(*name) { + builder = builder.header(*name, value); + } + } + builder + .body(IdleTimeoutBody::new(upstream.body).boxed()) + .expect("engine success response is always valid") + } + Ok(upstream) + if upstream.status == StatusCode::UNAUTHORIZED + || upstream.status == StatusCode::FORBIDDEN => + { + // Fail-closed: no 200 was opened, so no body has streamed. Raise + // the re-bootstrap status and return a CLEAN gateway error that + // leaks neither the upstream 401/403 nor its WWW-Authenticate. + rebootstrap.raise(); + warn!( + upstream_status = %upstream.status, + "bulk upstream rejected the read/control token; raising re-bootstrap status" + ); + gateway_error(StatusCode::BAD_GATEWAY, REBOOTSTRAP_MSG) + } + Ok(upstream) => { + // Any other non-2xx is treated as transient. Surface a gateway + // error the engine may retry; do NOT raise re-bootstrap. + warn!(upstream_status = %upstream.status, "bulk upstream returned a transient error"); + gateway_error(StatusCode::BAD_GATEWAY, "bulk upstream returned an error") + } + Err(e) => { + warn!(error = %e, "bulk upstream request failed"); + gateway_error(StatusCode::BAD_GATEWAY, "bulk upstream unreachable") + } + } +} + +// --------------------------------------------------------------------------- +// Loopback listener. +// --------------------------------------------------------------------------- + +/// Bind the loopback registry socket. Always `127.0.0.1` (never `0.0.0.0`): the +/// registry is device-local only, which is what keeps it trust-free for the +/// engine and unreachable from the LAN. +pub(crate) async fn bind_loopback(port: u16) -> Result { + let addr = SocketAddr::from((Ipv4Addr::LOCALHOST, port)); + TcpListener::bind(addr) + .await + .with_context(|| format!("binding loopback registry on {addr}")) +} + +/// Serve the loopback registry over `listener`, forwarding every read to +/// `upstream`. Runs until the listener errors. +/// The accept loop, over anything that can accept connections. +/// +/// Generic over [`Accept`] rather than taking a `TcpListener` so the retry +/// policy is reachable from a test: forcing repeated `accept()` failures on a +/// real listener means corrupting its fd, which is not portable. Mirrors the +/// [`Upstream`] seam this module already uses for the same reason. +pub(crate) async fn serve_on( + listener: L, + upstream: Arc, + rebootstrap: ReBootstrap, +) -> Result<()> { + // Consecutive accept failures. Reset by any successful accept, so a run has + // to be genuinely unbroken to reach the limit. + let mut consecutive_errors: u32 = 0; + loop { + let stream = match listener.accept().await { + Ok(stream) => { + consecutive_errors = 0; + stream + } + // A single accept error (a peer that vanished between the SYN and + // the accept, a momentary EMFILE) says nothing about the listener, + // and killing the proxy over one would leave every later Sync pull + // failing connection-refused. Retry those. + // + // But retrying forever is its own failure: on a permanently bad + // listener the task never returns, so the process never exits and + // `Restart=always` has nothing to restart - the proxy is dead and + // the supervisor cannot tell. Give up after an unbroken run and let + // the error propagate so systemd can do its job. + Err(e) => { + consecutive_errors += 1; + if consecutive_errors >= ACCEPT_ERROR_LIMIT { + return Err(e).with_context(|| { + format!("accepting loopback connections failed {consecutive_errors} times in a row") + }); + } + warn!( + error = %e, + consecutive_errors, + "accepting loopback connection failed; retrying" + ); + sleep(ACCEPT_ERROR_BACKOFF).await; + continue; + } + }; + let upstream = upstream.clone(); + let rebootstrap = rebootstrap.clone(); + tokio::spawn(async move { + let io = TokioIo::new(stream); + let service = service_fn(move |req: Request| { + let upstream = upstream.clone(); + let rebootstrap = rebootstrap.clone(); + async move { + let method = req.method().clone(); + let path = req + .uri() + .path_and_query() + .map(|pq| pq.as_str().to_owned()) + .unwrap_or_else(|| req.uri().path().to_owned()); + let range = req.headers().get(RANGE).cloned(); + Ok::<_, Infallible>( + proxy_read(method, path, range, upstream.as_ref(), &rebootstrap).await, + ) + } + }); + if let Err(e) = hyper::server::conn::http1::Builder::new() + .serve_connection(io, service) + .await + { + warn!(error = %e, "loopback registry connection error"); + } + }); + } +} + +/// Bind `127.0.0.1:port` and serve the loopback registry. The real device +/// entry point (called from `main`). +pub(crate) async fn serve_loopback( + port: u16, + upstream: Arc, + rebootstrap: ReBootstrap, +) -> Result<()> { + let listener = bind_loopback(port).await?; + match listener.local_addr() { + Ok(addr) => info!(%addr, "loopback registry proxy listening"), + Err(_) => info!("loopback registry proxy listening"), + } + serve_on(listener, upstream, rebootstrap).await +} + +#[cfg(test)] +mod tests { + use super::*; + + use std::sync::Arc; + use std::sync::atomic::AtomicUsize; + + /// A listener whose `accept()` fails a scripted number of times. + /// + /// `fail_forever` models a listener that has gone permanently bad - the case + /// the retry bound exists for. `failing_then_recovering` models a transient run + /// followed by recovery, so the counter's reset can be observed. + #[derive(Clone)] + struct ScriptedListener { + attempts: Arc, + fail_forever: bool, + succeed_after: usize, + } + + impl ScriptedListener { + fn failing_forever() -> Self { + Self { + attempts: Arc::new(AtomicUsize::new(0)), + fail_forever: true, + succeed_after: 0, + } + } + + fn failing_then_recovering(succeed_after: usize) -> Self { + Self { + attempts: Arc::new(AtomicUsize::new(0)), + fail_forever: false, + succeed_after, + } + } + + fn attempts(&self) -> usize { + self.attempts.load(Ordering::SeqCst) + } + } + + impl Accept for ScriptedListener { + type Conn = tokio::io::DuplexStream; + + async fn accept(&self) -> std::io::Result { + let n = self.attempts.fetch_add(1, Ordering::SeqCst); + if self.fail_forever || n < self.succeed_after { + return Err(std::io::Error::other("scripted accept failure")); + } + if n == self.succeed_after { + // One success in the middle: enough to reset the counter. + let (a, _b) = tokio::io::duplex(64); + return Ok(a); + } + if n <= self.succeed_after * 2 { + // A SECOND failing run after the success. Both runs are one + // short of the limit, so the loop survives only if the success + // reset the counter - cumulatively they are well past it. + return Err(std::io::Error::other("scripted accept failure")); + } + // Then park, the way a real listener waits for the next peer. A + // listener that kept returning instantly would spin the loop. + std::future::pending().await + } + } + + /// A listener that fails forever must end the loop rather than spin, so the + /// process exits and `Restart=always` can recover it. Before the bound was + /// added this returned never, which is what made the supervisor useless. + #[tokio::test(start_paused = true)] + async fn a_permanently_failing_listener_ends_the_serve_loop() { + let listener = ScriptedListener::failing_forever(); + let upstream = Arc::new(ok_upstream()); + let rebootstrap = ReBootstrap::default(); + + let started = tokio::time::Instant::now(); + let err = tokio::time::timeout( + std::time::Duration::from_secs(30), + serve_on(listener, upstream, rebootstrap), + ) + .await + .expect("the loop must terminate rather than spin forever") + .expect_err("a permanently failing listener must surface an error"); + + assert!( + err.to_string().contains("in a row"), + "the error should name the consecutive-failure run: {err}" + ); + + // The backoff is the only thing standing between a sustained accept + // error and a busy spin, and without this it had no coverage at all - + // deleting the sleep left every proxy test green. Time is virtual here + // (`start_paused`), so this asserts the sleep actually elapsed without + // the suite paying for it. + let elapsed = started.elapsed(); + let want = ACCEPT_ERROR_BACKOFF * (ACCEPT_ERROR_LIMIT - 1); + assert!( + elapsed >= want, + "each retry must back off: expected at least {want:?} across \ + {ACCEPT_ERROR_LIMIT} attempts, only {elapsed:?} elapsed" + ); + } + + /// The counter must reset on a successful accept, or a long-lived proxy that + /// sees scattered transient errors would eventually shut itself down for no + /// reason. Fails if the reset is removed. + #[tokio::test(start_paused = true)] + async fn a_successful_accept_resets_the_error_counter() { + // More total failures than the limit, but never a consecutive run that + // reaches it: two batches either side of a success. + let listener = ScriptedListener::failing_then_recovering(ACCEPT_ERROR_LIMIT as usize - 1); + let observed = listener.clone(); + let upstream = Arc::new(ok_upstream()); + + let handle = tokio::spawn(serve_on(listener, upstream, ReBootstrap::default())); + + // Wait for both failing runs plus the success between them to elapse. + let want = 2 * (ACCEPT_ERROR_LIMIT as usize - 1) + 1; + let deadline = std::time::Instant::now() + std::time::Duration::from_secs(60); + while observed.attempts() < want && !handle.is_finished() { + assert!(std::time::Instant::now() < deadline, "test timed out"); + tokio::time::sleep(std::time::Duration::from_millis(25)).await; + } + + assert!( + !handle.is_finished(), + "the loop gave up after {} accepts: a success between two runs of \ + {} failures must reset the counter, since neither run reaches the \ + limit of {ACCEPT_ERROR_LIMIT} on its own", + observed.attempts(), + ACCEPT_ERROR_LIMIT - 1, + ); + handle.abort(); + } + + /// A minimal upstream for the accept-loop tests, which never reach it. + fn ok_upstream() -> FakeUpstream { + FakeUpstream { + status: StatusCode::OK, + headers: Vec::new(), + body: "", + } + } + + use http_body_util::BodyExt; + use hyper::body::Incoming; + use hyper::service::service_fn; + use rcgen::{BasicConstraints, CertificateParams, IsCa, KeyPair}; + use tokio::net::TcpStream; + use tokio_rustls::TlsAcceptor; + use tokio_rustls::rustls::pki_types::{ + CertificateDer, PrivateKeyDer, PrivatePkcs8KeyDer, ServerName, + }; + + // --- helpers ----------------------------------------------------------- + + fn full(bytes: impl Into) -> RespBody { + Full::new(bytes.into()) + .map_err(|never| match never {}) + .boxed() + } + + /// A CA plus a leaf it signed for `127.0.0.1`. + struct Ca { + ca_pem: String, + leaf_cert: CertificateDer<'static>, + leaf_key: PrivateKeyDer<'static>, + } + + fn make_ca() -> Ca { + let mut ca_params = CertificateParams::new(Vec::new()).expect("ca params"); + ca_params.is_ca = IsCa::Ca(BasicConstraints::Unconstrained); + let ca_key = KeyPair::generate().expect("ca key"); + let ca_cert = ca_params.self_signed(&ca_key).expect("self-sign ca"); + + let leaf_params = + CertificateParams::new(vec!["127.0.0.1".to_string()]).expect("leaf params"); + let leaf_key = KeyPair::generate().expect("leaf key"); + let leaf_cert = leaf_params + .signed_by(&leaf_key, &ca_cert, &ca_key) + .expect("sign leaf"); + + Ca { + ca_pem: ca_cert.pem(), + leaf_cert: leaf_cert.der().clone(), + leaf_key: PrivateKeyDer::from(PrivatePkcs8KeyDer::from(leaf_key.serialize_der())), + } + } + + /// Start an in-process TLS "host bulk listener" presenting `cert`/`key`. + /// A path containing `unauth` returns 401 + `WWW-Authenticate`; anything + /// else returns 200 with a known body. + async fn start_tls_upstream( + cert: CertificateDer<'static>, + key: PrivateKeyDer<'static>, + ) -> SocketAddr { + let provider = Arc::new(tokio_rustls::rustls::crypto::ring::default_provider()); + let config = tokio_rustls::rustls::ServerConfig::builder_with_provider(provider) + .with_safe_default_protocol_versions() + .expect("server protocol versions") + .with_no_client_auth() + .with_single_cert(vec![cert], key) + .expect("server single cert"); + let acceptor = TlsAcceptor::from(Arc::new(config)); + + let listener = TcpListener::bind((Ipv4Addr::LOCALHOST, 0)) + .await + .expect("bind tls upstream"); + let addr = listener.local_addr().expect("upstream addr"); + + tokio::spawn(async move { + loop { + let (tcp, _) = match listener.accept().await { + Ok(pair) => pair, + Err(_) => return, + }; + let acceptor = acceptor.clone(); + tokio::spawn(async move { + let tls = match acceptor.accept(tcp).await { + Ok(tls) => tls, + Err(_) => return, + }; + let io = TokioIo::new(tls); + let service = service_fn(|req: Request| async move { + let resp = if req.uri().path().contains("unauth") { + Response::builder() + .status(StatusCode::UNAUTHORIZED) + .header("www-authenticate", "Bearer realm=\"https://evil/token\"") + .body(full("denied")) + .unwrap() + } else { + Response::builder() + .status(StatusCode::OK) + .header("content-type", "application/octet-stream") + .header("docker-content-digest", "sha256:deadbeef") + .body(full("BLOBDATA")) + .unwrap() + }; + Ok::<_, Infallible>(resp) + }); + let _ = hyper::server::conn::http1::Builder::new() + .serve_connection(io, service) + .await; + }); + } + }); + + addr + } + + /// A plain-HTTP (no TLS) GET against the loopback registry, standing in for + /// the device engine. Returns status, headers, and the collected body. + async fn plain_get(addr: SocketAddr, path: &str) -> (StatusCode, HeaderMap, Bytes) { + let tcp = TcpStream::connect(addr).await.expect("connect loopback"); + let io = TokioIo::new(tcp); + let (mut sender, conn) = hyper::client::conn::http1::handshake(io) + .await + .expect("loopback handshake"); + tokio::spawn(async move { + let _ = conn.await; + }); + let request = Request::builder() + .method(Method::GET) + .uri(path) + .header(HOST, addr.to_string()) + .body(Empty::::new()) + .expect("loopback request"); + let response = sender + .send_request(request) + .await + .expect("loopback response"); + let status = response.status(); + let headers = response.headers().clone(); + let body = response + .into_body() + .collect() + .await + .expect("collect") + .to_bytes(); + (status, headers, body) + } + + /// A fake upstream returning a canned response, for the pure proxy-logic + /// tests that need no network. + struct FakeUpstream { + status: StatusCode, + headers: Vec<(&'static str, &'static str)>, + body: &'static str, + } + + impl Upstream for FakeUpstream { + async fn fetch( + &self, + _method: Method, + _path: String, + _range: Option, + ) -> Result { + let mut headers = HeaderMap::new(); + for (name, value) in &self.headers { + headers.insert( + hyper::header::HeaderName::from_static(name), + HeaderValue::from_static(value), + ); + } + Ok(UpstreamResponse { + status: self.status, + headers, + body: full(self.body), + }) + } + } + + #[test] + fn the_production_body_idle_timeout_is_generous() { + // The suite runs with a 50ms override, so nothing else would notice the + // shipped value being wrong. It has to be long enough that a slow but + // live link is never cut - it bounds a stall, not a slow transfer. + assert!( + PRODUCTION_BODY_IDLE_TIMEOUT >= Duration::from_secs(30), + "a short production idle deadline would kill slow but healthy pulls" + ); + assert!( + UPSTREAM_BODY_IDLE_TIMEOUT < PRODUCTION_BODY_IDLE_TIMEOUT, + "the test override must be the shorter of the two" + ); + } + + /// A 200 whose body never delivers a byte. + struct StallingUpstream; + + impl Upstream for StallingUpstream { + async fn fetch( + &self, + _method: Method, + _path: String, + _range: Option, + ) -> Result { + Ok(UpstreamResponse { + status: StatusCode::OK, + headers: HeaderMap::new(), + body: StallingBody.boxed(), + }) + } + } + + #[tokio::test] + async fn a_stalled_upstream_body_fails_through_the_proxy() { + // The wiring, not just the wrapper: reverting `proxy_read` to hand the + // upstream body straight to the response leaves the two IdleTimeoutBody + // unit tests green while every real pull can still hang forever. + let resp = proxy_read( + Method::GET, + "/v2/app/blobs/sha256:abc".to_string(), + None, + &StallingUpstream, + &ReBootstrap::default(), + ) + .await; + assert_eq!(resp.status(), StatusCode::OK); + + // The outer bound is what makes an unwired wrapper FAIL rather than hang. + // With the wrapper in place the body's own 50ms deadline fires long + // first, so this costs nothing on the passing path. + let err = tokio::time::timeout(BODY_TEST_BOUND, resp.into_body().collect()) + .await + .expect("a stalled upstream body must not hang the engine's pull") + .expect_err("a stalled upstream body must surface as an error"); + assert_eq!(err.kind(), std::io::ErrorKind::TimedOut); + } + + async fn collect(resp: Response) -> (StatusCode, HeaderMap, Bytes) { + let status = resp.status(); + let headers = resp.headers().clone(); + let body = resp + .into_body() + .collect() + .await + .expect("collect") + .to_bytes(); + (status, headers, body) + } + + // --- proxy-logic tests ------------------------------------------------- + + /// A body that never produces a frame and never wakes its waker - a + /// half-open connection to a host that went to sleep mid-layer. + struct StallingBody; + + impl hyper::body::Body for StallingBody { + type Data = Bytes; + type Error = std::io::Error; + + fn poll_frame( + self: std::pin::Pin<&mut Self>, + _cx: &mut std::task::Context<'_>, + ) -> std::task::Poll, std::io::Error>>> + { + std::task::Poll::Pending + } + } + + #[tokio::test(start_paused = true)] + async fn a_raise_with_no_waiter_registered_is_still_delivered() { + // The reporting arm builds a fresh `notified()` future on every select + // iteration, so it counts as a registered waiter only while parked in + // the select. A 401 raised while the loop body ran another arm - + // mid-`send_frame`, say - found no waiter, and `notify_waiters` stores + // no permit, so the wake was dropped. The flag stayed set and surfaced + // at the next session start, which is exactly the mid-session delivery + // the Notify was added for. + let flag = ReBootstrap::default(); + flag.raise(); + + tokio::time::timeout(Duration::from_secs(5), flag.raised()) + .await + .expect("a raise with no waiter must still wake the next waiter"); + } + + /// Bound for the body tests: comfortably longer than the 50ms idle deadline + /// the suite runs with, short enough that a regression reports in seconds + /// rather than hanging the run. + const BODY_TEST_BOUND: Duration = Duration::from_secs(5); + + #[tokio::test] + async fn a_body_that_stalls_mid_stream_fails_instead_of_hanging() { + // The stage timeouts cover TCP, TLS, the HTTP/1 handshake and the + // request head - every stage EXCEPT the one a sleeping laptop actually + // stalls in. Unwrapped, this collect never returns: `docker pull` waits + // on a loopback response that never arrives, `on_sync` never returns, no + // Status is ever emitted, and systemd still reports active (running). + let err = tokio::time::timeout( + BODY_TEST_BOUND, + IdleTimeoutBody::new(StallingBody).collect(), + ) + .await + .expect("a stalled body must not hang forever") + .expect_err("a stalled body must surface as an error"); + assert_eq!( + err.kind(), + std::io::ErrorKind::TimedOut, + "a stalled body must fail as a timeout: {err}" + ); + assert!( + err.to_string().contains("no body data"), + "the error must say the stall was in the body: {err}" + ); + } + + #[tokio::test] + async fn a_body_that_delivers_is_passed_through_untouched() { + // The mirror: the wrapper must not truncate or corrupt a working + // stream, or every pull breaks. Without this the test above passes for a + // wrapper that errors unconditionally. + let bytes = IdleTimeoutBody::new(full("LAYER-BYTES")) + .collect() + .await + .expect("a delivering body must not be cut off") + .to_bytes(); + assert_eq!(&bytes[..], b"LAYER-BYTES"); + } + + #[tokio::test] + async fn proxy_forwards_upstream_200_body_verbatim() { + let upstream = FakeUpstream { + status: StatusCode::OK, + headers: vec![ + ("content-type", "application/octet-stream"), + ("docker-content-digest", "sha256:abc"), + ], + body: "LAYER-BYTES", + }; + let rebootstrap = ReBootstrap::default(); + let resp = proxy_read( + Method::GET, + "/v2/app/blobs/sha256:abc".to_string(), + None, + &upstream, + &rebootstrap, + ) + .await; + let (status, headers, body) = collect(resp).await; + + assert_eq!(status, StatusCode::OK); + assert_eq!(&body[..], b"LAYER-BYTES"); + assert_eq!( + headers.get("content-type").unwrap(), + "application/octet-stream" + ); + assert_eq!(headers.get("docker-content-digest").unwrap(), "sha256:abc"); + assert!(!rebootstrap.is_raised()); + } + + #[tokio::test] + async fn proxy_401_returns_clean_error_no_www_authenticate_and_raises_rebootstrap() { + let upstream = FakeUpstream { + status: StatusCode::UNAUTHORIZED, + headers: vec![("www-authenticate", "Bearer realm=\"https://evil/token\"")], + body: "denied", + }; + let rebootstrap = ReBootstrap::default(); + let resp = proxy_read( + Method::GET, + "/v2/app/blobs/sha256:abc".to_string(), + None, + &upstream, + &rebootstrap, + ) + .await; + let (status, headers, body) = collect(resp).await; + + // Engine never sees a 401 — it gets a clean gateway error. + assert_ne!(status, StatusCode::UNAUTHORIZED); + assert_eq!(status, StatusCode::BAD_GATEWAY); + // The upstream WWW-Authenticate is never proxied back. + assert!(headers.get("www-authenticate").is_none()); + // The clean body carries no upstream auth semantics. + assert_eq!(&body[..], REBOOTSTRAP_MSG.as_bytes()); + // The re-bootstrap status is raised for the agent to surface. + assert!(rebootstrap.is_raised()); + } + + #[tokio::test] + async fn proxy_transient_5xx_is_gateway_error_without_rebootstrap() { + let upstream = FakeUpstream { + status: StatusCode::SERVICE_UNAVAILABLE, + headers: vec![], + body: "later", + }; + let rebootstrap = ReBootstrap::default(); + let resp = proxy_read( + Method::GET, + "/v2/app/manifests/dev".to_string(), + None, + &upstream, + &rebootstrap, + ) + .await; + let (status, _headers, _body) = collect(resp).await; + + assert_eq!(status, StatusCode::BAD_GATEWAY); + // A transient error is retryable, not a terminal re-bootstrap signal. + assert!(!rebootstrap.is_raised()); + } + + // --- pinned-CA tests --------------------------------------------------- + + #[tokio::test] + async fn pinned_ca_accepts_matching_leaf_and_streams_200() { + let ca = make_ca(); + let addr = start_tls_upstream(ca.leaf_cert.clone(), ca.leaf_key.clone_key()).await; + + let bootstrap = Bootstrap { + bulk_endpoint: addr.to_string(), + read_token: "read-token".to_string(), + ca_cert_pem: ca.ca_pem.clone(), + ws_endpoint: String::new(), + }; + let upstream = HttpsUpstream::from_bootstrap(&bootstrap).expect("build upstream"); + + let resp = upstream + .fetch(Method::GET, "/v2/app/blobs/sha256:abc".to_string(), None) + .await + .expect("pinned CA accepts its own leaf"); + assert!(resp.status.is_success()); + let body = resp.body.collect().await.expect("collect").to_bytes(); + assert_eq!(&body[..], b"BLOBDATA"); + } + + #[tokio::test] + async fn pinned_ca_rejects_leaf_from_a_different_ca() { + // Server presents a leaf signed by CA "B"... + let server_ca = make_ca(); + let addr = + start_tls_upstream(server_ca.leaf_cert.clone(), server_ca.leaf_key.clone_key()).await; + + // ...but the client pins a DIFFERENT CA "A". + let client_ca = make_ca(); + let bootstrap = Bootstrap { + bulk_endpoint: addr.to_string(), + read_token: "read-token".to_string(), + ca_cert_pem: client_ca.ca_pem.clone(), + ws_endpoint: String::new(), + }; + let upstream = HttpsUpstream::from_bootstrap(&bootstrap).expect("build upstream"); + + let result = upstream + .fetch(Method::GET, "/v2/app/blobs/sha256:abc".to_string(), None) + .await; + assert!( + result.is_err(), + "handshake must fail: a host leaf not signed by the bootstrap CA is rejected" + ); + } + + // --- end-to-end loopback test ------------------------------------------ + + #[tokio::test] + async fn loopback_is_plain_http_and_proxies_over_tls_upstream() { + let ca = make_ca(); + let upstream_addr = start_tls_upstream(ca.leaf_cert.clone(), ca.leaf_key.clone_key()).await; + + let bootstrap = Bootstrap { + bulk_endpoint: upstream_addr.to_string(), + read_token: "read-token".to_string(), + ca_cert_pem: ca.ca_pem.clone(), + ws_endpoint: String::new(), + }; + let upstream = Arc::new(HttpsUpstream::from_bootstrap(&bootstrap).expect("build upstream")); + let rebootstrap = ReBootstrap::default(); + + let listener = bind_loopback(0).await.expect("bind loopback"); + let loopback_addr = listener.local_addr().expect("loopback addr"); + // The loopback socket must be 127.0.0.1, never 0.0.0.0. + assert!(loopback_addr.ip().is_loopback()); + + tokio::spawn(serve_on(listener, upstream, rebootstrap.clone())); + + // The engine speaks PLAIN HTTP to the loopback with zero TLS/trust + // config; the proxy forwards over the pinned-CA HTTPS upstream. + let (status, headers, body) = plain_get(loopback_addr, "/v2/app/blobs/sha256:abc").await; + assert_eq!(status, StatusCode::OK); + assert_eq!(&body[..], b"BLOBDATA"); + assert_eq!( + headers.get("docker-content-digest").unwrap(), + "sha256:deadbeef" + ); + assert!(!rebootstrap.is_raised()); + + // An upstream 401 surfaces as a clean gateway error with no + // WWW-Authenticate, and raises the re-bootstrap status. + let (status, headers, _body) = plain_get(loopback_addr, "/v2/app/blobs/unauth").await; + assert_eq!(status, StatusCode::BAD_GATEWAY); + assert!(headers.get("www-authenticate").is_none()); + assert!(rebootstrap.is_raised()); + } + + #[tokio::test] + async fn bind_loopback_binds_localhost_not_wildcard() { + let listener = bind_loopback(0).await.expect("bind loopback"); + let addr = listener.local_addr().expect("addr"); + assert!(addr.ip().is_loopback()); + assert_eq!(addr.ip(), std::net::IpAddr::V4(Ipv4Addr::LOCALHOST)); + } + + // Guard for ServerName import usage in non-test builds is unnecessary; keep + // the import honest by asserting a loopback IP parses as a server name. + #[test] + fn loopback_ip_is_a_valid_server_name() { + assert!(ServerName::try_from("127.0.0.1").is_ok()); + } +} diff --git a/agent/src/sync.rs b/agent/src/sync.rs new file mode 100644 index 0000000..19694f6 --- /dev/null +++ b/agent/src/sync.rs @@ -0,0 +1,1343 @@ +//! Device-side sync handler for Container Dev Mode (design D9; task 6.3). +//! +//! On a [`crate::HostFrame::Sync`] the agent, IN ORDER: +//! 1. **Pulls** the new image through the device loopback proxy +//! (`127.0.0.1:`, task 6.2). The pull rides the proxy -> host bulk +//! HTTPS path; it NEVER re-pulls over the control WebSocket (D9 splits +//! bulk from control). +//! 2. **Restarts the container** via the engine AFTER the pull, so the new +//! image takes effect. The read-only rootFS is untouched throughout. +//! 3. **Records the now-running digest** in [`crate::AgentState::running_digest`] +//! so task 6.1's reconnect `Hello` reports it. This lands BEFORE the pointer +//! write so a full or unwritable `/var` cannot stop the agent reporting what +//! it actually runs. +//! 4. **Rewrites the active-image pointer** — a small JSON file recording the +//! now-active `{image, tag, digest}` — on the **writable partition** +//! (`/var/lib/avocado/container-dev/active-image.json`), only once the +//! restart succeeded, so a failed restart never leaves the pointer ahead of +//! what the device actually runs. The path is derived from +//! [`WRITABLE_ROOT`], never a read-only rootFS path (`/usr`, `/etc`, the +//! image rootfs). This is the one step allowed to fail without failing the +//! sync, so no step may assume the pointer file exists - it can be absent +//! while the digest at step 3 is already set. +//! +//! The engine interaction is behind the [`Engine`] trait (mirroring task 6.2's +//! [`crate::proxy::Upstream`] pattern) so the ordering / pointer / digest logic +//! is unit-tested with a fake. The production [`CommandEngine`] shells out to +//! docker/podman via `tokio::process::Command` and is the only untested part — +//! no new TLS/HTTP deps are pulled in for the shell-out, keeping `aws-lc-rs` +//! out of the tree (A9). + +use std::future::Future; +use std::path::{Path, PathBuf}; +use std::time::Duration; + +use anyhow::{Context, Result, ensure}; +use serde::{Deserialize, Serialize}; +use tokio::process::Command; +use tracing::{info, warn}; + +use crate::AgentState; + +/// The writable-partition root for on-device mutable state. Consistent with +/// `main.rs`'s bootstrap root (`DEFAULT_BOOTSTRAP_PATH` lives under this same +/// directory). NEVER a read-only rootFS path. +pub(crate) const WRITABLE_ROOT: &str = "/var/lib/avocado/container-dev"; +/// The active-image pointer file name, written under [`WRITABLE_ROOT`]. +const ACTIVE_IMAGE_FILE: &str = "active-image.json"; +/// Default container engine binary when unset (docker; podman via the env var). +const DEFAULT_ENGINE: &str = "docker"; +/// Default container name to restart when unset. +const DEFAULT_CONTAINER: &str = "avocado-dev"; +/// Env var selecting the container engine binary (`docker` or `podman`). +const ENGINE_ENV: &str = "AVOCADO_CONTAINER_DEV_ENGINE"; +/// Env var selecting the container name to restart on sync. +const CONTAINER_ENV: &str = "AVOCADO_CONTAINER_DEV_CONTAINER"; +/// Env var overriding the writable state root (used by tests and staging). +const STATE_ROOT_ENV: &str = "AVOCADO_CONTAINER_DEV_STATE_ROOT"; +/// Env var naming the systemd unit that OWNS the container, restarted instead of +/// the container itself when set. A per-device override; the normal source is +/// the `Sync` frame's `service`, which the host fills from the project's own +/// `container_dev.images[].service`. +/// +/// ` restart ` re-executes the container's existing config, +/// which pins the image by ID at create time - so a freshly pulled image for the +/// same tag is ignored and the sync silently no-ops while reporting success. The +/// unit that launched the container re-runs ` run` on restart, which +/// re-resolves the tag and therefore adopts the new image. +/// +/// Nothing set this on a shipped device: the unit exports only `RUST_LOG` and +/// `AVOCADO_CONTAINER_DEV_ENGINE`, and the agent had no other way to learn a +/// unit name, so every installation took the container branch and every sync +/// no-opped. The `service:` field the host already parses is the answer, and it +/// now rides the frame rather than depending on an operator setting this by +/// hand. +const SERVICE_ENV: &str = "AVOCADO_CONTAINER_DEV_SERVICE"; + +// --------------------------------------------------------------------------- +// Active-image pointer. +// --------------------------------------------------------------------------- + +/// The active-image pointer: which `{image, tag, digest}` the device now runs. +/// Serialized to `active-image.json` on the writable partition. +#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)] +pub(crate) struct ActiveImage { + pub image: String, + pub tag: String, + pub digest: String, +} + +/// The active-image pointer path, derived from `writable_root`. The join makes +/// it structurally impossible to land on a rootFS constant: the caller supplies +/// the root, and production always supplies [`WRITABLE_ROOT`]. +fn active_image_path(writable_root: &Path) -> PathBuf { + writable_root.join(ACTIVE_IMAGE_FILE) +} + +/// Write the active-image pointer under `writable_root`, creating the directory +/// if needed. Returns the path written. +fn write_active_image(writable_root: &Path, active: &ActiveImage) -> Result { + let path = active_image_path(writable_root); + if let Some(parent) = path.parent() { + std::fs::create_dir_all(parent) + .with_context(|| format!("creating writable state dir {}", parent.display()))?; + } + let json = serde_json::to_string_pretty(active).context("serializing active-image pointer")?; + std::fs::write(&path, json) + .with_context(|| format!("writing active-image pointer {}", path.display()))?; + Ok(path) +} + +/// The digest-pinned loopback registry reference the engine pulls. The device +/// engine auto-trusts `127.0.0.1`, so this rides the plain-HTTP loopback proxy +/// (task 6.2) which forwards to the host bulk HTTPS listener. +fn loopback_reference(port: u16, image: &str, digest: &str) -> String { + format!("127.0.0.1:{port}/{image}@{digest}") +} + +// --------------------------------------------------------------------------- +// Engine abstraction (injectable for tests). +// --------------------------------------------------------------------------- + +/// The container engine: pull an image reference, restart a container. Async so +/// the production impl can shell out without blocking the runtime; abstracted so +/// the sync ordering / pointer / digest logic is unit-testable with a fake. +pub(crate) trait Engine: Send + Sync + 'static { + fn pull(&self, reference: &str) -> impl Future> + Send; + /// Point `target` (the ref the service names) at `source` (what we just pulled). + fn tag(&self, source: &str, target: &str) -> impl Future> + Send; + /// Restart whatever owns the image so the pulled build takes effect. + /// + /// `service` is the systemd unit the HOST says owns this image (the + /// `service:` field on the matching `container_dev.images` entry, carried on + /// the `Sync` frame). It is the only input that makes the default + /// installation work: see [`SERVICE_ENV`]. + fn restart( + &self, + container: &str, + service: Option<&str>, + ) -> impl Future> + Send; + /// The image ID `container` is currently running, or `None` when the + /// container does not exist. + /// + /// Exists so the sync can PROVE the restart adopted the new image rather + /// than assume it. ` restart` re-executes the container object, + /// which is bound to the image ID it was created with and does not + /// re-resolve the tag - so on the branch the shipped unit actually takes, + /// every step reported success while the container went on running the old + /// image. + fn running_image_id( + &self, + container: &str, + ) -> impl Future>> + Send; + /// The image ID `reference` resolves to locally, or `None` when absent. + fn resolve_image_id( + &self, + reference: &str, + ) -> impl Future>> + Send; +} + +/// The real engine: shells out to the docker/podman CLI. This is the only +/// untested part of task 6.3 (it execs a subprocess); the pure [`on_sync`] core +/// above it is fully covered. +pub(crate) struct CommandEngine { + binary: String, +} + +impl CommandEngine { + /// Resolve the engine binary from `$AVOCADO_CONTAINER_DEV_ENGINE`, defaulting + /// to `docker`. + pub(crate) fn from_env() -> Self { + let binary = std::env::var(ENGINE_ENV) + .ok() + .map(|v| v.trim().to_string()) + .filter(|v| !v.is_empty()) + .unwrap_or_else(|| DEFAULT_ENGINE.to_string()); + Self { binary } + } +} + +/// The engine binary the agent would use, resolved from the environment. +/// +/// Split out so `main` can inspect it once at startup without constructing an +/// engine. [`CommandEngine::from_env`] runs per `Sync` frame, which is the wrong +/// place to warn from: nothing is logged at `systemctl start`, and then the same +/// warning repeats before every pull failure. +pub(crate) fn engine_binary_from_env() -> String { + std::env::var(ENGINE_ENV) + .ok() + .map(|v| v.trim().to_string()) + .filter(|v| !v.is_empty()) + .unwrap_or_else(|| DEFAULT_ENGINE.to_string()) +} + +/// Whether `binary` names an engine the shipped systemd unit cannot run. +/// +/// Rootful podman needs CAP_SYS_ADMIN, namespace creation, cgroup writes and a +/// writable `/var/lib/containers`; the unit's sandbox denies all of them. Pure so +/// the classification is testable without an environment or a process. +pub(crate) fn is_unsupported_engine(binary: &str) -> bool { + binary.contains("podman") +} + +/// Warn once when the configured engine cannot work under the shipped unit. +/// +/// A warning rather than a refusal: the restriction belongs to the unit, not the +/// binary, so an agent run directly for debugging is unaffected. Without it the +/// failure is invisible in the worst way - podman fails at store init, `on_sync`'s +/// error is only logged by its caller, and every Sync silently no-ops while the +/// agent stays connected reporting a stale digest. +pub(crate) fn warn_if_unsupported_engine(binary: &str) { + if is_unsupported_engine(binary) { + warn!( + engine = %binary, + "podman is not supported under the container-agent-dev systemd unit \ + (its sandbox denies the privileges rootful podman needs); expect pulls \ + to fail unless this agent is running outside that unit" + ); + } +} + +/// Whether `binary` resolves to something executable on `PATH`. +/// +/// Split from the check below so the lookup rule is testable without an +/// environment: an absolute or relative path is probed directly, a bare name is +/// searched along `PATH` the way the kernel's exec would. +pub(crate) fn engine_on_path(binary: &str, path_var: Option<&str>) -> bool { + fn executable(p: &Path) -> bool { + #[cfg(unix)] + { + use std::os::unix::fs::PermissionsExt; + std::fs::metadata(p) + .map(|m| m.is_file() && m.permissions().mode() & 0o111 != 0) + .unwrap_or(false) + } + #[cfg(not(unix))] + { + p.is_file() + } + } + + if binary.contains('/') { + return executable(Path::new(binary)); + } + let Some(path_var) = path_var else { + return false; + }; + path_var + .split(':') + .filter(|dir| !dir.is_empty()) + .any(|dir| executable(&Path::new(dir).join(binary))) +} + +/// Fail startup when the configured engine binary is not present. +/// +/// Nothing verified this before, and the consequence was the shape the unit's +/// own `Environment=` comment calls the worst available: merging the dev +/// extension onto a device without the separate `docker` extension left the unit +/// `active (running)` and silent, because `warn_if_unsupported_engine` only +/// string-matches podman. The first `Sync` then failed at +/// `Command::new("docker").status()` with ENOENT, which the swallowed-error path +/// reduced to a `warn!` - so `running_digest` kept its old value, the WS stayed +/// up, the host saw a connected device, and every sync no-oped forever. +/// +/// Refusing here turns that into a unit that fails at start and says why, which +/// is visible in `systemctl status` without knowing to look for it. +pub(crate) fn ensure_engine_available(binary: &str) -> Result<()> { + ensure!( + engine_on_path(binary, std::env::var("PATH").ok().as_deref()), + "container engine `{binary}` was not found on PATH. The agent execs it directly on \ + every sync, so there is nothing it can do without it - install the `docker` extension \ + in this runtime, or set {ENGINE_ENV} to an engine that is present." + ); + Ok(()) +} + +/// The systemd unit owning the container, from `$AVOCADO_CONTAINER_DEV_SERVICE`. +/// +/// `None` keeps the container-restart path, so a device that does not launch its +/// container from a unit behaves exactly as before. +pub(crate) fn service_from_env() -> Option { + std::env::var(SERVICE_ENV) + .ok() + .map(|v| v.trim().to_string()) + .filter(|v| !v.is_empty()) +} + +/// Which systemd unit to restart, given the frame's `service` and the env +/// override. `None` keeps the container-restart path. +/// +/// Split out so the precedence is testable without mutating process env: the +/// frame wins because the host reads it from the project's own +/// `container_dev.images[].service`, and a blank or whitespace-only value on the +/// wire is treated as absent rather than as a unit named "". +fn restart_unit(frame_service: Option<&str>, env_service: Option) -> Option { + frame_service + .map(str::trim) + .filter(|v| !v.is_empty()) + .map(str::to_string) + .or(env_service) +} + +impl Engine for CommandEngine { + async fn pull(&self, reference: &str) -> Result<()> { + run_engine(&self.binary, &["pull", reference]).await + } + + async fn tag(&self, source: &str, target: &str) -> Result<()> { + run_engine(&self.binary, &["tag", source, target]).await + } + + /// Restart the owning systemd unit when one is known, else the container. + /// + /// The frame's service wins over the env var: the host reads it from the + /// project's own `container_dev.images[].service`, which is where the answer + /// actually lives, while the env var is a per-device override for a unit the + /// host does not know about. See [`SERVICE_ENV`] for why restarting the + /// container is not enough. + async fn restart(&self, container: &str, service: Option<&str>) -> Result<()> { + match restart_unit(service, service_from_env()) { + Some(service) => { + info!(service = %service, "restarting owning systemd unit to adopt the pulled image"); + run_engine("systemctl", &["restart", &service]).await + } + None => run_engine(&self.binary, &["restart", container]).await, + } + } + + async fn running_image_id(&self, container: &str) -> Result> { + engine_output( + &self.binary, + &["inspect", container, "--format", "{{.Image}}"], + ) + .await + } + + async fn resolve_image_id(&self, reference: &str) -> Result> { + engine_output( + &self.binary, + &["image", "inspect", reference, "--format", "{{.Id}}"], + ) + .await + } +} + +/// Run ` ` and capture stdout, mapping a non-zero exit to +/// `None` rather than an error. +/// +/// A non-zero exit from `inspect` means "no such object", which is a legitimate +/// answer here (the container has never been created) and not a failure to +/// report. +async fn engine_output(binary: &str, args: &[&str]) -> Result> { + let out = Command::new(binary) + .args(args) + .kill_on_drop(true) + .output() + .await + .with_context(|| format!("spawning `{binary} {}`", args.join(" ")))?; + if !out.status.success() { + return Ok(None); + } + let text = String::from_utf8_lossy(&out.stdout).trim().to_string(); + Ok(if text.is_empty() { None } else { Some(text) }) +} + +/// Run ` ` to completion, mapping a non-zero exit to an error. +/// +/// `kill_on_drop` because the sync runs on a task the session loop aborts when +/// the link drops (`main.rs`'s `AbortOnDrop`). An abort lands on whichever await +/// point is current, and without this the in-flight `docker pull` would keep +/// running with nothing owning it - burning bandwidth on a superseded digest +/// while the next session starts its own pull. +async fn run_engine(binary: &str, args: &[&str]) -> Result<()> { + let status = Command::new(binary) + .args(args) + .kill_on_drop(true) + .status() + .await + .with_context(|| format!("spawning `{binary} {}`", args.join(" ")))?; + ensure!( + status.success(), + "`{binary} {}` exited with {status}", + args.join(" ") + ); + Ok(()) +} + +// --------------------------------------------------------------------------- +// Sync config + core. +// --------------------------------------------------------------------------- + +/// The device-side sync configuration: the loopback port the engine pulls from, +/// the container to restart, and the writable state root the pointer lands in. +pub(crate) struct SyncConfig { + pub loopback_port: u16, + pub container: String, + pub writable_root: PathBuf, +} + +impl SyncConfig { + /// Resolve from env: loopback port shares `main.rs`'s resolver; the writable + /// root defaults to [`WRITABLE_ROOT`] (`$AVOCADO_CONTAINER_DEV_STATE_ROOT` + /// overrides for tests/staging); the container defaults to `avocado-dev` + /// (`$AVOCADO_CONTAINER_DEV_CONTAINER` overrides). + pub(crate) fn from_env() -> Self { + let writable_root = std::env::var_os(STATE_ROOT_ENV) + .map(PathBuf::from) + .unwrap_or_else(|| PathBuf::from(WRITABLE_ROOT)); + let container = std::env::var(CONTAINER_ENV) + .ok() + .map(|v| v.trim().to_string()) + .filter(|v| !v.is_empty()) + .unwrap_or_else(|| DEFAULT_CONTAINER.to_string()); + Self { + loopback_port: crate::loopback_port(), + container, + writable_root, + } + } +} + +/// How long a restarted unit gets to bring its container up before the sync +/// treats the absence as a failure. +/// +/// `systemctl restart` returns once the unit is active, which for a `Type=simple` +/// unit is before its ` run` has necessarily created the container. A +/// single immediate probe would race that and fail a sync that was about to +/// work, so the check waits rather than asking once. +const CONTAINER_APPEAR_TIMEOUT: Duration = Duration::from_secs(30); + +/// Gap between container-existence probes inside [`CONTAINER_APPEAR_TIMEOUT`]. +const CONTAINER_POLL_INTERVAL: Duration = Duration::from_millis(500); + +/// The image id running in `container`, waiting up to +/// [`CONTAINER_APPEAR_TIMEOUT`] for the container to exist at all. +/// +/// Returns `None` only when it never appeared, which the caller treats as a +/// failed sync. +async fn running_image_id_once_up( + engine: &E, + container: &str, +) -> Result> { + let deadline = tokio::time::Instant::now() + CONTAINER_APPEAR_TIMEOUT; + loop { + let running = engine + .running_image_id(container) + .await + .with_context(|| format!("reading the image id running in {container}"))?; + if running.is_some() || tokio::time::Instant::now() >= deadline { + return Ok(running); + } + tokio::time::sleep(CONTAINER_POLL_INTERVAL).await; + } +} + +/// Handle a `Sync`: pull -> restart -> record digest -> rewrite pointer. +/// +/// The order is load-bearing at both ends. A pull failure aborts before the +/// restart, the digest update and the pointer write, so a failed sync never +/// claims a new image is running and never mutates state. The digest is recorded +/// before the pointer write, so an unwritable `/var` costs the pointer file but +/// not a correct `Hello` - which makes the pointer write the only step that may +/// fail while the sync still returns `Ok`. All writes target the writable +/// partition; the read-only rootFS is never touched. +pub(crate) async fn on_sync( + engine: &E, + cfg: &SyncConfig, + image: &str, + tag: &str, + digest: &str, + service: Option<&str>, + state: &AgentState, +) -> Result<()> { + // 1. Pull through the device loopback proxy (never over the control WS). + let reference = loopback_reference(cfg.loopback_port, image, digest); + engine + .pull(&reference) + .await + .with_context(|| format!("pulling {reference} through loopback proxy"))?; + + // 2. Point the ref the service names at what we just pulled. + // + // The pull is digest-pinned, so it lands as an untagged image. The unit's + // ExecStart names `:`, which on this device either does not + // exist (the host built it, so the target never had that tag) or still + // resolves to the PREVIOUS build. Either way the restart below would not + // run what was just fetched, while every step still reported success. + let service_ref = format!("{image}:{tag}"); + engine + .tag(&reference, &service_ref) + .await + .with_context(|| format!("tagging {reference} as {service_ref}"))?; + + // 3. Restart the container AFTER the pull and tag so the new image takes effect. + engine + .restart(&cfg.container, service) + .await + .with_context(|| format!("restarting container {}", cfg.container))?; + + // 3b. PROVE the restart adopted the new image, rather than assume it. + // + // ` restart ` re-executes the existing container + // object, which is bound to the image ID it was created with and does + // not re-resolve the tag. That is the branch the shipped unit takes, + // because nothing sets AVOCADO_CONTAINER_DEV_SERVICE - so the pull + // succeeded, the tag succeeded, the restart succeeded, and the container + // went on running the previous image while step 4 recorded the new + // digest and the host's reconcile saw its desired state satisfied. The + // developer saw unchanged behaviour with every layer reporting success. + // + // Comparing IDs is engine-generic and catches the failure on whichever + // branch produced it. BOTH ids must be known: an unknown one is a + // failure, not a pass. Skipping the check when the container is absent + // read as tolerance for a unit that creates its container lazily, but + // there is no such case left - on the container branch `restart` has + // already failed with "no such container" before reaching here, and on + // the service branch the unit was just restarted, so a container that is + // still missing once it has had time to appear means the restart did not + // produce one. Passing there recorded the new digest for a device + // running nothing at all, which is the same false success this probe + // exists to end, reached by the other door. + let wanted = engine + .resolve_image_id(&service_ref) + .await + .with_context(|| format!("resolving the image id of {service_ref}"))? + .ok_or_else(|| { + anyhow::anyhow!( + "{service_ref} resolves to no image even though the tag step just succeeded; \ + there is nothing for the container to run" + ) + })?; + let running = running_image_id_once_up(engine, &cfg.container) + .await? + .ok_or_else(|| { + anyhow::anyhow!( + "restart succeeded but container {} does not exist after {}s, so nothing is \ + running the pulled image. Check that the restarted unit actually starts it", + cfg.container, + CONTAINER_APPEAR_TIMEOUT.as_secs(), + ) + })?; + ensure!( + wanted == running, + "restart did not adopt the pulled image: container {} still runs image {running}, \ + expected {wanted}. `{} restart {}` re-executes the existing container object, which \ + stays bound to its create-time image id; name the systemd unit that owns the container \ + in this image's `service:` field under `container_dev.images` so the restart re-runs \ + the unit and re-resolves the tag", + cfg.container, + engine_binary_from_env(), + cfg.container, + ); + + // 4. Record the now-running digest. This happens IMMEDIATELY after the + // restart, before the pointer write, because at this instant the + // container really is running `digest` - that is an observed fact, not a + // plan. Deferring it behind the pointer write would let a pointer failure + // leave `Hello` reporting an image the device is demonstrably not running, + // which is the mirror image of the skew that moving the pointer after the + // restart fixed. + *state + .running_digest + .lock() + .expect("running_digest mutex poisoned") = digest.to_string(); + + // 4. Rewrite the active-image pointer on the WRITABLE partition, only once + // the restart has actually succeeded. The pointer records what the device + // IS running, so writing it ahead of the restart would leave it claiming + // the new image while the container still ran the old one on any restart + // failure. Nothing consumes the pointer during startup, so it has no + // reason to precede the restart. The path is derived from + // cfg.writable_root, never a read-only rootFS path. + // + // A failure here is reported, not propagated. The pull and the restart + // both succeeded, so the sync did happen; failing the whole operation + // over the bookkeeping record would tell the host the sync did not take + // effect when it did, and the host's only recovery - retry the same + // digest - cannot fix a full or unwritable /var anyway. + let active = ActiveImage { + image: image.to_string(), + tag: tag.to_string(), + digest: digest.to_string(), + }; + // One terminal line per outcome, and only the Ok arm says "sync complete". + // A single unconditional success line here claimed "pointer rewritten" on the + // failure branch too, so grepping for the obvious success string reported a + // current pointer when no file had been written - the warning above it was + // right, but the success line is the one that reads as authoritative. + match write_active_image(&cfg.writable_root, &active) { + Ok(path) => info!( + %image, %tag, %digest, pointer = %path.display(), + "sync complete: pulled, container restarted, active-image pointer rewritten" + ), + Err(e) => warn!( + error = %e, + %image, %tag, %digest, + "sync incomplete: pulled and container restarted, but the active-image \ + pointer could not be written; the running digest is reported from memory \ + only and will be lost on restart" + ), + } + Ok(()) +} + +#[cfg(test)] +mod tests { + use super::*; + + use std::sync::{Arc, Mutex}; + + /// An in-process engine that records its ordered calls; optionally fails the + /// pull to exercise the abort path. Never execs a real engine. + struct FakeEngine { + calls: Arc>>, + fail_pull: bool, + fail_tag: bool, + fail_restart: bool, + /// What ` image inspect ` answers. + wanted_id: Option, + /// What ` inspect ` answers AFTER the restart. The + /// default matches `wanted_id`, i.e. a restart that adopted the image. + running_id: Option, + /// Probes that answer "no such container" before `running_id` is + /// returned - a unit whose container takes a moment to come up. + absent_probes: Mutex, + } + + impl FakeEngine { + fn new(calls: Arc>>) -> Self { + Self { + calls, + fail_pull: false, + fail_tag: false, + fail_restart: false, + wanted_id: Some("sha256:newid".to_string()), + running_id: Some("sha256:newid".to_string()), + absent_probes: Mutex::new(0), + } + } + + fn failing_pull(calls: Arc>>) -> Self { + Self { + fail_pull: true, + ..Self::new(calls) + } + } + + /// `docker tag` rejects some refs outright - an uppercase letter in a + /// host-supplied `image`, for instance - so the step CAN fail, and the + /// fake could not express it. Without this the `?` on `engine.tag(...)` + /// had zero coverage: replacing it with `let _ = ...` left all eight + /// sync tests green while production fell through to a restart that + /// re-ran the previous image and then recorded the new digest. + fn failing_tag(calls: Arc>>) -> Self { + Self { + fail_tag: true, + ..Self::new(calls) + } + } + + fn failing_restart(calls: Arc>>) -> Self { + Self { + fail_restart: true, + ..Self::new(calls) + } + } + + /// A restart that completed but left the container on its old image - + /// what ` restart` does on the branch the shipped unit takes. + fn restart_that_does_not_adopt(calls: Arc>>) -> Self { + Self { + running_id: Some("sha256:oldid".to_string()), + ..Self::new(calls) + } + } + + /// A restart that reported success but never produced a container. + fn container_never_appears(calls: Arc>>) -> Self { + Self { + running_id: None, + ..Self::new(calls) + } + } + + /// A unit whose container takes `probes` polls to come up - what a + /// `Type=simple` unit does, since `systemctl restart` returns before its + /// ` run` has necessarily created anything. + fn container_appears_after(calls: Arc>>, probes: usize) -> Self { + Self { + absent_probes: Mutex::new(probes), + ..Self::new(calls) + } + } + + /// A tag step that reported success but left the ref resolving to + /// nothing. + fn tag_resolves_to_nothing(calls: Arc>>) -> Self { + Self { + wanted_id: None, + ..Self::new(calls) + } + } + } + + impl Engine for FakeEngine { + async fn pull(&self, reference: &str) -> Result<()> { + self.calls.lock().unwrap().push(format!("pull:{reference}")); + if self.fail_pull { + anyhow::bail!("simulated pull failure"); + } + Ok(()) + } + + async fn tag(&self, source: &str, target: &str) -> Result<()> { + self.calls + .lock() + .unwrap() + .push(format!("tag:{source}->{target}")); + if self.fail_tag { + anyhow::bail!("simulated tag failure"); + } + Ok(()) + } + + async fn restart(&self, container: &str, service: Option<&str>) -> Result<()> { + self.calls.lock().unwrap().push(match service { + Some(service) => format!("restart-unit:{service}"), + None => format!("restart:{container}"), + }); + if self.fail_restart { + anyhow::bail!("simulated restart failure"); + } + Ok(()) + } + + async fn running_image_id(&self, container: &str) -> Result> { + self.calls + .lock() + .unwrap() + .push(format!("running_image_id:{container}")); + let mut absent = self.absent_probes.lock().unwrap(); + if *absent > 0 { + *absent -= 1; + return Ok(None); + } + Ok(self.running_id.clone()) + } + + async fn resolve_image_id(&self, reference: &str) -> Result> { + self.calls + .lock() + .unwrap() + .push(format!("resolve_image_id:{reference}")); + Ok(self.wanted_id.clone()) + } + } + + fn test_cfg(writable_root: PathBuf) -> SyncConfig { + SyncConfig { + loopback_port: 15151, + container: "avocado-dev".to_string(), + writable_root, + } + } + + // Falsifier 1: a sync must restart the container AFTER the pull. This test + // asserts BOTH happened and that pull strictly precedes restart, so an + // implementation that skipped the restart (or reordered it) fails here. + #[tokio::test] + async fn sync_pulls_then_restarts_in_order() { + let calls = Arc::new(Mutex::new(Vec::new())); + let engine = FakeEngine::new(calls.clone()); + let tmp = tempfile::tempdir().unwrap(); + let cfg = test_cfg(tmp.path().to_path_buf()); + let state = crate::AgentState::new("dev-01".to_string()); + + on_sync(&engine, &cfg, "my-app", "dev", "sha256:new", None, &state) + .await + .expect("sync succeeds"); + + let recorded = calls.lock().unwrap().clone(); + assert_eq!( + recorded.len(), + 5, + "expected pull, tag, restart, then the two adoption probes, got {recorded:?}" + ); + assert!( + recorded[0].starts_with("pull:"), + "first call must be the pull: {recorded:?}" + ); + assert!( + recorded[1].starts_with("tag:"), + "the tag must land between the pull and the restart: {recorded:?}" + ); + assert!( + recorded[2].starts_with("restart:"), + "the restart must come last: {recorded:?}" + ); + assert_eq!(recorded[0], "pull:127.0.0.1:15151/my-app@sha256:new"); + assert_eq!(recorded[2], "restart:avocado-dev"); + } + + // The pull lands a digest-pinned image. Unless something points the ref the + // service actually names at it, the restart re-runs whatever that ref already + // meant - the old image, or nothing at all when the target never had the tag. + #[tokio::test] + async fn sync_tags_the_pulled_digest_before_restarting() { + let calls = Arc::new(Mutex::new(Vec::new())); + let engine = FakeEngine::new(calls.clone()); + let tmp = tempfile::tempdir().unwrap(); + let cfg = test_cfg(tmp.path().to_path_buf()); + let state = crate::AgentState::new("dev-01".to_string()); + + on_sync(&engine, &cfg, "my-app", "dev", "sha256:new", None, &state) + .await + .expect("sync succeeds"); + + let recorded = calls.lock().unwrap().clone(); + assert_eq!( + recorded, + vec![ + "pull:127.0.0.1:15151/my-app@sha256:new".to_string(), + "tag:127.0.0.1:15151/my-app@sha256:new->my-app:dev".to_string(), + "restart:avocado-dev".to_string(), + "resolve_image_id:my-app:dev".to_string(), + "running_image_id:avocado-dev".to_string(), + ], + "the pulled digest must be tagged as the service's ref BEFORE the restart: {recorded:?}" + ); + } + + #[test] + fn a_missing_engine_binary_is_detected_before_the_first_sync() { + // The unit reported `active (running)` with no engine installed, and the + // first Sync failed at Command::new("docker") with ENOENT - reduced to a + // warn! by the swallowed-error path, so the host went on seeing a + // connected device whose syncs no-oped forever. + let tmp = tempfile::tempdir().unwrap(); + let dir = tmp.path().to_str().unwrap(); + + assert!( + !engine_on_path("docker", Some(dir)), + "an empty PATH dir must not resolve the engine" + ); + + let exe = tmp.path().join("docker"); + std::fs::write(&exe, b"#!/bin/sh\nexit 0\n").unwrap(); + #[cfg(unix)] + { + use std::os::unix::fs::PermissionsExt; + std::fs::set_permissions(&exe, std::fs::Permissions::from_mode(0o755)).unwrap(); + } + assert!( + engine_on_path("docker", Some(dir)), + "an executable on PATH must resolve" + ); + } + + #[test] + fn a_present_but_non_executable_engine_does_not_count() { + // Mode 0644 is not runnable; treating presence as availability would + // move the ENOENT to the first sync, which is the whole failure. + let tmp = tempfile::tempdir().unwrap(); + let exe = tmp.path().join("docker"); + std::fs::write(&exe, b"not executable").unwrap(); + #[cfg(unix)] + { + use std::os::unix::fs::PermissionsExt; + std::fs::set_permissions(&exe, std::fs::Permissions::from_mode(0o644)).unwrap(); + assert!(!engine_on_path( + "docker", + Some(tmp.path().to_str().unwrap()) + )); + } + } + + #[test] + fn an_absolute_engine_path_bypasses_path_lookup() { + let tmp = tempfile::tempdir().unwrap(); + let exe = tmp.path().join("myengine"); + std::fs::write(&exe, b"#!/bin/sh\nexit 0\n").unwrap(); + #[cfg(unix)] + { + use std::os::unix::fs::PermissionsExt; + std::fs::set_permissions(&exe, std::fs::Permissions::from_mode(0o755)).unwrap(); + assert!(engine_on_path(exe.to_str().unwrap(), None)); + } + assert!(!engine_on_path("/nonexistent/engine", None)); + } + + #[tokio::test] + async fn a_failed_tag_aborts_before_the_restart() { + // Replacing the `?` on `engine.tag(...)` with `let _ = ...` used to leave + // every sync test green, because the fake could not fail the step. In + // production that fall-through restarts the PREVIOUS image, then records + // the new digest and writes the pointer claiming it - the exact + // false-success the tag step exists to prevent. + let calls = Arc::new(Mutex::new(Vec::new())); + let engine = FakeEngine::failing_tag(calls.clone()); + let tmp = tempfile::tempdir().unwrap(); + let cfg = test_cfg(tmp.path().to_path_buf()); + let state = crate::AgentState::new("dev-01".to_string()); + + let err = on_sync(&engine, &cfg, "my-app", "dev", "sha256:new", None, &state) + .await + .expect_err("a failed tag must fail the sync"); + assert!( + format!("{err:#}").contains("tagging"), + "the error must name the tag step: {err:#}" + ); + + let recorded = calls.lock().unwrap().clone(); + assert!( + !recorded.iter().any(|c| c.starts_with("restart:")), + "a failed tag must abort BEFORE the restart: {recorded:?}" + ); + assert_eq!( + *state.running_digest.lock().unwrap(), + "", + "a failed tag must not record the new digest" + ); + assert!( + !active_image_path(&cfg.writable_root).exists(), + "a failed tag must not write the active-image pointer" + ); + } + + #[tokio::test] + async fn a_restart_that_does_not_adopt_the_image_fails_the_sync() { + // ` restart ` re-executes the existing container + // object, bound to its create-time image id, and does not re-resolve the + // tag. Nothing in the shipped unit sets AVOCADO_CONTAINER_DEV_SERVICE, so + // that is the branch every device takes - and every step reported success + // while the container went on running the old image. + let calls = Arc::new(Mutex::new(Vec::new())); + let engine = FakeEngine::restart_that_does_not_adopt(calls.clone()); + let tmp = tempfile::tempdir().unwrap(); + let cfg = test_cfg(tmp.path().to_path_buf()); + let state = crate::AgentState::new("dev-01".to_string()); + + let err = on_sync(&engine, &cfg, "my-app", "dev", "sha256:new", None, &state) + .await + .expect_err("a restart that kept the old image must fail the sync"); + assert!( + format!("{err:#}").contains("did not adopt"), + "the error must say the restart did not adopt the image: {err:#}" + ); + assert!( + format!("{err:#}").contains("`service:` field"), + "the error must name the remedy - the config field, not the env \ + override, since the field is what the host reads: {err:#}" + ); + assert_eq!( + *state.running_digest.lock().unwrap(), + "", + "a container still on the old image must not be reported as synced" + ); + assert!( + !active_image_path(&cfg.writable_root).exists(), + "the pointer must not claim a digest the container is not running" + ); + } + + #[tokio::test(start_paused = true)] + async fn a_container_that_never_appears_fails_the_sync() { + // The probe used to skip the check whenever either id was unknown, so a + // device with no container at all passed vacuously and recorded the new + // digest - reporting "running sha256:new" while running nothing. That is + // the same false success the probe exists to end, reached through the + // other branch, and it covered exactly the population the mismatch arm + // does not. + let calls = Arc::new(Mutex::new(Vec::new())); + let engine = FakeEngine::container_never_appears(calls); + let tmp = tempfile::tempdir().unwrap(); + let cfg = test_cfg(tmp.path().to_path_buf()); + let state = crate::AgentState::new("dev-01".to_string()); + + let err = on_sync(&engine, &cfg, "my-app", "dev", "sha256:new", None, &state) + .await + .expect_err("a container that never appears must fail the sync"); + assert!( + format!("{err:#}").contains("does not exist"), + "the error must say the container is absent: {err:#}" + ); + assert_eq!( + *state.running_digest.lock().unwrap(), + "", + "a device running no container must not report the new digest" + ); + assert!( + !active_image_path(&cfg.writable_root).exists(), + "the pointer must not claim an image nothing is running" + ); + } + + #[tokio::test(start_paused = true)] + async fn a_container_that_takes_a_moment_to_come_up_still_syncs() { + // `systemctl restart` returns when the unit is active, which for a + // Type=simple unit is before its ` run` has necessarily created + // the container. Asking once and failing would break every service-owned + // device the frame's `service:` field is meant to fix, so absence is + // only fatal after CONTAINER_APPEAR_TIMEOUT. + let calls = Arc::new(Mutex::new(Vec::new())); + let engine = FakeEngine::container_appears_after(calls.clone(), 3); + let tmp = tempfile::tempdir().unwrap(); + let cfg = test_cfg(tmp.path().to_path_buf()); + let state = crate::AgentState::new("dev-01".to_string()); + + on_sync(&engine, &cfg, "my-app", "dev", "sha256:new", None, &state) + .await + .expect("a container that comes up within the window must sync"); + + assert_eq!(*state.running_digest.lock().unwrap(), "sha256:new"); + let probes = calls + .lock() + .unwrap() + .iter() + .filter(|c| c.starts_with("running_image_id:")) + .count(); + assert_eq!(probes, 4, "three absent probes then the one that answered"); + } + + #[tokio::test(start_paused = true)] + async fn a_tag_that_resolves_to_nothing_fails_the_sync() { + // The tag step reported success, so the ref must resolve. If it does + // not, there is no image for the container to run and the old probe + // passed vacuously on the unknown id. + let calls = Arc::new(Mutex::new(Vec::new())); + let engine = FakeEngine::tag_resolves_to_nothing(calls); + let tmp = tempfile::tempdir().unwrap(); + let cfg = test_cfg(tmp.path().to_path_buf()); + let state = crate::AgentState::new("dev-01".to_string()); + + let err = on_sync(&engine, &cfg, "my-app", "dev", "sha256:new", None, &state) + .await + .expect_err("an unresolvable service ref must fail the sync"); + assert!( + format!("{err:#}").contains("resolves to no image"), + "the error must name the unresolvable ref: {err:#}" + ); + assert_eq!(*state.running_digest.lock().unwrap(), ""); + } + + #[tokio::test] + async fn the_frames_service_is_what_gets_restarted() { + // The whole point of carrying `service` on the wire: without it the + // agent restarts the container, which stays bound to its create-time + // image id, and the sync no-ops on every shipped device. + let calls = Arc::new(Mutex::new(Vec::new())); + let engine = FakeEngine::new(calls.clone()); + let tmp = tempfile::tempdir().unwrap(); + let cfg = test_cfg(tmp.path().to_path_buf()); + let state = crate::AgentState::new("dev-01".to_string()); + + on_sync( + &engine, + &cfg, + "my-app", + "dev", + "sha256:new", + Some("app.service"), + &state, + ) + .await + .expect("sync succeeds"); + + let recorded = calls.lock().unwrap().clone(); + assert!( + recorded.contains(&"restart-unit:app.service".to_string()), + "the frame's service must be what is restarted: {recorded:?}" + ); + assert!( + !recorded.iter().any(|c| c.starts_with("restart:")), + "the container must NOT be restarted when a service is named: {recorded:?}" + ); + } + + #[test] + fn the_frames_service_beats_the_env_override_and_blanks_are_ignored() { + // The host reads `service:` from the project's own config, so it is the + // better answer; the env var stays as a per-device override for a unit + // the host does not know about. A blank on the wire must fall through to + // it rather than name a unit called "". + assert_eq!( + restart_unit(Some("app.service"), Some("override.service".to_string())), + Some("app.service".to_string()) + ); + assert_eq!( + restart_unit(None, Some("override.service".to_string())), + Some("override.service".to_string()) + ); + assert_eq!( + restart_unit(Some(" "), Some("override.service".to_string())), + Some("override.service".to_string()), + "a blank service on the wire must not shadow the env override" + ); + assert_eq!(restart_unit(Some(""), None), None); + assert_eq!( + restart_unit(None, None), + None, + "no service anywhere keeps the container-restart path" + ); + } + + // Falsifier 2: the pointer lands on the writable partition, never a rootFS + // path, and round-trips {image, tag, digest}. + #[tokio::test] + async fn sync_writes_pointer_under_writable_root_and_round_trips() { + let calls = Arc::new(Mutex::new(Vec::new())); + let engine = FakeEngine::new(calls); + let tmp = tempfile::tempdir().unwrap(); + let root = tmp.path().to_path_buf(); + let cfg = test_cfg(root.clone()); + let state = crate::AgentState::new("dev-01".to_string()); + + on_sync(&engine, &cfg, "my-app", "dev", "sha256:new", None, &state) + .await + .expect("sync succeeds"); + + let pointer = active_image_path(&root); + assert!( + pointer.starts_with(&root), + "pointer must live under the writable root: {}", + pointer.display() + ); + assert!(pointer.exists(), "pointer file must be written"); + + let raw = std::fs::read_to_string(&pointer).unwrap(); + let back: ActiveImage = serde_json::from_str(&raw).unwrap(); + assert_eq!( + back, + ActiveImage { + image: "my-app".to_string(), + tag: "dev".to_string(), + digest: "sha256:new".to_string(), + } + ); + } + + // The pointer path is derived from the writable root, not a rootFS constant. + // The production root is on the writable partition and matches main.rs's + // bootstrap root. + #[test] + fn active_image_path_derives_from_writable_root_never_rootfs() { + assert!( + WRITABLE_ROOT.starts_with("/var/lib/avocado"), + "writable root must be on the writable partition" + ); + for ro in ["/usr", "/etc"] { + assert!( + !WRITABLE_ROOT.starts_with(ro), + "writable root must not be under the read-only rootFS path {ro}" + ); + } + let path = active_image_path(Path::new(WRITABLE_ROOT)); + assert!( + path.starts_with(WRITABLE_ROOT), + "pointer path must be derived from the writable root" + ); + assert_eq!(path.file_name().unwrap(), ACTIVE_IMAGE_FILE); + // Same writable partition as main.rs's bootstrap root. + assert!( + Path::new(crate::DEFAULT_BOOTSTRAP_PATH).starts_with(WRITABLE_ROOT), + "pointer root must match main.rs's bootstrap root partition" + ); + } + + // Falsifier 3 (positive): after a successful sync the running digest is the + // synced digest. + #[tokio::test] + async fn sync_updates_running_digest_on_success() { + let calls = Arc::new(Mutex::new(Vec::new())); + let engine = FakeEngine::new(calls); + let tmp = tempfile::tempdir().unwrap(); + let cfg = test_cfg(tmp.path().to_path_buf()); + let state = crate::AgentState::new("dev-01".to_string()); + + on_sync(&engine, &cfg, "my-app", "dev", "sha256:new", None, &state) + .await + .expect("sync succeeds"); + + assert_eq!(*state.running_digest.lock().unwrap(), "sha256:new"); + } + + // Edge: a pull failure aborts before the restart, does not write the + // pointer, and does not rewrite the running digest. + #[tokio::test] + async fn pull_failure_aborts_before_restart_and_keeps_digest() { + let calls = Arc::new(Mutex::new(Vec::new())); + let engine = FakeEngine::failing_pull(calls.clone()); + let tmp = tempfile::tempdir().unwrap(); + let cfg = test_cfg(tmp.path().to_path_buf()); + let state = crate::AgentState::new("dev-01".to_string()); + // A prior digest that must survive an aborted sync unchanged. + *state.running_digest.lock().unwrap() = "sha256:old".to_string(); + + let err = on_sync(&engine, &cfg, "my-app", "dev", "sha256:new", None, &state) + .await + .expect_err("pull failure must abort the sync"); + assert!( + err.to_string().contains("loopback proxy"), + "unexpected error: {err}" + ); + + let recorded = calls.lock().unwrap().clone(); + assert_eq!( + recorded.len(), + 1, + "only the pull should have been attempted: {recorded:?}" + ); + assert!(recorded[0].starts_with("pull:")); + assert!( + !active_image_path(tmp.path()).exists(), + "pointer must not be written when the pull fails" + ); + assert_eq!( + *state.running_digest.lock().unwrap(), + "sha256:old", + "running digest must be unchanged after an aborted sync" + ); + } + + // Edge: a restart failure must not leave the pointer claiming an image the + // device is not running. The pull succeeded so the bytes are on the device, + // but the container still runs the OLD image - a pointer written ahead of + // the restart would disagree with both reality and `running_digest`. + #[tokio::test] + async fn restart_failure_leaves_no_stale_pointer() { + let calls = Arc::new(Mutex::new(Vec::new())); + let engine = FakeEngine::failing_restart(calls.clone()); + let tmp = tempfile::tempdir().unwrap(); + let cfg = test_cfg(tmp.path().to_path_buf()); + let state = crate::AgentState::new("dev-01".to_string()); + *state.running_digest.lock().unwrap() = "sha256:old".to_string(); + + let err = on_sync(&engine, &cfg, "my-app", "dev", "sha256:new", None, &state) + .await + .expect_err("restart failure must fail the sync"); + assert!( + err.to_string().contains("restarting container"), + "unexpected error: {err}" + ); + + let recorded = calls.lock().unwrap().clone(); + assert_eq!( + recorded, + vec![ + "pull:127.0.0.1:15151/my-app@sha256:new".to_string(), + "tag:127.0.0.1:15151/my-app@sha256:new->my-app:dev".to_string(), + "restart:avocado-dev".to_string(), + ], + "a failed restart must abort before the adoption probes: {recorded:?}" + ); + assert!( + !active_image_path(tmp.path()).exists(), + "pointer must not claim the new image when the restart failed" + ); + assert_eq!( + *state.running_digest.lock().unwrap(), + "sha256:old", + "running digest must still report the image actually running" + ); + } + + // The mirror of `restart_failure_leaves_no_stale_pointer`: the restart + // SUCCEEDS and the pointer write then fails (ENOSPC, an unwritable /var, a + // state root pointing somewhere impossible). The container really is running + // the new image at that point, so `running_digest` must say so - reporting + // the old digest in the reconnect `Hello` would tell the host to re-sync an + // image the device already runs, and would be a claim contradicted by the + // device itself. + #[tokio::test] + async fn pointer_write_failure_after_a_successful_restart_still_reports_the_new_digest() { + let calls = Arc::new(Mutex::new(Vec::new())); + let engine = FakeEngine::new(calls.clone()); + let tmp = tempfile::tempdir().unwrap(); + + // A writable_root whose parent is a FILE: create_dir_all cannot succeed, + // so write_active_image fails while pull and restart do not. + let blocker = tmp.path().join("not-a-dir"); + std::fs::write(&blocker, b"x").unwrap(); + let cfg = test_cfg(blocker.join("state")); + + let state = crate::AgentState::new("dev-01".to_string()); + *state.running_digest.lock().unwrap() = "sha256:old".to_string(); + + on_sync(&engine, &cfg, "my-app", "dev", "sha256:new", None, &state) + .await + .expect("a pointer-write failure must not fail a sync that already took effect"); + + let recorded = calls.lock().unwrap().clone(); + assert_eq!( + recorded, + vec![ + "pull:127.0.0.1:15151/my-app@sha256:new".to_string(), + "tag:127.0.0.1:15151/my-app@sha256:new->my-app:dev".to_string(), + "restart:avocado-dev".to_string(), + "resolve_image_id:my-app:dev".to_string(), + "running_image_id:avocado-dev".to_string(), + ], + "the pull and restart must both have run: {recorded:?}" + ); + assert!( + !active_image_path(&cfg.writable_root).exists(), + "the pointer genuinely could not be written" + ); + assert_eq!( + *state.running_digest.lock().unwrap(), + "sha256:new", + "Hello must report the digest the container is actually running, \ + even though the pointer write failed" + ); + } + + // The engine classification the startup warning keys on. Pure, so the + // decision is testable without an environment or a spawned process - which + // is the whole reason it was split out of `CommandEngine::from_env`, where it + // ran once per Sync frame and so could not warn at startup at all. + #[test] + fn podman_is_classified_unsupported_and_docker_is_not() { + assert!(is_unsupported_engine("podman")); + assert!( + is_unsupported_engine("/usr/bin/podman"), + "an absolute path must classify the same as a bare name" + ); + assert!( + is_unsupported_engine("podman-remote"), + "a podman variant is still podman" + ); + + assert!(!is_unsupported_engine("docker")); + assert!(!is_unsupported_engine("/usr/bin/docker")); + assert!( + !is_unsupported_engine(DEFAULT_ENGINE), + "the default engine must never be classified unsupported" + ); + } + + #[test] + fn loopback_reference_pins_digest_on_loopback_host() { + let reference = loopback_reference(15151, "my-app", "sha256:abc"); + assert_eq!(reference, "127.0.0.1:15151/my-app@sha256:abc"); + } +} diff --git a/avocado.yaml b/avocado.yaml index dc5de79..2f544ac 100644 --- a/avocado.yaml +++ b/avocado.yaml @@ -1,35 +1,100 @@ -# Which targets this extension supports. '*' = any target. -# For a BSP, replace with the single board target and set default_target. -supported_targets: '*' +# Targets this extension is declared to support. +# +# Deliberately NOT '*'. The wildcard is a claim about every target in the +# platform, and only these two have produced an installable package. Add a +# target here when it has actually been built and the produced binary has been +# seen to exec on that architecture - not when it is expected to work. A +# wildcard here with a partial sweep behind it is how `supported_targets` +# stopped meaning anything on the in-tree version of this extension. +supported_targets: + - qemux86-64 + - imx93-frdm extensions: - # Rename to avocado-ext- (or avocado-bsp- for a BSP). - avocado-ext-template: + avocado-ext-container-agent-dev: version: 0.1.0 release: r0 - summary: One-line summary - description: One-line description + summary: Container Dev Mode device agent (development only) + description: | + Device-side agent for Container Dev Mode: holds a control WebSocket to the + host CLI, proxies image pulls from the host's registry over the + bootstrap-delivered pinned CA, and hot-reloads the target container on sync. + Its own extension so it is present ONLY in a dev runtime and absent from + prod. The per-project CA is NOT baked here; it arrives at bootstrap on the + writable partition when `avocado container dev up` runs. DO NOT USE IN + PRODUCTION. + + REQUIRES the `docker` extension in the same runtime. The agent execs the + engine CLI as its own child on every sync, so without it there is nothing + it can do. Declaring that dependency here would not enforce it: the schema + accepts an inter-extension dependency, but avocado-cli only parses and logs + it - `ext/install.rs` prints "should be installed via main install command" + and installs nothing, deliberately, to avoid circular dependencies. A + declaration would read like a guarantee and deliver none. Pulling `docker` + in as a plain package is worse: it installs the binary without + `docker.service` or the bridge kernel modules, turning a loud startup + failure into a silent connect failure on every sync. So the pairing is + stated here for whoever composes the runtime, and the agent's own startup + preflight is what makes a missing engine fail loudly. license: Apache-2.0 - url: https://github.com/avocadolinux/avocado-os + url: https://github.com/avocado-linux/ext-container-agent-dev vendor: Avocado Linux - # Runtime packages pulled from the target feed. - packages: {} - - # Optional, delete what you don't use: - # - # enable_services: - # - foo.service - # - # on_merge: - # - systemctl start --no-block foo.service - # - # sdk: - # packages: - # nativesdk-foo: '*' + # The crate is staged and cross-compiled at package time, so its sources are + # part of the payload rather than a prebuilt artifact. + package_files: + - agent + - overlay + - avocado.yaml + - cad-*.sh + + overlay: overlay + + enable_services: + - container-agent-dev.path + - container-agent-dev.service + + on_merge: + # Start the PATH unit, not the service. The service carries + # ConditionPathExists on the bootstrap, which does not exist at merge time, + # so starting it here is skipped and nothing re-triggers it: `avocado + # container dev up` delivers the bootstrap over SSH and issues no systemctl + # call. The path unit has no such condition, so it starts here and activates + # the service the moment the bootstrap lands. + - systemctl start --no-block container-agent-dev.path + + on_unmerge: + - systemctl stop container-agent-dev.path + - systemctl stop container-agent-dev.service + + packages: + avocado-container-agent-dev: + compile: avocado-container-agent-dev + install: cad-install.sh + + sdk: + packages: + nativesdk-binutils: '*' + nativesdk-cargo: '*' + nativesdk-gcc: '*' + nativesdk-glibc-dev: '*' + nativesdk-libgcc-dev: '*' + nativesdk-rust: '*' + packagegroup-rust-cross-canadian-avocado-{{ avocado.target }}: '*' # Release-scoped, not channel-scoped: this resolves to avocadolinux/sdk:2026. -# The SDK image is per-release only — appending the channel points at a tag -# that does not exist. +# The SDK image is per-release only - appending the channel points at a tag that +# does not exist. AVOCADO_DISTRO_RELEASE is set by the release workflow per +# matrix row; export it yourself for a local build. sdk: image: docker.io/avocadolinux/sdk:{{ env.AVOCADO_DISTRO_RELEASE }} + + compile: + avocado-container-agent-dev: + compile: cad-compile.sh + clean: cad-clean.sh + packages: + libstd-rs: '*' + libstd-rs-dev: '*' + packages: + avocado-sdk-toolchain: '*' diff --git a/cad-clean.sh b/cad-clean.sh new file mode 100755 index 0000000..772828a --- /dev/null +++ b/cad-clean.sh @@ -0,0 +1,10 @@ +#!/bin/bash +set -e + +echo "Cleaning avocado-container-agent-dev build artifacts" + +cd "$(dirname "$0")/agent" + +cargo clean --target-dir "$AVOCADO_BUILD_DIR" + +echo "avocado-container-agent-dev cleaned successfully" diff --git a/cad-compile.sh b/cad-compile.sh new file mode 100755 index 0000000..795aa61 --- /dev/null +++ b/cad-compile.sh @@ -0,0 +1,103 @@ +#!/bin/bash +set -euo pipefail + +RUST_TARGET="" +match_count=0 +matches="" + +# Resolve the Rust target from RUST_TARGET_PATH. Enumerate every candidate +# instead of stopping at the first: an OECORE_TARGET_ARCH that prefix-matches +# two target JSONs used to take whichever the glob happened to return first and +# discard the rest without a word. That is the worst failure shape available +# here - a binary for the wrong architecture packages, publishes and installs +# successfully, and only fails when the device tries to exec it, several layers +# from the cause. Refusing to guess keeps the failure local and attributable. +# +# devtool-debt: this block is duplicated, still unfixed, in the three sibling +# extensions that have not yet moved to their own ext-* repo - cli, connect and +# tunnels - in both their compile and install scripts. jtop and microclaw carry +# a copy too, but theirs are superseded: both migrated to ext-* repos, and +# ext-microclaw resolves the architecture with a plain case statement that does +# not have this defect at all. +# Ceiling: those three are safe by GLOB ORDERING ALONE, not by construction. +# Every target whose architecture matches the SDK host architecture has two +# matching JSONs - the device triple and the SDK's own -avocadosdk-* - and +# first-match-and-break survives only because "avocado-" sorts ahead of +# "avocadosdk-" on the hyphen. Nothing enforces that ordering. This is not a +# theoretical ceiling: it is exactly what broke qemux86-64 here the first time +# the guard below ran. +# Upgrade trigger: a vendor string that reorders those two, a third triple +# sorting between them, or any of the three gaining a target whose architecture +# prefix-matches more than one device triple. +for json_file in "$RUST_TARGET_PATH"/*.json; do + if [ -f "$json_file" ]; then + json_name=$(basename "$json_file" .json) + + # Skip the SDK's own nativesdk triple. It shares the architecture prefix + # with the device triple whenever the SDK host arch matches the target + # arch, which on x86_64 means RUST_TARGET_PATH holds both + # x86_64-avocado-linux-gnu and x86_64-avocadosdk-linux-gnu. The + # avocadosdk vendor is host-side by construction - avocado-cli tags every + # nativesdk artifact _avocadosdk - so it is never a candidate for a + # device binary and must not count as a competing match. + case "$json_name" in + "${OECORE_TARGET_ARCH}-avocadosdk-"*) continue ;; + esac + + if [[ "$json_name" == "${OECORE_TARGET_ARCH}-"* ]]; then + RUST_TARGET="$json_name" + match_count=$((match_count + 1)) + matches="$matches $json_name" + fi + fi +done + +if [ -z "$RUST_TARGET" ]; then + echo "Error: Could not find Rust target for $OECORE_TARGET_ARCH" >&2 + exit 1 +fi + +if [ "$match_count" -gt 1 ]; then + echo "Error: $OECORE_TARGET_ARCH matches $match_count Rust targets:$matches" >&2 + echo "Error: refusing to pick one - the wrong triple yields a binary that packages and installs but cannot exec on the device." >&2 + exit 1 +fi + +# Stable marker: the resolution fixture test greps for this to tell "resolution +# succeeded and a later step failed" from "resolution itself failed", which exit +# code alone cannot distinguish when the test runs without a real SDK. Load- +# bearing test surface, not a debug echo. +echo "resolved-target: $RUST_TARGET" + +echo "Compiling avocado-container-agent-dev for target: $RUST_TARGET" + +cd "$(dirname "$0")/agent" + +# Clear any rustflags that might cause conflicts with our .cargo/config.toml +unset RUSTFLAGS +unset CARGO_BUILD_RUSTFLAGS +for var in $(env | grep -o 'CARGO_TARGET_[A-Z0-9_]*_RUSTFLAGS'); do + unset "$var" +done + +# Remove any existing config that might conflict +rm -rf .cargo + +# Create config.toml with cross-compilation settings +mkdir -p .cargo +cat > .cargo/config.toml << EOF +[target.$RUST_TARGET] +rustflags = ["--sysroot=$SDKTARGETSYSROOT/usr", "-C", "link-arg=--sysroot=$SDKTARGETSYSROOT"] +EOF + +# Use a persistent cargo registry cache to avoid re-downloading crates +export CARGO_HOME="${AVOCADO_BUILD_DIR}/.cargo-cache" + +# Enforce the ring-only rustls crypto provider before building. TARGET is +# mandatory here so the guard resolves the same SDK triple the build ships; +# without it the guard would check the host triple instead. Runs under set -e, +# so a non-zero exit (aws-lc-rs present, or a guard error) fails the compile. +TARGET="$RUST_TARGET" scripts/check-provider.sh +cargo build --release --target "$RUST_TARGET" --target-dir "$AVOCADO_BUILD_DIR" + +echo "avocado-container-agent-dev compiled successfully" diff --git a/cad-install.sh b/cad-install.sh new file mode 100755 index 0000000..856cfcf --- /dev/null +++ b/cad-install.sh @@ -0,0 +1,62 @@ +#!/bin/bash +set -euo pipefail + +RUST_TARGET="" +match_count=0 +matches="" + +# Resolution must stay byte-for-byte equivalent to cad-compile.sh's. The two run +# in separate invocations against the same SDK, so a divergence here silently +# installs a different triple's binary than the one that was compiled - the +# install would look successful and ship the wrong artifact. Enumerate every +# candidate rather than stopping at the first; see cad-compile.sh for why an +# ambiguous match must refuse rather than pick. +for json_file in "$RUST_TARGET_PATH"/*.json; do + if [ -f "$json_file" ]; then + json_name=$(basename "$json_file" .json) + + # Skip the SDK's own nativesdk triple - see cad-compile.sh for why. This + # exclusion must stay identical in both scripts: if only one of them + # skipped it, that script would refuse while the other resolved, and the + # extension would compile but fail to install (or worse, install a + # host-triple binary onto the device). + case "$json_name" in + "${OECORE_TARGET_ARCH}-avocadosdk-"*) continue ;; + esac + + if [[ "$json_name" == "${OECORE_TARGET_ARCH}-"* ]]; then + RUST_TARGET="$json_name" + match_count=$((match_count + 1)) + matches="$matches $json_name" + fi + fi +done + +# Same guard cad-compile.sh has. Without it an unmatched loop leaves RUST_TARGET +# empty, BINARY_PATH collapses to "$AVOCADO_BUILD_DIR//release/...", and the -f +# check below reports a missing binary instead of the actual fault. +if [ -z "$RUST_TARGET" ]; then + echo "Error: Could not find Rust target for $OECORE_TARGET_ARCH" >&2 + exit 1 +fi + +if [ "$match_count" -gt 1 ]; then + echo "Error: $OECORE_TARGET_ARCH matches $match_count Rust targets:$matches" >&2 + echo "Error: refusing to pick one - installing a different triple's binary than was compiled would ship a wrong-architecture artifact that looks installed." >&2 + exit 1 +fi + +# Stable marker, same contract as cad-compile.sh's: the resolution fixture test +# greps for it to separate a successful resolution from a later failure. +echo "resolved-target: $RUST_TARGET" + +BINARY_PATH="$AVOCADO_BUILD_DIR/$RUST_TARGET/release/avocado-container-agent-dev" + +if [ ! -f "$BINARY_PATH" ]; then + echo "Error: Binary not found at $BINARY_PATH" + exit 1 +fi + +echo "Installing avocado-container-agent-dev into extension" +install -D -m 755 "$BINARY_PATH" "$AVOCADO_BUILD_EXT_SYSROOT/usr/bin/avocado-container-agent-dev" +echo "avocado-container-agent-dev installed successfully" diff --git a/overlay/usr/lib/systemd/system/container-agent-dev.path b/overlay/usr/lib/systemd/system/container-agent-dev.path new file mode 100644 index 0000000..3b08ea1 --- /dev/null +++ b/overlay/usr/lib/systemd/system/container-agent-dev.path @@ -0,0 +1,32 @@ +[Unit] +Description=Watch for the Container Dev Mode bootstrap and start the agent +Documentation=https://github.com/avocadolinux/avocado-os +# This unit exists because the service cannot start itself at merge time. +# +# The service carries ConditionPathExists=, and the extension's +# on_merge hook runs `systemctl start` BEFORE any bootstrap exists - so the +# condition skipped it, ExecStart never ran, and the journal recorded only +# "Condition check resulted in ... being skipped". Restart=always does not help: +# it applies to a unit that reached a running state, and this one never did. +# +# Nothing re-triggered a start afterwards either. `avocado container dev up` +# delivers bootstrap.json over SSH and issues no systemctl call at all (there is +# no `systemctl` anywhere in the CLI's container-dev paths), so the agent stayed +# down until an unrelated reboot while `up` reported success and every sync +# reached nobody. +# +# A .path unit is the systemd-native answer: it is startable at merge time +# because it has no such condition, and it activates the service the moment the +# file lands - which is exactly the event that was missing. + +[Path] +PathExists=/var/lib/avocado/container-dev/bootstrap.json +# Also fire when the file is REPLACED, not only when it first appears. Re-running +# `avocado container dev up` overwrites bootstrap.json with a freshly minted +# token; a device whose agent had exited then needs starting again, and one that +# is already running is unaffected (systemd will not re-start an active service). +PathChanged=/var/lib/avocado/container-dev/bootstrap.json +Unit=container-agent-dev.service + +[Install] +WantedBy=multi-user.target diff --git a/overlay/usr/lib/systemd/system/container-agent-dev.service b/overlay/usr/lib/systemd/system/container-agent-dev.service new file mode 100644 index 0000000..228cd9c --- /dev/null +++ b/overlay/usr/lib/systemd/system/container-agent-dev.service @@ -0,0 +1,132 @@ +[Unit] +Description=Avocado Container Dev Mode device agent (development only) +Documentation=https://github.com/avocadolinux/avocado-os +After=network.target +Wants=network.target +# The agent execs the docker CLI as its own child on every sync, so a daemon +# that has not started yet is a startup ordering problem, not a runtime one. +# Without this the first Sync after a boot raced dockerd and failed at +# `Command::new("docker").status()`; the swallowed-error path reduced that to a +# warn!, `running_digest` kept its old value, and the host went on seeing a +# connected device whose syncs no-oped. `Wants=` rather than `Requires=`: a +# device that is not running docker at all should still start the agent and fail +# loudly on the engine preflight, which names the missing binary. +After=docker.service +Wants=docker.service +# Stay inert until `avocado container dev up` delivers the bootstrap (bulk +# endpoint, read/control token, pinned CA) to the writable partition (design +# D5/5.2). No bootstrap means no session to join, so there is nothing to do. +ConditionPathExists=/var/lib/avocado/container-dev/bootstrap.json +# The default StartLimitIntervalSec=10s / StartLimitBurst=5 cannot trip at +# RestartSec=5: two starts per 10s window is well under five, so a unit that +# fails instantly - a truncated or malformed bootstrap.json makes +# `load_bootstrap` return Err and the process exit in milliseconds - sat in +# `activating (auto-restart)` forever. `systemctl is-failed` answers `activating` +# rather than `failed`, no OnFailure= hook ever runs, and the only evidence is +# one journal line every 5s. +# +# Widening the window is what makes the limit reachable: five starts inside five +# minutes at 5s apart trips in roughly 25 seconds, so a genuinely broken +# bootstrap lands the unit in `failed` where it can be seen and hooked, while a +# transient failure (the host briefly unreachable at boot) still gets its +# retries. +# +# Both keys live in [Unit], not [Service] - systemd moved them in v229 and +# silently ignores them under [Service], which `systemd-analyze verify` reports +# as "Unknown key ... ignoring". +StartLimitIntervalSec=300 +StartLimitBurst=5 + +[Service] +# StateDirectory replaces the former `ExecStartPre=/bin/mkdir -p`, which could +# not run under a read-only /var. Mode 0700 is NOT the default: systemd applies +# 0755 on every start, including to a directory that already exists, and this +# one holds bootstrap.json with the Bearer read/control token and the session +# CA. Without this line a `systemctl start` silently widens a 0700 directory +# created by `avocado container dev up`. +StateDirectory=avocado/container-dev +StateDirectoryMode=0700 +ExecStart=/usr/bin/avocado-container-agent-dev +Environment=RUST_LOG=info +# Pin the engine rather than leaving the DOCKER ONLY decision below to a comment. +# The agent only warns on a podman value and then execs it anyway (deliberately - +# the restriction is this unit's, not the binary's, so a debugging run outside the +# unit still works), which means an unset variable here let a DefaultEnvironment= +# in system.conf select podman. That failure is the worst shape available: podman +# dies at store init under this sandbox, on_sync's error is only logged, the unit +# stays active (running), and every Sync silently no-ops while the device keeps +# reporting healthy. A unit-level Environment= beats DefaultEnvironment=, so this +# closes that path. +Environment=AVOCADO_CONTAINER_DEV_ENGINE=docker + +# DOCKER ONLY. This unit deliberately does not support the podman engine. +# +# The agent runs as root: it listens on a loopback socket, dials outbound TLS, +# parses untrusted WS/HTTP frames, and execs the container engine as its own +# child. Confining that meant choosing, because the two goals are mutually +# exclusive. systemd.exec(5) is explicit that ProtectSystem is undone by a +# privileged process unless paired with CapabilityBoundingSet=~CAP_SYS_ADMIN - +# root can otherwise remount its own read-only binds read-write - and rootful +# podman needs exactly CAP_SYS_ADMIN, plus namespaces, cgroup writes and kernel +# knobs, all in-process. Keeping podman working meant the sandbox was decorative; +# a real sandbox means podman cannot run under this unit. +# +# We chose the sandbox. Docker is unaffected: its daemon is a separate unit +# holding the privilege, reached over an AF_UNIX socket, and a read-only mount +# does not block connect() to a socket. Restoring podman properly is tracked as +# a follow-up - it needs the engine out of this process, not a weaker unit. +CapabilityBoundingSet= +NoNewPrivileges=true +ProtectSystem=strict +ProtectHome=true +PrivateTmp=true +RestrictSUIDSGID=true +RestrictRealtime=true +LockPersonality=true +SystemCallArchitectures=native +RestrictNamespaces=true +# PrivateDevices, ProtectKernelTunables and ProtectControlGroups are the three +# directives systemd.exec(5) names as closing ProtectSystem=strict's /dev, /proc +# and /sys carve-outs; strict alone leaves all three writable. /dev is the one +# that matters most here: block devices are root:disk 0660 and this agent runs as +# root, so the owner bits alone allow opening /dev/mmcblk0 O_RDWR and overwriting +# the "read-only" rootfs. An empty CapabilityBoundingSet does not stop it and +# SystemCallFilter has nothing to deny - the open needs no capability and no +# privileged syscall. Without this line the untrusted WS/HTTP frame path can +# reach the exact hole the block above claims to have closed. +PrivateDevices=yes +ProtectControlGroups=true +ProtectKernelTunables=true +ProtectKernelModules=true +SystemCallFilter=@system-service +SystemCallErrorNumber=EPERM +# The agent writes active-image.json into StateDirectory, so this keeps that file +# off world-readable. It does NOT govern bootstrap.json: the host writes that one +# (see main.rs) and the agent only reads it, so the Bearer read/control token's +# mode is set by the delivery path, not here. StateDirectoryMode=0700 above is +# what keeps the token unreadable, and even that first applies at `systemctl +# start` - after ConditionPathExists has already proven bootstrap.json exists. +# Writing it 0600 belongs on the CLI side that delivers it. +UMask=0077 + +# StateDirectory is the only PERSISTENT writable path under ProtectSystem=strict, +# which is all the agent needs: bootstrap.json and active-image.json both live +# there. PrivateTmp above also grants a private /tmp and /var/tmp, discarded when +# the unit stops - so a new write path needs its own directive only if it must +# survive a restart. +# +# No AF_NETLINK. It was only ever required by netavark configuring a podman +# container's netns; the docker CLI just talks to its daemon over AF_UNIX, and +# the agent's own traffic is AF_INET/AF_INET6. +RestrictAddressFamilies=AF_UNIX AF_INET AF_INET6 + +# Raise the fd ceiling rather than only absorbing EMFILE in the proxy's accept +# loop; the default is the reason that error path exists at all. +LimitNOFILE=65536 + +Restart=always +RestartSec=5 +SyslogIdentifier=avocado-container-agent-dev + +[Install] +WantedBy=multi-user.target