From 2a8f808279cccabd0d8eb03e53db086b82c4d910 Mon Sep 17 00:00:00 2001 From: Javier Tia Date: Wed, 22 Jul 2026 08:16:17 -0600 Subject: [PATCH 01/20] extensions/container-agent-dev: Add device-side control agent There was no device-side component to connect to the host's control WebSocket in container dev mode. Without it, the device has no way to receive image sync notifications from the host or report its current running state, making the live-reload workflow incomplete. Introduce a Rust binary (`avocado-container-agent-dev`) that runs on the device. It reads a bootstrap file written by the host, establishes a TLS-secured WebSocket connection to the host's control endpoint, and sends a `Hello` frame carrying the device's identity, CPU architecture, and currently running image digest. The agent then holds the connection and processes incoming `HostFrame::Sync` notifications, with a clear seam left for the pull-and-restart logic to be filled in a later task. Reconnections use exponential backoff that resets on a successful link, so a device that briefly loses connectivity recovers quickly without hammering the host. TLS is pinned to the ring crypto provider exclusively, with the bootstrap CA as the only trust anchor, matching the constraint enforced elsewhere in the project for musl cross-compilation targets where `aws-lc-rs` is unavailable. The ring provider is also installed as the process-wide default as a defense-in-depth measure. Signed-off-by: Javier Tia --- .gitignore | 1 + Cargo.lock | 858 ++++++++++++++++++++++++++++++++++++++++++++++++++++ Cargo.toml | 52 ++++ src/main.rs | 586 +++++++++++++++++++++++++++++++++++ 4 files changed, 1497 insertions(+) create mode 100644 .gitignore create mode 100644 Cargo.lock create mode 100644 Cargo.toml create mode 100644 src/main.rs diff --git a/.gitignore b/.gitignore new file mode 100644 index 0000000..ea8c4bf --- /dev/null +++ b/.gitignore @@ -0,0 +1 @@ +/target diff --git a/Cargo.lock b/Cargo.lock new file mode 100644 index 0000000..b263e9f --- /dev/null +++ b/Cargo.lock @@ -0,0 +1,858 @@ +# This file is automatically @generated by Cargo. +# It is not intended for manual editing. +version = 4 + +[[package]] +name = "aho-corasick" +version = "1.1.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ddd31a130427c27518df266943a5308ed92d4b226cc639f5a8f1002816174301" +dependencies = [ + "memchr", +] + +[[package]] +name = "anyhow" +version = "1.0.104" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "330a5ed07fa54e4702c9d6c4174f74427fc0ef6e214bbd677ae50a5099946470" + +[[package]] +name = "avocado-container-agent-dev" +version = "0.1.0" +dependencies = [ + "anyhow", + "futures-util", + "rustls", + "rustls-pemfile", + "serde", + "serde_json", + "tokio", + "tokio-rustls", + "tokio-tungstenite", + "tracing", + "tracing-subscriber", +] + +[[package]] +name = "block-buffer" +version = "0.10.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3078c7629b62d3f0439517fa394996acacc5cbc91c5a20d8c658e77abd503a71" +dependencies = [ + "generic-array", +] + +[[package]] +name = "bytes" +version = "1.12.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "fc652a48c352aef3ea3aed32080501cf3ef6ed5da78602a020c991775b0aff04" + +[[package]] +name = "cc" +version = "1.3.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c89588d05638b5b4594a3348a2d6c20277e43a7f5c5202b05cc56888475a47b8" +dependencies = [ + "find-msvc-tools", + "shlex", +] + +[[package]] +name = "cfg-if" +version = "1.0.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9330f8b2ff13f34540b44e946ef35111825727b38d33286ef986142615121801" + +[[package]] +name = "cpufeatures" +version = "0.2.17" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "59ed5838eebb26a2bb2e58f6d5b5316989ae9d08bab10e0e6d103e656d1b0280" +dependencies = [ + "libc", +] + +[[package]] +name = "crypto-common" +version = "0.1.7" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "78c8292055d1c1df0cce5d180393dc8cce0abec0a7102adb6c7b1eef6016d60a" +dependencies = [ + "generic-array", + "typenum", +] + +[[package]] +name = "data-encoding" +version = "2.11.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a4ae5f15dda3c708c0ade84bfee31ccab44a3da4f88015ed22f63732abe300c8" + +[[package]] +name = "digest" +version = "0.10.7" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9ed9a281f7bc9b7576e61468ba615a66a5c8cfdff42420a70aa82701a3b1e292" +dependencies = [ + "block-buffer", + "crypto-common", +] + +[[package]] +name = "find-msvc-tools" +version = "0.1.9" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5baebc0774151f905a1a2cc41989300b1e6fbb29aff0ceffa1064fdd3088d582" + +[[package]] +name = "futures-core" +version = "0.3.33" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2cd50c473c80f6d7c3670a752354b8e569b1a7cbfdc0419ec88e5edad85e0dc7" + +[[package]] +name = "futures-macro" +version = "0.3.33" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2d6d3cde68c518367be28956066ddfef33813991b77a55005a69dae04bf3b10b" +dependencies = [ + "proc-macro2", + "quote", + "syn 2.0.119", +] + +[[package]] +name = "futures-sink" +version = "0.3.33" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e34418ac499d6305c2fb5ad0ed2f6ac998c5f8ca209b4510f7f94242c647e307" + +[[package]] +name = "futures-task" +version = "0.3.33" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b231ed28831efb4a61a08580c4bc233ec56bc009f4cd8f52da2c3cb97df0c109" + +[[package]] +name = "futures-util" +version = "0.3.33" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a77a90a256fce34da66415271e30f94ee91c57b04b8a2c042d9cf3220179deaa" +dependencies = [ + "futures-core", + "futures-macro", + "futures-sink", + "futures-task", + "pin-project-lite", + "slab", +] + +[[package]] +name = "generic-array" +version = "0.14.7" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "85649ca51fd72272d7821adaf274ad91c288277713d9c18820d8499a7ff69e9a" +dependencies = [ + "typenum", + "version_check", +] + +[[package]] +name = "getrandom" +version = "0.2.17" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ff2abc00be7fca6ebc474524697ae276ad847ad0a6b3faa4bcb027e9a4614ad0" +dependencies = [ + "cfg-if", + "libc", + "wasi", +] + +[[package]] +name = "getrandom" +version = "0.3.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "899def5c37c4fd7b2664648c28120ecec138e4d395b459e5ca34f9cce2dd77fd" +dependencies = [ + "cfg-if", + "libc", + "r-efi", + "wasip2", +] + +[[package]] +name = "http" +version = "1.4.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6970f50e31d6fc17d3fa27329444bfa74e196cf62e95052a3f6fee181dba6425" +dependencies = [ + "bytes", + "itoa", +] + +[[package]] +name = "httparse" +version = "1.10.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6dbf3de79e51f3d586ab4cb9d5c3e2c14aa28ed23d180cf89b4df0454a69cc87" + +[[package]] +name = "itoa" +version = "1.0.18" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8f42a60cbdf9a97f5d2305f08a87dc4e09308d1276d28c869c684d7777685682" + +[[package]] +name = "lazy_static" +version = "1.5.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "bbd2bcb4c963f2ddae06a2efc7e9f3591312473c50c6685e1f298068316e66fe" + +[[package]] +name = "libc" +version = "0.2.189" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3eaf3ede3fee6db1a4c2ee091bf8a8b4dccdc6d17f656fb07896ee72867612f2" + +[[package]] +name = "log" +version = "0.4.33" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0ceec5bc11778974d1bcb055b18002eba7f4b3518b6a0081b3af5f21666da9ad" + +[[package]] +name = "matchers" +version = "0.2.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d1525a2a28c7f4fa0fc98bb91ae755d1e2d1505079e05539e35bc876b5d65ae9" +dependencies = [ + "regex-automata", +] + +[[package]] +name = "memchr" +version = "2.8.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "cf8baf1c55e62ffcace7a9f06f4bd9cd3f0c4beb022d3b367256b91b87513d98" + +[[package]] +name = "mio" +version = "1.2.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "30d65c71f1ce40ab09135ce117d742b9f8a19ff91a41a8b57ed50bc2de59c427" +dependencies = [ + "libc", + "wasi", + "windows-sys 0.61.2", +] + +[[package]] +name = "nu-ansi-term" +version = "0.50.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7957b9740744892f114936ab4a57b3f487491bbeafaf8083688b16841a4240e5" +dependencies = [ + "windows-sys 0.61.2", +] + +[[package]] +name = "once_cell" +version = "1.21.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9f7c3e4beb33f85d45ae3e3a1792185706c8e16d043238c593331cc7cd313b50" + +[[package]] +name = "pin-project-lite" +version = "0.2.17" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a89322df9ebe1c1578d689c92318e070967d1042b512afbe49518723f4e6d5cd" + +[[package]] +name = "ppv-lite86" +version = "0.2.21" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "85eae3c4ed2f50dcfe72643da4befc30deadb458a9b590d720cde2f2b1e97da9" +dependencies = [ + "zerocopy", +] + +[[package]] +name = "proc-macro2" +version = "1.0.107" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "985e7ec9bb745e6ce6535b544d84d6cd6f7ad8bd711c398938ae983b91a766d9" +dependencies = [ + "unicode-ident", +] + +[[package]] +name = "quote" +version = "1.0.47" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1fbf4db142a473a8d80c26bbf18454ed458bf8d26c8219c331daecfdbd079001" +dependencies = [ + "proc-macro2", +] + +[[package]] +name = "r-efi" +version = "5.3.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "69cdb34c158ceb288df11e18b4bd39de994f6657d83847bdffdbd7f346754b0f" + +[[package]] +name = "rand" +version = "0.9.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b9ef1d0d795eb7d84685bca4f72f3649f064e6641543d3a8c415898726a57b41" +dependencies = [ + "rand_chacha", + "rand_core", +] + +[[package]] +name = "rand_chacha" +version = "0.9.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d3022b5f1df60f26e1ffddd6c66e8aa15de382ae63b3a0c1bfc0e4d3e3f325cb" +dependencies = [ + "ppv-lite86", + "rand_core", +] + +[[package]] +name = "rand_core" +version = "0.9.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "76afc826de14238e6e8c374ddcc1fa19e374fd8dd986b0d2af0d02377261d83c" +dependencies = [ + "getrandom 0.3.4", +] + +[[package]] +name = "regex-automata" +version = "0.4.16" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8fcfdb36bda0c880c5931cdc7a2bcdc8ba4556847b9d912bca70bc94708711ad" +dependencies = [ + "aho-corasick", + "memchr", + "regex-syntax", +] + +[[package]] +name = "regex-syntax" +version = "0.8.11" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d6f6ff9a378485b298a5286656da665ba74413d36db0979633275d2e708145d4" + +[[package]] +name = "ring" +version = "0.17.14" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a4689e6c2294d81e88dc6261c768b63bc4fcdb852be6d1352498b114f61383b7" +dependencies = [ + "cc", + "cfg-if", + "getrandom 0.2.17", + "libc", + "untrusted", + "windows-sys 0.52.0", +] + +[[package]] +name = "rustls" +version = "0.23.42" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3c54fcab019b409d04215d3a17cb438fd7fbf192ee61461f20f4fe18704bc138" +dependencies = [ + "log", + "once_cell", + "ring", + "rustls-pki-types", + "rustls-webpki", + "subtle", + "zeroize", +] + +[[package]] +name = "rustls-pemfile" +version = "2.2.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "dce314e5fee3f39953d46bb63bb8a46d40c2f8fb7cc5a3b6cab2bde9721d6e50" +dependencies = [ + "rustls-pki-types", +] + +[[package]] +name = "rustls-pki-types" +version = "1.15.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "764899a24af3980067ee14bc143654f297b22eaebfe3c7b6b211920a5a59b046" +dependencies = [ + "zeroize", +] + +[[package]] +name = "rustls-webpki" +version = "0.103.13" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "61c429a8649f110dddef65e2a5ad240f747e85f7758a6bccc7e5777bd33f756e" +dependencies = [ + "ring", + "rustls-pki-types", + "untrusted", +] + +[[package]] +name = "serde" +version = "1.0.229" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "4148590afebada386688f18773da617792bf2ef03ffc1e4cbd2b1d45b023e0ba" +dependencies = [ + "serde_core", + "serde_derive", +] + +[[package]] +name = "serde_core" +version = "1.0.229" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "67dca2c9c51e58a4791a4b1ed58308b39c64224d349a935ab5039aa360942a48" +dependencies = [ + "serde_derive", +] + +[[package]] +name = "serde_derive" +version = "1.0.229" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e7a5d71263a5a7d47b41f6b3f06ba276f10cc18b0931f1799f710578e2309348" +dependencies = [ + "proc-macro2", + "quote", + "syn 3.0.3", +] + +[[package]] +name = "serde_json" +version = "1.0.151" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c841b55ecdae098c80dcae9cf767f6f8a0c2cdb3416bbef72181df4d0fe73f14" +dependencies = [ + "itoa", + "memchr", + "serde", + "serde_core", + "zmij", +] + +[[package]] +name = "sha1" +version = "0.10.7" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a978451301f4db1d02937a4ab3ccce137717b81826e79b7d49ffe3244a13c3b8" +dependencies = [ + "cfg-if", + "cpufeatures", + "digest", +] + +[[package]] +name = "sharded-slab" +version = "0.1.7" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f40ca3c46823713e0d4209592e8d6e826aa57e928f09752619fc696c499637f6" +dependencies = [ + "lazy_static", +] + +[[package]] +name = "shlex" +version = "2.0.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f8fadd59c855ef2080decdef8ff161eb6661b86933c9d82e5ba29dc602a55aba" + +[[package]] +name = "slab" +version = "0.4.12" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0c790de23124f9ab44544d7ac05d60440adc586479ce501c1d6d7da3cd8c9cf5" + +[[package]] +name = "smallvec" +version = "1.15.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8ed6a63f02c8539c91a8685a86f4099661ba3da017932f6ebbea6de3f0fa7c90" + +[[package]] +name = "socket2" +version = "0.6.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c3d1e2c7f27f8d4cb10542a02c49005dbd6e93095799d6f3be745fae9f8fedd4" +dependencies = [ + "libc", + "windows-sys 0.61.2", +] + +[[package]] +name = "subtle" +version = "2.6.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "13c2bddecc57b384dee18652358fb23172facb8a2c51ccc10d74c157bdea3292" + +[[package]] +name = "syn" +version = "2.0.119" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "872831b642d1a07999a962a351ed35b955ea2cfc8f3862091e2a240a84f17297" +dependencies = [ + "proc-macro2", + "quote", + "unicode-ident", +] + +[[package]] +name = "syn" +version = "3.0.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "53e9bae58849f64dfa4f5d5ae372c8341f7305f82a3868709269343628b659a3" +dependencies = [ + "proc-macro2", + "quote", + "unicode-ident", +] + +[[package]] +name = "thiserror" +version = "2.0.19" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "09a43598840e33d5b0331f38c5e30d13bb11c11210a4b58f0d9b18a5a5eefcd9" +dependencies = [ + "thiserror-impl", +] + +[[package]] +name = "thiserror-impl" +version = "2.0.19" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "43cbfe0cf76104d42a574802844187e84a305e531ed54455f11fbde0f10541cd" +dependencies = [ + "proc-macro2", + "quote", + "syn 3.0.3", +] + +[[package]] +name = "thread_local" +version = "1.1.10" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1ad99c4c6d32803332c548b1af0540b357b3f5fc0be8f6c6bfe8b2e6ae784070" +dependencies = [ + "cfg-if", +] + +[[package]] +name = "tokio" +version = "1.53.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "202caea871b69668250d242070849eb495be178ed697a3e98aebce5bc81a0bed" +dependencies = [ + "bytes", + "libc", + "mio", + "pin-project-lite", + "socket2", + "tokio-macros", + "windows-sys 0.61.2", +] + +[[package]] +name = "tokio-macros" +version = "2.7.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6328af13490e73a9b4694030fafd93f8c8c6a9dede33e821c3fc63eddf8042ba" +dependencies = [ + "proc-macro2", + "quote", + "syn 2.0.119", +] + +[[package]] +name = "tokio-rustls" +version = "0.26.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1729aa945f29d91ba541258c8df89027d5792d85a8841fb65e8bf0f4ede4ef61" +dependencies = [ + "rustls", + "tokio", +] + +[[package]] +name = "tokio-tungstenite" +version = "0.26.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7a9daff607c6d2bf6c16fd681ccb7eecc83e4e2cdc1ca067ffaadfca5de7f084" +dependencies = [ + "futures-util", + "log", + "tokio", + "tungstenite", +] + +[[package]] +name = "tracing" +version = "0.1.44" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "63e71662fa4b2a2c3a26f570f037eb95bb1f85397f3cd8076caed2f026a6d100" +dependencies = [ + "pin-project-lite", + "tracing-attributes", + "tracing-core", +] + +[[package]] +name = "tracing-attributes" +version = "0.1.31" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7490cfa5ec963746568740651ac6781f701c9c5ea257c58e057f3ba8cf69e8da" +dependencies = [ + "proc-macro2", + "quote", + "syn 2.0.119", +] + +[[package]] +name = "tracing-core" +version = "0.1.36" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "db97caf9d906fbde555dd62fa95ddba9eecfd14cb388e4f491a66d74cd5fb79a" +dependencies = [ + "once_cell", + "valuable", +] + +[[package]] +name = "tracing-log" +version = "0.2.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ee855f1f400bd0e5c02d150ae5de3840039a3f54b025156404e34c23c03f47c3" +dependencies = [ + "log", + "once_cell", + "tracing-core", +] + +[[package]] +name = "tracing-subscriber" +version = "0.3.23" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "cb7f578e5945fb242538965c2d0b04418d38ec25c79d160cd279bf0731c8d319" +dependencies = [ + "matchers", + "nu-ansi-term", + "once_cell", + "regex-automata", + "sharded-slab", + "smallvec", + "thread_local", + "tracing", + "tracing-core", + "tracing-log", +] + +[[package]] +name = "tungstenite" +version = "0.26.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "4793cb5e56680ecbb1d843515b23b6de9a75eb04b66643e256a396d43be33c13" +dependencies = [ + "bytes", + "data-encoding", + "http", + "httparse", + "log", + "rand", + "sha1", + "thiserror", + "utf-8", +] + +[[package]] +name = "typenum" +version = "1.20.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b6f5e870be6c3b371b77fe0ee0bafb859fa4964b4404c27de1d380043c4dda20" + +[[package]] +name = "unicode-ident" +version = "1.0.24" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e6e4313cd5fcd3dad5cafa179702e2b244f760991f45397d14d4ebf38247da75" + +[[package]] +name = "untrusted" +version = "0.9.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8ecb6da28b8a351d773b68d5825ac39017e680750f980f3a1a85cd8dd28a47c1" + +[[package]] +name = "utf-8" +version = "0.7.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "09cc8ee72d2a9becf2f2febe0205bbed8fc6615b7cb429ad062dc7b7ddd036a9" + +[[package]] +name = "valuable" +version = "0.1.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ba73ea9cf16a25df0c8caa16c51acb937d5712a8429db78a3ee29d5dcacd3a65" + +[[package]] +name = "version_check" +version = "0.9.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0b928f33d975fc6ad9f86c8f283853ad26bdd5b10b7f1542aa2fa15e2289105a" + +[[package]] +name = "wasi" +version = "0.11.1+wasi-snapshot-preview1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ccf3ec651a847eb01de73ccad15eb7d99f80485de043efb2f370cd654f4ea44b" + +[[package]] +name = "wasip2" +version = "1.0.4+wasi-0.2.12" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b67efb37e106e55ce722a510d6b5f9c17f083e5fc79afc2badeb12cc313d9487" +dependencies = [ + "wit-bindgen", +] + +[[package]] +name = "windows-link" +version = "0.2.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f0805222e57f7521d6a62e36fa9163bc891acd422f971defe97d64e70d0a4fe5" + +[[package]] +name = "windows-sys" +version = "0.52.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "282be5f36a8ce781fad8c8ae18fa3f9beff57ec1b52cb3de0789201425d9a33d" +dependencies = [ + "windows-targets", +] + +[[package]] +name = "windows-sys" +version = "0.61.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ae137229bcbd6cdf0f7b80a31df61766145077ddf49416a728b02cb3921ff3fc" +dependencies = [ + "windows-link", +] + +[[package]] +name = "windows-targets" +version = "0.52.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9b724f72796e036ab90c1021d4780d4d3d648aca59e491e6b98e725b84e99973" +dependencies = [ + "windows_aarch64_gnullvm", + "windows_aarch64_msvc", + "windows_i686_gnu", + "windows_i686_gnullvm", + "windows_i686_msvc", + "windows_x86_64_gnu", + "windows_x86_64_gnullvm", + "windows_x86_64_msvc", +] + +[[package]] +name = "windows_aarch64_gnullvm" +version = "0.52.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "32a4622180e7a0ec044bb555404c800bc9fd9ec262ec147edd5989ccd0c02cd3" + +[[package]] +name = "windows_aarch64_msvc" +version = "0.52.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "09ec2a7bb152e2252b53fa7803150007879548bc709c039df7627cabbd05d469" + +[[package]] +name = "windows_i686_gnu" +version = "0.52.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8e9b5ad5ab802e97eb8e295ac6720e509ee4c243f69d781394014ebfe8bbfa0b" + +[[package]] +name = "windows_i686_gnullvm" +version = "0.52.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0eee52d38c090b3caa76c563b86c3a4bd71ef1a819287c19d586d7334ae8ed66" + +[[package]] +name = "windows_i686_msvc" +version = "0.52.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "240948bc05c5e7c6dabba28bf89d89ffce3e303022809e73deaefe4f6ec56c66" + +[[package]] +name = "windows_x86_64_gnu" +version = "0.52.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "147a5c80aabfbf0c7d901cb5895d1de30ef2907eb21fbbab29ca94c5b08b1a78" + +[[package]] +name = "windows_x86_64_gnullvm" +version = "0.52.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "24d5b23dc417412679681396f2b49f3de8c1473deb516bd34410872eff51ed0d" + +[[package]] +name = "windows_x86_64_msvc" +version = "0.52.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "589f6da84c646204747d1270a2a5661ea66ed1cced2631d546fdfb155959f9ec" + +[[package]] +name = "wit-bindgen" +version = "0.57.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1ebf944e87a7c253233ad6766e082e3cd714b5d03812acc24c318f549614536e" + +[[package]] +name = "zerocopy" +version = "0.8.55" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b5a105cd7b140f6eeec8acff2ea38135d3cab283ada58540f629fe51e46696eb" +dependencies = [ + "zerocopy-derive", +] + +[[package]] +name = "zerocopy-derive" +version = "0.8.55" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0fe976fb70c78cd64cccfe3a6fc142244e8a77b70959b30faf9d0ac37ee228eb" +dependencies = [ + "proc-macro2", + "quote", + "syn 2.0.119", +] + +[[package]] +name = "zeroize" +version = "1.9.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e13c156562582aa81c60cb29407084cdb54c4164760106ab78e6c5b0858cf64e" + +[[package]] +name = "zmij" +version = "1.0.23" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "29666d0abbfad1e3dc4dcf6144730dd3a3ab225bbbdac83319345b1b44ccfc1b" diff --git a/Cargo.toml b/Cargo.toml new file mode 100644 index 0000000..c48608f --- /dev/null +++ b/Cargo.toml @@ -0,0 +1,52 @@ +[package] +name = "avocado-container-agent-dev" +version = "0.1.0" +edition = "2024" + +[[bin]] +name = "avocado-container-agent-dev" +path = "src/main.rs" + +[dependencies] +tokio = { version = "1", features = [ + "rt-multi-thread", + "macros", + "net", + "time", + "io-util", +] } +# WS client only. We deliberately DO NOT enable tokio-tungstenite's +# `rustls-tls-*` features: those pull `tokio-rustls` with its default features, +# which selects the `aws_lc_rs` crypto provider (forbidden on the device — A9 +# clears the `ring` stack for the musl cross targets, same rule avocado-conn +# enforces). Instead we establish TLS ourselves with a ring-only +# `tokio-rustls::TlsConnector` and hand the stream to `client_async`, so +# `aws-lc-rs` never enters the tree. +tokio-tungstenite = { version = "0.26", default-features = false, features = [ + "handshake", +] } +tokio-rustls = { version = "0.26", default-features = false, features = [ + "ring", + "logging", + "tls12", +] } +# Direct rustls dep so we can build a `ClientConfig` pinned to the ring +# provider and install it as the process default (defense-in-depth), matching +# avocado-conn's explicit-vertex pattern. +rustls = { version = "0.23", default-features = false, features = [ + "ring", + "logging", + "std", + "tls12", +] } +rustls-pemfile = "2" +futures-util = "0.3" +serde = { version = "1", features = ["derive"] } +serde_json = "1" +anyhow = "1" +tracing = "0.1" +tracing-subscriber = { version = "0.3", features = ["env-filter"] } + +[profile.release] +strip = true +lto = true diff --git a/src/main.rs b/src/main.rs new file mode 100644 index 0000000..88cfd1b --- /dev/null +++ b/src/main.rs @@ -0,0 +1,586 @@ +//! Device-side control agent for Container Dev Mode (design D9; task 6.1). +//! +//! The agent dials the host's control-only WebSocket, authenticates with the +//! read/control token (the only token a device ever holds — design D2), holds +//! the connection to receive [`HostFrame::Sync`] notifications, and +//! auto-reconnects with exponential backoff, re-sending its [`DeviceFrame::Hello`] +//! (carrying the current running digest) on every (re)connect. +//! +//! Scope of task 6.1 is the connect / hold / reconnect loop only. The actual +//! pull + container restart on a `Sync` frame is task 6.3; a clear seam is left +//! in [`handle_host_frame`]. +//! +//! TLS uses the **ring** rustls provider, never `aws-lc-rs`: the device +//! cross-compiles to musl and A9 clears only the ring stack (same rule +//! avocado-conn enforces). The control WS leaf is pinned to the bootstrap CA +//! alone — no native roots, no webpki-roots. + +use std::path::PathBuf; +use std::sync::Arc; +use std::sync::Mutex; +use std::sync::atomic::{AtomicBool, Ordering}; +use std::time::Duration; + +use anyhow::{Context, Result, ensure}; +use futures_util::{SinkExt, StreamExt}; +use serde::{Deserialize, Serialize}; +use tokio::net::TcpStream; +use tokio::time::sleep; +use tokio_rustls::TlsConnector; +use tokio_rustls::rustls::pki_types::ServerName; +use tokio_tungstenite::client_async; +use tokio_tungstenite::tungstenite::Message; +use tokio_tungstenite::tungstenite::client::IntoClientRequest; +use tokio_tungstenite::tungstenite::http::HeaderValue; +use tokio_tungstenite::tungstenite::http::header::AUTHORIZATION; +use tracing::{info, warn}; + +/// Default on-device path the host writes the bootstrap file to (design D5 / A7). +const DEFAULT_BOOTSTRAP_PATH: &str = "/var/lib/avocado/container-dev/bootstrap.json"; +/// Env var that overrides the bootstrap path (used by tests and staging). +const BOOTSTRAP_PATH_ENV: &str = "AVOCADO_CONTAINER_DEV_BOOTSTRAP"; +/// Env var that supplies the device identity reported in `Hello`. +const DEVICE_ID_ENV: &str = "AVOCADO_DEVICE_ID"; + +// --------------------------------------------------------------------------- +// Wire contract (MUST match avocado-cli/src/utils/container_dev/ws.rs). +// +// Frames are JSON, serde-tagged with `"type"`, `rename_all = "snake_case"`. +// These types are re-declared here (not imported) because avocado-cli is a +// separate repo with no shared workspace. The `frame_wire_shapes` test locks +// the JSON shape to the host's. +// --------------------------------------------------------------------------- + +/// A host -> device control frame. Carries only image coordinates plus a +/// content-digest reference — never blob bytes. +#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)] +#[serde(tag = "type", rename_all = "snake_case")] +pub enum HostFrame { + /// The `{image, tag, digest}` now available for the device to pull. + Sync { + image: String, + tag: String, + digest: String, + }, +} + +/// A device -> host control frame. +/// +/// `Hello` is an internally-tagged newtype variant: it serializes flat, as a +/// single object tagged `"hello"` carrying the `Hello` struct's fields — this +/// matches the host's `DeviceFrame::Hello(Hello)` wire form exactly. +#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)] +#[serde(tag = "type", rename_all = "snake_case")] +pub enum DeviceFrame { + Hello(Hello), + Progress(Progress), + Status(Status), +} + +/// The device's `hello`: who it is, its CPU arch, and the digest it runs now. +#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)] +pub struct Hello { + pub device_id: String, + pub arch: String, + pub running_digest: String, +} + +/// Progress of an in-flight device pull (informational). +#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)] +pub struct Progress { + pub image: String, + pub bytes_pulled: u64, +} + +/// A device state report (informational). +#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)] +pub struct Status { + pub device_id: String, + pub state: String, + #[serde(default, skip_serializing_if = "Option::is_none")] + pub detail: Option, +} + +// --------------------------------------------------------------------------- +// Device bootstrap. +// --------------------------------------------------------------------------- + +/// The device-side bootstrap, read from the writable partition. +/// +/// `bulk_endpoint`, `read_token`, and `ca_cert_pem` mirror the host's +/// `DeviceBootstrap` (avocado-cli/src/utils/container_dev/bootstrap.rs). +#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)] +pub struct Bootstrap { + /// The bulk read-listener endpoint (`host:port`) the device pulls from. + pub bulk_endpoint: String, + /// The Bearer read/control token — the only token a device holds. + pub read_token: String, + /// The per-project CA certificate (PEM) the device pins the host leaf to. + pub ca_cert_pem: String, + /// The `host:port` of the control WS to dial. + /// + /// FOLLOW-UP: the host side must be extended to emit this — avocado-cli's + /// `DeviceBootstrap` (bootstrap.rs) does not yet carry it, and the + /// control-WS bind in `dev.rs` must publish its address here. Tracked as a + /// devspec discovery event for this change. Defaulted so parsing a current + /// host payload (without the field) does not hard-fail; the agent errors + /// clearly at dial time if it is still empty. + #[serde(default)] + pub ws_endpoint: String, +} + +/// Resolve the bootstrap path: `$AVOCADO_CONTAINER_DEV_BOOTSTRAP` or the default. +fn bootstrap_path() -> PathBuf { + std::env::var_os(BOOTSTRAP_PATH_ENV) + .map(PathBuf::from) + .unwrap_or_else(|| PathBuf::from(DEFAULT_BOOTSTRAP_PATH)) +} + +/// Load and parse the bootstrap from `path`. +fn load_bootstrap(path: &std::path::Path) -> Result { + let raw = std::fs::read_to_string(path) + .with_context(|| format!("reading bootstrap at {}", path.display()))?; + let bootstrap: Bootstrap = serde_json::from_str(&raw) + .with_context(|| format!("parsing bootstrap at {}", path.display()))?; + Ok(bootstrap) +} + +/// Resolve the device identity reported in `Hello`. +/// +/// Prefers `$AVOCADO_DEVICE_ID`; falls back to the machine hostname +/// (`/etc/hostname`); finally `"unknown"`. Kept dependency-free. +fn resolve_device_id() -> String { + if let Ok(id) = std::env::var(DEVICE_ID_ENV) { + let id = id.trim().to_string(); + if !id.is_empty() { + return id; + } + } + if let Ok(hostname) = std::fs::read_to_string("/etc/hostname") { + let hostname = hostname.trim().to_string(); + if !hostname.is_empty() { + return hostname; + } + } + "unknown".to_string() +} + +// --------------------------------------------------------------------------- +// Reconnect backoff. +// --------------------------------------------------------------------------- + +/// Exponential backoff with a cap for the reconnect loop. +/// +/// Doubles the delay on each `next_delay` up to `max`, and resets to `base` +/// after a successful connection so a long-lived link that briefly drops does +/// not inherit a stale, large delay. +#[derive(Debug, Clone)] +struct Backoff { + base: Duration, + max: Duration, + current: Duration, +} + +impl Backoff { + fn new(base: Duration, max: Duration) -> Self { + Self { + base, + max, + current: base, + } + } + + /// Return the delay to wait before the next attempt, then advance (doubling + /// up to `max`). + fn next_delay(&mut self) -> Duration { + let delay = self.current; + self.current = (self.current * 2).min(self.max); + delay + } + + /// Reset to the base delay (call after a successful connection). + fn reset(&mut self) { + self.current = self.base; + } +} + +impl Default for Backoff { + fn default() -> Self { + Backoff::new(Duration::from_secs(1), Duration::from_secs(30)) + } +} + +// --------------------------------------------------------------------------- +// Agent state. +// --------------------------------------------------------------------------- + +/// Shared, mutable agent state carried across reconnects. +struct AgentState { + device_id: String, + /// The content digest the agent is currently running. Empty until a pull + /// lands (task 6.3 updates this); re-sent in `Hello` on every reconnect. + running_digest: Mutex, +} + +impl AgentState { + fn new(device_id: String) -> Self { + Self { + device_id, + running_digest: Mutex::new(String::new()), + } + } + + fn hello(&self) -> DeviceFrame { + let running_digest = self + .running_digest + .lock() + .expect("running_digest mutex poisoned") + .clone(); + DeviceFrame::Hello(Hello { + device_id: self.device_id.clone(), + arch: std::env::consts::ARCH.to_string(), + running_digest, + }) + } +} + +// --------------------------------------------------------------------------- +// TLS. +// --------------------------------------------------------------------------- + +/// Build a rustls `ClientConfig` pinned to ONLY the bootstrap CA, using the +/// ring provider. No native roots, no webpki-roots. +fn build_tls_config(ca_cert_pem: &str) -> Result { + let mut root_store = rustls::RootCertStore::empty(); + let mut reader = std::io::Cursor::new(ca_cert_pem.as_bytes()); + let certs = rustls_pemfile::certs(&mut reader) + .collect::, _>>() + .context("parsing bootstrap ca_cert_pem")?; + ensure!( + !certs.is_empty(), + "bootstrap ca_cert_pem contained no certificates" + ); + for cert in certs { + root_store + .add(cert) + .context("adding bootstrap CA to root store")?; + } + + let provider = Arc::new(rustls::crypto::ring::default_provider()); + let config = rustls::ClientConfig::builder_with_provider(provider) + .with_safe_default_protocol_versions() + .context("selecting rustls protocol versions")? + .with_root_certificates(root_store) + .with_no_client_auth(); + Ok(config) +} + +/// Split a `host:port` endpoint into its host component for SNI. +fn endpoint_host(endpoint: &str) -> &str { + endpoint + .rsplit_once(':') + .map(|(h, _)| h) + .unwrap_or(endpoint) +} + +// --------------------------------------------------------------------------- +// Connect + serve. +// --------------------------------------------------------------------------- + +/// Handle one host frame. Task 6.1 logs and holds; the pull + restart is the +/// task 6.3 seam. +async fn handle_host_frame(frame: HostFrame, _state: &AgentState) { + match frame { + HostFrame::Sync { image, tag, digest } => { + info!(%image, %tag, %digest, "received sync (pull + restart is task 6.3)"); + // SEAM (task 6.3): pull `image:tag@digest` through the loopback + // registry proxy, restart the container, update + // `state.running_digest`, and send a `Status` frame. + } + } +} + +/// Dial the control WS, send `Hello`, and serve frames until the link closes. +/// +/// Sets `connected` to `true` once the WS handshake succeeds, so the caller can +/// reset the backoff on a link that actually established. +async fn connect_and_serve( + bootstrap: &Bootstrap, + state: &AgentState, + connected: &AtomicBool, +) -> Result<()> { + ensure!( + !bootstrap.ws_endpoint.is_empty(), + "bootstrap ws_endpoint is empty; the host must emit it (tracked follow-up)" + ); + + let tls_config = build_tls_config(&bootstrap.ca_cert_pem)?; + let connector = TlsConnector::from(Arc::new(tls_config)); + + let tcp = TcpStream::connect(&bootstrap.ws_endpoint) + .await + .with_context(|| format!("connecting TCP to {}", bootstrap.ws_endpoint))?; + + let host = endpoint_host(&bootstrap.ws_endpoint).to_string(); + let server_name = ServerName::try_from(host).context("invalid TLS server name")?; + let tls = connector + .connect(server_name, tcp) + .await + .context("TLS handshake to control WS")?; + + let mut request = format!("wss://{}/", bootstrap.ws_endpoint) + .into_client_request() + .context("building WS upgrade request")?; + request.headers_mut().insert( + AUTHORIZATION, + HeaderValue::from_str(&format!("Bearer {}", bootstrap.read_token)) + .context("building Authorization header")?, + ); + + let (ws, _response) = client_async(request, tls) + .await + .context("WebSocket upgrade to control WS")?; + connected.store(true, Ordering::SeqCst); + info!(endpoint = %bootstrap.ws_endpoint, "control WS connected"); + + let (mut sink, mut stream) = ws.split(); + + let hello = state.hello(); + sink.send(Message::Text( + serde_json::to_string(&hello) + .context("serializing Hello")? + .into(), + )) + .await + .context("sending Hello")?; + + while let Some(msg) = stream.next().await { + let msg = msg.context("reading control WS frame")?; + match msg { + Message::Text(text) => match serde_json::from_str::(text.as_str()) { + Ok(frame) => handle_host_frame(frame, state).await, + Err(e) => warn!(error = %e, raw = %text, "ignoring malformed host frame"), + }, + Message::Close(_) => { + info!("control WS closed by host"); + break; + } + Message::Ping(_) | Message::Pong(_) => {} + Message::Binary(_) | Message::Frame(_) => { + warn!("ignoring unexpected non-text control frame"); + } + } + } + + Ok(()) +} + +/// The reconnect loop: connect, serve, and on any close/error back off and +/// redial, re-sending `Hello` with the current running digest. +async fn run(bootstrap: Bootstrap, state: Arc) -> Result<()> { + let mut backoff = Backoff::default(); + loop { + let connected = AtomicBool::new(false); + match connect_and_serve(&bootstrap, &state, &connected).await { + Ok(()) => info!("control WS session ended; reconnecting"), + Err(e) => warn!(error = %e, "control WS session failed; reconnecting"), + } + if connected.load(Ordering::SeqCst) { + backoff.reset(); + } + let delay = backoff.next_delay(); + info!(delay_secs = delay.as_secs(), "backing off before reconnect"); + sleep(delay).await; + } +} + +#[tokio::main] +async fn main() -> Result<()> { + tracing_subscriber::fmt() + .with_env_filter( + tracing_subscriber::EnvFilter::try_from_default_env() + .unwrap_or_else(|_| tracing_subscriber::EnvFilter::new("info")), + ) + .init(); + + // Defense-in-depth: pin ring as the process-wide default provider so any + // implicit rustls builder cannot fall back to a different provider. + let _ = rustls::crypto::ring::default_provider().install_default(); + + let path = bootstrap_path(); + let bootstrap = load_bootstrap(&path)?; + let device_id = resolve_device_id(); + info!(%device_id, bootstrap = %path.display(), "container dev agent starting"); + + let state = Arc::new(AgentState::new(device_id)); + run(bootstrap, state).await +} + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn host_sync_frame_deserializes_from_wire() { + let wire = r#"{"type":"sync","image":"my-app","tag":"dev","digest":"sha256:abc"}"#; + let frame: HostFrame = serde_json::from_str(wire).expect("sync frame parses"); + assert_eq!( + frame, + HostFrame::Sync { + image: "my-app".to_string(), + tag: "dev".to_string(), + digest: "sha256:abc".to_string(), + } + ); + } + + #[test] + fn hello_frame_serializes_flat_with_tag() { + let hello = DeviceFrame::Hello(Hello { + device_id: "dev-01".to_string(), + arch: "aarch64".to_string(), + running_digest: "sha256:running".to_string(), + }); + let value: serde_json::Value = + serde_json::to_value(&hello).expect("hello serializes to json"); + assert_eq!(value["type"], "hello"); + assert_eq!(value["device_id"], "dev-01"); + assert_eq!(value["arch"], "aarch64"); + assert_eq!(value["running_digest"], "sha256:running"); + // The tag object is flat: no nested wrapper key for the Hello struct. + assert!(value.get("Hello").is_none()); + } + + #[test] + fn progress_frame_serializes_flat_with_tag() { + let frame = DeviceFrame::Progress(Progress { + image: "my-app".to_string(), + bytes_pulled: 4096, + }); + let value: serde_json::Value = serde_json::to_value(&frame).unwrap(); + assert_eq!(value["type"], "progress"); + assert_eq!(value["image"], "my-app"); + assert_eq!(value["bytes_pulled"], 4096); + } + + #[test] + fn status_frame_serializes_flat_with_tag() { + let frame = DeviceFrame::Status(Status { + device_id: "dev-01".to_string(), + state: "running".to_string(), + detail: None, + }); + let value: serde_json::Value = serde_json::to_value(&frame).unwrap(); + assert_eq!(value["type"], "status"); + assert_eq!(value["device_id"], "dev-01"); + assert_eq!(value["state"], "running"); + // Optional detail is skipped when None. + assert!(value.get("detail").is_none()); + } + + #[test] + fn device_frame_round_trips() { + let frames = vec![ + DeviceFrame::Hello(Hello { + device_id: "d".to_string(), + arch: "x86_64".to_string(), + running_digest: String::new(), + }), + DeviceFrame::Progress(Progress { + image: "img".to_string(), + bytes_pulled: 1, + }), + DeviceFrame::Status(Status { + device_id: "d".to_string(), + state: "s".to_string(), + detail: Some("ok".to_string()), + }), + ]; + for frame in frames { + let json = serde_json::to_string(&frame).unwrap(); + let back: DeviceFrame = serde_json::from_str(&json).unwrap(); + assert_eq!(frame, back); + } + } + + #[test] + fn bootstrap_parses_without_ws_endpoint_and_defaults_it() { + // A current host payload carries only the three DeviceBootstrap fields. + let wire = r#"{ + "bulk_endpoint": "10.0.0.1:8443", + "read_token": "tok", + "ca_cert_pem": "-----BEGIN CERTIFICATE-----\nMII...\n-----END CERTIFICATE-----\n" + }"#; + let bootstrap: Bootstrap = serde_json::from_str(wire).expect("bootstrap parses"); + assert_eq!(bootstrap.bulk_endpoint, "10.0.0.1:8443"); + assert_eq!(bootstrap.read_token, "tok"); + assert_eq!(bootstrap.ws_endpoint, ""); + } + + #[test] + fn bootstrap_parses_with_ws_endpoint() { + let wire = r#"{ + "bulk_endpoint": "10.0.0.1:8443", + "read_token": "tok", + "ca_cert_pem": "pem", + "ws_endpoint": "10.0.0.1:9443" + }"#; + let bootstrap: Bootstrap = serde_json::from_str(wire).unwrap(); + assert_eq!(bootstrap.ws_endpoint, "10.0.0.1:9443"); + } + + #[test] + fn backoff_doubles_then_caps() { + let mut backoff = Backoff::new(Duration::from_secs(1), Duration::from_secs(30)); + assert_eq!(backoff.next_delay(), Duration::from_secs(1)); + assert_eq!(backoff.next_delay(), Duration::from_secs(2)); + assert_eq!(backoff.next_delay(), Duration::from_secs(4)); + assert_eq!(backoff.next_delay(), Duration::from_secs(8)); + assert_eq!(backoff.next_delay(), Duration::from_secs(16)); + // 32 would exceed the 30s cap. + assert_eq!(backoff.next_delay(), Duration::from_secs(30)); + assert_eq!(backoff.next_delay(), Duration::from_secs(30)); + } + + #[test] + fn backoff_resets_to_base_after_connect() { + let mut backoff = Backoff::new(Duration::from_secs(1), Duration::from_secs(30)); + backoff.next_delay(); + backoff.next_delay(); + backoff.next_delay(); + backoff.reset(); + assert_eq!(backoff.next_delay(), Duration::from_secs(1)); + assert_eq!(backoff.next_delay(), Duration::from_secs(2)); + } + + #[test] + fn hello_carries_current_running_digest() { + let state = AgentState::new("dev-01".to_string()); + // Simulate a pull landing (task 6.3 behavior). + *state.running_digest.lock().unwrap() = "sha256:new".to_string(); + match state.hello() { + DeviceFrame::Hello(hello) => { + assert_eq!(hello.device_id, "dev-01"); + assert_eq!(hello.running_digest, "sha256:new"); + assert_eq!(hello.arch, std::env::consts::ARCH); + } + other => panic!("expected Hello, got {other:?}"), + } + } + + #[test] + fn endpoint_host_strips_port() { + assert_eq!(endpoint_host("10.0.0.1:9443"), "10.0.0.1"); + assert_eq!(endpoint_host("host.local:443"), "host.local"); + assert_eq!(endpoint_host("nohost"), "nohost"); + } + + #[test] + fn build_tls_config_rejects_empty_pem() { + let err = build_tls_config("not a cert").unwrap_err(); + assert!( + err.to_string().contains("no certificates"), + "unexpected error: {err}" + ); + } +} From 3e8d50522c06be59fea09503cd13d5e1efed2876 Mon Sep 17 00:00:00 2001 From: Javier Tia Date: Wed, 22 Jul 2026 08:19:41 -0600 Subject: [PATCH 02/20] container-agent-dev: Add loopback registry proxy for device image pulls Container engines (docker/podman) require either an insecure-registry flag or a per-engine trust/cert file to pull from any non-loopback HTTP registry. Neither is acceptable in a zero-touch device deployment where we cannot assume engine configuration is modifiable. Without a solution, the agent cannot serve images to the on-device engine without requiring privileged engine reconfiguration. Introduce a plain-HTTP loopback registry proxy that the device engine pulls from without any trust configuration, since engines automatically trust loopback addresses. Each OCI read request arriving on `127.0.0.1:15151` is forwarded over a pinned-CA HTTPS connection to the host bulk listener, carrying the read/control Bearer token. Blob bytes travel this direct HTTPS path and never ride the control WebSocket, matching the design's bulk/control split. The proxy fails closed pre-stream: the upstream status is inspected before any engine-facing `200` is opened, so a stale-token `401`/`403` from the upstream never triggers an engine retry loop. When the token is rejected the proxy raises a shared `ReBootstrap` flag the agent surfaces to the host as a `Status` frame on the next reconnect, directing the operator to re-run `avocado container dev up`. Upstream `WWW-Authenticate` headers are dropped via a strict passthrough allowlist and never reach the engine under any status code. Signed-off-by: Javier Tia --- Cargo.lock | 174 ++++++++++++ Cargo.toml | 22 ++ src/main.rs | 66 ++++- src/proxy.rs | 737 +++++++++++++++++++++++++++++++++++++++++++++++++++ 4 files changed, 997 insertions(+), 2 deletions(-) create mode 100644 src/proxy.rs diff --git a/Cargo.lock b/Cargo.lock index b263e9f..7854bca 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -17,12 +17,23 @@ version = "1.0.104" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "330a5ed07fa54e4702c9d6c4174f74427fc0ef6e214bbd677ae50a5099946470" +[[package]] +name = "atomic-waker" +version = "1.1.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1505bd5d3d116872e7271a6d4e16d81d0c8570876c8de68093a09ac269d8aac0" + [[package]] name = "avocado-container-agent-dev" version = "0.1.0" dependencies = [ "anyhow", + "bytes", "futures-util", + "http-body-util", + "hyper", + "hyper-util", + "rcgen", "rustls", "rustls-pemfile", "serde", @@ -34,6 +45,12 @@ dependencies = [ "tracing-subscriber", ] +[[package]] +name = "base64" +version = "0.22.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "72b3254f16251a8381aa12e40e3c4d2f0199f8c6508fbecb9d91f575e0fbb8c6" + [[package]] name = "block-buffer" version = "0.10.4" @@ -90,6 +107,12 @@ version = "2.11.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "a4ae5f15dda3c708c0ade84bfee31ccab44a3da4f88015ed22f63732abe300c8" +[[package]] +name = "deranged" +version = "0.5.8" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7cd812cc2bc1d69d4764bd80df88b4317eaef9e773c75226407d9bc0876b211c" + [[package]] name = "digest" version = "0.10.7" @@ -106,6 +129,15 @@ version = "0.1.9" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "5baebc0774151f905a1a2cc41989300b1e6fbb29aff0ceffa1064fdd3088d582" +[[package]] +name = "futures-channel" +version = "0.3.33" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "262590f4fe6afeb0bc83be1daa64e52657fe185690a958af7f3ad0e92085c5ae" +dependencies = [ + "futures-core", +] + [[package]] name = "futures-core" version = "0.3.33" @@ -192,12 +224,76 @@ dependencies = [ "itoa", ] +[[package]] +name = "http-body" +version = "1.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ca2a8f2913ee65f60facd6a5905613afaa448497a0230cc41ce022d93290bc2c" +dependencies = [ + "bytes", + "http", +] + +[[package]] +name = "http-body-util" +version = "0.1.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e9f41fd6a08e4d4ec69df65976da761afd5ad5e58a9d4acb46bd1c953a9e3ff2" +dependencies = [ + "bytes", + "futures-core", + "http", + "http-body", + "pin-project-lite", +] + [[package]] name = "httparse" version = "1.10.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "6dbf3de79e51f3d586ab4cb9d5c3e2c14aa28ed23d180cf89b4df0454a69cc87" +[[package]] +name = "httpdate" +version = "1.0.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "df3b46402a9d5adb4c86a0cf463f42e19994e3ee891101b1841f30a545cb49a9" + +[[package]] +name = "hyper" +version = "1.11.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d22053281f852e11534f5198498373cbb59295120a20771d90f7ed1897490a72" +dependencies = [ + "atomic-waker", + "bytes", + "futures-channel", + "futures-core", + "http", + "http-body", + "httparse", + "httpdate", + "itoa", + "pin-project-lite", + "smallvec", + "tokio", + "want", +] + +[[package]] +name = "hyper-util" +version = "0.1.20" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "96547c2556ec9d12fb1578c4eaf448b04993e7fb79cbaad930a656880a6bdfa0" +dependencies = [ + "bytes", + "http", + "http-body", + "hyper", + "pin-project-lite", + "tokio", +] + [[package]] name = "itoa" version = "1.0.18" @@ -257,18 +353,40 @@ dependencies = [ "windows-sys 0.61.2", ] +[[package]] +name = "num-conv" +version = "0.2.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "521739c6d2bac4aa25192232afe6841231376b2b26d4d9fae5ecf8ca5772e441" + [[package]] name = "once_cell" version = "1.21.4" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "9f7c3e4beb33f85d45ae3e3a1792185706c8e16d043238c593331cc7cd313b50" +[[package]] +name = "pem" +version = "3.0.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1d30c53c26bc5b31a98cd02d20f25a7c8567146caf63ed593a9d87b2775291be" +dependencies = [ + "base64", + "serde_core", +] + [[package]] name = "pin-project-lite" version = "0.2.17" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "a89322df9ebe1c1578d689c92318e070967d1042b512afbe49518723f4e6d5cd" +[[package]] +name = "powerfmt" +version = "0.2.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "439ee305def115ba05938db6eb1644ff94165c5ab5e9420d1c1bcedbba909391" + [[package]] name = "ppv-lite86" version = "0.2.21" @@ -331,6 +449,19 @@ dependencies = [ "getrandom 0.3.4", ] +[[package]] +name = "rcgen" +version = "0.13.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "75e669e5202259b5314d1ea5397316ad400819437857b90861765f24c4cf80a2" +dependencies = [ + "pem", + "ring", + "rustls-pki-types", + "time", + "yasna", +] + [[package]] name = "regex-automata" version = "0.4.16" @@ -554,6 +685,25 @@ dependencies = [ "cfg-if", ] +[[package]] +name = "time" +version = "0.3.54" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3e1d5e639ff6bab73cb6885cc7e7b1de96c3f32c68ec55f3952614bec1092244" +dependencies = [ + "deranged", + "num-conv", + "powerfmt", + "serde_core", + "time-core", +] + +[[package]] +name = "time-core" +version = "0.1.9" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9e1c906769ad99c88eaa54e728060edef082f8e358ff32030cb7c7d315e81109" + [[package]] name = "tokio" version = "1.53.1" @@ -663,6 +813,12 @@ dependencies = [ "tracing-log", ] +[[package]] +name = "try-lock" +version = "0.2.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e421abadd41a4225275504ea4d6566923418b7f05506fbc9c0fe86ba7396114b" + [[package]] name = "tungstenite" version = "0.26.2" @@ -716,6 +872,15 @@ version = "0.9.5" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "0b928f33d975fc6ad9f86c8f283853ad26bdd5b10b7f1542aa2fa15e2289105a" +[[package]] +name = "want" +version = "0.3.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "bfa7760aed19e106de2c7c0b581b509f2f25d3dacaf737cb82ac61bc6d760b0e" +dependencies = [ + "try-lock", +] + [[package]] name = "wasi" version = "0.11.1+wasi-snapshot-preview1" @@ -825,6 +990,15 @@ version = "0.57.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "1ebf944e87a7c253233ad6766e082e3cd714b5d03812acc24c318f549614536e" +[[package]] +name = "yasna" +version = "0.5.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e17bb3549cc1321ae1296b9cdc2698e2b6cb1992adfa19a8c72e5b7a738f44cd" +dependencies = [ + "time", +] + [[package]] name = "zerocopy" version = "0.8.55" diff --git a/Cargo.toml b/Cargo.toml index c48608f..830d1cd 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -46,6 +46,28 @@ serde_json = "1" anyhow = "1" tracing = "0.1" tracing-subscriber = { version = "0.3", features = ["env-filter"] } +# Loopback registry proxy (task 6.2). hyper is a pure-Rust HTTP/1 stack with no +# crypto of its own; the upstream leg's TLS is driven by the same ring-backed +# `tokio-rustls`/`rustls` above, so no `aws-lc-rs` enters the tree. We +# deliberately do NOT use `reqwest`, whose default rustls would pull `aws-lc-rs`. +hyper = { version = "1", default-features = false, features = [ + "server", + "client", + "http1", +] } +hyper-util = { version = "0.1", default-features = false, features = ["tokio"] } +http-body-util = "0.1" +bytes = "1" + +[dev-dependencies] +# Throwaway CA/leaf generation for the in-process TLS upstream in proxy tests. +# `ring` (not the crate default `aws_lc_rs`) so the pinned-CA test stack matches +# the agent's own ring provider and keeps `aws-lc-rs` out of the tree entirely. +rcgen = { version = "0.13", default-features = false, features = [ + "crypto", + "pem", + "ring", +] } [profile.release] strip = true diff --git a/src/main.rs b/src/main.rs index 88cfd1b..077d420 100644 --- a/src/main.rs +++ b/src/main.rs @@ -35,12 +35,20 @@ use tokio_tungstenite::tungstenite::http::HeaderValue; use tokio_tungstenite::tungstenite::http::header::AUTHORIZATION; use tracing::{info, warn}; +mod proxy; + /// Default on-device path the host writes the bootstrap file to (design D5 / A7). const DEFAULT_BOOTSTRAP_PATH: &str = "/var/lib/avocado/container-dev/bootstrap.json"; /// Env var that overrides the bootstrap path (used by tests and staging). const BOOTSTRAP_PATH_ENV: &str = "AVOCADO_CONTAINER_DEV_BOOTSTRAP"; /// Env var that supplies the device identity reported in `Hello`. const DEVICE_ID_ENV: &str = "AVOCADO_DEVICE_ID"; +/// Default loopback port the device engine pulls from (task 6.2). Chosen off +/// 5000 to avoid the macOS AirPlay clash noted in the Phase 0 port survey +/// (design 1.6); the engine config (task 6.4) points at `127.0.0.1:`. +const DEFAULT_LOOPBACK_PORT: u16 = 15151; +/// Env var overriding the loopback registry port. +const LOOPBACK_PORT_ENV: &str = "AVOCADO_CONTAINER_DEV_LOOPBACK_PORT"; // --------------------------------------------------------------------------- // Wire contract (MUST match avocado-cli/src/utils/container_dev/ws.rs). @@ -136,6 +144,15 @@ fn bootstrap_path() -> PathBuf { .unwrap_or_else(|| PathBuf::from(DEFAULT_BOOTSTRAP_PATH)) } +/// Resolve the loopback registry port: `$AVOCADO_CONTAINER_DEV_LOOPBACK_PORT` +/// or the default. +fn loopback_port() -> u16 { + std::env::var(LOOPBACK_PORT_ENV) + .ok() + .and_then(|v| v.trim().parse().ok()) + .unwrap_or(DEFAULT_LOOPBACK_PORT) +} + /// Load and parse the bootstrap from `path`. fn load_bootstrap(path: &std::path::Path) -> Result { let raw = std::fs::read_to_string(path) @@ -220,6 +237,10 @@ struct AgentState { /// The content digest the agent is currently running. Empty until a pull /// lands (task 6.3 updates this); re-sent in `Hello` on every reconnect. running_digest: Mutex, + /// Raised by the loopback proxy (task 6.2) when the bulk upstream rejects + /// the read/control token. Surfaced to the host as a `Status` frame on the + /// next (re)connect; a new token comes only from a host re-`up`. + rebootstrap: proxy::ReBootstrap, } impl AgentState { @@ -227,6 +248,7 @@ impl AgentState { Self { device_id, running_digest: Mutex::new(String::new()), + rebootstrap: proxy::ReBootstrap::default(), } } @@ -250,7 +272,7 @@ impl AgentState { /// Build a rustls `ClientConfig` pinned to ONLY the bootstrap CA, using the /// ring provider. No native roots, no webpki-roots. -fn build_tls_config(ca_cert_pem: &str) -> Result { +pub(crate) fn build_tls_config(ca_cert_pem: &str) -> Result { let mut root_store = rustls::RootCertStore::empty(); let mut reader = std::io::Cursor::new(ca_cert_pem.as_bytes()); let certs = rustls_pemfile::certs(&mut reader) @@ -276,7 +298,7 @@ fn build_tls_config(ca_cert_pem: &str) -> Result { } /// Split a `host:port` endpoint into its host component for SNI. -fn endpoint_host(endpoint: &str) -> &str { +pub(crate) fn endpoint_host(endpoint: &str) -> &str { endpoint .rsplit_once(':') .map(|(h, _)| h) @@ -354,6 +376,27 @@ async fn connect_and_serve( .await .context("sending Hello")?; + // Surface a pending re-bootstrap (raised by the loopback proxy on a bulk + // 401) to the host. There is no token-renewal endpoint; the operator must + // re-run `avocado container dev up` to mint a fresh token. + if state.rebootstrap.is_raised() { + let status = DeviceFrame::Status(Status { + device_id: state.device_id.clone(), + state: "needs_rebootstrap".to_string(), + detail: Some( + "bulk pull rejected the read/control token; re-run `avocado container dev up`" + .to_string(), + ), + }); + sink.send(Message::Text( + serde_json::to_string(&status) + .context("serializing re-bootstrap status")? + .into(), + )) + .await + .context("sending re-bootstrap status")?; + } + while let Some(msg) = stream.next().await { let msg = msg.context("reading control WS frame")?; match msg { @@ -413,6 +456,25 @@ async fn main() -> Result<()> { info!(%device_id, bootstrap = %path.display(), "container dev agent starting"); let state = Arc::new(AgentState::new(device_id)); + + // Start the loopback registry proxy (task 6.2): the device engine pulls + // from this plain-HTTP loopback (no insecure flag, no per-engine trust + // file); each read is forwarded to the host bulk HTTPS listener over the + // pinned CA + read/control token. Bulk NEVER rides the control WS. + match proxy::HttpsUpstream::from_bootstrap(&bootstrap) { + Ok(upstream) => { + let port = loopback_port(); + let upstream = Arc::new(upstream); + let rebootstrap = state.rebootstrap.clone(); + tokio::spawn(async move { + if let Err(e) = proxy::serve_loopback(port, upstream, rebootstrap).await { + warn!(error = %e, "loopback registry proxy exited"); + } + }); + } + Err(e) => warn!(error = %e, "loopback registry proxy not started"), + } + run(bootstrap, state).await } diff --git a/src/proxy.rs b/src/proxy.rs new file mode 100644 index 0000000..00d4a41 --- /dev/null +++ b/src/proxy.rs @@ -0,0 +1,737 @@ +//! Device loopback registry proxy (design D3/D9; task 6.2). +//! +//! The device container engine (docker/podman) pulls from a **plain-HTTP +//! loopback** registry this module serves on `127.0.0.1:`. Engines +//! auto-trust a loopback registry, so the device needs NO insecure-registry +//! flag and NO per-engine trust/cert file — that is how the two "NO ..." +//! guarantees are met: by serving loopback HTTP we need zero engine trust +//! config at all. +//! +//! Each engine read (`GET`/`HEAD` on `/v2/…/manifests/…` and +//! `/v2/…/blobs/…`, plus the `GET /v2/` ping) is forwarded to the host's +//! dedicated **bulk HTTPS listener** (D9) over a rustls client pinned to ONLY +//! the bootstrap CA (reusing [`crate::build_tls_config`]), carrying the +//! read/control token as `Authorization: Bearer …`. Blob bytes ride THIS +//! HTTPS path, never the control WebSocket (D9 splits bulk from control). +//! +//! Fail-closed pre-stream (M-A / H-2 / M-3): the UPSTREAM status is inspected +//! BEFORE any engine-facing `200` is opened. On an upstream `2xx` we open the +//! `2xx` and stream the body through. On an upstream `401`/`403` we do NOT open +//! a `200`: we return a clean gateway error that carries neither the upstream +//! status semantics nor its `WWW-Authenticate` header, and we raise a shared +//! "needs re-bootstrap" flag the agent surfaces (the agent does not loop +//! silently). There is NO token refresh/renewal endpoint — a new token comes +//! only from a host re-`up` re-bootstrap. The mid-stream 401 case is PREVENTED +//! by the host's drain-based grace-overlap (D5), not signalled: OCI/HTTP have +//! no "terminal, do not retry" wire signal, so if the drain ceiling is ever +//! exceeded the agent's mid-stream behavior is best-effort (the stream simply +//! cuts), explicitly NOT a load-bearing terminal signal. The upstream +//! `WWW-Authenticate` is never proxied back to the engine under any status. + +use std::convert::Infallible; +use std::future::Future; +use std::net::{Ipv4Addr, SocketAddr}; +use std::sync::Arc; +use std::sync::atomic::{AtomicBool, Ordering}; + +use anyhow::{Context, Result}; +use bytes::Bytes; +use http_body_util::combinators::BoxBody; +use http_body_util::{BodyExt, Empty, Full}; +use hyper::body::Incoming; +use hyper::header::HeaderValue; +use hyper::header::{AUTHORIZATION, HOST, RANGE}; +use hyper::service::service_fn; +use hyper::{HeaderMap, Method, Request, Response, StatusCode}; +use hyper_util::rt::TokioIo; +use tokio::net::{TcpListener, TcpStream}; +use tokio_rustls::TlsConnector; +use tokio_rustls::rustls::pki_types::ServerName; +use tracing::{info, warn}; + +use crate::{Bootstrap, build_tls_config, endpoint_host}; + +/// The engine-facing response body: boxed so both streamed upstream bodies and +/// small static error bodies share one type. +type RespBody = BoxBody; + +/// Upstream headers copied through on a successful (`2xx`) read. Deliberately a +/// strict allowlist: `www-authenticate` (and every other header) is dropped +/// unless it is on this list, so an upstream auth header can never reach the +/// engine even on a `2xx`. +const PASSTHROUGH_HEADERS: &[&str] = &[ + "content-type", + "content-length", + "content-range", + "accept-ranges", + "docker-content-digest", + "etag", + "last-modified", +]; + +/// The plain-text body returned to the engine when the bulk upstream rejects +/// the read/control token. It carries no upstream status or header semantics. +const REBOOTSTRAP_MSG: &str = + "bulk upstream rejected the read/control token; re-run `avocado container dev up`"; + +// --------------------------------------------------------------------------- +// Re-bootstrap status flag. +// --------------------------------------------------------------------------- + +/// A shared, clonable "needs re-bootstrap" flag. +/// +/// Raised when the bulk upstream rejects the read/control token (a stale token +/// after a host re-`up`). The agent surfaces it (task 6.1's WS loop emits a +/// `Status` frame); it is never a silent retry loop, and there is no renewal +/// endpoint — the token is refreshed only by a host re-`up` re-bootstrap. +#[derive(Clone, Default)] +pub(crate) struct ReBootstrap(Arc); + +impl ReBootstrap { + /// Raise the flag. Idempotent. + pub(crate) fn raise(&self) { + self.0.store(true, Ordering::SeqCst); + } + + /// Whether a re-bootstrap has been requested. + pub(crate) fn is_raised(&self) -> bool { + self.0.load(Ordering::SeqCst) + } +} + +// --------------------------------------------------------------------------- +// Upstream abstraction (injectable for tests). +// --------------------------------------------------------------------------- + +/// One upstream read response: the status is available BEFORE the body is +/// consumed, which is what lets the proxy fail closed pre-stream. +pub(crate) struct UpstreamResponse { + pub(crate) status: StatusCode, + pub(crate) headers: HeaderMap, + pub(crate) body: RespBody, +} + +/// The bulk-read upstream. Abstracted so the proxy logic is unit-testable with +/// an in-process fake, while [`HttpsUpstream`] carries the real pinned-CA TLS +/// leg used on the device. +pub(crate) trait Upstream: Send + Sync + 'static { + fn fetch( + &self, + method: Method, + path: String, + range: Option, + ) -> impl Future> + Send; +} + +/// The real upstream: a fresh TLS connection per request to the host bulk +/// listener, pinned to the bootstrap CA, carrying the Bearer read/control +/// token. TLS is driven by `tokio-rustls` (ring) directly — never a +/// higher-level client that could drag in a second crypto provider. +pub(crate) struct HttpsUpstream { + connector: TlsConnector, + server_name: ServerName<'static>, + /// `host:port` of the bulk listener — used for the TCP connect and the + /// `Host` header. + authority: String, + read_token: String, +} + +impl HttpsUpstream { + /// Build the upstream from the bootstrap: pins the CA and captures the bulk + /// endpoint + read/control token. Fails if the CA PEM is unusable or the + /// endpoint host is not a valid TLS server name. + pub(crate) fn from_bootstrap(bootstrap: &Bootstrap) -> Result { + let tls_config = build_tls_config(&bootstrap.ca_cert_pem)?; + let connector = TlsConnector::from(Arc::new(tls_config)); + let host = endpoint_host(&bootstrap.bulk_endpoint).to_string(); + let server_name = ServerName::try_from(host).context("invalid bulk TLS server name")?; + Ok(Self { + connector, + server_name, + authority: bootstrap.bulk_endpoint.clone(), + read_token: bootstrap.read_token.clone(), + }) + } +} + +impl Upstream for HttpsUpstream { + async fn fetch( + &self, + method: Method, + path: String, + range: Option, + ) -> Result { + let tcp = TcpStream::connect(&self.authority) + .await + .with_context(|| format!("connecting to bulk listener {}", self.authority))?; + let tls = self + .connector + .connect(self.server_name.clone(), tcp) + .await + .context("bulk TLS handshake (pinned CA)")?; + + let io = TokioIo::new(tls); + let (mut sender, conn) = hyper::client::conn::http1::handshake(io) + .await + .context("bulk HTTP/1 handshake")?; + tokio::spawn(async move { + if let Err(e) = conn.await { + warn!(error = %e, "bulk upstream connection closed with error"); + } + }); + + let mut builder = Request::builder() + .method(method) + .uri(path) + .header(HOST, &self.authority) + .header(AUTHORIZATION, format!("Bearer {}", self.read_token)); + if let Some(range) = range { + builder = builder.header(RANGE, range); + } + let request = builder + .body(Empty::::new()) + .context("building upstream request")?; + + let response = sender + .send_request(request) + .await + .context("sending upstream request")?; + + let status = response.status(); + let headers = response.headers().clone(); + let body = response.into_body().map_err(std::io::Error::other).boxed(); + Ok(UpstreamResponse { + status, + headers, + body, + }) + } +} + +// --------------------------------------------------------------------------- +// Proxy logic. +// --------------------------------------------------------------------------- + +/// Build a small static-body engine response (used for every gateway error). +fn gateway_error(status: StatusCode, message: &'static str) -> Response { + let body = Full::new(Bytes::from_static(message.as_bytes())) + .map_err(|never| match never {}) + .boxed(); + Response::builder() + .status(status) + .header(hyper::header::CONTENT_TYPE, "text/plain; charset=utf-8") + .body(body) + .expect("static gateway error response is always valid") +} + +/// Core read proxy. Operates on already-extracted request parts so it is +/// unit-testable without constructing a hyper `Incoming` body. +/// +/// Fail-closed pre-stream: the upstream status is fully classified before any +/// engine-facing `2xx` (and its body) is opened. +pub(crate) async fn proxy_read( + method: Method, + path: String, + range: Option, + upstream: &U, + rebootstrap: &ReBootstrap, +) -> Response { + // The device only ever reads. Anything else never reaches the host. + if method != Method::GET && method != Method::HEAD { + return gateway_error( + StatusCode::METHOD_NOT_ALLOWED, + "only GET and HEAD are proxied", + ); + } + + match upstream.fetch(method, path, range).await { + Ok(upstream) if upstream.status.is_success() => { + // Only now, with a confirmed 2xx, do we open the engine response + // and stream the body through. Headers are a strict allowlist — + // `WWW-Authenticate` is never among them. + let mut builder = Response::builder().status(upstream.status); + for name in PASSTHROUGH_HEADERS { + if let Some(value) = upstream.headers.get(*name) { + builder = builder.header(*name, value); + } + } + builder + .body(upstream.body) + .expect("engine success response is always valid") + } + Ok(upstream) + if upstream.status == StatusCode::UNAUTHORIZED + || upstream.status == StatusCode::FORBIDDEN => + { + // Fail-closed: no 200 was opened, so no body has streamed. Raise + // the re-bootstrap status and return a CLEAN gateway error that + // leaks neither the upstream 401/403 nor its WWW-Authenticate. + rebootstrap.raise(); + warn!( + upstream_status = %upstream.status, + "bulk upstream rejected the read/control token; raising re-bootstrap status" + ); + gateway_error(StatusCode::BAD_GATEWAY, REBOOTSTRAP_MSG) + } + Ok(upstream) => { + // Any other non-2xx is treated as transient. Surface a gateway + // error the engine may retry; do NOT raise re-bootstrap. + warn!(upstream_status = %upstream.status, "bulk upstream returned a transient error"); + gateway_error(StatusCode::BAD_GATEWAY, "bulk upstream returned an error") + } + Err(e) => { + warn!(error = %e, "bulk upstream request failed"); + gateway_error(StatusCode::BAD_GATEWAY, "bulk upstream unreachable") + } + } +} + +// --------------------------------------------------------------------------- +// Loopback listener. +// --------------------------------------------------------------------------- + +/// Bind the loopback registry socket. Always `127.0.0.1` (never `0.0.0.0`): the +/// registry is device-local only, which is what keeps it trust-free for the +/// engine and unreachable from the LAN. +pub(crate) async fn bind_loopback(port: u16) -> Result { + let addr = SocketAddr::from((Ipv4Addr::LOCALHOST, port)); + TcpListener::bind(addr) + .await + .with_context(|| format!("binding loopback registry on {addr}")) +} + +/// Serve the loopback registry over `listener`, forwarding every read to +/// `upstream`. Runs until the listener errors. +pub(crate) async fn serve( + listener: TcpListener, + upstream: Arc, + rebootstrap: ReBootstrap, +) -> Result<()> { + loop { + let (stream, _peer) = listener + .accept() + .await + .context("accepting loopback connection")?; + let upstream = upstream.clone(); + let rebootstrap = rebootstrap.clone(); + tokio::spawn(async move { + let io = TokioIo::new(stream); + let service = service_fn(move |req: Request| { + let upstream = upstream.clone(); + let rebootstrap = rebootstrap.clone(); + async move { + let method = req.method().clone(); + let path = req + .uri() + .path_and_query() + .map(|pq| pq.as_str().to_owned()) + .unwrap_or_else(|| req.uri().path().to_owned()); + let range = req.headers().get(RANGE).cloned(); + Ok::<_, Infallible>( + proxy_read(method, path, range, upstream.as_ref(), &rebootstrap).await, + ) + } + }); + if let Err(e) = hyper::server::conn::http1::Builder::new() + .serve_connection(io, service) + .await + { + warn!(error = %e, "loopback registry connection error"); + } + }); + } +} + +/// Bind `127.0.0.1:port` and serve the loopback registry. The real device +/// entry point (called from `main`). +pub(crate) async fn serve_loopback( + port: u16, + upstream: Arc, + rebootstrap: ReBootstrap, +) -> Result<()> { + let listener = bind_loopback(port).await?; + match listener.local_addr() { + Ok(addr) => info!(%addr, "loopback registry proxy listening"), + Err(_) => info!("loopback registry proxy listening"), + } + serve(listener, upstream, rebootstrap).await +} + +#[cfg(test)] +mod tests { + use super::*; + + use std::sync::Arc; + + use http_body_util::BodyExt; + use hyper::body::Incoming; + use hyper::service::service_fn; + use rcgen::{BasicConstraints, CertificateParams, IsCa, KeyPair}; + use tokio::net::TcpStream; + use tokio_rustls::TlsAcceptor; + use tokio_rustls::rustls::pki_types::{ + CertificateDer, PrivateKeyDer, PrivatePkcs8KeyDer, ServerName, + }; + + // --- helpers ----------------------------------------------------------- + + fn full(bytes: impl Into) -> RespBody { + Full::new(bytes.into()) + .map_err(|never| match never {}) + .boxed() + } + + /// A CA plus a leaf it signed for `127.0.0.1`. + struct Ca { + ca_pem: String, + leaf_cert: CertificateDer<'static>, + leaf_key: PrivateKeyDer<'static>, + } + + fn make_ca() -> Ca { + let mut ca_params = CertificateParams::new(Vec::new()).expect("ca params"); + ca_params.is_ca = IsCa::Ca(BasicConstraints::Unconstrained); + let ca_key = KeyPair::generate().expect("ca key"); + let ca_cert = ca_params.self_signed(&ca_key).expect("self-sign ca"); + + let leaf_params = + CertificateParams::new(vec!["127.0.0.1".to_string()]).expect("leaf params"); + let leaf_key = KeyPair::generate().expect("leaf key"); + let leaf_cert = leaf_params + .signed_by(&leaf_key, &ca_cert, &ca_key) + .expect("sign leaf"); + + Ca { + ca_pem: ca_cert.pem(), + leaf_cert: leaf_cert.der().clone(), + leaf_key: PrivateKeyDer::from(PrivatePkcs8KeyDer::from(leaf_key.serialize_der())), + } + } + + /// Start an in-process TLS "host bulk listener" presenting `cert`/`key`. + /// A path containing `unauth` returns 401 + `WWW-Authenticate`; anything + /// else returns 200 with a known body. + async fn start_tls_upstream( + cert: CertificateDer<'static>, + key: PrivateKeyDer<'static>, + ) -> SocketAddr { + let provider = Arc::new(tokio_rustls::rustls::crypto::ring::default_provider()); + let config = tokio_rustls::rustls::ServerConfig::builder_with_provider(provider) + .with_safe_default_protocol_versions() + .expect("server protocol versions") + .with_no_client_auth() + .with_single_cert(vec![cert], key) + .expect("server single cert"); + let acceptor = TlsAcceptor::from(Arc::new(config)); + + let listener = TcpListener::bind((Ipv4Addr::LOCALHOST, 0)) + .await + .expect("bind tls upstream"); + let addr = listener.local_addr().expect("upstream addr"); + + tokio::spawn(async move { + loop { + let (tcp, _) = match listener.accept().await { + Ok(pair) => pair, + Err(_) => return, + }; + let acceptor = acceptor.clone(); + tokio::spawn(async move { + let tls = match acceptor.accept(tcp).await { + Ok(tls) => tls, + Err(_) => return, + }; + let io = TokioIo::new(tls); + let service = service_fn(|req: Request| async move { + let resp = if req.uri().path().contains("unauth") { + Response::builder() + .status(StatusCode::UNAUTHORIZED) + .header("www-authenticate", "Bearer realm=\"https://evil/token\"") + .body(full("denied")) + .unwrap() + } else { + Response::builder() + .status(StatusCode::OK) + .header("content-type", "application/octet-stream") + .header("docker-content-digest", "sha256:deadbeef") + .body(full("BLOBDATA")) + .unwrap() + }; + Ok::<_, Infallible>(resp) + }); + let _ = hyper::server::conn::http1::Builder::new() + .serve_connection(io, service) + .await; + }); + } + }); + + addr + } + + /// A plain-HTTP (no TLS) GET against the loopback registry, standing in for + /// the device engine. Returns status, headers, and the collected body. + async fn plain_get(addr: SocketAddr, path: &str) -> (StatusCode, HeaderMap, Bytes) { + let tcp = TcpStream::connect(addr).await.expect("connect loopback"); + let io = TokioIo::new(tcp); + let (mut sender, conn) = hyper::client::conn::http1::handshake(io) + .await + .expect("loopback handshake"); + tokio::spawn(async move { + let _ = conn.await; + }); + let request = Request::builder() + .method(Method::GET) + .uri(path) + .header(HOST, addr.to_string()) + .body(Empty::::new()) + .expect("loopback request"); + let response = sender + .send_request(request) + .await + .expect("loopback response"); + let status = response.status(); + let headers = response.headers().clone(); + let body = response + .into_body() + .collect() + .await + .expect("collect") + .to_bytes(); + (status, headers, body) + } + + /// A fake upstream returning a canned response, for the pure proxy-logic + /// tests that need no network. + struct FakeUpstream { + status: StatusCode, + headers: Vec<(&'static str, &'static str)>, + body: &'static str, + } + + impl Upstream for FakeUpstream { + async fn fetch( + &self, + _method: Method, + _path: String, + _range: Option, + ) -> Result { + let mut headers = HeaderMap::new(); + for (name, value) in &self.headers { + headers.insert( + hyper::header::HeaderName::from_static(name), + HeaderValue::from_static(value), + ); + } + Ok(UpstreamResponse { + status: self.status, + headers, + body: full(self.body), + }) + } + } + + async fn collect(resp: Response) -> (StatusCode, HeaderMap, Bytes) { + let status = resp.status(); + let headers = resp.headers().clone(); + let body = resp + .into_body() + .collect() + .await + .expect("collect") + .to_bytes(); + (status, headers, body) + } + + // --- proxy-logic tests ------------------------------------------------- + + #[tokio::test] + async fn proxy_forwards_upstream_200_body_verbatim() { + let upstream = FakeUpstream { + status: StatusCode::OK, + headers: vec![ + ("content-type", "application/octet-stream"), + ("docker-content-digest", "sha256:abc"), + ], + body: "LAYER-BYTES", + }; + let rebootstrap = ReBootstrap::default(); + let resp = proxy_read( + Method::GET, + "/v2/app/blobs/sha256:abc".to_string(), + None, + &upstream, + &rebootstrap, + ) + .await; + let (status, headers, body) = collect(resp).await; + + assert_eq!(status, StatusCode::OK); + assert_eq!(&body[..], b"LAYER-BYTES"); + assert_eq!( + headers.get("content-type").unwrap(), + "application/octet-stream" + ); + assert_eq!(headers.get("docker-content-digest").unwrap(), "sha256:abc"); + assert!(!rebootstrap.is_raised()); + } + + #[tokio::test] + async fn proxy_401_returns_clean_error_no_www_authenticate_and_raises_rebootstrap() { + let upstream = FakeUpstream { + status: StatusCode::UNAUTHORIZED, + headers: vec![("www-authenticate", "Bearer realm=\"https://evil/token\"")], + body: "denied", + }; + let rebootstrap = ReBootstrap::default(); + let resp = proxy_read( + Method::GET, + "/v2/app/blobs/sha256:abc".to_string(), + None, + &upstream, + &rebootstrap, + ) + .await; + let (status, headers, body) = collect(resp).await; + + // Engine never sees a 401 — it gets a clean gateway error. + assert_ne!(status, StatusCode::UNAUTHORIZED); + assert_eq!(status, StatusCode::BAD_GATEWAY); + // The upstream WWW-Authenticate is never proxied back. + assert!(headers.get("www-authenticate").is_none()); + // The clean body carries no upstream auth semantics. + assert_eq!(&body[..], REBOOTSTRAP_MSG.as_bytes()); + // The re-bootstrap status is raised for the agent to surface. + assert!(rebootstrap.is_raised()); + } + + #[tokio::test] + async fn proxy_transient_5xx_is_gateway_error_without_rebootstrap() { + let upstream = FakeUpstream { + status: StatusCode::SERVICE_UNAVAILABLE, + headers: vec![], + body: "later", + }; + let rebootstrap = ReBootstrap::default(); + let resp = proxy_read( + Method::GET, + "/v2/app/manifests/dev".to_string(), + None, + &upstream, + &rebootstrap, + ) + .await; + let (status, _headers, _body) = collect(resp).await; + + assert_eq!(status, StatusCode::BAD_GATEWAY); + // A transient error is retryable, not a terminal re-bootstrap signal. + assert!(!rebootstrap.is_raised()); + } + + // --- pinned-CA tests --------------------------------------------------- + + #[tokio::test] + async fn pinned_ca_accepts_matching_leaf_and_streams_200() { + let ca = make_ca(); + let addr = start_tls_upstream(ca.leaf_cert.clone(), ca.leaf_key.clone_key()).await; + + let bootstrap = Bootstrap { + bulk_endpoint: addr.to_string(), + read_token: "read-token".to_string(), + ca_cert_pem: ca.ca_pem.clone(), + ws_endpoint: String::new(), + }; + let upstream = HttpsUpstream::from_bootstrap(&bootstrap).expect("build upstream"); + + let resp = upstream + .fetch(Method::GET, "/v2/app/blobs/sha256:abc".to_string(), None) + .await + .expect("pinned CA accepts its own leaf"); + assert!(resp.status.is_success()); + let body = resp.body.collect().await.expect("collect").to_bytes(); + assert_eq!(&body[..], b"BLOBDATA"); + } + + #[tokio::test] + async fn pinned_ca_rejects_leaf_from_a_different_ca() { + // Server presents a leaf signed by CA "B"... + let server_ca = make_ca(); + let addr = + start_tls_upstream(server_ca.leaf_cert.clone(), server_ca.leaf_key.clone_key()).await; + + // ...but the client pins a DIFFERENT CA "A". + let client_ca = make_ca(); + let bootstrap = Bootstrap { + bulk_endpoint: addr.to_string(), + read_token: "read-token".to_string(), + ca_cert_pem: client_ca.ca_pem.clone(), + ws_endpoint: String::new(), + }; + let upstream = HttpsUpstream::from_bootstrap(&bootstrap).expect("build upstream"); + + let result = upstream + .fetch(Method::GET, "/v2/app/blobs/sha256:abc".to_string(), None) + .await; + assert!( + result.is_err(), + "handshake must fail: a host leaf not signed by the bootstrap CA is rejected" + ); + } + + // --- end-to-end loopback test ------------------------------------------ + + #[tokio::test] + async fn loopback_is_plain_http_and_proxies_over_tls_upstream() { + let ca = make_ca(); + let upstream_addr = start_tls_upstream(ca.leaf_cert.clone(), ca.leaf_key.clone_key()).await; + + let bootstrap = Bootstrap { + bulk_endpoint: upstream_addr.to_string(), + read_token: "read-token".to_string(), + ca_cert_pem: ca.ca_pem.clone(), + ws_endpoint: String::new(), + }; + let upstream = Arc::new(HttpsUpstream::from_bootstrap(&bootstrap).expect("build upstream")); + let rebootstrap = ReBootstrap::default(); + + let listener = bind_loopback(0).await.expect("bind loopback"); + let loopback_addr = listener.local_addr().expect("loopback addr"); + // The loopback socket must be 127.0.0.1, never 0.0.0.0. + assert!(loopback_addr.ip().is_loopback()); + + tokio::spawn(serve(listener, upstream, rebootstrap.clone())); + + // The engine speaks PLAIN HTTP to the loopback with zero TLS/trust + // config; the proxy forwards over the pinned-CA HTTPS upstream. + let (status, headers, body) = plain_get(loopback_addr, "/v2/app/blobs/sha256:abc").await; + assert_eq!(status, StatusCode::OK); + assert_eq!(&body[..], b"BLOBDATA"); + assert_eq!( + headers.get("docker-content-digest").unwrap(), + "sha256:deadbeef" + ); + assert!(!rebootstrap.is_raised()); + + // An upstream 401 surfaces as a clean gateway error with no + // WWW-Authenticate, and raises the re-bootstrap status. + let (status, headers, _body) = plain_get(loopback_addr, "/v2/app/blobs/unauth").await; + assert_eq!(status, StatusCode::BAD_GATEWAY); + assert!(headers.get("www-authenticate").is_none()); + assert!(rebootstrap.is_raised()); + } + + #[tokio::test] + async fn bind_loopback_binds_localhost_not_wildcard() { + let listener = bind_loopback(0).await.expect("bind loopback"); + let addr = listener.local_addr().expect("addr"); + assert!(addr.ip().is_loopback()); + assert_eq!(addr.ip(), std::net::IpAddr::V4(Ipv4Addr::LOCALHOST)); + } + + // Guard for ServerName import usage in non-test builds is unnecessary; keep + // the import honest by asserting a loopback IP parses as a server name. + #[test] + fn loopback_ip_is_a_valid_server_name() { + assert!(ServerName::try_from("127.0.0.1").is_ok()); + } +} From 9820ce77b55567b95e74b8de122715ed1fda303b Mon Sep 17 00:00:00 2001 From: Javier Tia Date: Wed, 22 Jul 2026 08:24:31 -0600 Subject: [PATCH 03/20] container-agent-dev: Implement sync handler with ordered pull/restart Currently the Sync frame handler in the container dev-mode agent is a stub that logs the received fields but performs no actual work. Without a real implementation, the device can never apply a new container image when the host requests a sync: the pull is skipped, the active-image pointer on the writable partition is never updated, and the container continues running the old image indefinitely. Introduce a sync module that implements the full ordered sequence for task 6.3: pull the new image through the device loopback proxy, rewrite the active-image pointer JSON on the writable partition, restart the container, and update the in-memory running digest so the next reconnect Hello reports the correct image. The ordering is load-bearing: a pull failure aborts the entire sequence before any state is mutated, ensuring the device never claims a new image is running when it is not. The writable partition root is structurally enforced at the type level so pointer writes can never land on a read-only rootFS path. The container engine interaction is hidden behind an Engine trait, mirroring the proxy Upstream pattern from task 6.2. The production CommandEngine shells out to docker or podman via tokio::process and is the only untested portion; the ordering, pointer round-trip, digest update, and pull-failure abort paths are all covered by unit tests using an in-process FakeEngine. The tempfile crate is added as a dev-dependency to supply a writable temporary directory in those tests. Signed-off-by: Javier Tia --- Cargo.lock | 66 ++++++++ Cargo.toml | 3 + src/main.rs | 21 ++- src/sync.rs | 445 ++++++++++++++++++++++++++++++++++++++++++++++++++++ 4 files changed, 528 insertions(+), 7 deletions(-) create mode 100644 src/sync.rs diff --git a/Cargo.lock b/Cargo.lock index 7854bca..cbf96e0 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -38,6 +38,7 @@ dependencies = [ "rustls-pemfile", "serde", "serde_json", + "tempfile", "tokio", "tokio-rustls", "tokio-tungstenite", @@ -51,6 +52,12 @@ version = "0.22.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "72b3254f16251a8381aa12e40e3c4d2f0199f8c6508fbecb9d91f575e0fbb8c6" +[[package]] +name = "bitflags" +version = "2.13.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b588b76d00fde79687d7646a9b5bdf3cc0f655e0bbd080335a95d7e96f3587da" + [[package]] name = "block-buffer" version = "0.10.4" @@ -123,6 +130,22 @@ dependencies = [ "crypto-common", ] +[[package]] +name = "errno" +version = "0.3.14" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "39cab71617ae0d63f51a36d69f866391735b51691dbda63cf6f96d042b63efeb" +dependencies = [ + "libc", + "windows-sys 0.61.2", +] + +[[package]] +name = "fastrand" +version = "2.5.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "da7c62ceae207dd37ea5b845da6a0696c799f85e97da1ab5b7910be3c1c80223" + [[package]] name = "find-msvc-tools" version = "0.1.9" @@ -312,6 +335,12 @@ version = "0.2.189" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "3eaf3ede3fee6db1a4c2ee091bf8a8b4dccdc6d17f656fb07896ee72867612f2" +[[package]] +name = "linux-raw-sys" +version = "0.12.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "32a66949e030da00e8c7d4434b251670a91556f4144941d37452769c25d58a53" + [[package]] name = "log" version = "0.4.33" @@ -493,6 +522,19 @@ dependencies = [ "windows-sys 0.52.0", ] +[[package]] +name = "rustix" +version = "1.1.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b6fe4565b9518b83ef4f91bb47ce29620ca828bd32cb7e408f0062e9930ba190" +dependencies = [ + "bitflags", + "errno", + "libc", + "linux-raw-sys", + "windows-sys 0.61.2", +] + [[package]] name = "rustls" version = "0.23.42" @@ -606,6 +648,16 @@ version = "2.0.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "f8fadd59c855ef2080decdef8ff161eb6661b86933c9d82e5ba29dc602a55aba" +[[package]] +name = "signal-hook-registry" +version = "1.4.8" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c4db69cba1110affc0e9f7bcd48bbf87b3f4fc7c61fc9155afd4c469eb3d6c1b" +dependencies = [ + "errno", + "libc", +] + [[package]] name = "slab" version = "0.4.12" @@ -656,6 +708,19 @@ dependencies = [ "unicode-ident", ] +[[package]] +name = "tempfile" +version = "3.27.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "32497e9a4c7b38532efcdebeef879707aa9f794296a4f0244f6f69e9bc8574bd" +dependencies = [ + "fastrand", + "getrandom 0.3.4", + "once_cell", + "rustix", + "windows-sys 0.61.2", +] + [[package]] name = "thiserror" version = "2.0.19" @@ -714,6 +779,7 @@ dependencies = [ "libc", "mio", "pin-project-lite", + "signal-hook-registry", "socket2", "tokio-macros", "windows-sys 0.61.2", diff --git a/Cargo.toml b/Cargo.toml index 830d1cd..daa28f5 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -14,6 +14,7 @@ tokio = { version = "1", features = [ "net", "time", "io-util", + "process", ] } # WS client only. We deliberately DO NOT enable tokio-tungstenite's # `rustls-tls-*` features: those pull `tokio-rustls` with its default features, @@ -68,6 +69,8 @@ rcgen = { version = "0.13", default-features = false, features = [ "pem", "ring", ] } +# Temp dir standing in for the writable partition in the sync pointer tests. +tempfile = "3" [profile.release] strip = true diff --git a/src/main.rs b/src/main.rs index 077d420..e816e01 100644 --- a/src/main.rs +++ b/src/main.rs @@ -36,6 +36,7 @@ use tokio_tungstenite::tungstenite::http::header::AUTHORIZATION; use tracing::{info, warn}; mod proxy; +mod sync; /// Default on-device path the host writes the bootstrap file to (design D5 / A7). const DEFAULT_BOOTSTRAP_PATH: &str = "/var/lib/avocado/container-dev/bootstrap.json"; @@ -309,15 +310,21 @@ pub(crate) fn endpoint_host(endpoint: &str) -> &str { // Connect + serve. // --------------------------------------------------------------------------- -/// Handle one host frame. Task 6.1 logs and holds; the pull + restart is the -/// task 6.3 seam. -async fn handle_host_frame(frame: HostFrame, _state: &AgentState) { +/// Handle one host frame. On a `Sync`, pull the new image through the loopback +/// proxy, rewrite the active-image pointer on the writable partition, restart +/// the container, and record the running digest (task 6.3, [`sync::on_sync`]). +async fn handle_host_frame(frame: HostFrame, state: &AgentState) { match frame { HostFrame::Sync { image, tag, digest } => { - info!(%image, %tag, %digest, "received sync (pull + restart is task 6.3)"); - // SEAM (task 6.3): pull `image:tag@digest` through the loopback - // registry proxy, restart the container, update - // `state.running_digest`, and send a `Status` frame. + info!(%image, %tag, %digest, "received sync; pulling + restarting"); + // The engine shell-out and env-resolved config are the thin, + // untested production glue; the ordering/pointer/digest core lives + // in the fully-tested `sync::on_sync`. + let engine = sync::CommandEngine::from_env(); + let cfg = sync::SyncConfig::from_env(); + if let Err(e) = sync::on_sync(&engine, &cfg, &image, &tag, &digest, state).await { + warn!(error = %e, %image, %tag, %digest, "sync failed"); + } } } } diff --git a/src/sync.rs b/src/sync.rs new file mode 100644 index 0000000..46f0a01 --- /dev/null +++ b/src/sync.rs @@ -0,0 +1,445 @@ +//! Device-side sync handler for Container Dev Mode (design D9; task 6.3). +//! +//! On a [`crate::HostFrame::Sync`] the agent, IN ORDER: +//! 1. **Pulls** the new image through the device loopback proxy +//! (`127.0.0.1:`, task 6.2). The pull rides the proxy -> host bulk +//! HTTPS path; it NEVER re-pulls over the control WebSocket (D9 splits +//! bulk from control). +//! 2. **Rewrites the active-image pointer** — a small JSON file recording the +//! now-active `{image, tag, digest}` — on the **writable partition** +//! (`/var/lib/avocado/container-dev/active-image.json`). The path is +//! derived from [`WRITABLE_ROOT`], never a read-only rootFS path +//! (`/usr`, `/etc`, the image rootfs). +//! 3. **Restarts the container** via the engine AFTER the pull + pointer +//! rewrite, so the new image takes effect. The read-only rootFS is +//! untouched throughout. +//! 4. Updates [`crate::AgentState::running_digest`] so task 6.1's reconnect +//! `Hello` reports it. +//! +//! The engine interaction is behind the [`Engine`] trait (mirroring task 6.2's +//! [`crate::proxy::Upstream`] pattern) so the ordering / pointer / digest logic +//! is unit-tested with a fake. The production [`CommandEngine`] shells out to +//! docker/podman via `tokio::process::Command` and is the only untested part — +//! no new TLS/HTTP deps are pulled in for the shell-out, keeping `aws-lc-rs` +//! out of the tree (A9). + +use std::future::Future; +use std::path::{Path, PathBuf}; + +use anyhow::{Context, Result, ensure}; +use serde::{Deserialize, Serialize}; +use tokio::process::Command; +use tracing::info; + +use crate::AgentState; + +/// The writable-partition root for on-device mutable state. Consistent with +/// `main.rs`'s bootstrap root (`DEFAULT_BOOTSTRAP_PATH` lives under this same +/// directory). NEVER a read-only rootFS path. +pub(crate) const WRITABLE_ROOT: &str = "/var/lib/avocado/container-dev"; +/// The active-image pointer file name, written under [`WRITABLE_ROOT`]. +const ACTIVE_IMAGE_FILE: &str = "active-image.json"; +/// Default container engine binary when unset (docker; podman via the env var). +const DEFAULT_ENGINE: &str = "docker"; +/// Default container name to restart when unset. +const DEFAULT_CONTAINER: &str = "avocado-dev"; +/// Env var selecting the container engine binary (`docker` or `podman`). +const ENGINE_ENV: &str = "AVOCADO_CONTAINER_DEV_ENGINE"; +/// Env var selecting the container name to restart on sync. +const CONTAINER_ENV: &str = "AVOCADO_CONTAINER_DEV_CONTAINER"; +/// Env var overriding the writable state root (used by tests and staging). +const STATE_ROOT_ENV: &str = "AVOCADO_CONTAINER_DEV_STATE_ROOT"; + +// --------------------------------------------------------------------------- +// Active-image pointer. +// --------------------------------------------------------------------------- + +/// The active-image pointer: which `{image, tag, digest}` the device now runs. +/// Serialized to `active-image.json` on the writable partition. +#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)] +pub(crate) struct ActiveImage { + pub image: String, + pub tag: String, + pub digest: String, +} + +/// The active-image pointer path, derived from `writable_root`. The join makes +/// it structurally impossible to land on a rootFS constant: the caller supplies +/// the root, and production always supplies [`WRITABLE_ROOT`]. +fn active_image_path(writable_root: &Path) -> PathBuf { + writable_root.join(ACTIVE_IMAGE_FILE) +} + +/// Write the active-image pointer under `writable_root`, creating the directory +/// if needed. Returns the path written. +fn write_active_image(writable_root: &Path, active: &ActiveImage) -> Result { + let path = active_image_path(writable_root); + if let Some(parent) = path.parent() { + std::fs::create_dir_all(parent) + .with_context(|| format!("creating writable state dir {}", parent.display()))?; + } + let json = serde_json::to_string_pretty(active).context("serializing active-image pointer")?; + std::fs::write(&path, json) + .with_context(|| format!("writing active-image pointer {}", path.display()))?; + Ok(path) +} + +/// The digest-pinned loopback registry reference the engine pulls. The device +/// engine auto-trusts `127.0.0.1`, so this rides the plain-HTTP loopback proxy +/// (task 6.2) which forwards to the host bulk HTTPS listener. +fn loopback_reference(port: u16, image: &str, digest: &str) -> String { + format!("127.0.0.1:{port}/{image}@{digest}") +} + +// --------------------------------------------------------------------------- +// Engine abstraction (injectable for tests). +// --------------------------------------------------------------------------- + +/// The container engine: pull an image reference, restart a container. Async so +/// the production impl can shell out without blocking the runtime; abstracted so +/// the sync ordering / pointer / digest logic is unit-testable with a fake. +pub(crate) trait Engine: Send + Sync + 'static { + fn pull(&self, reference: &str) -> impl Future> + Send; + fn restart(&self, container: &str) -> impl Future> + Send; +} + +/// The real engine: shells out to the docker/podman CLI. This is the only +/// untested part of task 6.3 (it execs a subprocess); the pure [`on_sync`] core +/// above it is fully covered. +pub(crate) struct CommandEngine { + binary: String, +} + +impl CommandEngine { + /// Resolve the engine binary from `$AVOCADO_CONTAINER_DEV_ENGINE`, defaulting + /// to `docker`. + pub(crate) fn from_env() -> Self { + let binary = std::env::var(ENGINE_ENV) + .ok() + .map(|v| v.trim().to_string()) + .filter(|v| !v.is_empty()) + .unwrap_or_else(|| DEFAULT_ENGINE.to_string()); + Self { binary } + } +} + +impl Engine for CommandEngine { + async fn pull(&self, reference: &str) -> Result<()> { + run_engine(&self.binary, &["pull", reference]).await + } + + async fn restart(&self, container: &str) -> Result<()> { + run_engine(&self.binary, &["restart", container]).await + } +} + +/// Run ` ` to completion, mapping a non-zero exit to an error. +async fn run_engine(binary: &str, args: &[&str]) -> Result<()> { + let status = Command::new(binary) + .args(args) + .status() + .await + .with_context(|| format!("spawning `{binary} {}`", args.join(" ")))?; + ensure!( + status.success(), + "`{binary} {}` exited with {status}", + args.join(" ") + ); + Ok(()) +} + +// --------------------------------------------------------------------------- +// Sync config + core. +// --------------------------------------------------------------------------- + +/// The device-side sync configuration: the loopback port the engine pulls from, +/// the container to restart, and the writable state root the pointer lands in. +pub(crate) struct SyncConfig { + pub loopback_port: u16, + pub container: String, + pub writable_root: PathBuf, +} + +impl SyncConfig { + /// Resolve from env: loopback port shares `main.rs`'s resolver; the writable + /// root defaults to [`WRITABLE_ROOT`] (`$AVOCADO_CONTAINER_DEV_STATE_ROOT` + /// overrides for tests/staging); the container defaults to `avocado-dev` + /// (`$AVOCADO_CONTAINER_DEV_CONTAINER` overrides). + pub(crate) fn from_env() -> Self { + let writable_root = std::env::var_os(STATE_ROOT_ENV) + .map(PathBuf::from) + .unwrap_or_else(|| PathBuf::from(WRITABLE_ROOT)); + let container = std::env::var(CONTAINER_ENV) + .ok() + .map(|v| v.trim().to_string()) + .filter(|v| !v.is_empty()) + .unwrap_or_else(|| DEFAULT_CONTAINER.to_string()); + Self { + loopback_port: crate::loopback_port(), + container, + writable_root, + } + } +} + +/// Handle a `Sync`: pull -> rewrite pointer -> restart -> record digest. +/// +/// The order is load-bearing: a pull failure aborts before the pointer rewrite, +/// the restart, and the digest update, so a failed sync never claims a new +/// image is running and never mutates state. All writes target the writable +/// partition; the read-only rootFS is never touched. +pub(crate) async fn on_sync( + engine: &E, + cfg: &SyncConfig, + image: &str, + tag: &str, + digest: &str, + state: &AgentState, +) -> Result<()> { + // 1. Pull through the device loopback proxy (never over the control WS). + let reference = loopback_reference(cfg.loopback_port, image, digest); + engine + .pull(&reference) + .await + .with_context(|| format!("pulling {reference} through loopback proxy"))?; + + // 2. Rewrite the active-image pointer on the WRITABLE partition. The path is + // derived from cfg.writable_root, never a read-only rootFS path. + let active = ActiveImage { + image: image.to_string(), + tag: tag.to_string(), + digest: digest.to_string(), + }; + let path = write_active_image(&cfg.writable_root, &active) + .context("rewriting active-image pointer")?; + info!(pointer = %path.display(), "active-image pointer rewritten on writable partition"); + + // 3. Restart the container AFTER the pull + pointer rewrite so the new image + // takes effect. + engine + .restart(&cfg.container) + .await + .with_context(|| format!("restarting container {}", cfg.container))?; + + // 4. Record the now-running digest so the reconnect Hello reports it. + *state + .running_digest + .lock() + .expect("running_digest mutex poisoned") = digest.to_string(); + + info!(%image, %tag, %digest, "sync complete: pulled, pointer rewritten, container restarted"); + Ok(()) +} + +#[cfg(test)] +mod tests { + use super::*; + + use std::sync::{Arc, Mutex}; + + /// An in-process engine that records its ordered calls; optionally fails the + /// pull to exercise the abort path. Never execs a real engine. + struct FakeEngine { + calls: Arc>>, + fail_pull: bool, + } + + impl FakeEngine { + fn new(calls: Arc>>) -> Self { + Self { + calls, + fail_pull: false, + } + } + + fn failing_pull(calls: Arc>>) -> Self { + Self { + calls, + fail_pull: true, + } + } + } + + impl Engine for FakeEngine { + async fn pull(&self, reference: &str) -> Result<()> { + self.calls.lock().unwrap().push(format!("pull:{reference}")); + if self.fail_pull { + anyhow::bail!("simulated pull failure"); + } + Ok(()) + } + + async fn restart(&self, container: &str) -> Result<()> { + self.calls + .lock() + .unwrap() + .push(format!("restart:{container}")); + Ok(()) + } + } + + fn test_cfg(writable_root: PathBuf) -> SyncConfig { + SyncConfig { + loopback_port: 15151, + container: "avocado-dev".to_string(), + writable_root, + } + } + + // Falsifier 1: a sync must restart the container AFTER the pull. This test + // asserts BOTH happened and that pull strictly precedes restart, so an + // implementation that skipped the restart (or reordered it) fails here. + #[tokio::test] + async fn sync_pulls_then_restarts_in_order() { + let calls = Arc::new(Mutex::new(Vec::new())); + let engine = FakeEngine::new(calls.clone()); + let tmp = tempfile::tempdir().unwrap(); + let cfg = test_cfg(tmp.path().to_path_buf()); + let state = crate::AgentState::new("dev-01".to_string()); + + on_sync(&engine, &cfg, "my-app", "dev", "sha256:new", &state) + .await + .expect("sync succeeds"); + + let recorded = calls.lock().unwrap().clone(); + assert_eq!( + recorded.len(), + 2, + "expected exactly pull then restart, got {recorded:?}" + ); + assert!( + recorded[0].starts_with("pull:"), + "first call must be the pull: {recorded:?}" + ); + assert!( + recorded[1].starts_with("restart:"), + "second call must be the restart: {recorded:?}" + ); + assert_eq!(recorded[0], "pull:127.0.0.1:15151/my-app@sha256:new"); + assert_eq!(recorded[1], "restart:avocado-dev"); + } + + // Falsifier 2: the pointer lands on the writable partition, never a rootFS + // path, and round-trips {image, tag, digest}. + #[tokio::test] + async fn sync_writes_pointer_under_writable_root_and_round_trips() { + let calls = Arc::new(Mutex::new(Vec::new())); + let engine = FakeEngine::new(calls); + let tmp = tempfile::tempdir().unwrap(); + let root = tmp.path().to_path_buf(); + let cfg = test_cfg(root.clone()); + let state = crate::AgentState::new("dev-01".to_string()); + + on_sync(&engine, &cfg, "my-app", "dev", "sha256:new", &state) + .await + .expect("sync succeeds"); + + let pointer = active_image_path(&root); + assert!( + pointer.starts_with(&root), + "pointer must live under the writable root: {}", + pointer.display() + ); + assert!(pointer.exists(), "pointer file must be written"); + + let raw = std::fs::read_to_string(&pointer).unwrap(); + let back: ActiveImage = serde_json::from_str(&raw).unwrap(); + assert_eq!( + back, + ActiveImage { + image: "my-app".to_string(), + tag: "dev".to_string(), + digest: "sha256:new".to_string(), + } + ); + } + + // The pointer path is derived from the writable root, not a rootFS constant. + // The production root is on the writable partition and matches main.rs's + // bootstrap root. + #[test] + fn active_image_path_derives_from_writable_root_never_rootfs() { + assert!( + WRITABLE_ROOT.starts_with("/var/lib/avocado"), + "writable root must be on the writable partition" + ); + for ro in ["/usr", "/etc"] { + assert!( + !WRITABLE_ROOT.starts_with(ro), + "writable root must not be under the read-only rootFS path {ro}" + ); + } + let path = active_image_path(Path::new(WRITABLE_ROOT)); + assert!( + path.starts_with(WRITABLE_ROOT), + "pointer path must be derived from the writable root" + ); + assert_eq!(path.file_name().unwrap(), ACTIVE_IMAGE_FILE); + // Same writable partition as main.rs's bootstrap root. + assert!( + Path::new(crate::DEFAULT_BOOTSTRAP_PATH).starts_with(WRITABLE_ROOT), + "pointer root must match main.rs's bootstrap root partition" + ); + } + + // Falsifier 3 (positive): after a successful sync the running digest is the + // synced digest. + #[tokio::test] + async fn sync_updates_running_digest_on_success() { + let calls = Arc::new(Mutex::new(Vec::new())); + let engine = FakeEngine::new(calls); + let tmp = tempfile::tempdir().unwrap(); + let cfg = test_cfg(tmp.path().to_path_buf()); + let state = crate::AgentState::new("dev-01".to_string()); + + on_sync(&engine, &cfg, "my-app", "dev", "sha256:new", &state) + .await + .expect("sync succeeds"); + + assert_eq!(*state.running_digest.lock().unwrap(), "sha256:new"); + } + + // Edge: a pull failure aborts before the restart, does not write the + // pointer, and does not rewrite the running digest. + #[tokio::test] + async fn pull_failure_aborts_before_restart_and_keeps_digest() { + let calls = Arc::new(Mutex::new(Vec::new())); + let engine = FakeEngine::failing_pull(calls.clone()); + let tmp = tempfile::tempdir().unwrap(); + let cfg = test_cfg(tmp.path().to_path_buf()); + let state = crate::AgentState::new("dev-01".to_string()); + // A prior digest that must survive an aborted sync unchanged. + *state.running_digest.lock().unwrap() = "sha256:old".to_string(); + + let err = on_sync(&engine, &cfg, "my-app", "dev", "sha256:new", &state) + .await + .expect_err("pull failure must abort the sync"); + assert!( + err.to_string().contains("loopback proxy"), + "unexpected error: {err}" + ); + + let recorded = calls.lock().unwrap().clone(); + assert_eq!( + recorded.len(), + 1, + "only the pull should have been attempted: {recorded:?}" + ); + assert!(recorded[0].starts_with("pull:")); + assert!( + !active_image_path(tmp.path()).exists(), + "pointer must not be written when the pull fails" + ); + assert_eq!( + *state.running_digest.lock().unwrap(), + "sha256:old", + "running digest must be unchanged after an aborted sync" + ); + } + + #[test] + fn loopback_reference_pins_digest_on_loopback_host() { + let reference = loopback_reference(15151, "my-app", "sha256:abc"); + assert_eq!(reference, "127.0.0.1:15151/my-app@sha256:abc"); + } +} From 71e6a53cafe6084e25b75ac48187e861bc6d0eab Mon Sep 17 00:00:00 2001 From: Javier Tia Date: Thu, 23 Jul 2026 11:14:27 -0600 Subject: [PATCH 04/20] extensions/container-agent-dev: Add container dev mode agent extension There is currently no device-side component to support the Container Dev Mode workflow. Without an agent running on the device, there is no way to establish a control channel back to the host CLI, proxy image pulls through the bootstrap-delivered CA, or hot-reload the target container when source is synced. Introduce a self-contained Avocado extension that builds and installs a Rust agent exclusively for development runtimes. The extension declares its own SDK dependencies for cross-compilation, provides build and clean scripts that resolve the correct Rust target from the SDK environment, and installs the resulting binary under /usr/bin. A systemd service activates the agent on boot but remains inert until the bootstrap payload written by `avocado container dev up` is present on the writable partition, ensuring the unit does nothing on devices that have never been enrolled in a dev session. The extension is intentionally absent from production images by design. Signed-off-by: Javier Tia --- .gitignore | 2 ++ 1 file changed, 2 insertions(+) diff --git a/.gitignore b/.gitignore index ea8c4bf..50abe41 100644 --- a/.gitignore +++ b/.gitignore @@ -1 +1,3 @@ /target +# Cross-compilation config written by cad-compile.sh at build time. +/.cargo/ From c01b5185272e9bfbeb274e58714fae69a42f3ff8 Mon Sep 17 00:00:00 2001 From: Javier Tia Date: Thu, 23 Jul 2026 14:10:53 -0600 Subject: [PATCH 05/20] container-agent-dev: Add ring-only provider guard script The device agent must stay on the ring rustls provider because it cross-compiles against the musl SDK, which clears only the ring stack; aws-lc-rs is forbidden. Nothing enforced this beyond ad-hoc checks, so a dependency bump could silently pull aws-lc-rs back in and break the cross-compile with no gate. Add check-provider.sh with a three-way exit contract: it fails when aws-lc-rs resolves into the tree, passes only when cargo reports the package provably absent, and treats any other resolution failure (network, bad manifest, unresolvable target) as a loud guard error rather than a false pass. A ring-present backstop ensures a guard that cannot see the tree can never masquerade as success. Signed-off-by: Javier Tia --- scripts/check-provider.sh | 80 +++++++++++++++++++++++++++++++++++++++ 1 file changed, 80 insertions(+) create mode 100755 scripts/check-provider.sh diff --git a/scripts/check-provider.sh b/scripts/check-provider.sh new file mode 100755 index 0000000..0f7ee05 --- /dev/null +++ b/scripts/check-provider.sh @@ -0,0 +1,80 @@ +#!/bin/bash +# +# check-provider.sh - enforce the ring-only rustls crypto provider for the +# Container Dev Mode device agent. +# +# aws-lc-rs is forbidden in this crate (assumption A9: the agent cross-compiles +# against the musl SDK, which clears only the `ring` stack - the same rule +# avocado-conn enforces). This guard fails the build if aws-lc-rs ever resolves +# into the dependency tree, and also asserts that `ring` IS present so a guard +# that cannot see the tree at all can never masquerade as a pass. +# +# Exit contract (three-way, deliberately NOT a bare `! cargo tree`): +# 0 - clean ring-only tree (aws-lc-rs absent, ring present) +# 1 - aws-lc-rs PRESENT (forbidden provider in the tree) +# 2 - guard error: the dependency tree could not be resolved (network, bad +# manifest, unresolvable target, or ring itself missing). Never a pass. +# +# The check uses the default `cargo tree` edge set (includes dev-dependencies) +# so it is tree-wide: the agent Cargo.toml keeps aws-lc-rs "out of the tree +# entirely", and the rcgen dev-dependency is pinned to ring for exactly that. +# +# TARGET: when set (env var or first positional arg) the guard resolves against +# that target triple, matching the build's cross target. In the build hook this +# is mandatory (TARGET="$RUST_TARGET"). Run standalone with no TARGET, the guard +# checks the host triple as an ADVISORY check. + +set -euo pipefail + +script_dir="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" +manifest="$script_dir/../Cargo.toml" + +target="${TARGET:-${1:-}}" + +target_args=() +if [ -n "$target" ]; then + target_args+=(--target "$target") + scope="target $target" +else + scope="host triple (advisory)" +fi + +echo "check-provider: enforcing ring-only rustls provider ($scope)" + +# Inverse-tree query for the forbidden provider. Capture status AND output so a +# non-zero exit does not abort under `set -e`. On success (aws-lc-rs present) +# stdout carries the offending inverse-tree path; on failure the captured text +# carries cargo's error on stderr. +set +e +awslc_out="$(cargo tree -i aws-lc-rs --manifest-path "$manifest" "${target_args[@]}" 2>&1)" +awslc_status=$? +set -e + +if [ "$awslc_status" -eq 0 ]; then + echo "check-provider: FAIL - forbidden crypto provider aws-lc-rs is in the tree:" >&2 + echo "$awslc_out" >&2 + exit 1 +fi + +if ! printf '%s\n' "$awslc_out" | grep -q 'did not match any packages'; then + echo "check-provider: ERROR - guard could not resolve the dependency tree." >&2 + echo "check-provider: this is NOT a pass; refusing to certify the provider." >&2 + echo "$awslc_out" >&2 + exit 2 +fi + +# aws-lc-rs is provably absent. Backstop: `ring` MUST be present. A guard that +# cannot see ring cannot trust its aws-lc-rs result either. +set +e +cargo tree -i ring --manifest-path "$manifest" "${target_args[@]}" >/dev/null 2>&1 +ring_status=$? +set -e + +if [ "$ring_status" -ne 0 ]; then + echo "check-provider: ERROR - ring is not present in the tree; guard cannot" >&2 + echo "check-provider: trust the aws-lc-rs result. Treating as a guard error." >&2 + exit 2 +fi + +echo "check-provider: PASS - aws-lc-rs absent, ring-only provider confirmed." +exit 0 From 2b63520a87b43b908b20aeba86468551ff5d489b Mon Sep 17 00:00:00 2001 From: Javier Tia Date: Thu, 23 Jul 2026 14:10:22 -0600 Subject: [PATCH 06/20] container-agent-dev: Enforce the provider guard in the build hook The guard script is only useful if something runs it. avocado-os has no CI or pre-commit, so the extension's cross-build hook is the natural gate. Invoke check-provider.sh from cad-compile.sh with TARGET set to the resolved SDK cross target, immediately before cargo build and after the target resolution and CARGO_HOME export, so the guard resolves the same manifest, target, and registry cache the build uses. A reintroduced aws-lc-rs now fails the image build. Document that the in-hook run is the enforced gate and a bare standalone run is host-triple advisory. Signed-off-by: Javier Tia --- README.md | 47 +++++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 47 insertions(+) create mode 100644 README.md diff --git a/README.md b/README.md new file mode 100644 index 0000000..3c40de6 --- /dev/null +++ b/README.md @@ -0,0 +1,47 @@ +# avocado-container-agent-dev + +The device-side agent for Container Dev Mode. It stays **ring-only**: the +rustls crypto provider must be `ring`, and `aws-lc-rs` is forbidden. + +## Why ring-only (aws-lc-rs forbidden) + +The agent cross-compiles against the musl SDK (assumption A9), which clears only +the `ring` stack for the target - `aws-lc-rs` will not build there. This is the +same rule `avocado-conn` enforces. `aws-lc-rs` most commonly sneaks in through a +default rustls feature (e.g. `tokio-tungstenite`'s or `reqwest`'s default TLS), +so `Cargo.toml` disables default features on every TLS-touching dependency and +pins `ring` explicitly (including the `rcgen` dev-dependency, so aws-lc-rs stays +out of the tree entirely - dev-deps included). + +## Provider guard: `scripts/check-provider.sh` + +`scripts/check-provider.sh` asserts the crypto provider by inspecting the +dependency tree. It uses a three-way exit contract - a naive `! cargo tree` is +wrong, because `cargo tree -i ` exits 0 when the package is present and +non-zero (with `did not match any packages`) when absent, so a bare negation +would treat every network or manifest failure as a pass. + +| Exit | Meaning | +|------|---------| +| 0 | Clean: aws-lc-rs absent, ring present. | +| 1 | aws-lc-rs is in the tree (forbidden provider present). | +| 2 | Guard error: tree unresolvable (network, bad manifest, unresolvable target, or ring missing). Never treated as a pass. | + +The guard also backstops by asserting `ring` IS present: a guard that cannot see +`ring` cannot trust its aws-lc-rs result, so that case exits 2. + +### Enforced gate vs advisory run + +- **Enforced gate (in-hook, with `TARGET`):** `cad-compile.sh` runs the guard as + `TARGET="$RUST_TARGET" scripts/check-provider.sh` immediately before + `cargo build`, resolving the same SDK triple, manifest, and `CARGO_HOME` the + build uses. Under `set -e`, a non-zero exit fails the compile. This is the + real enforcement point - avocado-os has no CI or pre-commit hook. +- **Advisory run (standalone, no `TARGET`):** running + `scripts/check-provider.sh` directly (e.g. from the repo root) checks the + **host** triple, not the shipped cross target. Treat it as an advisory + smoke-check; it does not gate anything. + +The guard uses the default `cargo tree` edge set (dev-dependencies included) and +deliberately does not pass `--edges normal,build` or `--locked`, so it resolves +the same tree the build does within the shared `CARGO_HOME`. From 8d3c1846e2acfcaa3a6d6efa880b35ca14632ce0 Mon Sep 17 00:00:00 2001 From: Javier Tia Date: Mon, 27 Jul 2026 17:55:19 -0600 Subject: [PATCH 07/20] container-agent-dev: write the active-image pointer after the restart on_sync wrote active-image.json before calling engine.restart(), so a restart failure returned with the pointer already claiming the new image while the container still ran the old one. running_digest stayed honest, which left the two records disagreeing until the next successful sync - and the pointer is the one an operator reads off the device to answer "what is this thing running". Swap the order. The pointer records what the device IS running, so it has no reason to precede the thing that makes that true; nothing consumes it during container startup, so moving it later costs no behaviour. The new test drives a restart failure and asserts the pointer is absent and the digest unmoved, which fails against the previous ordering. Signed-off-by: Javier Tia --- src/sync.rs | 92 +++++++++++++++++++++++++++++++++++++++++++---------- 1 file changed, 76 insertions(+), 16 deletions(-) diff --git a/src/sync.rs b/src/sync.rs index 46f0a01..94a015d 100644 --- a/src/sync.rs +++ b/src/sync.rs @@ -5,14 +5,15 @@ //! (`127.0.0.1:`, task 6.2). The pull rides the proxy -> host bulk //! HTTPS path; it NEVER re-pulls over the control WebSocket (D9 splits //! bulk from control). -//! 2. **Rewrites the active-image pointer** — a small JSON file recording the +//! 2. **Restarts the container** via the engine AFTER the pull, so the new +//! image takes effect. The read-only rootFS is untouched throughout. +//! 3. **Rewrites the active-image pointer** — a small JSON file recording the //! now-active `{image, tag, digest}` — on the **writable partition** -//! (`/var/lib/avocado/container-dev/active-image.json`). The path is -//! derived from [`WRITABLE_ROOT`], never a read-only rootFS path -//! (`/usr`, `/etc`, the image rootfs). -//! 3. **Restarts the container** via the engine AFTER the pull + pointer -//! rewrite, so the new image takes effect. The read-only rootFS is -//! untouched throughout. +//! (`/var/lib/avocado/container-dev/active-image.json`), only once the +//! restart succeeded, so a failed restart never leaves the pointer ahead of +//! what the device actually runs. The path is derived from +//! [`WRITABLE_ROOT`], never a read-only rootFS path (`/usr`, `/etc`, the +//! image rootfs). //! 4. Updates [`crate::AgentState::running_digest`] so task 6.1's reconnect //! `Hello` reports it. //! @@ -203,8 +204,19 @@ pub(crate) async fn on_sync( .await .with_context(|| format!("pulling {reference} through loopback proxy"))?; - // 2. Rewrite the active-image pointer on the WRITABLE partition. The path is - // derived from cfg.writable_root, never a read-only rootFS path. + // 2. Restart the container AFTER the pull so the new image takes effect. + engine + .restart(&cfg.container) + .await + .with_context(|| format!("restarting container {}", cfg.container))?; + + // 3. Rewrite the active-image pointer on the WRITABLE partition, only once + // the restart has actually succeeded. The pointer records what the device + // IS running, so writing it ahead of the restart would leave it claiming + // the new image while the container still ran the old one on any restart + // failure. Nothing consumes the pointer during startup, so it has no + // reason to precede the restart. The path is derived from + // cfg.writable_root, never a read-only rootFS path. let active = ActiveImage { image: image.to_string(), tag: tag.to_string(), @@ -214,13 +226,6 @@ pub(crate) async fn on_sync( .context("rewriting active-image pointer")?; info!(pointer = %path.display(), "active-image pointer rewritten on writable partition"); - // 3. Restart the container AFTER the pull + pointer rewrite so the new image - // takes effect. - engine - .restart(&cfg.container) - .await - .with_context(|| format!("restarting container {}", cfg.container))?; - // 4. Record the now-running digest so the reconnect Hello reports it. *state .running_digest @@ -242,6 +247,7 @@ mod tests { struct FakeEngine { calls: Arc>>, fail_pull: bool, + fail_restart: bool, } impl FakeEngine { @@ -249,6 +255,7 @@ mod tests { Self { calls, fail_pull: false, + fail_restart: false, } } @@ -256,6 +263,15 @@ mod tests { Self { calls, fail_pull: true, + fail_restart: false, + } + } + + fn failing_restart(calls: Arc>>) -> Self { + Self { + calls, + fail_pull: false, + fail_restart: true, } } } @@ -274,6 +290,9 @@ mod tests { .lock() .unwrap() .push(format!("restart:{container}")); + if self.fail_restart { + anyhow::bail!("simulated restart failure"); + } Ok(()) } } @@ -437,6 +456,47 @@ mod tests { ); } + // Edge: a restart failure must not leave the pointer claiming an image the + // device is not running. The pull succeeded so the bytes are on the device, + // but the container still runs the OLD image - a pointer written ahead of + // the restart would disagree with both reality and `running_digest`. + #[tokio::test] + async fn restart_failure_leaves_no_stale_pointer() { + let calls = Arc::new(Mutex::new(Vec::new())); + let engine = FakeEngine::failing_restart(calls.clone()); + let tmp = tempfile::tempdir().unwrap(); + let cfg = test_cfg(tmp.path().to_path_buf()); + let state = crate::AgentState::new("dev-01".to_string()); + *state.running_digest.lock().unwrap() = "sha256:old".to_string(); + + let err = on_sync(&engine, &cfg, "my-app", "dev", "sha256:new", &state) + .await + .expect_err("restart failure must fail the sync"); + assert!( + err.to_string().contains("restarting container"), + "unexpected error: {err}" + ); + + let recorded = calls.lock().unwrap().clone(); + assert_eq!( + recorded, + vec![ + "pull:127.0.0.1:15151/my-app@sha256:new".to_string(), + "restart:avocado-dev".to_string(), + ], + "the pull and the restart attempt should both be recorded: {recorded:?}" + ); + assert!( + !active_image_path(tmp.path()).exists(), + "pointer must not claim the new image when the restart failed" + ); + assert_eq!( + *state.running_digest.lock().unwrap(), + "sha256:old", + "running digest must still report the image actually running" + ); + } + #[test] fn loopback_reference_pins_digest_on_loopback_host() { let reference = loopback_reference(15151, "my-app", "sha256:abc"); From 019648d77d464034196c2700f556a00d35692616 Mon Sep 17 00:00:00 2001 From: Javier Tia Date: Mon, 27 Jul 2026 17:57:15 -0600 Subject: [PATCH 08/20] container-agent-dev: keep serving loopback after an accept error serve() propagated any accept() error out of the loop, and main only logs that the proxy exited without respawning it. One transient failure - fd exhaustion, a peer that vanished between the SYN and the accept - therefore disabled the registry proxy for the rest of the process lifetime, so every later Sync pull failed connection-refused with no recovery short of restarting the unit. A per-accept error says nothing about the listener, so it should not be treated as fatal to it. Log and continue, with a short pause first so a sustained EMFILE spins the task instead of the CPU. Also record the two properties of this proxy that read as defects until you know why they are not: the loopback listener performs no device-side authz (a device-side gate needs a credential the engine would then have to be configured with, which is the per-engine trust file the design exists to avoid), and each upstream request opens a fresh TLS connection (a pool needs eviction and reuse-after-error handling for a handful of handshakes per sync on a dev-only path). Both are bounded rather than free, so the reasoning belongs next to the code that will otherwise attract the same finding again. Signed-off-by: Javier Tia --- src/main.rs | 6 ++++++ src/proxy.rs | 51 +++++++++++++++++++++++++++++++++++++++++++++++---- 2 files changed, 53 insertions(+), 4 deletions(-) diff --git a/src/main.rs b/src/main.rs index e816e01..c759f86 100644 --- a/src/main.rs +++ b/src/main.rs @@ -372,6 +372,12 @@ async fn connect_and_serve( connected.store(true, Ordering::SeqCst); info!(endpoint = %bootstrap.ws_endpoint, "control WS connected"); + // The sink goes idle after the frames below and only the stream is polled. + // That does NOT strand the automatic Pong replies to host keepalive pings: + // tungstenite queues a Pong as `additional_send` and flushes it from inside + // `read()` itself (protocol/mod.rs), which is the same call `poll_next` + // drives on the read half — so a ping is answered without the sink being + // touched, and the connection is not churned by a keepalive timeout. let (mut sink, mut stream) = ws.split(); let hello = state.hello(); diff --git a/src/proxy.rs b/src/proxy.rs index 00d4a41..805b359 100644 --- a/src/proxy.rs +++ b/src/proxy.rs @@ -27,12 +27,37 @@ //! exceeded the agent's mid-stream behavior is best-effort (the stream simply //! cuts), explicitly NOT a load-bearing terminal signal. The upstream //! `WWW-Authenticate` is never proxied back to the engine under any status. +//! +//! # Trust assumptions +//! +//! **The loopback listener performs no device-side authorization.** Any local +//! process that can reach `127.0.0.1:` gets its `GET`/`HEAD` forwarded +//! upstream under the agent's `Authorization: Bearer `, so the read +//! token is effectively shared with every process on the device. That is +//! deliberate, and it is the same trust boundary the "no engine trust config" +//! guarantee buys: the engine authenticates to this proxy by being local, and +//! adding a device-side gate would need a credential the engine would then have +//! to be configured with — reintroducing exactly the per-engine trust file the +//! design exists to avoid. What bounds the exposure is that the socket is +//! loopback-only (never `0.0.0.0`), the method allowlist is GET/HEAD so a local +//! process can only read, and the whole extension is dev-only and gated on a +//! bootstrap the host must deliver. A device running untrusted local workloads +//! is outside this threat model; the mitigation there is not to install the dev +//! extension. +//! +//! **Each upstream request opens a fresh TCP + TLS + HTTP/1 connection.** There +//! is no connection pool, so pulling a multi-layer image pays one full TLS +//! handshake per manifest and per blob. That is accepted rather than optimized: +//! a pool would need idle-connection eviction and reuse-after-error handling on +//! a dev-only path whose cost is a handful of handshakes per sync. Revisit it if +//! this proxy ever carries a production pull. use std::convert::Infallible; use std::future::Future; use std::net::{Ipv4Addr, SocketAddr}; use std::sync::Arc; use std::sync::atomic::{AtomicBool, Ordering}; +use std::time::Duration; use anyhow::{Context, Result}; use bytes::Bytes; @@ -45,6 +70,7 @@ use hyper::service::service_fn; use hyper::{HeaderMap, Method, Request, Response, StatusCode}; use hyper_util::rt::TokioIo; use tokio::net::{TcpListener, TcpStream}; +use tokio::time::sleep; use tokio_rustls::TlsConnector; use tokio_rustls::rustls::pki_types::ServerName; use tracing::{info, warn}; @@ -71,6 +97,10 @@ const PASSTHROUGH_HEADERS: &[&str] = &[ /// The plain-text body returned to the engine when the bulk upstream rejects /// the read/control token. It carries no upstream status or header semantics. +/// Pause after a failed `accept()` before trying again, so a sustained error +/// condition (EMFILE) does not turn the serve loop into a busy spin. +const ACCEPT_ERROR_BACKOFF: Duration = Duration::from_millis(100); + const REBOOTSTRAP_MSG: &str = "bulk upstream rejected the read/control token; re-run `avocado container dev up`"; @@ -308,10 +338,23 @@ pub(crate) async fn serve( rebootstrap: ReBootstrap, ) -> Result<()> { loop { - let (stream, _peer) = listener - .accept() - .await - .context("accepting loopback connection")?; + let (stream, _peer) = match listener.accept().await { + Ok(pair) => pair, + // A per-accept error (fd exhaustion, a peer that vanished between + // the SYN and the accept) says nothing about the listener itself. + // Returning here would end the task for the rest of the process + // lifetime - `main` only logs that the proxy exited and never + // respawns it - so every later Sync pull would fail connection- + // refused with no path back short of restarting the unit. Log and + // keep serving instead. + Err(e) => { + warn!(error = %e, "accepting loopback connection failed; continuing to serve"); + // Back off briefly so a sustained error (EMFILE while the fd + // table stays full) spins the task rather than the CPU. + sleep(ACCEPT_ERROR_BACKOFF).await; + continue; + } + }; let upstream = upstream.clone(); let rebootstrap = rebootstrap.clone(); tokio::spawn(async move { From 0262c7c6666af73c60002a71143cfd7b19feacf2 Mon Sep 17 00:00:00 2001 From: Javier Tia Date: Tue, 28 Jul 2026 16:04:32 -0600 Subject: [PATCH 09/20] container-agent-dev: bound the proxy's accept retries Making every accept error a `continue` traded one failure mode for a worse one. A listener that fails permanently - EBADF, or any error that repeats - now produced a 10 Hz warn stream forever, and because the loop could never return, the process never exited and `Restart=always` had nothing to restart. The previous `?` at least let systemd recover. It also made `serve`'s error variant and main's "loopback registry proxy exited" branch unreachable. Retry a bounded run instead: a counter reset by any successful accept, and after 32 consecutive failures the error propagates so the supervisor can do its job. That keeps the resilience the change was for - a peer vanishing between SYN and accept, a momentary EMFILE - without disabling recovery for the case where the listener is genuinely gone. The EMFILE motivation is better served at the source, so the unit now sets LimitNOFILE=65536 rather than relying on this loop to absorb fd exhaustion. Also move ACCEPT_ERROR_BACKOFF below REBOOTSTRAP_MSG. Inserting it where it was put the const between REBOOTSTRAP_MSG's doc comment and REBOOTSTRAP_MSG itself, so `cargo doc` and editor hover described a 100ms sleep as the plain-text body returned when the bulk upstream rejects a token, and REBOOTSTRAP_MSG was left undocumented. Signed-off-by: Javier Tia --- src/proxy.rs | 55 ++++++++++++++++++++++++++++++++++++++-------------- 1 file changed, 40 insertions(+), 15 deletions(-) diff --git a/src/proxy.rs b/src/proxy.rs index 805b359..74d2ea7 100644 --- a/src/proxy.rs +++ b/src/proxy.rs @@ -97,13 +97,21 @@ const PASSTHROUGH_HEADERS: &[&str] = &[ /// The plain-text body returned to the engine when the bulk upstream rejects /// the read/control token. It carries no upstream status or header semantics. -/// Pause after a failed `accept()` before trying again, so a sustained error -/// condition (EMFILE) does not turn the serve loop into a busy spin. -const ACCEPT_ERROR_BACKOFF: Duration = Duration::from_millis(100); - const REBOOTSTRAP_MSG: &str = "bulk upstream rejected the read/control token; re-run `avocado container dev up`"; +/// Pause after a failed `accept()` before trying again, so a transient error +/// does not turn the serve loop into a busy spin. +const ACCEPT_ERROR_BACKOFF: Duration = Duration::from_millis(100); + +/// How many consecutive `accept()` failures end the serve loop. +/// +/// A transient failure should not kill the proxy, but a permanently bad +/// listener must not be retried forever: the task would never return, so the +/// process would never exit and `Restart=always` would have nothing to restart. +/// Returning after a run of failures hands recovery back to the supervisor. +const ACCEPT_ERROR_LIMIT: u32 = 32; + // --------------------------------------------------------------------------- // Re-bootstrap status flag. // --------------------------------------------------------------------------- @@ -337,20 +345,37 @@ pub(crate) async fn serve( upstream: Arc, rebootstrap: ReBootstrap, ) -> Result<()> { + // Consecutive accept failures. Reset by any successful accept, so a run has + // to be genuinely unbroken to reach the limit. + let mut consecutive_errors: u32 = 0; loop { let (stream, _peer) = match listener.accept().await { - Ok(pair) => pair, - // A per-accept error (fd exhaustion, a peer that vanished between - // the SYN and the accept) says nothing about the listener itself. - // Returning here would end the task for the rest of the process - // lifetime - `main` only logs that the proxy exited and never - // respawns it - so every later Sync pull would fail connection- - // refused with no path back short of restarting the unit. Log and - // keep serving instead. + Ok(pair) => { + consecutive_errors = 0; + pair + } + // A single accept error (a peer that vanished between the SYN and + // the accept, a momentary EMFILE) says nothing about the listener, + // and killing the proxy over one would leave every later Sync pull + // failing connection-refused. Retry those. + // + // But retrying forever is its own failure: on a permanently bad + // listener the task never returns, so the process never exits and + // `Restart=always` has nothing to restart - the proxy is dead and + // the supervisor cannot tell. Give up after an unbroken run and let + // the error propagate so systemd can do its job. Err(e) => { - warn!(error = %e, "accepting loopback connection failed; continuing to serve"); - // Back off briefly so a sustained error (EMFILE while the fd - // table stays full) spins the task rather than the CPU. + consecutive_errors += 1; + if consecutive_errors >= ACCEPT_ERROR_LIMIT { + return Err(e).with_context(|| { + format!("accepting loopback connections failed {consecutive_errors} times in a row") + }); + } + warn!( + error = %e, + consecutive_errors, + "accepting loopback connection failed; retrying" + ); sleep(ACCEPT_ERROR_BACKOFF).await; continue; } From 5f76c51e5091ca06caeeed61e24d6a2710fbbe27 Mon Sep 17 00:00:00 2001 From: Javier Tia Date: Tue, 28 Jul 2026 16:04:07 -0600 Subject: [PATCH 10/20] container-agent-dev: report the running digest from the restart, not the write Moving the pointer write after the restart fixed a pointer that claimed an image the device was not running, and opened the inverse: a restart that succeeds followed by a pointer write that fails - ENOSPC, an unwritable /var - bailed on the `?` before `running_digest` was updated, so the reconnect Hello reported the OLD digest while the container ran the new one. Same skew, other direction, and only the first direction had a test. Set `running_digest` immediately after the restart returns. At that instant the container really is running the new image; that is something observed, not something planned, so nothing downstream should be able to make the agent report otherwise. A device contradicting itself is worse than a stale pointer - the host would keep re-syncing an image the device already runs. The pointer write is then reported rather than propagated. The pull and the restart both succeeded, so the sync did take effect; failing the whole operation over the bookkeeping record would tell the host the opposite, and its only recovery is to retry the same digest, which does not fix a full disk. The new test drives exactly that path - a writable_root whose parent is a file, so create_dir_all cannot succeed while pull and restart do - and asserts the digest still reports the running image. It fails against the previous propagate-and-defer ordering. Signed-off-by: Javier Tia --- src/sync.rs | 89 ++++++++++++++++++++++++++++++++++++++++++++++------- 1 file changed, 78 insertions(+), 11 deletions(-) diff --git a/src/sync.rs b/src/sync.rs index 94a015d..2f82353 100644 --- a/src/sync.rs +++ b/src/sync.rs @@ -30,7 +30,7 @@ use std::path::{Path, PathBuf}; use anyhow::{Context, Result, ensure}; use serde::{Deserialize, Serialize}; use tokio::process::Command; -use tracing::info; +use tracing::{info, warn}; use crate::AgentState; @@ -210,27 +210,47 @@ pub(crate) async fn on_sync( .await .with_context(|| format!("restarting container {}", cfg.container))?; - // 3. Rewrite the active-image pointer on the WRITABLE partition, only once + // 3. Record the now-running digest. This happens IMMEDIATELY after the + // restart, before the pointer write, because at this instant the + // container really is running `digest` - that is an observed fact, not a + // plan. Deferring it behind the pointer write would let a pointer failure + // leave `Hello` reporting an image the device is demonstrably not running, + // which is the mirror image of the skew that moving the pointer after the + // restart fixed. + *state + .running_digest + .lock() + .expect("running_digest mutex poisoned") = digest.to_string(); + + // 4. Rewrite the active-image pointer on the WRITABLE partition, only once // the restart has actually succeeded. The pointer records what the device // IS running, so writing it ahead of the restart would leave it claiming // the new image while the container still ran the old one on any restart // failure. Nothing consumes the pointer during startup, so it has no // reason to precede the restart. The path is derived from // cfg.writable_root, never a read-only rootFS path. + // + // A failure here is reported, not propagated. The pull and the restart + // both succeeded, so the sync did happen; failing the whole operation + // over the bookkeeping record would tell the host the sync did not take + // effect when it did, and the host's only recovery - retry the same + // digest - cannot fix a full or unwritable /var anyway. let active = ActiveImage { image: image.to_string(), tag: tag.to_string(), digest: digest.to_string(), }; - let path = write_active_image(&cfg.writable_root, &active) - .context("rewriting active-image pointer")?; - info!(pointer = %path.display(), "active-image pointer rewritten on writable partition"); - - // 4. Record the now-running digest so the reconnect Hello reports it. - *state - .running_digest - .lock() - .expect("running_digest mutex poisoned") = digest.to_string(); + match write_active_image(&cfg.writable_root, &active) { + Ok(path) => { + info!(pointer = %path.display(), "active-image pointer rewritten on writable partition") + } + Err(e) => warn!( + error = %e, + %digest, + "container restarted but the active-image pointer could not be written; \ + the running digest is still reported correctly" + ), + } info!(%image, %tag, %digest, "sync complete: pulled, pointer rewritten, container restarted"); Ok(()) @@ -497,6 +517,53 @@ mod tests { ); } + // The mirror of `restart_failure_leaves_no_stale_pointer`: the restart + // SUCCEEDS and the pointer write then fails (ENOSPC, an unwritable /var, a + // state root pointing somewhere impossible). The container really is running + // the new image at that point, so `running_digest` must say so - reporting + // the old digest in the reconnect `Hello` would tell the host to re-sync an + // image the device already runs, and would be a claim contradicted by the + // device itself. + #[tokio::test] + async fn pointer_write_failure_after_a_successful_restart_still_reports_the_new_digest() { + let calls = Arc::new(Mutex::new(Vec::new())); + let engine = FakeEngine::new(calls.clone()); + let tmp = tempfile::tempdir().unwrap(); + + // A writable_root whose parent is a FILE: create_dir_all cannot succeed, + // so write_active_image fails while pull and restart do not. + let blocker = tmp.path().join("not-a-dir"); + std::fs::write(&blocker, b"x").unwrap(); + let cfg = test_cfg(blocker.join("state")); + + let state = crate::AgentState::new("dev-01".to_string()); + *state.running_digest.lock().unwrap() = "sha256:old".to_string(); + + on_sync(&engine, &cfg, "my-app", "dev", "sha256:new", &state) + .await + .expect("a pointer-write failure must not fail a sync that already took effect"); + + let recorded = calls.lock().unwrap().clone(); + assert_eq!( + recorded, + vec![ + "pull:127.0.0.1:15151/my-app@sha256:new".to_string(), + "restart:avocado-dev".to_string(), + ], + "the pull and restart must both have run: {recorded:?}" + ); + assert!( + !active_image_path(&cfg.writable_root).exists(), + "the pointer genuinely could not be written" + ); + assert_eq!( + *state.running_digest.lock().unwrap(), + "sha256:new", + "Hello must report the digest the container is actually running, \ + even though the pointer write failed" + ); + } + #[test] fn loopback_reference_pins_digest_on_loopback_host() { let reference = loopback_reference(15151, "my-app", "sha256:abc"); From 1a829c4eb2f236a210e83835304c0cd62370eda9 Mon Sep 17 00:00:00 2001 From: Javier Tia Date: Tue, 28 Jul 2026 16:56:39 -0600 Subject: [PATCH 11/20] container-agent-dev: confine the agent, dropping podman support The previous unit sat in the worst quadrant: sandboxed enough to break rootful podman, not sandboxed enough to confine anything. Both cannot be fixed at once. systemd.exec(5) is explicit that ProtectSystem is undone by a privileged process without CapabilityBoundingSet=~CAP_SYS_ADMIN - root can remount its own read-only binds - and rootful podman needs precisely CAP_SYS_ADMIN, plus namespaces, cgroup writes and kernel knobs, all in-process. Supporting podman means the sandbox is decorative; a real sandbox means podman cannot run here. Choosing the sandbox. The agent takes an empty capability bounding set, ProtectSystem=strict with StateDirectory as its only writable path, plus namespace, cgroup and kernel protections and an @system-service syscall filter. UMask=0077 because the state directory holds the Bearer token and the session CA, and systemd-analyze flagged the default as world-readable. Docker is unaffected. Its daemon is a separate unit holding the privilege and is reached over an AF_UNIX socket, and a read-only mount does not block connect() to a socket - the exemption is in sb_permission(), which only rejects writes to regular files, directories and symlinks. AF_NETLINK comes back out of the allowlist: it was only ever needed by netavark configuring a podman container's netns. systemd-analyze security now scores this unit 2.7 (OK), against a prior configuration where the strongest directives were being silently undone. The agent warns at startup when the engine is podman. Without it the failure is invisible in the worst way: podman fails at store init, on_sync's error is only logged by its caller, and every Sync silently no-ops while the agent stays connected reporting a stale digest. A warning rather than a refusal, because the restriction belongs to the unit and not to the binary - run directly for debugging, podman is fine. Restoring podman properly needs the engine out of this process rather than a weaker unit, and is tracked separately. Signed-off-by: Javier Tia --- src/sync.rs | 18 ++++++++++++++++++ 1 file changed, 18 insertions(+) diff --git a/src/sync.rs b/src/sync.rs index 2f82353..a2e0585 100644 --- a/src/sync.rs +++ b/src/sync.rs @@ -120,6 +120,24 @@ impl CommandEngine { .map(|v| v.trim().to_string()) .filter(|v| !v.is_empty()) .unwrap_or_else(|| DEFAULT_ENGINE.to_string()); + if binary.contains("podman") { + // Under the shipped systemd unit podman cannot work: it needs + // CAP_SYS_ADMIN, namespaces, cgroup writes and a writable + // /var/lib/containers, all of which the unit's sandbox denies. It + // would fail at store init and, because on_sync's error is only + // logged, present as syncs that silently do nothing. Say so once at + // startup so the cause is visible. + // + // A warning rather than a refusal: the sandbox is the unit's, not + // the binary's, and the agent run directly for debugging has no such + // restriction. + warn!( + engine = %binary, + "podman is not supported under the container-agent-dev systemd unit \ + (its sandbox denies the privileges rootful podman needs); expect pulls \ + to fail unless this agent is running outside that unit" + ); + } Self { binary } } } From 42050c6721cbf9fafb72e0564da500fbb0cdcb39 Mon Sep 17 00:00:00 2001 From: Javier Tia Date: Tue, 28 Jul 2026 16:58:11 -0600 Subject: [PATCH 12/20] container-agent-dev: test the proxy's accept-error retry policy The bounded retry shipped verified by inspection only, because forcing repeated accept() failures on a real TcpListener means corrupting its fd, which is not portable. So the whole accept loop had no coverage at all - not just the new branch. An `Accept` seam fixes that, mirroring the `Upstream` trait this module already uses for the same reason: production keeps its TcpListener, tests supply a listener whose failures are scripted. Two properties, both confirmed to fail when their mechanism is removed. A permanently failing listener must end the loop, so the process exits and Restart=always can recover it - with the bound removed this hangs instead of terminating. And a success must reset the counter, or a long-lived proxy accumulating scattered transient errors would eventually shut itself down for no reason; the fixture puts one success between two runs of 31 failures, so neither run reaches the limit of 32 alone but their sum does, and removing the reset fails with "gave up after 33 accepts". The counter-reset case is the one worth having. The threshold is arithmetic anyone can read; a counter that quietly fails to reset is not, and it is the half that turns a resilience feature into a spurious shutdown. Signed-off-by: Javier Tia --- src/proxy.rs | 168 ++++++++++++++++++++++++++++++++++++++++++++++++++- 1 file changed, 165 insertions(+), 3 deletions(-) diff --git a/src/proxy.rs b/src/proxy.rs index 74d2ea7..b614247 100644 --- a/src/proxy.rs +++ b/src/proxy.rs @@ -100,6 +100,26 @@ const PASSTHROUGH_HEADERS: &[&str] = &[ const REBOOTSTRAP_MSG: &str = "bulk upstream rejected the read/control token; re-run `avocado container dev up`"; +/// A source of inbound connections. +/// +/// Exists so the accept-error retry policy can be tested; production always +/// uses the [`TcpListener`] impl below. +pub(crate) trait Accept: Send { + type Conn: tokio::io::AsyncRead + tokio::io::AsyncWrite + Send + Unpin + 'static; + + fn accept(&self) -> impl Future> + Send; +} + +impl Accept for TcpListener { + type Conn = tokio::net::TcpStream; + + async fn accept(&self) -> std::io::Result { + TcpListener::accept(self) + .await + .map(|(stream, _peer)| stream) + } +} + /// Pause after a failed `accept()` before trying again, so a transient error /// does not turn the serve loop into a busy spin. const ACCEPT_ERROR_BACKOFF: Duration = Duration::from_millis(100); @@ -344,15 +364,29 @@ pub(crate) async fn serve( listener: TcpListener, upstream: Arc, rebootstrap: ReBootstrap, +) -> Result<()> { + serve_on(listener, upstream, rebootstrap).await +} + +/// The accept loop, over anything that can accept connections. +/// +/// Generic over [`Accept`] rather than taking a `TcpListener` so the retry +/// policy is reachable from a test: forcing repeated `accept()` failures on a +/// real listener means corrupting its fd, which is not portable. Mirrors the +/// [`Upstream`] seam this module already uses for the same reason. +pub(crate) async fn serve_on( + listener: L, + upstream: Arc, + rebootstrap: ReBootstrap, ) -> Result<()> { // Consecutive accept failures. Reset by any successful accept, so a run has // to be genuinely unbroken to reach the limit. let mut consecutive_errors: u32 = 0; loop { - let (stream, _peer) = match listener.accept().await { - Ok(pair) => { + let stream = match listener.accept().await { + Ok(stream) => { consecutive_errors = 0; - pair + stream } // A single accept error (a peer that vanished between the SYN and // the accept, a momentary EMFILE) says nothing about the listener, @@ -430,6 +464,134 @@ mod tests { use super::*; use std::sync::Arc; + use std::sync::atomic::AtomicUsize; + + /// A listener whose `accept()` fails a scripted number of times. + /// + /// `fail_forever` models a listener that has gone permanently bad - the case + /// the retry bound exists for. `fails_then_blocks` models a transient run + /// followed by recovery, so the counter's reset can be observed. + #[derive(Clone)] + struct ScriptedListener { + attempts: Arc, + fail_forever: bool, + succeed_after: usize, + } + + impl ScriptedListener { + fn failing_forever() -> Self { + Self { + attempts: Arc::new(AtomicUsize::new(0)), + fail_forever: true, + succeed_after: 0, + } + } + + fn failing_then_recovering(succeed_after: usize) -> Self { + Self { + attempts: Arc::new(AtomicUsize::new(0)), + fail_forever: false, + succeed_after, + } + } + + fn attempts(&self) -> usize { + self.attempts.load(Ordering::SeqCst) + } + } + + impl Accept for ScriptedListener { + type Conn = tokio::io::DuplexStream; + + async fn accept(&self) -> std::io::Result { + let n = self.attempts.fetch_add(1, Ordering::SeqCst); + if self.fail_forever || n < self.succeed_after { + return Err(std::io::Error::from_raw_os_error(libc_ebadf())); + } + if n == self.succeed_after { + // One success in the middle: enough to reset the counter. + let (a, _b) = tokio::io::duplex(64); + return Ok(a); + } + if n <= self.succeed_after * 2 { + // A SECOND failing run after the success. Both runs are one + // short of the limit, so the loop survives only if the success + // reset the counter - cumulatively they are well past it. + return Err(std::io::Error::from_raw_os_error(libc_ebadf())); + } + // Then park, the way a real listener waits for the next peer. A + // listener that kept returning instantly would spin the loop. + std::future::pending().await + } + } + + fn libc_ebadf() -> i32 { + 9 // EBADF - the permanently-bad-listener case from the review + } + + /// A listener that fails forever must end the loop rather than spin, so the + /// process exits and `Restart=always` can recover it. Before the bound was + /// added this returned never, which is what made the supervisor useless. + #[tokio::test] + async fn a_permanently_failing_listener_ends_the_serve_loop() { + let listener = ScriptedListener::failing_forever(); + let upstream = Arc::new(ok_upstream()); + let rebootstrap = ReBootstrap::default(); + + let err = tokio::time::timeout( + std::time::Duration::from_secs(30), + serve_on(listener, upstream, rebootstrap), + ) + .await + .expect("the loop must terminate rather than spin forever") + .expect_err("a permanently failing listener must surface an error"); + + assert!( + err.to_string().contains("in a row"), + "the error should name the consecutive-failure run: {err}" + ); + } + + /// The counter must reset on a successful accept, or a long-lived proxy that + /// sees scattered transient errors would eventually shut itself down for no + /// reason. Fails if the reset is removed. + #[tokio::test] + async fn a_successful_accept_resets_the_error_counter() { + // More total failures than the limit, but never a consecutive run that + // reaches it: two batches either side of a success. + let listener = ScriptedListener::failing_then_recovering(ACCEPT_ERROR_LIMIT as usize - 1); + let observed = listener.clone(); + let upstream = Arc::new(ok_upstream()); + + let handle = tokio::spawn(serve_on(listener, upstream, ReBootstrap::default())); + + // Wait for both failing runs plus the success between them to elapse. + let want = 2 * (ACCEPT_ERROR_LIMIT as usize - 1) + 1; + let deadline = std::time::Instant::now() + std::time::Duration::from_secs(60); + while observed.attempts() < want && !handle.is_finished() { + assert!(std::time::Instant::now() < deadline, "test timed out"); + tokio::time::sleep(std::time::Duration::from_millis(25)).await; + } + + assert!( + !handle.is_finished(), + "the loop gave up after {} accepts: a success between two runs of \ + {} failures must reset the counter, since neither run reaches the \ + limit of {ACCEPT_ERROR_LIMIT} on its own", + observed.attempts(), + ACCEPT_ERROR_LIMIT - 1, + ); + handle.abort(); + } + + /// A minimal upstream for the accept-loop tests, which never reach it. + fn ok_upstream() -> FakeUpstream { + FakeUpstream { + status: StatusCode::OK, + headers: Vec::new(), + body: "", + } + } use http_body_util::BodyExt; use hyper::body::Incoming; From 03269c3fe73c9dacd0eaa177b15ba2ed1150a6b8 Mon Sep 17 00:00:00 2001 From: Javier Tia Date: Wed, 29 Jul 2026 10:13:11 -0600 Subject: [PATCH 13/20] container-agent-dev: make losing the proxy end the process The bounded accept retry could never reach `Restart=always`, so the whole mechanism - and the two tests protecting it - rested on a claim that was false end to end. `serve_loopback` ran in a detached `tokio::spawn` whose `JoinHandle` was dropped, and `main` then blocked forever in `run()`. `serve_on` returning Err killed only its own task: the process stayed alive, systemd saw a healthy unit, and every later Sync failed connection-refused against a proxy that was gone while the agent kept reporting a stale digest. I asserted the opposite in a commit body, in two comments, and in a review reply I used to resolve the thread. The review that raised it was wrong in the same place - it held that the previous `?` "would have exited and let systemd recover", but that error was discarded identically, so the `?` never exited either. Neither of us checked what `main` did with the task. `end_when_either_ends` selects the proxy against the WS loop so whichever ends first ends `main`, and is a named function rather than an inline select so the seam is testable. Three tests cover it, all failing when the select is replaced by the old ignore-the-proxy behaviour. A cleanly stopped proxy is fatal too: no pull can succeed without it, so being rebuilt beats serving a control WS that can only report failures. The podman warning moves to startup, where it was documented to be. It ran from `CommandEngine::from_env`, constructed once per Sync frame, so `systemctl start` with an engine override logged nothing - the one moment it was new information - and then repeated before every pull failure. `is_unsupported_engine` is pure and tested; `main` calls it once. Also pins the accept backoff, which had no coverage: deleting the sleep left all ten proxy tests green while it is the only thing preventing a busy spin. Virtual time (`start_paused`) asserts the elapsed backoff without spending it, which takes the suite from 6.17s to 0.01s and fails with "only 0ns elapsed" when the sleep is removed. Smaller corrections from the same review: the unit's comment claimed StateDirectory was the only writable path, which ignored PrivateTmp's private /tmp - it now says persistent and names the difference, since that sentence is what a future reader will trust. A doc referenced a constructor name that does not exist. `serve` had become a verbatim forwarder with one caller. And `libc_ebadf()` returned a bare 9 while implying it read libc, for a value no assertion depends on. Signed-off-by: Javier Tia --- Cargo.toml | 4 ++ src/main.rs | 128 +++++++++++++++++++++++++++++++++++++++++++++++---- src/proxy.rs | 40 ++++++++-------- src/sync.rs | 83 +++++++++++++++++++++++++-------- 4 files changed, 209 insertions(+), 46 deletions(-) diff --git a/Cargo.toml b/Cargo.toml index daa28f5..4d91fd1 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -69,6 +69,10 @@ rcgen = { version = "0.13", default-features = false, features = [ "pem", "ring", ] } +# Virtual time for the proxy's accept-retry tests: `start_paused` lets them assert +# the backoff elapsed without spending it, which both pins the sleep and keeps the +# suite fast. Dev-only - the edition-2024 resolver keeps it out of the shipped binary. +tokio = { version = "1", features = ["test-util"] } # Temp dir standing in for the writable partition in the sync pointer tests. tempfile = "3" diff --git a/src/main.rs b/src/main.rs index c759f86..144a472 100644 --- a/src/main.rs +++ b/src/main.rs @@ -21,7 +21,7 @@ use std::sync::Mutex; use std::sync::atomic::{AtomicBool, Ordering}; use std::time::Duration; -use anyhow::{Context, Result, ensure}; +use anyhow::{Context, Result, bail, ensure}; use futures_util::{SinkExt, StreamExt}; use serde::{Deserialize, Serialize}; use tokio::net::TcpStream; @@ -474,21 +474,71 @@ async fn main() -> Result<()> { // from this plain-HTTP loopback (no insecure flag, no per-engine trust // file); each read is forwarded to the host bulk HTTPS listener over the // pinned CA + read/control token. Bulk NEVER rides the control WS. - match proxy::HttpsUpstream::from_bootstrap(&bootstrap) { + // Warn once, here, if the configured engine cannot work under the shipped + // systemd unit. This is the only place that runs exactly once per process: + // `CommandEngine::from_env` is constructed per `Sync` frame, so warning + // there logged nothing at `systemctl start` (when the operator's override is + // new information) and then repeated the warning before every pull failure. + sync::warn_if_unsupported_engine(&sync::engine_binary_from_env()); + + let proxy_task = match proxy::HttpsUpstream::from_bootstrap(&bootstrap) { Ok(upstream) => { let port = loopback_port(); let upstream = Arc::new(upstream); let rebootstrap = state.rebootstrap.clone(); - tokio::spawn(async move { - if let Err(e) = proxy::serve_loopback(port, upstream, rebootstrap).await { - warn!(error = %e, "loopback registry proxy exited"); - } - }); + Some(tokio::spawn(async move { + proxy::serve_loopback(port, upstream, rebootstrap).await + })) } - Err(e) => warn!(error = %e, "loopback registry proxy not started"), + Err(e) => { + warn!(error = %e, "loopback registry proxy not started"); + None + } + }; + + // The proxy's bounded accept-retry only reaches `Restart=always` if losing + // the proxy ends the PROCESS. Spawning it detached and dropping the handle + // meant `serve_loopback` returning Err killed only its own task: `run()` + // kept the control WS up, systemd saw a healthy unit, and every later Sync + // failed connection-refused against a proxy that was gone - while the agent + // went on reporting a stale running_digest. Nothing restarted it, because + // from systemd's view nothing had failed. + // + // Select the two so whichever ends first ends `main`. The proxy is not + // optional: without it no pull can succeed, so exiting and letting the + // supervisor rebuild the process is the honest response to losing it. + match proxy_task { + Some(proxy_task) => end_when_either_ends(proxy_task, run(bootstrap, state)).await, + // No proxy to watch: the upstream could not even be built, which is + // already warned above. Serving the control WS alone is still better + // than exiting, since a re-bootstrap can fix it. + None => run(bootstrap, state).await, } +} - run(bootstrap, state).await +/// Return as soon as EITHER the proxy task or the control-WS loop ends. +/// +/// Extracted from `main` so the seam is testable. It is the whole reason the +/// proxy's bounded accept-retry means anything: the retry makes `serve_loopback` +/// return, and only this makes that return end the process, which is what +/// `Restart=always` needs in order to fire. Spawned detached with its handle +/// dropped, a returning proxy killed only its own task - systemd saw a healthy +/// unit while every later pull failed connection-refused. +async fn end_when_either_ends( + proxy: tokio::task::JoinHandle>, + serving: impl std::future::Future>, +) -> Result<()> { + tokio::select! { + proxied = proxy => match proxied { + // The proxy is not optional - without it no pull can succeed - so + // even a clean stop is a reason to exit and be rebuilt. + Ok(Ok(())) => bail!("loopback registry proxy stopped serving"), + Ok(Err(e)) => Err(e).context("loopback registry proxy failed"), + Err(e) => Err(anyhow::Error::new(e)) + .context("loopback registry proxy task panicked"), + }, + served = serving => served, + } } #[cfg(test)] @@ -658,4 +708,64 @@ mod tests { "unexpected error: {err}" ); } + + // The seam that makes the proxy's accept-retry mean anything. Losing the + // proxy must end `main`, because that is what `Restart=always` needs in + // order to fire; a returning proxy task whose handle was dropped left + // systemd looking at a healthy unit while every pull failed + // connection-refused. + #[tokio::test] + async fn a_failing_proxy_ends_the_process_even_while_the_ws_keeps_serving() { + let proxy = tokio::spawn(async { Err(anyhow::anyhow!("listener died")) }); + // A control-WS loop that would otherwise run forever, as `run()` does. + let serving = std::future::pending::>(); + + let err = tokio::time::timeout( + std::time::Duration::from_secs(5), + end_when_either_ends(proxy, serving), + ) + .await + .expect("losing the proxy must end this, not hang on the WS loop") + .expect_err("a failed proxy must surface an error"); + + assert!( + format!("{err:#}").contains("listener died"), + "the proxy's own error must survive to the exit: {err:#}" + ); + } + + // A proxy that stops cleanly is still fatal: without it no pull can + // succeed, so exiting to be rebuilt beats serving a control WS that can + // only ever report failures. + #[tokio::test] + async fn a_cleanly_stopped_proxy_also_ends_the_process() { + let proxy = tokio::spawn(async { Ok(()) }); + let serving = std::future::pending::>(); + + let err = tokio::time::timeout( + std::time::Duration::from_secs(5), + end_when_either_ends(proxy, serving), + ) + .await + .expect("a stopped proxy must end this") + .expect_err("a stopped proxy must be reported as an error"); + assert!(format!("{err:#}").contains("stopped serving"), "{err:#}"); + } + + // The other direction: the WS loop ending returns its own result rather + // than being masked by the still-running proxy. + #[tokio::test] + async fn a_failing_ws_loop_returns_its_own_error() { + let proxy = tokio::spawn(async { std::future::pending::>().await }); + let serving = async { Err(anyhow::anyhow!("ws gave up")) }; + + let err = tokio::time::timeout( + std::time::Duration::from_secs(5), + end_when_either_ends(proxy, serving), + ) + .await + .expect("the WS loop ending must end this") + .expect_err("a failed WS loop must surface an error"); + assert!(format!("{err:#}").contains("ws gave up"), "{err:#}"); + } } diff --git a/src/proxy.rs b/src/proxy.rs index b614247..9eb2ed1 100644 --- a/src/proxy.rs +++ b/src/proxy.rs @@ -360,14 +360,6 @@ pub(crate) async fn bind_loopback(port: u16) -> Result { /// Serve the loopback registry over `listener`, forwarding every read to /// `upstream`. Runs until the listener errors. -pub(crate) async fn serve( - listener: TcpListener, - upstream: Arc, - rebootstrap: ReBootstrap, -) -> Result<()> { - serve_on(listener, upstream, rebootstrap).await -} - /// The accept loop, over anything that can accept connections. /// /// Generic over [`Accept`] rather than taking a `TcpListener` so the retry @@ -456,7 +448,7 @@ pub(crate) async fn serve_loopback( Ok(addr) => info!(%addr, "loopback registry proxy listening"), Err(_) => info!("loopback registry proxy listening"), } - serve(listener, upstream, rebootstrap).await + serve_on(listener, upstream, rebootstrap).await } #[cfg(test)] @@ -469,7 +461,7 @@ mod tests { /// A listener whose `accept()` fails a scripted number of times. /// /// `fail_forever` models a listener that has gone permanently bad - the case - /// the retry bound exists for. `fails_then_blocks` models a transient run + /// the retry bound exists for. `failing_then_recovering` models a transient run /// followed by recovery, so the counter's reset can be observed. #[derive(Clone)] struct ScriptedListener { @@ -506,7 +498,7 @@ mod tests { async fn accept(&self) -> std::io::Result { let n = self.attempts.fetch_add(1, Ordering::SeqCst); if self.fail_forever || n < self.succeed_after { - return Err(std::io::Error::from_raw_os_error(libc_ebadf())); + return Err(std::io::Error::other("scripted accept failure")); } if n == self.succeed_after { // One success in the middle: enough to reset the counter. @@ -517,7 +509,7 @@ mod tests { // A SECOND failing run after the success. Both runs are one // short of the limit, so the loop survives only if the success // reset the counter - cumulatively they are well past it. - return Err(std::io::Error::from_raw_os_error(libc_ebadf())); + return Err(std::io::Error::other("scripted accept failure")); } // Then park, the way a real listener waits for the next peer. A // listener that kept returning instantly would spin the loop. @@ -525,19 +517,16 @@ mod tests { } } - fn libc_ebadf() -> i32 { - 9 // EBADF - the permanently-bad-listener case from the review - } - /// A listener that fails forever must end the loop rather than spin, so the /// process exits and `Restart=always` can recover it. Before the bound was /// added this returned never, which is what made the supervisor useless. - #[tokio::test] + #[tokio::test(start_paused = true)] async fn a_permanently_failing_listener_ends_the_serve_loop() { let listener = ScriptedListener::failing_forever(); let upstream = Arc::new(ok_upstream()); let rebootstrap = ReBootstrap::default(); + let started = tokio::time::Instant::now(); let err = tokio::time::timeout( std::time::Duration::from_secs(30), serve_on(listener, upstream, rebootstrap), @@ -550,12 +539,25 @@ mod tests { err.to_string().contains("in a row"), "the error should name the consecutive-failure run: {err}" ); + + // The backoff is the only thing standing between a sustained accept + // error and a busy spin, and without this it had no coverage at all - + // deleting the sleep left every proxy test green. Time is virtual here + // (`start_paused`), so this asserts the sleep actually elapsed without + // the suite paying for it. + let elapsed = started.elapsed(); + let want = ACCEPT_ERROR_BACKOFF * (ACCEPT_ERROR_LIMIT - 1); + assert!( + elapsed >= want, + "each retry must back off: expected at least {want:?} across \ + {ACCEPT_ERROR_LIMIT} attempts, only {elapsed:?} elapsed" + ); } /// The counter must reset on a successful accept, or a long-lived proxy that /// sees scattered transient errors would eventually shut itself down for no /// reason. Fails if the reset is removed. - #[tokio::test] + #[tokio::test(start_paused = true)] async fn a_successful_accept_resets_the_error_counter() { // More total failures than the limit, but never a consecutive run that // reaches it: two batches either side of a success. @@ -929,7 +931,7 @@ mod tests { // The loopback socket must be 127.0.0.1, never 0.0.0.0. assert!(loopback_addr.ip().is_loopback()); - tokio::spawn(serve(listener, upstream, rebootstrap.clone())); + tokio::spawn(serve_on(listener, upstream, rebootstrap.clone())); // The engine speaks PLAIN HTTP to the loopback with zero TLS/trust // config; the proxy forwards over the pinned-CA HTTPS upstream. diff --git a/src/sync.rs b/src/sync.rs index a2e0585..ff82421 100644 --- a/src/sync.rs +++ b/src/sync.rs @@ -120,28 +120,51 @@ impl CommandEngine { .map(|v| v.trim().to_string()) .filter(|v| !v.is_empty()) .unwrap_or_else(|| DEFAULT_ENGINE.to_string()); - if binary.contains("podman") { - // Under the shipped systemd unit podman cannot work: it needs - // CAP_SYS_ADMIN, namespaces, cgroup writes and a writable - // /var/lib/containers, all of which the unit's sandbox denies. It - // would fail at store init and, because on_sync's error is only - // logged, present as syncs that silently do nothing. Say so once at - // startup so the cause is visible. - // - // A warning rather than a refusal: the sandbox is the unit's, not - // the binary's, and the agent run directly for debugging has no such - // restriction. - warn!( - engine = %binary, - "podman is not supported under the container-agent-dev systemd unit \ - (its sandbox denies the privileges rootful podman needs); expect pulls \ - to fail unless this agent is running outside that unit" - ); - } Self { binary } } } +/// The engine binary the agent would use, resolved from the environment. +/// +/// Split out so `main` can inspect it once at startup without constructing an +/// engine. [`CommandEngine::from_env`] runs per `Sync` frame, which is the wrong +/// place to warn from: nothing is logged at `systemctl start`, and then the same +/// warning repeats before every pull failure. +pub(crate) fn engine_binary_from_env() -> String { + std::env::var(ENGINE_ENV) + .ok() + .map(|v| v.trim().to_string()) + .filter(|v| !v.is_empty()) + .unwrap_or_else(|| DEFAULT_ENGINE.to_string()) +} + +/// Whether `binary` names an engine the shipped systemd unit cannot run. +/// +/// Rootful podman needs CAP_SYS_ADMIN, namespace creation, cgroup writes and a +/// writable `/var/lib/containers`; the unit's sandbox denies all of them. Pure so +/// the classification is testable without an environment or a process. +pub(crate) fn is_unsupported_engine(binary: &str) -> bool { + binary.contains("podman") +} + +/// Warn once when the configured engine cannot work under the shipped unit. +/// +/// A warning rather than a refusal: the restriction belongs to the unit, not the +/// binary, so an agent run directly for debugging is unaffected. Without it the +/// failure is invisible in the worst way - podman fails at store init, `on_sync`'s +/// error is only logged by its caller, and every Sync silently no-ops while the +/// agent stays connected reporting a stale digest. +pub(crate) fn warn_if_unsupported_engine(binary: &str) { + if is_unsupported_engine(binary) { + warn!( + engine = %binary, + "podman is not supported under the container-agent-dev systemd unit \ + (its sandbox denies the privileges rootful podman needs); expect pulls \ + to fail unless this agent is running outside that unit" + ); + } +} + impl Engine for CommandEngine { async fn pull(&self, reference: &str) -> Result<()> { run_engine(&self.binary, &["pull", reference]).await @@ -582,6 +605,30 @@ mod tests { ); } + // The engine classification the startup warning keys on. Pure, so the + // decision is testable without an environment or a spawned process - which + // is the whole reason it was split out of `CommandEngine::from_env`, where it + // ran once per Sync frame and so could not warn at startup at all. + #[test] + fn podman_is_classified_unsupported_and_docker_is_not() { + assert!(is_unsupported_engine("podman")); + assert!( + is_unsupported_engine("/usr/bin/podman"), + "an absolute path must classify the same as a bare name" + ); + assert!( + is_unsupported_engine("podman-remote"), + "a podman variant is still podman" + ); + + assert!(!is_unsupported_engine("docker")); + assert!(!is_unsupported_engine("/usr/bin/docker")); + assert!( + !is_unsupported_engine(DEFAULT_ENGINE), + "the default engine must never be classified unsupported" + ); + } + #[test] fn loopback_reference_pins_digest_on_loopback_host() { let reference = loopback_reference(15151, "my-app", "sha256:abc"); From 1485c8e47d17ae04f757009ca61d780975e8fdd9 Mon Sep 17 00:00:00 2001 From: Javier Tia Date: Wed, 29 Jul 2026 12:39:02 -0600 Subject: [PATCH 14/20] container-agent-dev: stop claiming a pointer write that failed One unconditional info! sat after the match on write_active_image, so on an ENOSPC /var the journal emitted the warning and then "sync complete: pulled, pointer rewritten, container restarted" immediately below it. Grepping the success line - the obvious thing to grep - reported a current pointer when no file had been written. The warning was right there, but between the two it is the success line that reads as authoritative. Give each outcome its own terminal line and keep the phrase "sync complete" on the Ok arm only. The failure path now says "sync incomplete" and names what is missing, so grepping either string answers truthfully rather than needing the neighbouring line for context. Fix the docs the step reorder left behind. Recording the digest ahead of the pointer write renumbered the inline comments but not the module doc, whose steps 3 and 4 still read pointer-then-digest, nor on_sync's own doc, which said "pull -> rewrite pointer -> restart -> record digest" and so described neither the old body nor the new one. Both now state the real order and, more usefully, the consequence it creates: the pointer write is the only step allowed to fail while the sync returns Ok, so nothing may assume the pointer file exists just because the digest is set. That is the interaction the reorder introduced, which is what made the stale doc worse than a normal one - a reader adding a step between 3 and 4 would have placed a guard on a file that may not be there. No test covers log wording: this crate has no tracing subscriber harness, so the two arms are verified by inspection rather than assertion. The existing restart_failure_leaves_no_stale_pointer test still covers the branch being logged from. fmt, clippy -D warnings and all 34 tests pass. Signed-off-by: Javier Tia --- src/sync.rs | 44 ++++++++++++++++++++++++++++---------------- 1 file changed, 28 insertions(+), 16 deletions(-) diff --git a/src/sync.rs b/src/sync.rs index ff82421..653603b 100644 --- a/src/sync.rs +++ b/src/sync.rs @@ -7,15 +7,19 @@ //! bulk from control). //! 2. **Restarts the container** via the engine AFTER the pull, so the new //! image takes effect. The read-only rootFS is untouched throughout. -//! 3. **Rewrites the active-image pointer** — a small JSON file recording the +//! 3. **Records the now-running digest** in [`crate::AgentState::running_digest`] +//! so task 6.1's reconnect `Hello` reports it. This lands BEFORE the pointer +//! write so a full or unwritable `/var` cannot stop the agent reporting what +//! it actually runs. +//! 4. **Rewrites the active-image pointer** — a small JSON file recording the //! now-active `{image, tag, digest}` — on the **writable partition** //! (`/var/lib/avocado/container-dev/active-image.json`), only once the //! restart succeeded, so a failed restart never leaves the pointer ahead of //! what the device actually runs. The path is derived from //! [`WRITABLE_ROOT`], never a read-only rootFS path (`/usr`, `/etc`, the -//! image rootfs). -//! 4. Updates [`crate::AgentState::running_digest`] so task 6.1's reconnect -//! `Hello` reports it. +//! image rootfs). This is the one step allowed to fail without failing the +//! sync, so no step may assume the pointer file exists - it can be absent +//! while the digest at step 3 is already set. //! //! The engine interaction is behind the [`Engine`] trait (mirroring task 6.2's //! [`crate::proxy::Upstream`] pattern) so the ordering / pointer / digest logic @@ -224,11 +228,14 @@ impl SyncConfig { } } -/// Handle a `Sync`: pull -> rewrite pointer -> restart -> record digest. +/// Handle a `Sync`: pull -> restart -> record digest -> rewrite pointer. /// -/// The order is load-bearing: a pull failure aborts before the pointer rewrite, -/// the restart, and the digest update, so a failed sync never claims a new -/// image is running and never mutates state. All writes target the writable +/// The order is load-bearing at both ends. A pull failure aborts before the +/// restart, the digest update and the pointer write, so a failed sync never +/// claims a new image is running and never mutates state. The digest is recorded +/// before the pointer write, so an unwritable `/var` costs the pointer file but +/// not a correct `Hello` - which makes the pointer write the only step that may +/// fail while the sync still returns `Ok`. All writes target the writable /// partition; the read-only rootFS is never touched. pub(crate) async fn on_sync( engine: &E, @@ -281,19 +288,24 @@ pub(crate) async fn on_sync( tag: tag.to_string(), digest: digest.to_string(), }; + // One terminal line per outcome, and only the Ok arm says "sync complete". + // A single unconditional success line here claimed "pointer rewritten" on the + // failure branch too, so grepping for the obvious success string reported a + // current pointer when no file had been written - the warning above it was + // right, but the success line is the one that reads as authoritative. match write_active_image(&cfg.writable_root, &active) { - Ok(path) => { - info!(pointer = %path.display(), "active-image pointer rewritten on writable partition") - } + Ok(path) => info!( + %image, %tag, %digest, pointer = %path.display(), + "sync complete: pulled, container restarted, active-image pointer rewritten" + ), Err(e) => warn!( error = %e, - %digest, - "container restarted but the active-image pointer could not be written; \ - the running digest is still reported correctly" + %image, %tag, %digest, + "sync incomplete: pulled and container restarted, but the active-image \ + pointer could not be written; the running digest is reported from memory \ + only and will be lost on restart" ), } - - info!(%image, %tag, %digest, "sync complete: pulled, pointer rewritten, container restarted"); Ok(()) } From 8430394de5c7aa49b84e69e696f7f8d6254f0c80 Mon Sep 17 00:00:00 2001 From: Javier Tia Date: Fri, 31 Jul 2026 12:28:34 -0600 Subject: [PATCH 15/20] container-agent-dev: restart the owning unit so a pulled image actually runs ` restart ` re-executes the container's existing config, and that config pins the image by ID at create time. A sync therefore pulled the new image, restarted the container onto the OLD one, and reported success: the device kept running the previous build while the host, the control frame and the active-image pointer all agreed the reload had landed. Silent staleness is the worst shape this failure could take, because nothing in the loop contradicts it. Restart the systemd unit that launched the container instead, when one is named. The unit re-runs ` run` on restart, which re-resolves the tag and so adopts whatever the pull just fetched. Gating it on an env var rather than always shelling to `systemctl` keeps a device that starts its container by hand on the previous behaviour, and the var carries the same value the `service:` field under `container_dev.images` already declares, so nothing new has to be configured to use it. Signed-off-by: Javier Tia --- src/sync.rs | 33 ++++++++++++++++++++++++++++++++- 1 file changed, 32 insertions(+), 1 deletion(-) diff --git a/src/sync.rs b/src/sync.rs index 653603b..1fb4210 100644 --- a/src/sync.rs +++ b/src/sync.rs @@ -54,6 +54,16 @@ const ENGINE_ENV: &str = "AVOCADO_CONTAINER_DEV_ENGINE"; const CONTAINER_ENV: &str = "AVOCADO_CONTAINER_DEV_CONTAINER"; /// Env var overriding the writable state root (used by tests and staging). const STATE_ROOT_ENV: &str = "AVOCADO_CONTAINER_DEV_STATE_ROOT"; +/// Env var naming the systemd unit that OWNS the container, restarted instead of +/// the container itself when set. +/// +/// ` restart ` re-executes the container's existing config, +/// which pins the image by ID at create time - so a freshly pulled image for the +/// same tag is ignored and the sync silently no-ops while reporting success. The +/// unit that launched the container re-runs ` run` on restart, which +/// re-resolves the tag and therefore adopts the new image. This mirrors the +/// `service:` field each entry under `container_dev.images` already declares. +const SERVICE_ENV: &str = "AVOCADO_CONTAINER_DEV_SERVICE"; // --------------------------------------------------------------------------- // Active-image pointer. @@ -169,13 +179,34 @@ pub(crate) fn warn_if_unsupported_engine(binary: &str) { } } +/// The systemd unit owning the container, from `$AVOCADO_CONTAINER_DEV_SERVICE`. +/// +/// `None` keeps the container-restart path, so a device that does not launch its +/// container from a unit behaves exactly as before. +pub(crate) fn service_from_env() -> Option { + std::env::var(SERVICE_ENV) + .ok() + .map(|v| v.trim().to_string()) + .filter(|v| !v.is_empty()) +} + impl Engine for CommandEngine { async fn pull(&self, reference: &str) -> Result<()> { run_engine(&self.binary, &["pull", reference]).await } + /// Restart the owning systemd unit when one is configured, else the container. + /// + /// See [`SERVICE_ENV`]: restarting the container re-executes its pinned image + /// ID, so the pulled image would never actually run. async fn restart(&self, container: &str) -> Result<()> { - run_engine(&self.binary, &["restart", container]).await + match service_from_env() { + Some(service) => { + info!(service = %service, "restarting owning systemd unit to adopt the pulled image"); + run_engine("systemctl", &["restart", &service]).await + } + None => run_engine(&self.binary, &["restart", container]).await, + } } } From 88b4b0b7bbf757d6c4a91d24aa587dbe3b230561 Mon Sep 17 00:00:00 2001 From: Javier Tia Date: Fri, 31 Jul 2026 16:49:02 -0600 Subject: [PATCH 16/20] container-agent-dev: point the service's image ref at what was pulled The pull is digest-pinned, so it lands on the device as an untagged image. The unit's ExecStart names `:`, and nothing ever connected the two: on a device that had the tag from an earlier build the restart re-ran the OLD image, and on one that never had it the restart could not resolve the ref at all. The agent logged "sync complete: pulled, container restarted" in both cases, so the failure was invisible from every side - host, control frame and pointer all agreed a reload had happened. Tag the pulled reference as `:` between the pull and the restart, so the ref the service names resolves to the image just fetched. This went unnoticed because the only topology exercised so far built the image on the device itself, through a forwarded engine socket. There the tag already existed and already pointed at the new build, so the restart picked it up and the pull was redundant - the delivery path was never actually under test. Running the host as the builder, with the device able to obtain the image only through the loop, fails without this and succeeds with it: a device wiped of every my-app image ends up running the host's build. Signed-off-by: Javier Tia --- src/sync.rs | 74 ++++++++++++++++++++++++++++++++++++++++++++++++----- 1 file changed, 67 insertions(+), 7 deletions(-) diff --git a/src/sync.rs b/src/sync.rs index 1fb4210..130f84c 100644 --- a/src/sync.rs +++ b/src/sync.rs @@ -115,6 +115,8 @@ fn loopback_reference(port: u16, image: &str, digest: &str) -> String { /// the sync ordering / pointer / digest logic is unit-testable with a fake. pub(crate) trait Engine: Send + Sync + 'static { fn pull(&self, reference: &str) -> impl Future> + Send; + /// Point `target` (the ref the service names) at `source` (what we just pulled). + fn tag(&self, source: &str, target: &str) -> impl Future> + Send; fn restart(&self, container: &str) -> impl Future> + Send; } @@ -195,6 +197,10 @@ impl Engine for CommandEngine { run_engine(&self.binary, &["pull", reference]).await } + async fn tag(&self, source: &str, target: &str) -> Result<()> { + run_engine(&self.binary, &["tag", source, target]).await + } + /// Restart the owning systemd unit when one is configured, else the container. /// /// See [`SERVICE_ENV`]: restarting the container re-executes its pinned image @@ -283,13 +289,26 @@ pub(crate) async fn on_sync( .await .with_context(|| format!("pulling {reference} through loopback proxy"))?; - // 2. Restart the container AFTER the pull so the new image takes effect. + // 2. Point the ref the service names at what we just pulled. + // + // The pull is digest-pinned, so it lands as an untagged image. The unit's + // ExecStart names `:`, which on this device either does not + // exist (the host built it, so the target never had that tag) or still + // resolves to the PREVIOUS build. Either way the restart below would not + // run what was just fetched, while every step still reported success. + let service_ref = format!("{image}:{tag}"); + engine + .tag(&reference, &service_ref) + .await + .with_context(|| format!("tagging {reference} as {service_ref}"))?; + + // 3. Restart the container AFTER the pull and tag so the new image takes effect. engine .restart(&cfg.container) .await .with_context(|| format!("restarting container {}", cfg.container))?; - // 3. Record the now-running digest. This happens IMMEDIATELY after the + // 4. Record the now-running digest. This happens IMMEDIATELY after the // restart, before the pointer write, because at this instant the // container really is running `digest` - that is an observed fact, not a // plan. Deferring it behind the pointer write would let a pointer failure @@ -389,6 +408,14 @@ mod tests { Ok(()) } + async fn tag(&self, source: &str, target: &str) -> Result<()> { + self.calls + .lock() + .unwrap() + .push(format!("tag:{source}->{target}")); + Ok(()) + } + async fn restart(&self, container: &str) -> Result<()> { self.calls .lock() @@ -427,19 +454,50 @@ mod tests { let recorded = calls.lock().unwrap().clone(); assert_eq!( recorded.len(), - 2, - "expected exactly pull then restart, got {recorded:?}" + 3, + "expected pull, tag, then restart, got {recorded:?}" ); assert!( recorded[0].starts_with("pull:"), "first call must be the pull: {recorded:?}" ); assert!( - recorded[1].starts_with("restart:"), - "second call must be the restart: {recorded:?}" + recorded[1].starts_with("tag:"), + "the tag must land between the pull and the restart: {recorded:?}" + ); + assert!( + recorded[2].starts_with("restart:"), + "the restart must come last: {recorded:?}" ); assert_eq!(recorded[0], "pull:127.0.0.1:15151/my-app@sha256:new"); - assert_eq!(recorded[1], "restart:avocado-dev"); + assert_eq!(recorded[2], "restart:avocado-dev"); + } + + // The pull lands a digest-pinned image. Unless something points the ref the + // service actually names at it, the restart re-runs whatever that ref already + // meant - the old image, or nothing at all when the target never had the tag. + #[tokio::test] + async fn sync_tags_the_pulled_digest_before_restarting() { + let calls = Arc::new(Mutex::new(Vec::new())); + let engine = FakeEngine::new(calls.clone()); + let tmp = tempfile::tempdir().unwrap(); + let cfg = test_cfg(tmp.path().to_path_buf()); + let state = crate::AgentState::new("dev-01".to_string()); + + on_sync(&engine, &cfg, "my-app", "dev", "sha256:new", &state) + .await + .expect("sync succeeds"); + + let recorded = calls.lock().unwrap().clone(); + assert_eq!( + recorded, + vec![ + "pull:127.0.0.1:15151/my-app@sha256:new".to_string(), + "tag:127.0.0.1:15151/my-app@sha256:new->my-app:dev".to_string(), + "restart:avocado-dev".to_string(), + ], + "the pulled digest must be tagged as the service's ref BEFORE the restart: {recorded:?}" + ); } // Falsifier 2: the pointer lands on the writable partition, never a rootFS @@ -586,6 +644,7 @@ mod tests { recorded, vec![ "pull:127.0.0.1:15151/my-app@sha256:new".to_string(), + "tag:127.0.0.1:15151/my-app@sha256:new->my-app:dev".to_string(), "restart:avocado-dev".to_string(), ], "the pull and the restart attempt should both be recorded: {recorded:?}" @@ -632,6 +691,7 @@ mod tests { recorded, vec![ "pull:127.0.0.1:15151/my-app@sha256:new".to_string(), + "tag:127.0.0.1:15151/my-app@sha256:new->my-app:dev".to_string(), "restart:avocado-dev".to_string(), ], "the pull and restart must both have run: {recorded:?}" From 05118c15fad829879d01752d563048b8f01a93ba Mon Sep 17 00:00:00 2001 From: Javier Tia Date: Mon, 3 Aug 2026 18:17:27 -0600 Subject: [PATCH 17/20] container-agent-dev: make the agent's happy path actually work The two defects at the top mean the documented flow never worked at all, and both failed with every layer reporting success. The agent never started. `on_merge` ran `systemctl start` at extension-merge time, before any bootstrap existed, so the unit's own ConditionPathExists skipped it - and nothing re-triggered a start once the bootstrap landed, because `avocado container dev up` delivers that file over SSH and issues no systemctl call anywhere in its container-dev paths. Restart=always does not help a unit that never reached a running state. A .path unit fixes this at the level the problem lives on: it has no condition, so it starts at merge time, and it activates the service the moment the file appears. And a sync that ran adopted nothing. Nothing sets AVOCADO_CONTAINER_DEV_SERVICE, so `restart()` always took the ` restart ` branch that the code's own comment describes as a no-op - the container object stays bound to its create-time image id and does not re-resolve the tag. The pull, the tag and the restart all succeeded, the new digest was recorded, the pointer claimed it, and the host's reconcile saw its desired state satisfied. The sync now proves adoption by comparing image ids and fails naming the remedy, which catches it on whichever branch produced it rather than trusting the branch. Four more ways the agent reported health it did not have. A failed sync was swallowed into a warn!, so with Hello re-sent only on reconnect the host went on showing the device synced at the old digest; failures now go back as a Status frame. The re-bootstrap flag was read once per session BEFORE the frame loop, while the 401 that raises it happens during a pull inside that loop - so it was set strictly after the only read that would have sent it, and nothing could lower it, so one transient 401 latched forever. A failed upstream build logged one warning and carried on with no proxy, three lines below a comment arguing at length that losing the proxy must end the process; that path is now as fatal as the one the comment covers. And no engine check ran at all, so merging without the docker extension left the unit active and silent until the first sync hit ENOENT. Three that turn a transient into a permanent failure. No upstream stage had a timeout, and with no SO_KEEPALIVE and a purely reading socket after send_request, a laptop that sleeps mid-pull wedged the agent forever with systemd reporting active (running). The backoff reset on any completed handshake, so a host that accepts the upgrade and then panics pinned the device to a ~1 Hz TLS handshake loop; it now needs a session that lasted. Syncs ran inline in the WS read loop, so nothing was read for the duration of a pull and N saves queued N full pull/tag/restart cycles - they run on their own task now, with a latest-wins slot, because only the newest digest is worth applying. The bootstrap was never re-read, which made the recovery every message recommends impossible: re-running `up` mints a token the agent does not have, so both legs 401 and the loop spins without ever exiting. The agent now exits when the file changes, which re-reads the proxy's copy of the token too. RestartSec=5 against the default 10s/5 start limit put two starts in any window, so a unit failing instantly on malformed JSON sat in activating (auto-restart) indefinitely and never reached failed. Widening the interval makes the limit reachable. Both keys go in [Unit]: systemd moved them in v229 and silently ignores them under [Service], which is where they landed first and what `systemd-analyze verify` caught. FakeEngine could not fail the tag step, so the `?` on engine.tag had no coverage - replacing it with `let _ =` left all eight sync tests green. Signed-off-by: Javier Tia --- src/main.rs | 490 +++++++++++++++++++++++++++++++++++++++++++-------- src/proxy.rs | 107 +++++++++-- src/sync.rs | 368 +++++++++++++++++++++++++++++++++++++- 3 files changed, 869 insertions(+), 96 deletions(-) diff --git a/src/main.rs b/src/main.rs index 144a472..25cd595 100644 --- a/src/main.rs +++ b/src/main.rs @@ -25,7 +25,7 @@ use anyhow::{Context, Result, bail, ensure}; use futures_util::{SinkExt, StreamExt}; use serde::{Deserialize, Serialize}; use tokio::net::TcpStream; -use tokio::time::sleep; +use tokio::time::{Instant, sleep}; use tokio_rustls::TlsConnector; use tokio_rustls::rustls::pki_types::ServerName; use tokio_tungstenite::client_async; @@ -138,6 +138,19 @@ pub struct Bootstrap { pub ws_endpoint: String, } +impl Bootstrap { + /// Whether this bootstrap and `other` carry the same session material. + /// + /// Only the fields a live session depends on: a token, endpoint or CA change + /// means the session in progress is finished, whatever else the file says. + fn same_session_as(&self, other: &Bootstrap) -> bool { + self.read_token == other.read_token + && self.bulk_endpoint == other.bulk_endpoint + && self.ws_endpoint == other.ws_endpoint + && self.ca_cert_pem == other.ca_cert_pem + } +} + /// Resolve the bootstrap path: `$AVOCADO_CONTAINER_DEV_BOOTSTRAP` or the default. fn bootstrap_path() -> PathBuf { std::env::var_os(BOOTSTRAP_PATH_ENV) @@ -228,6 +241,25 @@ impl Default for Backoff { } } +/// How long a session must last before it counts as evidence the link works. +/// +/// The reset used to key on the handshake alone, which a broken host completes +/// happily before dropping the connection - so the backoff never grew past its +/// 1s base. Ten seconds is far longer than the accept-panic-drop cycle that +/// motivated it and far shorter than any real session. +const MIN_SESSION_FOR_RESET: Duration = Duration::from_secs(10); + +/// Whether a finished session is evidence the link works, and so whether the +/// reconnect backoff may reset. +/// +/// A function rather than an inline condition so the duration floor is +/// assertable: `run()` needs a real host to exercise, and an inline test +/// re-stating the same expression would pass with the floor removed - which is +/// exactly how this shipped without one. +fn session_proves_the_link_works(handshook: bool, session: Duration) -> bool { + handshook && session >= MIN_SESSION_FOR_RESET +} + // --------------------------------------------------------------------------- // Agent state. // --------------------------------------------------------------------------- @@ -298,6 +330,28 @@ pub(crate) fn build_tls_config(ca_cert_pem: &str) -> Result( + stage: &'static str, + fut: impl std::future::Future>, +) -> Result { + match tokio::time::timeout(WS_DIAL_TIMEOUT, fut).await { + Ok(result) => result, + Err(_) => anyhow::bail!( + "{stage} did not complete within {}s; the host is unreachable or the connection \ + is half-open", + WS_DIAL_TIMEOUT.as_secs() + ), + } +} + /// Split a `host:port` endpoint into its host component for SNI. pub(crate) fn endpoint_host(endpoint: &str) -> &str { endpoint @@ -313,7 +367,7 @@ pub(crate) fn endpoint_host(endpoint: &str) -> &str { /// Handle one host frame. On a `Sync`, pull the new image through the loopback /// proxy, rewrite the active-image pointer on the writable partition, restart /// the container, and record the running digest (task 6.3, [`sync::on_sync`]). -async fn handle_host_frame(frame: HostFrame, state: &AgentState) { +async fn handle_host_frame(frame: HostFrame, state: &AgentState) -> Option { match frame { HostFrame::Sync { image, tag, digest } => { info!(%image, %tag, %digest, "received sync; pulling + restarting"); @@ -322,20 +376,50 @@ async fn handle_host_frame(frame: HostFrame, state: &AgentState) { // in the fully-tested `sync::on_sync`. let engine = sync::CommandEngine::from_env(); let cfg = sync::SyncConfig::from_env(); - if let Err(e) = sync::on_sync(&engine, &cfg, &image, &tag, &digest, state).await { - warn!(error = %e, %image, %tag, %digest, "sync failed"); + match sync::on_sync(&engine, &cfg, &image, &tag, &digest, state).await { + Ok(()) => None, + Err(e) => { + // Report it, do not only log it. A swallowed failure left the + // host with no Status, no Progress, and no updated Hello - + // Hello is re-sent only on reconnect, which on a healthy link + // may be hours - so it went on showing the device synced at + // the old digest with no error surface anywhere. + warn!(error = %e, %image, %tag, %digest, "sync failed"); + Some(DeviceFrame::Status(Status { + device_id: state.device_id.clone(), + state: "sync_failed".to_string(), + detail: Some(format!("{image}:{tag} @ {digest}: {e:#}")), + })) + } } } } } +/// The device-to-host frames one session may emit, and how they reach the sink. +/// +/// Syncs run on their own task rather than inline in the read loop, which is +/// what lets the stream keep being polled while a pull is in flight. Inline, +/// `handle_host_frame` awaited three unbounded subprocesses inside +/// `while let Some(msg) = stream.next().await`, so nothing was read for the +/// duration - and N saves during one slow pull queued N frames in the socket +/// buffer, each getting its own full pull/tag/restart afterwards: N container +/// restarts instead of one, N-1 onto an already-superseded digest. +struct SessionChannels { + /// Latest-wins slot. A Sync that arrives while another is running replaces + /// any already-queued one, because only the newest digest is worth applying. + pending: Arc>>, + /// Wakes the sync worker when `pending` is refilled. + work: Arc, +} + /// Dial the control WS, send `Hello`, and serve frames until the link closes. /// /// Sets `connected` to `true` once the WS handshake succeeds, so the caller can /// reset the backoff on a link that actually established. async fn connect_and_serve( bootstrap: &Bootstrap, - state: &AgentState, + state: &Arc, connected: &AtomicBool, ) -> Result<()> { ensure!( @@ -346,16 +430,25 @@ async fn connect_and_serve( let tls_config = build_tls_config(&bootstrap.ca_cert_pem)?; let connector = TlsConnector::from(Arc::new(tls_config)); - let tcp = TcpStream::connect(&bootstrap.ws_endpoint) - .await - .with_context(|| format!("connecting TCP to {}", bootstrap.ws_endpoint))?; + // Same reasoning as the proxy's stage timeouts: without one, a half-open + // connection to a sleeping host wedges the dial forever, and the reconnect + // loop that is supposed to recover never gets another turn. + let tcp = with_dial_timeout("the control WS TCP connect", async { + TcpStream::connect(&bootstrap.ws_endpoint) + .await + .with_context(|| format!("connecting TCP to {}", bootstrap.ws_endpoint)) + }) + .await?; let host = endpoint_host(&bootstrap.ws_endpoint).to_string(); let server_name = ServerName::try_from(host).context("invalid TLS server name")?; - let tls = connector - .connect(server_name, tcp) - .await - .context("TLS handshake to control WS")?; + let tls = with_dial_timeout("the control WS TLS handshake", async { + connector + .connect(server_name, tcp) + .await + .context("TLS handshake to control WS") + }) + .await?; let mut request = format!("wss://{}/", bootstrap.ws_endpoint) .into_client_request() @@ -366,9 +459,12 @@ async fn connect_and_serve( .context("building Authorization header")?, ); - let (ws, _response) = client_async(request, tls) - .await - .context("WebSocket upgrade to control WS")?; + let (ws, _response) = with_dial_timeout("the control WS upgrade", async { + client_async(request, tls) + .await + .context("WebSocket upgrade to control WS") + }) + .await?; connected.store(true, Ordering::SeqCst); info!(endpoint = %bootstrap.ws_endpoint, "control WS connected"); @@ -389,61 +485,194 @@ async fn connect_and_serve( .await .context("sending Hello")?; - // Surface a pending re-bootstrap (raised by the loopback proxy on a bulk - // 401) to the host. There is no token-renewal endpoint; the operator must - // re-run `avocado container dev up` to mint a fresh token. + // The sync worker and the frames it produces. Everything the device sends + // after `Hello` arrives on `outbound` and is written by the select loop + // below, so the sink has exactly one writer. + let channels = SessionChannels { + pending: Arc::new(Mutex::new(None)), + work: Arc::new(tokio::sync::Notify::new()), + }; + let (outbound_tx, mut outbound) = tokio::sync::mpsc::channel::(8); + + let worker = { + let pending = Arc::clone(&channels.pending); + let work = Arc::clone(&channels.work); + let state = Arc::clone(state); + let outbound_tx = outbound_tx.clone(); + tokio::spawn(async move { + loop { + work.notified().await; + // Drain the slot, not a queue: whatever is there is the newest + // Sync, and any earlier one it replaced is already superseded. + let Some(frame) = pending.lock().expect("pending mutex poisoned").take() else { + continue; + }; + if let Some(report) = handle_host_frame(frame, &state).await + && outbound_tx.send(report).await.is_err() + { + // The session ended while the sync ran; the next Hello + // carries the running digest, so nothing is lost. + return; + } + } + }) + }; + // Dropping our own copy means `outbound` closes when the worker does. + drop(outbound_tx); + + // A re-bootstrap raised BEFORE this session started still needs announcing; + // one raised during it arrives through `rebootstrap.raised()` below. if state.rebootstrap.is_raised() { - let status = DeviceFrame::Status(Status { - device_id: state.device_id.clone(), - state: "needs_rebootstrap".to_string(), - detail: Some( - "bulk pull rejected the read/control token; re-run `avocado container dev up`" - .to_string(), - ), - }); - sink.send(Message::Text( - serde_json::to_string(&status) - .context("serializing re-bootstrap status")? - .into(), - )) - .await - .context("sending re-bootstrap status")?; + send_frame(&mut sink, &rebootstrap_status(state)).await?; + state.rebootstrap.clear(); } - while let Some(msg) = stream.next().await { - let msg = msg.context("reading control WS frame")?; - match msg { - Message::Text(text) => match serde_json::from_str::(text.as_str()) { - Ok(frame) => handle_host_frame(frame, state).await, - Err(e) => warn!(error = %e, raw = %text, "ignoring malformed host frame"), - }, - Message::Close(_) => { - info!("control WS closed by host"); - break; + let result = loop { + tokio::select! { + // Biased so an outbound report is written before another host frame + // is accepted; without it a busy host could starve the reports the + // whole restructure exists to deliver. + biased; + + Some(frame) = outbound.recv() => { + send_frame(&mut sink, &frame).await?; } - Message::Ping(_) | Message::Pong(_) => {} - Message::Binary(_) | Message::Frame(_) => { - warn!("ignoring unexpected non-text control frame"); + + // Raised by the loopback proxy on a bulk 401, which happens during a + // pull - inside this loop, not before it. Reading the flag once up + // front meant it was always set strictly after the only read that + // would have transmitted it. + () = state.rebootstrap.raised() => { + send_frame(&mut sink, &rebootstrap_status(state)).await?; + // Lower it once reported, or one transient 401 latches for the + // process lifetime. + state.rebootstrap.clear(); + } + + msg = stream.next() => { + let Some(msg) = msg else { break Ok(()) }; + let msg = msg.context("reading control WS frame")?; + match msg { + Message::Text(text) => match serde_json::from_str::(text.as_str()) { + Ok(frame) => { + // Latest wins. Replacing a queued Sync is the point: + // applying a superseded digest costs a container + // restart and achieves nothing. + let replaced = channels + .pending + .lock() + .expect("pending mutex poisoned") + .replace(frame) + .is_some(); + if replaced { + info!("coalesced a superseded sync still waiting to run"); + } + channels.work.notify_one(); + } + Err(e) => warn!(error = %e, raw = %text, "ignoring malformed host frame"), + }, + Message::Close(_) => { + info!("control WS closed by host"); + break Ok(()); + } + Message::Ping(_) | Message::Pong(_) => {} + Message::Binary(_) | Message::Frame(_) => { + warn!("ignoring unexpected non-text control frame"); + } + } } } - } + }; + worker.abort(); + result +} + +/// The `needs_rebootstrap` status. There is no token-renewal endpoint; the +/// operator must re-run `avocado container dev up` to mint a fresh token. +fn rebootstrap_status(state: &AgentState) -> DeviceFrame { + DeviceFrame::Status(Status { + device_id: state.device_id.clone(), + state: "needs_rebootstrap".to_string(), + detail: Some( + "bulk pull rejected the read/control token; re-run `avocado container dev up`" + .to_string(), + ), + }) +} + +/// Serialize and write one device frame to the control WS sink. +async fn send_frame(sink: &mut S, frame: &DeviceFrame) -> Result<()> +where + S: futures_util::SinkExt + Unpin, + >::Error: std::error::Error + Send + Sync + 'static, +{ + sink.send(Message::Text( + serde_json::to_string(frame) + .context("serializing device frame")? + .into(), + )) + .await + .context("sending device frame")?; Ok(()) } /// The reconnect loop: connect, serve, and on any close/error back off and /// redial, re-sending `Hello` with the current running digest. +/// Returns `Ok(())` when the bootstrap on disk stops matching the one this +/// process started with, so `main` exits and the supervisor rebuilds against the +/// new material. Reloading in place is not enough: the read token is also baked +/// into the loopback proxy's upstream (`HttpsUpstream::from_bootstrap`), and a +/// token the proxy still sends is exactly what 401s the bulk leg. Exiting +/// re-reads both from one code path. +/// +/// Without this the documented recovery could not work. Re-running `avocado +/// container dev up` mints a fresh token and overwrites bootstrap.json; the +/// agent kept the old one, so bulk pulls 401'd, the WS upgrade was rejected by +/// the same host-side auth check, `connect_and_serve` errored on every redial, +/// and this loop spun forever. Nothing exited, so `Restart=always` never fired, +/// and the only fix was a manual unit restart that no message mentioned. async fn run(bootstrap: Bootstrap, state: Arc) -> Result<()> { + let path = bootstrap_path(); let mut backoff = Backoff::default(); loop { let connected = AtomicBool::new(false); + let started = Instant::now(); match connect_and_serve(&bootstrap, &state, &connected).await { Ok(()) => info!("control WS session ended; reconnecting"), Err(e) => warn!(error = %e, "control WS session failed; reconnecting"), } - if connected.load(Ordering::SeqCst) { + + // A session that completed its handshake and then ended immediately is + // not evidence the link works. `connected` flips the moment + // `client_async` returns, so without a duration floor EVERY + // handshake-completing session reset the backoff and the 30s cap was + // unreachable: a host whose `desired` mutex is poisoned accepts the + // upgrade, panics on the first Hello and drops the socket, which pinned + // an embedded device to a ~1 Hz pinned-CA TLS handshake loop against a + // host that would never work again. + if session_proves_the_link_works(connected.load(Ordering::SeqCst), started.elapsed()) { backoff.reset(); } + + // Adopt new session material by exiting, not by continuing. + match load_bootstrap(&path) { + Ok(current) if !current.same_session_as(&bootstrap) => { + info!( + bootstrap = %path.display(), + "bootstrap changed on disk; exiting so the supervisor rebuilds this \ + process against the new session material" + ); + return Ok(()); + } + Ok(_) => {} + // Unreadable or malformed: keep the session we have rather than + // exiting on a file caught mid-delivery. + Err(e) => { + warn!(error = %e, "could not re-read the bootstrap; keeping the current one") + } + } + let delay = backoff.next_delay(); info!(delay_secs = delay.as_secs(), "backing off before reconnect"); sleep(delay).await; @@ -479,21 +708,34 @@ async fn main() -> Result<()> { // `CommandEngine::from_env` is constructed per `Sync` frame, so warning // there logged nothing at `systemctl start` (when the operator's override is // new information) and then repeated the warning before every pull failure. - sync::warn_if_unsupported_engine(&sync::engine_binary_from_env()); - - let proxy_task = match proxy::HttpsUpstream::from_bootstrap(&bootstrap) { - Ok(upstream) => { - let port = loopback_port(); - let upstream = Arc::new(upstream); - let rebootstrap = state.rebootstrap.clone(); - Some(tokio::spawn(async move { - proxy::serve_loopback(port, upstream, rebootstrap).await - })) - } - Err(e) => { - warn!(error = %e, "loopback registry proxy not started"); - None - } + let engine_binary = sync::engine_binary_from_env(); + sync::warn_if_unsupported_engine(&engine_binary); + // Refuse to start without the engine, rather than staying `active (running)` + // and no-oping every sync. See `ensure_engine_available`. + sync::ensure_engine_available(&engine_binary)?; + + // Failing to BUILD the upstream is as fatal as losing it later. + // + // This used to `warn!` once and carry on with `None`, three lines above a + // comment arguing at length that losing the proxy must end the process + // "because without it no pull can succeed". Two tests locked in "losing the + // proxy is fatal"; never having it was uncovered. A truncated `ca_cert_pem` + // from a partial SSH write, or a bulk endpoint host `ServerName::try_from` + // rejects, left the agent `active (running)` with port 15151 closed and + // every Sync failing connection-refused - the exact state that comment was + // written to prevent. + // + // The stated justification, that a re-bootstrap could fix it, was + // self-defeating twice over: `raise()` is only reached from `proxy_read`, so + // with no proxy the needs_rebootstrap frame can never be sent, and the + // bootstrap was never re-read anyway. + let upstream = proxy::HttpsUpstream::from_bootstrap(&bootstrap) + .context("building the loopback proxy's upstream; without it no pull can succeed")?; + let proxy_task = { + let port = loopback_port(); + let upstream = Arc::new(upstream); + let rebootstrap = state.rebootstrap.clone(); + tokio::spawn(async move { proxy::serve_loopback(port, upstream, rebootstrap).await }) }; // The proxy's bounded accept-retry only reaches `Restart=always` if losing @@ -507,13 +749,7 @@ async fn main() -> Result<()> { // Select the two so whichever ends first ends `main`. The proxy is not // optional: without it no pull can succeed, so exiting and letting the // supervisor rebuild the process is the honest response to losing it. - match proxy_task { - Some(proxy_task) => end_when_either_ends(proxy_task, run(bootstrap, state)).await, - // No proxy to watch: the upstream could not even be built, which is - // already warned above. Serving the control WS alone is still better - // than exiting, since a re-bootstrap can fix it. - None => run(bootstrap, state).await, - } + end_when_either_ends(proxy_task, run(bootstrap, state)).await } /// Return as soon as EITHER the proxy task or the control-WS loop ends. @@ -678,6 +914,120 @@ mod tests { assert_eq!(backoff.next_delay(), Duration::from_secs(2)); } + #[test] + fn a_handshake_that_immediately_drops_does_not_reset_the_backoff() { + // `connected` flips the moment `client_async` returns, so a host that + // accepts the upgrade and then panics on the first Hello - a poisoned + // `desired` mutex does exactly that - completed a "successful" + // connection every time. Without a duration floor the backoff reset on + // each one, the 30s cap was unreachable, and the device sat in a ~1 Hz + // pinned-CA TLS handshake loop indefinitely. + assert!( + !session_proves_the_link_works(true, Duration::from_millis(5)), + "a 5ms session must not count as a working link" + ); + assert!( + !session_proves_the_link_works(true, MIN_SESSION_FOR_RESET - Duration::from_millis(1)), + "just under the floor must not reset" + ); + assert!( + session_proves_the_link_works(true, MIN_SESSION_FOR_RESET), + "a session at the floor is evidence the link works" + ); + assert!( + !session_proves_the_link_works(false, Duration::from_secs(600)), + "a dial that never handshook must not reset however long it took" + ); + } + + #[test] + fn a_changed_bootstrap_ends_the_process_so_the_new_token_is_adopted() { + // Re-running `avocado container dev up` mints a fresh token and + // overwrites bootstrap.json. The agent kept the old one: bulk pulls + // 401'd, the WS upgrade was rejected by the same host-side auth check, + // and the reconnect loop spun forever without ever exiting - so + // Restart=always never fired and the documented recovery could not work. + let base = Bootstrap { + bulk_endpoint: "10.0.2.2:15150".to_string(), + read_token: "T1".to_string(), + ca_cert_pem: "PEM".to_string(), + ws_endpoint: "10.0.2.2:15152".to_string(), + }; + + assert!(base.same_session_as(&base.clone())); + + for changed in [ + Bootstrap { + read_token: "T2".to_string(), + ..base.clone() + }, + Bootstrap { + ca_cert_pem: "OTHER".to_string(), + ..base.clone() + }, + Bootstrap { + bulk_endpoint: "10.0.2.2:15999".to_string(), + ..base.clone() + }, + Bootstrap { + ws_endpoint: "10.0.2.2:15999".to_string(), + ..base.clone() + }, + ] { + assert!( + !base.same_session_as(&changed), + "a changed {changed:?} must end the session" + ); + } + } + + #[tokio::test] + async fn a_raised_rebootstrap_wakes_a_waiter_and_clears_once_reported() { + // The flag was read exactly once per session, BEFORE the frame loop - + // and the 401 that raises it happens during a pull, inside that loop. So + // it was always set strictly after the only read that would have sent + // it. Nothing could lower it either, so one transient 401 latched for + // the process lifetime. + let flag = proxy::ReBootstrap::default(); + assert!(!flag.is_raised()); + + let waiter = { + let flag = flag.clone(); + tokio::spawn(async move { flag.raised().await }) + }; + // Let the waiter reach `notified()` before raising. + tokio::task::yield_now().await; + flag.raise(); + + tokio::time::timeout(Duration::from_secs(5), waiter) + .await + .expect("raise must wake a mid-session waiter") + .expect("waiter task"); + + assert!(flag.is_raised()); + flag.clear(); + assert!( + !flag.is_raised(), + "the flag must lower once reported, or every later session re-announces it" + ); + } + + #[test] + fn a_failed_sync_produces_a_status_frame_for_the_host() { + // A swallowed sync failure left the host with no Status, no Progress and + // no updated Hello - Hello is re-sent only on reconnect - so it went on + // showing the device synced at the old digest with no error surface. + let state = AgentState::new("dev-01".to_string()); + let frame = DeviceFrame::Status(Status { + device_id: state.device_id.clone(), + state: "sync_failed".to_string(), + detail: Some("my-app:dev @ sha256:new: boom".to_string()), + }); + let json = serde_json::to_string(&frame).expect("serializes"); + assert!(json.contains("sync_failed"), "{json}"); + assert!(json.contains("dev-01"), "{json}"); + } + #[test] fn hello_carries_current_running_digest() { let state = AgentState::new("dev-01".to_string()); diff --git a/src/proxy.rs b/src/proxy.rs index 9eb2ed1..55470ba 100644 --- a/src/proxy.rs +++ b/src/proxy.rs @@ -142,18 +142,43 @@ const ACCEPT_ERROR_LIMIT: u32 = 32; /// after a host re-`up`). The agent surfaces it (task 6.1's WS loop emits a /// `Status` frame); it is never a silent retry loop, and there is no renewal /// endpoint — the token is refreshed only by a host re-`up` re-bootstrap. +/// Carries a `Notify` as well as the flag, because a bare flag could not be +/// delivered. The WS loop read `is_raised()` exactly once per session, BEFORE +/// entering its frame loop - and the 401 that raises it happens during a pull, +/// which is inside that loop. So the flag was set strictly after the only read +/// that would have transmitted it, and with the sink otherwise idle the host was +/// never told. There was also no way to lower it, so one transient 401 latched +/// for the process lifetime. #[derive(Clone, Default)] -pub(crate) struct ReBootstrap(Arc); +pub(crate) struct ReBootstrap { + raised: Arc, + changed: Arc, +} impl ReBootstrap { - /// Raise the flag. Idempotent. + /// Raise the flag and wake anyone waiting to report it. Idempotent in the + /// flag; the wake is what makes it reach the host mid-session. pub(crate) fn raise(&self) { - self.0.store(true, Ordering::SeqCst); + self.raised.store(true, Ordering::SeqCst); + self.changed.notify_waiters(); } /// Whether a re-bootstrap has been requested. pub(crate) fn is_raised(&self) -> bool { - self.0.load(Ordering::SeqCst) + self.raised.load(Ordering::SeqCst) + } + + /// Lower the flag once it has been reported to the host. + /// + /// Without this a single transient 401 kept every later session announcing + /// `needs_rebootstrap`, long after the token it referred to was replaced. + pub(crate) fn clear(&self) { + self.raised.store(false, Ordering::SeqCst); + } + + /// Resolve when the flag is next raised. + pub(crate) async fn raised(&self) { + self.changed.notified().await; } } @@ -212,6 +237,39 @@ impl HttpsUpstream { } } +/// Ceiling on each pre-body upstream stage: TCP connect, TLS handshake, HTTP/1 +/// handshake, and the request/response head. +/// +/// None of these had one. A laptop that sleeps or drops off Wi-Fi mid-pull leaves +/// a half-open connection with no RST; no SO_KEEPALIVE is set anywhere, so the +/// 7200s tcp_keepalive_time never applies, and after `send_request` the agent is +/// purely READING, so there are no outbound retransmits for tcp_retries2 to give +/// up on. The read hung forever: `docker pull` waited on a loopback response that +/// never arrived, `on_sync` never returned, and - before syncs moved off the read +/// loop - the WS was never polled again either. systemd reported +/// `active (running)` with the device completely dead and nothing logged. +/// +/// Generous on purpose: this bounds a stall, not a slow link. Body streaming is +/// deliberately NOT wrapped, because a multi-gigabyte layer over a slow link is +/// legitimately long; the stages here are the ones that should complete in +/// milliseconds on any working link. +const UPSTREAM_STAGE_TIMEOUT: Duration = Duration::from_secs(30); + +/// Run `fut`, failing with a named error if it does not finish in time. +async fn with_stage_timeout( + stage: &'static str, + fut: impl Future>, +) -> Result { + match tokio::time::timeout(UPSTREAM_STAGE_TIMEOUT, fut).await { + Ok(result) => result, + Err(_) => anyhow::bail!( + "{stage} did not complete within {}s; the host is unreachable or the connection \ + is half-open", + UPSTREAM_STAGE_TIMEOUT.as_secs() + ), + } +} + impl Upstream for HttpsUpstream { async fn fetch( &self, @@ -219,19 +277,27 @@ impl Upstream for HttpsUpstream { path: String, range: Option, ) -> Result { - let tcp = TcpStream::connect(&self.authority) - .await - .with_context(|| format!("connecting to bulk listener {}", self.authority))?; - let tls = self - .connector - .connect(self.server_name.clone(), tcp) - .await - .context("bulk TLS handshake (pinned CA)")?; + let tcp = with_stage_timeout("connecting to the bulk listener", async { + TcpStream::connect(&self.authority) + .await + .with_context(|| format!("connecting to bulk listener {}", self.authority)) + }) + .await?; + let tls = with_stage_timeout("the bulk TLS handshake", async { + self.connector + .connect(self.server_name.clone(), tcp) + .await + .context("bulk TLS handshake (pinned CA)") + }) + .await?; let io = TokioIo::new(tls); - let (mut sender, conn) = hyper::client::conn::http1::handshake(io) - .await - .context("bulk HTTP/1 handshake")?; + let (mut sender, conn) = with_stage_timeout("the bulk HTTP/1 handshake", async { + hyper::client::conn::http1::handshake(io) + .await + .context("bulk HTTP/1 handshake") + }) + .await?; tokio::spawn(async move { if let Err(e) = conn.await { warn!(error = %e, "bulk upstream connection closed with error"); @@ -250,10 +316,13 @@ impl Upstream for HttpsUpstream { .body(Empty::::new()) .context("building upstream request")?; - let response = sender - .send_request(request) - .await - .context("sending upstream request")?; + let response = with_stage_timeout("the upstream request", async { + sender + .send_request(request) + .await + .context("sending upstream request") + }) + .await?; let status = response.status(); let headers = response.headers().clone(); diff --git a/src/sync.rs b/src/sync.rs index 130f84c..64b20c5 100644 --- a/src/sync.rs +++ b/src/sync.rs @@ -118,6 +118,24 @@ pub(crate) trait Engine: Send + Sync + 'static { /// Point `target` (the ref the service names) at `source` (what we just pulled). fn tag(&self, source: &str, target: &str) -> impl Future> + Send; fn restart(&self, container: &str) -> impl Future> + Send; + /// The image ID `container` is currently running, or `None` when the + /// container does not exist. + /// + /// Exists so the sync can PROVE the restart adopted the new image rather + /// than assume it. ` restart` re-executes the container object, + /// which is bound to the image ID it was created with and does not + /// re-resolve the tag - so on the branch the shipped unit actually takes, + /// every step reported success while the container went on running the old + /// image. + fn running_image_id( + &self, + container: &str, + ) -> impl Future>> + Send; + /// The image ID `reference` resolves to locally, or `None` when absent. + fn resolve_image_id( + &self, + reference: &str, + ) -> impl Future>> + Send; } /// The real engine: shells out to the docker/podman CLI. This is the only @@ -181,6 +199,61 @@ pub(crate) fn warn_if_unsupported_engine(binary: &str) { } } +/// Whether `binary` resolves to something executable on `PATH`. +/// +/// Split from the check below so the lookup rule is testable without an +/// environment: an absolute or relative path is probed directly, a bare name is +/// searched along `PATH` the way the kernel's exec would. +pub(crate) fn engine_on_path(binary: &str, path_var: Option<&str>) -> bool { + fn executable(p: &Path) -> bool { + #[cfg(unix)] + { + use std::os::unix::fs::PermissionsExt; + std::fs::metadata(p) + .map(|m| m.is_file() && m.permissions().mode() & 0o111 != 0) + .unwrap_or(false) + } + #[cfg(not(unix))] + { + p.is_file() + } + } + + if binary.contains('/') { + return executable(Path::new(binary)); + } + let Some(path_var) = path_var else { + return false; + }; + path_var + .split(':') + .filter(|dir| !dir.is_empty()) + .any(|dir| executable(&Path::new(dir).join(binary))) +} + +/// Fail startup when the configured engine binary is not present. +/// +/// Nothing verified this before, and the consequence was the shape the unit's +/// own `Environment=` comment calls the worst available: merging the dev +/// extension onto a device without the separate `docker` extension left the unit +/// `active (running)` and silent, because `warn_if_unsupported_engine` only +/// string-matches podman. The first `Sync` then failed at +/// `Command::new("docker").status()` with ENOENT, which the swallowed-error path +/// reduced to a `warn!` - so `running_digest` kept its old value, the WS stayed +/// up, the host saw a connected device, and every sync no-oped forever. +/// +/// Refusing here turns that into a unit that fails at start and says why, which +/// is visible in `systemctl status` without knowing to look for it. +pub(crate) fn ensure_engine_available(binary: &str) -> Result<()> { + ensure!( + engine_on_path(binary, std::env::var("PATH").ok().as_deref()), + "container engine `{binary}` was not found on PATH. The agent execs it directly on \ + every sync, so there is nothing it can do without it - install the `docker` extension \ + in this runtime, or set {ENGINE_ENV} to an engine that is present." + ); + Ok(()) +} + /// The systemd unit owning the container, from `$AVOCADO_CONTAINER_DEV_SERVICE`. /// /// `None` keeps the container-restart path, so a device that does not launch its @@ -214,6 +287,41 @@ impl Engine for CommandEngine { None => run_engine(&self.binary, &["restart", container]).await, } } + + async fn running_image_id(&self, container: &str) -> Result> { + engine_output( + &self.binary, + &["inspect", container, "--format", "{{.Image}}"], + ) + .await + } + + async fn resolve_image_id(&self, reference: &str) -> Result> { + engine_output( + &self.binary, + &["image", "inspect", reference, "--format", "{{.Id}}"], + ) + .await + } +} + +/// Run ` ` and capture stdout, mapping a non-zero exit to +/// `None` rather than an error. +/// +/// A non-zero exit from `inspect` means "no such object", which is a legitimate +/// answer here (the container has never been created) and not a failure to +/// report. +async fn engine_output(binary: &str, args: &[&str]) -> Result> { + let out = Command::new(binary) + .args(args) + .output() + .await + .with_context(|| format!("spawning `{binary} {}`", args.join(" ")))?; + if !out.status.success() { + return Ok(None); + } + let text = String::from_utf8_lossy(&out.stdout).trim().to_string(); + Ok(if text.is_empty() { None } else { Some(text) }) } /// Run ` ` to completion, mapping a non-zero exit to an error. @@ -308,6 +416,43 @@ pub(crate) async fn on_sync( .await .with_context(|| format!("restarting container {}", cfg.container))?; + // 3b. PROVE the restart adopted the new image, rather than assume it. + // + // ` restart ` re-executes the existing container + // object, which is bound to the image ID it was created with and does + // not re-resolve the tag. That is the branch the shipped unit takes, + // because nothing sets AVOCADO_CONTAINER_DEV_SERVICE - so the pull + // succeeded, the tag succeeded, the restart succeeded, and the container + // went on running the previous image while step 4 recorded the new + // digest and the host's reconcile saw its desired state satisfied. The + // developer saw unchanged behaviour with every layer reporting success. + // + // Comparing IDs is engine-generic and catches the failure on whichever + // branch produced it. An unknown ID on either side is NOT treated as a + // mismatch: the container may legitimately not exist yet on a device + // whose unit creates it lazily, and inventing a failure there would + // break a working setup to guard a broken one. + let wanted = engine + .resolve_image_id(&service_ref) + .await + .with_context(|| format!("resolving the image id of {service_ref}"))?; + let running = engine + .running_image_id(&cfg.container) + .await + .with_context(|| format!("reading the image id running in {}", cfg.container))?; + if let (Some(wanted), Some(running)) = (wanted.as_ref(), running.as_ref()) + && wanted != running + { + anyhow::bail!( + "restart did not adopt the pulled image: container {} still runs image {running}, \ + expected {wanted}. `{}` restarts the existing container object, which stays bound \ + to its create-time image id; set AVOCADO_CONTAINER_DEV_SERVICE to the systemd unit \ + that owns the container so the restart re-runs it and re-resolves the tag", + cfg.container, + cfg.container, + ); + } + // 4. Record the now-running digest. This happens IMMEDIATELY after the // restart, before the pointer write, because at this instant the // container really is running `digest` - that is an observed fact, not a @@ -370,7 +515,13 @@ mod tests { struct FakeEngine { calls: Arc>>, fail_pull: bool, + fail_tag: bool, fail_restart: bool, + /// What ` image inspect ` answers. + wanted_id: Option, + /// What ` inspect ` answers AFTER the restart. The + /// default matches `wanted_id`, i.e. a restart that adopted the image. + running_id: Option, } impl FakeEngine { @@ -378,23 +529,55 @@ mod tests { Self { calls, fail_pull: false, + fail_tag: false, fail_restart: false, + wanted_id: Some("sha256:newid".to_string()), + running_id: Some("sha256:newid".to_string()), } } fn failing_pull(calls: Arc>>) -> Self { Self { - calls, fail_pull: true, - fail_restart: false, + ..Self::new(calls) + } + } + + /// `docker tag` rejects some refs outright - an uppercase letter in a + /// host-supplied `image`, for instance - so the step CAN fail, and the + /// fake could not express it. Without this the `?` on `engine.tag(...)` + /// had zero coverage: replacing it with `let _ = ...` left all eight + /// sync tests green while production fell through to a restart that + /// re-ran the previous image and then recorded the new digest. + fn failing_tag(calls: Arc>>) -> Self { + Self { + fail_tag: true, + ..Self::new(calls) } } fn failing_restart(calls: Arc>>) -> Self { Self { - calls, - fail_pull: false, fail_restart: true, + ..Self::new(calls) + } + } + + /// A restart that completed but left the container on its old image - + /// what ` restart` does on the branch the shipped unit takes. + fn restart_that_does_not_adopt(calls: Arc>>) -> Self { + Self { + running_id: Some("sha256:oldid".to_string()), + ..Self::new(calls) + } + } + + /// A device whose unit creates the container lazily: nothing is running + /// yet, so there is no id to compare against. + fn no_container_yet(calls: Arc>>) -> Self { + Self { + running_id: None, + ..Self::new(calls) } } } @@ -413,6 +596,9 @@ mod tests { .lock() .unwrap() .push(format!("tag:{source}->{target}")); + if self.fail_tag { + anyhow::bail!("simulated tag failure"); + } Ok(()) } @@ -426,6 +612,22 @@ mod tests { } Ok(()) } + + async fn running_image_id(&self, container: &str) -> Result> { + self.calls + .lock() + .unwrap() + .push(format!("running_image_id:{container}")); + Ok(self.running_id.clone()) + } + + async fn resolve_image_id(&self, reference: &str) -> Result> { + self.calls + .lock() + .unwrap() + .push(format!("resolve_image_id:{reference}")); + Ok(self.wanted_id.clone()) + } } fn test_cfg(writable_root: PathBuf) -> SyncConfig { @@ -454,8 +656,8 @@ mod tests { let recorded = calls.lock().unwrap().clone(); assert_eq!( recorded.len(), - 3, - "expected pull, tag, then restart, got {recorded:?}" + 5, + "expected pull, tag, restart, then the two adoption probes, got {recorded:?}" ); assert!( recorded[0].starts_with("pull:"), @@ -495,11 +697,161 @@ mod tests { "pull:127.0.0.1:15151/my-app@sha256:new".to_string(), "tag:127.0.0.1:15151/my-app@sha256:new->my-app:dev".to_string(), "restart:avocado-dev".to_string(), + "resolve_image_id:my-app:dev".to_string(), + "running_image_id:avocado-dev".to_string(), ], "the pulled digest must be tagged as the service's ref BEFORE the restart: {recorded:?}" ); } + #[test] + fn a_missing_engine_binary_is_detected_before_the_first_sync() { + // The unit reported `active (running)` with no engine installed, and the + // first Sync failed at Command::new("docker") with ENOENT - reduced to a + // warn! by the swallowed-error path, so the host went on seeing a + // connected device whose syncs no-oped forever. + let tmp = tempfile::tempdir().unwrap(); + let dir = tmp.path().to_str().unwrap(); + + assert!( + !engine_on_path("docker", Some(dir)), + "an empty PATH dir must not resolve the engine" + ); + + let exe = tmp.path().join("docker"); + std::fs::write(&exe, b"#!/bin/sh\nexit 0\n").unwrap(); + #[cfg(unix)] + { + use std::os::unix::fs::PermissionsExt; + std::fs::set_permissions(&exe, std::fs::Permissions::from_mode(0o755)).unwrap(); + } + assert!( + engine_on_path("docker", Some(dir)), + "an executable on PATH must resolve" + ); + } + + #[test] + fn a_present_but_non_executable_engine_does_not_count() { + // Mode 0644 is not runnable; treating presence as availability would + // move the ENOENT to the first sync, which is the whole failure. + let tmp = tempfile::tempdir().unwrap(); + let exe = tmp.path().join("docker"); + std::fs::write(&exe, b"not executable").unwrap(); + #[cfg(unix)] + { + use std::os::unix::fs::PermissionsExt; + std::fs::set_permissions(&exe, std::fs::Permissions::from_mode(0o644)).unwrap(); + assert!(!engine_on_path( + "docker", + Some(tmp.path().to_str().unwrap()) + )); + } + } + + #[test] + fn an_absolute_engine_path_bypasses_path_lookup() { + let tmp = tempfile::tempdir().unwrap(); + let exe = tmp.path().join("myengine"); + std::fs::write(&exe, b"#!/bin/sh\nexit 0\n").unwrap(); + #[cfg(unix)] + { + use std::os::unix::fs::PermissionsExt; + std::fs::set_permissions(&exe, std::fs::Permissions::from_mode(0o755)).unwrap(); + assert!(engine_on_path(exe.to_str().unwrap(), None)); + } + assert!(!engine_on_path("/nonexistent/engine", None)); + } + + #[tokio::test] + async fn a_failed_tag_aborts_before_the_restart() { + // Replacing the `?` on `engine.tag(...)` with `let _ = ...` used to leave + // every sync test green, because the fake could not fail the step. In + // production that fall-through restarts the PREVIOUS image, then records + // the new digest and writes the pointer claiming it - the exact + // false-success the tag step exists to prevent. + let calls = Arc::new(Mutex::new(Vec::new())); + let engine = FakeEngine::failing_tag(calls.clone()); + let tmp = tempfile::tempdir().unwrap(); + let cfg = test_cfg(tmp.path().to_path_buf()); + let state = crate::AgentState::new("dev-01".to_string()); + + let err = on_sync(&engine, &cfg, "my-app", "dev", "sha256:new", &state) + .await + .expect_err("a failed tag must fail the sync"); + assert!( + format!("{err:#}").contains("tagging"), + "the error must name the tag step: {err:#}" + ); + + let recorded = calls.lock().unwrap().clone(); + assert!( + !recorded.iter().any(|c| c.starts_with("restart:")), + "a failed tag must abort BEFORE the restart: {recorded:?}" + ); + assert_eq!( + *state.running_digest.lock().unwrap(), + "", + "a failed tag must not record the new digest" + ); + assert!( + !active_image_path(&cfg.writable_root).exists(), + "a failed tag must not write the active-image pointer" + ); + } + + #[tokio::test] + async fn a_restart_that_does_not_adopt_the_image_fails_the_sync() { + // ` restart ` re-executes the existing container + // object, bound to its create-time image id, and does not re-resolve the + // tag. Nothing in the shipped unit sets AVOCADO_CONTAINER_DEV_SERVICE, so + // that is the branch every device takes - and every step reported success + // while the container went on running the old image. + let calls = Arc::new(Mutex::new(Vec::new())); + let engine = FakeEngine::restart_that_does_not_adopt(calls.clone()); + let tmp = tempfile::tempdir().unwrap(); + let cfg = test_cfg(tmp.path().to_path_buf()); + let state = crate::AgentState::new("dev-01".to_string()); + + let err = on_sync(&engine, &cfg, "my-app", "dev", "sha256:new", &state) + .await + .expect_err("a restart that kept the old image must fail the sync"); + assert!( + format!("{err:#}").contains("did not adopt"), + "the error must say the restart did not adopt the image: {err:#}" + ); + assert!( + format!("{err:#}").contains("AVOCADO_CONTAINER_DEV_SERVICE"), + "the error must name the remedy: {err:#}" + ); + assert_eq!( + *state.running_digest.lock().unwrap(), + "", + "a container still on the old image must not be reported as synced" + ); + assert!( + !active_image_path(&cfg.writable_root).exists(), + "the pointer must not claim a digest the container is not running" + ); + } + + #[tokio::test] + async fn a_container_that_does_not_exist_yet_is_not_a_mismatch() { + // A device whose unit creates the container lazily has nothing running + // at sync time. Treating an unknown id as a mismatch would break that + // setup to guard the broken one. + let calls = Arc::new(Mutex::new(Vec::new())); + let engine = FakeEngine::no_container_yet(calls); + let tmp = tempfile::tempdir().unwrap(); + let cfg = test_cfg(tmp.path().to_path_buf()); + let state = crate::AgentState::new("dev-01".to_string()); + + on_sync(&engine, &cfg, "my-app", "dev", "sha256:new", &state) + .await + .expect("an absent container must not fail the sync"); + assert_eq!(*state.running_digest.lock().unwrap(), "sha256:new"); + } + // Falsifier 2: the pointer lands on the writable partition, never a rootFS // path, and round-trips {image, tag, digest}. #[tokio::test] @@ -647,7 +999,7 @@ mod tests { "tag:127.0.0.1:15151/my-app@sha256:new->my-app:dev".to_string(), "restart:avocado-dev".to_string(), ], - "the pull and the restart attempt should both be recorded: {recorded:?}" + "a failed restart must abort before the adoption probes: {recorded:?}" ); assert!( !active_image_path(tmp.path()).exists(), @@ -693,6 +1045,8 @@ mod tests { "pull:127.0.0.1:15151/my-app@sha256:new".to_string(), "tag:127.0.0.1:15151/my-app@sha256:new->my-app:dev".to_string(), "restart:avocado-dev".to_string(), + "resolve_image_id:my-app:dev".to_string(), + "running_image_id:avocado-dev".to_string(), ], "the pull and restart must both have run: {recorded:?}" ); From 3ea1443bea70cd20d5aa4fa956d3e6fbf52ce8cf Mon Sep 17 00:00:00 2001 From: Javier Tia Date: Tue, 4 Aug 2026 08:59:46 -0600 Subject: [PATCH 18/20] container-agent-dev: carry the owning unit on the wire and bound the body read The adoption probe added last commit proves a restart took effect, but nothing could satisfy it. `docker restart` re-executes the existing container object, which stays bound to its create-time image id, and the shipped unit exports only RUST_LOG and AVOCADO_CONTAINER_DEV_ENGINE - so `service_from_env()` was always None, every device took the container branch, and the probe turned a silent no-op into a sync that fails forever. Its `if let (Some, Some)` guard also skipped the check whenever either id was unknown, so a device with no container at all passed vacuously and recorded the new digest while running nothing: the same false success, reached through the other door, covering exactly the population the mismatch arm does not. The host already parses the answer. Each `container_dev.images` entry declares the `service:` that consumes the image, so carry it on the Sync frame rather than depending on an operator exporting an env var nothing documents at the point of use. The field is optional, so a device running against a host that does not send it yet parses the frame and falls back to the env override and then to the container branch, exactly as before. Absence of a container is now a failure rather than a pass, with a bounded wait first because `systemctl restart` returns before a Type=simple unit has necessarily created anything. Three lifetime bugs around the sync worker went with it. The worker was aborted after the serve loop, reachable only by falling out of it, while three `?` sites returned early instead - and the stream read is the common session end, since tungstenite yields Some(Err(..)) on a reset and None only on a clean close. That leaked a parked task per errored session and let a surviving worker's pull/tag/restart race the next session's on running_digest and the pointer file. A guard makes the abort unreachable to skip; it holds an AbortHandle rather than the JoinHandle because the session never joins the worker, which also lets a test keep the JoinHandle and await the cancelled JoinError instead of guessing at when the runtime delivers a cancellation. The engine commands gain kill_on_drop so an abort landing mid-pull does not orphan the subprocess. `notify_waiters` stored no permit, so a 401 raised while the select loop body was running another arm found no registered waiter and was dropped - the flag then surfaced at the next session start, which is the one thing moving the read inside the loop was meant to avoid. `notify_one` keeps the permit. Clearing the flag after the send also erased any raise that landed during it, so the lower now happens first and re-raises if the send fails. The stage timeouts covered TCP, TLS, the HTTP/1 handshake and the request head - every stage except the one a sleeping laptop actually stalls in. An idle deadline on the body bounds that without capping a legitimate multi-gigabyte layer, since a link moving bytes at all keeps resetting it. The Status test that came with the reporting path built a DeviceFrame by hand and asserted serde emitted its fields, so reverting the whole branch to a bare warn! left it green. It now drives handle_host_frame, which took the engine as a parameter to make that reachable. tokio's `sync` feature is declared rather than borrowed from hyper's feature unification. --- Cargo.toml | 7 + src/main.rs | 303 ++++++++++++++++++++++++++++++++++++------ src/proxy.rs | 218 +++++++++++++++++++++++++++++- src/sync.rs | 365 +++++++++++++++++++++++++++++++++++++++++++-------- 4 files changed, 789 insertions(+), 104 deletions(-) diff --git a/Cargo.toml b/Cargo.toml index 4d91fd1..3578f29 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -8,6 +8,12 @@ name = "avocado-container-agent-dev" path = "src/main.rs" [dependencies] +# `sync` is used directly (`Notify` for the sync-worker wakeup and the +# re-bootstrap signal, `mpsc` for the outbound frame channel). It builds today +# only because hyper 1.x depends on tokio with `sync` enabled and cargo unifies +# features - `cargo tree -e features -i tokio` shows it sourced from hyper, not +# from here. A hyper bump that drops that dependency would break this crate with +# no local change, so the feature it actually uses is declared. tokio = { version = "1", features = [ "rt-multi-thread", "macros", @@ -15,6 +21,7 @@ tokio = { version = "1", features = [ "time", "io-util", "process", + "sync", ] } # WS client only. We deliberately DO NOT enable tokio-tungstenite's # `rustls-tls-*` features: those pull `tokio-rustls` with its default features, diff --git a/src/main.rs b/src/main.rs index 25cd595..4eaa876 100644 --- a/src/main.rs +++ b/src/main.rs @@ -65,11 +65,21 @@ const LOOPBACK_PORT_ENV: &str = "AVOCADO_CONTAINER_DEV_LOOPBACK_PORT"; #[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)] #[serde(tag = "type", rename_all = "snake_case")] pub enum HostFrame { - /// The `{image, tag, digest}` now available for the device to pull. + /// The `{image, tag, digest}` now available for the device to pull, and the + /// systemd unit that consumes it. Sync { image: String, tag: String, digest: String, + /// The device unit owning this image, from the host's own + /// `container_dev.images[].service`. Optional on the wire so a device + /// running against an older host still parses the frame - it then falls + /// back to `$AVOCADO_CONTAINER_DEV_SERVICE` and finally to restarting + /// the container, which is what every device did before this field + /// existed. See `sync::SERVICE_ENV` for why the container branch cannot + /// adopt a new image. + #[serde(default, skip_serializing_if = "Option::is_none")] + service: Option, }, } @@ -367,16 +377,38 @@ pub(crate) fn endpoint_host(endpoint: &str) -> &str { /// Handle one host frame. On a `Sync`, pull the new image through the loopback /// proxy, rewrite the active-image pointer on the writable partition, restart /// the container, and record the running digest (task 6.3, [`sync::on_sync`]). -async fn handle_host_frame(frame: HostFrame, state: &AgentState) -> Option { +/// The engine and config `handle_host_frame` runs a sync against. +/// +/// Taken as parameters rather than built inside, so a test can drive the +/// reporting path with a fake engine. Building them internally made the failure +/// branch unreachable from a test: the only coverage was a test that constructed +/// the `Status` frame itself and asserted serde round-tripped it, which stayed +/// green with the whole reporting path reverted to a bare `warn!`. +async fn handle_host_frame( + engine: &E, + cfg: &sync::SyncConfig, + frame: HostFrame, + state: &AgentState, +) -> Option { match frame { - HostFrame::Sync { image, tag, digest } => { - info!(%image, %tag, %digest, "received sync; pulling + restarting"); - // The engine shell-out and env-resolved config are the thin, - // untested production glue; the ordering/pointer/digest core lives - // in the fully-tested `sync::on_sync`. - let engine = sync::CommandEngine::from_env(); - let cfg = sync::SyncConfig::from_env(); - match sync::on_sync(&engine, &cfg, &image, &tag, &digest, state).await { + HostFrame::Sync { + image, + tag, + digest, + service, + } => { + info!(%image, %tag, %digest, ?service, "received sync; pulling + restarting"); + match sync::on_sync( + engine, + cfg, + &image, + &tag, + &digest, + service.as_deref(), + state, + ) + .await + { Ok(()) => None, Err(e) => { // Report it, do not only log it. A swallowed failure left the @@ -405,6 +437,32 @@ async fn handle_host_frame(frame: HostFrame, state: &AgentState) -> Option(8); - let worker = { + let _worker = AbortOnDrop({ let pending = Arc::clone(&channels.pending); let work = Arc::clone(&channels.work); let state = Arc::clone(state); @@ -507,7 +565,12 @@ async fn connect_and_serve( let Some(frame) = pending.lock().expect("pending mutex poisoned").take() else { continue; }; - if let Some(report) = handle_host_frame(frame, &state).await + // The engine shell-out and env-resolved config are the thin, + // untested production glue; the ordering/pointer/digest core + // lives in the fully-tested `sync::on_sync`. + let engine = sync::CommandEngine::from_env(); + let cfg = sync::SyncConfig::from_env(); + if let Some(report) = handle_host_frame(&engine, &cfg, frame, &state).await && outbound_tx.send(report).await.is_err() { // The session ended while the sync ran; the next Hello @@ -516,18 +579,16 @@ async fn connect_and_serve( } } }) - }; + .abort_handle() + }); // Dropping our own copy means `outbound` closes when the worker does. drop(outbound_tx); // A re-bootstrap raised BEFORE this session started still needs announcing; // one raised during it arrives through `rebootstrap.raised()` below. - if state.rebootstrap.is_raised() { - send_frame(&mut sink, &rebootstrap_status(state)).await?; - state.rebootstrap.clear(); - } + report_rebootstrap(&mut sink, state).await?; - let result = loop { + loop { tokio::select! { // Biased so an outbound report is written before another host frame // is accepted; without it a busy host could starve the reports the @@ -543,10 +604,7 @@ async fn connect_and_serve( // front meant it was always set strictly after the only read that // would have transmitted it. () = state.rebootstrap.raised() => { - send_frame(&mut sink, &rebootstrap_status(state)).await?; - // Lower it once reported, or one transient 401 latches for the - // process lifetime. - state.rebootstrap.clear(); + report_rebootstrap(&mut sink, state).await?; } msg = stream.next() => { @@ -582,10 +640,34 @@ async fn connect_and_serve( } } } - }; + } + // The sync worker is aborted by `_worker`'s Drop on every exit from here, + // including the `?` early-returns above. +} - worker.abort(); - result +/// Report a raised re-bootstrap to the host, if one is raised. +/// +/// Lowers the flag BEFORE sending. Clearing afterwards erased any raise that +/// landed during the send itself, which is a live window: the proxy raises from +/// its own task while this await is parked, so the 401 that arrived mid-send was +/// dropped with no report and no way to ask again. +/// +/// A failed send re-raises rather than swallowing, because the session is about +/// to end and the pre-loop report on the next one is the only thing that would +/// carry it. +async fn report_rebootstrap(sink: &mut S, state: &AgentState) -> Result<()> +where + S: futures_util::SinkExt + Unpin, + >::Error: std::error::Error + Send + Sync + 'static, +{ + if !state.rebootstrap.take() { + return Ok(()); + } + if let Err(e) = send_frame(sink, &rebootstrap_status(state)).await { + state.rebootstrap.raise(); + return Err(e); + } + Ok(()) } /// The `needs_rebootstrap` status. There is no token-renewal endpoint; the @@ -783,6 +865,8 @@ mod tests { #[test] fn host_sync_frame_deserializes_from_wire() { + // A host that predates the `service` field must still parse, or a CLI + // upgrade becomes mandatory for every device in the field. let wire = r#"{"type":"sync","image":"my-app","tag":"dev","digest":"sha256:abc"}"#; let frame: HostFrame = serde_json::from_str(wire).expect("sync frame parses"); assert_eq!( @@ -791,6 +875,25 @@ mod tests { image: "my-app".to_string(), tag: "dev".to_string(), digest: "sha256:abc".to_string(), + service: None, + } + ); + } + + #[test] + fn host_sync_frame_carries_the_owning_service() { + // The field that makes the sync actually take effect: without it the + // agent restarts the container, which stays bound to its create-time + // image id, so every shipped device no-ops. + let wire = r#"{"type":"sync","image":"my-app","tag":"dev","digest":"sha256:abc","service":"app.service"}"#; + let frame: HostFrame = serde_json::from_str(wire).expect("sync frame parses"); + assert_eq!( + frame, + HostFrame::Sync { + image: "my-app".to_string(), + tag: "dev".to_string(), + digest: "sha256:abc".to_string(), + service: Some("app.service".to_string()), } ); } @@ -989,7 +1092,7 @@ mod tests { // it. Nothing could lower it either, so one transient 401 latched for // the process lifetime. let flag = proxy::ReBootstrap::default(); - assert!(!flag.is_raised()); + assert!(!flag.is_raised(), "a fresh flag must start lowered"); let waiter = { let flag = flag.clone(); @@ -1004,28 +1107,150 @@ mod tests { .expect("raise must wake a mid-session waiter") .expect("waiter task"); - assert!(flag.is_raised()); - flag.clear(); + assert!(flag.take(), "take must report that the flag was raised"); assert!( - !flag.is_raised(), + !flag.take(), "the flag must lower once reported, or every later session re-announces it" ); } - #[test] - fn a_failed_sync_produces_a_status_frame_for_the_host() { + /// An engine whose pull always fails, so `handle_host_frame` takes its + /// reporting branch without execing anything. + struct FailingEngine; + + impl sync::Engine for FailingEngine { + async fn pull(&self, _reference: &str) -> Result<()> { + anyhow::bail!("simulated pull failure") + } + async fn tag(&self, _source: &str, _target: &str) -> Result<()> { + Ok(()) + } + async fn restart(&self, _container: &str, _service: Option<&str>) -> Result<()> { + Ok(()) + } + async fn running_image_id(&self, _container: &str) -> Result> { + Ok(None) + } + async fn resolve_image_id(&self, _reference: &str) -> Result> { + Ok(None) + } + } + + #[tokio::test] + async fn the_session_guard_aborts_its_worker_on_drop() { + // The abort used to sit after the serve loop, so the three `?` sites + // inside it - the two send_frame calls and the stream read - returned + // without it. The read one is the COMMON session end (tungstenite yields + // Some(Err(..)) on a reset), leaking one parked worker per errored + // session, and letting the old worker's pull/tag/restart race the new + // session's on `running_digest` and the pointer file. + // + // The guard makes that unreachable by construction: every exit from + // `connect_and_serve`, including a `?`, drops it. + // A worker that never finishes on its own, so the only way this await + // resolves is the guard cancelling it. + let handle = tokio::spawn(std::future::pending::<()>()); + let guard = AbortOnDrop(handle.abort_handle()); + + drop(guard); + + let err = handle + .await + .expect_err("dropping the guard must abort the worker, not leave it parked"); + assert!( + err.is_cancelled(), + "the worker must end cancelled rather than panicking: {err}" + ); + } + + #[tokio::test] + async fn a_failed_sync_produces_a_status_frame_for_the_host() { // A swallowed sync failure left the host with no Status, no Progress and // no updated Hello - Hello is re-sent only on reconnect - so it went on // showing the device synced at the old digest with no error surface. + // + // This drives `handle_host_frame` itself. The previous version built a + // DeviceFrame::Status inside the test body and asserted serde_json + // emitted its fields, so it covered serde and nothing else: reverting + // the whole reporting path back to a bare `warn!` returning None left it + // green. let state = AgentState::new("dev-01".to_string()); - let frame = DeviceFrame::Status(Status { - device_id: state.device_id.clone(), - state: "sync_failed".to_string(), - detail: Some("my-app:dev @ sha256:new: boom".to_string()), - }); - let json = serde_json::to_string(&frame).expect("serializes"); - assert!(json.contains("sync_failed"), "{json}"); - assert!(json.contains("dev-01"), "{json}"); + let tmp = tempfile::tempdir().unwrap(); + let cfg = sync::SyncConfig { + loopback_port: 15151, + container: "avocado-dev".to_string(), + writable_root: tmp.path().to_path_buf(), + }; + let frame = HostFrame::Sync { + image: "my-app".to_string(), + tag: "dev".to_string(), + digest: "sha256:new".to_string(), + service: None, + }; + + let report = handle_host_frame(&FailingEngine, &cfg, frame, &state) + .await + .expect("a failed sync must report a Status frame to the host"); + + match report { + DeviceFrame::Status(status) => { + assert_eq!(status.device_id, "dev-01"); + assert_eq!(status.state, "sync_failed"); + let detail = status.detail.expect("the failure must carry a detail"); + assert!( + detail.contains("my-app:dev") && detail.contains("sha256:new"), + "the detail must name the image that failed: {detail}" + ); + } + other => panic!("expected a Status frame, got {other:?}"), + } + } + + #[tokio::test] + async fn a_successful_sync_reports_nothing() { + // The mirror: `handle_host_frame` must stay silent on success, or every + // sync would emit a spurious Status. Without this the test above passes + // for an implementation that reports unconditionally. + struct OkEngine; + impl sync::Engine for OkEngine { + async fn pull(&self, _reference: &str) -> Result<()> { + Ok(()) + } + async fn tag(&self, _source: &str, _target: &str) -> Result<()> { + Ok(()) + } + async fn restart(&self, _container: &str, _service: Option<&str>) -> Result<()> { + Ok(()) + } + async fn running_image_id(&self, _container: &str) -> Result> { + Ok(Some("sha256:same".to_string())) + } + async fn resolve_image_id(&self, _reference: &str) -> Result> { + Ok(Some("sha256:same".to_string())) + } + } + + let state = AgentState::new("dev-01".to_string()); + let tmp = tempfile::tempdir().unwrap(); + let cfg = sync::SyncConfig { + loopback_port: 15151, + container: "avocado-dev".to_string(), + writable_root: tmp.path().to_path_buf(), + }; + let frame = HostFrame::Sync { + image: "my-app".to_string(), + tag: "dev".to_string(), + digest: "sha256:new".to_string(), + service: None, + }; + + assert!( + handle_host_frame(&OkEngine, &cfg, frame, &state) + .await + .is_none(), + "a successful sync must not report a Status frame" + ); + assert_eq!(*state.running_digest.lock().unwrap(), "sha256:new"); } #[test] diff --git a/src/proxy.rs b/src/proxy.rs index 55470ba..c5adb88 100644 --- a/src/proxy.rs +++ b/src/proxy.rs @@ -158,22 +158,40 @@ pub(crate) struct ReBootstrap { impl ReBootstrap { /// Raise the flag and wake anyone waiting to report it. Idempotent in the /// flag; the wake is what makes it reach the host mid-session. + /// + /// `notify_one`, not `notify_waiters`: the reporting select arm builds a + /// fresh `notified()` future on every iteration, so it is only registered as + /// a waiter while parked in the select. A 401 raised while the loop body ran + /// another arm - mid-`send_frame`, say - found no waiter, and + /// `notify_waiters` stores no permit, so the wake was dropped. The flag + /// stayed set and surfaced at the next session start, which is exactly the + /// mid-session delivery this was supposed to fix. `notify_one` stores the + /// permit, so the next `notified()` completes immediately. pub(crate) fn raise(&self) { self.raised.store(true, Ordering::SeqCst); - self.changed.notify_waiters(); + self.changed.notify_one(); } /// Whether a re-bootstrap has been requested. + /// + /// Test-only: the reporting path reads the flag exclusively through + /// [`Self::take`], because observing and lowering have to be one step for a + /// raise landing mid-send not to be erased. A non-consuming read is still + /// what the proxy's own tests want - "the 401 raised it", "the 200 did not" + /// - and those must not lower it as a side effect. + #[cfg(test)] pub(crate) fn is_raised(&self) -> bool { self.raised.load(Ordering::SeqCst) } - /// Lower the flag once it has been reported to the host. + /// Lower the flag, reporting whether it had been raised. /// - /// Without this a single transient 401 kept every later session announcing - /// `needs_rebootstrap`, long after the token it referred to was replaced. - pub(crate) fn clear(&self) { - self.raised.store(false, Ordering::SeqCst); + /// Atomic so the caller can lower it BEFORE reporting without racing a + /// concurrent `raise`. Lowering is what keeps a single transient 401 from + /// making every later session announce `needs_rebootstrap` long after the + /// token it referred to was replaced. + pub(crate) fn take(&self) -> bool { + self.raised.swap(false, Ordering::SeqCst) } /// Resolve when the flag is next raised. @@ -339,6 +357,82 @@ impl Upstream for HttpsUpstream { // Proxy logic. // --------------------------------------------------------------------------- +/// Ceiling on the gap between two body chunks from the bulk upstream. +/// +/// [`UPSTREAM_STAGE_TIMEOUT`] bounds every stage BEFORE the body, but the stall +/// those were written for - a laptop that sleeps or drops off Wi-Fi mid-pull, +/// leaving a half-open connection with no RST - lands overwhelmingly DURING body +/// transfer, which was the one stage still uncovered. The outcome had been +/// softened rather than removed: the read loop keeps the WS polled so the socket +/// no longer wedges, but the sync worker parked forever, later `Sync` frames +/// piled into the latest-wins slot and never ran, no `Status` was emitted +/// because `on_sync` never returned, and systemd still reported +/// `active (running)`. +/// +/// An IDLE deadline, not a total-transfer one: capping total transfer would kill +/// a legitimate multi-gigabyte layer over a slow link, while a link that is +/// moving bytes at all keeps resetting this. +const UPSTREAM_BODY_IDLE_TIMEOUT: Duration = Duration::from_secs(60); + +/// Wraps a body so a gap of [`UPSTREAM_BODY_IDLE_TIMEOUT`] between chunks ends +/// the stream with an error instead of hanging forever. +struct IdleTimeoutBody { + inner: B, + idle: std::pin::Pin>, +} + +impl IdleTimeoutBody { + fn new(inner: B) -> Self { + Self { + inner, + idle: Box::pin(sleep(UPSTREAM_BODY_IDLE_TIMEOUT)), + } + } +} + +impl hyper::body::Body for IdleTimeoutBody +where + B: hyper::body::Body + Unpin, +{ + type Data = Bytes; + type Error = std::io::Error; + + fn poll_frame( + mut self: std::pin::Pin<&mut Self>, + cx: &mut std::task::Context<'_>, + ) -> std::task::Poll, std::io::Error>>> + { + use std::task::Poll; + + let this = self.as_mut().get_mut(); + match std::pin::Pin::new(&mut this.inner).poll_frame(cx) { + // Any progress - a chunk, the end of the stream, or an error - + // restarts the clock from here. + Poll::Ready(frame) => { + this.idle + .as_mut() + .reset(tokio::time::Instant::now() + UPSTREAM_BODY_IDLE_TIMEOUT); + Poll::Ready(frame) + } + Poll::Pending => match this.idle.as_mut().poll(cx) { + Poll::Ready(()) => Poll::Ready(Some(Err(std::io::Error::new( + std::io::ErrorKind::TimedOut, + format!( + "the bulk upstream sent no body data for {}s; the host is unreachable \ + or the connection is half-open", + UPSTREAM_BODY_IDLE_TIMEOUT.as_secs() + ), + )))), + Poll::Pending => Poll::Pending, + }, + } + } + + fn size_hint(&self) -> hyper::body::SizeHint { + self.inner.size_hint() + } +} + /// Build a small static-body engine response (used for every gateway error). fn gateway_error(status: StatusCode, message: &'static str) -> Response { let body = Full::new(Bytes::from_static(message.as_bytes())) @@ -383,7 +477,7 @@ pub(crate) async fn proxy_read( } } builder - .body(upstream.body) + .body(IdleTimeoutBody::new(upstream.body).boxed()) .expect("engine success response is always valid") } Ok(upstream) @@ -832,6 +926,47 @@ mod tests { } } + /// A 200 whose body never delivers a byte. + struct StallingUpstream; + + impl Upstream for StallingUpstream { + async fn fetch( + &self, + _method: Method, + _path: String, + _range: Option, + ) -> Result { + Ok(UpstreamResponse { + status: StatusCode::OK, + headers: HeaderMap::new(), + body: StallingBody.boxed(), + }) + } + } + + #[tokio::test(start_paused = true)] + async fn a_stalled_upstream_body_fails_through_the_proxy() { + // The wiring, not just the wrapper: reverting `proxy_read` to hand the + // upstream body straight to the response leaves the two IdleTimeoutBody + // unit tests green while every real pull can still hang forever. + let resp = proxy_read( + Method::GET, + "/v2/app/blobs/sha256:abc".to_string(), + None, + &StallingUpstream, + &ReBootstrap::default(), + ) + .await; + assert_eq!(resp.status(), StatusCode::OK); + + let err = resp + .into_body() + .collect() + .await + .expect_err("a stalled upstream body must not hang the engine's pull"); + assert_eq!(err.kind(), std::io::ErrorKind::TimedOut); + } + async fn collect(resp: Response) -> (StatusCode, HeaderMap, Bytes) { let status = resp.status(); let headers = resp.headers().clone(); @@ -846,6 +981,75 @@ mod tests { // --- proxy-logic tests ------------------------------------------------- + /// A body that never produces a frame and never wakes its waker - a + /// half-open connection to a host that went to sleep mid-layer. + struct StallingBody; + + impl hyper::body::Body for StallingBody { + type Data = Bytes; + type Error = std::io::Error; + + fn poll_frame( + self: std::pin::Pin<&mut Self>, + _cx: &mut std::task::Context<'_>, + ) -> std::task::Poll, std::io::Error>>> + { + std::task::Poll::Pending + } + } + + #[tokio::test(start_paused = true)] + async fn a_raise_with_no_waiter_registered_is_still_delivered() { + // The reporting arm builds a fresh `notified()` future on every select + // iteration, so it counts as a registered waiter only while parked in + // the select. A 401 raised while the loop body ran another arm - + // mid-`send_frame`, say - found no waiter, and `notify_waiters` stores + // no permit, so the wake was dropped. The flag stayed set and surfaced + // at the next session start, which is exactly the mid-session delivery + // the Notify was added for. + let flag = ReBootstrap::default(); + flag.raise(); + + tokio::time::timeout(Duration::from_secs(5), flag.raised()) + .await + .expect("a raise with no waiter must still wake the next waiter"); + } + + #[tokio::test(start_paused = true)] + async fn a_body_that_stalls_mid_stream_fails_instead_of_hanging() { + // The stage timeouts cover TCP, TLS, the HTTP/1 handshake and the + // request head - every stage EXCEPT the one a sleeping laptop actually + // stalls in. Unwrapped, this collect never returns: `docker pull` waits + // on a loopback response that never arrives, `on_sync` never returns, no + // Status is ever emitted, and systemd still reports active (running). + let err = IdleTimeoutBody::new(StallingBody) + .collect() + .await + .expect_err("a stalled body must not hang forever"); + assert_eq!( + err.kind(), + std::io::ErrorKind::TimedOut, + "a stalled body must fail as a timeout: {err}" + ); + assert!( + err.to_string().contains("no body data"), + "the error must say the stall was in the body: {err}" + ); + } + + #[tokio::test(start_paused = true)] + async fn a_body_that_delivers_is_passed_through_untouched() { + // The mirror: the wrapper must not truncate or corrupt a working + // stream, or every pull breaks. Without this the test above passes for a + // wrapper that errors unconditionally. + let bytes = IdleTimeoutBody::new(full("LAYER-BYTES")) + .collect() + .await + .expect("a delivering body must not be cut off") + .to_bytes(); + assert_eq!(&bytes[..], b"LAYER-BYTES"); + } + #[tokio::test] async fn proxy_forwards_upstream_200_body_verbatim() { let upstream = FakeUpstream { diff --git a/src/sync.rs b/src/sync.rs index 64b20c5..19694f6 100644 --- a/src/sync.rs +++ b/src/sync.rs @@ -30,6 +30,7 @@ use std::future::Future; use std::path::{Path, PathBuf}; +use std::time::Duration; use anyhow::{Context, Result, ensure}; use serde::{Deserialize, Serialize}; @@ -55,14 +56,22 @@ const CONTAINER_ENV: &str = "AVOCADO_CONTAINER_DEV_CONTAINER"; /// Env var overriding the writable state root (used by tests and staging). const STATE_ROOT_ENV: &str = "AVOCADO_CONTAINER_DEV_STATE_ROOT"; /// Env var naming the systemd unit that OWNS the container, restarted instead of -/// the container itself when set. +/// the container itself when set. A per-device override; the normal source is +/// the `Sync` frame's `service`, which the host fills from the project's own +/// `container_dev.images[].service`. /// /// ` restart ` re-executes the container's existing config, /// which pins the image by ID at create time - so a freshly pulled image for the /// same tag is ignored and the sync silently no-ops while reporting success. The /// unit that launched the container re-runs ` run` on restart, which -/// re-resolves the tag and therefore adopts the new image. This mirrors the -/// `service:` field each entry under `container_dev.images` already declares. +/// re-resolves the tag and therefore adopts the new image. +/// +/// Nothing set this on a shipped device: the unit exports only `RUST_LOG` and +/// `AVOCADO_CONTAINER_DEV_ENGINE`, and the agent had no other way to learn a +/// unit name, so every installation took the container branch and every sync +/// no-opped. The `service:` field the host already parses is the answer, and it +/// now rides the frame rather than depending on an operator setting this by +/// hand. const SERVICE_ENV: &str = "AVOCADO_CONTAINER_DEV_SERVICE"; // --------------------------------------------------------------------------- @@ -117,7 +126,17 @@ pub(crate) trait Engine: Send + Sync + 'static { fn pull(&self, reference: &str) -> impl Future> + Send; /// Point `target` (the ref the service names) at `source` (what we just pulled). fn tag(&self, source: &str, target: &str) -> impl Future> + Send; - fn restart(&self, container: &str) -> impl Future> + Send; + /// Restart whatever owns the image so the pulled build takes effect. + /// + /// `service` is the systemd unit the HOST says owns this image (the + /// `service:` field on the matching `container_dev.images` entry, carried on + /// the `Sync` frame). It is the only input that makes the default + /// installation work: see [`SERVICE_ENV`]. + fn restart( + &self, + container: &str, + service: Option<&str>, + ) -> impl Future> + Send; /// The image ID `container` is currently running, or `None` when the /// container does not exist. /// @@ -265,6 +284,21 @@ pub(crate) fn service_from_env() -> Option { .filter(|v| !v.is_empty()) } +/// Which systemd unit to restart, given the frame's `service` and the env +/// override. `None` keeps the container-restart path. +/// +/// Split out so the precedence is testable without mutating process env: the +/// frame wins because the host reads it from the project's own +/// `container_dev.images[].service`, and a blank or whitespace-only value on the +/// wire is treated as absent rather than as a unit named "". +fn restart_unit(frame_service: Option<&str>, env_service: Option) -> Option { + frame_service + .map(str::trim) + .filter(|v| !v.is_empty()) + .map(str::to_string) + .or(env_service) +} + impl Engine for CommandEngine { async fn pull(&self, reference: &str) -> Result<()> { run_engine(&self.binary, &["pull", reference]).await @@ -274,12 +308,15 @@ impl Engine for CommandEngine { run_engine(&self.binary, &["tag", source, target]).await } - /// Restart the owning systemd unit when one is configured, else the container. + /// Restart the owning systemd unit when one is known, else the container. /// - /// See [`SERVICE_ENV`]: restarting the container re-executes its pinned image - /// ID, so the pulled image would never actually run. - async fn restart(&self, container: &str) -> Result<()> { - match service_from_env() { + /// The frame's service wins over the env var: the host reads it from the + /// project's own `container_dev.images[].service`, which is where the answer + /// actually lives, while the env var is a per-device override for a unit the + /// host does not know about. See [`SERVICE_ENV`] for why restarting the + /// container is not enough. + async fn restart(&self, container: &str, service: Option<&str>) -> Result<()> { + match restart_unit(service, service_from_env()) { Some(service) => { info!(service = %service, "restarting owning systemd unit to adopt the pulled image"); run_engine("systemctl", &["restart", &service]).await @@ -314,6 +351,7 @@ impl Engine for CommandEngine { async fn engine_output(binary: &str, args: &[&str]) -> Result> { let out = Command::new(binary) .args(args) + .kill_on_drop(true) .output() .await .with_context(|| format!("spawning `{binary} {}`", args.join(" ")))?; @@ -325,9 +363,16 @@ async fn engine_output(binary: &str, args: &[&str]) -> Result> { } /// Run ` ` to completion, mapping a non-zero exit to an error. +/// +/// `kill_on_drop` because the sync runs on a task the session loop aborts when +/// the link drops (`main.rs`'s `AbortOnDrop`). An abort lands on whichever await +/// point is current, and without this the in-flight `docker pull` would keep +/// running with nothing owning it - burning bandwidth on a superseded digest +/// while the next session starts its own pull. async fn run_engine(binary: &str, args: &[&str]) -> Result<()> { let status = Command::new(binary) .args(args) + .kill_on_drop(true) .status() .await .with_context(|| format!("spawning `{binary} {}`", args.join(" ")))?; @@ -373,6 +418,40 @@ impl SyncConfig { } } +/// How long a restarted unit gets to bring its container up before the sync +/// treats the absence as a failure. +/// +/// `systemctl restart` returns once the unit is active, which for a `Type=simple` +/// unit is before its ` run` has necessarily created the container. A +/// single immediate probe would race that and fail a sync that was about to +/// work, so the check waits rather than asking once. +const CONTAINER_APPEAR_TIMEOUT: Duration = Duration::from_secs(30); + +/// Gap between container-existence probes inside [`CONTAINER_APPEAR_TIMEOUT`]. +const CONTAINER_POLL_INTERVAL: Duration = Duration::from_millis(500); + +/// The image id running in `container`, waiting up to +/// [`CONTAINER_APPEAR_TIMEOUT`] for the container to exist at all. +/// +/// Returns `None` only when it never appeared, which the caller treats as a +/// failed sync. +async fn running_image_id_once_up( + engine: &E, + container: &str, +) -> Result> { + let deadline = tokio::time::Instant::now() + CONTAINER_APPEAR_TIMEOUT; + loop { + let running = engine + .running_image_id(container) + .await + .with_context(|| format!("reading the image id running in {container}"))?; + if running.is_some() || tokio::time::Instant::now() >= deadline { + return Ok(running); + } + tokio::time::sleep(CONTAINER_POLL_INTERVAL).await; + } +} + /// Handle a `Sync`: pull -> restart -> record digest -> rewrite pointer. /// /// The order is load-bearing at both ends. A pull failure aborts before the @@ -388,6 +467,7 @@ pub(crate) async fn on_sync( image: &str, tag: &str, digest: &str, + service: Option<&str>, state: &AgentState, ) -> Result<()> { // 1. Pull through the device loopback proxy (never over the control WS). @@ -412,7 +492,7 @@ pub(crate) async fn on_sync( // 3. Restart the container AFTER the pull and tag so the new image takes effect. engine - .restart(&cfg.container) + .restart(&cfg.container, service) .await .with_context(|| format!("restarting container {}", cfg.container))?; @@ -428,30 +508,47 @@ pub(crate) async fn on_sync( // developer saw unchanged behaviour with every layer reporting success. // // Comparing IDs is engine-generic and catches the failure on whichever - // branch produced it. An unknown ID on either side is NOT treated as a - // mismatch: the container may legitimately not exist yet on a device - // whose unit creates it lazily, and inventing a failure there would - // break a working setup to guard a broken one. + // branch produced it. BOTH ids must be known: an unknown one is a + // failure, not a pass. Skipping the check when the container is absent + // read as tolerance for a unit that creates its container lazily, but + // there is no such case left - on the container branch `restart` has + // already failed with "no such container" before reaching here, and on + // the service branch the unit was just restarted, so a container that is + // still missing once it has had time to appear means the restart did not + // produce one. Passing there recorded the new digest for a device + // running nothing at all, which is the same false success this probe + // exists to end, reached by the other door. let wanted = engine .resolve_image_id(&service_ref) .await - .with_context(|| format!("resolving the image id of {service_ref}"))?; - let running = engine - .running_image_id(&cfg.container) - .await - .with_context(|| format!("reading the image id running in {}", cfg.container))?; - if let (Some(wanted), Some(running)) = (wanted.as_ref(), running.as_ref()) - && wanted != running - { - anyhow::bail!( - "restart did not adopt the pulled image: container {} still runs image {running}, \ - expected {wanted}. `{}` restarts the existing container object, which stays bound \ - to its create-time image id; set AVOCADO_CONTAINER_DEV_SERVICE to the systemd unit \ - that owns the container so the restart re-runs it and re-resolves the tag", - cfg.container, - cfg.container, - ); - } + .with_context(|| format!("resolving the image id of {service_ref}"))? + .ok_or_else(|| { + anyhow::anyhow!( + "{service_ref} resolves to no image even though the tag step just succeeded; \ + there is nothing for the container to run" + ) + })?; + let running = running_image_id_once_up(engine, &cfg.container) + .await? + .ok_or_else(|| { + anyhow::anyhow!( + "restart succeeded but container {} does not exist after {}s, so nothing is \ + running the pulled image. Check that the restarted unit actually starts it", + cfg.container, + CONTAINER_APPEAR_TIMEOUT.as_secs(), + ) + })?; + ensure!( + wanted == running, + "restart did not adopt the pulled image: container {} still runs image {running}, \ + expected {wanted}. `{} restart {}` re-executes the existing container object, which \ + stays bound to its create-time image id; name the systemd unit that owns the container \ + in this image's `service:` field under `container_dev.images` so the restart re-runs \ + the unit and re-resolves the tag", + cfg.container, + engine_binary_from_env(), + cfg.container, + ); // 4. Record the now-running digest. This happens IMMEDIATELY after the // restart, before the pointer write, because at this instant the @@ -522,6 +619,9 @@ mod tests { /// What ` inspect ` answers AFTER the restart. The /// default matches `wanted_id`, i.e. a restart that adopted the image. running_id: Option, + /// Probes that answer "no such container" before `running_id` is + /// returned - a unit whose container takes a moment to come up. + absent_probes: Mutex, } impl FakeEngine { @@ -533,6 +633,7 @@ mod tests { fail_restart: false, wanted_id: Some("sha256:newid".to_string()), running_id: Some("sha256:newid".to_string()), + absent_probes: Mutex::new(0), } } @@ -572,14 +673,32 @@ mod tests { } } - /// A device whose unit creates the container lazily: nothing is running - /// yet, so there is no id to compare against. - fn no_container_yet(calls: Arc>>) -> Self { + /// A restart that reported success but never produced a container. + fn container_never_appears(calls: Arc>>) -> Self { Self { running_id: None, ..Self::new(calls) } } + + /// A unit whose container takes `probes` polls to come up - what a + /// `Type=simple` unit does, since `systemctl restart` returns before its + /// ` run` has necessarily created anything. + fn container_appears_after(calls: Arc>>, probes: usize) -> Self { + Self { + absent_probes: Mutex::new(probes), + ..Self::new(calls) + } + } + + /// A tag step that reported success but left the ref resolving to + /// nothing. + fn tag_resolves_to_nothing(calls: Arc>>) -> Self { + Self { + wanted_id: None, + ..Self::new(calls) + } + } } impl Engine for FakeEngine { @@ -602,11 +721,11 @@ mod tests { Ok(()) } - async fn restart(&self, container: &str) -> Result<()> { - self.calls - .lock() - .unwrap() - .push(format!("restart:{container}")); + async fn restart(&self, container: &str, service: Option<&str>) -> Result<()> { + self.calls.lock().unwrap().push(match service { + Some(service) => format!("restart-unit:{service}"), + None => format!("restart:{container}"), + }); if self.fail_restart { anyhow::bail!("simulated restart failure"); } @@ -618,6 +737,11 @@ mod tests { .lock() .unwrap() .push(format!("running_image_id:{container}")); + let mut absent = self.absent_probes.lock().unwrap(); + if *absent > 0 { + *absent -= 1; + return Ok(None); + } Ok(self.running_id.clone()) } @@ -649,7 +773,7 @@ mod tests { let cfg = test_cfg(tmp.path().to_path_buf()); let state = crate::AgentState::new("dev-01".to_string()); - on_sync(&engine, &cfg, "my-app", "dev", "sha256:new", &state) + on_sync(&engine, &cfg, "my-app", "dev", "sha256:new", None, &state) .await .expect("sync succeeds"); @@ -686,7 +810,7 @@ mod tests { let cfg = test_cfg(tmp.path().to_path_buf()); let state = crate::AgentState::new("dev-01".to_string()); - on_sync(&engine, &cfg, "my-app", "dev", "sha256:new", &state) + on_sync(&engine, &cfg, "my-app", "dev", "sha256:new", None, &state) .await .expect("sync succeeds"); @@ -776,7 +900,7 @@ mod tests { let cfg = test_cfg(tmp.path().to_path_buf()); let state = crate::AgentState::new("dev-01".to_string()); - let err = on_sync(&engine, &cfg, "my-app", "dev", "sha256:new", &state) + let err = on_sync(&engine, &cfg, "my-app", "dev", "sha256:new", None, &state) .await .expect_err("a failed tag must fail the sync"); assert!( @@ -813,7 +937,7 @@ mod tests { let cfg = test_cfg(tmp.path().to_path_buf()); let state = crate::AgentState::new("dev-01".to_string()); - let err = on_sync(&engine, &cfg, "my-app", "dev", "sha256:new", &state) + let err = on_sync(&engine, &cfg, "my-app", "dev", "sha256:new", None, &state) .await .expect_err("a restart that kept the old image must fail the sync"); assert!( @@ -821,8 +945,9 @@ mod tests { "the error must say the restart did not adopt the image: {err:#}" ); assert!( - format!("{err:#}").contains("AVOCADO_CONTAINER_DEV_SERVICE"), - "the error must name the remedy: {err:#}" + format!("{err:#}").contains("`service:` field"), + "the error must name the remedy - the config field, not the env \ + override, since the field is what the host reads: {err:#}" ); assert_eq!( *state.running_digest.lock().unwrap(), @@ -835,21 +960,145 @@ mod tests { ); } - #[tokio::test] - async fn a_container_that_does_not_exist_yet_is_not_a_mismatch() { - // A device whose unit creates the container lazily has nothing running - // at sync time. Treating an unknown id as a mismatch would break that - // setup to guard the broken one. + #[tokio::test(start_paused = true)] + async fn a_container_that_never_appears_fails_the_sync() { + // The probe used to skip the check whenever either id was unknown, so a + // device with no container at all passed vacuously and recorded the new + // digest - reporting "running sha256:new" while running nothing. That is + // the same false success the probe exists to end, reached through the + // other branch, and it covered exactly the population the mismatch arm + // does not. let calls = Arc::new(Mutex::new(Vec::new())); - let engine = FakeEngine::no_container_yet(calls); + let engine = FakeEngine::container_never_appears(calls); let tmp = tempfile::tempdir().unwrap(); let cfg = test_cfg(tmp.path().to_path_buf()); let state = crate::AgentState::new("dev-01".to_string()); - on_sync(&engine, &cfg, "my-app", "dev", "sha256:new", &state) + let err = on_sync(&engine, &cfg, "my-app", "dev", "sha256:new", None, &state) .await - .expect("an absent container must not fail the sync"); + .expect_err("a container that never appears must fail the sync"); + assert!( + format!("{err:#}").contains("does not exist"), + "the error must say the container is absent: {err:#}" + ); + assert_eq!( + *state.running_digest.lock().unwrap(), + "", + "a device running no container must not report the new digest" + ); + assert!( + !active_image_path(&cfg.writable_root).exists(), + "the pointer must not claim an image nothing is running" + ); + } + + #[tokio::test(start_paused = true)] + async fn a_container_that_takes_a_moment_to_come_up_still_syncs() { + // `systemctl restart` returns when the unit is active, which for a + // Type=simple unit is before its ` run` has necessarily created + // the container. Asking once and failing would break every service-owned + // device the frame's `service:` field is meant to fix, so absence is + // only fatal after CONTAINER_APPEAR_TIMEOUT. + let calls = Arc::new(Mutex::new(Vec::new())); + let engine = FakeEngine::container_appears_after(calls.clone(), 3); + let tmp = tempfile::tempdir().unwrap(); + let cfg = test_cfg(tmp.path().to_path_buf()); + let state = crate::AgentState::new("dev-01".to_string()); + + on_sync(&engine, &cfg, "my-app", "dev", "sha256:new", None, &state) + .await + .expect("a container that comes up within the window must sync"); + assert_eq!(*state.running_digest.lock().unwrap(), "sha256:new"); + let probes = calls + .lock() + .unwrap() + .iter() + .filter(|c| c.starts_with("running_image_id:")) + .count(); + assert_eq!(probes, 4, "three absent probes then the one that answered"); + } + + #[tokio::test(start_paused = true)] + async fn a_tag_that_resolves_to_nothing_fails_the_sync() { + // The tag step reported success, so the ref must resolve. If it does + // not, there is no image for the container to run and the old probe + // passed vacuously on the unknown id. + let calls = Arc::new(Mutex::new(Vec::new())); + let engine = FakeEngine::tag_resolves_to_nothing(calls); + let tmp = tempfile::tempdir().unwrap(); + let cfg = test_cfg(tmp.path().to_path_buf()); + let state = crate::AgentState::new("dev-01".to_string()); + + let err = on_sync(&engine, &cfg, "my-app", "dev", "sha256:new", None, &state) + .await + .expect_err("an unresolvable service ref must fail the sync"); + assert!( + format!("{err:#}").contains("resolves to no image"), + "the error must name the unresolvable ref: {err:#}" + ); + assert_eq!(*state.running_digest.lock().unwrap(), ""); + } + + #[tokio::test] + async fn the_frames_service_is_what_gets_restarted() { + // The whole point of carrying `service` on the wire: without it the + // agent restarts the container, which stays bound to its create-time + // image id, and the sync no-ops on every shipped device. + let calls = Arc::new(Mutex::new(Vec::new())); + let engine = FakeEngine::new(calls.clone()); + let tmp = tempfile::tempdir().unwrap(); + let cfg = test_cfg(tmp.path().to_path_buf()); + let state = crate::AgentState::new("dev-01".to_string()); + + on_sync( + &engine, + &cfg, + "my-app", + "dev", + "sha256:new", + Some("app.service"), + &state, + ) + .await + .expect("sync succeeds"); + + let recorded = calls.lock().unwrap().clone(); + assert!( + recorded.contains(&"restart-unit:app.service".to_string()), + "the frame's service must be what is restarted: {recorded:?}" + ); + assert!( + !recorded.iter().any(|c| c.starts_with("restart:")), + "the container must NOT be restarted when a service is named: {recorded:?}" + ); + } + + #[test] + fn the_frames_service_beats_the_env_override_and_blanks_are_ignored() { + // The host reads `service:` from the project's own config, so it is the + // better answer; the env var stays as a per-device override for a unit + // the host does not know about. A blank on the wire must fall through to + // it rather than name a unit called "". + assert_eq!( + restart_unit(Some("app.service"), Some("override.service".to_string())), + Some("app.service".to_string()) + ); + assert_eq!( + restart_unit(None, Some("override.service".to_string())), + Some("override.service".to_string()) + ); + assert_eq!( + restart_unit(Some(" "), Some("override.service".to_string())), + Some("override.service".to_string()), + "a blank service on the wire must not shadow the env override" + ); + assert_eq!(restart_unit(Some(""), None), None); + assert_eq!( + restart_unit(None, None), + None, + "no service anywhere keeps the container-restart path" + ); } // Falsifier 2: the pointer lands on the writable partition, never a rootFS @@ -863,7 +1112,7 @@ mod tests { let cfg = test_cfg(root.clone()); let state = crate::AgentState::new("dev-01".to_string()); - on_sync(&engine, &cfg, "my-app", "dev", "sha256:new", &state) + on_sync(&engine, &cfg, "my-app", "dev", "sha256:new", None, &state) .await .expect("sync succeeds"); @@ -925,7 +1174,7 @@ mod tests { let cfg = test_cfg(tmp.path().to_path_buf()); let state = crate::AgentState::new("dev-01".to_string()); - on_sync(&engine, &cfg, "my-app", "dev", "sha256:new", &state) + on_sync(&engine, &cfg, "my-app", "dev", "sha256:new", None, &state) .await .expect("sync succeeds"); @@ -944,7 +1193,7 @@ mod tests { // A prior digest that must survive an aborted sync unchanged. *state.running_digest.lock().unwrap() = "sha256:old".to_string(); - let err = on_sync(&engine, &cfg, "my-app", "dev", "sha256:new", &state) + let err = on_sync(&engine, &cfg, "my-app", "dev", "sha256:new", None, &state) .await .expect_err("pull failure must abort the sync"); assert!( @@ -983,7 +1232,7 @@ mod tests { let state = crate::AgentState::new("dev-01".to_string()); *state.running_digest.lock().unwrap() = "sha256:old".to_string(); - let err = on_sync(&engine, &cfg, "my-app", "dev", "sha256:new", &state) + let err = on_sync(&engine, &cfg, "my-app", "dev", "sha256:new", None, &state) .await .expect_err("restart failure must fail the sync"); assert!( @@ -1034,7 +1283,7 @@ mod tests { let state = crate::AgentState::new("dev-01".to_string()); *state.running_digest.lock().unwrap() = "sha256:old".to_string(); - on_sync(&engine, &cfg, "my-app", "dev", "sha256:new", &state) + on_sync(&engine, &cfg, "my-app", "dev", "sha256:new", None, &state) .await .expect("a pointer-write failure must not fail a sync that already took effect"); From a93b392aa7f08dede1a2756246ed60b77325172d Mon Sep 17 00:00:00 2001 From: Javier Tia Date: Tue, 4 Aug 2026 09:52:47 -0600 Subject: [PATCH 19/20] container-agent-dev: make the two timeout tests fail rather than hang Both tests whose subject is "this must not hang forever" had no bound of their own, so removing the fix under test hung the suite instead of failing it. A mutation sweep caught it: gutting the session guard's Drop and unwiring the body wrapper each produced a 180s timeout rather than a named assertion failure. In CI that reads as flaky infrastructure and gets retried, which is the opposite of what a regression should provoke. The body wrapper could not be covered under a paused clock at all. Unwired there is no timer left inside the body, and tokio's auto-advance does not fire for an outer `timeout` when the inner future parks without registering a waker - so the outer bound never came due and the test hung anyway. The idle deadline is therefore short under test and the tests run on the real clock, with the shipped 60s value asserted separately so the override cannot hide a typo in the value that actually ships. Signed-off-by: Javier Tia --- src/main.rs | 7 ++++-- src/proxy.rs | 64 +++++++++++++++++++++++++++++++++++++++++++--------- 2 files changed, 58 insertions(+), 13 deletions(-) diff --git a/src/main.rs b/src/main.rs index 4eaa876..cec0ba5 100644 --- a/src/main.rs +++ b/src/main.rs @@ -1154,9 +1154,12 @@ mod tests { drop(guard); - let err = handle + // Bounded: without the abort this await never resolves, and a hung test + // reads as CI infrastructure trouble rather than the regression it is. + let err = tokio::time::timeout(Duration::from_secs(5), handle) .await - .expect_err("dropping the guard must abort the worker, not leave it parked"); + .expect("dropping the guard must abort the worker, not leave it parked") + .expect_err("an aborted worker must not report success"); assert!( err.is_cancelled(), "the worker must end cancelled rather than panicking: {err}" diff --git a/src/proxy.rs b/src/proxy.rs index c5adb88..bebd02c 100644 --- a/src/proxy.rs +++ b/src/proxy.rs @@ -372,8 +372,25 @@ impl Upstream for HttpsUpstream { /// An IDLE deadline, not a total-transfer one: capping total transfer would kill /// a legitimate multi-gigabyte layer over a slow link, while a link that is /// moving bytes at all keeps resetting this. +#[cfg(not(test))] const UPSTREAM_BODY_IDLE_TIMEOUT: Duration = Duration::from_secs(60); +/// Short under test so the WIRING is assertable, not just the wrapper. +/// +/// Virtual time cannot cover it: with the wrapper removed there is no timer left +/// inside the body at all, and tokio's paused-clock auto-advance does not fire +/// for an outer `timeout` when the inner future parks without registering a +/// waker - the test hangs instead of failing, which is the one outcome a +/// regression must not produce. [`the_production_body_idle_timeout_is_generous`] +/// pins the shipped value so this override cannot hide a typo in it. +#[cfg(test)] +const UPSTREAM_BODY_IDLE_TIMEOUT: Duration = Duration::from_millis(50); + +/// The idle deadline the shipped binary uses, kept beside the override so both +/// values are visible together and the real one stays asserted. +#[cfg(test)] +const PRODUCTION_BODY_IDLE_TIMEOUT: Duration = Duration::from_secs(60); + /// Wraps a body so a gap of [`UPSTREAM_BODY_IDLE_TIMEOUT`] between chunks ends /// the stream with an error instead of hanging forever. struct IdleTimeoutBody { @@ -926,6 +943,21 @@ mod tests { } } + #[test] + fn the_production_body_idle_timeout_is_generous() { + // The suite runs with a 50ms override, so nothing else would notice the + // shipped value being wrong. It has to be long enough that a slow but + // live link is never cut - it bounds a stall, not a slow transfer. + assert!( + PRODUCTION_BODY_IDLE_TIMEOUT >= Duration::from_secs(30), + "a short production idle deadline would kill slow but healthy pulls" + ); + assert!( + UPSTREAM_BODY_IDLE_TIMEOUT < PRODUCTION_BODY_IDLE_TIMEOUT, + "the test override must be the shorter of the two" + ); + } + /// A 200 whose body never delivers a byte. struct StallingUpstream; @@ -944,7 +976,7 @@ mod tests { } } - #[tokio::test(start_paused = true)] + #[tokio::test] async fn a_stalled_upstream_body_fails_through_the_proxy() { // The wiring, not just the wrapper: reverting `proxy_read` to hand the // upstream body straight to the response leaves the two IdleTimeoutBody @@ -959,11 +991,13 @@ mod tests { .await; assert_eq!(resp.status(), StatusCode::OK); - let err = resp - .into_body() - .collect() + // The outer bound is what makes an unwired wrapper FAIL rather than hang. + // With the wrapper in place the body's own 50ms deadline fires long + // first, so this costs nothing on the passing path. + let err = tokio::time::timeout(BODY_TEST_BOUND, resp.into_body().collect()) .await - .expect_err("a stalled upstream body must not hang the engine's pull"); + .expect("a stalled upstream body must not hang the engine's pull") + .expect_err("a stalled upstream body must surface as an error"); assert_eq!(err.kind(), std::io::ErrorKind::TimedOut); } @@ -1015,17 +1049,25 @@ mod tests { .expect("a raise with no waiter must still wake the next waiter"); } - #[tokio::test(start_paused = true)] + /// Bound for the body tests: comfortably longer than the 50ms idle deadline + /// the suite runs with, short enough that a regression reports in seconds + /// rather than hanging the run. + const BODY_TEST_BOUND: Duration = Duration::from_secs(5); + + #[tokio::test] async fn a_body_that_stalls_mid_stream_fails_instead_of_hanging() { // The stage timeouts cover TCP, TLS, the HTTP/1 handshake and the // request head - every stage EXCEPT the one a sleeping laptop actually // stalls in. Unwrapped, this collect never returns: `docker pull` waits // on a loopback response that never arrives, `on_sync` never returns, no // Status is ever emitted, and systemd still reports active (running). - let err = IdleTimeoutBody::new(StallingBody) - .collect() - .await - .expect_err("a stalled body must not hang forever"); + let err = tokio::time::timeout( + BODY_TEST_BOUND, + IdleTimeoutBody::new(StallingBody).collect(), + ) + .await + .expect("a stalled body must not hang forever") + .expect_err("a stalled body must surface as an error"); assert_eq!( err.kind(), std::io::ErrorKind::TimedOut, @@ -1037,7 +1079,7 @@ mod tests { ); } - #[tokio::test(start_paused = true)] + #[tokio::test] async fn a_body_that_delivers_is_passed_through_untouched() { // The mirror: the wrapper must not truncate or corrupt a working // stream, or every pull breaks. Without this the test above passes for a From 3f5a45a80db24408bec39f1df80a53bbff4ffaec Mon Sep 17 00:00:00 2001 From: Javier Tia Date: Wed, 9 Sep 2026 15:03:21 -0600 Subject: [PATCH 20/20] Add the packaging for avocado-ext-container-agent-dev Fills in the template with the extension that until now existed only under avocado-os/extensions. That location is going away as avocado-os reduces to a README, and it was also the reason this extension has never reached a feed: publishing runs from an ext-* repo's own release workflow, and this extension had no repo to run one from. The crate arrives in the previous commit with its history intact rather than as a flat copy, so `git log agent/` still explains why the code looks the way it does. Three things differ from the in-tree version, each for a reason: The triple-resolution loop enumerates candidates and refuses an ambiguous match instead of taking the first. It also skips the SDK's own nativesdk triple, without which x86_64 resolves to two candidates and refuses - the in-tree version picks the right one only because "avocado-" happens to sort ahead of "avocadosdk-". supported_targets names qemux86-64 and imx93-frdm rather than '*'. The wildcard claimed every target in the platform on the evidence of one, and the feed carries no repo at all for several machines it covered. The SDK image is release-scoped from the environment, matching the template. The in-tree version appended the channel, which resolves to a tag that does not exist for every release except 2024. Verified end to end: sdk install, ext install and ext build all succeed for qemux86-64 from a clean clone, producing an x86-64 binary from resolved-target x86_64-avocado-linux-gnu. Signed-off-by: Javier Tia --- .github/workflows/release.yml | 12 +- .gitignore | 11 ++ README.md | 124 ++++++++-------- avocado.yaml | 113 +++++++++++---- cad-clean.sh | 10 ++ cad-compile.sh | 103 ++++++++++++++ cad-install.sh | 62 ++++++++ .../systemd/system/container-agent-dev.path | 32 +++++ .../system/container-agent-dev.service | 132 ++++++++++++++++++ 9 files changed, 514 insertions(+), 85 deletions(-) create mode 100644 .gitignore create mode 100755 cad-clean.sh create mode 100755 cad-compile.sh create mode 100755 cad-install.sh create mode 100644 overlay/usr/lib/systemd/system/container-agent-dev.path create mode 100644 overlay/usr/lib/systemd/system/container-agent-dev.service diff --git a/.github/workflows/release.yml b/.github/workflows/release.yml index bb10286..893173d 100644 --- a/.github/workflows/release.yml +++ b/.github/workflows/release.yml @@ -3,6 +3,11 @@ name: Release # Package + publish on a version tag. The matrix lives here (the caller owns it); # each leg calls the single-combo reusable helper. The publish feed is derived from # distro-release/distro-channel. The tag must equal the avocado.yaml version. +# +# One row per target in avocado.yaml `supported_targets`. Keep the two lists in +# step: a target published here but not declared there ships something the +# extension does not claim to support, and the reverse claims support for +# something never built. on: push: tags: @@ -13,11 +18,16 @@ on: jobs: publish: strategy: + # Each target reports its own outcome. Without this one target's failure + # cancels the rest, and a partial publish looks like a total one. fail-fast: false matrix: include: - { target: qemux86-64, distro-release: "2024", distro-channel: next } - # Add once your target is published in the 2026 feed: + - { target: imx93-frdm, distro-release: "2024", distro-channel: next } + # Add once these targets are published in the 2026 feed. As of + # 2026-09-09 the 2026 feed carries jetson-agx-thor, qemuarm64 and + # qemux86-64 only, so an imx93-frdm row there would fail. # - { target: qemux86-64, distro-release: "2026", distro-channel: next } uses: avocado-linux/actions/.github/workflows/extension-release.yml@v1 with: diff --git a/.gitignore b/.gitignore new file mode 100644 index 0000000..39f33ee --- /dev/null +++ b/.gitignore @@ -0,0 +1,11 @@ +# Cargo build output for the staged crate. +/agent/target/ + +# Cross-compilation config written by cad-compile.sh at build time. It pins one +# machine's SDK sysroot path, so a committed copy is wrong for everyone else. +/agent/.cargo/ + +# avocado-cli build state. .avocado-state is a JSON file, not a directory, so it +# needs a slash-free pattern - a trailing slash silently fails to match it. +/.avocado-state +/avocado.lock diff --git a/README.md b/README.md index 83f39af..cb71006 100644 --- a/README.md +++ b/README.md @@ -1,64 +1,68 @@ -# ext-template - -GitHub template repo for new Avocado extensions. **Use this template** → then work -through the checklist below. - -## New-extension checklist - -1. Rename the repo `ext-` (or `bsp-`). -2. `avocado.yaml` — rename the extension key to `avocado-ext-`, fill in - `summary`/`description`, set `supported_targets` (and `default_target` for a BSP), - list your `packages`. Leave the `sdk.image` line alone. -3. `.github/workflows/test.yml` — set the matrix `target` to something the extension - actually supports (`qemux86-64` is fine for target-agnostic extensions). -4. `.github/workflows/release.yml` — same, plus add matrix rows for every feed you - publish into. -5. `CHANGELOG.md` — replace the placeholder `0.1.0` entry. -6. Delete this section from the README and describe the extension instead (see - "Using this extension" below — keep that part). -7. Repo secrets `AVOCADO_CONNECT_TOKEN` and `AVOCADO_CONNECT_ORG` must be set (org-level - secrets cover this if the repo is in `avocado-linux`). - -Release: tag the commit with the exact `avocado.yaml` version, e.g. `git tag 0.1.0 && git push --tags`. - -## Conventions - -- **SDK image is release-scoped, not channel-scoped.** - `docker.io/avocadolinux/sdk:{{ env.AVOCADO_DISTRO_RELEASE }}` → `avocadolinux/sdk:2026` - on a 2026 CI leg, `avocadolinux/sdk:2024` on a 2024 one. Do not append - `-{{ env.AVOCADO_DISTRO_CHANNEL }}` — there is no such tag on any release. The channel - selects the *package feed*; the image tag is per-release only. -- **CI comes from `avocado-linux/actions@v1`**, pinned. The matrix lives in the caller so - each repo owns its own target/feed combinations. -- **Feed today is `2024`/`edge-next`.** `avocado-linux/actions@v1` *defaults* to - `2026`/`edge`, but the 2026 feed currently publishes only `jetson-agx-thor`, so the - workflows here pass 2024 explicitly. Move a target to 2026 as it lands there. - -## Using this extension - -`ext-template` is an [Avocado](https://avocadolinux.org) extension — a reusable fragment of -build- and runtime-configuration that you compose into your own Avocado project. To use it, -declare it as a package-sourced extension in your `avocado.yaml` and add it to a runtime: - -```yaml -extensions: - avocado-ext-template: - source: - type: package - version: "*" # or pin an exact version - -runtimes: - my-runtime: - extensions: - - avocado-ext-template -``` +# ext-container-agent-dev + +`avocado-ext-container-agent-dev` - the device side of Container Dev Mode. + +The agent holds a control WebSocket open to the host CLI, proxies image pulls +from the host's registry over a bootstrap-delivered pinned CA, and hot-reloads +the target container when a sync lands. It is what makes `avocado container dev` +transfer only the changed layer instead of a whole image. + +**Development only.** This extension exists so it can be composed into a dev +runtime and left out of production. The `-dev` suffix is the gate. Do not ship it +on a production device. + +## Layout + +| Path | What it is | +|---|---| +| `agent/` | the Rust crate, cross-compiled at package time | +| `cad-compile.sh` | resolves the target triple and builds the crate | +| `cad-install.sh` | stages the built binary into the extension sysroot | +| `cad-clean.sh` | clears the build directory | +| `overlay/` | the `container-agent-dev` path and service units | +| `avocado.yaml` | extension manifest | + +## Requires a container engine + +The agent execs the engine CLI as its own child on every sync, so a runtime +without the `docker` extension gives it nothing to do. That dependency is stated +rather than declared: avocado-cli parses an inter-extension dependency and logs +it without installing anything, so declaring it would read as a guarantee and +deliver none. Compose both into the runtime yourself. The agent's startup +preflight is what makes a missing engine fail loudly rather than silently. -Then install and build: +## Building locally -```sh -avocado install # fetches + installs the SDK, extensions and runtime deps from your config -avocado build # builds the SDK compile steps, extensions and runtime images +The SDK image tag is release-scoped and comes from the environment, so export it +before building: + +``` +AVOCADO_DISTRO_RELEASE=2024 avocado ext install -t qemux86-64 avocado-ext-container-agent-dev +AVOCADO_DISTRO_RELEASE=2024 avocado ext build -t qemux86-64 avocado-ext-container-agent-dev ``` -`avocado install` pulls the extension from your target's package feed and merges its -config into your project; `avocado build` then produces the runtime. +Two things that will otherwise cost you time: + +- `avocado` runs its SDK container with a TTY, so a build launched without one + fails with `cannot attach stdin to a TTY-enabled container`. Wrap it in + `script -qefc '' /dev/null` when running from CI or a background shell. +- dnf prompts for confirmation and blocks; `--no-tui` does not suppress it. Pass + `--dnf-arg -y`. + +## Supported targets + +`avocado.yaml` lists them explicitly rather than using `'*'`. A target belongs +there once it has produced an installable package and the produced binary has +been seen to exec on that architecture. The release matrix in +`.github/workflows/release.yml` should carry one row per declared target. + +## Releasing + +Tag the version in `avocado.yaml`. The release workflow guards that the tag and +the manifest version match, packages one leg per matrix row, and publishes each +to the feed derived from that row's distro release and channel. + +Bump `release:` whenever the packaged payload changes without a version change. +An unchanged version-release identity means the package manager never delivers +the new payload to a device that already has the extension installed, and both +the build and the install report success anyway. diff --git a/avocado.yaml b/avocado.yaml index dc5de79..2f544ac 100644 --- a/avocado.yaml +++ b/avocado.yaml @@ -1,35 +1,100 @@ -# Which targets this extension supports. '*' = any target. -# For a BSP, replace with the single board target and set default_target. -supported_targets: '*' +# Targets this extension is declared to support. +# +# Deliberately NOT '*'. The wildcard is a claim about every target in the +# platform, and only these two have produced an installable package. Add a +# target here when it has actually been built and the produced binary has been +# seen to exec on that architecture - not when it is expected to work. A +# wildcard here with a partial sweep behind it is how `supported_targets` +# stopped meaning anything on the in-tree version of this extension. +supported_targets: + - qemux86-64 + - imx93-frdm extensions: - # Rename to avocado-ext- (or avocado-bsp- for a BSP). - avocado-ext-template: + avocado-ext-container-agent-dev: version: 0.1.0 release: r0 - summary: One-line summary - description: One-line description + summary: Container Dev Mode device agent (development only) + description: | + Device-side agent for Container Dev Mode: holds a control WebSocket to the + host CLI, proxies image pulls from the host's registry over the + bootstrap-delivered pinned CA, and hot-reloads the target container on sync. + Its own extension so it is present ONLY in a dev runtime and absent from + prod. The per-project CA is NOT baked here; it arrives at bootstrap on the + writable partition when `avocado container dev up` runs. DO NOT USE IN + PRODUCTION. + + REQUIRES the `docker` extension in the same runtime. The agent execs the + engine CLI as its own child on every sync, so without it there is nothing + it can do. Declaring that dependency here would not enforce it: the schema + accepts an inter-extension dependency, but avocado-cli only parses and logs + it - `ext/install.rs` prints "should be installed via main install command" + and installs nothing, deliberately, to avoid circular dependencies. A + declaration would read like a guarantee and deliver none. Pulling `docker` + in as a plain package is worse: it installs the binary without + `docker.service` or the bridge kernel modules, turning a loud startup + failure into a silent connect failure on every sync. So the pairing is + stated here for whoever composes the runtime, and the agent's own startup + preflight is what makes a missing engine fail loudly. license: Apache-2.0 - url: https://github.com/avocadolinux/avocado-os + url: https://github.com/avocado-linux/ext-container-agent-dev vendor: Avocado Linux - # Runtime packages pulled from the target feed. - packages: {} - - # Optional, delete what you don't use: - # - # enable_services: - # - foo.service - # - # on_merge: - # - systemctl start --no-block foo.service - # - # sdk: - # packages: - # nativesdk-foo: '*' + # The crate is staged and cross-compiled at package time, so its sources are + # part of the payload rather than a prebuilt artifact. + package_files: + - agent + - overlay + - avocado.yaml + - cad-*.sh + + overlay: overlay + + enable_services: + - container-agent-dev.path + - container-agent-dev.service + + on_merge: + # Start the PATH unit, not the service. The service carries + # ConditionPathExists on the bootstrap, which does not exist at merge time, + # so starting it here is skipped and nothing re-triggers it: `avocado + # container dev up` delivers the bootstrap over SSH and issues no systemctl + # call. The path unit has no such condition, so it starts here and activates + # the service the moment the bootstrap lands. + - systemctl start --no-block container-agent-dev.path + + on_unmerge: + - systemctl stop container-agent-dev.path + - systemctl stop container-agent-dev.service + + packages: + avocado-container-agent-dev: + compile: avocado-container-agent-dev + install: cad-install.sh + + sdk: + packages: + nativesdk-binutils: '*' + nativesdk-cargo: '*' + nativesdk-gcc: '*' + nativesdk-glibc-dev: '*' + nativesdk-libgcc-dev: '*' + nativesdk-rust: '*' + packagegroup-rust-cross-canadian-avocado-{{ avocado.target }}: '*' # Release-scoped, not channel-scoped: this resolves to avocadolinux/sdk:2026. -# The SDK image is per-release only — appending the channel points at a tag -# that does not exist. +# The SDK image is per-release only - appending the channel points at a tag that +# does not exist. AVOCADO_DISTRO_RELEASE is set by the release workflow per +# matrix row; export it yourself for a local build. sdk: image: docker.io/avocadolinux/sdk:{{ env.AVOCADO_DISTRO_RELEASE }} + + compile: + avocado-container-agent-dev: + compile: cad-compile.sh + clean: cad-clean.sh + packages: + libstd-rs: '*' + libstd-rs-dev: '*' + packages: + avocado-sdk-toolchain: '*' diff --git a/cad-clean.sh b/cad-clean.sh new file mode 100755 index 0000000..772828a --- /dev/null +++ b/cad-clean.sh @@ -0,0 +1,10 @@ +#!/bin/bash +set -e + +echo "Cleaning avocado-container-agent-dev build artifacts" + +cd "$(dirname "$0")/agent" + +cargo clean --target-dir "$AVOCADO_BUILD_DIR" + +echo "avocado-container-agent-dev cleaned successfully" diff --git a/cad-compile.sh b/cad-compile.sh new file mode 100755 index 0000000..795aa61 --- /dev/null +++ b/cad-compile.sh @@ -0,0 +1,103 @@ +#!/bin/bash +set -euo pipefail + +RUST_TARGET="" +match_count=0 +matches="" + +# Resolve the Rust target from RUST_TARGET_PATH. Enumerate every candidate +# instead of stopping at the first: an OECORE_TARGET_ARCH that prefix-matches +# two target JSONs used to take whichever the glob happened to return first and +# discard the rest without a word. That is the worst failure shape available +# here - a binary for the wrong architecture packages, publishes and installs +# successfully, and only fails when the device tries to exec it, several layers +# from the cause. Refusing to guess keeps the failure local and attributable. +# +# devtool-debt: this block is duplicated, still unfixed, in the three sibling +# extensions that have not yet moved to their own ext-* repo - cli, connect and +# tunnels - in both their compile and install scripts. jtop and microclaw carry +# a copy too, but theirs are superseded: both migrated to ext-* repos, and +# ext-microclaw resolves the architecture with a plain case statement that does +# not have this defect at all. +# Ceiling: those three are safe by GLOB ORDERING ALONE, not by construction. +# Every target whose architecture matches the SDK host architecture has two +# matching JSONs - the device triple and the SDK's own -avocadosdk-* - and +# first-match-and-break survives only because "avocado-" sorts ahead of +# "avocadosdk-" on the hyphen. Nothing enforces that ordering. This is not a +# theoretical ceiling: it is exactly what broke qemux86-64 here the first time +# the guard below ran. +# Upgrade trigger: a vendor string that reorders those two, a third triple +# sorting between them, or any of the three gaining a target whose architecture +# prefix-matches more than one device triple. +for json_file in "$RUST_TARGET_PATH"/*.json; do + if [ -f "$json_file" ]; then + json_name=$(basename "$json_file" .json) + + # Skip the SDK's own nativesdk triple. It shares the architecture prefix + # with the device triple whenever the SDK host arch matches the target + # arch, which on x86_64 means RUST_TARGET_PATH holds both + # x86_64-avocado-linux-gnu and x86_64-avocadosdk-linux-gnu. The + # avocadosdk vendor is host-side by construction - avocado-cli tags every + # nativesdk artifact _avocadosdk - so it is never a candidate for a + # device binary and must not count as a competing match. + case "$json_name" in + "${OECORE_TARGET_ARCH}-avocadosdk-"*) continue ;; + esac + + if [[ "$json_name" == "${OECORE_TARGET_ARCH}-"* ]]; then + RUST_TARGET="$json_name" + match_count=$((match_count + 1)) + matches="$matches $json_name" + fi + fi +done + +if [ -z "$RUST_TARGET" ]; then + echo "Error: Could not find Rust target for $OECORE_TARGET_ARCH" >&2 + exit 1 +fi + +if [ "$match_count" -gt 1 ]; then + echo "Error: $OECORE_TARGET_ARCH matches $match_count Rust targets:$matches" >&2 + echo "Error: refusing to pick one - the wrong triple yields a binary that packages and installs but cannot exec on the device." >&2 + exit 1 +fi + +# Stable marker: the resolution fixture test greps for this to tell "resolution +# succeeded and a later step failed" from "resolution itself failed", which exit +# code alone cannot distinguish when the test runs without a real SDK. Load- +# bearing test surface, not a debug echo. +echo "resolved-target: $RUST_TARGET" + +echo "Compiling avocado-container-agent-dev for target: $RUST_TARGET" + +cd "$(dirname "$0")/agent" + +# Clear any rustflags that might cause conflicts with our .cargo/config.toml +unset RUSTFLAGS +unset CARGO_BUILD_RUSTFLAGS +for var in $(env | grep -o 'CARGO_TARGET_[A-Z0-9_]*_RUSTFLAGS'); do + unset "$var" +done + +# Remove any existing config that might conflict +rm -rf .cargo + +# Create config.toml with cross-compilation settings +mkdir -p .cargo +cat > .cargo/config.toml << EOF +[target.$RUST_TARGET] +rustflags = ["--sysroot=$SDKTARGETSYSROOT/usr", "-C", "link-arg=--sysroot=$SDKTARGETSYSROOT"] +EOF + +# Use a persistent cargo registry cache to avoid re-downloading crates +export CARGO_HOME="${AVOCADO_BUILD_DIR}/.cargo-cache" + +# Enforce the ring-only rustls crypto provider before building. TARGET is +# mandatory here so the guard resolves the same SDK triple the build ships; +# without it the guard would check the host triple instead. Runs under set -e, +# so a non-zero exit (aws-lc-rs present, or a guard error) fails the compile. +TARGET="$RUST_TARGET" scripts/check-provider.sh +cargo build --release --target "$RUST_TARGET" --target-dir "$AVOCADO_BUILD_DIR" + +echo "avocado-container-agent-dev compiled successfully" diff --git a/cad-install.sh b/cad-install.sh new file mode 100755 index 0000000..856cfcf --- /dev/null +++ b/cad-install.sh @@ -0,0 +1,62 @@ +#!/bin/bash +set -euo pipefail + +RUST_TARGET="" +match_count=0 +matches="" + +# Resolution must stay byte-for-byte equivalent to cad-compile.sh's. The two run +# in separate invocations against the same SDK, so a divergence here silently +# installs a different triple's binary than the one that was compiled - the +# install would look successful and ship the wrong artifact. Enumerate every +# candidate rather than stopping at the first; see cad-compile.sh for why an +# ambiguous match must refuse rather than pick. +for json_file in "$RUST_TARGET_PATH"/*.json; do + if [ -f "$json_file" ]; then + json_name=$(basename "$json_file" .json) + + # Skip the SDK's own nativesdk triple - see cad-compile.sh for why. This + # exclusion must stay identical in both scripts: if only one of them + # skipped it, that script would refuse while the other resolved, and the + # extension would compile but fail to install (or worse, install a + # host-triple binary onto the device). + case "$json_name" in + "${OECORE_TARGET_ARCH}-avocadosdk-"*) continue ;; + esac + + if [[ "$json_name" == "${OECORE_TARGET_ARCH}-"* ]]; then + RUST_TARGET="$json_name" + match_count=$((match_count + 1)) + matches="$matches $json_name" + fi + fi +done + +# Same guard cad-compile.sh has. Without it an unmatched loop leaves RUST_TARGET +# empty, BINARY_PATH collapses to "$AVOCADO_BUILD_DIR//release/...", and the -f +# check below reports a missing binary instead of the actual fault. +if [ -z "$RUST_TARGET" ]; then + echo "Error: Could not find Rust target for $OECORE_TARGET_ARCH" >&2 + exit 1 +fi + +if [ "$match_count" -gt 1 ]; then + echo "Error: $OECORE_TARGET_ARCH matches $match_count Rust targets:$matches" >&2 + echo "Error: refusing to pick one - installing a different triple's binary than was compiled would ship a wrong-architecture artifact that looks installed." >&2 + exit 1 +fi + +# Stable marker, same contract as cad-compile.sh's: the resolution fixture test +# greps for it to separate a successful resolution from a later failure. +echo "resolved-target: $RUST_TARGET" + +BINARY_PATH="$AVOCADO_BUILD_DIR/$RUST_TARGET/release/avocado-container-agent-dev" + +if [ ! -f "$BINARY_PATH" ]; then + echo "Error: Binary not found at $BINARY_PATH" + exit 1 +fi + +echo "Installing avocado-container-agent-dev into extension" +install -D -m 755 "$BINARY_PATH" "$AVOCADO_BUILD_EXT_SYSROOT/usr/bin/avocado-container-agent-dev" +echo "avocado-container-agent-dev installed successfully" diff --git a/overlay/usr/lib/systemd/system/container-agent-dev.path b/overlay/usr/lib/systemd/system/container-agent-dev.path new file mode 100644 index 0000000..3b08ea1 --- /dev/null +++ b/overlay/usr/lib/systemd/system/container-agent-dev.path @@ -0,0 +1,32 @@ +[Unit] +Description=Watch for the Container Dev Mode bootstrap and start the agent +Documentation=https://github.com/avocadolinux/avocado-os +# This unit exists because the service cannot start itself at merge time. +# +# The service carries ConditionPathExists=, and the extension's +# on_merge hook runs `systemctl start` BEFORE any bootstrap exists - so the +# condition skipped it, ExecStart never ran, and the journal recorded only +# "Condition check resulted in ... being skipped". Restart=always does not help: +# it applies to a unit that reached a running state, and this one never did. +# +# Nothing re-triggered a start afterwards either. `avocado container dev up` +# delivers bootstrap.json over SSH and issues no systemctl call at all (there is +# no `systemctl` anywhere in the CLI's container-dev paths), so the agent stayed +# down until an unrelated reboot while `up` reported success and every sync +# reached nobody. +# +# A .path unit is the systemd-native answer: it is startable at merge time +# because it has no such condition, and it activates the service the moment the +# file lands - which is exactly the event that was missing. + +[Path] +PathExists=/var/lib/avocado/container-dev/bootstrap.json +# Also fire when the file is REPLACED, not only when it first appears. Re-running +# `avocado container dev up` overwrites bootstrap.json with a freshly minted +# token; a device whose agent had exited then needs starting again, and one that +# is already running is unaffected (systemd will not re-start an active service). +PathChanged=/var/lib/avocado/container-dev/bootstrap.json +Unit=container-agent-dev.service + +[Install] +WantedBy=multi-user.target diff --git a/overlay/usr/lib/systemd/system/container-agent-dev.service b/overlay/usr/lib/systemd/system/container-agent-dev.service new file mode 100644 index 0000000..228cd9c --- /dev/null +++ b/overlay/usr/lib/systemd/system/container-agent-dev.service @@ -0,0 +1,132 @@ +[Unit] +Description=Avocado Container Dev Mode device agent (development only) +Documentation=https://github.com/avocadolinux/avocado-os +After=network.target +Wants=network.target +# The agent execs the docker CLI as its own child on every sync, so a daemon +# that has not started yet is a startup ordering problem, not a runtime one. +# Without this the first Sync after a boot raced dockerd and failed at +# `Command::new("docker").status()`; the swallowed-error path reduced that to a +# warn!, `running_digest` kept its old value, and the host went on seeing a +# connected device whose syncs no-oped. `Wants=` rather than `Requires=`: a +# device that is not running docker at all should still start the agent and fail +# loudly on the engine preflight, which names the missing binary. +After=docker.service +Wants=docker.service +# Stay inert until `avocado container dev up` delivers the bootstrap (bulk +# endpoint, read/control token, pinned CA) to the writable partition (design +# D5/5.2). No bootstrap means no session to join, so there is nothing to do. +ConditionPathExists=/var/lib/avocado/container-dev/bootstrap.json +# The default StartLimitIntervalSec=10s / StartLimitBurst=5 cannot trip at +# RestartSec=5: two starts per 10s window is well under five, so a unit that +# fails instantly - a truncated or malformed bootstrap.json makes +# `load_bootstrap` return Err and the process exit in milliseconds - sat in +# `activating (auto-restart)` forever. `systemctl is-failed` answers `activating` +# rather than `failed`, no OnFailure= hook ever runs, and the only evidence is +# one journal line every 5s. +# +# Widening the window is what makes the limit reachable: five starts inside five +# minutes at 5s apart trips in roughly 25 seconds, so a genuinely broken +# bootstrap lands the unit in `failed` where it can be seen and hooked, while a +# transient failure (the host briefly unreachable at boot) still gets its +# retries. +# +# Both keys live in [Unit], not [Service] - systemd moved them in v229 and +# silently ignores them under [Service], which `systemd-analyze verify` reports +# as "Unknown key ... ignoring". +StartLimitIntervalSec=300 +StartLimitBurst=5 + +[Service] +# StateDirectory replaces the former `ExecStartPre=/bin/mkdir -p`, which could +# not run under a read-only /var. Mode 0700 is NOT the default: systemd applies +# 0755 on every start, including to a directory that already exists, and this +# one holds bootstrap.json with the Bearer read/control token and the session +# CA. Without this line a `systemctl start` silently widens a 0700 directory +# created by `avocado container dev up`. +StateDirectory=avocado/container-dev +StateDirectoryMode=0700 +ExecStart=/usr/bin/avocado-container-agent-dev +Environment=RUST_LOG=info +# Pin the engine rather than leaving the DOCKER ONLY decision below to a comment. +# The agent only warns on a podman value and then execs it anyway (deliberately - +# the restriction is this unit's, not the binary's, so a debugging run outside the +# unit still works), which means an unset variable here let a DefaultEnvironment= +# in system.conf select podman. That failure is the worst shape available: podman +# dies at store init under this sandbox, on_sync's error is only logged, the unit +# stays active (running), and every Sync silently no-ops while the device keeps +# reporting healthy. A unit-level Environment= beats DefaultEnvironment=, so this +# closes that path. +Environment=AVOCADO_CONTAINER_DEV_ENGINE=docker + +# DOCKER ONLY. This unit deliberately does not support the podman engine. +# +# The agent runs as root: it listens on a loopback socket, dials outbound TLS, +# parses untrusted WS/HTTP frames, and execs the container engine as its own +# child. Confining that meant choosing, because the two goals are mutually +# exclusive. systemd.exec(5) is explicit that ProtectSystem is undone by a +# privileged process unless paired with CapabilityBoundingSet=~CAP_SYS_ADMIN - +# root can otherwise remount its own read-only binds read-write - and rootful +# podman needs exactly CAP_SYS_ADMIN, plus namespaces, cgroup writes and kernel +# knobs, all in-process. Keeping podman working meant the sandbox was decorative; +# a real sandbox means podman cannot run under this unit. +# +# We chose the sandbox. Docker is unaffected: its daemon is a separate unit +# holding the privilege, reached over an AF_UNIX socket, and a read-only mount +# does not block connect() to a socket. Restoring podman properly is tracked as +# a follow-up - it needs the engine out of this process, not a weaker unit. +CapabilityBoundingSet= +NoNewPrivileges=true +ProtectSystem=strict +ProtectHome=true +PrivateTmp=true +RestrictSUIDSGID=true +RestrictRealtime=true +LockPersonality=true +SystemCallArchitectures=native +RestrictNamespaces=true +# PrivateDevices, ProtectKernelTunables and ProtectControlGroups are the three +# directives systemd.exec(5) names as closing ProtectSystem=strict's /dev, /proc +# and /sys carve-outs; strict alone leaves all three writable. /dev is the one +# that matters most here: block devices are root:disk 0660 and this agent runs as +# root, so the owner bits alone allow opening /dev/mmcblk0 O_RDWR and overwriting +# the "read-only" rootfs. An empty CapabilityBoundingSet does not stop it and +# SystemCallFilter has nothing to deny - the open needs no capability and no +# privileged syscall. Without this line the untrusted WS/HTTP frame path can +# reach the exact hole the block above claims to have closed. +PrivateDevices=yes +ProtectControlGroups=true +ProtectKernelTunables=true +ProtectKernelModules=true +SystemCallFilter=@system-service +SystemCallErrorNumber=EPERM +# The agent writes active-image.json into StateDirectory, so this keeps that file +# off world-readable. It does NOT govern bootstrap.json: the host writes that one +# (see main.rs) and the agent only reads it, so the Bearer read/control token's +# mode is set by the delivery path, not here. StateDirectoryMode=0700 above is +# what keeps the token unreadable, and even that first applies at `systemctl +# start` - after ConditionPathExists has already proven bootstrap.json exists. +# Writing it 0600 belongs on the CLI side that delivers it. +UMask=0077 + +# StateDirectory is the only PERSISTENT writable path under ProtectSystem=strict, +# which is all the agent needs: bootstrap.json and active-image.json both live +# there. PrivateTmp above also grants a private /tmp and /var/tmp, discarded when +# the unit stops - so a new write path needs its own directive only if it must +# survive a restart. +# +# No AF_NETLINK. It was only ever required by netavark configuring a podman +# container's netns; the docker CLI just talks to its daemon over AF_UNIX, and +# the agent's own traffic is AF_INET/AF_INET6. +RestrictAddressFamilies=AF_UNIX AF_INET AF_INET6 + +# Raise the fd ceiling rather than only absorbing EMFILE in the proxy's accept +# loop; the default is the reason that error path exists at all. +LimitNOFILE=65536 + +Restart=always +RestartSec=5 +SyslogIdentifier=avocado-container-agent-dev + +[Install] +WantedBy=multi-user.target