From a31209b2e2f7203f728b3a1e2e922c4ea59aed8b Mon Sep 17 00:00:00 2001 From: Matthew Date: Sat, 8 Aug 2026 12:57:31 -0700 Subject: [PATCH 1/3] ci: verify what we publish from the consumer side Nothing in this repo has ever checked the artifact a user downloads. ci.yml proves the code builds and its tests pass; release.yml's verify-assets proves the upload step believed itself, asserted from inside the run that did the uploading. Neither is evidence about the published thing. That gap shipped twice in this fleet. webrequest-hook v1.0.4 published as a zero-asset phantom: tag, Release object, nothing to download. headroom-hook's published Docker bundle could not boot the gateway, because its shipped docker/bundle/config.yaml still used config shapes busbar 1.5.3 retired, so the container exits 1 during config load and never binds a port. Both were green everywhere. Adds consumer-verify.yml, which calls the fleet's shared GetBusbar/busbar/.github/workflows/plugin-consumer-verify.yml. It checks out nothing, so a fix that is committed but never published still fails it. It asserts the Release is published rather than a still-draft, that every platform archive it owes is present and downloadable THROUGH /releases/latest/download/ (a count can never see a missing platform, only a name can), that one archive really is a plugin busbar would accept - manifest name/alias/kind/version, the sha256 binding the cdylib beside it, a real shared library, a non-empty signature - and, where a runnable bundle is published, that the image BOOTS AND SERVES. The same job is appended to release.yml as its final step, so a broken publish turns the RELEASE red rather than leaving a green release beside a red run nobody correlates. On failure the shared workflow opens or updates ONE labelled issue naming the failing check, its expected and observed values and the run URL, and closes it again when a run passes. The inputs are read off the PUBLISHED manifest rather than guessed from the crate name: this fleet genuinely disagrees with itself about naming (the store repos drop the trailing -plugin the auth repos keep, and store-valkey publishes as busbar-store-redis). --- .github/workflows/consumer-verify.yml | 55 +++++++++++++++++++++++++++ .github/workflows/release.yml | 37 ++++++++++++++++++ 2 files changed, 92 insertions(+) create mode 100644 .github/workflows/consumer-verify.yml diff --git a/.github/workflows/consumer-verify.yml b/.github/workflows/consumer-verify.yml new file mode 100644 index 0000000..95d7f6c --- /dev/null +++ b/.github/workflows/consumer-verify.yml @@ -0,0 +1,55 @@ +name: consumer-verify + +# Does what this repo PUBLISHED actually work when a user gets it? +# +# Every other workflow here reports on itself. ci.yml proves the code builds and its tests pass. +# release.yml's verify-assets proves the upload step believed it succeeded, asserted from inside the +# run that did the uploading. None of that is evidence about the artifact a user downloads, and the +# gap is not theoretical: webrequest-hook v1.0.4 published as a zero-asset phantom, and +# headroom-hook's published bundle could not boot the gateway because its shipped config used shapes +# busbar 1.5.3 retired. Both were green everywhere. Nothing anywhere noticed. +# +# The logic lives in ONE place for the whole fleet, exactly like plugin-ci.yml, so a fix reaches +# every plugin at once instead of being copied ten times and drifting nine. +# +# WHY BOTH TRIGGERS. release: published catches a broken publish immediately, and it fires whether or +# not the release workflow itself finished happy - which matters, because a verifier that only runs +# when everything already worked is not a verifier. The daily schedule catches ROT: an artifact that +# published fine can stop working later when nothing about it changed (a bundle that no longer boots +# against a newer engine, an asset deleted by hand, a release un-flagged as latest). A +# publish-time-only check structurally cannot see that class. +on: + release: + types: [published] + schedule: + - cron: " * * *" + workflow_dispatch: + inputs: + version: + description: "Version to verify (e.g. 1.0.4). Empty means the newest published release." + required: false + type: string + +permissions: + contents: read + issues: write + actions: read + +jobs: + consumer: + uses: GetBusbar/busbar/.github/workflows/plugin-consumer-verify.yml@dev + with: + version: ${{ inputs.version || '' }} + # Read off the PUBLISHED artifact, not guessed from the crate name: the filename prefix and the + # manifest name genuinely differ across this fleet (the store repos drop the trailing -plugin + # that the auth repos keep, and store-valkey publishes as busbar-store-redis). + asset_prefix: busbar-store-sqlite + plugin_name: busbar-store-sqlite-plugin + plugin_alias: sqlite + plugin_kind: store + # This repo publishes a runnable bundle, so the check does not stop at 'the tarball exists': + # the image is pulled fresh and the container must BOOT and answer /healthz. That is the + # assertion that was missing when the published bundle shipped a config busbar had retired. + bundle_image: 53 9 + bundle_env: "" + secrets: inherit diff --git a/.github/workflows/release.yml b/.github/workflows/release.yml index ea25126..6802b35 100644 --- a/.github/workflows/release.yml +++ b/.github/workflows/release.yml @@ -237,3 +237,40 @@ jobs: -f event_type=upstream-release \ -f "client_payload[repo]=${GITHUB_REPOSITORY}" \ -f "client_payload[tag]=${GITHUB_REF_NAME}" + + # THE LAST STEP: does what we just published work for a user? + # + # Everything above verifies what THIS RUN produced, from inside this run. verify-assets asserts the + # assets it uploaded are present, which proves the run believed itself. It cannot prove the tarball + # downloads for a stranger, unpacks into a plugin busbar will load, or (where a bundle is + # published) boots. Those are the failures that shipped: a zero-asset phantom release, and a + # published bundle that exits 1 on startup. + # + # Consumer verification is POST-PUBLICATION by nature - you cannot download an asset that was never + # uploaded - so this cannot block the publish and does not pretend to. What it does is make the + # verdict impossible to miss: a failure is THIS release run's failure, and the shared workflow also + # opens or updates one labelled issue naming the failing check, expected, observed and the run URL. + # + # !cancelled() because a needs: on a FAILED job skips its dependent by default - the exact shape + # that skips a release's own guard precisely when the release is broken. + consumer-verification: + name: consumer verification (LAST STEP) + needs: [verify-assets] + if: ${{ !cancelled() }} + uses: GetBusbar/busbar/.github/workflows/plugin-consumer-verify.yml@dev + with: + version: ${{ github.ref_name }} + asset_prefix: busbar-store-sqlite + plugin_name: busbar-store-sqlite-plugin + plugin_alias: sqlite + plugin_kind: store + # This repo publishes a runnable bundle, so the check does not stop at 'the tarball exists': + # the image is pulled fresh and the container must BOOT and answer /healthz. That is the + # assertion that was missing when the published bundle shipped a config busbar had retired. + bundle_image: 53 9 + bundle_env: "" + permissions: + contents: read + issues: write + actions: read + secrets: inherit From 471dc86bdd74f3ac42f284b358c332b52e95628f Mon Sep 17 00:00:00 2001 From: Matthew Date: Sat, 8 Aug 2026 13:16:19 -0700 Subject: [PATCH 2/3] ci: say which busbar branch we mean, instead of '/merge' On a pull_request event `github.ref_name` is '/merge', not a branch name, so plugin-ci asked GetBusbar/busbar for a branch called '5/merge' and the sibling checkout died with 'The process /usr/bin/git failed with exit code 1', naming neither the ref nor the reason. Every pull request to this repo has been red on that before running a line of the change it was testing. `github.base_ref` is the branch the PR targets and is empty on a push, so this means dev-against-dev and main-against-main as intended on both events. plugin-ci.yml also now falls back with a loud warning when a ref cannot be resolved, so this is belt and braces - but a caller should say what it means rather than lean on a fallback. --- .github/workflows/ci.yml | 7 ++++++- 1 file changed, 6 insertions(+), 1 deletion(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 78c0a3f..2725600 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -19,4 +19,9 @@ jobs: plugin_kind: store plugin_alias: sqlite service: none - busbar_ref: ${{ github.ref_name }} # same-branch: qa builds core qa, dev builds core dev (no stale main/tag) + busbar_ref: ${{ github.base_ref || github.ref_name }} + # base_ref FIRST: on a pull_request `github.ref_name` is '/merge', not a branch + # name, so this asked busbar for a branch called '5/merge' and the sibling checkout died + # with an unreadable git error on EVERY pull request to this repo. + # Same-branch intent is unchanged: qa builds core qa, dev builds core dev, never a + # stale main or tag. From 662358bb3df328d6ff7da17ab6ab91e4b4fb3f81 Mon Sep 17 00:00:00 2001 From: Matthew Jackson <1085847+MattJackson@users.noreply.github.com> Date: Sat, 8 Aug 2026 15:46:39 -0700 Subject: [PATCH 3/3] test(e2e): give --validate the env secret the fixture config names busbar 1.5.3 (core commit 879712a, "validate: resolve built-in secret references, and fail when one cannot") changed `--validate` to RESOLVE built-in `env:`/`file:` secret references and exit 1 when one cannot, rather than only checking the reference's shape. This test's fixture config names `providers.mock.api_key: { env: MOCK_KEY }`, copied from core's own `crates/busbar/tests/cli_validate.rs::write_configs` but without the `.env("MOCK_KEY", ...)` that fixture sets alongside it. So `--validate` now fails on any machine that does not happen to have MOCK_KEY set, which is every CI runner and most dev machines: [error] providers.mock.api_key: secret env:MOCK_KEY cannot resolve: environment variable 'MOCK_KEY' is unset The real boot later in the same test is unaffected, which is why only the validate assertion went red. The names are extracted from the config text rather than hardcoded, so the harness keeps working when this fixture or a future one names a different variable. GetBusbar/store-mysql already took exactly this approach for the same core change; this matches it so the two do not drift. --- store-sqlite-plugin/tests/e2e.rs | 71 ++++++++++++++++++++++++-------- 1 file changed, 53 insertions(+), 18 deletions(-) diff --git a/store-sqlite-plugin/tests/e2e.rs b/store-sqlite-plugin/tests/e2e.rs index 47fe0d3..f036891 100644 --- a/store-sqlite-plugin/tests/e2e.rs +++ b/store-sqlite-plugin/tests/e2e.rs @@ -100,6 +100,37 @@ fn plugin_path() -> Option { candidate } +/// Every `env:` secret-ref name a config text references, in first-seen order, de-duplicated. +/// +/// busbar 1.5.3 made `--validate` RESOLVE built-in (`env`/`file`) secret references and exit 1 when +/// one cannot resolve, rather than only checking the reference's SHAPE. A fixture config that names +/// a real-looking env var (here, `MOCK_KEY`) then fails `--validate` on any machine that doesn't +/// happen to have that var set -- which is every CI runner and most dev machines. Hardcoding +/// `MOCK_KEY` here would fix today's failure but rot the moment this fixture, or a future one, names +/// a different variable. Extracting the names generically (same approach `GetBusbar/store-mysql`'s +/// own `store-mysql-plugin/tests/e2e.rs` already took for this exact change, and the core repo's +/// `crates/busbar/tests/docs_examples.rs`) keeps the harness working no matter what the fixture +/// references. +fn referenced_env_vars(text: &str) -> Vec { + let mut v: Vec = Vec::new(); + for (i, _) in text.match_indices("env:") { + let rest = &text[i + 4..]; + let name: String = rest + .chars() + .skip_while(|c| c.is_whitespace()) + .take_while(|c| c.is_ascii_alphanumeric() || *c == '_') + .collect(); + if !name.is_empty() && !v.contains(&name) { + v.push(name); + } + } + v +} + +/// A placeholder value for a fixture-referenced secret: 64 hex chars, which is valid for +/// `auth.signing_key` and harmless as any other secret's value. +const SECRET_PLACEHOLDER: &str = "0000000000000000000000000000000000000000000000000000000000000001"; + /// The sibling busbarAI checkout's root (same convention this repo already uses for its path deps /// in Cargo.toml). fn busbarai_root() -> PathBuf { @@ -200,20 +231,17 @@ fn load_and_exercise_sqlite_plugin_via_file_drop() { "mock:\n protocol: anthropic\n base_url: \"http://127.0.0.1:9\"\n api_key_env: MOCK_KEY\n", ) .unwrap(); - std::fs::write( - &config, - format!( - "listen: \"127.0.0.1:0\"\n\ - store:\n module: sqlite\n settings: {{ db_path: \"{}\" }}\n\ - plugins:\n enabled: true\n dir: {}\n trust:\n allow_unsigned: true\n\ - auth:\n chain: []\n\ - providers:\n mock:\n api_key: {{ env: MOCK_KEY }}\n\ - models:\n test-model:\n provider: mock\n", - db_path.display(), - plugins_dir.display() - ), - ) - .unwrap(); + let config_text = format!( + "listen: \"127.0.0.1:0\"\n\ + store:\n module: sqlite\n settings: {{ db_path: \"{}\" }}\n\ + plugins:\n enabled: true\n dir: {}\n trust:\n allow_unsigned: true\n\ + auth:\n chain: []\n\ + providers:\n mock:\n api_key: {{ env: MOCK_KEY }}\n\ + models:\n test-model:\n provider: mock\n", + db_path.display(), + plugins_dir.display() + ); + std::fs::write(&config, &config_text).unwrap(); // `--validate` is DELIBERATELY not used for the load-proof itself: it is manifest-only by // design ("no server, no network, no state, no dlopen" -- crates/busbar/src/main.rs's own @@ -224,12 +252,19 @@ fn load_and_exercise_sqlite_plugin_via_file_drop() { // plugin passes the trust/manifest gate; then a REAL BOOT (no `--validate` flag) is the only // thing that actually `dlopen`s the plugin and runs `Store::open`/migration, so that's what // proves the persistence claim. - let out = Command::new(&busbar_bin) + // + // `--validate` RESOLVES built-in `env:` secret references (busbar 1.5.3); give every one this + // fixture names a placeholder so the gate tests the config's SHAPE, not this machine's + // environment. See `referenced_env_vars`'s doc comment for why this is generic, not hardcoded. + let mut validate_cmd = Command::new(&busbar_bin); + validate_cmd .arg("--validate") .env("BUSBAR_CONFIG", &config) - .env("BUSBAR_PROVIDERS", &providers) - .output() - .expect("run busbar --validate"); + .env("BUSBAR_PROVIDERS", &providers); + for name in referenced_env_vars(&config_text) { + validate_cmd.env(name, SECRET_PLACEHOLDER); + } + let out = validate_cmd.output().expect("run busbar --validate"); assert!( out.status.success(), "busbar --validate must succeed with the file-dropped sqlite plugin: stdout={} stderr={}",