ci: verify what we publish from the consumer side

.github/workflows/ci.yml

@@ -19,4 +19,9 @@ jobs:
       plugin_kind: store
       plugin_alias: sqlite
       service: none
-      busbar_ref: ${{ github.ref_name }} # same-branch: qa builds core qa, dev builds core dev (no stale main/tag)
+      busbar_ref: ${{ github.base_ref || github.ref_name }}
+      # base_ref FIRST: on a pull_request `github.ref_name` is '<number>/merge', not a branch
+      # name, so this asked busbar for a branch called '5/merge' and the sibling checkout died
+      # with an unreadable git error on EVERY pull request to this repo.
+      # Same-branch intent is unchanged: qa builds core qa, dev builds core dev, never a
+      # stale main or tag.

.github/workflows/consumer-verify.yml

@@ -0,0 +1,55 @@
+name: consumer-verify
+
+# Does what this repo PUBLISHED actually work when a user gets it?
+#
+# Every other workflow here reports on itself. ci.yml proves the code builds and its tests pass.
+# release.yml's verify-assets proves the upload step believed it succeeded, asserted from inside the
+# run that did the uploading. None of that is evidence about the artifact a user downloads, and the
+# gap is not theoretical: webrequest-hook v1.0.4 published as a zero-asset phantom, and
+# headroom-hook's published bundle could not boot the gateway because its shipped config used shapes
+# busbar 1.5.3 retired. Both were green everywhere. Nothing anywhere noticed.
+#
+# The logic lives in ONE place for the whole fleet, exactly like plugin-ci.yml, so a fix reaches
+# every plugin at once instead of being copied ten times and drifting nine.
+#
+# WHY BOTH TRIGGERS. release: published catches a broken publish immediately, and it fires whether or
+# not the release workflow itself finished happy - which matters, because a verifier that only runs
+# when everything already worked is not a verifier. The daily schedule catches ROT: an artifact that
+# published fine can stop working later when nothing about it changed (a bundle that no longer boots
+# against a newer engine, an asset deleted by hand, a release un-flagged as latest). A
+# publish-time-only check structurally cannot see that class.
+on:
+  release:
+    types: [published]
+  schedule:
+    - cron: " * * *"
+  workflow_dispatch:
+    inputs:
+      version:
+        description: "Version to verify (e.g. 1.0.4). Empty means the newest published release."
+        required: false
+        type: string
+
+permissions:
+  contents: read
+  issues: write
+  actions: read
+
+jobs:
+  consumer:
+    uses: GetBusbar/busbar/.github/workflows/plugin-consumer-verify.yml@dev
+    with:
+      version: ${{ inputs.version || '' }}
+      # Read off the PUBLISHED artifact, not guessed from the crate name: the filename prefix and the
+      # manifest name genuinely differ across this fleet (the store repos drop the trailing -plugin
+      # that the auth repos keep, and store-valkey publishes as busbar-store-redis).
+      asset_prefix: busbar-store-sqlite
+      plugin_name: busbar-store-sqlite-plugin
+      plugin_alias: sqlite
+      plugin_kind: store
+      # This repo publishes a runnable bundle, so the check does not stop at 'the tarball exists':
+      # the image is pulled fresh and the container must BOOT and answer /healthz. That is the
+      # assertion that was missing when the published bundle shipped a config busbar had retired.
+      bundle_image: 53 9
+      bundle_env: ""
+    secrets: inherit

.github/workflows/release.yml

@@ -237,3 +237,40 @@ jobs:
             -f event_type=upstream-release \
             -f "client_payload[repo]=${GITHUB_REPOSITORY}" \
             -f "client_payload[tag]=${GITHUB_REF_NAME}"
+
+  # THE LAST STEP: does what we just published work for a user?
+  #
+  # Everything above verifies what THIS RUN produced, from inside this run. verify-assets asserts the
+  # assets it uploaded are present, which proves the run believed itself. It cannot prove the tarball
+  # downloads for a stranger, unpacks into a plugin busbar will load, or (where a bundle is
+  # published) boots. Those are the failures that shipped: a zero-asset phantom release, and a
+  # published bundle that exits 1 on startup.
+  #
+  # Consumer verification is POST-PUBLICATION by nature - you cannot download an asset that was never
+  # uploaded - so this cannot block the publish and does not pretend to. What it does is make the
+  # verdict impossible to miss: a failure is THIS release run's failure, and the shared workflow also
+  # opens or updates one labelled issue naming the failing check, expected, observed and the run URL.
+  #
+  # !cancelled() because a needs: on a FAILED job skips its dependent by default - the exact shape
+  # that skips a release's own guard precisely when the release is broken.
+  consumer-verification:
+    name: consumer verification (LAST STEP)
+    needs: [verify-assets]
+    if: ${{ !cancelled() }}
+    uses: GetBusbar/busbar/.github/workflows/plugin-consumer-verify.yml@dev
+    with:
+      version: ${{ github.ref_name }}
+      asset_prefix: busbar-store-sqlite
+      plugin_name: busbar-store-sqlite-plugin
+      plugin_alias: sqlite
+      plugin_kind: store
+      # This repo publishes a runnable bundle, so the check does not stop at 'the tarball exists':
+      # the image is pulled fresh and the container must BOOT and answer /healthz. That is the
+      # assertion that was missing when the published bundle shipped a config busbar had retired.
+      bundle_image: 53 9
+      bundle_env: ""
+    permissions:
+      contents: read
+      issues: write
+      actions: read
+    secrets: inherit

store-sqlite-plugin/tests/e2e.rs

@@ -100,6 +100,37 @@ fn plugin_path() -> Option<PathBuf> {
     candidate
 }
 
+/// Every `env:` secret-ref name a config text references, in first-seen order, de-duplicated.
+///
+/// busbar 1.5.3 made `--validate` RESOLVE built-in (`env`/`file`) secret references and exit 1 when
+/// one cannot resolve, rather than only checking the reference's SHAPE. A fixture config that names
+/// a real-looking env var (here, `MOCK_KEY`) then fails `--validate` on any machine that doesn't
+/// happen to have that var set -- which is every CI runner and most dev machines. Hardcoding
+/// `MOCK_KEY` here would fix today's failure but rot the moment this fixture, or a future one, names
+/// a different variable. Extracting the names generically (same approach `GetBusbar/store-mysql`'s
+/// own `store-mysql-plugin/tests/e2e.rs` already took for this exact change, and the core repo's
+/// `crates/busbar/tests/docs_examples.rs`) keeps the harness working no matter what the fixture
+/// references.
+fn referenced_env_vars(text: &str) -> Vec<String> {
+    let mut v: Vec<String> = Vec::new();
+    for (i, _) in text.match_indices("env:") {
+        let rest = &text[i + 4..];
+        let name: String = rest
+            .chars()
+            .skip_while(|c| c.is_whitespace())
+            .take_while(|c| c.is_ascii_alphanumeric() || *c == '_')
+            .collect();
+        if !name.is_empty() && !v.contains(&name) {
+            v.push(name);
+        }
+    }
+    v
+}
+
+/// A placeholder value for a fixture-referenced secret: 64 hex chars, which is valid for
+/// `auth.signing_key` and harmless as any other secret's value.
+const SECRET_PLACEHOLDER: &str = "0000000000000000000000000000000000000000000000000000000000000001";
+
 /// The sibling busbarAI checkout's root (same convention this repo already uses for its path deps
 /// in Cargo.toml).
 fn busbarai_root() -> PathBuf {
@@ -200,20 +231,17 @@ fn load_and_exercise_sqlite_plugin_via_file_drop() {
         "mock:\n  protocol: anthropic\n  base_url: \"http://127.0.0.1:9\"\n  api_key_env: MOCK_KEY\n",
     )
     .unwrap();
-    std::fs::write(
-        &config,
-        format!(
-            "listen: \"127.0.0.1:0\"\n\
-             store:\n  module: sqlite\n  settings: {{ db_path: \"{}\" }}\n\
-             plugins:\n  enabled: true\n  dir: {}\n  trust:\n    allow_unsigned: true\n\
-             auth:\n  chain: []\n\
-             providers:\n  mock:\n    api_key: {{ env: MOCK_KEY }}\n\
-             models:\n  test-model:\n    provider: mock\n",
-            db_path.display(),
-            plugins_dir.display()
-        ),
-    )
-    .unwrap();
+    let config_text = format!(
+        "listen: \"127.0.0.1:0\"\n\
+         store:\n  module: sqlite\n  settings: {{ db_path: \"{}\" }}\n\
+         plugins:\n  enabled: true\n  dir: {}\n  trust:\n    allow_unsigned: true\n\
+         auth:\n  chain: []\n\
+         providers:\n  mock:\n    api_key: {{ env: MOCK_KEY }}\n\
+         models:\n  test-model:\n    provider: mock\n",
+        db_path.display(),
+        plugins_dir.display()
+    );
+    std::fs::write(&config, &config_text).unwrap();
 
     // `--validate` is DELIBERATELY not used for the load-proof itself: it is manifest-only by
     // design ("no server, no network, no state, no dlopen" -- crates/busbar/src/main.rs's own
@@ -224,12 +252,19 @@ fn load_and_exercise_sqlite_plugin_via_file_drop() {
     // plugin passes the trust/manifest gate; then a REAL BOOT (no `--validate` flag) is the only
     // thing that actually `dlopen`s the plugin and runs `Store::open`/migration, so that's what
     // proves the persistence claim.
-    let out = Command::new(&busbar_bin)
+    //
+    // `--validate` RESOLVES built-in `env:` secret references (busbar 1.5.3); give every one this
+    // fixture names a placeholder so the gate tests the config's SHAPE, not this machine's
+    // environment. See `referenced_env_vars`'s doc comment for why this is generic, not hardcoded.
+    let mut validate_cmd = Command::new(&busbar_bin);
+    validate_cmd
         .arg("--validate")
         .env("BUSBAR_CONFIG", &config)
-        .env("BUSBAR_PROVIDERS", &providers)
-        .output()
-        .expect("run busbar --validate");
+        .env("BUSBAR_PROVIDERS", &providers);
+    for name in referenced_env_vars(&config_text) {
+        validate_cmd.env(name, SECRET_PLACEHOLDER);
+    }
+    let out = validate_cmd.output().expect("run busbar --validate");
     assert!(
         out.status.success(),
         "busbar --validate must succeed with the file-dropped sqlite plugin: stdout={} stderr={}",