fix: preserve deployment failure evidence

README.md

@@ -82,7 +82,7 @@ The Release workflow publishes the promoted versions with npm provenance. It rej
 
 Do not merge `master` back into `dev`. `NPM_TOKEN` needs publish access to the `@understudy` scope.
 
-The Deploy workflow updates staging after every `dev` push. After the one-time protocol-3 cutover, it updates production after every `master` push. Production deployment first rebuilds the store extension and requires its normalized contents to match `apps/extension/store-release.json` in the `published` state.
+The Deploy workflow attempts a staging update after every `dev` push. The `staging` GitHub environment must provide its scoped `CLOUDFLARE_API_TOKEN` before the upload can run. After the one-time protocol-3 cutover, the workflow updates production after every `master` push. Production deployment first rebuilds the store extension and requires its normalized contents to match `apps/extension/store-release.json` in the `published` state.
 
 Use the guarded production wrapper for the first protocol-3 cutover and any later compatibility-contract change. It validates the protocol-3 device map, published extension, canary credential, immutable source snapshot, Worker provenance, and deployment evidence. Follow the [production rollout runbook](docs/unattended-production-rollout.md).
 

apps/backend/README.md

@@ -173,9 +173,9 @@ pnpm --filter @understudy/backend deploy:staging
 
 The command records local dirty provenance and writes evidence under `/tmp`. The next `dev` deployment can replace the local deployment.
 
-Before the first `dev` merge, create a GitHub `staging` environment restricted to `dev`, add a staging-scoped `CLOUDFLARE_API_TOKEN`, and run `provision:staging`. Create a separate `production` environment restricted to `master`, add a production-scoped token, and keep `PRODUCTION_AUTODEPLOY_ENABLED=false` until the manual compatibility cutover has passed. Workflow deployment tokens are exposed only to their deployment step.
+The GitHub `staging` environment exists, but its secret inventory was empty when last checked on 2026-08-04. Before rerunning the failed staging deployment, restrict the environment to `dev`, add a staging-scoped `CLOUDFLARE_API_TOKEN`, and run `provision:staging`. The separate `production` environment also exists with an empty secret inventory; restrict it to `master`, add a production-scoped token before production deployment, and keep `PRODUCTION_AUTODEPLOY_ENABLED=false` until the manual compatibility cutover has passed. Workflow deployment tokens are exposed only to their deployment step.
 
-Every deployment writes an `attempting`, `failed`, or `verified` evidence artifact. Failed post-upload evidence includes `priorDeployment`, the exact deployment state captured before upload. Recover staging with its prior 100% version:
+After the mode and evidence path are validated, every automated deployment writes an `attempting`, `failed`, or `verified` evidence artifact. Build, dry-run, prior-deployment lookup, source-ref, upload, and verification failures identify their stage. Failed post-upload evidence includes `priorDeployment`, the exact deployment state captured before upload. Recover staging with its prior 100% version:
 
 ```bash
 previous_version="$(jq -r '.priorDeployment.versions[] | select(.percentage == 100) | .version_id' /absolute/path/staging-deployment.json)"

apps/backend/scripts/deploy-lib.sh

@@ -53,6 +53,26 @@ understudy_deploy_dry_run() {
   fi
 }
 
+understudy_json_or_null() {
+  local value="${1:-null}"
+  if jq -e -n --argjson value "$value" '$value | type' >/dev/null 2>&1; then
+    printf '%s' "$value"
+  else
+    printf 'null'
+  fi
+}
+
+understudy_require_json_type() {
+  local value="$1"
+  local expected_type="$2"
+  local label="$3"
+  if ! jq -e -n --argjson value "$value" --arg expected "$expected_type" \
+    '$value | type == $expected' >/dev/null 2>&1; then
+    echo "$label did not return a JSON $expected_type" >&2
+    return 1
+  fi
+}
+
 understudy_versions_json() {
   understudy_wrangler_control_plane versions list --json
 }

apps/backend/scripts/deploy-production.sh

@@ -151,18 +151,16 @@ if [[ "$confirmation" != "DEPLOY" ]]; then
   exit 1
 fi
 
-assert_source_unchanged
-assert_current_master_head
-UNDERSTUDY_PRIOR_DEPLOYMENT="$(understudy_wrangler_control_plane deployments status --json)"
-prior_versions="$(understudy_versions_json)"
+UNDERSTUDY_PRIOR_DEPLOYMENT='null'
+prior_versions='null'
 health='null'
 active_version='null'
 source_release='null'
 deployment='null'
 device_tokens_secret_version='null'
 extension_id_secret_version='null'
 secret_derived='null'
-secret_mutation_possible='true'
+secret_mutation_possible='false'
 deployment_stage="prepared"
 
 write_evidence() {
@@ -227,13 +225,34 @@ record_failed_deployment() {
   local exit_code="$?"
   trap - EXIT
   set +e
+  compatibility_config="$(understudy_json_or_null "$compatibility_config")"
+  production_contract="$(understudy_json_or_null "$production_contract")"
+  store_release="$(understudy_json_or_null "$store_release")"
+  health="$(understudy_json_or_null "$health")"
+  source_release="$(understudy_json_or_null "$source_release")"
+  active_version="$(understudy_json_or_null "$active_version")"
+  deployment="$(understudy_json_or_null "$deployment")"
+  UNDERSTUDY_PRIOR_DEPLOYMENT="$(understudy_json_or_null "$UNDERSTUDY_PRIOR_DEPLOYMENT")"
+  prior_versions="$(understudy_json_or_null "$prior_versions")"
+  device_tokens_secret_version="$(understudy_json_or_null "$device_tokens_secret_version")"
+  extension_id_secret_version="$(understudy_json_or_null "$extension_id_secret_version")"
+  secret_derived="$(understudy_json_or_null "$secret_derived")"
   write_evidence "failed" "$deployment_stage" "$exit_code"
   cleanup
   exit "$exit_code"
 }
 
 write_evidence "attempting"
 trap record_failed_deployment EXIT
+deployment_stage="source-ref"
+assert_source_unchanged
+assert_current_master_head
+deployment_stage="prior-deployment"
+UNDERSTUDY_PRIOR_DEPLOYMENT="$(understudy_wrangler_control_plane deployments status --json)"
+understudy_require_json_type "$UNDERSTUDY_PRIOR_DEPLOYMENT" object "prior deployment"
+prior_versions="$(understudy_versions_json)"
+understudy_require_json_type "$prior_versions" array "prior version inventory"
+secret_mutation_possible='true'
 deployment_stage="device-token-secret"
 understudy_with_cloudflare_auth node "$backend_dir/scripts/put-validated-secret.mjs" \
   DEVICE_TOKENS "$device_tokens_sha256" <"$device_tokens_path"
@@ -252,6 +271,7 @@ extension_id_secret_version="$(
     '{before: $before, after: $after}' |
     node "$backend_dir/scripts/secret-version.mjs"
 )"
+deployment_stage="source-ref"
 assert_source_unchanged
 assert_current_master_head
 deployment_stage="upload"

apps/backend/scripts/deploy-target.integration.mjs

@@ -22,7 +22,7 @@ describe("deployment target integration", () => {
     const fixture = await deploymentFixture({ FAKE_SYSTEM_DNS: "missing" });
     const result = runDeployment(fixture);
 
-    assert.equal(result.status, 0, result.stderr);
+    assert.equal(result.status, 0, commandFailure(result));
     const evidence = JSON.parse(await readFile(fixture.evidence, "utf8"));
     assert.equal(evidence.outcome, "verified");
     assert.equal(evidence.activeWorkerVersion.id, "v1");
@@ -43,8 +43,7 @@ describe("deployment target integration", () => {
     assert.match(result.stderr, /expected source provenance/);
     assert.match(await readFile(fixture.log, "utf8"), /deploy --strict/);
     const evidence = JSON.parse(await readFile(fixture.evidence, "utf8"));
-    assert.equal(evidence.outcome, "failed");
-    assert.equal(evidence.failureStage, "verification");
+    assertFailedEvidence(evidence, result, "verification");
     assert.equal(evidence.priorDeployment.versions[0].version_id, "v0");
   });
 
@@ -55,33 +54,59 @@ describe("deployment target integration", () => {
     assert.notEqual(result.status, 0);
     assert.match(result.stderr, /health provenance did not converge/);
     const evidence = JSON.parse(await readFile(fixture.evidence, "utf8"));
-    assert.equal(evidence.outcome, "failed");
-    assert.equal(evidence.failureStage, "verification");
+    assertFailedEvidence(evidence, result, "verification");
     assert.equal(evidence.health, null);
   });
+
+  it("records failed evidence when deployment credentials are missing", async () => {
+    const fixture = await deploymentFixture({ FAKE_REQUIRE_AUTH: "true" });
+    delete fixture.env.CLOUDFLARE_API_TOKEN;
+    const result = runDeployment(fixture);
+
+    assert.notEqual(result.status, 0);
+    assert.match(
+      result.stderr,
+      /Cloudflare API token required/,
+      commandFailure(result),
+    );
+    const evidence = JSON.parse(await readFile(fixture.evidence, "utf8"));
+    assertFailedEvidence(evidence, result, "prior-deployment");
+    assert.equal(evidence.priorDeployment, null);
+  });
+
+  it("rejects malformed prior-deployment JSON before upload", async () => {
+    const fixture = await deploymentFixture({
+      FAKE_PRIOR_DEPLOYMENT_JSON: "{not-json",
+    });
+    const result = runDeployment(fixture);
+
+    assert.notEqual(result.status, 0);
+    assert.match(result.stderr, /prior deployment did not return a JSON object/);
+    const evidence = JSON.parse(await readFile(fixture.evidence, "utf8"));
+    assertFailedEvidence(evidence, result, "prior-deployment");
+    assert.equal(evidence.priorDeployment, null);
+    assert.doesNotMatch(await readFile(fixture.log, "utf8"), /deploy --strict/);
+  });
+
+  it("records build failures before any control-plane access", async () => {
+    const fixture = await deploymentFixture({ FAKE_BUILD_FAILURE: "true" });
+    const result = runDeployment(fixture);
+
+    assert.notEqual(result.status, 0);
+    assert.match(result.stderr, /protocol build failed/);
+    const evidence = JSON.parse(await readFile(fixture.evidence, "utf8"));
+    assertFailedEvidence(evidence, result, "build");
+    assert.equal(evidence.priorDeployment, null);
+    assert.doesNotMatch(await readFile(fixture.log, "utf8"), /deployments status/);
+  });
 });
 
 describe("manual production cutover integration", () => {
   it("attributes each new secret version from newest-first inventories", async () => {
     const fixture = await manualDeploymentFixture();
-    const result = spawnSync(
-      "bash",
-      [
-        MANUAL_SCRIPT,
-        fixture.evidence,
-        fixture.deviceTokens,
-        fixture.extensionId,
-        fixture.canaryCredential,
-      ],
-      {
-        cwd: REPO_ROOT,
-        env: fixture.env,
-        input: "DEPLOY\n",
-        encoding: "utf8",
-      },
-    );
+    const result = runManualDeployment(fixture);
 
-    assert.equal(result.status, 0, result.stderr);
+    assert.equal(result.status, 0, commandFailure(result));
     const evidence = JSON.parse(await readFile(fixture.evidence, "utf8"));
     assert.equal(evidence.outcome, "verified");
     assert.equal(evidence.deviceTokensSecretVersion.id, "device-secret-new");
@@ -99,6 +124,40 @@ describe("manual production cutover integration", () => {
     assert.match(log, /secret put DEVICE_TOKENS.*auth=present/);
     assert.match(log, /secret put EXTENSION_ID.*auth=present/);
   });
+
+  it("records missing production credentials before secret mutation", async () => {
+    const fixture = await manualDeploymentFixture({ FAKE_REQUIRE_AUTH: "true" });
+    delete fixture.env.CLOUDFLARE_API_TOKEN;
+    const result = runManualDeployment(fixture);
+
+    assert.notEqual(result.status, 0);
+    assert.match(
+      result.stderr,
+      /Cloudflare API token required/,
+      commandFailure(result),
+    );
+    const evidence = JSON.parse(await readFile(fixture.evidence, "utf8"));
+    assertFailedEvidence(evidence, result, "prior-deployment");
+    assert.equal(evidence.priorDeployment, null);
+    assert.equal(evidence.secretMutationPossible, false);
+    assert.doesNotMatch(await readFile(fixture.log, "utf8"), /secret put/);
+  });
+
+  it("attributes a post-secret source-ref failure", async () => {
+    const fixture = await manualDeploymentFixture({
+      FAKE_LATE_SOURCE_FAILURE: "true",
+    });
+    const result = runManualDeployment(fixture);
+
+    assert.notEqual(result.status, 0);
+    const evidence = JSON.parse(await readFile(fixture.evidence, "utf8"));
+    assertFailedEvidence(evidence, result, "source-ref");
+    assert.equal(evidence.secretMutationPossible, true);
+    const log = await readFile(fixture.log, "utf8");
+    assert.match(log, /secret put DEVICE_TOKENS/);
+    assert.match(log, /secret put EXTENSION_ID/);
+    assert.doesNotMatch(log, /deploy --strict/);
+  });
 });
 
 async function deploymentFixture(overrides = {}) {
@@ -131,14 +190,27 @@ auth=absent
 if [[ -n "\${CLOUDFLARE_API_TOKEN:-}" ]]; then auth=present; fi
 printf '%s auth=%s\\n' "$*" "$auth" >>"$FAKE_LOG"
 case "$*" in
-  "--filter @understudy/protocol build") ;;
+  "--filter @understudy/protocol build")
+    if [[ "\${FAKE_BUILD_FAILURE:-}" == "true" ]]; then
+      printf 'protocol build failed\n' >&2
+      exit 1
+    fi
+    ;;
   "--version") printf '11.5.2\\n' ;;
   *"wrangler deploy --dry-run"*) ;;
   *"wrangler deploy --strict"*) ;;
   *"wrangler versions list --json"*)
     printf '[{"id":"v1","annotations":{"workers/tag":"%s","workers/message":"source %s"}}]\\n' "\${FAKE_VERSION_TAG:-$FAKE_SHA}" "$FAKE_SHA"
     ;;
   *"wrangler deployments status --json"*)
+    if [[ "\${FAKE_REQUIRE_AUTH:-}" == "true" && "$auth" == "absent" ]]; then
+      printf 'Cloudflare API token required\n' >&2
+      exit 1
+    fi
+    if [[ -n "\${FAKE_PRIOR_DEPLOYMENT_JSON:-}" ]]; then
+      printf '%s\n' "$FAKE_PRIOR_DEPLOYMENT_JSON"
+      exit 0
+    fi
     count_file="$FAKE_STATE/status-count"
     count=0
     if [[ -f "$count_file" ]]; then count="$(<"$count_file")"; fi
@@ -184,7 +256,7 @@ printf '{"ok":true,"commit":"%s","versionId":"v1","deployedAt":"2030-01-01T00:00
   };
 }
 
-async function manualDeploymentFixture() {
+async function manualDeploymentFixture(overrides = {}) {
   const root = await mkdtemp(join(tmpdir(), "understudy-manual-deploy-test-"));
   temporary.push(root);
   const bin = join(root, "bin");
@@ -222,7 +294,18 @@ async function manualDeploymentFixture() {
 set -eu
 case "$*" in
   *"rev-parse --show-toplevel"*) printf '%s\\n' "$FAKE_REPO_ROOT" ;;
-  *"rev-parse refs/remotes/origin/master"*) printf '%s\\n' "$FAKE_SHA" ;;
+  *"rev-parse refs/remotes/origin/master"*)
+    count_file="$FAKE_STATE/master-ref-count"
+    count=0
+    if [[ -f "$count_file" ]]; then count="$(<"$count_file")"; fi
+    count=$((count + 1))
+    printf '%s' "$count" >"$count_file"
+    if [[ "\${FAKE_LATE_SOURCE_FAILURE:-}" == "true" && "$count" -ge 3 ]]; then
+      printf '%s\\n' "c\${FAKE_SHA:1}"
+    else
+      printf '%s\\n' "$FAKE_SHA"
+    fi
+    ;;
   *"rev-parse HEAD"*) printf '%s\\n' "$FAKE_SHA" ;;
   *"status --porcelain"*) ;;
   *"fetch --quiet origin"*) ;;
@@ -258,6 +341,10 @@ case "$*" in
   *"wrangler deploy --dry-run"*|*"wrangler deploy --strict"*) ;;
   *"wrangler secret put"*) IFS= read -r _ || true ;;
   *"wrangler deployments status --json"*)
+    if [[ "\${FAKE_REQUIRE_AUTH:-}" == "true" && "$auth" == "absent" ]]; then
+      printf 'Cloudflare API token required\n' >&2
+      exit 1
+    fi
     count_file="$FAKE_STATE/deployment-count"
     count=0
     if [[ -f "$count_file" ]]; then count="$(<"$count_file")"; fi
@@ -299,6 +386,7 @@ printf '{"ok":true,"commit":"%s","versionId":"v1","deployedAt":"2030-01-01T00:00
     log,
     env: {
       ...process.env,
+      ...overrides,
       PATH: `${bin}:${process.env.PATH}`,
       FAKE_LOG: log,
       FAKE_REPO_ROOT: REPO_ROOT,
@@ -318,6 +406,42 @@ function runDeployment(fixture) {
   });
 }
 
+function runManualDeployment(fixture) {
+  return spawnSync(
+    "bash",
+    [
+      MANUAL_SCRIPT,
+      fixture.evidence,
+      fixture.deviceTokens,
+      fixture.extensionId,
+      fixture.canaryCredential,
+    ],
+    {
+      cwd: REPO_ROOT,
+      env: fixture.env,
+      input: "DEPLOY\n",
+      encoding: "utf8",
+    },
+  );
+}
+
+function commandFailure(result) {
+  return JSON.stringify({
+    status: result.status,
+    signal: result.signal,
+    error: result.error?.message,
+    stdout: result.stdout,
+    stderr: result.stderr,
+  });
+}
+
+function assertFailedEvidence(evidence, result, failureStage) {
+  assert.notEqual(result.status, 0);
+  assert.equal(evidence.outcome, "failed");
+  assert.equal(evidence.failureStage, failureStage);
+  assert.equal(evidence.exitCode, result.status);
+}
+
 async function executable(path, contents) {
   await writeFile(path, contents);
   await chmod(path, 0o755);