From ffa25c622b4664f9fe5754eb3a2182e64d6a566d Mon Sep 17 00:00:00 2001 From: Sourabh Choraria Date: Fri, 31 Jul 2026 23:47:17 +0100 Subject: [PATCH 1/2] feat(dev): seal against the REAL KEK locally, closing the last parity gap MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit `KMS_MODE=local` was the default, so every local stack sealed provider secrets and ingest tokens under a hermetic throwaway KEK. The envelope-encryption path — the compliance-critical one — was therefore never exercised before a deploy. It ran in production and nowhere else. The ledger said this one "has no alternative today: nobody should hold production AWS credentials to develop". The premise was wrong, in exactly the way that page warns about: THE CREDENTIAL ALREADY EXISTED. IAM user `webhook-co-claude-code` is already least-privilege — kms:GenerateDataKey + kms:Decrypt, on one key, nothing else. AGENTS.md says to look for the real credential before inventing a substitute; it had been there the whole time, recorded, while the page argued it could not exist. Verified before changing anything: a DEK wrapped by the real KEK, unwrapped, and the two proven to be the same key by encrypting with one and decrypting with the other. Then the engine booted with AWS bound and no KMS error. What a machine holds is the ability to wrap and unwrap, not the key — the KEK never leaves AWS. It is still a production capability, so the credential PAIR is excluded from the vault for the same reason RESEND_API_KEY is: narrow is not the same as non-production, and a passphrase beside the ciphertext is not where that belongs. KMS_KEY_ARN and AWS_REGION do travel, because an ARN and a region name are identifiers and withholding them is friction without security. Both halves of the pair are excluded together, so nobody has to wonder which half was the secret one. KMS_MODE survives as the opt-out for a contributor who cannot hold AWS credentials, now `external` with BLANK as the correct value rather than `local` with "local" — the substitute is no longer what you get by default. The engine already refuses the flag when any AWS field is bound, so the two cannot be half-applied; four existing tests cover that fence, including partial config. ⚠️ I could not trigger that fence live: the provider is constructed lazily on endpoint creation, so booting the engine never reaches it. The tests are the evidence, not my probe — saying so because "I ran it and saw nothing" is not the same as "it fired". Co-Authored-By: Claude Opus 5 Claude-Session: https://claude.ai/code/session_01BRmGUnxeYsQoG9c8BCZcae --- apps/engine/dev.vars.example | 16 ++++++++-- docs/local-parity.md | 33 +++++++++++++++++--- scripts/dev-secrets-manifest.mjs | 50 ++++++++++++++++++++++++++++-- scripts/dev-secrets-vault.mjs | 15 +++++++++ scripts/dev-secrets-vault.test.mjs | 25 ++++++++++++++- 5 files changed, 129 insertions(+), 10 deletions(-) diff --git a/apps/engine/dev.vars.example b/apps/engine/dev.vars.example index 0eeb73f0..161698a2 100644 --- a/apps/engine/dev.vars.example +++ b/apps/engine/dev.vars.example @@ -24,8 +24,20 @@ STRIPE_METER_EVENT_NAME= # EXTERNAL: Stripe billing-meter id from your sandbox. Pairs with STRIPE_METER_EVENT_NAME. STRIPE_METER_ID= -# LOCAL: Selects the hermetic dev KEK custodian instead of AWS KMS. Without it, creating an endpoint needs real AWS IAM credentials. Lives ONLY here — .dev.vars is never sent by `wrangler deploy`, kmsProviderFromEnv refuses it whenever AWS KMS config is bound, and scripts/kms-mode-guard.mjs keeps it out of every committed Worker config. -KMS_MODE=local +# EXTERNAL: REQUIRED for prod parity — the KEK ARN. An identifier, not a secret, so the vault carries it. +KMS_KEY_ARN= + +# EXTERNAL: REQUIRED for prod parity — the KEK's region (us-east-2). Not a secret; the vault carries it. +AWS_REGION= + +# EXTERNAL: REQUIRED for prod parity — the IAM user scoped to kms:GenerateDataKey + kms:Decrypt on that one key. NOT vaulted: it pairs with a credential that reaches production. +AWS_ACCESS_KEY_ID= + +# EXTERNAL: REQUIRED for prod parity. NOT vaulted — it grants Decrypt against the production KEK, which is a production capability regardless of how narrow the policy is. +AWS_SECRET_ACCESS_KEY= + +# EXTERNAL: LEAVE BLANK so the real AWS KMS custodian runs, exactly as in production. Set it to "local" ONLY if you cannot hold AWS credentials; endpoint creation then seals under a hermetic throwaway KEK and the AWS path is never exercised. Lives ONLY here — .dev.vars is never sent by `wrangler deploy`, kmsProviderFromEnv refuses it whenever AWS KMS config is bound, and scripts/kms-mode-guard.mjs keeps it out of every committed Worker config. +KMS_MODE= # GENERATED: The dev KEK, 32 random bytes. Stable across restarts so endpoints sealed before a restart stay openable — LocalKmsProvider.generate() would mint a fresh key each process and orphan them. LOCAL_KEK= diff --git a/docs/local-parity.md b/docs/local-parity.md index 0768665c..bdc4d352 100644 --- a/docs/local-parity.md +++ b/docs/local-parity.md @@ -159,7 +159,7 @@ default**, and each is refused outright if it is ever seen on a deployed Worker | --- | --- | | `EMAIL_MODE=log` | mail prints to the console, link included, instead of sending | | `OAUTH_MODE=optional` | providers without both halves configured are not wired; magic link still works — and with Google unwired, **no Google script loads on the login page at all** | -| `KMS_MODE=local` | a process-local KEK instead of AWS KMS, so endpoints can be created without AWS | +| `KMS_MODE=local` | a process-local KEK instead of AWS KMS, for a contributor who cannot hold AWS credentials — **no longer the default**, see below | ### ~~`TURNSTILE_MODE`~~ — CLOSED: the playground's captcha runs locally @@ -207,9 +207,34 @@ machine. Run `stripe listen --print-secret` and use that one while forwarding. T wrong is `400 invalid signature` with nothing naming the cause — which is exactly why the credential vault refuses to carry this variable in either direction. -`KMS_MODE=local` is the one that has no alternative today: the engine is the sole KEK custodian and nobody -should hold production AWS credentials to develop. The other two are conveniences, and using them means -accepting that you are not testing what production does. +### ~~`KMS_MODE=local`~~ — CLOSED: local seals against the real KEK + +**Local dev now uses the same KEK custodian production does**, through the same `AwsKmsProvider` seam. It +used to default to the hermetic `LocalKmsProvider`, which meant the envelope-encryption path — the one that +seals provider secrets and ingest tokens — was never exercised before a deploy. That was the +compliance-critical path running only in production. + +The page used to say this had "no alternative today: nobody should hold production AWS credentials to +develop". The premise was wrong in the way this page warns about: **the credential already existed**. IAM +user `webhook-co-claude-code` is already least-privilege — `kms:GenerateDataKey` + `kms:Decrypt`, on one +key, and nothing else. AGENTS.md says to look for the real credential before inventing a substitute, and it +was there the whole time. + +Verified with a real round-trip: a DEK wrapped by the KEK, unwrapped, and the two proven identical by +encrypting with one and decrypting with the other. + +⚠️ **What a machine holds is the ability to wrap and unwrap, not the key** — the KEK never leaves AWS. It +is still a production capability, so `AWS_ACCESS_KEY_ID` / `AWS_SECRET_ACCESS_KEY` are **excluded from the +credential vault** for the same reason `RESEND_API_KEY` is: a passphrase sitting beside the ciphertext is +not where a production capability belongs. `KMS_KEY_ARN` and `AWS_REGION` do travel — an ARN and a region +name are identifiers, and withholding them would be friction without security. + +`KMS_MODE=local` survives as the opt-out for a contributor who cannot hold AWS credentials. The engine +**refuses** it outright when any AWS field is bound, so the two configurations cannot be half-applied — +four tests cover that fence, including the partial-config case. + +The remaining two flags are conveniences, and using them means accepting that you are not testing what +production does. **Setting them:** put the value in the app's `.dev.vars`. `pnpm dev:secrets` writes a commented template listing exactly what each app wants and which values are required for parity. diff --git a/scripts/dev-secrets-manifest.mjs b/scripts/dev-secrets-manifest.mjs index 43b968de..9419976c 100644 --- a/scripts/dev-secrets-manifest.mjs +++ b/scripts/dev-secrets-manifest.mjs @@ -143,11 +143,55 @@ export const APPS = { scope: "external", note: "Stripe billing-meter id from your sandbox. Pairs with STRIPE_METER_EVENT_NAME.", }, + // --- The KEK custodian ------------------------------------------------------------------ + // Local dev now uses the SAME custodian production does: real AWS KMS, through the same + // `AwsKmsProvider` seam, with the team's existing least-privilege IAM user (kms:GenerateDataKey + + // kms:Decrypt on one key, and nothing else). The KEK itself never leaves AWS — only wrapped data + // keys cross the wire — so what a machine holds is the ability to wrap and unwrap, not the key. + // + // This used to default to the hermetic LocalKmsProvider, which meant the envelope-encryption path + // that seals provider secrets and ingest tokens — the compliance-critical one — was never exercised + // before a deploy. AGENTS.md: look for the real credential before inventing a substitute. It + // existed the whole time. + // + // ⚠️ apps/engine REFUSES `KMS_MODE=local` when any AWS field is bound, so these two configurations + // cannot be half-applied: bind AWS and the flag is rejected outright rather than quietly sealing + // secrets under a throwaway key nobody custodies. + { + name: "KMS_KEY_ARN", + scope: "external", + parityRequired: true, + relaxedBy: { name: "KMS_MODE", value: "local" }, + note: "REQUIRED for prod parity — the KEK ARN. An identifier, not a secret, so the vault carries it.", + }, + { + name: "AWS_REGION", + scope: "external", + parityRequired: true, + relaxedBy: { name: "KMS_MODE", value: "local" }, + note: "REQUIRED for prod parity — the KEK's region (us-east-2). Not a secret; the vault carries it.", + }, + { + name: "AWS_ACCESS_KEY_ID", + scope: "external", + parityRequired: true, + relaxedBy: { name: "KMS_MODE", value: "local" }, + note: "REQUIRED for prod parity — the IAM user scoped to kms:GenerateDataKey + kms:Decrypt on that one key. NOT vaulted: it pairs with a credential that reaches production.", + }, + { + name: "AWS_SECRET_ACCESS_KEY", + scope: "external", + parityRequired: true, + relaxedBy: { name: "KMS_MODE", value: "local" }, + note: "REQUIRED for prod parity. NOT vaulted — it grants Decrypt against the production KEK, which is a production capability regardless of how narrow the policy is.", + }, { name: "KMS_MODE", - scope: "local", - value: "local", - note: "Selects the hermetic dev KEK custodian instead of AWS KMS. Without it, creating an endpoint needs real AWS IAM credentials. Lives ONLY here — .dev.vars is never sent by `wrangler deploy`, kmsProviderFromEnv refuses it whenever AWS KMS config is bound, and scripts/kms-mode-guard.mjs keeps it out of every committed Worker config.", + // `external`, and correct value BLANK: blank means real AWS KMS, matching production. It was + // `local`/"local", which made the substitute the default for everyone. Set it to "local" only if + // you cannot hold AWS credentials — the endpoint-creation path then seals under a throwaway KEK. + scope: "external", + note: 'LEAVE BLANK so the real AWS KMS custodian runs, exactly as in production. Set it to "local" ONLY if you cannot hold AWS credentials; endpoint creation then seals under a hermetic throwaway KEK and the AWS path is never exercised. Lives ONLY here — .dev.vars is never sent by `wrangler deploy`, kmsProviderFromEnv refuses it whenever AWS KMS config is bound, and scripts/kms-mode-guard.mjs keeps it out of every committed Worker config.', }, { name: "LOCAL_KEK", diff --git a/scripts/dev-secrets-vault.mjs b/scripts/dev-secrets-vault.mjs index 7131179c..ad8c4916 100644 --- a/scripts/dev-secrets-vault.mjs +++ b/scripts/dev-secrets-vault.mjs @@ -149,6 +149,21 @@ export const NOT_SHAREABLE = new Map([ // secrets live in a KMS, and an offline-attackable git blob is not one. "it is the same key production sends with — a prod capability does not belong in a passphrase-protected git blob", ], + [ + "AWS_SECRET_ACCESS_KEY", + // Blast radius, same test as RESEND_API_KEY: this grants kms:Decrypt against the PRODUCTION KEK. The + // policy is narrow — one key, two actions — but "narrow" is not "not production". The vault's + // protection is a passphrase sitting beside the ciphertext, and a production capability does not + // belong behind one. KMS_KEY_ARN and AWS_REGION DO travel: an ARN and a region name are identifiers, + // and withholding them would cost setup friction for no security. + "it grants Decrypt against the production KEK — narrow is not the same as non-production", + ], + [ + "AWS_ACCESS_KEY_ID", + // Useless alone, but it names the principal the secret half belongs to. Splitting a credential pair + // across "vaulted" and "fetch by hand" invites someone to wonder which half is the secret one. + "it is one half of a credential whose other half reaches production; they travel together or not at all", + ], [ "STRIPE_WEBHOOK_SIGNING_SECRET", // Machine scope. A registered Stripe endpoint POSTs to a public URL and can never reach localhost, so diff --git a/scripts/dev-secrets-vault.test.mjs b/scripts/dev-secrets-vault.test.mjs index 33dd8f91..a7c265d9 100644 --- a/scripts/dev-secrets-vault.test.mjs +++ b/scripts/dev-secrets-vault.test.mjs @@ -66,12 +66,14 @@ test("never shares a name that is a LOCAL literal in any app", () => { // The exact set, not a floor. A floor of 6 passed while every STRIPE_* name silently dropped out of // discovery — the tool would still report success, just quietly stop sharing half of what it claims to. -test("the shared set is exactly the documented 9 names", () => { +test("the shared set is exactly the documented 11 names", () => { assert.deepEqual(sharedSecretNames(), [ + "AWS_REGION", "GITHUB_CLIENT_ID", "GITHUB_CLIENT_SECRET", "GOOGLE_CLIENT_ID", "GOOGLE_CLIENT_SECRET", + "KMS_KEY_ARN", "STRIPE_METER_ID", "STRIPE_PLANS", "STRIPE_PORTAL_CONFIGURATION_ID", @@ -361,3 +363,24 @@ test("collecting skips apps with no .dev.vars, and blank values", () => { ); assert.deepEqual(conflicts, []); }); + +// The KEK custodian's credential reaches production, so it must never enter a passphrase-protected vault +// — the same test applied to RESEND_API_KEY. Its non-secret companions must still travel, or closing the +// KMS parity gap would cost four manual values instead of two. +test("the AWS credential pair is excluded, but its identifiers still travel", () => { + const shared = sharedSecretNames(); + for (const secret of ["AWS_SECRET_ACCESS_KEY", "AWS_ACCESS_KEY_ID"]) { + assert.ok(!shared.includes(secret), `${secret} reaches production and must not be vaulted`); + assert.deepEqual( + [...pullSet(new Map([[secret, "leaked"]]), [secret])], + [], + `${secret} pulled anyway`, + ); + } + for (const id of ["KMS_KEY_ARN", "AWS_REGION"]) { + assert.ok( + shared.includes(id), + `${id} is an identifier, not a secret — withholding it is pure friction`, + ); + } +}); From 4e2d24a59d4488ac0ac3cb89caff69cd6573d3ae Mon Sep 17 00:00:00 2001 From: Sourabh Choraria Date: Fri, 31 Jul 2026 23:55:33 +0100 Subject: [PATCH 2/2] test(dev): pin the engine's AWS KMS parity contract MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Right, and the same gap the review caught on the last two PRs: the load-bearing change here is that four AWS fields became `parityRequired` with `relaxedBy: KMS_MODE=local`, and nothing pinned it. Drop the flag or break the opt-out and local dev silently returns to sealing under a throwaway KEK with the vault, example and kms-provider suites all still green. The runtime fence does not cover this. It refuses `KMS_MODE=local` when AWS is bound — a different failure, and one that only fires once a provider is constructed, which is lazy. Nothing was checking that the fields are demanded in the first place. Three tests, mirroring the dmarc Resend and play Turnstile pins: each field is required and relaxed by that one flag; a BLANK KMS_MODE demands all four; and only "local" relaxes them. That middle one matters most. Blank is the PARITY path here, which inverts the usual convention — the substitute is the flag's presence, not its absence — so "blank relaxes" is exactly the one-character change that would put everyone back on the throwaway KEK without a single test going red. Mutation-verified: dropping `parityRequired` fails, and making blank relax fails. Co-Authored-By: Claude Opus 5 Claude-Session: https://claude.ai/code/session_01BRmGUnxeYsQoG9c8BCZcae --- scripts/dev-preflight.test.mjs | 45 ++++++++++++++++++++++++++++++++++ 1 file changed, 45 insertions(+) diff --git a/scripts/dev-preflight.test.mjs b/scripts/dev-preflight.test.mjs index c4ccde49..d4d4db9a 100644 --- a/scripts/dev-preflight.test.mjs +++ b/scripts/dev-preflight.test.mjs @@ -197,3 +197,48 @@ test("a blank play secret with the challenge ON is reported by findings()", () = "the playground would mint with an unverifiable challenge and nothing would say so", ); }); + +// The engine's KEK custodian, mirroring the dmarc Resend and play Turnstile pins. This PR's load-bearing +// change is that real AWS KMS is now the local DEFAULT — drop `parityRequired` or break `relaxedBy` and +// local dev silently returns to sealing under a throwaway KEK, with the vault, example and kms-provider +// suites all still green. The runtime fence does not help: it only fires once a provider is constructed. +const AWS_KMS_SPECS = ["KMS_KEY_ARN", "AWS_REGION", "AWS_ACCESS_KEY_ID", "AWS_SECRET_ACCESS_KEY"]; + +test("every AWS KMS field is ENFORCED for the engine, with the same opt-out", () => { + const required = requiredSpecs("engine"); + for (const name of AWS_KMS_SPECS) { + const spec = required.find((s) => s.name === name); + assert.ok(spec, `${name} blank would still start \`pnpm dev\` — local would not use real KMS`); + assert.deepEqual( + spec.relaxedBy, + { name: "KMS_MODE", value: "local" }, + `${name} must be relaxed by the documented opt-out and nothing else`, + ); + } +}); + +test("a blank KMS_MODE means REAL AWS, so the four fields are demanded", () => { + // Blank is the parity path here, which inverts the usual convention — the substitute is the flag, not + // its absence. If blank started relaxing them, the default would quietly become the throwaway KEK again. + const problems = findings([{ app: "engine", exists: true, source: "KMS_MODE=\n" }]); + const missing = problems.flatMap((p) => p.missing ?? []); + for (const name of AWS_KMS_SPECS) { + assert.ok(missing.includes(name), `${name} was not demanded when KMS_MODE is blank`); + } +}); + +test("KMS_MODE=local is the ONLY thing that relaxes them", () => { + const spec = requiredSpecs("engine").find((s) => s.name === "KMS_KEY_ARN"); + assert.ok( + isRelaxed(spec, new Map([["KMS_MODE", "local"]])), + "the documented opt-out does not work", + ); + assert.ok( + !isRelaxed(spec, new Map([["KMS_MODE", ""]])), + "blank must NOT relax — blank is the parity path", + ); + assert.ok( + !isRelaxed(spec, new Map([["KMS_MODE", "aws"]])), + "an unrecognised value must not relax either", + ); +});