diff --git a/.drive/projects/forcing-function-apps/plan.md b/.drive/projects/forcing-function-apps/plan.md index ff28814b..f77a9003 100644 --- a/.drive/projects/forcing-function-apps/plan.md +++ b/.drive/projects/forcing-function-apps/plan.md @@ -56,49 +56,64 @@ with that session, not a parallel spike against their moving target. ## Milestone 1: datahub on the framework -### S1 — Secrets as bindings - -Secret declared as a dependency, resolved to a binding; backing grounded -against the platform surface (Compute env vars vs management-API store) before -the design settles. ADR for the secrets model. - -- **Builds on:** nothing. -- **Hands to:** S2 — apps can declare secret inputs and receive typed bindings. - -### S2 — datahub port skeleton - -datahub deployed via `prisma-app deploy` from its own repo: ingest + web -services, postgres resource, secrets bindings, published/preview packages. -Scheduling unchanged (in-process tick) for this slice. - -- **Builds on:** S1; publishing pipeline. -- **Hands to:** S4 — a framework-deployed datahub verified equivalent to the - current deployment. - -### S3 — datahub consumes cron - -The cron design and mechanism moved to its own project — **[Config Params + -Cron](../config-params-and-cron/spec.md)** (ADR-0018/0019/0020): cron is a driver -System (a scheduler that depends on what it calls), built on a new schema-typed, -target-serialized config param. Designing cron for datahub is what surfaced the -config-model change, so that foundation was carved out. - -This slice is now just the datahub side: wire datahub's `/tick` to the cron -`cron-scheduler` + a `router`, with the schedule as a `defineSchedule` param. - -- **Builds on:** the Config Params + Cron project delivering a working cron; S2 - (datahub port skeleton). -- **Hands to:** S4 — datahub's scheduled ingest running on the framework's cron. - -### S4 — datahub on cron + cutover - -`/tick` driven by the cron resource; equivalence verified; the team's real -instance cut over. Closes M1. - -- **Builds on:** S2, S3. +### S1 — Secrets as bindings — RESOLVED, NO SLICE + +The Config Params + Cron project absorbed most of this: params carry a +`secret` facet (redacted in introspection — including defaults, fixed on +`claude/datahub-port` — and stored as encrypted env vars by the target). The +remaining piece, where secret *values* come from, is bridged in the port by +deployer-env-sourced defaults (`fromEnv()`; `system.ts` loads on the +deployer's machine). A first-class deploy-values mechanism is recorded under +Follow-ups; no separate secrets slice or ADR proved necessary for M1. + +### S2+S3 — datahub port (skeleton + cron) — DONE, one slice (TML-3012) + +Cron shipped as `@prisma/app-cron` ([prisma/app#45](https://github.com/prisma/app/pull/45)), +so S2 and S3 ran together. Spec: +[specs/s2s3-datahub-port.md](specs/s2s3-datahub-port.md). Datahub branch +`claude/prisma-app-port`: `system.ts` (postgres + cron(ingest-as-router) + +web), ingest params from the original zod schemas with secret facets, the +in-process `TICK_INTERVAL_MS` scheduler deleted (the cron scheduler is the +only clock), framework packages via pkg.pr.new previews of #45. Proven by a +`Load(system)` graph test and a `bootstrapService` boot of the real ingest +entry (health + trigger dispatch). Port evidence recorded under Follow-ups. + +### S4 — datahub live deploy + cutover + +Deploy the ported datahub via `prisma-app deploy` with the team's real secrets +and workspace credentials; verify equivalence against the current deployment; +cut the team's real instance over. Closes M1. **Operator-gated** — needs +credentials agents don't hold. + +- **Builds on:** S2+S3 (done); prisma/app#45 merged (or npm-published + packages replacing the preview pins). - **Hands to:** M2 — port mechanics proven, first emulated resource in production. +### Follow-ups (evidence from the S2+S3 port) + +- **pnPostgres conversion of `@workspace/db`** (ADR-0022): datahub's db layer + is prisma-next with a contract — exactly what `pnPostgres` types. Converting + would also eliminate the phantom-dependency fragility the port had to pin + around (`@prisma-next/*` + `pg` reached only via hoisting; adding + `@prisma/app-cloud` broke the hoist and silently degraded `db.orm` to `any`). +- **A blessed pattern for module-global DB clients.** The port bridges + `DATABASE_URL` from `load()` ad hoc (ingest: dynamic-import entry; web: + `instrumentation.ts` + a lazy client). Real apps will keep hitting this; + the framework should bless one pattern. +- **Deploy-time param values.** `fromEnv()` (deployer-env-as-default) works + but is pure app-side convention — no framework validation that a required + secret is present at deploy. A first-class values mechanism is the durable + fix (the old S1's remaining scope). +- **Next.js static generation never runs `instrumentation.ts`** (Next 16.1.6): + a page that queries the DB at build time can't get its config from `load()`; + the port went `force-dynamic`. Worth a documented stance for the nextjs + adapter. +- **bun tarball fan-in bug**: 3+ workspace packages depending on the same + pkg.pr.new tarball URL makes `bun install` fail non-deterministically until + the cache warms (repros on bun 1.3.13/1.3.14). Worth an upstream repro if it + recurs. + ### Parallelisation Two independent threads join at S4: diff --git a/.drive/projects/forcing-function-apps/specs/s2s3-datahub-port.md b/.drive/projects/forcing-function-apps/specs/s2s3-datahub-port.md new file mode 100644 index 00000000..59815a26 --- /dev/null +++ b/.drive/projects/forcing-function-apps/specs/s2s3-datahub-port.md @@ -0,0 +1,127 @@ +# S2+S3 — datahub onto @prisma/app: port skeleton + cron + +One slice, two repos. Runs the plan's S2 (port skeleton) and S3 (datahub +consumes cron) together — cron shipped as `@prisma/app-cron` +([PR #45](https://github.com/prisma/app/pull/45)), so nothing separates them +anymore. S4 (live deploy + cutover) is explicitly out: it needs the team's real +secrets and workspace credentials. Linear: TML-3012. + +Branches: `claude/prisma-app-port` in **datahub** (the port); +`claude/datahub-port` in **prisma/app** (framework fixes + these artifacts, +based on #45's branch). + +## What datahub is (verified against source) + +- **`apps/ingest`** — Bun + Hono service. POST endpoints per sync source + (Stripe customers/charges/products/prices/subscriptions/invoices, PostHog + persons/events, aggregate refreshes) plus **`POST /tick`** = one budgeted + step of every job (`tick.ts`; catch-up = more ticks; `DEFAULT_TICK_MS` = + 180_000). Today an in-process `setInterval` fires ticks when + `TICK_INTERVAL_MS > 0`. Config surface (`env.ts`, zod): `DATABASE_URL`, + `STRIPE_API_KEY`*, `POSTHOG_HOST`, `POSTHOG_PROJECT_ID`, `POSTHOG_API_KEY`*, + `CLICKHOUSE_URL`, `CLICKHOUSE_USER`, `CLICKHOUSE_PASSWORD`*, + `CLICKHOUSE_DATABASE`, `INGEST_POSTHOG_EVENTS` (string → string[] transform), + `PORT`, `TICK_INTERVAL_MS`. (* = secret.) +- **`apps/web`** — Next.js 16 dashboard, reads postgres through + `@workspace/db` in `lib/queries.ts`. +- **`packages/db`** — prisma-next client (`@prisma-next/postgres` + contract + files) over a hand-built `pg` Pool (custom idle-error listener for Compute + sleep/resume), **module-global**, reads `process.env.DATABASE_URL` at import. +- Current deploy: `prisma.compute.ts` (`@prisma/compute-sdk`), env from + per-app `.env` files. + +## Decisions of record + +1. **One schedule job.** datahub's tick model is already "one endpoint, one + budgeted step of everything", so the cron schedule is + `defineSchedule({ tick: '180s' })` and the trigger handler calls `tick()`. + Do NOT explode `INGEST_JOBS` into per-source cron jobs — budgets and + ordering live inside `tick.ts` on purpose. +2. **Ingest IS the cron router.** Ingest exposes + `{ trigger: triggerContract }` and mounts `serveSchedule`'s fetch handler + on its Hono app (route `POST /rpc/trigger` to it). No separate router + service — the scheduler is the only added instance. Composition: + + ```ts + // system.ts (datahub root) + export default system('datahub', {}, ({ provision }) => { + const db = provision('database', postgres({ name: 'database' })); + provision('cron', cron('cron', { schedule, router: ingestService }), { db }); + provision('web', webService, { db }); + return {}; + }); + ``` + + Ingest lives at address `cron.router`; its deps (`db`) forward through the + cron system's boundary (proven by app-cron's Load test). +3. **Secrets are `secret` params, values sourced from the deployer's env.** + `system.ts`/`service.ts` load on the deployer's machine, so a helper + (`fromEnv('STRIPE_API_KEY')`) supplies each secret/scalar param's `default` + from `process.env` at deploy-load. This replaces the `.env`-file mechanism. + The framework redacts secret defaults in `configOf` + (fixed on `claude/datahub-port`). A first-class deploy-values mechanism + stays deferred (this is the plan's S1 resolution — the config-params + project absorbed the rest of S1's need). +4. **Params keep datahub's zod schemas.** zod ≥3.24 implements Standard + Schema, and params take any Standard Schema — port `env.ts`'s field schemas + onto the params (including the `INGEST_POSTHOG_EVENTS` transform). This + deliberately exercises a second schema vendor through the config pipeline. +5. **`DATABASE_URL` is bridged from `load()`, not passthrough.** The deployed + instance only receives stashed config keys — nothing sets `DATABASE_URL`. + The module-global `@workspace/db` client stays (minimal diff); each + service's entry does `process.env.DATABASE_URL = service.load().db.url` + **before** any module that imports `@workspace/db` loads (ingest: entry + sets it, then dynamic-imports the app; web: `instrumentation.ts` or + equivalent — implementer grounds which runs first in Next 16 standalone). + Converting `@workspace/db` to #44's `pnPostgres` contract dep is the deep + port — record it as follow-up evidence, do not do it in this slice. +6. **Framework packages via pkg.pr.new previews of PR #45** + (`https://pkg.pr.new/prisma/app/@prisma/app-cron@45` etc. — the + owner/repo-scoped form; verified resolvable). The branch is a coherent + snapshot (S1 + #44 + cron). If the port needs a framework fix that lands on + `claude/datahub-port`, open a PR for that branch and switch to its preview + URLs. + +## Scope (datahub branch) + +- Root: `system.ts`, `prisma-app.config.ts` (prismaCloud + nodeBuild + + nextjsBuild + state, mirroring storefront-auth's), root `package.json` + deploy/destroy scripts, framework deps. +- `apps/ingest`: `service.ts` (compute: deps `{ db: postgres() }`, params per + env.ts with zod schemas + secret facets, expose `{ trigger }`, node build + adapter); entry refactor (config from `service.config()`, DATABASE_URL + bridge, serveSchedule mounted, in-process `setInterval` scheduler and + `TICK_INTERVAL_MS` **deleted** — the cron system is the clock now); delete + `env.ts` in favor of params (keep zod field schemas). +- `apps/web`: `service.ts` (compute: deps `{ db }`, nextjs build adapter); + DATABASE_URL bridge. +- Remove `prisma.compute.ts` + `@prisma/compute-sdk` usage for deploy + (runtime `@prisma/compute` KeepAwakeGuard **stays** — it is a platform + runtime API, orthogonal to the deploy path). + +## Definition of done + +- [ ] `Load(system)` test green in datahub: graph contains `database`, + `cron.router`, `cron.scheduler`, `web`; edges `database→cron.router`, + `database→web`, `cron.router→cron.scheduler`. +- [ ] Ingest's entry boots via `bootstrapService` with a supplied Config + (fake values): `/health` responds; `POST /rpc/trigger {jobId:'tick'}` + reaches the tick handler (tick's own DB work may be stubbed or may fail + cleanly past the dispatch point — the assertion is dispatch, not sync). +- [ ] Every param from `env.ts` is declared on the service with the right + secret/optional/default facets; no `.env`-file reads remain in the + deploy path; secret values come only from the deployer's env. +- [ ] `bun install` resolves the preview packages; datahub `typecheck` and + `build` green; no `@prisma/compute-sdk` deploy config remains. +- [ ] The in-process tick scheduler is gone; the only clock is + `cron.scheduler`. +- [ ] Follow-up evidence recorded in the project plan (pnPostgres conversion, + module-global-client pattern, deploy-values mechanism). + +## Non-goals + +- **Live deploy / cutover (S4)** — needs team secrets + workspace creds. +- **pnPostgres conversion of `@workspace/db`** — recorded follow-up. +- **ClickHouse/PostHog/Stripe as framework resources** — they are external + SaaS; params suffice. +- **open-chat / M2.** diff --git a/packages/app/src/__tests__/config.test.ts b/packages/app/src/__tests__/config.test.ts index 19e55a44..32fb8227 100644 --- a/packages/app/src/__tests__/config.test.ts +++ b/packages/app/src/__tests__/config.test.ts @@ -37,6 +37,25 @@ describe('configOf', () => { ]); }); + test("a secret param's default is redacted — values absent from introspection", () => { + const root = service({ + name: 'test-service', + extension: 'test/pack', + type: 'fake/app', + inputs: {}, + params: { + apiKey: string({ secret: true, default: 'sk-live-123' }), + port: number({ default: 3000 }), + }, + build, + }); + + expect(configOf(root)).toEqual([ + scalarDeclaration('service', 'apiKey', { secret: true }), + scalarDeclaration('service', 'port', { default: 3000 }), + ]); + }); + test('owner discriminates service vs input params — same name cannot collide', () => { const root = service({ name: 'test-service', diff --git a/packages/app/src/config.ts b/packages/app/src/config.ts index 8625b9d0..5ce13b1f 100644 --- a/packages/app/src/config.ts +++ b/packages/app/src/config.ts @@ -113,7 +113,9 @@ export function configOf(root: ServiceNode): readonly ConfigDeclaration[] { schema: projectSchema(param.schema), secret: param.secret === true, optional: param.optional === true, - default: param.default, + // A secret param's default is a value (possibly sourced from the + // deployer's environment) — redacted here like any other secret value. + default: param.secret === true ? undefined : param.default, }); } } @@ -125,7 +127,7 @@ export function configOf(root: ServiceNode): readonly ConfigDeclaration[] { schema: projectSchema(param.schema), secret: param.secret === true, optional: param.optional === true, - default: param.default, + default: param.secret === true ? undefined : param.default, }); }