From e4c9b5c99a102cb4c9e8eb869fbc6e01893c3bcd Mon Sep 17 00:00:00 2001 From: Dan Sutton Date: Sat, 18 Jul 2026 17:25:29 +0100 Subject: [PATCH 1/6] fix(webapp,run-engine,run-store,redis-worker): read-your-writes + global-scope idempotency correctness under the run-ops split MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Production code only — the guarding tests are in the stacked PR. 1) Read-your-writes: run-store reads that gate a mutation or feed a public GET / realtime response were routed to a lagging read replica, so a just-written run/waitpoint/batch could spuriously miss under replica lag. Route those reads to the owning primary (findRun/findWaitpoint/findBatchTaskRunByFriendlyId -> *OnPrimary, a primary re-read on a miss, or a retryable 404 where the SDK polls). Additive: the happy path is unchanged; a primary read happens only on a miss. 2) Global-scope idempotency across the split: a global-scope key carries no per-run salt, so the same (env, task, key) triggered concurrently from parents resident on different run-ops DBs could dedup-miss on each DB and create a duplicate run (the per-DB unique index can't enforce cross-DB uniqueness). Serialize such triggers (global scope, or scope-absent, while split is active) through the existing Redis idempotency claim, resolve the winner by id across both DBs, and reacquire the claim on the expired/failed clear-and-recreate path. run/attempt scope embed the run id in their hash and never contend. --- .../app/models/runtimeEnvironment.server.ts | 24 +- .../v3/ApiWaitpointPresenter.server.ts | 53 ++- .../presenters/v3/BatchPresenter.server.ts | 14 +- .../app/routes/api.v1.batches.$batchId.ts | 5 + ....runs.$runFriendlyId.input-streams.wait.ts | 30 +- ...uns.$runFriendlyId.session-streams.wait.ts | 28 +- .../app/routes/api.v1.runs.$runId.metadata.ts | 13 + ...i.v1.sessions.$session.end-and-continue.ts | 15 +- ...ens.$waitpointFriendlyId.callback.$hash.ts | 11 +- .../app/routes/api.v2.batches.$batchId.ts | 5 + .../routes/realtime.v1.batches.$batchId.ts | 5 + .../app/routes/realtime.v1.runs.$runId.ts | 28 +- .../realtime.v1.streams.$runId.$streamId.ts | 40 +- ...streams.$runId.$target.$streamId.append.ts | 40 +- ...ime.v1.streams.$runId.$target.$streamId.ts | 91 ++-- ...ltime.v1.streams.$runId.input.$streamId.ts | 59 +-- ...am.runs.$runParam.idempotencyKey.reset.tsx | 27 +- ...ram.realtime.v1.sessions.$sessionId.$io.ts | 41 +- ...am.realtime.v1.streams.$runId.$streamId.ts | 30 +- ...ltime.v1.streams.$runId.input.$streamId.ts | 30 +- .../route.tsx | 28 +- .../route.tsx | 10 +- .../resources.taskruns.$runParam.cancel.ts | 35 +- .../resources.taskruns.$runParam.replay.ts | 7 +- .../app/routes/sync.traces.runs.$traceId.ts | 11 +- .../concerns/idempotencyKeys.server.ts | 423 +++++++++++++----- .../app/services/dashboardAgent.server.ts | 4 +- .../realtime/sessionRunManager.server.ts | 6 +- .../v3/mollifier/idempotencyClaim.server.ts | 33 ++ .../app/v3/services/batchTriggerV3.server.ts | 23 +- .../src/engine/systems/runAttemptSystem.ts | 21 +- .../src/engine/systems/ttlSystem.ts | 34 +- .../run-store/src/runOpsStore.ts | 7 +- packages/redis-worker/src/mollifier/buffer.ts | 17 + 34 files changed, 818 insertions(+), 430 deletions(-) diff --git a/apps/webapp/app/models/runtimeEnvironment.server.ts b/apps/webapp/app/models/runtimeEnvironment.server.ts index 987394ea40c..939cd55e94d 100644 --- a/apps/webapp/app/models/runtimeEnvironment.server.ts +++ b/apps/webapp/app/models/runtimeEnvironment.server.ts @@ -274,19 +274,17 @@ export async function findEnvironmentFromRun( ): Promise { // Run-ops scalars (runTags/batchId/runtimeEnvironmentId) from the run store; the env half is // resolved via the control-plane resolver so the run-ops DB can split without a cross-DB join. - const taskRun = await runStore.findRun( - { - id: runId, - }, - { - select: { - runTags: true, - batchId: true, - runtimeEnvironmentId: true, - }, - }, - tx ?? $replica - ); + const select = { + runTags: true, + batchId: true, + runtimeEnvironmentId: true, + } as const; + let taskRun = await runStore.findRun({ id: runId }, { select }, tx ?? $replica); + if (!taskRun) { + // Read-your-writes: a just-created run may not have replicated. Re-read the owning primary before + // treating it as absent, so runMetadataUpdated doesn't drop a live run's final metadata + publish. + taskRun = await runStore.findRun({ id: runId }, { select }, prisma); + } if (!taskRun) { return null; } diff --git a/apps/webapp/app/presenters/v3/ApiWaitpointPresenter.server.ts b/apps/webapp/app/presenters/v3/ApiWaitpointPresenter.server.ts index c4477efefac..eb1d0dd0cdf 100644 --- a/apps/webapp/app/presenters/v3/ApiWaitpointPresenter.server.ts +++ b/apps/webapp/app/presenters/v3/ApiWaitpointPresenter.server.ts @@ -42,29 +42,36 @@ export class ApiWaitpointPresenter extends BasePresenter { return this.trace("call", async (span) => { // The store routes by the waitpointId's residency (id shape) and reads the owning // store's replica. waitpointId is pre-decoded from the friendlyId via WaitpointId.toId. - const waitpoint = await this.runStore.findWaitpoint({ - where: { - id: waitpointId, - environmentId: environment.id, - }, - select: { - id: true, - friendlyId: true, - type: true, - status: true, - idempotencyKey: true, - userProvidedIdempotencyKey: true, - idempotencyKeyExpiresAt: true, - inactiveIdempotencyKey: true, - output: true, - outputType: true, - outputIsError: true, - completedAfter: true, - completedAt: true, - createdAt: true, - tags: true, - }, - }); + const where = { + id: waitpointId, + environmentId: environment.id, + }; + const select = { + id: true, + friendlyId: true, + type: true, + status: true, + idempotencyKey: true, + userProvidedIdempotencyKey: true, + idempotencyKeyExpiresAt: true, + inactiveIdempotencyKey: true, + output: true, + outputType: true, + outputIsError: true, + completedAfter: true, + completedAt: true, + createdAt: true, + tags: true, + } as const; + + let waitpoint = await this.runStore.findWaitpoint({ where, select }); + + // Read-your-writes on a public GET: a just-minted token may not be on the owning store's + // replica yet, so a replica miss would 404 a live token. Re-read the owning primary before + // concluding it doesn't exist (mirrors the metadata GET loader + the complete/callback paths). + if (!waitpoint) { + waitpoint = await this.runStore.findWaitpointOnPrimary({ where, select }); + } if (!waitpoint) { logger.error(`WaitpointPresenter: Waitpoint not found`, { diff --git a/apps/webapp/app/presenters/v3/BatchPresenter.server.ts b/apps/webapp/app/presenters/v3/BatchPresenter.server.ts index b9190a6f686..3e9ef0be858 100644 --- a/apps/webapp/app/presenters/v3/BatchPresenter.server.ts +++ b/apps/webapp/app/presenters/v3/BatchPresenter.server.ts @@ -47,13 +47,25 @@ export class BatchPresenter extends BasePresenter { // The BatchTaskRun (run-ops) is read through the run store, which routes by residency. The // runtimeEnvironment (control-plane) is resolved separately because the cross-seam FK is // dropped, so the batch row cannot single-SQL join to control-plane RuntimeEnvironment. - const batch = await this.runStore.findBatchTaskRunByFriendlyId( + let batch = await this.runStore.findBatchTaskRunByFriendlyId( batchId, environmentId, { include: BATCH_INCLUDE }, this._replica ); + // Read-your-writes: findBatchTaskRunByFriendlyId defaults to (and here reads) the replica, so a + // batch created within the replica's apply window returns null under lag. Re-read from the owning + // primary on a miss so a live batch's detail page never spuriously 404s ("Batch not found"). + if (!batch) { + batch = await this.runStore.findBatchTaskRunByFriendlyId( + batchId, + environmentId, + { include: BATCH_INCLUDE }, + this._prisma + ); + } + if (!batch) { throw new Error("Batch not found"); } diff --git a/apps/webapp/app/routes/api.v1.batches.$batchId.ts b/apps/webapp/app/routes/api.v1.batches.$batchId.ts index 01acd3d140f..713716f20ee 100644 --- a/apps/webapp/app/routes/api.v1.batches.$batchId.ts +++ b/apps/webapp/app/routes/api.v1.batches.$batchId.ts @@ -12,6 +12,11 @@ export const loader = createLoaderApiRoute( params: ParamsSchema, allowJWT: true, corsStrategy: "all", + // A just-created batch may not yet have replicated to the read replica this client-less + // findBatchTaskRunByFriendlyId lookup routes to; return a retryable 404 so the SDK retries through + // replica lag rather than stranding a live batch on a permanent 404 (mirrors the run-get routes, + // e.g. api.v3.runs.$runId). + shouldRetryNotFound: true, findResource: (params, auth) => { return runStore.findBatchTaskRunByFriendlyId(params.batchId, auth.environment.id, { include: { errors: true }, diff --git a/apps/webapp/app/routes/api.v1.runs.$runFriendlyId.input-streams.wait.ts b/apps/webapp/app/routes/api.v1.runs.$runFriendlyId.input-streams.wait.ts index 3c081b2280b..024779ac666 100644 --- a/apps/webapp/app/routes/api.v1.runs.$runFriendlyId.input-streams.wait.ts +++ b/apps/webapp/app/routes/api.v1.runs.$runFriendlyId.input-streams.wait.ts @@ -33,21 +33,23 @@ const { action, loader } = createActionApiRoute( }, async ({ authentication, body, params }) => { try { - const run = await runStore.findRun( - { - friendlyId: params.runFriendlyId, - runtimeEnvironmentId: authentication.environment.id, + const where = { + friendlyId: params.runFriendlyId, + runtimeEnvironmentId: authentication.environment.id, + }; + const args = { + select: { + id: true, + friendlyId: true, + realtimeStreamsVersion: true, + streamBasinName: true, }, - { - select: { - id: true, - friendlyId: true, - realtimeStreamsVersion: true, - streamBasinName: true, - }, - }, - $replica - ); + }; + // Replica lag can null out a live run; a spurious 404 fails the .wait() registration on a run + // that exists. Re-read the owning primary on a replica miss. + const run = + (await runStore.findRun(where, args, $replica)) ?? + (await runStore.findRunOnPrimary(where, args)); if (!run) { return json({ error: "Run not found" }, { status: 404 }); diff --git a/apps/webapp/app/routes/api.v1.runs.$runFriendlyId.session-streams.wait.ts b/apps/webapp/app/routes/api.v1.runs.$runFriendlyId.session-streams.wait.ts index f6c58ab748b..c00ff51b3be 100644 --- a/apps/webapp/app/routes/api.v1.runs.$runFriendlyId.session-streams.wait.ts +++ b/apps/webapp/app/routes/api.v1.runs.$runFriendlyId.session-streams.wait.ts @@ -39,20 +39,22 @@ const { action, loader } = createActionApiRoute( }, async ({ authentication, body, params }) => { try { - const run = await runStore.findRun( - { - friendlyId: params.runFriendlyId, - runtimeEnvironmentId: authentication.environment.id, + const where = { + friendlyId: params.runFriendlyId, + runtimeEnvironmentId: authentication.environment.id, + }; + const args = { + select: { + id: true, + friendlyId: true, + realtimeStreamsVersion: true, }, - { - select: { - id: true, - friendlyId: true, - realtimeStreamsVersion: true, - }, - }, - $replica - ); + }; + // Replica lag can null out a live run; a spurious 404 fails the .wait() registration on a run + // that exists. Re-read the owning primary on a replica miss. + const run = + (await runStore.findRun(where, args, $replica)) ?? + (await runStore.findRunOnPrimary(where, args)); if (!run) { return json({ error: "Run not found" }, { status: 404 }); diff --git a/apps/webapp/app/routes/api.v1.runs.$runId.metadata.ts b/apps/webapp/app/routes/api.v1.runs.$runId.metadata.ts index 58cf572f44d..ac26302f8a3 100644 --- a/apps/webapp/app/routes/api.v1.runs.$runId.metadata.ts +++ b/apps/webapp/app/routes/api.v1.runs.$runId.metadata.ts @@ -64,6 +64,19 @@ export async function loader({ request, params }: LoaderFunctionArgs) { ); } + // Read-your-writes: a run drained from the buffer to the primary but not yet replicated misses + // both the replica read and the buffer. Re-read the owning primary before 404ing. + const primaryRun = await runStore.findRunOnPrimary( + { friendlyId: parsed.data.runId, runtimeEnvironmentId: env.id }, + { select: { metadata: true, metadataType: true } } + ); + if (primaryRun) { + return json( + { metadata: primaryRun.metadata, metadataType: primaryRun.metadataType }, + { status: 200 } + ); + } + return json({ error: "Run not found" }, { status: 404 }); } diff --git a/apps/webapp/app/routes/api.v1.sessions.$session.end-and-continue.ts b/apps/webapp/app/routes/api.v1.sessions.$session.end-and-continue.ts index 35770928c2d..c6e5a90f667 100644 --- a/apps/webapp/app/routes/api.v1.sessions.$session.end-and-continue.ts +++ b/apps/webapp/app/routes/api.v1.sessions.$session.end-and-continue.ts @@ -75,7 +75,7 @@ const { action, loader } = createActionApiRoute( // SDK exposes via `ctx.run.id`). Internally `Session.currentRunId` // stores the TaskRun.id cuid, so resolve before handing to the // optimistic-claim service. - const callingRun = await runStore.findRun( + let callingRun = await runStore.findRun( { friendlyId: body.callingRunId, runtimeEnvironmentId: authentication.environment.id, @@ -83,6 +83,19 @@ const { action, loader } = createActionApiRoute( { select: { id: true } }, $replica ); + if (!callingRun) { + // Replica lag: `callingRunId` is the agent's own live run (it is executing this request), so it + // exists on the owning primary even when the read replica has not caught up. Re-read the primary + // before 404ing — otherwise a lagging replica turns a legitimate handoff into a spurious + // "callingRunId not found in this environment". + callingRun = await runStore.findRunOnPrimary( + { + friendlyId: body.callingRunId, + runtimeEnvironmentId: authentication.environment.id, + }, + { select: { id: true } } + ); + } if (!callingRun) { return json({ error: "callingRunId not found in this environment" }, { status: 404 }); } diff --git a/apps/webapp/app/routes/api.v1.waitpoints.tokens.$waitpointFriendlyId.callback.$hash.ts b/apps/webapp/app/routes/api.v1.waitpoints.tokens.$waitpointFriendlyId.callback.$hash.ts index 502db2aca79..431a7eb2582 100644 --- a/apps/webapp/app/routes/api.v1.waitpoints.tokens.$waitpointFriendlyId.callback.$hash.ts +++ b/apps/webapp/app/routes/api.v1.waitpoints.tokens.$waitpointFriendlyId.callback.$hash.ts @@ -36,13 +36,22 @@ export async function action({ request, params }: ActionFunctionArgs) { // Resolve wherever the waitpoint resides. The store routes by the waitpoint id's residency // (id-shape) and probes both run-ops DBs, so a token on either store resolves; the env is // resolved below from the row via the control-plane resolver. - const waitpoint = await runStore.findWaitpoint({ + let waitpoint = await runStore.findWaitpoint({ where: { id: waitpointId, }, select: { id: true, status: true, environmentId: true }, }); + if (!waitpoint) { + // Read-your-writes: a token whose callback fires right after mint may not have replicated + // yet. Re-read the owning primary before 404ing (mirrors complete.ts's primary fallback). + waitpoint = await runStore.findWaitpointOnPrimary({ + where: { id: waitpointId }, + select: { id: true, status: true, environmentId: true }, + }); + } + if (!waitpoint) { return json({ error: "Waitpoint not found" }, { status: 404 }); } diff --git a/apps/webapp/app/routes/api.v2.batches.$batchId.ts b/apps/webapp/app/routes/api.v2.batches.$batchId.ts index b6ef4136bd4..4f23f14bca6 100644 --- a/apps/webapp/app/routes/api.v2.batches.$batchId.ts +++ b/apps/webapp/app/routes/api.v2.batches.$batchId.ts @@ -12,6 +12,11 @@ export const loader = createLoaderApiRoute( params: ParamsSchema, allowJWT: true, corsStrategy: "all", + // A just-created batch may not yet have replicated to the read replica this client-less + // findBatchTaskRunByFriendlyId lookup routes to; return a retryable 404 so the SDK retries through + // replica lag rather than stranding a live batch on a permanent 404 (mirrors the run-get routes, + // e.g. api.v3.runs.$runId). + shouldRetryNotFound: true, findResource: (params, auth) => { return runStore.findBatchTaskRunByFriendlyId(params.batchId, auth.environment.id, { include: { errors: true }, diff --git a/apps/webapp/app/routes/realtime.v1.batches.$batchId.ts b/apps/webapp/app/routes/realtime.v1.batches.$batchId.ts index 29ad61d6646..b95cd4491db 100644 --- a/apps/webapp/app/routes/realtime.v1.batches.$batchId.ts +++ b/apps/webapp/app/routes/realtime.v1.batches.$batchId.ts @@ -13,6 +13,11 @@ export const loader = createLoaderApiRoute( params: ParamsSchema, allowJWT: true, corsStrategy: "all", + // A just-created batch may not yet have replicated to the read replica this client-less + // findBatchTaskRunByFriendlyId lookup routes to; return a retryable 404 so the SDK retries through + // replica lag rather than stranding a live batch on a permanent 404 (mirrors the run-get routes, + // e.g. api.v3.runs.$runId). + shouldRetryNotFound: true, findResource: (params, auth) => { return runStore.findBatchTaskRunByFriendlyId(params.batchId, auth.environment.id); }, diff --git a/apps/webapp/app/routes/realtime.v1.runs.$runId.ts b/apps/webapp/app/routes/realtime.v1.runs.$runId.ts index 51b47b22afd..c65bf0ecd43 100644 --- a/apps/webapp/app/routes/realtime.v1.runs.$runId.ts +++ b/apps/webapp/app/routes/realtime.v1.runs.$runId.ts @@ -15,22 +15,24 @@ export const loader = createLoaderApiRoute( allowJWT: true, corsStrategy: "all", findResource: async (params, authentication) => { - return runStore.findRun( - { - friendlyId: params.runId, - runtimeEnvironmentId: authentication.environment.id, - }, - { - include: { - batch: { - select: { - friendlyId: true, - }, + const where = { + friendlyId: params.runId, + runtimeEnvironmentId: authentication.environment.id, + }; + const args = { + include: { + batch: { + select: { + friendlyId: true, }, }, }, - $replica - ); + }; + // Replica lag can null out a run that already exists on the owning primary. A spurious 404 + // here permanently fails the client's realtime subscription (the SSE client treats 404 as + // "stream gone" — nonRetryableStatuses). Re-read the primary on a replica miss. + const run = await runStore.findRun(where, args, $replica); + return run ?? runStore.findRunOnPrimary(where, args); }, authorization: { action: "read", diff --git a/apps/webapp/app/routes/realtime.v1.streams.$runId.$streamId.ts b/apps/webapp/app/routes/realtime.v1.streams.$runId.$streamId.ts index 895e336fd8b..ab941fd05ad 100644 --- a/apps/webapp/app/routes/realtime.v1.streams.$runId.$streamId.ts +++ b/apps/webapp/app/routes/realtime.v1.streams.$runId.$streamId.ts @@ -16,29 +16,29 @@ export const loader = createLoaderApiRoute( allowJWT: true, corsStrategy: "all", findResource: async (params, auth) => { - const run = await runStore.findRun( - { - friendlyId: params.runId, - runtimeEnvironmentId: auth.environment.id, - }, - { - select: { - id: true, - friendlyId: true, - taskIdentifier: true, - runTags: true, - realtimeStreamsVersion: true, - streamBasinName: true, - batch: { - select: { - friendlyId: true, - }, + const where = { + friendlyId: params.runId, + runtimeEnvironmentId: auth.environment.id, + }; + const args = { + select: { + id: true, + friendlyId: true, + taskIdentifier: true, + runTags: true, + realtimeStreamsVersion: true, + streamBasinName: true, + batch: { + select: { + friendlyId: true, }, }, }, - $replica - ); - return run; + }; + // Replica lag can null out a live run; a spurious 404 permanently fails the SSE subscription + // (the client treats 404 as "stream gone"). Re-read the owning primary on a replica miss. + const run = await runStore.findRun(where, args, $replica); + return run ?? runStore.findRunOnPrimary(where, args); }, authorization: { action: "read", diff --git a/apps/webapp/app/routes/realtime.v1.streams.$runId.$target.$streamId.append.ts b/apps/webapp/app/routes/realtime.v1.streams.$runId.$target.$streamId.append.ts index 12400e8d0b0..7195a4ae9a6 100644 --- a/apps/webapp/app/routes/realtime.v1.streams.$runId.$target.$streamId.append.ts +++ b/apps/webapp/app/routes/realtime.v1.streams.$runId.$target.$streamId.append.ts @@ -27,29 +27,31 @@ const { action } = createActionApiRoute( maxContentLength: MAX_APPEND_BODY_BYTES, }, async ({ request, params, authentication }) => { - const run = await runStore.findRun( - { - friendlyId: params.runId, - runtimeEnvironmentId: authentication.environment.id, - }, - { - select: { - id: true, - friendlyId: true, - parentTaskRun: { - select: { - friendlyId: true, - }, + const where = { + friendlyId: params.runId, + runtimeEnvironmentId: authentication.environment.id, + }; + const args = { + select: { + id: true, + friendlyId: true, + parentTaskRun: { + select: { + friendlyId: true, }, - rootTaskRun: { - select: { - friendlyId: true, - }, + }, + rootTaskRun: { + select: { + friendlyId: true, }, }, }, - $replica - ); + }; + // Replica lag can null out a live run; a spurious 404 permanently fails the append. Re-read the + // owning primary on a replica miss. + const run = + (await runStore.findRun(where, args, $replica)) ?? + (await runStore.findRunOnPrimary(where, args)); if (!run) { return new Response("Run not found", { status: 404 }); diff --git a/apps/webapp/app/routes/realtime.v1.streams.$runId.$target.$streamId.ts b/apps/webapp/app/routes/realtime.v1.streams.$runId.$target.$streamId.ts index 01a2b550d45..31c55645743 100644 --- a/apps/webapp/app/routes/realtime.v1.streams.$runId.$target.$streamId.ts +++ b/apps/webapp/app/routes/realtime.v1.streams.$runId.$target.$streamId.ts @@ -19,32 +19,34 @@ const { action } = createActionApiRoute( params: ParamsSchema, }, async ({ request, params, authentication }) => { - const run = await runStore.findRun( - { - friendlyId: params.runId, - runtimeEnvironmentId: authentication.environment.id, - }, - { - select: { - id: true, - friendlyId: true, - streamBasinName: true, - parentTaskRun: { - select: { - friendlyId: true, - streamBasinName: true, - }, + const where = { + friendlyId: params.runId, + runtimeEnvironmentId: authentication.environment.id, + }; + const args = { + select: { + id: true, + friendlyId: true, + streamBasinName: true, + parentTaskRun: { + select: { + friendlyId: true, + streamBasinName: true, }, - rootTaskRun: { - select: { - friendlyId: true, - streamBasinName: true, - }, + }, + rootTaskRun: { + select: { + friendlyId: true, + streamBasinName: true, }, }, }, - $replica - ); + }; + // Replica lag can null out a live run; a spurious 404 permanently fails the ingest client. + // Re-read the owning primary on a replica miss. + const run = + (await runStore.findRun(where, args, $replica)) ?? + (await runStore.findRunOnPrimary(where, args)); if (!run) { return new Response("Run not found", { status: 404 }); @@ -154,32 +156,33 @@ const loader = createLoaderApiRoute( allowJWT: false, corsStrategy: "none", findResource: async (params, authentication) => { - return runStore.findRun( - { - friendlyId: params.runId, - runtimeEnvironmentId: authentication.environment.id, - }, - { - select: { - id: true, - friendlyId: true, - streamBasinName: true, - parentTaskRun: { - select: { - friendlyId: true, - streamBasinName: true, - }, + const where = { + friendlyId: params.runId, + runtimeEnvironmentId: authentication.environment.id, + }; + const args = { + select: { + id: true, + friendlyId: true, + streamBasinName: true, + parentTaskRun: { + select: { + friendlyId: true, + streamBasinName: true, }, - rootTaskRun: { - select: { - friendlyId: true, - streamBasinName: true, - }, + }, + rootTaskRun: { + select: { + friendlyId: true, + streamBasinName: true, }, }, }, - $replica - ); + }; + // Replica lag can null out a live run; a spurious 404 permanently fails the HEAD probe. + // Re-read the owning primary on a replica miss. + const run = await runStore.findRun(where, args, $replica); + return run ?? runStore.findRunOnPrimary(where, args); }, }, async ({ request, params, resource: run, authentication }) => { diff --git a/apps/webapp/app/routes/realtime.v1.streams.$runId.input.$streamId.ts b/apps/webapp/app/routes/realtime.v1.streams.$runId.input.$streamId.ts index 78fe332b8af..3c67fdb94d4 100644 --- a/apps/webapp/app/routes/realtime.v1.streams.$runId.input.$streamId.ts +++ b/apps/webapp/app/routes/realtime.v1.streams.$runId.input.$streamId.ts @@ -39,22 +39,24 @@ const { action } = createActionApiRoute( }, }, async ({ request, params, authentication }) => { - const run = await runStore.findRun( - { - friendlyId: params.runId, - runtimeEnvironmentId: authentication.environment.id, - }, - { - select: { - id: true, - friendlyId: true, - completedAt: true, - realtimeStreamsVersion: true, - streamBasinName: true, - }, + const where = { + friendlyId: params.runId, + runtimeEnvironmentId: authentication.environment.id, + }; + const args = { + select: { + id: true, + friendlyId: true, + completedAt: true, + realtimeStreamsVersion: true, + streamBasinName: true, }, - $replica - ); + }; + // Replica lag can null out a live run; a spurious 404 permanently fails the input send. Re-read + // the owning primary on a replica miss. + const run = + (await runStore.findRun(where, args, $replica)) ?? + (await runStore.findRunOnPrimary(where, args)); if (!run) { return json({ ok: false, error: "Run not found" }, { status: 404 }); @@ -133,22 +135,23 @@ const loader = createLoaderApiRoute( allowJWT: true, corsStrategy: "all", findResource: async (params, auth) => { - return runStore.findRun( - { - friendlyId: params.runId, - runtimeEnvironmentId: auth.environment.id, - }, - { - include: { - batch: { - select: { - friendlyId: true, - }, + const where = { + friendlyId: params.runId, + runtimeEnvironmentId: auth.environment.id, + }; + const args = { + include: { + batch: { + select: { + friendlyId: true, }, }, }, - $replica - ); + }; + // Replica lag can null out a live run; a spurious 404 permanently fails the SSE tail (the + // client treats 404 as "stream gone"). Re-read the owning primary on a replica miss. + const run = await runStore.findRun(where, args, $replica); + return run ?? runStore.findRunOnPrimary(where, args); }, authorization: { action: "read", diff --git a/apps/webapp/app/routes/resources.orgs.$organizationSlug.projects.$projectParam.env.$envParam.runs.$runParam.idempotencyKey.reset.tsx b/apps/webapp/app/routes/resources.orgs.$organizationSlug.projects.$projectParam.env.$envParam.runs.$runParam.idempotencyKey.reset.tsx index 9567ce084e7..dea9ed18e60 100644 --- a/apps/webapp/app/routes/resources.orgs.$organizationSlug.projects.$projectParam.env.$envParam.runs.$runParam.idempotencyKey.reset.tsx +++ b/apps/webapp/app/routes/resources.orgs.$organizationSlug.projects.$projectParam.env.$envParam.runs.$runParam.idempotencyKey.reset.tsx @@ -12,20 +12,19 @@ export const action: ActionFunction = async ({ request, params }) => { const { projectParam, organizationSlug, envParam, runParam } = v3RunParamsSchema.parse(params); try { - const taskRun = await runStore.findRun( - { - friendlyId: runParam, - }, - { - select: { - id: true, - idempotencyKey: true, - taskIdentifier: true, - projectId: true, - runtimeEnvironmentId: true, - }, - } - ); + const resetSelect = { + id: true, + idempotencyKey: true, + taskIdentifier: true, + projectId: true, + runtimeEnvironmentId: true, + }; + let taskRun = await runStore.findRun({ friendlyId: runParam }, { select: resetSelect }); + if (!taskRun) { + // Read-your-writes: a just-created run may not have replicated. Re-read the owning primary + // before 404ing — this null gates the reset mutation below (mirrors cancel/replay). + taskRun = await runStore.findRunOnPrimary({ friendlyId: runParam }, { select: resetSelect }); + } if (!taskRun) { return jsonWithErrorMessage({}, request, "Run not found"); diff --git a/apps/webapp/app/routes/resources.orgs.$organizationSlug.projects.$projectParam.env.$envParam.runs.$runParam.realtime.v1.sessions.$sessionId.$io.ts b/apps/webapp/app/routes/resources.orgs.$organizationSlug.projects.$projectParam.env.$envParam.runs.$runParam.realtime.v1.sessions.$sessionId.$io.ts index c001d66d509..062b75503a6 100644 --- a/apps/webapp/app/routes/resources.orgs.$organizationSlug.projects.$projectParam.env.$envParam.runs.$runParam.realtime.v1.sessions.$sessionId.$io.ts +++ b/apps/webapp/app/routes/resources.orgs.$organizationSlug.projects.$projectParam.env.$envParam.runs.$runParam.realtime.v1.sessions.$sessionId.$io.ts @@ -1,6 +1,6 @@ import { type LoaderFunctionArgs } from "@remix-run/server-runtime"; import { z } from "zod"; -import { $replica } from "~/db.server"; +import { $replica, prisma } from "~/db.server"; import { runStore } from "~/v3/runStore.server"; import { findProjectBySlug } from "~/models/project.server"; import { findEnvironmentBySlug } from "~/models/runtimeEnvironment.server"; @@ -8,7 +8,7 @@ import { getRequestAbortSignal } from "~/services/httpAsyncStorage.server"; import { S2RealtimeStreams } from "~/services/realtime/s2realtimeStreams.server"; import { canonicalSessionAddressingKey, - resolveSessionByIdOrExternalId, + resolveSessionWithWriterFallback, } from "~/services/realtime/sessions.server"; import { getRealtimeStreamInstance } from "~/services/realtime/v1StreamsGlobal.server"; import { requireUserId } from "~/services/session.server"; @@ -51,22 +51,27 @@ export async function loader({ request, params }: LoaderFunctionArgs) { // Verify the run lives in this environment — keeps callers from // subscribing to arbitrary sessions via `/runs/$runParam/...`. - const run = await runStore.findRun( - { - friendlyId: runParam, - runtimeEnvironmentId: environment.id, - }, - { - select: { id: true, friendlyId: true }, - }, - $replica - ); + const runWhere = { + friendlyId: runParam, + runtimeEnvironmentId: environment.id, + }; + const runArgs = { + select: { id: true, friendlyId: true }, + }; + // Replica lag can null out a live run; a spurious 404 breaks the dashboard Agent tab subscription + // (useRealtimeStream surfaces the error and does not auto-retry). Re-read the primary on a miss. + const run = + (await runStore.findRun(runWhere, runArgs, $replica)) ?? + (await runStore.findRunOnPrimary(runWhere, runArgs)); if (!run) { return new Response("Run not found", { status: 404 }); } - const session = await resolveSessionByIdOrExternalId($replica, environment.id, sessionId); + // Replica lag can null out a just-created session; a spurious 404 breaks the dashboard Agent tab + // subscription (useRealtimeStream surfaces the error and does not auto-retry). Resolve replica-first + // with a writer fallback — the same helper the sibling `.in/append` route uses. + const session = await resolveSessionWithWriterFallback(environment.id, sessionId); if (!session) { return new Response("Session not found", { status: 404 }); @@ -76,10 +81,12 @@ export async function loader({ request, params }: LoaderFunctionArgs) { // this environment is enough to subscribe to any session in the same // environment — defeats the point of scoping subscriptions through the // run route. SessionRun.runId is indexed (@unique), so this is cheap. - const linkedSessionRun = await $replica.sessionRun.findFirst({ - where: { runId: run.id, sessionId: session.id }, - select: { id: true }, - }); + // Replica lag can null out the just-created run↔session linkage row; a spurious 404 breaks the + // dashboard Agent tab subscription (client does not auto-retry). Re-read the primary on a miss. + const linkWhere = { runId: run.id, sessionId: session.id }; + const linkedSessionRun = + (await $replica.sessionRun.findFirst({ where: linkWhere, select: { id: true } })) ?? + (await prisma.sessionRun.findFirst({ where: linkWhere, select: { id: true } })); if (!linkedSessionRun) { return new Response("Session not found for run", { status: 404 }); diff --git a/apps/webapp/app/routes/resources.orgs.$organizationSlug.projects.$projectParam.env.$envParam.runs.$runParam.realtime.v1.streams.$runId.$streamId.ts b/apps/webapp/app/routes/resources.orgs.$organizationSlug.projects.$projectParam.env.$envParam.runs.$runParam.realtime.v1.streams.$runId.$streamId.ts index 6c50b0f1463..4fbb309e2a8 100644 --- a/apps/webapp/app/routes/resources.orgs.$organizationSlug.projects.$projectParam.env.$envParam.runs.$runParam.realtime.v1.streams.$runId.$streamId.ts +++ b/apps/webapp/app/routes/resources.orgs.$organizationSlug.projects.$projectParam.env.$envParam.runs.$runParam.realtime.v1.streams.$runId.$streamId.ts @@ -45,21 +45,23 @@ export async function loader({ request, params }: LoaderFunctionArgs) { return new Response("Environment not found", { status: 404 }); } - const run = await runStore.findRun( - { - friendlyId: runId, - runtimeEnvironmentId: environment.id, + const runWhere = { + friendlyId: runId, + runtimeEnvironmentId: environment.id, + }; + const runArgs = { + select: { + id: true, + friendlyId: true, + realtimeStreamsVersion: true, + streamBasinName: true, }, - { - select: { - id: true, - friendlyId: true, - realtimeStreamsVersion: true, - streamBasinName: true, - }, - }, - $replica - ); + }; + // Replica lag can null out a live run; a spurious 404 breaks the dashboard Agent tab subscription + // (useRealtimeStream surfaces the error and does not auto-retry). Re-read the primary on a miss. + const run = + (await runStore.findRun(runWhere, runArgs, $replica)) ?? + (await runStore.findRunOnPrimary(runWhere, runArgs)); if (!run) { return new Response("Run not found", { status: 404 }); diff --git a/apps/webapp/app/routes/resources.orgs.$organizationSlug.projects.$projectParam.env.$envParam.runs.$runParam.realtime.v1.streams.$runId.input.$streamId.ts b/apps/webapp/app/routes/resources.orgs.$organizationSlug.projects.$projectParam.env.$envParam.runs.$runParam.realtime.v1.streams.$runId.input.$streamId.ts index 1ecc7819c23..7eae8f6fdcf 100644 --- a/apps/webapp/app/routes/resources.orgs.$organizationSlug.projects.$projectParam.env.$envParam.runs.$runParam.realtime.v1.streams.$runId.input.$streamId.ts +++ b/apps/webapp/app/routes/resources.orgs.$organizationSlug.projects.$projectParam.env.$envParam.runs.$runParam.realtime.v1.streams.$runId.input.$streamId.ts @@ -47,21 +47,23 @@ export async function loader({ request, params }: LoaderFunctionArgs) { return new Response("Environment not found", { status: 404 }); } - const run = await runStore.findRun( - { - friendlyId: runId, - runtimeEnvironmentId: environment.id, - }, - { - select: { - id: true, - friendlyId: true, - realtimeStreamsVersion: true, - streamBasinName: true, - }, + const runWhere = { + friendlyId: runId, + runtimeEnvironmentId: environment.id, + }; + const runArgs = { + select: { + id: true, + friendlyId: true, + realtimeStreamsVersion: true, + streamBasinName: true, }, - $replica - ); + }; + // Replica lag can null out a live run; a spurious 404 breaks the dashboard Agent tab input-stream + // subscription (useRealtimeStream surfaces the error, no auto-retry). Re-read the primary on a miss. + const run = + (await runStore.findRun(runWhere, runArgs, $replica)) ?? + (await runStore.findRunOnPrimary(runWhere, runArgs)); if (!run) { return new Response("Run not found", { status: 404 }); diff --git a/apps/webapp/app/routes/resources.orgs.$organizationSlug.projects.$projectParam.env.$envParam.runs.$runParam.streams.$streamKey/route.tsx b/apps/webapp/app/routes/resources.orgs.$organizationSlug.projects.$projectParam.env.$envParam.runs.$runParam.streams.$streamKey/route.tsx index 3d8cec515c5..43381613433 100644 --- a/apps/webapp/app/routes/resources.orgs.$organizationSlug.projects.$projectParam.env.$envParam.runs.$runParam.streams.$streamKey/route.tsx +++ b/apps/webapp/app/routes/resources.orgs.$organizationSlug.projects.$projectParam.env.$envParam.runs.$runParam.streams.$streamKey/route.tsx @@ -60,18 +60,22 @@ export const loader = async ({ request, params }: LoaderFunctionArgs) => { throw new Response("Not Found", { status: 404 }); } - const run = await runStore.findRun( - { friendlyId: runParam, projectId: project.id }, - { - select: { - id: true, - friendlyId: true, - realtimeStreamsVersion: true, - streamBasinName: true, - runtimeEnvironmentId: true, - }, - } - ); + const runWhere = { friendlyId: runParam, projectId: project.id }; + const runArgs = { + select: { + id: true, + friendlyId: true, + realtimeStreamsVersion: true, + streamBasinName: true, + runtimeEnvironmentId: true, + }, + }; + // Client-less findRun defaults to the read replica; replica lag can null out a live run and 404 a + // valid stream-viewer request (useRealtimeStream surfaces the error, no auto-retry). Re-read the + // owning primary on a replica miss. + const run = + (await runStore.findRun(runWhere, runArgs)) ?? + (await runStore.findRunOnPrimary(runWhere, runArgs)); if (!run) { throw new Response("Not Found", { status: 404 }); diff --git a/apps/webapp/app/routes/resources.orgs.$organizationSlug.projects.$projectParam.env.$envParam.waitpoints.$waitpointFriendlyId.complete/route.tsx b/apps/webapp/app/routes/resources.orgs.$organizationSlug.projects.$projectParam.env.$envParam.waitpoints.$waitpointFriendlyId.complete/route.tsx index 16592ba33f5..e1ed1b2cc70 100644 --- a/apps/webapp/app/routes/resources.orgs.$organizationSlug.projects.$projectParam.env.$envParam.waitpoints.$waitpointFriendlyId.complete/route.tsx +++ b/apps/webapp/app/routes/resources.orgs.$organizationSlug.projects.$projectParam.env.$envParam.waitpoints.$waitpointFriendlyId.complete/route.tsx @@ -81,7 +81,7 @@ export const action = async ({ request, params }: ActionFunctionArgs) => { const waitpointId = WaitpointId.toId(waitpointFriendlyId); - const waitpoint = await runStore.findWaitpoint({ + let waitpoint = await runStore.findWaitpoint({ select: { projectId: true, environmentId: true, @@ -90,6 +90,14 @@ export const action = async ({ request, params }: ActionFunctionArgs) => { id: waitpointId, }, }); + if (!waitpoint) { + // Read-your-writes: a just-minted token may not have replicated. Re-read the owning primary + // before the auth guard / "No waitpoint found" (mirrors the token complete/callback routes). + waitpoint = await runStore.findWaitpointOnPrimary({ + select: { projectId: true, environmentId: true }, + where: { id: waitpointId }, + }); + } if (waitpoint?.projectId !== project.id) { return redirectWithErrorMessage( diff --git a/apps/webapp/app/routes/resources.taskruns.$runParam.cancel.ts b/apps/webapp/app/routes/resources.taskruns.$runParam.cancel.ts index ce74becc7be..cf6b0317780 100644 --- a/apps/webapp/app/routes/resources.taskruns.$runParam.cancel.ts +++ b/apps/webapp/app/routes/resources.taskruns.$runParam.cancel.ts @@ -80,21 +80,26 @@ export const action = dashboardAction( // The project-scope + membership auth is a control-plane concern resolved // separately below; joining project/organization here is a cross-DB join // that returns nothing once the run lives in the run-ops DB. - const taskRun = await runStore.findRun( - { friendlyId: runParam }, - { - select: { - id: true, - engine: true, - status: true, - friendlyId: true, - taskEventStore: true, - createdAt: true, - completedAt: true, - projectId: true, - }, - } - ); + const cancelRunSelect = { + id: true, + engine: true, + status: true, + friendlyId: true, + taskEventStore: true, + createdAt: true, + completedAt: true, + projectId: true, + }; + let taskRun = await runStore.findRun({ friendlyId: runParam }, { select: cancelRunSelect }); + if (!taskRun) { + // Read-your-writes: a just-created run may not have replicated. Re-read the owning primary + // before treating it as absent (mirrors resolveRunOrganizationId's primary fallback above). + taskRun = await runStore.findRun( + { friendlyId: runParam }, + { select: cancelRunSelect }, + prisma + ); + } // Project-scope + membership auth is control-plane only — keyed by the // run's projectId. A miss is treated as not-found (mirrors the old where). diff --git a/apps/webapp/app/routes/resources.taskruns.$runParam.replay.ts b/apps/webapp/app/routes/resources.taskruns.$runParam.replay.ts index 8074a3d3b16..d6fe04be99f 100644 --- a/apps/webapp/app/routes/resources.taskruns.$runParam.replay.ts +++ b/apps/webapp/app/routes/resources.taskruns.$runParam.replay.ts @@ -323,7 +323,12 @@ export const action = dashboardAction( try { // Run-ops read keyed by friendlyId only; membership auth is re-checked on the // control plane below, keyed off the resolved run's projectId. - const pgRun = await runStore.findRun({ friendlyId: runParam }); + let pgRun = await runStore.findRun({ friendlyId: runParam }); + if (!pgRun) { + // Read-your-writes: a just-created run may not have replicated. Re-read the owning primary + // before falling back to the mollifier buffer (mirrors resolveRunOrganizationId above). + pgRun = await runStore.findRun({ friendlyId: runParam }, prisma); + } // Mollifier read-fallback: if the original isn't in PG yet, synthesise a // TaskRun from the buffered snapshot. Needs project/org/env slugs for the diff --git a/apps/webapp/app/routes/sync.traces.runs.$traceId.ts b/apps/webapp/app/routes/sync.traces.runs.$traceId.ts index e89772a103d..5fd1f7c65c0 100644 --- a/apps/webapp/app/routes/sync.traces.runs.$traceId.ts +++ b/apps/webapp/app/routes/sync.traces.runs.$traceId.ts @@ -23,7 +23,7 @@ export async function loader({ params, request }: LoaderFunctionArgs) { return new Response("No user found in cookie", { status: 401 }); } - const run = await runStore.findRun( + let run = await runStore.findRun( { traceId, }, @@ -35,6 +35,15 @@ export async function loader({ params, request }: LoaderFunctionArgs) { $replica ); + if (!run) { + // Read-your-writes: a just-created run may not have replicated yet. Re-read the owning + // primary before 404ing so a live run's realtime trace feed isn't spuriously not-found. + run = await runStore.findRunOnPrimary( + { traceId }, + { select: { runtimeEnvironmentId: true } } + ); + } + if (!run) { return new Response("No run found", { status: 404 }); } diff --git a/apps/webapp/app/runEngine/concerns/idempotencyKeys.server.ts b/apps/webapp/app/runEngine/concerns/idempotencyKeys.server.ts index ea50261a5a1..561e6a5d476 100644 --- a/apps/webapp/app/runEngine/concerns/idempotencyKeys.server.ts +++ b/apps/webapp/app/runEngine/concerns/idempotencyKeys.server.ts @@ -1,5 +1,5 @@ -import { RunId } from "@trigger.dev/core/v3/isomorphic"; -import type { PrismaClientOrTransaction, TaskRun } from "@trigger.dev/database"; +import { ownerEngine, RunId } from "@trigger.dev/core/v3/isomorphic"; +import type { PrismaClientOrTransaction, TaskRun, Waitpoint } from "@trigger.dev/database"; import { env } from "~/env.server"; import { logger } from "~/services/logger.server"; import { resolveIdempotencyKeyTTL } from "~/utils/idempotencyKeys.server"; @@ -8,7 +8,7 @@ import type { RunEngine } from "~/v3/runEngine.server"; import { shouldIdempotencyKeyBeCleared } from "~/v3/taskStatus"; import { getMollifierBuffer } from "~/v3/mollifier/mollifierBuffer.server"; import { findRunByIdWithMollifierFallback } from "~/v3/mollifier/readFallback.server"; -import { claimOrAwait } from "~/v3/mollifier/idempotencyClaim.server"; +import { claimOrAwait, resetResolvedClaim } from "~/v3/mollifier/idempotencyClaim.server"; import { makeResolveMollifierFlag } from "~/v3/mollifier/mollifierGate.server"; import { runStore } from "~/v3/runStore.server"; import { runOpsLegacyPrisma, runOpsNewPrisma } from "~/db.server"; @@ -24,6 +24,13 @@ import type { TraceEventConcern, TriggerTaskRequest } from "../types"; // handleTriggerRequest. const resolveOrgMollifierFlag = makeResolveMollifierFlag(); +// Cap on the claim-loser recreate re-acquisition loop (see +// reacquireClearedGlobalWinner). Each pass reopens a stale resolved slot and +// re-enters the claim; bounded so a pathological stream of expired/failed +// winners can't spin forever. On exhaustion we fall open to the create with +// PG's unique index as the backstop. +const MAX_CLEARED_WINNER_REACQUIRES = 5; + // Claim ownership context returned to the caller when the // IdempotencyKeyConcern won a pre-gate claim. Caller MUST publish the // winning runId on pipeline success (`publishClaim`) or release the @@ -209,107 +216,11 @@ export class IdempotencyKeyConcern { } if (existingRun) { - // The idempotency key has expired - if (existingRun.idempotencyKeyExpiresAt && existingRun.idempotencyKeyExpiresAt < new Date()) { - logger.debug("[TriggerTaskService][call] Idempotency key has expired", { - idempotencyKey: request.options?.idempotencyKey, - run: existingRun, - }); - - // Update the existing run to remove the idempotency key - await runStore.clearIdempotencyKey( - { byId: { runId: existingRun.id, idempotencyKey } }, - dedupClient - ); - - return { isCached: false, idempotencyKey, idempotencyKeyExpiresAt }; - } - - // If the existing run failed or was expired, we clear the key and do a new run - if (shouldIdempotencyKeyBeCleared(existingRun.status)) { - logger.debug("[TriggerTaskService][call] Idempotency key should be cleared", { - idempotencyKey: request.options?.idempotencyKey, - runStatus: existingRun.status, - runId: existingRun.id, - }); - - // Update the existing run to remove the idempotency key - await runStore.clearIdempotencyKey( - { byId: { runId: existingRun.id, idempotencyKey } }, - dedupClient - ); - - return { isCached: false, idempotencyKey, idempotencyKeyExpiresAt }; - } - - // We have an idempotent run, so we return it - const parentRunId = request.body.options?.parentRunId; - const resumeParentOnCompletion = request.body.options?.resumeParentOnCompletion; - - //We're using `andWait` so we need to block the parent run with a waitpoint - if (resumeParentOnCompletion && parentRunId) { - // `parentRunId` comes from the request body and isn't re-validated - // here, so confirm the parent run is in the caller's environment - // before wiring a waitpoint against it. - const parentRunInternalId = RunId.fromFriendlyId(parentRunId); - const parentRunInCallerEnv = await runStore.findRun( - { - id: parentRunInternalId, - runtimeEnvironmentId: request.environment.id, - }, - { select: { id: true } }, - this.prisma - ); - if (!parentRunInCallerEnv) { - throw new ServiceValidationError("Parent run not found in the calling environment", 404); - } - - // Get or create waitpoint lazily (existing run may not have one if it was standalone) - let associatedWaitpoint = existingRun.associatedWaitpoint; - if (!associatedWaitpoint) { - associatedWaitpoint = await this.engine.getOrCreateRunWaitpoint({ - runId: existingRun.id, - projectId: request.environment.projectId, - environmentId: request.environment.id, - }); - } - - await this.traceEventConcern.traceIdempotentRun( - request, - parentStore, - { - existingRun, - idempotencyKey, - incomplete: associatedWaitpoint.status === "PENDING", - isError: associatedWaitpoint.outputIsError, - }, - async (event) => { - const spanId = - request.options?.parentAsLinkType === "replay" - ? event.spanId - : event.traceparent?.spanId - ? `${event.traceparent.spanId}:${event.spanId}` - : event.spanId; - - await this.engine.blockRunWithWaitpoint({ - runId: parentRunInternalId, - waitpoints: associatedWaitpoint!.id, - spanIdToComplete: spanId, - batch: request.options?.batchId - ? { - id: request.options.batchId, - index: request.options.batchIndex ?? 0, - } - : undefined, - projectId: request.environment.projectId, - organizationId: request.environment.organizationId, - tx: dedupClient, - }); - } - ); - } - - return { isCached: true, run: existingRun }; + return await this.handleExistingRun(request, parentStore, existingRun, { + idempotencyKey, + idempotencyKeyExpiresAt, + dedupClient, + }); } // Pre-gate claim — closes the PG+buffer race during gate transition. @@ -325,20 +236,37 @@ export class IdempotencyKeyConcern { // claim's Redis SETNX keeps its RTT off the hot path for those requests // during staged rollout. The org-flag check is a pure in-memory read of // `Organization.featureFlags`, no DB query. + // + // Under the run-ops split the claim ALSO acts as the only cross-DB mutex a + // `global`-scope key has: that key is per (environment, task), so it can be + // triggered concurrently from two parents on DIFFERENT physical DBs where + // each probe misses and the per-DB unique index can't enforce uniqueness. + // For that case the claim is eligible regardless of the per-org flag AND of + // resumeParentOnCompletion (the loser wires its parent waitpoint against the + // winner in the resolved branch below). An absent scope (pre-hashed key / + // older SDK) is treated conservatively as possibly-global — harmless for a + // real run/attempt key, whose hash already embeds the parent id so two + // parents mint DISTINCT keys that never share a claim slot. + const idempotencyKeyScope = request.body.options?.idempotencyKeyOptions?.scope; + const globalUnderSplit = + (idempotencyKeyScope === "global" || idempotencyKeyScope === undefined) && + (await isSplitEnabled()); + const claimEligible = - !request.body.options?.resumeParentOnCompletion && !request.body.options?.debounce && !request.options?.oneTimeUseToken && - (await resolveOrgMollifierFlag({ - envId: request.environment.id, - orgId: request.environment.organizationId, - taskId: request.taskId, - orgFeatureFlags: - (request.environment.organization?.featureFlags as - | Record - | null - | undefined) ?? null, - })); + (globalUnderSplit || + (!request.body.options?.resumeParentOnCompletion && + (await resolveOrgMollifierFlag({ + envId: request.environment.id, + orgId: request.environment.organizationId, + taskId: request.taskId, + orgFeatureFlags: + (request.environment.organization?.featureFlags as + | Record + | null + | undefined) ?? null, + })))); if (claimEligible) { const ttlSeconds = Math.max( 1, @@ -356,6 +284,44 @@ export class IdempotencyKeyConcern { pollStepMs: env.TRIGGER_MOLLIFIER_CLAIM_POLL_MS, }); if (outcome.kind === "resolved") { + // Global-under-split loser: the winner lives on ITS parent's DB, which + // may differ from this loser's parent DB. Resolve the winner by id + // across both DBs (classify the winner friendlyId → NEW/LEGACY client, + // then let the router route+fall-back by id-shape) and feed it through + // the same existing-run handling the PG-hit path uses — so expiry-clear, + // status-clear, and the resumeParentOnCompletion waitpoint wiring all + // apply to the loser exactly as they would to a plain cached hit. + if (globalUnderSplit) { + const winner = await this.resolveWinnerAcrossDbs(outcome.runId, request.environment.id); + if (winner) { + const resolved = await this.handleExistingRun(request, parentStore, winner, { + idempotencyKey, + idempotencyKeyExpiresAt, + dedupClient, + }); + // A LIVE winner (cached hit, or andWait waitpoint wired) is terminal. + if (resolved.isCached) { + return resolved; + } + // CRITICAL (CodeRabbit): the resolved winner was EXPIRED or FAILED, so + // handleExistingRun cleared its key and would have us CREATE a new run. + // The initial create is serialised by the claim, but this clear-and- + // recreate is not — and under the split the per-DB unique index can't + // dedup a cross-residency recreate, so two concurrent losers clearing + // the SAME winner would each create → duplicate run. Re-serialise the + // recreate through the claim so exactly one caller recreates (and + // publishes) while the rest resolve to the fresh run. + return await this.reacquireClearedGlobalWinner(request, parentStore, { + idempotencyKey, + idempotencyKeyExpiresAt, + dedupClient, + ttlSeconds, + clearedRunId: outcome.runId, + safetyNetMs: env.TRIGGER_MOLLIFIER_CLAIM_WAIT_MS, + pollStepMs: env.TRIGGER_MOLLIFIER_CLAIM_POLL_MS, + }); + } + } // Another concurrent trigger committed first. Re-resolve via the // existing checks: writer-side PG findFirst first (defeats // replica lag), then buffer fallback for the buffered case. @@ -421,4 +387,231 @@ export class IdempotencyKeyConcern { return { isCached: false, idempotencyKey, idempotencyKeyExpiresAt }; } + + // Resolve an already-existing idempotent run: honour key expiry / status + // clearing, and for `andWait` (resumeParentOnCompletion) block the calling + // parent on the run's waitpoint. Extracted so both the PG-hit path and the + // cross-DB claim-loser path resolve an existing run identically. + private async handleExistingRun( + request: TriggerTaskRequest, + parentStore: string | undefined, + existingRun: TaskRun & { associatedWaitpoint?: Waitpoint | null }, + ctx: { + idempotencyKey: string; + idempotencyKeyExpiresAt: Date; + dedupClient: PrismaClientOrTransaction; + } + ): Promise { + const { idempotencyKey, idempotencyKeyExpiresAt, dedupClient } = ctx; + + // The idempotency key has expired + if (existingRun.idempotencyKeyExpiresAt && existingRun.idempotencyKeyExpiresAt < new Date()) { + logger.debug("[TriggerTaskService][call] Idempotency key has expired", { + idempotencyKey: request.options?.idempotencyKey, + run: existingRun, + }); + + // Update the existing run to remove the idempotency key + await runStore.clearIdempotencyKey( + { byId: { runId: existingRun.id, idempotencyKey } }, + dedupClient + ); + + return { isCached: false, idempotencyKey, idempotencyKeyExpiresAt }; + } + + // If the existing run failed or was expired, we clear the key and do a new run + if (shouldIdempotencyKeyBeCleared(existingRun.status)) { + logger.debug("[TriggerTaskService][call] Idempotency key should be cleared", { + idempotencyKey: request.options?.idempotencyKey, + runStatus: existingRun.status, + runId: existingRun.id, + }); + + // Update the existing run to remove the idempotency key + await runStore.clearIdempotencyKey( + { byId: { runId: existingRun.id, idempotencyKey } }, + dedupClient + ); + + return { isCached: false, idempotencyKey, idempotencyKeyExpiresAt }; + } + + // We have an idempotent run, so we return it + const parentRunId = request.body.options?.parentRunId; + const resumeParentOnCompletion = request.body.options?.resumeParentOnCompletion; + + //We're using `andWait` so we need to block the parent run with a waitpoint + if (resumeParentOnCompletion && parentRunId) { + // `parentRunId` comes from the request body and isn't re-validated + // here, so confirm the parent run is in the caller's environment + // before wiring a waitpoint against it. + const parentRunInternalId = RunId.fromFriendlyId(parentRunId); + const parentRunInCallerEnv = await runStore.findRun( + { + id: parentRunInternalId, + runtimeEnvironmentId: request.environment.id, + }, + { select: { id: true } }, + this.prisma + ); + if (!parentRunInCallerEnv) { + throw new ServiceValidationError("Parent run not found in the calling environment", 404); + } + + // Get or create waitpoint lazily (existing run may not have one if it was standalone) + let associatedWaitpoint = existingRun.associatedWaitpoint; + if (!associatedWaitpoint) { + associatedWaitpoint = await this.engine.getOrCreateRunWaitpoint({ + runId: existingRun.id, + projectId: request.environment.projectId, + environmentId: request.environment.id, + }); + } + + await this.traceEventConcern.traceIdempotentRun( + request, + parentStore, + { + existingRun, + idempotencyKey, + incomplete: associatedWaitpoint.status === "PENDING", + isError: associatedWaitpoint.outputIsError, + }, + async (event) => { + const spanId = + request.options?.parentAsLinkType === "replay" + ? event.spanId + : event.traceparent?.spanId + ? `${event.traceparent.spanId}:${event.spanId}` + : event.spanId; + + await this.engine.blockRunWithWaitpoint({ + runId: parentRunInternalId, + waitpoints: associatedWaitpoint!.id, + spanIdToComplete: spanId, + batch: request.options?.batchId + ? { + id: request.options.batchId, + index: request.options.batchIndex ?? 0, + } + : undefined, + projectId: request.environment.projectId, + organizationId: request.environment.organizationId, + tx: dedupClient, + }); + } + ); + } + + return { isCached: true, run: existingRun }; + } + + // Re-serialise a cross-DB recreate through the claim after a claim-loser's + // resolved winner turned out to be EXPIRED / FAILED (its key was cleared by + // handleExistingRun). Without this, concurrent losers clearing the same + // winner each create a new run on their own DB — the per-DB unique index + // can't dedup a cross-residency pair, so the initial-create's serialisation + // is lost on the recreate. Each pass: compare-and-delete the stale resolved + // slot (keyed on the cleared runId — never an unconditional DEL, so a + // reacquirer that already re-published a NEW winner is not wiped), then + // re-enter claimOrAwait. Exactly one caller wins the re-claim and recreates + // (returning its claim to publish); the rest resolve to the fresh run. If a + // re-claim resolves to ANOTHER cleared winner we advance and loop, bounded + // by MAX_CLEARED_WINNER_REACQUIRES; on exhaustion (or an unfindable + // resolution) we fall open to the create, matching the initial claim's + // fail-open posture (PG unique index as backstop). + private async reacquireClearedGlobalWinner( + request: TriggerTaskRequest, + parentStore: string | undefined, + ctx: { + idempotencyKey: string; + idempotencyKeyExpiresAt: Date; + dedupClient: PrismaClientOrTransaction; + ttlSeconds: number; + clearedRunId: string; + safetyNetMs: number; + pollStepMs: number; + } + ): Promise { + const { idempotencyKey, idempotencyKeyExpiresAt, dedupClient, ttlSeconds } = ctx; + const claimInput = { + envId: request.environment.id, + taskIdentifier: request.taskId, + idempotencyKey, + }; + let staleRunId = ctx.clearedRunId; + for (let attempt = 0; attempt < MAX_CLEARED_WINNER_REACQUIRES; attempt++) { + await resetResolvedClaim({ ...claimInput, runId: staleRunId }); + + const outcome = await claimOrAwait({ + ...claimInput, + ttlSeconds, + safetyNetMs: ctx.safetyNetMs, + pollStepMs: ctx.pollStepMs, + }); + + if (outcome.kind === "timed_out") { + throw new ServiceValidationError("Idempotency claim resolution timed out", 503); + } + if (outcome.kind === "claimed") { + // We own the recreate. Caller MUST publish the new runId / release on error. + return { + isCached: false, + idempotencyKey, + idempotencyKeyExpiresAt, + claim: { ...claimInput, token: outcome.token }, + }; + } + // resolved: another caller won the recreate. Honour it like any cached + // hit (incl. andWait wiring). If it too was cleared, advance and loop. + const winner = await this.resolveWinnerAcrossDbs(outcome.runId, request.environment.id); + if (!winner) { + logger.warn("idempotency reacquire resolved but runId not findable", { + envId: request.environment.id, + taskIdentifier: request.taskId, + claimedRunId: outcome.runId, + }); + break; + } + const resolved = await this.handleExistingRun(request, parentStore, winner, { + idempotencyKey, + idempotencyKeyExpiresAt, + dedupClient, + }); + if (resolved.isCached) { + return resolved; + } + staleRunId = outcome.runId; + } + return { isCached: false, idempotencyKey, idempotencyKeyExpiresAt }; + } + + // Resolve a claim winner (a run friendlyId) across both split DBs. Classify + // the id-shape to pick the writer client for read-your-writes, then read by + // id — the routing store routes to the owning store and falls back to the + // other, so a winner on either DB is found. Returns null when the id can't be + // classified or the row genuinely isn't there (caller falls through). + private async resolveWinnerAcrossDbs( + winnerFriendlyId: string, + environmentId: string + ): Promise<(TaskRun & { associatedWaitpoint?: Waitpoint | null }) | null> { + let internalId: string; + try { + internalId = RunId.fromFriendlyId(winnerFriendlyId); + } catch { + return null; + } + let client: PrismaClientOrTransaction; + try { + client = ownerEngine(internalId) === "NEW" ? runOpsNewPrisma : runOpsLegacyPrisma; + } catch { + client = this.prisma; + } + return runStore.findRun( + { id: internalId, runtimeEnvironmentId: environmentId }, + { include: { associatedWaitpoint: true } }, + client + ); + } } diff --git a/apps/webapp/app/services/dashboardAgent.server.ts b/apps/webapp/app/services/dashboardAgent.server.ts index 14eb51fcbef..039ee3c3ada 100644 --- a/apps/webapp/app/services/dashboardAgent.server.ts +++ b/apps/webapp/app/services/dashboardAgent.server.ts @@ -212,7 +212,9 @@ export async function resolveRunCommit( environmentId: string, runFriendlyId: string ): Promise<{ sha: string; version: string; dirty: boolean } | null> { - const run = await runStore.findRun( + // Read-your-writes: a just-locked run's lockedToVersionId may not have replicated. Read the owning + // primary so a live, pinned run resolves its commit instead of silently falling back to branch head. + const run = await runStore.findRunOnPrimary( { friendlyId: runFriendlyId, runtimeEnvironmentId: environmentId }, { select: { lockedToVersionId: true } } ); diff --git a/apps/webapp/app/services/realtime/sessionRunManager.server.ts b/apps/webapp/app/services/realtime/sessionRunManager.server.ts index 5ff462154bf..10cd966d95c 100644 --- a/apps/webapp/app/services/realtime/sessionRunManager.server.ts +++ b/apps/webapp/app/services/realtime/sessionRunManager.server.ts @@ -478,8 +478,10 @@ export async function swapSessionRun(params: SwapSessionRunParams): Promise { - // Use the read replica — this is a hot-path probe and stale-by-ms is - // fine. The append handler re-checks if it ends up reusing the runId. + // Use the read replica — hot-path probe, stale-by-ms is fine. The dangerous + // stale shape (looks-vanished → double-trigger) is closed by the writer re-probe + // in ensureRunForSession; a stale-non-final reuse self-heals via the durable S2 + // stream + next-append re-probe (there is no append-handler re-check). // `friendlyId` is fetched alongside `status` so the dead-run-detection // branch in `ensureRunForSession` can forward the public-form id as // `payload.previousRunId` without a second read. `Session.currentRunId` diff --git a/apps/webapp/app/v3/mollifier/idempotencyClaim.server.ts b/apps/webapp/app/v3/mollifier/idempotencyClaim.server.ts index 47c9733c927..196017b438d 100644 --- a/apps/webapp/app/v3/mollifier/idempotencyClaim.server.ts +++ b/apps/webapp/app/v3/mollifier/idempotencyClaim.server.ts @@ -213,6 +213,39 @@ export async function releaseClaim(input: { } } +// Reopen a stale RESOLVED claim slot whose winner was cleared (expired / +// failed), so the claim-loser reacquire path can re-serialise a cross-DB +// recreate through a fresh claim. Compare-and-delete on the observed winner +// runId (buffer-side) so a concurrent reacquirer's freshly published winner +// is never wiped. Best-effort: on buffer-null / error we no-op and let the +// reacquire's claimOrAwait proceed (fail-open — the caller caps attempts and +// ultimately falls through to the create, PG unique index as backstop). +export async function resetResolvedClaim(input: { + envId: string; + taskIdentifier: string; + idempotencyKey: string; + runId: string; + buffer?: MollifierBuffer | null; +}): Promise { + const buffer = input.buffer === undefined ? getMollifierBuffer() : input.buffer; + if (!buffer) return false; + try { + return await buffer.resetResolvedClaim({ + envId: input.envId, + taskIdentifier: input.taskIdentifier, + idempotencyKey: input.idempotencyKey, + expectedRunId: input.runId, + }); + } catch (err) { + logger.warn("idempotency resolved-claim reset failed", { + envId: input.envId, + taskIdentifier: input.taskIdentifier, + err: err instanceof Error ? err.message : String(err), + }); + return false; + } +} + function defaultSleep(ms: number): Promise { return new Promise((resolve) => setTimeout(resolve, ms)); } diff --git a/apps/webapp/app/v3/services/batchTriggerV3.server.ts b/apps/webapp/app/v3/services/batchTriggerV3.server.ts index a50ba637474..fdb87d2d2ca 100644 --- a/apps/webapp/app/v3/services/batchTriggerV3.server.ts +++ b/apps/webapp/app/v3/services/batchTriggerV3.server.ts @@ -195,18 +195,23 @@ export class BatchTriggerV3Service extends BaseService { span.setAttribute("batchId", batchId); const dependentAttempt = body?.dependentAttempt - ? await this.runStore.findTaskRunAttempt({ - // Scope to the caller's environment (see dependentAttemptWhere). - where: dependentAttemptWhere(body.dependentAttempt, environment.id), - include: { - taskRun: { - select: { - id: true, - status: true, + ? await this.runStore.findTaskRunAttempt( + { + // Scope to the caller's environment (see dependentAttemptWhere). + where: dependentAttemptWhere(body.dependentAttempt, environment.id), + include: { + taskRun: { + select: { + id: true, + status: true, + }, }, }, }, - }) + // Read-your-writes: the dependent parent attempt's terminal state must be seen even + // when the read replica lags, so route this read to the owning primary. + this._prisma + ) : undefined; if ( diff --git a/internal-packages/run-engine/src/engine/systems/runAttemptSystem.ts b/internal-packages/run-engine/src/engine/systems/runAttemptSystem.ts index 7f53c7648de..922b41cd225 100644 --- a/internal-packages/run-engine/src/engine/systems/runAttemptSystem.ts +++ b/internal-packages/run-engine/src/engine/systems/runAttemptSystem.ts @@ -170,7 +170,9 @@ export class RunAttemptSystem { } public async resolveTaskRunContext(runId: string): Promise { - const run = await this.$.runStore.findRun( + // read-your-writes: a just-created/locked run may not be on a replica yet; read the owning primary + // so a live run's execution context resolves instead of throwing a spurious 404. + const run = await this.$.runStore.findRunOnPrimary( { id: runId, }, @@ -711,8 +713,8 @@ export class RunAttemptSystem { const completedAt = new Date(); - // Read current usage values to calculate new totals (safe under runLock) - const currentRun = await this.$.runStore.findRun( + // Read current usage totals on the owning primary (read-your-writes; a replica lags) + const currentRun = await this.$.runStore.findRunOnPrimary( { id: runId }, { select: { @@ -904,7 +906,8 @@ export class RunAttemptSystem { const failedAt = new Date(); const retryResult = await retryOutcomeFromCompletion( - this.$.readOnlyPrisma, + // read-your-writes: lock-time maxAttempts/lockedRetryConfig may not be on a replica yet + this.$.prisma, this.$.runStore, { runId, @@ -917,7 +920,9 @@ export class RunAttemptSystem { // Force requeue means it was crashed so the attempt span needs to be closed if (forceRequeue) { - const minimalRun = await this.$.runStore.findRun( + // read-your-writes: the run was just written in this flow; read the owning primary so the + // requeue event re-read cannot false-miss on a lagging replica (mirrors the :906 read). + const minimalRun = await this.$.runStore.findRunOnPrimary( { id: runId, }, @@ -1375,7 +1380,7 @@ export class RunAttemptSystem { // Calculate updated usage if we have attempt duration data let usageUpdate: { usageDurationMs: number; costInCents: number } | undefined; if (attemptDurationMs !== undefined) { - const currentRun = await this.$.runStore.findRun( + const currentRun = await this.$.runStore.findRunOnPrimary( { id: runId }, { select: { @@ -1589,8 +1594,8 @@ export class RunAttemptSystem { const truncatedError = this.#truncateTaskRunError(error); - // Read current usage values to calculate new totals - const currentRun = await this.$.runStore.findRun( + // Read current usage totals on the owning primary (read-your-writes; a replica lags) + const currentRun = await this.$.runStore.findRunOnPrimary( { id: runId }, { select: { diff --git a/internal-packages/run-engine/src/engine/systems/ttlSystem.ts b/internal-packages/run-engine/src/engine/systems/ttlSystem.ts index af6f41eac2e..0fb3b8387cb 100644 --- a/internal-packages/run-engine/src/engine/systems/ttlSystem.ts +++ b/internal-packages/run-engine/src/engine/systems/ttlSystem.ts @@ -158,22 +158,26 @@ export class TtlSystem { const skipped: { runId: string; reason: string }[] = []; // Fetch all runs in a single query (no snapshot data needed) - const runs = await this.$.runStore.findRuns({ - where: { id: { in: runIds } }, - select: { - id: true, - spanId: true, - status: true, - lockedAt: true, - ttl: true, - taskEventStore: true, - createdAt: true, - associatedWaitpoint: { select: { id: true } }, - organizationId: true, - projectId: true, - runtimeEnvironmentId: true, + const runs = await this.$.runStore.findRuns( + { + where: { id: { in: runIds } }, + select: { + id: true, + spanId: true, + status: true, + lockedAt: true, + ttl: true, + taskEventStore: true, + createdAt: true, + associatedWaitpoint: { select: { id: true } }, + organizationId: true, + projectId: true, + runtimeEnvironmentId: true, + }, + // read-your-writes: the queue slot is already claimed; a lagging replica would orphan the run }, - }); + this.$.prisma + ); // Filter runs that can be expired const runsToExpire: typeof runs = []; diff --git a/internal-packages/run-store/src/runOpsStore.ts b/internal-packages/run-store/src/runOpsStore.ts index f6b613a393f..97c0759e5e1 100644 --- a/internal-packages/run-store/src/runOpsStore.ts +++ b/internal-packages/run-store/src/runOpsStore.ts @@ -1244,6 +1244,11 @@ export class RoutingRunStore implements RunStore { args as Record ); const id = RoutingRunStore.#waitpointId((args as { where?: unknown }).where); + // A colocated lookup (no id, resolved via `coLocateWithRunId`) is the (env,idempotencyKey) dedup + // probe of createDateTimeWaitpoint/createManualWaitpoint: read-your-writes within the owning + // run/tree. On a retry it must observe attempt 1's just-written waitpoint to short-circuit, so it + // reads the owning store's PRIMARY — the replica can lag and miss it, re-arming/re-blocking the run. + const coLocatedDedup = id === undefined && opts?.coLocateWithRunId !== undefined; const store = id !== undefined ? await this.#resolveWaitpointStore(id, client !== undefined) @@ -1254,7 +1259,7 @@ export class RoutingRunStore implements RunStore { store !== undefined ? ((await store.findWaitpoint( scalarArgs as typeof args, - RoutingRunStore.#ownPrimary(store, client) + coLocatedDedup ? store.primaryReadClient : RoutingRunStore.#ownPrimary(store, client) )) as Record | null) : (((await this.#new.findWaitpoint( scalarArgs as typeof args, diff --git a/packages/redis-worker/src/mollifier/buffer.ts b/packages/redis-worker/src/mollifier/buffer.ts index d2281cd4332..179de94eca3 100644 --- a/packages/redis-worker/src/mollifier/buffer.ts +++ b/packages/redis-worker/src/mollifier/buffer.ts @@ -462,6 +462,23 @@ export class MollifierBuffer { await this.redis.releaseMollifierClaim(claimKey, `${PENDING_PREFIX}${input.token}`); } + // Reopen a RESOLVED claim slot whose winner was cleared, so the claim-loser + // reacquire path can re-serialise a cross-DB recreate through a fresh claim. + // Compare-and-delete on the observed `expectedRunId` (never an unconditional + // DEL) so a concurrent reacquirer that already re-published a NEW winner is + // never wiped. Reuses the lookup self-heal's compare-and-delete Lua. Returns + // true if the stale slot was cleared. + async resetResolvedClaim( + input: IdempotencyLookupInput & { expectedRunId: string } + ): Promise { + const claimKey = makeIdempotencyClaimKey(input); + const deleted = (await this.redis.delMollifierKeyIfEquals( + claimKey, + input.expectedRunId + )) as number; + return deleted === 1; + } + // Read the current claim value, used by the wait/poll loop on losers // to detect "pending" → "resolved" transitions and timeouts. async readClaim(input: IdempotencyLookupInput): Promise { From fb8fcfe619d8d17c79d61c56d17ee3ac814c29b1 Mon Sep 17 00:00:00 2001 From: Dan Sutton Date: Sat, 18 Jul 2026 17:35:20 +0100 Subject: [PATCH 2/6] chore(run-store): remove the unused #routeWaitpointWrite helper MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Dead since #4272 inlined per-write residency routing at each call site — the private helper had zero callers (only doc-comments referenced it by name), tripping eslint no-unused-private-class-members. Remove it and reword the comments that referenced it. --- .../run-store/src/runOpsStore.ts | 22 ++++--------------- 1 file changed, 4 insertions(+), 18 deletions(-) diff --git a/internal-packages/run-store/src/runOpsStore.ts b/internal-packages/run-store/src/runOpsStore.ts index 97c0759e5e1..56f6addebdf 100644 --- a/internal-packages/run-store/src/runOpsStore.ts +++ b/internal-packages/run-store/src/runOpsStore.ts @@ -138,20 +138,6 @@ export class RoutingRunStore implements RunStore { return this.#routeOrNew(runId).runInTransaction(runId, fn); } - // A waitpoint WRITE co-locates with its run by id-shape (cuid → LEGACY, run-ops id → NEW, - // unclassifiable → LEGACY), mirroring how `blockRunWithWaitpointEdges` routes the edge by - // run id. The caller's `tx` is never forwarded: a routed write must run on the OWNING store's - // OWN client so the row lands in that store's database (same-store atomicity comes from the - // owning store opening its own transaction, never from a caller-supplied one). - #routeWaitpointWrite( - id: string | undefined, - _tx?: PrismaClientOrTransaction - ): { store: RunStore; tx?: PrismaClientOrTransaction } { - const store = - typeof id === "string" && this.#classifySafe(id) === "NEW" ? this.#new : this.#legacy; - return { store, tx: undefined }; - } - // Resolve which store ACTUALLY holds a waitpoint id: drain-on-read can relocate a cuid // waitpoint onto NEW while keeping its id, so probe the id-shape's home then the other. // `onPrimary` probes each store's own primary (read-your-writes callers; a fresh row may not @@ -1187,7 +1173,7 @@ export class RoutingRunStore implements RunStore { opts?.residency, RoutingRunStore.#waitpointId(data) ); - // Never forward the caller's tx into a routed write (matches #routeWaitpointWrite). + // Never forward the caller's tx into a routed write (it runs on the owning store's own client). return store.createWaitpoint(args, undefined); } @@ -1706,7 +1692,7 @@ export class RoutingRunStore implements RunStore { ): Promise { // Route by the batch's classifiable internal id: run-ops id→NEW, cuid→LEGACY. The caller's // `tx` is never forwarded — the create runs on the owning store's own client so the batch and - // its co-resident child runs/items land on the same DB. Mirrors #routeWaitpointWrite / + // its co-resident child runs/items land on the same DB. Mirrors the by-id waitpoint-write routing / // updateBatchTaskRun. const store = await this.#routeOrNewForWrite(data.id); return store.createBatchTaskRun(data, undefined); @@ -1723,7 +1709,7 @@ export class RoutingRunStore implements RunStore { const id = typeof args.where.id === "string" ? args.where.id : (args.where.friendlyId ?? undefined); // The caller's `tx` is never forwarded — the update runs on the owning store's own client so - // it targets the DB the batch actually lives on. Mirrors #routeWaitpointWrite. + // it targets the DB the batch actually lives on. Mirrors the by-id waitpoint-write routing. const store = this.#routeOrNew(id); return store.updateBatchTaskRun(args, undefined); } @@ -1885,7 +1871,7 @@ export class RoutingRunStore implements RunStore { // WaitpointTag — a standalone entity (no run/waitpoint FK) keyed by (environmentId, name). // --------------------------------------------------------------------------- - // Callers never mint a tag id (defaults to cuid), so #routeWaitpointWrite always resolves LEGACY + // Callers never mint a tag id (defaults to cuid), so a tag write always resolves LEGACY // today — deliberately single-homed, like standalone waitpoint tokens. If tag-id minting is ever made // residency-aware, findManyWaitpointTags must de-dupe by (environmentId, name) or names will duplicate. upsertWaitpointTag( From 1c625e17e9515316e639601d5beeca30736babe5 Mon Sep 17 00:00:00 2001 From: Dan Sutton Date: Sun, 19 Jul 2026 16:08:32 +0100 Subject: [PATCH 3/6] =?UTF-8?q?fix(webapp,run-store):=20review=20follow-up?= =?UTF-8?q?s=20=E2=80=94=20split=20idempotency=20+=20realtime=20batch=20ha?= =?UTF-8?q?rdening?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Production follow-ups from the CodeRabbit/Claude/Devin review. Caller-driven guard tests are in the stacked tests PR; the two test edits here are coupled to the production change (the publishClaim signature change invalidates a main assertion; the fan-out removal obsoletes a main test). - Claim TTL floor (TRIGGER_MOLLIFIER_CLAIM_MIN_TTL_SECONDS, default 5) independent of the customer key TTL, so a short key TTL can't expire the claim mid-pipeline and let a loser re-claim. - publishClaim returns the buffer CAS result; the trigger success path detects + logs a no-op'd publish. - reacquireClearedGlobalWinner fails closed with a retryable 503 (exhaustion / unfindable winner); the expired/failed clear-and-recreate path routes through it too (Devin), so that recreate is serialised. - Realtime batch route re-reads the owning primary on a replica miss (backend-agnostic; closes the Electric ShapeStream permanent-404 for self-hosters). - Remove the classifiable-id cross-store fan-out from findRun: a run's id-shape fixes its residency for life, so the single-store read is correct and the fan-out was dead code. --- apps/webapp/app/env.server.ts | 3 + .../routes/realtime.v1.batches.$batchId.ts | 15 +- .../concerns/idempotencyKeys.server.ts | 70 ++++- .../runEngine/services/triggerTask.server.ts | 13 +- apps/webapp/app/v3/mollifier/claimTtl.ts | 12 + .../v3/mollifier/idempotencyClaim.server.ts | 10 +- .../resolveBatchForRealtime.server.ts | 24 ++ .../test/mollifierIdempotencyClaim.test.ts | 8 +- ...OpsStore.residencyMismatchFallback.test.ts | 296 ------------------ .../run-store/src/runOpsStore.ts | 33 +- 10 files changed, 128 insertions(+), 356 deletions(-) create mode 100644 apps/webapp/app/v3/mollifier/claimTtl.ts create mode 100644 apps/webapp/app/v3/realtime/resolveBatchForRealtime.server.ts delete mode 100644 internal-packages/run-store/src/runOpsStore.residencyMismatchFallback.test.ts diff --git a/apps/webapp/app/env.server.ts b/apps/webapp/app/env.server.ts index 4e2d4c5845f..973560ca07f 100644 --- a/apps/webapp/app/env.server.ts +++ b/apps/webapp/app/env.server.ts @@ -1311,6 +1311,9 @@ const EnvironmentSchema = z // claim TTL), how long a waiter blocks before timing out, and the // waiter poll interval. TRIGGER_MOLLIFIER_CLAIM_TTL_SECONDS: z.coerce.number().int().positive().default(30), + // Pipeline floor: the claim never shrinks below this even for a short customer key TTL, so it + // can't expire mid-pipeline and let a loser re-claim (cross-DB duplicate under the split). + TRIGGER_MOLLIFIER_CLAIM_MIN_TTL_SECONDS: z.coerce.number().int().positive().default(5), TRIGGER_MOLLIFIER_CLAIM_WAIT_MS: z.coerce.number().int().positive().default(5_000), TRIGGER_MOLLIFIER_CLAIM_POLL_MS: z.coerce.number().int().positive().default(25), diff --git a/apps/webapp/app/routes/realtime.v1.batches.$batchId.ts b/apps/webapp/app/routes/realtime.v1.batches.$batchId.ts index b95cd4491db..8a761b32f2e 100644 --- a/apps/webapp/app/routes/realtime.v1.batches.$batchId.ts +++ b/apps/webapp/app/routes/realtime.v1.batches.$batchId.ts @@ -2,7 +2,7 @@ import { z } from "zod"; import { getRequestAbortSignal } from "~/services/httpAsyncStorage.server"; import { resolveRealtimeStreamClient } from "~/services/realtime/resolveRealtimeStreamClient.server"; import { anyResource, createLoaderApiRoute } from "~/services/routeBuilders/apiBuilder.server"; -import { runStore } from "~/v3/runStore.server"; +import { resolveBatchTaskRunForRealtime } from "~/v3/realtime/resolveBatchForRealtime.server"; const ParamsSchema = z.object({ batchId: z.string(), @@ -13,14 +13,13 @@ export const loader = createLoaderApiRoute( params: ParamsSchema, allowJWT: true, corsStrategy: "all", - // A just-created batch may not yet have replicated to the read replica this client-less - // findBatchTaskRunByFriendlyId lookup routes to; return a retryable 404 so the SDK retries through - // replica lag rather than stranding a live batch on a permanent 404 (mirrors the run-get routes, - // e.g. api.v3.runs.$runId). + // A just-created batch may not yet have replicated to the read replica the client-less lookup uses. + // shouldRetryNotFound stamps a retryable 404 for the zodfetch GET; the realtime resolver ALSO + // re-reads the owning primary on a replica miss, so the Electric ShapeStream consumer (which ignores + // x-should-retry) doesn't strand a live batch on a permanent 404. Mirrors the run-get routes. shouldRetryNotFound: true, - findResource: (params, auth) => { - return runStore.findBatchTaskRunByFriendlyId(params.batchId, auth.environment.id); - }, + findResource: (params, auth) => + resolveBatchTaskRunForRealtime(params.batchId, auth.environment.id), authorization: { action: "read", // See sibling note in api.v1.batches.$batchId.ts — `{type: "runs"}` diff --git a/apps/webapp/app/runEngine/concerns/idempotencyKeys.server.ts b/apps/webapp/app/runEngine/concerns/idempotencyKeys.server.ts index 561e6a5d476..f6696865e94 100644 --- a/apps/webapp/app/runEngine/concerns/idempotencyKeys.server.ts +++ b/apps/webapp/app/runEngine/concerns/idempotencyKeys.server.ts @@ -9,6 +9,7 @@ import { shouldIdempotencyKeyBeCleared } from "~/v3/taskStatus"; import { getMollifierBuffer } from "~/v3/mollifier/mollifierBuffer.server"; import { findRunByIdWithMollifierFallback } from "~/v3/mollifier/readFallback.server"; import { claimOrAwait, resetResolvedClaim } from "~/v3/mollifier/idempotencyClaim.server"; +import { computeClaimTtlSeconds } from "~/v3/mollifier/claimTtl"; import { makeResolveMollifierFlag } from "~/v3/mollifier/mollifierGate.server"; import { runStore } from "~/v3/runStore.server"; import { runOpsLegacyPrisma, runOpsNewPrisma } from "~/db.server"; @@ -180,6 +181,14 @@ export class IdempotencyKeyConcern { } ); + // `global`-scope (or scope-absent) keys under the split have no per-run salt, so the Redis claim is + // their only cross-DB dedup mutex. Computed here (not just in the claim block below) because the + // expired/failed clear-and-recreate path must serialise through it too. + const idempotencyKeyScope = request.body.options?.idempotencyKeyOptions?.scope; + const globalUnderSplit = + (idempotencyKeyScope === "global" || idempotencyKeyScope === undefined) && + (await isSplitEnabled()); + const existingRun = idempotencyKey ? await runStore.findRun( { @@ -216,11 +225,37 @@ export class IdempotencyKeyConcern { } if (existingRun) { - return await this.handleExistingRun(request, parentStore, existingRun, { + const handled = await this.handleExistingRun(request, parentStore, existingRun, { idempotencyKey, idempotencyKeyExpiresAt, dedupClient, }); + // A LIVE cached hit (or andWait waitpoint wiring) is terminal. + if (handled.isCached) { + return handled; + } + // isCached === false → the existing run was EXPIRED/FAILED, so handleExistingRun cleared its key + // and we must recreate. For a global-scope key under split that recreate has to be claim- + // serialised too — otherwise two concurrent cross-residency recreates each create a run the + // per-DB unique index can't dedup (the same hole reacquireClearedGlobalWinner closes on the + // claim-loser path). Non-split / non-global: the plain unserialised recreate is safe. + if (globalUnderSplit) { + return await this.reacquireClearedGlobalWinner(request, parentStore, { + idempotencyKey, + idempotencyKeyExpiresAt, + dedupClient, + ttlSeconds: computeClaimTtlSeconds({ + keyExpiresAt: idempotencyKeyExpiresAt, + now: Date.now(), + minTtlSeconds: env.TRIGGER_MOLLIFIER_CLAIM_MIN_TTL_SECONDS, + maxTtlSeconds: env.TRIGGER_MOLLIFIER_CLAIM_TTL_SECONDS, + }), + clearedRunId: existingRun.friendlyId, + safetyNetMs: env.TRIGGER_MOLLIFIER_CLAIM_WAIT_MS, + pollStepMs: env.TRIGGER_MOLLIFIER_CLAIM_POLL_MS, + }); + } + return handled; } // Pre-gate claim — closes the PG+buffer race during gate transition. @@ -247,11 +282,8 @@ export class IdempotencyKeyConcern { // older SDK) is treated conservatively as possibly-global — harmless for a // real run/attempt key, whose hash already embeds the parent id so two // parents mint DISTINCT keys that never share a claim slot. - const idempotencyKeyScope = request.body.options?.idempotencyKeyOptions?.scope; - const globalUnderSplit = - (idempotencyKeyScope === "global" || idempotencyKeyScope === undefined) && - (await isSplitEnabled()); - + // (idempotencyKeyScope / globalUnderSplit are computed above — they also gate the expired/failed + // recreate serialisation.) const claimEligible = !request.body.options?.debounce && !request.options?.oneTimeUseToken && @@ -268,13 +300,12 @@ export class IdempotencyKeyConcern { | undefined) ?? null, })))); if (claimEligible) { - const ttlSeconds = Math.max( - 1, - Math.min( - env.TRIGGER_MOLLIFIER_CLAIM_TTL_SECONDS, - Math.ceil((idempotencyKeyExpiresAt.getTime() - Date.now()) / 1000) - ) - ); + const ttlSeconds = computeClaimTtlSeconds({ + keyExpiresAt: idempotencyKeyExpiresAt, + now: Date.now(), + minTtlSeconds: env.TRIGGER_MOLLIFIER_CLAIM_MIN_TTL_SECONDS, + maxTtlSeconds: env.TRIGGER_MOLLIFIER_CLAIM_TTL_SECONDS, + }); const outcome = await claimOrAwait({ envId: request.environment.id, taskIdentifier: request.taskId, @@ -519,8 +550,8 @@ export class IdempotencyKeyConcern { // (returning its claim to publish); the rest resolve to the fresh run. If a // re-claim resolves to ANOTHER cleared winner we advance and loop, bounded // by MAX_CLEARED_WINNER_REACQUIRES; on exhaustion (or an unfindable - // resolution) we fall open to the create, matching the initial claim's - // fail-open posture (PG unique index as backstop). + // resolution) we fail CLOSED with a retryable 503 so the SDK retry re-serialises, + // rather than fall open to an unserialised cross-DB create. private async reacquireClearedGlobalWinner( request: TriggerTaskRequest, parentStore: string | undefined, @@ -584,7 +615,14 @@ export class IdempotencyKeyConcern { } staleRunId = outcome.runId; } - return { isCached: false, idempotencyKey, idempotencyKeyExpiresAt }; + // Exhausted the bounded reacquires (or the winner was unfindable). Rather than fall through to an + // UNSERIALISED create — which under global-scope-split can dual-create across DBs (the per-DB unique + // index can't dedup cross-residency) — fail closed with a retryable 503 so the SDK retry re-serialises + // through a fresh claim (mirrors the timed_out branch above). + throw new ServiceValidationError( + "Idempotency claim could not be re-serialised after repeated cleared winners", + 503 + ); } // Resolve a claim winner (a run friendlyId) across both split DBs. Classify diff --git a/apps/webapp/app/runEngine/services/triggerTask.server.ts b/apps/webapp/app/runEngine/services/triggerTask.server.ts index f4981fece16..ff22a479a1d 100644 --- a/apps/webapp/app/runEngine/services/triggerTask.server.ts +++ b/apps/webapp/app/runEngine/services/triggerTask.server.ts @@ -752,7 +752,7 @@ export class RunEngineTriggerTaskService { // Pipeline returned successfully — publish the claim if we held // one. Waiters polling for our key resolve to this runId. if (idempotencyClaim && result?.run?.friendlyId) { - await publishMollifierClaim({ + const published = await publishMollifierClaim({ envId: idempotencyClaim.envId, taskIdentifier: idempotencyClaim.taskIdentifier, idempotencyKey: idempotencyClaim.idempotencyKey, @@ -760,6 +760,17 @@ export class RunEngineTriggerTaskService { runId: result.run.friendlyId, ttlSeconds: env.TRIGGER_MOLLIFIER_CLAIM_TTL_SECONDS, }); + if (!published) { + // Our claim expired mid-pipeline and another claimant took it, so this publish no-op'd: a + // different run is now canonical for this key while we return ours (a cross-DB dup under the + // split). Rare now the claim TTL is floored (C1); surfaced for monitoring pending auto- + // convergence (re-resolve the current winner + cancel this orphan). + logger.warn("mollifier claim publish no-op'd; winner lost the claim mid-pipeline", { + envId: idempotencyClaim.envId, + taskIdentifier: idempotencyClaim.taskIdentifier, + runId: result.run.friendlyId, + }); + } } return result; } catch (err) { diff --git a/apps/webapp/app/v3/mollifier/claimTtl.ts b/apps/webapp/app/v3/mollifier/claimTtl.ts new file mode 100644 index 00000000000..7fc8687cec7 --- /dev/null +++ b/apps/webapp/app/v3/mollifier/claimTtl.ts @@ -0,0 +1,12 @@ +// The claim is a serialization lock that must outlive the winner's create-and-publish pipeline. A short +// customer key TTL must NOT shrink it below a pipeline floor (else the claim expires mid-pipeline and a +// polling loser re-claims → cross-DB duplicate). Floor at `minTtlSeconds` independent of key TTL; cap at max. +export function computeClaimTtlSeconds(input: { + keyExpiresAt: Date; + now: number; + minTtlSeconds: number; + maxTtlSeconds: number; +}): number { + const keyTtlSeconds = Math.ceil((input.keyExpiresAt.getTime() - input.now) / 1000); + return Math.min(input.maxTtlSeconds, Math.max(input.minTtlSeconds, keyTtlSeconds)); +} diff --git a/apps/webapp/app/v3/mollifier/idempotencyClaim.server.ts b/apps/webapp/app/v3/mollifier/idempotencyClaim.server.ts index 196017b438d..191ff62058b 100644 --- a/apps/webapp/app/v3/mollifier/idempotencyClaim.server.ts +++ b/apps/webapp/app/v3/mollifier/idempotencyClaim.server.ts @@ -163,12 +163,14 @@ export async function publishClaim(input: { runId: string; ttlSeconds?: number; buffer?: MollifierBuffer | null; -}): Promise { +}): Promise { const buffer = input.buffer === undefined ? getMollifierBuffer() : input.buffer; - if (!buffer) return; + if (!buffer) return true; const ttlSeconds = input.ttlSeconds ?? DEFAULT_CLAIM_TTL_SECONDS; try { - await buffer.publishClaim({ + // false = compare-and-set no-op: a stale claimant (our TTL expired) already moved in, so our + // publish did NOT set the winner. The caller decides how to converge. + return await buffer.publishClaim({ envId: input.envId, taskIdentifier: input.taskIdentifier, idempotencyKey: input.idempotencyKey, @@ -182,6 +184,8 @@ export async function publishClaim(input: { taskIdentifier: input.taskIdentifier, err: err instanceof Error ? err.message : String(err), }); + // Unknown publish state (transient error) — the claim TTL is the safety net; don't signal a no-op. + return true; } } diff --git a/apps/webapp/app/v3/realtime/resolveBatchForRealtime.server.ts b/apps/webapp/app/v3/realtime/resolveBatchForRealtime.server.ts new file mode 100644 index 00000000000..edd318f9230 --- /dev/null +++ b/apps/webapp/app/v3/realtime/resolveBatchForRealtime.server.ts @@ -0,0 +1,24 @@ +import { prisma } from "~/db.server"; +import { runStore } from "~/v3/runStore.server"; + +type BatchStore = Pick; + +// The realtime batch route reads the batch client-less (replica), which can miss a just-created batch +// under replica lag. `shouldRetryNotFound` covers the zodfetch GET, but the Electric ShapeStream +// consumer (self-hosters) ignores `x-should-retry`, so re-read the owning primary on a miss — passing a +// non-replica writer flips each store leg to its own primary — to avoid a permanent 404. +export function resolveBatchTaskRunForRealtime( + friendlyId: string, + environmentId: string, + deps?: { store?: BatchStore; writer?: unknown } +) { + const store = deps?.store ?? runStore; + const writer = deps?.writer ?? prisma; + return store + .findBatchTaskRunByFriendlyId(friendlyId, environmentId) + .then( + (onReplica) => + onReplica ?? + store.findBatchTaskRunByFriendlyId(friendlyId, environmentId, undefined, writer as never) + ); +} diff --git a/apps/webapp/test/mollifierIdempotencyClaim.test.ts b/apps/webapp/test/mollifierIdempotencyClaim.test.ts index 1b093250f1e..84083087a4b 100644 --- a/apps/webapp/test/mollifierIdempotencyClaim.test.ts +++ b/apps/webapp/test/mollifierIdempotencyClaim.test.ts @@ -181,13 +181,13 @@ describe("publishClaim", () => { expect(buffer.publishClaim).toHaveBeenCalledOnce(); }); - it("no-op when buffer is null", async () => { + it("no buffer → true (nothing to converge, not a no-op'd publish)", async () => { await expect( publishClaim({ ...baseInput, token: "owner-token", runId: "run_X", buffer: null }) - ).resolves.toBeUndefined(); + ).resolves.toBe(true); }); - it("swallows errors so trigger pipeline isn't broken by Redis hiccups", async () => { + it("swallows errors so trigger pipeline isn't broken by Redis hiccups (returns true, unknown state)", async () => { const buffer = { publishClaim: vi.fn(async () => { throw new Error("ECONNREFUSED"); @@ -195,7 +195,7 @@ describe("publishClaim", () => { } as unknown as MollifierBuffer; await expect( publishClaim({ ...baseInput, token: "owner-token", runId: "run_X", buffer }) - ).resolves.toBeUndefined(); + ).resolves.toBe(true); }); }); diff --git a/internal-packages/run-store/src/runOpsStore.residencyMismatchFallback.test.ts b/internal-packages/run-store/src/runOpsStore.residencyMismatchFallback.test.ts deleted file mode 100644 index 8897c295662..00000000000 --- a/internal-packages/run-store/src/runOpsStore.residencyMismatchFallback.test.ts +++ /dev/null @@ -1,296 +0,0 @@ -// RED→GREEN lock for RoutingRunStore.findRun's ON-MISS FAN-OUT for a CLASSIFIABLE id. -// -// THE BUG: findRun for a classifiable id routed to the SINGLE owning store (by id-shape -// classification) and returned whatever it gave — no on-miss fallback. When a run's PHYSICAL -// residency does not match its id-shape classification (e.g. a pre-#4154 27-char base62 run that -// lives on the NEW store but now classifies LEGACY), findRun routed to the wrong store, missed, -// and returned null → a spurious 404 — even though the run is physically present on the OTHER DB -// (and runs.list surfaces it). The unclassifiable path already fanned out NEW→LEGACY; this makes -// the classifiable path equally robust. -// -// Uses the REAL two-physical-DB split (heteroRunOpsPostgresTest: prisma14 = full/legacy on PG14, -// prisma17 = dedicated run-ops subset on PG17). NEVER mocked. The residency/classification MISMATCH -// is simulated deterministically by injecting a custom `classify` fn into the RoutingRunStore -// constructor — the physical row is written to the NEW store while `classify` reports its id LEGACY. - -import { heteroRunOpsPostgresTest } from "@internal/testcontainers"; -import { ownerEngine, type Residency } from "@trigger.dev/core/v3/isomorphic"; -import type { PrismaClient } from "@trigger.dev/database"; -import type { RunOpsPrismaClient } from "@internal/run-ops-database"; -import { describe, expect } from "vitest"; -import { PostgresRunStore } from "./PostgresRunStore.js"; -import { RoutingRunStore } from "./runOpsStore.js"; -import type { CreateRunInput, RunStore } from "./types.js"; - -// ownerEngine classifies by internal-id LENGTH/version char: 25 chars → cuid → LEGACY, -// a v1 body (version "1" at index 25) → run-ops id → NEW. -function cuidLegacy(seed: string): string { - return (seed + "c".repeat(25)).slice(0, 25); -} -function runOpsNew(seed: string): string { - return (seed.replace(/[^0-9a-v]/g, "0") + "k".repeat(24)).slice(0, 24) + "01"; -} - -function makeDedicatedStore(prisma17: RunOpsPrismaClient) { - return new PostgresRunStore({ - prisma: prisma17 as never, - readOnlyPrisma: prisma17 as never, - schemaVariant: "dedicated", - }); -} - -function makeLegacyStore(prisma14: PrismaClient) { - return new PostgresRunStore({ - prisma: prisma14, - readOnlyPrisma: prisma14, - schemaVariant: "legacy", - }); -} - -// Wrap a real store so findRun/findRunOnPrimary calls are COUNTED while every method still delegates -// to the REAL PostgresRunStore (this is instrumentation, not a behavior mock — the underlying reads, -// writes, getters all run for real). Lets us assert the FAST PATH does not touch the other store. -function countingReads( - inner: RunStore, - counts: { findRun: number; findRunOnPrimary: number } -): RunStore { - return new Proxy(inner, { - get(target, prop) { - // Read via target[prop] so getters (e.g. primaryReadClient) run with `this` = the real store. - const value = (target as unknown as Record)[prop]; - if (typeof value !== "function") return value; - if (prop === "findRun" || prop === "findRunOnPrimary") { - return (...args: unknown[]) => { - counts[prop as "findRun" | "findRunOnPrimary"] += 1; - return (value as (...a: unknown[]) => unknown).apply(target, args); - }; - } - return (value as (...a: unknown[]) => unknown).bind(target); - }, - }) as unknown as RunStore; -} - -async function seedLegacyEnvironment(prisma14: PrismaClient, suffix: string) { - const organization = await prisma14.organization.create({ - data: { title: `Org ${suffix}`, slug: `org-${suffix}` }, - }); - const project = await prisma14.project.create({ - data: { - name: `Project ${suffix}`, - slug: `project-${suffix}`, - externalRef: `proj_${suffix}`, - organizationId: organization.id, - }, - }); - const environment = await prisma14.runtimeEnvironment.create({ - data: { - type: "DEVELOPMENT", - slug: "dev", - projectId: project.id, - organizationId: organization.id, - apiKey: `tr_dev_${suffix}`, - pkApiKey: `pk_dev_${suffix}`, - shortcode: `short_${suffix}`, - }, - }); - return { - organizationId: organization.id, - projectId: project.id, - runtimeEnvironmentId: environment.id, - environmentId: environment.id, - }; -} - -function buildCreateRunInput(params: { - runId: string; - friendlyId: string; - organizationId: string; - projectId: string; - runtimeEnvironmentId: string; -}): CreateRunInput { - return { - data: { - id: params.runId, - engine: "V2", - status: "PENDING", - friendlyId: params.friendlyId, - runtimeEnvironmentId: params.runtimeEnvironmentId, - environmentType: "DEVELOPMENT", - organizationId: params.organizationId, - projectId: params.projectId, - taskIdentifier: "my-task", - payload: "{}", - payloadType: "application/json", - traceContext: {}, - traceId: `trace_${params.runId}`, - spanId: `span_${params.runId}`, - queue: "task/my-task", - isTest: false, - taskEventStore: "taskEvent", - depth: 0, - }, - snapshot: { - engine: "V2", - executionStatus: "RUN_CREATED", - description: "Run was created", - runStatus: "PENDING", - environmentId: params.runtimeEnvironmentId, - environmentType: "DEVELOPMENT", - projectId: params.projectId, - organizationId: params.organizationId, - }, - }; -} - -// Insert a TaskRun row DIRECTLY onto the NEW (dedicated) store, bypassing routing, so we can force a -// residency/classification MISMATCH: the row is physically on #new while `classify` calls its id LEGACY. -async function insertRunOnNewStore( - prisma17: RunOpsPrismaClient, - params: { - runId: string; - friendlyId: string; - environmentId: string; - organizationId: string; - projectId: string; - } -) { - await prisma17.taskRun.create({ - data: { - id: params.runId, - engine: "V2", - status: "PENDING", - friendlyId: params.friendlyId, - runtimeEnvironmentId: params.environmentId, - environmentType: "DEVELOPMENT", - organizationId: params.organizationId, - projectId: params.projectId, - taskIdentifier: "my-task", - payload: "{}", - payloadType: "application/json", - traceContext: {}, - traceId: `trace_${params.runId}`, - spanId: `span_${params.runId}`, - queue: "task/my-task", - isTest: false, - taskEventStore: "taskEvent", - depth: 0, - }, - }); -} - -describe("RoutingRunStore.findRun — on-miss fan-out for a classifiable id (residency ≠ classification)", () => { - // ── THE BUG: a run physically on #new whose id classifies LEGACY must still be found ── - // Without the on-miss fallback, findRun routes to #legacy (per classification), misses, returns null. - heteroRunOpsPostgresTest( - "returns a #new-resident run whose id classifies LEGACY (owning-store miss → other-store fallback)", - async ({ prisma14, prisma17 }) => { - const env = await seedLegacyEnvironment(prisma14, "mm1"); - const newStore = makeDedicatedStore(prisma17); - const legacyStore = makeLegacyStore(prisma14); - - // A run-ops-shaped id (so ownerEngine would say NEW), but we FORCE classify → LEGACY to model - // a residency/classification mismatch: physically on #new, classified LEGACY. - const mismatchId = runOpsNew("mm1"); - const classify = (id: string): Residency => (id === mismatchId ? "LEGACY" : ownerEngine(id)); - const router = new RoutingRunStore({ new: newStore, legacy: legacyStore, classify }); - - await insertRunOnNewStore(prisma17, { - runId: mismatchId, - friendlyId: "run_mm1", - environmentId: env.environmentId, - organizationId: env.organizationId, - projectId: env.projectId, - }); - - // Physical residency sanity: on #new only. - expect(await prisma17.taskRun.findUnique({ where: { id: mismatchId } })).not.toBeNull(); - expect(await prisma14.taskRun.findUnique({ where: { id: mismatchId } })).toBeNull(); - - // classify → LEGACY routes to #legacy (miss); the fix falls back to #new and finds the run. - const byId = (await router.findRun({ id: mismatchId }, { select: { id: true } })) as Record< - string, - unknown - > | null; - expect(byId?.id).toBe(mismatchId); - - // Same on the read-your-writes primary variant (a caller-passed writer → findRunOnPrimary). - const byIdPrimary = (await router.findRun( - { id: mismatchId }, - { select: { id: true } }, - prisma14 - )) as Record | null; - expect(byIdPrimary?.id).toBe(mismatchId); - } - ); - - // ── FAST PATH: a run found in its CLASSIFIED store is a SINGLE read (no second-store probe) ── - heteroRunOpsPostgresTest( - "does NOT read the other store when the classified (owning) store hits", - async ({ prisma14, prisma17 }) => { - const env = await seedLegacyEnvironment(prisma14, "mm2"); - - // NEW-resident run-ops-id run: owning store = #new. Wrap #legacy to catch any stray probe. - const newCounts = { findRun: 0, findRunOnPrimary: 0 }; - const legacyCounts = { findRun: 0, findRunOnPrimary: 0 }; - const newStore = countingReads(makeDedicatedStore(prisma17), newCounts); - const legacyStore = countingReads(makeLegacyStore(prisma14), legacyCounts); - const router = new RoutingRunStore({ new: newStore, legacy: legacyStore }); - - const newId = runOpsNew("mm2n"); // classifies NEW - await router.createRun( - buildCreateRunInput({ - runId: newId, - friendlyId: "run_mm2_new", - organizationId: env.organizationId, - projectId: env.projectId, - runtimeEnvironmentId: env.runtimeEnvironmentId, - }) - ); - - const hit = (await router.findRun({ id: newId }, { select: { id: true } })) as Record< - string, - unknown - > | null; - expect(hit?.id).toBe(newId); - // Owning store read exactly once; the other store NOT touched (fast path preserved). - expect(newCounts.findRun).toBe(1); - expect(legacyCounts.findRun).toBe(0); - expect(legacyCounts.findRunOnPrimary).toBe(0); - - // Symmetric: a cuid run whose owning store is #legacy must not probe #new on a hit. - const legacyId = cuidLegacy("mm2l"); // classifies LEGACY - await router.createRun( - buildCreateRunInput({ - runId: legacyId, - friendlyId: "run_mm2_legacy", - organizationId: env.organizationId, - projectId: env.projectId, - runtimeEnvironmentId: env.runtimeEnvironmentId, - }) - ); - newCounts.findRun = 0; - legacyCounts.findRun = 0; - - const hitLegacy = (await router.findRun( - { id: legacyId }, - { select: { id: true } } - )) as Record | null; - expect(hitLegacy?.id).toBe(legacyId); - expect(legacyCounts.findRun).toBe(1); - expect(newCounts.findRun).toBe(0); - } - ); - - // ── A genuine miss on BOTH stores still returns null (fan-out exhausted) ── - heteroRunOpsPostgresTest( - "returns null when the run is on neither store", - async ({ prisma14, prisma17 }) => { - const newStore = makeDedicatedStore(prisma17); - const legacyStore = makeLegacyStore(prisma14); - const router = new RoutingRunStore({ new: newStore, legacy: legacyStore }); - expect( - await router.findRun({ id: cuidLegacy("ghost") }, { select: { id: true } }) - ).toBeNull(); - } - ); -}); diff --git a/internal-packages/run-store/src/runOpsStore.ts b/internal-packages/run-store/src/runOpsStore.ts index 56f6addebdf..d14ae525224 100644 --- a/internal-packages/run-store/src/runOpsStore.ts +++ b/internal-packages/run-store/src/runOpsStore.ts @@ -229,9 +229,11 @@ export class RoutingRunStore implements RunStore { const onPrimary = readYourWrites(argsOrClient, _client); const id = idFromWhere(where); if (id !== undefined) { - // Residency-classifiable (id/friendlyId): route to the owning store, then fall back to the - // OTHER store on a miss. - return this.#findRunRouted(id, where, args, onPrimary); + // Residency-classifiable (id/friendlyId): id-shape is destiny for a run's whole life — a run-ops + // id lives on NEW, a cuid on LEGACY — so read the one owning store (no cross-store fan-out). + const store = this.#routeOrNew(id); + const method = onPrimary ? "findRunOnPrimary" : "findRun"; + return (store[method] as (...rest: unknown[]) => Promise)(where, args); } // Unclassifiable where (e.g. spanId, idempotencyKey): the run may live on either DB, // so fan out NEW-first then LEGACY rather than defaulting to NEW — defaulting silently @@ -239,31 +241,6 @@ export class RoutingRunStore implements RunStore { return this.#findRunUnrouted(where, args, onPrimary); } - // A classifiable id names its OWNING store by id-shape, but physical residency can diverge from - // classification (a pre-#4154 base62 run lives on #new yet classifies LEGACY). Read the owning - // store first — a hit is a SINGLE read (the fast path) — then, ONLY on a miss, probe the OTHER - // store before returning null, so a run whose residency ≠ its id-shape is found rather than 404'd. - // Both legs run the SAME index-covered TaskRun lookup; the fan-out cost is paid only on the (rare) - // miss. Mirrors #findRunUnrouted's shape but keyed on the classified owner. - async #findRunRouted( - id: string, - where: Prisma.TaskRunWhereInput, - args: unknown, - onPrimary: boolean - ): Promise { - const method = onPrimary ? "findRunOnPrimary" : "findRun"; - const owning = this.#routeOrNew(id); - const fromOwning = await (owning[method] as (...rest: unknown[]) => Promise)( - where, - args - ); - if (fromOwning != null) { - return fromOwning; - } - const other = owning === this.#new ? this.#legacy : this.#new; - return (other[method] as (...rest: unknown[]) => Promise)(where, args); - } - async #findRunUnrouted( where: Prisma.TaskRunWhereInput, args: unknown, From 82b42afb257fe10c0b0c109de51d327295a7483f Mon Sep 17 00:00:00 2001 From: Dan Sutton Date: Sat, 18 Jul 2026 17:25:51 +0100 Subject: [PATCH 4/6] test(webapp,run-engine,run-store): caller-driven replica-lag + idempotency guards Guards for the production fixes in the base PR. Each fixed site has a caller-driven test that drives the real exported route/presenter/service/engine caller against a real Postgres with the owning replica frozen (shared laggingReplica primitive), and goes RED when the fix is reverted; tolerated read-view sites carry a caller-driven GREEN proof of self-healing. The global-scope idempotency guard drives the real dedup + claim path through a real MollifierBuffer over a Redis testcontainer, incl. the cross-DB andWait and the expired/failed clear-and-recreate reacquire cases. --- .../test/batchServices.replicaLag.test.ts | 679 +++++++++++ .../test/bulkActionV2.replicaLag.test.ts | 205 ++++ .../test/cancelRouteReplicaLag.guard.test.ts | 266 +++++ ...ngineReplicaReads.replicaLag.guard.test.ts | 289 +++++ ...EnvironmentFromRunReplicaLag.guard.test.ts | 196 ++++ ...otencyGlobalScopeCrossDbConcurrent.test.ts | 930 +++++++++++++++ ...mpotencyResetRouteReplicaLag.guard.test.ts | 276 +++++ .../metadataRouteReplicaLag.guard.test.ts | 261 +++++ ...entersSessionBatchReplicaLag.guard.test.ts | 358 ++++++ .../test/readRunForEvent.replicaLag.test.ts | 198 ++++ .../test/realtimeServices.replicaLag.test.ts | 555 +++++++++ ...meSessionsIoRoute.replicaLag.guard.test.ts | 263 +++++ .../realtimeStreamRoutes.replicaLag.test.ts | 707 ++++++++++++ .../test/replayRouteReplicaLag.guard.test.ts | 261 +++++ .../routesBatchGetReplicaLag.guard.test.ts | 397 +++++++ .../runGetRoutes.replicaLag.guard.test.ts | 1021 +++++++++++++++++ .../test/runPresenters.replicaLag.test.ts | 533 +++++++++ .../runsRepositoryConvert.replicaLag.test.ts | 152 +++ ...onWaitpointRoutes.replicaLag.guard.test.ts | 351 ++++++ .../test/spanTraceRoutes.replicaLag.test.ts | 545 +++++++++ ...pointCallbackRouteReplicaLag.guard.test.ts | 193 ++++ ...pointCompleteRouteReplicaLag.guard.test.ts | 251 ++++ ...itpointPresenters.replicaLag.guard.test.ts | 568 +++++++++ ...yDecisionReadAfterWrite.replicaLag.test.ts | 185 +++ .../ttlSystemExpireReplicaLag.guard.test.ts | 154 +++ ...ncySweeperCallbackReplicaLag.guard.test.ts | 122 ++ .../runAttemptSystemReplicaLag.guard.test.ts | 516 +++++++++ ...ependentAttemptReadView.replicaLag.test.ts | 251 ++++ ...atchIdempotencyDedupReadAfterWrite.test.ts | 161 +++ ...tore.bulkActionReadView.replicaLag.test.ts | 253 ++++ ...cancelRunReadAfterWrite.replicaLag.test.ts | 221 ++++ ....dashboardAgentReadView.replicaLag.test.ts | 224 ++++ ...psStore.engineReadViews.replicaLag.test.ts | 359 ++++++ ...tore.idempotencyGlobalScopeCrossDb.test.ts | 328 ++++++ ...dempotencyResetReadView.replicaLag.test.ts | 225 ++++ ...modelRuntimeEnvReadView.replicaLag.test.ts | 183 +++ ...e.presentersRunReadView.replicaLag.test.ts | 435 +++++++ ...ersSessionBatchReadView.replicaLag.test.ts | 344 ++++++ ...entersWaitpointReadView.replicaLag.test.ts | 516 +++++++++ ...ealtimeServicesReadView.replicaLag.test.ts | 356 ++++++ ...re.replayReadAfterWrite.replicaLag.test.ts | 211 ++++ ...re.resolveRunForMutationReplicaLag.test.ts | 200 ++++ ....routesBatchGetReadView.replicaLag.test.ts | 312 +++++ ...sRealtimeStreamReadView.replicaLag.test.ts | 590 ++++++++++ ...re.routesRunGetReadView.replicaLag.test.ts | 664 +++++++++++ ...routesSpanTraceReadView.replicaLag.test.ts | 322 ++++++ ...viceBatchFamilyReadView.replicaLag.test.ts | 346 ++++++ ...onMetadataRouteReadView.replicaLag.test.ts | 231 ++++ ...tore.sessionRunProbeReadAfterWrite.test.ts | 211 ++++ ...e.storeReceiverReadView.replicaLag.test.ts | 288 +++++ .../run-store/src/runOpsStore.test.ts | 27 +- ...OpsStore.ttlExpireOrphanReplicaLag.test.ts | 222 ++++ ....usageCostUndercountReadAfterWrite.test.ts | 253 ++++ ...AndReplayLoaderReadView.replicaLag.test.ts | 246 ++++ ...itpointCompleteTokenReadAfterWrite.test.ts | 175 +++ ...Store.waitpointDedupReadAfterWrite.test.ts | 288 +++++ 56 files changed, 18863 insertions(+), 11 deletions(-) create mode 100644 apps/webapp/test/batchServices.replicaLag.test.ts create mode 100644 apps/webapp/test/bulkActionV2.replicaLag.test.ts create mode 100644 apps/webapp/test/cancelRouteReplicaLag.guard.test.ts create mode 100644 apps/webapp/test/engineReplicaReads.replicaLag.guard.test.ts create mode 100644 apps/webapp/test/findEnvironmentFromRunReplicaLag.guard.test.ts create mode 100644 apps/webapp/test/idempotencyGlobalScopeCrossDbConcurrent.test.ts create mode 100644 apps/webapp/test/idempotencyResetRouteReplicaLag.guard.test.ts create mode 100644 apps/webapp/test/metadataRouteReplicaLag.guard.test.ts create mode 100644 apps/webapp/test/presentersSessionBatchReplicaLag.guard.test.ts create mode 100644 apps/webapp/test/readRunForEvent.replicaLag.test.ts create mode 100644 apps/webapp/test/realtimeServices.replicaLag.test.ts create mode 100644 apps/webapp/test/realtimeSessionsIoRoute.replicaLag.guard.test.ts create mode 100644 apps/webapp/test/realtimeStreamRoutes.replicaLag.test.ts create mode 100644 apps/webapp/test/replayRouteReplicaLag.guard.test.ts create mode 100644 apps/webapp/test/routesBatchGetReplicaLag.guard.test.ts create mode 100644 apps/webapp/test/runGetRoutes.replicaLag.guard.test.ts create mode 100644 apps/webapp/test/runPresenters.replicaLag.test.ts create mode 100644 apps/webapp/test/runsRepositoryConvert.replicaLag.test.ts create mode 100644 apps/webapp/test/sessionWaitpointRoutes.replicaLag.guard.test.ts create mode 100644 apps/webapp/test/spanTraceRoutes.replicaLag.test.ts create mode 100644 apps/webapp/test/waitpointCallbackRouteReplicaLag.guard.test.ts create mode 100644 apps/webapp/test/waitpointCompleteRouteReplicaLag.guard.test.ts create mode 100644 apps/webapp/test/waitpointPresenters.replicaLag.guard.test.ts create mode 100644 internal-packages/run-engine/src/engine/retryDecisionReadAfterWrite.replicaLag.test.ts create mode 100644 internal-packages/run-engine/src/engine/systems/ttlSystemExpireReplicaLag.guard.test.ts create mode 100644 internal-packages/run-engine/src/engine/tests/concurrencySweeperCallbackReplicaLag.guard.test.ts create mode 100644 internal-packages/run-engine/src/engine/tests/runAttemptSystemReplicaLag.guard.test.ts create mode 100644 internal-packages/run-store/src/runOpsStore.batchDependentAttemptReadView.replicaLag.test.ts create mode 100644 internal-packages/run-store/src/runOpsStore.batchIdempotencyDedupReadAfterWrite.test.ts create mode 100644 internal-packages/run-store/src/runOpsStore.bulkActionReadView.replicaLag.test.ts create mode 100644 internal-packages/run-store/src/runOpsStore.cancelRunReadAfterWrite.replicaLag.test.ts create mode 100644 internal-packages/run-store/src/runOpsStore.dashboardAgentReadView.replicaLag.test.ts create mode 100644 internal-packages/run-store/src/runOpsStore.engineReadViews.replicaLag.test.ts create mode 100644 internal-packages/run-store/src/runOpsStore.idempotencyGlobalScopeCrossDb.test.ts create mode 100644 internal-packages/run-store/src/runOpsStore.idempotencyResetReadView.replicaLag.test.ts create mode 100644 internal-packages/run-store/src/runOpsStore.modelRuntimeEnvReadView.replicaLag.test.ts create mode 100644 internal-packages/run-store/src/runOpsStore.presentersRunReadView.replicaLag.test.ts create mode 100644 internal-packages/run-store/src/runOpsStore.presentersSessionBatchReadView.replicaLag.test.ts create mode 100644 internal-packages/run-store/src/runOpsStore.presentersWaitpointReadView.replicaLag.test.ts create mode 100644 internal-packages/run-store/src/runOpsStore.realtimeServicesReadView.replicaLag.test.ts create mode 100644 internal-packages/run-store/src/runOpsStore.replayReadAfterWrite.replicaLag.test.ts create mode 100644 internal-packages/run-store/src/runOpsStore.resolveRunForMutationReplicaLag.test.ts create mode 100644 internal-packages/run-store/src/runOpsStore.routesBatchGetReadView.replicaLag.test.ts create mode 100644 internal-packages/run-store/src/runOpsStore.routesRealtimeStreamReadView.replicaLag.test.ts create mode 100644 internal-packages/run-store/src/runOpsStore.routesRunGetReadView.replicaLag.test.ts create mode 100644 internal-packages/run-store/src/runOpsStore.routesSpanTraceReadView.replicaLag.test.ts create mode 100644 internal-packages/run-store/src/runOpsStore.serviceBatchFamilyReadView.replicaLag.test.ts create mode 100644 internal-packages/run-store/src/runOpsStore.sessionMetadataRouteReadView.replicaLag.test.ts create mode 100644 internal-packages/run-store/src/runOpsStore.sessionRunProbeReadAfterWrite.test.ts create mode 100644 internal-packages/run-store/src/runOpsStore.storeReceiverReadView.replicaLag.test.ts create mode 100644 internal-packages/run-store/src/runOpsStore.ttlExpireOrphanReplicaLag.test.ts create mode 100644 internal-packages/run-store/src/runOpsStore.usageCostUndercountReadAfterWrite.test.ts create mode 100644 internal-packages/run-store/src/runOpsStore.waitpointCompleteRouteAndReplayLoaderReadView.replicaLag.test.ts create mode 100644 internal-packages/run-store/src/runOpsStore.waitpointCompleteTokenReadAfterWrite.test.ts create mode 100644 internal-packages/run-store/src/runOpsStore.waitpointDedupReadAfterWrite.test.ts diff --git a/apps/webapp/test/batchServices.replicaLag.test.ts b/apps/webapp/test/batchServices.replicaLag.test.ts new file mode 100644 index 00000000000..cae6ab053ba --- /dev/null +++ b/apps/webapp/test/batchServices.replicaLag.test.ts @@ -0,0 +1,679 @@ +// Replica-lag guards for the batch-service reads. +// +// Each case drives the REAL exported caller (a webapp service method that contains the run-store +// read) against a real Postgres testcontainer, with the store's read replica FROZEN via the shared +// `laggingReplica` primitive so a replica-routed read misses the just-written row while the primary +// still holds it. Only deps orthogonal to the read path are mocked (engine enqueue/count, env +// resolution, object-store download, id minting, control-plane env assertion, worker enqueue). +// +// Reads under guard (PostgresRunStore client-less defaults): +// findBatchTaskRunById / findBatchTaskRunByIdempotencyKey / countBatchTaskRunItems → PRIMARY +// findTaskRunAttempt → REPLICA (client ?? this.readOnlyPrisma) +// +// Property: the PRIMARY-routed reads observe the live row under lag (asserted on output) and +// wasHit() === false proves they never touched the frozen replica; the dependent-attempt +// read is threaded the primary client so it still sees a live terminal parent and the "parent +// already in a terminal state" guard fires. + +import { heteroPostgresTest, laggingReplica } from "@internal/testcontainers"; +import { PostgresRunStore } from "@internal/run-store"; +import type { PrismaClient } from "@trigger.dev/database"; +import { BatchId } from "@trigger.dev/core/v3/isomorphic"; +import { describe, expect, vi } from "vitest"; + +vi.setConfig({ testTimeout: 60_000, hookTimeout: 60_000 }); + +// ---- Module mocks orthogonal to the run-store read path. ----------------------------------------- +const dbHolder = vi.hoisted(() => ({ prisma: undefined as any })); +vi.mock("~/db.server", () => ({ + get prisma() { + return dbHolder.prisma; + }, + get $replica() { + return dbHolder.prisma; + }, +})); +// Prevent the run engine + run-store singletons from constructing at import; every caller here is +// driven with an explicitly-injected store/engine, so these defaults are never used. +vi.mock("~/v3/runEngine.server", () => ({ engine: {} })); +vi.mock("~/v3/runStore.server", () => ({ runStore: {} })); +vi.mock("~/v3/batchTriggerWorker.server", () => ({ + batchTriggerWorker: { enqueue: vi.fn(async () => {}) }, +})); +vi.mock("~/services/platform.v3.server", async (importOriginal) => ({ + ...((await importOriginal()) as Record), + getEntitlement: vi.fn(async () => ({ hasAccess: true })), +})); +// Env resolution is downstream of the batch read in both processBatchTaskRun callers. +vi.mock("~/models/runtimeEnvironment.server", () => ({ + findEnvironmentById: vi.fn(async () => null), +})); +// Control-plane env existence assertion (batchTriggerV3.call preamble) — orthogonal to the reads. +vi.mock("~/v3/runOpsMigration/controlPlaneResolver.server", () => ({ + controlPlaneResolver: { assertEnvExists: vi.fn(async () => {}) }, +})); +// Batch friendly-id minting — deterministic id so the created batch is predictable. +const mintHolder = vi.hoisted(() => ({ id: "", friendlyId: "" })); +vi.mock("~/v3/runOpsMigration/mintBatchFriendlyId.server", () => ({ + mintBatchFriendlyId: vi.fn(async () => ({ + id: mintHolder.id, + friendlyId: mintHolder.friendlyId, + })), +})); +// Object-store payload download for the existing-batch response (idempotency dedup path). +const objHolder = vi.hoisted(() => ({ + packet: { data: "[]", dataType: "application/json" } as { data?: string; dataType: string }, +})); +vi.mock("~/v3/objectStore.server", () => ({ + downloadPacketFromObjectStore: vi.fn(async () => objHolder.packet), + uploadPacketToObjectStore: vi.fn(async () => "uploaded"), +})); + +import { findEnvironmentById } from "~/models/runtimeEnvironment.server"; +// REAL callers under guard. +import { + BatchTriggerV3Service, + tryCompleteBatchV3, +} from "../app/v3/services/batchTriggerV3.server"; +import { StreamBatchItemsService } from "../app/runEngine/services/streamBatchItems.server"; +import { RunEngineBatchTriggerService } from "../app/runEngine/services/batchTrigger.server"; + +let seq = 0; + +async function seedTenant(prisma: PrismaClient, suffix: string) { + const organization = await prisma.organization.create({ + data: { title: `Org ${suffix}`, slug: `org-${suffix}` }, + }); + const project = await prisma.project.create({ + data: { + name: `Project ${suffix}`, + slug: `project-${suffix}`, + externalRef: `proj_${suffix}`, + organizationId: organization.id, + }, + }); + const environment = await prisma.runtimeEnvironment.create({ + data: { + type: "DEVELOPMENT", + slug: "dev", + projectId: project.id, + organizationId: organization.id, + apiKey: `tr_dev_${suffix}`, + pkApiKey: `pk_dev_${suffix}`, + shortcode: `short_${suffix}`, + }, + }); + return { organization, project, environment }; +} + +// A PostgresRunStore whose PRIMARY is the live container and whose read replica is frozen for the +// given models (missing mode → replica-routed reads return null/[]/0). `wasHit(model)` proves whether +// a read actually consulted the frozen replica. +function storeWithFrozenReplica(prisma: PrismaClient, models: string[]) { + const replica = laggingReplica( + prisma, + models.map((model) => ({ model, mode: "missing" as const })) + ); + const store = new PostgresRunStore({ + prisma, + readOnlyPrisma: replica.client, + schemaVariant: "legacy", + }); + return { store, replica }; +} + +async function createBatchOnPrimary( + prisma: PrismaClient, + params: { + id: string; + friendlyId: string; + runtimeEnvironmentId: string; + runCount: number; + status?: "PENDING" | "PROCESSING" | "COMPLETED" | "PARTIAL_FAILED" | "ABORTED"; + sealed?: boolean; + expectedCount?: number; + runIds?: string[]; + idempotencyKey?: string; + idempotencyKeyExpiresAt?: Date | null; + payload?: string; + payloadType?: string; + processingCompletedAt?: Date | null; + } +) { + return prisma.batchTaskRun.create({ + data: { + id: params.id, + friendlyId: params.friendlyId, + runtimeEnvironmentId: params.runtimeEnvironmentId, + runCount: params.runCount, + expectedCount: params.expectedCount ?? params.runCount, + status: params.status ?? "PENDING", + sealed: params.sealed ?? false, + runIds: params.runIds ?? [], + batchVersion: "v3", + idempotencyKey: params.idempotencyKey ?? null, + idempotencyKeyExpiresAt: params.idempotencyKeyExpiresAt ?? null, + payload: params.payload, + payloadType: params.payloadType ?? "application/json", + processingCompletedAt: params.processingCompletedAt ?? null, + }, + }); +} + +// Seed a real TaskRun + a COMPLETED TaskRunAttempt (with its required FK graph) on the PRIMARY, whose +// parent run is in a FINAL status. Used to prove the dependent-attempt guard fires on a live row. +async function seedTerminalAttempt( + prisma: PrismaClient, + seed: { organization: { id: string }; project: { id: string }; environment: { id: string } }, + suffix: string +): Promise<{ attemptFriendlyId: string }> { + const envId = seed.environment.id; + const projectId = seed.project.id; + + const run = await prisma.taskRun.create({ + data: { + friendlyId: `run_parent_${suffix}`, + taskIdentifier: "parent-task", + payload: "{}", + payloadType: "application/json", + traceId: `t_${suffix}`, + spanId: `s_${suffix}`, + runtimeEnvironmentId: envId, + projectId, + organizationId: seed.organization.id, + environmentType: "DEVELOPMENT", + queue: "task/parent-task", + status: "COMPLETED_SUCCESSFULLY", + }, + }); + + const worker = await prisma.backgroundWorker.create({ + data: { + friendlyId: `worker_${suffix}`, + contentHash: `hash_${suffix}`, + projectId, + runtimeEnvironmentId: envId, + version: "20240101.1", + metadata: {}, + }, + }); + const queue = await prisma.taskQueue.create({ + data: { + friendlyId: `queue_${suffix}`, + name: "task/parent-task", + projectId, + runtimeEnvironmentId: envId, + }, + }); + const workerTask = await prisma.backgroundWorkerTask.create({ + data: { + friendlyId: `wt_${suffix}`, + slug: "parent-task", + filePath: "src/trigger/parent.ts", + workerId: worker.id, + projectId, + runtimeEnvironmentId: envId, + queueId: queue.id, + }, + }); + const attemptFriendlyId = `attempt_${suffix}`; + await prisma.taskRunAttempt.create({ + data: { + friendlyId: attemptFriendlyId, + number: 1, + taskRunId: run.id, + backgroundWorkerId: worker.id, + backgroundWorkerTaskId: workerTask.id, + runtimeEnvironmentId: envId, + queueId: queue.id, + status: "COMPLETED", + }, + }); + return { attemptFriendlyId }; +} + +async function* emptyItems() { + // no items +} + +// Minimal AuthenticatedEnvironment shape the traced callers read (attributesFromAuthenticatedEnv + +// the env-ownership check). Not the run-store read path — purely the tracing/ownership periphery. +function authEnv(seed: { + organization: { id: string; slug: string; title: string }; + project: { id: string; name: string; slug: string }; + environment: { id: string }; +}) { + return { + id: seed.environment.id, + type: "DEVELOPMENT", + slug: "dev", + organizationId: seed.organization.id, + projectId: seed.project.id, + organization: { + id: seed.organization.id, + slug: seed.organization.slug, + title: seed.organization.title, + featureFlags: {}, + }, + project: { id: seed.project.id, name: seed.project.name, slug: seed.project.slug }, + }; +} + +describe("batch-svc — run-ops replica-lag guards", () => { + // ================================================================================================ + // tryCompleteBatchV3 — findBatchTaskRunById + countBatchTaskRunItems (both PRIMARY-routed) + // With the batch + its items frozen on the replica, the completion function reads both on the + // primary, sees the sealed+fully-completed batch, and transitions it to COMPLETED. + // ================================================================================================ + heteroPostgresTest( + "tryCompleteBatchV3 completes a sealed batch whose row+items have not replicated (reads primary)", + async ({ prisma14 }) => { + const prisma = prisma14 as unknown as PrismaClient; + dbHolder.prisma = prisma; + const suffix = `trycomplete_${seq++}`; + const seed = await seedTenant(prisma, suffix); + + const { store, replica } = storeWithFrozenReplica(prisma, [ + "batchTaskRun", + "batchTaskRunItem", + ]); + + const { id: batchId, friendlyId } = BatchId.generate(); + await createBatchOnPrimary(prisma, { + id: batchId, + friendlyId, + runtimeEnvironmentId: seed.environment.id, + runCount: 2, + expectedCount: 2, + status: "PENDING", + sealed: true, + }); + + // Two COMPLETED items on the primary (each needs a real TaskRun FK). + for (const n of [1, 2]) { + const run = await prisma.taskRun.create({ + data: { + friendlyId: `run_${suffix}_${n}`, + taskIdentifier: "child-task", + payload: "{}", + payloadType: "application/json", + traceId: `t_${suffix}_${n}`, + spanId: `s_${suffix}_${n}`, + runtimeEnvironmentId: seed.environment.id, + projectId: seed.project.id, + organizationId: seed.organization.id, + environmentType: "DEVELOPMENT", + queue: "task/child-task", + status: "COMPLETED_SUCCESSFULLY", + }, + }); + await prisma.batchTaskRunItem.create({ + data: { batchTaskRunId: batchId, taskRunId: run.id, status: "COMPLETED" }, + }); + } + + await tryCompleteBatchV3(batchId, prisma as never, false, store); + + // The frozen replica was NEVER consulted for either read — both routed to the primary. + expect(replica.wasHit("batchTaskRun")).toBe(false); + expect(replica.wasHit("batchTaskRunItem")).toBe(false); + + // Observable outcome: the batch transitioned to COMPLETED with the full completed count. Had + // either read hit the frozen replica, findBatchTaskRunById→null (early return, no transition) or + // countBatchTaskRunItems→0 (< expectedCount, no transition). + const finalBatch = await prisma.batchTaskRun.findFirstOrThrow({ where: { id: batchId } }); + expect(finalBatch.status).toBe("COMPLETED"); + expect(finalBatch.completedCount).toBe(2); + } + ); + + // ================================================================================================ + // StreamBatchItemsService.call — initial findBatchTaskRunById (PRIMARY-routed) + // The entry batch lookup reads the primary, finds the live PENDING batch (frozen on the replica), + // and seals it — a replica-routed read would miss it. + // ================================================================================================ + heteroPostgresTest( + "streamBatchItems finds+seals a live batch whose row has not replicated (entry read → primary)", + async ({ prisma14 }) => { + const prisma = prisma14 as unknown as PrismaClient; + dbHolder.prisma = prisma; + const suffix = `stream134_${seq++}`; + const seed = await seedTenant(prisma, suffix); + + const { store, replica } = storeWithFrozenReplica(prisma, ["batchTaskRun"]); + const { id: batchId, friendlyId } = BatchId.generate(); + await createBatchOnPrimary(prisma, { + id: batchId, + friendlyId, + runtimeEnvironmentId: seed.environment.id, + runCount: 0, + status: "PENDING", + sealed: false, + }); + + const fakeEngine = { + runStore: store, + getBatchEnqueuedCount: vi.fn(async () => 0), + enqueueBatchItem: vi.fn(async () => ({ enqueued: true })), + }; + const service = new StreamBatchItemsService({ engine: fakeEngine as never }); + + const result = await service.call(authEnv(seed) as never, friendlyId, emptyItems(), { + maxItemBytes: 1024, + concurrency: 1, + }); + + expect(replica.wasHit("batchTaskRun")).toBe(false); + expect(result.sealed).toBe(true); + expect(result.id).toBe(friendlyId); + // The primary write actually sealed the row. + const sealed = await prisma.batchTaskRun.findFirstOrThrow({ where: { id: batchId } }); + expect(sealed.sealed).toBe(true); + expect(sealed.status).toBe("PROCESSING"); + } + ); + + // ================================================================================================ + // StreamBatchItemsService.call — count-mismatch re-query findBatchTaskRunById (PRIMARY-routed) + // A concurrent fast-completion marks the batch COMPLETED on the primary between the entry read and + // the count check; the re-query reads the primary, sees COMPLETED, and returns sealed:true. + // ================================================================================================ + heteroPostgresTest( + "streamBatchItems count-mismatch re-query reads the primary (fast-completed batch → sealed:true)", + async ({ prisma14 }) => { + const prisma = prisma14 as unknown as PrismaClient; + dbHolder.prisma = prisma; + const suffix = `stream206_${seq++}`; + const seed = await seedTenant(prisma, suffix); + + const { store, replica } = storeWithFrozenReplica(prisma, ["batchTaskRun"]); + const { id: batchId, friendlyId } = BatchId.generate(); + await createBatchOnPrimary(prisma, { + id: batchId, + friendlyId, + runtimeEnvironmentId: seed.environment.id, + runCount: 1, + status: "PENDING", + sealed: false, + }); + + const fakeEngine = { + runStore: store, + // Simulate the BatchQueue completing the batch on the PRIMARY before the count check, and + // return a mismatched enqueued count so the caller takes the re-query branch. + getBatchEnqueuedCount: vi.fn(async () => { + await prisma.batchTaskRun.update({ + where: { id: batchId }, + data: { status: "COMPLETED", processingCompletedAt: new Date() }, + }); + return 0; + }), + enqueueBatchItem: vi.fn(async () => ({ enqueued: true })), + }; + const service = new StreamBatchItemsService({ engine: fakeEngine as never }); + + const result = await service.call(authEnv(seed) as never, friendlyId, emptyItems(), { + maxItemBytes: 1024, + concurrency: 1, + }); + + expect(replica.wasHit("batchTaskRun")).toBe(false); + // Re-query found the COMPLETED batch on the primary → idempotent-retry success. + expect(result.sealed).toBe(true); + expect(result.id).toBe(friendlyId); + } + ); + + // ================================================================================================ + // StreamBatchItemsService.call — seal-race re-query findBatchTaskRunById (PRIMARY-routed) + // A concurrent request seals the batch (PROCESSING) on the primary before this request's + // conditional seal, so the conditional update matches 0 rows; the re-query reads the primary, sees + // PROCESSING, and returns sealed:true. + // ================================================================================================ + heteroPostgresTest( + "streamBatchItems seal-race re-query reads the primary (concurrently-sealed batch → sealed:true)", + async ({ prisma14 }) => { + const prisma = prisma14 as unknown as PrismaClient; + dbHolder.prisma = prisma; + const suffix = `stream294_${seq++}`; + const seed = await seedTenant(prisma, suffix); + + const { store, replica } = storeWithFrozenReplica(prisma, ["batchTaskRun"]); + const { id: batchId, friendlyId } = BatchId.generate(); + await createBatchOnPrimary(prisma, { + id: batchId, + friendlyId, + runtimeEnvironmentId: seed.environment.id, + runCount: 0, + status: "PENDING", + sealed: false, + }); + + const fakeEngine = { + runStore: store, + // A concurrent request seals the batch on the PRIMARY before this request's conditional seal, + // so the conditional update matches nothing and the caller takes the re-query branch. + getBatchEnqueuedCount: vi.fn(async () => { + await prisma.batchTaskRun.update({ + where: { id: batchId }, + data: { status: "PROCESSING", sealed: true, sealedAt: new Date() }, + }); + return 0; + }), + enqueueBatchItem: vi.fn(async () => ({ enqueued: true })), + }; + const service = new StreamBatchItemsService({ engine: fakeEngine as never }); + + const result = await service.call(authEnv(seed) as never, friendlyId, emptyItems(), { + maxItemBytes: 1024, + concurrency: 1, + }); + + expect(replica.wasHit("batchTaskRun")).toBe(false); + // Re-query found the concurrently-sealed batch on the primary → sealed:true (no spurious throw). + expect(result.sealed).toBe(true); + expect(result.id).toBe(friendlyId); + } + ); + + // ================================================================================================ + // RunEngineBatchTriggerService.processBatchTaskRun — findBatchTaskRunById (PRIMARY-routed) + // The worker reads the just-written batch on the primary and proceeds to resolve its environment. + // ================================================================================================ + heteroPostgresTest( + "runEngine processBatchTaskRun reads the batch on the primary when the replica lags", + async ({ prisma14 }) => { + const prisma = prisma14 as unknown as PrismaClient; + dbHolder.prisma = prisma; + const suffix = `retrigger_${seq++}`; + const seed = await seedTenant(prisma, suffix); + + const { store, replica } = storeWithFrozenReplica(prisma, ["batchTaskRun"]); + const { id: batchId, friendlyId } = BatchId.generate(); + await createBatchOnPrimary(prisma, { + id: batchId, + friendlyId, + runtimeEnvironmentId: seed.environment.id, + runCount: 1, + status: "PENDING", + }); + + vi.mocked(findEnvironmentById).mockClear(); + vi.mocked(findEnvironmentById).mockResolvedValue(null as never); + + const fakeEngine = { runStore: store }; + const service = new RunEngineBatchTriggerService( + "sequential", + prisma as never, + fakeEngine as never + ); + + await service.processBatchTaskRun({ + batchId, + processingId: "0", + range: { start: 0, count: 50 }, + attemptCount: 0, + strategy: "sequential", + }); + + expect(replica.wasHit("batchTaskRun")).toBe(false); + // The batch was found on the primary → env resolution was reached with the batch's env id. Had + // the read hit the frozen replica the batch would be null and findEnvironmentById never called. + expect(vi.mocked(findEnvironmentById)).toHaveBeenCalledWith(seed.environment.id); + } + ); + + // ================================================================================================ + // BatchTriggerV3Service.processBatchTaskRun — findBatchTaskRunById (PRIMARY-routed) + // Same property on the v3 worker path (injected runStore instead of an engine). + // ================================================================================================ + heteroPostgresTest( + "batchTriggerV3 processBatchTaskRun reads the batch on the primary when the replica lags", + async ({ prisma14 }) => { + const prisma = prisma14 as unknown as PrismaClient; + dbHolder.prisma = prisma; + const suffix = `v3process_${seq++}`; + const seed = await seedTenant(prisma, suffix); + + const { store, replica } = storeWithFrozenReplica(prisma, ["batchTaskRun"]); + const { id: batchId, friendlyId } = BatchId.generate(); + await createBatchOnPrimary(prisma, { + id: batchId, + friendlyId, + runtimeEnvironmentId: seed.environment.id, + runCount: 1, + status: "PENDING", + }); + + vi.mocked(findEnvironmentById).mockClear(); + vi.mocked(findEnvironmentById).mockResolvedValue(null as never); + + const service = new BatchTriggerV3Service(undefined, undefined, prisma as never, store); + + await service.processBatchTaskRun({ + batchId, + processingId: "0", + range: { start: 0, count: 50 }, + attemptCount: 0, + strategy: "sequential", + }); + + expect(replica.wasHit("batchTaskRun")).toBe(false); + expect(vi.mocked(findEnvironmentById)).toHaveBeenCalledWith(seed.environment.id); + } + ); + + // ================================================================================================ + // BatchTriggerV3Service.call — findBatchTaskRunByIdempotencyKey (PRIMARY-routed) + // The idempotency probe reads the primary, finds the just-written batch, and returns the cached + // response — so no duplicate batch is triggered. + // ================================================================================================ + heteroPostgresTest( + "batchTriggerV3.call dedups on the primary when the just-written batch has not replicated", + async ({ prisma14 }) => { + const prisma = prisma14 as unknown as PrismaClient; + dbHolder.prisma = prisma; + const suffix = `v3idem_${seq++}`; + const seed = await seedTenant(prisma, suffix); + + const { store, replica } = storeWithFrozenReplica(prisma, ["batchTaskRun"]); + const { id: batchId, friendlyId } = BatchId.generate(); + const idempotencyKey = `idem_${suffix}`; + await createBatchOnPrimary(prisma, { + id: batchId, + friendlyId, + runtimeEnvironmentId: seed.environment.id, + runCount: 1, + status: "COMPLETED", + runIds: ["run_cached_1"], + idempotencyKey, + idempotencyKeyExpiresAt: null, + payload: '[{"task":"my-task"}]', + payloadType: "application/json", + }); + objHolder.packet = { data: '[{"task":"my-task"}]', dataType: "application/json" }; + + const environment = { + id: seed.environment.id, + organizationId: seed.organization.id, + type: "DEVELOPMENT", + organization: { id: seed.organization.id, featureFlags: {} }, + project: { id: seed.project.id }, + }; + const service = new BatchTriggerV3Service( + undefined, + undefined, + prisma as never, + store, + (async () => "cuid") as never + ); + + const result = await service.call( + environment as never, + { items: [{ task: "my-task", payload: "{}", options: {} }] } as never, + { idempotencyKey } + ); + + expect(replica.wasHit("batchTaskRun")).toBe(false); + // The existing batch was found on the primary → returned as cached (no duplicate). + expect(result.isCached).toBe(true); + expect(result.id).toBe(friendlyId); + expect(result.idempotencyKey).toBe(idempotencyKey); + } + ); + + // ================================================================================================ + // BatchTriggerV3Service.call — findTaskRunAttempt (dependent-attempt read) + // The dependent-attempt read is threaded the primary client, so under replica lag it still finds a + // live TERMINAL parent attempt and the caller rejects the batch with a ServiceValidationError + // ("parent already in a terminal state") rather than building a batch for a dead parent. + // wasHit("taskRunAttempt") === false confirms the read hit the primary, not the frozen replica. + // ================================================================================================ + heteroPostgresTest( + "batchTriggerV3.call rejects a batch whose live terminal dependent-attempt has not replicated", + async ({ prisma14 }) => { + const prisma = prisma14 as unknown as PrismaClient; + dbHolder.prisma = prisma; + const suffix = `v3dep_${seq++}`; + const seed = await seedTenant(prisma, suffix); + + // The dependent (parent) attempt + its FINAL run live on the PRIMARY only. + const { attemptFriendlyId } = await seedTerminalAttempt(prisma, seed, suffix); + + const { store, replica } = storeWithFrozenReplica(prisma, ["taskRunAttempt"]); + mintHolder.id = `batch_${suffix}`; + mintHolder.friendlyId = `batch_friendly_${suffix}`; + + const environment = { + id: seed.environment.id, + organizationId: seed.organization.id, + type: "DEVELOPMENT", + organization: { id: seed.organization.id, featureFlags: {} }, + project: { id: seed.project.id }, + }; + const service = new BatchTriggerV3Service( + undefined, + undefined, + prisma as never, + store, + (async () => "cuid") as never + ); + + // The dependent-attempt read hits the primary, finds the live terminal attempt, and the caller + // rejects — the frozen replica is never consulted for it. + await expect( + service.call( + environment as never, + { + items: [{ task: "child-task", payload: "{}", options: {} }], + dependentAttempt: attemptFriendlyId, + } as never, + {} + ) + ).rejects.toThrow(/already in a terminal state/); + + // The dependent-attempt read hit the PRIMARY, never the frozen replica. + expect(replica.wasHit("taskRunAttempt")).toBe(false); + } + ); +}); diff --git a/apps/webapp/test/bulkActionV2.replicaLag.test.ts b/apps/webapp/test/bulkActionV2.replicaLag.test.ts new file mode 100644 index 00000000000..a38530e60b3 --- /dev/null +++ b/apps/webapp/test/bulkActionV2.replicaLag.test.ts @@ -0,0 +1,205 @@ +// Under replica lag, BulkActionService.process (CANCEL and REPLAY) tolerates a member whose row has +// not replicated: the per-batch member-hydration findRuns misses, the member is skipped this pass (no +// throw, no mutation — not cancelled, not replayed), and the run stays live on the primary. The id set +// comes from ClickHouse, fed downstream of Postgres and lagging more than the PG replica, so an id +// cannot reach this batch before its PG row is on the replica — the skip window is unreachable. +// +// Drives the REAL exported BulkActionService.process against a real Postgres testcontainer. Tenant + +// group are seeded on the primary; only orthogonal peripherals are mocked (ClickHouse-sourced id list, +// commonWorker enqueue, db/engine/run-store singletons so the module imports don't construct at load). +// The member-hydration store is a real PostgresRunStore whose read replica is frozen via the shared +// laggingReplica primitive. + +import { heteroPostgresTest, laggingReplica } from "@internal/testcontainers"; +import { PostgresRunStore } from "@internal/run-store"; +import { BulkActionStatus, BulkActionType, type PrismaClient } from "@trigger.dev/database"; +import { BulkActionId } from "@trigger.dev/core/v3/isomorphic"; +import { describe, expect, vi } from "vitest"; + +vi.setConfig({ testTimeout: 60_000, hookTimeout: 60_000 }); + +// Prevent the db / engine / run-store singletons from constructing at import (the service is driven with +// an explicitly-injected prisma + store). +vi.mock("~/db.server", () => ({ prisma: {}, $replica: {} })); +vi.mock("~/v3/runEngine.server", () => ({ engine: {} })); +vi.mock("~/v3/runStore.server", () => ({ runStore: {} })); +// commonWorker.enqueue is the "next batch" scheduling side effect — orthogonal to the hydration read. +vi.mock("~/v3/commonWorker.server", () => ({ commonWorker: { enqueue: vi.fn(async () => {}) } })); +// ClickHouse client factory — the id list is injected via the RunsRepository mock below. +vi.mock("~/services/clickhouse/clickhouseFactoryInstance.server", () => ({ + clickhouseFactory: { getClickhouseForOrganization: vi.fn(async () => ({})) }, +})); +// RunsRepository.listRunIds is the ClickHouse-sourced id page. Keep parseRunListInputOptions REAL; swap +// only the repository so listRunIds returns our controlled member id set + a terminal (null) cursor. +const listHolder = vi.hoisted(() => ({ runIds: [] as string[] })); +vi.mock("~/services/runsRepository/runsRepository.server", async (importOriginal) => ({ + ...((await importOriginal()) as Record), + RunsRepository: class { + constructor(_opts: any) {} + async listRunIds() { + return { runIds: listHolder.runIds, pagination: { nextCursor: null, previousCursor: null } }; + } + async countRuns() { + return listHolder.runIds.length; + } + }, +})); + +import { BulkActionService } from "~/v3/services/bulk/BulkActionV2.server"; + +let seq = 0; + +async function seedTenant(prisma: PrismaClient, suffix: string) { + const organization = await prisma.organization.create({ + data: { title: `Org ${suffix}`, slug: `org-${suffix}` }, + }); + const project = await prisma.project.create({ + data: { + name: `Project ${suffix}`, + slug: `project-${suffix}`, + externalRef: `proj_${suffix}`, + organizationId: organization.id, + }, + }); + const environment = await prisma.runtimeEnvironment.create({ + data: { + type: "DEVELOPMENT", + slug: "dev", + projectId: project.id, + organizationId: organization.id, + apiKey: `tr_dev_${suffix}`, + pkApiKey: `pk_dev_${suffix}`, + shortcode: `short_${suffix}`, + }, + }); + return { organization, project, environment }; +} + +async function seedRun( + prisma: PrismaClient, + seed: { organization: { id: string }; project: { id: string }; environment: { id: string } }, + suffix: string, + status: "EXECUTING" | "COMPLETED_SUCCESSFULLY" +) { + const runId = `run_${suffix}`; + await prisma.taskRun.create({ + data: { + id: runId, + engine: "V2", + status, + friendlyId: `run_fr_${suffix}`, + taskIdentifier: "my-task", + payload: "{}", + payloadType: "application/json", + traceId: `trace_${suffix}`, + spanId: `span_${suffix}`, + queue: "task/my-task", + runtimeEnvironmentId: seed.environment.id, + projectId: seed.project.id, + organizationId: seed.organization.id, + environmentType: "DEVELOPMENT", + }, + }); + return runId; +} + +async function seedGroup( + prisma: PrismaClient, + seed: { project: { id: string }; environment: { id: string } }, + type: BulkActionType +) { + const { id, friendlyId } = BulkActionId.generate(); + await prisma.bulkActionGroup.create({ + data: { + id, + friendlyId, + projectId: seed.project.id, + environmentId: seed.environment.id, + type, + params: {}, + queryName: "bulk_action_v1", + status: BulkActionStatus.PENDING, + totalCount: 1, + }, + }); + return id; +} + +describe("BulkActionV2.process tolerates members whose rows have not replicated", () => { + // CANCEL. + heteroPostgresTest( + "CANCEL skips a member whose row has not replicated, leaving the run live on the primary", + async ({ prisma14 }) => { + const prisma = prisma14 as unknown as PrismaClient; + const suffix = `bulk_cancel_${seq++}`; + const seed = await seedTenant(prisma, suffix); + const runId = await seedRun(prisma, seed, suffix, "EXECUTING"); + const groupId = await seedGroup(prisma, seed, BulkActionType.CANCEL); + + listHolder.runIds = [runId]; + + // Member-hydration store: primary live, taskRun frozen-missing on the replica. + const replica = laggingReplica(prisma, [{ model: "taskRun", mode: "missing" }]); + const store = new PostgresRunStore({ + prisma, + readOnlyPrisma: replica.client as never, + schemaVariant: "legacy", + }); + + const service = new BulkActionService(prisma as never, prisma as never, store); + await service.process(groupId, { continueInline: true }); + + // The hydration read really hit the (lagging) replica. + expect(replica.wasHit("taskRun")).toBe(true); + + // OBSERVABLE OUTPUT: the member was SKIPPED — the run was NOT cancelled (still EXECUTING). + const onPrimary = await prisma.taskRun.findFirstOrThrow({ where: { id: runId } }); + expect(onPrimary.status).toBe("EXECUTING"); + + // No progress was counted for the skipped member (success=0, failure=0). + const group = await prisma.bulkActionGroup.findFirstOrThrow({ where: { id: groupId } }); + expect(group.successCount).toBe(0); + expect(group.failureCount).toBe(0); + } + ); + + // REPLAY. + heteroPostgresTest( + "REPLAY skips a member whose row has not replicated, leaving the run live on the primary", + async ({ prisma14 }) => { + const prisma = prisma14 as unknown as PrismaClient; + const suffix = `bulk_replay_${seq++}`; + const seed = await seedTenant(prisma, suffix); + const runId = await seedRun(prisma, seed, suffix, "COMPLETED_SUCCESSFULLY"); + const groupId = await seedGroup(prisma, seed, BulkActionType.REPLAY); + + listHolder.runIds = [runId]; + + const replica = laggingReplica(prisma, [{ model: "taskRun", mode: "missing" }]); + const store = new PostgresRunStore({ + prisma, + readOnlyPrisma: replica.client as never, + schemaVariant: "legacy", + }); + + const service = new BulkActionService(prisma as never, prisma as never, store); + await service.process(groupId, { continueInline: true }); + + expect(replica.wasHit("taskRun")).toBe(true); + + // OBSERVABLE OUTPUT: the member was SKIPPED — no child replay run was created for this member. + const childRuns = await prisma.taskRun.findMany({ + where: { rootTaskRunId: runId }, + }); + expect(childRuns).toHaveLength(0); + + const group = await prisma.bulkActionGroup.findFirstOrThrow({ where: { id: groupId } }); + expect(group.successCount).toBe(0); + expect(group.failureCount).toBe(0); + + // The skip is pure lag: the member run is genuinely live on the primary. + const onPrimary = await prisma.taskRun.findFirstOrThrow({ where: { id: runId } }); + expect(onPrimary.friendlyId).toBe(`run_fr_${suffix}`); + } + ); +}); diff --git a/apps/webapp/test/cancelRouteReplicaLag.guard.test.ts b/apps/webapp/test/cancelRouteReplicaLag.guard.test.ts new file mode 100644 index 00000000000..f85717411e0 --- /dev/null +++ b/apps/webapp/test/cancelRouteReplicaLag.guard.test.ts @@ -0,0 +1,266 @@ +// Property: the CANCEL route resolves a just-created run under replica lag. The real exported `action` +// finds a run present only on the primary (replica frozen "missing", buffer disabled) via the primary +// re-read and cancels it, returning the success redirect rather than "Run not found". +// +// Drives the REAL route `action` (not a direct store call) against a REAL Postgres testcontainer with a +// REAL lagging replica (shared `laggingReplica`, `taskRun` frozen "missing"). Only peripherals are +// mocked: the dashboard auth gate (rbac/session), the downstream CancelTaskRunService, the mollifier +// buffer (disabled), and the toast/redirect formatting. The run read and the found/not-found decision +// run for real. + +import { heteroPostgresTest, laggingReplica } from "@internal/testcontainers"; +import type { PrismaClient } from "@trigger.dev/database"; +import { describe, expect, vi } from "vitest"; + +vi.setConfig({ testTimeout: 60_000, hookTimeout: 60_000 }); + +// ---- Holders wired into the mocked module singletons before each action() call. ------------------ +// `primaryHolder.client` -> the real container (the writer / owning primary). +// `replicaHolder.client` -> a lagging replica over the SAME container: `taskRun` reads come back empty +// (row "not replicated yet"), every other model + all writes forward to the real container. +const primaryHolder = vi.hoisted(() => ({ client: undefined as any })); +const replicaHolder = vi.hoisted(() => ({ client: undefined as any })); + +// Records every CancelTaskRunService.call(run) so the test can assert the cancel was reached with the +// right run. +const cancelCalls = vi.hoisted(() => ({ runs: [] as any[] })); + +// The user the (mocked) dashboard auth resolves to — must match the seeded OrgMember so the route's +// real control-plane membership check (`prisma.project.findFirst({ ... members: { some: { userId }}}`) +// authorizes the cancel. +const AUTH = vi.hoisted(() => ({ userId: "user_cancel_guard" })); + +// ~/db.server: point the two proxies the run-store / control-plane singletons read at our holders. +// Never mocks the DB itself — the proxies forward to real testcontainer clients. Run-ops split +// handles are left undefined so runStore.server falls back to the single control-plane store +// (buildRunStore split-off): writer = `prisma`, replica = `$replica`. That is the exact webapp +// single-DB topology this read-your-writes property lives in. +vi.mock("~/db.server", async () => { + const { Prisma } = await import("@trigger.dev/database"); + const lazyProxy = (holder: { client: any }, label: string) => + new Proxy( + {}, + { + get(_t, prop) { + if (!holder.client) throw new Error(`${label} not set for this test`); + const value = holder.client[prop]; + // The run-store singleton memoizes each Prisma delegate on first access; re-resolve through + // the holder so it always routes to the current test's client (mirrors the sibling + // waitpointCallback.controlPlane test's proxy). + if (value !== null && typeof value === "object") { + return new Proxy(value, { get: (_d, method) => holder.client[prop][method] }); + } + return value; + }, + } + ); + return { + prisma: lazyProxy(primaryHolder, "primaryHolder.client"), + $replica: lazyProxy(replicaHolder, "replicaHolder.client"), + // Split-off: leaving these undefined makes runStore.server build the single-DB passthrough store. + runOpsNewPrismaClient: undefined, + runOpsNewReplicaClient: undefined, + runOpsLegacyPrisma: undefined, + runOpsLegacyReplica: undefined, + sqlDatabaseSchema: Prisma.sql([`public`]), + }; +}); + +// Dashboard auth gate (peripheral): a passing ability + constant user id. The route's REAL +// control-plane membership check still runs against the seeded OrgMember. +vi.mock("~/services/rbac.server", () => ({ + rbac: { + authenticateSession: async () => ({ + ok: true, + user: { id: AUTH.userId, email: "guard@example.com", admin: false }, + ability: { can: () => true, canSuper: () => true }, + }), + }, +})); + +// Prevent the remix-auth strategy chain (auth.server validates secrets at module load) from loading +// via dashboardBuilder.server's static `getUserId` import. Auth outcome is owned by the rbac mock. +vi.mock("~/services/session.server", () => ({ + getUserId: async () => undefined, + requireUserId: async () => AUTH.userId, +})); + +// Buffer disabled: getEntry never returns an entry, so the buffer fallback is a clean miss and the +// only thing that can find the run is the run-store read (replica, then the primary re-read). +vi.mock("~/v3/mollifier/mollifierBuffer.server", () => ({ + getMollifierBuffer: () => null, +})); + +// Downstream cancel is engine work, not the read under test: record the call and no-op. +vi.mock("~/v3/services/cancelTaskRun.server", () => ({ + CancelTaskRunService: class { + async call(run: any) { + cancelCalls.runs.push(run); + } + }, +})); + +// Toast/redirect formatting is not under test; return marker Responses so assertions don't depend on +// SESSION_SECRET / cookie machinery. The success path returns a 302 here; the "Run not found" path +// returns a remix `json(...)` (200) from the real route code, which is how a failure to cancel shows up. +vi.mock("~/models/message.server", () => ({ + redirectWithSuccessMessage: async (path: string, _req: Request, message: string) => + new Response(null, { status: 302, headers: { "x-redirect": path, "x-toast": message } }), + redirectWithErrorMessage: async (path: string, _req: Request, message: string) => + new Response(null, { status: 302, headers: { "x-redirect": path, "x-error": message } }), +})); + +import { PostgresRunStore } from "@internal/run-store"; +import type { CreateRunInput } from "@internal/run-store"; +// The REAL route action under test. +import { action } from "~/routes/resources.taskruns.$runParam.cancel"; + +let seq = 0; + +async function seedTenant(prisma: PrismaClient, suffix: string) { + const organization = await prisma.organization.create({ + data: { title: `Org ${suffix}`, slug: `org-${suffix}` }, + }); + const project = await prisma.project.create({ + data: { + name: `Project ${suffix}`, + slug: `project-${suffix}`, + externalRef: `proj_${suffix}`, + organizationId: organization.id, + }, + }); + const environment = await prisma.runtimeEnvironment.create({ + data: { + type: "DEVELOPMENT", + slug: "dev", + projectId: project.id, + organizationId: organization.id, + apiKey: `tr_dev_${suffix}`, + pkApiKey: `pk_dev_${suffix}`, + shortcode: `short_${suffix}`, + }, + }); + // The user the auth mock resolves to, joined to the org so the route's real membership check passes. + await prisma.user.create({ + data: { + id: AUTH.userId, + email: `guard-${suffix}@example.com`, + authenticationMethod: "MAGIC_LINK", + }, + }); + await prisma.orgMember.create({ + data: { userId: AUTH.userId, organizationId: organization.id, role: "ADMIN" }, + }); + return { organization, project, environment }; +} + +function buildCreateRunInput(p: { + runId: string; + friendlyId: string; + organizationId: string; + projectId: string; + runtimeEnvironmentId: string; +}): CreateRunInput { + return { + data: { + id: p.runId, + engine: "V2", + status: "PENDING", + friendlyId: p.friendlyId, + runtimeEnvironmentId: p.runtimeEnvironmentId, + environmentType: "DEVELOPMENT", + organizationId: p.organizationId, + projectId: p.projectId, + taskIdentifier: "my-task", + payload: '{"hello":"world"}', + payloadType: "application/json", + context: { foo: "bar" }, + traceContext: { trace: "ctx" }, + traceId: "trace_1", + spanId: "span_1", + runTags: [], + queue: "task/my-task", + isTest: false, + taskEventStore: "taskEvent", + depth: 0, + createdAt: new Date("2024-01-01T00:00:00.000Z"), + }, + snapshot: { + engine: "V2", + executionStatus: "RUN_CREATED", + description: "Run was created", + runStatus: "PENDING", + environmentId: p.runtimeEnvironmentId, + environmentType: "DEVELOPMENT", + projectId: p.projectId, + organizationId: p.organizationId, + }, + }; +} + +function cancelRequest(redirectUrl: string) { + const body = new URLSearchParams({ redirectUrl }).toString(); + return new Request("http://localhost/resources/taskruns/x/cancel", { + method: "POST", + headers: { "content-type": "application/x-www-form-urlencoded" }, + body, + }); +} + +describe("cancel route resolves a just-created run under replica lag", () => { + heteroPostgresTest( + "cancels a live run whose row has not yet replicated", + async ({ prisma14 }) => { + const prisma = prisma14 as unknown as PrismaClient; + const suffix = `cancel_guard_${seq++}`; + + const seed = await seedTenant(prisma, suffix); + + // Seed the run on the PRIMARY (writer) only. The lagging replica will not see it. + const runId = `run_${"c".repeat(21)}`; // cuid-shaped -> legacy single store + const friendlyId = `run_${suffix}`; + const writerStore = new PostgresRunStore({ prisma, readOnlyPrisma: prisma }); + await writerStore.createRun( + buildCreateRunInput({ + runId, + friendlyId, + organizationId: seed.organization.id, + projectId: seed.project.id, + runtimeEnvironmentId: seed.environment.id, + }) + ); + + // A REAL lagging replica over the same container: taskRun reads miss; everything else forwards. + const replica = laggingReplica(prisma, [{ model: "taskRun", mode: "missing" }]); + + primaryHolder.client = prisma; + replicaHolder.client = replica.client; + + cancelCalls.runs.length = 0; + + const redirectUrl = "/orgs/x/projects/y/runs"; + const res = await action({ + request: cancelRequest(redirectUrl), + params: { runParam: friendlyId }, + context: {} as never, + }); + + // The replica WAS consulted (the read-your-writes hazard was really exercised)... + expect(replica.wasHit("taskRun")).toBe(true); + + // ...and the primary re-read found the run, so the cancel was reached with the right run. + expect(cancelCalls.runs).toHaveLength(1); + expect(cancelCalls.runs[0].friendlyId).toBe(friendlyId); + expect(cancelCalls.runs[0].id).toBe(runId); + + // The action returned the success redirect, NOT the "Run not found" json(200). + expect(res.status).toBe(302); + expect(res.headers.get("x-redirect")).toBe(redirectUrl); + expect(res.headers.get("x-toast")).toBe("Canceled run"); + + // Belt-and-braces: the body is not the "Run not found" field-error json. + const bodyText = await res.clone().text(); + expect(bodyText).not.toContain("Run not found"); + } + ); +}); diff --git a/apps/webapp/test/engineReplicaReads.replicaLag.guard.test.ts b/apps/webapp/test/engineReplicaReads.replicaLag.guard.test.ts new file mode 100644 index 00000000000..9520d0b6e39 --- /dev/null +++ b/apps/webapp/test/engineReplicaReads.replicaLag.guard.test.ts @@ -0,0 +1,289 @@ +// Replica-lag properties for engine-adjacent webapp reads. Each case drives the REAL exported caller +// (resolveRunForMutation, resolveRunCommit) through a real PostgresRunStore whose owning replica is +// FROZEN via the shared `laggingReplica` primitive; only orthogonal collaborators are mocked. + +import { heteroPostgresTest, laggingReplica } from "@internal/testcontainers"; +import { PostgresRunStore } from "@internal/run-store"; +import type { CreateRunInput } from "@internal/run-store"; +import type { PrismaClient } from "@trigger.dev/database"; +import { afterEach, describe, expect, vi } from "vitest"; + +vi.setConfig({ testTimeout: 60_000, hookTimeout: 60_000 }); + +// --------------------------------------------------------------------------------------------------- +// Module singletons the webapp-service callers close over. Filled per-test; a stable Proxy keeps the +// named import binding constant while forwarding to the per-test client/store. +// --------------------------------------------------------------------------------------------------- +const H = vi.hoisted(() => { + const proxyTo = (get: () => any, label: string) => + new Proxy( + {}, + { + get(_t, prop) { + const target = get(); + if (!target) throw new Error(`${label} not set for this test`); + const value = target[prop]; + return typeof value === "function" ? value.bind(target) : value; + }, + } + ); + return { + dbHolder: { primary: undefined as any, replica: undefined as any }, + storeHolder: { store: undefined as any }, + proxyTo, + }; +}); +const { dbHolder, storeHolder } = H; + +vi.mock("~/db.server", () => ({ + prisma: H.proxyTo(() => H.dbHolder.primary, "dbHolder.primary"), + $replica: H.proxyTo(() => H.dbHolder.replica, "dbHolder.replica"), +})); + +vi.mock("~/v3/runStore.server", () => ({ + runStore: H.proxyTo(() => H.storeHolder.store, "storeHolder.store"), +})); + +// The buffer is orthogonal to resolveRunForMutation; force a clean miss so the only recovery path is +// the writer probe under test. +vi.mock("~/v3/mollifier/mollifierBuffer.server", () => ({ + getMollifierBuffer: () => null, +})); + +// dashboardAgent.server pulls a heavy import graph (SDK, GitHub app, rbac) at module load that is +// entirely orthogonal to the findRun under test — stub those leaves so the module imports cleanly. +vi.mock("~/env.server", () => ({ + env: { API_ORIGIN: "https://api.local", APP_ORIGIN: "https://app.local" }, +})); +vi.mock("~/services/gitHub.server", () => ({ githubApp: {} })); +vi.mock("~/services/logger.server", () => ({ + logger: { warn: vi.fn(), info: vi.fn(), error: vi.fn(), debug: vi.fn() }, +})); +vi.mock("@trigger.dev/rbac", () => ({ signUserActorToken: vi.fn() })); +vi.mock("@trigger.dev/sdk", () => ({ TriggerClient: class {} })); +vi.mock("@trigger.dev/sdk/ai", () => ({ chat: {} })); + +// The REAL callers under guard. +import { resolveRunForMutation } from "~/v3/mollifier/resolveRunForMutation.server"; +import { resolveRunCommit } from "~/services/dashboardAgent.server"; + +let seq = 0; + +async function seedTenant(prisma: PrismaClient, suffix: string) { + const organization = await prisma.organization.create({ + data: { title: `Org ${suffix}`, slug: `org-${suffix}` }, + }); + const project = await prisma.project.create({ + data: { + name: `Project ${suffix}`, + slug: `project-${suffix}`, + externalRef: `proj_${suffix}`, + organizationId: organization.id, + }, + }); + const environment = await prisma.runtimeEnvironment.create({ + data: { + type: "DEVELOPMENT", + slug: "dev", + projectId: project.id, + organizationId: organization.id, + apiKey: `tr_dev_${suffix}`, + pkApiKey: `pk_dev_${suffix}`, + shortcode: `short_${suffix}`, + }, + }); + return { organization, project, environment }; +} + +function buildCreateRunInput(p: { + runId: string; + friendlyId: string; + organizationId: string; + projectId: string; + runtimeEnvironmentId: string; + lockedToVersionId?: string; +}): CreateRunInput { + return { + data: { + id: p.runId, + engine: "V2", + status: "PENDING", + friendlyId: p.friendlyId, + runtimeEnvironmentId: p.runtimeEnvironmentId, + environmentType: "DEVELOPMENT", + organizationId: p.organizationId, + projectId: p.projectId, + taskIdentifier: "my-task", + payload: '{"hello":"world"}', + payloadType: "application/json", + lockedToVersionId: p.lockedToVersionId, + context: { foo: "bar" }, + traceContext: { trace: "ctx" }, + traceId: "trace_1", + spanId: "span_1", + runTags: [], + queue: "task/my-task", + isTest: false, + taskEventStore: "taskEvent", + depth: 0, + createdAt: new Date("2024-01-01T00:00:00.000Z"), + }, + snapshot: { + engine: "V2", + executionStatus: "RUN_CREATED", + description: "Run was created", + runStatus: "PENDING", + environmentId: p.runtimeEnvironmentId, + environmentType: "DEVELOPMENT", + projectId: p.projectId, + organizationId: p.organizationId, + }, + }; +} + +afterEach(() => { + dbHolder.primary = undefined; + dbHolder.replica = undefined; + storeHolder.store = undefined; +}); + +// =================================================================================================== +// resolveRunForMutation findRun — a replica miss is recovered by the writer probe. Driving the REAL +// exported resolveRunForMutation under a frozen replica returns {source:"pg"} via the writer probe, +// with the replica genuinely consulted first. +// =================================================================================================== +describe("resolveRunForMutation — replica lag", () => { + heteroPostgresTest( + "a live run absent on the replica is recovered via the writer probe (source: pg)", + async ({ prisma14 }) => { + const prisma = prisma14 as unknown as PrismaClient; + const suffix = `rrfm_${seq++}`; + const seed = await seedTenant(prisma, suffix); + + const runId = `run_${"a".repeat(21)}`; + const friendlyId = `run_${suffix}`; + const writerStore = new PostgresRunStore({ prisma, readOnlyPrisma: prisma }); + await writerStore.createRun( + buildCreateRunInput({ + runId, + friendlyId, + organizationId: seed.organization.id, + projectId: seed.project.id, + runtimeEnvironmentId: seed.environment.id, + }) + ); + + const replica = laggingReplica(prisma, [{ model: "taskRun", mode: "missing" }]); + // The runStore singleton the caller reads through: writer=primary, replica used only via the + // client the caller threads. We route findRun's client explicitly through db.server handles. + storeHolder.store = new PostgresRunStore({ prisma, readOnlyPrisma: prisma }); + dbHolder.primary = prisma; + dbHolder.replica = replica.client; + + const result = await resolveRunForMutation({ + runParam: friendlyId, + environmentId: seed.environment.id, + organizationId: seed.organization.id, + }); + + // The replica was consulted first (read-your-writes hazard genuinely exercised)... + expect(replica.wasHit("taskRun")).toBe(true); + // ...and the writer probe recovered the live run. + expect(result).not.toBeNull(); + expect(result?.source).toBe("pg"); + expect(result?.friendlyId).toBe(friendlyId); + } + ); + + heteroPostgresTest( + "a genuinely absent run still resolves to null (writer probe misses too)", + async ({ prisma14 }) => { + const prisma = prisma14 as unknown as PrismaClient; + const suffix = `rrfm_absent_${seq++}`; + const seed = await seedTenant(prisma, suffix); + + const replica = laggingReplica(prisma, [{ model: "taskRun", mode: "missing" }]); + storeHolder.store = new PostgresRunStore({ prisma, readOnlyPrisma: prisma }); + dbHolder.primary = prisma; + dbHolder.replica = replica.client; + + const result = await resolveRunForMutation({ + runParam: "run_does_not_exist", + environmentId: seed.environment.id, + organizationId: seed.organization.id, + }); + expect(result).toBeNull(); + } + ); +}); + +// =================================================================================================== +// resolveRunCommit findRun — a live pinned run resolves its commit via the owning PRIMARY +// (findRunOnPrimary), so the frozen replica is never consulted. Routing this read to the replica would +// miss the row and return null, silently falling back to the branch head on a LIVE deployed+pinned run. +// =================================================================================================== +describe("resolveRunCommit — replica lag", () => { + heteroPostgresTest( + "a live pinned run resolves its commit via the primary", + async ({ prisma14 }) => { + const prisma = prisma14 as unknown as PrismaClient; + const suffix = `rrc_${seq++}`; + const seed = await seedTenant(prisma, suffix); + + // A deployed BackgroundWorker + WorkerDeployment carrying the commit the run is pinned to. + const worker = await prisma.backgroundWorker.create({ + data: { + friendlyId: `worker_${suffix}`, + contentHash: `hash_${suffix}`, + projectId: seed.project.id, + runtimeEnvironmentId: seed.environment.id, + version: "20240101.1", + metadata: {}, + }, + }); + await prisma.workerDeployment.create({ + data: { + friendlyId: `deployment_${suffix}`, + contentHash: `hash_${suffix}`, + version: "20240101.1", + shortCode: `sc_${suffix}`, + projectId: seed.project.id, + environmentId: seed.environment.id, + workerId: worker.id, + commitSHA: "deadbeefcafe", + git: { dirty: false }, + }, + }); + + const runId = `run_${"b".repeat(21)}`; + const friendlyId = `run_${suffix}`; + const writerStore = new PostgresRunStore({ prisma, readOnlyPrisma: prisma }); + await writerStore.createRun( + buildCreateRunInput({ + runId, + friendlyId, + organizationId: seed.organization.id, + projectId: seed.project.id, + runtimeEnvironmentId: seed.environment.id, + lockedToVersionId: worker.id, + }) + ); + + // The store the caller reads through: its REPLICA is frozen (row not visible), primary is live. + const replica = laggingReplica(prisma, [{ model: "taskRun", mode: "missing" }]); + storeHolder.store = new PostgresRunStore({ prisma, readOnlyPrisma: replica.client }); + dbHolder.primary = prisma; + dbHolder.replica = replica.client; + + const result = await resolveRunCommit(seed.environment.id, friendlyId); + + // The read routes to the owning PRIMARY (findRunOnPrimary), so the frozen replica is never + // consulted, and the live pinned run + its deployment commit resolve. + expect(replica.wasHit("taskRun")).toBe(false); + expect(result).not.toBeNull(); + expect(result?.sha).toBe("deadbeefcafe"); + expect(result?.version).toBe("20240101.1"); + expect(result?.dirty).toBe(false); + } + ); +}); diff --git a/apps/webapp/test/findEnvironmentFromRunReplicaLag.guard.test.ts b/apps/webapp/test/findEnvironmentFromRunReplicaLag.guard.test.ts new file mode 100644 index 00000000000..e233931ce1e --- /dev/null +++ b/apps/webapp/test/findEnvironmentFromRunReplicaLag.guard.test.ts @@ -0,0 +1,196 @@ +// Verifies findEnvironmentFromRun resolves a live, primary-resident run whose row has not yet +// replicated, so the runMetadataUpdated handler keeps the metadata write + realtime publish. Drives the +// REAL exported caller as the handler does (via $replica, no tx) over a real Postgres testcontainer with +// taskRun frozen "missing" on the replica; only the control-plane env resolver + db.server proxies are mocked. + +import { heteroPostgresTest, laggingReplica } from "@internal/testcontainers"; +import type { PrismaClient } from "@trigger.dev/database"; +import { describe, expect, vi } from "vitest"; + +vi.setConfig({ testTimeout: 60_000, hookTimeout: 60_000 }); + +// Holders wired into the mocked db.server singletons before each call. +// primaryHolder.client -> the real container (writer / owning primary). +// replicaHolder.client -> a lagging replica over the SAME container: taskRun reads come back empty. +const primaryHolder = vi.hoisted(() => ({ client: undefined as any })); +const replicaHolder = vi.hoisted(() => ({ client: undefined as any })); + +// The env the (mocked) control-plane resolver returns, plus a record of the ids it was asked for, so +// the test can prove the caller reached the env-resolve with the run's real runtimeEnvironmentId. +const resolver = vi.hoisted(() => ({ + env: undefined as any, + calls: [] as string[], +})); + +// ~/db.server: point the two proxies the run-store singleton reads at our holders. Never mocks the DB +// itself — the proxies forward to real testcontainer clients. Run-ops split handles left undefined so +// runStore.server builds the single-DB passthrough store (writer = prisma, replica = $replica) — the +// exact webapp single-DB topology these reads run in. +vi.mock("~/db.server", async () => { + const { Prisma } = await import("@trigger.dev/database"); + const lazyProxy = (holder: { client: any }, label: string) => + new Proxy( + {}, + { + get(_t, prop) { + if (!holder.client) throw new Error(`${label} not set for this test`); + const value = holder.client[prop]; + // The run-store singleton memoizes each Prisma delegate on first access; re-resolve through + // the holder so it always routes to the current test's client. + if (value !== null && typeof value === "object") { + return new Proxy(value, { get: (_d, method) => holder.client[prop][method] }); + } + return value; + }, + } + ); + return { + prisma: lazyProxy(primaryHolder, "primaryHolder.client"), + $replica: lazyProxy(replicaHolder, "replicaHolder.client"), + runOpsNewPrismaClient: undefined, + runOpsNewReplicaClient: undefined, + runOpsLegacyPrisma: undefined, + runOpsLegacyReplica: undefined, + sqlDatabaseSchema: Prisma.sql([`public`]), + }; +}); + +// Downstream env hydration is orthogonal to the run read under test: return a fixed env and record +// the environmentId it was asked to resolve. This is only reached when the run read succeeds, so it +// doubles as a witness that the caller got past the null-guard. +vi.mock("~/v3/runOpsMigration/controlPlaneResolver.server", () => ({ + controlPlaneResolver: { + resolveAuthenticatedEnv: async (environmentId: string) => { + resolver.calls.push(environmentId); + return resolver.env; + }, + }, +})); + +import { PostgresRunStore } from "@internal/run-store"; +import type { CreateRunInput } from "@internal/run-store"; +// The REAL exported caller under test. +import { findEnvironmentFromRun } from "~/models/runtimeEnvironment.server"; + +let seq = 0; + +async function seedTenant(prisma: PrismaClient, suffix: string) { + const organization = await prisma.organization.create({ + data: { title: `Org ${suffix}`, slug: `org-${suffix}` }, + }); + const project = await prisma.project.create({ + data: { + name: `Project ${suffix}`, + slug: `project-${suffix}`, + externalRef: `proj_${suffix}`, + organizationId: organization.id, + }, + }); + const environment = await prisma.runtimeEnvironment.create({ + data: { + type: "DEVELOPMENT", + slug: "dev", + projectId: project.id, + organizationId: organization.id, + apiKey: `tr_dev_${suffix}`, + pkApiKey: `pk_dev_${suffix}`, + shortcode: `short_${suffix}`, + }, + }); + return { organization, project, environment }; +} + +function buildCreateRunInput(p: { + runId: string; + friendlyId: string; + organizationId: string; + projectId: string; + runtimeEnvironmentId: string; +}): CreateRunInput { + return { + data: { + id: p.runId, + engine: "V2", + status: "PENDING", + friendlyId: p.friendlyId, + runtimeEnvironmentId: p.runtimeEnvironmentId, + environmentType: "DEVELOPMENT", + organizationId: p.organizationId, + projectId: p.projectId, + taskIdentifier: "my-task", + payload: '{"hello":"world"}', + payloadType: "application/json", + context: { foo: "bar" }, + traceContext: { trace: "ctx" }, + traceId: "trace_1", + spanId: "span_1", + runTags: ["alpha", "beta"], + queue: "task/my-task", + isTest: false, + taskEventStore: "taskEvent", + depth: 0, + createdAt: new Date("2024-01-01T00:00:00.000Z"), + }, + snapshot: { + engine: "V2", + executionStatus: "RUN_CREATED", + description: "Run was created", + runStatus: "PENDING", + environmentId: p.runtimeEnvironmentId, + environmentType: "DEVELOPMENT", + projectId: p.projectId, + organizationId: p.organizationId, + }, + }; +} + +describe("findEnvironmentFromRun resolves a primary-resident run under replica lag", () => { + heteroPostgresTest( + "a run not yet on the replica resolves via the primary re-read (non-null, keeping the metadata write)", + async ({ prisma14 }) => { + const prisma = prisma14 as unknown as PrismaClient; + const suffix = `envfromrun_${seq++}`; + + const seed = await seedTenant(prisma, suffix); + + // Seed the run on the PRIMARY (writer) only. The lagging replica will not see it. + const runId = `run_${"e".repeat(21)}`; + const friendlyId = `run_${suffix}`; + const writerStore = new PostgresRunStore({ prisma, readOnlyPrisma: prisma }); + await writerStore.createRun( + buildCreateRunInput({ + runId, + friendlyId, + organizationId: seed.organization.id, + projectId: seed.project.id, + runtimeEnvironmentId: seed.environment.id, + }) + ); + + // A REAL lagging replica over the same container: taskRun reads miss; everything else forwards. + const replica = laggingReplica(prisma, [{ model: "taskRun", mode: "missing" }]); + + primaryHolder.client = prisma; + replicaHolder.client = replica.client; + + // The env the downstream resolver returns once the run is found. + resolver.env = { id: seed.environment.id, __marker: "resolved-env" }; + resolver.calls.length = 0; + + // Drive the REAL exported caller exactly as the runMetadataUpdated handler does — no tx, so the + // read routes to $replica (the lagging replica). + const result = await findEnvironmentFromRun(runId); + + // The replica WAS consulted (the lag was really exercised on taskRun), yet the caller resolves + // this live, primary-resident run to a non-null result. + expect(replica.wasHit("taskRun")).toBe(true); + expect(result).not.toBeNull(); + expect(result!.runTags).toEqual(["alpha", "beta"]); + expect(result!.batchId).toBeNull(); + expect(result!.environment).toEqual(resolver.env); + + // The env-resolve was reached with the run's real runtimeEnvironmentId (past the null-guard). + expect(resolver.calls).toEqual([seed.environment.id]); + } + ); +}); diff --git a/apps/webapp/test/idempotencyGlobalScopeCrossDbConcurrent.test.ts b/apps/webapp/test/idempotencyGlobalScopeCrossDbConcurrent.test.ts new file mode 100644 index 00000000000..b501cb553e7 --- /dev/null +++ b/apps/webapp/test/idempotencyGlobalScopeCrossDbConcurrent.test.ts @@ -0,0 +1,930 @@ +// CONCURRENT cross-DB GLOBAL-scope idempotency dedup for the run-ops split: one global key triggered +// concurrently from two parents on DIFFERENT physical DBs still yields exactly ONE child. +// +// The hard case dedup fan-out alone cannot cover: both triggers' dedup probes run BEFORE either child +// is created, so each probe misses and the per-DB unique index on (runtimeEnvironmentId, +// taskIdentifier, idempotencyKey) cannot enforce cross-DB uniqueness. The Redis idempotency-claim +// primitive (claimOrAwait/publishClaim) is the cross-DB mutual-exclusion gate for global-scope keys +// while the split is active, regardless of the per-org mollifier flag: the claim loser resolves the +// winner by id across both DBs and returns it as a cached hit → one child. +// +// Setup: drives the REAL IdempotencyKeyConcern.handleTriggerRequest against a REAL two-physical-DB +// RoutingRunStore (heteroPostgresTest, never mocked on the read/dedup path) AND a REAL MollifierBuffer +// over a Redis testcontainer. SETNX arbitration runs for real — one claimant wins, the other pends +// and polls until the winner publishes; only the caller's create + publishClaim glue is played here. +// +// Determinism: the loser's contended claim genuinely returns "pending" (it has probed PG, missed, and +// is now blocked on the winner). A barrier opens on that real "pending" outcome so the winner holds +// its create+publish until both claimants contend, then publishes well inside the loser's poll +// deadline — so the loser resolves on every run, not on wall-clock luck. + +import { + heteroPostgresTest, + network, + redisContainer, + redisOptions, + type StartedNetwork, + type StartedRedisContainer, +} from "@internal/testcontainers"; +import { PostgresRunStore, RoutingRunStore } from "@internal/run-store"; +import { generateRunOpsId, RunId } from "@trigger.dev/core/v3/isomorphic"; +import type { PrismaClient, TaskRunStatus } from "@trigger.dev/database"; +import { MollifierBuffer } from "@trigger.dev/redis-worker"; +import type { IdempotencyClaimResult } from "@trigger.dev/redis-worker"; +import type { RedisOptions } from "ioredis"; +import { describe, expect, vi } from "vitest"; + +// hookTimeout is bumped because the per-test bufferHarness fixture closes a real Redis client +// (MollifierBuffer.close → redis.quit) during teardown, which can outrun the 10s default. +vi.setConfig({ testTimeout: 60_000, hookTimeout: 30_000 }); + +// --- Module wiring ----------------------------------------------------------------------------- +// Hoisted holders so the (import-time) mocks below can defer to per-test values. +const h = vi.hoisted(() => ({ + router: null as unknown, + buffer: null as unknown, + splitOn: true, +})); + +// The concern reads `runStore` (module singleton) for both the id-less dedup probe and the +// by-id winner resolution. Point it at the per-test RoutingRunStore built over the two real +// containers. +vi.mock("~/v3/runStore.server", () => ({ + runStore: new Proxy( + {}, + { + get(_t, prop) { + const target = h.router as Record; + const value = target[prop]; + return typeof value === "function" ? value.bind(target) : value; + }, + } + ), +})); + +// Real claim algorithm (claimOrAwait/publishClaim) backed by a REAL MollifierBuffer over a Redis +// testcontainer (see buildRealBuffer). The read/dedup path (PG findRun) stays real too. +vi.mock("~/v3/mollifier/mollifierBuffer.server", () => ({ + getMollifierBuffer: () => h.buffer, +})); + +// Split ON so resolveIdempotencyDedupClient routes the dedup client by the parent's residency and +// runStore behaves as the RoutingRunStore. +vi.mock("~/v3/runOpsMigration/splitMode.server", () => ({ + isSplitEnabled: async () => h.splitOn, +})); + +// The run-ops db.server handles are only ever used as the dedup-client SENTINEL under routing +// (their identity is never forwarded to a routed store — only their presence signals +// read-your-writes). Truthy placeholders suffice; the real containers are reached via the router. +vi.mock("~/db.server", () => ({ + prisma: {}, + $replica: {}, + runOpsNewPrisma: { __sentinel: "new" }, + runOpsLegacyPrisma: { __sentinel: "legacy" }, + runOpsNewReplica: {}, + runOpsLegacyReplica: {}, +})); + +import { IdempotencyKeyConcern } from "~/runEngine/concerns/idempotencyKeys.server"; +import { publishClaim } from "~/v3/mollifier/idempotencyClaim.server"; +import type { TraceEventConcern, TriggerTaskRequest } from "~/runEngine/types"; + +// --- Real MollifierBuffer over Redis, instrumented for the barrier ----------------------------- +// Builds the SAME MollifierBuffer the webapp uses (constructed with { redisOptions }), pointed at +// the per-test Redis container. The instrumentation is a thin observability wrapper only: it counts +// claim attempts and opens a barrier when a claim genuinely returns "pending" (a real SETNX loser). +// The SETNX itself, publishClaim and readClaim are the buffer's real Redis ops — nothing is faked. +type BufferHarness = { + buffer: MollifierBuffer; + secondClaimPending: Promise; + // Resolves once TWO claimants have genuinely returned "pending" — i.e. both + // losers of a winner + two-loser trio have probed PG (missed) and are + // blocked on the winner. Used by the EXPIRED/FAILED reacquire cases so the + // winner holds its create+publish until BOTH losers contend, guaranteeing + // both take the claim-loser path (not a PG hit) on the SAME resolved winner. + bothLosersPending: Promise; + readonly claimCalls: number; + close: () => Promise; +}; + +function buildRealBuffer(options: RedisOptions): BufferHarness { + const real = new MollifierBuffer({ redisOptions: options }); + let claimCalls = 0; + let pendingCount = 0; + let resolveSecondPending!: () => void; + const secondClaimPending = new Promise((r) => (resolveSecondPending = r)); + let resolveBothLosersPending!: () => void; + const bothLosersPending = new Promise((r) => (resolveBothLosersPending = r)); + + const buffer = new Proxy(real, { + get(target, prop, receiver) { + if (prop === "claimIdempotency") { + return async ( + input: Parameters[0] + ): Promise => { + claimCalls += 1; + const result = await target.claimIdempotency(input); + // A real "pending" means the SETNX already had a winner and THIS caller is the loser: it + // has probed PG (missed) and is now blocked on the winner. Open the barrier so the winner + // can proceed to create + publish, guaranteeing both claimants contend first. + if (result.kind === "pending") { + pendingCount += 1; + resolveSecondPending(); + if (pendingCount >= 2) resolveBothLosersPending(); + } + return result; + }; + } + const value = Reflect.get(target, prop, receiver); + return typeof value === "function" + ? (value as (...a: unknown[]) => unknown).bind(target) + : value; + }, + }) as unknown as MollifierBuffer; + + return { + buffer, + secondClaimPending, + bothLosersPending, + get claimCalls() { + return claimCalls; + }, + close: () => real.close(), + }; +} + +// heteroPostgresTest gives us the two physical Postgres DBs (PG14 legacy + PG17 new). Compose a +// per-test Redis container alongside (the isolatedRedisTest / postgresAndRedisTest precedent: +// network + redisContainer + redisOptions), then hand each test a real MollifierBuffer over it. The +// harness is a fixture so its Redis client is closed on teardown WHILE the container is still up +// (fixtures tear down in reverse: harness before redisContainer), never leaking a client. +const heteroPgWithRedisTest = heteroPostgresTest.extend<{ + network: StartedNetwork; + redisContainer: StartedRedisContainer; + redisOptions: RedisOptions; + bufferHarness: BufferHarness; +}>({ + network, + redisContainer, + redisOptions, + bufferHarness: async ({ redisOptions }, use) => { + const harness = buildRealBuffer(redisOptions); + try { + await use(harness); + } finally { + await harness.close(); + } + }, +}); + +// --- Real split RoutingRunStore over the two containers ---------------------------------------- +function makeSplitRouter(prisma14: PrismaClient, prisma17: PrismaClient) { + const legacyStore = new PostgresRunStore({ prisma: prisma14, readOnlyPrisma: prisma14 }); + const newStore = new PostgresRunStore({ prisma: prisma17, readOnlyPrisma: prisma17 }); + return new RoutingRunStore({ new: newStore, legacy: legacyStore }); +} + +type SharedEnv = { + organizationId: string; + projectId: string; + runtimeEnvironmentId: string; +}; + +// Seed the SAME logical env (identical scalar ids) on BOTH physical DBs so a child's FK-bearing +// create resolves whichever DB its id-shape routes it to. +async function seedSharedEnv( + prisma14: PrismaClient, + prisma17: PrismaClient, + suffix: string +): Promise { + const organizationId = `org_${suffix}`; + const projectId = `proj_${suffix}`; + const runtimeEnvironmentId = `env_${suffix}`; + for (const prisma of [prisma14, prisma17]) { + await prisma.organization.create({ + data: { id: organizationId, title: `Org ${suffix}`, slug: `org-${suffix}` }, + }); + await prisma.project.create({ + data: { + id: projectId, + name: `Project ${suffix}`, + slug: `project-${suffix}`, + externalRef: `proj_${suffix}`, + organizationId, + }, + }); + await prisma.runtimeEnvironment.create({ + data: { + id: runtimeEnvironmentId, + type: "DEVELOPMENT", + slug: "dev", + projectId, + organizationId, + apiKey: `tr_dev_${suffix}`, + pkApiKey: `pk_dev_${suffix}`, + shortcode: `short_${suffix}`, + }, + }); + } + return { organizationId, projectId, runtimeEnvironmentId }; +} + +function makeCreateRunInput(params: { + runId: string; + friendlyId: string; + env: SharedEnv; + idempotencyKey: string; + taskIdentifier: string; + // Winner-seed overrides: an EXPIRED-key winner (expiry in the past) or a + // FAILED-status winner, so the resolved-winner clear-and-recreate branch of + // handleExistingRun fires. Default is a live PENDING run with a 24h key. + status?: TaskRunStatus; + idempotencyKeyExpiresAt?: Date; +}) { + return { + data: { + id: params.runId, + engine: "V2" as const, + status: params.status ?? ("PENDING" as const), + friendlyId: params.friendlyId, + runtimeEnvironmentId: params.env.runtimeEnvironmentId, + environmentType: "DEVELOPMENT" as const, + organizationId: params.env.organizationId, + projectId: params.env.projectId, + idempotencyKey: params.idempotencyKey, + idempotencyKeyExpiresAt: + params.idempotencyKeyExpiresAt ?? new Date(Date.now() + 24 * 60 * 60 * 1000), + taskIdentifier: params.taskIdentifier, + payload: '{"hello":"world"}', + payloadType: "application/json", + traceId: `trace_${params.runId}`, + spanId: `span_${params.runId}`, + queue: "task/my-task", + isTest: false, + depth: 1, + createdAt: new Date("2024-01-01T00:00:00.000Z"), + }, + snapshot: { + engine: "V2" as const, + executionStatus: "RUN_CREATED" as const, + description: "Run was created", + runStatus: "PENDING" as const, + environmentId: params.env.runtimeEnvironmentId, + environmentType: "DEVELOPMENT" as const, + projectId: params.env.projectId, + organizationId: params.env.organizationId, + }, + }; +} + +type Residency = "NEW" | "LEGACY"; + +// A NEW-resident id (run-ops id) → routes to #new; a LEGACY-resident id (cuid) → routes to #legacy. +function mintChildId(residency: Residency): string { + return residency === "NEW" ? generateRunOpsId() : RunId.generate().id; +} +function mintParentFriendlyId(residency: Residency): string { + return `run_${residency === "NEW" ? generateRunOpsId() : RunId.generate().id}`; +} + +type TriggerSpec = { + parentResidency: Residency; + idempotencyKey: string; + scope?: "run" | "attempt" | "global"; + taskIdentifier: string; + // andWait: block this parent on the resolved run's waitpoint. + resumeParentOnCompletion?: boolean; + // Explicit parent friendlyId (a seeded parent) — overrides the random mint. Required for the + // andWait path so the parent-in-caller-env validation can find a real row. + parentRunFriendlyId?: string; +}; + +function makeRequest(env: SharedEnv, spec: TriggerSpec): TriggerTaskRequest { + return { + taskId: spec.taskIdentifier, + environment: { + id: env.runtimeEnvironmentId, + organizationId: env.organizationId, + projectId: env.projectId, + // No mollifierEnabled override → the per-org mollifier flag resolves false, so the ONLY thing + // that can make the claim eligible is the global-under-split rule. + organization: { featureFlags: {} }, + }, + options: {}, + body: { + options: { + idempotencyKey: spec.idempotencyKey, + parentRunId: spec.parentRunFriendlyId ?? mintParentFriendlyId(spec.parentResidency), + ...(spec.resumeParentOnCompletion ? { resumeParentOnCompletion: true } : {}), + ...(spec.scope ? { idempotencyKeyOptions: { key: "user-key", scope: spec.scope } } : {}), + }, + }, + } as unknown as TriggerTaskRequest; +} + +async function countChildren( + prisma14: PrismaClient, + prisma17: PrismaClient, + env: SharedEnv, + idempotencyKey: string, + taskIdentifier: string +) { + const where = { + runtimeEnvironmentId: env.runtimeEnvironmentId, + idempotencyKey, + taskIdentifier, + }; + const legacy = await prisma14.taskRun.count({ where }); + const nw = await prisma17.taskRun.count({ where }); + return { legacy, new: nw, total: legacy + nw }; +} + +// Drive two triggers whose dedup PROBES are both forced to run before either child is created. +// `A` is the first (winning) trigger; `B` is the second (losing) trigger. Returns each outcome. +async function driveConcurrentPair(opts: { + concern: IdempotencyKeyConcern; + router: RoutingRunStore; + buffer: { secondClaimPending: Promise }; + env: SharedEnv; + a: { spec: TriggerSpec; childResidency: Residency }; + b: { spec: TriggerSpec; childResidency: Residency }; +}) { + const { concern, router, env } = opts; + + const childA = mintChildId(opts.a.childResidency); + const childAFriendly = `run_${childA}`; + const childB = mintChildId(opts.b.childResidency); + const childBFriendly = `run_${childB}`; + + const reqA = makeRequest(env, opts.a.spec); + const reqB = makeRequest(env, opts.b.spec); + + // Phase 1: A probes + (maybe) claims. Runs to completion but creates NOTHING — the create is the + // caller's job, played below. A is the first claimant, so it wins the SETNX and returns immediately. + const aRes = await concern.handleTriggerRequest(reqA, undefined); + + let resolveBReturned!: () => void; + const bReturned = new Promise((r) => (resolveBReturned = r)); + + const createChild = (childId: string, friendlyId: string, spec: TriggerSpec) => + router.createRun( + makeCreateRunInput({ + runId: childId, + friendlyId, + env, + idempotencyKey: spec.idempotencyKey, + taskIdentifier: spec.taskIdentifier, + }) as never + ); + + // B: probe + (maybe) claim-wait. This blocks on the claim until A publishes. + const bFlow = (async () => { + const res = await concern.handleTriggerRequest(reqB, undefined); + resolveBReturned(); + if (!res.isCached) { + await createChild(childB, childBFriendly, opts.b.spec); + if (res.claim) { + await publishClaim({ ...res.claim, runId: childBFriendly }); + } + } + return res; + })(); + + // A: create + publish, but hold the create open until B has entered the claim wait (via the real + // "pending" barrier) or fully returned without a claim (the negative control) — so B's probe + // always precedes A's create. + const aFlow = (async () => { + if (!aRes.isCached) { + await Promise.race([opts.buffer.secondClaimPending, bReturned]); + await createChild(childA, childAFriendly, opts.a.spec); + if (aRes.claim) { + await publishClaim({ ...aRes.claim, runId: childAFriendly }); + } + } + })(); + + const [, bRes] = await Promise.all([aFlow, bFlow]); + return { aRes, bRes, childAFriendly, childBFriendly }; +} + +// Drive ONE winner + TWO losers against a single global key. The winner (A) wins the claim and +// creates an EXPIRED- or FAILED-status child, then publishes it — but HOLDS the create until BOTH +// losers (B, C) have probed PG (missed) and are blocked on the claim (`bothLosersPending`). Both +// losers then resolve to the winner's cleared child. A single loser recreating a cleared winner is +// correct; the DUPLICATE only appears when TWO losers clear the same winner and each recreates on a +// different DB (no cross-DB unique backstop) — hence a trio, not a pair. The winner-seed overrides +// (status / idempotencyKeyExpiresAt) pick which clear branch of handleExistingRun fires. +async function driveWinnerPlusTwoLosers(opts: { + concern: IdempotencyKeyConcern; + router: RoutingRunStore; + buffer: { bothLosersPending: Promise }; + env: SharedEnv; + idempotencyKey: string; + taskIdentifier: string; + winnerSeed: { status?: TaskRunStatus; idempotencyKeyExpiresAt?: Date }; + a: { spec: TriggerSpec; childResidency: Residency }; + b: { spec: TriggerSpec; childResidency: Residency }; + c: { spec: TriggerSpec; childResidency: Residency }; +}) { + const { concern, router, env } = opts; + + const childW = mintChildId(opts.a.childResidency); + const childWFriendly = `run_${childW}`; + const childB = mintChildId(opts.b.childResidency); + const childBFriendly = `run_${childB}`; + const childC = mintChildId(opts.c.childResidency); + const childCFriendly = `run_${childC}`; + + const reqA = makeRequest(env, opts.a.spec); + const reqB = makeRequest(env, opts.b.spec); + const reqC = makeRequest(env, opts.c.spec); + + const createChild = ( + childId: string, + friendlyId: string, + spec: TriggerSpec, + seed?: { status?: TaskRunStatus; idempotencyKeyExpiresAt?: Date } + ) => + router.createRun( + makeCreateRunInput({ + runId: childId, + friendlyId, + env, + idempotencyKey: spec.idempotencyKey, + taskIdentifier: spec.taskIdentifier, + ...seed, + }) as never + ); + + // Phase 1: A probes + wins the claim (no create yet — that's the caller's job below). + const aRes = await concern.handleTriggerRequest(reqA, undefined); + + // Loser flow: probe + claim-wait; on a NON-cached return, create its child and (if it holds a + // reacquired claim) publish it. Exactly one loser reacquires the claim and creates; the other + // resolves to that fresh run as a cached hit and creates nothing. + const loserFlow = ( + req: TriggerTaskRequest, + childId: string, + friendlyId: string, + spec: TriggerSpec + ) => + (async () => { + const res = await concern.handleTriggerRequest(req, undefined); + if (!res.isCached) { + await createChild(childId, friendlyId, spec); + if (res.claim) { + await publishClaim({ ...res.claim, runId: friendlyId }); + } + } + return res; + })(); + + const bFlow = loserFlow(reqB, childB, childBFriendly, opts.b.spec); + const cFlow = loserFlow(reqC, childC, childCFriendly, opts.c.spec); + + // A: hold the winner create+publish until BOTH losers are blocked on the claim, so both probe + // BEFORE the winner child exists and both take the claim-loser path on the SAME resolved winner. + const aFlow = (async () => { + if (!aRes.isCached) { + await opts.buffer.bothLosersPending; + await createChild(childW, childWFriendly, opts.a.spec, opts.winnerSeed); + if (aRes.claim) { + await publishClaim({ ...aRes.claim, runId: childWFriendly }); + } + } + })(); + + const [, bRes, cRes] = await Promise.all([aFlow, bFlow, cFlow]); + return { aRes, bRes, cRes, childWFriendly, childBFriendly, childCFriendly }; +} + +describe("run-ops split — CONCURRENT global-scope idempotency dedup across two different-residency parents", () => { + heteroPgWithRedisTest( + "global scope, NEW-parent winner + LEGACY-parent loser: exactly one child, loser resolves to winner", + async ({ prisma14, prisma17, bufferHarness }) => { + const env = await seedSharedEnv(prisma14, prisma17, "gcc_a"); + const router = makeSplitRouter(prisma14, prisma17); + h.router = router; + h.buffer = bufferHarness.buffer; + h.splitOn = true; + + const idempotencyKey = "global-key-concurrent-a"; + const taskIdentifier = "child-task"; + const concern = new IdempotencyKeyConcern(prisma14 as never, {} as never, {} as never); + + const { bRes, childAFriendly } = await driveConcurrentPair({ + concern, + router, + buffer: bufferHarness, + env, + a: { + spec: { parentResidency: "NEW", idempotencyKey, scope: "global", taskIdentifier }, + childResidency: "NEW", + }, + b: { + spec: { parentResidency: "LEGACY", idempotencyKey, scope: "global", taskIdentifier }, + childResidency: "LEGACY", + }, + }); + + const counts = await countChildren(prisma14, prisma17, env, idempotencyKey, taskIdentifier); + + // Load-bearing assertion: one global key ⇒ exactly ONE child, across two physical DBs. + expect(counts.total).toBe(1); + // The loser must be a cached hit resolving to the winner's run. + expect(bRes.isCached).toBe(true); + if (bRes.isCached === true) { + expect(bRes.run.friendlyId).toBe(childAFriendly); + } + // The claim was actually contended (both triggers hit the real SETNX) — proves the mutex was + // exercised for real. + expect(bufferHarness.claimCalls).toBeGreaterThanOrEqual(2); + } + ); + + heteroPgWithRedisTest( + "global scope, LEGACY-parent winner + NEW-parent loser: exactly one child (symmetric)", + async ({ prisma14, prisma17, bufferHarness }) => { + const env = await seedSharedEnv(prisma14, prisma17, "gcc_b"); + const router = makeSplitRouter(prisma14, prisma17); + h.router = router; + h.buffer = bufferHarness.buffer; + h.splitOn = true; + + const idempotencyKey = "global-key-concurrent-b"; + const taskIdentifier = "child-task"; + const concern = new IdempotencyKeyConcern(prisma14 as never, {} as never, {} as never); + + const { bRes, childAFriendly } = await driveConcurrentPair({ + concern, + router, + buffer: bufferHarness, + env, + a: { + spec: { parentResidency: "LEGACY", idempotencyKey, scope: "global", taskIdentifier }, + childResidency: "LEGACY", + }, + b: { + spec: { parentResidency: "NEW", idempotencyKey, scope: "global", taskIdentifier }, + childResidency: "NEW", + }, + }); + + const counts = await countChildren(prisma14, prisma17, env, idempotencyKey, taskIdentifier); + expect(counts.total).toBe(1); + expect(bRes.isCached).toBe(true); + if (bRes.isCached === true) { + expect(bRes.run.friendlyId).toBe(childAFriendly); + } + } + ); + + heteroPgWithRedisTest( + "global scope andWait loser: exactly one child AND the loser's parent (other DB) is blocked on the WINNER's run waitpoint", + async ({ prisma14, prisma17, bufferHarness }) => { + // The by-id cross-DB resolution (resolveWinnerAcrossDbs) the plain cases don't fully assert: + // the LOSER is an andWait trigger whose parent lives on the LEGACY DB, while the WINNER's child + // lives on the NEW DB. When the loser resolves the claim it must (1) dedup to the single winner + // child (across DBs), and (2) wire ITS parent's waitpoint against the WINNER's run — i.e. find + // the winner on the other DB, get/create the winner's run waitpoint, and block the loser's + // parent on that waitpoint. A real engine can't span two physical DBs, so we stub the two + // waitpoint methods to record their args; the resolution + routing that feeds them is real. + const env = await seedSharedEnv(prisma14, prisma17, "gcc_andwait"); + const router = makeSplitRouter(prisma14, prisma17); + h.router = router; + h.buffer = bufferHarness.buffer; + h.splitOn = true; + + const idempotencyKey = "global-key-concurrent-andwait"; + const taskIdentifier = "child-task"; + const parentTaskIdentifier = "parent-task"; + + // The loser's parent lives on the LEGACY (PG14) DB. Its friendlyId is a cuid-shaped run id so + // RunId.fromFriendlyId → routes the parent-in-caller-env validation to the legacy store. + const loserParent = RunId.generate(); + await prisma14.taskRun.create({ + data: { + id: loserParent.id, + friendlyId: loserParent.friendlyId, + engine: "V2", + status: "EXECUTING", + runtimeEnvironmentId: env.runtimeEnvironmentId, + environmentType: "DEVELOPMENT", + organizationId: env.organizationId, + projectId: env.projectId, + taskIdentifier: parentTaskIdentifier, + payload: "{}", + payloadType: "application/json", + traceId: `trace_${loserParent.id}`, + spanId: `span_${loserParent.id}`, + queue: "task/parent-task", + isTest: false, + depth: 0, + }, + }); + + // Stub engine: record the winner waitpoint get/create + the parent block. + const getOrCreateCalls: Array<{ runId: string; projectId: string; environmentId: string }> = + []; + const blockCalls: Array<{ runId: string; waitpoints: string | string[] }> = []; + const winnerWaitpoint = { + id: "waitpoint_winner_run", + status: "PENDING" as const, + outputIsError: false, + }; + const stubEngine = { + async getOrCreateRunWaitpoint(input: { + runId: string; + projectId: string; + environmentId: string; + }) { + getOrCreateCalls.push(input); + return winnerWaitpoint; + }, + async blockRunWithWaitpoint(input: { runId: string; waitpoints: string | string[] }) { + blockCalls.push({ runId: input.runId, waitpoints: input.waitpoints }); + return {}; + }, + }; + + // Stub traceEventConcern: just run the callback with a minimal span (the andWait path reads + // event.spanId / event.traceparent). + const stubTrace: Partial = { + async traceIdempotentRun(_request, _parentStore, _options, callback) { + return callback( + { + spanId: "span_trace", + traceparent: undefined, + traceId: "trace_x", + traceContext: {}, + setAttribute: () => {}, + failWithError: () => {}, + stop: () => {}, + } as never, + "test" + ); + }, + }; + + const concern = new IdempotencyKeyConcern( + prisma14 as never, + stubEngine as never, + stubTrace as never + ); + + const { bRes, childAFriendly } = await driveConcurrentPair({ + concern, + router, + buffer: bufferHarness, + env, + a: { + // Winner: NEW parent, plain global trigger, NEW-resident child (lands on the NEW DB). + spec: { parentResidency: "NEW", idempotencyKey, scope: "global", taskIdentifier }, + childResidency: "NEW", + }, + b: { + // Loser: andWait, LEGACY parent (seeded above, on the OTHER DB), same global key. + spec: { + parentResidency: "LEGACY", + idempotencyKey, + scope: "global", + taskIdentifier, + resumeParentOnCompletion: true, + parentRunFriendlyId: loserParent.friendlyId, + }, + childResidency: "LEGACY", + }, + }); + + const counts = await countChildren(prisma14, prisma17, env, idempotencyKey, taskIdentifier); + const winnerInternalId = RunId.fromFriendlyId(childAFriendly); + + // (1) Dedup held across DBs: exactly ONE child, and the loser is a cached hit to the winner. + expect(counts.total).toBe(1); + expect(counts.new).toBe(1); // the winner child lives on the NEW DB + expect(counts.legacy).toBe(0); // the loser NEVER created a legacy-DB child + expect(bRes.isCached).toBe(true); + if (bRes.isCached === true) { + expect(bRes.run.friendlyId).toBe(childAFriendly); + } + + // (2) Cross-DB waitpoint linkage: resolveWinnerAcrossDbs found the winner on the NEW DB, and + // the loser got/created the waitpoint against the WINNER's run id (not its own parent's). + expect(getOrCreateCalls).toHaveLength(1); + expect(getOrCreateCalls[0]!.runId).toBe(winnerInternalId); + expect(getOrCreateCalls[0]!.environmentId).toBe(env.runtimeEnvironmentId); + + // (3) …and the loser's parent (which lives on the LEGACY DB) is blocked on the WINNER's run + // waitpoint — the by-id cross-DB resolution wired one DB's parent to the other DB's run. + expect(blockCalls).toHaveLength(1); + expect(blockCalls[0]!.runId).toBe(loserParent.id); + expect(blockCalls[0]!.waitpoints).toBe(winnerWaitpoint.id); + + // The claim was contended for real (both triggers hit the SETNX). + expect(bufferHarness.claimCalls).toBeGreaterThanOrEqual(2); + } + ); + + heteroPgWithRedisTest( + "options-absent under split (pre-hashed key / older SDK): treated conservatively → one child", + async ({ prisma14, prisma17, bufferHarness }) => { + const env = await seedSharedEnv(prisma14, prisma17, "gcc_c"); + const router = makeSplitRouter(prisma14, prisma17); + h.router = router; + h.buffer = bufferHarness.buffer; + h.splitOn = true; + + const idempotencyKey = "prehashed-key-concurrent-c"; + const taskIdentifier = "child-task"; + const concern = new IdempotencyKeyConcern(prisma14 as never, {} as never, {} as never); + + const { bRes } = await driveConcurrentPair({ + concern, + router, + buffer: bufferHarness, + env, + a: { + // No `scope` → no idempotencyKeyOptions on the wire (pre-hashed key / older SDK). + spec: { parentResidency: "NEW", idempotencyKey, taskIdentifier }, + childResidency: "NEW", + }, + b: { + spec: { parentResidency: "LEGACY", idempotencyKey, taskIdentifier }, + childResidency: "LEGACY", + }, + }); + + const counts = await countChildren(prisma14, prisma17, env, idempotencyKey, taskIdentifier); + expect(counts.total).toBe(1); + expect(bRes.isCached).toBe(true); + } + ); + + heteroPgWithRedisTest( + "global scope, EXPIRED-key winner cleared by TWO concurrent losers: exactly one NEW child (reacquired)", + async ({ prisma14, prisma17, bufferHarness }) => { + // The critical residual this covers: the resolved winner's key is EXPIRED, so each loser's + // handleExistingRun clears it and would recreate a NEW run. Two losers on different-residency + // parents recreate on different DBs where the per-DB unique index can't dedup them, so the + // recreate must re-serialise through the claim — one loser reacquires + creates, the other + // resolves to it → exactly ONE new child. + const env = await seedSharedEnv(prisma14, prisma17, "gcc_expired"); + const router = makeSplitRouter(prisma14, prisma17); + h.router = router; + h.buffer = bufferHarness.buffer; + h.splitOn = true; + + const idempotencyKey = "global-key-expired-winner"; + const taskIdentifier = "child-task"; + const concern = new IdempotencyKeyConcern(prisma14 as never, {} as never, {} as never); + + const { bRes, cRes, childWFriendly, childBFriendly, childCFriendly } = + await driveWinnerPlusTwoLosers({ + concern, + router, + buffer: bufferHarness, + env, + idempotencyKey, + taskIdentifier, + // Winner child is a genuinely-expired run: an EXPIRED status AND an already-expired key, so + // handleExistingRun's expiry branch clears it. Both attributes matter for determinism: the + // expiry drives the clear; the EXPIRED status keeps the SECOND loser on the clear path even + // after the first loser's clear has NULLed the key + expiry (a plain PENDING winner would + // then look "live" to the second reader and dedup by accident, masking the duplicate). + winnerSeed: { status: "EXPIRED", idempotencyKeyExpiresAt: new Date(Date.now() - 60_000) }, + a: { + spec: { parentResidency: "NEW", idempotencyKey, scope: "global", taskIdentifier }, + childResidency: "NEW", + }, + b: { + spec: { parentResidency: "LEGACY", idempotencyKey, scope: "global", taskIdentifier }, + childResidency: "LEGACY", + }, + c: { + spec: { parentResidency: "NEW", idempotencyKey, scope: "global", taskIdentifier }, + childResidency: "NEW", + }, + }); + + // The winner's key was cleared, so only the recreated child(ren) still carry the key. Exactly + // ONE new child ⇒ the reacquire serialised the two losers' recreate across the split. + const counts = await countChildren(prisma14, prisma17, env, idempotencyKey, taskIdentifier); + expect(counts.total).toBe(1); + // Exactly one loser recreated (claim reacquired); the other resolved to it as a cached hit. + const cachedCount = [bRes, cRes].filter((r) => r.isCached).length; + expect(cachedCount).toBe(1); + // The cached loser must resolve to the RECREATED run (the other loser's fresh child), NOT the + // cleared winner — a stale isCached still pointing at childWFriendly would otherwise slip past. + const recreaterFriendly = bRes.isCached ? childCFriendly : childBFriendly; + const cachedLoser = bRes.isCached ? bRes : cRes; + if (cachedLoser.isCached === true) { + expect(cachedLoser.run.friendlyId).toBe(recreaterFriendly); + expect(cachedLoser.run.friendlyId).not.toBe(childWFriendly); + } + // Both losers genuinely contended on the claim (winner + two losers ⇒ ≥3 attempts). + expect(bufferHarness.claimCalls).toBeGreaterThanOrEqual(3); + } + ); + + heteroPgWithRedisTest( + "global scope, FAILED-status winner cleared by TWO concurrent losers: exactly one NEW child (reacquired)", + async ({ prisma14, prisma17, bufferHarness }) => { + // Same critical residual via the OTHER clear branch: the resolved winner is in a failed status + // (shouldIdempotencyKeyBeCleared === true), so handleExistingRun clears its key and each loser + // would recreate → the recreate re-serialises through the claim to yield exactly one child. + const env = await seedSharedEnv(prisma14, prisma17, "gcc_failed"); + const router = makeSplitRouter(prisma14, prisma17); + h.router = router; + h.buffer = bufferHarness.buffer; + h.splitOn = true; + + const idempotencyKey = "global-key-failed-winner"; + const taskIdentifier = "child-task"; + const concern = new IdempotencyKeyConcern(prisma14 as never, {} as never, {} as never); + + const { bRes, cRes, childWFriendly, childBFriendly, childCFriendly } = + await driveWinnerPlusTwoLosers({ + concern, + router, + buffer: bufferHarness, + env, + idempotencyKey, + taskIdentifier, + // Winner child carries a LIVE (future) key but a failed status → status-clear branch fires. + winnerSeed: { status: "COMPLETED_WITH_ERRORS" }, + a: { + spec: { parentResidency: "NEW", idempotencyKey, scope: "global", taskIdentifier }, + childResidency: "NEW", + }, + b: { + spec: { parentResidency: "LEGACY", idempotencyKey, scope: "global", taskIdentifier }, + childResidency: "LEGACY", + }, + c: { + spec: { parentResidency: "NEW", idempotencyKey, scope: "global", taskIdentifier }, + childResidency: "NEW", + }, + }); + + const counts = await countChildren(prisma14, prisma17, env, idempotencyKey, taskIdentifier); + expect(counts.total).toBe(1); + const cachedCount = [bRes, cRes].filter((r) => r.isCached).length; + expect(cachedCount).toBe(1); + // The cached loser must resolve to the RECREATED run (the other loser's fresh child), NOT the + // cleared winner — a stale isCached still pointing at childWFriendly would otherwise slip past. + const recreaterFriendly = bRes.isCached ? childCFriendly : childBFriendly; + const cachedLoser = bRes.isCached ? bRes : cRes; + if (cachedLoser.isCached === true) { + expect(cachedLoser.run.friendlyId).toBe(recreaterFriendly); + expect(cachedLoser.run.friendlyId).not.toBe(childWFriendly); + } + expect(bufferHarness.claimCalls).toBeGreaterThanOrEqual(3); + } + ); + + heteroPgWithRedisTest( + "NEGATIVE CONTROL — run scope, distinct per-parent keys: TWO children, no claim contention", + async ({ prisma14, prisma17, bufferHarness }) => { + const env = await seedSharedEnv(prisma14, prisma17, "gcc_d"); + const router = makeSplitRouter(prisma14, prisma17); + h.router = router; + h.buffer = bufferHarness.buffer; + h.splitOn = true; + + const taskIdentifier = "child-task"; + // Run-scope keys embed the parent run id in their hash, so two parents produce DIFFERENT + // hashed keys. These are genuinely distinct runs and must NOT be deduped. + const keyA = "run-scope-hash-A"; + const keyB = "run-scope-hash-B"; + const concern = new IdempotencyKeyConcern(prisma14 as never, {} as never, {} as never); + + const { aRes, bRes } = await driveConcurrentPair({ + concern, + router, + buffer: bufferHarness, + env, + a: { + spec: { parentResidency: "NEW", idempotencyKey: keyA, scope: "run", taskIdentifier }, + childResidency: "NEW", + }, + b: { + spec: { parentResidency: "LEGACY", idempotencyKey: keyB, scope: "run", taskIdentifier }, + childResidency: "LEGACY", + }, + }); + + const a = await countChildren(prisma14, prisma17, env, keyA, taskIdentifier); + const b = await countChildren(prisma14, prisma17, env, keyB, taskIdentifier); + + // Two distinct keys ⇒ two distinct children, one per DB. Neither is a cached hit. + expect(a.total).toBe(1); + expect(b.total).toBe(1); + expect(aRes.isCached).toBe(false); + expect(bRes.isCached).toBe(false); + // Run-scope under split is NOT global-under-split and the org isn't mollifier-enabled, so the + // claim mutex is never touched — the negative control never contends on the claim. + expect(bufferHarness.claimCalls).toBe(0); + } + ); +}); diff --git a/apps/webapp/test/idempotencyResetRouteReplicaLag.guard.test.ts b/apps/webapp/test/idempotencyResetRouteReplicaLag.guard.test.ts new file mode 100644 index 00000000000..2d2b99f7ee7 --- /dev/null +++ b/apps/webapp/test/idempotencyResetRouteReplicaLag.guard.test.ts @@ -0,0 +1,276 @@ +// Property: the idempotency-key RESET route action resolves a live run under replica lag. When the +// replica-first findRun misses a just-created run, the action re-reads the owning primary via +// findRunOnPrimary, reaches ResetIdempotencyKeyService, and returns the "Idempotency key reset +// successfully" toast — never a "Run not found" short-circuit for a run the user can see. +// +// Drives the REAL exported route action against a real split store (heteroRunOpsPostgresTest, two +// Postgres containers) whose owning replica is frozen with the shared lagging-replica primitive. Only +// module seams that inject the store / assert the outcome are mocked (dependency injection at the +// module boundary, not reimplementation — the store classes and findRun/findRunOnPrimary are real): +// runStore (the RoutingRunStore built here), db (the live legacy container the tenant is seeded on), +// session (a fixed userId), and resetIdempotencyKey (a recording stub of the reached service). + +import type * as RemixNode from "@remix-run/node"; +import { heteroRunOpsPostgresTest, laggingReplica } from "@internal/testcontainers"; +import { describe, expect, vi } from "vitest"; +import { PostgresRunStore, RoutingRunStore } from "@internal/run-store"; +import type { CreateRunInput } from "@internal/run-store"; + +vi.setConfig({ testTimeout: 60_000 }); + +// Mutable holders shared with the hoisted vi.mock factories below. The route module is imported +// dynamically inside each test AFTER these are populated, and the mock factories read them lazily via +// getters, so each test injects its own live store + prisma client. +const h = vi.hoisted(() => ({ + runStore: undefined as unknown, + prisma: undefined as unknown, + userId: "usr_reset_guard", + resetCalls: [] as Array<{ idempotencyKey: string; taskIdentifier: string; envId: string }>, +})); + +vi.mock("~/services/session.server", () => ({ + requireUserId: vi.fn(async () => h.userId), +})); + +vi.mock("~/v3/runStore.server", () => ({ + get runStore() { + return h.runStore; + }, +})); + +vi.mock("~/db.server", () => ({ + get prisma() { + return h.prisma; + }, +})); + +vi.mock("~/v3/services/resetIdempotencyKey.server", () => ({ + ResetIdempotencyKeyService: class { + async call(idempotencyKey: string, taskIdentifier: string, authenticatedEnv: { id: string }) { + h.resetCalls.push({ idempotencyKey, taskIdentifier, envId: authenticatedEnv?.id }); + return { id: idempotencyKey }; + } + }, +})); + +// Test-safe cookie session (fixed secret, real @remix-run/node storage) so the route's toast +// round-trip is exercised end-to-end without loading app env config (`env.server`/SESSION_SECRET). +vi.mock("~/models/message.server", async () => { + const { createCookieSessionStorage, json } = (await vi.importActual( + "@remix-run/node" + )) as typeof RemixNode; + const { getSession, commitSession } = createCookieSessionStorage({ + cookie: { + name: "__message", + path: "/", + httpOnly: true, + sameSite: "lax", + secrets: ["test-secret"], + }, + }); + const jsonWith = async ( + data: unknown, + request: Request, + message: string, + type: "success" | "error" + ) => { + const session = await getSession(request.headers.get("cookie")); + session.flash("toastMessage", { message, type, options: { ephemeral: true } }); + return json(data, { headers: { "Set-Cookie": await commitSession(session) } }); + }; + return { + getSession, + commitSession, + jsonWithSuccessMessage: (data: unknown, request: Request, message: string) => + jsonWith(data, request, message, "success"), + jsonWithErrorMessage: (data: unknown, request: Request, message: string) => + jsonWith(data, request, message, "error"), + }; +}); + +const RESET_SELECT = { + id: true, + idempotencyKey: true, + taskIdentifier: true, + projectId: true, + runtimeEnvironmentId: true, +} as const; + +function buildCreateRunInput(params: { + runId: string; + friendlyId: string; + taskIdentifier: string; + organizationId: string; + projectId: string; + runtimeEnvironmentId: string; + idempotencyKey: string; +}): CreateRunInput { + return { + data: { + id: params.runId, + engine: "V2", + status: "COMPLETED_SUCCESSFULLY", + friendlyId: params.friendlyId, + runtimeEnvironmentId: params.runtimeEnvironmentId, + environmentType: "DEVELOPMENT", + organizationId: params.organizationId, + projectId: params.projectId, + taskIdentifier: params.taskIdentifier, + idempotencyKey: params.idempotencyKey, + payload: '{"hello":"world"}', + payloadType: "application/json", + context: { foo: "bar" }, + traceContext: { trace: "ctx" }, + traceId: "trace_1", + spanId: "span_1", + runTags: [], + queue: "task/my-task", + isTest: false, + taskEventStore: "taskEvent", + depth: 0, + createdAt: new Date("2024-01-01T00:00:00.000Z"), + }, + snapshot: { + engine: "V2", + executionStatus: "RUN_CREATED", + description: "Run was created", + runStatus: "PENDING", + environmentId: params.runtimeEnvironmentId, + environmentType: "DEVELOPMENT", + projectId: params.projectId, + organizationId: params.organizationId, + }, + } as CreateRunInput; +} + +// Decode the flashed toast from the action's Set-Cookie so we assert the user-visible outcome, not an +// internal. Uses the mocked test-safe message.server session above (no env.server / SESSION_SECRET). +async function readToast( + response: Response +): Promise<{ message: string; type: string } | undefined> { + const { getSession } = await import("~/models/message.server"); + const setCookie = response.headers.get("Set-Cookie"); + const session = await getSession(setCookie); + return session.get("toastMessage") as { message: string; type: string } | undefined; +} + +describe("idempotency-key reset route action reads-your-writes under a lagging split replica", () => { + heteroRunOpsPostgresTest( + "returns the success toast for a run not yet replicated on the owning replica", + async ({ prisma14, prisma17 }) => { + h.resetCalls = []; + + // Seed the org/project/env + an authorized user on the LEGACY control-plane container. + const suffix = "reset_route_guard"; + const organization = await prisma14.organization.create({ + data: { title: `Org ${suffix}`, slug: `org-${suffix}` }, + }); + const project = await prisma14.project.create({ + data: { + name: `Project ${suffix}`, + slug: `project-${suffix}`, + externalRef: `proj_${suffix}`, + organizationId: organization.id, + }, + }); + const environment = await prisma14.runtimeEnvironment.create({ + data: { + type: "DEVELOPMENT", + slug: "dev", + projectId: project.id, + organizationId: organization.id, + apiKey: `tr_dev_${suffix}`, + pkApiKey: `pk_dev_${suffix}`, + shortcode: `short_${suffix}`, + }, + }); + const user = await prisma14.user.create({ + data: { + id: h.userId, + email: `${suffix}@example.com`, + authenticationMethod: "MAGIC_LINK", + }, + }); + await prisma14.orgMember.create({ + data: { organizationId: organization.id, userId: user.id, role: "ADMIN" }, + }); + + // Build the REAL split store exactly as runStore.server holds it: the LEGACY store's replica lags + // (frozen), its writer is live; the NEW store is live but empty. friendlyId reads fan out across + // both stores' replicas (miss under lag) then, on the primary re-read, both writers (legacy hits). + const legacyReplica = laggingReplica(prisma14, [{ model: "taskRun", mode: "missing" }]); + const legacyStore = new PostgresRunStore({ + prisma: prisma14, + readOnlyPrisma: legacyReplica.client, + schemaVariant: "legacy", + }); + const newStore = new PostgresRunStore({ + prisma: prisma17 as never, + readOnlyPrisma: prisma17 as never, + schemaVariant: "dedicated", + }); + const router = new RoutingRunStore({ new: newStore, legacy: legacyStore }); + + const runId = `run_${"c".repeat(25)}`; // cuid body -> LEGACY-resident + const friendlyId = "run_reset_route_guard"; + const idempotencyKey = "user-supplied-key-guard"; + const taskIdentifier = "my-task"; + await legacyStore.createRun( + buildCreateRunInput({ + runId, + friendlyId, + taskIdentifier, + organizationId: organization.id, + projectId: project.id, + runtimeEnvironmentId: environment.id, + idempotencyKey, + }) + ); + + // Sanity: the run exists on the owning primary but NOT on the (lagging) replica — the exact + // read-your-writes window under test. + expect(await router.findRun({ friendlyId }, { select: RESET_SELECT })).toBeNull(); + expect( + await router.findRunOnPrimary({ friendlyId }, { select: RESET_SELECT }) + ).not.toBeNull(); + + const primarySpy = vi.spyOn(router, "findRunOnPrimary"); + + // Inject the live store + prisma the route will hold, then drive the REAL exported action. + h.runStore = router; + h.prisma = prisma14; + + const { action } = + await import("~/routes/resources.orgs.$organizationSlug.projects.$projectParam.env.$envParam.runs.$runParam.idempotencyKey.reset"); + + const request = new Request("http://localhost/reset", { method: "POST" }); + const response: Response = await action({ + request, + params: { + organizationSlug: organization.slug, + projectParam: project.slug, + envParam: environment.slug, + runParam: friendlyId, + }, + context: {}, + } as never); + + const toast = await readToast(response); + + // Property: the action re-reads the owning primary on the replica miss, finds the run, and + // reaches the reset service — rather than returning not-found for a run the user can see. + expect(legacyReplica.wasHit()).toBe(true); // the lagging replica path really executed in the action + expect(primarySpy).toHaveBeenCalled(); // the owning-primary re-read was taken + expect(h.resetCalls).toHaveLength(1); // ResetIdempotencyKeyService.call was reached + expect(h.resetCalls[0]).toMatchObject({ + idempotencyKey, + taskIdentifier, + envId: environment.id, + }); + expect(toast?.type).toBe("success"); + expect(toast?.message).toBe("Idempotency key reset successfully"); + // And it is NOT the not-found short-circuit. + expect(toast?.message).not.toBe("Run not found"); + } + ); +}); diff --git a/apps/webapp/test/metadataRouteReplicaLag.guard.test.ts b/apps/webapp/test/metadataRouteReplicaLag.guard.test.ts new file mode 100644 index 00000000000..9a27055c33f --- /dev/null +++ b/apps/webapp/test/metadataRouteReplicaLag.guard.test.ts @@ -0,0 +1,261 @@ +// Property: the metadata GET loader resolves a live run on a replica+buffer double-miss via its primary +// fallback, returning 200 + the run's metadata. Drives the REAL exported `loader` end-to-end: +// 1. runStore.findRun(..., $replica) → REPLICA (lagging) → null +// 2. findRunByIdWithMollifierFallback(...) → buffer MISS (null) +// 3. runStore.findRunOnPrimary(...) → owning PRIMARY → HIT +// Store is REAL: a split RoutingRunStore over two testcontainer Postgres DBs, the owning +// (legacy/control-plane) REPLICA frozen behind the shared laggingReplica. Only the loader's webapp +// singletons (auth, $replica brand, mollifier buffer, route builder, logging) are stubbed. + +import { describe, expect, vi } from "vitest"; +import { heteroRunOpsPostgresTest, laggingReplica } from "@internal/testcontainers"; +import type { PrismaClient } from "@trigger.dev/database"; +import type { RunOpsPrismaClient } from "@internal/run-ops-database"; +import { PostgresRunStore, RoutingRunStore } from "@internal/run-store"; +import type { CreateRunInput } from "@internal/run-store"; + +// vi.mock factories are hoisted above the imports, so anything they reference must be created inside +// vi.hoisted. `holder.store` is filled in per-test with the REAL split router; the mocked +// `~/v3/runStore.server` export is a stable Proxy that forwards every property access to it. The +// branded `$replica` marker uses the global-registry symbol the run-store brands replicas with +// (readReplicaClient.ts) — a branded client makes the routing store keep the read on the owning +// REPLICA (no primary escalation), which under lag is exactly the miss the primary fallback recovers. +const { holder } = vi.hoisted(() => { + const REPLICA_BRAND = Symbol.for("trigger.dev/run-store/read-replica"); + return { + holder: { + store: undefined as unknown, + brandedReplica: { [REPLICA_BRAND]: true } as object, + environment: undefined as unknown, + bufferResult: null as unknown, + }, + }; +}); + +vi.mock("~/db.server", () => ({ prisma: {}, $replica: holder.brandedReplica })); +vi.mock("~/env.server", () => ({ + env: { + TASK_RUN_METADATA_MAXIMUM_SIZE: 256 * 1024, + TRIGGER_MOLLIFIER_METADATA_MAX_RETRIES: 3, + TRIGGER_MOLLIFIER_METADATA_BACKOFF_BASE_MS: 10, + TRIGGER_MOLLIFIER_METADATA_BACKOFF_STEP_MS: 10, + }, +})); +// The route module builds its action at import time via createActionApiRoute; stub the builder so the +// heavy platform/auth middleware graph never evaluates. We drive the exported `loader` directly. +vi.mock("~/services/routeBuilders/apiBuilder.server", () => ({ + createActionApiRoute: () => ({ action: vi.fn() }), +})); +vi.mock("~/services/apiAuth.server", () => ({ + authenticateApiRequest: vi.fn(async () => ({ environment: holder.environment })), +})); +// Inject the REAL split router. A stable Proxy keeps the named import binding constant while +// forwarding every method to the per-test router set in holder.store. +vi.mock("~/v3/runStore.server", () => ({ + runStore: new Proxy( + {}, + { + get(_target, prop) { + const store = holder.store as Record; + if (!store) throw new Error("test bug: holder.store not initialised before loader ran"); + const value = store[prop]; + return typeof value === "function" + ? (value as (...a: unknown[]) => unknown).bind(store) + : value; + }, + } + ), +})); +// Buffer fallback returns whatever the test set (null = buffer miss, the double-miss scenario). +vi.mock("~/v3/mollifier/readFallback.server", () => ({ + findRunByIdWithMollifierFallback: vi.fn(async () => holder.bufferResult), +})); +// Action-only leaf; stub to keep the import graph light. +vi.mock("~/v3/mollifier/applyMetadataMutation.server", () => ({ + applyMetadataMutationToBufferedRun: vi.fn(), +})); +vi.mock("~/services/metadata/updateMetadataInstance.server", () => ({ + updateMetadataService: { call: vi.fn(async () => undefined) }, +})); +vi.mock("~/v3/services/common.server", () => ({ + ServiceValidationError: class extends Error {}, +})); +vi.mock("~/services/logger.server", () => ({ + logger: { warn: vi.fn(), info: vi.fn(), error: vi.fn(), debug: vi.fn() }, +})); + +import { loader } from "~/routes/api.v1.runs.$runId.metadata"; + +// A cuid (25 chars after `run_`) classifies LEGACY, so both the create and the friendlyId-keyed reads +// route to the legacy (control-plane) store — the store that owns this run. +const CUID_25 = "c".repeat(25); + +async function seedEnvironmentLegacy(prisma: PrismaClient, suffix: string) { + const organization = await prisma.organization.create({ + data: { title: `Org ${suffix}`, slug: `org-${suffix}` }, + }); + const project = await prisma.project.create({ + data: { + name: `Project ${suffix}`, + slug: `project-${suffix}`, + externalRef: `proj_${suffix}`, + organizationId: organization.id, + }, + }); + const environment = await prisma.runtimeEnvironment.create({ + data: { + type: "DEVELOPMENT", + slug: "dev", + projectId: project.id, + organizationId: organization.id, + apiKey: `tr_dev_${suffix}`, + pkApiKey: `pk_dev_${suffix}`, + shortcode: `short_${suffix}`, + }, + }); + return { organization, project, environment }; +} + +function buildCreateRunInput(params: { + runId: string; + friendlyId: string; + organizationId: string; + projectId: string; + runtimeEnvironmentId: string; + metadata?: string; + metadataType?: string; +}): CreateRunInput { + return { + data: { + id: params.runId, + engine: "V2", + status: "PENDING", + friendlyId: params.friendlyId, + runtimeEnvironmentId: params.runtimeEnvironmentId, + environmentType: "DEVELOPMENT", + organizationId: params.organizationId, + projectId: params.projectId, + taskIdentifier: "my-task", + payload: '{"hello":"world"}', + payloadType: "application/json", + metadata: params.metadata, + metadataType: params.metadataType, + context: { foo: "bar" }, + traceContext: { trace: "ctx" }, + traceId: "trace_1", + spanId: "span_1", + runTags: ["alpha"], + queue: "task/my-task", + isTest: false, + taskEventStore: "taskEvent", + depth: 0, + createdAt: new Date("2024-01-01T00:00:00.000Z"), + }, + snapshot: { + engine: "V2", + executionStatus: "RUN_CREATED", + description: "Run was created", + runStatus: "PENDING", + environmentId: params.runtimeEnvironmentId, + environmentType: "DEVELOPMENT", + projectId: params.projectId, + organizationId: params.organizationId, + }, + }; +} + +function buildRouterWithLaggingLegacyReplica(prisma14: PrismaClient, prisma17: RunOpsPrismaClient) { + const legacyReplica = laggingReplica(prisma14, [{ model: "taskRun", mode: "missing" }]); + const legacyStore = new PostgresRunStore({ + prisma: prisma14, + readOnlyPrisma: legacyReplica.client, + schemaVariant: "legacy", + }); + const newStore = new PostgresRunStore({ + prisma: prisma17 as never, + readOnlyPrisma: prisma17 as never, + schemaVariant: "dedicated", + }); + const router = new RoutingRunStore({ new: newStore, legacy: legacyStore }); + return { router, legacyStore, legacyReplica }; +} + +async function callLoader(friendlyId: string) { + const response = await loader({ + request: new Request("https://api.trigger.dev/api/v1/runs/" + friendlyId + "/metadata", { + headers: { Authorization: "Bearer tr_dev_meta" }, + }), + params: { runId: friendlyId }, + context: {} as never, + }); + return response as Response; +} + +describe("metadata GET loader under replica lag", () => { + heteroRunOpsPostgresTest( + "metadata GET loader resolves a live run on a replica and buffer double-miss", + async ({ prisma14, prisma17 }) => { + const { router, legacyStore, legacyReplica } = buildRouterWithLaggingLegacyReplica( + prisma14, + prisma17 + ); + + const seed = await seedEnvironmentLegacy(prisma14, "meta"); + const runId = `run_${CUID_25}`; // cuid → LEGACY-owned + const friendlyId = "run_meta_live"; + const metadata = '{"phase":"one"}'; + await legacyStore.createRun( + buildCreateRunInput({ + runId, + friendlyId, + organizationId: seed.organization.id, + projectId: seed.project.id, + runtimeEnvironmentId: seed.environment.id, + metadata, + metadataType: "application/json", + }) + ); + + // Wire the loader's module singletons: the real split router, the authenticated env, and a + // buffer that MISSES (the run has drained from the buffer to the primary but the replica has + // not caught up — the exact double-miss under test). + holder.store = router; + holder.environment = { id: seed.environment.id, organizationId: seed.organization.id }; + holder.bufferResult = null; + + const response = await callLoader(friendlyId); + const body = (await response.json()) as { + metadata?: unknown; + metadataType?: unknown; + error?: string; + }; + + // The replica WAS consulted (and, being frozen, missed) — proving the read genuinely went + // through the lagging replica and the recovery is the primary fallback, not a lucky replica hit. + expect(legacyReplica.wasHit()).toBe(true); + + // The property: the loader re-reads the owning primary and returns the live run's metadata. + expect(response.status).toBe(200); + expect(body.metadata).toBe(metadata); + expect(body.metadataType).toBe("application/json"); + } + ); + + // Negative control: when the run truly does not exist anywhere (replica miss + buffer miss + + // primary miss), the loader must still 404. This pins the behavior to "recover a LIVE run" rather + // than "never 404". + heteroRunOpsPostgresTest( + "metadata GET loader 404s when the run is absent on the primary too", + async ({ prisma14, prisma17 }) => { + const { router } = buildRouterWithLaggingLegacyReplica(prisma14, prisma17); + const seed = await seedEnvironmentLegacy(prisma14, "meta_absent"); + + holder.store = router; + holder.environment = { id: seed.environment.id, organizationId: seed.organization.id }; + holder.bufferResult = null; + + const response = await callLoader("run_does_not_exist"); + expect(response.status).toBe(404); + } + ); +}); diff --git a/apps/webapp/test/presentersSessionBatchReplicaLag.guard.test.ts b/apps/webapp/test/presentersSessionBatchReplicaLag.guard.test.ts new file mode 100644 index 00000000000..a9d0fe14182 --- /dev/null +++ b/apps/webapp/test/presentersSessionBatchReplicaLag.guard.test.ts @@ -0,0 +1,358 @@ +// Replica-lag properties for the AI-session + batch DASHBOARD presenters. Drives the REAL exported +// presenter classes (SessionListPresenter, SessionPresenter, BatchPresenter) end-to-end via `.call()` +// against a real Postgres (heteroPostgresTest) whose READ replica is FROZEN via the shared +// `laggingReplica` primitive (run/batch row on the primary, not yet replicated); only orthogonal +// peripherals are mocked (ClickHouse session list, displayable-env, worker-deployment probe, presign). + +import { heteroPostgresTest, laggingReplica } from "@internal/testcontainers"; +import type { PrismaClient } from "@trigger.dev/database"; +import { describe, expect, vi } from "vitest"; + +vi.setConfig({ testTimeout: 60_000, hookTimeout: 60_000 }); + +// ~/db.server: the two proxies the run-store / control-plane singletons read from point at our +// per-test holders (never mocks the DB itself — the proxies forward to a real testcontainer client). +// Run-ops split handles left undefined => runStore.server builds the single-DB passthrough store +// (writer = `prisma`, replica = `$replica`), the exact webapp single-DB topology these reads live in. +const primaryHolder = vi.hoisted(() => ({ client: undefined as any })); +const replicaHolder = vi.hoisted(() => ({ client: undefined as any })); + +vi.mock("~/db.server", async () => { + const { Prisma } = await import("@trigger.dev/database"); + const lazyProxy = (holder: { client: any }, label: string) => + new Proxy( + {}, + { + get(_t, prop) { + if (!holder.client) throw new Error(`${label} not set for this test`); + const value = holder.client[prop]; + if (value !== null && typeof value === "object") { + return new Proxy(value, { get: (_d, method) => holder.client[prop][method] }); + } + return value; + }, + } + ); + return { + prisma: lazyProxy(primaryHolder, "primaryHolder.client"), + $replica: lazyProxy(replicaHolder, "replicaHolder.client"), + runOpsNewPrismaClient: undefined, + runOpsNewReplicaClient: undefined, + runOpsLegacyPrisma: undefined, + runOpsLegacyReplica: undefined, + sqlDatabaseSchema: Prisma.sql([`public`]), + }; +}); + +// ---- Orthogonal peripherals --------------------------------------------------------------------- + +// A stub displayable environment (control-plane resolve, orthogonal to the run-ops run/batch read). +const STUB_ENV = { + id: "env_stub", + type: "DEVELOPMENT" as const, + slug: "dev", + organizationId: "org_stub", + projectId: "proj_stub", + userId: undefined, + branchName: null, + git: null, +}; + +vi.mock("~/models/runtimeEnvironment.server", () => ({ + findDisplayableEnvironment: async () => STUB_ENV, +})); + +// SessionListPresenter's possibleTasks probe — orthogonal; no worker seeded -> empty task list. +vi.mock("~/v3/models/workerDeployment.server", () => ({ + findCurrentWorkerFromEnvironment: async () => null, +})); + +// SessionListPresenter's session list comes from ClickHouse via SessionsRepository — orthogonal to +// its run-store run read. The mock returns a controlled session whose currentRunId points at the run +// we seed on the primary only, so that findRuns read is exercised for real against the lag. +const sessionListHolder = vi.hoisted(() => ({ sessions: [] as any[] })); +vi.mock("~/services/sessionsRepository/sessionsRepository.server", () => ({ + LEGACY_PLAYGROUND_TAG: "__playground__", + SessionsRepository: class { + constructor(_deps: any) {} + async listSessions() { + return { + sessions: sessionListHolder.sessions, + pagination: { nextCursor: null, previousCursor: null }, + }; + } + }, +})); + +// SessionPresenter snapshot presign — orthogonal object-store call; a miss is handled gracefully. +vi.mock("~/v3/objectStore.server", () => ({ + generatePresignedUrl: async () => ({ success: false as const, error: "stub" }), +})); + +import { PostgresRunStore } from "@internal/run-store"; +import type { CreateRunInput } from "@internal/run-store"; +// The REAL callers under guard. +import { SessionListPresenter } from "~/presenters/v3/SessionListPresenter.server"; +import { SessionPresenter } from "~/presenters/v3/SessionPresenter.server"; +import { BatchPresenter } from "~/presenters/v3/BatchPresenter.server"; + +let seq = 0; + +async function seedTenant(prisma: PrismaClient, suffix: string) { + const organization = await prisma.organization.create({ + data: { title: `Org ${suffix}`, slug: `org-${suffix}` }, + }); + const project = await prisma.project.create({ + data: { + name: `Project ${suffix}`, + slug: `project-${suffix}`, + externalRef: `proj_${suffix}`, + organizationId: organization.id, + }, + }); + const environment = await prisma.runtimeEnvironment.create({ + data: { + type: "DEVELOPMENT", + slug: "dev", + projectId: project.id, + organizationId: organization.id, + apiKey: `tr_dev_${suffix}`, + pkApiKey: `pk_dev_${suffix}`, + shortcode: `short_${suffix}`, + }, + }); + return { organization, project, environment }; +} + +function buildCreateRunInput(p: { + runId: string; + friendlyId: string; + organizationId: string; + projectId: string; + runtimeEnvironmentId: string; +}): CreateRunInput { + return { + data: { + id: p.runId, + engine: "V2", + status: "EXECUTING", + friendlyId: p.friendlyId, + runtimeEnvironmentId: p.runtimeEnvironmentId, + environmentType: "DEVELOPMENT", + organizationId: p.organizationId, + projectId: p.projectId, + taskIdentifier: "my-task", + payload: '{"hello":"world"}', + payloadType: "application/json", + context: { foo: "bar" }, + traceContext: { trace: "ctx" }, + traceId: `trace_${p.runId}`, + spanId: `span_${p.runId}`, + runTags: [], + queue: "task/my-task", + isTest: false, + taskEventStore: "taskEvent", + depth: 0, + createdAt: new Date("2024-01-01T00:00:00.000Z"), + }, + snapshot: { + engine: "V2", + executionStatus: "EXECUTING", + description: "Run is executing", + runStatus: "EXECUTING", + environmentId: p.runtimeEnvironmentId, + environmentType: "DEVELOPMENT", + projectId: p.projectId, + organizationId: p.organizationId, + }, + }; +} + +describe("session/batch dashboard presenters under replica lag", () => { + // Read 1 — SessionListPresenter findRuns. + heteroPostgresTest( + "SessionListPresenter.call renders the list row with currentRunFriendlyId undefined under lag", + async ({ prisma14 }) => { + const prisma = prisma14 as unknown as PrismaClient; + const suffix = `sesslist_${seq++}`; + const seed = await seedTenant(prisma, suffix); + + // Seed the current run on the PRIMARY only. + const runId = `run_sesslist_${suffix}`; + const runFriendlyId = `run_sl_${suffix}`; + const writerStore = new PostgresRunStore({ prisma, readOnlyPrisma: prisma }); + await writerStore.createRun( + buildCreateRunInput({ + runId, + friendlyId: runFriendlyId, + organizationId: seed.organization.id, + projectId: seed.project.id, + runtimeEnvironmentId: seed.environment.id, + }) + ); + + // ClickHouse session list (mocked) yields ONE session pointing at that run. + sessionListHolder.sessions = [ + { + id: `sess_${suffix}`, + friendlyId: `session_${suffix}`, + externalId: null, + type: "chat", + taskIdentifier: "my-agent", + isTest: false, + tags: [], + closedAt: null, + closedReason: null, + expiresAt: null, + createdAt: new Date("2024-01-01T00:00:00.000Z"), + updatedAt: new Date("2024-01-01T00:00:00.000Z"), + currentRunId: runId, + }, + ]; + + // A REAL lagging replica: taskRun reads miss; everything else forwards to the real container. + const replica = laggingReplica(prisma, [{ model: "taskRun", mode: "missing" }]); + primaryHolder.client = prisma; + replicaHolder.client = replica.client; + + const presenter = new SessionListPresenter(replica.client as any, {} as any); + const result = await presenter.call(seed.organization.id, seed.environment.id, { + projectId: seed.project.id, + }); + + // The run read really hit the (lagging) replica. + expect(replica.wasHit("taskRun")).toBe(true); + + // OBSERVABLE OUTPUT: the session still renders; the run-link is simply absent under lag. + expect(result.sessions).toHaveLength(1); + expect(result.sessions[0].id).toBe(`sess_${suffix}`); + expect(result.sessions[0].currentRunFriendlyId).toBeUndefined(); + + // The omission is pure replica lag: the run exists on the PRIMARY. + const onPrimary = await prisma.taskRun.findFirst({ where: { id: runId } }); + expect(onPrimary?.friendlyId).toBe(runFriendlyId); + } + ); + + // Read 2 — SessionPresenter findRuns + findRun (currentRun fallback). + heteroPostgresTest( + "SessionPresenter.call returns the detail with currentRun and history run null under lag", + async ({ prisma14 }) => { + const prisma = prisma14 as unknown as PrismaClient; + const suffix = `sessdetail_${seq++}`; + const seed = await seedTenant(prisma, suffix); + + // Seed the run on the PRIMARY only. + const runId = `run_sessdetail_${suffix}`; + const runFriendlyId = `run_sd_${suffix}`; + const writerStore = new PostgresRunStore({ prisma, readOnlyPrisma: prisma }); + await writerStore.createRun( + buildCreateRunInput({ + runId, + friendlyId: runFriendlyId, + organizationId: seed.organization.id, + projectId: seed.project.id, + runtimeEnvironmentId: seed.environment.id, + }) + ); + + // Seed the Session (currentRunId -> the run) + a SessionRun history row (runId -> the run). + const sessionFriendlyId = `session_${suffix}`; + const session = await prisma.session.create({ + data: { + friendlyId: sessionFriendlyId, + type: "chat", + projectId: seed.project.id, + runtimeEnvironmentId: seed.environment.id, + environmentType: "DEVELOPMENT", + organizationId: seed.organization.id, + taskIdentifier: "my-agent", + triggerConfig: { basePayload: {} }, + currentRunId: runId, + }, + }); + await prisma.sessionRun.create({ + data: { sessionId: session.id, runId, reason: "initial" }, + }); + + // taskRun frozen-missing on the replica; Session + SessionRun forward to the primary. + const replica = laggingReplica(prisma, [{ model: "taskRun", mode: "missing" }]); + primaryHolder.client = prisma; + replicaHolder.client = replica.client; + + const presenter = new SessionPresenter(replica.client as any); + const result = await presenter.call({ + userId: "user_x", + environmentId: seed.environment.id, + sessionParam: sessionFriendlyId, + projectExternalRef: seed.project.externalRef, + environmentSlug: "dev", + }); + + // The run reads really hit the (lagging) replica. + expect(replica.wasHit("taskRun")).toBe(true); + + // OBSERVABLE OUTPUT: the detail resolves (session found via the forwarding replica)... + expect(result).not.toBeNull(); + expect(result!.friendlyId).toBe(sessionFriendlyId); + // ...currentRun link self-heals to null under lag (both the map read AND the fallback miss)... + expect(result!.currentRun).toBeNull(); + // ...and the history row renders with a null run summary. + expect(result!.runs).toHaveLength(1); + expect(result!.runs[0].run).toBeNull(); + + // The nulls are pure replica lag: the run exists on the PRIMARY. + const onPrimary = await prisma.taskRun.findFirst({ where: { id: runId } }); + expect(onPrimary?.friendlyId).toBe(runFriendlyId); + } + ); + + // Read 3 — BatchPresenter findBatchTaskRunByFriendlyId (recovered via primary re-read). + heteroPostgresTest( + "BatchPresenter.call returns a live batch under lag via the primary re-read", + async ({ prisma14 }) => { + const prisma = prisma14 as unknown as PrismaClient; + const suffix = `batch_${seq++}`; + const seed = await seedTenant(prisma, suffix); + + // Seed the batch on the PRIMARY only. + const batchId = `batch_${suffix}`; + const batchFriendlyId = `batch_fr_${suffix}`; + const writerStore = new PostgresRunStore({ prisma, readOnlyPrisma: prisma }); + await writerStore.createBatchTaskRun({ + id: batchId, + friendlyId: batchFriendlyId, + runtimeEnvironmentId: seed.environment.id, + }); + + // batchTaskRun frozen-missing on the replica. + const replica = laggingReplica(prisma, [{ model: "batchTaskRun", mode: "missing" }]); + + // BatchPresenter reads via an INJECTED real store: writer = primary, read-replica = lagging. + const store = new PostgresRunStore({ prisma, readOnlyPrisma: replica.client as any }); + const presenter = new BatchPresenter( + prisma as any, // _prisma -> the owning primary (the re-read target) + replica.client as any, // _replica -> the lagging replica (the read under test) + { resolveDisplayableEnvironment: async () => STUB_ENV as any }, + store + ); + + const result = await presenter.call({ + environmentId: seed.environment.id, + batchId: batchFriendlyId, + userId: "user_x", + }); + + // The read really hit the (lagging) replica (the read-your-writes hazard was exercised)... + expect(replica.wasHit("batchTaskRun")).toBe(true); + + // ...and the presenter returned the LIVE batch via the primary re-read, not "Batch not found". + expect(result.id).toBe(batchId); + expect(result.friendlyId).toBe(batchFriendlyId); + + // The row is genuinely on the PRIMARY (so the recovery is a real primary read, not the replica). + const onPrimary = await prisma.batchTaskRun.findFirst({ where: { id: batchId } }); + expect(onPrimary?.friendlyId).toBe(batchFriendlyId); + } + ); +}); diff --git a/apps/webapp/test/readRunForEvent.replicaLag.test.ts b/apps/webapp/test/readRunForEvent.replicaLag.test.ts new file mode 100644 index 00000000000..877f920e40f --- /dev/null +++ b/apps/webapp/test/readRunForEvent.replicaLag.test.ts @@ -0,0 +1,198 @@ +// Verifies readRunForEvent tolerates replica lag on its event-enrichment read (the store.findRun +// closures inside readThroughRun, which in single-DB passthrough route to the replica). Drives the REAL +// exported function over a real Postgres testcontainer with the replica FROZEN via laggingReplica. +// Two properties: (a) PRESENT-BUT-STALE — a completed run whose UPDATE has not replicated returns the row +// with correct immutable identity and stale status, no throw; (b) MISSING — an unreplicated INSERT returns +// null (no throw), the event fires without enrichment. Both self-heal; the row is live on the primary. + +import { containerTest, laggingReplica } from "@internal/testcontainers"; +import { PostgresRunStore } from "@internal/run-store"; +import type { PrismaClient } from "@trigger.dev/database"; +import { describe, expect, vi } from "vitest"; +import { readRunForEvent, type EventReadDeps } from "~/v3/runEngineHandlersShared.server"; + +vi.setConfig({ testTimeout: 60_000, hookTimeout: 60_000 }); + +// Exactly the immutable+mutable mix a real event-bus enrichment read selects. +const EVENT_SELECT = { + id: true, + friendlyId: true, + traceId: true, + spanId: true, + createdAt: true, + completedAt: true, + taskIdentifier: true, + status: true, + organizationId: true, +} as const; + +async function seedEnvironment(prisma: PrismaClient, slugSuffix: string) { + const organization = await prisma.organization.create({ + data: { title: `Org ${slugSuffix}`, slug: `org-${slugSuffix}` }, + }); + const project = await prisma.project.create({ + data: { + name: `Project ${slugSuffix}`, + slug: `project-${slugSuffix}`, + externalRef: `proj_${slugSuffix}`, + organizationId: organization.id, + }, + }); + const environment = await prisma.runtimeEnvironment.create({ + data: { + type: "DEVELOPMENT", + slug: "dev", + projectId: project.id, + organizationId: organization.id, + apiKey: `tr_dev_${slugSuffix}`, + pkApiKey: `pk_dev_${slugSuffix}`, + shortcode: `short_${slugSuffix}`, + }, + }); + return { organization, project, environment }; +} + +describe("readRunForEvent tolerates replica lag on its event-enrichment read", () => { + // (a) PRESENT-BUT-STALE. + containerTest( + "readRunForEvent returns a present-but-stale run under lag — correct immutable fields, stale status, no throw", + async ({ prisma }) => { + const { organization, project, environment } = await seedEnvironment(prisma, "rrfe_stale"); + + const runId = "c".repeat(25); // cuid-shaped → LEGACY; passthrough reads it as-is + const friendlyId = "run_rrfe_stale"; + const createdAt = new Date("2024-01-01T00:00:00.000Z"); + const completedAt = new Date("2024-01-01T00:05:00.000Z"); + + // The run is COMPLETED on the PRIMARY (completion is an UPDATE applied on the primary). + await prisma.taskRun.create({ + data: { + id: runId, + engine: "V2", + status: "COMPLETED_SUCCESSFULLY", + friendlyId, + taskIdentifier: "my-task", + payload: "{}", + payloadType: "application/json", + traceId: "trace_rrfe", + spanId: "span_rrfe", + queue: "task/my-task", + runtimeEnvironmentId: environment.id, + projectId: project.id, + organizationId: organization.id, + environmentType: "DEVELOPMENT", + isTest: false, + taskEventStore: "taskEvent", + createdAt, + completedAt, + }, + }); + + // A FROZEN replica: the taskRun row is present but PRE-completion (status EXECUTING, completedAt + // null) — the UPDATE has not replicated. Immutable fields match the live row (they were set at + // INSERT, before the lag window). Everything else forwards to the real container. + const replica = laggingReplica(prisma, [ + { + model: "taskRun", + mode: "frozen", + rows: [ + { + id: runId, + friendlyId, + traceId: "trace_rrfe", + spanId: "span_rrfe", + createdAt, + taskIdentifier: "my-task", + organizationId: organization.id, + // STALE mutable fields (pre-completion snapshot): + status: "EXECUTING", + completedAt: null, + }, + ], + }, + ]); + + const store = new PostgresRunStore({ prisma, readOnlyPrisma: replica.client as never }); + const deps: EventReadDeps = { + store, + newReplica: replica.client as never, + legacyReplica: replica.client as never, + splitEnabled: false, + }; + + const run = await readRunForEvent(runId, environment.id, EVENT_SELECT, deps); + + // The enrichment read really hit the (lagging) replica. + expect(replica.wasHit("taskRun")).toBe(true); + + // OBSERVABLE OUTPUT: the run resolves (present-but-stale) — no throw, no null. + expect(run).not.toBeNull(); + // Immutable identity is CORRECT even off the stale replica: + expect(run!.id).toBe(runId); + expect(run!.friendlyId).toBe(friendlyId); + expect(run!.traceId).toBe("trace_rrfe"); + expect(run!.spanId).toBe("span_rrfe"); + expect(run!.taskIdentifier).toBe("my-task"); + expect(run!.createdAt).toEqual(createdAt); + // Mutable completion fields are STALE off the replica (this is the tolerated staleness): + expect(run!.status).toBe("EXECUTING"); + expect(run!.completedAt).toBeNull(); + + // The staleness is pure lag: on the PRIMARY the completion is applied. + const onPrimary = await prisma.taskRun.findFirstOrThrow({ where: { id: runId } }); + expect(onPrimary.status).toBe("COMPLETED_SUCCESSFULLY"); + expect(onPrimary.completedAt).toEqual(completedAt); + // Immutable fields are identical on primary and the stale replica read. + expect(onPrimary.friendlyId).toBe(run!.friendlyId); + } + ); + + // (b) MISSING (INSERT not yet replicated). + containerTest( + "readRunForEvent returns null (no throw) when a live run's INSERT has not replicated", + async ({ prisma }) => { + const { organization, project, environment } = await seedEnvironment(prisma, "rrfe_missing"); + + const runId = "d".repeat(25); + await prisma.taskRun.create({ + data: { + id: runId, + engine: "V2", + status: "EXECUTING", + friendlyId: "run_rrfe_missing", + taskIdentifier: "my-task", + payload: "{}", + payloadType: "application/json", + traceId: "trace_m", + spanId: "span_m", + queue: "task/my-task", + runtimeEnvironmentId: environment.id, + projectId: project.id, + organizationId: organization.id, + environmentType: "DEVELOPMENT", + isTest: false, + taskEventStore: "taskEvent", + }, + }); + + const replica = laggingReplica(prisma, [{ model: "taskRun", mode: "missing" }]); + const store = new PostgresRunStore({ prisma, readOnlyPrisma: replica.client as never }); + const deps: EventReadDeps = { + store, + newReplica: replica.client as never, + legacyReplica: replica.client as never, + splitEnabled: false, + }; + + const run = await readRunForEvent(runId, environment.id, EVENT_SELECT, deps); + + expect(replica.wasHit("taskRun")).toBe(true); + // OBSERVABLE OUTPUT: not-found degrades to null (no throw); the event fires without enrichment. + expect(run).toBeNull(); + + // The null is pure lag: the run is live on the PRIMARY. + const onPrimary = await prisma.taskRun.findFirstOrThrow({ where: { id: runId } }); + expect(onPrimary.friendlyId).toBe("run_rrfe_missing"); + } + ); +}); diff --git a/apps/webapp/test/realtimeServices.replicaLag.test.ts b/apps/webapp/test/realtimeServices.replicaLag.test.ts new file mode 100644 index 00000000000..7df81778db9 --- /dev/null +++ b/apps/webapp/test/realtimeServices.replicaLag.test.ts @@ -0,0 +1,555 @@ +// Replica-lag guards for the realtime-services reads. +// +// Drives the REAL exported callers end-to-end against a real Postgres (heteroPostgresTest) whose +// replica is FROZEN via the shared `laggingReplica` primitive (taskRun mode:"missing"), asserting a +// concrete observable on each caller's output. Every read here is a DISPLAY / probe resolve with a +// real tolerance the caller owns — none is a read-your-writes gate a stale read corrupts: +// +// 1. RunHydrator.hydrateByIds (findRuns on $replica) — under lag the un-replicated row is OMITTED +// from the frame ([] returned, no throw); the feed self-heals on the next hydrate tick. +// 2. RunHydrator.getRunById / #fetch (findRun on $replica) — returns null ("not yet visible"), no +// throw; the short-TTL cache re-fetches. A read-only display resolve, never a decision. +// 3. ensureRunForSession → getRunStatusAndFriendlyId (findRun on $replica) — the replica probe +// misses the live currentRun, the caller's WRITER re-probe recovers it, and the run is reused +// (triggered:false) without a second trigger. +// 4. swapSessionRun → resolveRunFriendlyId (findRun on $replica) — the replica misses the calling +// run's friendlyId and the caller falls back to the cuid (?? runId); the swap still COMPLETES +// (swapped:true). +// 5. serializeSessionWithFriendlyRunId (client-less findRun → replica) — a pre-existing +// currentRunId pointer resolves to null on the wire (safe degraded direction); GET/PATCH only +// serialize pre-existing pointers, so this display staleness self-heals on the next GET. +// 6. serializeSessionsWithFriendlyRunIds (client-less findRuns → replica) — the un-replicated run +// drops out of the id→friendlyId map so that session's currentRunId serializes null. Same +// self-healing display resolve, batched. +// +// Only webapp singletons orthogonal to the read (db.server handles, the runStore singleton, logger, +// the downstream Trigger/Cancel services) are mocked; the read path and the found/not-found + +// fallback decisions are the genuine article. Reads 1/2 and 5/6 take their store/replica by +// injection, so they drive the real caller with NO module mocking. + +import { heteroPostgresTest, laggingReplica } from "@internal/testcontainers"; +import type { PrismaClient, Session } from "@trigger.dev/database"; +import { describe, expect, vi } from "vitest"; + +vi.setConfig({ testTimeout: 60_000, hookTimeout: 60_000 }); + +// --- Holders wired per-test into the mocked sessionRunManager singletons (reads 3 & 4 only). -------- +const primaryHolder = vi.hoisted(() => ({ client: undefined as any })); +const replicaHolder = vi.hoisted(() => ({ client: undefined as any })); +const storeHolder = vi.hoisted(() => ({ store: undefined as any })); +// Records every TriggerTaskService.call so read 3 can assert NO double-trigger and read 4 can assert +// which previousRunId the resolveRunFriendlyId fallback forwarded. +const triggerState = vi.hoisted(() => ({ + calls: [] as Array<{ taskIdentifier: string; body: any; options: any }>, + result: { run: { id: "", friendlyId: "" } } as { run: { id: string; friendlyId: string } }, +})); + +// db.server: two lazy proxies forwarding to the per-test holders. Never mocks the DB — the proxies +// forward to real testcontainer clients (primary = writer, replica = the frozen lagging client). +vi.mock("~/db.server", () => { + const lazyProxy = (holder: { client: any }, label: string) => + new Proxy( + {}, + { + get(_t, prop) { + if (!holder.client) throw new Error(`${label} not set for this test`); + const value = holder.client[prop]; + if (value !== null && typeof value === "object") { + return new Proxy(value, { get: (_d, method) => holder.client[prop][method] }); + } + return value; + }, + } + ); + return { + prisma: lazyProxy(primaryHolder, "primaryHolder.client"), + $replica: lazyProxy(replicaHolder, "replicaHolder.client"), + }; +}); + +// runStore singleton: a stable Proxy forwarding every method to the per-test real PostgresRunStore. +vi.mock("~/v3/runStore.server", () => ({ + runStore: new Proxy( + {}, + { + get(_t, prop) { + const store = storeHolder.store as Record; + if (!store) throw new Error("test bug: storeHolder.store not set before caller ran"); + const value = store[prop]; + return typeof value === "function" + ? (value as (...a: unknown[]) => unknown).bind(store) + : value; + }, + } + ), +})); + +vi.mock("~/services/logger.server", () => ({ + logger: { warn: vi.fn(), info: vi.fn(), error: vi.fn(), debug: vi.fn() }, +})); + +// Downstream trigger is engine work, not the read under test: record the call, return the seeded run. +vi.mock("~/v3/services/triggerTask.server", () => ({ + TriggerTaskService: class { + async call(taskIdentifier: string, _environment: any, body: any, options: any) { + triggerState.calls.push({ taskIdentifier, body, options }); + return triggerState.result; + } + }, +})); + +vi.mock("~/v3/services/cancelTaskRun.server", () => ({ + CancelTaskRunService: class { + async call() {} + }, +})); + +import { PostgresRunStore } from "@internal/run-store"; +import type { CreateRunInput } from "@internal/run-store"; +// The REAL exported callers under guard. +import { RunHydrator } from "~/services/realtime/runReader.server"; +import { ensureRunForSession, swapSessionRun } from "~/services/realtime/sessionRunManager.server"; +import { + serializeSessionWithFriendlyRunId, + serializeSessionsWithFriendlyRunIds, +} from "~/services/realtime/sessions.server"; +import type { AuthenticatedEnvironment } from "~/services/apiAuth.server"; + +let seq = 0; + +async function seedTenant(prisma: PrismaClient, suffix: string) { + const organization = await prisma.organization.create({ + data: { title: `Org ${suffix}`, slug: `org-${suffix}` }, + }); + const project = await prisma.project.create({ + data: { + name: `Project ${suffix}`, + slug: `project-${suffix}`, + externalRef: `proj_${suffix}`, + organizationId: organization.id, + }, + }); + const environment = await prisma.runtimeEnvironment.create({ + data: { + type: "DEVELOPMENT", + slug: "dev", + projectId: project.id, + organizationId: organization.id, + apiKey: `tr_dev_${suffix}`, + pkApiKey: `pk_dev_${suffix}`, + shortcode: `short_${suffix}`, + }, + }); + return { organization, project, environment }; +} + +function buildCreateRunInput(p: { + runId: string; + friendlyId: string; + organizationId: string; + projectId: string; + runtimeEnvironmentId: string; + status?: CreateRunInput["data"]["status"]; +}): CreateRunInput { + return { + data: { + id: p.runId, + engine: "V2", + status: p.status ?? "PENDING", + friendlyId: p.friendlyId, + runtimeEnvironmentId: p.runtimeEnvironmentId, + environmentType: "DEVELOPMENT", + organizationId: p.organizationId, + projectId: p.projectId, + taskIdentifier: "my-task", + payload: '{"hello":"world"}', + payloadType: "application/json", + context: { foo: "bar" }, + traceContext: { trace: "ctx" }, + traceId: "trace_1", + spanId: "span_1", + runTags: [], + queue: "task/my-task", + isTest: false, + taskEventStore: "taskEvent", + depth: 0, + createdAt: new Date("2024-01-01T00:00:00.000Z"), + }, + snapshot: { + engine: "V2", + executionStatus: "RUN_CREATED", + description: "Run was created", + runStatus: p.status ?? "PENDING", + environmentId: p.runtimeEnvironmentId, + environmentType: "DEVELOPMENT", + projectId: p.projectId, + organizationId: p.organizationId, + }, + }; +} + +// A cuid-shaped run id (Session.currentRunId stores the internal cuid). +const cuidRunId = (suffix: string) => `run_${suffix.padEnd(24, "x").slice(0, 24)}`; + +describe("realtime-svc — replica-lag guards", () => { + // RunHydrator.hydrateByIds + heteroPostgresTest( + "hydrateByIds omits an un-replicated run from the frame ([]), never throws", + async ({ prisma14 }) => { + const prisma = prisma14 as unknown as PrismaClient; + const suffix = `rr_hydrate_${seq++}`; + const seed = await seedTenant(prisma, suffix); + + const runId = cuidRunId(`h${seq}`); + const friendlyId = `run_${suffix}`; + const writerStore = new PostgresRunStore({ prisma, readOnlyPrisma: prisma }); + await writerStore.createRun( + buildCreateRunInput({ + runId, + friendlyId, + organizationId: seed.organization.id, + projectId: seed.project.id, + runtimeEnvironmentId: seed.environment.id, + }) + ); + + const replica = laggingReplica(prisma, [{ model: "taskRun", mode: "missing" }]); + // The hydrator holds the real store; hydrateByIds passes `options.replica` as the read client. + const hydrator = new RunHydrator({ + replica: replica.client as PrismaClient, + runStore: writerStore, + cacheTtlMs: 0, + }); + + const rows = await hydrator.hydrateByIds(seed.environment.id, [runId]); + + // Observable: the frame is empty (row not visible on the replica), no throw. + expect(rows).toEqual([]); + expect(replica.wasHit("taskRun")).toBe(true); + + // The row IS on the primary — the next hydrate tick (writer/primary) recovers it. + const onPrimary = await writerStore.findRuns( + { + where: { runtimeEnvironmentId: seed.environment.id, id: { in: [runId] } }, + select: { id: true, friendlyId: true }, + }, + prisma + ); + expect(onPrimary).toHaveLength(1); + expect(onPrimary[0]!.friendlyId).toBe(friendlyId); + } + ); + + // RunHydrator.getRunById / #fetch + heteroPostgresTest( + "getRunById returns null for an un-replicated run (absent frame), never throws", + async ({ prisma14 }) => { + const prisma = prisma14 as unknown as PrismaClient; + const suffix = `rr_fetch_${seq++}`; + const seed = await seedTenant(prisma, suffix); + + const runId = cuidRunId(`f${seq}`); + const friendlyId = `run_${suffix}`; + const writerStore = new PostgresRunStore({ prisma, readOnlyPrisma: prisma }); + await writerStore.createRun( + buildCreateRunInput({ + runId, + friendlyId, + organizationId: seed.organization.id, + projectId: seed.project.id, + runtimeEnvironmentId: seed.environment.id, + }) + ); + + const replica = laggingReplica(prisma, [{ model: "taskRun", mode: "missing" }]); + const hydrator = new RunHydrator({ + replica: replica.client as PrismaClient, + runStore: writerStore, + cacheTtlMs: 0, + }); + + const row = await hydrator.getRunById(seed.environment.id, runId); + + // Observable: null (not yet visible), no throw. + expect(row).toBeNull(); + expect(replica.wasHit("taskRun")).toBe(true); + + const onPrimary = await writerStore.findRunOnPrimary( + { id: runId, runtimeEnvironmentId: seed.environment.id }, + { select: { friendlyId: true } } + ); + expect(onPrimary?.friendlyId).toBe(friendlyId); + } + ); + + // ensureRunForSession → getRunStatusAndFriendlyId + heteroPostgresTest( + "ensureRunForSession reuses a live run whose row missed the replica (writer re-probe) — NO double-trigger", + async ({ prisma14 }) => { + const prisma = prisma14 as unknown as PrismaClient; + const suffix = `srm_ensure_${seq++}`; + const seed = await seedTenant(prisma, suffix); + + // The live current run — PENDING (non-final) — present on the PRIMARY only. + const runId = cuidRunId(`e${seq}`); + const friendlyId = `run_${suffix}`; + const writerStore = new PostgresRunStore({ prisma, readOnlyPrisma: prisma }); + await writerStore.createRun( + buildCreateRunInput({ + runId, + friendlyId, + organizationId: seed.organization.id, + projectId: seed.project.id, + runtimeEnvironmentId: seed.environment.id, + status: "PENDING", + }) + ); + + const replica = laggingReplica(prisma, [{ model: "taskRun", mode: "missing" }]); + primaryHolder.client = prisma; // the mocked `prisma` (writer re-probe target) + replicaHolder.client = replica.client; // the mocked `$replica` (probe target — lags) + storeHolder.store = writerStore; + triggerState.calls.length = 0; + triggerState.result = { run: { id: cuidRunId(`e2${seq}`), friendlyId: `run_${suffix}_2` } }; + + const session = { + id: `session_${suffix}`, + friendlyId: `session_${suffix}`, + taskIdentifier: "my-task", + triggerConfig: { basePayload: {} }, + currentRunId: runId, + currentRunVersion: 0, + } as unknown as Pick< + Session, + | "id" + | "friendlyId" + | "taskIdentifier" + | "triggerConfig" + | "currentRunId" + | "currentRunVersion" + >; + + const result = await ensureRunForSession({ + session, + environment: { id: seed.environment.id } as unknown as AuthenticatedEnvironment, + reason: "manual", + }); + + // Observable: the writer re-probe recovered the live run → reuse it, do NOT trigger a second run. + expect(result).toEqual({ runId, triggered: false }); + expect(triggerState.calls).toHaveLength(0); + // The replica WAS consulted first (and, frozen, missed) — proving the recovery is the writer + // re-probe, not a lucky replica hit. + expect(replica.wasHit("taskRun")).toBe(true); + + // Proof the run is a live row on the primary (writer read returns it non-final). + const onPrimary = await writerStore.findRunOnPrimary( + { id: runId }, + { select: { status: true, friendlyId: true } } + ); + expect(onPrimary?.status).toBe("PENDING"); + } + ); + + // swapSessionRun → resolveRunFriendlyId + heteroPostgresTest( + "swapSessionRun completes under lag; resolveRunFriendlyId falls back to the cuid for previousRunId", + async ({ prisma14 }) => { + const prisma = prisma14 as unknown as PrismaClient; + const suffix = `srm_swap_${seq++}`; + const seed = await seedTenant(prisma, suffix); + + // The calling run — present on the PRIMARY only; its friendlyId will NOT be visible on the replica. + const callingRunId = cuidRunId(`s${seq}`); + const callingFriendlyId = `run_${suffix}_calling`; + const writerStore = new PostgresRunStore({ prisma, readOnlyPrisma: prisma }); + await writerStore.createRun( + buildCreateRunInput({ + runId: callingRunId, + friendlyId: callingFriendlyId, + organizationId: seed.organization.id, + projectId: seed.project.id, + runtimeEnvironmentId: seed.environment.id, + }) + ); + + // A real Session row whose optimistic claim (currentRunId === callingRunId, version 0) can succeed. + const sessionRow = await prisma.session.create({ + data: { + friendlyId: `session_${suffix}`, + type: "chat.agent", + projectId: seed.project.id, + runtimeEnvironmentId: seed.environment.id, + environmentType: "DEVELOPMENT", + organizationId: seed.organization.id, + taskIdentifier: "my-task", + triggerConfig: { basePayload: {} }, + currentRunId: callingRunId, + currentRunVersion: 0, + }, + }); + + const replica = laggingReplica(prisma, [{ model: "taskRun", mode: "missing" }]); + primaryHolder.client = prisma; + replicaHolder.client = replica.client; + storeHolder.store = writerStore; + triggerState.calls.length = 0; + const newRunId = cuidRunId(`sn${seq}`); + const newFriendlyId = `run_${suffix}_new`; + triggerState.result = { run: { id: newRunId, friendlyId: newFriendlyId } }; + + const result = await swapSessionRun({ + session: sessionRow, + callingRunId, + environment: { id: seed.environment.id } as unknown as AuthenticatedEnvironment, + reason: "upgrade", + }); + + // Observable 1: the swap COMPLETED — the replica miss did not fail it. + expect(result).toEqual({ runId: newRunId, swapped: true }); + + // Observable 2: resolveRunFriendlyId missed on the replica and degraded to the cuid, so the + // previousRunId forwarded to the triggered run is the calling run's cuid (documented fallback). + expect(triggerState.calls).toHaveLength(1); + expect(triggerState.calls[0]!.body.payload.previousRunId).toBe(callingRunId); + expect(replica.wasHit("taskRun")).toBe(true); + + // Proof the null was lag-induced: the primary holds the resolvable friendlyId (≠ the cuid). + const onPrimary = await writerStore.findRunOnPrimary( + { id: callingRunId }, + { select: { friendlyId: true } } + ); + expect(onPrimary?.friendlyId).toBe(callingFriendlyId); + expect(callingFriendlyId).not.toBe(callingRunId); + + // Wait for the fire-and-forget SessionRun audit write (keyed by the new runId) to land before + // teardown, so it can't race a closing pool. Poll for the row rather than sleep a fixed interval. + await vi.waitFor(async () => { + expect(await prisma.sessionRun.findFirst({ where: { runId: newRunId } })).not.toBeNull(); + }); + } + ); + + // serializeSessionWithFriendlyRunId + heteroPostgresTest( + "serializeSessionWithFriendlyRunId serializes currentRunId=null when the run row lags the replica", + async ({ prisma14 }) => { + const prisma = prisma14 as unknown as PrismaClient; + const suffix = `sess_one_${seq++}`; + const seed = await seedTenant(prisma, suffix); + + const runId = cuidRunId(`o${seq}`); + const friendlyId = `run_${suffix}`; + const writerStore = new PostgresRunStore({ prisma, readOnlyPrisma: prisma }); + await writerStore.createRun( + buildCreateRunInput({ + runId, + friendlyId, + organizationId: seed.organization.id, + projectId: seed.project.id, + runtimeEnvironmentId: seed.environment.id, + }) + ); + + const sessionRow = await prisma.session.create({ + data: { + friendlyId: `session_${suffix}`, + type: "chat.agent", + projectId: seed.project.id, + runtimeEnvironmentId: seed.environment.id, + environmentType: "DEVELOPMENT", + organizationId: seed.organization.id, + taskIdentifier: "my-task", + triggerConfig: { basePayload: {} }, + currentRunId: runId, + currentRunVersion: 0, + }, + }); + + const replica = laggingReplica(prisma, [{ model: "taskRun", mode: "missing" }]); + // Inject a store whose REPLICA lags; the serializer's client-less findRun reads it. + const laggingStore = new PostgresRunStore({ prisma, readOnlyPrisma: replica.client }); + + const item = await serializeSessionWithFriendlyRunId(sessionRow, laggingStore); + + // Observable: the pre-existing currentRunId pointer resolves to null (safe degraded direction). + expect(item.currentRunId).toBeNull(); + expect(replica.wasHit("taskRun")).toBe(true); + expect(item.id).toBe(sessionRow.friendlyId); + + // Proof the row exists on the primary — the client's next GET (replica caught up) resolves it. + const onPrimary = await writerStore.findRunOnPrimary( + { id: runId, projectId: seed.project.id, runtimeEnvironmentId: seed.environment.id }, + { select: { friendlyId: true } } + ); + expect(onPrimary?.friendlyId).toBe(friendlyId); + } + ); + + // serializeSessionsWithFriendlyRunIds + heteroPostgresTest( + "serializeSessionsWithFriendlyRunIds serializes currentRunId=null for a session whose run lags the replica", + async ({ prisma14 }) => { + const prisma = prisma14 as unknown as PrismaClient; + const suffix = `sess_list_${seq++}`; + const seed = await seedTenant(prisma, suffix); + + const runId = cuidRunId(`l${seq}`); + const friendlyId = `run_${suffix}`; + const writerStore = new PostgresRunStore({ prisma, readOnlyPrisma: prisma }); + await writerStore.createRun( + buildCreateRunInput({ + runId, + friendlyId, + organizationId: seed.organization.id, + projectId: seed.project.id, + runtimeEnvironmentId: seed.environment.id, + }) + ); + + const sessionRow = await prisma.session.create({ + data: { + friendlyId: `session_${suffix}`, + type: "chat.agent", + projectId: seed.project.id, + runtimeEnvironmentId: seed.environment.id, + environmentType: "DEVELOPMENT", + organizationId: seed.organization.id, + taskIdentifier: "my-task", + triggerConfig: { basePayload: {} }, + currentRunId: runId, + currentRunVersion: 0, + }, + }); + + const replica = laggingReplica(prisma, [{ model: "taskRun", mode: "missing" }]); + const laggingStore = new PostgresRunStore({ prisma, readOnlyPrisma: replica.client }); + + const items = await serializeSessionsWithFriendlyRunIds( + [sessionRow], + { projectId: seed.project.id, runtimeEnvironmentId: seed.environment.id }, + laggingStore + ); + + // Observable: the un-replicated run drops out of the id→friendlyId map → currentRunId null. + expect(items).toHaveLength(1); + expect(items[0]!.currentRunId).toBeNull(); + expect(replica.wasHit("taskRun")).toBe(true); + + // Proof the row exists on the primary — the next list fetch resolves it. + const onPrimary = await writerStore.findRuns( + { + where: { + id: { in: [runId] }, + projectId: seed.project.id, + runtimeEnvironmentId: seed.environment.id, + }, + select: { id: true, friendlyId: true }, + }, + prisma + ); + expect(onPrimary).toHaveLength(1); + expect(onPrimary[0]!.friendlyId).toBe(friendlyId); + } + ); +}); diff --git a/apps/webapp/test/realtimeSessionsIoRoute.replicaLag.guard.test.ts b/apps/webapp/test/realtimeSessionsIoRoute.replicaLag.guard.test.ts new file mode 100644 index 00000000000..11e0bc24e9d --- /dev/null +++ b/apps/webapp/test/realtimeSessionsIoRoute.replicaLag.guard.test.ts @@ -0,0 +1,263 @@ +// Property: under a lagging control-plane replica the dashboard Agent-tab SSE subscribe loader still +// resolves a just-created Session / SessionRun and establishes the stream (200), never surfacing a +// "Session not found" / "Session not found for run" 404 for a live subscription. The Session read uses +// a replica-first writer fallback and the run<->session linkage read re-reads the primary on a replica +// miss. +// +// Drives the REAL exported route loader against a real Postgres testcontainer whose control-plane read +// replica is a real lagging replica (the shared laggingReplica primitive); the DB is never mocked. Only +// orthogonal deps are mocked (dashboard auth/session, project/environment slug resolution, the realtime +// stream instance, the request abort signal). Case A freezes Session on the replica; Case B freezes +// SessionRun; wasHit proves the frozen replica was really consulted, so no green is a lucky primary hit. + +import { heteroPostgresTest, laggingReplica } from "@internal/testcontainers"; +import type { PrismaClient } from "@trigger.dev/database"; +import { describe, expect, vi } from "vitest"; + +vi.setConfig({ testTimeout: 60_000, hookTimeout: 60_000 }); + +// ---- Holders wired into the mocked module singletons before each loader() call. ------------------ +// `primaryHolder.client` -> the real container (the writer / owning primary). +// `replicaHolder.client` -> a lagging replica over the SAME container: the frozen model's reads come +// back empty (row "not replicated yet"); every other model + all writes forward to the real container. +const primaryHolder = vi.hoisted(() => ({ client: undefined as any })); +const replicaHolder = vi.hoisted(() => ({ client: undefined as any })); + +// The user the (mocked) dashboard auth resolves to. +const AUTH = vi.hoisted(() => ({ userId: "user_sessions_io_guard" })); + +// Staged fix-orthogonal control-plane resolutions (project + environment slug lookups). +const cpLookups = vi.hoisted(() => ({ project: undefined as any, environment: undefined as any })); + +// ~/db.server: point the two proxies the run-store / control-plane singletons read at our holders. +// Never mocks the DB itself — the proxies forward to real testcontainer clients. Run-ops split handles +// are left undefined so runStore.server falls back to the single control-plane store (writer = `prisma`, +// replica = `$replica`) — the exact webapp single-DB topology the read-your-writes hazard lives in. +vi.mock("~/db.server", async () => { + const { Prisma } = await import("@trigger.dev/database"); + const lazyProxy = (holder: { client: any }, label: string) => + new Proxy( + {}, + { + get(_t, prop) { + if (!holder.client) throw new Error(`${label} not set for this test`); + const value = holder.client[prop]; + // The run-store singleton memoizes each Prisma delegate on first access; re-resolve through + // the holder so it always routes to the current test's client (mirrors the sibling guards). + if (value !== null && typeof value === "object") { + return new Proxy(value, { get: (_d, method) => holder.client[prop][method] }); + } + return value; + }, + } + ); + return { + prisma: lazyProxy(primaryHolder, "primaryHolder.client"), + $replica: lazyProxy(replicaHolder, "replicaHolder.client"), + // Split-off: leaving these undefined makes runStore.server build the single-DB passthrough store. + runOpsNewPrismaClient: undefined, + runOpsNewReplicaClient: undefined, + runOpsLegacyPrisma: undefined, + runOpsLegacyReplica: undefined, + sqlDatabaseSchema: Prisma.sql([`public`]), + }; +}); + +// Dashboard auth (orthogonal): a fixed user id. +vi.mock("~/services/session.server", () => ({ + getUserId: async () => AUTH.userId, + requireUserId: async () => AUTH.userId, +})); + +// Project / environment slug resolution (orthogonal control-plane auth reads): return what's staged. +vi.mock("~/models/project.server", () => ({ + findProjectBySlug: async () => cpLookups.project, +})); +vi.mock("~/models/runtimeEnvironment.server", () => ({ + findEnvironmentBySlug: async () => cpLookups.environment, +})); + +// Realtime stream backend (orthogonal): make `getRealtimeStreamInstance` return a REAL instance of the +// (mocked) S2RealtimeStreams class so the loader's `instanceof S2RealtimeStreams` gate passes, and its +// `streamResponseFromSessionStream` returns a marker 200 Response — proof the loader reached the stream +// rather than 404'ing on a session/linkage read. +vi.mock("~/services/realtime/s2realtimeStreams.server", () => { + class S2RealtimeStreams { + streamResponseFromSessionStream() { + return new Response("stream-established", { + status: 200, + headers: { "x-stream": "established" }, + }); + } + } + return { S2RealtimeStreams }; +}); +vi.mock("~/services/realtime/v1StreamsGlobal.server", async () => { + const { S2RealtimeStreams } = await import("~/services/realtime/s2realtimeStreams.server"); + return { getRealtimeStreamInstance: () => new (S2RealtimeStreams as any)() }; +}); + +// Request abort signal is sourced from AsyncLocalStorage in prod; not under test. +vi.mock("~/services/httpAsyncStorage.server", () => ({ + getRequestAbortSignal: () => new AbortController().signal, +})); + +// The REAL loader under test. +import { loader } from "~/routes/resources.orgs.$organizationSlug.projects.$projectParam.env.$envParam.runs.$runParam.realtime.v1.sessions.$sessionId.$io"; + +let seq = 0; + +async function seedTenant(prisma: PrismaClient, suffix: string) { + const organization = await prisma.organization.create({ + data: { title: `Org ${suffix}`, slug: `org-${suffix}` }, + }); + const project = await prisma.project.create({ + data: { + name: `Project ${suffix}`, + slug: `project-${suffix}`, + externalRef: `proj_${suffix}`, + organizationId: organization.id, + }, + }); + const environment = await prisma.runtimeEnvironment.create({ + data: { + type: "DEVELOPMENT", + slug: "dev", + projectId: project.id, + organizationId: organization.id, + apiKey: `tr_dev_${suffix}`, + pkApiKey: `pk_dev_${suffix}`, + shortcode: `short_${suffix}`, + }, + }); + return { organization, project, environment }; +} + +async function seedSessionRunLinkage( + prisma: PrismaClient, + seed: { organization: any; project: any; environment: any }, + suffix: string +) { + const session = await prisma.session.create({ + data: { + friendlyId: `session_${suffix}`, + type: "chat", + projectId: seed.project.id, + runtimeEnvironmentId: seed.environment.id, + environmentType: "DEVELOPMENT", + organizationId: seed.organization.id, + taskIdentifier: "agent-task", + triggerConfig: { basePayload: {} }, + }, + }); + const run = await prisma.taskRun.create({ + data: { + friendlyId: `run_${suffix}`, + engine: "V2", + taskIdentifier: "agent-task", + payload: "{}", + payloadType: "application/json", + traceId: `trace_${suffix}`, + spanId: `span_${suffix}`, + queue: "task/agent-task", + projectId: seed.project.id, + organizationId: seed.organization.id, + runtimeEnvironmentId: seed.environment.id, + runTags: [], + }, + }); + const sessionRun = await prisma.sessionRun.create({ + data: { sessionId: session.id, runId: run.id, reason: "initial" }, + }); + return { session, run, sessionRun }; +} + +function subscribeRequest() { + return new Request( + "http://localhost/resources/orgs/o/projects/p/env/dev/runs/r/realtime/v1/sessions/s/out" + ); +} + +function loaderParams( + seed: { organization: any; project: any }, + runFriendlyId: string, + sessionFriendlyId: string +) { + return { + organizationSlug: seed.organization.slug, + projectParam: seed.project.slug, + envParam: "dev", + runParam: runFriendlyId, + sessionId: sessionFriendlyId, + io: "out", + }; +} + +function stageCpLookups(seed: { organization: any; project: any; environment: any }) { + cpLookups.project = { id: seed.project.id, organizationId: seed.organization.id }; + cpLookups.environment = { id: seed.environment.id, type: "DEVELOPMENT", slug: "dev" }; +} + +describe("sessions.$sessionId.$io SSE subscribe loader under control-plane replica lag", () => { + heteroPostgresTest( + "establishes the stream for a Session not yet replicated via the writer fallback", + async ({ prisma14 }) => { + const prisma = prisma14 as unknown as PrismaClient; + const suffix = `sessguard_a_${seq++}`; + + const seed = await seedTenant(prisma, suffix); + const { session, run } = await seedSessionRunLinkage(prisma, seed, suffix); + stageCpLookups(seed); + + // Freeze `Session` on the replica: the just-created row is not visible there, only on the writer. + const replica = laggingReplica(prisma, [{ model: "session", mode: "missing" }]); + primaryHolder.client = prisma; + replicaHolder.client = replica.client; + + const res = (await loader({ + request: subscribeRequest(), + params: loaderParams(seed, run.friendlyId, session.friendlyId), + context: {} as never, + })) as Response; + + // Frozen replica really consulted (not a lucky primary hit); writer fallback then resolved it. + expect(replica.wasHit("session")).toBe(true); + expect(res.status).not.toBe(404); + expect(res.status).toBe(200); + expect(res.headers.get("x-stream")).toBe("established"); + const body = await res.clone().text(); + expect(body).not.toContain("Session not found"); + } + ); + + heteroPostgresTest( + "establishes the stream for a SessionRun linkage not yet replicated via the primary re-read", + async ({ prisma14 }) => { + const prisma = prisma14 as unknown as PrismaClient; + const suffix = `sessguard_b_${seq++}`; + + const seed = await seedTenant(prisma, suffix); + const { session, run } = await seedSessionRunLinkage(prisma, seed, suffix); + stageCpLookups(seed); + + // Freeze `SessionRun` on the replica: the linkage row is missing there, but the Session is present. + const replica = laggingReplica(prisma, [{ model: "sessionRun", mode: "missing" }]); + primaryHolder.client = prisma; + replicaHolder.client = replica.client; + + const res = (await loader({ + request: subscribeRequest(), + params: loaderParams(seed, run.friendlyId, session.friendlyId), + context: {} as never, + })) as Response; + + // Frozen replica really consulted; primary re-read then resolved the linkage. + expect(replica.wasHit("sessionRun")).toBe(true); + expect(res.status).not.toBe(404); + expect(res.status).toBe(200); + expect(res.headers.get("x-stream")).toBe("established"); + const body = await res.clone().text(); + expect(body).not.toContain("Session not found for run"); + } + ); +}); diff --git a/apps/webapp/test/realtimeStreamRoutes.replicaLag.test.ts b/apps/webapp/test/realtimeStreamRoutes.replicaLag.test.ts new file mode 100644 index 00000000000..fb8cf5caf37 --- /dev/null +++ b/apps/webapp/test/realtimeStreamRoutes.replicaLag.test.ts @@ -0,0 +1,707 @@ +// Property: every realtime-stream route resolves a live run under replica lag. Each case drives the +// REAL exported loader/action from a webapp route module against a REAL Postgres testcontainer whose +// replica is FROZEN via the shared `laggingReplica` (taskRun "missing"). The REAL webapp `runStore` +// singleton is built over the mocked `~/db.server` handles as the single-DB webapp holds it +// (prisma = writer/primary, $replica = frozen replica; split handles undefined → single passthrough +// store). Only peripherals are mocked: bearer/rbac + dashboard-session auth, project/env slug +// resolvers, the downstream realtime backends (S2 / native), the run engine + waitpoint caches, the +// control-plane env resolver. The run READ and the found/not-found decision run for real. +// +// Each route reads the run from the REPLICA (`runStore.findRun(..., $replica)` or a client-less +// findRun served from readOnlyPrisma). Under lag a run present on the owning PRIMARY reads back null; +// the primary re-read (`run ?? runStore.findRunOnPrimary(where, args)`) resolves it, so every case +// gets PAST the run 404 (asserted on the caller's real Response). + +import { heteroPostgresTest, laggingReplica } from "@internal/testcontainers"; +import type { PrismaClient } from "@trigger.dev/database"; +import { describe, expect, vi } from "vitest"; + +vi.setConfig({ testTimeout: 60_000, hookTimeout: 60_000 }); + +// --- Holders wired per-test into the mocked module singletons ------------------------------------ +// `primaryHolder.client` -> the real container (writer / owning primary). +// `replicaHolder.client` -> the frozen replica over the SAME container (taskRun reads come back null). +const primaryHolder = vi.hoisted(() => ({ client: undefined as any })); +const replicaHolder = vi.hoisted(() => ({ client: undefined as any })); + +// The authenticated environment the (mocked) auth resolves to + the dashboard project/session it +// resolves. Filled per-test from the seeded tenant so the routes' run reads scope correctly. +const ctx = vi.hoisted(() => ({ + environment: undefined as any, + project: undefined as any, + session: undefined as any, + userId: "user_realtime_guard", +})); + +// A single marker class used for BOTH the S2 backend mock and the getRealtimeStreamInstance mock, so +// the routes' `instanceof S2RealtimeStreams` checks pass. Every method returns a marker Response / +// benign value — reaching any of them proves the run read was tolerated (got past the 404). +const streamMarker = vi.hoisted(() => { + class S2Marker { + streamResponse() { + return new Response("marker:streamResponse", { status: 200 }); + } + streamResponseFromSessionStream() { + return new Response("marker:sessionStream", { status: 200 }); + } + ingestData() { + return new Response("marker:ingestData", { status: 200 }); + } + async initializeStream() { + return { responseHeaders: {} as Record }; + } + async appendPart() { + return undefined; + } + async getLastChunkIndex() { + return 0; + } + async readRecords() { + return [] as any[]; + } + async readSessionStreamRecords() { + return [] as any[]; + } + } + return { S2Marker }; +}); + +// ~/db.server: stable lazy proxies (captured by the runStore singleton at first import) that forward +// every access to the current test's client. Run-ops split handles undefined → single passthrough +// store (writer = prisma, replica = $replica) — the exact single-DB webapp topology this property +// lives in. Never mocks the DB itself. +vi.mock("~/db.server", async () => { + const { Prisma } = await import("@trigger.dev/database"); + const lazyProxy = (holder: { client: any }, label: string) => + new Proxy( + {}, + { + get(_t, prop) { + if (!holder.client) throw new Error(`${label} not set for this test`); + const value = holder.client[prop]; + if (value !== null && typeof value === "object") { + return new Proxy(value, { get: (_d, method) => holder.client[prop][method] }); + } + return value; + }, + } + ); + return { + prisma: lazyProxy(primaryHolder, "primaryHolder.client"), + $replica: lazyProxy(replicaHolder, "replicaHolder.client"), + runOpsNewPrismaClient: undefined, + runOpsNewReplicaClient: undefined, + runOpsLegacyPrisma: undefined, + runOpsLegacyReplica: undefined, + sqlDatabaseSchema: Prisma.sql([`public`]), + }; +}); + +// Bearer/JWT auth for the API-builder routes (createLoaderApiRoute / createActionApiRoute call +// rbac.authenticateBearer). Passing auth + permissive ability; the run read is unaffected. +vi.mock("~/services/rbac.server", () => ({ + rbac: { + authenticateBearer: async () => ({ + ok: true, + environment: ctx.environment, + subject: { type: "private" }, + ability: { can: () => true, canSuper: () => true }, + jwt: undefined, + }), + }, +})); + +// Dashboard-session auth + slug resolvers for the resources.* routes (peripheral). +vi.mock("~/services/session.server", () => ({ + requireUserId: async () => ctx.userId, + getUserId: async () => ctx.userId, +})); +vi.mock("~/models/project.server", () => ({ + findProjectBySlug: async () => ctx.project, +})); +vi.mock("~/models/runtimeEnvironment.server", () => ({ + findEnvironmentBySlug: async () => ctx.environment, +})); + +// Downstream realtime backends: markers. Reaching them proves the run read was tolerated. +vi.mock("~/services/realtime/s2realtimeStreams.server", () => ({ + S2RealtimeStreams: streamMarker.S2Marker, +})); +vi.mock("~/services/realtime/v1StreamsGlobal.server", () => ({ + getRealtimeStreamInstance: () => new streamMarker.S2Marker(), +})); +vi.mock("~/services/realtime/resolveRealtimeStreamClient.server", () => ({ + resolveRealtimeStreamClient: async () => ({ + streamRun: async () => new Response("marker:streamRun", { status: 200 }), + }), +})); + +// Session resolution (peripheral): return a marker session so the run↔session linkage check is +// what runs next (and, absent a seeded SessionRun row, yields a DISTINCT 404 body we assert on). +vi.mock("~/services/realtime/sessions.server", () => ({ + resolveSessionByIdOrExternalId: async () => ctx.session, + resolveSessionWithWriterFallback: async () => ctx.session, + canonicalSessionAddressingKey: () => "addr_marker", + isSessionFriendlyIdForm: () => false, + serializeSession: (s: any) => s, +})); + +// Control-plane env resolver used by the plain-action stream route + the streamKey dashboard route. +vi.mock("~/v3/runOpsMigration/controlPlaneResolver.server", () => ({ + controlPlaneResolver: { + resolveAuthenticatedEnv: async () => ctx.environment, + }, +})); + +// Run engine + waitpoint caches for the .wait() routes (peripheral). +vi.mock("~/v3/runEngine.server", () => ({ + engine: { + createManualWaitpoint: async () => ({ waitpoint: { id: "waitpoint_marker" }, isCached: false }), + completeWaitpoint: async () => ({}), + }, +})); +vi.mock("~/services/inputStreamWaitpointCache.server", () => ({ + getInputStreamWaitpoint: async () => null, + setInputStreamWaitpoint: async () => undefined, + deleteInputStreamWaitpoint: async () => undefined, +})); +vi.mock("~/services/sessionStreamWaitpointCache.server", () => ({ + addSessionStreamWaitpoint: async () => undefined, + removeSessionStreamWaitpoint: async () => undefined, +})); +vi.mock("~/models/waitpointTag.server", () => ({ + createWaitpointTag: async () => ({}), + MAX_TAGS_PER_WAITPOINT: 10, +})); + +vi.mock("~/services/httpAsyncStorage.server", () => ({ + getRequestAbortSignal: () => undefined, +})); +vi.mock("~/services/logger.server", () => ({ + logger: { warn: vi.fn(), info: vi.fn(), error: vi.fn(), debug: vi.fn() }, +})); + +import { PostgresRunStore } from "@internal/run-store"; +import type { CreateRunInput } from "@internal/run-store"; + +// The REAL exported callers under guard. +import { loader as realtimeRunLoader } from "~/routes/realtime.v1.runs.$runId"; +import { loader as streamStreamIdLoader } from "~/routes/realtime.v1.streams.$runId.$streamId"; +import { + action as streamTargetAction, + loader as streamTargetLoader, +} from "~/routes/realtime.v1.streams.$runId.$target.$streamId"; +import { action as streamTargetAppendAction } from "~/routes/realtime.v1.streams.$runId.$target.$streamId.append"; +import { + action as streamInputAction, + loader as streamInputLoader, +} from "~/routes/realtime.v1.streams.$runId.input.$streamId"; +import { action as inputStreamsWaitAction } from "~/routes/api.v1.runs.$runFriendlyId.input-streams.wait"; +import { action as sessionStreamsWaitAction } from "~/routes/api.v1.runs.$runFriendlyId.session-streams.wait"; +import { loader as dashSessionIoLoader } from "~/routes/resources.orgs.$organizationSlug.projects.$projectParam.env.$envParam.runs.$runParam.realtime.v1.sessions.$sessionId.$io"; +import { loader as dashStreamLoader } from "~/routes/resources.orgs.$organizationSlug.projects.$projectParam.env.$envParam.runs.$runParam.realtime.v1.streams.$runId.$streamId"; +import { loader as dashInputStreamLoader } from "~/routes/resources.orgs.$organizationSlug.projects.$projectParam.env.$envParam.runs.$runParam.realtime.v1.streams.$runId.input.$streamId"; +import { loader as dashStreamKeyLoader } from "~/routes/resources.orgs.$organizationSlug.projects.$projectParam.env.$envParam.runs.$runParam.streams.$streamKey/route"; + +let seq = 0; + +async function seedTenant(prisma: PrismaClient, suffix: string) { + const organization = await prisma.organization.create({ + data: { title: `Org ${suffix}`, slug: `org-${suffix}` }, + }); + const project = await prisma.project.create({ + data: { + name: `Project ${suffix}`, + slug: `project-${suffix}`, + externalRef: `proj_${suffix}`, + organizationId: organization.id, + }, + }); + const environment = await prisma.runtimeEnvironment.create({ + data: { + type: "DEVELOPMENT", + slug: "dev", + projectId: project.id, + organizationId: organization.id, + apiKey: `tr_dev_${suffix}`, + pkApiKey: `pk_dev_${suffix}`, + shortcode: `short_${suffix}`, + }, + }); + // The streamKey dashboard route queries `$replica.project` with a member filter directly (not the + // mocked findProjectBySlug), so seed a real user + org membership the resolved userId matches. + const userId = `user_${suffix}`; + await prisma.user.create({ + data: { id: userId, email: `guard-${suffix}@example.com`, authenticationMethod: "MAGIC_LINK" }, + }); + await prisma.orgMember.create({ + data: { userId, organizationId: organization.id, role: "ADMIN" }, + }); + return { organization, project, environment, userId }; +} + +function buildCreateRunInput(p: { + runId: string; + friendlyId: string; + organizationId: string; + projectId: string; + runtimeEnvironmentId: string; +}): CreateRunInput { + return { + data: { + id: p.runId, + engine: "V2", + status: "EXECUTING", + friendlyId: p.friendlyId, + runtimeEnvironmentId: p.runtimeEnvironmentId, + environmentType: "DEVELOPMENT", + organizationId: p.organizationId, + projectId: p.projectId, + taskIdentifier: "my-task", + payload: '{"hello":"world"}', + payloadType: "application/json", + context: { foo: "bar" }, + traceContext: { trace: "ctx" }, + traceId: "trace_1", + spanId: "span_1", + runTags: [], + queue: "task/my-task", + isTest: false, + taskEventStore: "taskEvent", + depth: 0, + createdAt: new Date("2024-01-01T00:00:00.000Z"), + }, + snapshot: { + engine: "V2", + executionStatus: "EXECUTING", + description: "Run is executing", + runStatus: "EXECUTING", + environmentId: p.runtimeEnvironmentId, + environmentType: "DEVELOPMENT", + projectId: p.projectId, + organizationId: p.organizationId, + }, + }; +} + +// Seed a live run on the PRIMARY only, freeze the replica, and wire the module singletons. +async function setupLiveRunWithFrozenReplica(prisma: PrismaClient) { + const suffix = `rt_${seq++}`; + const seed = await seedTenant(prisma, suffix); + const runId = `run_${suffix}_${"a".repeat(16)}`; + const friendlyId = `run_${suffix}`; + + const writerStore = new PostgresRunStore({ prisma, readOnlyPrisma: prisma }); + await writerStore.createRun( + buildCreateRunInput({ + runId, + friendlyId, + organizationId: seed.organization.id, + projectId: seed.project.id, + runtimeEnvironmentId: seed.environment.id, + }) + ); + + const replica = laggingReplica(prisma, [{ model: "taskRun", mode: "missing" }]); + primaryHolder.client = prisma; + replicaHolder.client = replica.client; + + ctx.environment = { + id: seed.environment.id, + type: "DEVELOPMENT", + slug: "dev", + apiKey: seed.environment.apiKey, + organizationId: seed.organization.id, + projectId: seed.project.id, + organization: { id: seed.organization.id, slug: seed.organization.slug, featureFlags: {} }, + project: { + id: seed.project.id, + slug: seed.project.slug, + externalRef: seed.project.externalRef, + }, + }; + ctx.project = { id: seed.project.id, slug: seed.project.slug }; + ctx.session = { id: "session_marker", friendlyId: "session_marker", externalId: null }; + ctx.userId = seed.userId; + + return { seed, runId, friendlyId, replica }; +} + +// Normalize: some dashboard loaders THROW a Response on 404, others RETURN it. +async function invoke(fn: () => Promise): Promise { + try { + return await fn(); + } catch (e) { + if (e instanceof Response) return e; + throw e; + } +} + +// The API-builder enforces `maxContentLength` by REQUIRING a content-length header (missing → 413 +// before the handler). Set it so the run read under test is actually reached. +function jsonPost(url: string, body: unknown) { + const payload = JSON.stringify(body); + return new Request(url, { + method: "POST", + headers: { + "content-type": "application/json", + "content-length": String(Buffer.byteLength(payload)), + }, + body: payload, + }); +} + +function rawPost(url: string, body: string) { + return new Request(url, { + method: "POST", + headers: { "content-length": String(Buffer.byteLength(body)) }, + body, + }); +} + +describe("realtime-stream routes under replica lag", () => { + // 1. realtime.v1.runs.$runId loader (createLoaderApiRoute findResource) — SDK subscribeToRun. + heteroPostgresTest( + "realtime run loader resolves a live run under replica lag", + async ({ prisma14 }) => { + const prisma = prisma14 as unknown as PrismaClient; + const { friendlyId, replica } = await setupLiveRunWithFrozenReplica(prisma); + + const res = await invoke( + () => + realtimeRunLoader({ + request: new Request(`http://localhost/realtime/v1/runs/${friendlyId}`), + params: { runId: friendlyId }, + context: {} as never, + }) as Promise + ); + + expect(replica.wasHit("taskRun")).toBe(true); + // findResource re-reads the primary → run found → handler reaches the stream client marker. + expect(res.status).toBe(200); + expect(await res.clone().text()).toBe("marker:streamRun"); + } + ); + + // 2. realtime.v1.streams.$streamId loader (createLoaderApiRoute findResource) — SSE subscribe. + heteroPostgresTest( + "realtime stream loader resolves a live run under replica lag", + async ({ prisma14 }) => { + const prisma = prisma14 as unknown as PrismaClient; + const { friendlyId, replica } = await setupLiveRunWithFrozenReplica(prisma); + + const res = await invoke( + () => + streamStreamIdLoader({ + request: new Request(`http://localhost/realtime/v1/streams/${friendlyId}/s1`), + params: { runId: friendlyId, streamId: "s1" }, + context: {} as never, + }) as Promise + ); + + expect(replica.wasHit("taskRun")).toBe(true); + expect(res.status).toBe(200); + expect(await res.clone().text()).toBe("marker:streamResponse"); + } + ); + + // 4. realtime.v1.streams.$target.$streamId action (createActionApiRoute). + heteroPostgresTest( + "realtime stream-target action resolves a live run under replica lag", + async ({ prisma14 }) => { + const prisma = prisma14 as unknown as PrismaClient; + const { friendlyId, replica } = await setupLiveRunWithFrozenReplica(prisma); + + const res = await invoke( + () => + streamTargetAction({ + request: new Request(`http://localhost/realtime/v1/streams/${friendlyId}/self/s1`, { + method: "POST", + body: "chunk-data", + }), + params: { runId: friendlyId, target: "self", streamId: "s1" }, + context: {} as never, + }) as Promise + ); + + expect(replica.wasHit("taskRun")).toBe(true); + expect(res.status).toBe(200); + expect(await res.clone().text()).toBe("marker:ingestData"); + } + ); + + // 5. realtime.v1.streams.$target.$streamId loader (createLoaderApiRoute findResource, HEAD). + heteroPostgresTest( + "realtime stream-target loader resolves a live run under replica lag", + async ({ prisma14 }) => { + const prisma = prisma14 as unknown as PrismaClient; + const { friendlyId, replica } = await setupLiveRunWithFrozenReplica(prisma); + + const res = await invoke( + () => + streamTargetLoader({ + request: new Request(`http://localhost/realtime/v1/streams/${friendlyId}/self/s1`, { + method: "HEAD", + }), + params: { runId: friendlyId, target: "self", streamId: "s1" }, + context: {} as never, + }) as Promise + ); + + expect(replica.wasHit("taskRun")).toBe(true); + // Run found → handler reaches the HEAD getLastChunkIndex path (200 + X-Last-Chunk-Index). + expect(res.status).toBe(200); + expect(res.headers.get("X-Last-Chunk-Index")).toBe("0"); + } + ); + + // 6. realtime.v1.streams.$target.$streamId.append action (createActionApiRoute). + heteroPostgresTest( + "realtime stream-target append action resolves a live run under replica lag", + async ({ prisma14 }) => { + const prisma = prisma14 as unknown as PrismaClient; + const { friendlyId, replica } = await setupLiveRunWithFrozenReplica(prisma); + + const res = await invoke( + () => + streamTargetAppendAction({ + request: rawPost( + `http://localhost/realtime/v1/streams/${friendlyId}/self/s1/append`, + "part-data" + ), + params: { runId: friendlyId, target: "self", streamId: "s1" }, + context: {} as never, + }) as Promise + ); + + expect(replica.wasHit("taskRun")).toBe(true); + // The first (replica) run read is recovered via the primary re-read → the second read (already on + // primary) + appendPart marker → {ok:true} 200. + expect(res.status).toBe(200); + expect(await res.clone().json()).toEqual({ ok: true }); + } + ); + + // 7. realtime.v1.streams.input.$streamId action (createActionApiRoute, .send()). + heteroPostgresTest( + "realtime input-stream action resolves a live run under replica lag", + async ({ prisma14 }) => { + const prisma = prisma14 as unknown as PrismaClient; + const { friendlyId, replica } = await setupLiveRunWithFrozenReplica(prisma); + + const res = await invoke( + () => + streamInputAction({ + request: jsonPost(`http://localhost/realtime/v1/streams/${friendlyId}/input/s1`, { + data: { hello: "world" }, + }), + params: { runId: friendlyId, streamId: "s1" }, + context: {} as never, + }) as Promise + ); + + expect(replica.wasHit("taskRun")).toBe(true); + // Run found → appendPart marker → {ok:true}. + expect(res.status).toBe(200); + expect(await res.clone().json()).toEqual({ ok: true }); + } + ); + + // 8. realtime.v1.streams.input.$streamId loader (createLoaderApiRoute findResource, SSE tail). + heteroPostgresTest( + "realtime input-stream loader resolves a live run under replica lag", + async ({ prisma14 }) => { + const prisma = prisma14 as unknown as PrismaClient; + const { friendlyId, replica } = await setupLiveRunWithFrozenReplica(prisma); + + const res = await invoke( + () => + streamInputLoader({ + request: new Request(`http://localhost/realtime/v1/streams/${friendlyId}/input/s1`), + params: { runId: friendlyId, streamId: "s1" }, + context: {} as never, + }) as Promise + ); + + expect(replica.wasHit("taskRun")).toBe(true); + expect(res.status).toBe(200); + expect(await res.clone().text()).toBe("marker:streamResponse"); + } + ); + + // 9. api.v1 input-streams.wait action (createActionApiRoute, .wait()). + heteroPostgresTest( + "input-streams wait action resolves a live run under replica lag", + async ({ prisma14 }) => { + const prisma = prisma14 as unknown as PrismaClient; + const { friendlyId, replica } = await setupLiveRunWithFrozenReplica(prisma); + + const res = await invoke( + () => + inputStreamsWaitAction({ + request: jsonPost(`http://localhost/api/v1/runs/${friendlyId}/input-streams/wait`, { + streamId: "s1", + }), + params: { runFriendlyId: friendlyId }, + context: {} as never, + }) as Promise + ); + + expect(replica.wasHit("taskRun")).toBe(true); + // Run found → createManualWaitpoint marker → waitpoint json. + expect(res.status).toBe(200); + expect(await res.clone().json()).toMatchObject({ isCached: false }); + } + ); + + // 10. api.v1 session-streams.wait action (createActionApiRoute, .wait()). + heteroPostgresTest( + "session-streams wait action resolves a live run under replica lag", + async ({ prisma14 }) => { + const prisma = prisma14 as unknown as PrismaClient; + const { friendlyId, replica } = await setupLiveRunWithFrozenReplica(prisma); + + const res = await invoke( + () => + sessionStreamsWaitAction({ + request: jsonPost(`http://localhost/api/v1/runs/${friendlyId}/session-streams/wait`, { + session: "session_marker", + io: "out", + }), + params: { runFriendlyId: friendlyId }, + context: {} as never, + }) as Promise + ); + + expect(replica.wasHit("taskRun")).toBe(true); + // Run found → createManualWaitpoint marker → waitpoint json. + expect(res.status).toBe(200); + expect(await res.clone().json()).toMatchObject({ isCached: false }); + } + ); + + // 11. dashboard sessions.$sessionId.$io loader. + heteroPostgresTest( + "dashboard session-io loader resolves a live run under replica lag", + async ({ prisma14 }) => { + const prisma = prisma14 as unknown as PrismaClient; + const { friendlyId, replica } = await setupLiveRunWithFrozenReplica(prisma); + + const res = await invoke( + () => + dashSessionIoLoader({ + request: new Request( + `http://localhost/resources/.../runs/${friendlyId}/realtime/v1/sessions/session_marker/out` + ), + params: { + organizationSlug: ctx.environment.organization.slug, + projectParam: ctx.project.slug, + envParam: "dev", + runParam: friendlyId, + sessionId: "session_marker", + io: "out", + }, + context: {} as never, + }) as Promise + ); + + expect(replica.wasHit("taskRun")).toBe(true); + const body = await res.clone().text(); + // Run read recovered via the primary re-read → past the run 404 → the next gate (run↔session + // linkage, no seeded SessionRun) returns "Session not found for run". Asserting on body text pins + // this to the run-read decision specifically. + expect(body).not.toContain("Run not found"); + expect(body).toContain("Session not found for run"); + } + ); + + // 12. dashboard streams.$streamId loader. + heteroPostgresTest( + "dashboard stream loader resolves a live run under replica lag", + async ({ prisma14 }) => { + const prisma = prisma14 as unknown as PrismaClient; + const { friendlyId, replica } = await setupLiveRunWithFrozenReplica(prisma); + + const res = await invoke( + () => + dashStreamLoader({ + request: new Request(`http://localhost/resources/.../streams/${friendlyId}/s1`), + params: { + organizationSlug: ctx.environment.organization.slug, + projectParam: ctx.project.slug, + envParam: "dev", + runParam: friendlyId, + runId: friendlyId, + streamId: "s1", + }, + context: {} as never, + }) as Promise + ); + + expect(replica.wasHit("taskRun")).toBe(true); + expect(res.status).toBe(200); + expect(await res.clone().text()).toBe("marker:streamResponse"); + } + ); + + // 13. dashboard streams.input.$streamId loader. + heteroPostgresTest( + "dashboard input-stream loader resolves a live run under replica lag", + async ({ prisma14 }) => { + const prisma = prisma14 as unknown as PrismaClient; + const { friendlyId, replica } = await setupLiveRunWithFrozenReplica(prisma); + + const res = await invoke( + () => + dashInputStreamLoader({ + request: new Request(`http://localhost/resources/.../streams/${friendlyId}/input/s1`), + params: { + organizationSlug: ctx.environment.organization.slug, + projectParam: ctx.project.slug, + envParam: "dev", + runParam: friendlyId, + runId: friendlyId, + streamId: "s1", + }, + context: {} as never, + }) as Promise + ); + + expect(replica.wasHit("taskRun")).toBe(true); + expect(res.status).toBe(200); + expect(await res.clone().text()).toBe("marker:streamResponse"); + } + ); + + // 14. dashboard streams.$streamKey loader (CLIENT-LESS findRun → replica). + heteroPostgresTest( + "dashboard stream-key loader resolves a live run under replica lag", + async ({ prisma14 }) => { + const prisma = prisma14 as unknown as PrismaClient; + const { friendlyId, replica } = await setupLiveRunWithFrozenReplica(prisma); + + const res = await invoke( + () => + dashStreamKeyLoader({ + request: new Request( + `http://localhost/resources/.../streams/${friendlyId}/streams/my-stream` + ), + params: { + organizationSlug: ctx.environment.organization.slug, + projectParam: ctx.project.slug, + envParam: "dev", + runParam: friendlyId, + streamKey: "my-stream", + }, + context: {} as never, + }) as Promise + ); + + expect(replica.wasHit("taskRun")).toBe(true); + // The client-less findRun (replica) is recovered via findRunOnPrimary → env resolves (slug + // matches) → streamResponse marker. + expect(res.status).toBe(200); + expect(await res.clone().text()).toBe("marker:streamResponse"); + } + ); +}); diff --git a/apps/webapp/test/replayRouteReplicaLag.guard.test.ts b/apps/webapp/test/replayRouteReplicaLag.guard.test.ts new file mode 100644 index 00000000000..e9bba9d666c --- /dev/null +++ b/apps/webapp/test/replayRouteReplicaLag.guard.test.ts @@ -0,0 +1,261 @@ +// Property: the replay ACTION resolves a source run that exists only on the owning primary and REPLAYS +// it (findRun by friendlyId, re-read on `prisma` when the replica misses) rather than reporting "Run +// not found". Drives the REAL exported route `action` on a real split topology +// (heteroRunOpsPostgresTest) with the owning legacy replica FROZEN and an empty mollifier buffer, so +// the primary re-read is the only path that resolves the run. Only orthogonal deps are stubbed (the +// auth/RBAC wrapper, downstream ReplayTaskRunService.call, the mollifier buffer, redirect helpers). + +import { heteroRunOpsPostgresTest, laggingReplica } from "@internal/testcontainers"; +import type { PrismaClient } from "@trigger.dev/database"; +import type { RunOpsPrismaClient } from "@internal/run-ops-database"; +import { describe, expect, vi } from "vitest"; + +// ---- hoisted holders (vi.mock factories run before imports) --------------------------------- +const routerHolder = vi.hoisted(() => ({ router: undefined as any })); +const primaryHolder = vi.hoisted(() => ({ client: undefined as any })); +const replayCallSpy = vi.hoisted(() => vi.fn()); + +// The route reads the source run through the `runStore` singleton. Point it at the split +// RoutingRunStore we build per-test (frozen legacy replica + real primary). +vi.mock("~/v3/runStore.server", () => ({ + runStore: new Proxy( + {}, + { + get(_t, prop) { + const router = routerHolder.router; + if (!router) throw new Error("routerHolder.router not set for this test"); + const value = (router as any)[prop]; + return typeof value === "function" ? value.bind(router) : value; + }, + } + ), +})); + +// The `prisma` singleton is the client the action passes as the primary re-read override, and the +// one controlPlaneResolver + the org-membership auth query read. Point every db.server handle at +// the real control-plane (PG14) container. The DB is never mocked — the proxy forwards to a real +// testcontainer client. (Re-resolving delegates per access mirrors waitpointCallback.controlPlane.) +vi.mock("~/db.server", async () => { + const { Prisma } = await import("@trigger.dev/database"); + const lazyProxy = (holder: { client: any }, label: string) => + new Proxy( + {}, + { + get(_t, prop) { + if (!holder.client) throw new Error(`${label} not set for this test`); + const value = holder.client[prop]; + if (value !== null && typeof value === "object") { + return new Proxy(value, { get: (_d, method) => holder.client[prop][method] }); + } + return value; + }, + } + ); + const cp = lazyProxy(primaryHolder, "primaryHolder.client"); + return { + prisma: cp, + $replica: cp, + runOpsNewPrisma: cp, + runOpsNewReplica: cp, + runOpsLegacyPrisma: cp, + runOpsLegacyReplica: cp, + runOpsNewPrismaClient: cp, + runOpsNewReplicaClient: cp, + runOpsLegacyPrismaClient: cp, + runOpsLegacyReplicaClient: cp, + runOpsSplitReadEnabled: false, + sqlDatabaseSchema: Prisma.sql([`public`]), + Prisma, + }; +}); + +// Bypass ONLY the auth/RBAC wrapper — return the handler verbatim so the REAL action body runs. +vi.mock("~/services/routeBuilders/dashboardBuilder", () => ({ + dashboardAction: (_options: unknown, handler: unknown) => handler, + dashboardLoader: (_options: unknown, handler: unknown) => handler, +})); + +// Downstream collaborator: observe whether the action reached the replay, without triggering one. +vi.mock("~/v3/services/replayTaskRun.server", () => ({ + ReplayTaskRunService: class { + call = replayCallSpy; + }, +})); + +// Force the mollifier buffer empty so the ONLY way pgRun resolves is the primary re-read. +vi.mock("~/v3/mollifier/mollifierBuffer.server", () => ({ + getMollifierBuffer: () => null, +})); + +// Make the redirect helpers session-free so the returned Response is deterministic regardless of +// SESSION_SECRET; keep every other export real. +vi.mock("~/models/message.server", async (importOriginal) => { + const actual = (await importOriginal()) as Record; + return { + ...actual, + redirectWithSuccessMessage: (path: string) => + new Response(null, { status: 302, headers: { location: path } }), + redirectWithErrorMessage: (path: string) => + new Response(null, { status: 302, headers: { location: path } }), + }; +}); + +import { PostgresRunStore, RoutingRunStore } from "@internal/run-store"; +import { action } from "~/routes/resources.taskruns.$runParam.replay"; + +vi.setConfig({ testTimeout: 60_000, hookTimeout: 60_000 }); + +const CUID_25 = "c".repeat(25); // classifies LEGACY + +let seedN = 0; +async function seedTenant(prisma: PrismaClient) { + const suffix = `replay_guard_${seedN++}`; + const organization = await prisma.organization.create({ + data: { title: `Org ${suffix}`, slug: `org-${suffix}` }, + }); + const project = await prisma.project.create({ + data: { + name: `Project ${suffix}`, + slug: `project-${suffix}`, + externalRef: `proj_${suffix}`, + organizationId: organization.id, + }, + }); + const environment = await prisma.runtimeEnvironment.create({ + data: { + type: "DEVELOPMENT", + slug: "dev", + projectId: project.id, + organizationId: organization.id, + apiKey: `tr_dev_${suffix}`, + pkApiKey: `pk_dev_${suffix}`, + shortcode: `short_${suffix}`, + }, + }); + const user = await prisma.user.create({ + data: { + email: `${suffix}@test.local`, + authenticationMethod: "MAGIC_LINK", + admin: false, + }, + }); + // Org membership: the action's authorization query is + // project.findFirst({ where: { id, organization: { members: { some: { userId } } } } }) + await prisma.orgMember.create({ + data: { userId: user.id, organizationId: organization.id, role: "ADMIN" }, + }); + return { organization, project, environment, user }; +} + +function sourceRunData(opts: { + id: string; + friendlyId: string; + organizationId: string; + projectId: string; + runtimeEnvironmentId: string; +}) { + return { + id: opts.id, + engine: "V2" as const, + status: "COMPLETED_SUCCESSFULLY" as const, + friendlyId: opts.friendlyId, + runtimeEnvironmentId: opts.runtimeEnvironmentId, + environmentType: "DEVELOPMENT" as const, + organizationId: opts.organizationId, + projectId: opts.projectId, + taskIdentifier: "my-task", + payload: JSON.stringify({ hello: "world" }), + payloadType: "application/json", + traceContext: {}, + traceId: `trace_${opts.id}`, + spanId: `span_${opts.id}`, + queue: "task/my-task", + isTest: false, + taskEventStore: "taskEvent", + depth: 0, + }; +} + +// LEGACY-owning router whose legacy replica is frozen; the NEW store is a real (empty) store so the +// on-miss fan-out to the other store's replica also misses. Mirrors the split topology db.server +// builds in production (buildRunStore split path). +function buildRouterWithFrozenLegacyReplica(prisma14: PrismaClient, prisma17: RunOpsPrismaClient) { + const legacyReplica = laggingReplica(prisma14, [{ model: "taskRun", mode: "missing" }]); + const legacyStore = new PostgresRunStore({ + prisma: prisma14, + readOnlyPrisma: legacyReplica.client, + schemaVariant: "legacy", + }); + const newStore = new PostgresRunStore({ + prisma: prisma17 as never, + readOnlyPrisma: prisma17 as never, + schemaVariant: "dedicated", + }); + const router = new RoutingRunStore({ new: newStore, legacy: legacyStore }); + return { router, legacyReplica }; +} + +const FAILED_REDIRECT = "/failed-redirect-sentinel"; + +function replayRequest() { + const body = new URLSearchParams({ failedRedirect: FAILED_REDIRECT }); + return new Request("http://localhost/replay", { + method: "POST", + headers: { "content-type": "application/x-www-form-urlencoded" }, + body, + }); +} + +describe("replay action resolves a source run under owning-replica lag via the primary re-read", () => { + heteroRunOpsPostgresTest( + "resolves a source run that exists only on the owning primary and replays it", + async ({ prisma14, prisma17 }) => { + const seed = await seedTenant(prisma14 as unknown as PrismaClient); + const runId = `run_${CUID_25}`; + const friendlyId = "run_replay_guard"; + await (prisma14 as unknown as PrismaClient).taskRun.create({ + data: sourceRunData({ + id: runId, + friendlyId, + organizationId: seed.organization.id, + projectId: seed.project.id, + runtimeEnvironmentId: seed.environment.id, + }), + }); + + // Wire the singletons the mocked modules read. + primaryHolder.client = prisma14; + const { router, legacyReplica } = buildRouterWithFrozenLegacyReplica( + prisma14 as unknown as PrismaClient, + prisma17 + ); + routerHolder.router = router; + replayCallSpy.mockReset(); + replayCallSpy.mockResolvedValue({ + id: "run_new_internal", + friendlyId: "run_new_friendly", + spanId: "span_new", + }); + + // Drive the REAL action handler (auth wrapper stubbed to a passthrough). + const response = (await action({ + request: replayRequest(), + params: { runParam: friendlyId }, + user: { id: seed.user.id }, + } as never)) as Response; + + // The frozen replica was consulted (proves we exercised the lag path, not a warm replica). + expect(legacyReplica.wasHit()).toBe(true); + + // The primary re-read resolved the source run, so the action reached the replay with the correct + // source run (a replica-only read would leave pgRun null and report "Run not found" instead). + expect(replayCallSpy).toHaveBeenCalledTimes(1); + expect(replayCallSpy.mock.calls[0][0]).toMatchObject({ friendlyId }); + + // User-facing: a success redirect to the new run's path, NOT the failedRedirect sentinel. + expect(response.status).toBe(302); + expect(response.headers.get("location")).not.toBe(FAILED_REDIRECT); + expect(response.headers.get("location")).toContain("run_new_friendly"); + } + ); +}); diff --git a/apps/webapp/test/routesBatchGetReplicaLag.guard.test.ts b/apps/webapp/test/routesBatchGetReplicaLag.guard.test.ts new file mode 100644 index 00000000000..1777fe65c02 --- /dev/null +++ b/apps/webapp/test/routesBatchGetReplicaLag.guard.test.ts @@ -0,0 +1,397 @@ +// Verifies the routes-batch-get read paths under replica lag — five external-API call sites across two +// store methods with different default routing. Each case drives the REAL exported route caller +// (loader/action) end-to-end against a real split Postgres whose owning LEGACY replica is FROZEN via +// laggingReplica; the run-store read path is real, never mocked. Only orthogonal deps are stubbed +// (bearer auth, request-idempotency cache, realtime stream, abort signal). +// +// findBatchTaskRunByFriendlyId (GET/subscribe) routes to the REPLICA: under lag a live batch reads as +// null and the resource gate returns a 404 carrying x-should-retry, which the SDK obeys — so the property +// is that the header is "true" (retryable, self-heals through lag), matching the sibling run-get routes. +// findBatchTaskRunById (dedup) routes to the PRIMARY: under owning-replica lag the just-written batch is +// still found (frozen replica never consulted), so the retried request dedups to the cached batch (200) +// instead of minting a duplicate — proven by wasHit("batchTaskRun") === false and the 200 dedup body. + +import { heteroRunOpsPostgresTest, laggingReplica } from "@internal/testcontainers"; +import type { PrismaClient } from "@trigger.dev/database"; +import type { RunOpsPrismaClient } from "@internal/run-ops-database"; +import { PostgresRunStore, RoutingRunStore } from "@internal/run-store"; +import type { CreateBatchTaskRunData } from "@internal/run-store"; +import { describe, expect, vi } from "vitest"; + +vi.setConfig({ testTimeout: 60_000, hookTimeout: 60_000 }); + +// ---- Hoisted singleton holder + a stable Proxy that forwards every run-store method to the per-test +// REAL RoutingRunStore set in holder.store. vi.mock factories (hoisted above the imports) reference +// these, so they must be created inside vi.hoisted. +const { holder, storeProxy } = vi.hoisted(() => { + const holder = { + store: undefined as unknown, + environment: undefined as unknown, + cachedRequest: null as { id: string } | null, + }; + const storeProxy = new Proxy( + {}, + { + get(_target, prop) { + const store = holder.store as Record | undefined; + if (!store) throw new Error("test bug: holder.store not initialised before caller ran"); + const value = store[prop]; + return typeof value === "function" + ? (value as (...a: unknown[]) => unknown).bind(store) + : value; + }, + } + ); + return { holder, storeProxy }; +}); + +// db.server is imported transitively by the api-builder graph; provide inert stubs so importing the +// real routes never constructs a live Prisma client. The routes never read from these — the run-store +// read path is the injected RoutingRunStore below. +vi.mock("~/db.server", async () => { + const { Prisma } = await import("@trigger.dev/database"); + return { prisma: {}, $replica: {}, sqlDatabaseSchema: Prisma.sql([`public`]) }; +}); + +// Bearer auth (orthogonal): return ok with the seeded environment + a permissive ability. The +// friendlyId loaders 404 at the resource gate BEFORE authorization, so the ability only matters for the +// two PRIMARY action sites (authorization runs before their handler). +vi.mock("~/services/rbac.server", () => ({ + rbac: { + authenticateBearer: async () => ({ + ok: true, + environment: holder.environment, + subject: { type: "private" }, + ability: { can: () => true, canSuper: () => true }, + jwt: undefined, + }), + }, +})); + +// Inject the REAL split router as the app-level run store (used by v1/v2 batch loaders, the realtime +// loader, and the v2.tasks.batch action). +vi.mock("~/v3/runStore.server", () => ({ runStore: storeProxy })); + +// api.v3.batches reads through `engine.runStore`. Override test/setup.ts's no-op engine so `runStore` +// resolves to the injected router while any other engine access stays a no-op (dedup path touches none). +vi.mock("~/v3/runEngine.server", () => ({ + engine: new Proxy( + {}, + { + get(_target, prop) { + if (prop === "runStore") return storeProxy; + return () => Promise.resolve(undefined); + }, + } + ), +})); + +// Request-idempotency cache lookup (orthogonal): return whatever the test seeded. The REAL +// findCachedEntity callback then drives runStore.findBatchTaskRunById against the injected router. +vi.mock("~/services/requestIdempotencyInstance.server", () => ({ + requestIdempotency: { + checkRequest: async () => holder.cachedRequest, + saveRequest: async () => {}, + }, +})); + +// Realtime stream client (orthogonal). Under lag the realtime loader 404s at the resource gate +// before reaching it; stubbing it just keeps the import light. `getRequestAbortSignal` is likewise only +// reached on the found path, so httpAsyncStorage.server is left real (the logger needs getHttpContext). +vi.mock("~/services/realtime/resolveRealtimeStreamClient.server", () => ({ + resolveRealtimeStreamClient: async () => ({ + streamBatch: async () => new Response("stream", { status: 200 }), + }), +})); + +// The REAL callers under test. +import { loader as loaderV1 } from "~/routes/api.v1.batches.$batchId"; +import { loader as loaderV2 } from "~/routes/api.v2.batches.$batchId"; +import { loader as loaderRealtime } from "~/routes/realtime.v1.batches.$batchId"; +import { action as actionTasksBatch } from "~/routes/api.v2.tasks.batch"; +import { action as actionCreateBatch } from "~/routes/api.v3.batches"; + +// A cuid (25 chars after the `batch_` prefix) classifies LEGACY, so the batch is owned by the legacy +// (control-plane) store; the client-less reads then land on the LEGACY leg. +const CUID_25 = "c".repeat(25); + +let seq = 0; + +async function seedEnvironment(prisma: PrismaClient, suffix: string) { + const organization = await prisma.organization.create({ + data: { title: `Org ${suffix}`, slug: `org-${suffix}` }, + }); + const project = await prisma.project.create({ + data: { + name: `Project ${suffix}`, + slug: `project-${suffix}`, + externalRef: `proj_${suffix}`, + organizationId: organization.id, + }, + }); + const environment = await prisma.runtimeEnvironment.create({ + data: { + type: "DEVELOPMENT", + slug: "dev", + projectId: project.id, + organizationId: organization.id, + apiKey: `tr_dev_${suffix}`, + pkApiKey: `pk_dev_${suffix}`, + shortcode: `short_${suffix}`, + }, + }); + return { organization, project, environment }; +} + +// Build the RoutingRunStore exactly as the webapp holds it, with the LEGACY replica frozen in "missing" +// mode for the batch models: a REPLICA-routed batch read comes back null AND flips wasHit — so a null +// result + wasHit(true) proves REPLICA routing (the friendlyId reads above), and a correct row + +// wasHit(false) proves PRIMARY routing (the dedup reads above). +function buildLaggingRouter(prisma14: PrismaClient, prisma17: RunOpsPrismaClient) { + const legacyReplica = laggingReplica(prisma14, [ + { model: "batchTaskRun", mode: "missing" }, + { model: "batchTaskRunItem", mode: "missing" }, + ]); + const legacyStore = new PostgresRunStore({ + prisma: prisma14, + readOnlyPrisma: legacyReplica.client, + schemaVariant: "legacy", + }); + const newStore = new PostgresRunStore({ + prisma: prisma17 as never, + readOnlyPrisma: prisma17 as never, + schemaVariant: "dedicated", + }); + const router = new RoutingRunStore({ new: newStore, legacy: legacyStore }); + return { router, legacyStore, legacyReplica }; +} + +function batchData(overrides: Partial): CreateBatchTaskRunData { + return { + id: `batch_${CUID_25}`, + friendlyId: "batch_route_get_f", + runtimeEnvironmentId: "PLACEHOLDER", + status: "PENDING", + runCount: 1, + runIds: ["run_child_1"], + expectedCount: 1, + batchVersion: "runengine:v2", + sealed: false, + ...overrides, + }; +} + +function getRequest(url: string, apiKey: string) { + return new Request(url, { headers: { Authorization: `Bearer ${apiKey}` } }); +} + +// The slim AuthenticatedEnvironment the routes + tenantContext read. Only orthogonal fields — the +// actual batch read is keyed off `.id`. +function authEnv(seed: Awaited>) { + return { + id: seed.environment.id, + apiKey: seed.environment.apiKey, + type: seed.environment.type, + slug: seed.environment.slug, + organization: { + id: seed.organization.id, + slug: seed.organization.slug, + title: seed.organization.title, + }, + project: { + id: seed.project.id, + slug: seed.project.slug, + externalRef: seed.project.externalRef, + }, + }; +} + +describe("routes-batch-get callers under a lagging replica", () => { + // ── api.v1.batches loader — findBatchTaskRunByFriendlyId (REPLICA) ─────────────────── + heteroRunOpsPostgresTest( + "api.v1.batches loader: a live batch stale on the replica returns a retryable 404 (x-should-retry:true)", + async ({ prisma14, prisma17 }) => { + const { router, legacyStore, legacyReplica } = buildLaggingRouter(prisma14, prisma17); + const suffix = `bget_v1_${seq++}`; + const seed = await seedEnvironment(prisma14, suffix); + const friendlyId = `batch_${suffix}_f`; + await legacyStore.createBatchTaskRun( + batchData({ friendlyId, runtimeEnvironmentId: seed.environment.id }) + ); + + holder.store = router; + holder.environment = authEnv(seed); + + const res = (await loaderV1({ + request: getRequest( + `http://localhost/api/v1/batches/${friendlyId}`, + seed.environment.apiKey + ), + params: { batchId: friendlyId }, + context: {} as never, + })) as Response; + + // The owning REPLICA was genuinely consulted and (frozen) missed. + expect(legacyReplica.wasHit("batchTaskRun")).toBe(true); + // Real loader output: a 404 for a LIVE batch stale on the replica... + expect(res.status).toBe(404); + // ...but retryable → the SDK retries through lag until the replica catches up. + expect(res.headers.get("x-should-retry")).toBe("true"); + } + ); + + // ── api.v2.batches loader — findBatchTaskRunByFriendlyId (REPLICA) ─────────────────── + heteroRunOpsPostgresTest( + "api.v2.batches loader: client-less replica read, stale-null returns a retryable 404", + async ({ prisma14, prisma17 }) => { + const { router, legacyStore, legacyReplica } = buildLaggingRouter(prisma14, prisma17); + const suffix = `bget_v2_${seq++}`; + const seed = await seedEnvironment(prisma14, suffix); + const friendlyId = `batch_${suffix}_f`; + await legacyStore.createBatchTaskRun( + batchData({ + friendlyId, + runtimeEnvironmentId: seed.environment.id, + processingCompletedAt: new Date(), + }) + ); + + holder.store = router; + holder.environment = authEnv(seed); + + const res = (await loaderV2({ + request: getRequest( + `http://localhost/api/v2/batches/${friendlyId}`, + seed.environment.apiKey + ), + params: { batchId: friendlyId }, + context: {} as never, + })) as Response; + + expect(legacyReplica.wasHit("batchTaskRun")).toBe(true); + expect(res.status).toBe(404); + expect(res.headers.get("x-should-retry")).toBe("true"); + } + ); + + // ── realtime.v1.batches loader — findBatchTaskRunByFriendlyId (REPLICA) ───────────── + heteroRunOpsPostgresTest( + "realtime.v1.batches loader: stale-null returns a retryable 404 at the resource gate before streamBatch", + async ({ prisma14, prisma17 }) => { + const { router, legacyStore, legacyReplica } = buildLaggingRouter(prisma14, prisma17); + const suffix = `bget_rt_${seq++}`; + const seed = await seedEnvironment(prisma14, suffix); + const friendlyId = `batch_${suffix}_f`; + const batchId = `batch_${CUID_25}`; + await legacyStore.createBatchTaskRun( + batchData({ id: batchId, friendlyId, runtimeEnvironmentId: seed.environment.id }) + ); + + holder.store = router; + holder.environment = authEnv(seed); + + const res = (await loaderRealtime({ + request: getRequest( + `http://localhost/realtime/v1/batches/${friendlyId}`, + seed.environment.apiKey + ), + params: { batchId: friendlyId }, + context: {} as never, + })) as Response; + + expect(legacyReplica.wasHit("batchTaskRun")).toBe(true); + // 404 at the resource gate — the subscription never starts (streamBatch would return a 200 body). + expect(res.status).toBe(404); + expect(res.headers.get("x-should-retry")).toBe("true"); + } + ); + + // ── api.v2.tasks.batch action — findBatchTaskRunById (PRIMARY) ─────────────────────────────────────── + heteroRunOpsPostgresTest( + "api.v2.tasks.batch action: request-idempotency dedup reads the owning primary, finding the cached batch under lag (200 dedup, no duplicate)", + async ({ prisma14, prisma17 }) => { + const { router, legacyStore, legacyReplica } = buildLaggingRouter(prisma14, prisma17); + const suffix = `bget_tb_${seq++}`; + const seed = await seedEnvironment(prisma14, suffix); + const batchId = `batch_${CUID_25}`; + const friendlyId = `batch_${suffix}_f`; + await legacyStore.createBatchTaskRun( + batchData({ + id: batchId, + friendlyId, + runtimeEnvironmentId: seed.environment.id, + runCount: 5, + }) + ); + + holder.store = router; + holder.environment = authEnv(seed); + holder.cachedRequest = { id: batchId }; + + const body = JSON.stringify({ items: [{ task: "my-task" }] }); + const res = (await actionTasksBatch({ + request: new Request("http://localhost/api/v2/tasks/batch", { + method: "POST", + headers: { + "content-type": "application/json", + "content-length": String(Buffer.byteLength(body)), + authorization: `Bearer ${seed.environment.apiKey}`, + "x-trigger-request-idempotency-key": "req_idem_tb", + }, + body, + }), + params: {}, + context: {} as never, + })) as Response; + + // PRIMARY routing tolerance: the frozen owning replica was NEVER consulted... + expect(legacyReplica.wasHit("batchTaskRun")).toBe(false); + // ...and the real action returned the cached batch (dedup), not a duplicate create. + expect(res.status).toBe(200); + expect(await res.json()).toEqual({ id: friendlyId, runCount: 5 }); + } + ); + + // ── api.v3.batches action — findBatchTaskRunById (PRIMARY) ─────────────────────────────────────────── + heteroRunOpsPostgresTest( + "api.v3.batches action: create-batch idempotency dedup reads the primary, finding the cached batch under lag (isCached 200, no duplicate)", + async ({ prisma14, prisma17 }) => { + const { router, legacyStore, legacyReplica } = buildLaggingRouter(prisma14, prisma17); + const suffix = `bget_v3_${seq++}`; + const seed = await seedEnvironment(prisma14, suffix); + const batchId = `batch_${CUID_25}`; + const friendlyId = `batch_${suffix}_f`; + await legacyStore.createBatchTaskRun( + batchData({ + id: batchId, + friendlyId, + runtimeEnvironmentId: seed.environment.id, + runCount: 3, + }) + ); + + holder.store = router; + holder.environment = authEnv(seed); + holder.cachedRequest = { id: batchId }; + + const body = JSON.stringify({ runCount: 3, idempotencyKey: "idem_v3" }); + const res = (await actionCreateBatch({ + request: new Request("http://localhost/api/v3/batches", { + method: "POST", + headers: { + "content-type": "application/json", + "content-length": String(Buffer.byteLength(body)), + authorization: `Bearer ${seed.environment.apiKey}`, + }, + body, + }), + params: {}, + context: {} as never, + })) as Response; + + expect(legacyReplica.wasHit("batchTaskRun")).toBe(false); + expect(res.status).toBe(200); + expect(await res.json()).toEqual({ id: friendlyId, runCount: 3, isCached: true }); + } + ); +}); diff --git a/apps/webapp/test/runGetRoutes.replicaLag.guard.test.ts b/apps/webapp/test/runGetRoutes.replicaLag.guard.test.ts new file mode 100644 index 00000000000..8d844f550b3 --- /dev/null +++ b/apps/webapp/test/runGetRoutes.replicaLag.guard.test.ts @@ -0,0 +1,1021 @@ +// Replica-lag proof for the "routes-run-get" reads — every run-GET read that RoutingRunStore serves +// from the OWNING store's REPLICA. This file imports and drives the REAL exported loaders/actions +// from the route modules, against a REAL split RoutingRunStore over two Postgres testcontainers with +// the owning (legacy) store's REPLICA FROZEN via the shared laggingReplica primitive. Only deps +// orthogonal to the read are mocked (auth/session, control-plane env resolution, redirect/toast +// formatting, downstream engine/clickhouse/event/presenter services); the run-store read path each +// case's decision hangs on is the genuine article. +// +// For every case, the concrete proof driven through the real caller is the CORRECT observable output +// under lag: +// * frozen-STALE replica (the run WAS replicated but carries an out-of-date snapshot): the caller's +// routing/redirect/auth/queue-key DECISION is driven only by IMMUTABLE fields (projectId, +// runtimeEnvironmentId, spanId, traceId, organizationId, engine, queue, concurrencyKey, createdAt), +// so the real caller returns the CORRECT redirect / 200 payload even off the lagging replica — the +// only staleness is cosmetic (status/completedAt) which the UI self-heals on the next poll. +// * frozen-MISSING replica + a documented recovery: logs.$logId returns 200 with runStatus undefined +// (optional annotation); the finished-attempt read returns null → empty output on a run that still +// renders; the cancel/replay action context-resolvers fall back to the owning PRIMARY and resolve +// the org (proven by the action NOT spuriously denying). +// The run-store read really goes through the frozen replica in each case — asserted with +// `wasHit("taskRun")` — so no proof is a lucky primary hit. + +import { describe, expect, vi } from "vitest"; +import { heteroRunOpsPostgresTest, laggingReplica } from "@internal/testcontainers"; +import type { PrismaClient } from "@trigger.dev/database"; +import type { RunOpsPrismaClient } from "@internal/run-ops-database"; +import { PostgresRunStore, RoutingRunStore } from "@internal/run-store"; +import type { CreateRunInput } from "@internal/run-store"; + +vi.setConfig({ testTimeout: 120_000, hookTimeout: 120_000 }); + +// ── Hoisted holders wired per-test before each real caller runs ─────────────────────────────────── +// `cp.client` -> the real control-plane container (prisma14): backs the mocked ~/db.server `prisma` +// AND (branded) `$replica`, used for the orthogonal project/orgMember auth reads. +// `router.store` -> the real split RoutingRunStore whose legacy replica is frozen. +// `authUser`, `resolvedEnv`, `logDetail`, `project`, `environment` feed the mocked fix-orthogonal deps. +const cp = vi.hoisted(() => ({ client: undefined as any })); +const router = vi.hoisted(() => ({ store: undefined as any })); +const authUser = vi.hoisted(() => ({ id: "user_rrg_guard", admin: false, isImpersonating: false })); +const resolved = vi.hoisted(() => ({ + authEnv: undefined as any, + env: undefined as any, + lockedWorker: null as any, +})); +const logDetail = vi.hoisted(() => ({ result: undefined as any })); +const cpLookups = vi.hoisted(() => ({ project: undefined as any, environment: undefined as any })); +const cancelCalls = vi.hoisted(() => ({ runs: [] as any[] })); +const replayCalls = vi.hoisted(() => ({ runs: [] as any[] })); + +const READ_REPLICA_BRAND = Symbol.for("trigger.dev/run-store/read-replica"); + +// ~/db.server: `prisma` -> real writer; `$replica` -> a BRANDED proxy over the real writer. The brand +// makes RoutingRunStore treat a `$replica`-arg read as a replica read (no primary escalation) exactly +// as production does, while property access (project/orgMember/taskSchedule.findFirst) forwards to the +// real control-plane client so the orthogonal auth reads work. Run-ops split handles are left +// undefined; the router is injected directly via the ~/v3/runStore.server mock below. +vi.mock("~/db.server", () => { + const brandedReplica = new Proxy( + {}, + { + get(_t, prop) { + if (prop === READ_REPLICA_BRAND) return true; + const c = cp.client; + if (!c) throw new Error("cp.client not set for this test"); + const value = c[prop]; + return typeof value === "function" ? value.bind(c) : value; + }, + } + ); + const prismaProxy = new Proxy( + {}, + { + get(_t, prop) { + const c = cp.client; + if (!c) throw new Error("cp.client not set for this test"); + const value = c[prop]; + return typeof value === "function" ? value.bind(c) : value; + }, + } + ); + return { + prisma: prismaProxy, + $replica: brandedReplica, + runOpsNewPrismaClient: undefined, + runOpsNewReplicaClient: undefined, + runOpsLegacyPrisma: undefined, + runOpsLegacyReplica: undefined, + }; +}); + +// Inject the REAL split router. A stable Proxy keeps the named import binding constant while +// forwarding every method to the per-test router in `router.store`. +vi.mock("~/v3/runStore.server", () => ({ + runStore: new Proxy( + {}, + { + get(_t, prop) { + const store = router.store; + if (!store) throw new Error("router.store not set for this test"); + const value = store[prop]; + return typeof value === "function" ? value.bind(store) : value; + }, + } + ), +})); + +// Auth/session (orthogonal): fixed user id. +vi.mock("~/services/session.server", () => ({ + requireUserId: async () => authUser.id, + requireUser: async () => authUser, + getUserId: async () => authUser.id, +})); + +// Control-plane env resolution (a downstream cross-DB lookup, orthogonal to the run-store read): +// returns whatever the test staged. +vi.mock("~/v3/runOpsMigration/controlPlaneResolver.server", () => ({ + controlPlaneResolver: { + resolveAuthenticatedEnv: async () => resolved.authEnv, + resolveEnv: async () => resolved.env, + resolveRunLockedWorker: async () => resolved.lockedWorker, + }, +})); + +// Redirect/toast formatting (orthogonal): marker Responses so assertions don't need cookie machinery. +vi.mock("~/models/message.server", () => ({ + redirectWithSuccessMessage: async (path: string, _req: Request, message: string) => + new Response(null, { status: 302, headers: { "x-redirect": path, "x-toast": message } }), + redirectWithErrorMessage: async (path: string, _req: Request, message: string) => + new Response(null, { status: 302, headers: { "x-redirect": path, "x-error": message } }), +})); + +// Buffer disabled everywhere so a run-store miss is a clean miss (no buffer masking the read outcome). +vi.mock("~/v3/mollifier/mollifierBuffer.server", () => ({ getMollifierBuffer: () => null })); + +// debug-loader downstream: the run-queue Redis introspection is not the read under test. +vi.mock("~/v3/runEngine.server", () => ({ + engine: { + runQueue: { + getQueueConcurrencyLimit: async () => 10, + getEnvConcurrencyLimit: async () => 20, + currentConcurrencyOfQueue: async () => 1, + currentConcurrencyOfEnvironment: async () => 2, + keys: { + queueCurrentConcurrencyKey: () => "qcc", + envCurrentConcurrencyKey: () => "ecc", + queueConcurrencyLimitKey: () => "qcl", + envConcurrencyLimitKey: () => "ecl", + }, + }, + }, +})); + +// logs.$logId downstream: ClickHouse + presenter + slug lookups are orthogonal. +vi.mock("~/services/clickhouse/clickhouseFactoryInstance.server", () => ({ + clickhouseFactory: { getClickhouseForOrganization: async () => ({}) }, +})); +vi.mock("~/presenters/v3/LogDetailPresenter.server", () => ({ + LogDetailPresenter: class { + async call() { + return logDetail.result; + } + }, +})); +vi.mock("~/models/project.server", () => ({ + findProjectBySlug: async () => cpLookups.project, +})); +vi.mock("~/models/runtimeEnvironment.server", () => ({ + findEnvironmentBySlug: async () => cpLookups.environment, + displayableEnvironment: (env: any) => ({ id: env.id, type: env.type, slug: env.slug }), +})); + +// logs.download downstream: event repository + trace export are orthogonal. +vi.mock("~/v3/eventRepository/index.server", () => ({ + getEventRepositoryForStore: async () => ({ + streamTraceEvents: async function* () { + yield {} as any; + }, + }), +})); +vi.mock("~/v3/eventRepository/traceExport.server", () => ({ + getTraceExportFormat: () => ({ extension: "log" }), + streamTraceExport: async function* () { + yield "trace-line\n"; + }, +})); +vi.mock("~/v3/taskEventStore.server", () => ({ + getTaskEventStoreTableForRun: () => "taskEvent", +})); +vi.mock("~/env.server", () => ({ + env: { APP_ORIGIN: "https://app.test", TASK_RUN_METADATA_MAXIMUM_SIZE: 262144 }, +})); + +// replay-loader downstream: regions/worker/queue lookups are orthogonal. +vi.mock("~/presenters/v3/RegionsPresenter.server", () => ({ + RegionsPresenter: class { + async call() { + return { regions: [] }; + } + }, +})); +vi.mock("~/v3/models/workerDeployment.server", () => ({ + findCurrentWorkerDeployment: async () => null, +})); +vi.mock("~/runEngine/concerns/workerQueueSplit.server", () => ({ + regionForDisplay: () => undefined, +})); + +// Downstream cancel/replay services (engine work, orthogonal): record the call. +vi.mock("~/v3/services/cancelTaskRun.server", () => ({ + CancelTaskRunService: class { + async call(run: any) { + cancelCalls.runs.push(run); + } + }, +})); +vi.mock("~/v3/services/replayTaskRun.server", () => ({ + ReplayTaskRunService: class { + async call(run: any) { + replayCalls.runs.push(run); + return { id: "new_run", friendlyId: "run_replayed", spanId: "span_new" }; + } + }, +})); + +// RBAC gate (orthogonal) for the dashboardAction routes (cancel/replay). Passing ability; the route's +// own control-plane membership check still runs against seeded data. +vi.mock("~/services/rbac.server", () => ({ + rbac: { + authenticateSession: async () => ({ + ok: true, + user: { id: authUser.id, email: "guard@example.com", admin: false }, + ability: { can: () => true, canSuper: () => true }, + }), + }, +})); +vi.mock("~/services/logger.server", () => ({ + logger: { warn: vi.fn(), info: vi.fn(), error: vi.fn(), debug: vi.fn() }, +})); + +// ── Real callers under guard ────────────────────────────────────────────────────────────────────── +import { loader as orgsRedirectLoader } from "~/routes/orgs.$organizationSlug.projects.$projectParam.runs.$runParam"; +import { loader as shortLinkLoader } from "~/routes/runs.$runParam"; +import { loader as inspectorLoader } from "~/routes/resources.runs.$runParam"; +import { loader as debugLoader } from "~/routes/resources.taskruns.$runParam.debug"; +import { loader as logDetailLoader } from "~/routes/resources.orgs.$organizationSlug.projects.$projectParam.env.$envParam.logs.$logId"; +import { loader as logsDownloadLoader } from "~/routes/resources.runs.$runParam.logs.download"; +import { + loader as replayLoader, + action as replayAction, +} from "~/routes/resources.taskruns.$runParam.replay"; +import { action as cancelAction } from "~/routes/resources.taskruns.$runParam.cancel"; + +type AnyClient = PrismaClient | RunOpsPrismaClient; + +// A cuid-shaped id (25 chars, no run-ops marker) classifies LEGACY → friendlyId-keyed reads route to +// the legacy (control-plane / prisma14) store, whose replica we freeze. +const CUID_25 = "e".repeat(25); +let seq = 0; + +async function seedTenant(prisma: PrismaClient, suffix: string) { + const organization = await prisma.organization.create({ + data: { title: `Org ${suffix}`, slug: `org-${suffix}` }, + }); + const project = await prisma.project.create({ + data: { + name: `Project ${suffix}`, + slug: `project-${suffix}`, + externalRef: `proj_${suffix}`, + organizationId: organization.id, + }, + }); + await prisma.user.create({ + data: { + id: authUser.id, + email: `guard-${suffix}@example.com`, + authenticationMethod: "MAGIC_LINK", + }, + }); + const orgMember = await prisma.orgMember.create({ + data: { userId: authUser.id, organizationId: organization.id, role: "ADMIN" }, + }); + const environment = await prisma.runtimeEnvironment.create({ + data: { + type: "DEVELOPMENT", + slug: "dev", + projectId: project.id, + organizationId: organization.id, + // Link the dev env to the org member so the replay loader's DEVELOPMENT env-list query + // (orgMember.userId === userId) surfaces it. + orgMemberId: orgMember.id, + apiKey: `tr_dev_${suffix}`, + pkApiKey: `pk_dev_${suffix}`, + shortcode: `short_${suffix}`, + }, + }); + return { organization, project, environment }; +} + +function buildCreateRunInput(p: { + runId: string; + friendlyId: string; + organizationId: string; + projectId: string; + runtimeEnvironmentId: string; + status?: string; + spanId?: string; + traceId?: string; + createdAt?: Date; + completedAt?: Date | null; + concurrencyKey?: string | null; +}): CreateRunInput { + return { + data: { + id: p.runId, + engine: "V2", + status: (p.status ?? "PENDING") as never, + friendlyId: p.friendlyId, + runtimeEnvironmentId: p.runtimeEnvironmentId, + environmentType: "DEVELOPMENT", + organizationId: p.organizationId, + projectId: p.projectId, + taskIdentifier: "my-task", + payload: '{"hello":"world"}', + payloadType: "application/json", + context: { foo: "bar" }, + traceContext: { trace: "ctx" }, + traceId: p.traceId ?? (p.spanId ? `trace_${p.friendlyId}` : "trace_1"), + spanId: p.spanId ?? "span_1", + runTags: [], + queue: "task/my-task", + isTest: false, + taskEventStore: "taskEvent", + depth: 0, + concurrencyKey: p.concurrencyKey ?? undefined, + createdAt: p.createdAt ?? new Date("2024-01-01T00:00:00.000Z"), + completedAt: p.completedAt ?? undefined, + }, + snapshot: { + engine: "V2", + executionStatus: "RUN_CREATED", + description: "Run was created", + runStatus: (p.status ?? "PENDING") as never, + environmentId: p.runtimeEnvironmentId, + environmentType: "DEVELOPMENT", + projectId: p.projectId, + organizationId: p.organizationId, + }, + }; +} + +// Build the router as runStore.server holds it: legacy store (prisma14) + dedicated store (prisma17). +// The legacy replica is a frozen laggingReplica; a friendlyId/cuid-routed read is served stale/missing. +function buildRouterWithFrozenLegacyReplica( + prisma14: PrismaClient, + prisma17: RunOpsPrismaClient, + legacyReplica: AnyClient +) { + const legacyStore = new PostgresRunStore({ + prisma: prisma14, + readOnlyPrisma: legacyReplica as never, + schemaVariant: "legacy", + }); + const newStore = new PostgresRunStore({ + prisma: prisma17 as never, + readOnlyPrisma: prisma17 as never, + schemaVariant: "dedicated", + }); + return new RoutingRunStore({ new: newStore, legacy: legacyStore }); +} + +function authEnvFor(seed: { organization: any; project: any; environment: any }) { + return { + id: seed.environment.id, + slug: seed.environment.slug, + type: seed.environment.type, + apiKey: seed.environment.apiKey, + organizationId: seed.organization.id, + organization: { + id: seed.organization.id, + slug: seed.organization.slug, + title: seed.organization.title, + }, + project: { + id: seed.project.id, + slug: seed.project.slug, + name: seed.project.name, + externalRef: seed.project.externalRef, + }, + git: null, + }; +} + +describe("routes-run-get — REAL loaders/actions vs a frozen owning replica", () => { + // orgs redirect loader — canonical run redirect + heteroRunOpsPostgresTest( + "orgs redirect loader: frozen-STALE replica → 302 to the correct v3 path (immutable projectId/runtimeEnvironmentId); frozen-MISSING → transient 404", + async ({ prisma14, prisma17 }) => { + const suffix = `rrg1_${seq++}`; + cp.client = prisma14; + const seed = await seedTenant(prisma14 as unknown as PrismaClient, suffix); + const friendlyId = `run_${suffix}`; + const runId = `run_${CUID_25}`; + const writer = new PostgresRunStore({ + prisma: prisma14, + readOnlyPrisma: prisma14, + schemaVariant: "legacy", + }); + await writer.createRun( + buildCreateRunInput({ + runId, + friendlyId, + organizationId: seed.organization.id, + projectId: seed.project.id, + runtimeEnvironmentId: seed.environment.id, + status: "EXECUTING", + }) + ); + resolved.authEnv = authEnvFor(seed); + + const req = (fid: string) => + new Request( + `http://localhost/orgs/${seed.organization.slug}/projects/${seed.project.slug}/runs/${fid}` + ); + const params = (fid: string) => ({ + organizationSlug: seed.organization.slug, + projectParam: seed.project.slug, + runParam: fid, + }); + + // (a) frozen-STALE: the run IS on the replica but with an out-of-date snapshot. projectId + + // runtimeEnvironmentId are immutable, so the redirect target is correct off the lagging replica. + const stale = laggingReplica(prisma14, [ + { + model: "taskRun", + mode: "frozen", + rows: [ + { + friendlyId, + projectId: seed.project.id, + runtimeEnvironmentId: seed.environment.id, + status: "PENDING", + }, + ], + }, + ]); + router.store = buildRouterWithFrozenLegacyReplica( + prisma14 as any, + prisma17 as any, + stale.client + ); + const staleRes = (await orgsRedirectLoader({ + request: req(friendlyId), + params: params(friendlyId), + context: {} as never, + })) as Response; + expect(stale.wasHit("taskRun")).toBe(true); + expect(staleRes.status).toBe(302); + expect(staleRes.headers.get("location")).toBe( + `/orgs/${seed.organization.slug}/projects/${seed.project.slug}/env/${seed.environment.slug}/runs/${friendlyId}` + ); + + // (b) frozen-MISSING: transient not-found (drives no mutation; self-heals on next load). + const missing = laggingReplica(prisma14, [{ model: "taskRun", mode: "missing" }]); + router.store = buildRouterWithFrozenLegacyReplica( + prisma14 as any, + prisma17 as any, + missing.client + ); + let missStatus = 0; + try { + await orgsRedirectLoader({ + request: req(friendlyId), + params: params(friendlyId), + context: {} as never, + }); + } catch (e) { + missStatus = (e as Response).status; + } + expect(missing.wasHit("taskRun")).toBe(true); + expect(missStatus).toBe(404); + } + ); + + // short-link loader — public short-link redirect + heteroRunOpsPostgresTest( + "short-link loader: frozen-STALE → 302 to correct v3 path with ?span (immutable spanId/projectId); frozen-MISSING → error-redirect (no crash)", + async ({ prisma14, prisma17 }) => { + const suffix = `rrg2_${seq++}`; + cp.client = prisma14; + const seed = await seedTenant(prisma14 as unknown as PrismaClient, suffix); + const friendlyId = `run_${suffix}`; + const runId = `run_${CUID_25}`; + const spanId = "span_short_fixed"; + const writer = new PostgresRunStore({ + prisma: prisma14, + readOnlyPrisma: prisma14, + schemaVariant: "legacy", + }); + await writer.createRun( + buildCreateRunInput({ + runId, + friendlyId, + organizationId: seed.organization.id, + projectId: seed.project.id, + runtimeEnvironmentId: seed.environment.id, + status: "EXECUTING", + spanId, + }) + ); + resolved.authEnv = authEnvFor(seed); + + const stale = laggingReplica(prisma14, [ + { + model: "taskRun", + mode: "frozen", + rows: [ + { + friendlyId, + spanId, + projectId: seed.project.id, + runtimeEnvironmentId: seed.environment.id, + }, + ], + }, + ]); + router.store = buildRouterWithFrozenLegacyReplica( + prisma14 as any, + prisma17 as any, + stale.client + ); + const staleRes = (await shortLinkLoader({ + request: new Request(`http://localhost/runs/${friendlyId}`), + params: { runParam: friendlyId }, + context: {} as never, + })) as Response; + expect(stale.wasHit("taskRun")).toBe(true); + expect(staleRes.status).toBe(302); + const loc = staleRes.headers.get("location")!; + expect(loc).toContain( + `/orgs/${seed.organization.slug}/projects/${seed.project.slug}/env/${seed.environment.slug}/runs/${friendlyId}` + ); + expect(loc).toContain(`span=${spanId}`); + + // frozen-MISSING → the loader returns the error-redirect marker (302, x-error), never throws/500. + const missing = laggingReplica(prisma14, [{ model: "taskRun", mode: "missing" }]); + router.store = buildRouterWithFrozenLegacyReplica( + prisma14 as any, + prisma17 as any, + missing.client + ); + const missRes = (await shortLinkLoader({ + request: new Request(`http://localhost/runs/${friendlyId}`), + params: { runParam: friendlyId }, + context: {} as never, + })) as Response; + expect(missing.wasHit("taskRun")).toBe(true); + expect(missRes.status).toBe(302); + expect(missRes.headers.get("x-error")).toContain("doesn't exist"); + } + ); + + // inspector loader — findRun + the finished-attempt read + heteroRunOpsPostgresTest( + "inspector loader: frozen-STALE run → 200 typedjson with correct immutable friendlyId; the finished-attempt read missing on the replica → empty output, run still renders", + async ({ prisma14, prisma17 }) => { + const suffix = `rrg3_${seq++}`; + cp.client = prisma14; + const seed = await seedTenant(prisma14 as unknown as PrismaClient, suffix); + const friendlyId = `run_${suffix}`; + const runId = `run_${CUID_25}`; + const createdAt = new Date("2024-02-02T00:00:00.000Z"); + const writer = new PostgresRunStore({ + prisma: prisma14, + readOnlyPrisma: prisma14, + schemaVariant: "legacy", + }); + await writer.createRun( + buildCreateRunInput({ + runId, + friendlyId, + organizationId: seed.organization.id, + projectId: seed.project.id, + runtimeEnvironmentId: seed.environment.id, + status: "COMPLETED_SUCCESSFULLY", + createdAt, + }) + ); + resolved.authEnv = authEnvFor(seed); + resolved.lockedWorker = null; + + // Stale run row (immutable id/projectId/createdAt/friendlyId correct) AND the finished-attempt + // read misses on the replica → output undefined, but the run still renders 200. + const stale = laggingReplica(prisma14, [ + { + model: "taskRun", + mode: "frozen", + rows: [ + { + id: runId, + friendlyId, + status: "COMPLETED_SUCCESSFULLY", + projectId: seed.project.id, + runtimeEnvironmentId: seed.environment.id, + createdAt, + queue: "task/my-task", + payload: '{"hello":"world"}', + payloadType: "application/json", + runTags: [], + baseCostInCents: 0, + costInCents: 0, + }, + ], + }, + { model: "taskRunAttempt", mode: "missing" }, + ]); + router.store = buildRouterWithFrozenLegacyReplica( + prisma14 as any, + prisma17 as any, + stale.client + ); + const res = (await inspectorLoader({ + request: new Request(`http://localhost/resources/runs/${friendlyId}`), + params: { runParam: friendlyId }, + context: {} as never, + })) as Response; + expect(stale.wasHit("taskRun")).toBe(true); + expect(stale.wasHit("taskRunAttempt")).toBe(true); // the finished-attempt read hit the replica + expect(res.status).toBe(200); + const body = await res.json(); + // typedjson envelope: payload is under .json in remix-typedjson serialisation + const data = body.json ?? body; + expect(data.friendlyId).toBe(friendlyId); + expect(data.isFinished).toBe(true); + // finished-attempt read missed on the replica → output absent (typedjson encodes the + // undefined as null over the wire), but the run itself rendered. + expect(data.output ?? null).toBeNull(); + } + ); + + // debug loader — admin queue-debug + heteroRunOpsPostgresTest( + "debug loader: frozen-STALE → 200 with the run + queue keys derived from immutable engine/queue/concurrencyKey", + async ({ prisma14, prisma17 }) => { + const suffix = `rrg4_${seq++}`; + cp.client = prisma14; + const seed = await seedTenant(prisma14 as unknown as PrismaClient, suffix); + const friendlyId = `run_${suffix}`; + const runId = `run_${CUID_25}`; + const writer = new PostgresRunStore({ + prisma: prisma14, + readOnlyPrisma: prisma14, + schemaVariant: "legacy", + }); + await writer.createRun( + buildCreateRunInput({ + runId, + friendlyId, + organizationId: seed.organization.id, + projectId: seed.project.id, + runtimeEnvironmentId: seed.environment.id, + status: "EXECUTING", + concurrencyKey: "ck-site4", + }) + ); + resolved.authEnv = authEnvFor(seed); + + const stale = laggingReplica(prisma14, [ + { + model: "taskRun", + mode: "frozen", + rows: [ + { + id: runId, + friendlyId, + engine: "V2", + queue: "task/my-task", + concurrencyKey: "ck-site4", + queueTimestamp: null, + runtimeEnvironmentId: seed.environment.id, + projectId: seed.project.id, + }, + ], + }, + ]); + router.store = buildRouterWithFrozenLegacyReplica( + prisma14 as any, + prisma17 as any, + stale.client + ); + const res = (await debugLoader({ + request: new Request(`http://localhost/resources/taskruns/${friendlyId}/debug`), + params: { runParam: friendlyId }, + context: {} as never, + })) as Response; + expect(stale.wasHit("taskRun")).toBe(true); + expect(res.status).toBe(200); + const body = await res.json(); + const data = body.json ?? body; + expect(data.engine).toBe("V2"); + expect(data.run.queue).toBe("task/my-task"); + expect(data.run.concurrencyKey).toBe("ck-site4"); + } + ); + + // log-detail loader — run-status annotation (branded $replica) + heteroRunOpsPostgresTest( + "log-detail loader: findRun($replica-branded) MISSING on the replica → 200 with runStatus undefined (optional annotation self-heals)", + async ({ prisma14, prisma17 }) => { + const suffix = `rrg5_${seq++}`; + cp.client = prisma14; + const seed = await seedTenant(prisma14 as unknown as PrismaClient, suffix); + const friendlyId = `run_${suffix}`; + const runId = `run_${CUID_25}`; + const writer = new PostgresRunStore({ + prisma: prisma14, + readOnlyPrisma: prisma14, + schemaVariant: "legacy", + }); + await writer.createRun( + buildCreateRunInput({ + runId, + friendlyId, + organizationId: seed.organization.id, + projectId: seed.project.id, + runtimeEnvironmentId: seed.environment.id, + status: "COMPLETED_SUCCESSFULLY", + }) + ); + cpLookups.project = { id: seed.project.id, organizationId: seed.organization.id }; + cpLookups.environment = { id: seed.environment.id, type: "DEVELOPMENT", slug: "dev" }; + logDetail.result = { runId: friendlyId, message: "hello", someField: 1 }; + + const missing = laggingReplica(prisma14, [{ model: "taskRun", mode: "missing" }]); + router.store = buildRouterWithFrozenLegacyReplica( + prisma14 as any, + prisma17 as any, + missing.client + ); + + const logId = encodeURIComponent(`trace_x::span_x::${friendlyId}::2024-01-01T00:00:00.000Z`); + const res = (await logDetailLoader({ + request: new Request(`http://localhost/resources/orgs/o/projects/p/env/dev/logs/${logId}`), + params: { + organizationSlug: seed.organization.slug, + projectParam: seed.project.slug, + envParam: "dev", + logId, + }, + context: {} as never, + })) as Response; + expect(missing.wasHit("taskRun")).toBe(true); + expect(res.status).toBe(200); + const body = await res.json(); + const data = body.json ?? body; + // The log detail is returned; the run-status annotation the lagging replica couldn't supply is + // simply undefined (the caller optional-chains `run?.status`). + expect(data.message).toBe("hello"); + expect(data.runStatus ?? null).toBeNull(); + } + ); + + // trace-download loader — trace-export download + heteroRunOpsPostgresTest( + "trace-download loader: frozen-STALE (completedAt null) → 200 gzip stream; immutable traceId/org/createdAt drive the export window, stale-null completedAt = open-ended (superset) not lossy", + async ({ prisma14, prisma17 }) => { + const suffix = `rrg6_${seq++}`; + cp.client = prisma14; + const seed = await seedTenant(prisma14 as unknown as PrismaClient, suffix); + const friendlyId = `run_${suffix}`; + const runId = `run_${CUID_25}`; + const traceId = "trace_dl_fixed"; + const createdAt = new Date("2024-03-03T00:00:00.000Z"); + const writer = new PostgresRunStore({ + prisma: prisma14, + readOnlyPrisma: prisma14, + schemaVariant: "legacy", + }); + await writer.createRun( + buildCreateRunInput({ + runId, + friendlyId, + organizationId: seed.organization.id, + projectId: seed.project.id, + runtimeEnvironmentId: seed.environment.id, + status: "COMPLETED_SUCCESSFULLY", + traceId, + createdAt, + completedAt: new Date("2024-03-03T00:05:00.000Z"), + }) + ); + resolved.authEnv = authEnvFor(seed); + + const stale = laggingReplica(prisma14, [ + { + model: "taskRun", + mode: "frozen", + rows: [ + { + friendlyId, + traceId, + organizationId: seed.organization.id, + runtimeEnvironmentId: seed.environment.id, + createdAt, + completedAt: null, + taskEventStore: "taskEvent", + taskIdentifier: "my-task", + }, + ], + }, + ]); + router.store = buildRouterWithFrozenLegacyReplica( + prisma14 as any, + prisma17 as any, + stale.client + ); + const res = (await logsDownloadLoader({ + request: new Request(`http://localhost/resources/runs/${friendlyId}/logs/download`), + params: { runParam: friendlyId }, + context: {} as never, + })) as Response; + expect(stale.wasHit("taskRun")).toBe(true); + expect(res.status).toBe(200); + expect(res.headers.get("Content-Encoding")).toBe("gzip"); + expect(res.headers.get("Content-Disposition")).toContain(`${friendlyId}.log`); + } + ); + + // replay loader — replay dialog + heteroRunOpsPostgresTest( + "replay loader: frozen-STALE run → 200 typedjson replay payload built from immutable seed fields (payload/tags/queue)", + async ({ prisma14, prisma17 }) => { + const suffix = `rrg8_${seq++}`; + cp.client = prisma14; + const seed = await seedTenant(prisma14 as unknown as PrismaClient, suffix); + const friendlyId = `run_${suffix}`; + const runId = `run_${CUID_25}`; + // Seed a project row the replay loader's loadProjectEnvironments ($replica.project.findFirst) finds + // with an environment matching runtimeEnvironmentId. + const writer = new PostgresRunStore({ + prisma: prisma14, + readOnlyPrisma: prisma14, + schemaVariant: "legacy", + }); + await writer.createRun( + buildCreateRunInput({ + runId, + friendlyId, + organizationId: seed.organization.id, + projectId: seed.project.id, + runtimeEnvironmentId: seed.environment.id, + status: "COMPLETED_SUCCESSFULLY", + }) + ); + + const stale = laggingReplica(prisma14, [ + { + model: "taskRun", + mode: "frozen", + rows: [ + { + friendlyId, + payload: '{"hello":"world"}', + payloadType: "application/json", + seedMetadata: null, + seedMetadataType: null, + runtimeEnvironmentId: seed.environment.id, + projectId: seed.project.id, + concurrencyKey: null, + maxAttempts: null, + maxDurationInSeconds: null, + machinePreset: null, + workerQueue: null, + region: null, + ttl: null, + idempotencyKey: null, + runTags: [], + queue: "task/my-task", + taskIdentifier: "my-task", + }, + ], + }, + ]); + router.store = buildRouterWithFrozenLegacyReplica( + prisma14 as any, + prisma17 as any, + stale.client + ); + const res = (await replayLoader({ + request: new Request(`http://localhost/resources/taskruns/${friendlyId}/replay`), + params: { runParam: friendlyId }, + context: {} as never, + })) as Response; + expect(stale.wasHit("taskRun")).toBe(true); + expect(res.status).toBe(200); + const body = await res.json(); + const data = body.json ?? body; + expect(data.queue).toBe("task/my-task"); + expect(data.runTags).toEqual([]); + } + ); + + // cancel action — resolveRunOrganizationId (dashboardAction) + // The action's `context` resolver reads the run by friendlyId (client-less → REPLICA) to resolve + // the org that scopes the RBAC check. Under a frozen-MISSING replica (+ drained buffer) the first + // read misses and the PRIMARY FALLBACK re-reads the owning primary to recover the org. This is + // load-bearing: authenticateAndAuthorize denies a scoped action when `ctx.organizationId` is absent + // (hasScope=false → fail-closed). Driving the REAL action under lag returns the 302 success + // redirect and reaches the cancel, reachable only if the fallback resolved the org. + heteroRunOpsPostgresTest( + "cancel action ctx resolver: frozen-MISSING replica → primary fallback resolves the org → action authorizes and cancels (not fail-closed 403)", + async ({ prisma14, prisma17 }) => { + const suffix = `rrg7_${seq++}`; + cp.client = prisma14; + const seed = await seedTenant(prisma14 as unknown as PrismaClient, suffix); + const friendlyId = `run_${suffix}`; + const runId = `run_${CUID_25}`; + const writer = new PostgresRunStore({ + prisma: prisma14, + readOnlyPrisma: prisma14, + schemaVariant: "legacy", + }); + await writer.createRun( + buildCreateRunInput({ + runId, + friendlyId, + organizationId: seed.organization.id, + projectId: seed.project.id, + runtimeEnvironmentId: seed.environment.id, + status: "EXECUTING", + }) + ); + // The context resolver's resolveEnv (mocked control-plane) returns the org for the primary-found run. + resolved.env = { organizationId: seed.organization.id }; + cancelCalls.runs.length = 0; + + const missing = laggingReplica(prisma14, [{ model: "taskRun", mode: "missing" }]); + router.store = buildRouterWithFrozenLegacyReplica( + prisma14 as any, + prisma17 as any, + missing.client + ); + + const redirectUrl = `/orgs/${seed.organization.slug}/projects/${seed.project.slug}/runs`; + const res = (await cancelAction({ + request: new Request(`http://localhost/resources/taskruns/${friendlyId}/cancel`, { + method: "POST", + headers: { "content-type": "application/x-www-form-urlencoded" }, + body: new URLSearchParams({ redirectUrl }).toString(), + }), + params: { runParam: friendlyId }, + context: {} as never, + })) as Response; + + expect(missing.wasHit("taskRun")).toBe(true); + // Authorized + cancelled — impossible unless the ctx resolver recovered the org via the primary + // fallback (a missing scope would have failed authorization closed with a redirect/403). + expect(res.status).toBe(302); + expect(res.headers.get("x-toast")).toBe("Canceled run"); + expect(cancelCalls.runs).toHaveLength(1); + expect(cancelCalls.runs[0].friendlyId).toBe(friendlyId); + } + ); + + // replay action — resolveRunOrganizationId (dashboardAction) + // Identical shape to the cancel action above: the ctx resolver's client-less findRun misses the + // frozen replica, the PRIMARY FALLBACK recovers the org, and the scoped RBAC check therefore + // authorizes. The REAL replay action under lag returns the 302 success redirect and reaches the + // replay service (only reachable with a resolved org scope). + heteroRunOpsPostgresTest( + "replay action ctx resolver: frozen-MISSING replica → primary fallback resolves the org → action authorizes and replays (not fail-closed 403)", + async ({ prisma14, prisma17 }) => { + const suffix = `rrg8b_${seq++}`; + cp.client = prisma14; + const seed = await seedTenant(prisma14 as unknown as PrismaClient, suffix); + const friendlyId = `run_${suffix}`; + const runId = `run_${CUID_25}`; + const writer = new PostgresRunStore({ + prisma: prisma14, + readOnlyPrisma: prisma14, + schemaVariant: "legacy", + }); + await writer.createRun( + buildCreateRunInput({ + runId, + friendlyId, + organizationId: seed.organization.id, + projectId: seed.project.id, + runtimeEnvironmentId: seed.environment.id, + status: "COMPLETED_SUCCESSFULLY", + }) + ); + resolved.env = { organizationId: seed.organization.id }; + resolved.authEnv = authEnvFor(seed); + replayCalls.runs.length = 0; + + const missing = laggingReplica(prisma14, [{ model: "taskRun", mode: "missing" }]); + router.store = buildRouterWithFrozenLegacyReplica( + prisma14 as any, + prisma17 as any, + missing.client + ); + + const failedRedirect = `/orgs/${seed.organization.slug}/projects/${seed.project.slug}/runs/${friendlyId}`; + const res = (await replayAction({ + request: new Request(`http://localhost/resources/taskruns/${friendlyId}/replay`, { + method: "POST", + headers: { "content-type": "application/x-www-form-urlencoded" }, + body: new URLSearchParams({ failedRedirect }).toString(), + }), + params: { runParam: friendlyId }, + context: {} as never, + })) as Response; + + expect(missing.wasHit("taskRun")).toBe(true); + expect(res.status).toBe(302); + expect(res.headers.get("x-toast")).toBe("Replaying run"); + expect(replayCalls.runs).toHaveLength(1); + expect(replayCalls.runs[0].friendlyId).toBe(friendlyId); + } + ); +}); diff --git a/apps/webapp/test/runPresenters.replicaLag.test.ts b/apps/webapp/test/runPresenters.replicaLag.test.ts new file mode 100644 index 00000000000..1eaaeb054f2 --- /dev/null +++ b/apps/webapp/test/runPresenters.replicaLag.test.ts @@ -0,0 +1,533 @@ +// Replica-lag proof for seven run-detail PRESENTER reads. Imports and drives the REAL exported presenter +// method / SSE loader that contains each read against a real Postgres (prisma14) whose run-store replica +// is frozen by the shared laggingReplica primitive (taskRun "missing"). Only orthogonal deps are stubbed +// (auth/session, presign, the mollifier buffer/read-fallback, the event repository); the run-store read +// path is the genuine article — a single PostgresRunStore wired as the webapp's single-DB runStore +// singleton (writer = prisma14, replica = frozen). Each case asserts the REAL caller's observable output +// under lag and separately confirms the run is live on the primary, so every miss is purely replica lag. +// +// Every read is a read-only dashboard/API GET whose stale/absent value is a documented fallback +// (buffer / typed-error / retryable-404) or a cosmetic omission that self-heals on the next poll once +// the replica catches up — none drives a mutation, a wrong terminal state, or a non-self-healing +// failure. Each case's per-test comment names the tolerating mechanism. + +import { heteroPostgresTest, laggingReplica } from "@internal/testcontainers"; +import type { PrismaClient } from "@trigger.dev/database"; +import { PostgresRunStore } from "@internal/run-store"; +import { describe, expect, vi } from "vitest"; + +vi.setConfig({ testTimeout: 90_000, hookTimeout: 90_000 }); + +// ---- Holders wired per-test before each real-caller invocation. ----------------------------------- +// dbHolder.$replica -> the BRANDED frozen replica (taskRun reads miss; every other model forwards to +// the real container). dbHolder.prisma -> the real writer (prisma14). +// storeHolder.store -> the single PostgresRunStore the mocked `runStore` singleton forwards to. +// bufferHolder.result-> the mollifier read-fallback result (null = buffer miss, the realistic double-miss). +// bufferHolder.calls -> counts findRunByIdWithMollifierFallback invocations (proves the buffer fallback +// path actually ran for read 1, not merely that null happened). +// sessionHolder.userId -> the id requireUserId resolves to (read 4). +// eventRepoHolder.repo -> the event repository getEventRepositoryForStore returns (read 7). +const dbHolder = vi.hoisted(() => ({ prisma: null as any, $replica: null as any })); +const storeHolder = vi.hoisted(() => ({ store: undefined as any })); +const bufferHolder = vi.hoisted(() => ({ result: null as any, calls: 0 })); +const sessionHolder = vi.hoisted(() => ({ userId: "user_presenters_guard" })); +const eventRepoHolder = vi.hoisted(() => ({ repo: undefined as any })); + +// ~/db.server: redirect the module-level handles to the real test clients. NOT a DB mock — reads hit a +// real Postgres container; this only chooses which real client each module resolves. $replica is the +// frozen replica; the run-ops split handles are inert (single-DB webapp topology). +vi.mock("~/db.server", () => ({ + get prisma() { + return dbHolder.prisma; + }, + get $replica() { + return dbHolder.$replica; + }, + runOpsLegacyReplica: undefined, + runOpsNewReplica: undefined, + runOpsSplitReadEnabled: false, +})); + +// The ONE wiring boundary: inject the real single-DB store the presenters read through. A stable getter +// keeps the named import binding constant while returning the per-test store. +vi.mock("~/v3/runStore.server", () => ({ + get runStore() { + return storeHolder.store; + }, +})); + +// Presign is never exercised (payloads are inline JSON on the null path); inert stub keeps the import light. +vi.mock("~/v3/objectStore.server", () => ({ + generatePresignedUrl: vi.fn(async () => ({ success: false, error: "not-used" })), +})); + +// The mollifier read-fallback is a downstream service ORTHOGONAL to the run-store read under test. It +// returns bufferHolder.result (null = buffer miss) and counts calls so read 1 can prove the fallback ran. +vi.mock("~/v3/mollifier/readFallback.server", () => ({ + findRunByIdWithMollifierFallback: vi.fn(async () => { + bufferHolder.calls++; + return bufferHolder.result; + }), +})); + +// The mollifier buffer (read 4) — disabled so the stream loader's fallback is a clean miss and the only +// thing that could resolve a traceId is the run-store read (frozen -> null -> 404). +vi.mock("~/v3/mollifier/mollifierBuffer.server", () => ({ + getMollifierBuffer: () => null, +})); + +// Dashboard session auth (read 4) — resolves a fixed user id. Orthogonal to the run read. +vi.mock("~/services/session.server", () => ({ + requireUserId: vi.fn(async () => sessionHolder.userId), + getUserId: vi.fn(async () => sessionHolder.userId), + requireUser: vi.fn(async () => ({ id: sessionHolder.userId })), + getUser: vi.fn(async () => ({ id: sessionHolder.userId })), +})); + +// The event repository (read 7) — a downstream store, not the run read. getSpan returns eventRepoHolder.repo's +// span so #getSpan proceeds to the run-store findRuns({parentSpanId}) read under test. +vi.mock("~/v3/eventRepository/index.server", () => ({ + getEventRepositoryForStore: vi.fn(async () => eventRepoHolder.repo), +})); + +// The REAL exported callers under proof. +import { ApiRetrieveRunPresenter } from "~/presenters/v3/ApiRetrieveRunPresenter.server"; +import { ApiRunResultPresenter } from "~/presenters/v3/ApiRunResultPresenter.server"; +import { RunPresenter, RunNotInPgError } from "~/presenters/v3/RunPresenter.server"; +import { RunStreamPresenter } from "~/presenters/v3/RunStreamPresenter.server"; +import { PlaygroundPresenter } from "~/presenters/v3/PlaygroundPresenter.server"; +import { SpanPresenter } from "~/presenters/v3/SpanPresenter.server"; + +let seq = 0; + +type Seed = { + organizationId: string; + projectId: string; + projectSlug: string; + environmentId: string; + environmentSlug: string; +}; + +async function seedTenant(prisma: PrismaClient, suffix: string): Promise { + const organization = await prisma.organization.create({ + data: { title: `Org ${suffix}`, slug: `org-${suffix}` }, + }); + const project = await prisma.project.create({ + data: { + name: `Project ${suffix}`, + slug: `project-${suffix}`, + externalRef: `proj_${suffix}`, + organizationId: organization.id, + }, + }); + const environment = await prisma.runtimeEnvironment.create({ + data: { + type: "DEVELOPMENT", + slug: "dev", + projectId: project.id, + organizationId: organization.id, + apiKey: `tr_dev_${suffix}`, + pkApiKey: `pk_dev_${suffix}`, + shortcode: `short_${suffix}`, + }, + }); + await prisma.user.create({ + data: { + id: sessionHolder.userId, + email: `guard-${suffix}@example.com`, + authenticationMethod: "MAGIC_LINK", + }, + }); + await prisma.orgMember.create({ + data: { userId: sessionHolder.userId, organizationId: organization.id, role: "ADMIN" }, + }); + return { + organizationId: organization.id, + projectId: project.id, + projectSlug: project.slug, + environmentId: environment.id, + environmentSlug: environment.slug, + }; +} + +async function seedRun( + prisma: PrismaClient, + seed: Seed, + opts: { + id: string; + friendlyId: string; + status?: string; + spanId?: string; + parentSpanId?: string; + } +) { + await prisma.taskRun.create({ + data: { + id: opts.id, + engine: "V2", + status: (opts.status ?? "EXECUTING") as never, + friendlyId: opts.friendlyId, + runtimeEnvironmentId: seed.environmentId, + environmentType: "DEVELOPMENT", + organizationId: seed.organizationId, + projectId: seed.projectId, + taskIdentifier: "my-task", + payload: "{}", + payloadType: "application/json", + traceContext: {}, + traceId: `trace_${opts.id}`, + spanId: opts.spanId ?? `span_${opts.id}`, + ...(opts.parentSpanId !== undefined ? { parentSpanId: opts.parentSpanId } : {}), + queue: "task/my-task", + isTest: false, + taskEventStore: "taskEvent", + depth: 0, + }, + }); +} + +// Wire the per-test holders: the branded frozen replica over `prisma`, and the single PostgresRunStore +// (writer = prisma, replica = frozen) the mocked runStore singleton forwards to. Returns the frozen +// handle so a test can assert wasHit("taskRun"). +function wireFrozenStore(prisma: PrismaClient) { + const frozen = laggingReplica(prisma, [{ model: "taskRun", mode: "missing" }]); + const store = new PostgresRunStore({ prisma, readOnlyPrisma: frozen.client as never }); + dbHolder.prisma = prisma; + dbHolder.$replica = frozen.client; + storeHolder.store = store; + bufferHolder.result = null; + bufferHolder.calls = 0; + return frozen; +} + +describe("run-detail presenter reads against a frozen run-store replica", () => { + // ApiRetrieveRunPresenter findRun (+$replica) — public GET /api/v3/runs/:runId. Drive the REAL static + // findRun. Under lag the replica misses and the buffer fallback (findRunByIdWithMollifierFallback) + // fires (proven by call count), buffer misses too, so the caller returns null. The retrieve route maps + // null to a 404 carrying `x-should-retry: true`, so the SDK retrieve/poll loop re-fetches and the next + // poll (replica caught up) succeeds. Self-healing; read-only. + heteroPostgresTest( + "ApiRetrieveRunPresenter.findRun returns null for a live run absent on the replica (retryable-404 contract)", + async ({ prisma14 }) => { + const prisma = prisma14 as unknown as PrismaClient; + const suffix = `retrieve_${seq++}`; + const seed = await seedTenant(prisma, suffix); + const runId = `run_id_${suffix}`; + const friendlyId = `run_${suffix}`; + await seedRun(prisma, seed, { id: runId, friendlyId, status: "COMPLETED_SUCCESSFULLY" }); + + const frozen = wireFrozenStore(prisma); + + const env = { id: seed.environmentId, organizationId: seed.organizationId } as any; + const result = await ApiRetrieveRunPresenter.findRun(friendlyId, env); + + // The frozen replica WAS consulted (the read-your-writes window is really exercised)... + expect(frozen.wasHit("taskRun")).toBe(true); + // ...the presenter's documented buffer fallback FIRED (not merely that null happened)... + expect(bufferHolder.calls).toBe(1); + // ...and the observable caller output is null (retrieve route -> retryable 404). + expect(result).toBeNull(); + + // Primary contrast: the run is genuinely live — the miss is purely replica lag. + const onPrimary = await prisma.taskRun.findFirst({ + where: { friendlyId }, + select: { id: true }, + }); + expect(onPrimary?.id).toBe(runId); + } + ); + + // ApiRunResultPresenter findRun (no client) — GET /api/v1/runs/:runId/result poll (SDK waitForRun + // result). Drive the REAL call(); the store is injected via the presenter's own constructor seam (4th + // arg). Under lag the replica misses -> null -> the presenter returns undefined, the poll's "not + // finished yet" signal — the SDK result-poll loop retries and succeeds once the replica catches up. + // Read-only. + heteroPostgresTest( + "ApiRunResultPresenter.call returns undefined for a live run absent on the replica (poll retries)", + async ({ prisma14 }) => { + const prisma = prisma14 as unknown as PrismaClient; + const suffix = `result_${seq++}`; + const seed = await seedTenant(prisma, suffix); + const runId = `run_id_${suffix}`; + const friendlyId = `run_${suffix}`; + await seedRun(prisma, seed, { id: runId, friendlyId, status: "COMPLETED_SUCCESSFULLY" }); + + const frozen = wireFrozenStore(prisma); + + // Construct exactly as the route does, but inject the real frozen store through the ctor seam. + const presenter = new ApiRunResultPresenter( + prisma, + frozen.client as any, + undefined, + storeHolder.store + ); + // call() wraps in traceWithEnv, which reads env slug/org/project attributes. + const env = { + id: seed.environmentId, + type: "DEVELOPMENT", + slug: seed.environmentSlug, + organizationId: seed.organizationId, + organization: { id: seed.organizationId, slug: seed.projectSlug, title: "Org" }, + projectId: seed.projectId, + project: { id: seed.projectId, name: "Project" }, + } as any; + const result = await presenter.call(friendlyId, env); + + expect(frozen.wasHit("taskRun")).toBe(true); + expect(result).toBeUndefined(); + + const onPrimary = await prisma.taskRun.findFirst({ + where: { friendlyId }, + select: { id: true }, + }); + expect(onPrimary?.id).toBe(runId); + } + ); + + // RunPresenter findRun (no client) — dashboard run-detail page loader. Drive the REAL call(). Under + // lag the replica misses -> null -> the presenter throws the typed RunNotInPgError, the route's signal + // to fall back to the synthesised mollifier-buffer view (and off the noisy Prisma error path); the + // page self-heals on the next poll. Read-only. + heteroPostgresTest( + "RunPresenter.call throws RunNotInPgError for a live run absent on the replica (route buffer view)", + async ({ prisma14 }) => { + const prisma = prisma14 as unknown as PrismaClient; + const suffix = `detail_${seq++}`; + const seed = await seedTenant(prisma, suffix); + const runId = `run_id_${suffix}`; + const friendlyId = `run_${suffix}`; + await seedRun(prisma, seed, { id: runId, friendlyId }); + + const frozen = wireFrozenStore(prisma); + + const presenter = new RunPresenter(prisma); + const call = presenter.call({ + userId: sessionHolder.userId, + projectSlug: seed.projectSlug, + environmentSlug: seed.environmentSlug, + runFriendlyId: friendlyId, + showDeletedLogs: false, + showDebug: false, + }); + + await expect(call).rejects.toBeInstanceOf(RunNotInPgError); + expect(frozen.wasHit("taskRun")).toBe(true); + + const onPrimary = await prisma.taskRun.findFirst({ + where: { friendlyId }, + select: { id: true }, + }); + expect(onPrimary?.id).toBe(runId); + } + ); + + // RunStreamPresenter findRun (no client) — run-detail SSE trace-stream loader. Drive the REAL loader + // from createLoader(). Under lag the replica misses -> run null -> not authorized -> traceId null -> + // buffer disabled -> the handler throws Response(404), which createSSELoader rethrows. 404 is the + // SSE-reconnect contract: the dashboard reconnects and the next connection (replica caught up) + // attaches. + heteroPostgresTest( + "RunStreamPresenter loader throws Response 404 for a live run absent on the replica (SSE reconnect)", + async ({ prisma14 }) => { + const prisma = prisma14 as unknown as PrismaClient; + const suffix = `stream_${seq++}`; + const seed = await seedTenant(prisma, suffix); + const runId = `run_id_${suffix}`; + const friendlyId = `run_${suffix}`; + await seedRun(prisma, seed, { id: runId, friendlyId }); + + const frozen = wireFrozenStore(prisma); + + const loader = new RunStreamPresenter(prisma).createLoader(); + const request = new Request(`http://localhost/resources/runs/${friendlyId}/stream`); + + let thrown: unknown; + try { + await loader({ request, params: { runParam: friendlyId } } as any); + } catch (e) { + thrown = e; + } + + expect(thrown).toBeInstanceOf(Response); + expect((thrown as Response).status).toBe(404); + expect(frozen.wasHit("taskRun")).toBe(true); + + const onPrimary = await prisma.taskRun.findFirst({ + where: { friendlyId }, + select: { id: true }, + }); + expect(onPrimary?.id).toBe(runId); + } + ); + + // PlaygroundPresenter findRuns (no client) — agent-playground conversation list. Drive the REAL + // getRecentConversations. The conversation row is read off $replica (playgroundConversation not + // frozen), then each backing run's scalars via a client-less findRuns over the id set (frozen -> []). + // The missing run is absent from runsById, so the conversation renders with runFriendlyId=null / + // runStatus=null / isActive=false — a cosmetic "status unknown" on one row that self-heals next load. + // The row still renders. + heteroPostgresTest( + "PlaygroundPresenter.getRecentConversations renders the row with null run fields when the run is absent on the replica", + async ({ prisma14 }) => { + const prisma = prisma14 as unknown as PrismaClient; + const suffix = `playground_${seq++}`; + const seed = await seedTenant(prisma, suffix); + const runId = `run_id_${suffix}`; + const friendlyId = `run_${suffix}`; + await seedRun(prisma, seed, { id: runId, friendlyId, status: "EXECUTING" }); + + await prisma.playgroundConversation.create({ + data: { + chatId: `chat_${suffix}`, + title: "Conversation", + agentSlug: "my-agent", + runId, + projectId: seed.projectId, + runtimeEnvironmentId: seed.environmentId, + userId: sessionHolder.userId, + }, + }); + + const frozen = wireFrozenStore(prisma); + + const presenter = new PlaygroundPresenter(); + const conversations = await presenter.getRecentConversations({ + environmentId: seed.environmentId, + agentSlug: "my-agent", + userId: sessionHolder.userId, + }); + + expect(frozen.wasHit("taskRun")).toBe(true); + // The conversation row IS returned (not dropped)... + expect(conversations).toHaveLength(1); + // ...but its backing-run fields self-heal to null because the run missed on the replica. + expect(conversations[0].chatId).toBe(`chat_${suffix}`); + expect(conversations[0].runFriendlyId).toBeNull(); + expect(conversations[0].runStatus).toBeNull(); + expect(conversations[0].isActive).toBe(false); + + const onPrimary = await prisma.taskRun.findFirst({ + where: { id: runId }, + select: { id: true }, + }); + expect(onPrimary?.id).toBe(runId); + } + ); + + // SpanPresenter findRun (+this._replica) — span-detail (run inspector) panel. Drive the REAL public + // findRun with the originalRunId branch; it passes the branded this._replica. Under lag the replica + // misses -> null. The span-detail panel renders its not-found/loading state for this poll and + // self-heals next tick. Read-only. + heteroPostgresTest( + "SpanPresenter.findRun returns null for a live run absent on the replica (span-detail not-found this poll)", + async ({ prisma14 }) => { + const prisma = prisma14 as unknown as PrismaClient; + const suffix = `span_${seq++}`; + const seed = await seedTenant(prisma, suffix); + const runId = `run_id_${suffix}`; + const friendlyId = `run_${suffix}`; + await seedRun(prisma, seed, { id: runId, friendlyId }); + + const frozen = wireFrozenStore(prisma); + + // _prisma = writer, _replica = branded frozen (exactly how the webapp constructs it). + const presenter = new SpanPresenter(prisma, frozen.client as any); + const run = await presenter.findRun({ + originalRunId: friendlyId, + spanId: `span_${runId}`, + environmentId: seed.environmentId, + }); + + expect(frozen.wasHit("taskRun")).toBe(true); + expect(run).toBeNull(); + + const onPrimary = await prisma.taskRun.findFirst({ + where: { friendlyId }, + select: { id: true }, + }); + expect(onPrimary?.id).toBe(runId); + } + ); + + // SpanPresenter findRuns {parentSpanId} (+this._replica) — the "triggered runs" list on the + // span-detail panel. Drive the REAL call() end-to-end: it resolves the parent run on the primary, + // gets the span from the (stubbed) event repository, then reads the child runs via + // runStore.findRuns({parentSpanId}, this._replica) — the frozen replica. Under lag the just-triggered + // child is invisible -> span.triggeredRuns === [] — a cosmetic "one fewer child shown" that self-heals + // next poll; the child still exists and executes. Read-only. + heteroPostgresTest( + "SpanPresenter.call yields empty triggeredRuns when the child run is absent on the replica", + async ({ prisma14 }) => { + const prisma = prisma14 as unknown as PrismaClient; + const suffix = `triggered_${seq++}`; + const seed = await seedTenant(prisma, suffix); + + const parentRunId = `run_parent_${suffix}`; + const parentFriendlyId = `run_${suffix}`; + const spanId = `span_detail_${suffix}`; + await seedRun(prisma, seed, { + id: parentRunId, + friendlyId: parentFriendlyId, + spanId, + }); + + // The child run whose parentSpanId is the inspected span — the one that should appear in the + // triggered-runs list but is missing on the frozen replica. + const childRunId = `run_child_${suffix}`; + const childFriendlyId = `run_child_fr_${suffix}`; + await seedRun(prisma, seed, { + id: childRunId, + friendlyId: childFriendlyId, + spanId: `span_child_${suffix}`, + parentSpanId: spanId, + }); + + const frozen = wireFrozenStore(prisma); + + // Stub the event repository so getSpan yields a minimal "attempt" span; this lets #getSpan proceed + // to the run-store findRuns({parentSpanId}) read under test without ClickHouse. + eventRepoHolder.repo = { + getSpan: async () => ({ + spanId, + parentId: undefined, + message: "attempt", + isError: false, + isPartial: false, + isCancelled: false, + level: "TRACE", + startTime: new Date(), + duration: 0, + events: [], + style: {}, + properties: {}, + resourceProperties: {}, + entity: { type: "attempt", id: undefined }, + metadata: {}, + }), + // Not reached on this path (getRun returns undefined before a trace summary is needed). + getTraceSummary: async () => null, + }; + + const presenter = new SpanPresenter(prisma, frozen.client as any); + const result = await presenter.call({ + userId: sessionHolder.userId, + projectSlug: seed.projectSlug, + envSlug: seed.environmentSlug, + spanId, + runFriendlyId: parentFriendlyId, + }); + + expect(frozen.wasHit("taskRun")).toBe(true); + expect(result?.type).toBe("span"); + // The child triggered by this span is omitted from the list under replica lag. + expect((result as any).span.triggeredRuns).toEqual([]); + + // Primary contrast: the child genuinely exists with this parentSpanId — the omission is pure lag. + const childOnPrimary = await prisma.taskRun.findFirst({ + where: { parentSpanId: spanId }, + select: { friendlyId: true }, + }); + expect(childOnPrimary?.friendlyId).toBe(childFriendlyId); + } + ); +}); diff --git a/apps/webapp/test/runsRepositoryConvert.replicaLag.test.ts b/apps/webapp/test/runsRepositoryConvert.replicaLag.test.ts new file mode 100644 index 00000000000..7b5187c2613 --- /dev/null +++ b/apps/webapp/test/runsRepositoryConvert.replicaLag.test.ts @@ -0,0 +1,152 @@ +// Property: convertRunListInputOptionsToFilterRunsOptions tolerates a batch friendlyId → id resolution +// miss under replica lag. Drives the REAL exported function (not a reimplementation) with a store whose +// replica is FROZEN via the shared `laggingReplica`, so the batch lookup misses the just-written row +// while the owning primary still holds it. +// +// The read (store.findBatchTaskRunByFriendlyId, no client → REPLICA) resolves a `batch_` friendlyId to +// an internal id for a ClickHouse list FILTER, behind an `if (batch)` guard. Under lag it returns null, +// the guard is skipped, and the returned FilterRunsOptions keeps `batchId` as the UNRESOLVED friendlyId +// (matches no ClickHouse batch_id this load; the next revalidation resolves it once replicated). The +// function does NOT throw and mutates nothing; the batch is live on the primary (internal id differs), +// proving the miss is pure lag. + +import { heteroPostgresTest, laggingReplica } from "@internal/testcontainers"; +import { PostgresRunStore } from "@internal/run-store"; +import type { PrismaClient } from "@trigger.dev/database"; +import { describe, expect, vi } from "vitest"; + +vi.setConfig({ testTimeout: 60_000, hookTimeout: 60_000 }); + +// Prevent the run-store / db singletons from constructing at import; the function under test is driven +// with an explicitly-injected store + prisma, so these module defaults are never used at runtime. +vi.mock("~/db.server", () => ({})); +vi.mock("~/v3/runStore.server", () => ({ runStore: {} })); + +import { convertRunListInputOptionsToFilterRunsOptions } from "~/services/runsRepository/runsRepository.server"; + +let seq = 0; + +async function seedTenant(prisma: PrismaClient, suffix: string) { + const organization = await prisma.organization.create({ + data: { title: `Org ${suffix}`, slug: `org-${suffix}` }, + }); + const project = await prisma.project.create({ + data: { + name: `Project ${suffix}`, + slug: `project-${suffix}`, + externalRef: `proj_${suffix}`, + organizationId: organization.id, + }, + }); + const environment = await prisma.runtimeEnvironment.create({ + data: { + type: "DEVELOPMENT", + slug: "dev", + projectId: project.id, + organizationId: organization.id, + apiKey: `tr_dev_${suffix}`, + pkApiKey: `pk_dev_${suffix}`, + shortcode: `short_${suffix}`, + }, + }); + return { organization, project, environment }; +} + +describe("convertRunListInputOptionsToFilterRunsOptions under replica lag", () => { + heteroPostgresTest( + "leaves a not-yet-replicated batch friendlyId unresolved in the filter without throwing", + async ({ prisma14 }) => { + const prisma = prisma14 as unknown as PrismaClient; + const suffix = `convert_${seq++}`; + const seed = await seedTenant(prisma, suffix); + + // Seed the batch on the PRIMARY only. Internal id differs from the friendlyId. + const batchId = `batchinternal_${suffix}`; + const batchFriendlyId = `batch_${suffix}`; + await prisma.batchTaskRun.create({ + data: { + id: batchId, + friendlyId: batchFriendlyId, + runtimeEnvironmentId: seed.environment.id, + status: "PENDING", + batchVersion: "v3", + }, + }); + + // batchTaskRun frozen-missing on the replica; the client-less read routes there. + const replica = laggingReplica(prisma, [{ model: "batchTaskRun", mode: "missing" }]); + const store = new PostgresRunStore({ + prisma, + readOnlyPrisma: replica.client as never, + schemaVariant: "legacy", + }); + + const result = await convertRunListInputOptionsToFilterRunsOptions( + { + organizationId: seed.organization.id, + projectId: seed.project.id, + environmentId: seed.environment.id, + batchId: batchFriendlyId, + }, + prisma as never, + store + ); + + // The batch resolution really hit the (lagging) replica. + expect(replica.wasHit("batchTaskRun")).toBe(true); + + // OBSERVABLE OUTPUT: the friendlyId is left UNRESOLVED (not swapped to the internal id) — the + // ClickHouse batch_id filter matches nothing this load; the function did not throw. + expect(result.batchId).toBe(batchFriendlyId); + expect(result.batchId).not.toBe(batchId); + + // The miss is pure lag: the batch is live on the PRIMARY (and would resolve on revalidation). + const onPrimary = await prisma.batchTaskRun.findFirst({ + where: { friendlyId: batchFriendlyId }, + }); + expect(onPrimary?.id).toBe(batchId); + } + ); + + // Control: with a live replica the SAME function resolves the friendlyId to the internal id — proving + // the unresolved result above is caused solely by the replica lag, not by the code path being dead. + heteroPostgresTest( + "control: resolves the batch friendlyId to the internal id with a caught-up replica", + async ({ prisma14 }) => { + const prisma = prisma14 as unknown as PrismaClient; + const suffix = `convert_ok_${seq++}`; + const seed = await seedTenant(prisma, suffix); + + const batchId = `batchinternal_${suffix}`; + const batchFriendlyId = `batch_${suffix}`; + await prisma.batchTaskRun.create({ + data: { + id: batchId, + friendlyId: batchFriendlyId, + runtimeEnvironmentId: seed.environment.id, + status: "PENDING", + batchVersion: "v3", + }, + }); + + const store = new PostgresRunStore({ + prisma, + readOnlyPrisma: prisma, + schemaVariant: "legacy", + }); + + const result = await convertRunListInputOptionsToFilterRunsOptions( + { + organizationId: seed.organization.id, + projectId: seed.project.id, + environmentId: seed.environment.id, + batchId: batchFriendlyId, + }, + prisma as never, + store + ); + + expect(result.batchId).toBe(batchId); + } + ); +}); diff --git a/apps/webapp/test/sessionWaitpointRoutes.replicaLag.guard.test.ts b/apps/webapp/test/sessionWaitpointRoutes.replicaLag.guard.test.ts new file mode 100644 index 00000000000..9a622b6b510 --- /dev/null +++ b/apps/webapp/test/sessionWaitpointRoutes.replicaLag.guard.test.ts @@ -0,0 +1,351 @@ +// Replica-lag properties for the route-session-waitpoint reads. Each case drives the REAL route handler +// createActionApiRoute wraps + exports as `action` (captured from the builder, not reimplemented) +// against a real Postgres (heteroPostgresTest) whose read replica is FROZEN via the shared +// `laggingReplica` primitive; only orthogonal deps are mocked (auth/session resolution, route-builder +// middleware, downstream swap/engine services, logging). +// +// end-and-continue calling-run resolve: a live calling run absent on the replica is recovered via the +// owning-primary re-read (findRunOnPrimary), so the handoff swap is reached rather than a 404. +// waitpoint-token complete lookup: a client-less findWaitpoint misses the just-minted token on the +// replica, yet the owning-primary re-read (findWaitpointOnPrimary) resolves it and the waitpoint +// completes (200), reaching engine.completeWaitpoint. + +import { heteroPostgresTest, laggingReplica } from "@internal/testcontainers"; +import type { PrismaClient } from "@trigger.dev/database"; +import { WaitpointId } from "@trigger.dev/core/v3/isomorphic"; +import { describe, expect, vi } from "vitest"; + +vi.setConfig({ testTimeout: 60_000, hookTimeout: 60_000 }); + +// ---- Hoisted holders wired per-test before the captured handler runs. --------------------------- +const H = vi.hoisted(() => ({ + store: undefined as any, // the REAL PostgresRunStore for the current case (mocked runStore forwards here) + primary: undefined as any, // db.server `prisma` -> the real container (owning primary / writer) + replica: undefined as any, // db.server `$replica` -> the lagging replica over the same container + swap: { calls: [] as any[], result: undefined as any }, // swapSessionRun recorder + canned result + engine: { calls: [] as any[] }, // engine.completeWaitpoint recorder + handlers: [] as Array<{ config: any; handler: any }>, // captured inner handlers, one per route import +})); + +// ~/db.server: lazy proxies that always resolve through the holder so the run-store singleton's +// memoized delegates route to the current test's client. Never mocks the DB itself. +vi.mock("~/db.server", () => { + const lazyProxy = (key: "primary" | "replica", label: string) => + new Proxy( + {}, + { + get(_t, prop) { + const client = H[key]; + if (!client) throw new Error(`${label} not set for this test`); + const value = client[prop]; + if (value !== null && typeof value === "object") { + return new Proxy(value, { get: (_d, method) => H[key][prop][method] }); + } + return value; + }, + } + ); + return { + prisma: lazyProxy("primary", "H.primary"), + $replica: lazyProxy("replica", "H.replica"), + runOpsNewPrismaClient: undefined, + runOpsNewReplicaClient: undefined, + runOpsLegacyPrisma: undefined, + runOpsLegacyReplica: undefined, + }; +}); + +// The REAL split router / PostgresRunStore is injected per-test into H.store; the mocked singleton is +// a stable Proxy forwarding every method to it. This is what the routes read/write through. +vi.mock("~/v3/runStore.server", () => ({ + runStore: new Proxy( + {}, + { + get(_t, prop) { + const store = H.store; + if (!store) throw new Error("test bug: H.store not initialised before handler ran"); + const value = store[prop]; + return typeof value === "function" ? value.bind(store) : value; + }, + } + ), +})); + +// Route builder: capture the inner handler each route hands to createActionApiRoute so we can drive +// the REAL caller directly, bypassing only the auth/body middleware (orthogonal). +vi.mock("~/services/routeBuilders/apiBuilder.server", () => ({ + anyResource: (x: unknown) => x, + createActionApiRoute: (config: any, handler: any) => { + H.handlers.push({ config, handler }); + return { action: vi.fn(), loader: vi.fn() }; + }, +})); + +// Session resolution lives in `findResource` (which we bypass by passing `resource` directly); stub so +// the heavy realtime import graph never evaluates. +vi.mock("~/services/realtime/sessions.server", () => ({ + resolveSessionByIdOrExternalId: vi.fn(async () => null), +})); + +// Downstream swap is engine/realtime work, not the read under test: record the call + return a canned +// result so the success branch (and its read-after-write findRun on the primary) is exercised. +vi.mock("~/services/realtime/sessionRunManager.server", () => ({ + swapSessionRun: vi.fn(async (params: any) => { + H.swap.calls.push(params); + return H.swap.result; + }), +})); + +// Waitpoint completion downstream: orthogonal. Record the engine call; canned packet. +vi.mock("~/v3/runEngine.server", () => ({ + engine: { + completeWaitpoint: vi.fn(async (args: any) => { + H.engine.calls.push(args); + return { id: args.id }; + }), + }, +})); +vi.mock("~/runEngine/concerns/waitpointCompletionPacket.server", () => ({ + processWaitpointCompletionPacket: vi.fn(async () => ({ + data: "OK", + dataType: "application/json", + })), +})); + +vi.mock("~/env.server", () => ({ + env: { TASK_PAYLOAD_MAXIMUM_SIZE: 3 * 1024 * 1024 }, +})); +vi.mock("~/services/logger.server", () => ({ + logger: { info: vi.fn(), warn: vi.fn(), error: vi.fn(), debug: vi.fn() }, +})); + +import { PostgresRunStore } from "@internal/run-store"; +import type { CreateRunInput } from "@internal/run-store"; + +let seq = 0; + +async function seedTenant(prisma: PrismaClient, suffix: string) { + const organization = await prisma.organization.create({ + data: { title: `Org ${suffix}`, slug: `org-${suffix}` }, + }); + const project = await prisma.project.create({ + data: { + name: `Project ${suffix}`, + slug: `project-${suffix}`, + externalRef: `proj_${suffix}`, + organizationId: organization.id, + }, + }); + const environment = await prisma.runtimeEnvironment.create({ + data: { + type: "DEVELOPMENT", + slug: "dev", + projectId: project.id, + organizationId: organization.id, + apiKey: `tr_dev_${suffix}`, + pkApiKey: `pk_dev_${suffix}`, + shortcode: `short_${suffix}`, + }, + }); + return { organization, project, environment }; +} + +function buildCreateRunInput(p: { + runId: string; + friendlyId: string; + organizationId: string; + projectId: string; + runtimeEnvironmentId: string; +}): CreateRunInput { + return { + data: { + id: p.runId, + engine: "V2", + status: "PENDING", + friendlyId: p.friendlyId, + runtimeEnvironmentId: p.runtimeEnvironmentId, + environmentType: "DEVELOPMENT", + organizationId: p.organizationId, + projectId: p.projectId, + taskIdentifier: "my-task", + payload: '{"hello":"world"}', + payloadType: "application/json", + context: { foo: "bar" }, + traceContext: { trace: "ctx" }, + traceId: "trace_1", + spanId: "span_1", + runTags: [], + queue: "task/my-task", + isTest: false, + taskEventStore: "taskEvent", + depth: 0, + createdAt: new Date("2024-01-01T00:00:00.000Z"), + }, + snapshot: { + engine: "V2", + executionStatus: "RUN_CREATED", + description: "Run was created", + runStatus: "PENDING", + environmentId: p.runtimeEnvironmentId, + environmentType: "DEVELOPMENT", + projectId: p.projectId, + organizationId: p.organizationId, + }, + }; +} + +// Import the target route once and return the inner handler the builder captured for it. Matching on +// the zod params shape keeps this robust regardless of import order. +async function loadHandler(modulePath: string, paramKey: string) { + await import(modulePath); + const entry = H.handlers.find((h) => { + try { + return Boolean(h.config?.params?.shape?.[paramKey]); + } catch { + return false; + } + }); + if (!entry) throw new Error(`handler for ${modulePath} (param ${paramKey}) not captured`); + return entry.handler as (args: any) => Promise; +} + +describe("route-session-waitpoint reads under replica lag", () => { + // end-and-continue calling-run resolve + heteroPostgresTest( + "end-and-continue: a live calling run absent on the replica is recovered via the primary re-read", + async ({ prisma14 }) => { + const prisma = prisma14 as unknown as PrismaClient; + const suffix = `eac_${seq++}`; + const seed = await seedTenant(prisma, suffix); + + // Seed the calling run on the PRIMARY only; the lagging replica will not see it. + const runId = `run_${"a".repeat(21)}${seq}`; + const friendlyId = `run_call_${suffix}`; + const writerStore = new PostgresRunStore({ prisma, readOnlyPrisma: prisma }); + await writerStore.createRun( + buildCreateRunInput({ + runId, + friendlyId, + organizationId: seed.organization.id, + projectId: seed.project.id, + runtimeEnvironmentId: seed.environment.id, + }) + ); + + const replica = laggingReplica(prisma, [{ model: "taskRun", mode: "missing" }]); + + // The run-store the mocked singleton forwards to: writer + readReplica both the real container, + // so findRunOnPrimary (this.prisma) hits and the read-after-write findRun(prisma) hits. + H.store = new PostgresRunStore({ prisma, readOnlyPrisma: prisma }); + H.primary = prisma; // db.server `prisma` + H.replica = replica.client; // db.server `$replica` -> lagging (the client the route passes to findRun) + H.swap = { calls: [], result: { runId, swapped: true } }; + + const handler = await loadHandler( + "~/routes/api.v1.sessions.$session.end-and-continue", + "session" + ); + + const session = { + id: `session_${suffix}`, + friendlyId: `session_${suffix}`, + externalId: null, + closedAt: null, + expiresAt: null, + }; + + const res = await handler({ + authentication: { environment: seed.environment }, + params: { session: session.friendlyId }, + body: { callingRunId: friendlyId, reason: "upgrade" }, + resource: session, + }); + + // The replica WAS consulted (frozen -> missed): the read genuinely went through the lagging + // replica, so recovery is the primary re-read, not a lucky replica hit. + expect(replica.wasHit("taskRun")).toBe(true); + + // The primary re-read resolved the calling run, so the swap was reached with its cuid... + expect(H.swap.calls).toHaveLength(1); + expect(H.swap.calls[0].callingRunId).toBe(runId); + + // ...and the handler returned 200 with the handoff body, not a 404. + expect(res.status).toBe(200); + const body = (await res.clone().json()) as { + runId?: string; + swapped?: boolean; + error?: string; + }; + expect(body.swapped).toBe(true); + expect(body.runId).toBe(friendlyId); + expect(body.error).toBeUndefined(); + } + ); + + // waitpoint-token complete lookup + heteroPostgresTest( + "waitpoint-token complete: a token absent on the replica is resolved via the primary re-read and completes", + async ({ prisma14 }) => { + const prisma = prisma14 as unknown as PrismaClient; + const suffix = `wctok_${seq++}`; + const seed = await seedTenant(prisma, suffix); + + const waitpointFriendlyId = `waitpoint_${"b".repeat(21)}${seq}`; + const waitpointId = WaitpointId.toId(waitpointFriendlyId); + + // Seed a still-PENDING MANUAL token waitpoint on the PRIMARY only. + const writerStore = new PostgresRunStore({ prisma, readOnlyPrisma: prisma }); + await writerStore.upsertWaitpoint({ + where: { + environmentId_idempotencyKey: { + environmentId: seed.environment.id, + idempotencyKey: waitpointId, + }, + }, + create: { + id: waitpointId, + friendlyId: waitpointFriendlyId, + type: "MANUAL", + status: "PENDING", + idempotencyKey: waitpointId, + userProvidedIdempotencyKey: false, + projectId: seed.project.id, + environmentId: seed.environment.id, + }, + update: {}, + }); + + // findWaitpoint is client-less in the route -> owning REPLICA (readOnlyPrisma). Freeze it. + const replica = laggingReplica(prisma, [{ model: "waitpoint", mode: "missing" }]); + H.store = new PostgresRunStore({ prisma, readOnlyPrisma: replica.client }); + H.primary = prisma; + H.replica = prisma; // not used by this route, but a valid client + H.engine = { calls: [] }; + + const handler = await loadHandler( + "~/routes/api.v1.waitpoints.tokens.$waitpointFriendlyId.complete", + "waitpointFriendlyId" + ); + + const res = await handler({ + authentication: { environment: seed.environment }, + params: { waitpointFriendlyId }, + body: { data: { ok: true } }, + }); + + // The replica WAS consulted (frozen -> missed) on the client-less findWaitpoint... + expect(replica.wasHit("waitpoint")).toBe(true); + + // ...yet the existing findWaitpointOnPrimary fallback resolved the token, so the real handler + // completed the waitpoint (200 success) rather than 404ing it. + expect(res.status).toBe(200); + const body = (await res.clone().json()) as { success?: boolean; error?: string }; + expect(body.success).toBe(true); + expect(body.error).toBeUndefined(); + + // The completion actually reached the engine with the resolved waitpoint id. + expect(H.engine.calls).toHaveLength(1); + expect(H.engine.calls[0].id).toBe(waitpointId); + } + ); +}); diff --git a/apps/webapp/test/spanTraceRoutes.replicaLag.test.ts b/apps/webapp/test/spanTraceRoutes.replicaLag.test.ts new file mode 100644 index 00000000000..f3df2d2ac7d --- /dev/null +++ b/apps/webapp/test/spanTraceRoutes.replicaLag.test.ts @@ -0,0 +1,545 @@ +// Replica-lag properties for the run-trace / span-detail reads, each driven through its REAL exported +// route caller (never a reimplemented read) against a real split Postgres with the owning LEGACY replica +// FROZEN via laggingReplica. The routes pass a branded $replica, so reads stay on the owning replica. +// Only orthogonal webapp singletons are mocked (bearer auth, mollifier buffer, ClickHouse repository, +// formatters, session cookie, control-plane resolver, longPollingFetch). +// +// Properties per read: the spans/trace findResource reads emit a RETRYABLE 404 (x-should-retry:true) on a +// replica+buffer miss and return 200 once the replica catches up (proven with a caught-up store); the +// triggeredRuns list simply omits a just-triggered child under lag (200, eventually consistent); the sync +// trace-runs loader recovers a live run via a primary re-read on a replica miss and still 404s a truly-absent run. + +import { describe, expect, vi } from "vitest"; +import { heteroRunOpsPostgresTest, laggingReplica } from "@internal/testcontainers"; +import type { PrismaClient } from "@trigger.dev/database"; +import type { RunOpsPrismaClient } from "@internal/run-ops-database"; +import { PostgresRunStore, RoutingRunStore } from "@internal/run-store"; +import type { CreateRunInput } from "@internal/run-store"; + +vi.setConfig({ testTimeout: 60_000, hookTimeout: 60_000 }); + +// ---- Hoisted holders wired into the mocked module singletons before each loader call. ------------- +// The branded `$replica` marker uses the global-registry symbol the run-store brands replicas with +// (readReplicaClient.ts) so the routing store keeps the read on the owning REPLICA. It also carries a +// live `orgMember` delegate (pointed at the real prisma14) because the sync route reads +// `$replica.orgMember.findFirst` directly — orthogonal to the run read under test. +const { holder } = vi.hoisted(() => { + const REPLICA_BRAND = Symbol.for("trigger.dev/run-store/read-replica"); + return { + holder: { + REPLICA_BRAND, + store: undefined as unknown, + replicaMarker: undefined as unknown, + environment: undefined as unknown, + bufferResult: null as unknown, + userId: undefined as unknown, + resolvedEnv: undefined as unknown, + span: undefined as unknown, + traceSummary: undefined as unknown, + }, + }; +}); + +// Run-store singleton: a stable Proxy forwarding every method to the per-test RoutingRunStore. +vi.mock("~/v3/runStore.server", () => ({ + runStore: new Proxy( + {}, + { + get(_t, prop) { + const store = holder.store as Record; + if (!store) throw new Error("test bug: holder.store not initialised before loader ran"); + const value = store[prop]; + return typeof value === "function" + ? (value as (...a: unknown[]) => unknown).bind(store) + : value; + }, + } + ), +})); + +// `$replica` brand marker (routes it to the owning replica) + a live orgMember delegate for the sync route. +vi.mock("~/db.server", () => ({ + prisma: {}, + $replica: new Proxy( + {}, + { + get(_t, prop) { + if (prop === holder.REPLICA_BRAND) return true; + const marker = holder.replicaMarker as Record | undefined; + return marker ? marker[prop] : undefined; + }, + } + ), +})); + +// Bearer auth + ability (orthogonal): resolve to the seeded environment and grant every check. +vi.mock("~/services/rbac.server", () => ({ + rbac: { + authenticateBearer: async () => ({ + ok: true, + environment: holder.environment, + subject: { type: "privateKey" }, + jwt: undefined, + ability: { can: () => true, canSuper: () => true }, + }), + }, +})); + +// Buffer fallback (mollifier): a clean MISS so the only run source is the run-store read path. +vi.mock("~/v3/mollifier/readFallback.server", () => ({ + findRunByIdWithMollifierFallback: vi.fn(async () => holder.bufferResult), +})); + +// ClickHouse event repository (downstream, orthogonal): return synthetic span / trace shapes. +vi.mock("~/v3/eventRepository/index.server", () => ({ + getEventRepositoryForStore: async () => ({ + getSpan: async () => holder.span, + getTraceDetailedSubtreeSummary: async () => holder.traceSummary, + }), +})); +vi.mock("~/v3/taskEventStore.server", () => ({ + getTaskEventStoreTableForRun: () => "taskEvent", +})); +vi.mock("~/components/runs/v3/ai", () => ({ + extractAISpanData: () => undefined, +})); +vi.mock("~/v3/mollifier/syntheticApiResponses.server", () => ({ + buildSyntheticSpanDetailBody: (r: unknown) => ({ synthetic: true, run: r }), + buildSyntheticTraceBody: (r: unknown) => ({ synthetic: true, run: r }), +})); + +// Sync-route peripherals (orthogonal to the run read under test). +vi.mock("~/services/session.server", () => ({ + getUserId: async () => holder.userId, +})); +vi.mock("~/v3/runOpsMigration/controlPlaneResolver.server", () => ({ + controlPlaneResolver: { resolveEnv: async () => holder.resolvedEnv }, +})); +vi.mock("~/utils/longPollingFetch", () => ({ + longPollingFetch: async () => + new Response("shape-stream-ok", { status: 200, headers: { "x-longpoll": "hit" } }), +})); +// Ensure ELECTRIC_ORIGIN is defined for the sync route's happy path; keep every other env value real +// so the heavy apiBuilder import graph still loads. +vi.mock("~/env.server", async (importOriginal) => { + const actual = (await importOriginal()) as { env: Record }; + return { env: { ...actual.env, ELECTRIC_ORIGIN: "http://electric.test" } }; +}); + +import { loader as spansLoader } from "~/routes/api.v1.runs.$runId.spans.$spanId"; +import { loader as traceLoader } from "~/routes/api.v1.runs.$runId.trace"; +import { loader as syncTraceRunsLoader } from "~/routes/sync.traces.runs.$traceId"; + +// A cuid (25 chars after `run_`) classifies LEGACY, so both the create and the friendlyId/traceId +// reads route to the legacy (control-plane) store — the store that owns these runs. +const CUID_25 = "c".repeat(25); +let seq = 0; + +async function seedTenant(prisma: PrismaClient, suffix: string) { + const organization = await prisma.organization.create({ + data: { title: `Org ${suffix}`, slug: `org-${suffix}` }, + }); + const project = await prisma.project.create({ + data: { + name: `Project ${suffix}`, + slug: `project-${suffix}`, + externalRef: `proj_${suffix}`, + organizationId: organization.id, + }, + }); + const environment = await prisma.runtimeEnvironment.create({ + data: { + type: "DEVELOPMENT", + slug: "dev", + projectId: project.id, + organizationId: organization.id, + apiKey: `tr_dev_${suffix}`, + pkApiKey: `pk_dev_${suffix}`, + shortcode: `short_${suffix}`, + }, + }); + return { organization, project, environment }; +} + +// The AuthenticatedEnvironment shape the real wrapper + handlers read. +function authEnvironment(seed: Awaited>) { + return { + id: seed.environment.id, + apiKey: seed.environment.apiKey, + type: "DEVELOPMENT", + slug: "dev", + organizationId: seed.organization.id, + organization: { id: seed.organization.id, slug: seed.organization.slug }, + project: { + id: seed.project.id, + slug: seed.project.slug, + externalRef: seed.project.externalRef, + }, + }; +} + +function buildCreateRunInput(p: { + runId: string; + friendlyId: string; + organizationId: string; + projectId: string; + runtimeEnvironmentId: string; + traceId: string; + spanId: string; + parentSpanId?: string; + taskIdentifier?: string; +}): CreateRunInput { + return { + data: { + id: p.runId, + engine: "V2", + status: "PENDING", + friendlyId: p.friendlyId, + runtimeEnvironmentId: p.runtimeEnvironmentId, + environmentType: "DEVELOPMENT", + organizationId: p.organizationId, + projectId: p.projectId, + taskIdentifier: p.taskIdentifier ?? "my-task", + payload: '{"hello":"world"}', + payloadType: "application/json", + context: { foo: "bar" }, + traceContext: { trace: "ctx" }, + traceId: p.traceId, + spanId: p.spanId, + parentSpanId: p.parentSpanId, + runTags: [], + queue: "task/my-task", + isTest: false, + taskEventStore: "taskEvent", + depth: 0, + createdAt: new Date("2024-01-01T00:00:00.000Z"), + }, + snapshot: { + engine: "V2", + executionStatus: "RUN_CREATED", + description: "Run was created", + runStatus: "PENDING", + environmentId: p.runtimeEnvironmentId, + environmentType: "DEVELOPMENT", + projectId: p.projectId, + organizationId: p.organizationId, + }, + }; +} + +// Build the split router. `legacyLag` configures the legacy store's frozen replica. +function buildRouter( + prisma14: PrismaClient, + prisma17: RunOpsPrismaClient, + legacyLag: Parameters[1] +) { + const legacyReplica = laggingReplica(prisma14, legacyLag); + const legacyStore = new PostgresRunStore({ + prisma: prisma14, + readOnlyPrisma: legacyReplica.client, + schemaVariant: "legacy", + }); + const newStore = new PostgresRunStore({ + prisma: prisma17 as never, + readOnlyPrisma: prisma17 as never, + schemaVariant: "dedicated", + }); + const router = new RoutingRunStore({ new: newStore, legacy: legacyStore }); + return { router, legacyStore, legacyReplica }; +} + +function spansRequest(runId: string, spanId: string) { + return { + request: new Request(`https://api.trigger.dev/api/v1/runs/${runId}/spans/${spanId}`, { + headers: { Authorization: "Bearer tr_dev_x" }, + }), + params: { runId, spanId }, + context: {} as never, + }; +} +function traceRequest(runId: string) { + return { + request: new Request(`https://api.trigger.dev/api/v1/runs/${runId}/trace`, { + headers: { Authorization: "Bearer tr_dev_x" }, + }), + params: { runId }, + context: {} as never, + }; +} +function syncRequest(traceId: string) { + return { + request: new Request(`https://app.trigger.dev/sync/traces/runs/${traceId}?live=true`), + params: { traceId }, + context: {} as never, + } as never; +} + +describe("run-trace/span-detail route loaders under a lagging replica", () => { + // spans loader — findResource findRun ($replica) + heteroRunOpsPostgresTest( + "spans loader: replica+buffer double-miss returns a retryable 404 (x-should-retry:true), self-healing to 200 once the replica catches up", + async ({ prisma14, prisma17 }) => { + const suffix = `spans_find_${seq++}`; + const seed = await seedTenant(prisma14, suffix); + const runId = `run_${CUID_25}`; + const friendlyId = `run_${suffix}`; + + // Seed the live run on the LEGACY primary (writer) only. + const lagged = buildRouter(prisma14, prisma17, [{ model: "taskRun", mode: "missing" }]); + await lagged.legacyStore.createRun( + buildCreateRunInput({ + runId, + friendlyId, + organizationId: seed.organization.id, + projectId: seed.project.id, + runtimeEnvironmentId: seed.environment.id, + traceId: `trace_${suffix}`, + spanId: `span_${suffix}`, + }) + ); + + holder.store = lagged.router; + holder.environment = authEnvironment(seed); + holder.bufferResult = null; + + const res = (await spansLoader(spansRequest(friendlyId, `span_${suffix}`))) as Response; + + // The frozen replica WAS consulted (the lag was really exercised). + expect(lagged.legacyReplica.wasHit("taskRun")).toBe(true); + // The caller emits the documented RETRYABLE not-found, not a terminal 404. + expect(res.status).toBe(404); + expect(res.headers.get("x-should-retry")).toBe("true"); + + // Self-heal proof: point the store at a NON-lagging replica (replica caught up == the SDK retry + // landing after replication) and the SAME loader now resolves the run and returns 200. + holder.span = { + spanId: `span_${suffix}`, + parentId: undefined, + message: "root", + isError: false, + isPartial: false, + isCancelled: false, + level: "TRACE", + startTime: new Date(), + duration: 1_000_000, + properties: undefined, + events: undefined, + entity: { type: "task" }, + }; + const caughtUp = buildRouter(prisma14, prisma17, []); // no models frozen + holder.store = caughtUp.router; + const res2 = (await spansLoader(spansRequest(friendlyId, `span_${suffix}`))) as Response; + expect(res2.status).toBe(200); + const body2 = (await res2.json()) as { runId?: string; spanId?: string }; + expect(body2.runId).toBe(friendlyId); + expect(body2.spanId).toBe(`span_${suffix}`); + } + ); + + // spans loader — handler findRuns triggeredRuns ($replica) + heteroRunOpsPostgresTest( + "spans loader: a just-triggered child on the primary is omitted from triggeredRuns under lag (200, list self-heals)", + async ({ prisma14, prisma17 }) => { + const suffix = `spans_children_${seq++}`; + const seed = await seedTenant(prisma14, suffix); + const parentRunId = `run_${CUID_25}`; + const parentFriendlyId = `run_${suffix}_p`; + const spanId = `span_${suffix}`; + + // Seed the parent run AND a child run (parentSpanId = spanId) on the LEGACY primary. + const writerStore = new PostgresRunStore({ + prisma: prisma14, + readOnlyPrisma: prisma14, + schemaVariant: "legacy", + }); + await writerStore.createRun( + buildCreateRunInput({ + runId: parentRunId, + friendlyId: parentFriendlyId, + organizationId: seed.organization.id, + projectId: seed.project.id, + runtimeEnvironmentId: seed.environment.id, + traceId: `trace_${suffix}`, + spanId, + }) + ); + const childRunId = `run_${"d".repeat(25)}`; + const childFriendlyId = `run_${suffix}_c`; + await writerStore.createRun( + buildCreateRunInput({ + runId: childRunId, + friendlyId: childFriendlyId, + organizationId: seed.organization.id, + projectId: seed.project.id, + runtimeEnvironmentId: seed.environment.id, + traceId: `trace_${suffix}`, + spanId: `childspan_${suffix}`, + parentSpanId: spanId, + taskIdentifier: "child-task", + }) + ); + + // Capture the PARENT row as the frozen replica snapshot (== "parent replicated, child written + // after and not yet replicated"). The parent resolves on the replica; the child does not. + const parentSnapshot = (await prisma14.taskRun.findFirstOrThrow({ + where: { id: parentRunId }, + })) as unknown as Record; + + const lagged = buildRouter(prisma14, prisma17, [ + { model: "taskRun", mode: "frozen", rows: [parentSnapshot] }, + ]); + holder.store = lagged.router; + holder.environment = authEnvironment(seed); + holder.bufferResult = null; + holder.span = { + spanId, + parentId: undefined, + message: "root", + isError: false, + isPartial: false, + isCancelled: false, + level: "TRACE", + startTime: new Date(), + duration: 2_000_000, + properties: undefined, + events: undefined, + entity: { type: "task" }, + }; + + const res = (await spansLoader(spansRequest(parentFriendlyId, spanId))) as Response; + const body = (await res.json()) as { runId?: string; triggeredRuns?: unknown }; + + // The parent resolved via the (frozen) replica, so the read genuinely went through it. + expect(lagged.legacyReplica.wasHit("taskRun")).toBe(true); + // 200 with the span, and the lagging child is simply OMITTED from the list. + expect(res.status).toBe(200); + expect(body.runId).toBe(parentFriendlyId); + expect(body.triggeredRuns).toBeUndefined(); + + // Prove the omission is lag (the child is present on the primary right now). + const onPrimary = await prisma14.taskRun.findMany({ + where: { runtimeEnvironmentId: seed.environment.id, parentSpanId: spanId }, + select: { friendlyId: true }, + }); + expect(onPrimary.map((r) => r.friendlyId)).toContain(childFriendlyId); + } + ); + + // trace loader — findResource findRun ($replica) + heteroRunOpsPostgresTest( + "trace loader: replica+buffer double-miss returns a retryable 404 (x-should-retry:true), self-healing to 200 once the replica catches up", + async ({ prisma14, prisma17 }) => { + const suffix = `trace_find_${seq++}`; + const seed = await seedTenant(prisma14, suffix); + const runId = `run_${CUID_25}`; + const friendlyId = `run_${suffix}`; + + const lagged = buildRouter(prisma14, prisma17, [{ model: "taskRun", mode: "missing" }]); + await lagged.legacyStore.createRun( + buildCreateRunInput({ + runId, + friendlyId, + organizationId: seed.organization.id, + projectId: seed.project.id, + runtimeEnvironmentId: seed.environment.id, + traceId: `trace_${suffix}`, + spanId: `span_${suffix}`, + }) + ); + + holder.store = lagged.router; + holder.environment = authEnvironment(seed); + holder.bufferResult = null; + + const res = (await traceLoader(traceRequest(friendlyId))) as Response; + expect(lagged.legacyReplica.wasHit("taskRun")).toBe(true); + expect(res.status).toBe(404); + expect(res.headers.get("x-should-retry")).toBe("true"); + + // Self-heal proof: caught-up replica -> the same loader resolves the run and returns the trace. + holder.traceSummary = { rootSpanId: `span_${suffix}`, spans: [] }; + const caughtUp = buildRouter(prisma14, prisma17, []); + holder.store = caughtUp.router; + const res2 = (await traceLoader(traceRequest(friendlyId))) as Response; + expect(res2.status).toBe(200); + const body2 = (await res2.json()) as { trace?: unknown }; + expect(body2.trace).toEqual({ rootSpanId: `span_${suffix}`, spans: [] }); + } + ); + + // sync trace-runs loader — findRun by traceId ($replica), with a primary fallback + heteroRunOpsPostgresTest( + "sync trace-runs loader: a live run lagging the replica is recovered via the primary fallback", + async ({ prisma14, prisma17 }) => { + const suffix = `sync_${seq++}`; + const seed = await seedTenant(prisma14, suffix); + const runId = `run_${CUID_25}`; + const friendlyId = `run_${suffix}`; + const traceId = `trace_${suffix}`; + const userId = `user_${suffix}`; + + // The dashboard user, joined to the org so the route's real orgMember check passes. + await prisma14.user.create({ + data: { id: userId, email: `u-${suffix}@example.com`, authenticationMethod: "MAGIC_LINK" }, + }); + await prisma14.orgMember.create({ + data: { userId, organizationId: seed.organization.id, role: "ADMIN" }, + }); + + const lagged = buildRouter(prisma14, prisma17, [{ model: "taskRun", mode: "missing" }]); + await lagged.legacyStore.createRun( + buildCreateRunInput({ + runId, + friendlyId, + organizationId: seed.organization.id, + projectId: seed.project.id, + runtimeEnvironmentId: seed.environment.id, + traceId, + spanId: `span_${suffix}`, + }) + ); + + holder.store = lagged.router; + holder.environment = authEnvironment(seed); + holder.userId = userId; + holder.resolvedEnv = { organizationId: seed.organization.id }; + holder.replicaMarker = { orgMember: prisma14.orgMember }; + + const res = (await syncTraceRunsLoader(syncRequest(traceId))) as Response; + + // The frozen replica WAS consulted (the lag was really exercised). + expect(lagged.legacyReplica.wasHit("taskRun")).toBe(true); + + // The primary fallback recovers the live run and the loader proceeds to the shape stream (200). + expect(res.status).toBe(200); + expect(res.headers.get("x-longpoll")).toBe("hit"); + } + ); + + // Negative control for the sync route: a truly-absent run must still 404 (the primary fallback + // recovers a LIVE run, it does not turn every miss into a 200). + heteroRunOpsPostgresTest( + "sync trace-runs loader: 404s when the run is absent on the primary too", + async ({ prisma14, prisma17 }) => { + const suffix = `sync_absent_${seq++}`; + const seed = await seedTenant(prisma14, suffix); + const userId = `user_${suffix}`; + await prisma14.user.create({ + data: { id: userId, email: `u-${suffix}@example.com`, authenticationMethod: "MAGIC_LINK" }, + }); + await prisma14.orgMember.create({ + data: { userId, organizationId: seed.organization.id, role: "ADMIN" }, + }); + + const lagged = buildRouter(prisma14, prisma17, [{ model: "taskRun", mode: "missing" }]); + holder.store = lagged.router; + holder.environment = authEnvironment(seed); + holder.userId = userId; + holder.resolvedEnv = { organizationId: seed.organization.id }; + holder.replicaMarker = { orgMember: prisma14.orgMember }; + + const res = (await syncTraceRunsLoader(syncRequest("trace_does_not_exist"))) as Response; + expect(res.status).toBe(404); + } + ); +}); diff --git a/apps/webapp/test/waitpointCallbackRouteReplicaLag.guard.test.ts b/apps/webapp/test/waitpointCallbackRouteReplicaLag.guard.test.ts new file mode 100644 index 00000000000..8ede36fcb69 --- /dev/null +++ b/apps/webapp/test/waitpointCallbackRouteReplicaLag.guard.test.ts @@ -0,0 +1,193 @@ +// Property: the HTTP-callback route reads-your-writes via a primary fallback. +// +// When the replica-routed `runStore.findWaitpoint({ where: { id } })` returns null (a token whose +// callback fires right after mint has not replicated yet), the route re-reads the OWNING PRIMARY via +// `runStore.findWaitpointOnPrimary(...)` before giving up, so a just-minted token still resolves. +// +// This drives the REAL exported route `action` end-to-end against a lagging split replica (rather +// than calling the store methods directly), so it verifies the route itself — not just the store — +// honours the fallback. +// +// The DB is never mocked: `~/db.server`'s `$replica`/`prisma` proxies forward to a real testcontainer +// (PG17). The replica proxy is wrapped so `waitpoint` reads come back empty (replication lag) while +// every other model — crucially `runtimeEnvironment`, which control-plane env resolution reads — +// forwards to the real row. The primary proxy is the unwrapped client, so `findWaitpointOnPrimary` +// sees the token. `~/v3/runEngine.server` is stubbed to a no-op by test/setup.ts, so the PENDING +// happy path reaches a real 200 without a live engine. +import { heteroPostgresTest, laggingReplica } from "@internal/testcontainers"; +import { describe, expect, vi } from "vitest"; + +const replicaHolder = vi.hoisted(() => ({ client: undefined as any })); +const primaryHolder = vi.hoisted(() => ({ client: undefined as any })); + +vi.mock("~/db.server", async () => { + const { Prisma } = await import("@trigger.dev/database"); + const lazyProxy = (holder: { client: any }, label: string) => + new Proxy( + {}, + { + get(_t, prop) { + if (!holder.client) { + throw new Error(`${label} not set for this test`); + } + const value = holder.client[prop]; + // The `runStore` singleton memoizes each Prisma delegate on first access, pinning it to + // the first test's (later-dropped) DB. Re-resolve so it routes to the current client. + if (value !== null && typeof value === "object") { + return new Proxy(value, { + get: (_d, method) => holder.client[prop][method], + }); + } + return value; + }, + } + ); + return { + prisma: lazyProxy(primaryHolder, "primaryHolder.client"), + $replica: lazyProxy(replicaHolder, "replicaHolder.client"), + runOpsNewPrisma: lazyProxy(replicaHolder, "replicaHolder.client"), + runOpsNewReplica: lazyProxy(replicaHolder, "replicaHolder.client"), + runOpsLegacyReplica: lazyProxy(replicaHolder, "replicaHolder.client"), + runOpsSplitReadEnabled: true, + sqlDatabaseSchema: Prisma.sql([`public`]), + }; +}); + +import type { PrismaClient } from "@trigger.dev/database"; +import { WaitpointId } from "@trigger.dev/core/v3/isomorphic"; +import { action } from "~/routes/api.v1.waitpoints.tokens.$waitpointFriendlyId.callback.$hash"; +import { generateHttpCallbackUrl } from "~/services/httpCallback.server"; + +vi.setConfig({ testTimeout: 60_000, hookTimeout: 60_000 }); + +function callbackRequest(body: unknown) { + const payload = JSON.stringify(body); + return new Request("http://localhost/callback", { + method: "POST", + headers: { "content-type": "application/json", "content-length": String(payload.length) }, + body: payload, + }); +} + +// Derives the same hash `verifyHttpCallbackHash` checks, via the production URL helper. +function hashFor(waitpointId: string, apiKey: string) { + const url = generateHttpCallbackUrl(waitpointId, apiKey); + return url.split("/").pop()!; +} + +let n = 0; +async function seedControlPlane(prisma: PrismaClient) { + const s = n++; + const organization = await prisma.organization.create({ + data: { title: `Org ${s}`, slug: `org-lag-${s}` }, + }); + const project = await prisma.project.create({ + data: { + name: `P ${s}`, + slug: `p-lag-${s}`, + externalRef: `proj_lag_${s}`, + organizationId: organization.id, + }, + }); + const environment = await prisma.runtimeEnvironment.create({ + data: { + type: "PRODUCTION", + slug: `env-lag-${s}`, + projectId: project.id, + organizationId: organization.id, + apiKey: `tr_lag_${s}`, + pkApiKey: `pk_lag_${s}`, + shortcode: `sc_lag_${s}`, + }, + }); + return { organization, project, environment }; +} + +// A just-minted MANUAL token whose `id` matches `WaitpointId.toId(friendlyId)`, so the route's +// friendlyId->id conversion + the hash (computed over `id`) line up. +async function seedJustMintedToken( + prisma: PrismaClient, + ctx: { environmentId: string; projectId: string }, + status: "PENDING" | "COMPLETED" = "PENDING" +) { + const s = n++; + const { id, friendlyId } = WaitpointId.generate(); + await prisma.waitpoint.create({ + data: { + id, + friendlyId, + type: "MANUAL", + status, + idempotencyKey: `idem_lag_${s}`, + userProvidedIdempotencyKey: false, + environmentId: ctx.environmentId, + projectId: ctx.projectId, + }, + }); + return { id, friendlyId }; +} + +describe("HTTP-callback route: read-your-writes primary fallback under replica lag (real route action)", () => { + // Happy path: a just-minted PENDING token is invisible on the (lagging) replica; the route resolves + // it on the primary and completes it -> 200. + heteroPostgresTest( + "PENDING token invisible on replica resolves via primary and completes -> 200", + async ({ prisma17 }) => { + const cp = await seedControlPlane(prisma17 as unknown as PrismaClient); + const token = await seedJustMintedToken(prisma17 as unknown as PrismaClient, { + environmentId: cp.environment.id, + projectId: cp.project.id, + }); + + const replica = laggingReplica(prisma17 as unknown as PrismaClient, [ + { model: "waitpoint", mode: "missing" }, + ]); + primaryHolder.client = prisma17; // findWaitpointOnPrimary sees the token + replicaHolder.client = replica.client; // findWaitpoint (+ env resolution) reads here; waitpoint starved + + // No parent env -> the hash is computed over the env's own apiKey. + const hash = hashFor(token.id, cp.environment.apiKey); + + const res = await action({ + request: callbackRequest({ ok: true }), + params: { waitpointFriendlyId: token.friendlyId, hash }, + context: {} as never, + }); + + expect(res.status).toBe(200); + expect(await res.json()).toEqual({ success: true }); + // The replica read genuinely fired and missed — the token was invisible there. + expect(replica.wasHit()).toBe(true); + } + ); + + // Independent-of-engine proof: a PENDING token with the WRONG hash resolves on the primary and the + // hash check runs -> 401. The 401 (not a not-found) pins that the primary fallback resolves the row, + // not that it merely masks the engine's no-op. + heteroPostgresTest( + "PENDING token invisible on replica resolves via primary then fails hash -> 401", + async ({ prisma17 }) => { + const cp = await seedControlPlane(prisma17 as unknown as PrismaClient); + const token = await seedJustMintedToken(prisma17 as unknown as PrismaClient, { + environmentId: cp.environment.id, + projectId: cp.project.id, + }); + + const replica = laggingReplica(prisma17 as unknown as PrismaClient, [ + { model: "waitpoint", mode: "missing" }, + ]); + primaryHolder.client = prisma17; + replicaHolder.client = replica.client; + + const res = await action({ + request: callbackRequest({ ok: true }), + params: { waitpointFriendlyId: token.friendlyId, hash: "not-the-right-hash" }, + context: {} as never, + }); + + expect(res.status).toBe(401); + expect(await res.json()).toEqual({ error: "Invalid URL, hash doesn't match" }); + expect(replica.wasHit()).toBe(true); + } + ); +}); diff --git a/apps/webapp/test/waitpointCompleteRouteReplicaLag.guard.test.ts b/apps/webapp/test/waitpointCompleteRouteReplicaLag.guard.test.ts new file mode 100644 index 00000000000..438d1a94dcc --- /dev/null +++ b/apps/webapp/test/waitpointCompleteRouteReplicaLag.guard.test.ts @@ -0,0 +1,251 @@ +// Property: under split replica lag the dashboard "complete waitpoint" route action still completes a +// just-minted token. It resolves the waitpoint by id via findWaitpoint (owning REPLICA), and on a null +// re-reads via findWaitpointOnPrimary before the projectId guard, so a token invisible on the lagging +// replica passes the guard and completion proceeds instead of failing with "No waitpoint found". +// Drives the REAL exported action; only peripheral collaborators are mocked. The seam — runStore over a +// split topology whose owning replica is frozen — is a REAL RoutingRunStore over real testcontainer +// Postgres. + +import { describe, expect, vi } from "vitest"; + +const runStoreHolder = vi.hoisted(() => ({ store: undefined as any })); +// $replica.project.findUnique — return the seeded project id so the guard's only remaining variable +// is whether the waitpoint read resolved (this read hits `project`, never the lagging `waitpoint`). +const projectHolder = vi.hoisted(() => ({ id: undefined as string | undefined })); +const engineHolder = vi.hoisted(() => ({ calls: [] as any[] })); + +vi.mock("~/v3/runStore.server", () => ({ + get runStore() { + return runStoreHolder.store; + }, +})); + +vi.mock("~/db.server", () => ({ + $replica: { + project: { + findUnique: async () => (projectHolder.id ? { id: projectHolder.id } : null), + }, + }, +})); + +vi.mock("~/v3/runEngine.server", () => ({ + engine: { + completeWaitpoint: async (args: any) => { + engineHolder.calls.push(args); + return { id: args.id }; + }, + }, +})); + +vi.mock("~/services/session.server", () => ({ + requireUserId: async () => "user_test", +})); + +vi.mock("~/env.server", () => ({ + env: { TASK_PAYLOAD_MAXIMUM_SIZE: 3_000_000 }, +})); + +vi.mock("~/services/logger.server", () => ({ + logger: { error: () => {}, info: () => {}, debug: () => {}, warn: () => {} }, +})); + +// Distinguishable stand-ins for the redirect helpers so we can read the outcome + message off the +// action's return without the real session-cookie machinery. +vi.mock("~/models/message.server", () => ({ + redirectWithErrorMessage: (redirect: string, _req: unknown, message: string) => + new Response(null, { + status: 302, + headers: { location: redirect, "x-outcome": "error", "x-message": message }, + }), + redirectWithSuccessMessage: (redirect: string, _req: unknown, message: string) => + new Response(null, { + status: 302, + headers: { location: redirect, "x-outcome": "success", "x-message": message }, + }), +})); + +// MANUAL-branch collaborators — the token completion path resolves the env then completes. Return an +// env whose id matches the seeded waitpoint's environmentId so the env guard passes. +const envHolder = vi.hoisted(() => ({ id: undefined as string | undefined })); +vi.mock("~/models/runtimeEnvironment.server", () => ({ + findEnvironmentBySlug: async () => (envHolder.id ? { id: envHolder.id } : null), +})); +vi.mock("~/runEngine/concerns/waitpointCompletionPacket.server", () => ({ + processWaitpointCompletionPacket: async () => ({ data: undefined, dataType: "application/json" }), +})); + +import { heteroRunOpsPostgresTest, laggingReplica } from "@internal/testcontainers"; +import { PostgresRunStore, RoutingRunStore } from "@internal/run-store"; +import type { PrismaClient } from "@trigger.dev/database"; +import { WaitpointId } from "@trigger.dev/core/v3/isomorphic"; +import { action } from "~/routes/resources.orgs.$organizationSlug.projects.$projectParam.env.$envParam.waitpoints.$waitpointFriendlyId.complete/route"; + +vi.setConfig({ testTimeout: 120_000, hookTimeout: 120_000 }); + +const WAITPOINT_CROSS_SEAM_FKS = [ + "Waitpoint_environmentId_fkey", + "Waitpoint_projectId_fkey", +] as const; + +async function dropWaitpointCrossSeamFks(prisma: PrismaClient) { + for (const c of WAITPOINT_CROSS_SEAM_FKS) { + await prisma.$executeRawUnsafe(`ALTER TABLE "Waitpoint" DROP CONSTRAINT IF EXISTS "${c}"`); + } +} + +// Seed a standalone PENDING MANUAL token on the writer, exactly as minting a resume token does. +async function seedPendingTokenWaitpoint( + store: PostgresRunStore, + params: { + id: string; + friendlyId: string; + projectId: string; + environmentId: string; + type?: "MANUAL" | "DATETIME"; + } +) { + await store.upsertWaitpoint({ + where: { + environmentId_idempotencyKey: { + environmentId: params.environmentId, + idempotencyKey: params.id, + }, + }, + create: { + id: params.id, + friendlyId: params.friendlyId, + type: params.type ?? "MANUAL", + status: "PENDING", + idempotencyKey: params.id, + userProvidedIdempotencyKey: false, + projectId: params.projectId, + environmentId: params.environmentId, + }, + update: {}, + }); +} + +function completeRequest(kind: "MANUAL" | "DATETIME") { + const body = new URLSearchParams(); + body.set("type", kind); + if (kind === "MANUAL") body.set("payload", "{}"); + body.set("successRedirect", "/success"); + body.set("failureRedirect", "/failure"); + return new Request("http://localhost/complete", { + method: "POST", + headers: { "content-type": "application/x-www-form-urlencoded" }, + body: body.toString(), + }); +} + +const params = (friendlyId: string) => ({ + organizationSlug: "org-slug", + projectParam: "proj-slug", + envParam: "dev", + waitpointFriendlyId: friendlyId, +}); + +describe("complete-waitpoint dashboard route reads-your-writes under split replica lag", () => { + // LEGACY-resident (cuid) token minted on the control-plane writer; its replica lags. The action's + // findWaitpoint(id) misses, and the findWaitpointOnPrimary fallback must resolve it so the just- + // minted token passes the projectId guard and completes — NOT "No waitpoint found". + heteroRunOpsPostgresTest( + "MANUAL token invisible on the lagging owning replica completes via the primary fallback", + async ({ prisma14, prisma17 }) => { + const legacyReplica = laggingReplica(prisma14, [{ model: "waitpoint", mode: "missing" }]); + const legacyStore = new PostgresRunStore({ + prisma: prisma14, + readOnlyPrisma: legacyReplica.client, + schemaVariant: "legacy", + }); + const newStore = new PostgresRunStore({ + prisma: prisma17 as never, + readOnlyPrisma: prisma17 as never, + schemaVariant: "dedicated", + }); + runStoreHolder.store = new RoutingRunStore({ new: newStore, legacy: legacyStore }); + + await dropWaitpointCrossSeamFks(prisma14 as unknown as PrismaClient); + + // id = WaitpointId.toId(friendlyId): the route computes the internal id from the friendly id, + // so the DB id must be exactly that bare cuid (classifies LEGACY -> control-plane store). + const { id: waitpointId, friendlyId } = WaitpointId.generate(); + const projectId = "proj_wc_route_leg"; + const environmentId = "env_wc_route_leg"; + projectHolder.id = projectId; + envHolder.id = environmentId; + engineHolder.calls = []; + + await seedPendingTokenWaitpoint(legacyStore, { + id: waitpointId, + friendlyId, + projectId, + environmentId, + }); + + const res = (await action({ + request: completeRequest("MANUAL"), + params: params(friendlyId), + context: {} as never, + })) as Response; + + // Property: the frozen replica was consulted (so the miss is real), yet the token completed via + // the owning-primary re-read — success, engine invoked, and NOT "No waitpoint found". + expect(legacyReplica.wasHit()).toBe(true); + expect(res.headers.get("x-outcome")).toBe("success"); + expect(res.headers.get("x-message")).toBe("Waitpoint completed"); + expect(res.headers.get("x-message")).not.toBe("No waitpoint found"); + expect(engineHolder.calls).toHaveLength(1); + expect(engineHolder.calls[0].id).toBe(waitpointId); + } + ); + + // Same seam via the DATETIME "skip" branch (also gated by the shared projectId guard). Kept as a + // second, mock-light assertion of the fallback so the guard doesn't hinge on MANUAL-branch helpers. + heteroRunOpsPostgresTest( + "DATETIME skip on a lag-invisible token resolves via the primary fallback", + async ({ prisma14, prisma17 }) => { + const legacyReplica = laggingReplica(prisma14, [{ model: "waitpoint", mode: "missing" }]); + const legacyStore = new PostgresRunStore({ + prisma: prisma14, + readOnlyPrisma: legacyReplica.client, + schemaVariant: "legacy", + }); + const newStore = new PostgresRunStore({ + prisma: prisma17 as never, + readOnlyPrisma: prisma17 as never, + schemaVariant: "dedicated", + }); + runStoreHolder.store = new RoutingRunStore({ new: newStore, legacy: legacyStore }); + + await dropWaitpointCrossSeamFks(prisma14 as unknown as PrismaClient); + + const { id: waitpointId, friendlyId } = WaitpointId.generate(); + const projectId = "proj_wc_route_dt"; + const environmentId = "env_wc_route_dt"; + projectHolder.id = projectId; + envHolder.id = environmentId; + engineHolder.calls = []; + + await seedPendingTokenWaitpoint(legacyStore, { + id: waitpointId, + friendlyId, + projectId, + environmentId, + type: "DATETIME", + }); + + const res = (await action({ + request: completeRequest("DATETIME"), + params: params(friendlyId), + context: {} as never, + })) as Response; + + expect(legacyReplica.wasHit()).toBe(true); + expect(res.headers.get("x-outcome")).toBe("success"); + expect(res.headers.get("x-message")).toBe("Waitpoint skipped"); + expect(engineHolder.calls).toHaveLength(1); + expect(engineHolder.calls[0].id).toBe(waitpointId); + } + ); +}); diff --git a/apps/webapp/test/waitpointPresenters.replicaLag.guard.test.ts b/apps/webapp/test/waitpointPresenters.replicaLag.guard.test.ts new file mode 100644 index 00000000000..f81ce6fa2fd --- /dev/null +++ b/apps/webapp/test/waitpointPresenters.replicaLag.guard.test.ts @@ -0,0 +1,568 @@ +// Property: the waitpoint-family READ presenters behave correctly under replica lag. Drives the REAL +// exported presenter classes (ApiWaitpointPresenter, WaitpointListPresenter, WaitpointPresenter, +// WaitpointTagListPresenter) through the REAL split RoutingRunStore over two testcontainer Postgres DBs +// (heteroRunOpsPostgresTest), the owning (LEGACY / control-plane) REPLICA frozen behind the shared +// laggingReplica, injected as the presenter's `runStore` arg. Only peripheral webapp singletons +// (db.server clients, runStore.server, logger, httpCallback URL, engine-version gate, clickhouse +// factory, control-plane env resolver, NextRunListPresenter leaf, ServiceValidationError) are stubbed. +// +// What each case proves, from the real caller's observable output under lag: +// 1 ApiWaitpointPresenter findWaitpoint — recovers a LIVE token on a replica miss via a +// findWaitpointOnPrimary fallback (returns the token; absent the fallback it throws +// "Waitpoint not found"). +// 2 WaitpointListPresenter findManyWaitpoints — the token list omits the just-minted token +// (tokens: []); eventual-consistency render, no write gated. +// 3 WaitpointListPresenter findWaitpoint — #probeAnyToken → hasAnyTokens false → cosmetic +// "no tokens yet" copy; self-heals next fetch. +// 4 WaitpointPresenter findWaitpoint — dashboard DETAIL loader returns null → not-found render; +// the user reloads. No write. +// 5 WaitpointPresenter findWaitpointConnectedRunIds — connected-runs sub-list empty on the detail +// render (connectedRuns: []); display only. +// 6 WaitpointPresenter findRuns — connected-run friendlyIds unresolved (connectedRuns: []) though +// the connection id was gathered; display only. +// 7 WaitpointTagListPresenter findManyWaitpointTags — tag-filter dropdown omits a just-created tag +// (tags: []); eventual-consistency list, no write. +// +// For cases 2-7, the test additionally drives the SAME presenter over a non-lagging router to prove +// the stale value is pure lag (the row is live on the primary), not absence. + +import { heteroRunOpsPostgresTest } from "@internal/testcontainers"; +import type { PrismaClient } from "@trigger.dev/database"; +import type { RunOpsPrismaClient } from "@internal/run-ops-database"; +import { describe, expect, vi } from "vitest"; + +// ── Fix-orthogonal module stubs (never the run-store read path) ──────────────────────────────────── +const { holder } = vi.hoisted(() => ({ + holder: { env: undefined as unknown }, +})); + +vi.mock("~/db.server", () => ({ prisma: {}, $replica: {} })); +vi.mock("~/v3/runStore.server", () => ({ runStore: {} })); +vi.mock("~/services/logger.server", () => ({ + logger: { warn: vi.fn(), info: vi.fn(), error: vi.fn(), debug: vi.fn() }, +})); +vi.mock("~/services/httpCallback.server", () => ({ + generateHttpCallbackUrl: () => "https://api.trigger.dev/http-callback", +})); +// ApiWaitpointPresenter imports ServiceValidationError from baseService.server, which itself drags the +// run-engine singleton at import; stub to a light Error subclass so the import graph stays cheap. +vi.mock("~/v3/services/baseService.server", () => ({ + ServiceValidationError: class ServiceValidationError extends Error {}, +})); +// Engine-version gate for the token LIST presenter — return V2 so it proceeds to the read under test. +vi.mock("~/v3/engineVersion.server", () => ({ + determineEngineVersion: vi.fn(async () => "V2"), +})); +// Reached by WaitpointPresenter only when connectedRuns is non-empty — never under lag; stub so import +// resolves without the real clickhouse client. +vi.mock("~/services/clickhouse/clickhouseFactoryInstance.server", () => ({ + clickhouseFactory: { getClickhouseForOrganization: vi.fn(async () => ({})) }, +})); +// The detail presenter resolves the env-derived fields (apiKey/organizationId) off this control-plane +// singleton after the waitpoint read succeeds. Peripheral to the run-store read path. +vi.mock("~/v3/runOpsMigration/controlPlaneResolver.server", () => ({ + controlPlaneResolver: { resolveAuthenticatedEnv: vi.fn(async () => holder.env) }, +})); +// Connected-run hydration leaf; never instantiated under lag (connectedRuns empty), but the import +// must resolve without its heavy graph. +vi.mock("~/presenters/v3/NextRunListPresenter.server", () => ({ + NextRunListPresenter: class { + async call() { + return { runs: [] }; + } + }, +})); + +import { PostgresRunStore, RoutingRunStore } from "@internal/run-store"; +import { ApiWaitpointPresenter } from "~/presenters/v3/ApiWaitpointPresenter.server"; +import { WaitpointListPresenter } from "~/presenters/v3/WaitpointListPresenter.server"; +import { WaitpointPresenter } from "~/presenters/v3/WaitpointPresenter.server"; +import { WaitpointTagListPresenter } from "~/presenters/v3/WaitpointTagListPresenter.server"; + +vi.setConfig({ testTimeout: 60_000, hookTimeout: 60_000 }); + +type AnyClient = PrismaClient | RunOpsPrismaClient; + +const CUID_25 = "e".repeat(25); // cuid id-shape → LEGACY (control-plane / prisma14) + +// A recording "replica" that has NOT caught up: reads for the SELECTED models/raw come back empty and +// flip `wasHit`, so a replica-routed read misses the just-written row. Everything else forwards to the +// real client (writes always land on the PRIMARY, so freezing the readOnly client never affects seeding). +function laggingReplica( + real: C, + freeze: { waitpoint?: boolean; taskRun?: boolean; waitpointTag?: boolean; raw?: boolean } +): { client: C; wasHit: () => boolean } { + let hit = false; + function frozenModel(target: any) { + return new Proxy(target, { + get(innerTarget, prop) { + if (prop === "findFirst" || prop === "findMany" || prop === "findUnique") { + return async () => { + hit = true; + return prop === "findMany" ? [] : null; + }; + } + if (prop === "findFirstOrThrow" || prop === "findUniqueOrThrow") { + return async () => { + hit = true; + throw new Error("lagging replica: row not visible"); + }; + } + return (innerTarget as any)[prop]; + }, + }); + } + const frozenWaitpoint = freeze.waitpoint ? frozenModel((real as any).waitpoint) : undefined; + const frozenTaskRun = freeze.taskRun ? frozenModel((real as any).taskRun) : undefined; + const frozenWaitpointTag = freeze.waitpointTag + ? frozenModel((real as any).waitpointTag) + : undefined; + const client = new Proxy(real, { + get(target, prop) { + if (prop === "waitpoint" && frozenWaitpoint) return frozenWaitpoint; + if (prop === "taskRun" && frozenTaskRun) return frozenTaskRun; + if (prop === "waitpointTag" && frozenWaitpointTag) return frozenWaitpointTag; + if (freeze.raw && (prop === "$queryRaw" || prop === "$queryRawUnsafe")) { + return async () => { + hit = true; + return []; + }; + } + return (target as any)[prop]; + }, + }) as C; + return { client, wasHit: () => hit }; +} + +// LEGACY-owning router whose legacy replica is frozen per `freeze`; the NEW store is real (non-lagging), +// so the on-miss fan-out to the other store's replica also legitimately misses. +function buildRouter( + prisma14: PrismaClient, + prisma17: RunOpsPrismaClient, + freeze: { waitpoint?: boolean; taskRun?: boolean; waitpointTag?: boolean; raw?: boolean } +) { + const legacyReplica = laggingReplica(prisma14, freeze); + const legacyStore = new PostgresRunStore({ + prisma: prisma14, + readOnlyPrisma: legacyReplica.client, + schemaVariant: "legacy", + }); + const newStore = new PostgresRunStore({ + prisma: prisma17 as never, + readOnlyPrisma: prisma17 as never, + schemaVariant: "dedicated", + }); + const router = new RoutingRunStore({ new: newStore, legacy: legacyStore }); + return { router, legacyStore, legacyReplica }; +} + +// A router with a fully non-lagging legacy replica — used to prove each case's stale value is +// pure lag (the row is live on the primary), by driving the SAME presenter against it. +function buildHealthyRouter(prisma14: PrismaClient, prisma17: RunOpsPrismaClient) { + return buildRouter(prisma14, prisma17, {}); +} + +async function seedEnvironmentLegacy(prisma: PrismaClient, suffix: string) { + const organization = await prisma.organization.create({ + data: { title: `Org ${suffix}`, slug: `org-${suffix}` }, + }); + const project = await prisma.project.create({ + data: { + name: `Project ${suffix}`, + slug: `project-${suffix}`, + externalRef: `proj_${suffix}`, + organizationId: organization.id, + }, + }); + const environment = await prisma.runtimeEnvironment.create({ + data: { + type: "DEVELOPMENT", + slug: "dev", + projectId: project.id, + organizationId: organization.id, + apiKey: `tr_dev_${suffix}`, + pkApiKey: `pk_dev_${suffix}`, + shortcode: `short_${suffix}`, + }, + }); + return { organization, project, environment }; +} + +async function seedManualWaitpoint( + store: PostgresRunStore, + params: { + id: string; + friendlyId: string; + projectId: string; + environmentId: string; + tags?: string[]; + } +) { + await store.upsertWaitpoint({ + where: { + environmentId_idempotencyKey: { + environmentId: params.environmentId, + idempotencyKey: params.id, + }, + }, + create: { + id: params.id, + friendlyId: params.friendlyId, + type: "MANUAL", + status: "PENDING", + idempotencyKey: params.id, + userProvidedIdempotencyKey: false, + projectId: params.projectId, + environmentId: params.environmentId, + ...(params.tags ? { tags: params.tags } : {}), + }, + update: {}, + }); +} + +function taskRunData(opts: { + id: string; + friendlyId: string; + organizationId: string; + projectId: string; + runtimeEnvironmentId: string; +}) { + return { + id: opts.id, + engine: "V2" as const, + status: "COMPLETED_SUCCESSFULLY" as const, + friendlyId: opts.friendlyId, + runtimeEnvironmentId: opts.runtimeEnvironmentId, + environmentType: "DEVELOPMENT" as const, + organizationId: opts.organizationId, + projectId: opts.projectId, + taskIdentifier: "my-task", + payload: JSON.stringify({ hello: "world" }), + payloadType: "application/json", + traceContext: {}, + traceId: `trace_${opts.id}`, + spanId: `span_${opts.id}`, + queue: "task/my-task", + isTest: false, + taskEventStore: "taskEvent", + depth: 0, + }; +} + +function apiEnvironment(env: { id: string; apiKey: string; projectId: string }) { + return { + id: env.id, + type: "DEVELOPMENT" as const, + project: { id: env.projectId, engine: "V2" as const }, + apiKey: env.apiKey, + }; +} + +describe("waitpoint-family read presenters under replica lag", () => { + // ApiWaitpointPresenter findWaitpoint — recovers a live token via a primary fallback + heteroRunOpsPostgresTest( + "ApiWaitpointPresenter.call resolves a live token under replica lag", + async ({ prisma14, prisma17 }) => { + const { router, legacyStore, legacyReplica } = buildRouter(prisma14, prisma17, { + waitpoint: true, + }); + const seed = await seedEnvironmentLegacy(prisma14, "api_wp"); + const waitpointId = `waitpoint_${CUID_25}`; // cuid → LEGACY + await seedManualWaitpoint(legacyStore, { + id: waitpointId, + friendlyId: "waitpoint_api_wp", + projectId: seed.project.id, + environmentId: seed.environment.id, + }); + + const presenter = new ApiWaitpointPresenter(prisma14, prisma14, undefined, router); + + // The presenter's replica read misses, then re-reads the owning primary and returns the LIVE + // token via the findWaitpointOnPrimary fallback. + const result = await presenter.call( + apiEnvironment({ + id: seed.environment.id, + apiKey: seed.environment.apiKey, + projectId: seed.project.id, + }), + waitpointId + ); + + // The replica WAS consulted (and, frozen, missed) — proving the recovery is the primary fallback. + expect(legacyReplica.wasHit()).toBe(true); + expect(result.id).toBe("waitpoint_api_wp"); + expect(result.type).toBe("MANUAL"); + expect(result.status).toBe("WAITING"); + } + ); + + // Negative control: a token that truly does not exist anywhere still throws (the fallback recovers a + // LIVE token, it does not blanket-swallow the not-found). + heteroRunOpsPostgresTest( + "control: ApiWaitpointPresenter.call throws when the token is absent on the primary too", + async ({ prisma14, prisma17 }) => { + const { router } = buildRouter(prisma14, prisma17, { waitpoint: true }); + const seed = await seedEnvironmentLegacy(prisma14, "api_absent"); + const presenter = new ApiWaitpointPresenter(prisma14, prisma14, undefined, router); + await expect( + presenter.call( + apiEnvironment({ + id: seed.environment.id, + apiKey: seed.environment.apiKey, + projectId: seed.project.id, + }), + `waitpoint_${"f".repeat(25)}` + ) + ).rejects.toThrow(/not found/i); + } + ); + + // WaitpointListPresenter findManyWaitpoints / findWaitpoint + heteroRunOpsPostgresTest( + "WaitpointListPresenter.call omits a just-minted token under replica lag", + async ({ prisma14, prisma17 }) => { + const { router, legacyStore, legacyReplica } = buildRouter(prisma14, prisma17, { + waitpoint: true, + }); + const seed = await seedEnvironmentLegacy(prisma14, "list"); + await seedManualWaitpoint(legacyStore, { + id: `waitpoint_${CUID_25}`, + friendlyId: "waitpoint_list", + projectId: seed.project.id, + environmentId: seed.environment.id, + }); + + const env = apiEnvironment({ + id: seed.environment.id, + apiKey: seed.environment.apiKey, + projectId: seed.project.id, + }); + + const laggy = await new WaitpointListPresenter(prisma14, prisma14, undefined, router).call({ + environment: env, + }); + + // The list render omits the token this pass (findManyWaitpoints), and the empty-state probe + // reports hasAnyTokens=false (the #probeAnyToken findWaitpoint read, cosmetic copy). No write + // gated; self-heals next fetch. + expect(laggy.success).toBe(true); + if (!laggy.success) throw new Error("unreachable"); + expect(laggy.tokens).toEqual([]); + expect(laggy.hasAnyTokens).toBe(false); + expect(legacyReplica.wasHit()).toBe(true); + + // Prove pure lag: the SAME presenter over a healthy router surfaces the live token. + const healthy = await new WaitpointListPresenter( + prisma14, + prisma14, + undefined, + buildHealthyRouter(prisma14, prisma17).router + ).call({ environment: env }); + expect(healthy.success).toBe(true); + if (!healthy.success) throw new Error("unreachable"); + expect(healthy.tokens).toHaveLength(1); + expect(healthy.tokens[0]!.id).toBe("waitpoint_list"); + expect(healthy.hasAnyTokens).toBe(true); + } + ); + + // WaitpointPresenter findWaitpoint + heteroRunOpsPostgresTest( + "WaitpointPresenter.call returns null for a token detail under replica lag", + async ({ prisma14, prisma17 }) => { + const { router, legacyStore, legacyReplica } = buildRouter(prisma14, prisma17, { + waitpoint: true, + }); + const seed = await seedEnvironmentLegacy(prisma14, "detail"); + const friendlyId = "waitpoint_detail"; + await seedManualWaitpoint(legacyStore, { + id: `waitpoint_${CUID_25}`, + friendlyId, + projectId: seed.project.id, + environmentId: seed.environment.id, + }); + holder.env = { + id: seed.environment.id, + organizationId: seed.organization.id, + apiKey: seed.environment.apiKey, + }; + + const laggy = await new WaitpointPresenter(prisma14, prisma14, undefined, router).call({ + friendlyId, + environmentId: seed.environment.id, + projectId: seed.project.id, + }); + + // The detail loader returns null → not-found page → user reloads. No write. + expect(laggy).toBeNull(); + expect(legacyReplica.wasHit()).toBe(true); + + const healthy = await new WaitpointPresenter( + prisma14, + prisma14, + undefined, + buildHealthyRouter(prisma14, prisma17).router + ).call({ friendlyId, environmentId: seed.environment.id, projectId: seed.project.id }); + expect(healthy).not.toBeNull(); + expect(healthy!.id).toBe(friendlyId); + expect(healthy!.type).toBe("MANUAL"); + } + ); + + // WaitpointPresenter findWaitpointConnectedRunIds + heteroRunOpsPostgresTest( + "WaitpointPresenter.call renders an empty connected-runs list when the connection join lags", + async ({ prisma14, prisma17 }) => { + // Freeze only the raw connection JOIN: the waitpoint read succeeds, so the caller reaches the + // connection-id gather, which comes back empty under lag. + const { router, legacyStore, legacyReplica } = buildRouter(prisma14, prisma17, { raw: true }); + const seed = await seedEnvironmentLegacy(prisma14, "conn68"); + const waitpointId = `waitpoint_${CUID_25}`; + const runId = `run_${CUID_25}`; // cuid → LEGACY, co-located with the token + const friendlyId = "waitpoint_conn68"; + await seedManualWaitpoint(legacyStore, { + id: waitpointId, + friendlyId, + projectId: seed.project.id, + environmentId: seed.environment.id, + }); + await prisma14.taskRun.create({ + data: taskRunData({ + id: runId, + friendlyId: "run_conn68", + organizationId: seed.organization.id, + projectId: seed.project.id, + runtimeEnvironmentId: seed.environment.id, + }), + }); + await router.blockRunWithWaitpointEdges({ + runId, + waitpointIds: [waitpointId], + projectId: seed.project.id, + }); + holder.env = { + id: seed.environment.id, + organizationId: seed.organization.id, + apiKey: seed.environment.apiKey, + }; + + const laggy = await new WaitpointPresenter(prisma14, prisma14, undefined, router).call({ + friendlyId, + environmentId: seed.environment.id, + projectId: seed.project.id, + }); + + // The token detail renders (found via replica) but the connected-runs list is empty this pass — + // display only, no decision on the stale set. + expect(laggy).not.toBeNull(); + expect(laggy!.id).toBe(friendlyId); + expect(laggy!.connectedRuns).toEqual([]); + expect(legacyReplica.wasHit()).toBe(true); + + // Prove pure lag: over a healthy router the connection id is gathered. + const healthyRouter = buildHealthyRouter(prisma14, prisma17).router; + const connectedIds = await healthyRouter.findWaitpointConnectedRunIds(waitpointId); + expect(connectedIds).toEqual([runId]); + } + ); + + // WaitpointPresenter findRuns + heteroRunOpsPostgresTest( + "WaitpointPresenter.call leaves connected-runs empty when the run lookup lags", + async ({ prisma14, prisma17 }) => { + // Freeze only taskRun reads: the waitpoint read and the connection gather succeed, so the caller + // reaches the friendlyId resolution, which comes back empty. + const { router, legacyStore, legacyReplica } = buildRouter(prisma14, prisma17, { + taskRun: true, + }); + const seed = await seedEnvironmentLegacy(prisma14, "conn72"); + const waitpointId = `waitpoint_${CUID_25}`; + const runId = `run_${CUID_25}`; + const friendlyId = "waitpoint_conn72"; + await seedManualWaitpoint(legacyStore, { + id: waitpointId, + friendlyId, + projectId: seed.project.id, + environmentId: seed.environment.id, + }); + await prisma14.taskRun.create({ + data: taskRunData({ + id: runId, + friendlyId: "run_conn72", + organizationId: seed.organization.id, + projectId: seed.project.id, + runtimeEnvironmentId: seed.environment.id, + }), + }); + await router.blockRunWithWaitpointEdges({ + runId, + waitpointIds: [waitpointId], + projectId: seed.project.id, + }); + holder.env = { + id: seed.environment.id, + organizationId: seed.organization.id, + apiKey: seed.environment.apiKey, + }; + + // Confirm this read is genuinely reached: the connection gather ran (raw NOT frozen); the taskRun + // read is the one that lags. + const connectedIds = await router.findWaitpointConnectedRunIds(waitpointId); + expect(connectedIds).toEqual([runId]); + + const laggy = await new WaitpointPresenter(prisma14, prisma14, undefined, router).call({ + friendlyId, + environmentId: seed.environment.id, + projectId: seed.project.id, + }); + + // Connected-run friendlyId resolution missed under lag → connectedRuns empty this render; + // display only. + expect(laggy).not.toBeNull(); + expect(laggy!.connectedRuns).toEqual([]); + expect(legacyReplica.wasHit()).toBe(true); + + // Prove pure lag: over a healthy router findRuns resolves the run's friendlyId. + const healthyRouter = buildHealthyRouter(prisma14, prisma17).router; + const runs = (await healthyRouter.findRuns({ + where: { id: { in: [runId] } }, + select: { friendlyId: true }, + take: 5, + })) as Array<{ friendlyId: string }>; + expect(runs).toHaveLength(1); + expect(runs[0]!.friendlyId).toBe("run_conn72"); + } + ); + + // WaitpointTagListPresenter findManyWaitpointTags + heteroRunOpsPostgresTest( + "WaitpointTagListPresenter.call omits a just-created tag under replica lag", + async ({ prisma14, prisma17 }) => { + const { router, legacyStore, legacyReplica } = buildRouter(prisma14, prisma17, { + waitpointTag: true, + }); + const seed = await seedEnvironmentLegacy(prisma14, "tags"); + await legacyStore.upsertWaitpointTag({ + environmentId: seed.environment.id, + name: "my-tag", + projectId: seed.project.id, + }); + + const laggy = await new WaitpointTagListPresenter(prisma14, prisma14, undefined, router).call( + { environmentId: seed.environment.id } + ); + + // The tag-filter dropdown omits the new tag this render; eventual-consistency list. + expect(laggy.tags).toEqual([]); + expect(legacyReplica.wasHit()).toBe(true); + + const healthy = await new WaitpointTagListPresenter( + prisma14, + prisma14, + undefined, + buildHealthyRouter(prisma14, prisma17).router + ).call({ environmentId: seed.environment.id }); + expect(healthy.tags).toEqual([{ name: "my-tag" }]); + } + ); +}); diff --git a/internal-packages/run-engine/src/engine/retryDecisionReadAfterWrite.replicaLag.test.ts b/internal-packages/run-engine/src/engine/retryDecisionReadAfterWrite.replicaLag.test.ts new file mode 100644 index 00000000000..6833f0bcd05 --- /dev/null +++ b/internal-packages/run-engine/src/engine/retryDecisionReadAfterWrite.replicaLag.test.ts @@ -0,0 +1,185 @@ +// Property: the retry-vs-fail decision must read maxAttempts / lockedRetryConfig from the OWNING +// PRIMARY. Those fields are written at lock time (a recent primary write); served from a lagging +// replica they read null and a run WITH retries remaining is permanently failed instead of retried. +// Freeze the replica at the pre-lock snapshot via the shared `laggingReplica` primitive and drive the +// REAL retryOutcomeFromCompletion: reading via the primary yields "retry", reading via the stale +// replica yields "fail_run" — the replica lag alone is what flips the decision. + +import { laggingReplica, postgresTest } from "@internal/testcontainers"; +import { PostgresRunStore } from "@internal/run-store"; +import type { CreateRunInput } from "@internal/run-store"; +import type { PrismaClient } from "@trigger.dev/database"; +import type { TaskRunError, TaskRunExecutionRetry } from "@trigger.dev/core/v3"; +import { describe, expect } from "vitest"; +import { retryOutcomeFromCompletion } from "./retrying.js"; + +async function seedEnvironment(prisma: PrismaClient, suffix: string) { + const organization = await prisma.organization.create({ + data: { title: `Org ${suffix}`, slug: `org-${suffix}` }, + }); + const project = await prisma.project.create({ + data: { + name: `Project ${suffix}`, + slug: `project-${suffix}`, + externalRef: `proj_${suffix}`, + organizationId: organization.id, + }, + }); + const environment = await prisma.runtimeEnvironment.create({ + data: { + type: "DEVELOPMENT", + slug: "dev", + projectId: project.id, + organizationId: organization.id, + apiKey: `tr_dev_${suffix}`, + pkApiKey: `pk_dev_${suffix}`, + shortcode: `short_${suffix}`, + }, + }); + return { organization, project, environment }; +} + +function buildCreateRunInput(params: { + runId: string; + friendlyId: string; + organizationId: string; + projectId: string; + runtimeEnvironmentId: string; +}): CreateRunInput { + return { + data: { + id: params.runId, + engine: "V2", + status: "EXECUTING", + friendlyId: params.friendlyId, + runtimeEnvironmentId: params.runtimeEnvironmentId, + environmentType: "DEVELOPMENT", + organizationId: params.organizationId, + projectId: params.projectId, + taskIdentifier: "my-task", + payload: '{"hello":"world"}', + payloadType: "application/json", + context: { foo: "bar" }, + traceContext: { trace: "ctx" }, + traceId: "trace_1", + spanId: "span_1", + runTags: [], + queue: "task/my-task", + isTest: false, + taskEventStore: "taskEvent", + depth: 0, + createdAt: new Date("2024-01-01T00:00:00.000Z"), + }, + snapshot: { + engine: "V2", + executionStatus: "EXECUTING", + description: "Run is executing", + runStatus: "EXECUTING", + environmentId: params.runtimeEnvironmentId, + environmentType: "DEVELOPMENT", + projectId: params.projectId, + organizationId: params.organizationId, + }, + }; +} + +// A plain retriable error (BUILT_IN_ERROR is retriable and not OOM), so the decision falls through +// to the maxAttempts / lockedRetryConfig gates — exactly the read this test targets. +const RETRIABLE_ERROR: TaskRunError = { + type: "BUILT_IN_ERROR", + name: "Error", + message: "boom", + stackTrace: "at handler (task.ts:1:1)", +}; + +// The completion carries explicit retry settings (the normal SDK-failure path), so once the run's +// maxAttempts is read as present the outcome is deterministically "retry" — isolating the read as +// the ONLY thing that flips the decision. +const RETRY_SETTINGS: TaskRunExecutionRetry = { timestamp: Date.now() + 60_000, delay: 60_000 }; + +describe("retry-vs-fail decision under replica lag", () => { + postgresTest( + "a retriable attempt retries when lock-time maxAttempts is read from the primary, not the lagging replica", + async ({ prisma }) => { + const { organization, project, environment } = await seedEnvironment(prisma, "retry_lag"); + const runId = `run_${"a".repeat(24)}`; + + // Build the store the way the engine does; its readOnlyPrisma is a replica (set below). + // The write client is the primary. + const runId_friendly = "run_retrylag"; + const primaryStore = new PostgresRunStore({ + prisma, + readOnlyPrisma: prisma, + schemaVariant: "legacy", + }); + await primaryStore.createRun( + buildCreateRunInput({ + runId, + friendlyId: runId_friendly, + organizationId: organization.id, + projectId: project.id, + runtimeEnvironmentId: environment.id, + }) + ); + + // Snapshot the run as the replica still sees it: PRE-LOCK, so maxAttempts / lockedRetryConfig + // are still null (the run was created but not yet locked to a worker). + const staleRun = await prisma.taskRun.findFirstOrThrow({ where: { id: runId } }); + expect(staleRun.maxAttempts).toBeNull(); + expect(staleRun.lockedRetryConfig).toBeNull(); + + // The attempt is locked: the primary now records maxAttempts + lockedRetryConfig (the recent + // lock-time write). This run has 3 attempts and is on attempt 1 -> it SHOULD retry. + await prisma.taskRun.update({ + where: { id: runId }, + data: { + maxAttempts: 3, + lockedRetryConfig: { + maxAttempts: 3, + minTimeoutInMs: 1000, + maxTimeoutInMs: 10000, + factor: 2, + }, + machinePreset: "small-1x", + usageDurationMs: 100, + costInCents: 0, + }, + }); + + // ...but the replica lags, frozen at the pre-lock snapshot (maxAttempts = null). + const replica = laggingReplica(prisma, [ + { model: "taskRun", mode: "frozen", rows: [staleRun] }, + ]); + + // The store as the engine holds it: reads default to the (lagging) replica. + const store = new PostgresRunStore({ + prisma, + readOnlyPrisma: replica.client, + schemaVariant: "legacy", + }); + + // Mirrors what the engine does — it passes `this.$.readOnlyPrisma` (the REPLICA) as the first + // arg. `readClient` models what the engine threads in. + const decide = (readClient: typeof prisma | typeof replica.client) => + retryOutcomeFromCompletion(readClient, store, { + runId, + attemptNumber: 1, + error: RETRIABLE_ERROR, + retryUsingQueue: false, + retrySettings: RETRY_SETTINGS, + }); + + // Reading the decision on the owning PRIMARY (this.$.prisma) sees maxAttempts = 3, so the run + // retries — the behaviour the engine must get. + const fromPrimary = await decide(prisma); + expect(fromPrimary.outcome).toBe("retry"); + + // Reading the same decision off the lagging replica (this.$.readOnlyPrisma) nulls maxAttempts, so + // a run with retries remaining is permanently FAILED. Pinning fail_run documents the misrouting + // the engine must never do. + const fromReplica = await decide(replica.client); + expect(replica.wasHit("taskRun")).toBe(true); // the decision read hit the stale replica + expect(fromReplica.outcome).toBe("fail_run"); + } + ); +}); diff --git a/internal-packages/run-engine/src/engine/systems/ttlSystemExpireReplicaLag.guard.test.ts b/internal-packages/run-engine/src/engine/systems/ttlSystemExpireReplicaLag.guard.test.ts new file mode 100644 index 00000000000..10414f3aa91 --- /dev/null +++ b/internal-packages/run-engine/src/engine/systems/ttlSystemExpireReplicaLag.guard.test.ts @@ -0,0 +1,154 @@ +// Verifies TtlSystem.expireRunsBatch reads its findRuns existence check off the owning store's WRITER, +// so a PENDING run whose row has not yet replicated is still found and EXPIRED (never orphaned as +// not_found). Because the TTL Lua script has already dequeued the run, a stale replica miss would not +// self-heal. Drives the REAL engine method against a PostgresRunStore whose replica is frozen; real +// Postgres + Redis via containerTest, laggingReplica from @internal/testcontainers, never mocked. + +import { containerTest, laggingReplica } from "@internal/testcontainers"; +import { trace } from "@internal/tracing"; +import { expect } from "vitest"; +import { PostgresRunStore } from "@internal/run-store"; +import type { RunStore, CreateRunInput } from "@internal/run-store"; +import type { PrismaClient } from "@trigger.dev/database"; +import { RunEngine } from "../index.js"; +import { setupAuthenticatedEnvironment } from "../tests/setup.js"; + +vi.setConfig({ testTimeout: 60_000 }); + +function createEngine(prisma: PrismaClient, redisOptions: any, store: RunStore) { + return new RunEngine({ + prisma, + store, + worker: { + redis: redisOptions, + workers: 1, + tasksPerWorker: 10, + pollIntervalMs: 100, + }, + queue: { + redis: redisOptions, + processWorkerQueueDebounceMs: 50, + masterQueueConsumersDisabled: true, + // Disable the automatic TTL sweep so the ONLY caller of expireRunsBatch is our explicit + // invocation — nothing races it or re-expires the run behind our back. + ttlSystem: { disabled: true }, + }, + runLock: { + redis: redisOptions, + }, + machines: { + defaultMachine: "small-1x", + machines: { + "small-1x": { + name: "small-1x" as const, + cpu: 0.5, + memory: 0.5, + centsPerMs: 0.0001, + }, + }, + baseCostInCents: 0.0001, + }, + tracer: trace.getTracer("test", "0.0.0"), + }); +} + +function buildCreateRunInput(params: { + runId: string; + friendlyId: string; + organizationId: string; + projectId: string; + runtimeEnvironmentId: string; +}): CreateRunInput { + return { + data: { + id: params.runId, + engine: "V2", + status: "PENDING", + friendlyId: params.friendlyId, + runtimeEnvironmentId: params.runtimeEnvironmentId, + environmentType: "PRODUCTION", + organizationId: params.organizationId, + projectId: params.projectId, + taskIdentifier: "my-task", + payload: '{"hello":"world"}', + payloadType: "application/json", + context: { foo: "bar" }, + traceContext: { trace: "ctx" }, + traceId: "trace_ttl_lag", + spanId: "span_ttl_lag", + runTags: [], + queue: "task/my-task", + isTest: false, + taskEventStore: "taskEvent", + depth: 0, + ttl: "1s", + createdAt: new Date("2024-01-01T00:00:00.000Z"), + }, + snapshot: { + engine: "V2", + executionStatus: "QUEUED", + description: "Run was created", + runStatus: "PENDING", + environmentId: params.runtimeEnvironmentId, + environmentType: "PRODUCTION", + projectId: params.projectId, + organizationId: params.organizationId, + }, + }; +} + +describe("TtlSystem.expireRunsBatch — read-your-writes under replica lag", () => { + containerTest( + "expires a PENDING run whose row is not yet visible on the lagging replica", + async ({ prisma, redisOptions }) => { + const authenticatedEnvironment = await setupAuthenticatedEnvironment(prisma, "PRODUCTION"); + + // The store's replica lags: `taskRun` reads come back empty, exactly as just after a + // very-short-TTL run is created and immediately swept. The WRITER (prisma) is up to date. + const replica = laggingReplica(prisma, [{ model: "taskRun", mode: "missing" }]); + const store = new PostgresRunStore({ prisma, readOnlyPrisma: replica.client }); + + const engine = createEngine(prisma, redisOptions, store); + + try { + // Seed the run directly through the store's WRITER so its row exists on the primary but is NOT + // visible on the lagging replica. (Created directly rather than via engine.trigger so that + // setup performs no replica reads that could interfere with the single read under test.) + const runId = "run_ttl_lag_guard_0000001"; + await store.createRun( + buildCreateRunInput({ + runId, + friendlyId: "run_ttllag1", + organizationId: authenticatedEnvironment.organization.id, + projectId: authenticatedEnvironment.project.id, + runtimeEnvironmentId: authenticatedEnvironment.id, + }) + ); + + // Sanity: the run really IS an expirable PENDING run on the writer, so the only reason it + // could fail to expire is a misrouted read against the stale replica. + const onPrimary = await store.findRun({ id: runId }, prisma); + expect(onPrimary).not.toBeNull(); + expect(onPrimary!.status).toBe("PENDING"); + + // Drive the REAL engine method. Its findRuns read is handed the writer (this.$.prisma) and + // finds the run rather than hitting the lagging replica. + const result = await engine.ttlSystem.expireRunsBatch([runId]); + + // The run is expired and nothing is skipped. + expect(result.expired).toEqual([runId]); + expect(result.skipped).toEqual([]); + + // The run is durably EXPIRED on the writer (the persisted outcome under test). + const persisted = await prisma.taskRun.findUniqueOrThrow({ where: { id: runId } }); + expect(persisted.status).toBe("EXPIRED"); + + // The existence-check read routes to the writer, so the stale replica is NEVER consulted for the + // expiry check — the seam these assertions pin. + expect(replica.wasHit("taskRun")).toBe(false); + } finally { + await engine.quit(); + } + } + ); +}); diff --git a/internal-packages/run-engine/src/engine/tests/concurrencySweeperCallbackReplicaLag.guard.test.ts b/internal-packages/run-engine/src/engine/tests/concurrencySweeperCallbackReplicaLag.guard.test.ts new file mode 100644 index 00000000000..4dac600ed0d --- /dev/null +++ b/internal-packages/run-engine/src/engine/tests/concurrencySweeperCallbackReplicaLag.guard.test.ts @@ -0,0 +1,122 @@ +// Replica-lag guard for the run-store read inside the private #concurrencySweeperCallback. The +// callback cannot be called by name (it is private), but the engine wires the REAL bound method as +// the RunQueue's concurrency-sweeper callback (`callback: this.#concurrencySweeperCallback.bind(this)`) +// and `RunQueue.options` is public-readonly, so this test invokes +// `engine.runQueue.options.concurrencySweeper.callback([...])` — the exact production callback object, +// driven end-to-end over the engine's real runStore. RunQueue's own Redis scan is the only thing +// bypassed (it merely feeds the callback the runIds it already receives from +// `processCurrentConcurrencyRunIds`). +// +// The read: findRuns for runs finished > 10 min ago with a final status and an org set (client-less → +// REPLICA); the callback releases the stale concurrency those runs still hold. +// +// Property: this lag is tolerable and self-healing. Under lag the read misses the finished run and the +// callback returns [] (no runs marked for ack), so the run keeps its concurrency slot THIS scan; the +// sweeper runs periodically, so the next scan (once replicated) releases it — a miss delays cleanup by +// one scan, never wrongly releases a live run, and never leaks permanently. A second engine over a +// caught-up replica proves the SAME wired callback returns the run. + +import { containerTest, laggingReplica } from "@internal/testcontainers"; +import { trace } from "@internal/tracing"; +import { PostgresRunStore } from "@internal/run-store"; +import type { PrismaClient } from "@trigger.dev/database"; +import { describe, expect } from "vitest"; +import { RunEngine } from "../index.js"; +import { setupAuthenticatedEnvironment } from "./setup.js"; + +function baseEngineOptions(redisOptions: any) { + return { + worker: { redis: redisOptions, workers: 1, tasksPerWorker: 10, pollIntervalMs: 100 }, + queue: { + redis: redisOptions, + masterQueueConsumersDisabled: true, + processWorkerQueueDebounceMs: 50, + }, + runLock: { redis: redisOptions }, + machines: { + defaultMachine: "small-1x" as const, + machines: { + "small-1x": { name: "small-1x" as const, cpu: 0.5, memory: 0.5, centsPerMs: 0.0001 }, + }, + baseCostInCents: 0.0001, + }, + tracer: trace.getTracer("test", "0.0.0"), + }; +} + +async function seedFinishedRun(prisma: PrismaClient, environment: any, runId: string) { + // Completed 30 min ago, final status, org set — matches the callback's predicate exactly. On the PRIMARY. + await prisma.taskRun.create({ + data: { + id: runId, + engine: "V2", + status: "COMPLETED_SUCCESSFULLY", + friendlyId: `fr_${runId}`, + taskIdentifier: "my-task", + payload: "{}", + payloadType: "application/json", + traceId: `trace_${runId}`, + spanId: `span_${runId}`, + queue: "task/my-task", + runtimeEnvironmentId: environment.id, + projectId: environment.project.id, + organizationId: environment.organization.id, + environmentType: "PRODUCTION", + createdAt: new Date(Date.now() - 60 * 60 * 1000), + completedAt: new Date(Date.now() - 30 * 60 * 1000), + }, + }); +} + +describe("concurrency sweeper callback — replica-lag guard", () => { + containerTest( + "the wired sweeper callback misses a finished run under lag (returns []) — self-healing; a caught-up replica proves the same callback finds it", + async ({ prisma, redisOptions }) => { + const environment = await setupAuthenticatedEnvironment(prisma, "PRODUCTION"); + const runId = "run_nnnnnnnnnnnnnnnnnnnnnn91"; + await seedFinishedRun(prisma, environment, runId); + + // Engine A — taskRun frozen-missing on the replica; the callback's client-less findRuns routes there. + const laggingReplicaHandle = laggingReplica(prisma, [{ model: "taskRun", mode: "missing" }]); + const laggingStore = new PostgresRunStore({ + prisma, + readOnlyPrisma: laggingReplicaHandle.client as never, + }); + const laggingEngine = new RunEngine({ + prisma, + readOnlyPrisma: laggingReplicaHandle.client as never, + store: laggingStore, + ...baseEngineOptions(redisOptions), + }); + + // Engine B — caught-up replica (control), proving the wired callback path is live. + const liveStore = new PostgresRunStore({ prisma, readOnlyPrisma: prisma }); + const liveEngine = new RunEngine({ + prisma, + readOnlyPrisma: prisma, + store: liveStore, + ...baseEngineOptions(redisOptions), + }); + + try { + // Lagging: the REAL wired callback (engine's private #concurrencySweeperCallback) misses the + // finished run and returns [] — the run keeps its concurrency slot this scan (no throw). + const lagging = await laggingEngine.runQueue.options.concurrencySweeper!.callback([runId]); + expect(lagging).toEqual([]); + expect(laggingReplicaHandle.wasHit("taskRun")).toBe(true); + + // Caught-up control: the SAME wired callback finds the finished run and marks it for ack. + const caughtUp = await liveEngine.runQueue.options.concurrencySweeper!.callback([runId]); + expect(caughtUp.map((r) => r.id)).toEqual([runId]); + expect(caughtUp[0]!.orgId).toBe(environment.organization.id); + + // The miss is pure lag: the run is genuinely finished on the PRIMARY (so a later scan releases it). + const onPrimary = await prisma.taskRun.findFirstOrThrow({ where: { id: runId } }); + expect(onPrimary.status).toBe("COMPLETED_SUCCESSFULLY"); + } finally { + await laggingEngine.quit(); + await liveEngine.quit(); + } + } + ); +}); diff --git a/internal-packages/run-engine/src/engine/tests/runAttemptSystemReplicaLag.guard.test.ts b/internal-packages/run-engine/src/engine/tests/runAttemptSystemReplicaLag.guard.test.ts new file mode 100644 index 00000000000..1ebabdc8272 --- /dev/null +++ b/internal-packages/run-engine/src/engine/tests/runAttemptSystemReplicaLag.guard.test.ts @@ -0,0 +1,516 @@ +// Property: the completion-path reads in runAttemptSystem hit the owning primary, not a lagging replica. +// These drive the REAL RunEngine methods end-to-end (trigger → dequeue → startRunAttempt → +// complete/cancel), not a reimplementation. The store is a real PostgresRunStore whose primary is the +// live container and whose replica is a lagging proxy serving a STALE row for exactly the read under +// test. The same proxy is the engine's readOnlyPrisma, so a read routed to the owning primary +// (this.$.prisma / findRunOnPrimary) never touches the proxy, while a replica-routed read +// (this.$.readOnlyPrisma / client-less findRun) hits the stale value. Unlike the retry test that passes +// its own client into retryOutcomeFromCompletion, this exercises WHICH client the engine threads. + +import { assertNonNullable, containerTest, laggingReplica } from "@internal/testcontainers"; +import { trace } from "@internal/tracing"; +import { PostgresRunStore } from "@internal/run-store"; +import type { PrismaClient } from "@trigger.dev/database"; +import { setTimeout } from "node:timers/promises"; +import { describe, expect } from "vitest"; +import { RunEngine } from "../index.js"; +import { setupAuthenticatedEnvironment, setupBackgroundWorker } from "./setup.js"; + +// A read-replica that lags its primary for exactly two completion-path reads, matched by the engine's +// own `select` shape so no unrelated read is disturbed. `retryStale` fabricates the pre-lock snapshot +// (maxAttempts/lockedRetryConfig still null) for the retry-decision read; `usageStale` fabricates a +// pre-accumulation snapshot (usage totals still at their earlier value) for the usage RMW read. Every +// other property/method forwards to the real client. `hits` proves whether the lagging replica was +// actually consulted (0 when the reads go to the primary; >0 when they hit the replica). +type LagState = { + retryStale: boolean; + usageStale: { usageDurationMs: number; costInCents: number; machinePreset: string | null } | null; +}; + +function laggingReadReplica( + real: C +): { client: C; state: LagState; hits: { retry: number; usage: number } } { + const state: LagState = { retryStale: false, usageStale: null }; + const hits = { retry: 0, usage: 0 }; + + const isRetryDecisionSelect = (select: any) => + !!select && select.maxAttempts === true && select.lockedRetryConfig === true; + + // The usage RMW read (attemptSucceeded / cancelRun / permanentlyFailRun) selects exactly these three + // scalars and nothing else — and crucially never maxAttempts, which distinguishes it from the retry + // read above. + const isUsageSelect = (select: any) => + !!select && + select.usageDurationMs === true && + select.costInCents === true && + select.machinePreset === true && + !("maxAttempts" in select) && + !("lockedRetryConfig" in select); + + const realTaskRun = (real as any).taskRun; + + const taskRunProxy = new Proxy(realTaskRun, { + get(target, prop) { + if (prop === "findFirst") { + return async (args?: { select?: any }) => { + const select = args?.select; + if (state.retryStale && isRetryDecisionSelect(select)) { + hits.retry++; + // Replica has not applied the lock-time write yet: no retry config visible. + return { + maxAttempts: null, + lockedRetryConfig: null, + usageDurationMs: 0, + costInCents: 0, + machinePreset: null, + }; + } + if (state.usageStale && isUsageSelect(select)) { + hits.usage++; + // Replica has not applied the accumulated-usage write yet: the earlier totals. + return { ...state.usageStale }; + } + return target.findFirst(args); + }; + } + const value = target[prop]; + return typeof value === "function" ? value.bind(target) : value; + }, + }); + + const client = new Proxy(real, { + get(target, prop) { + if (prop === "taskRun") { + return taskRunProxy; + } + const value = (target as any)[prop]; + return typeof value === "function" ? value.bind(target) : value; + }, + }) as C; + + return { client, state, hits }; +} + +function baseEngineOptions(redisOptions: any) { + return { + worker: { redis: redisOptions, workers: 1, tasksPerWorker: 10, pollIntervalMs: 100 }, + queue: { + redis: redisOptions, + masterQueueConsumersDisabled: true, + processWorkerQueueDebounceMs: 50, + }, + runLock: { redis: redisOptions }, + machines: { + defaultMachine: "small-1x" as const, + machines: { + "small-1x": { name: "small-1x" as const, cpu: 0.5, memory: 0.5, centsPerMs: 0.0001 }, + }, + baseCostInCents: 0.0001, + }, + tracer: trace.getTracer("test", "0.0.0"), + }; +} + +const triggerParams = (friendlyId: string, environment: any, taskIdentifier: string) => ({ + number: 1, + friendlyId, + environment, + taskIdentifier, + payload: "{}", + payloadType: "application/json", + context: {}, + traceContext: {}, + traceId: "t12345", + spanId: "s12345", + workerQueue: "main", + queue: `task/${taskIdentifier}`, + isTest: false, + tags: [] as string[], +}); + +// Drive the real engine to a locked, EXECUTING first attempt over a store whose replica is the +// lagging proxy. Returns everything the individual guards need. +async function triggerAndStartAttempt( + prisma: PrismaClient, + redisOptions: any, + friendlyId: string, + retryOptions?: any +) { + const environment = await setupAuthenticatedEnvironment(prisma, "PRODUCTION"); + + const replica = laggingReadReplica(prisma); + const store = new PostgresRunStore({ + prisma, + readOnlyPrisma: replica.client as never, + }); + const engine = new RunEngine({ + prisma, + readOnlyPrisma: replica.client as never, + store, + ...baseEngineOptions(redisOptions), + }); + + try { + const taskIdentifier = "test-task"; + await setupBackgroundWorker(engine, environment, taskIdentifier, undefined, retryOptions); + + const run = await engine.trigger( + triggerParams(friendlyId, environment, taskIdentifier), + prisma + ); + + await setTimeout(500); + const dequeued = await engine.dequeueFromWorkerQueue({ + consumerId: "test_guard", + workerQueue: "main", + }); + expect(dequeued.length).toBe(1); + + const attempt = await engine.startRunAttempt({ + runId: dequeued[0].run.id, + snapshotId: dequeued[0].snapshot.id, + }); + + return { engine, environment, run, attempt, store, replica }; + } catch (e) { + // Setup threw after the engine was constructed: quit it here so a failed helper does not leak the + // engine (the callers only run `await engine.quit()` on the success path). + await engine.quit(); + throw e; + } +} + +describe("RunAttemptSystem completion-path reads must hit the owning primary, not a lagging replica", () => { + // Retry-vs-fail decision. A retriable failure with attempts remaining must RETRY. The retry config + // (maxAttempts / lockedRetryConfig) is a lock-time write; if the decision read is served by a lagging + // replica it reads null and the run is permanently FAILED instead. Reading via the owning primary + // (this.$.prisma) sees the config and the run retries; a replica-routed read fails it. + containerTest( + "attemptFailed retries a retriable run whose lock-time retry config has not replicated", + async ({ prisma, redisOptions }) => { + const { engine, run, attempt, replica } = await triggerAndStartAttempt( + prisma, + redisOptions, + "run_nnnnnnnnnnnnnnnnnnnnnnnn01" + ); + + try { + // The primary now holds maxAttempts=3 (set at lock time). Freeze the replica at the pre-lock + // snapshot: the ONLY thing that could flip the outcome is which client serves the decision read. + const primaryRun = await prisma.taskRun.findFirstOrThrow({ where: { id: run.id } }); + expect(primaryRun.maxAttempts).toBe(3); + replica.state.retryStale = true; + + const result = await engine.completeRunAttempt({ + runId: run.id, + snapshotId: attempt.snapshot.id, + completion: { + ok: false, + id: run.id, + error: { + type: "BUILT_IN_ERROR", + name: "UserError", + message: "boom", + stackTrace: "Error: boom\n at :1:1", + }, + retry: { timestamp: Date.now(), delay: 0 }, + }, + }); + + // Decision read hit the primary → maxAttempts=3 seen → the run retries. + expect(result.attemptStatus).toBe("RETRY_IMMEDIATELY"); + expect(result.snapshot.executionStatus).toBe("EXECUTING"); + expect(result.run.status).toBe("EXECUTING"); + // The lagging replica was never consulted for the decision (a replica-routed read would make this > 0). + expect(replica.hits.retry).toBe(0); + } finally { + await engine.quit(); + } + } + ); + + // attemptSucceeded usage/cost RMW. The cumulative usage total is a read-modify-write: read current + // total, add this attempt, persist. If the read is served by a lagging replica it misses a + // just-persisted earlier-attempt total and the final usage undercounts. Reading via findRunOnPrimary + // accumulates on the primary total; a replica-routed read undercounts. + containerTest( + "attemptSucceeded accumulates usage on top of the primary total, not a stale replica total", + async ({ prisma, redisOptions }) => { + const { engine, run, attempt, replica } = await triggerAndStartAttempt( + prisma, + redisOptions, + "run_nnnnnnnnnnnnnnnnnnnnnnnn11" + ); + + try { + // Simulate an earlier attempt's usage already persisted on the PRIMARY (1000ms) that the + // lagging replica has not yet applied (it still reports 0). + await prisma.taskRun.update({ + where: { id: run.id }, + data: { usageDurationMs: 1000, costInCents: 5, machinePreset: "small-1x" }, + }); + replica.state.usageStale = { + usageDurationMs: 0, + costInCents: 0, + machinePreset: "small-1x", + }; + + const result = await engine.completeRunAttempt({ + runId: run.id, + snapshotId: attempt.snapshot.id, + completion: { + ok: true, + id: run.id, + output: `{"ok":true}`, + outputType: "application/json", + usage: { durationMs: 500 }, + }, + }); + expect(result.run.status).toBe("COMPLETED_SUCCESSFULLY"); + + // Primary read: 1000 (primary) + 500 (this attempt) = 1500. A replica read would see 0 → 500 (undercount). + const finalRun = await prisma.taskRun.findFirstOrThrow({ where: { id: run.id } }); + expect(finalRun.usageDurationMs).toBe(1500); + expect(replica.hits.usage).toBe(0); + } finally { + await engine.quit(); + } + } + ); + + // #permanentlyFailRun usage/cost RMW. A non-retriable failure permanently fails the run and still + // accumulates the final attempt's usage. Reached with a non-retriable error so + // retryOutcomeFromCompletion short-circuits to fail_run BEFORE its own read — isolating this to the + // #permanentlyFailRun usage read. Reading via findRunOnPrimary accumulates on the primary total; a + // replica-routed read undercounts. + containerTest( + "permanentlyFailRun accumulates usage on top of the primary total, not a stale replica total", + async ({ prisma, redisOptions }) => { + const { engine, run, attempt, replica } = await triggerAndStartAttempt( + prisma, + redisOptions, + "run_nnnnnnnnnnnnnnnnnnnnnnnn21" + ); + + try { + await prisma.taskRun.update({ + where: { id: run.id }, + data: { usageDurationMs: 1000, costInCents: 5, machinePreset: "small-1x" }, + }); + replica.state.usageStale = { + usageDurationMs: 0, + costInCents: 0, + machinePreset: "small-1x", + }; + + const result = await engine.completeRunAttempt({ + runId: run.id, + snapshotId: attempt.snapshot.id, + completion: { + ok: false, + id: run.id, + // Non-retriable internal error → straight to fail_run → #permanentlyFailRun. + error: { type: "INTERNAL_ERROR", code: "DISK_SPACE_EXCEEDED" }, + usage: { durationMs: 500 }, + }, + }); + expect(result.attemptStatus).toBe("RUN_FINISHED"); + expect(result.snapshot.executionStatus).toBe("FINISHED"); + + // Primary read: 1000 + 500 = 1500. A replica read would see 0 → 500 (undercount). + const finalRun = await prisma.taskRun.findFirstOrThrow({ where: { id: run.id } }); + expect(finalRun.usageDurationMs).toBe(1500); + expect(replica.hits.usage).toBe(0); + } finally { + await engine.quit(); + } + } + ); + + // cancelRun usage/cost RMW. Cancelling a run with attempt-duration data accumulates that usage. Driven + // through the real RunAttemptSystem.cancelRun (the engine's public cancelRun does not expose + // attemptDurationMs, so the real method is invoked via engine.runAttemptSystem). Reading via + // findRunOnPrimary accumulates on the primary total; a replica-routed read undercounts. + containerTest( + "cancelRun accumulates usage on top of the primary total, not a stale replica total", + async ({ prisma, redisOptions }) => { + const { engine, run, replica } = await triggerAndStartAttempt( + prisma, + redisOptions, + "run_nnnnnnnnnnnnnnnnnnnnnnnn31" + ); + + try { + await prisma.taskRun.update({ + where: { id: run.id }, + data: { usageDurationMs: 1000, costInCents: 5, machinePreset: "small-1x" }, + }); + replica.state.usageStale = { + usageDurationMs: 0, + costInCents: 0, + machinePreset: "small-1x", + }; + + const result = await engine.runAttemptSystem.cancelRun({ + runId: run.id, + reason: "guard test cancel", + finalizeRun: true, + attemptDurationMs: 500, + }); + assertNonNullable(result); + + // Primary read: 1000 + 500 = 1500. A replica read would see 0 → 500 (undercount). + const finalRun = await prisma.taskRun.findFirstOrThrow({ where: { id: run.id } }); + expect(finalRun.usageDurationMs).toBe(1500); + expect(replica.hits.usage).toBe(0); + } finally { + await engine.quit(); + } + } + ); +}); + +// engine-other read sites — two reads (resolveTaskRunContext, the attemptFailed forceRequeue re-read) +// that must hit the owning primary. Unlike the completion-path cases above (which fabricate STALE reads +// for two `select` shapes via `laggingReadReplica`), these need the whole taskRun row frozen-MISSING on +// the replica, so they use the shared `laggingReplica` ("missing") primitive behind a gate: it forwards +// to the live client during setup so the run reaches EXECUTING, then flips to the frozen replica for +// exactly the read under test. `wasHit()` proves whether the frozen replica was consulted for that read. +function gatedLaggingReplica(prisma: PrismaClient) { + const frozen = { on: false }; + const lagging = laggingReplica(prisma, [{ model: "taskRun", mode: "missing" }]); + const client = new Proxy(prisma, { + get(_t, prop) { + const src: any = frozen.on ? lagging.client : prisma; + const value = src[prop]; + return typeof value === "function" ? value.bind(src) : value; + }, + }) as unknown as PrismaClient; + return { client, frozen, wasHit: (m?: string) => lagging.wasHit(m) }; +} + +// Drive the real engine to a locked, EXECUTING first attempt over a store whose replica is the gated +// frozen-missing proxy. Mirrors `triggerAndStartAttempt` above but returns the gated replica handle. +async function triggerAndStartAttemptGated( + prisma: PrismaClient, + redisOptions: any, + friendlyId: string, + retryOptions?: any +) { + const environment = await setupAuthenticatedEnvironment(prisma, "PRODUCTION"); + + const replica = gatedLaggingReplica(prisma); + const store = new PostgresRunStore({ + prisma, + readOnlyPrisma: replica.client as never, + }); + const engine = new RunEngine({ + prisma, + readOnlyPrisma: replica.client as never, + store, + ...baseEngineOptions(redisOptions), + }); + + try { + const taskIdentifier = "test-task"; + await setupBackgroundWorker(engine, environment, taskIdentifier, undefined, retryOptions); + + const run = await engine.trigger( + triggerParams(friendlyId, environment, taskIdentifier), + prisma + ); + + await setTimeout(500); + const dequeued = await engine.dequeueFromWorkerQueue({ + consumerId: "test_guard", + workerQueue: "main", + }); + expect(dequeued.length).toBe(1); + + const attempt = await engine.startRunAttempt({ + runId: dequeued[0].run.id, + snapshotId: dequeued[0].snapshot.id, + }); + + return { engine, environment, run, attempt, store, replica }; + } catch (e) { + // Setup threw after the engine was constructed: quit it here so a failed helper does not leak the + // engine (the callers only run `await engine.quit()` on the success path). + await engine.quit(); + throw e; + } +} + +describe("RunEngine engine-other reads must hit the owning primary, not a lagging replica", () => { + // resolveTaskRunContext. Under lag the run's row is not visible on the replica, so a client-less + // findRun returns null and resolveTaskRunContext throws ServiceValidationError("Task run not found", + // 404) for a LIVE, EXECUTING run — a spurious 404 on the SpanPresenter's context read. Reading via + // findRunOnPrimary hits the primary and resolves the context; a replica-routed read 404s. + containerTest( + "resolveTaskRunContext resolves a live executing run whose row has not replicated", + async ({ prisma, redisOptions }) => { + const { engine, run, replica } = await triggerAndStartAttemptGated( + prisma, + redisOptions, + "run_nnnnnnnnnnnnnnnnnnnnnn73" + ); + try { + // Freeze the owning replica for the taskRun read: the ONLY thing that can flip the outcome is + // which client serves resolveTaskRunContext's read. + replica.frozen.on = true; + const ctx = await engine.resolveTaskRunContext(run.id); + + // The read routed to the primary → the live run's context resolved (friendlyId echoed), and the + // frozen replica was never consulted for the taskRun read. + expect(ctx.run.id).toBe(run.friendlyId); + expect(replica.wasHit("taskRun")).toBe(false); + } finally { + await engine.quit(); + } + } + ); + + // attemptFailed(forceRequeue) re-read. The forceRequeue branch re-reads the run (status/spanId/... for + // the runAttemptFailed event). Under lag a client-less findRun returns null and attemptFailed throws + // ServiceValidationError("Run not found", 404), aborting the crash-requeue of a LIVE run. Reading via + // findRunOnPrimary hits the primary; a replica-routed read 404s. + containerTest( + "attemptFailed(forceRequeue) re-reads a live run on the primary", + async ({ prisma, redisOptions }) => { + const { engine, run, attempt, replica } = await triggerAndStartAttemptGated( + prisma, + redisOptions, + "run_nnnnnnnnnnnnnnnnnnnnnn83", + { maxAttempts: 3 } + ); + try { + replica.frozen.on = true; + const result = await engine.runAttemptSystem.attemptFailed({ + runId: run.id, + snapshotId: attempt.snapshot.id, + completion: { + ok: false, + id: run.id, + error: { + type: "BUILT_IN_ERROR", + name: "UserError", + message: "boom", + stackTrace: "Error: boom\n at :1:1", + }, + retry: { timestamp: Date.now(), delay: 0 }, + }, + forceRequeue: true, + tx: prisma, + }); + + // The forceRequeue re-read hit the primary → attemptFailed completed (no 404) and returned a + // live result for this run; the frozen replica was never consulted for the taskRun read. + expect(result.run.id).toBe(run.id); + expect(result.attemptStatus).toBeDefined(); + expect(replica.wasHit("taskRun")).toBe(false); + } finally { + await engine.quit(); + } + } + ); +}); diff --git a/internal-packages/run-store/src/runOpsStore.batchDependentAttemptReadView.replicaLag.test.ts b/internal-packages/run-store/src/runOpsStore.batchDependentAttemptReadView.replicaLag.test.ts new file mode 100644 index 00000000000..281559c1080 --- /dev/null +++ b/internal-packages/run-store/src/runOpsStore.batchDependentAttemptReadView.replicaLag.test.ts @@ -0,0 +1,251 @@ +// Property: the batchTriggerV3 dependent-attempt read tolerates a miss under replica lag. The +// client-less findTaskRunAttempt resolves the PARENT run's attempt (by attempt friendlyId, env-scoped) +// to feed a terminal-state pre-check (throw ServiceValidationError if the parent attempt/run is +// terminal). With no `taskRunId` in the where the router fans out UNROUTED (NEW→LEGACY), each leg with a +// null client → readOnlyPrisma → genuinely REPLICA-routed (never defaults to primary). +// +// Under owning-replica lag the read returns null. Tolerated: the parent attempt is a long-committed row +// (the parent is mid-execution when it triggers the batch — never a read-your-write), and its only +// consumer is a best-effort TOCTOU guard that is racy even against the primary and whose resume path is +// a no-op on an already-terminal parent. A null/stale read skips the guard but drives no wrong outcome. +// +// Builds the router as the webapp holds it, seeds the parent run + attempt on the owning LEGACY primary, +// freezes the LEGACY replica with the shared laggingReplica, invokes findTaskRunAttempt with the EXACT +// caller args and NO client. + +import { heteroPostgresTest, laggingReplica } from "@internal/testcontainers"; +import type { PrismaClient } from "@trigger.dev/database"; +import { describe, expect } from "vitest"; +import { PostgresRunStore } from "./PostgresRunStore.js"; +import { RoutingRunStore } from "./runOpsStore.js"; +import type { CreateRunInput } from "./types.js"; + +// A cuid (25 chars after the `run_` prefix) classifies LEGACY, so the parent run + attempt are owned +// by the legacy (control-plane) store. The env-scoped, taskRunId-free read then fans out NEW→LEGACY. +const CUID_25 = "c".repeat(25); + +async function seedEnvironment(prisma: PrismaClient, slugSuffix: string) { + const organization = await prisma.organization.create({ + data: { title: `Org ${slugSuffix}`, slug: `org-${slugSuffix}` }, + }); + const project = await prisma.project.create({ + data: { + name: `Project ${slugSuffix}`, + slug: `project-${slugSuffix}`, + externalRef: `proj_${slugSuffix}`, + organizationId: organization.id, + }, + }); + const environment = await prisma.runtimeEnvironment.create({ + data: { + type: "DEVELOPMENT", + slug: "dev", + projectId: project.id, + organizationId: organization.id, + apiKey: `tr_dev_${slugSuffix}`, + pkApiKey: `pk_dev_${slugSuffix}`, + shortcode: `short_${slugSuffix}`, + }, + }); + return { organization, project, environment }; +} + +// Full CreateRunInput for the parent run (mirrors the routing test's builder) — used via +// store.createRun so we don't hand-maintain the TaskRun column list. +function buildCreateRunInput(params: { + runId: string; + friendlyId: string; + organizationId: string; + projectId: string; + runtimeEnvironmentId: string; +}): CreateRunInput { + return { + data: { + id: params.runId, + engine: "V2", + status: "EXECUTING", + friendlyId: params.friendlyId, + runtimeEnvironmentId: params.runtimeEnvironmentId, + environmentType: "DEVELOPMENT", + organizationId: params.organizationId, + projectId: params.projectId, + taskIdentifier: "parent-task", + payload: "{}", + payloadType: "application/json", + context: {}, + traceContext: {}, + traceId: `trace_${params.runId}`, + spanId: `span_${params.runId}`, + queue: "task/parent-task", + isTest: false, + taskEventStore: "taskEvent", + depth: 0, + createdAt: new Date("2024-01-01T00:00:00.000Z"), + }, + snapshot: { + engine: "V2", + executionStatus: "EXECUTING", + description: "Run is executing", + runStatus: "EXECUTING", + environmentId: params.runtimeEnvironmentId, + environmentType: "DEVELOPMENT", + projectId: params.projectId, + organizationId: params.organizationId, + }, + }; +} + +// Seed a TaskRunAttempt with FK triggers disabled. `session_replication_role` is session-scoped, so the +// `SET` and the insert must share one connection — separate pooled calls can split across connections, +// leaving the insert with FK triggers on. One transaction with `SET LOCAL` keeps them co-connected. +async function seedAttempt( + prisma: PrismaClient, + opts: { + attemptId: string; + friendlyId: string; + runId: string; + runtimeEnvironmentId: string; + status: string; + } +) { + await prisma.$transaction(async (tx) => { + await tx.$executeRawUnsafe(`SET LOCAL session_replication_role = replica`); + await tx.$executeRawUnsafe( + `INSERT INTO "TaskRunAttempt" (id, number, "friendlyId", "taskRunId", "backgroundWorkerId", "backgroundWorkerTaskId", "runtimeEnvironmentId", "queueId", status, "createdAt", "updatedAt", "usageDurationMs", "outputType") + VALUES ($1, 1, $2, $3, 'synthetic-worker', 'synthetic-worker-task', $4, 'synthetic-queue', $5::"TaskRunAttemptStatus", NOW(), NOW(), 0, 'application/json')`, + opts.attemptId, + opts.friendlyId, + opts.runId, + opts.runtimeEnvironmentId, + opts.status + ); + }); +} + +// Mirror of dependentAttemptScope#dependentAttemptWhere — replicated here to keep this test free of a +// webapp import while matching the exact caller shape. +function dependentAttemptWhere(friendlyId: string, environmentId: string) { + return { friendlyId, taskRun: { runtimeEnvironmentId: environmentId } } as const; +} + +// The EXACT args the batchTriggerV3 caller passes (where builder + include), for a given attempt. +function callSiteArgs(attemptFriendlyId: string, environmentId: string) { + return { + where: dependentAttemptWhere(attemptFriendlyId, environmentId), + include: { taskRun: { select: { id: true, status: true } } }, + } as const; +} + +describe("batchTriggerV3 dependentAttempt read — findTaskRunAttempt(no client) under replica lag", () => { + heteroPostgresTest( + "env-scoped dependent-attempt read returns null under replica lag, skipping the terminal-state pre-check", + async ({ prisma14, prisma17 }) => { + // LEGACY store's replica is FROZEN; its primary (prisma14) is real. + const legacyReplica = laggingReplica(prisma14, [ + { model: "taskRunAttempt", mode: "missing" }, + ]); + const legacyStore = new PostgresRunStore({ + prisma: prisma14, + readOnlyPrisma: legacyReplica.client, + }); + // NEW store non-lagging (and holds no attempt) — the fan-out probes it first and misses. + const newStore = new PostgresRunStore({ prisma: prisma17, readOnlyPrisma: prisma17 }); + const router = new RoutingRunStore({ new: newStore, legacy: legacyStore }); + + const seed = await seedEnvironment(prisma14, "batch_dep_lag"); + const parentRunId = `run_${CUID_25}`; // cuid → LEGACY-owned + await legacyStore.createRun( + buildCreateRunInput({ + runId: parentRunId, + friendlyId: "run_batch_dep_parent", + runtimeEnvironmentId: seed.environment.id, + organizationId: seed.organization.id, + projectId: seed.project.id, + }) + ); + + const attemptId = "attempt_batch_dep_lag"; + const attemptFriendlyId = "attempt_batch_dep_lag_f"; + // Parent attempt is COMPLETED — a TERMINAL attempt status. If the read saw it, the caller WOULD + // throw the ServiceValidationError. This makes the tolerance question sharp. + await seedAttempt(prisma14, { + attemptId, + friendlyId: attemptFriendlyId, + runId: parentRunId, + runtimeEnvironmentId: seed.environment.id, + status: "COMPLETED", + }); + + const args = callSiteArgs(attemptFriendlyId, seed.environment.id); + + // HAZARD proof: the call site's exact invocation (no client) misses the attempt under lag. + const underLag = await router.findTaskRunAttempt(args); + expect(underLag).toBeNull(); + // Prove the read was genuinely REPLICA-routed (the frozen owning replica was consulted). + expect(legacyReplica.wasHit()).toBe(true); + + // ROUTING proof (replica, not primary): the SAME store, SAME args, but a WRITER client forces the + // owning primary (#ownPrimary), which sees the row. So the null above is purely replica lag + + // replica routing — not a missing row or a bad query. + const onPrimary = await router.findTaskRunAttempt(args, prisma14); + expect(onPrimary?.id).toBe(attemptId); + expect(onPrimary?.status).toBe("COMPLETED"); + expect(onPrimary?.taskRun.id).toBe(parentRunId); + + // TOLERANCE assertion: reproduce the caller's `dependentAttempt ? throw-if-terminal : proceed` + // branch. Under lag dependentAttempt is null, so the terminal-state guard is SKIPPED. + const dependentAttempt = underLag as { status: string; taskRun: { status: string } } | null; + const wouldThrowTerminal = + !!dependentAttempt && + (["COMPLETED", "FAILED", "CANCELED"].includes(dependentAttempt.status) || + ["COMPLETED_SUCCESSFULLY", "COMPLETED_WITH_ERRORS", "CANCELED", "FAILED"].includes( + dependentAttempt.taskRun.status + )); + expect(wouldThrowTerminal).toBe(false); // guard skipped under lag — tolerated, not a wrong outcome + } + ); + + heteroPostgresTest( + "steady state: the read returns the terminal parent attempt from a caught-up replica", + async ({ prisma14, prisma17 }) => { + // Non-lagging: the legacy store reads its real replica (prisma14 itself as the replica handle). + const legacyStore = new PostgresRunStore({ prisma: prisma14, readOnlyPrisma: prisma14 }); + const newStore = new PostgresRunStore({ prisma: prisma17, readOnlyPrisma: prisma17 }); + const router = new RoutingRunStore({ new: newStore, legacy: legacyStore }); + + const seed = await seedEnvironment(prisma14, "batch_dep_ok"); + const parentRunId = `run_${CUID_25}`; + await legacyStore.createRun( + buildCreateRunInput({ + runId: parentRunId, + friendlyId: "run_batch_dep_parent_ok", + runtimeEnvironmentId: seed.environment.id, + organizationId: seed.organization.id, + projectId: seed.project.id, + }) + ); + + const attemptId = "attempt_batch_dep_ok"; + const attemptFriendlyId = "attempt_batch_dep_ok_f"; + await seedAttempt(prisma14, { + attemptId, + friendlyId: attemptFriendlyId, + runId: parentRunId, + runtimeEnvironmentId: seed.environment.id, + status: "COMPLETED", + }); + + const found = await router.findTaskRunAttempt( + callSiteArgs(attemptFriendlyId, seed.environment.id) + ); + expect(found?.id).toBe(attemptId); + expect(found?.status).toBe("COMPLETED"); + expect(found?.taskRun.id).toBe(parentRunId); + // Env-scope enforced: a wrong environment id in the where must not resolve the foreign attempt. + const wrongEnv = await router.findTaskRunAttempt( + callSiteArgs(attemptFriendlyId, "env_does_not_exist") + ); + expect(wrongEnv).toBeNull(); + } + ); +}); diff --git a/internal-packages/run-store/src/runOpsStore.batchIdempotencyDedupReadAfterWrite.test.ts b/internal-packages/run-store/src/runOpsStore.batchIdempotencyDedupReadAfterWrite.test.ts new file mode 100644 index 00000000000..32686f6f889 --- /dev/null +++ b/internal-packages/run-store/src/runOpsStore.batchIdempotencyDedupReadAfterWrite.test.ts @@ -0,0 +1,161 @@ +// Property: the client-less batch idempotency dedup probe must resolve attempt-1's just-written batch +// from the OWNING store's WRITER, not its lagging replica. batchTriggerV3 dedups via +// findBatchTaskRunByIdempotencyKey(environment.id, idempotencyKey) with NO client, so through the +// router each leg resolves via the owning store's default. Seed attempt-1's batch on the owning WRITER, +// freeze that store's replica (mode "missing") on the real split topology (heteroRunOpsPostgresTest), +// issue the client-less probe, and assert it finds the batch with the replica never consulted — proven +// for BOTH residencies. + +import { heteroRunOpsPostgresTest, laggingReplica } from "@internal/testcontainers"; +import type { PrismaClient } from "@trigger.dev/database"; +import type { RunOpsPrismaClient } from "@internal/run-ops-database"; +import { generateRunOpsId } from "@trigger.dev/core/v3/isomorphic"; +import { describe, expect } from "vitest"; +import { PostgresRunStore } from "./PostgresRunStore.js"; +import { RoutingRunStore } from "./runOpsStore.js"; +import type { RunStoreSchemaVariant } from "./types.js"; + +type AnyClient = PrismaClient | RunOpsPrismaClient; + +// ownerEngine (classifyResidency) routes a run-ops v1 body → NEW, everything else → LEGACY. +const CUID_25 = "c".repeat(25); // → LEGACY (#legacy / prisma14, full schema) + +// On the dedicated subset there are no Organization/Project/RuntimeEnvironment models (the run-ops +// rows carry FK-free scalar ids), so we mint synthetic owning ids. On legacy we seed the real rows +// the kept FKs require. +async function seedEnvironment( + prisma: AnyClient, + schemaVariant: RunStoreSchemaVariant, + slugSuffix: string +) { + if (schemaVariant === "dedicated") { + return { + organization: { id: `org_${slugSuffix}` }, + project: { id: `proj_${slugSuffix}` }, + environment: { id: `env_${slugSuffix}` }, + }; + } + const organization = await (prisma as PrismaClient).organization.create({ + data: { title: `Org ${slugSuffix}`, slug: `org-${slugSuffix}` }, + }); + const project = await (prisma as PrismaClient).project.create({ + data: { + name: `Project ${slugSuffix}`, + slug: `project-${slugSuffix}`, + externalRef: `proj_${slugSuffix}`, + organizationId: organization.id, + }, + }); + const environment = await (prisma as PrismaClient).runtimeEnvironment.create({ + data: { + type: "DEVELOPMENT", + slug: "dev", + projectId: project.id, + organizationId: organization.id, + apiKey: `tr_dev_${slugSuffix}`, + pkApiKey: `pk_dev_${slugSuffix}`, + shortcode: `short_${slugSuffix}`, + }, + }); + return { organization, project, environment }; +} + +// Seed attempt-1's batch (user-provided idempotencyKey) on the WRITER, exactly as batchTriggerV3's +// create arm would have written it on the first attempt. +async function seedBatch( + store: PostgresRunStore, + params: { id: string; friendlyId: string; environmentId: string; idempotencyKey: string } +) { + await store.createBatchTaskRun({ + id: params.id, + friendlyId: params.friendlyId, + runtimeEnvironmentId: params.environmentId, + idempotencyKey: params.idempotencyKey, + idempotencyKeyExpiresAt: new Date(Date.now() + 60 * 60_000), + }); +} + +describe("run-ops split — batch idempotency dedup reads the OWNING store's WRITER, not its lagging replica", () => { + // The dedup probe issued by batchTriggerV3 on a RETRY: + // findBatchTaskRunByIdempotencyKey(environmentId, idempotencyKey) // NO client + // must resolve attempt-1's just-written batch, so the retry returns the cached batch instead of + // triggering a duplicate one. Proven for BOTH residencies. + + // (a) LEGACY-resident (cuid) batch: attempt-1's batch was committed to the control-plane writer; the + // control-plane replica lags. The client-less dedup must find it on the owning (legacy) store. + heteroRunOpsPostgresTest( + "LEGACY cuid: client-less dedup finds attempt-1's batch despite replica lag", + async ({ prisma14, prisma17 }) => { + const legacyReplica = laggingReplica(prisma14, [{ model: "batchTaskRun", mode: "missing" }]); + const legacyStore = new PostgresRunStore({ + prisma: prisma14, + readOnlyPrisma: legacyReplica.client, + schemaVariant: "legacy", + }); + const newStore = new PostgresRunStore({ + prisma: prisma17 as never, + readOnlyPrisma: prisma17 as never, + schemaVariant: "dedicated", + }); + const router = new RoutingRunStore({ new: newStore, legacy: legacyStore }); + + const seed = await seedEnvironment(prisma14, "legacy", "batch_leg"); + const idempotencyKey = "batch-idem-legacy"; + await seedBatch(legacyStore, { + id: `batch_${CUID_25}`, // cuid → LEGACY + friendlyId: "batch_batch_leg", + environmentId: seed.environment.id, + idempotencyKey, + }); + + // The exact dedup probe: no client. + const found = await router.findBatchTaskRunByIdempotencyKey( + seed.environment.id, + idempotencyKey + ); + + // Resolved on the owning store's writer; the lagging replica must never be consulted. + expect(found).not.toBeNull(); + expect(found!.idempotencyKey).toBe(idempotencyKey); + expect(legacyReplica.wasHit("batchTaskRun")).toBe(false); + } + ); + + // (b) NEW-resident (run-ops id) batch on the dedicated subset schema: the NEW replica lags. The + // client-less dedup must resolve on the NEW writer. + heteroRunOpsPostgresTest( + "NEW run-ops id: client-less dedup finds attempt-1's batch despite NEW replica lag", + async ({ prisma14, prisma17 }) => { + const newReplica = laggingReplica(prisma17, [{ model: "batchTaskRun", mode: "missing" }]); + const newStore = new PostgresRunStore({ + prisma: prisma17 as never, + readOnlyPrisma: newReplica.client as never, + schemaVariant: "dedicated", + }); + const legacyStore = new PostgresRunStore({ + prisma: prisma14, + readOnlyPrisma: prisma14, + schemaVariant: "legacy", + }); + const router = new RoutingRunStore({ new: newStore, legacy: legacyStore }); + + const seed = await seedEnvironment(prisma17, "dedicated", "batch_new"); + const idempotencyKey = "batch-idem-new"; + await seedBatch(newStore, { + id: `batch_${generateRunOpsId()}`, // run-ops id → NEW + friendlyId: "batch_batch_new", + environmentId: seed.environment.id, + idempotencyKey, + }); + + const found = await router.findBatchTaskRunByIdempotencyKey( + seed.environment.id, + idempotencyKey + ); + + expect(found).not.toBeNull(); + expect(found!.idempotencyKey).toBe(idempotencyKey); + expect(newReplica.wasHit("batchTaskRun")).toBe(false); + } + ); +}); diff --git a/internal-packages/run-store/src/runOpsStore.bulkActionReadView.replicaLag.test.ts b/internal-packages/run-store/src/runOpsStore.bulkActionReadView.replicaLag.test.ts new file mode 100644 index 00000000000..392f5a132a9 --- /dev/null +++ b/internal-packages/run-store/src/runOpsStore.bulkActionReadView.replicaLag.test.ts @@ -0,0 +1,253 @@ +// Lagging-replica coverage for BULK ACTION member hydration on the run-ops split. BulkActionV2's +// CANCEL and REPLAY branches hydrate each batch's runs via a client-less id-set findRuns, so the bounded +// id-set path reads each OWNING store's REPLICA: a run on its owning PRIMARY but not yet on that store's +// REPLICA is dropped → a PARTIAL merge. Both are tolerated: the id set comes from a ClickHouse list fed +// DOWNSTREAM of Postgres, so a not-yet-replicated run isn't in ClickHouse yet and lands in a later page. +// The tests FORCE the (in-practice-unreachable) stale window at the store, prove the merge drops the +// lagging-store row, and prove that row really exists on the primary. +// Real split topology via heteroRunOpsPostgresTest — NEVER mocked. + +import { heteroRunOpsPostgresTest, laggingReplica } from "@internal/testcontainers"; +import type { PrismaClient } from "@trigger.dev/database"; +import type { RunOpsPrismaClient } from "@internal/run-ops-database"; +import { generateRunOpsId } from "@trigger.dev/core/v3/isomorphic"; +import { describe, expect } from "vitest"; +import { PostgresRunStore } from "./PostgresRunStore.js"; +import { RoutingRunStore } from "./runOpsStore.js"; +import type { CreateRunInput, RunStoreSchemaVariant } from "./types.js"; + +type AnyClient = PrismaClient | RunOpsPrismaClient; + +// classifyResidency routes a run-ops v1 body → NEW, everything else → LEGACY. +const CUID_25 = "c".repeat(25); // → LEGACY (#legacy / prisma14, full schema) + +async function seedEnvironment( + prisma: AnyClient, + schemaVariant: RunStoreSchemaVariant, + slugSuffix: string +) { + if (schemaVariant === "dedicated") { + return { + organization: { id: `org_${slugSuffix}` }, + project: { id: `proj_${slugSuffix}` }, + environment: { id: `env_${slugSuffix}` }, + }; + } + const organization = await (prisma as PrismaClient).organization.create({ + data: { title: `Org ${slugSuffix}`, slug: `org-${slugSuffix}` }, + }); + const project = await (prisma as PrismaClient).project.create({ + data: { + name: `Project ${slugSuffix}`, + slug: `project-${slugSuffix}`, + externalRef: `proj_${slugSuffix}`, + organizationId: organization.id, + }, + }); + const environment = await (prisma as PrismaClient).runtimeEnvironment.create({ + data: { + type: "DEVELOPMENT", + slug: "dev", + projectId: project.id, + organizationId: organization.id, + apiKey: `tr_dev_${slugSuffix}`, + pkApiKey: `pk_dev_${slugSuffix}`, + shortcode: `short_${slugSuffix}`, + }, + }); + return { organization, project, environment }; +} + +function buildCreateRunInput(params: { + runId: string; + friendlyId: string; + organizationId: string; + projectId: string; + runtimeEnvironmentId: string; +}): CreateRunInput { + return { + data: { + id: params.runId, + engine: "V2", + status: "COMPLETED_SUCCESSFULLY", + friendlyId: params.friendlyId, + runtimeEnvironmentId: params.runtimeEnvironmentId, + environmentType: "DEVELOPMENT", + organizationId: params.organizationId, + projectId: params.projectId, + taskIdentifier: "my-task", + payload: '{"hello":"world"}', + payloadType: "application/json", + traceContext: { trace: "ctx" }, + traceId: `trace_${params.runId}`, + spanId: `span_${params.runId}`, + queue: "task/my-task", + isTest: false, + taskEventStore: "taskEvent", + depth: 0, + createdAt: new Date("2024-01-01T00:00:00.000Z"), + }, + snapshot: { + engine: "V2", + executionStatus: "FINISHED", + description: "Run finished", + runStatus: "COMPLETED_SUCCESSFULLY", + environmentId: params.runtimeEnvironmentId, + environmentType: "DEVELOPMENT", + projectId: params.projectId, + organizationId: params.organizationId, + }, + }; +} + +// Exactly the projection the CANCEL branch reads. +const CANCEL_SELECT = { + id: true, + engine: true, + friendlyId: true, + status: true, + createdAt: true, + completedAt: true, + taskEventStore: true, +} as const; + +describe("run-ops split — bulk-action member hydration (id-set findRuns) vs. a lagging replica", () => { + // CANCEL branch. The id set is ClickHouse-sourced; here we FORCE the (in practice + // unreachable) stale window: freeze the LEGACY replica while a LEGACY-resident run lives on the + // LEGACY primary, and put a second, NEW-resident run whose replica is live in the same batch. + // The no-client findRuns(id-set) drops the lagging-store row and keeps the other → PARTIAL merge. + // Tolerated: the drop is safe because an id only reaches this batch after ClickHouse (which lags + // MORE than the PG replica) has it, by which point the PG replica already has the row. + heteroRunOpsPostgresTest( + "cancel: id-set findRuns (no client) drops the lagging-legacy-replica run, keeps the live-new-replica run", + async ({ prisma14, prisma17 }) => { + // LEGACY replica frozen; NEW replica live. + const legacyReplica = laggingReplica(prisma14, [{ model: "taskRun", mode: "missing" }]); + const legacyStore = new PostgresRunStore({ + prisma: prisma14, + readOnlyPrisma: legacyReplica.client, + schemaVariant: "legacy", + }); + const newStore = new PostgresRunStore({ + prisma: prisma17 as never, + readOnlyPrisma: prisma17 as never, + schemaVariant: "dedicated", + }); + const router = new RoutingRunStore({ new: newStore, legacy: legacyStore }); + + const legacySeed = await seedEnvironment(prisma14, "legacy", "bulk_cancel_leg"); + const laggingRunId = `run_${CUID_25}`; // cuid → LEGACY (owning replica is frozen) + await legacyStore.createRun( + buildCreateRunInput({ + runId: laggingRunId, + friendlyId: "run_bulk_cancel_leg", + organizationId: legacySeed.organization.id, + projectId: legacySeed.project.id, + runtimeEnvironmentId: legacySeed.environment.id, + }) + ); + + const newSeed = await seedEnvironment(prisma17, "dedicated", "bulk_cancel_new"); + const liveRunId = `run_${generateRunOpsId()}`; // v1 body → NEW (owning replica is live) + await newStore.createRun( + buildCreateRunInput({ + runId: liveRunId, + friendlyId: "run_bulk_cancel_new", + organizationId: newSeed.organization.id, + projectId: newSeed.project.id, + runtimeEnvironmentId: newSeed.environment.id, + }) + ); + + // EXACT call site — id set from listRunIds, select projection, NO client. + const runIdsToProcess = [laggingRunId, liveRunId]; + const runs = (await router.findRuns({ + where: { id: { in: runIdsToProcess } }, + select: CANCEL_SELECT, + })) as Array<{ id: string }>; + + // Fact: the lagging LEGACY replica returned nothing for its run, so the merge is + // PARTIAL — only the live-replica (NEW) run survives. The cancel branch would iterate just + // that one run this batch; the lagging one is silently absent from `runs`. + const ids = runs.map((r) => r.id).sort(); + expect(ids).toEqual([liveRunId]); + expect(ids).not.toContain(laggingRunId); + expect(legacyReplica.wasHit()).toBe(true); + + // The dropped run genuinely exists on the LEGACY primary — prove the miss is pure lag/routing, + // not a nonexistent row: re-read with the WRITER (→ owning primaries) returns BOTH. + const viaPrimary = (await router.findRuns( + { where: { id: { in: runIdsToProcess } }, select: CANCEL_SELECT }, + prisma14 as never + )) as Array<{ id: string }>; + expect(viaPrimary.map((r) => r.id).sort()).toEqual([laggingRunId, liveRunId].sort()); + // Tolerated: batch ids are ClickHouse-sourced (lags more than the PG replica) — see header. + } + ); + + // REPLAY branch. Same id-set read, full row (no select), roles swapped so the OTHER + // owning store lags: freeze the NEW replica, keep LEGACY live. Same tolerance as above. + heteroRunOpsPostgresTest( + "replay: id-set findRuns (no client, full row) drops the lagging-new-replica run, keeps the live-legacy-replica run", + async ({ prisma14, prisma17 }) => { + // NEW replica frozen; LEGACY replica live. + const newReplica = laggingReplica(prisma17, [{ model: "taskRun", mode: "missing" }]); + const newStore = new PostgresRunStore({ + prisma: prisma17 as never, + readOnlyPrisma: newReplica.client as never, + schemaVariant: "dedicated", + }); + const legacyStore = new PostgresRunStore({ + prisma: prisma14, + readOnlyPrisma: prisma14, + schemaVariant: "legacy", + }); + const router = new RoutingRunStore({ new: newStore, legacy: legacyStore }); + + const newSeed = await seedEnvironment(prisma17, "dedicated", "bulk_replay_new"); + const laggingRunId = `run_${generateRunOpsId()}`; // v1 body → NEW (owning replica is frozen) + await newStore.createRun( + buildCreateRunInput({ + runId: laggingRunId, + friendlyId: "run_bulk_replay_new", + organizationId: newSeed.organization.id, + projectId: newSeed.project.id, + runtimeEnvironmentId: newSeed.environment.id, + }) + ); + + const legacySeed = await seedEnvironment(prisma14, "legacy", "bulk_replay_leg"); + const liveRunId = `run_${CUID_25}`; // cuid → LEGACY (owning replica is live) + await legacyStore.createRun( + buildCreateRunInput({ + runId: liveRunId, + friendlyId: "run_bulk_replay_leg", + organizationId: legacySeed.organization.id, + projectId: legacySeed.project.id, + runtimeEnvironmentId: legacySeed.environment.id, + }) + ); + + // EXACT call site — id set from listRunIds, FULL row (no select), NO client. + const runIdsToProcess = [laggingRunId, liveRunId]; + const runs = (await router.findRuns({ + where: { id: { in: runIdsToProcess } }, + })) as Array<{ id: string }>; + + // Fact: the frozen NEW replica returns nothing for its run; only the live LEGACY + // run survives the merge. The replay branch would replay just that one run this batch. + const ids = runs.map((r) => r.id).sort(); + expect(ids).toEqual([liveRunId]); + expect(ids).not.toContain(laggingRunId); + expect(newReplica.wasHit()).toBe(true); + + // The dropped NEW run genuinely exists on the NEW primary: re-read with the WRITER returns BOTH. + const viaPrimary = (await router.findRuns( + { where: { id: { in: runIdsToProcess } } }, + prisma17 as never + )) as Array<{ id: string }>; + expect(viaPrimary.map((r) => r.id).sort()).toEqual([laggingRunId, liveRunId].sort()); + // Tolerated: batch ids are ClickHouse-sourced (lags more than the PG replica) — see header. + } + ); +}); diff --git a/internal-packages/run-store/src/runOpsStore.cancelRunReadAfterWrite.replicaLag.test.ts b/internal-packages/run-store/src/runOpsStore.cancelRunReadAfterWrite.replicaLag.test.ts new file mode 100644 index 00000000000..1e5c438507f --- /dev/null +++ b/internal-packages/run-store/src/runOpsStore.cancelRunReadAfterWrite.replicaLag.test.ts @@ -0,0 +1,221 @@ +// Lagging-replica coverage for the CANCEL route's run lookups on the run-ops split. +// +// The cancel route issues two friendlyId-keyed run reads through the shared runStore — the +// org-resolution read (resolveRunOrganizationId) and the cancel-decision read (action body) — both +// client-less, so both route to the OWNING store's REPLICA. Property: reading with the owning WRITER +// (the primary fallback / findRunOnPrimary) resolves a just-written run the replica has not yet +// received, so the cancel decision does not act on a stale miss. +// +// This is the same read-your-writes class as the other lag guards: a replica read whose result drives +// a decision (cancel vs. not-found). These tests freeze the owning replica with the shared +// laggingReplica primitive and prove, at the store seam, exactly what the route sees — the client-less +// read is null under lag, the writer read finds the row. Real split topology via +// heteroRunOpsPostgresTest — NEVER mocked. + +import { heteroRunOpsPostgresTest, laggingReplica } from "@internal/testcontainers"; +import type { PrismaClient } from "@trigger.dev/database"; +import type { RunOpsPrismaClient } from "@internal/run-ops-database"; +import { generateRunOpsId } from "@trigger.dev/core/v3/isomorphic"; +import { describe, expect } from "vitest"; +import { PostgresRunStore } from "./PostgresRunStore.js"; +import { RoutingRunStore } from "./runOpsStore.js"; +import type { CreateRunInput, RunStoreSchemaVariant } from "./types.js"; + +type AnyClient = PrismaClient | RunOpsPrismaClient; + +// ownerEngine (classifyResidency) routes a run-ops v1 body → NEW, everything else → LEGACY. +const CUID_25 = "c".repeat(25); // → LEGACY (#legacy / prisma14, full schema) +const NEW_RUN_ID = `run_${generateRunOpsId()}`; // valid v1 body → NEW (#new / prisma17, dedicated) + +async function seedEnvironment( + prisma: AnyClient, + schemaVariant: RunStoreSchemaVariant, + slugSuffix: string +) { + if (schemaVariant === "dedicated") { + return { + organization: { id: `org_${slugSuffix}` }, + project: { id: `proj_${slugSuffix}` }, + environment: { id: `env_${slugSuffix}` }, + }; + } + const organization = await (prisma as PrismaClient).organization.create({ + data: { title: `Org ${slugSuffix}`, slug: `org-${slugSuffix}` }, + }); + const project = await (prisma as PrismaClient).project.create({ + data: { + name: `Project ${slugSuffix}`, + slug: `project-${slugSuffix}`, + externalRef: `proj_${slugSuffix}`, + organizationId: organization.id, + }, + }); + const environment = await (prisma as PrismaClient).runtimeEnvironment.create({ + data: { + type: "DEVELOPMENT", + slug: "dev", + projectId: project.id, + organizationId: organization.id, + apiKey: `tr_dev_${slugSuffix}`, + pkApiKey: `pk_dev_${slugSuffix}`, + shortcode: `short_${slugSuffix}`, + }, + }); + return { organization, project, environment }; +} + +function buildCreateRunInput(params: { + runId: string; + friendlyId: string; + taskIdentifier: string; + organizationId: string; + projectId: string; + runtimeEnvironmentId: string; +}): CreateRunInput { + return { + data: { + id: params.runId, + engine: "V2", + status: "PENDING", + friendlyId: params.friendlyId, + runtimeEnvironmentId: params.runtimeEnvironmentId, + environmentType: "DEVELOPMENT", + organizationId: params.organizationId, + projectId: params.projectId, + taskIdentifier: params.taskIdentifier, + payload: '{"hello":"world"}', + payloadType: "application/json", + context: { foo: "bar" }, + traceContext: { trace: "ctx" }, + traceId: "trace_1", + spanId: "span_1", + runTags: ["alpha", "beta"], + queue: "task/my-task", + isTest: false, + taskEventStore: "taskEvent", + depth: 0, + createdAt: new Date("2024-01-01T00:00:00.000Z"), + }, + snapshot: { + engine: "V2", + executionStatus: "RUN_CREATED", + description: "Run was created", + runStatus: "PENDING", + environmentId: params.runtimeEnvironmentId, + environmentType: "DEVELOPMENT", + projectId: params.projectId, + organizationId: params.organizationId, + }, + }; +} + +// Exactly the projection the cancel action reads (subset that exists on both variants). +const CANCEL_SELECT = { + id: true, + engine: true, + status: true, + friendlyId: true, + projectId: true, +} as const; + +describe("run-ops split — cancel-route run lookup vs. a lagging replica (read-your-writes)", () => { + // (a) LEGACY-resident (cuid) run committed to the control-plane WRITER; the control-plane replica + // lags. Mirrors the moment a buffered run has just drained to the primary but not yet replicated. + heteroRunOpsPostgresTest( + "LEGACY cuid: no-client findRun is STALE under lag; the primary re-read finds it", + async ({ prisma14, prisma17 }) => { + const legacyReplica = laggingReplica(prisma14, [{ model: "taskRun", mode: "missing" }]); + const legacyStore = new PostgresRunStore({ + prisma: prisma14, + readOnlyPrisma: legacyReplica.client, + schemaVariant: "legacy", + }); + const newStore = new PostgresRunStore({ + prisma: prisma17 as never, + readOnlyPrisma: prisma17 as never, + schemaVariant: "dedicated", + }); + const router = new RoutingRunStore({ new: newStore, legacy: legacyStore }); + + const seed = await seedEnvironment(prisma14, "legacy", "cancel_leg"); + const runId = `run_${CUID_25}`; // cuid → LEGACY + const friendlyId = "run_cancel_leg"; + await legacyStore.createRun( + buildCreateRunInput({ + runId, + friendlyId, + taskIdentifier: "my-task", + organizationId: seed.organization.id, + projectId: seed.project.id, + runtimeEnvironmentId: seed.environment.id, + }) + ); + + // The exact cancel-route read — friendlyId, select, NO client → replica. + const staleRead = await router.findRun({ friendlyId }, { select: CANCEL_SELECT }); + + // The run exists on the primary but the replica lags, so the client-less read returns null. + // Re-reading the primary on this miss (as the org-resolution read does) resolves the live run, + // so the cancel does not act on a stale miss. + expect(staleRead).toBeNull(); + expect(legacyReplica.wasHit()).toBe(true); + + // Re-read via the control-plane WRITER (the same primary fallback the org-resolution read uses, + // routing to the owning store's primary) — this finds the run. + const primaryRead = await router.findRun( + { friendlyId }, + { select: CANCEL_SELECT }, + prisma14 as never + ); + expect(primaryRead).not.toBeNull(); + expect(primaryRead!.friendlyId).toBe(friendlyId); + expect(primaryRead!.id).toBe(runId); + } + ); + + // (b) NEW-resident (ksuid) run on the dedicated subset schema; the NEW replica lags. + heteroRunOpsPostgresTest( + "NEW ksuid: no-client findRun is STALE under NEW replica lag; the primary re-read finds it", + async ({ prisma14, prisma17 }) => { + const newReplica = laggingReplica(prisma17, [{ model: "taskRun", mode: "missing" }]); + const newStore = new PostgresRunStore({ + prisma: prisma17 as never, + readOnlyPrisma: newReplica.client as never, + schemaVariant: "dedicated", + }); + const legacyStore = new PostgresRunStore({ + prisma: prisma14, + readOnlyPrisma: prisma14, + schemaVariant: "legacy", + }); + const router = new RoutingRunStore({ new: newStore, legacy: legacyStore }); + + const seed = await seedEnvironment(prisma17, "dedicated", "cancel_new"); + const runId = NEW_RUN_ID; // v1 body → NEW + const friendlyId = "run_cancel_new"; + await newStore.createRun( + buildCreateRunInput({ + runId, + friendlyId, + taskIdentifier: "my-task", + organizationId: seed.organization.id, + projectId: seed.project.id, + runtimeEnvironmentId: seed.environment.id, + }) + ); + + const staleRead = await router.findRun({ friendlyId }, { select: CANCEL_SELECT }); + expect(staleRead).toBeNull(); + expect(newReplica.wasHit()).toBe(true); + + const primaryRead = await router.findRun( + { friendlyId }, + { select: CANCEL_SELECT }, + prisma17 as never + ); + expect(primaryRead).not.toBeNull(); + expect(primaryRead!.friendlyId).toBe(friendlyId); + expect(primaryRead!.id).toBe(runId); + } + ); +}); diff --git a/internal-packages/run-store/src/runOpsStore.dashboardAgentReadView.replicaLag.test.ts b/internal-packages/run-store/src/runOpsStore.dashboardAgentReadView.replicaLag.test.ts new file mode 100644 index 00000000000..1689009dc32 --- /dev/null +++ b/internal-packages/run-store/src/runOpsStore.dashboardAgentReadView.replicaLag.test.ts @@ -0,0 +1,224 @@ +// Replica-lag coverage for the dashboard-agent run->commit resolver read. dashboardAgent +// resolveRunCommit issues a client-less runStore.findRun({friendlyId, runtimeEnvironmentId}, {select: +// {lockedToVersionId}}) — no client, so readYourWrites() is false and the friendlyId-classified read +// routes to the owning store's REPLICA (miss => probe the other store's replica). Under lag a +// freshly-locked run is not yet visible, so findRun returns null and the resolver returns null. +// +// The stale null is tolerated: the sole consumer is a read-only resolver (the external repo-snapshot +// endpoint), where null becomes a 404 "no deployed commit" / branch-head fallback, not a mutation +// findResource guard. And the runId is dashboard-sourced (a ClickHouse view that lags more than the PG +// replica) and only carries a truthy lockedToVersionId after trigger + dequeue + lock (seconds of +// lifecycle), so by the time this read fires the PG replica has caught up — the stale-null window is +// unreachable for a run that would have returned a commit. +// +// dashboardAgent is a webapp module, so this exercises the exact underlying runStore.findRun pattern +// (same method, no-client arg, friendlyId+environment where, same select) with the owning replica +// frozen by the shared laggingReplica primitive. Asserts both the stale-null under lag and that the +// SAME read on the owning primary recovers the row's lockedToVersionId — so the null is purely replica +// lag. + +import { heteroRunOpsPostgresTest, laggingReplica } from "@internal/testcontainers"; +import type { PrismaClient } from "@trigger.dev/database"; +import { describe, expect } from "vitest"; +import { PostgresRunStore } from "./PostgresRunStore.js"; +import { RoutingRunStore } from "./runOpsStore.js"; + +// A cuid (25 chars after the `run_` prefix) classifies LEGACY, so the friendlyId-keyed read routes +// to the legacy (control-plane) store first — the owning store for this seeded run. +const CUID_25 = "c".repeat(25); +const FRIENDLY_25 = "d".repeat(25); + +async function seedEnvironmentLegacy(prisma: PrismaClient, suffix: string) { + const organization = await prisma.organization.create({ + data: { title: `Org ${suffix}`, slug: `org-${suffix}` }, + }); + const project = await prisma.project.create({ + data: { + name: `Project ${suffix}`, + slug: `project-${suffix}`, + externalRef: `proj_${suffix}`, + organizationId: organization.id, + }, + }); + const environment = await prisma.runtimeEnvironment.create({ + data: { + type: "DEVELOPMENT", + slug: "dev", + projectId: project.id, + organizationId: organization.id, + apiKey: `tr_dev_${suffix}`, + pkApiKey: `pk_dev_${suffix}`, + shortcode: `short_${suffix}`, + }, + }); + return { organization, project, environment }; +} + +async function seedBackgroundWorker( + prisma: PrismaClient, + opts: { suffix: string; organizationId: string; projectId: string; runtimeEnvironmentId: string } +) { + return prisma.backgroundWorker.create({ + data: { + friendlyId: `worker_${opts.suffix}`, + contentHash: `hash_${opts.suffix}`, + version: "20260717.1", + metadata: {}, + projectId: opts.projectId, + runtimeEnvironmentId: opts.runtimeEnvironmentId, + }, + }); +} + +function taskRunData(opts: { + id: string; + friendlyId: string; + organizationId: string; + projectId: string; + runtimeEnvironmentId: string; + lockedToVersionId?: string; +}) { + return { + id: opts.id, + engine: "V2" as const, + status: "PENDING" as const, + friendlyId: opts.friendlyId, + runtimeEnvironmentId: opts.runtimeEnvironmentId, + environmentType: "DEVELOPMENT" as const, + organizationId: opts.organizationId, + projectId: opts.projectId, + taskIdentifier: "my-task", + payload: "{}", + payloadType: "application/json", + traceContext: {}, + traceId: `trace_${opts.id}`, + spanId: `span_${opts.id}`, + queue: "task/my-task", + isTest: false, + taskEventStore: "taskEvent", + depth: 0, + lockedToVersionId: opts.lockedToVersionId ?? null, + }; +} + +// Exactly the resolveRunCommit read: findRun keyed on {friendlyId, runtimeEnvironmentId}, selecting +// lockedToVersionId, with NO client argument. Returns the resolver's early-return decision: +// null when the run (or its lockedToVersionId) is not visible, else the version id it would resolve. +async function resolveRunCommitPgRead( + router: RoutingRunStore, + where: { friendlyId: string; runtimeEnvironmentId: string } +): Promise { + const run = (await router.findRun(where, { select: { lockedToVersionId: true } })) as { + lockedToVersionId: string | null; + } | null; + if (!run?.lockedToVersionId) return null; // dashboardAgent resolveRunCommit early-return + return run.lockedToVersionId; +} + +describe("dashboardAgent resolveRunCommit — findRun (no client) reads the owning replica", () => { + heteroRunOpsPostgresTest( + "resolveRunCommit returns null for a fresh locked run not yet on the lagging replica (tolerated read-only fallback)", + async ({ prisma14, prisma17 }) => { + const legacyReplica = laggingReplica(prisma14, [{ model: "taskRun", mode: "missing" }]); + const legacyStore = new PostgresRunStore({ + prisma: prisma14, + readOnlyPrisma: legacyReplica.client, + schemaVariant: "legacy", + }); + const newReplica = laggingReplica(prisma17 as never, [{ model: "taskRun", mode: "missing" }]); + const newStore = new PostgresRunStore({ + prisma: prisma17 as never, + // Freeze the OTHER store's replica too, so the #findRunRouted miss-probe cannot mask the + // owning-replica staleness with a phantom hit. (The row lives only on legacy anyway.) + readOnlyPrisma: newReplica.client as never, + schemaVariant: "dedicated", + }); + const router = new RoutingRunStore({ new: newStore, legacy: legacyStore }); + + const seed = await seedEnvironmentLegacy(prisma14, "dash_lag"); + const worker = await seedBackgroundWorker(prisma14, { + suffix: "dash_lag", + organizationId: seed.organization.id, + projectId: seed.project.id, + runtimeEnvironmentId: seed.environment.id, + }); + const runId = `run_${CUID_25}`; // cuid => LEGACY owner + const friendlyId = `run_${FRIENDLY_25}`; // classifies LEGACY too => routes to owning store first + // The row EXISTS on the primary and is fully locked to a deployed version — resolveRunCommit + // would return a commit for it. Only replica lag hides it. + await prisma14.taskRun.create({ + data: taskRunData({ + id: runId, + friendlyId, + organizationId: seed.organization.id, + projectId: seed.project.id, + runtimeEnvironmentId: seed.environment.id, + lockedToVersionId: worker.id, + }), + }); + + const where = { friendlyId, runtimeEnvironmentId: seed.environment.id }; + + // HAZARD proof: the caller's read (no client => replica route) misses the fresh row under lag. + const underLag = await resolveRunCommitPgRead(router, where); + expect(underLag).toBeNull(); + // Prove it actually consulted the owning (legacy) replica — i.e. it is replica-routed, and the + // null is replica lag, not a routing accident. + expect(legacyReplica.wasHit()).toBe(true); + + // GROUND TRUTH: the SAME logical read against a healthy store (replica caught up) resolves the + // locked version id — so the null above is PURELY replica lag, and a route-to-primary would + // recover it. The caller tolerates the transient null (read-only 404 / branch-head fallback), so + // the property holds without any primary re-read. + const healthyLegacy = new PostgresRunStore({ + prisma: prisma14, + readOnlyPrisma: prisma14, // caught-up replica + schemaVariant: "legacy", + }); + const healthyRouter = new RoutingRunStore({ new: newStore, legacy: healthyLegacy }); + const resolved = await resolveRunCommitPgRead(healthyRouter, where); + expect(resolved).toBe(worker.id); + } + ); + + heteroRunOpsPostgresTest( + "resolveRunCommit resolves the locked version id from the replica in steady state", + async ({ prisma14, prisma17 }) => { + const legacyStore = new PostgresRunStore({ + prisma: prisma14, + readOnlyPrisma: prisma14, // caught-up replica handle + schemaVariant: "legacy", + }); + const newStore = new PostgresRunStore({ + prisma: prisma17 as never, + readOnlyPrisma: prisma17 as never, + schemaVariant: "dedicated", + }); + const router = new RoutingRunStore({ new: newStore, legacy: legacyStore }); + + const seed = await seedEnvironmentLegacy(prisma14, "dash_ok"); + const worker = await seedBackgroundWorker(prisma14, { + suffix: "dash_ok", + organizationId: seed.organization.id, + projectId: seed.project.id, + runtimeEnvironmentId: seed.environment.id, + }); + const runId = `run_${CUID_25}`; + const friendlyId = `run_${FRIENDLY_25}`; + await prisma14.taskRun.create({ + data: taskRunData({ + id: runId, + friendlyId, + organizationId: seed.organization.id, + projectId: seed.project.id, + runtimeEnvironmentId: seed.environment.id, + lockedToVersionId: worker.id, + }), + }); + + const where = { friendlyId, runtimeEnvironmentId: seed.environment.id }; + const resolved = await resolveRunCommitPgRead(router, where); + expect(resolved).toBe(worker.id); + } + ); +}); diff --git a/internal-packages/run-store/src/runOpsStore.engineReadViews.replicaLag.test.ts b/internal-packages/run-store/src/runOpsStore.engineReadViews.replicaLag.test.ts new file mode 100644 index 00000000000..7296c6373a3 --- /dev/null +++ b/internal-packages/run-store/src/runOpsStore.engineReadViews.replicaLag.test.ts @@ -0,0 +1,359 @@ +// Property: three run-engine read views tolerate stale/absent values under replica lag. Each issues a +// client-less runStore read, so RoutingRunStore routes it to the OWNING store's REPLICA (readYourWrites +// signal absent). With the replica frozen by the shared laggingReplica, this file asserts what each read +// returns under lag and documents the caller that makes the stale/absent value tolerable. Store built as +// the engine holds it (RoutingRunStore over a legacy + dedicated PostgresRunStore); reads invoked +// exactly as the caller does (same method, no client arg). +// +// Reads covered (one case each): +// 1. concurrencySweeper findRuns — #concurrencySweeperCallback (completedAt <= now-10min) +// 2. resolveTaskRunContext findRun +// 3. forceRequeue span-close findRun (telemetry) +// +// Routing recap: a client-less findRun with an id-classifiable `where` → #findRunRouted → owning store +// findRun → readOnlyPrisma (REPLICA); a client-less findRuns over an id set → #findRunsByIdSet → each +// store's findRuns with client undefined → readOnlyPrisma (REPLICA). All three hit the owning REPLICA. + +import { heteroRunOpsPostgresTest, laggingReplica } from "@internal/testcontainers"; +import type { PrismaClient } from "@trigger.dev/database"; +import type { RunOpsPrismaClient } from "@internal/run-ops-database"; +import { describe, expect } from "vitest"; +import { PostgresRunStore } from "./PostgresRunStore.js"; +import { RoutingRunStore } from "./runOpsStore.js"; + +type AnyClient = PrismaClient | RunOpsPrismaClient; + +// A cuid-shaped id (no run-ops v1 marker) classifies LEGACY, so both the create and the id-keyed reads +// route to the legacy (control-plane / prisma14, full schema) store first. +const CUID_25 = "e".repeat(25); + +// Mirror of the engine's final-run-status set (getFinalRunStatuses). Inlined to avoid a +// run-store -> run-engine test dependency; concurrencySweeper's `status: { in: ... }` filter only affects the +// PRIMARY-contrast query — the missing-mode replica ignores the where entirely and returns []. +const FINAL_RUN_STATUSES = [ + "CANCELED", + "INTERRUPTED", + "COMPLETED_SUCCESSFULLY", + "COMPLETED_WITH_ERRORS", + "SYSTEM_FAILURE", + "CRASHED", + "EXPIRED", + "TIMED_OUT", +] as const; + +async function seedEnvironment(prisma: PrismaClient, suffix: string) { + const organization = await prisma.organization.create({ + data: { title: `Org ${suffix}`, slug: `org-${suffix}` }, + }); + const project = await prisma.project.create({ + data: { + name: `Project ${suffix}`, + slug: `project-${suffix}`, + externalRef: `proj_${suffix}`, + organizationId: organization.id, + }, + }); + const environment = await prisma.runtimeEnvironment.create({ + data: { + type: "DEVELOPMENT", + slug: "dev", + projectId: project.id, + organizationId: organization.id, + apiKey: `tr_dev_${suffix}`, + pkApiKey: `pk_dev_${suffix}`, + shortcode: `short_${suffix}`, + }, + }); + return { organization, project, environment }; +} + +function taskRunData(opts: { + id: string; + friendlyId: string; + organizationId: string; + projectId: string; + runtimeEnvironmentId: string; + status?: string; + completedAt?: Date | null; + spanId?: string; + maxAttempts?: number | null; + createdAt?: Date; +}) { + return { + id: opts.id, + engine: "V2" as const, + status: (opts.status ?? "PENDING") as never, + friendlyId: opts.friendlyId, + runtimeEnvironmentId: opts.runtimeEnvironmentId, + environmentType: "DEVELOPMENT" as const, + organizationId: opts.organizationId, + projectId: opts.projectId, + taskIdentifier: "my-task", + payload: "{}", + payloadType: "application/json", + traceContext: {}, + traceId: `trace_${opts.id}`, + spanId: opts.spanId ?? `span_${opts.id}`, + queue: "task/my-task", + isTest: false, + taskEventStore: "taskEvent", + depth: 0, + ...(opts.completedAt !== undefined ? { completedAt: opts.completedAt } : {}), + ...(opts.maxAttempts !== undefined ? { maxAttempts: opts.maxAttempts } : {}), + ...(opts.createdAt !== undefined ? { createdAt: opts.createdAt } : {}), + }; +} + +function buildRouter( + prisma14: PrismaClient, + prisma17: unknown, + legacyReplicaClient: AnyClient +): RoutingRunStore { + const legacyStore = new PostgresRunStore({ + prisma: prisma14, + readOnlyPrisma: legacyReplicaClient, + schemaVariant: "legacy", + }); + const newStore = new PostgresRunStore({ + prisma: prisma17 as never, + readOnlyPrisma: prisma17 as never, + schemaVariant: "dedicated", + }); + return new RoutingRunStore({ new: newStore, legacy: legacyStore }); +} + +describe("run-engine read views under replica lag (client-less -> owning REPLICA)", () => { + // concurrencySweeper findRuns. + // The concurrencySweeper callback is handed a set of runIds (from the Redis concurrency set) and asks + // the store which are FINISHED and completed MORE THAN 10 MINUTES AGO, so it can release their held + // concurrency. The read is a client-less findRuns → owning REPLICA. Two caller facts make a + // stale/absent replica tolerable, and this test proves the store-level fact + asserts both: + // (a) the `completedAt <= now - 10min` filter: any row that MATCHES was committed to the primary at + // least 10 minutes ago, so real replica lag (sub-second..seconds) cannot hide a matching row. + // (b) concurrencySweeper runs periodically: a run transiently missed on one scan is re-evaluated on + // the next — a miss self-heals, it does not leak concurrency permanently. + // The consequence of a miss is at worst "concurrency released one scan later", never a wrong release. + heteroRunOpsPostgresTest( + "concurrencySweeper findRuns misses the finished run under replica lag", + async ({ prisma14, prisma17 }) => { + const seed = await seedEnvironment(prisma14, "sweeper_leg"); + const runId = `run_${CUID_25}`; // cuid → LEGACY + // A genuinely-finished run whose completedAt is 30 min in the past — matches the concurrencySweeper filter. + const completedAt = new Date(Date.now() - 1000 * 60 * 30); + await prisma14.taskRun.create({ + data: taskRunData({ + id: runId, + friendlyId: "run_sweeper_leg", + organizationId: seed.organization.id, + projectId: seed.project.id, + runtimeEnvironmentId: seed.environment.id, + status: "COMPLETED_SUCCESSFULLY", + completedAt, + }), + }); + + const completedAtOffsetMs = 1000 * 60 * 10; // the caller's default + + // Line-for-line mirror of #concurrencySweeperCallback's read + post-filter. + const sweeperCallback = async ( + runStore: RoutingRunStore, + runIds: string[], + readClient?: unknown + ) => { + const runs = (await runStore.findRuns( + { + where: { + id: { in: runIds }, + completedAt: { lte: new Date(Date.now() - completedAtOffsetMs) }, + organizationId: { not: null }, + status: { in: FINAL_RUN_STATUSES as unknown as never }, + }, + select: { id: true, status: true, organizationId: true }, + }, + readClient as never + )) as Array<{ id: string; status: string; organizationId: string | null }>; + return runs + .filter((r) => !!r.organizationId) + .map((r) => ({ id: r.id, orgId: r.organizationId! })); + }; + + // PRIMARY contrast: thread the writer (as read-your-writes callers do) → the run IS seen. Proves + // the run genuinely matches the concurrencySweeper predicate, so any miss below is purely the replica read. + const okReplica = laggingReplica(prisma14, [{ model: "taskRun", mode: "missing" }]); + const routerForPrimary = buildRouter(prisma14, prisma17, okReplica.client); + const viaPrimary = await sweeperCallback(routerForPrimary, [runId], prisma14); + expect(viaPrimary).toEqual([{ id: runId, orgId: seed.organization.id }]); + + // ACTUAL caller behavior: client-less findRuns → owning REPLICA. Freeze the legacy replica so the + // finished run is not visible. + const lagReplica = laggingReplica(prisma14, [{ model: "taskRun", mode: "missing" }]); + const router = buildRouter(prisma14, prisma17, lagReplica.client); + const viaReplica = await sweeperCallback(router, [runId]); + + // Store-level fact under lag: concurrencySweeper's read misses the run entirely and returns []. + expect(lagReplica.wasHit("taskRun")).toBe(true); + expect(viaReplica).toEqual([]); + // TOLERATED because (a) a real row matching `completedAt <= now-10min` (here -30min) committed >=10 + // min ago and has surely replicated, and (b) concurrencySweeper re-scans on schedule so a transient miss + // self-heals — the worst case is concurrency freed one scan later, never a wrong release. + } + ); + + // resolveTaskRunContext findRun. + // resolveTaskRunContext builds a V4 TaskRunContext for a run. Its ONLY non-test caller is + // SpanPresenter #getV4TaskRunContext, a READ-ONLY dashboard presenter that has ALREADY loaded `run` + // for display and passes run.id. The read is client-less → owning REPLICA. Under lag the store returns + // a STALE row (asserted below); the resolver copies its scalar fields straight into the display + // context. Because the sole caller is a read-only span-detail page, a stale field is a cosmetic + // display artifact that self-heals on the next page load — it drives no mutation or control decision. + // The extreme case (row wholly absent on the replica) throws a transient 404, likewise a display miss. + heteroRunOpsPostgresTest( + "resolveTaskRunContext findRun returns a stale run row under replica lag", + async ({ prisma14, prisma17 }) => { + const seed = await seedEnvironment(prisma14, "ctx_leg"); + const runId = `run_${CUID_25}`; // cuid → LEGACY + const friendlyId = "run_ctx_leg"; + // The run has since PROGRESSED on the primary to EXECUTING… + await prisma14.taskRun.create({ + data: taskRunData({ + id: runId, + friendlyId, + organizationId: seed.organization.id, + projectId: seed.project.id, + runtimeEnvironmentId: seed.environment.id, + status: "EXECUTING", + }), + }); + + // …but the replica still shows the OLD PENDING snapshot (frozen row). Provide the fields the site + // asserts on; the legacy read returns this row verbatim (select projection is applied by Postgres, + // which the frozen replica bypasses, so it returns exactly this shape). + const staleRow = { + id: runId, + friendlyId, + status: "PENDING", + runtimeEnvironmentId: seed.environment.id, + createdAt: new Date("2024-01-01T00:00:00.000Z"), + }; + const lagReplica = laggingReplica(prisma14, [ + { model: "taskRun", mode: "frozen", rows: [staleRow] }, + ]); + const router = buildRouter(prisma14, prisma17, lagReplica.client); + + // Invoke EXACTLY as resolveTaskRunContext does: client-less findRun({ id }, { select }). + const run = (await router.findRun( + { id: runId }, + { + select: { + id: true, + status: true, + friendlyId: true, + createdAt: true, + runtimeEnvironmentId: true, + }, + } + )) as { id: string; status: string; friendlyId: string } | null; + + expect(lagReplica.wasHit("taskRun")).toBe(true); + // Store-level fact: the client-less read is served by the lagging replica → STALE status. + expect(run).not.toBeNull(); + expect(run!.id).toBe(runId); + expect(run!.status).toBe("PENDING"); // stale — primary is already EXECUTING + + // Proof the staleness is confined to the replica: the primary read returns the fresh status. + const onPrimary = (await router.findRun( + { id: runId }, + { select: { status: true } }, + prisma14 + )) as { status: string } | null; + expect(onPrimary?.status).toBe("EXECUTING"); + // TOLERATED: the only caller (SpanPresenter #getV4TaskRunContext) is a read-only dashboard + // presenter — a stale status on the span-detail page self-heals on refresh and drives no mutation. + } + ); + + // forceRequeue span-close findRun. + // Inside completeRunAttempt's forceRequeue branch, AFTER the retry/requeue decision has already been + // made on the primary (retryOutcomeFromCompletion threads this.$.prisma), this client-less findRun + // re-reads the run purely to shape the `runAttemptFailed` telemetry event (eventBus.emit: status, + // spanId, createdAt, completedAt, updatedAt). The read is client-less → owning REPLICA. Under lag the + // store returns a STALE row (asserted below); the stale scalars only populate the emitted span-close + // event, so a slightly-old status/updatedAt is a cosmetic telemetry artifact — it does not feed the + // requeue control flow (already decided on the primary). The immutable fields the span close needs + // (spanId, createdAt) are set at creation and long replicated, so they are correct even under lag. + heteroRunOpsPostgresTest( + "forceRequeue span-close findRun returns stale status but correct immutable spanId and createdAt under replica lag", + async ({ prisma14, prisma17 }) => { + const seed = await seedEnvironment(prisma14, "requeue_leg"); + const runId = `run_${CUID_25}`; // cuid → LEGACY + const spanId = "span_requeue_fixed"; + const createdAt = new Date("2024-01-01T00:00:00.000Z"); + // Primary: the run has been requeued back to PENDING with a fresh updatedAt. + await prisma14.taskRun.create({ + data: taskRunData({ + id: runId, + friendlyId: "run_requeue_leg", + organizationId: seed.organization.id, + projectId: seed.project.id, + runtimeEnvironmentId: seed.environment.id, + status: "PENDING", + spanId, + maxAttempts: 3, + createdAt, + }), + }); + + // Replica still shows the pre-requeue EXECUTING snapshot with an old updatedAt — but the immutable + // spanId/createdAt match the primary (they never change after creation). + const staleRow = { + id: runId, + status: "EXECUTING", + spanId, + maxAttempts: 3, + taskEventStore: "taskEvent", + createdAt, + completedAt: null as Date | null, + updatedAt: new Date("2024-01-01T00:00:05.000Z"), + }; + const lagReplica = laggingReplica(prisma14, [ + { model: "taskRun", mode: "frozen", rows: [staleRow] }, + ]); + const router = buildRouter(prisma14, prisma17, lagReplica.client); + + // Invoke EXACTLY as the forceRequeue branch does: client-less findRun({ id }, { select }). + const minimalRun = (await router.findRun( + { id: runId }, + { + select: { + status: true, + spanId: true, + maxAttempts: true, + taskEventStore: true, + createdAt: true, + completedAt: true, + updatedAt: true, + }, + } + )) as { status: string; spanId: string; createdAt: Date } | null; + + expect(lagReplica.wasHit("taskRun")).toBe(true); + expect(minimalRun).not.toBeNull(); + // Store-level fact: mutable fields are STALE off the replica… + expect(minimalRun!.status).toBe("EXECUTING"); // stale — primary already PENDING (requeued) + // …but the fields the span close actually depends on are immutable and correct even under lag. + expect(minimalRun!.spanId).toBe(spanId); + expect(minimalRun!.createdAt).toEqual(createdAt); + + // Confirm the staleness is replica-only: the primary read shows the requeued PENDING status. + const onPrimary = (await router.findRun( + { id: runId }, + { select: { status: true } }, + prisma14 + )) as { status: string } | null; + expect(onPrimary?.status).toBe("PENDING"); + // TOLERATED: the read's result is only fed to eventBus.emit("runAttemptFailed", ...) (span-close + // telemetry). The requeue decision was already made on the primary; a stale status/updatedAt in + // the emitted event is cosmetic and does not alter control flow. + } + ); +}); diff --git a/internal-packages/run-store/src/runOpsStore.idempotencyGlobalScopeCrossDb.test.ts b/internal-packages/run-store/src/runOpsStore.idempotencyGlobalScopeCrossDb.test.ts new file mode 100644 index 00000000000..63a972c63c6 --- /dev/null +++ b/internal-packages/run-store/src/runOpsStore.idempotencyGlobalScopeCrossDb.test.ts @@ -0,0 +1,328 @@ +// Property: a GLOBAL-scope idempotency key (per environment+task, NOT per parent) maps to exactly ONE +// child run even when triggered from two parents of DIFFERENT residency. The dedup client is routed by +// the PARENT run's residency (NEW parent → NEW DB, LEGACY parent → LEGACY DB), and the trigger hot path +// probes runStore.findRun({env, idempotencyKey, task}) before minting. Exercised on the REAL +// two-physical-DB split (heteroRunOpsPostgresTest, never mocked): drives the actual +// RoutingRunStore.findRun probe (dedup client chosen by parent residency) + createRun mint in real +// sequential trigger order, asserting one child — the id-less findRun fans out across BOTH DBs, so the +// second probe sees the first child. A routing/topology property, not replica lag. + +import { heteroRunOpsPostgresTest } from "@internal/testcontainers"; +import type { PrismaClient, PrismaClientOrTransaction } from "@trigger.dev/database"; +import type { RunOpsPrismaClient } from "@internal/run-ops-database"; +import { describe, expect } from "vitest"; +import { PostgresRunStore } from "./PostgresRunStore.js"; +import { RoutingRunStore } from "./runOpsStore.js"; +import type { CreateRunInput, RunStoreSchemaVariant } from "./types.js"; + +type AnyClient = PrismaClient | RunOpsPrismaClient; +type Residency = "NEW" | "LEGACY"; + +// ownerEngine classifies by internal-id LENGTH after stripping a leading `_`: +// 25 chars (no internal underscore) → cuid → LEGACY (#legacy / prisma14), +// a v1 body (version "1" at index 25) → run-ops id → NEW (#new / prisma17). +function cuidLegacy(seed: string): string { + return (seed + "c".repeat(25)).slice(0, 25); // 25 chars → LEGACY +} +function runOpsNew(seed: string): string { + return (seed.replace(/[^0-9a-v]/g, "0") + "k".repeat(24)).slice(0, 24) + "01"; +} + +async function seedEnvironment( + prisma: AnyClient, + schemaVariant: RunStoreSchemaVariant, + suffix: string +) { + if (schemaVariant === "dedicated") { + return { + organization: { id: `org_${suffix}` }, + project: { id: `proj_${suffix}` }, + environment: { id: `env_${suffix}` }, + }; + } + const organization = await (prisma as PrismaClient).organization.create({ + data: { title: `Org ${suffix}`, slug: `org-${suffix}` }, + }); + const project = await (prisma as PrismaClient).project.create({ + data: { + name: `Project ${suffix}`, + slug: `project-${suffix}`, + externalRef: `proj_${suffix}`, + organizationId: organization.id, + }, + }); + const environment = await (prisma as PrismaClient).runtimeEnvironment.create({ + data: { + type: "DEVELOPMENT", + slug: "dev", + projectId: project.id, + organizationId: organization.id, + apiKey: `tr_dev_${suffix}`, + pkApiKey: `pk_dev_${suffix}`, + shortcode: `short_${suffix}`, + }, + }); + return { organization, project, environment }; +} + +// One logical environment whose scalar env/project/org ids are shared by both physical DBs, with real +// owning rows seeded on #legacy (kept FKs) and the same scalar ids valid on the FK-free #new subset. +async function seedSharedEnv(prisma14: PrismaClient, suffix: string) { + const legacy = await seedEnvironment(prisma14, "legacy", suffix); + return { + organizationId: legacy.organization.id, + projectId: legacy.project.id, + runtimeEnvironmentId: legacy.environment.id, + environmentId: legacy.environment.id, + }; +} + +function buildCreateRunInput(params: { + runId: string; + friendlyId: string; + organizationId: string; + projectId: string; + runtimeEnvironmentId: string; + idempotencyKey: string; + taskIdentifier: string; +}): CreateRunInput { + return { + data: { + id: params.runId, + engine: "V2", + status: "PENDING", + friendlyId: params.friendlyId, + runtimeEnvironmentId: params.runtimeEnvironmentId, + environmentType: "DEVELOPMENT", + organizationId: params.organizationId, + projectId: params.projectId, + idempotencyKey: params.idempotencyKey, + idempotencyKeyExpiresAt: new Date(Date.now() + 24 * 60 * 60 * 1000), + taskIdentifier: params.taskIdentifier, + payload: '{"hello":"world"}', + payloadType: "application/json", + context: { foo: "bar" }, + traceContext: { trace: "ctx" }, + traceId: `trace_${params.runId}`, + spanId: `span_${params.runId}`, + runTags: [], + queue: "task/my-task", + isTest: false, + taskEventStore: "taskEvent", + depth: 1, // a child run (has a parent) + createdAt: new Date("2024-01-01T00:00:00.000Z"), + }, + snapshot: { + engine: "V2", + executionStatus: "RUN_CREATED", + description: "Run was created", + runStatus: "PENDING", + environmentId: params.runtimeEnvironmentId, + environmentType: "DEVELOPMENT", + projectId: params.projectId, + organizationId: params.organizationId, + }, + }; +} + +function makeSplitRouter(prisma14: PrismaClient, prisma17: RunOpsPrismaClient) { + const legacyStore = new PostgresRunStore({ + prisma: prisma14, + readOnlyPrisma: prisma14, + schemaVariant: "legacy", + }); + const newStore = new PostgresRunStore({ + prisma: prisma17 as never, + readOnlyPrisma: prisma17 as never, + schemaVariant: "dedicated", + }); + return { router: new RoutingRunStore({ new: newStore, legacy: legacyStore }) }; +} + +// A faithful port of resolveIdempotencyDedupClient's parent-residency branch: the dedup client is the +// parent's own DB writer. This is the exact routing decision the code makes. +function resolveDedupClient( + parentResidency: Residency, + newClient: PrismaClientOrTransaction, + legacyClient: PrismaClientOrTransaction +): PrismaClientOrTransaction { + return parentResidency === "NEW" ? newClient : legacyClient; +} + +describe("run-ops split — GLOBAL-scope idempotency dedup across two different-residency parents", () => { + // The real trigger dedup flow for one trigger with an idempotency key, minus the webapp glue: + // 1. resolve the dedup client by the PARENT's residency, + // 2. probe runStore.findRun({ env, idempotencyKey, task }, { include }, dedupClient) — the EXACT + // id-less probe the trigger hot path issues, + // 3. if a run comes back → CACHED hit, mint nothing, + // 4. else createRun a fresh child on the store its (parent-inherited) id-shape names. + // A child inherits its parent's residency, so a NEW parent's child gets a run-ops id (→ #new) and a + // LEGACY parent's child gets a cuid (→ #legacy). + async function triggerChild( + router: RoutingRunStore, + params: { + parentResidency: Residency; + childId: string; + childFriendlyId: string; + idempotencyKey: string; + taskIdentifier: string; + env: { + organizationId: string; + projectId: string; + runtimeEnvironmentId: string; + environmentId: string; + }; + newClient: PrismaClientOrTransaction; + legacyClient: PrismaClientOrTransaction; + } + ): Promise<{ cached: boolean; runId: string }> { + const dedupClient = resolveDedupClient( + params.parentResidency, + params.newClient, + params.legacyClient + ); + + const existing = (await router.findRun( + { + runtimeEnvironmentId: params.env.runtimeEnvironmentId, + idempotencyKey: params.idempotencyKey, + taskIdentifier: params.taskIdentifier, + }, + { include: { associatedWaitpoint: true } }, + dedupClient + )) as Record | null; + + if (existing) { + return { cached: true, runId: existing.id as string }; + } + + await router.createRun( + buildCreateRunInput({ + runId: params.childId, + friendlyId: params.childFriendlyId, + organizationId: params.env.organizationId, + projectId: params.env.projectId, + runtimeEnvironmentId: params.env.runtimeEnvironmentId, + idempotencyKey: params.idempotencyKey, + taskIdentifier: params.taskIdentifier, + }) + ); + return { cached: false, runId: params.childId }; + } + + async function countChildrenForKey( + prisma14: PrismaClient, + prisma17: RunOpsPrismaClient, + environmentId: string, + idempotencyKey: string, + taskIdentifier: string + ): Promise<{ legacy: number; new: number; total: number }> { + const where = { + runtimeEnvironmentId: environmentId, + idempotencyKey, + taskIdentifier, + }; + const legacy = await prisma14.taskRun.count({ where }); + const nw = await prisma17.taskRun.count({ where }); + return { legacy, new: nw, total: legacy + nw }; + } + + // (a) NEW parent triggers first, then LEGACY parent — same GLOBAL key. If the LEGACY parent's probe + // only read the LEGACY DB it would miss the NEW child and mint a second one → two children. + heteroRunOpsPostgresTest( + "NEW-parent-first then LEGACY-parent: one global key must yield exactly one child", + async ({ prisma14, prisma17 }) => { + const { router } = makeSplitRouter(prisma14, prisma17); + const env = await seedSharedEnv(prisma14, "gsx_a"); + const idempotencyKey = "global-scope-key-a"; + const taskIdentifier = "child-task"; + + // Trigger 1: from a NEW-resident parent → child born on #new (run-ops id). + const first = await triggerChild(router, { + parentResidency: "NEW", + childId: runOpsNew("gxan"), + childFriendlyId: "run_gsx_a_new_child", + idempotencyKey, + taskIdentifier, + env, + newClient: prisma17 as unknown as PrismaClientOrTransaction, + legacyClient: prisma14, + }); + expect(first.cached).toBe(false); + + // Trigger 2: from a LEGACY-resident parent → child would be born on #legacy (cuid) UNLESS the + // dedup probe sees trigger 1's child on #new and returns it cached. + const second = await triggerChild(router, { + parentResidency: "LEGACY", + childId: cuidLegacy("gxal"), + childFriendlyId: "run_gsx_a_legacy_child", + idempotencyKey, + taskIdentifier, + env, + newClient: prisma17 as unknown as PrismaClientOrTransaction, + legacyClient: prisma14, + }); + + const counts = await countChildrenForKey( + prisma14, + prisma17, + env.environmentId, + idempotencyKey, + taskIdentifier + ); + + // The load-bearing assertion: a GLOBAL idempotency key must map to exactly ONE child run, + // no matter that the two parents live on different physical DBs. + expect(counts.total).toBe(1); + // And the second trigger must be a cached hit resolving to the FIRST child. + expect(second.cached).toBe(true); + expect(second.runId).toBe(first.runId); + } + ); + + // (b) Reverse order: LEGACY parent first, then NEW parent. Symmetric — guards the other fan-out leg. + heteroRunOpsPostgresTest( + "LEGACY-parent-first then NEW-parent: one global key must yield exactly one child", + async ({ prisma14, prisma17 }) => { + const { router } = makeSplitRouter(prisma14, prisma17); + const env = await seedSharedEnv(prisma14, "gsx_b"); + const idempotencyKey = "global-scope-key-b"; + const taskIdentifier = "child-task"; + + const first = await triggerChild(router, { + parentResidency: "LEGACY", + childId: cuidLegacy("gxbl"), + childFriendlyId: "run_gsx_b_legacy_child", + idempotencyKey, + taskIdentifier, + env, + newClient: prisma17 as unknown as PrismaClientOrTransaction, + legacyClient: prisma14, + }); + expect(first.cached).toBe(false); + + const second = await triggerChild(router, { + parentResidency: "NEW", + childId: runOpsNew("gxbn"), + childFriendlyId: "run_gsx_b_new_child", + idempotencyKey, + taskIdentifier, + env, + newClient: prisma17 as unknown as PrismaClientOrTransaction, + legacyClient: prisma14, + }); + + const counts = await countChildrenForKey( + prisma14, + prisma17, + env.environmentId, + idempotencyKey, + taskIdentifier + ); + + expect(counts.total).toBe(1); + expect(second.cached).toBe(true); + expect(second.runId).toBe(first.runId); + } + ); +}); diff --git a/internal-packages/run-store/src/runOpsStore.idempotencyResetReadView.replicaLag.test.ts b/internal-packages/run-store/src/runOpsStore.idempotencyResetReadView.replicaLag.test.ts new file mode 100644 index 00000000000..0081e307df2 --- /dev/null +++ b/internal-packages/run-store/src/runOpsStore.idempotencyResetReadView.replicaLag.test.ts @@ -0,0 +1,225 @@ +// Lagging-replica coverage for the IDEMPOTENCY-KEY RESET route's source-run lookup on the run-ops split. +// The reset action reads the run by friendlyId with NO client (→ OWNING store's REPLICA, fan-out to the +// other store's replica on a miss) and that result gates the whole reset — a DECISION-DRIVING read. +// Under lag the read returns null for a run that exists on the primary. These tests build the store as +// the route holds it, seed the run on the OWNING primary, freeze the OWNING replica via laggingReplica, +// and invoke the read EXACTLY as the route does, then show the owning-WRITER read escalates to the +// primary and resolves the run + its idempotencyKey. Real split topology via heteroRunOpsPostgresTest — NEVER mocked. + +import { heteroRunOpsPostgresTest, laggingReplica } from "@internal/testcontainers"; +import type { PrismaClient } from "@trigger.dev/database"; +import type { RunOpsPrismaClient } from "@internal/run-ops-database"; +import { generateRunOpsId } from "@trigger.dev/core/v3/isomorphic"; +import { describe, expect } from "vitest"; +import { PostgresRunStore } from "./PostgresRunStore.js"; +import { RoutingRunStore } from "./runOpsStore.js"; +import type { CreateRunInput, RunStoreSchemaVariant } from "./types.js"; + +type AnyClient = PrismaClient | RunOpsPrismaClient; + +// ownerEngine (classifyResidency) routes a run-ops v1 body → NEW, everything else → LEGACY. +const CUID_25 = "c".repeat(25); // → LEGACY (#legacy / prisma14, full schema) +const NEW_RUN_ID = `run_${generateRunOpsId()}`; // valid v1 body → NEW (#new / prisma17, dedicated) + +async function seedEnvironment( + prisma: AnyClient, + schemaVariant: RunStoreSchemaVariant, + slugSuffix: string +) { + if (schemaVariant === "dedicated") { + return { + organization: { id: `org_${slugSuffix}` }, + project: { id: `proj_${slugSuffix}` }, + environment: { id: `env_${slugSuffix}` }, + }; + } + const organization = await (prisma as PrismaClient).organization.create({ + data: { title: `Org ${slugSuffix}`, slug: `org-${slugSuffix}` }, + }); + const project = await (prisma as PrismaClient).project.create({ + data: { + name: `Project ${slugSuffix}`, + slug: `project-${slugSuffix}`, + externalRef: `proj_${slugSuffix}`, + organizationId: organization.id, + }, + }); + const environment = await (prisma as PrismaClient).runtimeEnvironment.create({ + data: { + type: "DEVELOPMENT", + slug: "dev", + projectId: project.id, + organizationId: organization.id, + apiKey: `tr_dev_${slugSuffix}`, + pkApiKey: `pk_dev_${slugSuffix}`, + shortcode: `short_${slugSuffix}`, + }, + }); + return { organization, project, environment }; +} + +function buildCreateRunInput(params: { + runId: string; + friendlyId: string; + taskIdentifier: string; + organizationId: string; + projectId: string; + runtimeEnvironmentId: string; + idempotencyKey: string; +}): CreateRunInput { + return { + data: { + id: params.runId, + engine: "V2", + status: "COMPLETED_SUCCESSFULLY", + friendlyId: params.friendlyId, + runtimeEnvironmentId: params.runtimeEnvironmentId, + environmentType: "DEVELOPMENT", + organizationId: params.organizationId, + projectId: params.projectId, + taskIdentifier: params.taskIdentifier, + // The reset route only makes sense for a run triggered WITH an idempotency key. + idempotencyKey: params.idempotencyKey, + payload: '{"hello":"world"}', + payloadType: "application/json", + context: { foo: "bar" }, + traceContext: { trace: "ctx" }, + traceId: "trace_1", + spanId: "span_1", + runTags: ["alpha", "beta"], + queue: "task/my-task", + isTest: false, + taskEventStore: "taskEvent", + depth: 0, + createdAt: new Date("2024-01-01T00:00:00.000Z"), + }, + snapshot: { + engine: "V2", + executionStatus: "RUN_CREATED", + description: "Run was created", + runStatus: "PENDING", + environmentId: params.runtimeEnvironmentId, + environmentType: "DEVELOPMENT", + projectId: params.projectId, + organizationId: params.organizationId, + }, + }; +} + +// Exactly the projection the reset action reads. +const RESET_SELECT = { + id: true, + idempotencyKey: true, + taskIdentifier: true, + projectId: true, + runtimeEnvironmentId: true, +} as const; + +describe("run-ops split — idempotency-key reset source-run lookup vs. a lagging replica (read-your-writes)", () => { + // LEGACY cuid resident. The run is committed to the control-plane WRITER; the control-plane replica + // lags (the buffer-drained-but-not-yet-replicated window). The route's no-client findRun hits that + // replica and returns null for a run that exists on the primary; the owning-writer re-read resolves it. + heteroRunOpsPostgresTest( + "LEGACY cuid: no-client findRun is stale under lag; owning-primary re-read finds it", + async ({ prisma14, prisma17 }) => { + const legacyReplica = laggingReplica(prisma14, [{ model: "taskRun", mode: "missing" }]); + const legacyStore = new PostgresRunStore({ + prisma: prisma14, + readOnlyPrisma: legacyReplica.client, + schemaVariant: "legacy", + }); + const newStore = new PostgresRunStore({ + prisma: prisma17 as never, + readOnlyPrisma: prisma17 as never, + schemaVariant: "dedicated", + }); + const router = new RoutingRunStore({ new: newStore, legacy: legacyStore }); + + const seed = await seedEnvironment(prisma14, "legacy", "idem_reset_leg"); + const runId = `run_${CUID_25}`; // cuid → LEGACY + const friendlyId = "run_idem_reset_leg"; + const idempotencyKey = "user-supplied-key-leg"; + await legacyStore.createRun( + buildCreateRunInput({ + runId, + friendlyId, + taskIdentifier: "my-task", + organizationId: seed.organization.id, + projectId: seed.project.id, + runtimeEnvironmentId: seed.environment.id, + idempotencyKey, + }) + ); + + // The exact reset-route read — friendlyId + RESET_SELECT, NO client → replica. + const staleRead = await router.findRun({ friendlyId }, { select: RESET_SELECT }); + + // The run exists on the primary but the replica lags, so the no-client read returns null. In the + // route this is the "Run not found" short-circuit, before ResetIdempotencyKeyService runs. The + // owning-writer re-read below is the property that resolves the run under lag. + expect(staleRead).toBeNull(); + expect(legacyReplica.wasHit()).toBe(true); + + // Pass the control-plane WRITER to findRun so it escalates to findRunOnPrimary. This resolves the + // run + its idempotencyKey under lag. + const primaryRead = await router.findRun( + { friendlyId }, + { select: RESET_SELECT }, + prisma14 as never + ); + expect(primaryRead).not.toBeNull(); + expect(primaryRead!.id).toBe(runId); + // The idempotencyKey the reset service needs is only visible via the primary. + expect(primaryRead!.idempotencyKey).toBe(idempotencyKey); + } + ); + + // Same property on a NEW-resident (ksuid) run; the NEW replica lags. Not legacy-specific: whichever + // store owns the run, the no-client read hits its lagging replica and the owning-writer re-read resolves it. + heteroRunOpsPostgresTest( + "NEW ksuid: no-client findRun is stale under NEW replica lag; owning-primary re-read finds it", + async ({ prisma14, prisma17 }) => { + const newReplica = laggingReplica(prisma17, [{ model: "taskRun", mode: "missing" }]); + const newStore = new PostgresRunStore({ + prisma: prisma17 as never, + readOnlyPrisma: newReplica.client as never, + schemaVariant: "dedicated", + }); + const legacyStore = new PostgresRunStore({ + prisma: prisma14, + readOnlyPrisma: prisma14, + schemaVariant: "legacy", + }); + const router = new RoutingRunStore({ new: newStore, legacy: legacyStore }); + + const seed = await seedEnvironment(prisma17, "dedicated", "idem_reset_new"); + const runId = NEW_RUN_ID; // v1 body → NEW + const friendlyId = "run_idem_reset_new"; + const idempotencyKey = "user-supplied-key-new"; + await newStore.createRun( + buildCreateRunInput({ + runId, + friendlyId, + taskIdentifier: "my-task", + organizationId: seed.organization.id, + projectId: seed.project.id, + runtimeEnvironmentId: seed.environment.id, + idempotencyKey, + }) + ); + + const staleRead = await router.findRun({ friendlyId }, { select: RESET_SELECT }); + expect(staleRead).toBeNull(); + expect(newReplica.wasHit()).toBe(true); + + const primaryRead = await router.findRun( + { friendlyId }, + { select: RESET_SELECT }, + prisma17 as never + ); + expect(primaryRead).not.toBeNull(); + expect(primaryRead!.id).toBe(runId); + expect(primaryRead!.idempotencyKey).toBe(idempotencyKey); + } + ); +}); diff --git a/internal-packages/run-store/src/runOpsStore.modelRuntimeEnvReadView.replicaLag.test.ts b/internal-packages/run-store/src/runOpsStore.modelRuntimeEnvReadView.replicaLag.test.ts new file mode 100644 index 00000000000..45c4eeaf91a --- /dev/null +++ b/internal-packages/run-store/src/runOpsStore.modelRuntimeEnvReadView.replicaLag.test.ts @@ -0,0 +1,183 @@ +// Lagging-replica coverage for the model-runtime-env read view on the run-ops split. +// +// findEnvironmentFromRun's findRun reads the run-ops scalars {runTags, batchId, runtimeEnvironmentId} +// off a run and resolves the authenticated env from them. The webapp calls it with NO tx from the +// `runMetadataUpdated` engine handler, so the client arg is the branded `$replica` handle and the +// read is REPLICA-routed. +// +// Routing (against runOpsStore.ts): findRun(where {id}, select, BRANDED $replica) → id classifiable → +// owning store first; a branded read-replica client is not a write signal, so the read stays on the +// owning store's REPLICA, then fans out to the OTHER store's replica on a miss. A cuid run owns +// LEGACY, so its frozen legacy replica is the one hit. +// +// Why read-your-writes matters here: the handler treats a null result as "Failed to find environment" +// and RETURNS, dropping BOTH the final-metadata write (updateMetadataService.call, which itself +// reads/writes the PRIMARY within the post-completion grace window) AND the realtime run-changed +// publish. The event is one-shot at attempt completion — no retry loop, no primary fallback. For a +// fast run whose lifetime is shorter than replica lag the owning replica lacks the row, so a +// replica-routed read misses a live, primary-resident run. Routing this read to the owning PRIMARY +// keeps the precheck at least as consistent as the mutation it gates (which already reads the primary). +// +// This proves the store fact at the seam: freeze the OWNING replica with the shared laggingReplica +// primitive, invoke findRun EXACTLY as the caller does (same where + select + branded-replica +// client), assert the null under lag AND that the identical row IS on the owning primary (so the miss +// is provably lag-induced and the primary re-read recovers it). Real split topology via +// heteroRunOpsPostgresTest — NEVER mocked. + +import { heteroRunOpsPostgresTest, laggingReplica } from "@internal/testcontainers"; +import type { PrismaClient } from "@trigger.dev/database"; +import { describe, expect } from "vitest"; +import { PostgresRunStore } from "./PostgresRunStore.js"; +import { RoutingRunStore } from "./runOpsStore.js"; +import { markReadReplicaClient } from "./readReplicaClient.js"; +import type { CreateRunInput } from "./types.js"; + +// ownerEngine (classifyResidency) routes a run-ops v1 body → NEW, everything else → LEGACY. A cuid +// run owns LEGACY (the control-plane DB), the residency of a run whose completion emits +// runMetadataUpdated, so the owning store is LEGACY and its frozen replica is the one this read hits. +const CUID_25 = "c".repeat(25); // → LEGACY (#legacy / prisma14, full schema) + +async function seedEnvironment(prisma: PrismaClient, slugSuffix: string) { + const organization = await prisma.organization.create({ + data: { title: `Org ${slugSuffix}`, slug: `org-${slugSuffix}` }, + }); + const project = await prisma.project.create({ + data: { + name: `Project ${slugSuffix}`, + slug: `project-${slugSuffix}`, + externalRef: `proj_${slugSuffix}`, + organizationId: organization.id, + }, + }); + const environment = await prisma.runtimeEnvironment.create({ + data: { + type: "DEVELOPMENT", + slug: "dev", + projectId: project.id, + organizationId: organization.id, + apiKey: `tr_dev_${slugSuffix}`, + pkApiKey: `pk_dev_${slugSuffix}`, + shortcode: `short_${slugSuffix}`, + }, + }); + return { organization, project, environment }; +} + +function buildCreateRunInput(params: { + runId: string; + friendlyId: string; + organizationId: string; + projectId: string; + runtimeEnvironmentId: string; +}): CreateRunInput { + return { + data: { + id: params.runId, + engine: "V2", + status: "PENDING", + friendlyId: params.friendlyId, + runtimeEnvironmentId: params.runtimeEnvironmentId, + environmentType: "DEVELOPMENT", + organizationId: params.organizationId, + projectId: params.projectId, + taskIdentifier: "my-task", + payload: '{"hello":"world"}', + payloadType: "application/json", + context: { foo: "bar" }, + traceContext: { trace: "ctx" }, + traceId: "trace_1", + spanId: "span_1", + runTags: ["alpha", "beta"], + queue: "task/my-task", + isTest: false, + taskEventStore: "taskEvent", + depth: 0, + createdAt: new Date("2024-01-01T00:00:00.000Z"), + }, + snapshot: { + engine: "V2", + executionStatus: "RUN_CREATED", + description: "Run was created", + runStatus: "PENDING", + environmentId: params.runtimeEnvironmentId, + environmentType: "DEVELOPMENT", + projectId: params.projectId, + organizationId: params.organizationId, + }, + }; +} + +// The webapp always passes its $replica handle, which is markReadReplicaClient()'d. A branded client +// tells RoutingRunStore "do not escalate to primary" — the read stays on the owning store's replica. +// Its identity is irrelevant; the router discards it and reads the owning store's readOnlyPrisma. +const BRANDED_REPLICA = markReadReplicaClient({}) as never; + +describe("run-ops split — model-runtime-env (findEnvironmentFromRun) read view vs. a lagging replica", () => { + // The findEnvironmentFromRun read (branded $replica). + heteroRunOpsPostgresTest( + "findEnvironmentFromRun findRun(branded $replica) is NULL under owning-replica lag — runMetadataUpdated handler drops the final-metadata write + realtime publish for a live run; primary re-read recovers it", + async ({ prisma14, prisma17 }) => { + // Build the router the way the webapp holds it: a LEGACY store on the control-plane DB whose + // replica LAGS, plus a fresh NEW store. The run is a cuid → LEGACY-resident. + const legacyReplica = laggingReplica(prisma14, [{ model: "taskRun", mode: "missing" }]); + const legacyStore = new PostgresRunStore({ + prisma: prisma14, + readOnlyPrisma: legacyReplica.client, + schemaVariant: "legacy", + }); + const newStore = new PostgresRunStore({ + prisma: prisma17 as never, + readOnlyPrisma: prisma17 as never, + schemaVariant: "dedicated", + }); + const router = new RoutingRunStore({ new: newStore, legacy: legacyStore }); + + const seed = await seedEnvironment(prisma14, "modelenv"); + const runId = `run_${CUID_25}`; // cuid → LEGACY + const friendlyId = "run_modelenv"; + await legacyStore.createRun( + buildCreateRunInput({ + runId, + friendlyId, + organizationId: seed.organization.id, + projectId: seed.project.id, + runtimeEnvironmentId: seed.environment.id, + }) + ); + + // The EXACT env-resolve read: where {id}, select {runTags, batchId, runtimeEnvironmentId}, $replica. + const staleRead = (await router.findRun( + { id: runId }, + { select: { runTags: true, batchId: true, runtimeEnvironmentId: true } }, + BRANDED_REPLICA + )) as { runTags: string[]; batchId: string | null; runtimeEnvironmentId: string } | null; + + // Store fact: the read is REPLICA-routed and the owning replica lags → null. The fan-out probes + // the LEGACY replica (owner) first, then the NEW replica; the frozen legacy replica records a hit. + expect(staleRead).toBeNull(); + expect(legacyReplica.wasHit()).toBe(true); + + // Caller decision, exercised: findEnvironmentFromRun returns null on the miss, so the + // runMetadataUpdated handler logs "Failed to find environment" and RETURNS — never calling + // updateMetadataService.call (the final-metadata write) or publishChangeRecord (realtime). For + // this LIVE run that is a dropped mutation, not a tolerated stale display value. + const environmentFromRun = staleRead + ? { runtimeEnvironmentId: staleRead.runtimeEnvironmentId } + : null; + expect(environmentFromRun).toBeNull(); // → handler aborts: metadata write + realtime publish dropped + + // Read-your-writes: re-reading the owning PRIMARY (pass the WRITER `prisma14` → the router + // escalates to findRunOnPrimary) returns the live run with the exact scalars the env-resolve + // needs. Routing findEnvironmentFromRun to the owning primary keeps this consistent with + // updateMetadataService.call's own primary read. + const primaryRead = (await router.findRun( + { id: runId }, + { select: { runTags: true, batchId: true, runtimeEnvironmentId: true } }, + prisma14 as never + )) as { runTags: string[]; batchId: string | null; runtimeEnvironmentId: string } | null; + expect(primaryRead).not.toBeNull(); + expect(primaryRead!.runtimeEnvironmentId).toBe(seed.environment.id); + expect(primaryRead!.runTags).toEqual(["alpha", "beta"]); + } + ); +}); diff --git a/internal-packages/run-store/src/runOpsStore.presentersRunReadView.replicaLag.test.ts b/internal-packages/run-store/src/runOpsStore.presentersRunReadView.replicaLag.test.ts new file mode 100644 index 00000000000..67baabc1565 --- /dev/null +++ b/internal-packages/run-store/src/runOpsStore.presentersRunReadView.replicaLag.test.ts @@ -0,0 +1,435 @@ +// Lagging-replica coverage for seven run-detail presenter reads. Every one is a client-less (or +// branded-replica) runStore read, so RoutingRunStore serves it from the OWNING store's REPLICA. This +// proves — with the replica frozen by the shared laggingReplica primitive — what each read returns +// under lag, and names the caller mechanism that makes the stale/absent value tolerable. Builds the +// store as the webapp holds it (RoutingRunStore over a legacy + dedicated PostgresRunStore) and invokes +// each read EXACTLY as the presenter does (same method, where, and client arg — none, or a branded +// $replica). +// +// Routing recap (verified against RoutingRunStore): findRun with NO client OR a BRANDED replica has +// readYourWrites false (a branded replica does not escalate), so it reads the owning store's +// readOnlyPrisma (REPLICA) and probes the other store's replica on a miss. findRuns over a bounded +// id-set reads each store's REPLICA; findRuns over an open predicate ({parentSpanId}) queries BOTH +// stores' replicas. So all seven reads hit a REPLICA. + +import { heteroRunOpsPostgresTest, laggingReplica } from "@internal/testcontainers"; +import type { PrismaClient } from "@trigger.dev/database"; +import type { RunOpsPrismaClient } from "@internal/run-ops-database"; +import { describe, expect } from "vitest"; +import { PostgresRunStore } from "./PostgresRunStore.js"; +import { RoutingRunStore } from "./runOpsStore.js"; +import { markReadReplicaClient } from "./readReplicaClient.js"; + +type AnyClient = PrismaClient | RunOpsPrismaClient; + +// A cuid-shaped id/friendlyId (no run-ops v1 marker) classifies LEGACY, so both the create and the +// keyed reads route to the legacy (control-plane / prisma14, full schema) store as the owner. +const CUID_25 = "e".repeat(25); +const FRIENDLY_25 = "f".repeat(25); + +async function seedEnvironment(prisma: PrismaClient, suffix: string) { + const organization = await prisma.organization.create({ + data: { title: `Org ${suffix}`, slug: `org-${suffix}` }, + }); + const project = await prisma.project.create({ + data: { + name: `Project ${suffix}`, + slug: `project-${suffix}`, + externalRef: `proj_${suffix}`, + organizationId: organization.id, + }, + }); + const environment = await prisma.runtimeEnvironment.create({ + data: { + type: "DEVELOPMENT", + slug: "dev", + projectId: project.id, + organizationId: organization.id, + apiKey: `tr_dev_${suffix}`, + pkApiKey: `pk_dev_${suffix}`, + shortcode: `short_${suffix}`, + }, + }); + return { organization, project, environment }; +} + +function taskRunData(opts: { + id: string; + friendlyId: string; + organizationId: string; + projectId: string; + runtimeEnvironmentId: string; + status?: string; + spanId?: string; + parentSpanId?: string; +}) { + return { + id: opts.id, + engine: "V2" as const, + status: (opts.status ?? "EXECUTING") as never, + friendlyId: opts.friendlyId, + runtimeEnvironmentId: opts.runtimeEnvironmentId, + environmentType: "DEVELOPMENT" as const, + organizationId: opts.organizationId, + projectId: opts.projectId, + taskIdentifier: "my-task", + payload: "{}", + payloadType: "application/json", + traceContext: {}, + traceId: `trace_${opts.id}`, + spanId: opts.spanId ?? `span_${opts.id}`, + ...(opts.parentSpanId !== undefined ? { parentSpanId: opts.parentSpanId } : {}), + queue: "task/my-task", + isTest: false, + taskEventStore: "taskEvent", + depth: 0, + }; +} + +// Build the router exactly as buildRunStore does (RoutingRunStore over legacy + dedicated stores), +// with the LEGACY store's REPLICA swapped for the frozen `legacyReplicaClient`. The NEW store keeps +// the real prisma17 as its replica — the seeded run lives only on legacy, so the routed miss-probe of +// the new store returns null/[] naturally (no phantom hit masking the legacy-replica staleness). +function buildRouter( + prisma14: PrismaClient, + prisma17: unknown, + legacyReplicaClient: AnyClient +): RoutingRunStore { + const legacyStore = new PostgresRunStore({ + prisma: prisma14, + readOnlyPrisma: legacyReplicaClient, + schemaVariant: "legacy", + }); + const newStore = new PostgresRunStore({ + prisma: prisma17 as never, + readOnlyPrisma: prisma17 as never, + schemaVariant: "dedicated", + }); + return new RoutingRunStore({ new: newStore, legacy: legacyStore }); +} + +describe("run-detail presenter read views route to the owning replica under lag", () => { + // ApiRetrieveRunPresenter findRun (+$replica) — public GET /runs/:runId retrieve. Passes the BRANDED + // $replica, which stays on a replica (no escalation), so the read is served by the owning REPLICA. + // Under lag a freshly triggered run's row is not yet on the replica, so findRun returns null and the + // presenter's `if (pgRow)` guard falls through → not-found for this poll. Tolerated: the SDK's + // runs.retrieve/poll loop re-fetches on the next poll once the replica catches up. Read-only. + heteroRunOpsPostgresTest( + "ApiRetrieveRunPresenter findRun (+$replica) returns null for a fresh run missed on the frozen replica (SDK retrieve/poll re-fetches)", + async ({ prisma14, prisma17 }) => { + const seed = await seedEnvironment(prisma14, "retrieve_leg"); + const runId = `run_${CUID_25}`; + const friendlyId = `run_${FRIENDLY_25}`; + await prisma14.taskRun.create({ + data: taskRunData({ + id: runId, + friendlyId, + organizationId: seed.organization.id, + projectId: seed.project.id, + runtimeEnvironmentId: seed.environment.id, + }), + }); + + const where = { friendlyId, runtimeEnvironmentId: seed.environment.id }; + const select = { select: { id: true, friendlyId: true, status: true } }; + + // PRIMARY contrast: an unbranded writer client → readYourWrites true → findRunOnPrimary → the + // run IS seen. Proves the run genuinely exists and any miss below is purely replica lag. + const okReplica = laggingReplica(prisma14, [{ model: "taskRun", mode: "missing" }]); + const primaryRouter = buildRouter(prisma14, prisma17, okReplica.client); + const viaPrimary = (await primaryRouter.findRun(where, select, prisma14 as never)) as { + id: string; + } | null; + expect(viaPrimary?.id).toBe(runId); + + // ACTUAL caller behavior: pass the BRANDED $replica handle → owning REPLICA (frozen). + const lagReplica = laggingReplica(prisma14, [{ model: "taskRun", mode: "missing" }]); + const $replica = markReadReplicaClient(lagReplica.client); + const router = buildRouter(prisma14, prisma17, lagReplica.client); + const pgRow = (await router.findRun(where, select, $replica as never)) as { + id: string; + } | null; + + expect(lagReplica.wasHit("taskRun")).toBe(true); + expect(pgRow).toBeNull(); + } + ); + + // ApiRunResultPresenter findRun (no client) — GET /runs/:runId/result poll (SDK waitForRun result). + // No client → owning REPLICA. Under lag the run is invisible → findRun null → the presenter returns + // undefined, which the SDK treats as "not finished yet, keep polling". A run that has produced a + // result committed that write well before the poll, so the transient undefined only lengthens the + // poll by one tick. Read-only. + heteroRunOpsPostgresTest( + "ApiRunResultPresenter findRun (no client) returns null for a run missed on the frozen replica (result poll retries)", + async ({ prisma14, prisma17 }) => { + const seed = await seedEnvironment(prisma14, "result_leg"); + const runId = `run_${CUID_25}`; + const friendlyId = `run_${FRIENDLY_25}`; + await prisma14.taskRun.create({ + data: taskRunData({ + id: runId, + friendlyId, + organizationId: seed.organization.id, + projectId: seed.project.id, + runtimeEnvironmentId: seed.environment.id, + status: "COMPLETED_SUCCESSFULLY", + }), + }); + + const where = { friendlyId, runtimeEnvironmentId: seed.environment.id }; + // Exact caller shape: an `include` of attempts. Under "missing" mode the replica returns null + // regardless of the projection; routing depends only on where + (absent) client. + const args = { include: { attempts: { orderBy: { createdAt: "desc" as const } } } }; + + const okReplica = laggingReplica(prisma14, [{ model: "taskRun", mode: "missing" }]); + const primaryRouter = buildRouter(prisma14, prisma17, okReplica.client); + const viaPrimary = (await primaryRouter.findRun(where, args, prisma14 as never)) as { + id: string; + } | null; + expect(viaPrimary?.id).toBe(runId); + + const lagReplica = laggingReplica(prisma14, [{ model: "taskRun", mode: "missing" }]); + const router = buildRouter(prisma14, prisma17, lagReplica.client); + // Invoke EXACTLY as the caller does: findRun(where, { include }) — NO client argument. + const taskRun = (await router.findRun(where, args)) as { id: string } | null; + + expect(lagReplica.wasHit("taskRun")).toBe(true); + expect(taskRun).toBeNull(); + // Presenter maps null → undefined: "not finished yet, keep polling". + expect(taskRun ?? undefined).toBeUndefined(); + } + ); + + // RunPresenter findRun (no client) — dashboard run-detail page loader. No client → owning REPLICA. + // Under lag the fresh run is invisible → findRun null → the presenter throws RunNotInPgError, which + // the route CATCHES to fall back to the synthesised mollifier-buffer view. That buffer holds exactly + // the just-triggered runs whose rows have not yet drained/replicated — the read-your-writes safety net + // for this window; the page self-heals on the next poll once the replica catches up. + heteroRunOpsPostgresTest( + "RunPresenter findRun (no client) returns null for a run missed on the frozen replica (mollifier-buffer fallback)", + async ({ prisma14, prisma17 }) => { + const seed = await seedEnvironment(prisma14, "detail_leg"); + const runId = `run_${CUID_25}`; + const friendlyId = `run_${FRIENDLY_25}`; + await prisma14.taskRun.create({ + data: taskRunData({ + id: runId, + friendlyId, + organizationId: seed.organization.id, + projectId: seed.project.id, + runtimeEnvironmentId: seed.environment.id, + }), + }); + + const where = { friendlyId }; + const select = { + select: { + id: true, + projectId: true, + friendlyId: true, + status: true, + runtimeEnvironmentId: true, + }, + }; + + const okReplica = laggingReplica(prisma14, [{ model: "taskRun", mode: "missing" }]); + const primaryRouter = buildRouter(prisma14, prisma17, okReplica.client); + const viaPrimary = (await primaryRouter.findRun(where, select, prisma14 as never)) as { + id: string; + } | null; + expect(viaPrimary?.id).toBe(runId); + + const lagReplica = laggingReplica(prisma14, [{ model: "taskRun", mode: "missing" }]); + const router = buildRouter(prisma14, prisma17, lagReplica.client); + const run = (await router.findRun(where, select)) as { id: string } | null; + + expect(lagReplica.wasHit("taskRun")).toBe(true); + expect(run).toBeNull(); // → presenter throws RunNotInPgError → route's mollifier-buffer fallback + } + ); + + // RunStreamPresenter findRun (no client) — run-detail SSE trace stream loader. No client → owning + // REPLICA. Under lag the fresh run is invisible → findRun null → traceId stays null and the presenter + // falls back to the mollifier buffer for a traceId; if still unresolved it 404s, but the SSE loader + // RECONNECTS (closing with 404 forces the dashboard to keep retrying). Worst case is a reconnect one + // tick later — the stream attaches as soon as the replica (or buffer) yields the traceId. Read-only. + heteroRunOpsPostgresTest( + "RunStreamPresenter findRun (no client) returns null for a run missed on the frozen replica (buffer/404-then-reconnect)", + async ({ prisma14, prisma17 }) => { + const seed = await seedEnvironment(prisma14, "stream_leg"); + const runId = `run_${CUID_25}`; + const friendlyId = `run_${FRIENDLY_25}`; + await prisma14.taskRun.create({ + data: taskRunData({ + id: runId, + friendlyId, + organizationId: seed.organization.id, + projectId: seed.project.id, + runtimeEnvironmentId: seed.environment.id, + }), + }); + + const where = { friendlyId }; + const select = { select: { traceId: true, projectId: true } }; + + const okReplica = laggingReplica(prisma14, [{ model: "taskRun", mode: "missing" }]); + const primaryRouter = buildRouter(prisma14, prisma17, okReplica.client); + const viaPrimary = (await primaryRouter.findRun(where, select, prisma14 as never)) as { + traceId: string; + } | null; + expect(viaPrimary?.traceId).toBe(`trace_${runId}`); + + const lagReplica = laggingReplica(prisma14, [{ model: "taskRun", mode: "missing" }]); + const router = buildRouter(prisma14, prisma17, lagReplica.client); + const run = (await router.findRun(where, select)) as { traceId: string } | null; + + expect(lagReplica.wasHit("taskRun")).toBe(true); + expect(run).toBeNull(); // traceId stays null → buffer fallback → 404-then-SSE-reconnect + } + ); + + // PlaygroundPresenter findRuns (no client) — agent-playground conversation list. Reads conversations + // off $replica, then resolves each conversation's backing run scalars via a client-less findRuns over + // the id set → owning REPLICA (bounded id-set path). Under lag a run missing on the replica is absent + // from runsById, so that conversation renders runFriendlyId=null / runStatus=null / isActive=false — + // a cosmetic "status unknown" on one list row that self-heals on the next list load; the row itself + // still renders. + heteroRunOpsPostgresTest( + "PlaygroundPresenter findRuns (no client) omits a run missed on the frozen replica (null status on that row)", + async ({ prisma14, prisma17 }) => { + const seed = await seedEnvironment(prisma14, "playground_leg"); + const runId = `run_${CUID_25}`; + const friendlyId = `run_${FRIENDLY_25}`; + await prisma14.taskRun.create({ + data: taskRunData({ + id: runId, + friendlyId, + organizationId: seed.organization.id, + projectId: seed.project.id, + runtimeEnvironmentId: seed.environment.id, + status: "EXECUTING", + }), + }); + + const findRunsArgs = { + where: { id: { in: [runId] } }, + select: { id: true, friendlyId: true, status: true }, + }; + + const okReplica = laggingReplica(prisma14, [{ model: "taskRun", mode: "missing" }]); + const primaryRouter = buildRouter(prisma14, prisma17, okReplica.client); + const viaPrimary = (await primaryRouter.findRuns(findRunsArgs, prisma14 as never)) as Array<{ + id: string; + }>; + expect(viaPrimary.map((r) => r.id)).toEqual([runId]); + + const lagReplica = laggingReplica(prisma14, [{ model: "taskRun", mode: "missing" }]); + const router = buildRouter(prisma14, prisma17, lagReplica.client); + // Invoke EXACTLY as the caller does: findRuns({ where:{id:{in}}, select }) — NO client argument. + const runs = (await router.findRuns(findRunsArgs)) as Array<{ id: string }>; + + expect(lagReplica.wasHit("taskRun")).toBe(true); + expect(runs).toEqual([]); // run absent → conversation row shows null friendlyId/status + } + ); + + // SpanPresenter findRun (+this._replica) — span-detail (run inspector) panel. Passes the BRANDED + // this._replica → owning REPLICA (no escalation). Under lag the fresh run is invisible → findRun null + // → the panel renders its not-found/loading state for this poll and self-heals on the next tick. + // Read-only. (The alternate `{spanId, runtimeEnvironmentId}` branch is unclassifiable → fans + // NEW→LEGACY, still each store's REPLICA — same replica-served conclusion.) + heteroRunOpsPostgresTest( + "SpanPresenter findRun (+this._replica) returns null for a run missed on the frozen replica (span-detail not-found this poll)", + async ({ prisma14, prisma17 }) => { + const seed = await seedEnvironment(prisma14, "span_leg"); + const runId = `run_${CUID_25}`; + const friendlyId = `run_${FRIENDLY_25}`; + await prisma14.taskRun.create({ + data: taskRunData({ + id: runId, + friendlyId, + organizationId: seed.organization.id, + projectId: seed.project.id, + runtimeEnvironmentId: seed.environment.id, + }), + }); + + // originalRunId branch: where = {friendlyId, runtimeEnvironmentId}. Representative select subset + // (the full caller select is ~60 scalar/relation fields; routing depends only on where + client, + // and "missing" mode ignores the projection). + const where = { friendlyId, runtimeEnvironmentId: seed.environment.id }; + const select = { select: { id: true, friendlyId: true, status: true, spanId: true } }; + + const okReplica = laggingReplica(prisma14, [{ model: "taskRun", mode: "missing" }]); + const primaryRouter = buildRouter(prisma14, prisma17, okReplica.client); + const viaPrimary = (await primaryRouter.findRun(where, select, prisma14 as never)) as { + id: string; + } | null; + expect(viaPrimary?.id).toBe(runId); + + const lagReplica = laggingReplica(prisma14, [{ model: "taskRun", mode: "missing" }]); + const _replica = markReadReplicaClient(lagReplica.client); + const router = buildRouter(prisma14, prisma17, lagReplica.client); + const run = (await router.findRun(where, select, _replica as never)) as { id: string } | null; + + expect(lagReplica.wasHit("taskRun")).toBe(true); + expect(run).toBeNull(); + } + ); + + // SpanPresenter findRuns {parentSpanId} (+this._replica) — the "triggered runs" list on a span-detail + // panel. Passes the BRANDED this._replica; the where is an OPEN predicate ({parentSpanId}) with no id + // set, so it queries BOTH stores' REPLICAS and dedupes. Under lag a just-triggered child run is not + // yet on the replica, so it is absent from the triggered-runs list for this render — a cosmetic "one + // fewer child shown" that self-heals on the next poll; the child still exists and executes. + heteroRunOpsPostgresTest( + "SpanPresenter findRuns {parentSpanId} (+this._replica) omits a child run missed on the frozen replica (triggered-runs list)", + async ({ prisma14, prisma17 }) => { + const seed = await seedEnvironment(prisma14, "triggered_leg"); + const parentSpanId = "span_parent_fixed"; + const childRunId = `run_${CUID_25}`; + const childFriendlyId = `run_${FRIENDLY_25}`; + await prisma14.taskRun.create({ + data: taskRunData({ + id: childRunId, + friendlyId: childFriendlyId, + organizationId: seed.organization.id, + projectId: seed.project.id, + runtimeEnvironmentId: seed.environment.id, + spanId: "span_child_fixed", + parentSpanId, + }), + }); + + const findRunsArgs = { + where: { parentSpanId }, + select: { + friendlyId: true, + taskIdentifier: true, + spanId: true, + createdAt: true, + status: true, + }, + }; + + // PRIMARY contrast: unbranded writer → each leg reads its primary → child run IS listed. + const okReplica = laggingReplica(prisma14, [{ model: "taskRun", mode: "missing" }]); + const primaryRouter = buildRouter(prisma14, prisma17, okReplica.client); + const viaPrimary = (await primaryRouter.findRuns(findRunsArgs, prisma14 as never)) as Array<{ + friendlyId: string; + }>; + expect(viaPrimary.map((r) => r.friendlyId)).toEqual([childFriendlyId]); + + const lagReplica = laggingReplica(prisma14, [{ model: "taskRun", mode: "missing" }]); + const _replica = markReadReplicaClient(lagReplica.client); + const router = buildRouter(prisma14, prisma17, lagReplica.client); + const triggeredRuns = (await router.findRuns(findRunsArgs, _replica as never)) as Array<{ + friendlyId: string; + }>; + + expect(lagReplica.wasHit("taskRun")).toBe(true); + expect(triggeredRuns).toEqual([]); // child omitted from the triggered-runs list this render + } + ); +}); diff --git a/internal-packages/run-store/src/runOpsStore.presentersSessionBatchReadView.replicaLag.test.ts b/internal-packages/run-store/src/runOpsStore.presentersSessionBatchReadView.replicaLag.test.ts new file mode 100644 index 00000000000..dbd1b59bdb3 --- /dev/null +++ b/internal-packages/run-store/src/runOpsStore.presentersSessionBatchReadView.replicaLag.test.ts @@ -0,0 +1,344 @@ +// Property: the four run-store reads issued by the AI-session + batch DASHBOARD presenters behave +// correctly under replica lag. Each passes the webapp's BRANDED `$replica` (markReadReplicaClient) into +// the run store; a branded replica makes the routing store's `#ownPrimary(store, client)` return +// undefined, so every leg reads the OWNING store's REPLICA (no primary escalation). Under lag each read +// misses a row present on the owning primary. Real split topology via heteroRunOpsPostgresTest; the +// branded arg is `markReadReplicaClient({})`, whose object is never forwarded across DBs — only its +// BRAND is read, exactly as in production. +// +// Reads covered (one case each): +// 1. SessionListPresenter findRuns (id-set fan-out, each leg owning REPLICA). Consumer builds +// `runById` and emits `currentRunFriendlyId`. A miss drops the row → the link is simply undefined, +// presenter returns normally. Display-view omission on an env-scoped list that revalidates. +// Tolerated. +// 2. SessionPresenter findRuns (id-set fan-out, each leg owning REPLICA). Consumer maps each +// sessionRun to `run: run ? {...} : null`. A miss yields `run: null` for that history row — the +// detail page still renders. Display GET view, no throw. Tolerated. +// 3. SessionPresenter findRun currentRun fallback (id-classified → owning REPLICA, then fan-out). +// The `runsById.get(currentRunId) ?? findRun(...)` fallback. A miss yields `currentRun: null` — +// the detail page renders with no current run highlighted until revalidation. Tolerated. +// 4. BatchPresenter findBatchTaskRunByFriendlyId (env-scoped friendlyId fan-out, each leg owning +// REPLICA). findBatchTaskRunByFriendlyId defaults to `this.readOnlyPrisma` — the lone batch-family +// read that defaults to the replica; findBatchTaskRunById and findBatchTaskRunByIdempotencyKey +// default to `this.prisma` (primary). On a bare null the presenter throws `Error("Batch not found")` +// → a 400 error page. This case pins the desired property: a LIVE batch's detail resolves under lag +// via a primary re-read (recovered below on the owning primary). +// +// Method: build the router as the webapp holds it, seed on the owning LEGACY primary, freeze the owning +// LEGACY replica with the shared laggingReplica, invoke each read with the EXACT caller's args + the +// branded replica client, assert the under-lag behavior, then prove the row is recoverable on the owning +// PRIMARY (unbranded writer → #ownPrimary → owning primary) so the null is purely replica lag + replica +// routing, not a missing row or bad query. + +import { heteroRunOpsPostgresTest, laggingReplica } from "@internal/testcontainers"; +import type { PrismaClient } from "@trigger.dev/database"; +import type { RunOpsPrismaClient } from "@internal/run-ops-database"; +import { describe, expect } from "vitest"; +import { PostgresRunStore } from "./PostgresRunStore.js"; +import { RoutingRunStore } from "./runOpsStore.js"; +import { markReadReplicaClient } from "./readReplicaClient.js"; +import type { CreateRunInput } from "./types.js"; + +// A cuid (25 chars after the id prefix) classifies LEGACY, so create + read both route to the legacy +// (control-plane) store first — the store that owns these session/run/batch rows in production. +const CUID_25 = "c".repeat(25); + +async function seedEnvironmentLegacy(prisma: PrismaClient, suffix: string) { + const organization = await prisma.organization.create({ + data: { title: `Org ${suffix}`, slug: `org-${suffix}` }, + }); + const project = await prisma.project.create({ + data: { + name: `Project ${suffix}`, + slug: `project-${suffix}`, + externalRef: `proj_${suffix}`, + organizationId: organization.id, + }, + }); + const environment = await prisma.runtimeEnvironment.create({ + data: { + type: "DEVELOPMENT", + slug: "dev", + projectId: project.id, + organizationId: organization.id, + apiKey: `tr_dev_${suffix}`, + pkApiKey: `pk_dev_${suffix}`, + shortcode: `short_${suffix}`, + }, + }); + return { organization, project, environment }; +} + +function buildCreateRunInput(params: { + runId: string; + friendlyId: string; + organizationId: string; + projectId: string; + runtimeEnvironmentId: string; +}): CreateRunInput { + return { + data: { + id: params.runId, + engine: "V2", + status: "EXECUTING", + friendlyId: params.friendlyId, + runtimeEnvironmentId: params.runtimeEnvironmentId, + environmentType: "DEVELOPMENT", + organizationId: params.organizationId, + projectId: params.projectId, + taskIdentifier: "my-task", + payload: '{"hello":"world"}', + payloadType: "application/json", + context: { foo: "bar" }, + traceContext: { trace: "ctx" }, + traceId: `trace_${params.runId}`, + spanId: `span_${params.runId}`, + queue: "task/my-task", + isTest: false, + taskEventStore: "taskEvent", + depth: 0, + createdAt: new Date("2024-01-01T00:00:00.000Z"), + }, + snapshot: { + engine: "V2", + executionStatus: "EXECUTING", + description: "Run is executing", + runStatus: "EXECUTING", + environmentId: params.runtimeEnvironmentId, + environmentType: "DEVELOPMENT", + projectId: params.projectId, + organizationId: params.organizationId, + }, + }; +} + +// Build the router the way the webapp holds it, with the LEGACY (control-plane) replica FROZEN for the +// given models (mode "missing" — the just-written row has not replicated). NEW is non-lagging + empty. +function buildRouter( + prisma14: PrismaClient, + prisma17: RunOpsPrismaClient, + frozenModels: readonly ("taskRun" | "batchTaskRun")[] +) { + const legacyReplica = laggingReplica( + prisma14, + frozenModels.map((model) => ({ model, mode: "missing" as const })) + ); + const legacyStore = new PostgresRunStore({ + prisma: prisma14, + readOnlyPrisma: legacyReplica.client, + schemaVariant: "legacy", + }); + const newStore = new PostgresRunStore({ + prisma: prisma17 as never, + readOnlyPrisma: prisma17 as never, + schemaVariant: "dedicated", + }); + const router = new RoutingRunStore({ new: newStore, legacy: legacyStore }); + return { router, legacyStore, legacyReplica }; +} + +describe("session and batch presenter read-views under replica lag", () => { + // SessionListPresenter findRuns (branded $replica) — tolerated. + heteroRunOpsPostgresTest( + "SessionListPresenter findRuns returns empty under replica lag, dropping the currentRunFriendlyId link", + async ({ prisma14, prisma17 }) => { + const { router, legacyStore, legacyReplica } = buildRouter(prisma14, prisma17, ["taskRun"]); + const seed = await seedEnvironmentLegacy(prisma14, "sesslist"); + const runId = `run_${CUID_25}`; // cuid → LEGACY + const friendlyId = "run_sesslist_current"; + await legacyStore.createRun( + buildCreateRunInput({ + runId, + friendlyId, + organizationId: seed.organization.id, + projectId: seed.project.id, + runtimeEnvironmentId: seed.environment.id, + }) + ); + + const brandedReplica = markReadReplicaClient({} as object); + const args = { + where: { + id: { in: [runId] }, + projectId: seed.project.id, + runtimeEnvironmentId: seed.environment.id, + }, + select: { id: true, friendlyId: true }, + } as const; + + // Under lag the owning-replica id-set fan-out misses → empty list. + const underLag = (await router.findRuns(args, brandedReplica)) as Array<{ + id: string; + friendlyId: string; + }>; + expect(underLag).toEqual([]); + expect(legacyReplica.wasHit("taskRun")).toBe(true); + + // TOLERANCE: the consumer's map lookup — a missing row just yields an undefined link, no throw. + const runById = new Map(underLag.map((r) => [r.id, r] as const)); + const currentRunFriendlyId = runById.get(runId)?.friendlyId; + expect(currentRunFriendlyId).toBeUndefined(); + + // ROUTING proof (replica, not primary): the unbranded WRITER forces the owning primary and finds it. + const onPrimary = (await router.findRuns(args, prisma14 as never)) as Array<{ + id: string; + friendlyId: string; + }>; + expect(onPrimary.map((r) => r.id)).toEqual([runId]); + expect(onPrimary[0]?.friendlyId).toBe(friendlyId); + } + ); + + // SessionPresenter findRuns (branded $replica) — tolerated. + heteroRunOpsPostgresTest( + "SessionPresenter findRuns returns empty under replica lag, rendering run:null for the history row", + async ({ prisma14, prisma17 }) => { + const { router, legacyStore, legacyReplica } = buildRouter(prisma14, prisma17, ["taskRun"]); + const seed = await seedEnvironmentLegacy(prisma14, "sesshist"); + const runId = `run_${CUID_25}`; // cuid → LEGACY + const friendlyId = "run_sesshist_row"; + await legacyStore.createRun( + buildCreateRunInput({ + runId, + friendlyId, + organizationId: seed.organization.id, + projectId: seed.project.id, + runtimeEnvironmentId: seed.environment.id, + }) + ); + + const brandedReplica = markReadReplicaClient({} as object); + const args = { + where: { id: { in: [runId] } }, + select: { id: true, friendlyId: true, status: true }, + } as const; + + const underLag = (await router.findRuns(args, brandedReplica)) as Array<{ + id: string; + friendlyId: string; + status: string; + }>; + expect(underLag).toEqual([]); + expect(legacyReplica.wasHit("taskRun")).toBe(true); + + // TOLERANCE: the consumer's mapping — the history row's `run` is null, page still renders. + const runsById = new Map(underLag.map((r) => [r.id, r] as const)); + const hydratedRow = runsById.get(runId); + const rowRun = hydratedRow + ? { friendlyId: hydratedRow.friendlyId, status: hydratedRow.status } + : null; + expect(rowRun).toBeNull(); + + // ROUTING proof: WRITER → owning primary → hydrated. + const onPrimary = (await router.findRuns(args, prisma14 as never)) as Array<{ + id: string; + friendlyId: string; + status: string; + }>; + expect(onPrimary.map((r) => r.id)).toEqual([runId]); + expect(onPrimary[0]?.status).toBe("EXECUTING"); + } + ); + + // SessionPresenter findRun currentRun fallback (branded $replica) — tolerated. + heteroRunOpsPostgresTest( + "SessionPresenter findRun currentRun fallback returns null under replica lag, emitting currentRun:null", + async ({ prisma14, prisma17 }) => { + const { router, legacyStore, legacyReplica } = buildRouter(prisma14, prisma17, ["taskRun"]); + const seed = await seedEnvironmentLegacy(prisma14, "currun"); + const currentRunId = `run_${CUID_25}`; // cuid → LEGACY + const friendlyId = "run_currun"; + await legacyStore.createRun( + buildCreateRunInput({ + runId: currentRunId, + friendlyId, + organizationId: seed.organization.id, + projectId: seed.project.id, + runtimeEnvironmentId: seed.environment.id, + }) + ); + + const brandedReplica = markReadReplicaClient({} as object); + + // The EXACT fallback read: { id }, select {id,friendlyId,status}, branded $replica. + const underLag = (await router.findRun( + { id: currentRunId }, + { select: { id: true, friendlyId: true, status: true } }, + brandedReplica + )) as { id: string; friendlyId: string; status: string } | null; + + // Replica-routed (id-classified → owning legacy replica, then other-store probe) → null under lag. + expect(underLag).toBeNull(); + expect(legacyReplica.wasHit("taskRun")).toBe(true); + + // TOLERANCE: the consumer's emit — currentRun is null, detail page still renders. + const currentRun = underLag + ? { friendlyId: underLag.friendlyId, status: underLag.status } + : null; + expect(currentRun).toBeNull(); + + // ROUTING proof: WRITER → readYourWrites → owning primary (findRunOnPrimary) → found. + const onPrimary = (await router.findRun( + { id: currentRunId }, + { select: { id: true, friendlyId: true, status: true } }, + prisma14 as never + )) as { id: string; friendlyId: string; status: string } | null; + expect(onPrimary).not.toBeNull(); + expect(onPrimary!.id).toBe(currentRunId); + expect(onPrimary!.friendlyId).toBe(friendlyId); + } + ); + + // BatchPresenter findBatchTaskRunByFriendlyId (branded $replica) — recovers via primary re-read. + heteroRunOpsPostgresTest( + "BatchPresenter findBatchTaskRunByFriendlyId resolves a live batch under replica lag", + async ({ prisma14, prisma17 }) => { + const { router, legacyStore, legacyReplica } = buildRouter(prisma14, prisma17, [ + "batchTaskRun", + ]); + const seed = await seedEnvironmentLegacy(prisma14, "batchview"); + const batchId = `batch_${CUID_25}`; // cuid → LEGACY + const batchFriendlyId = "batch_batchview"; + await legacyStore.createBatchTaskRun({ + id: batchId, + friendlyId: batchFriendlyId, + runtimeEnvironmentId: seed.environment.id, + }); + + const brandedReplica = markReadReplicaClient({} as object); + + // The EXACT read: friendlyId + environmentId + include, branded $replica. + const underLag = await router.findBatchTaskRunByFriendlyId( + batchFriendlyId, + seed.environment.id, + { include: { errors: true } }, + brandedReplica as never + ); + + // Store fact: findBatchTaskRunByFriendlyId defaults to `this.readOnlyPrisma`, so through the + // router (branded → #ownPrimary undefined) BOTH legs read their replica. Under lag → null. + expect(underLag).toBeNull(); + expect(legacyReplica.wasHit("batchTaskRun")).toBe(true); + + // On a bare null the presenter's guard throws "Batch not found" (→ a 400 error page from the route + // loader) for a batch ALIVE on the owning primary — reproduced here on the stale null. + expect(() => { + if (!underLag) { + throw new Error("Batch not found"); + } + }).toThrowError("Batch not found"); + + // The property: the row EXISTS on the owning primary — reading it there (unbranded WRITER → + // #ownPrimary → owning primary) returns the live batch, so the primary re-read resolves the detail. + const onPrimary = await router.findBatchTaskRunByFriendlyId( + batchFriendlyId, + seed.environment.id, + { include: { errors: true } }, + prisma14 as never + ); + expect(onPrimary).not.toBeNull(); + expect(onPrimary!.id).toBe(batchId); + expect(onPrimary!.friendlyId).toBe(batchFriendlyId); + } + ); +}); diff --git a/internal-packages/run-store/src/runOpsStore.presentersWaitpointReadView.replicaLag.test.ts b/internal-packages/run-store/src/runOpsStore.presentersWaitpointReadView.replicaLag.test.ts new file mode 100644 index 00000000000..0c7419b353a --- /dev/null +++ b/internal-packages/run-store/src/runOpsStore.presentersWaitpointReadView.replicaLag.test.ts @@ -0,0 +1,516 @@ +// Replica-lag properties for the waitpoint-family dashboard/API read presenters. Every read here is +// served by the OWNING store's REPLICA (no client passed) and backs a display/GET/list view, so each +// tolerates the lag. On the real split topology (heteroRunOpsPostgresTest, never mocked) we freeze the +// owning (LEGACY) replica via a local proxy that also freezes the $queryRaw connected-run lookup (which +// the shared laggingReplica primitive can't intercept), invoke each read EXACTLY as its +// caller does, assert the stale-under-lag value, and prove the row exists on the PRIMARY (so the miss +// is pure lag). A caller-passed primary client flips the read to the owning primary. + +import { heteroRunOpsPostgresTest } from "@internal/testcontainers"; +import type { PrismaClient } from "@trigger.dev/database"; +import type { RunOpsPrismaClient } from "@internal/run-ops-database"; +import { describe, expect } from "vitest"; +import { PostgresRunStore } from "./PostgresRunStore.js"; +import { RoutingRunStore } from "./runOpsStore.js"; + +type AnyClient = PrismaClient | RunOpsPrismaClient; + +const CUID_25 = "e".repeat(25); // cuid id-shape -> LEGACY (#legacy / prisma14, full schema) + +// A recording "replica" that has NOT caught up: `taskRun`, `waitpoint`, and `waitpointTag` MODEL reads +// come back empty, and `$queryRaw`/`$queryRawUnsafe` (the connection-join lookups) come back empty too, +// so any replica-routed read misses the just-written row. Everything else forwards to the real client. +// `wasHit` flips true iff an intercepted read was routed here. Writes always land on the PRIMARY +// (this.prisma), so freezing the readOnly client never affects seeding. +function laggingReplica(real: C): { client: C; wasHit: () => boolean } { + let hit = false; + function wrapModel(target: any) { + return new Proxy(target, { + get(innerTarget, prop) { + if (prop === "findFirst" || prop === "findMany" || prop === "findUnique") { + return async () => { + hit = true; + return prop === "findMany" ? [] : null; + }; + } + if (prop === "findFirstOrThrow" || prop === "findUniqueOrThrow") { + return async () => { + hit = true; + throw new Error("lagging replica: row not visible"); + }; + } + return (innerTarget as any)[prop]; + }, + }); + } + const laggingTaskRun = wrapModel((real as any).taskRun); + const laggingWaitpoint = wrapModel((real as any).waitpoint); + const laggingWaitpointTag = wrapModel((real as any).waitpointTag); + const client = new Proxy(real, { + get(target, prop) { + if (prop === "taskRun") return laggingTaskRun; + if (prop === "waitpoint") return laggingWaitpoint; + if (prop === "waitpointTag") return laggingWaitpointTag; + // The connection-id gather runs a raw JOIN (findWaitpointConnectedRunIds); a lagging replica + // has none of those rows yet, so freeze raw reads to empty as well. + if (prop === "$queryRaw" || prop === "$queryRawUnsafe") { + return async () => { + hit = true; + return []; + }; + } + return (target as any)[prop]; + }, + }) as C; + return { client, wasHit: () => hit }; +} + +async function seedEnvironmentLegacy(prisma: PrismaClient, suffix: string) { + const organization = await prisma.organization.create({ + data: { title: `Org ${suffix}`, slug: `org-${suffix}` }, + }); + const project = await prisma.project.create({ + data: { + name: `Project ${suffix}`, + slug: `project-${suffix}`, + externalRef: `proj_${suffix}`, + organizationId: organization.id, + }, + }); + const environment = await prisma.runtimeEnvironment.create({ + data: { + type: "DEVELOPMENT", + slug: "dev", + projectId: project.id, + organizationId: organization.id, + apiKey: `tr_dev_${suffix}`, + pkApiKey: `pk_dev_${suffix}`, + shortcode: `short_${suffix}`, + }, + }); + return { organization, project, environment }; +} + +// Build a LEGACY-owning router whose legacy replica is frozen (lagging); the NEW store is real +// (non-lagging) so the on-miss fan-out to the other store's replica also legitimately misses. +function buildRouterWithLaggingLegacyReplica(prisma14: PrismaClient, prisma17: RunOpsPrismaClient) { + const legacyReplica = laggingReplica(prisma14); + const legacyStore = new PostgresRunStore({ + prisma: prisma14, + readOnlyPrisma: legacyReplica.client, + schemaVariant: "legacy", + }); + const newStore = new PostgresRunStore({ + prisma: prisma17 as never, + readOnlyPrisma: prisma17 as never, + schemaVariant: "dedicated", + }); + const router = new RoutingRunStore({ new: newStore, legacy: legacyStore }); + return { router, legacyStore, legacyReplica }; +} + +// Seed a standalone MANUAL token waitpoint on the WRITER (exactly as minting a resume token does). +async function seedManualWaitpoint( + store: PostgresRunStore, + params: { + id: string; + friendlyId: string; + projectId: string; + environmentId: string; + tags?: string[]; + } +) { + await store.upsertWaitpoint({ + where: { + environmentId_idempotencyKey: { + environmentId: params.environmentId, + idempotencyKey: params.id, + }, + }, + create: { + id: params.id, + friendlyId: params.friendlyId, + type: "MANUAL", + status: "PENDING", + idempotencyKey: params.id, + userProvidedIdempotencyKey: false, + projectId: params.projectId, + environmentId: params.environmentId, + ...(params.tags ? { tags: params.tags } : {}), + }, + update: {}, + }); +} + +function taskRunData(opts: { + id: string; + friendlyId: string; + organizationId: string; + projectId: string; + runtimeEnvironmentId: string; +}) { + return { + id: opts.id, + engine: "V2" as const, + status: "COMPLETED_SUCCESSFULLY" as const, + friendlyId: opts.friendlyId, + runtimeEnvironmentId: opts.runtimeEnvironmentId, + environmentType: "DEVELOPMENT" as const, + organizationId: opts.organizationId, + projectId: opts.projectId, + taskIdentifier: "my-task", + payload: JSON.stringify({ hello: "world" }), + payloadType: "application/json", + traceContext: {}, + traceId: `trace_${opts.id}`, + spanId: `span_${opts.id}`, + queue: "task/my-task", + isTest: false, + taskEventStore: "taskEvent", + depth: 0, + }; +} + +// ── Exact projections the call sites read ───────────────────────────────────────────────────────── +const API_WAITPOINT_SELECT = { + id: true, + friendlyId: true, + type: true, + status: true, + idempotencyKey: true, + userProvidedIdempotencyKey: true, + idempotencyKeyExpiresAt: true, + inactiveIdempotencyKey: true, + output: true, + outputType: true, + outputIsError: true, + completedAfter: true, + completedAt: true, + createdAt: true, + tags: true, +} as const; + +const LIST_WAITPOINT_SELECT = { + id: true, + friendlyId: true, + status: true, + completedAt: true, + completedAfter: true, + outputIsError: true, + idempotencyKey: true, + idempotencyKeyExpiresAt: true, + inactiveIdempotencyKey: true, + userProvidedIdempotencyKey: true, + tags: true, + createdAt: true, +} as const; + +const DETAIL_WAITPOINT_SELECT = { + id: true, + friendlyId: true, + type: true, + status: true, + idempotencyKey: true, + userProvidedIdempotencyKey: true, + idempotencyKeyExpiresAt: true, + inactiveIdempotencyKey: true, + output: true, + outputType: true, + outputIsError: true, + completedAfter: true, + completedAt: true, + createdAt: true, + tags: true, + environmentId: true, +} as const; + +describe("waitpoint-family dashboard/API read presenters under replica lag", () => { + // ApiWaitpointPresenter (GET retrieve loader). + heteroRunOpsPostgresTest( + "ApiWaitpointPresenter findWaitpoint(id) is null under lag; the owning primary resolves it", + async ({ prisma14, prisma17 }) => { + const { router, legacyStore, legacyReplica } = buildRouterWithLaggingLegacyReplica( + prisma14, + prisma17 + ); + const seed = await seedEnvironmentLegacy(prisma14, "api_wp"); + const waitpointId = `waitpoint_${CUID_25}`; // cuid → LEGACY + await seedManualWaitpoint(legacyStore, { + id: waitpointId, + friendlyId: "waitpoint_api_wp", + projectId: seed.project.id, + environmentId: seed.environment.id, + }); + + // Exact caller read: select the api projection, where {id, environmentId}, NO client → owning replica. + const fromReplica = await router.findWaitpoint({ + where: { id: waitpointId, environmentId: seed.environment.id }, + select: API_WAITPOINT_SELECT, + }); + // Stale: owning replica lags → null. The route's `if (!waitpoint) throw "Waitpoint not found"` + // fires — a transient error on a read-only GET loader; the client re-requests the retrieve. + // This gates NO write (the mint POST + complete path use their own primary reads), so tolerated. + expect(fromReplica).toBeNull(); + expect(legacyReplica.wasHit()).toBe(true); + + // Prove the null is lag, not absence: the same call on the owning PRIMARY resolves the token. + const fromPrimary = await router.findWaitpoint( + { + where: { id: waitpointId, environmentId: seed.environment.id }, + select: API_WAITPOINT_SELECT, + }, + prisma14 + ); + expect(fromPrimary).not.toBeNull(); + expect(fromPrimary!.friendlyId).toBe("waitpoint_api_wp"); + expect(fromPrimary!.type).toBe("MANUAL"); + } + ); + + // WaitpointListPresenter findManyWaitpoints. + heteroRunOpsPostgresTest( + "WaitpointListPresenter findManyWaitpoints is empty under lag; the primary returns the token", + async ({ prisma14, prisma17 }) => { + const { router, legacyStore, legacyReplica } = buildRouterWithLaggingLegacyReplica( + prisma14, + prisma17 + ); + const seed = await seedEnvironmentLegacy(prisma14, "list_many"); + const waitpointId = `waitpoint_${CUID_25}`; + await seedManualWaitpoint(legacyStore, { + id: waitpointId, + friendlyId: "waitpoint_list_many", + projectId: seed.project.id, + environmentId: seed.environment.id, + }); + + // Exact caller read: MANUAL scan, keyset order, over-fetch window, NO client → both replicas. + const fromReplica = await router.findManyWaitpoints({ + where: { environmentId: seed.environment.id, type: "MANUAL" }, + orderBy: { id: "desc" }, + take: 26, + select: LIST_WAITPOINT_SELECT, + }); + // Stale: both replicas empty (new has no row; legacy replica frozen). The list simply omits the + // token this render; the next fetch (once caught up) shows it. No write/decision on the stale set. + expect(fromReplica).toEqual([]); + expect(legacyReplica.wasHit()).toBe(true); + + // Prove pure lag: the same scan on the owning primary returns the token. + const fromPrimary = await router.findManyWaitpoints( + { + where: { environmentId: seed.environment.id, type: "MANUAL" }, + orderBy: { id: "desc" }, + take: 26, + select: LIST_WAITPOINT_SELECT, + }, + prisma14 + ); + expect(fromPrimary).toHaveLength(1); + expect(fromPrimary[0]!.friendlyId).toBe("waitpoint_list_many"); + } + ); + + // WaitpointListPresenter #probeAnyToken findWaitpoint. + heteroRunOpsPostgresTest( + "WaitpointListPresenter probe findWaitpoint(MANUAL) is null under lag; the primary finds the token", + async ({ prisma14, prisma17 }) => { + const { router, legacyStore, legacyReplica } = buildRouterWithLaggingLegacyReplica( + prisma14, + prisma17 + ); + const seed = await seedEnvironmentLegacy(prisma14, "probe_any"); + const waitpointId = `waitpoint_${CUID_25}`; + await seedManualWaitpoint(legacyStore, { + id: waitpointId, + friendlyId: "waitpoint_probe_any", + projectId: seed.project.id, + environmentId: seed.environment.id, + }); + + // Exact caller read: no id, no select — fan NEW-then-LEGACY on each store's replica. + const fromReplica = await router.findWaitpoint({ + where: { environmentId: seed.environment.id, type: "MANUAL" }, + }); + // Stale null → hasAnyTokens=false. The only user-visible effect is which empty-state copy renders + // ("get started" vs "no matches"); it drives no write and self-corrects on the next render. + expect(fromReplica).toBeNull(); + const hasAnyTokensUnderLag = Boolean(fromReplica); + expect(hasAnyTokensUnderLag).toBe(false); + expect(legacyReplica.wasHit()).toBe(true); + + // Prove pure lag: the same probe on the owning primary finds a MANUAL token (hasAnyTokens=true). + const fromPrimary = await router.findWaitpoint( + { where: { environmentId: seed.environment.id, type: "MANUAL" } }, + prisma14 + ); + expect(fromPrimary).not.toBeNull(); + } + ); + + // WaitpointPresenter #findWaitpoint(friendlyId). + heteroRunOpsPostgresTest( + "WaitpointPresenter findWaitpoint(friendlyId) is null under lag; the primary resolves it", + async ({ prisma14, prisma17 }) => { + const { router, legacyStore, legacyReplica } = buildRouterWithLaggingLegacyReplica( + prisma14, + prisma17 + ); + const seed = await seedEnvironmentLegacy(prisma14, "detail_wp"); + const waitpointId = `waitpoint_${CUID_25}`; + const friendlyId = "waitpoint_detail_wp"; + await seedManualWaitpoint(legacyStore, { + id: waitpointId, + friendlyId, + projectId: seed.project.id, + environmentId: seed.environment.id, + }); + + // Exact caller read: where {friendlyId, environmentId} (no id) + detail select, NO client → replicas. + const fromReplica = await router.findWaitpoint({ + where: { friendlyId, environmentId: seed.environment.id }, + select: DETAIL_WAITPOINT_SELECT, + }); + // Stale null → presenter.call logs "Waitpoint not found" and returns null; the detail page shows + // not-found and the user reloads. Read-only render, no mutation gated. Tolerated. + expect(fromReplica).toBeNull(); + expect(legacyReplica.wasHit()).toBe(true); + + const fromPrimary = await router.findWaitpoint( + { + where: { friendlyId, environmentId: seed.environment.id }, + select: DETAIL_WAITPOINT_SELECT, + }, + prisma14 + ); + expect(fromPrimary).not.toBeNull(); + expect(fromPrimary!.environmentId).toBe(seed.environment.id); + } + ); + + // WaitpointPresenter findWaitpointConnectedRunIds. + heteroRunOpsPostgresTest( + "WaitpointPresenter findWaitpointConnectedRunIds is empty under lag; the primary returns the connection", + async ({ prisma14, prisma17 }) => { + const { router, legacyStore, legacyReplica } = buildRouterWithLaggingLegacyReplica( + prisma14, + prisma17 + ); + const seed = await seedEnvironmentLegacy(prisma14, "conn_ids"); + const waitpointId = `waitpoint_${CUID_25}`; + const runId = `run_${CUID_25}`; // cuid → LEGACY, co-located with the token + await seedManualWaitpoint(legacyStore, { + id: waitpointId, + friendlyId: "waitpoint_conn_ids", + projectId: seed.project.id, + environmentId: seed.environment.id, + }); + await prisma14.taskRun.create({ + data: taskRunData({ + id: runId, + friendlyId: "run_conn_ids", + organizationId: seed.organization.id, + projectId: seed.project.id, + runtimeEnvironmentId: seed.environment.id, + }), + }); + // The run↔waitpoint join is written on the run's DB (blockRunWithWaitpointEdges, routed by runId). + await router.blockRunWithWaitpointEdges({ + runId, + waitpointIds: [waitpointId], + projectId: seed.project.id, + }); + + // Exact caller read: fan the connection JOIN across both stores' replicas, NO client. + const fromReplica = await router.findWaitpointConnectedRunIds(waitpointId); + // Stale: both replicas' join reads empty (new has none; legacy replica raw frozen). The detail + // page shows an empty/partial "connected runs" list; a refresh fills it. Pure display. + expect(fromReplica).toEqual([]); + expect(legacyReplica.wasHit()).toBe(true); + + // Prove pure lag: the same fan-out on the owning primary returns the connected run id. + const fromPrimary = await router.findWaitpointConnectedRunIds(waitpointId, prisma14); + expect(fromPrimary).toEqual([runId]); + } + ); + + // WaitpointPresenter findRuns({id:{in}}). + heteroRunOpsPostgresTest( + "WaitpointPresenter findRuns(id in) is empty under lag; the primary returns the run", + async ({ prisma14, prisma17 }) => { + const { router, legacyReplica } = buildRouterWithLaggingLegacyReplica(prisma14, prisma17); + const seed = await seedEnvironmentLegacy(prisma14, "conn_runs"); + const runId = `run_${CUID_25}`; // cuid → LEGACY + await prisma14.taskRun.create({ + data: taskRunData({ + id: runId, + friendlyId: "run_conn_runs", + organizationId: seed.organization.id, + projectId: seed.project.id, + runtimeEnvironmentId: seed.environment.id, + }), + }); + + // Exact caller read: id-set + friendlyId projection + take, NO client → owning replica(s). + const fromReplica = (await router.findRuns({ + where: { id: { in: [runId] } }, + select: { friendlyId: true }, + take: 5, + })) as Array<{ friendlyId: string }>; + // Stale: owning replica lags → []; the detail page's connected-runs block renders nothing this + // pass. The set is only mapped to friendlyIds for display — no decision on the stale result. + expect(fromReplica).toEqual([]); + expect(legacyReplica.wasHit()).toBe(true); + + // Prove pure lag: the same query on the owning primary returns the run's friendlyId. + const fromPrimary = (await router.findRuns( + { where: { id: { in: [runId] } }, select: { friendlyId: true }, take: 5 }, + prisma14 + )) as Array<{ friendlyId: string }>; + expect(fromPrimary).toHaveLength(1); + expect(fromPrimary[0]!.friendlyId).toBe("run_conn_runs"); + } + ); + + // WaitpointTagListPresenter findManyWaitpointTags. + heteroRunOpsPostgresTest( + "WaitpointTagListPresenter findManyWaitpointTags is empty under lag; the primary returns the tag", + async ({ prisma14, prisma17 }) => { + const { router, legacyStore, legacyReplica } = buildRouterWithLaggingLegacyReplica( + prisma14, + prisma17 + ); + const seed = await seedEnvironmentLegacy(prisma14, "tag_list"); + await legacyStore.upsertWaitpointTag({ + environmentId: seed.environment.id, + name: "my-tag", + projectId: seed.project.id, + }); + + // Exact caller read: env + optional name filter, orderBy id desc, over-fetch window, NO client. + const fromReplica = await router.findManyWaitpointTags({ + where: { environmentId: seed.environment.id, name: undefined }, + orderBy: { id: "desc" }, + take: 26, + skip: 0, + }); + // Stale: both replicas' waitpointTag reads empty. The tag filter dropdown omits the new tag this + // render; it appears once the replica catches up. No write/decision. + expect(fromReplica).toEqual([]); + expect(legacyReplica.wasHit()).toBe(true); + + // Prove pure lag: the same query on the owning primary returns the tag. + const fromPrimary = await router.findManyWaitpointTags( + { + where: { environmentId: seed.environment.id, name: undefined }, + orderBy: { id: "desc" }, + take: 26, + skip: 0, + }, + prisma14 + ); + expect(fromPrimary).toHaveLength(1); + expect(fromPrimary[0]!.name).toBe("my-tag"); + } + ); +}); diff --git a/internal-packages/run-store/src/runOpsStore.realtimeServicesReadView.replicaLag.test.ts b/internal-packages/run-store/src/runOpsStore.realtimeServicesReadView.replicaLag.test.ts new file mode 100644 index 00000000000..9a3b19e2cc2 --- /dev/null +++ b/internal-packages/run-store/src/runOpsStore.realtimeServicesReadView.replicaLag.test.ts @@ -0,0 +1,356 @@ +// Lagging-replica coverage for the REALTIME-SERVICES read views on the run-ops split. Five run lookups +// feed realtime session/feed serialization; none is a read-your-writes mutation gate — each resolves an +// id/friendlyId for DISPLAY and tolerates a stale/missing read by construction. This file freezes the +// OWNING store's replica via laggingReplica, invokes each read EXACTLY as its caller does (same method + +// client arg), and asserts the concrete under-lag value plus a primary re-read that recovers the row (so +// the null/empty is provably lag-induced). No-client and branded-$replica reads both stay on the owning +// replica; a writer/tx read escalates to the primary. Real split topology via heteroRunOpsPostgresTest — NEVER mocked. + +import { heteroRunOpsPostgresTest, laggingReplica } from "@internal/testcontainers"; +import type { PrismaClient } from "@trigger.dev/database"; +import type { RunOpsPrismaClient } from "@internal/run-ops-database"; +import { describe, expect } from "vitest"; +import { PostgresRunStore } from "./PostgresRunStore.js"; +import { RoutingRunStore } from "./runOpsStore.js"; +import { markReadReplicaClient } from "./readReplicaClient.js"; +import type { CreateRunInput } from "./types.js"; + +// ownerEngine (classifyResidency) routes a run-ops v1 body → NEW, everything else → LEGACY. +// These sites all resolve pre-existing session/feed runs; LEGACY-resident cuid runs exercise the +// owning-store-first → other-store fan-out on the control-plane DB, the common realtime residency. +const CUID_25 = "c".repeat(25); // → LEGACY (#legacy / prisma14, full schema) + +async function seedEnvironment(prisma: PrismaClient, slugSuffix: string) { + const organization = await prisma.organization.create({ + data: { title: `Org ${slugSuffix}`, slug: `org-${slugSuffix}` }, + }); + const project = await prisma.project.create({ + data: { + name: `Project ${slugSuffix}`, + slug: `project-${slugSuffix}`, + externalRef: `proj_${slugSuffix}`, + organizationId: organization.id, + }, + }); + const environment = await prisma.runtimeEnvironment.create({ + data: { + type: "DEVELOPMENT", + slug: "dev", + projectId: project.id, + organizationId: organization.id, + apiKey: `tr_dev_${slugSuffix}`, + pkApiKey: `pk_dev_${slugSuffix}`, + shortcode: `short_${slugSuffix}`, + }, + }); + return { organization, project, environment }; +} + +function buildCreateRunInput(params: { + runId: string; + friendlyId: string; + taskIdentifier: string; + organizationId: string; + projectId: string; + runtimeEnvironmentId: string; +}): CreateRunInput { + return { + data: { + id: params.runId, + engine: "V2", + status: "PENDING", + friendlyId: params.friendlyId, + runtimeEnvironmentId: params.runtimeEnvironmentId, + environmentType: "DEVELOPMENT", + organizationId: params.organizationId, + projectId: params.projectId, + taskIdentifier: params.taskIdentifier, + payload: '{"hello":"world"}', + payloadType: "application/json", + context: { foo: "bar" }, + traceContext: { trace: "ctx" }, + traceId: "trace_1", + spanId: "span_1", + runTags: ["alpha", "beta"], + queue: "task/my-task", + isTest: false, + taskEventStore: "taskEvent", + depth: 0, + createdAt: new Date("2024-01-01T00:00:00.000Z"), + }, + snapshot: { + engine: "V2", + executionStatus: "RUN_CREATED", + description: "Run was created", + runStatus: "PENDING", + environmentId: params.runtimeEnvironmentId, + environmentType: "DEVELOPMENT", + projectId: params.projectId, + organizationId: params.organizationId, + }, + }; +} + +// Build the router the way the realtime services hold it: a LEGACY store on the control-plane DB +// whose replica LAGS, plus a fresh NEW store. Returns the router + the seeded run + the lagging +// replica probe. All five sites resolve a LEGACY-resident (cuid) run, so the owning store is LEGACY +// and its frozen replica is the one the read hits. +async function setupLaggingLegacy( + prisma14: PrismaClient, + prisma17: RunOpsPrismaClient, + slug: string +) { + const legacyReplica = laggingReplica(prisma14, [{ model: "taskRun", mode: "missing" }]); + const legacyStore = new PostgresRunStore({ + prisma: prisma14, + readOnlyPrisma: legacyReplica.client, + schemaVariant: "legacy", + }); + const newStore = new PostgresRunStore({ + prisma: prisma17 as never, + readOnlyPrisma: prisma17 as never, + schemaVariant: "dedicated", + }); + const router = new RoutingRunStore({ new: newStore, legacy: legacyStore }); + + const seed = await seedEnvironment(prisma14, slug); + const runId = `run_${CUID_25}`; // cuid → LEGACY + const friendlyId = `run_${slug}`; + await legacyStore.createRun( + buildCreateRunInput({ + runId, + friendlyId, + taskIdentifier: "my-task", + organizationId: seed.organization.id, + projectId: seed.project.id, + runtimeEnvironmentId: seed.environment.id, + }) + ); + + return { router, legacyReplica, seed, runId, friendlyId }; +} + +// The webapp always passes its $replica handle, which is markReadReplicaClient()'d. +// A branded client tells RoutingRunStore "do not escalate to primary" — so the read stays on the +// owning store's own replica. We pass a branded stand-in exactly as the hydrator/session manager do; +// its identity is irrelevant because the router discards it and reads the owning store's readOnlyPrisma. +const BRANDED_REPLICA = markReadReplicaClient({}) as never; + +describe("run-ops split — realtime-services read views vs. a lagging replica", () => { + // --- hydrateByIds findRuns (runReader) --------------------------------------------------------- + // The realtime feed hydrates a ClickHouse-resolved id-set for DISPLAY, passing $replica. Under lag + // the owning replica returns no rows. Tolerated: the id-set is sourced from ClickHouse, which lags + // the Postgres replica by MORE (an id can't appear in CH before it is on the PG replica), and wake + // hydrates are additionally held by the envChangeRouter replica-lag gate; a row missed on one + // hydrate tick reappears on the next. hydrateByIds returns whatever rows exist — a missing row is + // simply absent from the feed frame, never a wrong value. Store fact under lag: []. + heteroRunOpsPostgresTest( + "hydrateByIds findRuns(branded $replica) is empty under owning-replica lag; primary re-read hydrates the row", + async ({ prisma14, prisma17 }) => { + const { router, legacyReplica, seed, runId, friendlyId } = await setupLaggingLegacy( + prisma14, + prisma17, + "rr_hydrate" + ); + + // Exactly the hydrateByIds read — where {runtimeEnvironmentId, id:{in}}, select, $replica. + const staleRows = await router.findRuns( + { + where: { + runtimeEnvironmentId: seed.environment.id, + id: { in: [runId] }, + }, + select: { id: true, friendlyId: true, status: true }, + }, + BRANDED_REPLICA + ); + expect(staleRows).toEqual([]); + expect(legacyReplica.wasHit()).toBe(true); + + // The row IS on the primary — a writer-client read (escalates to owning primary) hydrates it. + const primaryRows = await router.findRuns( + { + where: { + runtimeEnvironmentId: seed.environment.id, + id: { in: [runId] }, + }, + select: { id: true, friendlyId: true, status: true }, + }, + prisma14 as never + ); + expect(primaryRows).toHaveLength(1); + expect(primaryRows[0]!.friendlyId).toBe(friendlyId); + } + ); + + // --- #fetch / getRunById findRun (runReader) --------------------------------------------------- + // Single-run hydrate for the feed, passing $replica. Under lag the owning replica (then the + // cross-store fan-out on miss) returns null. Tolerated: RunHydror.getRunById caches a null hit for + // only cacheTtlMs (250ms default) and the feed re-fetches; #fetch returns (run ?? null) — a null is + // "not yet visible", rendered as an absent frame in a read-only display feed, never a decision. + // Store fact under lag: null. + heteroRunOpsPostgresTest( + "#fetch findRun(branded $replica) is null under owning-replica lag; primary re-read finds the row", + async ({ prisma14, prisma17 }) => { + const { router, legacyReplica, seed, runId, friendlyId } = await setupLaggingLegacy( + prisma14, + prisma17, + "rr_fetch" + ); + + // Exactly the #fetch read — where {id, runtimeEnvironmentId}, select, $replica. + const stale = await router.findRun( + { id: runId, runtimeEnvironmentId: seed.environment.id }, + { select: { id: true, friendlyId: true, status: true } }, + BRANDED_REPLICA + ); + expect(stale).toBeNull(); + expect(legacyReplica.wasHit()).toBe(true); + + const primary = await router.findRun( + { id: runId, runtimeEnvironmentId: seed.environment.id }, + { select: { id: true, friendlyId: true, status: true } }, + prisma14 as never + ); + expect(primary).not.toBeNull(); + expect((primary as { friendlyId: string }).friendlyId).toBe(friendlyId); + } + ); + + // --- resolveRunFriendlyId findRun (sessionRunManager) ------------------------------------------ + // Resolves a run cuid → friendlyId for `payload.previousRunId`, passing $replica. Under lag the + // read is null. Tolerated: the caller is `return row?.friendlyId ?? runId` — on a null it falls back + // to the cuid, and previousRunId is customer-visible bookkeeping only, so a stale-but-non-null value + // is acceptable degraded behavior. Store fact under lag: null → caller returns the cuid. + heteroRunOpsPostgresTest( + "resolveRunFriendlyId findRun(branded $replica) is null under lag; caller falls back to the cuid", + async ({ prisma14, prisma17 }) => { + const { router, legacyReplica, runId, friendlyId } = await setupLaggingLegacy( + prisma14, + prisma17, + "srm_resolve" + ); + + // Exactly the resolveRunFriendlyId read — where {id}, select {friendlyId}, $replica. + const stale = await router.findRun( + { id: runId }, + { select: { friendlyId: true } }, + BRANDED_REPLICA + ); + expect(stale).toBeNull(); + expect(legacyReplica.wasHit()).toBe(true); + + // The caller's tolerance, exercised: `row?.friendlyId ?? runId`. + const resolved = (stale as { friendlyId: string } | null)?.friendlyId ?? runId; + expect(resolved).toBe(runId); // degraded: the cuid, not the friendlyId + + // Proof the row exists on the primary (writer read would have returned the friendlyId). + const primary = await router.findRun( + { id: runId }, + { select: { friendlyId: true } }, + prisma14 as never + ); + expect((primary as { friendlyId: string }).friendlyId).toBe(friendlyId); + } + ); + + // --- serializeSessionWithFriendlyRunId findRun (sessions) -------------------------------------- + // Resolves Session.currentRunId (internal cuid) → friendlyId for single-row session responses, + // NO client → owning replica. Under lag the read is null. Tolerated: serialized on GET/PATCH/close + // routes over a PRE-EXISTING currentRunId (the PATCH never writes currentRunId — the pointer was + // written by an earlier append), so this is a read-only display resolve, not a same-request + // read-your-writes. The caller is `currentRunId: run?.friendlyId ?? null`: null is the SAFE degraded + // direction, and the tenant-scoped where {projectId, runtimeEnvironmentId} guarantees a stale read + // never mis-resolves a run in another env. The client re-fetches. Store fact under lag: null. + heteroRunOpsPostgresTest( + "serializeSessionWithFriendlyRunId findRun(no client) is null under lag → currentRunId null", + async ({ prisma14, prisma17 }) => { + const { router, legacyReplica, seed, runId, friendlyId } = await setupLaggingLegacy( + prisma14, + prisma17, + "sess_one" + ); + + // Exactly the serializeSessionWithFriendlyRunId read — where {id, projectId, runtimeEnvironmentId}, select, NO client. + const stale = await router.findRun( + { + id: runId, + projectId: seed.project.id, + runtimeEnvironmentId: seed.environment.id, + }, + { select: { friendlyId: true } } + ); + expect(stale).toBeNull(); + expect(legacyReplica.wasHit()).toBe(true); + + // Caller tolerance, exercised. + const currentRunId = (stale as { friendlyId: string } | null)?.friendlyId ?? null; + expect(currentRunId).toBeNull(); + + // Proof the row exists on the primary. + const primary = await router.findRun( + { + id: runId, + projectId: seed.project.id, + runtimeEnvironmentId: seed.environment.id, + }, + { select: { friendlyId: true } }, + prisma14 as never + ); + expect((primary as { friendlyId: string }).friendlyId).toBe(friendlyId); + } + ); + + // --- serializeSessionsWithFriendlyRunIds findRuns (sessions) ----------------------------------- + // Batched currentRunId → friendlyId resolve for the session LIST endpoint, NO client → owning + // replica. Under lag the id-set read returns no rows. Tolerated: same read-only display resolve of + // pre-existing pointers as the single-session case above, batched. The caller is + // `friendlyIdByRunId.get(session.currentRunId) ?? null` over a Map built only from rows that came + // back, so a missing run serializes currentRunId=null — the safe degraded direction. The where is + // tenant-scoped {projectId, runtimeEnvironmentId}, so a stale id never resolves a run in another env. + // Store fact under lag: [] → empty map → currentRunId null. + heteroRunOpsPostgresTest( + "serializeSessionsWithFriendlyRunIds findRuns(no client) is empty under lag → currentRunId null", + async ({ prisma14, prisma17 }) => { + const { router, legacyReplica, seed, runId, friendlyId } = await setupLaggingLegacy( + prisma14, + prisma17, + "sess_list" + ); + + // Exactly the serializeSessionsWithFriendlyRunIds read — where {id:{in}, projectId, runtimeEnvironmentId}, select, NO client. + const staleRuns = await router.findRuns({ + where: { + id: { in: [runId] }, + projectId: seed.project.id, + runtimeEnvironmentId: seed.environment.id, + }, + select: { id: true, friendlyId: true }, + }); + expect(staleRuns).toEqual([]); + expect(legacyReplica.wasHit()).toBe(true); + + // Caller tolerance, exercised. + const friendlyIdByRunId = new Map( + (staleRuns as Array<{ id: string; friendlyId: string }>).map((r) => [r.id, r.friendlyId]) + ); + const currentRunId = friendlyIdByRunId.get(runId) ?? null; + expect(currentRunId).toBeNull(); + + // Proof the row exists on the primary. + const primaryRuns = await router.findRuns( + { + where: { + id: { in: [runId] }, + projectId: seed.project.id, + runtimeEnvironmentId: seed.environment.id, + }, + select: { id: true, friendlyId: true }, + }, + prisma14 as never + ); + expect(primaryRuns).toHaveLength(1); + expect((primaryRuns as Array<{ friendlyId: string }>)[0]!.friendlyId).toBe(friendlyId); + } + ); +}); diff --git a/internal-packages/run-store/src/runOpsStore.replayReadAfterWrite.replicaLag.test.ts b/internal-packages/run-store/src/runOpsStore.replayReadAfterWrite.replicaLag.test.ts new file mode 100644 index 00000000000..297449412bf --- /dev/null +++ b/internal-packages/run-store/src/runOpsStore.replayReadAfterWrite.replicaLag.test.ts @@ -0,0 +1,211 @@ +// Replica-lag coverage for the REPLAY route's source-run reads. The loader, resolveRunOrganizationId, +// and the action all call runStore.findRun({ friendlyId }) with NO client, so the read routes to the +// owning store's REPLICA (fan-out to the other store's replica on a miss). Passing the control-plane +// WRITER instead escalates to findRunOnPrimary (read-your-writes). +// +// The route is a Remix loader/action not unit-testable with an injected store here, so we exercise the +// exact findRun pattern each caller uses against a real split topology with the owning replica frozen. + +import { heteroRunOpsPostgresTest, laggingReplica } from "@internal/testcontainers"; +import type { PrismaClient } from "@trigger.dev/database"; +import type { RunOpsPrismaClient } from "@internal/run-ops-database"; +import { describe, expect } from "vitest"; +import { PostgresRunStore } from "./PostgresRunStore.js"; +import { RoutingRunStore } from "./runOpsStore.js"; + +const CUID_25 = "c".repeat(25); // → LEGACY (#legacy / prisma14, full schema) + +async function seedEnvironmentLegacy(prisma: PrismaClient, suffix: string) { + const organization = await prisma.organization.create({ + data: { title: `Org ${suffix}`, slug: `org-${suffix}` }, + }); + const project = await prisma.project.create({ + data: { + name: `Project ${suffix}`, + slug: `project-${suffix}`, + externalRef: `proj_${suffix}`, + organizationId: organization.id, + }, + }); + const environment = await prisma.runtimeEnvironment.create({ + data: { + type: "DEVELOPMENT", + slug: "dev", + projectId: project.id, + organizationId: organization.id, + apiKey: `tr_dev_${suffix}`, + pkApiKey: `pk_dev_${suffix}`, + shortcode: `short_${suffix}`, + }, + }); + return { organization, project, environment }; +} + +function taskRunData(opts: { + id: string; + friendlyId: string; + organizationId: string; + projectId: string; + runtimeEnvironmentId: string; +}) { + return { + id: opts.id, + engine: "V2" as const, + status: "COMPLETED_SUCCESSFULLY" as const, + friendlyId: opts.friendlyId, + runtimeEnvironmentId: opts.runtimeEnvironmentId, + environmentType: "DEVELOPMENT" as const, + organizationId: opts.organizationId, + projectId: opts.projectId, + taskIdentifier: "my-task", + payload: JSON.stringify({ hello: "world" }), + payloadType: "application/json", + traceContext: {}, + traceId: `trace_${opts.id}`, + spanId: `span_${opts.id}`, + queue: "task/my-task", + isTest: false, + taskEventStore: "taskEvent", + depth: 0, + }; +} + +// Build a LEGACY-owning router whose legacy replica is frozen (lagging), and a real (non-lagging) NEW +// store so the on-miss fan-out to the other store's replica also misses. Returns the router + a probe. +function buildRouterWithLaggingLegacyReplica(prisma14: PrismaClient, prisma17: RunOpsPrismaClient) { + const legacyReplica = laggingReplica(prisma14, [{ model: "taskRun", mode: "missing" }]); + const legacyStore = new PostgresRunStore({ + prisma: prisma14, + readOnlyPrisma: legacyReplica.client, + schemaVariant: "legacy", + }); + const newStore = new PostgresRunStore({ + prisma: prisma17 as never, + readOnlyPrisma: prisma17 as never, + schemaVariant: "dedicated", + }); + const router = new RoutingRunStore({ new: newStore, legacy: legacyStore }); + return { router, legacyReplica }; +} + +// The replay route reads the source run keyed by friendlyId (runParam) — mirror that exactly. +const REPLAY_SELECT = { + payload: true, + payloadType: true, + runtimeEnvironmentId: true, + projectId: true, + taskIdentifier: true, +} as const; + +describe("replay route source-run reads under replica lag (findRun by friendlyId)", () => { + // The action does `runStore.findRun({ friendlyId })` with NO client, so the read routes to the + // lagging replica and MISSES; its only miss-handler is the mollifier buffer, so once the buffered + // run has drained to the primary but not the replica the action would fall through to "Run not + // found" for a run that exists on the primary. Passing the control-plane WRITER (like + // resolveRunOrganizationId's primary fallback) makes the read resolve the fresh run. + heteroRunOpsPostgresTest( + "action read (no client) hits the lagging replica and MISSES a run that exists on the primary", + async ({ prisma14, prisma17 }) => { + const seed = await seedEnvironmentLegacy(prisma14, "replay_action"); + const runId = `run_${CUID_25}`; + const friendlyId = "run_replay_action"; + await prisma14.taskRun.create({ + data: taskRunData({ + id: runId, + friendlyId, + organizationId: seed.organization.id, + projectId: seed.project.id, + runtimeEnvironmentId: seed.environment.id, + }), + }); + + // Exactly the action's call: findRun({ friendlyId }) with NO client → owning store's replica, + // then fan-out to the other store's replica. Both miss under lag → null → "Run not found". + const { router, legacyReplica } = buildRouterWithLaggingLegacyReplica(prisma14, prisma17); + const viaReplica = await router.findRun({ friendlyId }); + expect(viaReplica).toBeNull(); // replica misses; the action re-reads the primary + expect(legacyReplica.wasHit()).toBe(true); + + // The action passes the control-plane WRITER (like resolveRunOrganizationId's primary fallback) + // → resolves the fresh run on the owning primary, never the replica. + const { router: routerPrimary, legacyReplica: legacyReplicaPrimary } = + buildRouterWithLaggingLegacyReplica(prisma14, prisma17); + const viaPrimary = await routerPrimary.findRun({ friendlyId }, prisma14); + expect(viaPrimary).not.toBeNull(); + expect((viaPrimary as { friendlyId: string }).friendlyId).toBe(friendlyId); + expect(legacyReplicaPrimary.wasHit()).toBe(false); + } + ); + + // resolveRunOrganizationId reads the replica first (misses under lag), checks the buffer, then FALLS + // BACK to the primary via `runStore.findRun(..., prisma)`. This documents that the primary-fallback + // leg resolves the run (and thus runtimeEnvironmentId → organizationId for the RBAC scope) even when + // the replica is frozen and the buffer is drained. + heteroRunOpsPostgresTest( + "resolveRunOrganizationId: replica leg misses under lag, primary-fallback leg resolves the run", + async ({ prisma14, prisma17 }) => { + const seed = await seedEnvironmentLegacy(prisma14, "replay_org"); + const runId = `run_${CUID_25}`; + const friendlyId = "run_replay_org"; + await prisma14.taskRun.create({ + data: taskRunData({ + id: runId, + friendlyId, + organizationId: seed.organization.id, + projectId: seed.project.id, + runtimeEnvironmentId: seed.environment.id, + }), + }); + + const { router, legacyReplica } = buildRouterWithLaggingLegacyReplica(prisma14, prisma17); + + // Leg 1 (the route's first read): replica, no client → miss under lag. + const replicaLeg = await router.findRun( + { friendlyId }, + { select: { runtimeEnvironmentId: true } } + ); + expect(replicaLeg).toBeNull(); + expect(legacyReplica.wasHit()).toBe(true); + + // Leg 2 (the route's primary fallback): findRun(..., prisma) → findRunOnPrimary resolves the + // run so the org scope is never left unresolved under replica lag. + const primaryLeg = await router.findRun( + { friendlyId }, + { select: { runtimeEnvironmentId: true } }, + prisma14 + ); + expect(primaryLeg).not.toBeNull(); + expect((primaryLeg as { runtimeEnvironmentId: string }).runtimeEnvironmentId).toBe( + seed.environment.id + ); + } + ); + + // The loader read is a read-view, included only to document the routing (no staleness assertion). + // The loader renders the replay DIALOG from the source run; on a replica miss it uses the mollifier + // buffer, else the user retries. Eventual consistency is the intended contract for rendering a form + // for an (already-existing, historical) run — it drives no write. Kept here so the full group of + // replay reads is represented; the primary-logic coverage is the two cases above. + heteroRunOpsPostgresTest( + "loader read (no client) routes to the replica (read-view — documents routing only)", + async ({ prisma14, prisma17 }) => { + const seed = await seedEnvironmentLegacy(prisma14, "replay_loader"); + const runId = `run_${CUID_25}`; + const friendlyId = "run_replay_loader"; + await prisma14.taskRun.create({ + data: taskRunData({ + id: runId, + friendlyId, + organizationId: seed.organization.id, + projectId: seed.project.id, + runtimeEnvironmentId: seed.environment.id, + }), + }); + + const { router, legacyReplica } = buildRouterWithLaggingLegacyReplica(prisma14, prisma17); + await router.findRun({ friendlyId }, { select: REPLAY_SELECT }); + // No client → the loader's read went to the replica, as designed for a display read. + expect(legacyReplica.wasHit()).toBe(true); + } + ); +}); diff --git a/internal-packages/run-store/src/runOpsStore.resolveRunForMutationReplicaLag.test.ts b/internal-packages/run-store/src/runOpsStore.resolveRunForMutationReplicaLag.test.ts new file mode 100644 index 00000000000..efa1dab8b44 --- /dev/null +++ b/internal-packages/run-store/src/runOpsStore.resolveRunForMutationReplicaLag.test.ts @@ -0,0 +1,200 @@ +// Store-seam characterization of the read primitive behind mollifier resolveRunForMutation — NOT a +// caller guard. The resolver reads findRun on the branded $replica, then re-probes the writer on a +// miss; here we reproduce just that two-step read against real Postgres with the owning replica frozen, +// to show the branded-replica read genuinely misses a fresh row (readYourWrites=false) while the writer +// re-probe recovers it. The caller contract (resolveRunForMutation returns source:"pg" instead of a +// spurious 404 under lag) is locked separately by the real caller-driven guards that drive the exported +// resolver end-to-end. + +import { heteroRunOpsPostgresTest, laggingReplica } from "@internal/testcontainers"; +import type { PrismaClient } from "@trigger.dev/database"; +import { describe, expect } from "vitest"; +import { PostgresRunStore } from "./PostgresRunStore.js"; +import { RoutingRunStore } from "./runOpsStore.js"; +import { markReadReplicaClient } from "./readReplicaClient.js"; + +// A cuid (25 chars after the `run_` prefix) classifies LEGACY, so both the create and the +// friendlyId-keyed read route to the legacy (control-plane) store first. +const CUID_25 = "c".repeat(25); +const FRIENDLY_25 = "d".repeat(25); + +async function seedEnvironmentLegacy(prisma: PrismaClient, suffix: string) { + const organization = await prisma.organization.create({ + data: { title: `Org ${suffix}`, slug: `org-${suffix}` }, + }); + const project = await prisma.project.create({ + data: { + name: `Project ${suffix}`, + slug: `project-${suffix}`, + externalRef: `proj_${suffix}`, + organizationId: organization.id, + }, + }); + const environment = await prisma.runtimeEnvironment.create({ + data: { + type: "DEVELOPMENT", + slug: "dev", + projectId: project.id, + organizationId: organization.id, + apiKey: `tr_dev_${suffix}`, + pkApiKey: `pk_dev_${suffix}`, + shortcode: `short_${suffix}`, + }, + }); + return { organization, project, environment }; +} + +function taskRunData(opts: { + id: string; + friendlyId: string; + organizationId: string; + projectId: string; + runtimeEnvironmentId: string; +}) { + return { + id: opts.id, + engine: "V2" as const, + status: "PENDING" as const, + friendlyId: opts.friendlyId, + runtimeEnvironmentId: opts.runtimeEnvironmentId, + environmentType: "DEVELOPMENT" as const, + organizationId: opts.organizationId, + projectId: opts.projectId, + taskIdentifier: "my-task", + payload: "{}", + payloadType: "application/json", + traceContext: {}, + traceId: `trace_${opts.id}`, + spanId: `span_${opts.id}`, + queue: "task/my-task", + isTest: false, + taskEventStore: "taskEvent", + depth: 0, + }; +} + +// Mimic the mollifier resolver's PG portion exactly: read the BRANDED replica first, then re-probe the +// WRITER on a miss. (The buffer probe between the two is orthogonal to the replica-lag question.) +async function resolveViaRunStorePgReads( + router: RoutingRunStore, + where: { friendlyId: string; runtimeEnvironmentId: string }, + brandedReplica: unknown, + writer: PrismaClient +): Promise<{ friendlyId: string } | null> { + const pgRun = (await router.findRun(where, { select: { friendlyId: true } }, brandedReplica)) as { + friendlyId: string; + } | null; + if (pgRun) return { friendlyId: pgRun.friendlyId }; + + const writerRun = (await router.findRun(where, { select: { friendlyId: true } }, writer)) as { + friendlyId: string; + } | null; + if (writerRun) return { friendlyId: writerRun.friendlyId }; + + return null; +} + +describe("store reads behind mollifier resolveRunForMutation — replica-first, writer-probe recovers", () => { + heteroRunOpsPostgresTest( + "fresh run invisible on the lagging replica is still resolved via the writer probe", + async ({ prisma14, prisma17 }) => { + const legacyReplica = laggingReplica(prisma14, [{ model: "taskRun", mode: "missing" }]); + const legacyStore = new PostgresRunStore({ + prisma: prisma14, + readOnlyPrisma: legacyReplica.client, + schemaVariant: "legacy", + }); + const newStore = new PostgresRunStore({ + prisma: prisma17 as never, + readOnlyPrisma: prisma17 as never, + schemaVariant: "dedicated", + }); + const router = new RoutingRunStore({ new: newStore, legacy: legacyStore }); + + const seed = await seedEnvironmentLegacy(prisma14, "mollifier_leg"); + const runId = `run_${CUID_25}`; // cuid → LEGACY + const friendlyId = `run_${FRIENDLY_25}`; // classifies LEGACY too → routes to owning store first + await prisma14.taskRun.create({ + data: taskRunData({ + id: runId, + friendlyId, + organizationId: seed.organization.id, + projectId: seed.project.id, + runtimeEnvironmentId: seed.environment.id, + }), + }); + + const where = { friendlyId, runtimeEnvironmentId: seed.environment.id }; + + // The resolver's phase-1 client: the BRANDED $replica (readYourWrites=false → replica routing). + const brandedReplica = markReadReplicaClient({} as object); + + // HAZARD proof: the phase-1 replica read alone misses the fresh row under lag. + const phase1Only = (await router.findRun( + where, + { select: { friendlyId: true } }, + brandedReplica + )) as { friendlyId: string } | null; + expect(phase1Only).toBeNull(); + expect(legacyReplica.wasHit()).toBe(true); + + // TOLERANCE proof: the full resolver pattern (replica-first, then writer-probe) resolves the run + // despite the lagging replica, so the mutation route does NOT wrongly 404. + const legacyReplica2 = laggingReplica(prisma14, [{ model: "taskRun", mode: "missing" }]); + const legacyStore2 = new PostgresRunStore({ + prisma: prisma14, + readOnlyPrisma: legacyReplica2.client, + schemaVariant: "legacy", + }); + const router2 = new RoutingRunStore({ new: newStore, legacy: legacyStore2 }); + + const resolved = await resolveViaRunStorePgReads(router2, where, brandedReplica, prisma14); + expect(resolved).not.toBeNull(); + expect(resolved?.friendlyId).toBe(friendlyId); + // The replica was consulted (phase 1) but the writer probe (phase 2) is what recovered the row. + expect(legacyReplica2.wasHit()).toBe(true); + } + ); + + heteroRunOpsPostgresTest( + "phase-1 replica read alone resolves the run in steady state (writer probe not needed)", + async ({ prisma14, prisma17 }) => { + // Non-lagging: legacy store reads its real replica (prisma14 itself here as the replica handle). + const legacyStore = new PostgresRunStore({ + prisma: prisma14, + readOnlyPrisma: prisma14, + schemaVariant: "legacy", + }); + const newStore = new PostgresRunStore({ + prisma: prisma17 as never, + readOnlyPrisma: prisma17 as never, + schemaVariant: "dedicated", + }); + const router = new RoutingRunStore({ new: newStore, legacy: legacyStore }); + + const seed = await seedEnvironmentLegacy(prisma14, "mollifier_ok"); + const runId = `run_${CUID_25}`; + const friendlyId = `run_${FRIENDLY_25}`; + await prisma14.taskRun.create({ + data: taskRunData({ + id: runId, + friendlyId, + organizationId: seed.organization.id, + projectId: seed.project.id, + runtimeEnvironmentId: seed.environment.id, + }), + }); + + const where = { friendlyId, runtimeEnvironmentId: seed.environment.id }; + const brandedReplica = markReadReplicaClient({} as object); + + const phase1 = (await router.findRun( + where, + { select: { friendlyId: true } }, + brandedReplica + )) as { friendlyId: string } | null; + expect(phase1).not.toBeNull(); + expect(phase1?.friendlyId).toBe(friendlyId); + } + ); +}); diff --git a/internal-packages/run-store/src/runOpsStore.routesBatchGetReadView.replicaLag.test.ts b/internal-packages/run-store/src/runOpsStore.routesBatchGetReadView.replicaLag.test.ts new file mode 100644 index 00000000000..f34101eac60 --- /dev/null +++ b/internal-packages/run-store/src/runOpsStore.routesBatchGetReadView.replicaLag.test.ts @@ -0,0 +1,312 @@ +// Coverage for the batch-GET route/loader reads issued with NO client, grouped as "routes-batch-get". +// Five external-API callers, two store methods routed differently: +// +// api.v1.batches.$batchId findBatchTaskRunByFriendlyId(id, env.id, {errors}) [GET] +// api.v2.batches.$batchId findBatchTaskRunByFriendlyId(id, env.id, {errors}) [GET] +// realtime.v1.batches.$batchId findBatchTaskRunByFriendlyId(id, env.id) [subscribe] +// api.v3.batches findBatchTaskRunById(cachedRequestId) [request-idempotency dedup] +// api.v2.tasks.batch findBatchTaskRunById(cachedRequestId) [request-idempotency dedup] +// +// Routing. The router fans BOTH methods out NEW→LEGACY, each leg via #ownPrimary(store, client). For a +// NO-client call #ownPrimary returns undefined, so each leg falls to the store default — and the two +// methods default DIFFERENTLY in PostgresRunStore: +// - findBatchTaskRunByFriendlyId `client ?? this.readOnlyPrisma` → REPLICA +// - findBatchTaskRunById `client ?? this.prisma` → PRIMARY +// So the three friendlyId GET/subscribe callers read the owning REPLICA; the two byId dedup callers read +// the owning PRIMARY. Every case proves its routing (frozen owning replica consulted, or not). +// +// findBatchTaskRunByFriendlyId (REPLICA) — api.v1 / api.v2 / realtime.v1: +// The friendlyId is looked up on the owning REPLICA. Under lag a just-created batch (committed to the +// owning PRIMARY) is not yet visible → the read returns null → createLoaderApiRoute short-circuits with +// `{ error: "Not found" }, status 404` and header `x-should-retry: false`, which the SDK obeys verbatim +// (no retry). The batch is live on the owning primary, recoverable via a primary re-read (asserted +// below). Contrast the sibling RUN-get routes, which set `shouldRetryNotFound: true` so the SDK retries +// the 404 through lag. +// +// findBatchTaskRunById (PRIMARY) — api.v3.batches / api.v2.tasks.batch — tolerated: +// The request-idempotency dedup (handleRequestIdempotency.findCachedEntity) looks up the cached batch +// by id on the owning PRIMARY. Under owning-replica lag the just-written batch is still found, so the +// retried create returns isCached instead of minting a DUPLICATE batch. Tolerance = PRIMARY routing, +// proven by wasHit("batchTaskRun") === false (the frozen replica is never consulted) AND a correct row. +// +// Build the RoutingRunStore as the webapp holds it (NEW = dedicated subset store / prisma17; LEGACY = +// control-plane store / prisma14 whose REPLICA is the frozen lagging client). Seed the batch (cuid id → +// LEGACY-owned) on the LEGACY PRIMARY, freeze the LEGACY replica (batchTaskRun + batchTaskRunItem, +// "missing") with the shared laggingReplica, then invoke each read EXACTLY as the caller does — same +// method, same args, NO client. Real split topology via heteroRunOpsPostgresTest — never mocked. + +import { heteroRunOpsPostgresTest, laggingReplica } from "@internal/testcontainers"; +import type { PrismaClient } from "@trigger.dev/database"; +import type { RunOpsPrismaClient } from "@internal/run-ops-database"; +import { describe, expect } from "vitest"; +import { PostgresRunStore } from "./PostgresRunStore.js"; +import { RoutingRunStore } from "./runOpsStore.js"; +import type { CreateBatchTaskRunData } from "./types.js"; + +// A cuid (25 chars after the `batch_` prefix) classifies LEGACY, so the batch is owned by the legacy +// (control-plane) store; the client-less reads then fan out / land on the LEGACY leg. +const CUID_25 = "c".repeat(25); + +async function seedEnvironment(prisma: PrismaClient, slugSuffix: string) { + const organization = await prisma.organization.create({ + data: { title: `Org ${slugSuffix}`, slug: `org-${slugSuffix}` }, + }); + const project = await prisma.project.create({ + data: { + name: `Project ${slugSuffix}`, + slug: `project-${slugSuffix}`, + externalRef: `proj_${slugSuffix}`, + organizationId: organization.id, + }, + }); + const environment = await prisma.runtimeEnvironment.create({ + data: { + type: "DEVELOPMENT", + slug: "dev", + projectId: project.id, + organizationId: organization.id, + apiKey: `tr_dev_${slugSuffix}`, + pkApiKey: `pk_dev_${slugSuffix}`, + shortcode: `short_${slugSuffix}`, + }, + }); + return { organization, project, environment }; +} + +// Build the RoutingRunStore the same way the webapp does. The LEGACY replica is frozen in "missing" mode +// for the batch models: any REPLICA-routed batch read comes back null/[]/0 AND flips wasHit — so +// wasHit(true) + a null result proves REPLICA routing (the findBatchTaskRunByFriendlyId cases above), +// and wasHit(false) + a correct row proves PRIMARY routing (the findBatchTaskRunById cases). +function buildLaggingRouter(prisma14: PrismaClient, prisma17: RunOpsPrismaClient) { + const legacyReplica = laggingReplica(prisma14, [ + { model: "batchTaskRun", mode: "missing" }, + { model: "batchTaskRunItem", mode: "missing" }, + ]); + const legacyStore = new PostgresRunStore({ + prisma: prisma14, + readOnlyPrisma: legacyReplica.client, + schemaVariant: "legacy", + }); + const newStore = new PostgresRunStore({ + prisma: prisma17 as never, + readOnlyPrisma: prisma17 as never, + schemaVariant: "dedicated", + }); + const router = new RoutingRunStore({ new: newStore, legacy: legacyStore }); + return { router, legacyStore, legacyReplica }; +} + +function batchData(overrides: Partial): CreateBatchTaskRunData { + return { + id: `batch_${CUID_25}`, + friendlyId: "batch_route_get_f", + runtimeEnvironmentId: "PLACEHOLDER", + status: "PENDING", + runCount: 1, + runIds: ["run_child_1"], + expectedCount: 1, + batchVersion: "runengine:v2", + sealed: false, + ...overrides, + }; +} + +// Reproduce the createLoaderApiRoute resource gate + the SDK's header obedience: a null findResource → +// 404 with x-should-retry "false" → the SDK returns { retry: false }. This is the mechanism that turns a +// stale replica null into a NON-RETRYABLE 404. `shouldRetryNotFound` is UNSET on all three batch routes. +function routeOutcomeForResource( + resource: unknown, + opts: { shouldRetryNotFound?: boolean } = {} +): { status: number; xShouldRetry: string | null; sdkWillRetry: boolean } { + if (!resource) { + const xShouldRetry = opts.shouldRetryNotFound ? "true" : "false"; + return { status: 404, xShouldRetry, sdkWillRetry: xShouldRetry === "true" }; + } + return { status: 200, xShouldRetry: null, sdkWillRetry: false }; +} + +describe("batch-GET route reads under replica lag", () => { + // api.v1.batches — findBatchTaskRunByFriendlyId (REPLICA). + heteroRunOpsPostgresTest( + "api.v1.batches findBatchTaskRunByFriendlyId reads stale-null under replica lag, yielding a non-retryable 404", + async ({ prisma14, prisma17 }) => { + const { router, legacyStore, legacyReplica } = buildLaggingRouter(prisma14, prisma17); + const seed = await seedEnvironment(prisma14, "bget_v1"); + const friendlyId = "batch_bget_v1_f"; + await legacyStore.createBatchTaskRun( + batchData({ friendlyId, runtimeEnvironmentId: seed.environment.id }) + ); + + // Exact caller invocation: (friendlyId, environment.id, { include: { errors: true } }), NO client. + const underLag = await router.findBatchTaskRunByFriendlyId(friendlyId, seed.environment.id, { + include: { errors: true }, + }); + + // Stale null for a batch that exists on the primary → the route's resource gate fires. + expect(underLag).toBeNull(); + // REPLICA routing proof: the frozen owning replica WAS consulted (this is the split hazard). + expect(legacyReplica.wasHit("batchTaskRun")).toBe(true); + + // The replica-routed miss yields a non-retryable 404 of a LIVE batch. + const outcome = routeOutcomeForResource(underLag); // shouldRetryNotFound unset on this route + expect(outcome.status).toBe(404); + expect(outcome.xShouldRetry).toBe("false"); + expect(outcome.sdkWillRetry).toBe(false); + + // The null is PURELY replica lag + replica routing, not a missing row / bad query. Passing the + // owning WRITER escalates the LEGACY leg to its PRIMARY (#ownPrimary) and the batch is found — a + // primary route / primary re-read resolves it. + const onPrimary = await router.findBatchTaskRunByFriendlyId( + friendlyId, + seed.environment.id, + { include: { errors: true } }, + prisma14 as never + ); + expect(onPrimary).not.toBeNull(); + expect(onPrimary!.friendlyId).toBe(friendlyId); + expect(Array.isArray(onPrimary!.errors)).toBe(true); + expect(routeOutcomeForResource(onPrimary).status).toBe(200); + } + ); + + // api.v2.batches — findBatchTaskRunByFriendlyId (REPLICA). + heteroRunOpsPostgresTest( + "api.v2.batches findBatchTaskRunByFriendlyId reads stale-null under replica lag, yielding a non-retryable 404", + async ({ prisma14, prisma17 }) => { + const { router, legacyStore, legacyReplica } = buildLaggingRouter(prisma14, prisma17); + const seed = await seedEnvironment(prisma14, "bget_v2"); + const friendlyId = "batch_bget_v2_f"; + await legacyStore.createBatchTaskRun( + batchData({ + friendlyId, + runtimeEnvironmentId: seed.environment.id, + processingCompletedAt: new Date(), + }) + ); + + const underLag = await router.findBatchTaskRunByFriendlyId(friendlyId, seed.environment.id, { + include: { errors: true }, + }); + expect(underLag).toBeNull(); + expect(legacyReplica.wasHit("batchTaskRun")).toBe(true); + + const outcome = routeOutcomeForResource(underLag); + expect(outcome.status).toBe(404); + expect(outcome.sdkWillRetry).toBe(false); + + const onPrimary = await router.findBatchTaskRunByFriendlyId( + friendlyId, + seed.environment.id, + { include: { errors: true } }, + prisma14 as never + ); + expect(onPrimary).not.toBeNull(); + expect(onPrimary!.friendlyId).toBe(friendlyId); + } + ); + + // realtime.v1.batches — findBatchTaskRunByFriendlyId (REPLICA). + heteroRunOpsPostgresTest( + "realtime.v1.batches findBatchTaskRunByFriendlyId reads stale-null under replica lag, 404ing before the subscription starts", + async ({ prisma14, prisma17 }) => { + const { router, legacyStore, legacyReplica } = buildLaggingRouter(prisma14, prisma17); + const seed = await seedEnvironment(prisma14, "bget_rt"); + const friendlyId = "batch_bget_rt_f"; + const batchId = `batch_${CUID_25}`; + await legacyStore.createBatchTaskRun( + batchData({ id: batchId, friendlyId, runtimeEnvironmentId: seed.environment.id }) + ); + + // Exact caller invocation: (friendlyId, environment.id) — NO include, NO client. + const underLag = await router.findBatchTaskRunByFriendlyId(friendlyId, seed.environment.id); + expect(underLag).toBeNull(); + expect(legacyReplica.wasHit("batchTaskRun")).toBe(true); + + // The loader returns 404 at the resource gate BEFORE reaching streamBatch — the subscription + // never starts. shouldRetryNotFound is unset here too → non-retryable. + const outcome = routeOutcomeForResource(underLag); + expect(outcome.status).toBe(404); + expect(outcome.sdkWillRetry).toBe(false); + + // Owning WRITER escalates to the LEGACY primary; batchRun.id (fed to streamBatch) resolves. + const onPrimary = await router.findBatchTaskRunByFriendlyId( + friendlyId, + seed.environment.id, + undefined, + prisma14 as never + ); + expect(onPrimary).not.toBeNull(); + expect(onPrimary!.id).toBe(batchId); + } + ); + + // api.v3.batches — findBatchTaskRunById (PRIMARY) — tolerated. + heteroRunOpsPostgresTest( + "api.v3.batches findBatchTaskRunById reads the primary for idempotency dedup, finding the cached batch under replica lag", + async ({ prisma14, prisma17 }) => { + const { router, legacyStore, legacyReplica } = buildLaggingRouter(prisma14, prisma17); + const seed = await seedEnvironment(prisma14, "bget_v3"); + const batchId = `batch_${CUID_25}`; + const friendlyId = "batch_bget_v3_f"; + await legacyStore.createBatchTaskRun( + batchData({ + id: batchId, + friendlyId, + runtimeEnvironmentId: seed.environment.id, + runCount: 3, + }) + ); + + // Exact caller invocation: findCachedEntity(cachedRequestId) → findBatchTaskRunById(id), NO client. + const cached = await router.findBatchTaskRunById(batchId); + + // PRIMARY routing proof: the just-written batch is found DESPITE the frozen replica... + expect(cached).not.toBeNull(); + expect(cached!.id).toBe(batchId); + // ...and the frozen owning replica was NEVER consulted (tolerance = primary routing, not lag-toleration). + expect(legacyReplica.wasHit("batchTaskRun")).toBe(false); + + // Reproduce the caller's dedup branch (findCachedEntity): env-match + non-null → buildResponse + // returns isCached, so the retried create does NOT mint a duplicate batch. + const dedupHit = !!cached && cached.runtimeEnvironmentId === seed.environment.id; + expect(dedupHit).toBe(true); + expect(cached!.friendlyId).toBe(friendlyId); + expect(cached!.runCount).toBe(3); + + // Env-scope guard: a foreign environment must reject the cached entity (→ null → re-create). + const foreignEnv = + !!cached && cached.runtimeEnvironmentId === "env_does_not_exist" ? cached : null; + expect(foreignEnv).toBeNull(); + } + ); + + // api.v2.tasks.batch — findBatchTaskRunById (PRIMARY) — tolerated. + heteroRunOpsPostgresTest( + "api.v2.tasks.batch findBatchTaskRunById reads the primary for dedup, finding the cached batch under replica lag", + async ({ prisma14, prisma17 }) => { + const { router, legacyStore, legacyReplica } = buildLaggingRouter(prisma14, prisma17); + const seed = await seedEnvironment(prisma14, "bget_tb"); + const batchId = `batch_${CUID_25}`; + const friendlyId = "batch_bget_tb_f"; + await legacyStore.createBatchTaskRun( + batchData({ + id: batchId, + friendlyId, + runtimeEnvironmentId: seed.environment.id, + runCount: 5, + }) + ); + + const cached = await router.findBatchTaskRunById(batchId); + + expect(cached).not.toBeNull(); + expect(cached!.id).toBe(batchId); + expect(legacyReplica.wasHit("batchTaskRun")).toBe(false); + + // Dedup branch: env-match + non-null → buildResponse returns { id: friendlyId, runCount }. + expect(cached!.runtimeEnvironmentId).toBe(seed.environment.id); + expect(cached!.friendlyId).toBe(friendlyId); + expect(cached!.runCount).toBe(5); + } + ); +}); diff --git a/internal-packages/run-store/src/runOpsStore.routesRealtimeStreamReadView.replicaLag.test.ts b/internal-packages/run-store/src/runOpsStore.routesRealtimeStreamReadView.replicaLag.test.ts new file mode 100644 index 00000000000..b1388bd2292 --- /dev/null +++ b/internal-packages/run-store/src/runOpsStore.routesRealtimeStreamReadView.replicaLag.test.ts @@ -0,0 +1,590 @@ +// Replica-lag properties for the realtime-stream route read views on the run-ops split. Fourteen +// route handlers resolve a run by friendlyId via runStore.findRun with a branded $replica handle or no +// client — both of which RoutingRunStore routes to the OWNING store's REPLICA, so under lag each can +// miss a freshly-created run and return null. Built as the webapp holds the RoutingRunStore, with the +// owning (legacy) replica FROZEN (taskRun "missing") via the shared `laggingReplica` primitive; each +// read is invoked EXACTLY as its caller does, asserting the under-lag value plus a primary re-read of +// the same row (so the null is provably lag). Thirteen reads tolerate the stale null (the run is +// already established when the route fires); subscribeToRun's findResource is a genuine +// read-your-writes window that re-reads the owning PRIMARY on a null to recover the just-triggered run. + +import { heteroRunOpsPostgresTest, laggingReplica } from "@internal/testcontainers"; +import type { PrismaClient } from "@trigger.dev/database"; +import type { RunOpsPrismaClient } from "@internal/run-ops-database"; +import { describe, expect } from "vitest"; +import { PostgresRunStore } from "./PostgresRunStore.js"; +import { RoutingRunStore } from "./runOpsStore.js"; +import { markReadReplicaClient } from "./readReplicaClient.js"; + +// A cuid-shaped id (no run-ops v1 marker) classifies LEGACY, so both the create and the friendlyId-keyed +// reads route to the legacy (control-plane / prisma14, full schema) store first — the common realtime +// residency and the store whose frozen replica the read hits. +const CUID_25 = "c".repeat(25); + +// The webapp always passes its `$replica` handle, which is markReadReplicaClient()'d. A branded client +// tells RoutingRunStore "do not escalate to primary" — so the read stays on the owning store's replica. +// We pass a branded stand-in exactly as the routes do; its identity is irrelevant because the router +// discards it and reads the owning store's readOnlyPrisma (the frozen replica in these tests). +const BRANDED_REPLICA = markReadReplicaClient({}) as never; + +async function seedEnvironment(prisma: PrismaClient, suffix: string) { + const organization = await prisma.organization.create({ + data: { title: `Org ${suffix}`, slug: `org-${suffix}` }, + }); + const project = await prisma.project.create({ + data: { + name: `Project ${suffix}`, + slug: `project-${suffix}`, + externalRef: `proj_${suffix}`, + organizationId: organization.id, + }, + }); + const environment = await prisma.runtimeEnvironment.create({ + data: { + type: "DEVELOPMENT", + slug: "dev", + projectId: project.id, + organizationId: organization.id, + apiKey: `tr_dev_${suffix}`, + pkApiKey: `pk_dev_${suffix}`, + shortcode: `short_${suffix}`, + }, + }); + return { organization, project, environment }; +} + +function taskRunData(opts: { + id: string; + friendlyId: string; + organizationId: string; + projectId: string; + runtimeEnvironmentId: string; +}) { + return { + id: opts.id, + engine: "V2" as const, + status: "PENDING" as never, + friendlyId: opts.friendlyId, + runtimeEnvironmentId: opts.runtimeEnvironmentId, + environmentType: "DEVELOPMENT" as const, + organizationId: opts.organizationId, + projectId: opts.projectId, + taskIdentifier: "my-task", + payload: "{}", + payloadType: "application/json", + traceContext: {}, + traceId: `trace_${opts.id}`, + spanId: `span_${opts.id}`, + queue: "task/my-task", + isTest: false, + taskEventStore: "taskEvent", + depth: 0, + realtimeStreamsVersion: "v1", + realtimeStreams: [], + }; +} + +// Build the router the realtime routes hold it as: a LEGACY store on the control-plane DB whose replica +// LAGS (taskRun frozen "missing"), plus a fresh NEW store. Seed a LEGACY-resident (cuid) run on the +// legacy PRIMARY. Returns the router + probe + seeded ids. Every test below resolves this run. +async function setupLaggingLegacy( + prisma14: PrismaClient, + prisma17: RunOpsPrismaClient, + slug: string +) { + const legacyReplica = laggingReplica(prisma14, [{ model: "taskRun", mode: "missing" }]); + const legacyStore = new PostgresRunStore({ + prisma: prisma14, + readOnlyPrisma: legacyReplica.client, + schemaVariant: "legacy", + }); + const newStore = new PostgresRunStore({ + prisma: prisma17 as never, + readOnlyPrisma: prisma17 as never, + schemaVariant: "dedicated", + }); + const router = new RoutingRunStore({ new: newStore, legacy: legacyStore }); + + const seed = await seedEnvironment(prisma14, slug); + const runId = `run_${CUID_25}`; // cuid → LEGACY + const friendlyId = `run_${slug}`; + await prisma14.taskRun.create({ + data: taskRunData({ + id: runId, + friendlyId, + organizationId: seed.organization.id, + projectId: seed.project.id, + runtimeEnvironmentId: seed.environment.id, + }), + }); + + return { router, legacyReplica, seed, runId, friendlyId }; +} + +describe("run-ops split — realtime-STREAM route read views vs. a lagging replica", () => { + // runs.$runId findRun — subscribeToRun's findResource. + // where {friendlyId, runtimeEnvironmentId}, include {batch:{select:{friendlyId}}}, BRANDED $replica. + // The canonical trigger→subscribe read-your-writes window: the replica miss is the first leg (a bare + // null would make apiBuilder 404 → terminal, since Electric retries only 429 and RunSubscription + // closes on FetchError), so findResource re-reads the owning PRIMARY on a null. Proves both legs: + // (a) the store returns null under lag, and (b) the owning-primary re-read recovers the run. + heteroRunOpsPostgresTest( + "runs.$runId findRun(branded $replica) is null under lag; the owning-primary re-read recovers the live run", + async ({ prisma14, prisma17 }) => { + const { router, legacyReplica, seed, friendlyId } = await setupLaggingLegacy( + prisma14, + prisma17, + "rt_run_subscribe" + ); + + const where = { friendlyId, runtimeEnvironmentId: seed.environment.id }; + const args = { include: { batch: { select: { friendlyId: true } } } }; + + // The stale replica read the auth findResource performs. + const stale = await router.findRun(where, args, BRANDED_REPLICA); + expect(stale).toBeNull(); // findResource null → apiBuilder returns 404 (x-should-retry:"false") + expect(legacyReplica.wasHit("taskRun")).toBe(true); + + // The run IS live on the primary — findResource re-reads it there (the owning-primary fallback, + // like resolveRunForMutation's writer re-probe) and recovers the just-triggered run. + const recovered = (await router.findRun(where, args, prisma14 as never)) as { + friendlyId: string; + } | null; + expect(recovered).not.toBeNull(); + expect(recovered!.friendlyId).toBe(friendlyId); + } + ); + + // streams.$streamId ingest findRun — plain action ingest. + // where {friendlyId} (NO env scope, back-compat), select {id,friendlyId,streamBasinName, + // runtimeEnvironmentId}, BRANDED $replica. Null → 404. Tolerated: this is the stream write/ingest + // side, invoked by the producer from INSIDE the executing run — the run was dequeued (primary-gated) + // long after creation, so it is replicated by the time it writes to its own stream; friendlyId is + // immutable. + heteroRunOpsPostgresTest( + "streams.$streamId ingest findRun(branded $replica) is null under lag; the primary sees the executing run", + async ({ prisma14, prisma17 }) => { + const { router, legacyReplica, runId, friendlyId } = await setupLaggingLegacy( + prisma14, + prisma17, + "rt_ingest_plain" + ); + const where = { friendlyId }; + const args = { + select: { id: true, friendlyId: true, streamBasinName: true, runtimeEnvironmentId: true }, + }; + + const stale = await router.findRun(where, args, BRANDED_REPLICA); + expect(stale).toBeNull(); + expect(legacyReplica.wasHit("taskRun")).toBe(true); + + const primary = (await router.findRun(where, args, prisma14 as never)) as { + id: string; + } | null; + expect(primary).not.toBeNull(); + expect(primary!.id).toBe(runId); + } + ); + + // streams.$streamId GET-SSE findResource — loader (GET SSE). + // where {friendlyId, runtimeEnvironmentId}, select {id,friendlyId,taskIdentifier,runTags, + // realtimeStreamsVersion,streamBasinName,batch:{...}}, BRANDED $replica. Null → 404. Tolerated: the + // public stream read follows a successful subscribeToRun that already resolved the run (the previous + // test covers that gate); stream chunks only exist once the run is executing, so the run is + // replicated by the time a reader arrives. + heteroRunOpsPostgresTest( + "streams.$streamId GET-SSE findResource(branded $replica) is null under lag; the primary sees the run", + async ({ prisma14, prisma17 }) => { + const { router, legacyReplica, seed, friendlyId } = await setupLaggingLegacy( + prisma14, + prisma17, + "rt_stream_get" + ); + const where = { friendlyId, runtimeEnvironmentId: seed.environment.id }; + const args = { + select: { + id: true, + friendlyId: true, + taskIdentifier: true, + runTags: true, + realtimeStreamsVersion: true, + streamBasinName: true, + batch: { select: { friendlyId: true } }, + }, + }; + + const stale = await router.findRun(where, args, BRANDED_REPLICA); + expect(stale).toBeNull(); + expect(legacyReplica.wasHit("taskRun")).toBe(true); + + const primary = (await router.findRun(where, args, prisma14 as never)) as { + friendlyId: string; + } | null; + expect(primary).not.toBeNull(); + expect(primary!.friendlyId).toBe(friendlyId); + } + ); + + // streams.$target action findRun — action (ingest/PUT). + // where {friendlyId, runtimeEnvironmentId}, select {id,friendlyId,streamBasinName,parentTaskRun:{...}, + // rootTaskRun:{...}}, BRANDED $replica. Null → 404. Tolerated: write side from inside the executing + // run; and the PUT completedAt gate re-reads the PRIMARY (`prisma`), so the authoritative decision + // never depends on this replica read. Immutable ids resolved here. + heteroRunOpsPostgresTest( + "streams.$target action findRun(branded $replica) is null under lag; the completedAt gate re-reads the primary", + async ({ prisma14, prisma17 }) => { + const { router, legacyReplica, seed, friendlyId } = await setupLaggingLegacy( + prisma14, + prisma17, + "rt_target_action" + ); + const where = { friendlyId, runtimeEnvironmentId: seed.environment.id }; + const args = { + select: { + id: true, + friendlyId: true, + streamBasinName: true, + parentTaskRun: { select: { friendlyId: true, streamBasinName: true } }, + rootTaskRun: { select: { friendlyId: true, streamBasinName: true } }, + }, + }; + + const stale = await router.findRun(where, args, BRANDED_REPLICA); + expect(stale).toBeNull(); + expect(legacyReplica.wasHit("taskRun")).toBe(true); + + const primary = (await router.findRun(where, args, prisma14 as never)) as { + friendlyId: string; + } | null; + expect(primary).not.toBeNull(); + expect(primary!.friendlyId).toBe(friendlyId); + } + ); + + // streams.$target HEAD findRun — loader (HEAD). + // Same select as the ingest/PUT action above, BRANDED $replica; resolves parent/root friendlyId for a + // HEAD last-chunk-index probe. Null → 404. Tolerated: read side of an established run's stream + // (subscribeToRun gated it first); read-only, immutable ids. + heteroRunOpsPostgresTest( + "streams.$target HEAD findResource(branded $replica) is null under lag; the primary sees the run", + async ({ prisma14, prisma17 }) => { + const { router, legacyReplica, seed, friendlyId } = await setupLaggingLegacy( + prisma14, + prisma17, + "rt_target_head" + ); + const where = { friendlyId, runtimeEnvironmentId: seed.environment.id }; + const args = { + select: { + id: true, + friendlyId: true, + streamBasinName: true, + parentTaskRun: { select: { friendlyId: true, streamBasinName: true } }, + rootTaskRun: { select: { friendlyId: true, streamBasinName: true } }, + }, + }; + + const stale = await router.findRun(where, args, BRANDED_REPLICA); + expect(stale).toBeNull(); + expect(legacyReplica.wasHit("taskRun")).toBe(true); + + const primary = (await router.findRun(where, args, prisma14 as never)) as { + friendlyId: string; + } | null; + expect(primary).not.toBeNull(); + expect(primary!.friendlyId).toBe(friendlyId); + } + ); + + // streams.$target append findRun — action. + // where {friendlyId, runtimeEnvironmentId}, select {id,friendlyId,parentTaskRun:{select:{friendlyId}}, + // rootTaskRun:{select:{friendlyId}}}, BRANDED $replica. Null → 404. Tolerated: append write from + // inside the executing run; the authoritative target completedAt gate re-reads the PRIMARY. + heteroRunOpsPostgresTest( + "streams.$target append findRun(branded $replica) is null under lag; the completedAt gate re-reads the primary", + async ({ prisma14, prisma17 }) => { + const { router, legacyReplica, seed, friendlyId } = await setupLaggingLegacy( + prisma14, + prisma17, + "rt_target_append" + ); + const where = { friendlyId, runtimeEnvironmentId: seed.environment.id }; + const args = { + select: { + id: true, + friendlyId: true, + parentTaskRun: { select: { friendlyId: true } }, + rootTaskRun: { select: { friendlyId: true } }, + }, + }; + + const stale = await router.findRun(where, args, BRANDED_REPLICA); + expect(stale).toBeNull(); + expect(legacyReplica.wasHit("taskRun")).toBe(true); + + const primary = (await router.findRun(where, args, prisma14 as never)) as { + friendlyId: string; + } | null; + expect(primary).not.toBeNull(); + expect(primary!.friendlyId).toBe(friendlyId); + } + ); + + // streams.input send findRun — action (send input). + // where {friendlyId, runtimeEnvironmentId}, select {id,friendlyId,completedAt,realtimeStreamsVersion, + // streamBasinName}, BRANDED $replica. Null → 404. Tolerated: the `.send()` writer targets an + // executing run's input stream, so it is replicated. The completedAt read off the replica fails safe: + // a stale completedAt=null only allows the append to proceed (a harmless write), never wrongly + // blocks a live run. + heteroRunOpsPostgresTest( + "streams.input send findRun(branded $replica) is null under lag; a stale completedAt fails safe by allowing the append", + async ({ prisma14, prisma17 }) => { + const { router, legacyReplica, seed, friendlyId } = await setupLaggingLegacy( + prisma14, + prisma17, + "rt_input_send" + ); + const where = { friendlyId, runtimeEnvironmentId: seed.environment.id }; + const args = { + select: { + id: true, + friendlyId: true, + completedAt: true, + realtimeStreamsVersion: true, + streamBasinName: true, + }, + }; + + const stale = await router.findRun(where, args, BRANDED_REPLICA); + expect(stale).toBeNull(); + expect(legacyReplica.wasHit("taskRun")).toBe(true); + + const primary = (await router.findRun(where, args, prisma14 as never)) as { + friendlyId: string; + } | null; + expect(primary).not.toBeNull(); + expect(primary!.friendlyId).toBe(friendlyId); + } + ); + + // streams.input GET-SSE findResource — loader (GET SSE tail). + // where {friendlyId, runtimeEnvironmentId}, include {batch:{select:{friendlyId}}}, BRANDED $replica. + // Null → 404. Tolerated: read side of an established run's input stream; read-only display tail, + // immutable ids, follows a resolved run. + heteroRunOpsPostgresTest( + "streams.input GET-SSE findResource(branded $replica) is null under lag; the primary sees the run", + async ({ prisma14, prisma17 }) => { + const { router, legacyReplica, seed, friendlyId } = await setupLaggingLegacy( + prisma14, + prisma17, + "rt_input_get" + ); + const where = { friendlyId, runtimeEnvironmentId: seed.environment.id }; + const args = { include: { batch: { select: { friendlyId: true } } } }; + + const stale = await router.findRun(where, args, BRANDED_REPLICA); + expect(stale).toBeNull(); + expect(legacyReplica.wasHit("taskRun")).toBe(true); + + const primary = (await router.findRun(where, args, prisma14 as never)) as { + friendlyId: string; + } | null; + expect(primary).not.toBeNull(); + expect(primary!.friendlyId).toBe(friendlyId); + } + ); + + // input-streams.wait findRun — create waitpoint. + // where {friendlyId, runtimeEnvironmentId}, select {id,friendlyId,realtimeStreamsVersion, + // streamBasinName}, BRANDED $replica. run.id then feeds engine.createManualWaitpoint (a mutation). + // Null → 404 → waitpoint not created. Tolerated: `.wait()` on an input stream is called from inside + // the executing run, so the run is long-since replicated; the mutation reads its own live run's id. + heteroRunOpsPostgresTest( + "input-streams.wait findRun(branded $replica) is null under lag; the primary sees the executing run", + async ({ prisma14, prisma17 }) => { + const { router, legacyReplica, seed, runId, friendlyId } = await setupLaggingLegacy( + prisma14, + prisma17, + "rt_input_wait" + ); + const where = { friendlyId, runtimeEnvironmentId: seed.environment.id }; + const args = { + select: { id: true, friendlyId: true, realtimeStreamsVersion: true, streamBasinName: true }, + }; + + const stale = await router.findRun(where, args, BRANDED_REPLICA); + expect(stale).toBeNull(); + expect(legacyReplica.wasHit("taskRun")).toBe(true); + + const primary = (await router.findRun(where, args, prisma14 as never)) as { + id: string; + } | null; + expect(primary).not.toBeNull(); + expect(primary!.id).toBe(runId); // the run.id createManualWaitpoint would co-locate the waitpoint on + } + ); + + // session-streams.wait findRun — create waitpoint. + // where {friendlyId, runtimeEnvironmentId}, select {id,friendlyId,realtimeStreamsVersion}, BRANDED + // $replica. run.id feeds engine.createManualWaitpoint. Null → 404. Tolerated: same reasoning as the + // input-streams wait above — the agent's `.wait()` on a session stream runs inside the executing run, + // so the run is replicated. + heteroRunOpsPostgresTest( + "session-streams.wait findRun(branded $replica) is null under lag; the primary sees the executing run", + async ({ prisma14, prisma17 }) => { + const { router, legacyReplica, seed, runId, friendlyId } = await setupLaggingLegacy( + prisma14, + prisma17, + "rt_session_wait" + ); + const where = { friendlyId, runtimeEnvironmentId: seed.environment.id }; + const args = { select: { id: true, friendlyId: true, realtimeStreamsVersion: true } }; + + const stale = await router.findRun(where, args, BRANDED_REPLICA); + expect(stale).toBeNull(); + expect(legacyReplica.wasHit("taskRun")).toBe(true); + + const primary = (await router.findRun(where, args, prisma14 as never)) as { + id: string; + } | null; + expect(primary).not.toBeNull(); + expect(primary!.id).toBe(runId); + } + ); + + // resources sessions.$io findRun — dashboard SSE. + // where {friendlyId, runtimeEnvironmentId}, select {id,friendlyId}, BRANDED $replica. Verifies the run + // lives in this env before subscribing to a session channel. Null → 404. Tolerated: this + // dashboard-auth route is opened by a human in the span inspector Agent tab, on a run whose detail + // page already resolved — the navigation latency dwarfs replica lag; read-only display; the error + // self-heals on reconnect/navigation. + heteroRunOpsPostgresTest( + "resources sessions.$io findRun(branded $replica) is null under lag; the primary sees the run", + async ({ prisma14, prisma17 }) => { + const { router, legacyReplica, seed, runId, friendlyId } = await setupLaggingLegacy( + prisma14, + prisma17, + "rt_res_session_io" + ); + const where = { friendlyId, runtimeEnvironmentId: seed.environment.id }; + const args = { select: { id: true, friendlyId: true } }; + + const stale = await router.findRun(where, args, BRANDED_REPLICA); + expect(stale).toBeNull(); + expect(legacyReplica.wasHit("taskRun")).toBe(true); + + const primary = (await router.findRun(where, args, prisma14 as never)) as { + id: string; + } | null; + expect(primary).not.toBeNull(); + expect(primary!.id).toBe(runId); + } + ); + + // resources streams.$streamId findRun — dashboard SSE. + // where {friendlyId, runtimeEnvironmentId}, select {id,friendlyId,realtimeStreamsVersion, + // streamBasinName}, BRANDED $replica. Null → 404. Tolerated: same dashboard-navigation reasoning as + // above (Agent tab output-stream viewer); read-only. + heteroRunOpsPostgresTest( + "resources streams.$streamId findRun(branded $replica) is null under lag; the primary sees the run", + async ({ prisma14, prisma17 }) => { + const { router, legacyReplica, seed, friendlyId } = await setupLaggingLegacy( + prisma14, + prisma17, + "rt_res_stream" + ); + const where = { friendlyId, runtimeEnvironmentId: seed.environment.id }; + const args = { + select: { + id: true, + friendlyId: true, + realtimeStreamsVersion: true, + streamBasinName: true, + }, + }; + + const stale = await router.findRun(where, args, BRANDED_REPLICA); + expect(stale).toBeNull(); + expect(legacyReplica.wasHit("taskRun")).toBe(true); + + const primary = (await router.findRun(where, args, prisma14 as never)) as { + friendlyId: string; + } | null; + expect(primary).not.toBeNull(); + expect(primary!.friendlyId).toBe(friendlyId); + } + ); + + // resources streams.input findRun — dashboard SSE. + // where {friendlyId, runtimeEnvironmentId}, select {id,friendlyId,realtimeStreamsVersion, + // streamBasinName}, BRANDED $replica. Null → 404. Tolerated: same dashboard-navigation reasoning as + // above (Agent tab input-stream viewer); read-only. + heteroRunOpsPostgresTest( + "resources streams.input findRun(branded $replica) is null under lag; the primary sees the run", + async ({ prisma14, prisma17 }) => { + const { router, legacyReplica, seed, friendlyId } = await setupLaggingLegacy( + prisma14, + prisma17, + "rt_res_input" + ); + const where = { friendlyId, runtimeEnvironmentId: seed.environment.id }; + const args = { + select: { + id: true, + friendlyId: true, + realtimeStreamsVersion: true, + streamBasinName: true, + }, + }; + + const stale = await router.findRun(where, args, BRANDED_REPLICA); + expect(stale).toBeNull(); + expect(legacyReplica.wasHit("taskRun")).toBe(true); + + const primary = (await router.findRun(where, args, prisma14 as never)) as { + friendlyId: string; + } | null; + expect(primary).not.toBeNull(); + expect(primary!.friendlyId).toBe(friendlyId); + } + ); + + // route.tsx findRun — dashboard SSE, no client argument. + // where {friendlyId, projectId} (note: projectId, not env), select {id,friendlyId, + // realtimeStreamsVersion,streamBasinName,runtimeEnvironmentId}, NO client → owning REPLICA. Null → + // throw Response 404. Tolerated: same dashboard-navigation reasoning as above; the run's own detail + // page resolved it already; read-only stream viewer. This caller passes no client at all (versus + // the branded $replica in the tests above) but routes to the same owning replica, so the under-lag + // behavior is identical. + heteroRunOpsPostgresTest( + "route.tsx findRun(no client) is null under lag; the primary sees the run", + async ({ prisma14, prisma17 }) => { + const { router, legacyReplica, seed, friendlyId } = await setupLaggingLegacy( + prisma14, + prisma17, + "rt_route_tsx" + ); + const where = { friendlyId, projectId: seed.project.id }; + const args = { + select: { + id: true, + friendlyId: true, + realtimeStreamsVersion: true, + streamBasinName: true, + runtimeEnvironmentId: true, + }, + }; + + // NO client — exactly as the route.tsx caller. + const stale = await router.findRun(where, args); + expect(stale).toBeNull(); + expect(legacyReplica.wasHit("taskRun")).toBe(true); + + const primary = (await router.findRun(where, args, prisma14 as never)) as { + friendlyId: string; + } | null; + expect(primary).not.toBeNull(); + expect(primary!.friendlyId).toBe(friendlyId); + } + ); +}); diff --git a/internal-packages/run-store/src/runOpsStore.routesRunGetReadView.replicaLag.test.ts b/internal-packages/run-store/src/runOpsStore.routesRunGetReadView.replicaLag.test.ts new file mode 100644 index 00000000000..32515068c43 --- /dev/null +++ b/internal-packages/run-store/src/runOpsStore.routesRunGetReadView.replicaLag.test.ts @@ -0,0 +1,664 @@ +// Lagging-replica coverage for the run-GET / run-detail read views ("routes-run-get"). Each read is +// exercised against real Postgres with the owning store's replica FROZEN via laggingReplica, driving the +// exact exported route caller that issues it. Every read RoutingRunStore serves from the OWNING store's +// REPLICA — a client-less findRun/findTaskRunAttempt, or a findRun passed the BRANDED $replica (the brand +// keeps the read on a replica, it does not escalate). We build the router exactly as ~/v3/runStore.server +// holds it (legacy + dedicated PostgresRunStore) and invoke each read as the caller does. +// +// Shared tolerance, asserted concretely per read: the fields that drive any control decision +// (routing/auth/redirect/queue keys — projectId, runtimeEnvironmentId, organizationId, spanId, traceId, +// createdAt, engine, queue, concurrencyKey, taskEventStore) are IMMUTABLE, so even a STALE replica row +// carries correct values. The only lag effect is transient ABSENCE → a not-found/error-redirect/empty +// display that self-heals on the next load and drives no mutation. Displayed mutable fields (status, +// cost, completedAt, output) are cosmetic and self-heal. + +import { heteroRunOpsPostgresTest, laggingReplica } from "@internal/testcontainers"; +import type { PrismaClient } from "@trigger.dev/database"; +import type { RunOpsPrismaClient } from "@internal/run-ops-database"; +import { describe, expect } from "vitest"; +import { PostgresRunStore } from "./PostgresRunStore.js"; +import { RoutingRunStore } from "./runOpsStore.js"; +import { markReadReplicaClient } from "./readReplicaClient.js"; + +type AnyClient = PrismaClient | RunOpsPrismaClient; + +// A cuid-shaped id (25 chars, no run-ops v1 marker) classifies LEGACY, so a friendlyId-keyed read +// routes to the legacy (control-plane / prisma14, full schema) store's replica first. +const CUID_25 = "e".repeat(25); +// A v1-marked 26-char body classifies NEW, routing to the dedicated (#new / prisma17) store — used for +// the findTaskRunAttempt case so we can freeze the NEW replica without control-plane FK seeding. +const NEW_ID_26 = "k".repeat(24) + "01"; + +// Final attempt statuses (FINAL_ATTEMPT_STATUSES). Inlined to avoid a run-store→webapp +// test dependency; only used to mirror the caller's findTaskRunAttempt `where`. +const FINAL_ATTEMPT_STATUSES = ["COMPLETED", "FAILED", "CANCELED"] as const; + +async function seedLegacyEnvironment(prisma: PrismaClient, suffix: string) { + const organization = await prisma.organization.create({ + data: { title: `Org ${suffix}`, slug: `org-${suffix}` }, + }); + const project = await prisma.project.create({ + data: { + name: `Project ${suffix}`, + slug: `project-${suffix}`, + externalRef: `proj_${suffix}`, + organizationId: organization.id, + }, + }); + const environment = await prisma.runtimeEnvironment.create({ + data: { + type: "DEVELOPMENT", + slug: "dev", + projectId: project.id, + organizationId: organization.id, + apiKey: `tr_dev_${suffix}`, + pkApiKey: `pk_dev_${suffix}`, + shortcode: `short_${suffix}`, + }, + }); + return { organization, project, environment }; +} + +function taskRunData(opts: { + id: string; + friendlyId: string; + organizationId: string; + projectId: string; + runtimeEnvironmentId: string; + status?: string; + spanId?: string; + engine?: "V1" | "V2"; + createdAt?: Date; + completedAt?: Date | null; + concurrencyKey?: string | null; + traceId?: string; +}) { + return { + id: opts.id, + engine: (opts.engine ?? "V2") as "V1" | "V2", + status: (opts.status ?? "PENDING") as never, + friendlyId: opts.friendlyId, + runtimeEnvironmentId: opts.runtimeEnvironmentId, + environmentType: "DEVELOPMENT" as const, + organizationId: opts.organizationId, + projectId: opts.projectId, + taskIdentifier: "my-task", + payload: "{}", + payloadType: "application/json", + traceContext: {}, + traceId: opts.traceId ?? `trace_${opts.id}`, + spanId: opts.spanId ?? `span_${opts.id}`, + queue: "task/my-task", + isTest: false, + taskEventStore: "taskEvent", + depth: 0, + ...(opts.concurrencyKey !== undefined ? { concurrencyKey: opts.concurrencyKey } : {}), + ...(opts.createdAt !== undefined ? { createdAt: opts.createdAt } : {}), + ...(opts.completedAt !== undefined ? { completedAt: opts.completedAt } : {}), + }; +} + +// Build the router exactly as runStore.server holds it: a legacy store (prisma14) + a dedicated store +// (prisma17). Either store's REPLICA (readOnlyPrisma) can be swapped for a frozen client so a +// friendlyId/id-routed read is served stale. +function buildRouter( + prisma14: PrismaClient, + prisma17: unknown, + opts?: { legacyReplica?: AnyClient; newReplica?: AnyClient } +): RoutingRunStore { + const legacyStore = new PostgresRunStore({ + prisma: prisma14, + readOnlyPrisma: (opts?.legacyReplica ?? prisma14) as never, + schemaVariant: "legacy", + }); + const newStore = new PostgresRunStore({ + prisma: prisma17 as never, + readOnlyPrisma: (opts?.newReplica ?? prisma17) as never, + schemaVariant: "dedicated", + }); + return new RoutingRunStore({ new: newStore, legacy: legacyStore }); +} + +describe("routes-run-get read views under replica lag (owning REPLICA)", () => { + // orgs redirect — legacy canonical redirect. + // Reads a run by friendlyId to get { projectId, runtimeEnvironmentId } and redirects to the v3 run + // path. Both selected fields are IMMUTABLE (set at creation). + // (a) frozen-missing replica → null → the loader throws 404. Tolerated: this is a GET navigation, the + // run list the link came from is served by the same replica, and a transient not-found self-heals + // on the next load; it drives no mutation. + // (b) frozen-STALE replica (run has since progressed to EXECUTING) → the decision fields projectId / + // runtimeEnvironmentId are STILL CORRECT because they never change — so the redirect target is + // right even off a lagging replica. + heteroRunOpsPostgresTest( + "orgs redirect findRun: frozen-missing → null (transient 404, self-heals); frozen-stale → immutable projectId/runtimeEnvironmentId still correct", + async ({ prisma14, prisma17 }) => { + const seed = await seedLegacyEnvironment(prisma14, "site1_leg"); + const runId = `run_${CUID_25}`; // cuid → LEGACY + const friendlyId = "run_site1"; + await prisma14.taskRun.create({ + data: taskRunData({ + id: runId, + friendlyId, + organizationId: seed.organization.id, + projectId: seed.project.id, + runtimeEnvironmentId: seed.environment.id, + status: "EXECUTING", // primary is already progressed + }), + }); + + const site1 = (router: RoutingRunStore) => + router.findRun( + { friendlyId }, + { select: { projectId: true, runtimeEnvironmentId: true } } + ) as Promise<{ projectId: string; runtimeEnvironmentId: string } | null>; + + // (a) missing: replica has not replicated the run → null → 404. + const missing = laggingReplica(prisma14, [{ model: "taskRun", mode: "missing" }]); + const viaMissing = await site1( + buildRouter(prisma14, prisma17, { legacyReplica: missing.client }) + ); + expect(missing.wasHit("taskRun")).toBe(true); + expect(viaMissing).toBeNull(); + + // Primary contrast: the run genuinely exists (so the null above was purely the lagging replica). + const onPrimary = (await buildRouter(prisma14, prisma17).findRun( + { friendlyId }, + { select: { projectId: true, runtimeEnvironmentId: true } }, + prisma14 + )) as { projectId: string } | null; + expect(onPrimary?.projectId).toBe(seed.project.id); + + // (b) stale: replica shows the OLD PENDING snapshot — but the redirect's decision fields are + // immutable, so the stale row still carries the correct projectId / runtimeEnvironmentId. + const staleRow = { + friendlyId, + projectId: seed.project.id, + runtimeEnvironmentId: seed.environment.id, + status: "PENDING", + }; + const frozen = laggingReplica(prisma14, [ + { model: "taskRun", mode: "frozen", rows: [staleRow] }, + ]); + const viaStale = await site1( + buildRouter(prisma14, prisma17, { legacyReplica: frozen.client }) + ); + expect(frozen.wasHit("taskRun")).toBe(true); + expect(viaStale).not.toBeNull(); + expect(viaStale!.projectId).toBe(seed.project.id); + expect(viaStale!.runtimeEnvironmentId).toBe(seed.environment.id); + } + ); + + // run inspector loader. + // The run inspector fetcher reads a run by friendlyId for a large DISPLAY projection. + // frozen-missing → null → 404 on the fetcher, which self-heals on the next poll/refresh. + // frozen-stale → a stale status/cost is displayed (cosmetic); the immutable id/projectId/ + // runtimeEnvironmentId/createdAt that gate the follow-on auth + attempt reads are still correct. + heteroRunOpsPostgresTest( + "run inspector findRun: frozen-missing → null (fetcher 404 self-heals); frozen-stale → status cosmetic-stale but immutable id/projectId/createdAt correct", + async ({ prisma14, prisma17 }) => { + const seed = await seedLegacyEnvironment(prisma14, "site2_leg"); + const runId = `run_${CUID_25}`; + const friendlyId = "run_site2"; + const createdAt = new Date("2024-01-01T00:00:00.000Z"); + await prisma14.taskRun.create({ + data: taskRunData({ + id: runId, + friendlyId, + organizationId: seed.organization.id, + projectId: seed.project.id, + runtimeEnvironmentId: seed.environment.id, + status: "COMPLETED_SUCCESSFULLY", // primary finished + createdAt, + }), + }); + + const select = { + id: true, + friendlyId: true, + status: true, + projectId: true, + runtimeEnvironmentId: true, + createdAt: true, + } as const; + const site2 = (router: RoutingRunStore) => + router.findRun({ friendlyId }, { select }) as Promise<{ + id: string; + status: string; + projectId: string; + createdAt: Date; + runtimeEnvironmentId: string; + } | null>; + + const missing = laggingReplica(prisma14, [{ model: "taskRun", mode: "missing" }]); + const viaMissing = await site2( + buildRouter(prisma14, prisma17, { legacyReplica: missing.client }) + ); + expect(missing.wasHit("taskRun")).toBe(true); + expect(viaMissing).toBeNull(); + + const staleRow = { + id: runId, + friendlyId, + status: "EXECUTING", // stale — primary is COMPLETED_SUCCESSFULLY + projectId: seed.project.id, + runtimeEnvironmentId: seed.environment.id, + createdAt, + }; + const frozen = laggingReplica(prisma14, [ + { model: "taskRun", mode: "frozen", rows: [staleRow] }, + ]); + const viaStale = await site2( + buildRouter(prisma14, prisma17, { legacyReplica: frozen.client }) + ); + expect(frozen.wasHit("taskRun")).toBe(true); + expect(viaStale).not.toBeNull(); + expect(viaStale!.status).toBe("EXECUTING"); // cosmetic-stale (display only) + // Immutable decision fields correct even off the lagging replica: + expect(viaStale!.id).toBe(runId); + expect(viaStale!.projectId).toBe(seed.project.id); + expect(viaStale!.runtimeEnvironmentId).toBe(seed.environment.id); + expect(viaStale!.createdAt).toEqual(createdAt); + + // Primary shows the fresh status — proving the staleness is confined to the replica. + const onPrimary = (await buildRouter(prisma14, prisma17).findRun( + { friendlyId }, + { select: { status: true } }, + prisma14 + )) as { status: string } | null; + expect(onPrimary?.status).toBe("COMPLETED_SUCCESSFULLY"); + } + ); + + // finished-attempt output findTaskRunAttempt. + // On a finished run, the inspector reads the final TaskRunAttempt (client-less) purely to display its + // output/error. A classifiable taskRunId routes to the owning store's REPLICA. Routed here via a + // run-ops id so we can freeze the #new replica without control-plane FK seeding. Under lag the read + // misses the live attempt (returns null) → the panel shows empty output on a finished run, which + // self-heals on the next load; it drives no mutation. + heteroRunOpsPostgresTest( + "finished-attempt findTaskRunAttempt: frozen-missing NEW replica misses the live attempt (null) — empty-output display self-heals; primary sees it", + async ({ prisma14, prisma17 }) => { + const runId = `run_${NEW_ID_26}`; // run-ops id → NEW (#new / prisma17) + const attemptId = `attempt_${NEW_ID_26}`; + // Seed run + attempt on the dedicated (#new) primary. The dedicated subset schema does not enforce + // the control-plane FKs, so synthetic org/proj/env ids are fine (mirrors batchProbeReadClient). + await prisma17.taskRun.create({ + data: taskRunData({ + id: runId, + friendlyId: "run_site3", + organizationId: "org_site3", + projectId: "proj_site3", + runtimeEnvironmentId: "env_site3", + status: "COMPLETED_SUCCESSFULLY", + }) as never, + }); + await prisma17.taskRunAttempt.create({ + data: { + id: attemptId, + number: 1, + friendlyId: "attempt_site3", + taskRunId: runId, + backgroundWorkerId: `bw_${NEW_ID_26}`, + backgroundWorkerTaskId: `bwt_${NEW_ID_26}`, + runtimeEnvironmentId: "env_site3", + queueId: `queue_${NEW_ID_26}`, + status: "COMPLETED", + output: '{"ok":true}', + outputType: "application/json", + } as never, + }); + + const site3 = (router: RoutingRunStore) => + router.findTaskRunAttempt({ + select: { output: true, outputType: true, error: true }, + where: { + status: { in: FINAL_ATTEMPT_STATUSES as unknown as never }, + taskRunId: runId, + }, + orderBy: { createdAt: "desc" }, + }) as Promise<{ output: string | null } | null>; + + // Freeze the NEW replica (taskRunAttempt missing) → client-less read served there misses it. + const missing = laggingReplica(prisma17 as AnyClient, [ + { model: "taskRunAttempt", mode: "missing" }, + ]); + const viaReplica = await site3( + buildRouter(prisma14, prisma17, { newReplica: missing.client }) + ); + expect(missing.wasHit("taskRunAttempt")).toBe(true); + expect(viaReplica).toBeNull(); // empty output shown on a finished run; self-heals + + // Primary contrast: threading the writer finds the live attempt (proves the miss was pure lag). + const onPrimary = await site3(buildRouter(prisma14, prisma17)).then(() => + buildRouter(prisma14, prisma17).findTaskRunAttempt( + { + select: { output: true, outputType: true, error: true }, + where: { + status: { in: FINAL_ATTEMPT_STATUSES as unknown as never }, + taskRunId: runId, + }, + orderBy: { createdAt: "desc" }, + }, + prisma17 as never + ) + ); + expect(onPrimary).not.toBeNull(); + expect((onPrimary as { output: string }).output).toBe('{"ok":true}'); + } + ); + + // public short-link redirect. + // Reads a run by friendlyId for { spanId, projectId, runtimeEnvironmentId } and redirects to the v3 + // run path (attaching ?span=). All three fields are IMMUTABLE. frozen-missing → + // null → redirectWithErrorMessage ("run doesn't exist or no permission") — a transient error redirect + // that self-heals; frozen-stale → spanId/projectId/runtimeEnvironmentId still correct. + heteroRunOpsPostgresTest( + "short-link redirect findRun: frozen-missing → null (transient error redirect); frozen-stale → immutable spanId/projectId/runtimeEnvironmentId correct", + async ({ prisma14, prisma17 }) => { + const seed = await seedLegacyEnvironment(prisma14, "site4_leg"); + const runId = `run_${CUID_25}`; + const friendlyId = "run_site4"; + const spanId = "span_site4_fixed"; + await prisma14.taskRun.create({ + data: taskRunData({ + id: runId, + friendlyId, + organizationId: seed.organization.id, + projectId: seed.project.id, + runtimeEnvironmentId: seed.environment.id, + status: "EXECUTING", + spanId, + }), + }); + + const site4 = (router: RoutingRunStore) => + router.findRun( + { friendlyId }, + { select: { spanId: true, projectId: true, runtimeEnvironmentId: true } } + ) as Promise<{ spanId: string; projectId: string; runtimeEnvironmentId: string } | null>; + + const missing = laggingReplica(prisma14, [{ model: "taskRun", mode: "missing" }]); + const viaMissing = await site4( + buildRouter(prisma14, prisma17, { legacyReplica: missing.client }) + ); + expect(missing.wasHit("taskRun")).toBe(true); + expect(viaMissing).toBeNull(); + + const staleRow = { + friendlyId, + spanId, + projectId: seed.project.id, + runtimeEnvironmentId: seed.environment.id, + }; + const frozen = laggingReplica(prisma14, [ + { model: "taskRun", mode: "frozen", rows: [staleRow] }, + ]); + const viaStale = await site4( + buildRouter(prisma14, prisma17, { legacyReplica: frozen.client }) + ); + expect(frozen.wasHit("taskRun")).toBe(true); + expect(viaStale).not.toBeNull(); + expect(viaStale!.spanId).toBe(spanId); + expect(viaStale!.projectId).toBe(seed.project.id); + expect(viaStale!.runtimeEnvironmentId).toBe(seed.environment.id); + + const onPrimary = (await buildRouter(prisma14, prisma17).findRun( + { friendlyId }, + { select: { spanId: true } }, + prisma14 + )) as { spanId: string } | null; + expect(onPrimary?.spanId).toBe(spanId); + } + ); + + // admin queue-debug loader. + // Reads a run by friendlyId for { id, engine, queue, concurrencyKey, runtimeEnvironmentId, projectId } + // and introspects the run-queue Redis sets. frozen-missing → null → 404 on an + // admin debug tool (self-heals); frozen-stale → engine/queue/concurrencyKey (all immutable) still + // correct, so the Redis keys it builds are the right ones even off a lagging replica. + heteroRunOpsPostgresTest( + "debug findRun: frozen-missing → null (admin-tool 404 self-heals); frozen-stale → immutable engine/queue/concurrencyKey correct", + async ({ prisma14, prisma17 }) => { + const seed = await seedLegacyEnvironment(prisma14, "site5_leg"); + const runId = `run_${CUID_25}`; + const friendlyId = "run_site5"; + await prisma14.taskRun.create({ + data: taskRunData({ + id: runId, + friendlyId, + organizationId: seed.organization.id, + projectId: seed.project.id, + runtimeEnvironmentId: seed.environment.id, + status: "EXECUTING", + engine: "V2", + concurrencyKey: "ck-site5", + }), + }); + + const select = { + id: true, + engine: true, + friendlyId: true, + queue: true, + concurrencyKey: true, + runtimeEnvironmentId: true, + projectId: true, + } as const; + const site5 = (router: RoutingRunStore) => + router.findRun({ friendlyId }, { select }) as Promise<{ + engine: string; + queue: string; + concurrencyKey: string | null; + projectId: string; + } | null>; + + const missing = laggingReplica(prisma14, [{ model: "taskRun", mode: "missing" }]); + const viaMissing = await site5( + buildRouter(prisma14, prisma17, { legacyReplica: missing.client }) + ); + expect(missing.wasHit("taskRun")).toBe(true); + expect(viaMissing).toBeNull(); + + const staleRow = { + id: runId, + friendlyId, + engine: "V2", + queue: "task/my-task", + concurrencyKey: "ck-site5", + runtimeEnvironmentId: seed.environment.id, + projectId: seed.project.id, + }; + const frozen = laggingReplica(prisma14, [ + { model: "taskRun", mode: "frozen", rows: [staleRow] }, + ]); + const viaStale = await site5( + buildRouter(prisma14, prisma17, { legacyReplica: frozen.client }) + ); + expect(frozen.wasHit("taskRun")).toBe(true); + expect(viaStale).not.toBeNull(); + expect(viaStale!.engine).toBe("V2"); + expect(viaStale!.queue).toBe("task/my-task"); + expect(viaStale!.concurrencyKey).toBe("ck-site5"); + expect(viaStale!.projectId).toBe(seed.project.id); + + const onPrimary = (await buildRouter(prisma14, prisma17).findRun( + { friendlyId }, + { select: { queue: true } }, + prisma14 + )) as { queue: string } | null; + expect(onPrimary?.queue).toBe("task/my-task"); + } + ); + + // log-detail run-status annotation. + // This caller is DISTINCT: it passes the BRANDED $replica as the 3rd findRun arg. The brand is a + // read-replica signal, so readYourWrites=false and the read STAYS on the owning REPLICA (the brand + // never escalates to the primary). The result is used only as `runStatus = run?.status` — an OPTIONAL + // display annotation on a log-detail row. First prove the branded client does + // NOT route to primary (frozen-stale → the read returns the STALE status), then that missing → null → + // runStatus undefined (the caller optional-chains and tolerates it). Contrast: an UNBRANDED writer + // escalates to the primary → fresh status. + heteroRunOpsPostgresTest( + "log-detail findRun(branded $replica): brand keeps read on REPLICA → stale status (tolerated via run?.status); unbranded writer escalates to fresh", + async ({ prisma14, prisma17 }) => { + const seed = await seedLegacyEnvironment(prisma14, "site6_leg"); + const runId = `run_${CUID_25}`; + const friendlyId = "run_site6"; + await prisma14.taskRun.create({ + data: taskRunData({ + id: runId, + friendlyId, + organizationId: seed.organization.id, + projectId: seed.project.id, + runtimeEnvironmentId: seed.environment.id, + status: "COMPLETED_SUCCESSFULLY", // primary finished + }), + }); + + // Frozen replica shows the stale EXECUTING snapshot and backs the legacy store's readOnlyPrisma. + const staleRow = { + friendlyId, + runtimeEnvironmentId: seed.environment.id, + status: "EXECUTING", + }; + const frozen = laggingReplica(prisma14, [ + { model: "taskRun", mode: "frozen", rows: [staleRow] }, + ]); + const router = buildRouter(prisma14, prisma17, { legacyReplica: frozen.client }); + + // The 3rd findRun arg is a ROUTING SIGNAL only — the router reads its presence + brand and never + // forwards it to the store (the store always reads its own readOnlyPrisma). So a standalone + // branded object faithfully stands in for the branded $replica the caller passes, without leaking + // the brand onto prisma14 (branding the frozen Proxy would forward the symbol to its writer target + // and wrongly mark prisma14 as a replica). + const brandedReplicaSignal = markReadReplicaClient({}) as AnyClient; + + // Invoke EXACTLY as the caller does: findRun(where, {select:{status}}, $replica-branded). + const run = (await router.findRun( + { friendlyId, runtimeEnvironmentId: seed.environment.id }, + { select: { status: true } }, + brandedReplicaSignal as never + )) as { status: string } | null; + const runStatus = run?.status; // the caller's runStatus + + expect(frozen.wasHit("taskRun")).toBe(true); + // The BRANDED client did NOT escalate to primary: the read is served by the (stale) replica. + expect(runStatus).toBe("EXECUTING"); // stale — primary is COMPLETED_SUCCESSFULLY + // Tolerated: runStatus is only a display annotation on the log row (`run?.status`), self-heals. + + // Contrast: an UNBRANDED writer IS a read-your-writes signal → escalates to primary → fresh. + const viaWriter = (await router.findRun( + { friendlyId, runtimeEnvironmentId: seed.environment.id }, + { select: { status: true } }, + prisma14 + )) as { status: string } | null; + expect(viaWriter?.status).toBe("COMPLETED_SUCCESSFULLY"); + + // And missing → null → runStatus undefined (optional, tolerated). + const missing = laggingReplica(prisma14, [{ model: "taskRun", mode: "missing" }]); + const routerMissing = buildRouter(prisma14, prisma17, { legacyReplica: missing.client }); + const runMissing = (await routerMissing.findRun( + { friendlyId, runtimeEnvironmentId: seed.environment.id }, + { select: { status: true } }, + brandedReplicaSignal as never + )) as { status: string } | null; + expect(missing.wasHit("taskRun")).toBe(true); + expect(runMissing?.status).toBeUndefined(); + } + ); + + // trace-export download. + // Reads a run by friendlyId for the trace-export window + auth: { friendlyId, traceId, organizationId, + // runtimeEnvironmentId, createdAt, completedAt, taskEventStore, taskIdentifier }. + // frozen-missing → null → the buffer fallback (or 404) handles it, self-heals. frozen-stale is SAFE: + // every field the ClickHouse trace query keys on — traceId, organizationId, runtimeEnvironmentId, + // createdAt, taskEventStore — is IMMUTABLE; the only mutable field, completedAt, is used as the export + // END bound (`run.completedAt ?? undefined`), and a stale null just yields an OPEN-ENDED window (a + // superset of events), never a truncated/lossy export. + heteroRunOpsPostgresTest( + "trace-download findRun: frozen-missing → null (buffer fallback self-heals); frozen-stale → immutable traceId/org/createdAt correct, stale null completedAt = open-ended (superset) not lossy", + async ({ prisma14, prisma17 }) => { + const seed = await seedLegacyEnvironment(prisma14, "site7_leg"); + const runId = `run_${CUID_25}`; + const friendlyId = "run_site7"; + const traceId = "trace_site7_fixed"; + const createdAt = new Date("2024-01-01T00:00:00.000Z"); + const completedAt = new Date("2024-01-01T00:05:00.000Z"); + await prisma14.taskRun.create({ + data: taskRunData({ + id: runId, + friendlyId, + organizationId: seed.organization.id, + projectId: seed.project.id, + runtimeEnvironmentId: seed.environment.id, + status: "COMPLETED_SUCCESSFULLY", + traceId, + createdAt, + completedAt, // primary: the run has completed + }), + }); + + const select = { + friendlyId: true, + traceId: true, + organizationId: true, + runtimeEnvironmentId: true, + createdAt: true, + completedAt: true, + taskEventStore: true, + taskIdentifier: true, + } as const; + const site7 = (router: RoutingRunStore) => + router.findRun({ friendlyId }, { select }) as Promise<{ + traceId: string; + organizationId: string | null; + createdAt: Date; + completedAt: Date | null; + taskEventStore: string; + } | null>; + + const missing = laggingReplica(prisma14, [{ model: "taskRun", mode: "missing" }]); + const viaMissing = await site7( + buildRouter(prisma14, prisma17, { legacyReplica: missing.client }) + ); + expect(missing.wasHit("taskRun")).toBe(true); + expect(viaMissing).toBeNull(); + + // Stale replica: still-RUNNING snapshot, completedAt null (not yet replicated), old status. + const staleRow = { + friendlyId, + traceId, + organizationId: seed.organization.id, + runtimeEnvironmentId: seed.environment.id, + createdAt, + completedAt: null as Date | null, // stale — primary already has a completedAt + taskEventStore: "taskEvent", + taskIdentifier: "my-task", + }; + const frozen = laggingReplica(prisma14, [ + { model: "taskRun", mode: "frozen", rows: [staleRow] }, + ]); + const viaStale = await site7( + buildRouter(prisma14, prisma17, { legacyReplica: frozen.client }) + ); + expect(frozen.wasHit("taskRun")).toBe(true); + expect(viaStale).not.toBeNull(); + // Immutable trace-query keys correct even off the lagging replica: + expect(viaStale!.traceId).toBe(traceId); + expect(viaStale!.organizationId).toBe(seed.organization.id); + expect(viaStale!.createdAt).toEqual(createdAt); + expect(viaStale!.taskEventStore).toBe("taskEvent"); + // The one mutable field is the export END bound: a stale null → open-ended window (superset), the + // export streams MORE events, never fewer — not a lossy/truncated download. + expect(viaStale!.completedAt).toBeNull(); + + const onPrimary = (await buildRouter(prisma14, prisma17).findRun( + { friendlyId }, + { select: { completedAt: true } }, + prisma14 + )) as { completedAt: Date | null } | null; + expect(onPrimary?.completedAt).toEqual(completedAt); // primary has the real end bound + } + ); +}); diff --git a/internal-packages/run-store/src/runOpsStore.routesSpanTraceReadView.replicaLag.test.ts b/internal-packages/run-store/src/runOpsStore.routesSpanTraceReadView.replicaLag.test.ts new file mode 100644 index 00000000000..15754d55724 --- /dev/null +++ b/internal-packages/run-store/src/runOpsStore.routesSpanTraceReadView.replicaLag.test.ts @@ -0,0 +1,322 @@ +// Lagging-replica coverage for the SPAN-DETAIL / TRACE / TRACE-SYNC route read views on the run-ops +// split. Four run lookups back three GET routes; NONE is a read-your-writes mutation gate — every one +// resolves a run for a DISPLAY/GET response and tolerates a stale/missing read by a named caller +// mechanism. This builds the store as each route holds it (`runStore` via a RoutingRunStore), freezes +// the OWNING store's replica with the shared laggingReplica primitive, invokes each read EXACTLY as +// the caller does (same method + branded-$replica client arg + where/select), and asserts the +// under-lag value AND that the SAME read against a caught-up replica recovers the row — so the +// null/empty is provably lag-induced, not a routing accident or true absence. +// +// Routing recap (against runOpsStore.ts on this branch): +// findRun/findRuns with NO client → owning store's REPLICA. +// findRun/findRuns with a BRANDED $replica → stays on the owning REPLICA (a read-replica brand is +// not a write signal). The webapp `$replica` handle is markReadReplicaClient()'d, so passing it +// is routing-equivalent to passing no client. +// findRun/findRuns with a WRITER/tx → owning store's PRIMARY (read-your-writes escalation). +// The routing store never forwards the client across DBs; only its brand/presence is read, so the +// branded stand-in below (markReadReplicaClient({})) faithfully reproduces the webapp arg. +// +// Reads covered (all read the owning REPLICA; each case names the exact tolerance mechanism): +// 1. spans.$spanId findRun({friendlyId, runtimeEnvironmentId}) — classifiable → routed lookup +// (owning replica, miss-probe the other store). +// 2. spans.$spanId findRuns({where:{runtimeEnvironmentId, parentSpanId}, take:50, select}) — open +// predicate → both stores' replicas, merged. +// 3. trace findRun({friendlyId, runtimeEnvironmentId}) — same shape as read 1. +// 4. sync.traces findRun({traceId}, {select:{runtimeEnvironmentId}}) — unclassifiable where → +// unrouted lookup (NEW-first then LEGACY, both replicas). +// +// Real split topology via heteroRunOpsPostgresTest — NEVER mocked. The laggingReplica primitive +// freezes findFirst/findMany/findUnique on a chosen Prisma model so a replica-routed read misses the +// just-written row (see the helper below). + +import { heteroRunOpsPostgresTest, laggingReplica } from "@internal/testcontainers"; +import type { PrismaClient } from "@trigger.dev/database"; +import type { RunOpsPrismaClient } from "@internal/run-ops-database"; +import { describe, expect } from "vitest"; +import { PostgresRunStore } from "./PostgresRunStore.js"; +import { RoutingRunStore } from "./runOpsStore.js"; +import { markReadReplicaClient } from "./readReplicaClient.js"; + +// A cuid (25 chars after the `run_` prefix) classifies LEGACY, so id/friendlyId-keyed reads route to +// the legacy (control-plane) store first — the owning store for the runs seeded here. Trace/span +// detail runs are overwhelmingly legacy-resident (draining DB) in the split window. +const CUID_25 = "c".repeat(25); + +// The webapp passes its `$replica` handle, markReadReplicaClient()'d. Its object identity is +// irrelevant to routing (the router discards it and reads the owning store's readOnlyPrisma) — only +// the brand matters. This stand-in reproduces the arg exactly. +const BRANDED_REPLICA = markReadReplicaClient({}) as never; + +async function seedEnvironment(prisma: PrismaClient, suffix: string) { + const organization = await prisma.organization.create({ + data: { title: `Org ${suffix}`, slug: `org-${suffix}` }, + }); + const project = await prisma.project.create({ + data: { + name: `Project ${suffix}`, + slug: `project-${suffix}`, + externalRef: `proj_${suffix}`, + organizationId: organization.id, + }, + }); + const environment = await prisma.runtimeEnvironment.create({ + data: { + type: "DEVELOPMENT", + slug: "dev", + projectId: project.id, + organizationId: organization.id, + apiKey: `tr_dev_${suffix}`, + pkApiKey: `pk_dev_${suffix}`, + shortcode: `short_${suffix}`, + }, + }); + return { organization, project, environment }; +} + +function taskRunData(opts: { + id: string; + friendlyId: string; + organizationId: string; + projectId: string; + runtimeEnvironmentId: string; + traceId: string; + spanId: string; + parentSpanId?: string; +}) { + return { + id: opts.id, + engine: "V2" as const, + status: "PENDING" as const, + friendlyId: opts.friendlyId, + runtimeEnvironmentId: opts.runtimeEnvironmentId, + environmentType: "DEVELOPMENT" as const, + organizationId: opts.organizationId, + projectId: opts.projectId, + taskIdentifier: "my-task", + payload: "{}", + payloadType: "application/json", + traceContext: {}, + traceId: opts.traceId, + spanId: opts.spanId, + parentSpanId: opts.parentSpanId ?? null, + queue: "task/my-task", + isTest: false, + taskEventStore: "taskEvent", + depth: 0, + }; +} + +// Build the router the way the routes hold it: a LEGACY store on the control-plane DB whose replica +// LAGS, plus a NEW store on the (empty) dedicated DB. Also returns a HEALTHY router (identical, but +// the legacy store reads a caught-up replica handle) sharing the SAME seeded rows in prisma14, so a +// ground-truth re-read proves the under-lag null/empty is purely replica lag. The NEW store's replica +// is left un-frozen (the dedicated DB is empty), so the cross-store miss-probe returns null naturally +// and cannot mask the owning-replica staleness with a phantom hit. +function makeRouters(prisma14: PrismaClient, prisma17: RunOpsPrismaClient) { + const legacyReplica = laggingReplica(prisma14, [{ model: "taskRun", mode: "missing" }]); + const laggingLegacy = new PostgresRunStore({ + prisma: prisma14, + readOnlyPrisma: legacyReplica.client, + schemaVariant: "legacy", + }); + const newStore = new PostgresRunStore({ + prisma: prisma17 as never, + readOnlyPrisma: prisma17 as never, + schemaVariant: "dedicated", + }); + const laggingRouter = new RoutingRunStore({ new: newStore, legacy: laggingLegacy }); + + const healthyLegacy = new PostgresRunStore({ + prisma: prisma14, + readOnlyPrisma: prisma14, // caught-up replica handle + schemaVariant: "legacy", + }); + const healthyRouter = new RoutingRunStore({ new: newStore, legacy: healthyLegacy }); + + return { laggingRouter, healthyRouter, legacyReplica }; +} + +describe("run-ops split — span-detail / trace / trace-sync route read views vs. a lagging replica", () => { + // --- spans.$spanId findPgRun -------------------------------------------------------------------- + // findResource resolves the run for the span-detail GET. Under lag the owning (legacy) replica + // returns null; the route falls through to the mollifier buffer fallback and, failing that, returns + // a retryable 404 (x-should-retry: true). The SDK re-requests, and by the retry the PG replica has + // caught up. No mutation is guarded by this read — a replica-stale null, tolerated by + // eventual-consistency-by-contract (retryable 404 + mollifier buffer fallback). Store fact under + // lag: null. + heteroRunOpsPostgresTest( + "spans.$spanId findPgRun findRun(branded $replica) is NULL under owning-replica lag; caught-up replica recovers the run (retryable 404 + buffer fallback tolerate the stale null)", + async ({ prisma14, prisma17 }) => { + const { laggingRouter, healthyRouter, legacyReplica } = makeRouters(prisma14, prisma17); + const seed = await seedEnvironment(prisma14, "spans_find"); + const runId = `run_${CUID_25}`; + const friendlyId = `run_spansfind0000000000000`; + await prisma14.taskRun.create({ + data: taskRunData({ + id: runId, + friendlyId, + organizationId: seed.organization.id, + projectId: seed.project.id, + runtimeEnvironmentId: seed.environment.id, + traceId: "trace_spans_find", + spanId: "span_spans_find", + }), + }); + + const where = { friendlyId, runtimeEnvironmentId: seed.environment.id }; + + // Exactly the call site: findRun(where, $replica) — no select, branded replica client. + const stale = await laggingRouter.findRun(where, BRANDED_REPLICA); + expect(stale).toBeNull(); + expect(legacyReplica.wasHit()).toBe(true); // proves the read was owning-replica-routed + + // Ground truth: the SAME read against a caught-up replica resolves the run — so the null above + // is purely replica lag. This is a read-only span-detail GET (no mutation depends on it), so the + // tolerated stale null is the intended behaviour. + const recovered = (await healthyRouter.findRun(where, BRANDED_REPLICA)) as { + friendlyId: string; + } | null; + expect(recovered).not.toBeNull(); + expect(recovered!.friendlyId).toBe(friendlyId); + } + ); + + // --- spans.$spanId triggeredRuns ---------------------------------------------------------------- + // The span-detail body lists the CHILD runs a span triggered (where {runtimeEnvironmentId, + // parentSpanId}). Under lag freshly-triggered children are not yet on the owning replica, so the + // list is empty/short. This is a DISPLAY list in a polled span-detail response — a missing child + // simply appears on the next poll; nothing decides or mutates on it. Replica-stale empty list, + // tolerated (display list, re-polled). Store fact under lag: []. + heteroRunOpsPostgresTest( + "spans.$spanId triggeredRuns findRuns(branded $replica) is EMPTY under owning-replica lag; caught-up replica lists the child run (display list, re-polled)", + async ({ prisma14, prisma17 }) => { + const { laggingRouter, healthyRouter, legacyReplica } = makeRouters(prisma14, prisma17); + const seed = await seedEnvironment(prisma14, "spans_children"); + const parentSpanId = "span_parent_children"; + const childId = `run_${CUID_25}`; + const childFriendlyId = `run_spanschild00000000000`; + await prisma14.taskRun.create({ + data: taskRunData({ + id: childId, + friendlyId: childFriendlyId, + organizationId: seed.organization.id, + projectId: seed.project.id, + runtimeEnvironmentId: seed.environment.id, + traceId: "trace_spans_children", + spanId: "span_child_children", + parentSpanId, + }), + }); + + const args = { + take: 50, + select: { + friendlyId: true, + taskIdentifier: true, + status: true, + createdAt: true, + }, + where: { + runtimeEnvironmentId: seed.environment.id, + parentSpanId, + }, + }; + + // Exactly the call site: findRuns(args, $replica) — open predicate, branded replica. + const staleRows = await laggingRouter.findRuns(args, BRANDED_REPLICA); + expect(staleRows).toEqual([]); + expect(legacyReplica.wasHit()).toBe(true); + + // Ground truth: the same open-predicate read against a caught-up replica lists the child. + const recoveredRows = (await healthyRouter.findRuns(args, BRANDED_REPLICA)) as Array<{ + friendlyId: string; + }>; + expect(recoveredRows).toHaveLength(1); + expect(recoveredRows[0]!.friendlyId).toBe(childFriendlyId); + } + ); + + // --- trace findPgRun ---------------------------------------------------------------------------- + // Identical read shape to the spans.$spanId findPgRun read above but for the trace GET. findResource + // resolves the run; a null falls through to the mollifier buffer fallback then a retryable 404. No + // mutation is guarded. Replica-stale null, tolerated by eventual-consistency-by-contract (retryable + // 404 + mollifier buffer fallback). Store fact under lag: null. + heteroRunOpsPostgresTest( + "trace route findPgRun findRun(branded $replica) is NULL under owning-replica lag; caught-up replica recovers the run (retryable 404 + buffer fallback tolerate the stale null)", + async ({ prisma14, prisma17 }) => { + const { laggingRouter, healthyRouter, legacyReplica } = makeRouters(prisma14, prisma17); + const seed = await seedEnvironment(prisma14, "trace_find"); + const runId = `run_${CUID_25}`; + const friendlyId = `run_tracefind00000000000000`; + await prisma14.taskRun.create({ + data: taskRunData({ + id: runId, + friendlyId, + organizationId: seed.organization.id, + projectId: seed.project.id, + runtimeEnvironmentId: seed.environment.id, + traceId: "trace_trace_find", + spanId: "span_trace_find", + }), + }); + + const where = { friendlyId, runtimeEnvironmentId: seed.environment.id }; + + const stale = await laggingRouter.findRun(where, BRANDED_REPLICA); + expect(stale).toBeNull(); + expect(legacyReplica.wasHit()).toBe(true); + + const recovered = (await healthyRouter.findRun(where, BRANDED_REPLICA)) as { + friendlyId: string; + } | null; + expect(recovered).not.toBeNull(); + expect(recovered!.friendlyId).toBe(friendlyId); + } + ); + + // --- sync.traces findRun ------------------------------------------------------------------------ + // The dashboard trace-sync loader resolves a run by traceId to authorize the user + resolve the + // env, then long-polls Electric for the "TaskRun" shape. The where carries ONLY traceId + // (unclassifiable) → unrouted lookup (NEW-first then LEGACY, both on their replicas). Under lag the + // owning replica returns null → the loader returns a 404 "No run found". This is a live-sync loader + // the dashboard re-establishes (long-poll); a run reaching it is user-navigation sourced and has + // existed for seconds, so the stale-null window is transient and re-polled. Tolerated (display/sync + // loader, re-polled; no mutation). Store fact under lag: null. + heteroRunOpsPostgresTest( + "sync.traces findRun(traceId, select, branded $replica) is NULL under owning-replica lag; caught-up replica recovers the run (re-polled sync loader tolerates the stale 404)", + async ({ prisma14, prisma17 }) => { + const { laggingRouter, healthyRouter, legacyReplica } = makeRouters(prisma14, prisma17); + const seed = await seedEnvironment(prisma14, "trace_sync"); + const runId = `run_${CUID_25}`; + const traceId = "trace_sync_unique"; + await prisma14.taskRun.create({ + data: taskRunData({ + id: runId, + friendlyId: `run_tracesync00000000000000`, + organizationId: seed.organization.id, + projectId: seed.project.id, + runtimeEnvironmentId: seed.environment.id, + traceId, + spanId: "span_trace_sync", + }), + }); + + // Exactly the call site: findRun({traceId}, {select:{runtimeEnvironmentId}}, $replica). + const stale = await laggingRouter.findRun( + { traceId }, + { select: { runtimeEnvironmentId: true } }, + BRANDED_REPLICA + ); + expect(stale).toBeNull(); + expect(legacyReplica.wasHit()).toBe(true); + + const recovered = (await healthyRouter.findRun( + { traceId }, + { select: { runtimeEnvironmentId: true } }, + BRANDED_REPLICA + )) as { runtimeEnvironmentId: string } | null; + expect(recovered).not.toBeNull(); + expect(recovered!.runtimeEnvironmentId).toBe(seed.environment.id); + } + ); +}); diff --git a/internal-packages/run-store/src/runOpsStore.serviceBatchFamilyReadView.replicaLag.test.ts b/internal-packages/run-store/src/runOpsStore.serviceBatchFamilyReadView.replicaLag.test.ts new file mode 100644 index 00000000000..6345b86cdff --- /dev/null +++ b/internal-packages/run-store/src/runOpsStore.serviceBatchFamilyReadView.replicaLag.test.ts @@ -0,0 +1,346 @@ +// Lagging-replica coverage for the batch-family reads issued by the batch services with NO client: +// findBatchTaskRunByIdempotencyKey (dedup), findBatchTaskRunById (batchTriggerV3 complete arm + +// tryCompleteBatchV3, runEngine batchTrigger, and three streamBatchItems checks), and +// countBatchTaskRunItems (tryCompleteBatchV3). +// +// The property every case proves: these three store methods are the batch-family PRIMARY defaulters. In +// the router, findBatchTaskRunById / findBatchTaskRunByIdempotencyKey fan out NEW→LEGACY and +// countBatchTaskRunItems routes by batchTaskRunId, each leg via #ownPrimary(store, undefined), which +// returns undefined for a null client; each PostgresRunStore method then falls back to +// `client ?? this.prisma` — the OWNING store's PRIMARY, not readOnlyPrisma. (Contrast +// findBatchTaskRunByFriendlyId / findManyBatchTaskRunItems, which default to readOnlyPrisma — covered +// separately.) +// +// So under owning-replica lag the just-written batch/items are found and no downstream decision is +// driven by a stale value: the dedup does not trigger a DUPLICATE batch; a stale null does not DROP +// batch completion, abort sealing, throw a spurious "Batch not found", or defeat the idempotent-retry +// short-circuit; a stale count does not fail to seal a finished batch. Each case asserts the frozen +// owning replica is NEVER consulted (wasHit === false) AND the row/count is correct — proving PRIMARY +// routing, not replica tolerance. +// +// Builds the RoutingRunStore as batchTriggerV3 / RunEngine hold it (a NEW dedicated store + a LEGACY +// control-plane store), seeds the batch (cuid id → LEGACY-owned) on the LEGACY PRIMARY, freezes the +// LEGACY replica with the shared laggingReplica primitive (batchTaskRun + batchTaskRunItem in "missing" +// mode), then invokes each read EXACTLY as the caller does (same method, same args, NO client). + +import { heteroRunOpsPostgresTest, laggingReplica } from "@internal/testcontainers"; +import type { PrismaClient } from "@trigger.dev/database"; +import type { RunOpsPrismaClient } from "@internal/run-ops-database"; +import { describe, expect } from "vitest"; +import { PostgresRunStore } from "./PostgresRunStore.js"; +import { RoutingRunStore } from "./runOpsStore.js"; +import type { CreateBatchTaskRunData } from "./types.js"; + +// A cuid (25 chars after the `batch_` prefix) classifies LEGACY, so the batch + its items are owned by +// the legacy (control-plane) store; the client-less reads then fan out / route through the LEGACY leg. +const CUID_25 = "c".repeat(25); + +async function seedEnvironment(prisma: PrismaClient, slugSuffix: string) { + const organization = await prisma.organization.create({ + data: { title: `Org ${slugSuffix}`, slug: `org-${slugSuffix}` }, + }); + const project = await prisma.project.create({ + data: { + name: `Project ${slugSuffix}`, + slug: `project-${slugSuffix}`, + externalRef: `proj_${slugSuffix}`, + organizationId: organization.id, + }, + }); + const environment = await prisma.runtimeEnvironment.create({ + data: { + type: "DEVELOPMENT", + slug: "dev", + projectId: project.id, + organizationId: organization.id, + apiKey: `tr_dev_${slugSuffix}`, + pkApiKey: `pk_dev_${slugSuffix}`, + shortcode: `short_${slugSuffix}`, + }, + }); + return { organization, project, environment }; +} + +// Build the RoutingRunStore exactly as the seam wires it: NEW = dedicated subset store (prisma17), +// LEGACY = control-plane store (prisma14) whose REPLICA is the frozen lagging client. The batch models +// are frozen in "missing" mode: if any batch read were replica-routed it would come back null/[]/0 AND +// flip wasHit — so wasHit(false) + a correct result together prove PRIMARY routing. +function buildLaggingRouter(prisma14: PrismaClient, prisma17: RunOpsPrismaClient) { + const legacyReplica = laggingReplica(prisma14, [ + { model: "batchTaskRun", mode: "missing" }, + { model: "batchTaskRunItem", mode: "missing" }, + ]); + const legacyStore = new PostgresRunStore({ + prisma: prisma14, + readOnlyPrisma: legacyReplica.client, + schemaVariant: "legacy", + }); + const newStore = new PostgresRunStore({ + prisma: prisma17 as never, + readOnlyPrisma: prisma17 as never, + schemaVariant: "dedicated", + }); + const router = new RoutingRunStore({ new: newStore, legacy: legacyStore }); + return { router, legacyStore, legacyReplica }; +} + +function batchData(overrides: Partial): CreateBatchTaskRunData { + return { + id: `batch_${CUID_25}`, + friendlyId: "batch_svc_family_f", + runtimeEnvironmentId: "PLACEHOLDER", + status: "PENDING", + runCount: 1, + expectedCount: 1, + batchVersion: "runengine:v2", + sealed: false, + ...overrides, + }; +} + +// Seed BatchTaskRunItem rows on the LEGACY primary with FK triggers disabled (no real TaskRun needed — +// countBatchTaskRunItems only matches batchTaskRunId + status). +async function seedItems( + prisma: PrismaClient, + opts: { batchTaskRunId: string; status: string; count: number; idPrefix: string } +) { + // `session_replication_role` is session-scoped, so `SET` and the inserts must share one connection — + // separate pooled Prisma calls can land on different connections, leaving the inserts with FK triggers + // still enabled. Run them in a single transaction with `SET LOCAL`, which applies for the duration of + // this transaction (same connection) and auto-resets on commit. + await prisma.$transaction(async (tx) => { + await tx.$executeRawUnsafe(`SET LOCAL session_replication_role = replica`); + for (let i = 0; i < opts.count; i++) { + await tx.$executeRawUnsafe( + `INSERT INTO "BatchTaskRunItem" (id, status, "batchTaskRunId", "taskRunId", "createdAt", "updatedAt") + VALUES ($1, $2::"BatchTaskRunItemStatus", $3, $4, NOW(), NOW())`, + `${opts.idPrefix}_${i}`, + opts.status, + opts.batchTaskRunId, + `run_item_${opts.idPrefix}_${i}` + ); + } + }); +} + +describe("batch-service family batch reads (no client) route to the owning PRIMARY under replica lag", () => { + // ── findBatchTaskRunByIdempotencyKey (dedup) ────────────────────── + heteroRunOpsPostgresTest( + "findBatchTaskRunByIdempotencyKey dedup hits the LEGACY primary, never the frozen replica (no duplicate batch)", + async ({ prisma14, prisma17 }) => { + const { router, legacyStore, legacyReplica } = buildLaggingRouter(prisma14, prisma17); + const seed = await seedEnvironment(prisma14, "bf_idem"); + const idempotencyKey = "batch-idem-key"; + await legacyStore.createBatchTaskRun( + batchData({ + friendlyId: "batch_bf_idem_f", + runtimeEnvironmentId: seed.environment.id, + idempotencyKey, + idempotencyKeyExpiresAt: new Date(Date.now() + 60 * 60_000), + }) + ); + + // Exact caller invocation: (environment.id, idempotencyKey), NO client. + const found = await router.findBatchTaskRunByIdempotencyKey( + seed.environment.id, + idempotencyKey + ); + + expect(found).not.toBeNull(); + expect(found!.idempotencyKey).toBe(idempotencyKey); + // Primary routing proof: the frozen LEGACY replica was never consulted. + expect(legacyReplica.wasHit("batchTaskRun")).toBe(false); + } + ); + + // ── findBatchTaskRunById (complete arm) ──────────────────────────── + heteroRunOpsPostgresTest( + "findBatchTaskRunById complete-arm lookup hits the LEGACY primary, never the frozen replica", + async ({ prisma14, prisma17 }) => { + const { router, legacyStore, legacyReplica } = buildLaggingRouter(prisma14, prisma17); + const seed = await seedEnvironment(prisma14, "bf_666"); + const batchId = `batch_${CUID_25}`; + await legacyStore.createBatchTaskRun( + batchData({ + id: batchId, + friendlyId: "batch_bf_666_f", + runtimeEnvironmentId: seed.environment.id, + }) + ); + + const batch = await router.findBatchTaskRunById(batchId); + + expect(batch).not.toBeNull(); + expect(batch!.id).toBe(batchId); + expect(legacyReplica.wasHit("batchTaskRun")).toBe(false); + } + ); + + // ── findBatchTaskRunById (tryCompleteBatchV3) ───────────────────── + heteroRunOpsPostgresTest( + "findBatchTaskRunById in tryCompleteBatchV3 hits the LEGACY primary, never the frozen replica", + async ({ prisma14, prisma17 }) => { + const { router, legacyStore, legacyReplica } = buildLaggingRouter(prisma14, prisma17); + const seed = await seedEnvironment(prisma14, "bf_1016"); + const batchId = `batch_${CUID_25}`; + await legacyStore.createBatchTaskRun( + batchData({ + id: batchId, + friendlyId: "batch_bf_1016_f", + runtimeEnvironmentId: seed.environment.id, + status: "PENDING", + sealed: true, + expectedCount: 1, + }) + ); + + const batch = await router.findBatchTaskRunById(batchId); + + expect(batch).not.toBeNull(); + expect(batch!.id).toBe(batchId); + // tryCompleteBatchV3 reads batch.status/sealed/expectedCount off this row — a stale null returns early. + expect(batch!.sealed).toBe(true); + expect(legacyReplica.wasHit("batchTaskRun")).toBe(false); + } + ); + + // ── countBatchTaskRunItems (tryCompleteBatchV3) ─────────────────── + heteroRunOpsPostgresTest( + "countBatchTaskRunItems in tryCompleteBatchV3 counts on the LEGACY primary, never the frozen replica", + async ({ prisma14, prisma17 }) => { + const { router, legacyStore, legacyReplica } = buildLaggingRouter(prisma14, prisma17); + const seed = await seedEnvironment(prisma14, "bf_1033"); + const batchId = `batch_${CUID_25}`; + await legacyStore.createBatchTaskRun( + batchData({ + id: batchId, + friendlyId: "batch_bf_1033_f", + runtimeEnvironmentId: seed.environment.id, + expectedCount: 3, + runCount: 3, + }) + ); + // 3 COMPLETED items on the LEGACY primary; the frozen replica would report 0. + await seedItems(prisma14, { + batchTaskRunId: batchId, + status: "COMPLETED", + count: 3, + idPrefix: "bti_1033", + }); + + // Exact caller invocation: ({ batchTaskRunId, status }), NO client. + const completedCount = await router.countBatchTaskRunItems({ + batchTaskRunId: batchId, + status: "COMPLETED", + }); + + expect(completedCount).toBe(3); + // Primary routing proof: the frozen LEGACY item replica was never consulted (else it returns 0). + expect(legacyReplica.wasHit("batchTaskRunItem")).toBe(false); + } + ); + + // ── runEngine batchTrigger findBatchTaskRunById ────────────────────────── + heteroRunOpsPostgresTest( + "findBatchTaskRunById in runEngine batchTrigger hits the LEGACY primary, never the frozen replica", + async ({ prisma14, prisma17 }) => { + const { router, legacyStore, legacyReplica } = buildLaggingRouter(prisma14, prisma17); + const seed = await seedEnvironment(prisma14, "bf_360"); + const batchId = `batch_${CUID_25}`; + await legacyStore.createBatchTaskRun( + batchData({ + id: batchId, + friendlyId: "batch_bf_360_f", + runtimeEnvironmentId: seed.environment.id, + }) + ); + + const batch = await router.findBatchTaskRunById(batchId); + + expect(batch).not.toBeNull(); + expect(batch!.id).toBe(batchId); + expect(legacyReplica.wasHit("batchTaskRun")).toBe(false); + } + ); + + // ── streamBatchItems findBatchTaskRunById (validate exists) ─────────────────────── + heteroRunOpsPostgresTest( + "findBatchTaskRunById existence check in streamBatchItems hits the LEGACY primary, never the frozen replica", + async ({ prisma14, prisma17 }) => { + const { router, legacyStore, legacyReplica } = buildLaggingRouter(prisma14, prisma17); + const seed = await seedEnvironment(prisma14, "bf_134"); + const batchId = `batch_${CUID_25}`; + await legacyStore.createBatchTaskRun( + batchData({ + id: batchId, + friendlyId: "batch_bf_134_f", + runtimeEnvironmentId: seed.environment.id, + }) + ); + + const batch = await router.findBatchTaskRunById(batchId); + + // The caller does: `if (!batch || batch.runtimeEnvironmentId !== environment.id) throw 404`. + expect(batch).not.toBeNull(); + expect(batch!.runtimeEnvironmentId).toBe(seed.environment.id); + const wouldThrow404 = !batch || batch.runtimeEnvironmentId !== seed.environment.id; + expect(wouldThrow404).toBe(false); + expect(legacyReplica.wasHit("batchTaskRun")).toBe(false); + } + ); + + // ── streamBatchItems findBatchTaskRunById (idempotent-retry recheck) ────────────── + heteroRunOpsPostgresTest( + "findBatchTaskRunById idempotent-retry recheck in streamBatchItems hits the LEGACY primary, never the frozen replica", + async ({ prisma14, prisma17 }) => { + const { router, legacyStore, legacyReplica } = buildLaggingRouter(prisma14, prisma17); + const seed = await seedEnvironment(prisma14, "bf_206"); + const batchId = `batch_${CUID_25}`; + await legacyStore.createBatchTaskRun( + batchData({ + id: batchId, + friendlyId: "batch_bf_206_f", + runtimeEnvironmentId: seed.environment.id, + status: "COMPLETED", + sealed: true, + processingCompletedAt: new Date(), + }) + ); + + const currentBatch = await router.findBatchTaskRunById(batchId); + + expect(currentBatch).not.toBeNull(); + // isIdempotentRetrySuccess(status, sealed, processingCompletedAt) reads these three off the row. + expect(currentBatch!.status).toBe("COMPLETED"); + expect(currentBatch!.sealed).toBe(true); + expect(legacyReplica.wasHit("batchTaskRun")).toBe(false); + } + ); + + // ── streamBatchItems findBatchTaskRunById (idempotent-retry recheck, second) ────────────── + heteroRunOpsPostgresTest( + "findBatchTaskRunById idempotent-retry recheck (second) in streamBatchItems hits the LEGACY primary, never the frozen replica", + async ({ prisma14, prisma17 }) => { + const { router, legacyStore, legacyReplica } = buildLaggingRouter(prisma14, prisma17); + const seed = await seedEnvironment(prisma14, "bf_294"); + const batchId = `batch_${CUID_25}`; + await legacyStore.createBatchTaskRun( + batchData({ + id: batchId, + friendlyId: "batch_bf_294_f", + runtimeEnvironmentId: seed.environment.id, + status: "COMPLETED", + sealed: true, + processingCompletedAt: new Date(), + }) + ); + + const currentBatch = await router.findBatchTaskRunById(batchId); + + expect(currentBatch).not.toBeNull(); + expect(currentBatch!.status).toBe("COMPLETED"); + expect(currentBatch!.sealed).toBe(true); + expect(legacyReplica.wasHit("batchTaskRun")).toBe(false); + } + ); +}); diff --git a/internal-packages/run-store/src/runOpsStore.sessionMetadataRouteReadView.replicaLag.test.ts b/internal-packages/run-store/src/runOpsStore.sessionMetadataRouteReadView.replicaLag.test.ts new file mode 100644 index 00000000000..17a133bee7e --- /dev/null +++ b/internal-packages/run-store/src/runOpsStore.sessionMetadataRouteReadView.replicaLag.test.ts @@ -0,0 +1,231 @@ +// Coverage for two friendlyId-keyed run reads that pass the webapp's BRANDED `$replica` into +// `runStore.findRun`. A branded replica makes `readYourWrites` return false, so the routing store keeps +// the read on the OWNING store's REPLICA (no primary escalation). Under lag both miss a row present on +// the owning primary. Real split topology via heteroRunOpsPostgresTest; the branded arg is +// `markReadReplicaClient({})`, whose object is never forwarded across DBs — only its BRAND is read. +// +// Reads covered (one case each): +// 1. end-and-continue callingRun lookup +// runStore.findRun({ friendlyId: callingRunId, runtimeEnvironmentId }, { select:{id} }, $replica) +// → owning REPLICA. On null → 404 "callingRunId not found in this environment". +// SAFE: `callingRunId` is `ctx.run.id` of the run CURRENTLY EXECUTING and issuing this API call. +// An executing run was materialised to the primary and dequeued long before it runs user code, so +// its row replicated well before this request — bounded lag cannot miss it. (Contrast the same +// route's read of the just-swapped run, which correctly passes the WRITER `prisma`.) +// 2. metadata GET loader +// runStore.findRun({ friendlyId, runtimeEnvironmentId }, { select:{metadata,metadataType} }, $replica) +// → owning REPLICA. On null the loader's only miss fallback is the mollifier buffer +// (buffer.getEntry — no primary re-read). Once a run has drained from the buffer to the primary +// but the replica has not caught up, it exists on NEITHER the replica NOR the buffer. The primary +// re-read (pass the WRITER `prisma`, i.e. findRunOnPrimary) recovers the live run before the 404. + +import { heteroRunOpsPostgresTest, laggingReplica } from "@internal/testcontainers"; +import type { PrismaClient } from "@trigger.dev/database"; +import type { RunOpsPrismaClient } from "@internal/run-ops-database"; +import { describe, expect } from "vitest"; +import { PostgresRunStore } from "./PostgresRunStore.js"; +import { RoutingRunStore } from "./runOpsStore.js"; +import { markReadReplicaClient } from "./readReplicaClient.js"; +import type { CreateRunInput } from "./types.js"; + +// A cuid (25 chars after the `run_` prefix) classifies LEGACY, so create + friendlyId-keyed read +// both route to the legacy (control-plane) store first — the store that owns these session/run rows. +const CUID_25 = "c".repeat(25); + +async function seedEnvironmentLegacy(prisma: PrismaClient, suffix: string) { + const organization = await prisma.organization.create({ + data: { title: `Org ${suffix}`, slug: `org-${suffix}` }, + }); + const project = await prisma.project.create({ + data: { + name: `Project ${suffix}`, + slug: `project-${suffix}`, + externalRef: `proj_${suffix}`, + organizationId: organization.id, + }, + }); + const environment = await prisma.runtimeEnvironment.create({ + data: { + type: "DEVELOPMENT", + slug: "dev", + projectId: project.id, + organizationId: organization.id, + apiKey: `tr_dev_${suffix}`, + pkApiKey: `pk_dev_${suffix}`, + shortcode: `short_${suffix}`, + }, + }); + return { organization, project, environment }; +} + +function buildCreateRunInput(params: { + runId: string; + friendlyId: string; + organizationId: string; + projectId: string; + runtimeEnvironmentId: string; + metadata?: string; + metadataType?: string; +}): CreateRunInput { + return { + data: { + id: params.runId, + engine: "V2", + status: "PENDING", + friendlyId: params.friendlyId, + runtimeEnvironmentId: params.runtimeEnvironmentId, + environmentType: "DEVELOPMENT", + organizationId: params.organizationId, + projectId: params.projectId, + taskIdentifier: "my-task", + payload: '{"hello":"world"}', + payloadType: "application/json", + metadata: params.metadata, + metadataType: params.metadataType, + context: { foo: "bar" }, + traceContext: { trace: "ctx" }, + traceId: "trace_1", + spanId: "span_1", + runTags: ["alpha"], + queue: "task/my-task", + isTest: false, + taskEventStore: "taskEvent", + depth: 0, + createdAt: new Date("2024-01-01T00:00:00.000Z"), + }, + snapshot: { + engine: "V2", + executionStatus: "RUN_CREATED", + description: "Run was created", + runStatus: "PENDING", + environmentId: params.runtimeEnvironmentId, + environmentType: "DEVELOPMENT", + projectId: params.projectId, + organizationId: params.organizationId, + }, + }; +} + +// Build the router the way the webapp holds it, with a lagging legacy (control-plane) replica. +function buildRouterWithLaggingLegacyReplica(prisma14: PrismaClient, prisma17: RunOpsPrismaClient) { + const legacyReplica = laggingReplica(prisma14, [{ model: "taskRun", mode: "missing" }]); + const legacyStore = new PostgresRunStore({ + prisma: prisma14, + readOnlyPrisma: legacyReplica.client, + schemaVariant: "legacy", + }); + const newStore = new PostgresRunStore({ + prisma: prisma17 as never, + readOnlyPrisma: prisma17 as never, + schemaVariant: "dedicated", + }); + const router = new RoutingRunStore({ new: newStore, legacy: legacyStore }); + return { router, legacyStore, legacyReplica }; +} + +describe("session and metadata route findRun read-views under replica lag", () => { + // end-and-continue callingRun lookup. + heteroRunOpsPostgresTest( + "end-and-continue callingRun read is stale-null under replica lag", + async ({ prisma14, prisma17 }) => { + const { router, legacyStore, legacyReplica } = buildRouterWithLaggingLegacyReplica( + prisma14, + prisma17 + ); + + const seed = await seedEnvironmentLegacy(prisma14, "eac_leg"); + const runId = `run_${CUID_25}`; // cuid → LEGACY + const friendlyId = "run_eac_calling"; + await legacyStore.createRun( + buildCreateRunInput({ + runId, + friendlyId, + organizationId: seed.organization.id, + projectId: seed.project.id, + runtimeEnvironmentId: seed.environment.id, + }) + ); + + // The branded $replica client arg (its object is never forwarded; only its brand is read). + const brandedReplica = markReadReplicaClient({} as object); + + // The EXACT :78 read: friendlyId + runtimeEnvironmentId, select {id}, branded $replica. + const staleRead = (await router.findRun( + { friendlyId, runtimeEnvironmentId: seed.environment.id }, + { select: { id: true } }, + brandedReplica + )) as { id: string } | null; + + // Store fact: branded-replica read is REPLICA-routed, so under lag it misses → null. + // A null return leads to a 404 "callingRunId not found in this environment". + expect(staleRead).toBeNull(); + expect(legacyReplica.wasHit()).toBe(true); + + // TOLERANCE premise proof: the row DOES exist on the owning primary. In production + // `callingRunId` = ctx.run.id of the CURRENTLY-EXECUTING caller, materialised + dequeued long + // before it runs user code, so it replicated well before the request — bounded lag can't miss + // it. The WRITER read (unbranded → readYourWrites → owning primary) recovers it here. + const primaryRead = (await router.findRun( + { friendlyId, runtimeEnvironmentId: seed.environment.id }, + { select: { id: true } }, + prisma14 as never + )) as { id: string } | null; + expect(primaryRead).not.toBeNull(); + expect(primaryRead!.id).toBe(runId); + } + ); + + // metadata GET loader. + heteroRunOpsPostgresTest( + "metadata loader read is stale-null under replica lag", + async ({ prisma14, prisma17 }) => { + const { router, legacyStore, legacyReplica } = buildRouterWithLaggingLegacyReplica( + prisma14, + prisma17 + ); + + const seed = await seedEnvironmentLegacy(prisma14, "meta_leg"); + const runId = `run_${CUID_25}`; // cuid → LEGACY + const friendlyId = "run_meta_view"; + const metadata = '{"phase":"one"}'; + await legacyStore.createRun( + buildCreateRunInput({ + runId, + friendlyId, + organizationId: seed.organization.id, + projectId: seed.project.id, + runtimeEnvironmentId: seed.environment.id, + metadata, + metadataType: "application/json", + }) + ); + + const brandedReplica = markReadReplicaClient({} as object); + + // The EXACT :43 read: friendlyId + runtimeEnvironmentId, select {metadata, metadataType}, branded $replica. + const staleRead = (await router.findRun( + { friendlyId, runtimeEnvironmentId: seed.environment.id }, + { select: { metadata: true, metadataType: true } }, + brandedReplica + )) as { metadata: string | null; metadataType: string } | null; + + // Store fact: branded-replica read is REPLICA-routed → stale/null under lag. The loader then + // consults ONLY the mollifier buffer (buffer.getEntry, no primary read); once the run has drained + // from the buffer to the primary, the buffer miss + this replica miss together yield a 404 + // "Run not found" for a run that exists on the owning primary. + expect(staleRead).toBeNull(); + expect(legacyReplica.wasHit()).toBe(true); + + // The property: re-reading the owning PRIMARY (pass the WRITER `prisma` → findRunOnPrimary) returns + // the live run with its metadata, so the primary re-read on the double-miss resolves it. + const primaryRead = (await router.findRun( + { friendlyId, runtimeEnvironmentId: seed.environment.id }, + { select: { metadata: true, metadataType: true } }, + prisma14 as never + )) as { metadata: string | null; metadataType: string } | null; + expect(primaryRead).not.toBeNull(); + expect(primaryRead!.metadata).toBe(metadata); + expect(primaryRead!.metadataType).toBe("application/json"); + } + ); +}); diff --git a/internal-packages/run-store/src/runOpsStore.sessionRunProbeReadAfterWrite.test.ts b/internal-packages/run-store/src/runOpsStore.sessionRunProbeReadAfterWrite.test.ts new file mode 100644 index 00000000000..11de9b0345f --- /dev/null +++ b/internal-packages/run-store/src/runOpsStore.sessionRunProbeReadAfterWrite.test.ts @@ -0,0 +1,211 @@ +// Store-seam characterization of the read primitive behind the session-run liveness probe — NOT a +// caller guard. Under owning-replica lag, findRun({id},{select},$replica) misses the just-written run +// (→ null) and the same findRun on the WRITER recovers it. This exercises only the store reads against +// real Postgres. The caller contract (ensureRunForSession reuses the live run and does NOT double- +// trigger the session) is locked separately by the real caller-driven realtime-services replica-lag +// guard, which drives ensureRunForSession end-to-end. + +import { heteroRunOpsPostgresTest, laggingReplica } from "@internal/testcontainers"; +import type { PrismaClient } from "@trigger.dev/database"; +import { describe, expect } from "vitest"; +import { PostgresRunStore } from "./PostgresRunStore.js"; +import { RoutingRunStore } from "./runOpsStore.js"; + +// ownerEngine classifies by internal-id LENGTH: 25 chars → cuid → LEGACY, 26 → run-ops id → NEW. +const CUID_25 = "c".repeat(25); // → LEGACY (#legacy / prisma14, full schema) +const NEW_ID_26 = "k".repeat(24) + "01"; // → NEW (#new / prisma17, dedicated subset schema) + +// The probe select mirrors getRunStatusAndFriendlyId exactly. +const PROBE_SELECT = { select: { status: true, friendlyId: true } } as const; + +async function seedEnvironmentLegacy(prisma: PrismaClient, suffix: string) { + const organization = await prisma.organization.create({ + data: { title: `Org ${suffix}`, slug: `org-${suffix}` }, + }); + const project = await prisma.project.create({ + data: { + name: `Project ${suffix}`, + slug: `project-${suffix}`, + externalRef: `proj_${suffix}`, + organizationId: organization.id, + }, + }); + const environment = await prisma.runtimeEnvironment.create({ + data: { + type: "DEVELOPMENT", + slug: "dev", + projectId: project.id, + organizationId: organization.id, + apiKey: `tr_dev_${suffix}`, + pkApiKey: `pk_dev_${suffix}`, + shortcode: `short_${suffix}`, + }, + }); + return { organization, project, environment }; +} + +function seedEnvironmentDedicated(suffix: string) { + return { + organization: { id: `org_${suffix}` }, + project: { id: `proj_${suffix}` }, + environment: { id: `env_${suffix}` }, + }; +} + +function taskRunData(opts: { + id: string; + friendlyId: string; + organizationId: string; + projectId: string; + runtimeEnvironmentId: string; + status?: "PENDING" | "EXECUTING" | "COMPLETED_SUCCESSFULLY"; +}) { + return { + id: opts.id, + engine: "V2" as const, + // Non-final by default: this is the "run is still alive, reuse it" case the probe must recover. + status: opts.status ?? "EXECUTING", + friendlyId: opts.friendlyId, + runtimeEnvironmentId: opts.runtimeEnvironmentId, + environmentType: "DEVELOPMENT" as const, + organizationId: opts.organizationId, + projectId: opts.projectId, + taskIdentifier: "my-task", + payload: "{}", + payloadType: "application/json", + traceContext: {}, + traceId: `trace_${opts.id}`, + spanId: `span_${opts.id}`, + queue: "task/my-task", + isTest: false, + taskEventStore: "taskEvent", + depth: 0, + }; +} + +describe("store reads behind the session-run liveness probe — replica misses, writer recovers", () => { + // (a) LEGACY-resident (cuid) session run: triggered moments ago on the control-plane writer; the + // control-plane replica lags. The branded-$replica probe misses; the writer re-probe finds the + // fresh EXECUTING run → ensureRunForSession reuses it instead of double-triggering. + heteroRunOpsPostgresTest( + "LEGACY cuid: branded-replica probe misses under lag; writer re-probe finds the live run", + async ({ prisma14, prisma17 }) => { + const legacyReplica = laggingReplica(prisma14, [{ model: "taskRun", mode: "missing" }]); + const legacyStore = new PostgresRunStore({ + prisma: prisma14, + readOnlyPrisma: legacyReplica.client, + schemaVariant: "legacy", + }); + const newStore = new PostgresRunStore({ + prisma: prisma17 as never, + readOnlyPrisma: prisma17 as never, + schemaVariant: "dedicated", + }); + const router = new RoutingRunStore({ new: newStore, legacy: legacyStore }); + + const seed = await seedEnvironmentLegacy(prisma14, "sess_leg"); + const runId = `run_${CUID_25}`; // cuid → LEGACY + await prisma14.taskRun.create({ + data: taskRunData({ + id: runId, + friendlyId: "run_sess_leg", + organizationId: seed.organization.id, + projectId: seed.project.id, + runtimeEnvironmentId: seed.environment.id, + status: "EXECUTING", + }), + }); + + // Step 1 — the getRunStatusAndFriendlyId probe. The call site passes the branded $replica; at + // the routing layer that is identical to omitting the client (both route to the owning store's + // OWN replica and neither escalates to the primary — the brand only suppresses escalation). + // Under lag the owning replica hasn't applied the fresh row → MISS. A naive caller treating this + // null as "run vanished" would double-trigger the session. + const viaReplica = await router.findRun({ id: runId }, PROBE_SELECT); + expect(viaReplica).toBeNull(); + expect(legacyReplica.wasHit()).toBe(true); + + // Step 2 — ensureRunForSession's compensation: re-read on the control-plane WRITER before + // deciding liveness. Fresh row is found with its non-final status → the session reuses the + // still-alive run and does NOT double-trigger. + const legacyReplica2 = laggingReplica(prisma14, [{ model: "taskRun", mode: "missing" }]); + const legacyStore2 = new PostgresRunStore({ + prisma: prisma14, + readOnlyPrisma: legacyReplica2.client, + schemaVariant: "legacy", + }); + const router2 = new RoutingRunStore({ new: newStore, legacy: legacyStore2 }); + const viaWriter = (await router2.findRun({ id: runId }, PROBE_SELECT, prisma14)) as { + status: string; + friendlyId: string; + } | null; + expect(viaWriter).not.toBeNull(); + expect(viaWriter!.friendlyId).toBe("run_sess_leg"); + // Non-final status ⇒ isFinalRunStatus(probe.status) is false ⇒ reuse branch (return existing). + expect(viaWriter!.status).toBe("EXECUTING"); + // The re-probe hit the WRITER, never the replica. + expect(legacyReplica2.wasHit()).toBe(false); + } + ); + + // (b) NEW-resident (run-ops id) session run under split: the session row + currentRunId live on the + // control-plane, but the run itself is on the NEW DB whose replica lags. The probe passes the + // BRANDED control-plane $replica; the router routes by residency to the NEW store's OWN replica → + // miss. The writer re-probe (control-plane writer identity) routes to the NEW store's OWN writer → + // finds the fresh run. Mirrors the live shape (sessions pass the control-plane clients; the run may + // be NEW-resident under split-ON). + heteroRunOpsPostgresTest( + "NEW run-ops id: branded-replica probe misses under NEW-replica lag; writer re-probe finds the live run", + async ({ prisma14, prisma17 }) => { + const newReplica = laggingReplica(prisma17, [{ model: "taskRun", mode: "missing" }]); + const newStore = new PostgresRunStore({ + prisma: prisma17 as never, + readOnlyPrisma: newReplica.client as never, + schemaVariant: "dedicated", + }); + const legacyStore = new PostgresRunStore({ + prisma: prisma14, + readOnlyPrisma: prisma14, + schemaVariant: "legacy", + }); + const router = new RoutingRunStore({ new: newStore, legacy: legacyStore }); + + const seed = seedEnvironmentDedicated("sess_new"); + const runId = `run_${NEW_ID_26}`; // run-ops id → NEW + await prisma17.taskRun.create({ + data: taskRunData({ + id: runId, + friendlyId: "run_sess_new", + organizationId: seed.organization.id, + projectId: seed.project.id, + runtimeEnvironmentId: seed.environment.id, + status: "EXECUTING", + }), + }); + + // Step 1 — the probe routes by residency to the NEW store's OWN (lagging) replica → miss. + const viaReplica = await router.findRun({ id: runId }, PROBE_SELECT); + expect(viaReplica).toBeNull(); + expect(newReplica.wasHit()).toBe(true); + + // Step 2 — writer re-probe. Passing the control-plane WRITER (writer identity) routes to the + // NEW store's OWN writer (never forwarding the control-plane client into the NEW DB) → finds + // the fresh run with its non-final status → reuse, no double-trigger. + const newReplica2 = laggingReplica(prisma17, [{ model: "taskRun", mode: "missing" }]); + const newStore2 = new PostgresRunStore({ + prisma: prisma17 as never, + readOnlyPrisma: newReplica2.client as never, + schemaVariant: "dedicated", + }); + const router2 = new RoutingRunStore({ new: newStore2, legacy: legacyStore }); + const viaWriter = (await router2.findRun({ id: runId }, PROBE_SELECT, prisma14)) as { + status: string; + friendlyId: string; + } | null; + expect(viaWriter).not.toBeNull(); + expect(viaWriter!.friendlyId).toBe("run_sess_new"); + expect(viaWriter!.status).toBe("EXECUTING"); + expect(newReplica2.wasHit()).toBe(false); + } + ); +}); diff --git a/internal-packages/run-store/src/runOpsStore.storeReceiverReadView.replicaLag.test.ts b/internal-packages/run-store/src/runOpsStore.storeReceiverReadView.replicaLag.test.ts new file mode 100644 index 00000000000..58f788f9d80 --- /dev/null +++ b/internal-packages/run-store/src/runOpsStore.storeReceiverReadView.replicaLag.test.ts @@ -0,0 +1,288 @@ +// Lagging-replica coverage for two RunStore reads whose receiver is `store` / `deps.store`. Both are +// split-mode reads RoutingRunStore serves from the OWNING store's REPLICA (no writer / branded replica). +// This file freezes the owning replica via laggingReplica, invokes each read with the EXACT caller args, +// asserts the under-lag behavior, then recovers the row on the owning PRIMARY (proving pure lag + routing). +// (1) findBatchTaskRunByFriendlyId (no client): under lag returns null and the caller's `if(batch)` guard +// falls through — no throw — leaving the friendlyId unresolved for a ClickHouse filter that lags more. +// (2) readThroughRun's readLegacy leg (deps.store.findRun): a LEGACY-resident run's pre-cutover row is +// long replicated, so findRun returns the OLD row (stale scalars, correct immutable identity), never +// null; the handler enriches from event-supplied endTime/updatedAt. + +import { + heteroRunOpsPostgresTest, + laggingReplica, + type LaggingModel, +} from "@internal/testcontainers"; +import type { PrismaClient } from "@trigger.dev/database"; +import type { RunOpsPrismaClient } from "@internal/run-ops-database"; +import { describe, expect } from "vitest"; +import { PostgresRunStore } from "./PostgresRunStore.js"; +import { RoutingRunStore } from "./runOpsStore.js"; +import { markReadReplicaClient } from "./readReplicaClient.js"; + +// A cuid (25 chars after the `run_`/`batch_` prefix) classifies LEGACY, so create + read both route to +// the legacy (control-plane / prisma14, full schema) store — the store that owns these rows in prod. +const CUID_25 = "c".repeat(25); + +async function seedEnvironmentLegacy(prisma: PrismaClient, suffix: string) { + const organization = await prisma.organization.create({ + data: { title: `Org ${suffix}`, slug: `org-${suffix}` }, + }); + const project = await prisma.project.create({ + data: { + name: `Project ${suffix}`, + slug: `project-${suffix}`, + externalRef: `proj_${suffix}`, + organizationId: organization.id, + }, + }); + const environment = await prisma.runtimeEnvironment.create({ + data: { + type: "DEVELOPMENT", + slug: "dev", + projectId: project.id, + organizationId: organization.id, + apiKey: `tr_dev_${suffix}`, + pkApiKey: `pk_dev_${suffix}`, + shortcode: `short_${suffix}`, + }, + }); + return { organization, project, environment }; +} + +// Build the router the way the webapp holds it, with the LEGACY (control-plane) replica FROZEN per the +// given configs. NEW is non-lagging + empty, so the router's miss-probe of NEW returns null/[] naturally +// (no phantom hit masking the legacy-replica staleness). +function buildRouter( + prisma14: PrismaClient, + prisma17: RunOpsPrismaClient, + configs: readonly LaggingModel[] +) { + const legacyReplica = laggingReplica(prisma14, configs); + const legacyStore = new PostgresRunStore({ + prisma: prisma14, + readOnlyPrisma: legacyReplica.client, + schemaVariant: "legacy", + }); + const newStore = new PostgresRunStore({ + prisma: prisma17 as never, + readOnlyPrisma: prisma17 as never, + schemaVariant: "dedicated", + }); + const router = new RoutingRunStore({ new: newStore, legacy: legacyStore }); + return { router, legacyStore, legacyReplica }; +} + +describe("run-ops split — store/deps.store receiver read-views vs. a lagging replica", () => { + // ── findBatchTaskRunByFriendlyId (no client) — runsRepository runs-list batch filter ─────────────── + heteroRunOpsPostgresTest( + "findBatchTaskRunByFriendlyId (no client) → owning REPLICA; under lag returns null, the caller's `if(batch)` guard falls through leaving the friendlyId unresolved for the ClickHouse filter (tolerated: CH lags more, list revalidates). Primary resolves the id.", + async ({ prisma14, prisma17 }) => { + const { router, legacyStore, legacyReplica } = buildRouter(prisma14, prisma17, [ + { model: "batchTaskRun", mode: "missing" }, + ]); + const seed = await seedEnvironmentLegacy(prisma14, "batchfilter"); + const batchId = `batch_${CUID_25}`; // cuid → LEGACY-resident + const batchFriendlyId = "batch_batchfilter"; + await legacyStore.createBatchTaskRun({ + id: batchId, + friendlyId: batchFriendlyId, + runtimeEnvironmentId: seed.environment.id, + }); + + // Invoke EXACTLY as convertRunListInputOptionsToFilterRunsOptions:329 does — friendlyId + envId, + // NO client, NO include. + const underLag = await router.findBatchTaskRunByFriendlyId( + batchFriendlyId, + seed.environment.id + ); + + // Store-level fact: default readOnlyPrisma → owning REPLICA (both legs). Frozen → null. + expect(underLag).toBeNull(); + expect(legacyReplica.wasHit("batchTaskRun")).toBe(true); + + // Reproduce the caller's guard and its downstream effect on the ClickHouse filter: on null it + // leaves `batchId` as the unresolved friendlyId — a filter value, not an error (no throw here). + const convertedBatchId = (() => { + let out: string = batchFriendlyId; // convertedOptions.batchId starts as options.batchId + const batch = underLag; // store.findBatchTaskRunByFriendlyId(...) + if (batch) { + out = (batch as { id: string }).id; + } + return out; + })(); + // The friendlyId flows UNRESOLVED into the ClickHouse `batch_id` filter (matches nothing → empty + // list) — consistent with ClickHouse's own laggier state (no child run of a just-created batch is + // ingested yet), and self-heals on the next poll once the replica catches up. + expect(convertedBatchId).toBe(batchFriendlyId); + expect(convertedBatchId.startsWith("batch_")).toBe(true); + + // PRIMARY contrast: an unbranded WRITER client → #ownPrimary → owning primary → the batch IS + // resolved to its internal id. Proves the null above is purely replica lag + replica routing. + const onPrimary = await router.findBatchTaskRunByFriendlyId( + batchFriendlyId, + seed.environment.id, + undefined, + prisma14 as never + ); + expect(onPrimary).not.toBeNull(); + expect(onPrimary!.id).toBe(batchId); + expect(onPrimary!.friendlyId).toBe(batchFriendlyId); + // With the primary read, the caller WOULD resolve the filter to the internal id. + const resolvedWithPrimary = onPrimary ? onPrimary.id : batchFriendlyId; + expect(resolvedWithPrimary).toBe(batchId); + } + ); + + // ── readLegacy-leg findRun({id},{select},replica) — readThroughRun for a LEGACY run ──────────── + // The readLegacy leg of readThroughRun for a LEGACY-resident run. Real behavior: the run row is OLD + // (pre-cutover, long replicated), so findRun returns it with STALE mutable scalars — never null — and + // the event handler enriches from event-supplied time + the row's immutable identifiers. + heteroRunOpsPostgresTest( + "readLegacy-leg findRun({id}) (branded $replica) → owning REPLICA; a LEGACY run's OLD row is present-but-STALE under lag (not a missing-row RYW); immutable spanId/traceId/friendlyId/createdAt are correct and the handler uses event-supplied endTime/updatedAt. Primary shows the fresh status.", + async ({ prisma14, prisma17 }) => { + const runId = `run_${CUID_25}`; // cuid → LEGACY-resident (pre-cutover row) + const friendlyId = "run_evtenrich_leg"; + const spanId = "span_evtenrich_fixed"; + const parentSpanId = "span_evtenrich_parent"; + const traceId = `trace_${runId}`; + const createdAt = new Date("2024-01-01T00:00:00.000Z"); // long before cut-over + + // The exact select the runSucceeded handler uses. + const eventSelect = { + id: true, + friendlyId: true, + traceId: true, + spanId: true, + parentSpanId: true, + createdAt: true, + completedAt: true, + taskIdentifier: true, + projectId: true, + runtimeEnvironmentId: true, + environmentType: true, + isTest: true, + organizationId: true, + taskEventStore: true, + runTags: true, + batchId: true, + } as const; + + // Frozen replica: an OLD snapshot of the SAME row — the selected mutable scalar stale (no + // completedAt yet), immutable identifiers identical to the primary. laggingReplica "frozen" returns + // this verbatim (bypasses Postgres projection), matched on top-level `where` id equality. + const staleRow = { + id: runId, + friendlyId, + traceId, + spanId, + parentSpanId, + createdAt, + completedAt: null as Date | null, // STALE — primary already has a completedAt + taskIdentifier: "my-task", + projectId: "", // filled after seed + runtimeEnvironmentId: "", // filled after seed + environmentType: "DEVELOPMENT", + isTest: false, + organizationId: "", // filled after seed + taskEventStore: "taskEvent", + runTags: [] as string[], + batchId: null as string | null, + }; + + const { router, legacyReplica } = buildRouter(prisma14, prisma17, [ + { model: "taskRun", mode: "frozen", rows: [staleRow] }, + ]); + const seed = await seedEnvironmentLegacy(prisma14, "evtenrich"); + staleRow.projectId = seed.project.id; + staleRow.runtimeEnvironmentId = seed.environment.id; + staleRow.organizationId = seed.organization.id; + + const completedAt = new Date("2024-06-01T00:00:00.000Z"); + await prisma14.taskRun.create({ + data: { + id: runId, + engine: "V2", + status: "COMPLETED_SUCCESSFULLY", // the terminal state the event fires on + friendlyId, + runtimeEnvironmentId: seed.environment.id, + environmentType: "DEVELOPMENT", + organizationId: seed.organization.id, + projectId: seed.project.id, + taskIdentifier: "my-task", + payload: "{}", + payloadType: "application/json", + traceContext: {}, + traceId, + spanId, + parentSpanId, + queue: "task/my-task", + isTest: false, + taskEventStore: "taskEvent", + depth: 0, + createdAt, + completedAt, + }, + }); + + // Invoke EXACTLY as the readLegacy closure does: findRun({id},{select}, ). + const brandedReplica = markReadReplicaClient({} as object); + const underLag = (await router.findRun( + { id: runId }, + { select: eventSelect }, + brandedReplica as never + )) as typeof staleRow | null; + + expect(legacyReplica.wasHit("taskRun")).toBe(true); + // KEY REAL-BEHAVIOR FACT: the read returns the OLD row (present), NOT null — a legacy-resident run's + // row was inserted pre-cutover and is long replicated. So this is a stale-row read, not a + // missing-row read-your-write. + expect(underLag).not.toBeNull(); + expect(underLag!.id).toBe(runId); + // The selected mutable scalar is STALE off the frozen replica… + expect(underLag!.completedAt).toBeNull(); + // …but every IMMUTABLE identifier the event enrichment actually depends on is correct under lag. + expect(underLag!.friendlyId).toBe(friendlyId); + expect(underLag!.spanId).toBe(spanId); + expect(underLag!.parentSpanId).toBe(parentSpanId); + expect(underLag!.traceId).toBe(traceId); + expect(underLag!.createdAt).toEqual(createdAt); + expect(underLag!.taskEventStore).toBe("taskEvent"); + expect(underLag!.organizationId).toBe(seed.organization.id); + + // TOLERANCE assertion: reproduce the handler's use of the enrichment read. endTime and updatedAtMs + // come from the EVENT payload (not the stale row); the span-close/publish keys are immutable row + // fields. So the emitted completion event + realtime publish are CORRECT despite the stale status. + const eventTime = new Date("2024-06-01T00:00:01.000Z"); // engine event `time` + const eventRunUpdatedAt = new Date("2024-06-01T00:00:01.500Z"); // engine event `run.updatedAt` + const completionEvent = { + endTime: eventTime, // completeSuccessfulRunEvent({ run, endTime: time }) + runId: underLag!.id, + spanId: underLag!.spanId, + traceId: underLag!.traceId, + friendlyId: underLag!.friendlyId, + taskEventStore: underLag!.taskEventStore, + }; + const publish = { + runId: underLag!.id, + envId: seed.environment.id, + tags: underLag!.runTags, + batchId: underLag!.batchId, + updatedAtMs: eventRunUpdatedAt.getTime(), // from event, not the stale row + }; + expect(completionEvent.endTime).toEqual(eventTime); // not derived from the stale row + expect(completionEvent.spanId).toBe(spanId); // immutable → correct under lag + expect(publish.updatedAtMs).toBe(eventRunUpdatedAt.getTime()); + + // PRIMARY contrast: an unbranded WRITER client → #ownPrimary → owning primary → the FRESH terminal + // status. Proves the staleness above is confined to the replica. + const onPrimary = (await router.findRun( + { id: runId }, + { select: { status: true, completedAt: true } }, + prisma14 as never + )) as { status: string; completedAt: Date | null } | null; + expect(onPrimary?.status).toBe("COMPLETED_SUCCESSFULLY"); + expect(onPrimary?.completedAt).toEqual(completedAt); + } + ); +}); diff --git a/internal-packages/run-store/src/runOpsStore.test.ts b/internal-packages/run-store/src/runOpsStore.test.ts index e64bf9f28c3..75f838deb34 100644 --- a/internal-packages/run-store/src/runOpsStore.test.ts +++ b/internal-packages/run-store/src/runOpsStore.test.ts @@ -1643,17 +1643,22 @@ describe("RoutingRunStore.findTaskRunAttempt residency routing", () => { status?: string; } ) { - await prisma.$executeRawUnsafe(`SET session_replication_role = replica`); - await prisma.$executeRawUnsafe( - `INSERT INTO "TaskRunAttempt" (id, number, "friendlyId", "taskRunId", "backgroundWorkerId", "backgroundWorkerTaskId", "runtimeEnvironmentId", "queueId", status, "createdAt", "updatedAt", "usageDurationMs", "outputType") - VALUES ($1, 1, $2, $3, 'synthetic-worker', 'synthetic-worker-task', $4, 'synthetic-queue', $5::"TaskRunAttemptStatus", NOW(), NOW(), 0, 'application/json')`, - opts.attemptId, - opts.friendlyId, - opts.runId, - opts.runtimeEnvironmentId, - opts.status ?? "COMPLETED" - ); - await prisma.$executeRawUnsafe(`SET session_replication_role = DEFAULT`); + // `session_replication_role` is session-scoped, so the `SET` and the insert must share one + // connection — separate pooled calls can split across connections, leaving the insert with FK + // triggers on (the synthetic worker/queue ids have no backing rows). One transaction with + // `SET LOCAL` keeps them co-connected and auto-resets on commit. + await prisma.$transaction(async (tx) => { + await tx.$executeRawUnsafe(`SET LOCAL session_replication_role = replica`); + await tx.$executeRawUnsafe( + `INSERT INTO "TaskRunAttempt" (id, number, "friendlyId", "taskRunId", "backgroundWorkerId", "backgroundWorkerTaskId", "runtimeEnvironmentId", "queueId", status, "createdAt", "updatedAt", "usageDurationMs", "outputType") + VALUES ($1, 1, $2, $3, 'synthetic-worker', 'synthetic-worker-task', $4, 'synthetic-queue', $5::"TaskRunAttemptStatus", NOW(), NOW(), 0, 'application/json')`, + opts.attemptId, + opts.friendlyId, + opts.runId, + opts.runtimeEnvironmentId, + opts.status ?? "COMPLETED" + ); + }); } heteroPostgresTest( diff --git a/internal-packages/run-store/src/runOpsStore.ttlExpireOrphanReplicaLag.test.ts b/internal-packages/run-store/src/runOpsStore.ttlExpireOrphanReplicaLag.test.ts new file mode 100644 index 00000000000..bcc0b1e07fc --- /dev/null +++ b/internal-packages/run-store/src/runOpsStore.ttlExpireOrphanReplicaLag.test.ts @@ -0,0 +1,222 @@ +// Guard for the run-ops split read-after-write property on the TTL BATCH EXPIRY path. +// +// TtlSystem.expireRunsBatch reads the runs it is about to expire with +// runStore.findRuns({ where: { id: { in: runIds } }, select: {...} }, this.$.prisma) +// threading the owning primary (as the single-run path expireRun does), so it reads the run it just +// wrote. Why that matters: a client-less findRuns is NOT a read-your-writes signal, so RoutingRunStore +// routes each leg to the owning store's REPLICA. A run whose row is not yet visible there comes back +// absent, is bucketed `not_found`, skipped, and never expired — and since the TTL Lua script has +// ALREADY removed the run from the Redis queue before this guard runs, that miss does not self-heal: +// the run would be permanently ORPHANED (never expired, never emits `runExpired`, never re-queued). +// +// This is the weakest of the read-after-write class: the run row is usually written well before TTL +// fires, so a lagging replica has normally caught up. The window only exists for a very-short-TTL run +// swept moments after creation. This test forces the lag with the shared lagging-replica primitive +// and pins both paths: the primary-threaded read (the production path) expires the run, while the +// client-less path misses it on the replica and buckets it not_found. +// +// Deterministic via heteroRunOpsPostgresTest (real split topology, NEVER mocked): a LEGACY-resident +// (cuid) run is created on the control-plane writer; the control-plane replica lags (its `taskRun` +// reads come back empty). We drive the EXACT guard logic ttlSystem.expireRunsBatch uses. + +import { heteroRunOpsPostgresTest, laggingReplica } from "@internal/testcontainers"; +import type { PrismaClient } from "@trigger.dev/database"; +import { describe, expect } from "vitest"; +import { PostgresRunStore } from "./PostgresRunStore.js"; +import { RoutingRunStore } from "./runOpsStore.js"; +import type { CreateRunInput, ReadClient } from "./types.js"; + +// ownerEngine (classifyResidency) routes a run-ops v1 body → NEW, everything else → LEGACY. +const CUID_25 = "d".repeat(25); // → LEGACY (control-plane / prisma14, full schema) + +async function seedEnvironment(prisma: PrismaClient, slugSuffix: string) { + const organization = await prisma.organization.create({ + data: { title: `Org ${slugSuffix}`, slug: `org-${slugSuffix}` }, + }); + const project = await prisma.project.create({ + data: { + name: `Project ${slugSuffix}`, + slug: `project-${slugSuffix}`, + externalRef: `proj_${slugSuffix}`, + organizationId: organization.id, + }, + }); + const environment = await prisma.runtimeEnvironment.create({ + data: { + type: "DEVELOPMENT", + slug: "dev", + projectId: project.id, + organizationId: organization.id, + apiKey: `tr_dev_${slugSuffix}`, + pkApiKey: `pk_dev_${slugSuffix}`, + shortcode: `short_${slugSuffix}`, + }, + }); + return { organization, project, environment }; +} + +function buildCreateRunInput(params: { + runId: string; + friendlyId: string; + organizationId: string; + projectId: string; + runtimeEnvironmentId: string; +}): CreateRunInput { + return { + data: { + id: params.runId, + engine: "V2", + status: "PENDING", + friendlyId: params.friendlyId, + runtimeEnvironmentId: params.runtimeEnvironmentId, + environmentType: "DEVELOPMENT", + organizationId: params.organizationId, + projectId: params.projectId, + taskIdentifier: "my-task", + payload: '{"hello":"world"}', + payloadType: "application/json", + context: { foo: "bar" }, + traceContext: { trace: "ctx" }, + traceId: "trace_1", + spanId: "span_1", + runTags: [], + queue: "task/my-task", + isTest: false, + taskEventStore: "taskEvent", + depth: 0, + ttl: "1s", + createdAt: new Date("2024-01-01T00:00:00.000Z"), + }, + snapshot: { + engine: "V2", + executionStatus: "RUN_CREATED", + description: "Run was created", + runStatus: "PENDING", + environmentId: params.runtimeEnvironmentId, + environmentType: "DEVELOPMENT", + projectId: params.projectId, + organizationId: params.organizationId, + }, + }; +} + +// A line-for-line mirror of the guard in TtlSystem.expireRunsBatch: read the candidate runs, filter +// to PENDING+unlocked, and bucket every candidate id whose row is absent as `not_found`. `readClient` +// models the two paths: the client-less batch guard passes nothing (replica default); the +// read-your-writes path threads the primary (as single-run expireRun does). +async function guardExpireBatch( + runStore: RoutingRunStore, + runIds: string[], + readClient?: ReadClient +): Promise<{ expired: string[]; skipped: { runId: string; reason: string }[] }> { + const expired: string[] = []; + const skipped: { runId: string; reason: string }[] = []; + + const runs = (await runStore.findRuns( + { + where: { id: { in: runIds } }, + select: { + id: true, + spanId: true, + status: true, + lockedAt: true, + ttl: true, + taskEventStore: true, + createdAt: true, + associatedWaitpoint: { select: { id: true } }, + organizationId: true, + projectId: true, + runtimeEnvironmentId: true, + }, + }, + readClient + )) as Array<{ id: string; status: string; lockedAt: Date | null }>; + + const runsToExpire: typeof runs = []; + for (const run of runs) { + if (run.status !== "PENDING") { + skipped.push({ runId: run.id, reason: `status_${run.status}` }); + continue; + } + if (run.lockedAt) { + skipped.push({ runId: run.id, reason: "locked" }); + continue; + } + runsToExpire.push(run); + } + + const foundRunIds = new Set(runs.map((r) => r.id)); + for (const runId of runIds) { + if (!foundRunIds.has(runId)) { + skipped.push({ runId, reason: "not_found" }); + } + } + + // The rest of expireRunsBatch would write EXPIRED + emit runExpired for runsToExpire only. + for (const run of runsToExpire) { + expired.push(run.id); + } + + return { expired, skipped }; +} + +describe("run-ops split — TTL batch expiry guard threads the owning primary; the client-less path orphans the run", () => { + // A LEGACY-resident (cuid) run sits PENDING on the control-plane WRITER; the control-plane REPLICA + // lags (its `taskRun` reads come back empty, as just after a very-short-TTL run is created). The + // TTL Lua script has already dequeued the run, so this guard is its only chance to be expired. + heteroRunOpsPostgresTest( + "LEGACY cuid: primary-threaded guard expires the run; the client-less control path misses it on the lagging replica -> orphaned as not_found", + async ({ prisma14, prisma17 }) => { + // Control-plane replica lags on `taskRun`: the just-created run row is not visible yet. + const legacyReplica = laggingReplica(prisma14, [{ model: "taskRun", mode: "missing" }]); + const legacyStore = new PostgresRunStore({ + prisma: prisma14, + readOnlyPrisma: legacyReplica.client, + schemaVariant: "legacy", + }); + const newStore = new PostgresRunStore({ + prisma: prisma17 as never, + readOnlyPrisma: prisma17 as never, + schemaVariant: "dedicated", + }); + const router = new RoutingRunStore({ new: newStore, legacy: legacyStore }); + + const seed = await seedEnvironment(prisma14, "ttl_leg"); + const runId = `run_${CUID_25}`; // cuid → LEGACY + await legacyStore.createRun( + buildCreateRunInput({ + runId, + friendlyId: "run_ttl_leg", + organizationId: seed.organization.id, + projectId: seed.project.id, + runtimeEnvironmentId: seed.environment.id, + }) + ); + + // Sanity: the run really is PENDING on the primary (so the only reason a guard could skip it is + // the misrouted replica read, not a genuinely-missing/non-PENDING run). + const onPrimary = await legacyStore.findRun({ id: runId }, prisma14); + expect(onPrimary).not.toBeNull(); + expect(onPrimary!.status).toBe("PENDING"); + + // Contrast — the read-your-writes path (thread the primary, as single-run expireRun does): the + // run is found on the owning store's writer and IS expired. Establishes the run is genuinely + // expirable, so any miss below is purely the misrouted read. + const viaPrimary = await guardExpireBatch(router, [runId], prisma14); + expect(viaPrimary.expired).toEqual([runId]); + expect(viaPrimary.skipped).toEqual([]); + + // The client-less control path: findRuns with NO client → owning store's REPLICA. + const guard = await guardExpireBatch(router, [runId]); + + // Proves the misrouted read is what happened: the (stale) legacy replica was consulted. + expect(legacyReplica.wasHit("taskRun")).toBe(true); + + // The property the primary read secures: a client-less findRuns misses the run on the lagging + // replica and buckets it `not_found`, so the run would be ORPHANED — never expired. Pinning it + // here documents why expireRunsBatch threads this.$.prisma (the viaPrimary path above). + expect(guard.expired).toEqual([]); + expect(guard.skipped).toEqual([{ runId, reason: "not_found" }]); + } + ); +}); diff --git a/internal-packages/run-store/src/runOpsStore.usageCostUndercountReadAfterWrite.test.ts b/internal-packages/run-store/src/runOpsStore.usageCostUndercountReadAfterWrite.test.ts new file mode 100644 index 00000000000..fcbe19eea5a --- /dev/null +++ b/internal-packages/run-store/src/runOpsStore.usageCostUndercountReadAfterWrite.test.ts @@ -0,0 +1,253 @@ +// Read-after-write coverage for the USAGE/COST read-modify-write in the attempt-completion path +// (attemptSucceeded, and the same shape in cancelRun / #permanentlyFailRun). The completion path reads +// the run's CUMULATIVE usageDurationMs/costInCents, recomputes, and writes back. Across a run's retries +// usage accumulates on the OWNING store's PRIMARY (each completion is a read-modify-write); if the read +// were served by a lagging replica pinned at a pre-delta snapshot, the recomputed total would be built +// on the STALE cumulative and the prior delta LOST -> usage/cost UNDERCOUNT (a classic lost update). +// Reading via findRunOnPrimary (the owning store's PRIMARY) keeps the total correct. +// +// Deterministic via heteroRunOpsPostgresTest (real split topology, never mocked) + the shared +// laggingReplica primitive (frozen mode: the owning store's replica is pinned at the pre-delta snapshot +// while the primary advances). The property: the primary read sees the fresher cumulative and the write +// preserves it; a client-less (replica) read would clobber it downward. + +import { heteroRunOpsPostgresTest, laggingReplica } from "@internal/testcontainers"; +import type { PrismaClient } from "@trigger.dev/database"; +import type { RunOpsPrismaClient } from "@internal/run-ops-database"; +import { describe, expect } from "vitest"; +import { PostgresRunStore } from "./PostgresRunStore.js"; +import { RoutingRunStore } from "./runOpsStore.js"; +import type { CompletionSnapshotInput, CreateRunInput, RunStoreSchemaVariant } from "./types.js"; + +type AnyClient = PrismaClient | RunOpsPrismaClient; + +// ownerEngine (classifyResidency) routes a cuid → LEGACY (#legacy / prisma14, full schema). +const CUID_25 = "d".repeat(25); + +async function seedEnvironment( + prisma: AnyClient, + schemaVariant: RunStoreSchemaVariant, + slugSuffix: string +) { + if (schemaVariant === "dedicated") { + return { + organization: { id: `org_${slugSuffix}` }, + project: { id: `proj_${slugSuffix}` }, + environment: { id: `env_${slugSuffix}` }, + }; + } + const organization = await (prisma as PrismaClient).organization.create({ + data: { title: `Org ${slugSuffix}`, slug: `org-${slugSuffix}` }, + }); + const project = await (prisma as PrismaClient).project.create({ + data: { + name: `Project ${slugSuffix}`, + slug: `project-${slugSuffix}`, + externalRef: `proj_${slugSuffix}`, + organizationId: organization.id, + }, + }); + const environment = await (prisma as PrismaClient).runtimeEnvironment.create({ + data: { + type: "DEVELOPMENT", + slug: "dev", + projectId: project.id, + organizationId: organization.id, + apiKey: `tr_dev_${slugSuffix}`, + pkApiKey: `pk_dev_${slugSuffix}`, + shortcode: `short_${slugSuffix}`, + }, + }); + return { organization, project, environment }; +} + +function buildCreateRunInput(params: { + runId: string; + friendlyId: string; + taskIdentifier: string; + organizationId: string; + projectId: string; + runtimeEnvironmentId: string; +}): CreateRunInput { + return { + data: { + id: params.runId, + engine: "V2", + status: "EXECUTING", + friendlyId: params.friendlyId, + runtimeEnvironmentId: params.runtimeEnvironmentId, + environmentType: "DEVELOPMENT", + organizationId: params.organizationId, + projectId: params.projectId, + taskIdentifier: params.taskIdentifier, + payload: '{"hello":"world"}', + payloadType: "application/json", + context: { foo: "bar" }, + traceContext: { trace: "ctx" }, + traceId: "trace_1", + spanId: "span_1", + runTags: [], + queue: "task/my-task", + isTest: false, + taskEventStore: "taskEvent", + depth: 0, + createdAt: new Date("2024-01-01T00:00:00.000Z"), + }, + snapshot: { + engine: "V2", + executionStatus: "EXECUTING", + description: "Run is executing", + runStatus: "EXECUTING", + environmentId: params.runtimeEnvironmentId, + environmentType: "DEVELOPMENT", + projectId: params.projectId, + organizationId: params.organizationId, + }, + }; +} + +// Line-for-line mirror of runAttemptSystem.attemptSucceeded's usage read-modify-write (dev-mode: +// cost is a pure passthrough of the read `currentCostInCents`, per #calculateUpdatedUsage's +// `environmentType !== "DEVELOPMENT"` guard). The cumulative read goes to the owning primary via +// findRunOnPrimary, exactly as the engine issues it. +async function attemptSucceededRecomputeUsage( + router: RoutingRunStore, + params: { + runId: string; + attemptDurationMs: number; + snapshot: CompletionSnapshotInput; + } +): Promise { + // Read current usage totals via findRunOnPrimary (owning store's PRIMARY), as + // attemptSucceeded/cancelRun/#permanentlyFailRun do. + const currentRun = (await router.findRunOnPrimary( + { id: params.runId }, + { + select: { + usageDurationMs: true, + costInCents: true, + machinePreset: true, + }, + } + )) as { usageDurationMs: number; costInCents: number; machinePreset: string | null } | null; + + if (!currentRun) { + throw new Error("Run not found"); + } + + // #calculateUpdatedUsage, DEVELOPMENT branch: usage accumulates, cost passes through unchanged. + const usageDurationMs = currentRun.usageDurationMs + params.attemptDurationMs; + const costInCents = currentRun.costInCents; + + await router.completeAttemptSuccess( + params.runId, + { + completedAt: new Date(), + output: '{"done":true}', + outputType: "application/json", + usageDurationMs, + costInCents, + snapshot: params.snapshot, + }, + { select: { id: true, usageDurationMs: true, costInCents: true } } + ); +} + +describe("run-ops split — attempt-completion usage/cost read-modify-write must read the OWNING store's WRITER, not its lagging replica", () => { + // A LEGACY-resident (cuid) run whose earlier attempts already accumulated usage/cost on the + // control-plane PRIMARY, while the control-plane REPLICA still lags at the pre-delta snapshot. + // The final attemptSucceeded reads the cumulative usage on the owning primary, recomputes, and + // writes back — so the fresher primary total is preserved (no lost update); a replica-routed read + // would clobber it downward. + heteroRunOpsPostgresTest( + "LEGACY cuid: attemptSucceeded accumulates usage on the primary total, not the lagging replica", + async ({ prisma14, prisma17 }) => { + const seed = await seedEnvironment(prisma14, "legacy", "usage_leg"); + const runId = `run_${CUID_25}`; // cuid → LEGACY + + // Build the store on the PRIMARY first so createRun + the "prior attempt" write both land there. + const primaryLegacyStore = new PostgresRunStore({ + prisma: prisma14, + readOnlyPrisma: prisma14, + schemaVariant: "legacy", + }); + await primaryLegacyStore.createRun( + buildCreateRunInput({ + runId, + friendlyId: "run_usage_leg", + taskIdentifier: "my-task", + organizationId: seed.organization.id, + projectId: seed.project.id, + runtimeEnvironmentId: seed.environment.id, + }) + ); + + // The pre-delta snapshot the replica is still pinned at: usage/cost as freshly created (both 0). + const staleRun = await prisma14.taskRun.findFirstOrThrow({ where: { id: runId } }); + const PRIOR_USAGE = staleRun.usageDurationMs; // 0 at create + const PRIOR_COST = staleRun.costInCents; // 0 at create + + // A prior attempt's read-modify-write already committed a cumulative delta to the PRIMARY. + const PRIOR_ATTEMPT_USAGE_MS = 1000; + const PRIOR_ATTEMPT_COST = 250; + await prisma14.taskRun.update({ + where: { id: runId }, + data: { + usageDurationMs: PRIOR_USAGE + PRIOR_ATTEMPT_USAGE_MS, + costInCents: PRIOR_COST + PRIOR_ATTEMPT_COST, + }, + }); + + // The control-plane replica lags: frozen at the pre-delta snapshot (usage/cost = 0). + const legacyReplica = laggingReplica(prisma14, [ + { + model: "taskRun", + mode: "frozen", + rows: [staleRun as unknown as Record], + }, + ]); + const legacyStore = new PostgresRunStore({ + prisma: prisma14, + readOnlyPrisma: legacyReplica.client, + schemaVariant: "legacy", + }); + const newStore = new PostgresRunStore({ + prisma: prisma17 as never, + readOnlyPrisma: prisma17 as never, + schemaVariant: "dedicated", + }); + const router = new RoutingRunStore({ new: newStore, legacy: legacyStore }); + + // The final successful attempt adds its own delta on top of the accumulated total. + const FINAL_ATTEMPT_USAGE_MS = 500; + await attemptSucceededRecomputeUsage(router, { + runId, + attemptDurationMs: FINAL_ATTEMPT_USAGE_MS, + snapshot: { + executionStatus: "FINISHED", + description: "Task completed successfully", + runStatus: "COMPLETED_SUCCESSFULLY", + attemptNumber: 2, + environmentId: seed.environment.id, + environmentType: "DEVELOPMENT", + projectId: seed.project.id, + organizationId: seed.organization.id, + }, + }); + + // The cumulative-usage read hits the owning PRIMARY, so the lagging replica is never consulted + // (a client-less read would hit it and undercount). + expect(legacyReplica.wasHit("taskRun")).toBe(false); + + // Ground truth lives on the PRIMARY. Correct cumulative total = prior accumulated + this attempt. + const finalRun = await prisma14.taskRun.findFirstOrThrow({ where: { id: runId } }); + + // The primary read sees the accumulated 1000ms/250c, so the RMW writes 1000 + 500 = 1500 (no + // undercount). A client-less (replica) read would recompute 0 + 500 = 500. + expect(finalRun.usageDurationMs).toBe( + PRIOR_USAGE + PRIOR_ATTEMPT_USAGE_MS + FINAL_ATTEMPT_USAGE_MS + ); + expect(finalRun.costInCents).toBe(PRIOR_COST + PRIOR_ATTEMPT_COST); + } + ); +}); diff --git a/internal-packages/run-store/src/runOpsStore.waitpointCompleteRouteAndReplayLoaderReadView.replicaLag.test.ts b/internal-packages/run-store/src/runOpsStore.waitpointCompleteRouteAndReplayLoaderReadView.replicaLag.test.ts new file mode 100644 index 00000000000..8493c66b052 --- /dev/null +++ b/internal-packages/run-store/src/runOpsStore.waitpointCompleteRouteAndReplayLoaderReadView.replicaLag.test.ts @@ -0,0 +1,246 @@ +// Coverage for two dashboard read call sites: +// +// complete-waitpoint route (dashboard "Complete waitpoint" action) +// runStore.findWaitpoint({ select: { projectId, environmentId }, where: { id } }) (no client) +// replay loader (renders the Replay dialog) +// runStore.findRun({ friendlyId }, { select: {...} }) (no client) +// +// Both pass NO client, so per the per-method default they route to the OWNING store's REPLICA (a `{ id }` +// waitpoint lookup even resolves its owning store via a replica probe first — #resolveWaitpointStore(id, +// onPrimary=false) — so a not-yet-replicated row is invisible on BOTH the resolution probe and the read). +// The owning replica is frozen with the shared laggingReplica on the REAL split topology +// (heteroRunOpsPostgresTest — NEVER mocked); each case asserts exactly what the caller sees. +// +// The complete-waitpoint route GATES a mutation in the SAME request: +// const waitpoint = await runStore.findWaitpoint({ ..., where: { id } }); +// if (waitpoint?.projectId !== project.id) return "No waitpoint found"; +// ... engine.completeWaitpoint({ id }) // the write +// Under lag the replica read is null (the first leg). A bare null makes `undefined !== project.id` true, +// which would return "No waitpoint found" for a waitpoint present on the primary. So on a null read the +// route re-reads the owning primary via runStore.findWaitpointOnPrimary({ where: { id } }) before +// deciding. The test pins both legs: the replica read is null, and the owning-primary re-read resolves +// the same waitpoint. +// +// The replay loader read is a tolerated read-view: it only renders a form; on a miss it falls through to +// the mollifier buffer and finally a 404 the user retries. It drives NO write and no decision on stale +// data — eventual consistency is the intended contract for opening a dialog for an (already-existing, +// historical) run. The consequential write path (the replay ACTION) has its OWN primary re-read, so a +// stale loader render never causes a wrong replay. The test asserts the store-level fact (null under +// lag, replica hit) AND proves the run exists on the primary (so the null is pure lag, not absence). + +import { heteroRunOpsPostgresTest, laggingReplica } from "@internal/testcontainers"; +import type { PrismaClient } from "@trigger.dev/database"; +import type { RunOpsPrismaClient } from "@internal/run-ops-database"; +import { describe, expect } from "vitest"; +import { PostgresRunStore } from "./PostgresRunStore.js"; +import { RoutingRunStore } from "./runOpsStore.js"; + +const CUID_25 = "e".repeat(25); // cuid id-shape -> LEGACY (#legacy / prisma14, full schema) + +async function seedEnvironmentLegacy(prisma: PrismaClient, suffix: string) { + const organization = await prisma.organization.create({ + data: { title: `Org ${suffix}`, slug: `org-${suffix}` }, + }); + const project = await prisma.project.create({ + data: { + name: `Project ${suffix}`, + slug: `project-${suffix}`, + externalRef: `proj_${suffix}`, + organizationId: organization.id, + }, + }); + const environment = await prisma.runtimeEnvironment.create({ + data: { + type: "DEVELOPMENT", + slug: "dev", + projectId: project.id, + organizationId: organization.id, + apiKey: `tr_dev_${suffix}`, + pkApiKey: `pk_dev_${suffix}`, + shortcode: `short_${suffix}`, + }, + }); + return { organization, project, environment }; +} + +// Build a LEGACY-owning router whose legacy replica is frozen (lagging); the NEW store is real +// (non-lagging) so the on-miss fan-out to the other store's replica also legitimately misses. +function buildRouterWithLaggingLegacyReplica(prisma14: PrismaClient, prisma17: RunOpsPrismaClient) { + const legacyReplica = laggingReplica(prisma14, [ + { model: "taskRun", mode: "missing" }, + { model: "waitpoint", mode: "missing" }, + ]); + const legacyStore = new PostgresRunStore({ + prisma: prisma14, + readOnlyPrisma: legacyReplica.client, + schemaVariant: "legacy", + }); + const newStore = new PostgresRunStore({ + prisma: prisma17 as never, + readOnlyPrisma: prisma17 as never, + schemaVariant: "dedicated", + }); + const router = new RoutingRunStore({ new: newStore, legacy: legacyStore }); + return { router, legacyStore, legacyReplica }; +} + +// Seed a standalone, still-PENDING MANUAL token waitpoint on the WRITER (exactly as minting a resume +// token does). The complete route looks it up by id right after; under replica lag that read misses. +async function seedPendingTokenWaitpoint( + store: PostgresRunStore, + params: { id: string; friendlyId: string; projectId: string; environmentId: string } +) { + await store.upsertWaitpoint({ + where: { + environmentId_idempotencyKey: { + environmentId: params.environmentId, + idempotencyKey: params.id, + }, + }, + create: { + id: params.id, + friendlyId: params.friendlyId, + type: "MANUAL", + status: "PENDING", + idempotencyKey: params.id, + userProvidedIdempotencyKey: false, + projectId: params.projectId, + environmentId: params.environmentId, + }, + update: {}, + }); +} + +function taskRunData(opts: { + id: string; + friendlyId: string; + organizationId: string; + projectId: string; + runtimeEnvironmentId: string; +}) { + return { + id: opts.id, + engine: "V2" as const, + status: "COMPLETED_SUCCESSFULLY" as const, + friendlyId: opts.friendlyId, + runtimeEnvironmentId: opts.runtimeEnvironmentId, + environmentType: "DEVELOPMENT" as const, + organizationId: opts.organizationId, + projectId: opts.projectId, + taskIdentifier: "my-task", + payload: JSON.stringify({ hello: "world" }), + payloadType: "application/json", + traceContext: {}, + traceId: `trace_${opts.id}`, + spanId: `span_${opts.id}`, + queue: "task/my-task", + isTest: false, + taskEventStore: "taskEvent", + depth: 0, + }; +} + +// The exact projection the complete route reads to run its project/env auth guard. +const COMPLETE_WAITPOINT_SELECT = { projectId: true, environmentId: true } as const; + +// The loader's source-run projection, subset that exists on both variants. +const REPLAY_LOADER_SELECT = { + payload: true, + payloadType: true, + runtimeEnvironmentId: true, + projectId: true, + taskIdentifier: true, +} as const; + +describe("waitpoint-complete route + replay loader read-view under replica lag", () => { + // ── complete-waitpoint route ──────────────────────────────────────────────────────────────────── + // findWaitpoint(no client) → owning REPLICA. Under lag it returns null (the first leg); a bare null + // makes the auth guard `waitpoint?.projectId !== project.id` refuse a real waitpoint with + // "No waitpoint found". So the route re-reads the owning primary (findWaitpointOnPrimary), which + // resolves the same waitpoint. + heteroRunOpsPostgresTest( + "complete-waitpoint route findWaitpoint is stale-null under replica lag and resolves on the primary re-read", + async ({ prisma14, prisma17 }) => { + const { router, legacyStore, legacyReplica } = buildRouterWithLaggingLegacyReplica( + prisma14, + prisma17 + ); + const seed = await seedEnvironmentLegacy(prisma14, "wpcomplete_leg"); + const waitpointId = `waitpoint_${CUID_25}`; // cuid → LEGACY + await seedPendingTokenWaitpoint(legacyStore, { + id: waitpointId, + friendlyId: "waitpoint_wpcomplete_leg", + projectId: seed.project.id, + environmentId: seed.environment.id, + }); + + // The EXACT complete-route read: select {projectId, environmentId}, where {id}, NO client. + const fromReplica = await router.findWaitpoint({ + select: COMPLETE_WAITPOINT_SELECT, + where: { id: waitpointId }, + }); + + // Store seam: the owning replica lags → null. This is precisely what drives the route to fail: + // (fromReplica?.projectId !== project.id) == (undefined !== ) == true + // → "No waitpoint found", and engine.completeWaitpoint is never called. The waitpoint exists. + expect(fromReplica).toBeNull(); + expect(legacyReplica.wasHit()).toBe(true); + // Reproduce the route's guard evaluation on the stale value: it wrongly fails. + const guardFailsUnderLag = fromReplica?.projectId !== seed.project.id; + expect(guardFailsUnderLag).toBe(true); + + // The route re-reads the owning primary on the miss. It finds the waitpoint, its projectId + // matches, and the guard passes — the completion proceeds. + const fromPrimary = await router.findWaitpointOnPrimary({ + select: COMPLETE_WAITPOINT_SELECT, + where: { id: waitpointId }, + }); + expect(fromPrimary).not.toBeNull(); + expect(fromPrimary!.projectId).toBe(seed.project.id); + expect(fromPrimary!.environmentId).toBe(seed.environment.id); + } + ); + + // ── replay loader (tolerated read-view) ────────────────────────────────────────────────────────── + // loader findRun(no client) → owning REPLICA. Under lag it returns null. Unlike the complete-waitpoint + // route above, this drives NO write and no decision on stale data: on a null the loader tries the + // mollifier buffer then 404s and the user retries; the consequential replay ACTION re-reads the + // primary on its own. So a stale loader render is safe. We assert the store-level fact (null + replica + // hit) AND prove the run truly exists on the primary — the null is pure lag, tolerated by a display + // render. + heteroRunOpsPostgresTest( + "replay loader findRun is null under replica lag while the run exists on the primary", + async ({ prisma14, prisma17 }) => { + const { router, legacyReplica } = buildRouterWithLaggingLegacyReplica(prisma14, prisma17); + const seed = await seedEnvironmentLegacy(prisma14, "replayloader_leg"); + const runId = `run_${CUID_25}`; + const friendlyId = "run_replayloader_leg"; + await prisma14.taskRun.create({ + data: taskRunData({ + id: runId, + friendlyId, + organizationId: seed.organization.id, + projectId: seed.project.id, + runtimeEnvironmentId: seed.environment.id, + }), + }); + + // The EXACT loader read: findRun by friendlyId with the display select, NO client → owning replica. + const fromReplica = await router.findRun({ friendlyId }, { select: REPLAY_LOADER_SELECT }); + expect(fromReplica).toBeNull(); // loader falls to the buffer / 404 (no write is driven) + expect(legacyReplica.wasHit()).toBe(true); + + // Prove the null is lag, not absence: the run is on the primary. The loader tolerates the stale + // miss because it only renders a dialog; the replay ACTION guards its write with its own primary + // re-read, so eventual consistency of this display read is correct-by-design. + const fromPrimary = await router.findRun( + { friendlyId }, + { select: REPLAY_LOADER_SELECT }, + prisma14 + ); + expect(fromPrimary).not.toBeNull(); + expect((fromPrimary as { projectId: string }).projectId).toBe(seed.project.id); + expect((fromPrimary as { taskIdentifier: string }).taskIdentifier).toBe("my-task"); + } + ); +}); diff --git a/internal-packages/run-store/src/runOpsStore.waitpointCompleteTokenReadAfterWrite.test.ts b/internal-packages/run-store/src/runOpsStore.waitpointCompleteTokenReadAfterWrite.test.ts new file mode 100644 index 00000000000..7cdf434fa58 --- /dev/null +++ b/internal-packages/run-store/src/runOpsStore.waitpointCompleteTokenReadAfterWrite.test.ts @@ -0,0 +1,175 @@ +// Property: the waitpoint-complete token read tolerates replica lag. The three completion routes read +// the waitpoint by id (findWaitpoint with NO client) before completing it, which routes to the OWNING +// store's REPLICA — and a `{id}` lookup resolves the owning store via a replica probe first, so a +// just-minted, not-yet-replicated token is invisible on both. Each route re-reads via +// findWaitpointOnPrimary on a null. Pins both legs with the shared lagging-replica primitive on the +// real split topology (never mocked): findWaitpoint(id) is null under lag; findWaitpointOnPrimary(id) +// resolves it. + +import { heteroRunOpsPostgresTest, laggingReplica } from "@internal/testcontainers"; +import type { PrismaClient } from "@trigger.dev/database"; +import type { RunOpsPrismaClient } from "@internal/run-ops-database"; +import { generateRunOpsId } from "@trigger.dev/core/v3/isomorphic"; +import { describe, expect } from "vitest"; +import { PostgresRunStore } from "./PostgresRunStore.js"; +import { RoutingRunStore } from "./runOpsStore.js"; +import type { RunStoreSchemaVariant } from "./types.js"; + +type AnyClient = PrismaClient | RunOpsPrismaClient; + +const CUID_25 = "d".repeat(25); // waitpoint id shape -> LEGACY (#legacy / prisma14, full schema) + +// On the dedicated subset there are no Organization/Project/RuntimeEnvironment models (run-ops rows +// carry FK-free scalar ids), so mint synthetic owning ids; on legacy seed the real rows the FKs need. +async function seedEnvironment( + prisma: AnyClient, + schemaVariant: RunStoreSchemaVariant, + slugSuffix: string +) { + if (schemaVariant === "dedicated") { + return { + organization: { id: `org_${slugSuffix}` }, + project: { id: `proj_${slugSuffix}` }, + environment: { id: `env_${slugSuffix}` }, + }; + } + const organization = await (prisma as PrismaClient).organization.create({ + data: { title: `Org ${slugSuffix}`, slug: `org-${slugSuffix}` }, + }); + const project = await (prisma as PrismaClient).project.create({ + data: { + name: `Project ${slugSuffix}`, + slug: `project-${slugSuffix}`, + externalRef: `proj_${slugSuffix}`, + organizationId: organization.id, + }, + }); + const environment = await (prisma as PrismaClient).runtimeEnvironment.create({ + data: { + type: "DEVELOPMENT", + slug: "dev", + projectId: project.id, + organizationId: organization.id, + apiKey: `tr_dev_${slugSuffix}`, + pkApiKey: `pk_dev_${slugSuffix}`, + shortcode: `short_${slugSuffix}`, + }, + }); + return { organization, project, environment }; +} + +// Seed a standalone, still-PENDING MANUAL token waitpoint on the WRITER, exactly as minting a resume +// token does. The completion routes look it up by id right after; under replica lag that read misses. +async function seedPendingTokenWaitpoint( + store: PostgresRunStore, + params: { id: string; friendlyId: string; projectId: string; environmentId: string } +) { + await store.upsertWaitpoint({ + where: { + environmentId_idempotencyKey: { + environmentId: params.environmentId, + idempotencyKey: params.id, + }, + }, + create: { + id: params.id, + friendlyId: params.friendlyId, + type: "MANUAL", + status: "PENDING", + idempotencyKey: params.id, + userProvidedIdempotencyKey: false, + projectId: params.projectId, + environmentId: params.environmentId, + }, + update: {}, + }); +} + +describe("run-ops split — waitpoint-complete token read: replica misses a just-minted token, primary resolves it", () => { + // (a) LEGACY-resident (cuid) token: minted on the control-plane writer; its replica lags. The + // completion routes' `findWaitpoint({ where: { id } })` must NOT strand the token. + heteroRunOpsPostgresTest( + "LEGACY cuid: findWaitpoint(id) is null under replica lag; findWaitpointOnPrimary(id) resolves it", + async ({ prisma14, prisma17 }) => { + const legacyReplica = laggingReplica(prisma14, [{ model: "waitpoint", mode: "missing" }]); + const legacyStore = new PostgresRunStore({ + prisma: prisma14, + readOnlyPrisma: legacyReplica.client, + schemaVariant: "legacy", + }); + const newStore = new PostgresRunStore({ + prisma: prisma17 as never, + readOnlyPrisma: prisma17 as never, + schemaVariant: "dedicated", + }); + const router = new RoutingRunStore({ new: newStore, legacy: legacyStore }); + + const seed = await seedEnvironment(prisma14, "legacy", "wctok_leg"); + const waitpointId = `waitpoint_${CUID_25}`; // cuid -> LEGACY + await seedPendingTokenWaitpoint(legacyStore, { + id: waitpointId, + friendlyId: "waitpoint_wctok_leg", + projectId: seed.project.id, + environmentId: seed.environment.id, + }); + + // The exact completion-route read (no client, by id). Under lag it misses the just-minted token; + // a bare null would strand a token that exists on the primary. + const fromReplica = await router.findWaitpoint({ + where: { id: waitpointId, environmentId: seed.environment.id }, + }); + expect(fromReplica).toBeNull(); + expect(legacyReplica.wasHit()).toBe(true); + + // The read-your-writes fallback each completion route applies: a re-read on the owning primary + // resolves the token. + const fromPrimary = await router.findWaitpointOnPrimary({ + where: { id: waitpointId, environmentId: seed.environment.id }, + }); + expect(fromPrimary).not.toBeNull(); + expect(fromPrimary!.id).toBe(waitpointId); + expect(fromPrimary!.status).toBe("PENDING"); + } + ); + + // (b) NEW-resident (run-ops id) token on the dedicated subset schema: the NEW replica lags. + heteroRunOpsPostgresTest( + "NEW id: findWaitpoint(id) is null under NEW replica lag; findWaitpointOnPrimary(id) resolves it", + async ({ prisma14, prisma17 }) => { + const newReplica = laggingReplica(prisma17, [{ model: "waitpoint", mode: "missing" }]); + const newStore = new PostgresRunStore({ + prisma: prisma17 as never, + readOnlyPrisma: newReplica.client as never, + schemaVariant: "dedicated", + }); + const legacyStore = new PostgresRunStore({ + prisma: prisma14, + readOnlyPrisma: prisma14, + schemaVariant: "legacy", + }); + const router = new RoutingRunStore({ new: newStore, legacy: legacyStore }); + + const seed = await seedEnvironment(prisma17, "dedicated", "wctok_new"); + const waitpointId = `waitpoint_${generateRunOpsId()}`; // run-ops id -> NEW + await seedPendingTokenWaitpoint(newStore, { + id: waitpointId, + friendlyId: "waitpoint_wctok_new", + projectId: seed.project.id, + environmentId: seed.environment.id, + }); + + const fromReplica = await router.findWaitpoint({ + where: { id: waitpointId, environmentId: seed.environment.id }, + }); + expect(fromReplica).toBeNull(); + expect(newReplica.wasHit()).toBe(true); + + const fromPrimary = await router.findWaitpointOnPrimary({ + where: { id: waitpointId, environmentId: seed.environment.id }, + }); + expect(fromPrimary).not.toBeNull(); + expect(fromPrimary!.id).toBe(waitpointId); + expect(fromPrimary!.status).toBe("PENDING"); + } + ); +}); diff --git a/internal-packages/run-store/src/runOpsStore.waitpointDedupReadAfterWrite.test.ts b/internal-packages/run-store/src/runOpsStore.waitpointDedupReadAfterWrite.test.ts new file mode 100644 index 00000000000..80247950fd9 --- /dev/null +++ b/internal-packages/run-store/src/runOpsStore.waitpointDedupReadAfterWrite.test.ts @@ -0,0 +1,288 @@ +// Verifies the waitpoint idempotency dedup reads the OWNING store's WRITER, not its lagging replica. +// createDateTimeWaitpoint/createManualWaitpoint dedup on a retry via findWaitpoint({environmentId, +// idempotencyKey}, undefined, {coLocateWithRunId}); colocated with the owning run it must resolve +// attempt-1's just-written waitpoint on the primary so isCached fires and the run does not re-arm. +// Deterministic via heteroRunOpsPostgresTest (real split topology, never mocked) + a lagging-replica +// proxy whose `waitpoint` reads return empty and record that they ran. + +import { heteroRunOpsPostgresTest, laggingReplica } from "@internal/testcontainers"; +import type { PrismaClient } from "@trigger.dev/database"; +import type { RunOpsPrismaClient } from "@internal/run-ops-database"; +import { generateRunOpsId } from "@trigger.dev/core/v3/isomorphic"; +import { describe, expect } from "vitest"; +import { PostgresRunStore } from "./PostgresRunStore.js"; +import { RoutingRunStore } from "./runOpsStore.js"; +import type { CreateRunInput, RunStoreSchemaVariant } from "./types.js"; + +type AnyClient = PrismaClient | RunOpsPrismaClient; + +// ownerEngine (classifyResidency) routes a run-ops v1 body → NEW, everything else → LEGACY. +const CUID_25 = "c".repeat(25); // → LEGACY (#legacy / prisma14, full schema) +const NEW_RUN_ID = `run_${generateRunOpsId()}`; // valid v1 body → NEW (#new / prisma17, dedicated) + +// On the dedicated subset there are no Organization/Project/RuntimeEnvironment models (the run-ops +// rows carry FK-free scalar ids), so we mint synthetic owning ids. On legacy we seed the real rows +// the kept FKs require. +async function seedEnvironment( + prisma: AnyClient, + schemaVariant: RunStoreSchemaVariant, + slugSuffix: string +) { + if (schemaVariant === "dedicated") { + return { + organization: { id: `org_${slugSuffix}` }, + project: { id: `proj_${slugSuffix}` }, + environment: { id: `env_${slugSuffix}` }, + }; + } + const organization = await (prisma as PrismaClient).organization.create({ + data: { title: `Org ${slugSuffix}`, slug: `org-${slugSuffix}` }, + }); + const project = await (prisma as PrismaClient).project.create({ + data: { + name: `Project ${slugSuffix}`, + slug: `project-${slugSuffix}`, + externalRef: `proj_${slugSuffix}`, + organizationId: organization.id, + }, + }); + const environment = await (prisma as PrismaClient).runtimeEnvironment.create({ + data: { + type: "DEVELOPMENT", + slug: "dev", + projectId: project.id, + organizationId: organization.id, + apiKey: `tr_dev_${slugSuffix}`, + pkApiKey: `pk_dev_${slugSuffix}`, + shortcode: `short_${slugSuffix}`, + }, + }); + return { organization, project, environment }; +} + +function buildCreateRunInput(params: { + runId: string; + friendlyId: string; + taskIdentifier: string; + organizationId: string; + projectId: string; + runtimeEnvironmentId: string; +}): CreateRunInput { + return { + data: { + id: params.runId, + engine: "V2", + status: "PENDING", + friendlyId: params.friendlyId, + runtimeEnvironmentId: params.runtimeEnvironmentId, + environmentType: "DEVELOPMENT", + organizationId: params.organizationId, + projectId: params.projectId, + taskIdentifier: params.taskIdentifier, + payload: '{"hello":"world"}', + payloadType: "application/json", + context: { foo: "bar" }, + traceContext: { trace: "ctx" }, + traceId: "trace_1", + spanId: "span_1", + runTags: ["alpha", "beta"], + queue: "task/my-task", + isTest: false, + taskEventStore: "taskEvent", + depth: 0, + createdAt: new Date("2024-01-01T00:00:00.000Z"), + }, + snapshot: { + engine: "V2", + executionStatus: "RUN_CREATED", + description: "Run was created", + runStatus: "PENDING", + environmentId: params.runtimeEnvironmentId, + environmentType: "DEVELOPMENT", + projectId: params.projectId, + organizationId: params.organizationId, + }, + }; +} + +// Seed attempt-1's already-completed DATETIME waitpoint (user-provided idempotencyKey) on the WRITER, +// exactly as `createDateTimeWaitpoint`'s upsert would have written it on the first attempt. +async function seedCompletedDateTimeWaitpoint( + store: PostgresRunStore, + params: { + id: string; + friendlyId: string; + projectId: string; + environmentId: string; + idempotencyKey: string; + } +) { + const completedAfter = new Date(Date.now() - 60_000); + await store.upsertWaitpoint({ + where: { + environmentId_idempotencyKey: { + environmentId: params.environmentId, + idempotencyKey: params.idempotencyKey, + }, + }, + create: { + id: params.id, + friendlyId: params.friendlyId, + type: "DATETIME", + status: "COMPLETED", + idempotencyKey: params.idempotencyKey, + userProvidedIdempotencyKey: true, + idempotencyKeyExpiresAt: new Date(Date.now() + 60 * 60_000), + projectId: params.projectId, + environmentId: params.environmentId, + completedAfter, + completedAt: completedAfter, + }, + update: {}, + }); +} + +describe("run-ops split — waitpoint idempotency dedup reads the OWNING store's WRITER, not its lagging replica", () => { + // The dedup probe issued by createDateTimeWaitpoint/createManualWaitpoint on a RETRY: + // findWaitpoint({ where: { environmentId, idempotencyKey } }, undefined, { coLocateWithRunId }) + // must resolve attempt-1's just-written waitpoint via the OWNING store's primary, so isCached fires + // and the run does NOT re-arm/re-block. Proven for BOTH residencies + a routing guard. + + // (a) LEGACY-resident (cuid) run: attempt-1's waitpoint was committed to the control-plane writer; + // the control-plane replica lags. The colocated dedup must find it on the owning (legacy) writer. + heteroRunOpsPostgresTest( + "LEGACY cuid: colocated dedup finds attempt-1's waitpoint despite replica lag", + async ({ prisma14, prisma17 }) => { + const legacyReplica = laggingReplica(prisma14, [{ model: "waitpoint", mode: "missing" }]); + const legacyStore = new PostgresRunStore({ + prisma: prisma14, + readOnlyPrisma: legacyReplica.client, + schemaVariant: "legacy", + }); + const newStore = new PostgresRunStore({ + prisma: prisma17 as never, + readOnlyPrisma: prisma17 as never, + schemaVariant: "dedicated", + }); + const router = new RoutingRunStore({ new: newStore, legacy: legacyStore }); + + const seed = await seedEnvironment(prisma14, "legacy", "wp_leg"); + const runId = `run_${CUID_25}`; // cuid → LEGACY + await legacyStore.createRun( + buildCreateRunInput({ + runId, + friendlyId: "run_wp_leg", + taskIdentifier: "my-task", + organizationId: seed.organization.id, + projectId: seed.project.id, + runtimeEnvironmentId: seed.environment.id, + }) + ); + const idempotencyKey = "wuik-legacy"; + await seedCompletedDateTimeWaitpoint(legacyStore, { + id: `waitpoint_${CUID_25}`, + friendlyId: "waitpoint_wp_leg", + projectId: seed.project.id, + environmentId: seed.environment.id, + idempotencyKey, + }); + + // The exact dedup probe: no client, colocated with the owning run. + const found = await router.findWaitpoint( + { where: { environmentId: seed.environment.id, idempotencyKey } }, + undefined, + { coLocateWithRunId: runId } + ); + + // Resolved on the owning store's writer; the replica is never consulted. + expect(found).not.toBeNull(); + expect(found!.idempotencyKey).toBe(idempotencyKey); + expect(found!.status).toBe("COMPLETED"); + expect(legacyReplica.wasHit()).toBe(false); + } + ); + + // (b) NEW-resident (ksuid) run on the dedicated subset schema: the NEW replica lags. The colocated + // dedup must resolve on the NEW writer. + heteroRunOpsPostgresTest( + "NEW ksuid: colocated dedup finds attempt-1's waitpoint despite NEW replica lag", + async ({ prisma14, prisma17 }) => { + const newReplica = laggingReplica(prisma17, [{ model: "waitpoint", mode: "missing" }]); + const newStore = new PostgresRunStore({ + prisma: prisma17 as never, + readOnlyPrisma: newReplica.client as never, + schemaVariant: "dedicated", + }); + const legacyStore = new PostgresRunStore({ + prisma: prisma14, + readOnlyPrisma: prisma14, + schemaVariant: "legacy", + }); + const router = new RoutingRunStore({ new: newStore, legacy: legacyStore }); + + const seed = await seedEnvironment(prisma17, "dedicated", "wp_new"); + const runId = NEW_RUN_ID; // v1 body → NEW + await newStore.createRun( + buildCreateRunInput({ + runId, + friendlyId: "run_wp_new", + taskIdentifier: "my-task", + organizationId: seed.organization.id, + projectId: seed.project.id, + runtimeEnvironmentId: seed.environment.id, + }) + ); + const idempotencyKey = "wuik-new"; + await seedCompletedDateTimeWaitpoint(newStore, { + id: `waitpoint_${generateRunOpsId()}`, + friendlyId: "waitpoint_wp_new", + projectId: seed.project.id, + environmentId: seed.environment.id, + idempotencyKey, + }); + + const found = await router.findWaitpoint( + { where: { environmentId: seed.environment.id, idempotencyKey } }, + undefined, + { coLocateWithRunId: runId } + ); + + expect(found).not.toBeNull(); + expect(found!.idempotencyKey).toBe(idempotencyKey); + expect(found!.status).toBe("COMPLETED"); + expect(newReplica.wasHit()).toBe(false); + } + ); + + // Guard: a NON-colocated, no-client findWaitpoint (no read-your-writes intent) still routes to the + // replica — colocated dedup must not turn every waitpoint read into a primary read (that would defeat + // replica offload). + heteroRunOpsPostgresTest( + "plain non-colocated reads still route to the replica (no read-your-writes escalation)", + async ({ prisma14, prisma17 }) => { + const legacyReplica = laggingReplica(prisma14, [{ model: "waitpoint", mode: "missing" }]); + const legacyStore = new PostgresRunStore({ + prisma: prisma14, + readOnlyPrisma: legacyReplica.client, + schemaVariant: "legacy", + }); + const newReplica = laggingReplica(prisma17, [{ model: "waitpoint", mode: "missing" }]); + const newStore = new PostgresRunStore({ + prisma: prisma17 as never, + readOnlyPrisma: newReplica.client as never, + schemaVariant: "dedicated", + }); + const router = new RoutingRunStore({ new: newStore, legacy: legacyStore }); + + // No id, no colocation → the cross-DB scan probes each store's replica. + const found = await router.findWaitpoint({ + where: { environmentId: "env_none", idempotencyKey: "does-not-exist" }, + }); + + expect(found).toBeNull(); + // Both legs must be replica-probed — an `||` would hide a primary-routing regression on one leg. + expect(legacyReplica.wasHit()).toBe(true); + expect(newReplica.wasHit()).toBe(true); + } + ); +}); From d1f1a8520e4ccfbd38bbcd432cd60633b6ab1af4 Mon Sep 17 00:00:00 2001 From: Dan Sutton Date: Sun, 19 Jul 2026 16:11:07 +0100 Subject: [PATCH 5/6] test(webapp): guards for the split-idempotency + realtime-batch review fixes Caller-driven / unit guards for the production hardening in the base PR: claim-TTL floor, publishClaim CAS result, reacquire fail-closed 503, expired/failed global-scope recreate re-serialisation, and the realtime batch primary re-read. Split out of the base PR so the production change stays reviewable on its own. --- apps/webapp/test/claimTtl.test.ts | 55 +++++++++++ ...mpotencyExpiredRecreateReserialize.test.ts | 93 +++++++++++++++++++ .../test/mollifierClaimResolution.test.ts | 26 ++++++ apps/webapp/test/publishClaimResult.test.ts | 33 +++++++ .../test/reacquireClearedGlobalWinner.test.ts | 44 +++++++++ .../test/resolveBatchForRealtime.test.ts | 47 ++++++++++ 6 files changed, 298 insertions(+) create mode 100644 apps/webapp/test/claimTtl.test.ts create mode 100644 apps/webapp/test/idempotencyExpiredRecreateReserialize.test.ts create mode 100644 apps/webapp/test/publishClaimResult.test.ts create mode 100644 apps/webapp/test/reacquireClearedGlobalWinner.test.ts create mode 100644 apps/webapp/test/resolveBatchForRealtime.test.ts diff --git a/apps/webapp/test/claimTtl.test.ts b/apps/webapp/test/claimTtl.test.ts new file mode 100644 index 00000000000..36da2bbfabc --- /dev/null +++ b/apps/webapp/test/claimTtl.test.ts @@ -0,0 +1,55 @@ +import { describe, expect, it } from "vitest"; +import { computeClaimTtlSeconds } from "~/v3/mollifier/claimTtl"; + +// The mollifier idempotency claim is a serialization lock that must outlive the winner's +// create-and-publish pipeline. If its TTL is derived from the customer's key TTL alone, a short key +// TTL expires the claim mid-pipeline and a polling loser re-claims → a second creator → cross-DB dup. +describe("computeClaimTtlSeconds", () => { + const now = Date.parse("2026-01-01T00:00:00.000Z"); + const minTtlSeconds = 5; + const maxTtlSeconds = 30; + + it("floors a short key TTL at the pipeline minimum (claim can't expire mid-pipeline)", () => { + expect( + computeClaimTtlSeconds({ + keyExpiresAt: new Date(now + 2_000), + now, + minTtlSeconds, + maxTtlSeconds, + }) + ).toBe(5); + }); + + it("caps a long key TTL at the maximum", () => { + expect( + computeClaimTtlSeconds({ + keyExpiresAt: new Date(now + 3_600_000), + now, + minTtlSeconds, + maxTtlSeconds, + }) + ).toBe(30); + }); + + it("uses the key TTL when it sits between the floor and the cap", () => { + expect( + computeClaimTtlSeconds({ + keyExpiresAt: new Date(now + 12_000), + now, + minTtlSeconds, + maxTtlSeconds, + }) + ).toBe(12); + }); + + it("floors an already-expired key at the minimum", () => { + expect( + computeClaimTtlSeconds({ + keyExpiresAt: new Date(now - 1_000), + now, + minTtlSeconds, + maxTtlSeconds, + }) + ).toBe(5); + }); +}); diff --git a/apps/webapp/test/idempotencyExpiredRecreateReserialize.test.ts b/apps/webapp/test/idempotencyExpiredRecreateReserialize.test.ts new file mode 100644 index 00000000000..364ce468599 --- /dev/null +++ b/apps/webapp/test/idempotencyExpiredRecreateReserialize.test.ts @@ -0,0 +1,93 @@ +import { describe, expect, it, vi } from "vitest"; + +// Devin #2: a global-scope (or scope-absent) key whose existing run is EXPIRED/FAILED gets its key +// cleared and recreated. Under the run-ops split that recreate must serialise through the claim (same +// cross-DB dup risk as the claim-loser cleared path) rather than fall through to an unserialised create. +vi.mock("~/db.server", () => ({ + prisma: {}, + $replica: {}, + runOpsNewPrisma: {}, + runOpsLegacyPrisma: {}, + runOpsNewReplica: {}, + runOpsLegacyReplica: {}, +})); + +const h = vi.hoisted(() => ({ existingRun: null as unknown, splitEnabled: true })); + +vi.mock("~/v3/runStore.server", () => ({ + runStore: { findRun: vi.fn(async () => h.existingRun) }, +})); +vi.mock("~/v3/mollifier/mollifierBuffer.server", () => ({ getMollifierBuffer: () => null })); +vi.mock("~/v3/mollifier/mollifierGate.server", () => ({ + makeResolveMollifierFlag: () => async () => false, +})); +vi.mock("~/v3/runOpsMigration/splitMode.server", () => ({ + isSplitEnabled: async () => h.splitEnabled, +})); +vi.mock("~/runEngine/concerns/idempotencyResidency.server", () => ({ + resolveIdempotencyDedupClient: async () => ({}), +})); + +import { IdempotencyKeyConcern } from "~/runEngine/concerns/idempotencyKeys.server"; +import type { TriggerTaskRequest } from "~/runEngine/types"; + +function makeRequest(): TriggerTaskRequest { + return { + taskId: "my-task", + environment: { id: "env_a", organizationId: "org_1", organization: { featureFlags: {} } }, + options: {}, + body: { options: { idempotencyKey: "k-1" } }, // scope absent → treated as global + } as unknown as TriggerTaskRequest; +} + +// handleExistingRun's documented contract for an expired/failed run: clear the key, return isCached:false. +const CLEARED = { + isCached: false as const, + idempotencyKey: "k-1", + idempotencyKeyExpiresAt: new Date(Date.now() + 60_000), +}; + +describe("IdempotencyKeyConcern · expired/failed recreate re-serialisation", () => { + it("routes the cleared recreate through the claim under global-scope split (no unserialised create)", async () => { + h.existingRun = { id: "run_internal", friendlyId: "run_friendly" }; + h.splitEnabled = true; + const concern = new IdempotencyKeyConcern({} as never, {} as never, {} as never); + vi.spyOn( + concern as never as { handleExistingRun: () => unknown }, + "handleExistingRun" + ).mockResolvedValue(CLEARED); + const SENTINEL = { ...CLEARED, claim: { token: "t" } }; + const reacquire = vi + .spyOn( + concern as never as { reacquireClearedGlobalWinner: () => unknown }, + "reacquireClearedGlobalWinner" + ) + .mockResolvedValue(SENTINEL); + + const result = await concern.handleTriggerRequest(makeRequest(), undefined); + + expect(reacquire).toHaveBeenCalledOnce(); + expect(result).toBe(SENTINEL); + }); + + it("does NOT re-serialise when the split is off — plain recreate", async () => { + h.existingRun = { id: "run_internal", friendlyId: "run_friendly" }; + h.splitEnabled = false; + const concern = new IdempotencyKeyConcern({} as never, {} as never, {} as never); + vi.spyOn( + concern as never as { handleExistingRun: () => unknown }, + "handleExistingRun" + ).mockResolvedValue(CLEARED); + const reacquire = vi + .spyOn( + concern as never as { reacquireClearedGlobalWinner: () => unknown }, + "reacquireClearedGlobalWinner" + ) + .mockResolvedValue({} as never); + + const result = await concern.handleTriggerRequest(makeRequest(), undefined); + + expect(reacquire).not.toHaveBeenCalled(); + expect(result).toBe(CLEARED); + }); +}); diff --git a/apps/webapp/test/mollifierClaimResolution.test.ts b/apps/webapp/test/mollifierClaimResolution.test.ts index c35c24c1c84..597f264e281 100644 --- a/apps/webapp/test/mollifierClaimResolution.test.ts +++ b/apps/webapp/test/mollifierClaimResolution.test.ts @@ -156,4 +156,30 @@ describe("IdempotencyKeyConcern · claim resolution", () => { h.orgFlag = true; // restore for any later tests in this file } }); + + it("floors the claim TTL for a short idempotency-key TTL so the claim can't expire mid-pipeline", async () => { + // A 2s customer key TTL must NOT shrink the claim below the pipeline floor — else the claim expires + // while the winner is still creating and a polling loser re-claims (cross-DB dup under the split). + // Capture the ttlSeconds the concern hands the buffer's claim. + let capturedTtl: number | undefined; + h.buffer = { + claimIdempotency: vi.fn(async (input: { ttlSeconds: number }) => { + capturedTtl = input.ttlSeconds; + return { kind: "claimed" as const }; + }), + lookupIdempotency: vi.fn(async () => null), + } as unknown as MollifierBuffer; + + const findFirst = vi.fn(async () => null); + const concern = makeConcern({ findFirst }); + + const request = makeRequest(); + (request.body.options as { idempotencyKeyTTL?: string }).idempotencyKeyTTL = "2s"; + + const result = await concern.handleTriggerRequest(request, undefined); + + expect(result.isCached).toBe(false); + // Floored at TRIGGER_MOLLIFIER_CLAIM_MIN_TTL_SECONDS (default 5), NOT the ~2s key TTL. + expect(capturedTtl).toBeGreaterThanOrEqual(5); + }); }); diff --git a/apps/webapp/test/publishClaimResult.test.ts b/apps/webapp/test/publishClaimResult.test.ts new file mode 100644 index 00000000000..a0725855072 --- /dev/null +++ b/apps/webapp/test/publishClaimResult.test.ts @@ -0,0 +1,33 @@ +import { describe, expect, it, vi } from "vitest"; +import { publishClaim } from "~/v3/mollifier/idempotencyClaim.server"; +import type { MollifierBuffer } from "@trigger.dev/redis-worker"; + +// publishClaim compare-and-sets on the caller's token; a `false` result means a stale claimant already +// moved in (the winner's claim expired mid-pipeline), so the publish no-op'd. The caller must be able to +// SEE that rather than silently assuming its run is canonical. +function fakeBuffer(publishResult: boolean): MollifierBuffer { + return { publishClaim: vi.fn(async () => publishResult) } as unknown as MollifierBuffer; +} + +const base = { + envId: "env_1", + taskIdentifier: "task", + idempotencyKey: "k", + token: "tok", + runId: "run_1", + ttlSeconds: 30, +}; + +describe("publishClaim result", () => { + it("returns false when the buffer compare-and-set no-ops (a stale claimant already moved in)", async () => { + expect(await publishClaim({ ...base, buffer: fakeBuffer(false) })).toBe(false); + }); + + it("returns true when the publish sets the winner", async () => { + expect(await publishClaim({ ...base, buffer: fakeBuffer(true) })).toBe(true); + }); + + it("returns true when the mollifier buffer is unavailable (nothing to converge)", async () => { + expect(await publishClaim({ ...base, buffer: null })).toBe(true); + }); +}); diff --git a/apps/webapp/test/reacquireClearedGlobalWinner.test.ts b/apps/webapp/test/reacquireClearedGlobalWinner.test.ts new file mode 100644 index 00000000000..9bf7492c0ad --- /dev/null +++ b/apps/webapp/test/reacquireClearedGlobalWinner.test.ts @@ -0,0 +1,44 @@ +import { describe, expect, it, vi } from "vitest"; + +// Force every reacquire attempt to RESOLVE to a winner (never `claimed`/`timed_out`); the spied +// handleExistingRun keeps reporting that winner as cleared, so the bounded loop advances to exhaustion. +vi.mock("~/v3/mollifier/idempotencyClaim.server", () => ({ + resetResolvedClaim: vi.fn(async () => {}), + claimOrAwait: vi.fn(async () => ({ kind: "resolved", runId: "run_cleared" })), +})); + +import { IdempotencyKeyConcern } from "~/runEngine/concerns/idempotencyKeys.server"; + +describe("reacquireClearedGlobalWinner — fail closed on exhaustion", () => { + it("throws a retryable 503 after exhausting reacquires (never falls through to an unserialised create)", async () => { + const concern = new IdempotencyKeyConcern({} as never, {} as never, {} as never); + // Winner is findable but perpetually cleared → each pass advances staleRunId, exhausting the bound. + vi.spyOn( + concern as unknown as { resolveWinnerAcrossDbs: () => unknown }, + "resolveWinnerAcrossDbs" + ).mockResolvedValue({ friendlyId: "run_cleared" }); + vi.spyOn( + concern as unknown as { handleExistingRun: () => unknown }, + "handleExistingRun" + ).mockResolvedValue({ isCached: false }); + + const request = { environment: { id: "env_1" }, taskId: "task" } as never; + const ctx = { + idempotencyKey: "k", + idempotencyKeyExpiresAt: new Date(Date.now() + 60_000), + dedupClient: {} as never, + ttlSeconds: 30, + clearedRunId: "run_cleared", + safetyNetMs: 5_000, + pollStepMs: 25, + }; + + await expect( + ( + concern as unknown as { + reacquireClearedGlobalWinner: (...a: unknown[]) => Promise; + } + ).reacquireClearedGlobalWinner(request, undefined, ctx) + ).rejects.toMatchObject({ name: "ServiceValidationError", status: 503 }); + }); +}); diff --git a/apps/webapp/test/resolveBatchForRealtime.test.ts b/apps/webapp/test/resolveBatchForRealtime.test.ts new file mode 100644 index 00000000000..40bfab816dc --- /dev/null +++ b/apps/webapp/test/resolveBatchForRealtime.test.ts @@ -0,0 +1,47 @@ +import { describe, expect, it, vi } from "vitest"; +import { resolveBatchTaskRunForRealtime } from "~/v3/realtime/resolveBatchForRealtime.server"; + +// The realtime batch route reads the batch client-less (replica). Under replica lag a just-created batch +// misses; `shouldRetryNotFound` covers the zodfetch GET, but the Electric ShapeStream consumer +// (self-hosters) ignores `x-should-retry`, so the route must re-read the owning PRIMARY on a miss. +// Passing a (non-replica) writer client flips each store leg to its own primary. +function laggingStore(batch: { id: string; friendlyId: string }) { + return { + findBatchTaskRunByFriendlyId: vi.fn( + async (_friendlyId: string, _envId: string, _args: unknown, client?: unknown) => + client ? batch : null + ), + }; +} + +describe("resolveBatchTaskRunForRealtime", () => { + it("re-reads the primary when the replica misses a fresh batch", async () => { + const store = laggingStore({ id: "b_1", friendlyId: "batch_1" }); + const found = await resolveBatchTaskRunForRealtime("batch_1", "env_1", { + store: store as never, + writer: {} as never, + }); + expect(found).toEqual({ id: "b_1", friendlyId: "batch_1" }); + expect(store.findBatchTaskRunByFriendlyId).toHaveBeenCalledTimes(2); + }); + + it("returns null when the batch is genuinely absent on both replica and primary", async () => { + const store = { findBatchTaskRunByFriendlyId: vi.fn(async () => null) }; + const found = await resolveBatchTaskRunForRealtime("nope", "env_1", { + store: store as never, + writer: {} as never, + }); + expect(found).toBeNull(); + }); + + it("does not re-read the primary when the replica already has the batch", async () => { + const store = { + findBatchTaskRunByFriendlyId: vi.fn(async () => ({ id: "b_2", friendlyId: "batch_2" })), + }; + await resolveBatchTaskRunForRealtime("batch_2", "env_1", { + store: store as never, + writer: {} as never, + }); + expect(store.findBatchTaskRunByFriendlyId).toHaveBeenCalledTimes(1); + }); +}); From f1351bb86754ebeb9d1bdccab9e4cdea2be9315e Mon Sep 17 00:00:00 2001 From: Dan Sutton Date: Sun, 19 Jul 2026 16:30:58 +0100 Subject: [PATCH 6/6] test: reconcile guards with base-PR production changes - cancelRun / idempotencyReset NEW-ksuid cases: use a run-ops-shaped friendlyId so the by-friendlyId read routes to NEW single-store (the fan-out that masked the fake LEGACY-classifying friendlyId is gone; a real NEW run's friendlyId classifies NEW). - routesBatchGet realtime.v1.batches case: assert the loader now recovers a stale-on-replica batch via the owning-primary re-read (reaches streamBatch, 200) instead of a resource-gate 404. --- .../test/routesBatchGetReplicaLag.guard.test.ts | 13 ++++++++----- ...Store.cancelRunReadAfterWrite.replicaLag.test.ts | 4 +++- ...tore.idempotencyResetReadView.replicaLag.test.ts | 4 +++- 3 files changed, 14 insertions(+), 7 deletions(-) diff --git a/apps/webapp/test/routesBatchGetReplicaLag.guard.test.ts b/apps/webapp/test/routesBatchGetReplicaLag.guard.test.ts index 1777fe65c02..6fe7a646708 100644 --- a/apps/webapp/test/routesBatchGetReplicaLag.guard.test.ts +++ b/apps/webapp/test/routesBatchGetReplicaLag.guard.test.ts @@ -274,9 +274,11 @@ describe("routes-batch-get callers under a lagging replica", () => { } ); - // ── realtime.v1.batches loader — findBatchTaskRunByFriendlyId (REPLICA) ───────────── + // ── realtime.v1.batches loader — replica miss recovered by the owning-primary re-read ───────────── + // The realtime loader re-reads the owning primary on a replica miss (the ShapeStream consumer ignores + // x-should-retry, so a 404 here would strand), so a stale-on-replica batch reaches streamBatch. heteroRunOpsPostgresTest( - "realtime.v1.batches loader: stale-null returns a retryable 404 at the resource gate before streamBatch", + "realtime.v1.batches loader: a batch stale on the replica is recovered via the owning-primary re-read (reaches streamBatch, no 404)", async ({ prisma14, prisma17 }) => { const { router, legacyStore, legacyReplica } = buildLaggingRouter(prisma14, prisma17); const suffix = `bget_rt_${seq++}`; @@ -299,10 +301,11 @@ describe("routes-batch-get callers under a lagging replica", () => { context: {} as never, })) as Response; + // Replica was consulted first (the miss)… expect(legacyReplica.wasHit("batchTaskRun")).toBe(true); - // 404 at the resource gate — the subscription never starts (streamBatch would return a 200 body). - expect(res.status).toBe(404); - expect(res.headers.get("x-should-retry")).toBe("true"); + // …then the owning-primary re-read found the live batch, so the subscription starts (streamBatch + // returns 200) instead of a resource-gate 404. + expect(res.status).toBe(200); } ); diff --git a/internal-packages/run-store/src/runOpsStore.cancelRunReadAfterWrite.replicaLag.test.ts b/internal-packages/run-store/src/runOpsStore.cancelRunReadAfterWrite.replicaLag.test.ts index 1e5c438507f..b99dfa6f619 100644 --- a/internal-packages/run-store/src/runOpsStore.cancelRunReadAfterWrite.replicaLag.test.ts +++ b/internal-packages/run-store/src/runOpsStore.cancelRunReadAfterWrite.replicaLag.test.ts @@ -192,7 +192,9 @@ describe("run-ops split — cancel-route run lookup vs. a lagging replica (read- const seed = await seedEnvironment(prisma17, "dedicated", "cancel_new"); const runId = NEW_RUN_ID; // v1 body → NEW - const friendlyId = "run_cancel_new"; + // friendlyId classifies identically to the id, so a NEW run's friendlyId must be run-ops-shaped + // for the by-friendlyId read to route to NEW (single-store; no cross-store fan-out). + const friendlyId = `run_${generateRunOpsId()}`; await newStore.createRun( buildCreateRunInput({ runId, diff --git a/internal-packages/run-store/src/runOpsStore.idempotencyResetReadView.replicaLag.test.ts b/internal-packages/run-store/src/runOpsStore.idempotencyResetReadView.replicaLag.test.ts index 0081e307df2..ad48e3d6935 100644 --- a/internal-packages/run-store/src/runOpsStore.idempotencyResetReadView.replicaLag.test.ts +++ b/internal-packages/run-store/src/runOpsStore.idempotencyResetReadView.replicaLag.test.ts @@ -194,7 +194,9 @@ describe("run-ops split — idempotency-key reset source-run lookup vs. a laggin const seed = await seedEnvironment(prisma17, "dedicated", "idem_reset_new"); const runId = NEW_RUN_ID; // v1 body → NEW - const friendlyId = "run_idem_reset_new"; + // A NEW run's friendlyId must be run-ops-shaped so the by-friendlyId read routes to NEW + // (single-store; friendlyId classifies identically to the id). + const friendlyId = `run_${generateRunOpsId()}`; const idempotencyKey = "user-supplied-key-new"; await newStore.createRun( buildCreateRunInput({