diff --git a/.changeset/lost-payload-terminal.md b/.changeset/lost-payload-terminal.md new file mode 100644 index 0000000000..42870d157b --- /dev/null +++ b/.changeset/lost-payload-terminal.md @@ -0,0 +1,5 @@ +--- +'@workflow/world-vercel': patch +--- + +Fail a run instead of retrying it forever when its event log references a payload the backend can no longer read diff --git a/docs/content/docs/v5/foundations/errors-and-retries.mdx b/docs/content/docs/v5/foundations/errors-and-retries.mdx index 4ea7a6b4c6..c7eb91296d 100644 --- a/docs/content/docs/v5/foundations/errors-and-retries.mdx +++ b/docs/content/docs/v5/foundations/errors-and-retries.mdx @@ -195,7 +195,7 @@ try { | `MAX_DELIVERIES_EXCEEDED` | The run exceeded the maximum number of queue deliveries | | `REPLAY_TIMEOUT` | A workflow replay exceeded the maximum allowed duration | | `REPLAY_DIVERGENCE` | A replay could not consume the event log deterministically, usually because of non-deterministic workflow code. | -| `CORRUPTED_EVENT_LOG` | The event log contains orphaned or mismatched events and cannot be replayed. If you see this, please [file an issue](https://github.com/vercel/workflow/issues) | +| `CORRUPTED_EVENT_LOG` | The event log cannot be replayed: it contains orphaned or mismatched events, or one of its stored payloads is no longer readable from the World's storage. If you see this, please [file an issue](https://github.com/vercel/workflow/issues) | | `WORLD_CONTRACT_ERROR` | A World response violated the SDK contract; points at a World implementation bug | | `RUNTIME_ERROR` | An internal runtime error. If you see this, please [file an issue](https://github.com/vercel/workflow/issues) | diff --git a/packages/core/src/classify-error.test.ts b/packages/core/src/classify-error.test.ts index aef330dd3d..439c506600 100644 --- a/packages/core/src/classify-error.test.ts +++ b/packages/core/src/classify-error.test.ts @@ -199,6 +199,18 @@ describe('isRetryableWorldError', () => { ).toBe(true); }); + it('does NOT treat a lost event payload as retryable', () => { + // The world layer raises this when the backend reports that an event's + // stored payload is gone (`payload-missing` terminal frame). Redelivering + // re-reads the same absent object forever, which is what turned one lost + // payload into 12,932 reads of the same run in 26 minutes. + expect( + isRetryableWorldError( + new CorruptedEventLogError('payload no longer exists in storage') + ) + ).toBe(false); + }); + it('does NOT treat 4xx (other than 429) as retryable', () => { expect( isRetryableWorldError( diff --git a/packages/world-vercel/src/events-v4.test.ts b/packages/world-vercel/src/events-v4.test.ts index 911e1a57e3..07c9f0a48b 100644 --- a/packages/world-vercel/src/events-v4.test.ts +++ b/packages/world-vercel/src/events-v4.test.ts @@ -1,5 +1,6 @@ import { Buffer } from 'node:buffer'; import { + CorruptedEventLogError, EntityConflictError, PreconditionFailedError, RunExpiredError, @@ -316,6 +317,99 @@ describe('getWorkflowRunEventsV4 over HTTP', () => { agent.assertNoPendingInterceptors(); }); + // A permanently missing payload arrives as a terminal frame rather than a + // truncated body, because a truncated body is what a dropped socket looks + // like: the runtime cannot tell "retry me" from "this can never work" and + // redelivers forever. One production run re-read a single missing payload + // 12,932 times in 26 minutes before this frame existed. + it('fails the run on a terminal payload-missing frame instead of retrying', async () => { + const origin = + WORKFLOW_SERVER_URL_OVERRIDE || 'https://vercel-workflow.com'; + const agent = new MockAgent(); + agent.disableNetConnect(); + + agent + .get(origin) + .intercept({ + path: '/api/v4/runs/wrun_1/events?returnAll=true', + method: 'GET', + }) + .reply( + 200, + encodeFrame( + { + _error: 1, + code: 'payload-missing', + message: + 'Event payload object is missing from storage: s3rf:t:p:production:wrun_1:wf:01ABC', + }, + new Uint8Array(0) + ), + { headers: { 'content-type': V4_FRAME_CONTENT_TYPE } } + ); + + const error = await getWorkflowRunEventsV4( + 'wrun_1', + {}, + { token: 'test-token', dispatcher: agent } + ).then( + () => undefined, + (err: unknown) => err + ); + + expect(CorruptedEventLogError.is(error)).toBe(true); + expect((error as Error).message).toContain('s3rf:t:p:production:wrun_1'); + // The classification is what makes it terminal: the runtime only + // redelivers a `WorkflowWorldError` with a 5xx status or a TRANSPORT / + // TIMEOUT code (see `isRetryableWorldError`), and it maps + // `CorruptedEventLogError` straight to the CORRUPTED_EVENT_LOG run + // failure. Asserted in `packages/core` rather than here, since core is + // not a dependency of this package. + expect(WorkflowWorldError.is(error)).toBe(false); + agent.assertNoPendingInterceptors(); + }); + + it('treats an unknown terminal error code as terminal, not retryable', async () => { + // Forward compatibility: a code this client has never heard of must not + // become a redelivery loop just because it is unrecognized. + const origin = + WORKFLOW_SERVER_URL_OVERRIDE || 'https://vercel-workflow.com'; + const agent = new MockAgent(); + agent.disableNetConnect(); + + agent + .get(origin) + .intercept({ + path: '/api/v4/runs/wrun_1/events?returnAll=true', + method: 'GET', + }) + .reply( + 200, + encodeFrame( + { _error: 1, code: 'some-future-condition', message: 'nope' }, + new Uint8Array(0) + ), + { headers: { 'content-type': V4_FRAME_CONTENT_TYPE } } + ); + + const error = await getWorkflowRunEventsV4( + 'wrun_1', + {}, + { token: 'test-token', dispatcher: agent } + ).then( + () => undefined, + (err: unknown) => err + ); + + expect(WorkflowWorldError.is(error)).toBe(true); + expect((error as Error).message).toContain('some-future-condition'); + // Neither a 5xx status nor a retryable code, so the runtime fails the run + // rather than redelivering it. + expect((error as WorkflowWorldError).status).toBeUndefined(); + expect((error as WorkflowWorldError).code).toBe('INVALID_RESPONSE'); + agent.assertNoPendingInterceptors(); + }); + it.each([ ['an unknown event type', { eventType: 'future_event', eventData: {} }], ['invalid event metadata', { eventType: 'run_created', eventData: {} }], diff --git a/packages/world-vercel/src/events-v4.ts b/packages/world-vercel/src/events-v4.ts index 2d8bb2c4f9..732398aba0 100644 --- a/packages/world-vercel/src/events-v4.ts +++ b/packages/world-vercel/src/events-v4.ts @@ -22,7 +22,7 @@ */ import assert from 'node:assert/strict'; -import { WorkflowWorldError } from '@workflow/errors'; +import { CorruptedEventLogError, WorkflowWorldError } from '@workflow/errors'; import { type Event, type EventResult, @@ -376,6 +376,24 @@ const EventStreamEndSchema = z.object({ hasMore: z.boolean(), }); +/** + * Terminal error frame. The backend sends this when it cannot finish a frame + * stream and retrying will not help — the response already committed to `200` + * with its first byte, so there is no status code left to carry the failure. + */ +const EventStreamErrorSchema = z.object({ + _error: z.literal(1), + code: z.string(), + message: z.string().optional(), +}); + +/** + * An event's payload object is gone from the backend's blob storage. The + * event row still references it, so every later read of this log fails the + * same way. + */ +const PAYLOAD_MISSING_ERROR_CODE = 'payload-missing'; + // Stable runtimes stored these errors as CBOR StructuredError objects rather // than the format-prefixed serialized bytes emitted by current runtimes. const legacyStructuredErrorEventTypes = new Set([ @@ -1378,6 +1396,49 @@ export interface ListEventsV4Result { hasMore: boolean; } +/** + * Turn a terminal error frame into the error the runtime should act on. + * + * `payload-missing` means an event's stored payload is gone, so this run can + * never replay: it must fail, not retry. That distinction is the whole point + * of the frame. Without it a permanent failure arrived as a truncated body, + * which is what a dropped socket looks like too, so the runtime kept + * redelivering the same doomed replay (one production run re-read a single + * missing payload 12,932 times in 26 minutes). + * + * `CorruptedEventLogError` is the right shape for it: the log references a + * payload nothing can produce, `isRetryableWorldError` leaves it alone, and + * `classifyRunError` already maps it to `CORRUPTED_EVENT_LOG`. + * + * An unrecognized code keeps the conservative reading — a `WorkflowWorldError` + * with no retryable code, so it is terminal rather than a redelivery loop, and + * a future code can be handled explicitly without a client release being + * required first. + */ +function streamErrorFrameToError( + meta: Record, + opName: string +): Error { + const parsed = EventStreamErrorSchema.safeParse(meta); + if (!parsed.success) { + return new WorkflowWorldError( + `v4 ${opName}: malformed terminal error frame`, + { code: 'SCHEMA_VALIDATION', cause: parsed.error } + ); + } + const { code, message } = parsed.data; + const detail = message ?? '(no detail)'; + if (code === PAYLOAD_MISSING_ERROR_CODE) { + return new CorruptedEventLogError( + `the event log references a payload that no longer exists in storage: ${detail}` + ); + } + return new WorkflowWorldError( + `v4 ${opName}: stream ended with terminal error "${code}": ${detail}`, + { code: 'INVALID_RESPONSE' } + ); +} + async function consumeEventFrameStream( response: Response, opName: string, @@ -1397,6 +1458,9 @@ async function consumeEventFrameStream( const end = EventStreamEndSchema.parse(frame.meta); return { cursor: end.next ?? null, hasMore: end.hasMore }; } + if (frame.meta._error === 1) { + throw streamErrorFrameToError(frame.meta, opName); + } if (Object.keys(frame.meta).some((key) => key.startsWith('_'))) { throw new Error(`v4 ${opName}: unexpected control frame`); } diff --git a/packages/world-vercel/src/utils.ts b/packages/world-vercel/src/utils.ts index f89873864b..65cb493187 100644 --- a/packages/world-vercel/src/utils.ts +++ b/packages/world-vercel/src/utils.ts @@ -39,8 +39,10 @@ import { version } from './version.js'; * `main`. It is rewritten by external CI for branch-deployment testing. * Prefer `VERCEL_WORKFLOW_SERVER_URL` for deployment-time configuration. */ +// TEMPORARY: points at the world-vercel backend preview that emits the +// terminal `payload-missing` frame this PR consumes. Revert to '' before merge. // biome-ignore format: External CI replaces only this line with a deployment URL that may exceed the formatter width. -export const WORKFLOW_SERVER_URL_OVERRIDE = ''; +export const WORKFLOW_SERVER_URL_OVERRIDE = 'https://workflow-server-git-peter-s3-no-retry-404.vercel.sh'; /** * HTTP methods that are safe to transparently re-issue inside the adapter.