[world-vercel] Fail the run on a lost event payload instead of retrying forever

.changeset/lost-payload-terminal.md

@@ -0,0 +1,5 @@
+---
+'@workflow/world-vercel': patch
+---
+
+Fail a run instead of retrying it forever when its event log references a payload the backend can no longer read

docs/content/docs/v5/foundations/errors-and-retries.mdx

@@ -195,7 +195,7 @@ try {
 | `MAX_DELIVERIES_EXCEEDED` | The run exceeded the maximum number of queue deliveries |
 | `REPLAY_TIMEOUT` | A workflow replay exceeded the maximum allowed duration |
 | `REPLAY_DIVERGENCE` | A replay could not consume the event log deterministically, usually because of non-deterministic workflow code. |
-| `CORRUPTED_EVENT_LOG` | The event log contains orphaned or mismatched events and cannot be replayed. If you see this, please [file an issue](https://github.com/vercel/workflow/issues) |
+| `CORRUPTED_EVENT_LOG` | The event log cannot be replayed: it contains orphaned or mismatched events, or one of its stored payloads is no longer readable from the World's storage. If you see this, please [file an issue](https://github.com/vercel/workflow/issues) |
 | `WORLD_CONTRACT_ERROR` | A World response violated the SDK contract; points at a World implementation bug |
 | `RUNTIME_ERROR` | An internal runtime error. If you see this, please [file an issue](https://github.com/vercel/workflow/issues) |
 

packages/core/src/classify-error.test.ts

@@ -199,6 +199,18 @@ describe('isRetryableWorldError', () => {
     ).toBe(true);
   });
 
+  it('does NOT treat a lost event payload as retryable', () => {
+    // The world layer raises this when the backend reports that an event's
+    // stored payload is gone (`payload-missing` terminal frame). Redelivering
+    // re-reads the same absent object forever, which is what turned one lost
+    // payload into 12,932 reads of the same run in 26 minutes.
+    expect(
+      isRetryableWorldError(
+        new CorruptedEventLogError('payload no longer exists in storage')
+      )
+    ).toBe(false);
+  });
+
   it('does NOT treat 4xx (other than 429) as retryable', () => {
     expect(
       isRetryableWorldError(

packages/world-vercel/src/events-v4.test.ts

@@ -1,5 +1,6 @@
 import { Buffer } from 'node:buffer';
 import {
+  CorruptedEventLogError,
   EntityConflictError,
   PreconditionFailedError,
   RunExpiredError,
@@ -316,6 +317,99 @@ describe('getWorkflowRunEventsV4 over HTTP', () => {
     agent.assertNoPendingInterceptors();
   });
 
+  // A permanently missing payload arrives as a terminal frame rather than a
+  // truncated body, because a truncated body is what a dropped socket looks
+  // like: the runtime cannot tell "retry me" from "this can never work" and
+  // redelivers forever. One production run re-read a single missing payload
+  // 12,932 times in 26 minutes before this frame existed.
+  it('fails the run on a terminal payload-missing frame instead of retrying', async () => {
+    const origin =
+      WORKFLOW_SERVER_URL_OVERRIDE || 'https://vercel-workflow.com';
+    const agent = new MockAgent();
+    agent.disableNetConnect();
+
+    agent
+      .get(origin)
+      .intercept({
+        path: '/api/v4/runs/wrun_1/events?returnAll=true',
+        method: 'GET',
+      })
+      .reply(
+        200,
+        encodeFrame(
+          {
+            _error: 1,
+            code: 'payload-missing',
+            message:
+              'Event payload object is missing from storage: s3rf:t:p:production:wrun_1:wf:01ABC',
+          },
+          new Uint8Array(0)
+        ),
+        { headers: { 'content-type': V4_FRAME_CONTENT_TYPE } }
+      );
+
+    const error = await getWorkflowRunEventsV4(
+      'wrun_1',
+      {},
+      { token: 'test-token', dispatcher: agent }
+    ).then(
+      () => undefined,
+      (err: unknown) => err
+    );
+
+    expect(CorruptedEventLogError.is(error)).toBe(true);
+    expect((error as Error).message).toContain('s3rf:t:p:production:wrun_1');
+    // The classification is what makes it terminal: the runtime only
+    // redelivers a `WorkflowWorldError` with a 5xx status or a TRANSPORT /
+    // TIMEOUT code (see `isRetryableWorldError`), and it maps
+    // `CorruptedEventLogError` straight to the CORRUPTED_EVENT_LOG run
+    // failure. Asserted in `packages/core` rather than here, since core is
+    // not a dependency of this package.
+    expect(WorkflowWorldError.is(error)).toBe(false);
+    agent.assertNoPendingInterceptors();
+  });
+
+  it('treats an unknown terminal error code as terminal, not retryable', async () => {
+    // Forward compatibility: a code this client has never heard of must not
+    // become a redelivery loop just because it is unrecognized.
+    const origin =
+      WORKFLOW_SERVER_URL_OVERRIDE || 'https://vercel-workflow.com';
+    const agent = new MockAgent();
+    agent.disableNetConnect();
+
+    agent
+      .get(origin)
+      .intercept({
+        path: '/api/v4/runs/wrun_1/events?returnAll=true',
+        method: 'GET',
+      })
+      .reply(
+        200,
+        encodeFrame(
+          { _error: 1, code: 'some-future-condition', message: 'nope' },
+          new Uint8Array(0)
+        ),
+        { headers: { 'content-type': V4_FRAME_CONTENT_TYPE } }
+      );
+
+    const error = await getWorkflowRunEventsV4(
+      'wrun_1',
+      {},
+      { token: 'test-token', dispatcher: agent }
+    ).then(
+      () => undefined,
+      (err: unknown) => err
+    );
+
+    expect(WorkflowWorldError.is(error)).toBe(true);
+    expect((error as Error).message).toContain('some-future-condition');
+    // Neither a 5xx status nor a retryable code, so the runtime fails the run
+    // rather than redelivering it.
+    expect((error as WorkflowWorldError).status).toBeUndefined();
+    expect((error as WorkflowWorldError).code).toBe('INVALID_RESPONSE');
+    agent.assertNoPendingInterceptors();
+  });
+
   it.each([
     ['an unknown event type', { eventType: 'future_event', eventData: {} }],
     ['invalid event metadata', { eventType: 'run_created', eventData: {} }],

packages/world-vercel/src/events-v4.ts

@@ -22,7 +22,7 @@
  */
 
 import assert from 'node:assert/strict';
-import { WorkflowWorldError } from '@workflow/errors';
+import { CorruptedEventLogError, WorkflowWorldError } from '@workflow/errors';
 import {
   type Event,
   type EventResult,
@@ -376,6 +376,24 @@ const EventStreamEndSchema = z.object({
   hasMore: z.boolean(),
 });
 
+/**
+ * Terminal error frame. The backend sends this when it cannot finish a frame
+ * stream and retrying will not help — the response already committed to `200`
+ * with its first byte, so there is no status code left to carry the failure.
+ */
+const EventStreamErrorSchema = z.object({
+  _error: z.literal(1),
+  code: z.string(),
+  message: z.string().optional(),
+});
+
+/**
+ * An event's payload object is gone from the backend's blob storage. The
+ * event row still references it, so every later read of this log fails the
+ * same way.
+ */
+const PAYLOAD_MISSING_ERROR_CODE = 'payload-missing';
+
 // Stable runtimes stored these errors as CBOR StructuredError objects rather
 // than the format-prefixed serialized bytes emitted by current runtimes.
 const legacyStructuredErrorEventTypes = new Set<EventType>([
@@ -1378,6 +1396,49 @@ export interface ListEventsV4Result {
   hasMore: boolean;
 }
 
+/**
+ * Turn a terminal error frame into the error the runtime should act on.
+ *
+ * `payload-missing` means an event's stored payload is gone, so this run can
+ * never replay: it must fail, not retry. That distinction is the whole point
+ * of the frame. Without it a permanent failure arrived as a truncated body,
+ * which is what a dropped socket looks like too, so the runtime kept
+ * redelivering the same doomed replay (one production run re-read a single
+ * missing payload 12,932 times in 26 minutes).
+ *
+ * `CorruptedEventLogError` is the right shape for it: the log references a
+ * payload nothing can produce, `isRetryableWorldError` leaves it alone, and
+ * `classifyRunError` already maps it to `CORRUPTED_EVENT_LOG`.
+ *
+ * An unrecognized code keeps the conservative reading — a `WorkflowWorldError`
+ * with no retryable code, so it is terminal rather than a redelivery loop, and
+ * a future code can be handled explicitly without a client release being
+ * required first.
+ */
+function streamErrorFrameToError(
+  meta: Record<string, unknown>,
+  opName: string
+): Error {
+  const parsed = EventStreamErrorSchema.safeParse(meta);
+  if (!parsed.success) {
+    return new WorkflowWorldError(
+      `v4 ${opName}: malformed terminal error frame`,
+      { code: 'SCHEMA_VALIDATION', cause: parsed.error }
+    );
+  }
+  const { code, message } = parsed.data;
+  const detail = message ?? '(no detail)';
+  if (code === PAYLOAD_MISSING_ERROR_CODE) {
+    return new CorruptedEventLogError(
+      `the event log references a payload that no longer exists in storage: ${detail}`
+    );
+  }
+  return new WorkflowWorldError(
+    `v4 ${opName}: stream ended with terminal error "${code}": ${detail}`,
+    { code: 'INVALID_RESPONSE' }
+  );
+}
+
 async function consumeEventFrameStream(
   response: Response,
   opName: string,
@@ -1397,6 +1458,9 @@ async function consumeEventFrameStream(
       const end = EventStreamEndSchema.parse(frame.meta);
       return { cursor: end.next ?? null, hasMore: end.hasMore };
     }
+    if (frame.meta._error === 1) {
+      throw streamErrorFrameToError(frame.meta, opName);
+    }
     if (Object.keys(frame.meta).some((key) => key.startsWith('_'))) {
       throw new Error(`v4 ${opName}: unexpected control frame`);
     }

packages/world-vercel/src/utils.ts

@@ -39,8 +39,10 @@ import { version } from './version.js';
  * `main`. It is rewritten by external CI for branch-deployment testing.
  * Prefer `VERCEL_WORKFLOW_SERVER_URL` for deployment-time configuration.
  */
+// TEMPORARY: points at the world-vercel backend preview that emits the
+// terminal `payload-missing` frame this PR consumes. Revert to '' before merge.
 // biome-ignore format: External CI replaces only this line with a deployment URL that may exceed the formatter width.
-export const WORKFLOW_SERVER_URL_OVERRIDE = '';
+export const WORKFLOW_SERVER_URL_OVERRIDE = 'https://workflow-server-git-peter-s3-no-retry-404.vercel.sh';
 
 /**
  * HTTP methods that are safe to transparently re-issue inside the adapter.