vercel · VaguelySerious · Aug 10, 2026
@@ -0,0 +1,5 @@
+---
+'@workflow/core': patch
+---
+
+Fix replay divergence when a step result overtook an earlier sleep or hook delivery that was parked behind an unread hook's payload
@@ -28,6 +28,12 @@
  * ULIDs in the order the committed log recorded. A regression surfaces as the
  * production `ReplayDivergenceError`.
  *
+ * Section 3 asserts the registry's other job directly, over a registry built
+ * by hand rather than by a replay: which entries an idle check may ignore. Get
+ * that wrong in one direction and a chain parked on an unclaimed hook payload
+ * deadlocks; wrong in the other and a suspension preempts a batch of parked
+ * step results.
+ *
  * The final section covers the SUSPENSION side of the registry
  * (vercel/workflow#3183): an idle check must not observe idle — and raise a
  * `WorkflowSuspension` — while a delivery that is committed to reaching the
@@ -49,6 +55,7 @@ import { WorkflowSuspension } from './global.js';
 import {
   awaitEarlierDeliveries,
   registerDeliveryBarrier,
+  scheduleWhenIdle,
   type WorkflowOrchestratorContext,
 } from './private.js';
 import { dehydrateStepReturnValue } from './serialization.js';
@@ -394,41 +401,121 @@ describe('hook payload delivery ordering against an earlier step result', () =>
   }
 });
 
-// ─── 3. registry scan cost ─────────────────────────────────────────────────
+// ─── 3. idle reachability over the barrier registry ────────────────────────
+//
+// `hasParkedCommittedDelivery` decides whether an idle check may observe idle,
+// and it is the only remaining caller of the recursive `resolvesOnItsOwn`
+// walk. Two opposite answers are load-bearing, and neither is covered by the
+// replay sections above, which exercise the walk only through whichever shape
+// their fixture happens to build:
 //
-// `resolvesOnItsOwn` walks the registry recursively: an armed hook re-checks
-// every earlier wait and step, an armed wait every earlier hook and step, and
-// so on. Unmemoized that is T(n) = Σ T(j) — exponential — and the registry is
-// not small by construction: `EventsConsumer` drains consecutively consumable
-// events synchronously while barriers only retire on microtask-driven
-// deliveries, so a fan-out of `Promise.race([hook, sleep(watchdog)])` branches
-// accumulates one barrier per branch per kind (measured: 49 live barriers for
-// 24 branches).
+//  - A PARKED CHAIN must not be counted. An unclaimed buffered hook payload is
+//    retired by the idle safety net in `registerDeliveryBarrier`, so counting
+//    it would gate its own retirement — and that extends to the wait parked
+//    behind it and the step gated on that wait. If any link were counted, idle
+//    would be unreachable, no net could fire, and the chain would never
+//    deliver: a deadlock, not a divergence.
+//  - An ALL-ARMED BATCH must be counted (vercel/workflow#3183). Parallel step
+//    results parked between their queue slots and their detached `resolve()`
+//    are invisible to `pendingDeliveries`, and an idle check that observed idle
+//    there would raise a `WorkflowSuspension` carrying none of the follow-up
+//    work the batch was about to create.
 //
-// The scan runs synchronously, before `awaitEarlierDeliveries` first awaits,
-// so timing the call alone measures it. Unmemoized, 40 alternating armed
-// hook/wait barriers is ~10^8 recursive calls — minutes. Memoized it is
-// linear. The bound is deliberately loose; this is an order-of-magnitude
-// guard, not a benchmark.
-describe('delivery-barrier registry scan cost', () => {
-  it('stays linear in registry size for a step delivery', () => {
-    const ctx = {
+// Asserted through `scheduleWhenIdle`, which is the coupling that matters, and
+// which makes both cases unambiguous: nothing in these registries ever
+// delivers, so the callback can only fire if the registry was excluded from
+// the idle count AND the barriers' own nets then retired it.
+//
+// This replaces a timing guard that no longer measured anything. It timed
+// `awaitEarlierDeliveries(ctx, 40, 'step')` against 40 alternating armed
+// hook/wait barriers to catch an unmemoized exponential walk (4.3e8 recursive
+// calls, 84s). That call site is gone: a step now tests `armed` directly, so
+// the call is a flat loop. The surviving caller cannot reach an exponential
+// shape at all — it returns at the first self-resolving entry, so it only
+// advances past entries that short-circuit on their first false child
+// (measured: 98 recursive calls unmemoized for the worst 40-barrier shape).
+// Timing it would assert nothing; see the memo note on `resolvesOnItsOwn`.
+describe('delivery-barrier idle reachability', () => {
+  function emptyCtx(): WorkflowOrchestratorContext {
+    return {
       pendingDeliveries: 0,
       promiseQueue: Promise.resolve(),
       pendingDeliveryBarriers: new Map(),
     } as unknown as WorkflowOrchestratorContext;
+  }
+
+  /** Whether `scheduleWhenIdle` observes idle within `rounds` timer ticks. */
+  async function reachesIdle(
+    ctx: WorkflowOrchestratorContext,
+    rounds = 10
+  ): Promise<boolean> {
+    let idle = false;
+    scheduleWhenIdle(ctx, () => {
+      idle = true;
+    });
+    for (let round = 0; round < rounds && !idle; round++) {
+      await ctx.promiseQueue;
+      await new Promise((resolve) => setTimeout(resolve, 0));
+    }
+    return idle;
+  }
+
+  it('unwinds a step parked behind a wait parked on an unclaimed payload, in log order', async () => {
+    const ctx = emptyCtx();
+    const order: string[] = [];
+    let payloadRetiredBeforeWait: boolean | undefined;
+
+    // The shape `step-delivery-ordering.test.ts` replays, as a registry: an
+    // unread hook's payload at index 0, a wait behind it, a step gated on that
+    // wait. Only the payload lacks a delivery chain — nothing in the workflow
+    // ever claims it, so the idle safety net is the only thing that can retire
+    // it. The wait and the step get the unconditional chain their real call
+    // sites attach at event-consumption time, as the INVARIANT on
+    // `registerDeliveryBarrier` requires of any armed barrier.
+    registerDeliveryBarrier(ctx, 0, 'hook', { armed: false });
+    const wait = registerDeliveryBarrier(ctx, 1, 'wait');
+    const step = registerDeliveryBarrier(ctx, 2, 'step');
+    const chains = [
+      awaitEarlierDeliveries(ctx, 1, 'wait').then(() => {
+        order.push('wait');
+        payloadRetiredBeforeWait = !ctx.pendingDeliveryBarriers?.has(0);
+        wait.markDelivered();
+      }),
+      awaitEarlierDeliveries(ctx, 2, 'step').then(() => {
+        order.push('step');
+        step.markDelivered();
+      }),
+    ];
+    expect(ctx.pendingDeliveryBarriers?.size).toBe(3);
+
+    // Neither chain can run yet: the wait gates on the unclaimed payload, and
+    // the step gates on the wait (it skips the payload directly, but the skip
+    // is not transitive through the armed wait).
+    await Promise.resolve();
+    expect(order).toEqual([]);
+
+    // Idle must stay reachable for the payload's net to fire at all. If any
+    // link of the chain were counted against idle, this would hang.
+    expect(await reachesIdle(ctx)).toBe(true);
+    await Promise.all(chains);
+
+    expect(payloadRetiredBeforeWait).toBe(true);
+    expect(order).toEqual(['wait', 'step']);
+    expect(ctx.pendingDeliveryBarriers?.size).toBe(0);
+  });
 
-    const BARRIERS = 40;
-    for (let index = 0; index < BARRIERS; index++) {
-      registerDeliveryBarrier(ctx, index, index % 2 ? 'hook' : 'wait');
+  it('is blocked by an all-armed batch of step results', async () => {
+    const ctx = emptyCtx();
+    // Armed and undelivered is exactly the window #3183 is about: the batch's
+    // queue slots have released `pendingDeliveries` and their detached
+    // `resolve()` calls have not run yet.
+    for (let index = 0; index < 3; index++) {
+      registerDeliveryBarrier(ctx, index, 'step');
     }
-    expect(ctx.pendingDeliveryBarriers?.size).toBe(BARRIERS);
 
-    const startedAt = performance.now();
-    // The floating promise never settles (nothing delivers these barriers);
-    // only the synchronous scan inside the call is under test.
-    void awaitEarlierDeliveries(ctx, BARRIERS, 'step');
-    expect(performance.now() - startedAt).toBeLessThan(1_000);
+    expect(await reachesIdle(ctx)).toBe(false);
+    // The nets are idle-gated too, so nothing retires behind our back.
+    expect(ctx.pendingDeliveryBarriers?.size).toBe(3);
   });
 });
 

@@ -271,24 +271,69 @@ const DEFER_BEHIND: Record<DeliveryKind, readonly DeliveryKind[]> = {
   step: ['wait', 'hook', 'step'],
 };
 
+/**
+ * Whether a delivery of `kind` at log index `index` gates on the earlier
+ * registry entry `other` (at `otherIndex`).
+ *
+ * Single source of truth for that question, called by both
+ * {@link awaitEarlierDeliveries} (which awaits what it gates on) and
+ * {@link computeResolvesOnItsOwn} (which recurses into what it gates on).
+ * Those two MUST agree exactly, and the doc block on
+ * {@link awaitEarlierDeliveries} stakes deadlock-freedom on it, so the
+ * condition lives here rather than being spelled out twice.
+ */
+function gatesOn(
+  kind: DeliveryKind,
+  index: number,
+  otherIndex: number,
+  other: DeliveryBarrierEntry
+): boolean {
+  if (otherIndex >= index || !DEFER_BEHIND[kind].includes(other.kind)) {
+    return false;
+  }
+  // A step skips an UNARMED earlier entry (an unclaimed buffered hook
+  // payload) — see the asymmetry described on `awaitEarlierDeliveries`. The
+  // skip is direct, never transitive: armed entries are still gated on, even
+  // when they are themselves parked behind such a payload.
+  return !(kind === 'step' && !other.armed);
+}
+
 /**
  * Whether `entry` will resolve on its own — it is armed, and every earlier
- * delivery it defers behind will likewise resolve on its own.
+ * delivery it actually gates on ({@link gatesOn}) will likewise resolve on its
+ * own.
  *
- * A step delivery is always self-resolving: it skips uncommitted deliveries
- * (see {@link awaitEarlierDeliveries}), and the earlier steps it does defer
- * behind are self-resolving by the same argument, inducting down on index.
+ * A step does not gate on an unclaimed buffered payload, so such a payload
+ * cannot keep it from resolving. A step DOES gate on earlier armed waits and
+ * hooks, so one parked behind an unclaimed payload makes the step
+ * non-self-resolving in turn. Disagreeing with {@link awaitEarlierDeliveries}
+ * here would not be a cosmetic problem: this predicate is what
+ * {@link hasParkedCommittedDelivery} uses to decide whether idle is reachable,
+ * and an entry reported self-resolving while it is in fact parked behind a
+ * payload that only the idle safety net can retire would gate its own
+ * retirement.
  *
  * Recursion terminates because every edge points to a strictly smaller index.
- * `memo` is required rather than an optimization: without it the walk is
- * exponential in the number of live hook/wait barriers (each armed entry
- * re-walks every earlier entry of the opposite kind, T(n) = Σ T(j)), and the
- * registry is not small by construction — `EventsConsumer` drains
- * consecutively consumable events synchronously while barriers only retire on
- * microtask-driven deliveries, so a fan-out of `Promise.race([hook, sleep])`
- * branches accumulates one barrier per branch per kind. Memoized, the walk is
- * linear in registry size. The memo MUST be per-call: `armed` mutates between
+ * `memo` keeps the walk linear in registry size, and the registry is not small
+ * by construction — `EventsConsumer` drains consecutively consumable events
+ * synchronously while barriers only retire on microtask-driven deliveries, so
+ * a fan-out of `Promise.race([hook, sleep])` branches accumulates one barrier
+ * per branch per kind. The memo MUST be per-call: `armed` mutates between
  * calls as buffered payloads are claimed.
+ *
+ * The memo is an optimization, not a correctness requirement. It once was one:
+ * `awaitEarlierDeliveries` used to run this walk for every earlier entry of a
+ * step delivery, with no early exit, which unmemoized is T(n) = Σ T(j) —
+ * measured at 4.3e8 recursive calls (84s) for 40 alternating armed hook/wait
+ * barriers. That call site is gone; a step now tests `armed` directly. The one
+ * surviving caller, {@link hasParkedCommittedDelivery}, cannot reach that
+ * shape: it returns at the FIRST self-resolving entry, so it only ever
+ * advances past entries that are non-self-resolving, and those short-circuit
+ * on their first false child. Every entry it evaluates therefore has
+ * all-false predecessors and returns after one child, degenerating the walk to
+ * a chain (measured: 98 calls unmemoized for the worst 40-barrier shape, 1
+ * call for the registry above). Do not restore an exponential claim here
+ * without restoring a caller that can produce it.
  */
 function resolvesOnItsOwn(
   barriers: Map<number, DeliveryBarrierEntry>,
@@ -314,16 +359,11 @@ function computeResolvesOnItsOwn(
   if (!entry.armed) {
     return false;
   }
-  if (entry.kind === 'step') {
-    return true;
-  }
-  const deferBehind = DEFER_BEHIND[entry.kind];
   for (const [otherIndex, other] of barriers) {
-    if (
-      otherIndex < index &&
-      deferBehind.includes(other.kind) &&
-      !resolvesOnItsOwn(barriers, otherIndex, other, memo)
-    ) {
+    if (!gatesOn(entry.kind, index, otherIndex, other)) {
+      continue;
+    }
+    if (!resolvesOnItsOwn(barriers, otherIndex, other, memo)) {
       return false;
     }
   }
@@ -343,31 +383,44 @@ function computeResolvesOnItsOwn(
  * suspension point first; see the comment at that `await` for why ordering the
  * `resolve()` calls alone is not enough.
  *
- * One asymmetry: a STEP result additionally skips any earlier delivery that
- * will not resolve on its own, i.e. one blocked (directly or transitively) on
- * a buffered hook payload no consumer has claimed. Such a payload is delivered
- * only when the workflow next reads the hook, and reaching that read very
- * commonly requires the step result itself (`await stepX()` before the read).
- * Gating the step on it would stall the workflow until the barrier's idle
- * safety net fires, which then releases every delivery queued behind that
+ * What counts as "defers behind" is {@link gatesOn}, shared with
+ * {@link computeResolvesOnItsOwn} so the two cannot drift.
+ *
+ * One asymmetry: a STEP result skips any earlier delivery that is UNARMED,
+ * i.e. a buffered hook payload no consumer has claimed. Such a payload is
+ * delivered only when the workflow next reads the hook, and reaching that read
+ * very commonly requires the step result itself (`await stepX()` before the
+ * read). Gating the step on it would stall the workflow until the barrier's
+ * idle safety net fires, which then releases every delivery queued behind that
  * payload at once — losing exactly the race this ordering exists to protect.
  * Waits and hooks keep gating on unclaimed payloads: for them, waiting for the
  * claim IS the ordering guarantee (a `wait_completed` must not preempt a
  * payload the log ordered first).
  *
- * A delivery that does gate on an unclaimed payload still delivers in log
- * order once that payload's barrier is retired, and that rests on the
- * PAYLOAD's barrier being retired before that of anything parked behind it.
- * That order is structural: safety-net retirements go through one per-context
- * dispenser that only ever retires the lowest-index entry at delivery idle,
- * and every retirement that wakes a chain flips
- * {@link hasParkedCommittedDelivery} back to true, re-blocking the dispenser
- * until the chain has drained — see {@link ensureBarrierSafetyNet}. (This
- * used to rest on the FIFO of one idle poll per barrier, which held for a
- * single parked segment but decayed to timing noise with several — the
- * release order, and therefore the ULIDs drawn by the woken branches, then
- * depended on how much log the replay had loaded. storm-log-replay.test.ts
- * replays a production log corrupted exactly that way.)
+ * The skip is direct, never transitive. A step still gates on an earlier ARMED
+ * wait or hook, including one that is itself parked behind an unclaimed
+ * payload. Skipping those too would invert log order for the commonest shape
+ * there is: a workflow that creates a hook it does not read on this branch,
+ * races `step` against `sleep`, and has the log say the sleep won. The step
+ * would then overtake the wait, both branches would swap the correlation ids
+ * they draw next, and replay would diverge — see
+ * `step-delivery-ordering.test.ts`. Waiting instead is safe because the
+ * payload's own idle safety net retires it and the whole chain then delivers
+ * in log order; {@link hasParkedCommittedDelivery} deliberately reports such a
+ * step as not self-resolving so that idle stays reachable.
+ *
+ * "The whole chain then delivers in log order" rests on the PAYLOAD's barrier
+ * being retired before that of anything parked behind it. That order is
+ * structural: safety-net retirements go through one per-context dispenser that
+ * only ever retires the lowest-index entry at delivery idle, and every
+ * retirement that wakes a chain flips {@link hasParkedCommittedDelivery} back
+ * to true, re-blocking the dispenser until the chain has drained — see
+ * {@link ensureBarrierSafetyNet}. (This used to rest on the FIFO of one idle
+ * poll per barrier, which held for a single parked segment but decayed to
+ * timing noise with several — the release order, and therefore the ULIDs
+ * drawn by the woken branches, then depended on how much log the replay had
+ * loaded. storm-log-replay.test.ts replays a production log corrupted exactly
+ * that way.)
  */
 export async function awaitEarlierDeliveries(
   ctx: WorkflowOrchestratorContext,
@@ -383,18 +436,9 @@ export async function awaitEarlierDeliveries(
     return;
   }
   const barriers = ctx.pendingDeliveryBarriers;
-  const deferBehind = DEFER_BEHIND[kind];
   const earlier: Promise<void>[] = [];
-  // Shared across this call only — see `resolvesOnItsOwn`.
-  const selfResolving = new Map<number, boolean>();
   for (const [index, entry] of barriers) {
-    if (index >= eventIndex || !deferBehind.includes(entry.kind)) {
-      continue;
-    }
-    if (
-      kind === 'step' &&
-      !resolvesOnItsOwn(barriers, index, entry, selfResolving)
-    ) {
+    if (!gatesOn(kind, eventIndex, index, entry)) {
       continue;
     }
     earlier.push(entry.delivered);
@@ -642,7 +686,10 @@ function canRetireAbandonedBarriers(ctx: WorkflowOrchestratorContext): boolean {
  * Deliveries that do NOT resolve on their own must be excluded, not for
  * accuracy but for termination: an unclaimed buffered hook payload is retired
  * BY the idle safety net in {@link registerDeliveryBarrier}, so counting it
- * here would gate its own retirement. Self-resolving deliveries always
+ * here would gate its own retirement. That reasoning extends to whatever is
+ * parked behind such a payload — a wait, and a step gating on that wait — for
+ * the same reason: the whole chain moves only once the net fires, and it
+ * cannot fire while the chain is counted. Self-resolving deliveries always
  * deliver from their own chains (see the INVARIANT on
  * {@link registerDeliveryBarrier}) and never need that net, so waiting on
  * them is deadlock-free.