From e01c03f3589e9b15d1022c0e4d5ba52e14c47c42 Mon Sep 17 00:00:00 2001 From: Ethan Dickson Date: Thu, 3 Sep 2026 19:01:19 +0000 Subject: [PATCH 01/26] fix: bash-monitor wakes are a level, never a queued edge A monitored bash task matching mid-step queued a tool-end wake that cut the stream, then task_await showed the lines and the wake was withdrawn, leaving the turn to end silently with finishReason tool-calls. The stream now reads the reconciler level (hasPendingToolEndInput) at each tool boundary and the reconciler dispatches only when the owner is idle, so the queued-wake cancellation protocol is deleted. --- ...code-cross-session-messaging-comparison.md | 2 +- .../agentSession.queueDispatch.test.ts | 445 +++--------------- src/node/services/agentSession.testHarness.ts | 2 + src/node/services/agentSession.ts | 288 ++++-------- .../builtInSkillContent.generated.ts | 2 +- .../bashMonitorWakeReconciler.test.ts | 133 ++++-- .../services/bashMonitorWakeReconciler.ts | 105 ++--- src/node/services/messageQueue.test.ts | 19 - src/node/services/messageQueue.ts | 45 +- src/node/services/streamManager.test.ts | 31 +- src/node/services/streamManager.ts | 19 +- src/node/services/taskService.test.ts | 25 +- src/node/services/taskService.ts | 4 +- .../services/taskWorkspaceSeam.testUtils.ts | 3 +- src/node/services/taskWorkspaceSeam.ts | 8 +- src/node/services/turnRequestBuilder.ts | 7 +- src/node/services/workspaceService.test.ts | 273 +++++------ src/node/services/workspaceService.ts | 114 ++--- .../services/workspaceTurnManager.test.ts | 2 +- src/node/services/workspaceTurnManager.ts | 18 +- 20 files changed, 545 insertions(+), 1000 deletions(-) diff --git a/docs/research/claude-code-cross-session-messaging-comparison.md b/docs/research/claude-code-cross-session-messaging-comparison.md index 2395ea38fa..d07198ac96 100644 --- a/docs/research/claude-code-cross-session-messaging-comparison.md +++ b/docs/research/claude-code-cross-session-messaging-comparison.md @@ -54,7 +54,7 @@ Mux's unit is not a terminal session bound to a socket; it is a **workspace** (w Messages to a busy workspace enter its `MessageQueue` (`src/node/services/messageQueue.ts`) and dispatch at a boundary chosen by `queueDispatchMode`: -- `tool-end`: the stream's stop conditions include `hasQueuedMessages("tool-end")`, evaluated by the AI SDK only after every sibling tool result in the current step settles (`createStopWhenCondition`, `src/node/services/streamManager.ts`); `AgentSession` soft-stops only once `activeToolCallIds` is empty. **A running tool call is never interrupted** — same guarantee as Claude Code. +- `tool-end`: the stream's stop conditions include `hasPendingToolEndInput()` (a queued tool-end message or an outstanding bash-monitor wake, read live), evaluated by the AI SDK only after every sibling tool result in the current step settles (`createStopWhenCondition`, `src/node/services/streamManager.ts`); `AgentSession` soft-stops only once `activeToolCallIds` is empty. **A running tool call is never interrupted** — same guarantee as Claude Code. - `turn-end`: dispatches after the current turn completes. - Idle target: the message starts a new turn immediately. diff --git a/src/node/services/agentSession.queueDispatch.test.ts b/src/node/services/agentSession.queueDispatch.test.ts index 08c6774a27..2e2e66680e 100644 --- a/src/node/services/agentSession.queueDispatch.test.ts +++ b/src/node/services/agentSession.queueDispatch.test.ts @@ -1,7 +1,7 @@ import { describe, expect, mock, spyOn, test } from "bun:test"; import type { MuxMessageMetadata } from "@/common/types/message"; -import { Err, Ok } from "@/common/types/result"; +import { Ok } from "@/common/types/result"; import type { WorkspaceGoalService } from "./workspaceGoalService"; import { createAgentSessionHarness, createStartedTurnHandle } from "./agentSession.testHarness"; import type { AIService } from "./aiService"; @@ -372,87 +372,6 @@ describe("AgentSession queued message tool-call dispatch", () => { } }); - test("withdrawn tool-end entry neither soft-stops nor hides a later entry's mode", async () => { - const workspaceId = "queue-dispatch-withdrawn-head"; - const queuedSignals: boolean[] = []; - const { session, cleanup, aiEmitter, aiService } = await createAgentSessionHarness({ - workspaceId, - backgroundProcessManagerOverrides: { - setMessageQueued: mock((_workspaceId: string, queued: boolean) => { - queuedSignals.push(queued); - }), - }, - }); - const stopStream = spyOn(aiService, "stopStream").mockResolvedValue(Ok(undefined)); - - try { - aiEmitter.emit("stream-start", streamStartEvent(workspaceId)); - const controller = new AbortController(); - session.queueMessage( - "Background monitor wake", - { model: TEST_MODEL, agentId: "exec", queueDispatchMode: "tool-end" }, - { synthetic: true, agentInitiated: true, cancelSignal: controller.signal } - ); - expect(session.hasQueuedMessages("tool-end")).toBe(true); - - controller.abort("monitor withdrawn"); - expect(session.hasQueuedMessages("tool-end")).toBe(false); - expect(session.hasQueuedMessages()).toBe(false); - - aiEmitter.emit("tool-call-end", { - ...toolCallEndEvent(workspaceId), - toolName: "web_search", - providerExecuted: true, - }); - expect(stopStream).not.toHaveBeenCalled(); - - session.queueMessage("follow up", { - model: TEST_MODEL, - agentId: "exec", - queueDispatchMode: "turn-end", - }); - expect(session.hasQueuedMessages("tool-end")).toBe(false); - expect(session.hasQueuedMessages("turn-end")).toBe(true); - expect(queuedSignals).toEqual([true, false]); - } finally { - stopStream.mockRestore(); - session.dispose(); - await cleanup(); - } - }); - - test.each([ - ["turn-end", "tool-end"], - ["tool-end", "turn-end"], - ] as const)( - "queueMessage reports the live entry's mode behind a withdrawn %s head", - async (withdrawnMode, liveMode) => { - const { session, cleanup } = await createAgentSessionHarness({ - workspaceId: "queue-dispatch-withdrawn-" + withdrawnMode + "-head", - }); - try { - const controller = new AbortController(); - session.queueMessage( - "Background monitor wake", - { model: TEST_MODEL, agentId: "exec", queueDispatchMode: withdrawnMode }, - { synthetic: true, agentInitiated: true, cancelSignal: controller.signal } - ); - controller.abort("monitor withdrawn"); - - expect( - session.queueMessage("follow up", { - model: TEST_MODEL, - agentId: "exec", - queueDispatchMode: liveMode, - }) - ).toBe(liveMode); - } finally { - session.dispose(); - await cleanup(); - } - } - ); - test("waits for every known sibling before stopping after a provider-executed result", async () => { const workspaceId = "queue-dispatch-provider-siblings"; const { session, cleanup, aiEmitter, aiService } = await createAgentSessionHarness({ @@ -706,316 +625,70 @@ describe("AgentSession queued message tool-call dispatch", () => { } }); - test("cancel signal retracts a synthetic entry after dequeue while history append is preparing", async () => { - const workspaceId = "queue-dispatch-cancel-preparing"; - const { session, cleanup, historyService, events } = await createAgentSessionHarness({ + test("hasPendingToolEndInput unions the queued tool-end head with the live wake level", async () => { + const workspaceId = "queue-dispatch-pending-tool-end-input"; + let level: () => Promise = () => Promise.resolve(false); + const { session, cleanup } = await createAgentSessionHarness({ workspaceId, - captureEvents: true, - }); - const originalAppend = historyService.appendToHistory.bind(historyService); - let markAppendStarted: () => void = () => undefined; - const appendStarted = new Promise((resolve) => { - markAppendStarted = resolve; - }); - let releaseAppend: () => void = () => undefined; - const appendRelease = new Promise((resolve) => { - releaseAppend = resolve; - }); - const appendSpy = spyOn(historyService, "appendToHistory").mockImplementation( - async (...args) => { - markAppendStarted(); - await appendRelease; - return originalAppend(...args); - } - ); - - try { - const controller = new AbortController(); - const cancelState = { canceledBeforeAcceptance: false }; - const canceledReasons: string[] = []; - session.queueMessage( - "Background monitor wake", - { model: TEST_MODEL, agentId: "exec" }, - { - synthetic: true, - agentInitiated: true, - cancelState, - cancelSignal: controller.signal, - onCanceled: (reason) => { - canceledReasons.push(reason); - }, - } - ); - - session.sendQueuedMessages(); - await appendStarted; - controller.abort("monitor canceled"); - releaseAppend(); - - expect(await waitForCondition(() => canceledReasons.length === 1)).toBe(true); - expect(await waitForCondition(() => !session.isBusy())).toBe(true); - expect(canceledReasons).toEqual(["monitor canceled"]); - - const history = await historyService.getHistoryFromLatestBoundary(workspaceId); - expect(history.success).toBe(true); - if (history.success) { - expect( - history.data.some((message) => - message.parts.some( - (part) => part.type === "text" && part.text === "Background monitor wake" - ) - ) - ).toBe(false); - } - expect( - events.some( - (event) => - event.type === "message" && - event.role === "user" && - event.parts.some( - (part) => part.type === "text" && part.text === "Background monitor wake" - ) - ) - ).toBe(false); - } finally { - releaseAppend(); - appendSpy.mockRestore(); - session.dispose(); - await cleanup(); - } - }); - - test("rollback failure preserves the wake and continues acceptance", async () => { - const workspaceId = "queue-dispatch-cancel-rollback-failure"; - const { session, cleanup, historyService } = await createAgentSessionHarness({ workspaceId }); - const originalAppend = historyService.appendToHistory.bind(historyService); - let markAppendStarted: () => void = () => undefined; - const appendStarted = new Promise((resolve) => { - markAppendStarted = resolve; - }); - let releaseAppend: () => void = () => undefined; - const appendRelease = new Promise((resolve) => { - releaseAppend = resolve; - }); - const appendSpy = spyOn(historyService, "appendToHistory").mockImplementation( - async (...args) => { - markAppendStarted(); - await appendRelease; - return originalAppend(...args); - } - ); - const deleteMessagesSpy = spyOn(historyService, "deleteMessages").mockResolvedValue( - Err("injected rollback failure") - ); - - try { - const controller = new AbortController(); - const cancelState = { canceledBeforeAcceptance: false }; - const canceledReasons: string[] = []; - let accepted = false; - const sendPromise = session.sendMessage( - "Background monitor wake", - { model: TEST_MODEL, agentId: "exec" }, - { - synthetic: true, - agentInitiated: true, - cancelState, - cancelSignal: controller.signal, - onCanceled: (reason) => { - canceledReasons.push(reason); - }, - onAccepted: () => { - accepted = true; - }, - } - ); - - await appendStarted; - controller.abort("monitor canceled"); - releaseAppend(); - const result = await sendPromise; - - expect(result.success).toBe(true); - expect(deleteMessagesSpy).toHaveBeenCalledTimes(1); - expect(canceledReasons).toEqual([]); - expect(cancelState.canceledBeforeAcceptance).toBe(false); - expect(accepted).toBe(true); - - const history = await historyService.getHistoryFromLatestBoundary(workspaceId); - expect(history.success).toBe(true); - if (history.success) { - expect( - history.data.some((message) => - message.parts.some( - (part) => part.type === "text" && part.text === "Background monitor wake" - ) - ) - ).toBe(true); - } - } finally { - releaseAppend(); - deleteMessagesSpy.mockRestore(); - appendSpy.mockRestore(); - session.dispose(); - await cleanup(); - } - }); - - test("verifies a committed rollback when batch deletion reports a post-write failure", async () => { - const workspaceId = "queue-dispatch-cancel-post-write-failure"; - const { session, cleanup, historyService } = await createAgentSessionHarness({ workspaceId }); - const originalAppend = historyService.appendToHistory.bind(historyService); - const originalDeleteMessages = historyService.deleteMessages.bind(historyService); - let markAppendStarted: () => void = () => undefined; - const appendStarted = new Promise((resolve) => { - markAppendStarted = resolve; + hasOutstandingBashMonitorWake: () => level(), }); - let releaseAppend: () => void = () => undefined; - const appendRelease = new Promise((resolve) => { - releaseAppend = resolve; - }); - const appendSpy = spyOn(historyService, "appendToHistory").mockImplementation( - async (...args) => { - markAppendStarted(); - await appendRelease; - return originalAppend(...args); - } - ); - const deleteMessagesSpy = spyOn(historyService, "deleteMessages").mockImplementation( - async (...args) => { - const result = await originalDeleteMessages(...args); - expect(result.success).toBe(true); - return Err("injected post-write failure"); - } - ); - try { - const controller = new AbortController(); - const cancelState = { canceledBeforeAcceptance: false }; - const canceledReasons: string[] = []; - let accepted = false; - const sendPromise = session.sendMessage( - "Background monitor wake", - { model: TEST_MODEL, agentId: "exec" }, - { - synthetic: true, - agentInitiated: true, - cancelState, - cancelSignal: controller.signal, - onCanceled: (reason) => { - canceledReasons.push(reason); - }, - onAccepted: () => { - accepted = true; - }, - } - ); + expect(await session.hasPendingToolEndInput()).toBe(false); - await appendStarted; - controller.abort("monitor canceled"); - releaseAppend(); - const result = await sendPromise; + // The level is read live — no snapshot survives from one boundary to the next. + level = () => Promise.resolve(true); + expect(await session.hasPendingToolEndInput()).toBe(true); + level = () => Promise.resolve(false); + expect(await session.hasPendingToolEndInput()).toBe(false); - expect(result.success).toBe(true); - expect(deleteMessagesSpy).toHaveBeenCalledTimes(1); - expect(canceledReasons).toEqual(["monitor canceled"]); - expect(cancelState.canceledBeforeAcceptance).toBe(true); - expect(accepted).toBe(false); + // A failing level read must not cut the stream. + level = () => Promise.reject(new Error("watermark read failed")); + expect(await session.hasPendingToolEndInput()).toBe(false); - const history = await historyService.getHistoryFromLatestBoundary(workspaceId); - expect(history.success).toBe(true); - if (history.success) { - expect( - history.data.some((message) => - message.parts.some( - (part) => part.type === "text" && part.text === "Background monitor wake" - ) - ) - ).toBe(false); - } + // Queue head decides independently of the level, and only for tool-end. + level = () => Promise.resolve(false); + session.queueMessage("later", { + model: TEST_MODEL, + agentId: "exec", + queueDispatchMode: "turn-end", + }); + expect(await session.hasPendingToolEndInput()).toBe(false); + session.clearQueue(); + session.queueMessage("now", { model: TEST_MODEL, agentId: "exec" }); + expect(await session.hasPendingToolEndInput()).toBe(true); } finally { - releaseAppend(); - deleteMessagesSpy.mockRestore(); - appendSpy.mockRestore(); session.dispose(); await cleanup(); } }); - test("cancellation during goal sync crosses the acceptance point of no return", async () => { - const workspaceId = "queue-dispatch-cancel-goal-reconcile"; - let markInitialSyncStarted: () => void = () => undefined; - const initialSyncStarted = new Promise((resolve) => { - markInitialSyncStarted = resolve; - }); - let releaseInitialSync: () => void = () => undefined; - const initialSyncRelease = new Promise((resolve) => { - releaseInitialSync = resolve; - }); - let syncCalls = 0; - const syncGoalModeWithChatTail = mock(async () => { - syncCalls += 1; - if (syncCalls === 1) { - markInitialSyncStarted(); - await initialSyncRelease; - } - return null; - }); - const workspaceGoalService = { - assertPricedModelForBudgetedGoal: mock(() => Promise.resolve(Ok(undefined))), - syncGoalModeWithChatTail, - } as unknown as WorkspaceGoalService; - const { session, cleanup, historyService } = await createAgentSessionHarness({ + test("the wake level and the queue head jointly drive the bash early-return flag", async () => { + const workspaceId = "queue-dispatch-yield-flag"; + const flags: boolean[] = []; + const { session, cleanup } = await createAgentSessionHarness({ workspaceId, - workspaceGoalService, + backgroundProcessManagerOverrides: { + setMessageQueued: (_workspaceId: string, queued: boolean) => { + flags.push(queued); + }, + }, }); - + const lastFlag = () => flags.at(-1); try { - const controller = new AbortController(); - const cancelState = { canceledBeforeAcceptance: false }; - const canceledReasons: string[] = []; - let accepted = false; - const sendPromise = session.sendMessage( - "Background monitor wake", - { model: TEST_MODEL, agentId: "exec" }, - { - synthetic: true, - agentInitiated: true, - cancelState, - cancelSignal: controller.signal, - onCanceled: (reason) => { - canceledReasons.push(reason); - }, - onAccepted: () => { - accepted = true; - }, - } - ); - - await initialSyncStarted; - controller.abort("monitor canceled"); - releaseInitialSync(); - const result = await sendPromise; - - expect(result.success).toBe(true); - expect(syncGoalModeWithChatTail).toHaveBeenCalledTimes(1); - expect(canceledReasons).toEqual([]); - expect(cancelState.canceledBeforeAcceptance).toBe(false); - expect(accepted).toBe(true); + session.setBashMonitorWakeOutstanding(true); + expect(lastFlag()).toBe(true); + session.setBashMonitorWakeOutstanding(false); + expect(lastFlag()).toBe(false); - const history = await historyService.getHistoryFromLatestBoundary(workspaceId); - expect(history.success).toBe(true); - if (history.success) { - expect( - history.data.some((message) => - message.parts.some( - (part) => part.type === "text" && part.text === "Background monitor wake" - ) - ) - ).toBe(true); - } + // Clearing the queue while the level is high must not drop the flag. + session.queueMessage("follow up", { model: TEST_MODEL, agentId: "exec" }); + expect(lastFlag()).toBe(true); + session.setBashMonitorWakeOutstanding(true); + session.clearQueue(); + expect(lastFlag()).toBe(true); + session.setBashMonitorWakeOutstanding(false); + expect(lastFlag()).toBe(false); } finally { - releaseInitialSync(); session.dispose(); await cleanup(); } @@ -1047,17 +720,17 @@ describe("AgentSession queued message tool-call dispatch", () => { let disposed = false; try { - const controller = new AbortController(); - const cancelState = { canceledBeforeAcceptance: false }; let accepted = false; const sendPromise = session.sendMessage( "Background monitor wake", - { model: TEST_MODEL, agentId: "exec" }, + { + model: TEST_MODEL, + agentId: "exec", + muxMetadata: { type: "bash-monitor-wake", records: [] }, + }, { synthetic: true, agentInitiated: true, - cancelState, - cancelSignal: controller.signal, onAccepted: () => { accepted = true; }, @@ -1072,7 +745,6 @@ describe("AgentSession queued message tool-call dispatch", () => { expect(result.success).toBe(true); expect(accepted).toBe(true); - expect(cancelState.canceledBeforeAcceptance).toBe(false); } finally { releaseSync(); if (!disposed) session.dispose(); @@ -1105,18 +777,18 @@ describe("AgentSession queued message tool-call dispatch", () => { }); try { - const controller = new AbortController(); - const cancelState = { canceledBeforeAcceptance: false }; const canceledReasons: string[] = []; let accepted = false; const sendPromise = session.sendMessage( "Background monitor wake", - { model: TEST_MODEL, agentId: "exec" }, + { + model: TEST_MODEL, + agentId: "exec", + muxMetadata: { type: "bash-monitor-wake", records: [] }, + }, { synthetic: true, agentInitiated: true, - cancelState, - cancelSignal: controller.signal, onCanceled: (reason) => { canceledReasons.push(reason); }, @@ -1139,7 +811,6 @@ describe("AgentSession queued message tool-call dispatch", () => { expect(accepted).toBe(true); expect(canceledReasons).toEqual([]); - expect(cancelState.canceledBeforeAcceptance).toBe(false); const history = await historyService.getHistoryFromLatestBoundary(workspaceId); expect(history.success).toBe(true); if (history.success) { diff --git a/src/node/services/agentSession.testHarness.ts b/src/node/services/agentSession.testHarness.ts index fac756ca28..c38911886e 100644 --- a/src/node/services/agentSession.testHarness.ts +++ b/src/node/services/agentSession.testHarness.ts @@ -110,6 +110,7 @@ export interface AgentSessionHarnessOptions { workspaceGoalService?: WorkspaceGoalService; mcpServerManager?: MCPServerManager; onCompactionComplete?: (metadata: CompactionCompletionMetadata) => void; + hasOutstandingBashMonitorWake?: () => Promise; captureEvents?: boolean; } @@ -154,6 +155,7 @@ export async function createAgentSessionHarness( workspaceGoalService: options.workspaceGoalService, backgroundProcessManager, onCompactionComplete: options.onCompactionComplete, + hasOutstandingBashMonitorWake: options.hasOutstandingBashMonitorWake, }); const events: WorkspaceChatMessage[] = []; diff --git a/src/node/services/agentSession.ts b/src/node/services/agentSession.ts index 29f406899a..7264f69819 100644 --- a/src/node/services/agentSession.ts +++ b/src/node/services/agentSession.ts @@ -113,7 +113,7 @@ import { createRuntimeContextForWorkspace, createRuntimeForWorkspace, } from "@/node/runtime/runtimeHelpers"; -import { MessageQueue, cancelReasonBeforeAcceptance } from "./messageQueue"; +import { MessageQueue } from "./messageQueue"; import type { QueueCutCutter } from "./messageQueue"; import { copyStreamLifecycleSnapshot, @@ -382,9 +382,9 @@ function hasSameWorkspaceTurnCorrelation( * Find the still-open workspace-turn correlation for a bash-monitor-wake * continuation stream. * - * A queued monitor wake dispatched at a tool boundary cuts the in-flight - * stream (finishReason "tool-calls") and immediately continues the same - * delegated work in a new stream. That continuation must inherit the cut + * An outstanding monitor wake makes the in-flight stream yield at a tool + * boundary (finishReason "tool-calls"); the wake turn sent once the owner is + * idle continues the same delegated work in a new stream. That continuation must inherit the cut * stream's workspace-turn metadata — otherwise the delegating parent sees the * cut as a premature turn failure ("Workspace turn ended before completion") * and the turn's real outcome can never settle the task handle (see @@ -623,6 +623,12 @@ interface AgentSessionOptions { * to yield to a manual send that is still awaiting pricing/settings. */ hasExternalSendPreflight?: () => boolean; + /** + * The bash-monitor wake level for this workspace (BashMonitorWakeReconciler + * .hasOutstandingWake). Read live at every tool boundary; wakes are never + * queued as messages, so this is the only way a stream learns one is pending. + */ + hasOutstandingBashMonitorWake?: () => Promise; } enum TurnPhase { @@ -664,6 +670,12 @@ export class AgentSession { private readonly sanitizeCliWorkspaceRegistration?: AgentSessionOptions["sanitizeCliWorkspaceRegistration"]; private readonly onPostCompactionStateChange?: () => void; private readonly hasExternalSendPreflight?: () => boolean; + private readonly hasOutstandingBashMonitorWake?: () => Promise; + /** + * Last published wake level (see setBashMonitorWakeOutstanding). Feeds the + * tool-end yield flag together with the queue head. + */ + private bashMonitorWakeOutstanding = false; private readonly emitter = new EventEmitter(); private readonly aiListeners: Array<{ event: string; handler: (...args: unknown[]) => void }> = []; @@ -845,8 +857,8 @@ export class AgentSession { /** * muxMetadata of the queued entry currently being dispatched, held from * dequeue until its sendMessage settles (the stream has started or failed). - * Lets hasPendingBashMonitorWakeContinuation see a wake continuation during - * the dequeue→stream-start window without consulting stale stream context. + * Lets hasPendingWorkspaceTurnContinuation / hasQueuedOrDispatchingEntry see the + * dequeue→stream-start window without consulting stale stream context. */ private dispatchingQueuedEntry = false; private dispatchingQueuedEntryMuxMetadata?: unknown; @@ -903,6 +915,7 @@ export class AgentSession { onIdleCompactionOutcome, onPostCompactionStateChange, hasExternalSendPreflight, + hasOutstandingBashMonitorWake, } = options; assert(typeof workspaceId === "string", "workspaceId must be a string"); @@ -931,6 +944,7 @@ export class AgentSession { this.sanitizeCliWorkspaceRegistration = sanitizeCliWorkspaceRegistration; this.onPostCompactionStateChange = onPostCompactionStateChange; this.hasExternalSendPreflight = hasExternalSendPreflight; + this.hasOutstandingBashMonitorWake = hasOutstandingBashMonitorWake; this.compactionHandler = new CompactionHandler({ workspaceId: this.workspaceId, @@ -3024,8 +3038,6 @@ export class AgentSession { onAccepted?: () => Promise | void; onAcceptedPreStreamFailure?: (error: SendMessageError) => Promise | void; onCanceled?: (reason: string) => Promise | void; - cancelState?: { canceledBeforeAcceptance: boolean }; - cancelSignal?: AbortSignal; /** * For queue-dispatched sends: when the user last added to the queued * entry. Goal safety compares it against the goal's explicit @@ -3040,8 +3052,8 @@ export class AgentSession { * turn claims PREPARING (isBusy() becomes true). WorkspaceService keeps * its session-invisible preflight reservation armed until this fires so * follow-up recovery cannot observe the idle gap between the service - * handoff and the busy claim (cancelBeforeAcceptance and the other - * admission awaits yield) and admit a synthetic turn ahead of the + * handoff and the busy claim (the admission awaits yield) and admit a + * synthetic turn ahead of the * accepted manual send. Refusal paths never fire it — the service's * scoped disposal releases the reservation when the call returns. */ @@ -3095,8 +3107,7 @@ export class AgentSession { const isAdmissionStale = () => internal?.admissionEpochStale?.() === true || internal?.admissionStale?.() === true; - const cancelSignal = internal?.cancelSignal; - const persistedCancelableMessageIds: string[] = []; + const persistedTurnRowMessageIds: string[] = []; // Roll back synthetic snapshots if the invoking user row fails to persist, or // later provider requests could consume orphaned context. /** @@ -3107,10 +3118,10 @@ export class AgentSession { * counting against the sender's budget. */ const rollbackPersistedTurnRows = async (): Promise => { - if (persistedCancelableMessageIds.length === 0) return true; + if (persistedTurnRowMessageIds.length === 0) return true; const rollbackResult = await this.historyService.deleteMessages( this.workspaceId, - persistedCancelableMessageIds + persistedTurnRowMessageIds ); if (rollbackResult.success) return true; log.error("Failed to roll back partially persisted turn rows", { @@ -3122,65 +3133,11 @@ export class AgentSession { ); return ( historyResult.success && - persistedCancelableMessageIds.every( + persistedTurnRowMessageIds.every( (messageId) => !historyResult.data.some((message) => message.id === messageId) ) ); }; - let cancellationHandled = false; - let cancellationDisabled = false; - const cancelBeforeAcceptance = async (): Promise => { - if (cancelSignal?.aborted !== true || cancellationDisabled) return false; - if (cancellationHandled) return true; - - if (persistedCancelableMessageIds.length > 0) { - // History also has non-session writers (for example goal pause boundaries). Delete exactly - // this preparing turn's rows in one atomic rewrite so later concurrent rows are preserved. - const rollbackResult = await this.historyService.deleteMessages( - this.workspaceId, - persistedCancelableMessageIds - ); - if (!rollbackResult.success) { - // deleteMessages can fail after its atomic rewrite (for example while refreshing - // sequence metadata). Verify the durable result before deciding whether cancellation won. - const historyResult = await this.historyService.getHistoryFromLatestBoundary( - this.workspaceId - ); - const rollbackCommitted = - historyResult.success && - persistedCancelableMessageIds.every( - (messageId) => !historyResult.data.some((message) => message.id === messageId) - ); - if (!rollbackCommitted) { - // Do not report cancellation (which would supersede the durable monitor wake) unless the - // not-yet-accepted row is actually gone. Continue accepting this wake instead of leaving - // a hidden synthetic row that can leak into a later provider request. - cancellationDisabled = true; - log.error("Failed to roll back canceled preparing turn; continuing acceptance", { - workspaceId: this.workspaceId, - error: rollbackResult.error, - verificationError: historyResult.success ? undefined : historyResult.error, - }); - return false; - } - log.warn("Preparing-turn rollback reported failure after its rewrite committed", { - workspaceId: this.workspaceId, - error: rollbackResult.error, - }); - } - } - - cancellationHandled = true; - await internal?.onCanceled?.(cancelReasonBeforeAcceptance(cancelSignal)); - if (internal?.cancelState != null) { - internal.cancelState.canceledBeforeAcceptance = true; - } - return true; - }; - - if (await cancelBeforeAcceptance()) { - return Ok(undefined); - } // Last-line-of-defence pricing gate: every dispatch path (initial sends, // sendQueuedMessages, dispatchPendingFollowUp, @@ -3203,9 +3160,6 @@ export class AgentSession { this.workspaceId, options?.model ); - if (await cancelBeforeAcceptance()) { - return Ok(undefined); - } if (!pricingGate.success) { if (isManualUserMessage) { const persisted = await this.preserveRejectedManualSend( @@ -3617,10 +3571,6 @@ export class AgentSession { // File changes after this point are surfaced via diffs instead. const snapshotResult = await this.materializeFileAtMentionsSnapshot(trimmedMessage); - if (await cancelBeforeAcceptance()) { - return Ok(undefined); - } - // Check compaction threshold BEFORE persisting the user message. // Skill snapshots are materialized AFTER this decision (below): when on-send // compaction defers the turn, the follow-up re-enters sendMessage with the same @@ -3645,9 +3595,6 @@ export class AgentSession { // so the compaction monitor can detect context limits even before any live // stream events have populated lastUsageState. await this.seedUsageStateFromHistory(); - if (await cancelBeforeAcceptance()) { - return Ok(undefined); - } const providersConfigForCompaction = this.getProvidersConfigSafe(); const compactionResult = this.compactionMonitor.checkBeforeSend({ @@ -3752,10 +3699,7 @@ export class AgentSession { if (!appendCompactionResult.success) { return Err(createUnknownSendMessageError(appendCompactionResult.error)); } - persistedCancelableMessageIds.push(autoCompactionMessage.id); - if (await cancelBeforeAcceptance()) { - return Ok(undefined); - } + persistedTurnRowMessageIds.push(autoCompactionMessage.id); this.emitChatEvent({ type: "auto-compaction-triggered", @@ -3799,15 +3743,11 @@ export class AgentSession { ); mcpPromptSnapshotMessages = await this.materializeMcpPromptSnapshots( typedMuxMetadata, - userMessage.id, - cancelSignal + userMessage.id ); } catch (error) { return Err(createUnknownSendMessageError(getErrorMessage(error))); } - if (await cancelBeforeAcceptance()) { - return Ok(undefined); - } } if (shouldPersistTurnSnapshots && snapshotResult?.snapshotMessage) { @@ -3818,10 +3758,7 @@ export class AgentSession { if (!snapshotAppendResult.success) { return Err(createUnknownSendMessageError(snapshotAppendResult.error)); } - persistedCancelableMessageIds.push(snapshotResult.snapshotMessage.id); - if (await cancelBeforeAcceptance()) { - return Ok(undefined); - } + persistedTurnRowMessageIds.push(snapshotResult.snapshotMessage.id); } if (shouldPersistTurnSnapshots && skillSnapshotMessages.length > 0) { @@ -3834,10 +3771,7 @@ export class AgentSession { await rollbackPersistedTurnRows(); return Err(createUnknownSendMessageError(skillSnapshotAppendResult.error)); } - persistedCancelableMessageIds.push(snapshotMessage.id); - if (await cancelBeforeAcceptance()) { - return Ok(undefined); - } + persistedTurnRowMessageIds.push(snapshotMessage.id); } } @@ -3851,10 +3785,7 @@ export class AgentSession { await rollbackPersistedTurnRows(); return Err(createUnknownSendMessageError(appendResult.error)); } - persistedCancelableMessageIds.push(snapshotMessage.id); - if (await cancelBeforeAcceptance()) { - return Ok(undefined); - } + persistedTurnRowMessageIds.push(snapshotMessage.id); } } @@ -3883,13 +3814,10 @@ export class AgentSession { await rollbackPersistedTurnRows(); return Err(createUnknownSendMessageError(batchAppendResult.error)); } - persistedCancelableMessageIds.push( + persistedTurnRowMessageIds.push( ...internal.preTurnMessages.map((message) => message.id), userMessage.id ); - if (await cancelBeforeAcceptance()) { - return Ok(undefined); - } } else if (!autoCompactionMessage) { // When on-send compaction triggers, the user message is NOT persisted to // history (it's sent as follow-up after compaction). Otherwise, persist @@ -3899,10 +3827,7 @@ export class AgentSession { await rollbackPersistedTurnRows(); return Err(createUnknownSendMessageError(appendResult.error)); } - persistedCancelableMessageIds.push(userMessage.id); - if (await cancelBeforeAcceptance()) { - return Ok(undefined); - } + persistedTurnRowMessageIds.push(userMessage.id); } // Caller-probe staleness (peer sends racing a Stop) must resolve BEFORE the pre-turn batch @@ -3936,12 +3861,10 @@ export class AgentSession { ); } - // Goal synchronization can mutate goal.json based on this durable user row. Once it begins, the - // turn has crossed the cancellation point-of-no-return: a concurrent monitor stop must let this - // wake finish acceptance rather than delete the row after goal state has already observed it. - if (cancelSignal != null) { - cancellationDisabled = true; - } + // The user row is durable from here on. A bash-monitor wake whose row is durable must be + // accepted even if a later step throws: otherwise the reconciler re-derives the same wake + // and delivers it twice (startup recovery resumes the durable row without redelivery). + const finalizeDurableWakeOnFailure = typedMuxMetadata?.type === "bash-monitor-wake"; // r54: the pre-turn batch is now irrevocable — rollbackPersistedTurnRows // is never invoked past this point, so even a failure in goal sync or // acceptance leaves the payload + trigger rows durable in the transcript. @@ -3951,9 +3874,7 @@ export class AgentSession { try { await this.workspaceGoalService?.syncGoalModeWithChatTail(this.workspaceId); } catch (error) { - if (cancelSignal != null) { - // The durable row crossed the point of no return, so every later goal-sync failure must still - // finalize this monitor wake. Startup recovery can resume the row without redelivering it. + if (finalizeDurableWakeOnFailure) { await internal?.onAccepted?.(); } throw error; @@ -3967,10 +3888,10 @@ export class AgentSession { } // Workspace may be tearing down while we await filesystem IO. - // If so, skip event emission + streaming to avoid races with dispose(). A cancelable monitor - // wake past the point of no return is already durable, so finalize it before leaving. + // If so, skip event emission + streaming to avoid races with dispose(). A monitor wake's + // row is already durable, so finalize it before leaving. if (this.disposed) { - if (cancelSignal != null && cancellationDisabled) { + if (finalizeDurableWakeOnFailure) { await internal?.onAccepted?.(); } return Ok(undefined); @@ -4096,7 +4017,7 @@ export class AgentSession { return Ok(undefined); } // Background processes are workspace-scoped, not context-scoped. Compaction must preserve - // processes, monitors, and queued wakes so a waiting agent is not stranded. + // processes, monitors, and pending wakes so a waiting agent is not stranded. // Note: Follow-up content for compaction is now stored on the summary message // and dispatched via dispatchPendingFollowUp() after compaction completes. // This provides crash safety - the follow-up survives app restarts. @@ -5149,7 +5070,7 @@ export class AgentSession { const optionsMuxMetadata = options?.muxMetadata as MuxMessageMetadata | undefined; const retryMuxMetadata = lastUserMessage?.metadata?.muxMetadata; // Bash-monitor-wake continuations inherit the correlation of a delegated - // workspace turn that was cut mid-work by the wake's queued dispatch, so + // workspace turn whose stream yielded mid-work to the wake, so // the turn's eventual terminal stream-end can settle the parent's handle. const streamMuxMetadata = optionsMuxMetadata?.type === "workspace-turn-task" @@ -5206,7 +5127,7 @@ export class AgentSession { experiments: options?.experiments, disableWorkspaceAgents: options?.disableWorkspaceAgents, strictAgentResolution: options?.strictAgentResolution, - hasQueuedMessages: this.hasQueuedMessages.bind(this), + hasPendingToolEndInput: () => this.hasPendingToolEndInput(), openaiTruncationModeOverride, // Mid-turn thinking overrides clamp against the same floor as the // send-time level above (single source of truth for the floor). @@ -6134,7 +6055,7 @@ export class AgentSession { if (this.deferQueuedFlushUntilAfterEdit) { this.queuedProviderToolEndAbortInFlight = false; // Clear the queued-message signal while the edit flow owns the next dispatch. - this.backgroundProcessManager.setMessageQueued(this.workspaceId, false); + this.syncToolEndYieldRequested(false); // Do not dispatch stream-end follow-ups while the edit flow is waiting // for IDLE; truncation must run before any synthetic turn resumes. } else { @@ -6501,8 +6422,6 @@ export class AgentSession { onAccepted?: () => Promise | void; onAcceptedPreStreamFailure?: (error: SendMessageError) => Promise | void; onCanceled?: (reason: string) => Promise | void; - cancelState?: { canceledBeforeAcceptance: boolean }; - cancelSignal?: AbortSignal; /** Synthetic assistant rows persisted just before the dispatched turn's user row. */ preTurnMessages?: MuxMessage[]; /** r54: fired once pre-turn rows cross the rollback horizon at dispatch. */ @@ -6521,15 +6440,11 @@ export class AgentSession { } this.emitQueuedMessageChanged(); // Signal to bash_output that it should return early to process queued messages - // only for tool-end dispatches. Return the same mode so the caller's foreground - // task waits follow the entry that will actually run, not a withdrawn FIFO head. + // only for tool-end dispatches. Return the FIFO head's mode so the caller's foreground + // task waits follow the entry that will actually run next, not the one just added. const nextDispatchableMode = this.messageQueue.getNextDispatchableMode(); - this.backgroundProcessManager.setMessageQueued( - this.workspaceId, - nextDispatchableMode === "tool-end" - ); - // Undefined only if the entry just added is itself withdrawn; WorkspaceService.sendMessage - // refuses those before enqueue, so null keeps its "nothing pending was queued" meaning. + this.syncToolEndYieldRequested(nextDispatchableMode === "tool-end"); + // The queue is non-empty right after a successful add, so this is never null here. return nextDispatchableMode ?? null; } @@ -6538,7 +6453,7 @@ export class AgentSession { const callbackSets = this.messageQueue.getClearCallbacks(); this.messageQueue.clear(); this.emitQueuedMessageChanged(); - this.backgroundProcessManager.setMessageQueued(this.workspaceId, false); + this.syncToolEndYieldRequested(false); for (const callbacks of callbackSets) { this.notifyQueuedMessageCleared(callbacks, cancelReason); } @@ -6554,10 +6469,7 @@ export class AgentSession { this.emitQueuedMessageChanged(); // Only the FIFO head can dispatch next; later hidden entries must not pull an earlier // user-authored turn-end entry forward to a step boundary. - this.backgroundProcessManager.setMessageQueued( - this.workspaceId, - this.messageQueue.getNextDispatchableMode() === "tool-end" - ); + this.syncToolEndYieldRequested(); return true; } @@ -6591,10 +6503,7 @@ export class AgentSession { return 0; } this.emitQueuedMessageChanged(); - this.backgroundProcessManager.setMessageQueued( - this.workspaceId, - this.messageQueue.getNextDispatchableMode() === "tool-end" - ); + this.syncToolEndYieldRequested(); for (const callbacks of removal.callbacks) { this.notifyQueuedMessageCleared(callbacks, cancelReason); } @@ -6619,20 +6528,61 @@ export class AgentSession { return false; } this.emitQueuedMessageChanged(); - this.backgroundProcessManager.setMessageQueued( - this.workspaceId, - this.messageQueue.getNextDispatchableMode() === "tool-end" - ); + this.syncToolEndYieldRequested(); this.notifyQueuedMessageCleared(callbacks, cancelReason); return true; } - /** Pending work only: withdrawn (aborted) entries still occupy the queue but never start a turn. */ + /** Whether the FIFO head (the only entry the next drain can send) matches `dispatchMode`. */ hasQueuedMessages(dispatchMode?: "tool-end" | "turn-end"): boolean { const nextMode = this.messageQueue.getNextDispatchableMode(); return nextMode != null && (dispatchMode == null || nextMode === dispatchMode); } + /** + * Stream stop condition: input that must run at a tool boundary is pending. A queued + * tool-end message is an edge we hold; a bash-monitor wake is a level read from the + * reconciler, so a wake whose lines this very step showed (task_await on the monitored + * process) is already gone by the time the SDK asks — the stream keeps going. + */ + async hasPendingToolEndInput(): Promise { + if (this.hasQueuedMessages("tool-end")) return true; + if (this.hasOutstandingBashMonitorWake == null) return false; + try { + return await this.hasOutstandingBashMonitorWake(); + } catch (error) { + log.debug("hasPendingToolEndInput: wake level read failed; not yielding", { + workspaceId: this.workspaceId, + error, + }); + return false; + } + } + + /** + * Mirror the reconciler's wake level. While high, long-polling bash reads return early + * so the stream reaches a tool boundary (same lever a queued tool-end message pulls). + */ + setBashMonitorWakeOutstanding(outstanding: boolean): void { + if (this.bashMonitorWakeOutstanding === outstanding) return; + this.bashMonitorWakeOutstanding = outstanding; + this.syncToolEndYieldRequested(); + } + + /** + * Tool-end yield flag = queue head is tool-end ∪ wake level. `queueHeadToolEnd` lets + * stream-end / clear paths assert the queue contribution is gone before the queue itself + * is observed empty. + */ + private syncToolEndYieldRequested( + queueHeadToolEnd = this.messageQueue.getNextDispatchableMode() === "tool-end" + ): void { + this.backgroundProcessManager.setMessageQueued( + this.workspaceId, + queueHeadToolEnd || this.bashMonitorWakeOutstanding + ); + } + /** Queued intra-tree agent peer messages awaiting dispatch (peer-message queue cap input). */ countQueuedAgentPeerMessages(): number { return this.messageQueue.countAgentPeerMessageEntries(); @@ -6675,24 +6625,6 @@ export class AgentSession { return false; } - /** - * Whether a bash-monitor-wake continuation is pending dispatch: the next - * queued entry is a wake, or a dequeued wake is mid-dispatch (dequeue → - * stream start). Wake sends are the only input that inherits an open - * delegated workspace turn's correlation, so TaskService uses this — not - * generic queued/preparing state — to decide whether a correlated - * "tool-calls" queue cut will be continued rather than superseded. Once the - * wake's stream starts, TaskService matches the active stream's inherited - * correlation instead (see hasSameTurnWakeContinuation). - */ - hasPendingBashMonitorWakeContinuation(): boolean { - if (this.messageQueue.isNextEntryBashMonitorWake()) { - return true; - } - const dispatching = this.dispatchingQueuedEntryMuxMetadata as MuxMessageMetadata | undefined; - return dispatching?.type === "bash-monitor-wake"; - } - /** * Whether a queued or dispatching entry continues the exact workspace-turn correlation. */ @@ -6799,7 +6731,6 @@ export class AgentSession { return false; } - // Physical check: withdrawn entries must still drain so their onCanceled fires. const shouldDispatch = abortReason !== "user" && !this.deferQueuedFlushUntilAfterEdit && @@ -6953,7 +6884,7 @@ export class AgentSession { this.queuedProviderToolEndAbortInFlight = false; // Clear the queued message flag (even if queue is empty, to handle race conditions) - this.backgroundProcessManager.setMessageQueued(this.workspaceId, false); + this.syncToolEndYieldRequested(false); if (!this.messageQueue.isEmpty()) { // Entries dispatch one at a time (FIFO): special sends (compaction, agent @@ -6966,10 +6897,7 @@ export class AgentSession { // Re-arm dispatch signals for the remaining entries so the stream we are // about to start drains them at its next tool end (or stream end). - this.backgroundProcessManager.setMessageQueued( - this.workspaceId, - this.messageQueue.getNextDispatchableMode() === "tool-end" - ); + this.syncToolEndYieldRequested(); // Set PREPARING synchronously before the async sendMessage to prevent // incoming messages from bypassing the queue during the await gap. @@ -6994,14 +6922,6 @@ export class AgentSession { this.sendQueuedMessages(); return; } - if (internal?.cancelState?.canceledBeforeAcceptance === true) { - // Cancellation can arrive after dequeue while sendMessage is validating or writing - // history. No stream will start, so release PREPARING and continue with later entries. - if (this.turnPhase === TurnPhase.PREPARING) { - this.setTurnPhase(TurnPhase.IDLE); - } - this.sendQueuedMessages(); - } }) .catch(async (error: unknown) => { // A REJECTED sendMessage (thrown, not returned Err — e.g. an awaited history or goal @@ -7846,8 +7766,7 @@ export class AgentSession { private async materializeMcpPromptSnapshots( muxMetadata: MuxMessageMetadata | undefined, - invokingMessageId: string, - cancelSignal: AbortSignal | undefined + invokingMessageId: string ): Promise { const mcpServerManager = this.mcpServerManager; if (!mcpServerManager) return []; @@ -7860,8 +7779,7 @@ export class AgentSession { this.workspaceId, ref.serverName, ref.promptName, - ref.arguments ?? {}, - cancelSignal !== undefined ? { signal: cancelSignal } : undefined + ref.arguments ?? {} ); return createMuxMessage(createMcpPromptSnapshotMessageId(), "user", prompt.text, { timestamp: Date.now(), @@ -7875,8 +7793,6 @@ export class AgentSession { }, }); } catch (error) { - // Cancellation is handled by cancelBeforeAcceptance after this returns. - if (cancelSignal?.aborted) return null; // A slash-invoked prompt was explicitly selected; sending the turn // without its expansion would silently change what the user asked // for. Inline references degrade to the authored text instead. diff --git a/src/node/services/agentSkills/builtInSkillContent.generated.ts b/src/node/services/agentSkills/builtInSkillContent.generated.ts index 61eae9eeb0..4706e71ed8 100644 --- a/src/node/services/agentSkills/builtInSkillContent.generated.ts +++ b/src/node/services/agentSkills/builtInSkillContent.generated.ts @@ -7721,7 +7721,7 @@ export const BUILTIN_SKILL_FILES: Record> = { "", "Messages to a busy workspace enter its `MessageQueue` (`src/node/services/messageQueue.ts`) and dispatch at a boundary chosen by `queueDispatchMode`:", "", - '- `tool-end`: the stream\'s stop conditions include `hasQueuedMessages("tool-end")`, evaluated by the AI SDK only after every sibling tool result in the current step settles (`createStopWhenCondition`, `src/node/services/streamManager.ts`); `AgentSession` soft-stops only once `activeToolCallIds` is empty. **A running tool call is never interrupted** — same guarantee as Claude Code.', + "- `tool-end`: the stream's stop conditions include `hasPendingToolEndInput()` (a queued tool-end message or an outstanding bash-monitor wake, read live), evaluated by the AI SDK only after every sibling tool result in the current step settles (`createStopWhenCondition`, `src/node/services/streamManager.ts`); `AgentSession` soft-stops only once `activeToolCallIds` is empty. **A running tool call is never interrupted** — same guarantee as Claude Code.", "- `turn-end`: dispatches after the current turn completes.", "- Idle target: the message starts a new turn immediately.", "", diff --git a/src/node/services/bashMonitorWakeReconciler.test.ts b/src/node/services/bashMonitorWakeReconciler.test.ts index 4348bd3c33..3133753ea3 100644 --- a/src/node/services/bashMonitorWakeReconciler.test.ts +++ b/src/node/services/bashMonitorWakeReconciler.test.ts @@ -130,51 +130,32 @@ describe("BashMonitorWakeReconciler", () => { expect(dispatches).toHaveLength(2); }); - test("re-dispatches unchanged signals after a queued delivery is canceled", async () => { + test("re-dispatches unchanged signals after the owner defers an in-flight wake", async () => { live = [liveSnapshot()]; await reconciler.reconcile(OWNER); - const queued = dispatches[0]; + const inFlight = dispatches[0]; - await queued.onDeferred(); + await inFlight.onDeferred(); await reconciler.reconcile(OWNER); expect(dispatches).toHaveLength(2); }); - test("superseding a queued wake uses a distinct queue key", async () => { - const queuedKeys = new Set(); - const queuedDispatches: BashMonitorWakeDispatch[] = []; - const queueing = new BashMonitorWakeReconciler({ - sessionsDir: root, - processManager: { - pullMonitorWakeSignals: () => live, - getMonitorWakeDeliveryState: () => Promise.resolve(deliveryState), - acknowledgeMonitorWake: () => undefined, - dropRetiredMonitor: () => undefined, - }, - registry: { - listAll: () => Promise.resolve([]), - remove: () => undefined, - recordTerminal: () => undefined, - }, - onWake: (dispatch) => { - if (queuedKeys.has(dispatch.dedupeKey)) return "deferred"; - queuedKeys.add(dispatch.dedupeKey); - queuedDispatches.push(dispatch); - return "in-flight"; - }, - }); + test("hands out one wake at a time: a newer match waits for the in-flight acceptance", async () => { live = [liveSnapshot()]; - await queueing.reconcile(OWNER); + await reconciler.reconcile(OWNER); + expect(dispatches).toHaveLength(1); + live = [ liveSnapshot({ match: { throughOffset: 24, lines: ["READY again"], totalMatches: 2 } }), ]; + await reconciler.reconcile(OWNER); + expect(dispatches).toHaveLength(1); - await queueing.reconcile(OWNER); - - expect(queuedDispatches).toHaveLength(2); - expect(queuedKeys.size).toBe(2); - expect(queuedDispatches[0].cancelSignal.aborted).toBe(true); + await dispatches[0].onAccepted(); + await reconciler.reconcile(OWNER); + expect(dispatches).toHaveLength(2); + expect(dispatches[1].prompt).toContain("READY again"); }); test("keeps dead registry evidence until the queued wake is accepted", async () => { @@ -198,25 +179,83 @@ describe("BashMonitorWakeReconciler", () => { expect(dispatches).toHaveLength(1); }); - test("cancels a queued wake when the level no longer has an outstanding signal", async () => { + test("hasOutstandingWake reads the level without dispatching", async () => { live = [liveSnapshot()]; - await reconciler.reconcile(OWNER); - const queued = dispatches[0]; + expect(await reconciler.hasOutstandingWake(OWNER)).toBe(true); + expect(dispatches).toHaveLength(0); - deliveryState = { status: "settled", shownThroughOffset: 12, terminalStatusShown: false }; - await reconciler.reconcile(OWNER); + // A same-process blocking read will show the lines itself: not outstanding. + deliveryState = { status: "blocked", readSettled: new Promise(() => undefined) }; + expect(await reconciler.hasOutstandingWake(OWNER)).toBe(false); - expect(queued.cancelSignal.aborted).toBe(true); - await queued.onAccepted(); - deliveryState = { status: "settled", shownThroughOffset: 0, terminalStatusShown: false }; + // Shown frontier past the match: the wake is gone, nothing to deliver later either. + deliveryState = { status: "settled", shownThroughOffset: 12, terminalStatusShown: false }; + expect(await reconciler.hasOutstandingWake(OWNER)).toBe(false); await reconciler.reconcile(OWNER); - expect(dispatches).toHaveLength(1); + expect(dispatches).toHaveLength(0); live = [ liveSnapshot({ match: { throughOffset: 24, lines: ["READY again"], totalMatches: 2 } }), ]; - await reconciler.reconcile(OWNER); - expect(dispatches).toHaveLength(2); + expect(await reconciler.hasOutstandingWake(OWNER)).toBe(true); + }); + + test("publishes the level on every read", async () => { + const levels: boolean[] = []; + const publishing = new BashMonitorWakeReconciler({ + sessionsDir: root, + processManager: { + pullMonitorWakeSignals: () => live, + getMonitorWakeDeliveryState: () => Promise.resolve(deliveryState), + acknowledgeMonitorWake: () => undefined, + dropRetiredMonitor: () => undefined, + }, + registry: { + listAll: () => Promise.resolve([]), + remove: () => undefined, + recordTerminal: () => undefined, + }, + onWake: () => "deferred", + onOutstandingChanged: (_owner, outstanding) => { + levels.push(outstanding); + }, + }); + live = [liveSnapshot()]; + await publishing.reconcile(OWNER); + deliveryState = { status: "settled", shownThroughOffset: 12, terminalStatusShown: false }; + await publishing.hasOutstandingWake(OWNER); + + expect(levels).toEqual([true, false]); + }); + + test("a full-history clear lowers the level without a follow-up read", async () => { + const levels: boolean[] = []; + const publishing = new BashMonitorWakeReconciler({ + sessionsDir: root, + processManager: { + pullMonitorWakeSignals: () => live, + getMonitorWakeDeliveryState: () => Promise.resolve(deliveryState), + acknowledgeMonitorWake: () => undefined, + dropRetiredMonitor: () => undefined, + }, + registry: { + listAll: () => Promise.resolve([]), + remove: () => undefined, + recordTerminal: () => undefined, + }, + onWake: () => "deferred", + onOutstandingChanged: (_owner, outstanding) => { + levels.push(outstanding); + }, + }); + live = [liveSnapshot()]; + await publishing.reconcile(OWNER); + expect(levels).toEqual([true]); + + // The consume path retires the signals it collects, so it must not republish them as + // outstanding (that would leave the owner's tool-end yield flag stuck high). + await publishing.beginFullHistoryClear(OWNER); + expect(levels).toEqual([true, false]); }); test("advances the watermark only on acceptance and later delivers a newer match", async () => { @@ -329,18 +368,18 @@ describe("BashMonitorWakeReconciler", () => { expect(restartedDispatches).toHaveLength(1); }); - test("explicit cancellation retracts a queued wake without consuming a later generation", async () => { + test("accepting a wake whose processes vanished does not consume a later generation", async () => { live = [liveSnapshot()]; rows = [registryRecord()]; await reconciler.reconcile(OWNER); - const queued = dispatches[0]; + const inFlight = dispatches[0]; live = []; rows = []; await reconciler.reconcile(OWNER); + expect(dispatches).toHaveLength(1); - expect(queued.cancelSignal.aborted).toBe(true); - await queued.onAccepted(); + await inFlight.onAccepted(); live = [ liveSnapshot({ createdAt: "2026-08-31T12:03:00.000Z", diff --git a/src/node/services/bashMonitorWakeReconciler.ts b/src/node/services/bashMonitorWakeReconciler.ts index 46728fc4be..03f22bf554 100644 --- a/src/node/services/bashMonitorWakeReconciler.ts +++ b/src/node/services/bashMonitorWakeReconciler.ts @@ -101,12 +101,16 @@ export interface BashMonitorWakeReconcilerRegistry { export type BashMonitorWakeDispatchOutcome = "in-flight" | "deferred"; +/** + * A wake handed to the owner. Wakes are a LEVEL derived from process state, never a + * queued edge: the receiver either starts a turn from it now (`onAccepted`) or leaves it + * pending (`onDeferred` / "deferred") and re-reconciles later. There is nothing to cancel — + * a wake whose matched output is shown meanwhile simply stops deriving on the next read. + */ export interface BashMonitorWakeDispatch { ownerWorkspaceId: string; prompt: string; muxMetadata: Extract; - dedupeKey: string; - cancelSignal: AbortSignal; onAccepted(): Promise; onDeferred(): Promise; } @@ -150,10 +154,8 @@ interface DerivedSignal { retired: boolean; } +/** A wake handed to `onWake` whose acceptance/deferral has not settled yet. */ interface DispatchState { - id: string; - signature: string; - controller: AbortController; signals: readonly DerivedSignal[]; accepted: boolean; } @@ -370,9 +372,25 @@ export class BashMonitorWakeReconciler { onWake( dispatch: BashMonitorWakeDispatch ): Promise | BashMonitorWakeDispatchOutcome; + /** + * Published on every level read (reconcile, snapshot, hasOutstandingWake) so the + * owner can mirror "pending input wants a tool boundary" side effects (early + * long-poll return, backgrounding foreground waits) from the level itself. + */ + onOutstandingChanged?(ownerWorkspaceId: string, outstanding: boolean): void; } ) {} + /** + * The wake level: whether the owner has a wake it has not seen yet. Same-process + * blocking reads (deferredReads) are not outstanding — the read shows the lines itself. + * Consumers: the stream's tool-boundary stop condition and delegated-turn settlement. + */ + async hasOutstandingWake(ownerWorkspaceId: string): Promise { + if (this.defunctWorkspaces.has(ownerWorkspaceId)) return false; + return (await this.snapshot(ownerWorkspaceId)).pendingWakeKinds.size > 0; + } + scheduleReconcile(ownerWorkspaceId: string): void { if (this.defunctWorkspaces.has(ownerWorkspaceId)) return; const state = this.state(ownerWorkspaceId); @@ -421,26 +439,7 @@ export class BashMonitorWakeReconciler { return snapshot.pendingWakeKinds.get(processId); } - async discardProcess( - ownerWorkspaceId: string, - processId: string, - createdAt: string - ): Promise { - await this.locks.withLock(ownerWorkspaceId, () => { - const state = this.state(ownerWorkspaceId); - if ( - state.dispatch?.signals.some( - (signal) => signal.processId === processId && signal.createdAt === createdAt - ) === true - ) { - state.dispatch.controller.abort(); - state.dispatch = undefined; - } - return Promise.resolve(); - }); - } async beginFullHistoryClear(ownerWorkspaceId: string): Promise { - this.abortDispatch(ownerWorkspaceId); await this.consumeCurrent(ownerWorkspaceId); return { ownerWorkspaceId }; } @@ -453,8 +452,6 @@ export class BashMonitorWakeReconciler { this.defunctWorkspaces.add(ownerWorkspaceId); this.resetRetry(ownerWorkspaceId); await this.locks.withLock(ownerWorkspaceId, () => { - const state = this.states.get(ownerWorkspaceId); - state?.dispatch?.controller.abort(); this.states.delete(ownerWorkspaceId); return Promise.resolve(); }); @@ -514,32 +511,11 @@ export class BashMonitorWakeReconciler { await this.cleanup(collected.autoConsumed); const state = this.state(ownerWorkspaceId); - if (collected.signals.length === 0) { - state.dispatch?.controller.abort(); - state.dispatch = undefined; - return undefined; - } - - const signature = JSON.stringify( - collected.signals.map((signal) => [ - signal.key, - signal.kind, - signal.matchOffset, - signal.terminal?.settledAt, - signal.matchedOutputAlreadyShown, - ]) - ); - if (state.dispatch?.signature === signature && !state.dispatch.controller.signal.aborted) { - return undefined; - } - state.dispatch?.controller.abort(); - const next: DispatchState = { - id: randomUUID(), - signature, - controller: new AbortController(), - signals: collected.signals, - accepted: false, - }; + // A wake already handed to the owner settles on its own (accept → watermarks advance + // and a reconcile is scheduled; defer → the owner re-arms a reconcile). Handing out a + // second one meanwhile could only duplicate or supersede the first. + if (collected.signals.length === 0 || state.dispatch != null) return undefined; + const next: DispatchState = { signals: collected.signals, accepted: false }; state.dispatch = next; return next; }); @@ -550,18 +526,12 @@ export class BashMonitorWakeReconciler { ownerWorkspaceId, prompt: buildPrompt(dispatch.signals), muxMetadata: buildMetadata(dispatch.signals), - dedupeKey: "bash-monitor-wake:" + ownerWorkspaceId + ":" + dispatch.id, - cancelSignal: dispatch.controller.signal, onAccepted: async () => this.accept(ownerWorkspaceId, dispatch), onDeferred: async () => this.defer(ownerWorkspaceId, dispatch), }); if (outcome === "deferred") await this.defer(ownerWorkspaceId, dispatch); } catch (error) { - await this.locks.withLock(ownerWorkspaceId, () => { - const state = this.state(ownerWorkspaceId); - if (state.dispatch === dispatch) state.dispatch = undefined; - return Promise.resolve(); - }); + await this.defer(ownerWorkspaceId, dispatch); throw error; } } @@ -575,7 +545,9 @@ export class BashMonitorWakeReconciler { } private async accept(ownerWorkspaceId: string, dispatch: DispatchState): Promise { await this.locks.withLock(ownerWorkspaceId, async () => { - if (dispatch.accepted || dispatch.controller.signal.aborted) return; + // The prompt reached the model, so its signals are consumed even if a full-history + // clear or process discard forgot this dispatch meanwhile. + if (dispatch.accepted) return; dispatch.accepted = true; const watermarks = await this.readWatermarks(ownerWorkspaceId); await this.advanceWatermarks(ownerWorkspaceId, watermarks, dispatch.signals); @@ -586,19 +558,15 @@ export class BashMonitorWakeReconciler { this.scheduleReconcile(ownerWorkspaceId); } - private abortDispatch(ownerWorkspaceId: string): void { - const state = this.state(ownerWorkspaceId); - state.dispatch?.controller.abort(); - state.dispatch = undefined; - } - private async consumeCurrent(ownerWorkspaceId: string): Promise { await this.locks.withLock(ownerWorkspaceId, async () => { - this.abortDispatch(ownerWorkspaceId); const collected = await this.collect(ownerWorkspaceId, false); const consumed = [...collected.signals, ...collected.autoConsumed]; await this.advanceWatermarks(ownerWorkspaceId, collected.watermarks, consumed); await this.cleanup(consumed); + // Everything collected is consumed, so the level is low by construction; publish it + // here because no read follows a consume. + this.args.onOutstandingChanged?.(ownerWorkspaceId, false); }); } @@ -674,6 +642,9 @@ export class BashMonitorWakeReconciler { signals.sort( (a, b) => a.createdAt.localeCompare(b.createdAt) || a.processId.localeCompare(b.processId) ); + // Only level reads publish; the consume path (applyFrontier=false) is about to retire + // these very signals and publishes low itself. + if (applyFrontier) this.args.onOutstandingChanged?.(ownerWorkspaceId, signals.length > 0); return { signals, autoConsumed, deferredReads, watermarks }; } diff --git a/src/node/services/messageQueue.test.ts b/src/node/services/messageQueue.test.ts index 82106e398c..7755dace1f 100644 --- a/src/node/services/messageQueue.test.ts +++ b/src/node/services/messageQueue.test.ts @@ -488,25 +488,6 @@ describe("MessageQueue", () => { expect(queue.getQueueDispatchMode()).toBe("tool-end"); }); - it("skips withdrawn entries when reporting the next dispatchable mode", () => { - const validOptions: SendMessageOptions = { model: "gpt-4", agentId: "exec" }; - const withdrawn = new AbortController(); - queue.add( - "withdrawn wake", - { ...validOptions, queueDispatchMode: "tool-end" }, - { synthetic: true, agentInitiated: true, cancelSignal: withdrawn.signal } - ); - expect(queue.getNextDispatchableMode()).toBe("tool-end"); - - withdrawn.abort(); - expect(queue.getNextDispatchableMode()).toBeUndefined(); - expect(queue.isEmpty()).toBe(false); - - queue.add("follow up", { ...validOptions, queueDispatchMode: "turn-end" }); - expect(queue.getNextDispatchableMode()).toBe("turn-end"); - expect(queue.getNextQueueDispatchMode()).toBe("tool-end"); - }); - it("should reset mode to tool-end when cleared", () => { queue.add("Follow up", { model: "gpt-4", diff --git a/src/node/services/messageQueue.ts b/src/node/services/messageQueue.ts index 93593b9f5d..2af2227b49 100644 --- a/src/node/services/messageQueue.ts +++ b/src/node/services/messageQueue.ts @@ -93,13 +93,6 @@ type GoalInterventionPolicy = NonNullable; -/** onCanceled text for a send whose cancel signal fired before the turn was accepted. */ -export function cancelReasonBeforeAcceptance(signal: AbortSignal): string { - return typeof signal.reason === "string" - ? signal.reason - : "Queued message canceled before acceptance."; -} - /** * Input poised to take over a session at a queue cut (see * AgentSession.getQueueCutCutter). Engaged stages win over the queue head; an @@ -132,10 +125,6 @@ interface QueuedMessageInternalOptions { onAccepted?: () => Promise | void; onAcceptedPreStreamFailure?: (error: SendMessageError) => Promise | void; onCanceled?: (reason: string) => Promise | void; - /** Mutable dispatch outcome shared with sendQueuedMessages. */ - cancelState?: { canceledBeforeAcceptance: boolean }; - /** Cancels a queued entry even after it has been dequeued into PREPARING. */ - cancelSignal?: AbortSignal; /** * Synthetic rows persisted by AgentSession.sendMessage immediately before the * turn's user row (family-message payloads). Deferring them with the trigger @@ -198,8 +187,6 @@ interface QueueEntry { onCanceled?: (reason: string) => Promise | void; onAccepted?: () => Promise | void; onAcceptedPreStreamFailure?: (error: SendMessageError) => Promise | void; - cancelState?: { canceledBeforeAcceptance: boolean }; - cancelSignal?: AbortSignal; /** Pre-turn rows delivered with this entry (entries carrying them are sealed). */ preTurnMessages?: MuxMessage[]; /** r54: fired once this entry's pre-turn rows cross the rollback horizon. */ @@ -271,13 +258,9 @@ export class MessageQueue { return this.entries[0]?.dispatchMode ?? "tool-end"; } - /** - * Dispatch mode of the first entry whose cancel signal has not fired, or undefined - * when none remains. Aborted entries still drain FIFO (as no-ops that fire - * onCanceled), but they are not pending work and must not arm a tool-end stop. - */ + /** Dispatch mode of the FIFO head, or undefined when the queue is empty. */ getNextDispatchableMode(): QueueDispatchMode | undefined { - return this.entries.find((entry) => entry.cancelSignal?.aborted !== true)?.dispatchMode; + return this.entries[0]?.dispatchMode; } /** @@ -341,18 +324,6 @@ export class MessageQueue { return { muxMetadata: head.muxMetadata, dispatchMode: head.dispatchMode }; } - /** - * Whether the next entry to dispatch is a bash-monitor wake. Wake sends are - * the only queued input that continues an open delegated workspace turn - * (see AgentSession.inheritOpenWorkspaceTurnMetadata); any other head entry - * supersedes the turn when it dispatches. - */ - isNextEntryBashMonitorWake(): boolean { - const muxMetadata = this.entries[0]?.muxMetadata; - if (typeof muxMetadata !== "object" || muxMetadata === null) return false; - return (muxMetadata as Record).type === "bash-monitor-wake"; - } - /** * Effective dispatch mode across pending entries: any entry queued for tool-end * makes the whole queue dispatch at tool-end (sticky, matching pre-entry behavior), @@ -512,8 +483,7 @@ export class MessageQueue { const incomingHasAcceptedCallbacks = internal?.onAccepted != null || internal?.onAcceptedPreStreamFailure != null || - internal?.onCanceled != null || - internal?.cancelSignal != null; + internal?.onCanceled != null; const incomingIsUserAuthored = internal?.synthetic !== true && internal?.agentInitiated !== true; // Sealed entries must own their turn end-to-end: workspace-turn metadata and @@ -626,12 +596,6 @@ export class MessageQueue { }; } - if (internal?.cancelState != null) { - entry.cancelState = internal.cancelState; - } - if (internal?.cancelSignal != null) { - entry.cancelSignal = internal.cancelSignal; - } if (internal?.admissionStale != null) { entry.admissionStale = internal.admissionStale; } @@ -891,7 +855,6 @@ export class MessageQueue { entry.onAccepted != null || entry.onAcceptedPreStreamFailure != null || entry.onCanceled != null || - entry.cancelSignal != null || entry.admissionStale != null || (entry.preTurnMessages?.length ?? 0) > 0; const internal = hasInternalOptions @@ -899,8 +862,6 @@ export class MessageQueue { ...(allAddsAreSynthetic ? { synthetic: true } : {}), ...(allAddsAreAgentInitiated ? { agentInitiated: true } : {}), ...(entry.onCanceled != null ? { onCanceled: entry.onCanceled } : {}), - ...(entry.cancelState != null ? { cancelState: entry.cancelState } : {}), - ...(entry.cancelSignal != null ? { cancelSignal: entry.cancelSignal } : {}), ...(entry.onAccepted != null ? { onAccepted: entry.onAccepted } : {}), ...(entry.onAcceptedPreStreamFailure != null ? { onAcceptedPreStreamFailure: entry.onAcceptedPreStreamFailure } diff --git a/src/node/services/streamManager.test.ts b/src/node/services/streamManager.test.ts index ff51f1e4db..51135af0c6 100644 --- a/src/node/services/streamManager.test.ts +++ b/src/node/services/streamManager.test.ts @@ -1224,9 +1224,9 @@ describe("StreamManager - stream resource scope", () => { }); describe("StreamManager - stopWhen configuration", () => { - type StopWhenCondition = (options: { steps: unknown[] }) => boolean; + type StopWhenCondition = (options: { steps: unknown[] }) => boolean | Promise; type BuildStopWhenCondition = (request: { - hasQueuedMessages?: (dispatchMode?: "tool-end" | "turn-end") => boolean; + hasPendingToolEndInput?: () => Promise | boolean; toolPolicy?: ToolPolicy; }) => StopWhenCondition[]; @@ -1239,7 +1239,7 @@ describe("StreamManager - stopWhen configuration", () => { function requiredToolConditionForTests(toolPolicy: ToolPolicy): StopWhenCondition { const [, , requiredToolCondition] = buildStopWhenForTests()({ - hasQueuedMessages: () => false, + hasPendingToolEndInput: () => false, toolPolicy, }); return requiredToolCondition; @@ -1249,23 +1249,30 @@ describe("StreamManager - stopWhen configuration", () => { return { steps: [{ toolResults: [{ toolName, output }] }] }; } - test("returns step-cap and queued-message conditions with no policy", () => { - let queued = false; - const stopWhen = buildStopWhenForTests()({ hasQueuedMessages: () => queued }); + test("returns step-cap and pending tool-end input conditions with no policy", async () => { + // The hook is a live level read (queued tool-end message or outstanding bash-monitor + // wake), evaluated after the step's tool results settle — so it is awaited per step. + let pending: Promise | boolean = false; + const stopWhen = buildStopWhenForTests()({ hasPendingToolEndInput: () => pending }); expect(stopWhen).toHaveLength(3); - const [maxStepCondition, queuedMessageCondition, requiredToolCondition] = stopWhen; + const [maxStepCondition, pendingInputCondition, requiredToolCondition] = stopWhen; expect(maxStepCondition({ steps: new Array(99999) })).toBe(false); expect(maxStepCondition({ steps: new Array(100000) })).toBe(true); - expect(queuedMessageCondition({ steps: [] })).toBe(false); - queued = true; - expect(queuedMessageCondition({ steps: [] })).toBe(true); + expect(await pendingInputCondition({ steps: [] })).toBe(false); + pending = Promise.resolve(true); + expect(await pendingInputCondition({ steps: [] })).toBe(true); expect(requiredToolCondition(stepsWithToolResult("agent_report", { success: true }))).toBe( false ); }); + test("omitting the pending-input hook never stops the step loop", async () => { + const [, pendingInputCondition] = buildStopWhenForTests()({}); + expect(await pendingInputCondition({ steps: [] })).toBe(false); + }); + const requiredToolCases: Array<{ name: string; toolPolicy: ToolPolicy; @@ -1797,7 +1804,7 @@ describe("StreamManager - sequential tool execution", () => { headers?: Record; maxOutputTokens?: number; streamCallSettings?: Record; - hasQueuedMessages?: (dispatchMode?: "tool-end" | "turn-end") => boolean; + hasPendingToolEndInput?: () => Promise | boolean; toolPolicy?: ToolPolicy; toolChoice?: { type: "tool"; toolName: string }; } @@ -1906,7 +1913,7 @@ describe("StreamManager - sequential tool execution", () => { messages: [{ role: "user", content: "hello" }], system: "system", tools, - hasQueuedMessages: () => false, + hasPendingToolEndInput: () => false, }); createStreamResult(request, new AbortController()); diff --git a/src/node/services/streamManager.ts b/src/node/services/streamManager.ts index d12db74e0a..6b28517348 100644 --- a/src/node/services/streamManager.ts +++ b/src/node/services/streamManager.ts @@ -244,7 +244,12 @@ interface StreamRequestOptions { maxOutputTokens?: number; callSettingsOverrides?: ResolvedCallSettingsOverrides; toolPolicy?: ToolPolicy; - hasQueuedMessages?: (dispatchMode?: "tool-end" | "turn-end") => boolean; + /** + * Whether input that must run at a tool boundary is pending: a queued tool-end message + * or an outstanding bash-monitor wake. Read (not snapshotted) after every step's tool + * results settle, so a wake whose lines the step itself just showed no longer counts. + */ + hasPendingToolEndInput?: () => Promise | boolean; headers?: Record; onChunk?: StreamTextOnChunk; onStepMessages?: (messages: ModelMessage[]) => void; @@ -289,7 +294,7 @@ interface StreamRequestConfig { headers?: Record; maxOutputTokens?: number; streamCallSettings?: Omit; - hasQueuedMessages?: (dispatchMode?: "tool-end" | "turn-end") => boolean; + hasPendingToolEndInput?: () => Promise | boolean; /** Optional hook for callers that need chunk-level visibility during streaming. */ onChunk?: StreamTextOnChunk; /** Optional hook for callers that need the live prepared step transcript. */ @@ -2068,7 +2073,7 @@ export class StreamManager { maxOutputTokens, callSettingsOverrides, toolPolicy, - hasQueuedMessages, + hasPendingToolEndInput, headers, onChunk, onStepMessages, @@ -2118,7 +2123,7 @@ export class StreamManager { maxOutputTokens: effectiveMaxOutputTokens, streamCallSettings: Object.keys(streamCallSettings).length > 0 ? streamCallSettings : undefined, - hasQueuedMessages, + hasPendingToolEndInput, onChunk, onStepMessages, toolPolicy, @@ -2131,7 +2136,7 @@ export class StreamManager { } private createStopWhenCondition( - request: Pick + request: Pick ): Array> { // Completion-tool stop check: completion/routing tools use explicit // success/ok markers (agent_report, propose_plan). @@ -2177,7 +2182,7 @@ export class StreamManager { // The SDK evaluates stop conditions only after every sibling tool result in the // model's current step settles. Do not move this to individual tool-call-end events: // that would abort the remaining calls the model emitted in the same batch. - () => request.hasQueuedMessages?.("tool-end") ?? false, + async () => (await request.hasPendingToolEndInput?.()) ?? false, hasSuccessfulRequiredToolResult, ]; } @@ -3155,7 +3160,7 @@ export class StreamManager { maxOutputTokens: fallbackState.original.maxOutputTokens, callSettingsOverrides: prepared.data.callSettingsOverrides, toolPolicy: streamInfo.request.toolPolicy, - hasQueuedMessages: streamInfo.request.hasQueuedMessages, + hasPendingToolEndInput: streamInfo.request.hasPendingToolEndInput, headers: prepared.data.headers, onChunk: streamInfo.request.onChunk, onStepMessages: streamInfo.request.onStepMessages, diff --git a/src/node/services/taskService.test.ts b/src/node/services/taskService.test.ts index 4dc2b3b77f..682ec99ea7 100644 --- a/src/node/services/taskService.test.ts +++ b/src/node/services/taskService.test.ts @@ -347,7 +347,7 @@ describe("TaskService", () => { isStreaming?: ReturnType; hasQueuedMessages?: ReturnType; hasPendingQueuedOrPreparingTurn?: ReturnType; - hasPendingBashMonitorWakeContinuation?: ReturnType; + hasOutstandingBashMonitorWake?: ReturnType; hasPendingWorkspaceTurnContinuation?: ReturnType; getQueueCutCutter?: ReturnType; hasPendingAutoRetry?: ReturnType; @@ -14116,7 +14116,7 @@ describe("TaskService", () => { expect(count2).toBe(0); }); - test("backgrounds waiters when tool-end message was already queued", async () => { + test("backgrounds waiters when tool-end input was already pending", async () => { const config = await createTestConfig(rootDir); const parentId = "parent-ws"; @@ -14140,8 +14140,9 @@ describe("TaskService", () => { testTaskSettings(2, 3) ); - const hasQueuedMessages = mock(() => true); - const { workspaceService } = createWorkspaceServiceMocks({ hasQueuedMessages }); + // The union flag: a queued tool-end message or an outstanding bash-monitor wake. + const isToolEndYieldRequested = mock(() => true); + const { workspaceService } = createWorkspaceServiceMocks({ isToolEndYieldRequested }); const { taskService } = createTaskServiceHarness(config, { workspaceService }); const internal = taskService as unknown as { backgroundableForegroundWaitersByWorkspaceId: Map>; @@ -14157,7 +14158,7 @@ describe("TaskService", () => { .catch((error: unknown) => error); expect(waitError).toBeInstanceOf(ForegroundWaitBackgroundedError); - expect(hasQueuedMessages).toHaveBeenCalledWith(parentId, "tool-end"); + expect(isToolEndYieldRequested).toHaveBeenCalledWith(parentId); expect(taskService.backgroundForegroundWaitsForWorkspace(parentId)).toBe(0); expect(internal.backgroundableForegroundWaitersByWorkspaceId.has(parentId)).toBe(false); expect(internal.pendingStartWaitersByTaskId.has(childId)).toBe(false); @@ -23851,15 +23852,15 @@ describe("TaskService", () => { expect(snapshot?.reportMarkdown).toBeUndefined(); }); - test("workspace-turn tool-calls stream-end defers to a queued wake continuation", async () => { - // A queued bash-monitor wake cuts the correlated stream at a tool boundary - // (finishReason "tool-calls") while the child seamlessly continues the - // same turn — the handle must stay running. - const hasPendingBashMonitorWakeContinuation = mock( - (workspaceId: string) => workspaceId === "childworkspace" + test("workspace-turn tool-calls stream-end defers to an outstanding wake", async () => { + // An outstanding bash-monitor wake makes the correlated stream yield at a tool + // boundary (finishReason "tool-calls"); the wake turn then continues the same + // turn — the handle must stay running. + const hasOutstandingBashMonitorWake = mock((workspaceId: string) => + Promise.resolve(workspaceId === "childworkspace") ); const { parentId, taskService } = await startWorkspaceTurnForTest({ - hasPendingBashMonitorWakeContinuation, + hasOutstandingBashMonitorWake, }); const internal = taskService as unknown as { handleStreamEnd: (event: StreamEndEvent) => Promise; diff --git a/src/node/services/taskService.ts b/src/node/services/taskService.ts index 4fbf97844f..1f75ba0f45 100644 --- a/src/node/services/taskService.ts +++ b/src/node/services/taskService.ts @@ -7768,7 +7768,7 @@ export class TaskService implements AgentTaskIntegration { /** * Background any registered foreground waits for the requesting workspace when a - * tool-end message is already queued. Shared by both wait-registration paths + * tool-end message is already queued or a bash-monitor wake is outstanding. Shared by both wait-registration paths * (workspace-turn and task await): the auto-backgrounding signal is edge-triggered * on enqueue, so a message queued before the waiter registered must be re-checked * here. No-op when backgrounding is disabled or no requesting workspace is set. @@ -7780,7 +7780,7 @@ export class TaskService implements AgentTaskIntegration { if ( shouldBackgroundOnQueuedMessage && requestingWorkspaceId && - this.workspaceService.hasQueuedMessages(requestingWorkspaceId, "tool-end") + this.workspaceService.isToolEndYieldRequested(requestingWorkspaceId) ) { this.backgroundForegroundWaitsForWorkspace(requestingWorkspaceId); } diff --git a/src/node/services/taskWorkspaceSeam.testUtils.ts b/src/node/services/taskWorkspaceSeam.testUtils.ts index 5b81af0977..84bad1689c 100644 --- a/src/node/services/taskWorkspaceSeam.testUtils.ts +++ b/src/node/services/taskWorkspaceSeam.testUtils.ts @@ -14,7 +14,8 @@ export function makeWorkspaceHostFake(overrides: Partial = {}): W hasQueuedMessages: () => false, hasPendingQueuedOrPreparingTurn: () => false, hasPendingAutoRetry: () => false, - hasPendingBashMonitorWakeContinuation: () => false, + hasOutstandingBashMonitorWake: () => Promise.resolve(false), + isToolEndYieldRequested: () => false, hasPendingWorkspaceTurnContinuation: () => false, hasQueuedWorkspaceTurn: () => false, removeQueuedWorkspaceTurn: () => Ok(true), diff --git a/src/node/services/taskWorkspaceSeam.ts b/src/node/services/taskWorkspaceSeam.ts index f156763a52..342e8de393 100644 --- a/src/node/services/taskWorkspaceSeam.ts +++ b/src/node/services/taskWorkspaceSeam.ts @@ -322,9 +322,6 @@ export interface SendMessageInternalOptions { onAccepted?: () => Promise | void; onCanceled?: (reason: string) => Promise | void; onAcceptedPreStreamFailure?: (error: SendMessageError) => Promise | void; - cancelState?: { canceledBeforeAcceptance: boolean }; - /** Cancels a synthetic send even after it has left MessageQueue for PREPARING. */ - cancelSignal?: AbortSignal; /** * Synchronous staleness probe from the caller, re-evaluated at the real admission points * (the enqueue block and the session's turn-admission gates) in addition to the @@ -399,7 +396,10 @@ export interface TurnAdmissionHost { hasQueuedMessages(workspaceId: string, dispatchMode?: "tool-end" | "turn-end"): boolean; hasPendingQueuedOrPreparingTurn(workspaceId: string): boolean; hasPendingAutoRetry(workspaceId: string): boolean; - hasPendingBashMonitorWakeContinuation(workspaceId: string): boolean; + /** Bash-monitor wake level (see WorkspaceService.hasOutstandingBashMonitorWake). */ + hasOutstandingBashMonitorWake(workspaceId: string): Promise; + /** Pending input (queued tool-end message or outstanding wake) wants a tool boundary. */ + isToolEndYieldRequested(workspaceId: string): boolean; hasPendingWorkspaceTurnContinuation( workspaceId: string, metadata: Extract diff --git a/src/node/services/turnRequestBuilder.ts b/src/node/services/turnRequestBuilder.ts index e851b13662..1f3f8aef16 100644 --- a/src/node/services/turnRequestBuilder.ts +++ b/src/node/services/turnRequestBuilder.ts @@ -272,7 +272,8 @@ export interface StreamMessageOptions { allowAgentSetGoal?: boolean; workspaceGoalService?: WorkspaceGoalService; disableWorkspaceAgents?: boolean; - hasQueuedMessages?: (dispatchMode?: "tool-end" | "turn-end") => boolean; + /** Whether input that must run at a tool boundary is pending (see StreamRequestInput). */ + hasPendingToolEndInput?: () => Promise | boolean; muxMetadata?: MuxMessageMetadata; openaiTruncationModeOverride?: "auto" | "disabled"; /** @@ -736,7 +737,7 @@ export class TurnRequestBuilder { allowAgentSetGoal, workspaceGoalService, disableWorkspaceAgents, - hasQueuedMessages, + hasPendingToolEndInput, openaiTruncationModeOverride, muxMetadata, minThinkingLevel: providedMinThinkingLevel, @@ -2856,7 +2857,7 @@ export class TurnRequestBuilder { maxOutputTokens, toolPolicy: effectiveToolPolicy, providedStreamToken: streamToken, - hasQueuedMessages, + hasPendingToolEndInput, workspaceName: metadata.name, thinkingLevel: streamThinkingLevel, headers: requestHeaders, diff --git a/src/node/services/workspaceService.test.ts b/src/node/services/workspaceService.test.ts index c885ff3dec..4e03176222 100644 --- a/src/node/services/workspaceService.test.ts +++ b/src/node/services/workspaceService.test.ts @@ -239,6 +239,7 @@ describe("WorkspaceService bash monitor wake reconciler wiring", () => { acknowledgeMonitorWake: mock(() => undefined), dropRetiredMonitor: mock(() => undefined), setMessageQueued: mock(() => undefined), + cleanup: mock(() => Promise.resolve()), }) as unknown as BackgroundProcessManager; const service = createWorkspaceServiceForTest({ config, @@ -255,7 +256,6 @@ describe("WorkspaceService bash monitor wake reconciler wiring", () => { test("monitor lifecycle and shown-output events poke the reconciler", async () => { const { service, events, cleanup } = await createWakeWiringService(); const scheduleReconcile = mock(() => undefined); - const discardProcess = mock(() => Promise.resolve()); const upsert = mock(() => Promise.resolve()); const remove = mock(() => Promise.resolve()); const recordTerminal = mock(() => Promise.resolve()); @@ -263,7 +263,6 @@ describe("WorkspaceService bash monitor wake reconciler wiring", () => { bashMonitorRecoveryPromise: Promise; bashMonitorWakeReconciler: { scheduleReconcile: typeof scheduleReconcile; - discardProcess: typeof discardProcess; }; bashMonitorRegistryStore: { upsert: typeof upsert; @@ -273,7 +272,7 @@ describe("WorkspaceService bash monitor wake reconciler wiring", () => { }; try { await internal.bashMonitorRecoveryPromise; - internal.bashMonitorWakeReconciler = { scheduleReconcile, discardProcess }; + internal.bashMonitorWakeReconciler = { scheduleReconcile }; internal.bashMonitorRegistryStore = { upsert, remove, recordTerminal }; const armed = { processId: "proc", @@ -297,7 +296,6 @@ describe("WorkspaceService bash monitor wake reconciler wiring", () => { } expect(upsert).toHaveBeenCalledWith(armed); - expect(discardProcess).toHaveBeenCalledWith("owner", "proc", armed.createdAt); expect(remove).toHaveBeenCalledWith("owner", "proc", armed.createdAt); expect(scheduleReconcile).toHaveBeenCalledTimes(4); } finally { @@ -505,7 +503,6 @@ describe("WorkspaceService bash monitor wake reconciler wiring", () => { test("cancellation invalidates a scheduled runtime failure persistence retry", async () => { const { service, events, cleanup } = await createWakeWiringService(); const scheduleReconcile = mock(() => undefined); - const discardProcess = mock(() => Promise.resolve()); const upsert = mock(() => Promise.resolve()); const remove = mock(() => Promise.resolve()); const recordLost = mock(() => Promise.reject(new Error("transient registry write failure"))); @@ -513,7 +510,6 @@ describe("WorkspaceService bash monitor wake reconciler wiring", () => { bashMonitorRecoveryPromise: Promise; bashMonitorWakeReconciler: { scheduleReconcile: typeof scheduleReconcile; - discardProcess: typeof discardProcess; }; bashMonitorRegistryStore: { upsert: typeof upsert; @@ -524,7 +520,7 @@ describe("WorkspaceService bash monitor wake reconciler wiring", () => { }; try { await internal.bashMonitorRecoveryPromise; - internal.bashMonitorWakeReconciler = { scheduleReconcile, discardProcess }; + internal.bashMonitorWakeReconciler = { scheduleReconcile }; internal.bashMonitorRegistryStore = { upsert, remove, @@ -560,11 +556,6 @@ describe("WorkspaceService bash monitor wake reconciler wiring", () => { expect(recordLost).toHaveBeenCalledTimes(1); expect(upsert).toHaveBeenCalledTimes(1); expect(remove).toHaveBeenCalledWith("owner", armMetadata.processId, armMetadata.createdAt); - expect(discardProcess).toHaveBeenCalledWith( - "owner", - armMetadata.processId, - armMetadata.createdAt - ); // The invalidated chain must also release its tracking entry so the // per-process failure-persist map stays bounded by in-flight chains. const tracking = ( @@ -748,8 +739,6 @@ describe("WorkspaceService bash monitor wake reconciler wiring", () => { ownerWorkspaceId: string; prompt: string; muxMetadata: { type: "bash-monitor-wake"; records: [] }; - dedupeKey: string; - cancelSignal: AbortSignal; onAccepted(): Promise; onDeferred(): Promise; }): Promise<"in-flight" | "deferred">; @@ -761,8 +750,6 @@ describe("WorkspaceService bash monitor wake reconciler wiring", () => { ownerWorkspaceId: workspaceId, prompt: "wake", muxMetadata: { type: "bash-monitor-wake", records: [] }, - dedupeKey: "wake", - cancelSignal: new AbortController().signal, onAccepted, onDeferred, }); @@ -774,7 +761,7 @@ describe("WorkspaceService bash monitor wake reconciler wiring", () => { } }); - test("active session-backed streams queue monitor wakes at tool end", async () => { + test("streaming owners never queue a wake: the stream yields on the level instead", async () => { const { config, service, cleanup } = await createWakeWiringService(); const workspaceId = "streaming-wake-owner"; await config.addWorkspace("/tmp/streaming-wake-project", { @@ -784,34 +771,18 @@ describe("WorkspaceService bash monitor wake reconciler wiring", () => { projectPath: "/tmp/streaming-wake-project", runtimeConfig: { type: "local" }, }); - let queuedMode: string | undefined; - let queuedCancelState: { canceledBeforeAcceptance: boolean } | undefined; - const sendMessage = mock( - ( - _workspaceId: string, - _prompt: string, - options: { queueDispatchMode?: string }, - internal?: { cancelState?: { canceledBeforeAcceptance: boolean } } - ) => { - queuedMode = options.queueDispatchMode; - queuedCancelState = internal?.cancelState; - return Promise.resolve(Ok(undefined)); - } - ); + const sendMessage = mock(() => Promise.resolve(Ok(undefined))); const afterIdle = mock(() => undefined); const internal = service as unknown as { aiService: { isStreaming(workspaceId: string): boolean }; hasPendingQueuedOrPreparingTurn(workspaceId: string): boolean; isBusyForMessage(workspaceId: string): boolean; scheduleBashMonitorWakeReconcileAfterIdle(workspaceId: string): void; - getDelegatedTurnContinuationSendOptions(workspaceId: string): Promise; sendMessage: typeof sendMessage; dispatchBashMonitorWake(dispatch: { ownerWorkspaceId: string; prompt: string; muxMetadata: { type: "bash-monitor-wake"; records: [] }; - dedupeKey: string; - cancelSignal: AbortSignal; onAccepted(): Promise; onDeferred(): Promise; }): Promise<"in-flight" | "deferred">; @@ -821,71 +792,98 @@ describe("WorkspaceService bash monitor wake reconciler wiring", () => { internal.hasPendingQueuedOrPreparingTurn = () => false; internal.isBusyForMessage = () => true; internal.scheduleBashMonitorWakeReconcileAfterIdle = afterIdle; - internal.getDelegatedTurnContinuationSendOptions = () => Promise.resolve({}); internal.sendMessage = sendMessage; const outcome = await internal.dispatchBashMonitorWake({ ownerWorkspaceId: workspaceId, prompt: "wake", muxMetadata: { type: "bash-monitor-wake", records: [] }, - dedupeKey: "wake", - cancelSignal: new AbortController().signal, onAccepted: () => Promise.resolve(), onDeferred: () => Promise.resolve(), }); - expect(outcome).toBe("in-flight"); - expect(sendMessage).toHaveBeenCalledTimes(1); - expect(queuedMode).toBe("tool-end"); - expect(queuedCancelState).toEqual({ canceledBeforeAcceptance: false }); + expect(outcome).toBe("deferred"); + expect(sendMessage).not.toHaveBeenCalled(); + expect(afterIdle).toHaveBeenCalledWith(workspaceId); + } finally { + await cleanup(); + } + }); + + test("a stream without a busy session defers without re-arming the idle wait", async () => { + // The idle wait resolves immediately when no session is busy, so re-arming here would + // spin reconcile → defer → re-arm until the stream ends; stream-end schedules instead. + const { config, service, cleanup } = await createWakeWiringService(); + const workspaceId = "orphan-stream-wake-owner"; + await config.addWorkspace("/tmp/orphan-stream-wake-project", { + id: workspaceId, + name: workspaceId, + projectName: "orphan-stream-wake-project", + projectPath: "/tmp/orphan-stream-wake-project", + runtimeConfig: { type: "local" }, + }); + const sendMessage = mock(() => Promise.resolve(Ok(undefined))); + const afterIdle = mock(() => undefined); + const internal = service as unknown as { + aiService: { isStreaming(workspaceId: string): boolean }; + hasPendingQueuedOrPreparingTurn(workspaceId: string): boolean; + isBusyForMessage(workspaceId: string): boolean; + scheduleBashMonitorWakeReconcileAfterIdle(workspaceId: string): void; + sendMessage: typeof sendMessage; + dispatchBashMonitorWake(dispatch: { + ownerWorkspaceId: string; + prompt: string; + muxMetadata: { type: "bash-monitor-wake"; records: [] }; + onAccepted(): Promise; + onDeferred(): Promise; + }): Promise<"in-flight" | "deferred">; + }; + try { + internal.aiService = { isStreaming: () => true }; + internal.hasPendingQueuedOrPreparingTurn = () => false; + internal.isBusyForMessage = () => false; + internal.scheduleBashMonitorWakeReconcileAfterIdle = afterIdle; + internal.sendMessage = sendMessage; + + const outcome = await internal.dispatchBashMonitorWake({ + ownerWorkspaceId: workspaceId, + prompt: "wake", + muxMetadata: { type: "bash-monitor-wake", records: [] }, + onAccepted: () => Promise.resolve(), + onDeferred: () => Promise.resolve(), + }); + + expect(outcome).toBe("deferred"); + expect(sendMessage).not.toHaveBeenCalled(); expect(afterIdle).not.toHaveBeenCalled(); } finally { await cleanup(); } }); - test("withdrawing a queued monitor wake removes it and releases its dedupe key", async () => { + test("idle owners receive the wake as a direct synthetic turn", async () => { const { config, service, cleanup } = await createWakeWiringService(); - const workspaceId = "withdrawn-wake-owner"; - await config.addWorkspace("/tmp/withdrawn-wake-project", { + const workspaceId = "idle-wake-owner"; + await config.addWorkspace("/tmp/idle-wake-project", { id: workspaceId, name: workspaceId, - projectName: "withdrawn-wake-project", - projectPath: "/tmp/withdrawn-wake-project", + projectName: "idle-wake-project", + projectPath: "/tmp/idle-wake-project", runtimeConfig: { type: "local" }, }); - const session = service.getOrCreateSession(workspaceId); - const queuedModes: Array<"tool-end" | "turn-end" | null> = []; - // The real sendMessage queues behind a busy session; mirror only that branch. + let sentOptions: { queueDispatchMode?: string; muxMetadata?: unknown } | undefined; const sendMessage = mock( ( _workspaceId: string, - prompt: string, - options: SendMessageOptions, - internal?: { - synthetic?: boolean; - agentInitiated?: boolean; - queueDedupeKey?: string; - removableQueueDedupeKey?: boolean; - cancelState?: { canceledBeforeAcceptance: boolean }; - cancelSignal?: AbortSignal; - onCanceled?: (reason: string) => Promise | void; - } + _prompt: string, + options: { queueDispatchMode?: string; muxMetadata?: unknown }, + internal?: { onAccepted?: () => Promise } ) => { - queuedModes.push( - session.queueMessage(prompt, options, { - synthetic: internal?.synthetic, - agentInitiated: internal?.agentInitiated, - dedupeKey: internal?.queueDedupeKey, - removableDedupeKey: internal?.removableQueueDedupeKey, - cancelState: internal?.cancelState, - cancelSignal: internal?.cancelSignal, - onCanceled: internal?.onCanceled, - }) - ); - return Promise.resolve(Ok(undefined)); + sentOptions = options; + return internal?.onAccepted?.().then(() => Ok(undefined)) ?? Promise.resolve(Ok(undefined)); } ); + const onAccepted = mock(() => Promise.resolve()); const internal = service as unknown as { aiService: { isStreaming(workspaceId: string): boolean }; hasPendingQueuedOrPreparingTurn(workspaceId: string): boolean; @@ -896,50 +894,92 @@ describe("WorkspaceService bash monitor wake reconciler wiring", () => { ownerWorkspaceId: string; prompt: string; muxMetadata: { type: "bash-monitor-wake"; records: [] }; - dedupeKey: string; - cancelSignal: AbortSignal; onAccepted(): Promise; onDeferred(): Promise; }): Promise<"in-flight" | "deferred">; }; - const dedupeKey = "bash-monitor-wake:" + workspaceId + ":dispatch-1"; - const onDeferred = mock(() => Promise.resolve()); - const dispatch = (cancelSignal: AbortSignal) => - internal.dispatchBashMonitorWake({ - ownerWorkspaceId: workspaceId, - prompt: "wake", - muxMetadata: { type: "bash-monitor-wake", records: [] }, - dedupeKey, - cancelSignal, - onAccepted: () => Promise.resolve(), - onDeferred, - }); try { - internal.aiService = { isStreaming: () => true }; + internal.aiService = { isStreaming: () => false }; internal.hasPendingQueuedOrPreparingTurn = () => false; - internal.isBusyForMessage = () => true; + internal.isBusyForMessage = () => false; internal.getDelegatedTurnContinuationSendOptions = () => Promise.resolve({}); internal.sendMessage = sendMessage; - const controller = new AbortController(); - expect(await dispatch(controller.signal)).toBe("in-flight"); - expect(session.hasQueuedMessages("tool-end")).toBe(true); - - controller.abort("output already shown"); - expect(session.hasQueuedMessages()).toBe(false); - expect(service.removeQueuedMessagesByDedupeKeyPrefix(workspaceId, dedupeKey)).toEqual(Ok(0)); - await new Promise((resolve) => setTimeout(resolve, 0)); - expect(onDeferred).toHaveBeenCalledTimes(1); + const outcome = await internal.dispatchBashMonitorWake({ + ownerWorkspaceId: workspaceId, + prompt: "wake", + muxMetadata: { type: "bash-monitor-wake", records: [] }, + onAccepted, + onDeferred: () => Promise.resolve(), + }); - // Already withdrawn at dispatch: never reaches the send, so nothing can be enqueued. - expect(await dispatch(controller.signal)).toBe("deferred"); + expect(outcome).toBe("in-flight"); expect(sendMessage).toHaveBeenCalledTimes(1); + expect(sentOptions?.queueDispatchMode).toBeUndefined(); + expect(sentOptions?.muxMetadata).toEqual({ type: "bash-monitor-wake", records: [] }); + expect(onAccepted).toHaveBeenCalledTimes(1); + } finally { + await cleanup(); + } + }); + + test("the stream yields on the wake level, not on a queued snapshot of it", async () => { + // Regression: a monitored bash matched mid-step while the same step's task_await showed + // the matched lines. The old queued tool-end wake still cut the stream (finishReason + // "tool-calls") and was then withdrawn, ending the turn with no assistant text. + const { service, cleanup } = await createWakeWiringService(); + const workspaceId = "level-yield-owner"; + const createdAt = "2026-08-31T12:00:00.000Z"; + const internal = service as unknown as { + backgroundProcessManager: { + pullMonitorWakeSignals: ReturnType; + getMonitorWakeDeliveryState: ReturnType; + setMessageQueued: ReturnType; + }; + }; + const processManager = internal.backgroundProcessManager; + const session = service.getOrCreateSession(workspaceId); + try { + processManager.pullMonitorWakeSignals.mockImplementation(() => + Promise.resolve([ + { + processId: "proc", + taskId: "bash:proc", + ownerWorkspaceId: workspaceId, + filter: "READY", + filterExclude: false, + script: "run", + createdAt, + match: { throughOffset: 12, lines: ["READY"], totalMatches: 1 }, + retired: false, + }, + ]) + ); + let shownThroughOffset = 0; + processManager.getMonitorWakeDeliveryState.mockImplementation(() => + Promise.resolve({ status: "settled", shownThroughOffset, terminalStatusShown: false }) + ); + + // Match not yet shown: the boundary yields and bash long-polls return early. + expect(await session.hasPendingToolEndInput()).toBe(true); + expect(processManager.setMessageQueued).toHaveBeenLastCalledWith(workspaceId, true); expect(session.hasQueuedMessages()).toBe(false); - expect(onDeferred).toHaveBeenCalledTimes(1); - expect(await dispatch(new AbortController().signal)).toBe("in-flight"); - expect(queuedModes).toEqual(["tool-end", "tool-end"]); + // task_await showed the lines before the SDK asked: no yield, no wake turn. + shownThroughOffset = 12; + expect(await session.hasPendingToolEndInput()).toBe(false); + expect(processManager.setMessageQueued).toHaveBeenLastCalledWith(workspaceId, false); + expect(await service.hasOutstandingBashMonitorWake(workspaceId)).toBe(false); + + // A queued tool-end message still yields on its own. + session.queueMessage("follow up", { + model: "anthropic:claude-sonnet-4-5", + agentId: "exec", + queueDispatchMode: "tool-end", + }); + expect(await session.hasPendingToolEndInput()).toBe(true); } finally { + session.dispose(); await cleanup(); } }); @@ -9177,35 +9217,6 @@ describe("WorkspaceService sendMessage status clearing", () => { } }); - test("refuses to queue a send whose cancel signal already fired", async () => { - fakeSession.isBusy.mockReturnValue(true); - const controller = new AbortController(); - controller.abort("monitor withdrawn"); - const onCanceled = mock(() => undefined); - const cancelState = { canceledBeforeAcceptance: false }; - - const result = await workspaceService.sendMessage( - "test-workspace", - "wake", - { model: "openai:gpt-4o-mini", agentId: "exec" }, - { - synthetic: true, - agentInitiated: true, - cancelSignal: controller.signal, - cancelState, - onCanceled, - queueDedupeKey: "bash-monitor-wake:test-workspace:1", - removableQueueDedupeKey: true, - } - ); - - expect(result.success).toBe(true); - expect(fakeSession.queueMessage).not.toHaveBeenCalled(); - expect(onCanceled).toHaveBeenCalledTimes(1); - expect(onCanceled).toHaveBeenCalledWith("monitor withdrawn"); - expect(cancelState.canceledBeforeAcceptance).toBe(true); - }); - test("strips stale workspace-turn correlation behind an earlier queued entry", async () => { fakeSession.hasQueuedOrDispatchingEntry.mockReturnValue(true); const onCanceled = mock(() => undefined); diff --git a/src/node/services/workspaceService.ts b/src/node/services/workspaceService.ts index c1d93b8fba..7bc9e7b46a 100644 --- a/src/node/services/workspaceService.ts +++ b/src/node/services/workspaceService.ts @@ -42,7 +42,6 @@ import { type StreamErrorRecoveryOutcome, } from "@/node/services/agentSession"; import type { QueueCutCutter } from "@/node/services/messageQueue"; -import { cancelReasonBeforeAcceptance } from "@/node/services/messageQueue"; import type { HistoryService } from "@/node/services/historyService"; import type { AIService } from "@/node/services/aiService"; import type { StreamManager } from "@/node/services/streamManager"; @@ -1925,11 +1924,6 @@ export class WorkspaceService extends EventEmitter implements WorkspaceHost { const persist = async (): Promise => { if (payload.reason === "canceled") { if (createdAt == null) return false; - await this.bashMonitorWakeReconciler.discardProcess( - workspaceId, - payload.processId, - createdAt - ); await this.bashMonitorRegistryStore.remove(workspaceId, payload.processId, createdAt); return true; } @@ -2380,6 +2374,15 @@ export class WorkspaceService extends EventEmitter implements WorkspaceHost { }, registry: this.bashMonitorRegistryStore, onWake: (dispatch) => this.dispatchBashMonitorWake(dispatch), + onOutstandingChanged: (ownerWorkspaceId, outstanding) => { + // The level drives the same tool-boundary side effects a queued tool-end message + // does: long-polling bash reads return early and foreground agent-task waits are + // backgrounded so the stream can reach the boundary where it yields to the wake. + this.sessions.get(ownerWorkspaceId)?.setBashMonitorWakeOutstanding(outstanding); + if (outstanding) { + this.agentTaskIntegration?.backgroundForegroundWaitsForWorkspace(ownerWorkspaceId); + } + }, }); if (typeof this.backgroundProcessManager.on === "function") { this.backgroundProcessManager.on("output:shown", this.bashOutputShownListener); @@ -2504,6 +2507,12 @@ export class WorkspaceService extends EventEmitter implements WorkspaceHost { this.pendingBashMonitorWakeIdleWaitsByOwner.set(ownerWorkspaceId, promise); } + /** + * Start a wake turn from the level, or leave it pending. A wake is never queued as a + * message: while the owner streams, the stream itself reads the level at each tool + * boundary (AgentSession.hasPendingToolEndInput) and yields with finishReason + * "tool-calls"; the after-idle reconcile then lands here again and sends directly. + */ private async dispatchBashMonitorWake( dispatch: BashMonitorWakeDispatch ): Promise { @@ -2515,16 +2524,17 @@ export class WorkspaceService extends EventEmitter implements WorkspaceHost { this.notifyBashMonitorWakeStateChanged(ownerWorkspaceId); return "in-flight"; } - const hasPendingTurn = this.hasPendingQueuedOrPreparingTurn(ownerWorkspaceId); - const hasSessionBackedBusyState = this.isBusyForMessage(ownerWorkspaceId); - const hasAiServiceStream = this.aiService.isStreaming(ownerWorkspaceId); - if (hasPendingTurn || (hasSessionBackedBusyState && !hasAiServiceStream)) { + if ( + this.hasPendingQueuedOrPreparingTurn(ownerWorkspaceId) || + this.isBusyForMessage(ownerWorkspaceId) + ) { this.scheduleBashMonitorWakeReconcileAfterIdle(ownerWorkspaceId); return "deferred"; } - if (hasAiServiceStream && !hasSessionBackedBusyState) { - return "deferred"; - } + // Streaming without a busy session (teardown window, or no session at all): the + // after-idle wait would resolve immediately and spin. The stream-end/abort/error + // listeners schedule the next reconcile instead. + if (this.aiService.isStreaming(ownerWorkspaceId)) return "deferred"; const sendOptions = (await this.getDelegatedTurnContinuationSendOptions(ownerWorkspaceId)) ?? (await this.getWorkflowContinuationSendOptions(ownerWorkspaceId)); @@ -2533,43 +2543,15 @@ export class WorkspaceService extends EventEmitter implements WorkspaceHost { return "deferred"; } - // Withdrawn while awaiting send options above: the abort listener below would never - // fire, and send preflight (which persists AI settings) has nothing left to admit. - if (dispatch.cancelSignal.aborted) return "deferred"; - let accepted = false; - // A queued wake can be superseded after dequeue. Share cancellation state so - // AgentSession can release PREPARING when cancellation wins before acceptance. - const cancelState = { canceledBeforeAcceptance: false }; - // Withdrawal (output already shown, process discarded, history cleared) must - // free the queue slot now, not at stream end: a lingering entry keeps the - // workspace reported busy and its dedupe key held. The key is unique per - // dispatch, so this cannot drop a newer wake's entry. - dispatch.cancelSignal.addEventListener( - "abort", - () => { - this.removeQueuedMessagesByDedupeKeyPrefix(ownerWorkspaceId, dispatch.dedupeKey, { - cancelReason: "Bash monitor wake withdrawn before dispatch.", - }); - }, - { once: true } - ); const sendResult = await this.sendMessage( ownerWorkspaceId, dispatch.prompt, - { - ...sendOptions, - queueDispatchMode: "tool-end", - muxMetadata: dispatch.muxMetadata, - }, + { ...sendOptions, muxMetadata: dispatch.muxMetadata }, { skipAutoResumeReset: true, synthetic: true, agentInitiated: true, - cancelState, - cancelSignal: dispatch.cancelSignal, - queueDedupeKey: dispatch.dedupeKey, - removableQueueDedupeKey: true, onAccepted: async () => { accepted = true; await dispatch.onAccepted(); @@ -2578,10 +2560,12 @@ export class WorkspaceService extends EventEmitter implements WorkspaceHost { onAcceptedPreStreamFailure: async () => { if (accepted) await dispatch.onAccepted(); }, + // Idle owner, so this send does not queue; still, a manual send racing past the + // idle checks above can push it into the queue and clear it before dispatch. onCanceled: async () => { if (!accepted) { await dispatch.onDeferred(); - this.scheduleBashMonitorWakeReconcile(ownerWorkspaceId); + this.scheduleBashMonitorWakeReconcileAfterIdle(ownerWorkspaceId); } }, } @@ -4030,6 +4014,8 @@ export class WorkspaceService extends EventEmitter implements WorkspaceHost { // is released at its queue/session handoff so a follow-up dispatched // from within that turn does not veto itself. hasExternalSendPreflight: () => this.hasSessionInvisiblePreflight(workspaceId), + hasOutstandingBashMonitorWake: () => + this.bashMonitorWakeReconciler.hasOutstandingWake(workspaceId), }); } @@ -10766,8 +10752,6 @@ export class WorkspaceService extends EventEmitter implements WorkspaceHost { agentInitiated: internal?.agentInitiated, goalKind: internal?.goalKind, goalId: internal?.goalId, - cancelState: internal?.cancelState, - cancelSignal: internal?.cancelSignal, onCanceled: internal?.onCanceled, onAccepted: internal?.onAccepted, onAcceptedPreStreamFailure: internal?.onAcceptedPreStreamFailure, @@ -10845,18 +10829,6 @@ export class WorkspaceService extends EventEmitter implements WorkspaceHost { } if (shouldQueue) { - // Mirrors AgentSession's cancelBeforeAcceptance for the queue path: a send withdrawn - // during the preflight awaits above must not occupy a queue slot (and hold its dedupe - // key) until the stream drains it. Nothing is persisted yet, so only the handshake runs. - if (internal?.cancelSignal?.aborted === true) { - await getContinuationSendState().onCanceled?.( - cancelReasonBeforeAcceptance(internal.cancelSignal) - ); - if (internal.cancelState != null) { - internal.cancelState.canceledBeforeAcceptance = true; - } - return Ok(undefined); - } // Everything from here to queueMessage is synchronous, so a probe pass here cannot go // stale before the entry is enqueued. if (internal?.admissionStale?.() === true) { @@ -10945,8 +10917,6 @@ export class WorkspaceService extends EventEmitter implements WorkspaceHost { workspaceTurnContinuation: internal?.workspaceTurnContinuation, dedupeKey: internal?.queueDedupeKey, removableDedupeKey: internal?.removableQueueDedupeKey, - cancelState: internal?.cancelState, - cancelSignal: internal?.cancelSignal, onCanceled: continuationSendState.onCanceled, onAccepted: internal?.onAccepted, onAcceptedPreStreamFailure: continuationSendState.onAcceptedPreStreamFailure, @@ -11045,9 +11015,9 @@ export class WorkspaceService extends EventEmitter implements WorkspaceHost { // follow-up redispatched from within this very turn (on-send compaction // completion) does not veto itself — while the admission awaits between // here and the busy claim stay covered. Codex P2 (PRRT_kwDOPxxmWM6cSRkH): - // releasing at the handoff itself left AgentSession's - // cancelBeforeAcceptance yield observable as idle, letting follow-up - // recovery admit an exec turn ahead of the accepted manual send. Refusal + // releasing at the handoff itself left AgentSession's pre-acceptance + // yield observable as idle, letting follow-up recovery admit an exec + // turn ahead of the accepted manual send. Refusal // paths never fire the callback; the scoped disposal releases on return. const result = await session.sendMessage(message, continuationSendState.options, { onTurnAdmissionCommitted: () => sessionInvisiblePreflight.release(), @@ -11057,8 +11027,6 @@ export class WorkspaceService extends EventEmitter implements WorkspaceHost { goalId: internal?.goalId, goalContinuation: internal?.goalContinuation, startStreamInBackground: internal?.startStreamInBackground, - cancelState: internal?.cancelState, - cancelSignal: internal?.cancelSignal, // Same authoring-time race as the queued path: the goal-creating // stream can end during the preflight awaits above, making a fresh // goal visible after the user hit enter but before this dispatch. @@ -11867,12 +11835,20 @@ export class WorkspaceService extends EventEmitter implements WorkspaceHost { } /** - * Whether a bash-monitor-wake continuation is queued next or mid-dispatch. - * See AgentSession.hasPendingBashMonitorWakeContinuation for semantics. + * The bash-monitor wake level: a wake the workspace has not seen yet. A stream that + * ended with "tool-calls" while this is high yielded to the wake and will be continued + * by it (BashMonitorWakeReconciler.hasOutstandingWake). */ - hasPendingBashMonitorWakeContinuation(workspaceId: string): boolean { - const session = this.sessions.get(workspaceId.trim()); - return session?.hasPendingBashMonitorWakeContinuation() ?? false; + hasOutstandingBashMonitorWake(workspaceId: string): Promise { + return this.bashMonitorWakeReconciler.hasOutstandingWake(workspaceId.trim()); + } + + /** + * Whether pending input (queued tool-end message or outstanding wake) wants the active + * stream to reach a tool boundary — the union flag long-polling bash reads consult. + */ + isToolEndYieldRequested(workspaceId: string): boolean { + return this.backgroundProcessManager.hasQueuedMessage(workspaceId.trim()); } /** diff --git a/src/node/services/workspaceTurnManager.test.ts b/src/node/services/workspaceTurnManager.test.ts index 20588945ad..23e66a40bd 100644 --- a/src/node/services/workspaceTurnManager.test.ts +++ b/src/node/services/workspaceTurnManager.test.ts @@ -317,7 +317,7 @@ describe("WorkspaceTurnManager", () => { isStreaming?: ReturnType; hasQueuedMessages?: ReturnType; hasPendingQueuedOrPreparingTurn?: ReturnType; - hasPendingBashMonitorWakeContinuation?: ReturnType; + hasOutstandingBashMonitorWake?: ReturnType; hasPendingWorkspaceTurnContinuation?: ReturnType; getQueueCutCutter?: ReturnType; hasPendingAutoRetry?: ReturnType; diff --git a/src/node/services/workspaceTurnManager.ts b/src/node/services/workspaceTurnManager.ts index b74f3b9d6e..b81bc3a842 100644 --- a/src/node/services/workspaceTurnManager.ts +++ b/src/node/services/workspaceTurnManager.ts @@ -4310,10 +4310,10 @@ export class WorkspaceTurnManager { * Whether a continuation of this exact delegated turn is pending or streaming. * Pending entries must carry the same correlation metadata as the ended stream. */ - private hasSameTurnContinuation( + private async hasSameTurnContinuation( event: StreamEndEvent, correlation: { taskHandleId: string; ownerWorkspaceId: string; turnId: string } - ): boolean { + ): Promise { if ( this.workspaceService.hasPendingWorkspaceTurnContinuation(event.workspaceId, { type: "workspace-turn-task", @@ -4322,7 +4322,9 @@ export class WorkspaceTurnManager { ) { return true; } - if (this.workspaceService.hasPendingBashMonitorWakeContinuation(event.workspaceId)) { + // A stream that ended with "tool-calls" while a wake is outstanding yielded to that + // wake; the wake turn inherits this correlation (inheritOpenWorkspaceTurnMetadata). + if (await this.workspaceService.hasOutstandingBashMonitorWake(event.workspaceId)) { return true; } const activeStream = this.streamManager?.getStreamInfo(event.workspaceId); @@ -4480,16 +4482,16 @@ export class WorkspaceTurnManager { return true; } - // A queued continuation can stop the in-flight stream at a tool boundary with - // finishReason "tool-calls" and continue the same delegated turn. Report - // wake-ups carry the exact correlation explicitly; bash-monitor wakes inherit - // it from history. Defer settlement until the continuation's terminal + // A queued continuation (or an outstanding bash-monitor wake) can stop the + // in-flight stream at a tool boundary with finishReason "tool-calls" and + // continue the same delegated turn. Report wake-ups carry the exact + // correlation explicitly; bash-monitor wakes inherit it from history. Defer settlement until the continuation's terminal // stream-end instead of reporting a false completion failure to the owner. // Any other queued input (manual message, /compact) supersedes the turn and // must settle the old outcome here. if ( event.metadata.finishReason === "tool-calls" && - this.hasSameTurnContinuation(event, metadata) + (await this.hasSameTurnContinuation(event, metadata)) ) { await this.markWorkspaceTurnStreamEndDeferred(event); return true; From c9d7894d5589eaa55acce02eb4f014047e6ad0af Mon Sep 17 00:00:00 2001 From: Ethan Dickson Date: Thu, 3 Sep 2026 19:32:37 +0000 Subject: [PATCH 02/26] fix: keep wake continuations visible through PREPARING; retire handed-off wakes on clear/dispose Codex review on #4071: - P1: a direct wake send lowers the reconciler level at row persistence, before its stream is observable. AgentSession now marks a bash-monitor-wake send in PREPARING (hasPendingBashMonitorWakeTurn) and WorkspaceService folds it into hasOutstandingBashMonitorWake so delegated-turn settlement still sees the continuation. - P1: a full-history clear or disposal forgets the in-flight dispatch; the receiver checks dispatch.isCurrent() under the history lock before sending. - P2: session dispose and reconciler dispose lower the mirrored tool-end yield flag so a re-created session does not inherit a stale early-return. --- .../agentSession.queueDispatch.test.ts | 85 +++++++++++++++++++ src/node/services/agentSession.ts | 55 +++++++++--- .../bashMonitorWakeReconciler.test.ts | 28 ++++++ .../services/bashMonitorWakeReconciler.ts | 12 +++ src/node/services/workspaceService.test.ts | 76 +++++++++++++++++ src/node/services/workspaceService.ts | 29 +++++-- 6 files changed, 267 insertions(+), 18 deletions(-) diff --git a/src/node/services/agentSession.queueDispatch.test.ts b/src/node/services/agentSession.queueDispatch.test.ts index 2e2e66680e..bcc30642a3 100644 --- a/src/node/services/agentSession.queueDispatch.test.ts +++ b/src/node/services/agentSession.queueDispatch.test.ts @@ -694,6 +694,91 @@ describe("AgentSession queued message tool-call dispatch", () => { } }); + test("disposing a session lowers the mirrored wake level", async () => { + // The flag lives in BackgroundProcessManager keyed by workspace id and outlives the + // session; a stale true would make a re-created session's bash reads return early. + const workspaceId = "queue-dispatch-dispose-clears-level"; + const flags: boolean[] = []; + const { session, cleanup } = await createAgentSessionHarness({ + workspaceId, + backgroundProcessManagerOverrides: { + setMessageQueued: (_workspaceId: string, queued: boolean) => { + flags.push(queued); + }, + }, + }); + try { + session.setBashMonitorWakeOutstanding(true); + expect(flags.at(-1)).toBe(true); + session.dispose(); + expect(flags.at(-1)).toBe(false); + } finally { + session.dispose(); + await cleanup(); + } + }); + + test("a wake send in PREPARING is a pending wake turn until it streams or ends", async () => { + const workspaceId = "queue-dispatch-preparing-wake-turn"; + let markStreamRequested: () => void = () => undefined; + const streamRequested = new Promise((resolve) => { + markStreamRequested = resolve; + }); + let releaseStream: () => void = () => undefined; + const streamRelease = new Promise((resolve) => { + releaseStream = resolve; + }); + const { session, cleanup } = await createAgentSessionHarness({ + workspaceId, + aiServiceOverrides: { + // Blocks at the provider call: the user row is durable (onAccepted has run, so the + // reconciler level is already low) but no stream is observable yet. + streamMessage: mock(async () => { + markStreamRequested(); + await streamRelease; + return Ok(createStartedTurnHandle("test-assistant-message")); + }), + }, + }); + + let disposed = false; + try { + expect(session.hasPendingBashMonitorWakeTurn()).toBe(false); + let accepted = false; + const sendPromise = session.sendMessage( + "Background monitor wake", + { + model: TEST_MODEL, + agentId: "exec", + muxMetadata: { type: "bash-monitor-wake", records: [] }, + }, + { + synthetic: true, + agentInitiated: true, + onAccepted: () => { + accepted = true; + }, + } + ); + + await streamRequested; + expect(accepted).toBe(true); + expect(session.isBusy()).toBe(true); + expect(session.hasPendingBashMonitorWakeTurn()).toBe(true); + + // Turn end (here: teardown to IDLE) clears it. + session.dispose(); + disposed = true; + expect(session.hasPendingBashMonitorWakeTurn()).toBe(false); + releaseStream(); + await sendPromise; + } finally { + releaseStream(); + if (!disposed) session.dispose(); + await cleanup(); + } + }); + test("disposed sessions finalize durable wakes after goal sync completes", async () => { const workspaceId = "queue-dispatch-disposed-after-goal-sync"; let markSyncStarted: () => void = () => undefined; diff --git a/src/node/services/agentSession.ts b/src/node/services/agentSession.ts index 7264f69819..6523b91c40 100644 --- a/src/node/services/agentSession.ts +++ b/src/node/services/agentSession.ts @@ -865,6 +865,12 @@ export class AgentSession { /** Correlation of the direct send currently in the PREPARING phase, if any. */ private preparingWorkspaceTurnMetadata?: WorkspaceTurnMuxMetadata; + /** + * The send in the PREPARING phase is a bash-monitor wake. Its `onAccepted` lowers the + * reconciler level once the user row is durable, before the stream is observable, so + * this marker is what keeps the wake continuation visible across that window. + */ + private preparingBashMonitorWake = false; /** Context needed to retry the current stream (cleared on stream end/abort/error). */ private activeStreamContext?: { @@ -992,6 +998,16 @@ export class AgentSession { this.activePreparedTurnAbortController?.abort(); this.activePreparedTurnAbortController = null; + // The bash early-return flag is keyed by workspace id and outlives this session; a + // stale wake level would otherwise make a re-created session's long-polling reads + // return early forever. (The reconciler's own dispose lowers the level too.) Only + // touched when this session raised it: tests dispose sessions built on partial + // BackgroundProcessManager stubs. + if (this.bashMonitorWakeOutstanding) { + this.bashMonitorWakeOutstanding = false; + this.syncToolEndYieldRequested(false); + } + // Ensure any callers blocked on waitForIdle() can continue during teardown. this.setTurnPhase(TurnPhase.IDLE); @@ -4002,8 +4018,7 @@ export class AgentSession { const preparedTurnAbortController = new AbortController(); this.activePreparedTurnAbortController = preparedTurnAbortController; - this.preparingWorkspaceTurnMetadata = getWorkspaceTurnMuxMetadata(optionsForStream.muxMetadata); - this.setTurnPhase(TurnPhase.PREPARING); + this.enterPreparing(optionsForStream.muxMetadata); // From this synchronous point isBusy() reports the turn — release the // service-side preflight reservation (see onTurnAdmissionCommitted doc). internal?.onTurnAdmissionCommitted?.(); @@ -4155,8 +4170,7 @@ export class AgentSession { internal?.goalKind, internal?.goalId ); - this.preparingWorkspaceTurnMetadata = getWorkspaceTurnMuxMetadata(optionsForStream.muxMetadata); - this.setTurnPhase(TurnPhase.PREPARING); + this.enterPreparing(optionsForStream.muxMetadata); // Open the mid-turn thinking override window for the resumed turn (after // setTurnPhase(PREPARING), which clears the holder on the IDLE transition). const turnThinkingOverride: ActiveTurnThinkingOverride = {}; @@ -5376,10 +5390,7 @@ export class AgentSession { retryGoalKind, retryGoalId ); - this.preparingWorkspaceTurnMetadata = getWorkspaceTurnMuxMetadata( - retryOptionsForResume.muxMetadata - ); - this.setTurnPhase(TurnPhase.PREPARING); + this.enterPreparing(retryOptionsForResume.muxMetadata); let retryResult: Result; try { retryResult = await this.streamWithHistory( @@ -5485,8 +5496,7 @@ export class AgentSession { } // Retry the same request, but without post-compaction injection. - this.preparingWorkspaceTurnMetadata = getWorkspaceTurnMuxMetadata(context.options?.muxMetadata); - this.setTurnPhase(TurnPhase.PREPARING); + this.enterPreparing(context.options?.muxMetadata); let retryResult: Result; try { retryResult = await this.streamWithHistory( @@ -5707,6 +5717,7 @@ export class AgentSession { this.dispatchingQueuedEntry = false; this.dispatchingQueuedEntryMuxMetadata = undefined; this.preparingWorkspaceTurnMetadata = undefined; + this.preparingBashMonitorWake = false; this.activeStreamStartedAtMs = payload.startTime; // Codex P1 (PRRT_kwDOPxxmWM6cClKS): a new live stream makes mid-stream // setGoal deferral meaningful again — clear the goal service's settled @@ -6205,6 +6216,7 @@ export class AgentSession { this.dispatchingQueuedEntry = false; this.dispatchingQueuedEntryMuxMetadata = undefined; this.preparingWorkspaceTurnMetadata = undefined; + this.preparingBashMonitorWake = false; // Turn ended: expire any mid-turn thinking override. Safe unconditionally // because a replacement turn (e.g. an edit) only creates its holder after // the preempted turn has already been transitioned to IDLE. @@ -6625,6 +6637,26 @@ export class AgentSession { return false; } + /** Claim PREPARING for a send and record what kind of input it carries. */ + private enterPreparing(muxMetadata: unknown): void { + this.preparingWorkspaceTurnMetadata = getWorkspaceTurnMuxMetadata(muxMetadata); + this.preparingBashMonitorWake = + (muxMetadata as MuxMessageMetadata | undefined)?.type === "bash-monitor-wake"; + this.setTurnPhase(TurnPhase.PREPARING); + } + + /** + * A bash-monitor wake turn between admission and stream start (direct send in PREPARING, + * or a dequeued wake entry). The reconciler level is already low here — `onAccepted` + * ran when the user row became durable — but no replacement stream is observable yet, + * so delegated-turn settlement must still see the continuation. + */ + hasPendingBashMonitorWakeTurn(): boolean { + if (this.preparingBashMonitorWake) return true; + const dispatching = this.dispatchingQueuedEntryMuxMetadata as MuxMessageMetadata | undefined; + return dispatching?.type === "bash-monitor-wake"; + } + /** * Whether a queued or dispatching entry continues the exact workspace-turn correlation. */ @@ -6901,8 +6933,7 @@ export class AgentSession { // Set PREPARING synchronously before the async sendMessage to prevent // incoming messages from bypassing the queue during the await gap. - this.preparingWorkspaceTurnMetadata = getWorkspaceTurnMuxMetadata(options?.muxMetadata); - this.setTurnPhase(TurnPhase.PREPARING); + this.enterPreparing(options?.muxMetadata); void this.sendMessage(message, options, { ...internal, enqueuedAtMs }) .then(async (result) => { diff --git a/src/node/services/bashMonitorWakeReconciler.test.ts b/src/node/services/bashMonitorWakeReconciler.test.ts index 3133753ea3..53574aa00b 100644 --- a/src/node/services/bashMonitorWakeReconciler.test.ts +++ b/src/node/services/bashMonitorWakeReconciler.test.ts @@ -158,6 +158,34 @@ describe("BashMonitorWakeReconciler", () => { expect(dispatches[1].prompt).toContain("READY again"); }); + test("a full-history clear retires a wake already in the owner's hands", async () => { + live = [liveSnapshot()]; + await reconciler.reconcile(OWNER); + expect(dispatches).toHaveLength(1); + expect(dispatches[0].isCurrent()).toBe(true); + + // The owner has not sent yet (it is waiting on its own history lock); the clear + // consumes the signals, so the receiver must find the wake stale and drop it. + const token = await reconciler.beginFullHistoryClear(OWNER); + expect(dispatches[0].isCurrent()).toBe(false); + expect(acknowledged).toEqual([{ processId: "proc", matchedThroughOffset: 12 }]); + await reconciler.finishFullHistoryClear(token); + + // Deferring the stale wake is a no-op, and the level stays low. + await dispatches[0].onDeferred(); + expect(await reconciler.hasOutstandingWake(OWNER)).toBe(false); + }); + + test("disposal lowers the published level and retires the in-flight wake", async () => { + live = [liveSnapshot()]; + await reconciler.reconcile(OWNER); + expect(dispatches).toHaveLength(1); + + await reconciler.dispose(OWNER); + expect(dispatches[0].isCurrent()).toBe(false); + expect(await reconciler.hasOutstandingWake(OWNER)).toBe(false); + }); + test("keeps dead registry evidence until the queued wake is accepted", async () => { rows = [registryRecord()]; diff --git a/src/node/services/bashMonitorWakeReconciler.ts b/src/node/services/bashMonitorWakeReconciler.ts index 03f22bf554..b6e7e979d9 100644 --- a/src/node/services/bashMonitorWakeReconciler.ts +++ b/src/node/services/bashMonitorWakeReconciler.ts @@ -111,6 +111,13 @@ export interface BashMonitorWakeDispatch { ownerWorkspaceId: string; prompt: string; muxMetadata: Extract; + /** + * False once a full-history clear or disposal retired the signals behind this wake + * while it was in the receiver's hands. The receiver re-checks it after taking its own + * locks (the clear runs under the same history lock) and before sending, so a stale + * prompt is never appended to freshly cleared history. + */ + isCurrent(): boolean; onAccepted(): Promise; onDeferred(): Promise; } @@ -455,6 +462,7 @@ export class BashMonitorWakeReconciler { this.states.delete(ownerWorkspaceId); return Promise.resolve(); }); + this.args.onOutstandingChanged?.(ownerWorkspaceId, false); } revive(ownerWorkspaceId: string): void { @@ -526,6 +534,7 @@ export class BashMonitorWakeReconciler { ownerWorkspaceId, prompt: buildPrompt(dispatch.signals), muxMetadata: buildMetadata(dispatch.signals), + isCurrent: () => this.states.get(ownerWorkspaceId)?.dispatch === dispatch, onAccepted: async () => this.accept(ownerWorkspaceId, dispatch), onDeferred: async () => this.defer(ownerWorkspaceId, dispatch), }); @@ -560,6 +569,9 @@ export class BashMonitorWakeReconciler { private async consumeCurrent(ownerWorkspaceId: string): Promise { await this.locks.withLock(ownerWorkspaceId, async () => { + // A wake already handed to the owner describes signals this consume retires; + // forgetting it flips its isCurrent() so the owner drops it instead of sending. + this.state(ownerWorkspaceId).dispatch = undefined; const collected = await this.collect(ownerWorkspaceId, false); const consumed = [...collected.signals, ...collected.autoConsumed]; await this.advanceWatermarks(ownerWorkspaceId, collected.watermarks, consumed); diff --git a/src/node/services/workspaceService.test.ts b/src/node/services/workspaceService.test.ts index 4e03176222..bc9d75d56d 100644 --- a/src/node/services/workspaceService.test.ts +++ b/src/node/services/workspaceService.test.ts @@ -739,6 +739,7 @@ describe("WorkspaceService bash monitor wake reconciler wiring", () => { ownerWorkspaceId: string; prompt: string; muxMetadata: { type: "bash-monitor-wake"; records: [] }; + isCurrent(): boolean; onAccepted(): Promise; onDeferred(): Promise; }): Promise<"in-flight" | "deferred">; @@ -750,6 +751,7 @@ describe("WorkspaceService bash monitor wake reconciler wiring", () => { ownerWorkspaceId: workspaceId, prompt: "wake", muxMetadata: { type: "bash-monitor-wake", records: [] }, + isCurrent: () => true, onAccepted, onDeferred, }); @@ -783,6 +785,7 @@ describe("WorkspaceService bash monitor wake reconciler wiring", () => { ownerWorkspaceId: string; prompt: string; muxMetadata: { type: "bash-monitor-wake"; records: [] }; + isCurrent(): boolean; onAccepted(): Promise; onDeferred(): Promise; }): Promise<"in-flight" | "deferred">; @@ -798,6 +801,7 @@ describe("WorkspaceService bash monitor wake reconciler wiring", () => { ownerWorkspaceId: workspaceId, prompt: "wake", muxMetadata: { type: "bash-monitor-wake", records: [] }, + isCurrent: () => true, onAccepted: () => Promise.resolve(), onDeferred: () => Promise.resolve(), }); @@ -834,6 +838,7 @@ describe("WorkspaceService bash monitor wake reconciler wiring", () => { ownerWorkspaceId: string; prompt: string; muxMetadata: { type: "bash-monitor-wake"; records: [] }; + isCurrent(): boolean; onAccepted(): Promise; onDeferred(): Promise; }): Promise<"in-flight" | "deferred">; @@ -849,6 +854,7 @@ describe("WorkspaceService bash monitor wake reconciler wiring", () => { ownerWorkspaceId: workspaceId, prompt: "wake", muxMetadata: { type: "bash-monitor-wake", records: [] }, + isCurrent: () => true, onAccepted: () => Promise.resolve(), onDeferred: () => Promise.resolve(), }); @@ -894,6 +900,7 @@ describe("WorkspaceService bash monitor wake reconciler wiring", () => { ownerWorkspaceId: string; prompt: string; muxMetadata: { type: "bash-monitor-wake"; records: [] }; + isCurrent(): boolean; onAccepted(): Promise; onDeferred(): Promise; }): Promise<"in-flight" | "deferred">; @@ -909,6 +916,7 @@ describe("WorkspaceService bash monitor wake reconciler wiring", () => { ownerWorkspaceId: workspaceId, prompt: "wake", muxMetadata: { type: "bash-monitor-wake", records: [] }, + isCurrent: () => true, onAccepted, onDeferred: () => Promise.resolve(), }); @@ -923,6 +931,74 @@ describe("WorkspaceService bash monitor wake reconciler wiring", () => { } }); + test("a wake retired by a history clear while waiting for the lock is dropped, not sent", async () => { + const { config, service, cleanup } = await createWakeWiringService(); + const workspaceId = "stale-wake-owner"; + await config.addWorkspace("/tmp/stale-wake-project", { + id: workspaceId, + name: workspaceId, + projectName: "stale-wake-project", + projectPath: "/tmp/stale-wake-project", + runtimeConfig: { type: "local" }, + }); + const sendMessage = mock(() => Promise.resolve(Ok(undefined))); + const internal = service as unknown as { + aiService: { isStreaming(workspaceId: string): boolean }; + hasPendingQueuedOrPreparingTurn(workspaceId: string): boolean; + isBusyForMessage(workspaceId: string): boolean; + getDelegatedTurnContinuationSendOptions(workspaceId: string): Promise; + sendMessage: typeof sendMessage; + dispatchBashMonitorWake(dispatch: { + ownerWorkspaceId: string; + prompt: string; + muxMetadata: { type: "bash-monitor-wake"; records: [] }; + isCurrent(): boolean; + onAccepted(): Promise; + onDeferred(): Promise; + }): Promise<"in-flight" | "deferred">; + }; + try { + internal.aiService = { isStreaming: () => false }; + internal.hasPendingQueuedOrPreparingTurn = () => false; + internal.isBusyForMessage = () => false; + internal.getDelegatedTurnContinuationSendOptions = () => Promise.resolve({}); + internal.sendMessage = sendMessage; + + const outcome = await internal.dispatchBashMonitorWake({ + ownerWorkspaceId: workspaceId, + prompt: "wake", + muxMetadata: { type: "bash-monitor-wake", records: [] }, + isCurrent: () => false, + onAccepted: () => Promise.resolve(), + onDeferred: () => Promise.resolve(), + }); + + expect(outcome).toBe("deferred"); + expect(sendMessage).not.toHaveBeenCalled(); + } finally { + await cleanup(); + } + }); + + test("a wake turn in PREPARING keeps the outstanding level visible to turn settlement", async () => { + const { service, cleanup } = await createWakeWiringService(); + const workspaceId = "preparing-wake-owner"; + const session = service.getOrCreateSession(workspaceId); + const sessionInternal = session as unknown as { + hasPendingBashMonitorWakeTurn(): boolean; + }; + try { + expect(await service.hasOutstandingBashMonitorWake(workspaceId)).toBe(false); + // The reconciler level is already low here (onAccepted ran at row persistence); + // the session marker is what keeps the continuation visible until stream start. + sessionInternal.hasPendingBashMonitorWakeTurn = () => true; + expect(await service.hasOutstandingBashMonitorWake(workspaceId)).toBe(true); + } finally { + session.dispose(); + await cleanup(); + } + }); + test("the stream yields on the wake level, not on a queued snapshot of it", async () => { // Regression: a monitored bash matched mid-step while the same step's task_await showed // the matched lines. The old queued tool-end wake still cut the stream (finishReason diff --git a/src/node/services/workspaceService.ts b/src/node/services/workspaceService.ts index 7bc9e7b46a..f6202b6e4a 100644 --- a/src/node/services/workspaceService.ts +++ b/src/node/services/workspaceService.ts @@ -2378,7 +2378,18 @@ export class WorkspaceService extends EventEmitter implements WorkspaceHost { // The level drives the same tool-boundary side effects a queued tool-end message // does: long-polling bash reads return early and foreground agent-task waits are // backgrounded so the stream can reach the boundary where it yields to the wake. - this.sessions.get(ownerWorkspaceId)?.setBashMonitorWakeOutstanding(outstanding); + const session = this.sessions.get(ownerWorkspaceId); + if (session != null) { + session.setBashMonitorWakeOutstanding(outstanding); + } else if ( + !outstanding && + // Partial BackgroundProcessManager stubs in tests (see the constructor guards). + typeof this.backgroundProcessManager.setMessageQueued === "function" + ) { + // No session, no queue: the flag can only be a stale mirror (e.g. reconciler + // disposal after the session went away), so drop it directly. + this.backgroundProcessManager.setMessageQueued(ownerWorkspaceId, false); + } if (outstanding) { this.agentTaskIntegration?.backgroundForegroundWaitsForWorkspace(ownerWorkspaceId); } @@ -2518,6 +2529,9 @@ export class WorkspaceService extends EventEmitter implements WorkspaceHost { ): Promise { return this.bashMonitorHistoryLocks.withLock(dispatch.ownerWorkspaceId, async () => { const ownerWorkspaceId = dispatch.ownerWorkspaceId; + // A full-history clear (same lock) may have retired these signals while this wake + // waited for the lock; sending now would append a stale prompt to cleared history. + if (!dispatch.isCurrent()) return "deferred"; const entry = findWorkspaceEntry(this.config.loadConfigOrDefault(), ownerWorkspaceId); if (entry == null) { await dispatch.onAccepted(); @@ -11835,12 +11849,15 @@ export class WorkspaceService extends EventEmitter implements WorkspaceHost { } /** - * The bash-monitor wake level: a wake the workspace has not seen yet. A stream that - * ended with "tool-calls" while this is high yielded to the wake and will be continued - * by it (BashMonitorWakeReconciler.hasOutstandingWake). + * The bash-monitor wake level: a wake the workspace has not seen yet, or a wake turn + * admitted but not yet streaming (AgentSession.hasPendingBashMonitorWakeTurn — the + * reconciler level is already consumed there). A stream that ended with "tool-calls" + * while this is high yielded to the wake and will be continued by it. */ - hasOutstandingBashMonitorWake(workspaceId: string): Promise { - return this.bashMonitorWakeReconciler.hasOutstandingWake(workspaceId.trim()); + async hasOutstandingBashMonitorWake(workspaceId: string): Promise { + const id = workspaceId.trim(); + if (this.sessions.get(id)?.hasPendingBashMonitorWakeTurn() === true) return true; + return this.bashMonitorWakeReconciler.hasOutstandingWake(id); } /** From a5944da47485320c93acd7959113fc08a1022316 Mon Sep 17 00:00:00 2001 From: Ethan Dickson Date: Thu, 3 Sep 2026 19:36:21 +0000 Subject: [PATCH 03/26] fix: retire a handed-off wake when its monitor is canceled; revalidate before send Codex security P2 on #4071: monitor cancellation cleared the level but a dispatch already in the receiver's hands was still sent. discardProcess (pull-side, no abort signal) forgets such a dispatch so isCurrent() turns false, and dispatchBashMonitorWake re-checks isCurrent() after awaiting the continuation send options, right before sendMessage. --- .../bashMonitorWakeReconciler.test.ts | 24 +++++++ .../services/bashMonitorWakeReconciler.ts | 24 +++++++ src/node/services/workspaceService.test.ts | 68 ++++++++++++++++++- src/node/services/workspaceService.ts | 8 +++ 4 files changed, 122 insertions(+), 2 deletions(-) diff --git a/src/node/services/bashMonitorWakeReconciler.test.ts b/src/node/services/bashMonitorWakeReconciler.test.ts index 53574aa00b..4ab3411c73 100644 --- a/src/node/services/bashMonitorWakeReconciler.test.ts +++ b/src/node/services/bashMonitorWakeReconciler.test.ts @@ -176,6 +176,30 @@ describe("BashMonitorWakeReconciler", () => { expect(await reconciler.hasOutstandingWake(OWNER)).toBe(false); }); + test("canceling a monitor retires a wake carrying its output and re-derives the rest", async () => { + live = [liveSnapshot()]; + rows = [registryRecord()]; + await reconciler.reconcile(OWNER); + expect(dispatches).toHaveLength(1); + expect(dispatches[0].isCurrent()).toBe(true); + + // The wake combines the live match and the dead registry row. Canceling the live + // monitor must retire the whole handed-out wake (its prompt embeds that output)... + await reconciler.discardProcess(OWNER, "proc", CREATED_AT); + expect(dispatches[0].isCurrent()).toBe(false); + + // ...and the unrelated dead-process signal comes back on its own in a fresh wake. + live = []; + await reconciler.reconcile(OWNER); + expect(dispatches).toHaveLength(2); + expect(dispatches[1].prompt).not.toContain("READY"); + expect(dispatches[1].isCurrent()).toBe(true); + + // Canceling an unrelated process leaves the current wake alone. + await reconciler.discardProcess(OWNER, "someone-else", CREATED_AT); + expect(dispatches[1].isCurrent()).toBe(true); + }); + test("disposal lowers the published level and retires the in-flight wake", async () => { live = [liveSnapshot()]; await reconciler.reconcile(OWNER); diff --git a/src/node/services/bashMonitorWakeReconciler.ts b/src/node/services/bashMonitorWakeReconciler.ts index b6e7e979d9..7aacd6d4b8 100644 --- a/src/node/services/bashMonitorWakeReconciler.ts +++ b/src/node/services/bashMonitorWakeReconciler.ts @@ -446,6 +446,30 @@ export class BashMonitorWakeReconciler { return snapshot.pendingWakeKinds.get(processId); } + /** + * The operator canceled a monitor: a wake already handed to the owner that carries this + * process's output must not be sent (its isCurrent() turns false). Its other signals, if + * any, re-derive on the next reconcile. + */ + async discardProcess( + ownerWorkspaceId: string, + processId: string, + createdAt: string + ): Promise { + await this.locks.withLock(ownerWorkspaceId, () => { + const state = this.states.get(ownerWorkspaceId); + if ( + state?.dispatch?.signals.some( + (signal) => signal.processId === processId && signal.createdAt === createdAt + ) === true + ) { + state.dispatch = undefined; + } + return Promise.resolve(); + }); + this.scheduleReconcile(ownerWorkspaceId); + } + async beginFullHistoryClear(ownerWorkspaceId: string): Promise { await this.consumeCurrent(ownerWorkspaceId); return { ownerWorkspaceId }; diff --git a/src/node/services/workspaceService.test.ts b/src/node/services/workspaceService.test.ts index bc9d75d56d..ddb1c38f0e 100644 --- a/src/node/services/workspaceService.test.ts +++ b/src/node/services/workspaceService.test.ts @@ -256,6 +256,7 @@ describe("WorkspaceService bash monitor wake reconciler wiring", () => { test("monitor lifecycle and shown-output events poke the reconciler", async () => { const { service, events, cleanup } = await createWakeWiringService(); const scheduleReconcile = mock(() => undefined); + const discardProcess = mock(() => Promise.resolve()); const upsert = mock(() => Promise.resolve()); const remove = mock(() => Promise.resolve()); const recordTerminal = mock(() => Promise.resolve()); @@ -263,6 +264,7 @@ describe("WorkspaceService bash monitor wake reconciler wiring", () => { bashMonitorRecoveryPromise: Promise; bashMonitorWakeReconciler: { scheduleReconcile: typeof scheduleReconcile; + discardProcess: typeof discardProcess; }; bashMonitorRegistryStore: { upsert: typeof upsert; @@ -272,7 +274,7 @@ describe("WorkspaceService bash monitor wake reconciler wiring", () => { }; try { await internal.bashMonitorRecoveryPromise; - internal.bashMonitorWakeReconciler = { scheduleReconcile }; + internal.bashMonitorWakeReconciler = { scheduleReconcile, discardProcess }; internal.bashMonitorRegistryStore = { upsert, remove, recordTerminal }; const armed = { processId: "proc", @@ -296,6 +298,9 @@ describe("WorkspaceService bash monitor wake reconciler wiring", () => { } expect(upsert).toHaveBeenCalledWith(armed); + // Cancellation retires any wake already handed out for this process before the + // registry row goes, so its captured output is never sent as a turn. + expect(discardProcess).toHaveBeenCalledWith("owner", "proc", armed.createdAt); expect(remove).toHaveBeenCalledWith("owner", "proc", armed.createdAt); expect(scheduleReconcile).toHaveBeenCalledTimes(4); } finally { @@ -510,6 +515,7 @@ describe("WorkspaceService bash monitor wake reconciler wiring", () => { bashMonitorRecoveryPromise: Promise; bashMonitorWakeReconciler: { scheduleReconcile: typeof scheduleReconcile; + discardProcess(workspaceId: string, processId: string, createdAt: string): Promise; }; bashMonitorRegistryStore: { upsert: typeof upsert; @@ -520,7 +526,10 @@ describe("WorkspaceService bash monitor wake reconciler wiring", () => { }; try { await internal.bashMonitorRecoveryPromise; - internal.bashMonitorWakeReconciler = { scheduleReconcile }; + internal.bashMonitorWakeReconciler = { + scheduleReconcile, + discardProcess: () => Promise.resolve(), + }; internal.bashMonitorRegistryStore = { upsert, remove, @@ -931,6 +940,61 @@ describe("WorkspaceService bash monitor wake reconciler wiring", () => { } }); + test("a monitor canceled while send options resolve retires the wake before it is sent", async () => { + const { config, service, cleanup } = await createWakeWiringService(); + const workspaceId = "canceled-mid-dispatch-wake-owner"; + await config.addWorkspace("/tmp/canceled-mid-dispatch-project", { + id: workspaceId, + name: workspaceId, + projectName: "canceled-mid-dispatch-project", + projectPath: "/tmp/canceled-mid-dispatch-project", + runtimeConfig: { type: "local" }, + }); + const sendMessage = mock(() => Promise.resolve(Ok(undefined))); + let current = true; + const internal = service as unknown as { + aiService: { isStreaming(workspaceId: string): boolean }; + hasPendingQueuedOrPreparingTurn(workspaceId: string): boolean; + isBusyForMessage(workspaceId: string): boolean; + getDelegatedTurnContinuationSendOptions(workspaceId: string): Promise; + sendMessage: typeof sendMessage; + dispatchBashMonitorWake(dispatch: { + ownerWorkspaceId: string; + prompt: string; + muxMetadata: { type: "bash-monitor-wake"; records: [] }; + isCurrent(): boolean; + onAccepted(): Promise; + onDeferred(): Promise; + }): Promise<"in-flight" | "deferred">; + }; + try { + internal.aiService = { isStreaming: () => false }; + internal.hasPendingQueuedOrPreparingTurn = () => false; + internal.isBusyForMessage = () => false; + // The operator cancels the monitor (discardProcess) while the continuation options + // are being resolved: the wake passed the entry check but must not reach sendMessage. + internal.getDelegatedTurnContinuationSendOptions = () => { + current = false; + return Promise.resolve({}); + }; + internal.sendMessage = sendMessage; + + const outcome = await internal.dispatchBashMonitorWake({ + ownerWorkspaceId: workspaceId, + prompt: "wake", + muxMetadata: { type: "bash-monitor-wake", records: [] }, + isCurrent: () => current, + onAccepted: () => Promise.resolve(), + onDeferred: () => Promise.resolve(), + }); + + expect(outcome).toBe("deferred"); + expect(sendMessage).not.toHaveBeenCalled(); + } finally { + await cleanup(); + } + }); + test("a wake retired by a history clear while waiting for the lock is dropped, not sent", async () => { const { config, service, cleanup } = await createWakeWiringService(); const workspaceId = "stale-wake-owner"; diff --git a/src/node/services/workspaceService.ts b/src/node/services/workspaceService.ts index f6202b6e4a..20d8151767 100644 --- a/src/node/services/workspaceService.ts +++ b/src/node/services/workspaceService.ts @@ -1924,6 +1924,11 @@ export class WorkspaceService extends EventEmitter implements WorkspaceHost { const persist = async (): Promise => { if (payload.reason === "canceled") { if (createdAt == null) return false; + await this.bashMonitorWakeReconciler.discardProcess( + workspaceId, + payload.processId, + createdAt + ); await this.bashMonitorRegistryStore.remove(workspaceId, payload.processId, createdAt); return true; } @@ -2556,6 +2561,9 @@ export class WorkspaceService extends EventEmitter implements WorkspaceHost { log.debug("Bash monitor wake has no send options; leaving pending", { ownerWorkspaceId }); return "deferred"; } + // Re-checked after the awaits above: a monitor canceled meanwhile (discardProcess) must + // not have its captured stdout submitted as an agent-initiated turn. + if (!dispatch.isCurrent()) return "deferred"; let accepted = false; const sendResult = await this.sendMessage( From 6b64c5f31b0438ee9d4276c5c85e9055aee87821 Mon Sep 17 00:00:00 2001 From: Ethan Dickson Date: Thu, 3 Sep 2026 20:06:53 +0000 Subject: [PATCH 04/26] Codex round 2: queue-head arbitration, never-queue wake send, wake-cut latch - hasPendingToolEndInput/yield flag: a non-empty queue arbitrates alone; the wake level only cuts over an empty queue (turn-end head is not promoted). - dispatchBashMonitorWake sends with requireIdle + admissionStale=!isCurrent(), so a racing manual send makes the wake a skip (re-armed after idle) instead of a queued entry the level can no longer retract. - AgentSession records a wake-caused cut (streamYieldedToBashMonitorWake) until the next admitted turn so delegated-turn settlement sees the cut even if the monitor is canceled before the stream-end handler runs. --- .../agentSession.queueDispatch.test.ts | 87 +++++++++++++++- src/node/services/agentSession.ts | 36 +++++-- src/node/services/workspaceService.test.ts | 98 +++++++++++++++++-- src/node/services/workspaceService.ts | 16 +-- 4 files changed, 210 insertions(+), 27 deletions(-) diff --git a/src/node/services/agentSession.queueDispatch.test.ts b/src/node/services/agentSession.queueDispatch.test.ts index bcc30642a3..04996e8740 100644 --- a/src/node/services/agentSession.queueDispatch.test.ts +++ b/src/node/services/agentSession.queueDispatch.test.ts @@ -645,8 +645,9 @@ describe("AgentSession queued message tool-call dispatch", () => { level = () => Promise.reject(new Error("watermark read failed")); expect(await session.hasPendingToolEndInput()).toBe(false); - // Queue head decides independently of the level, and only for tool-end. - level = () => Promise.resolve(false); + // A non-empty queue arbitrates alone: a turn-end head is not promoted to tool-end by + // a high wake level (the wake dispatcher waits for the queue to drain anyway). + level = () => Promise.resolve(true); session.queueMessage("later", { model: TEST_MODEL, agentId: "exec", @@ -654,6 +655,8 @@ describe("AgentSession queued message tool-call dispatch", () => { }); expect(await session.hasPendingToolEndInput()).toBe(false); session.clearQueue(); + expect(await session.hasPendingToolEndInput()).toBe(true); + level = () => Promise.resolve(false); session.queueMessage("now", { model: TEST_MODEL, agentId: "exec" }); expect(await session.hasPendingToolEndInput()).toBe(true); } finally { @@ -662,6 +665,73 @@ describe("AgentSession queued message tool-call dispatch", () => { } }); + test("a stream cut for the wake level stays a pending wake turn until a turn is admitted", async () => { + // Settlement of a delegated turn runs after the cut; an operator canceling the monitor in + // between lowers the level, so the cut itself must remain the evidence. + const workspaceId = "queue-dispatch-wake-cut-latch"; + let level = false; + let markStreamRequested: () => void = () => undefined; + const streamRequested = new Promise((resolve) => { + markStreamRequested = resolve; + }); + let releaseStream: () => void = () => undefined; + const streamRelease = new Promise((resolve) => { + releaseStream = resolve; + }); + const { session, cleanup } = await createAgentSessionHarness({ + workspaceId, + hasOutstandingBashMonitorWake: () => Promise.resolve(level), + aiServiceOverrides: { + streamMessage: mock(async () => { + markStreamRequested(); + await streamRelease; + return Ok(createStartedTurnHandle("test-assistant-message")); + }), + }, + }); + let disposed = false; + try { + level = true; + expect(await session.hasPendingToolEndInput()).toBe(true); + level = false; + expect(session.hasPendingBashMonitorWakeTurn()).toBe(true); + // Reading a low level later does not retract the recorded cut. + expect(await session.hasPendingToolEndInput()).toBe(false); + expect(session.hasPendingBashMonitorWakeTurn()).toBe(true); + + // Whatever turn is admitted next settles the cut — here a manual send, which is not + // itself a wake turn. + const sendPromise = session.sendMessage("hello", { model: TEST_MODEL, agentId: "exec" }); + await streamRequested; + expect(session.isBusy()).toBe(true); + expect(session.hasPendingBashMonitorWakeTurn()).toBe(false); + session.dispose(); + disposed = true; + releaseStream(); + await sendPromise; + } finally { + releaseStream(); + if (!disposed) session.dispose(); + await cleanup(); + } + }); + + test("a queued tool-end head is not recorded as a wake cut", async () => { + const workspaceId = "queue-dispatch-queue-cut-not-wake"; + const { session, cleanup } = await createAgentSessionHarness({ + workspaceId, + hasOutstandingBashMonitorWake: () => Promise.resolve(true), + }); + try { + session.queueMessage("now", { model: TEST_MODEL, agentId: "exec" }); + expect(await session.hasPendingToolEndInput()).toBe(true); + expect(session.hasPendingBashMonitorWakeTurn()).toBe(false); + } finally { + session.dispose(); + await cleanup(); + } + }); + test("the wake level and the queue head jointly drive the bash early-return flag", async () => { const workspaceId = "queue-dispatch-yield-flag"; const flags: boolean[] = []; @@ -688,6 +758,19 @@ describe("AgentSession queued message tool-call dispatch", () => { expect(lastFlag()).toBe(true); session.setBashMonitorWakeOutstanding(false); expect(lastFlag()).toBe(false); + + // A turn-end head owns the next dispatch: the level must not pull the early-return + // lever for it (mirrors hasPendingToolEndInput's arbitration). + session.setBashMonitorWakeOutstanding(true); + session.queueMessage("later", { + model: TEST_MODEL, + agentId: "exec", + queueDispatchMode: "turn-end", + }); + expect(lastFlag()).toBe(false); + session.clearQueue(); + expect(lastFlag()).toBe(true); + session.setBashMonitorWakeOutstanding(false); } finally { session.dispose(); await cleanup(); diff --git a/src/node/services/agentSession.ts b/src/node/services/agentSession.ts index 6523b91c40..b38edee309 100644 --- a/src/node/services/agentSession.ts +++ b/src/node/services/agentSession.ts @@ -871,6 +871,16 @@ export class AgentSession { * this marker is what keeps the wake continuation visible across that window. */ private preparingBashMonitorWake = false; + /** + * The last stream cut itself for the wake level (hasPendingToolEndInput returned true + * from the level) and no turn has been admitted since. Delegated-turn settlement needs the + * cut's cause, not the live level: an operator canceling the monitor between the cut and + * the stream-end handler lowers the level, and the "tool-calls" end would otherwise read as + * ended-before-completion instead of the same deferral a still-high level produces. A wake + * that never arrives is recovered by the turn manager's stale-deferred path (Codex P2 + * PRRT_kwDOPxxmWM6fDmpR). + */ + private streamYieldedToBashMonitorWake = false; /** Context needed to retry the current stream (cleared on stream end/abort/error). */ private activeStreamContext?: { @@ -5718,6 +5728,8 @@ export class AgentSession { this.dispatchingQueuedEntryMuxMetadata = undefined; this.preparingWorkspaceTurnMetadata = undefined; this.preparingBashMonitorWake = false; + // A live stream is the continuation (or a superseding turn) the cut waited for. + this.streamYieldedToBashMonitorWake = false; this.activeStreamStartedAtMs = payload.startTime; // Codex P1 (PRRT_kwDOPxxmWM6cClKS): a new live stream makes mid-stream // setGoal deferral meaningful again — clear the goal service's settled @@ -6556,12 +6568,20 @@ export class AgentSession { * tool-end message is an edge we hold; a bash-monitor wake is a level read from the * reconciler, so a wake whose lines this very step showed (task_await on the monitored * process) is already gone by the time the SDK asks — the stream keeps going. + * + * A non-empty queue arbitrates alone: its head runs next whatever the level says (the + * wake dispatcher waits for an empty queue), so cutting for the wake behind a turn-end + * head would only promote that entry to tool-end (Codex P2 PRRT_kwDOPxxmWM6fDmpV). */ async hasPendingToolEndInput(): Promise { - if (this.hasQueuedMessages("tool-end")) return true; + const nextMode = this.messageQueue.getNextDispatchableMode(); + if (nextMode != null) return nextMode === "tool-end"; if (this.hasOutstandingBashMonitorWake == null) return false; try { - return await this.hasOutstandingBashMonitorWake(); + const outstanding = await this.hasOutstandingBashMonitorWake(); + // The SDK only asks when the loop would otherwise continue, so a true here IS the cut. + if (outstanding) this.streamYieldedToBashMonitorWake = true; + return outstanding; } catch (error) { log.debug("hasPendingToolEndInput: wake level read failed; not yielding", { workspaceId: this.workspaceId, @@ -6582,16 +6602,16 @@ export class AgentSession { } /** - * Tool-end yield flag = queue head is tool-end ∪ wake level. `queueHeadToolEnd` lets - * stream-end / clear paths assert the queue contribution is gone before the queue itself - * is observed empty. + * Tool-end yield flag = queue head is tool-end ∪ (queue empty ∧ wake level), mirroring + * hasPendingToolEndInput's arbitration. `queueHeadToolEnd` lets stream-end / clear paths + * assert the queue contribution is gone before the queue itself is observed empty. */ private syncToolEndYieldRequested( queueHeadToolEnd = this.messageQueue.getNextDispatchableMode() === "tool-end" ): void { this.backgroundProcessManager.setMessageQueued( this.workspaceId, - queueHeadToolEnd || this.bashMonitorWakeOutstanding + queueHeadToolEnd || (this.bashMonitorWakeOutstanding && this.messageQueue.isEmpty()) ); } @@ -6642,6 +6662,8 @@ export class AgentSession { this.preparingWorkspaceTurnMetadata = getWorkspaceTurnMuxMetadata(muxMetadata); this.preparingBashMonitorWake = (muxMetadata as MuxMessageMetadata | undefined)?.type === "bash-monitor-wake"; + // Whatever is admitted now (the wake turn, or input superseding it) settles the cut. + this.streamYieldedToBashMonitorWake = false; this.setTurnPhase(TurnPhase.PREPARING); } @@ -6652,7 +6674,7 @@ export class AgentSession { * so delegated-turn settlement must still see the continuation. */ hasPendingBashMonitorWakeTurn(): boolean { - if (this.preparingBashMonitorWake) return true; + if (this.preparingBashMonitorWake || this.streamYieldedToBashMonitorWake) return true; const dispatching = this.dispatchingQueuedEntryMuxMetadata as MuxMessageMetadata | undefined; return dispatching?.type === "bash-monitor-wake"; } diff --git a/src/node/services/workspaceService.test.ts b/src/node/services/workspaceService.test.ts index ddb1c38f0e..46dc7ecff3 100644 --- a/src/node/services/workspaceService.test.ts +++ b/src/node/services/workspaceService.test.ts @@ -887,18 +887,28 @@ describe("WorkspaceService bash monitor wake reconciler wiring", () => { runtimeConfig: { type: "local" }, }); let sentOptions: { queueDispatchMode?: string; muxMetadata?: unknown } | undefined; + let sentInternal: + | { requireIdle?: boolean; admissionStale?: () => boolean; onCanceled?: unknown } + | undefined; const sendMessage = mock( ( _workspaceId: string, _prompt: string, options: { queueDispatchMode?: string; muxMetadata?: unknown }, - internal?: { onAccepted?: () => Promise } + internal?: { + onAccepted?: () => Promise; + requireIdle?: boolean; + admissionStale?: () => boolean; + onCanceled?: unknown; + } ) => { sentOptions = options; + sentInternal = internal; return internal?.onAccepted?.().then(() => Ok(undefined)) ?? Promise.resolve(Ok(undefined)); } ); const onAccepted = mock(() => Promise.resolve()); + let current = true; const internal = service as unknown as { aiService: { isStreaming(workspaceId: string): boolean }; hasPendingQueuedOrPreparingTurn(workspaceId: string): boolean; @@ -925,7 +935,7 @@ describe("WorkspaceService bash monitor wake reconciler wiring", () => { ownerWorkspaceId: workspaceId, prompt: "wake", muxMetadata: { type: "bash-monitor-wake", records: [] }, - isCurrent: () => true, + isCurrent: () => current, onAccepted, onDeferred: () => Promise.resolve(), }); @@ -935,6 +945,14 @@ describe("WorkspaceService bash monitor wake reconciler wiring", () => { expect(sentOptions?.queueDispatchMode).toBeUndefined(); expect(sentOptions?.muxMetadata).toEqual({ type: "bash-monitor-wake", records: [] }); expect(onAccepted).toHaveBeenCalledTimes(1); + // The wake is never queued (a racing manual send makes it a skip instead), and the + // admission probe tracks the wake's validity through every pre-durability gate so a + // monitor canceled mid-admission refuses the send. + expect(sentInternal?.requireIdle).toBe(true); + expect(sentInternal?.onCanceled).toBeUndefined(); + expect(sentInternal?.admissionStale?.()).toBe(false); + current = false; + expect(sentInternal?.admissionStale?.()).toBe(true); } finally { await cleanup(); } @@ -1044,6 +1062,63 @@ describe("WorkspaceService bash monitor wake reconciler wiring", () => { } }); + test("a wake skipped by a manual send that won the race re-arms instead of queuing", async () => { + const { config, service, cleanup } = await createWakeWiringService(); + const workspaceId = "skipped-wake-owner"; + await config.addWorkspace("/tmp/skipped-wake-project", { + id: workspaceId, + name: workspaceId, + projectName: "skipped-wake-project", + projectPath: "/tmp/skipped-wake-project", + runtimeConfig: { type: "local" }, + }); + // requireIdle skip: the send never queued and onAccepted never fired. + const sendMessage = mock(() => + Promise.resolve( + Err({ type: "unknown", raw: "Workspace is busy; idle-only send was skipped." }) + ) + ); + const afterIdle = mock(() => undefined); + const internal = service as unknown as { + aiService: { isStreaming(workspaceId: string): boolean }; + hasPendingQueuedOrPreparingTurn(workspaceId: string): boolean; + isBusyForMessage(workspaceId: string): boolean; + getDelegatedTurnContinuationSendOptions(workspaceId: string): Promise; + scheduleBashMonitorWakeReconcileAfterIdle(workspaceId: string): void; + sendMessage: typeof sendMessage; + dispatchBashMonitorWake(dispatch: { + ownerWorkspaceId: string; + prompt: string; + muxMetadata: { type: "bash-monitor-wake"; records: [] }; + isCurrent(): boolean; + onAccepted(): Promise; + onDeferred(): Promise; + }): Promise<"in-flight" | "deferred">; + }; + try { + internal.aiService = { isStreaming: () => false }; + internal.hasPendingQueuedOrPreparingTurn = () => false; + internal.isBusyForMessage = () => false; + internal.getDelegatedTurnContinuationSendOptions = () => Promise.resolve({}); + internal.scheduleBashMonitorWakeReconcileAfterIdle = afterIdle; + internal.sendMessage = sendMessage; + + const outcome = await internal.dispatchBashMonitorWake({ + ownerWorkspaceId: workspaceId, + prompt: "wake", + muxMetadata: { type: "bash-monitor-wake", records: [] }, + isCurrent: () => true, + onAccepted: () => Promise.resolve(), + onDeferred: () => Promise.resolve(), + }); + + expect(outcome).toBe("deferred"); + expect(afterIdle).toHaveBeenCalledTimes(1); + } finally { + await cleanup(); + } + }); + test("a wake turn in PREPARING keeps the outstanding level visible to turn settlement", async () => { const { service, cleanup } = await createWakeWiringService(); const workspaceId = "preparing-wake-owner"; @@ -1095,21 +1170,16 @@ describe("WorkspaceService bash monitor wake reconciler wiring", () => { }, ]) ); - let shownThroughOffset = 0; + let shownThroughOffset = 12; processManager.getMonitorWakeDeliveryState.mockImplementation(() => Promise.resolve({ status: "settled", shownThroughOffset, terminalStatusShown: false }) ); - // Match not yet shown: the boundary yields and bash long-polls return early. - expect(await session.hasPendingToolEndInput()).toBe(true); - expect(processManager.setMessageQueued).toHaveBeenLastCalledWith(workspaceId, true); - expect(session.hasQueuedMessages()).toBe(false); - // task_await showed the lines before the SDK asked: no yield, no wake turn. - shownThroughOffset = 12; expect(await session.hasPendingToolEndInput()).toBe(false); - expect(processManager.setMessageQueued).toHaveBeenLastCalledWith(workspaceId, false); + expect(processManager.setMessageQueued).not.toHaveBeenCalledWith(workspaceId, true); expect(await service.hasOutstandingBashMonitorWake(workspaceId)).toBe(false); + expect(session.hasQueuedMessages()).toBe(false); // A queued tool-end message still yields on its own. session.queueMessage("follow up", { @@ -1118,6 +1188,14 @@ describe("WorkspaceService bash monitor wake reconciler wiring", () => { queueDispatchMode: "tool-end", }); expect(await session.hasPendingToolEndInput()).toBe(true); + session.clearQueue(); + + // Match not shown (a different process, or a filtered read): the boundary yields, + // bash long-polls return early, and the cut stays visible to turn settlement. + shownThroughOffset = 0; + expect(await session.hasPendingToolEndInput()).toBe(true); + expect(processManager.setMessageQueued).toHaveBeenLastCalledWith(workspaceId, true); + expect(await service.hasOutstandingBashMonitorWake(workspaceId)).toBe(true); } finally { session.dispose(); await cleanup(); diff --git a/src/node/services/workspaceService.ts b/src/node/services/workspaceService.ts index 20d8151767..1d54ad9f1b 100644 --- a/src/node/services/workspaceService.ts +++ b/src/node/services/workspaceService.ts @@ -2574,6 +2574,14 @@ export class WorkspaceService extends EventEmitter implements WorkspaceHost { skipAutoResumeReset: true, synthetic: true, agentInitiated: true, + // Never queue: a manual send racing past the idle checks above would otherwise + // park this wake behind it, out of reach of the level (a later monitor cancel + // could not retract it). requireIdle turns that race into a skip (Err), and the + // admission probe re-validates the wake at every gate before the user row is + // durable; both fall through to the after-idle re-arm below (Codex P2 + // PRRT_kwDOPxxmWM6fDmpJ). + requireIdle: true, + admissionStale: () => !dispatch.isCurrent(), onAccepted: async () => { accepted = true; await dispatch.onAccepted(); @@ -2582,14 +2590,6 @@ export class WorkspaceService extends EventEmitter implements WorkspaceHost { onAcceptedPreStreamFailure: async () => { if (accepted) await dispatch.onAccepted(); }, - // Idle owner, so this send does not queue; still, a manual send racing past the - // idle checks above can push it into the queue and clear it before dispatch. - onCanceled: async () => { - if (!accepted) { - await dispatch.onDeferred(); - this.scheduleBashMonitorWakeReconcileAfterIdle(ownerWorkspaceId); - } - }, } ); if (!sendResult.success && !accepted) { From 48afcd8a463a6f54e7129ae9921c4abeda627ce0 Mon Sep 17 00:00:00 2001 From: Ethan Dickson Date: Thu, 3 Sep 2026 20:55:10 +0000 Subject: [PATCH 05/26] Codex round 3: shown-frontier invalidation, post-await queue recheck, wake cut as cutter stage, probe try/catch - Reconciler.outputShown(owner, processId) forgets a handed-out dispatch whose signal covers that process, so a wake still resolving send options while a manual turn showed its lines fails isCurrent()/admissionStale instead of admitting stale output. - hasPendingToolEndInput re-reads the queue head after the async level read; a message queued meanwhile arbitrates (turn-end head => no cut). - The wake-cut latch is now cut attribution only: getQueueCutCutter reports { stage: "bash-monitor-wake" } and hasPendingBashMonitorWakeTurn no longer consults it. A wake retracted after the cut (monitor canceled) settles the delegated handle as interrupted with a wake-specific supersede reason instead of deferring until the waiter times out. - hasSameTurnContinuation catches wake-probe failures and settles normally. --- .../agentSession.queueDispatch.test.ts | 44 ++++++++++--- src/node/services/agentSession.ts | 24 ++++--- .../bashMonitorWakeReconciler.test.ts | 22 +++++++ .../services/bashMonitorWakeReconciler.ts | 31 +++++++-- src/node/services/messageQueue.ts | 7 +- src/node/services/taskService.test.ts | 66 +++++++++++++++++++ src/node/services/workspaceService.test.ts | 8 ++- src/node/services/workspaceService.ts | 13 +++- src/node/services/workspaceTurnManager.ts | 39 +++++++++-- 9 files changed, 220 insertions(+), 34 deletions(-) diff --git a/src/node/services/agentSession.queueDispatch.test.ts b/src/node/services/agentSession.queueDispatch.test.ts index 04996e8740..e8adb0be04 100644 --- a/src/node/services/agentSession.queueDispatch.test.ts +++ b/src/node/services/agentSession.queueDispatch.test.ts @@ -665,9 +665,11 @@ describe("AgentSession queued message tool-call dispatch", () => { } }); - test("a stream cut for the wake level stays a pending wake turn until a turn is admitted", async () => { + test("a stream cut for the wake level is attributed as the cutter until a turn is admitted", async () => { // Settlement of a delegated turn runs after the cut; an operator canceling the monitor in - // between lowers the level, so the cut itself must remain the evidence. + // between lowers the level, so the cut itself must name its cause. It is attribution + // only: a continuation is read live (hasPendingBashMonitorWakeTurn stays false), so a + // retracted wake settles the handle instead of deferring it forever. const workspaceId = "queue-dispatch-wake-cut-latch"; let level = false; let markStreamRequested: () => void = () => undefined; @@ -691,20 +693,22 @@ describe("AgentSession queued message tool-call dispatch", () => { }); let disposed = false; try { + expect(session.getQueueCutCutter()).toBeUndefined(); level = true; expect(await session.hasPendingToolEndInput()).toBe(true); level = false; - expect(session.hasPendingBashMonitorWakeTurn()).toBe(true); + expect(session.getQueueCutCutter()).toEqual({ stage: "bash-monitor-wake" }); + expect(session.hasPendingBashMonitorWakeTurn()).toBe(false); // Reading a low level later does not retract the recorded cut. expect(await session.hasPendingToolEndInput()).toBe(false); - expect(session.hasPendingBashMonitorWakeTurn()).toBe(true); + expect(session.getQueueCutCutter()).toEqual({ stage: "bash-monitor-wake" }); // Whatever turn is admitted next settles the cut — here a manual send, which is not // itself a wake turn. const sendPromise = session.sendMessage("hello", { model: TEST_MODEL, agentId: "exec" }); await streamRequested; expect(session.isBusy()).toBe(true); - expect(session.hasPendingBashMonitorWakeTurn()).toBe(false); + expect(session.getQueueCutCutter()).toEqual({ stage: "preparing", muxMetadata: undefined }); session.dispose(); disposed = true; releaseStream(); @@ -716,16 +720,38 @@ describe("AgentSession queued message tool-call dispatch", () => { } }); - test("a queued tool-end head is not recorded as a wake cut", async () => { + test("a queued head is not recorded as a wake cut", async () => { const workspaceId = "queue-dispatch-queue-cut-not-wake"; + let releaseLevel: () => void = () => undefined; const { session, cleanup } = await createAgentSessionHarness({ workspaceId, - hasOutstandingBashMonitorWake: () => Promise.resolve(true), + hasOutstandingBashMonitorWake: () => + new Promise((resolve) => { + releaseLevel = () => resolve(true); + }), }); try { + // A message queued while the level is being read arbitrates like one queued before: + // a turn-end head means no cut (and no wake attribution), a tool-end head cuts as + // queued input. + const pendingTurnEnd = session.hasPendingToolEndInput(); + session.queueMessage("later", { + model: TEST_MODEL, + agentId: "exec", + queueDispatchMode: "turn-end", + }); + releaseLevel(); + expect(await pendingTurnEnd).toBe(false); + expect(session.getQueueCutCutter()).toMatchObject({ stage: "queued" }); + session.clearQueue(); + + const pendingToolEnd = session.hasPendingToolEndInput(); session.queueMessage("now", { model: TEST_MODEL, agentId: "exec" }); - expect(await session.hasPendingToolEndInput()).toBe(true); - expect(session.hasPendingBashMonitorWakeTurn()).toBe(false); + releaseLevel(); + expect(await pendingToolEnd).toBe(true); + expect(session.getQueueCutCutter()).toMatchObject({ stage: "queued" }); + session.clearQueue(); + expect(session.getQueueCutCutter()).toBeUndefined(); } finally { session.dispose(); await cleanup(); diff --git a/src/node/services/agentSession.ts b/src/node/services/agentSession.ts index b38edee309..2b6a2e71f6 100644 --- a/src/node/services/agentSession.ts +++ b/src/node/services/agentSession.ts @@ -873,12 +873,13 @@ export class AgentSession { private preparingBashMonitorWake = false; /** * The last stream cut itself for the wake level (hasPendingToolEndInput returned true - * from the level) and no turn has been admitted since. Delegated-turn settlement needs the - * cut's cause, not the live level: an operator canceling the monitor between the cut and - * the stream-end handler lowers the level, and the "tool-calls" end would otherwise read as - * ended-before-completion instead of the same deferral a still-high level produces. A wake - * that never arrives is recovered by the turn manager's stale-deferred path (Codex P2 - * PRRT_kwDOPxxmWM6fDmpR). + * from the level) and no turn has been admitted since. This is cut *attribution* + * (getQueueCutCutter), not a continuation marker: whether the wake still arrives is read + * live from the level / the admitted wake turn. An operator canceling the monitor between + * the cut and the stream-end handler lowers the level with no wake turn to follow, so + * settlement must not defer on it (the parent's wait would hang until timeout); the + * attribution lets it settle the "tool-calls" end as a wake cut instead of a truncation + * failure (Codex P2 PRRT_kwDOPxxmWM6fDmpR, PRRT_kwDOPxxmWM6fEQIf). */ private streamYieldedToBashMonitorWake = false; @@ -6579,6 +6580,10 @@ export class AgentSession { if (this.hasOutstandingBashMonitorWake == null) return false; try { const outstanding = await this.hasOutstandingBashMonitorWake(); + // A message queued during the level read arbitrates the same way: the stream-end + // drain would dispatch it whatever the level says (Codex P2 PRRT_kwDOPxxmWM6fEQIk). + const modeAfterRead = this.messageQueue.getNextDispatchableMode(); + if (modeAfterRead != null) return modeAfterRead === "tool-end"; // The SDK only asks when the loop would otherwise continue, so a true here IS the cut. if (outstanding) this.streamYieldedToBashMonitorWake = true; return outstanding; @@ -6674,7 +6679,7 @@ export class AgentSession { * so delegated-turn settlement must still see the continuation. */ hasPendingBashMonitorWakeTurn(): boolean { - if (this.preparingBashMonitorWake || this.streamYieldedToBashMonitorWake) return true; + if (this.preparingBashMonitorWake) return true; const dispatching = this.dispatchingQueuedEntryMuxMetadata as MuxMessageMetadata | undefined; return dispatching?.type === "bash-monitor-wake"; } @@ -6729,7 +6734,10 @@ export class AgentSession { return { stage: "dispatching", muxMetadata: this.dispatchingQueuedEntryMuxMetadata }; } const candidate = this.messageQueue.getNextQueueCutCandidate(); - return candidate != null ? { stage: "queued", ...candidate } : undefined; + if (candidate != null) return { stage: "queued", ...candidate }; + // No input holds the session: the stream itself yielded to the wake level. Settlement + // reads whether the wake still arrives from the level; this only names the cause. + return this.streamYieldedToBashMonitorWake ? { stage: "bash-monitor-wake" } : undefined; } /** Whether a message queued with this dedupe key is still pending (see MessageQueue.addOnce). */ diff --git a/src/node/services/bashMonitorWakeReconciler.test.ts b/src/node/services/bashMonitorWakeReconciler.test.ts index 4ab3411c73..4d2f673b42 100644 --- a/src/node/services/bashMonitorWakeReconciler.test.ts +++ b/src/node/services/bashMonitorWakeReconciler.test.ts @@ -200,6 +200,28 @@ describe("BashMonitorWakeReconciler", () => { expect(dispatches[1].isCurrent()).toBe(true); }); + test("a shown-frontier advance retires a handed-out wake so a stale send is refused", async () => { + // The owner ran a manual turn that task_await-ed the monitored process while this wake + // was still resolving send options: the reconcile that would re-derive it is queued + // behind the hand-off, so the frontier transition must invalidate the dispatch itself. + live = [liveSnapshot()]; + await reconciler.reconcile(OWNER); + expect(dispatches).toHaveLength(1); + expect(dispatches[0].isCurrent()).toBe(true); + + await reconciler.outputShown(OWNER, "someone-else"); + expect(dispatches[0].isCurrent()).toBe(true); + + deliveryState = { status: "settled", shownThroughOffset: 12, terminalStatusShown: false }; + await reconciler.outputShown(OWNER, "proc"); + expect(dispatches[0].isCurrent()).toBe(false); + + // Nothing derives any more: the lines were shown, so no replacement wake is handed out. + await reconciler.reconcile(OWNER); + expect(dispatches).toHaveLength(1); + expect(await reconciler.hasOutstandingWake(OWNER)).toBe(false); + }); + test("disposal lowers the published level and retires the in-flight wake", async () => { live = [liveSnapshot()]; await reconciler.reconcile(OWNER); diff --git a/src/node/services/bashMonitorWakeReconciler.ts b/src/node/services/bashMonitorWakeReconciler.ts index 7aacd6d4b8..927d0122e3 100644 --- a/src/node/services/bashMonitorWakeReconciler.ts +++ b/src/node/services/bashMonitorWakeReconciler.ts @@ -455,16 +455,33 @@ export class BashMonitorWakeReconciler { ownerWorkspaceId: string, processId: string, createdAt: string + ): Promise { + await this.forgetDispatchFor( + ownerWorkspaceId, + (signal) => signal.processId === processId && signal.createdAt === createdAt + ); + } + + /** + * A model-visible read advanced this process's shown frontier (or showed its terminal + * status). A wake already handed to the owner may now describe lines the owner has seen: + * the owner could have run a manual turn and returned idle while the wake was still + * resolving send options, so the reconcile that would re-derive it is queued behind that + * very hand-off. Forget the dispatch so its isCurrent() turns false at every admission + * gate; whatever still derives is re-handed by the reconcile scheduled here (Codex P2 + * PRRT_kwDOPxxmWM6fEQIa). + */ + async outputShown(ownerWorkspaceId: string, processId: string): Promise { + await this.forgetDispatchFor(ownerWorkspaceId, (signal) => signal.processId === processId); + } + + private async forgetDispatchFor( + ownerWorkspaceId: string, + covers: (signal: DerivedSignal) => boolean ): Promise { await this.locks.withLock(ownerWorkspaceId, () => { const state = this.states.get(ownerWorkspaceId); - if ( - state?.dispatch?.signals.some( - (signal) => signal.processId === processId && signal.createdAt === createdAt - ) === true - ) { - state.dispatch = undefined; - } + if (state?.dispatch?.signals.some(covers) === true) state.dispatch = undefined; return Promise.resolve(); }); this.scheduleReconcile(ownerWorkspaceId); diff --git a/src/node/services/messageQueue.ts b/src/node/services/messageQueue.ts index 2af2227b49..d8fde86e13 100644 --- a/src/node/services/messageQueue.ts +++ b/src/node/services/messageQueue.ts @@ -98,12 +98,15 @@ export type QueueDispatchMode = NonNullable { }); }); + test("a wake retracted after the cut settles the handle as a wake cut instead of deferring", async () => { + // The stream yielded to the wake level, then the operator canceled the monitor before + // this stream-end was processed: the level is low and no wake turn was admitted, so no + // continuation will ever arrive. Deferring would hang the owner's wait; the session's + // cut attribution settles it as a wake cut (not a truncation failure). + const { config, parentId, taskService, workspaceMocks } = await startWorkspaceTurnForTest({ + hasOutstandingBashMonitorWake: mock(() => Promise.resolve(false)), + }); + workspaceMocks.getQueueCutCutter.mockImplementation(() => ({ + stage: "bash-monitor-wake" as const, + })); + const internal = taskService as unknown as { + handleStreamEnd: (event: StreamEndEvent) => Promise; + }; + + await internal.handleStreamEnd({ + type: "stream-end", + workspaceId: "childworkspace", + messageId: "msg_retracted_wake_cut", + metadata: { + model: "anthropic:claude-opus-4-6", + agentId: "exec", + finishReason: "tool-calls", + muxMetadata: workspaceTurnMuxMetadata(parentId), + }, + parts: [{ type: "text", text: "Waiting on the build" }], + }); + + expect( + await new TaskHandleStore(config).getWorkspaceTurn(parentId, "wst_handle") + ).toMatchObject({ + status: "interrupted", + messageId: "msg_retracted_wake_cut", + error: + "Workspace turn yielded at a tool boundary to a bash-monitor wake that was retracted before delivery; the target workspace is idle and this delegated turn did not complete", + }); + }); + + test("a failing wake probe settles the handle instead of leaving it running", async () => { + // The probe is advisory: its I/O failing must not escape finalization with the terminal + // stream-end already consumed (the handle would stay running until the waiter timed out). + const { parentId, taskService } = await startWorkspaceTurnForTest({ + hasOutstandingBashMonitorWake: mock(() => Promise.reject(new Error("watermark read failed"))), + }); + const internal = taskService as unknown as { + handleStreamEnd: (event: StreamEndEvent) => Promise; + }; + + await internal.handleStreamEnd({ + type: "stream-end", + workspaceId: "childworkspace", + messageId: "msg_probe_failed", + metadata: { + model: "anthropic:claude-opus-4-6", + agentId: "exec", + finishReason: "tool-calls", + muxMetadata: workspaceTurnMuxMetadata(parentId), + }, + parts: [{ type: "text", text: "Kicked off verification" }], + }); + + const settled = await workspaceTurnSnapshot(taskService, parentId); + expect(settled?.status).not.toBe("running"); + expect(settled).toMatchObject({ messageId: "msg_probe_failed" }); + }); + test("nested agent progress preserves workspace-turn correlation", async () => { const hasPendingWorkspaceTurnContinuation = mock( ( diff --git a/src/node/services/workspaceService.test.ts b/src/node/services/workspaceService.test.ts index 46dc7ecff3..ce14751e53 100644 --- a/src/node/services/workspaceService.test.ts +++ b/src/node/services/workspaceService.test.ts @@ -257,6 +257,7 @@ describe("WorkspaceService bash monitor wake reconciler wiring", () => { const { service, events, cleanup } = await createWakeWiringService(); const scheduleReconcile = mock(() => undefined); const discardProcess = mock(() => Promise.resolve()); + const outputShown = mock(() => Promise.resolve()); const upsert = mock(() => Promise.resolve()); const remove = mock(() => Promise.resolve()); const recordTerminal = mock(() => Promise.resolve()); @@ -265,6 +266,7 @@ describe("WorkspaceService bash monitor wake reconciler wiring", () => { bashMonitorWakeReconciler: { scheduleReconcile: typeof scheduleReconcile; discardProcess: typeof discardProcess; + outputShown: typeof outputShown; }; bashMonitorRegistryStore: { upsert: typeof upsert; @@ -274,7 +276,7 @@ describe("WorkspaceService bash monitor wake reconciler wiring", () => { }; try { await internal.bashMonitorRecoveryPromise; - internal.bashMonitorWakeReconciler = { scheduleReconcile, discardProcess }; + internal.bashMonitorWakeReconciler = { scheduleReconcile, discardProcess, outputShown }; internal.bashMonitorRegistryStore = { upsert, remove, recordTerminal }; const armed = { processId: "proc", @@ -286,7 +288,7 @@ describe("WorkspaceService bash monitor wake reconciler wiring", () => { createdAt: "2026-08-31T12:00:00.000Z", }; events.emit("monitor:match", "owner", {}); - events.emit("output:shown", "owner", {}); + events.emit("output:shown", "owner", { processId: "proc", shownThroughOffset: 12 }); events.emit("monitor:armed", "owner", armed); events.emit("monitor:stopped", "owner", { processId: "proc", @@ -302,6 +304,8 @@ describe("WorkspaceService bash monitor wake reconciler wiring", () => { // registry row goes, so its captured output is never sent as a turn. expect(discardProcess).toHaveBeenCalledWith("owner", "proc", armed.createdAt); expect(remove).toHaveBeenCalledWith("owner", "proc", armed.createdAt); + // A shown-frontier advance revalidates any wake already handed out for the process. + expect(outputShown).toHaveBeenCalledWith("owner", "proc"); expect(scheduleReconcile).toHaveBeenCalledTimes(4); } finally { await cleanup(); diff --git a/src/node/services/workspaceService.ts b/src/node/services/workspaceService.ts index 1d54ad9f1b..cfdfd79407 100644 --- a/src/node/services/workspaceService.ts +++ b/src/node/services/workspaceService.ts @@ -1870,8 +1870,14 @@ export class WorkspaceService extends EventEmitter implements WorkspaceHost { } private readonly bashOutputShownListener = ( workspaceId: string, - _payload: OutputShownPayload + payload: OutputShownPayload ): void => { + if (this.removingWorkspaces.has(workspaceId)) return; + this.bashMonitorWakeReconciler + .outputShown(workspaceId, payload.processId) + .catch((error: unknown) => { + log.debug("Bash monitor output-shown revalidation failed", { workspaceId, error }); + }); this.scheduleBashMonitorWakeReconcile(workspaceId); }; private readonly bashMonitorMatchListener = ( @@ -11860,7 +11866,10 @@ export class WorkspaceService extends EventEmitter implements WorkspaceHost { * The bash-monitor wake level: a wake the workspace has not seen yet, or a wake turn * admitted but not yet streaming (AgentSession.hasPendingBashMonitorWakeTurn — the * reconciler level is already consumed there). A stream that ended with "tool-calls" - * while this is high yielded to the wake and will be continued by it. + * while this is high yielded to the wake and will be continued by it. Deliberately not + * the session's cut latch: a wake retracted after the cut (monitor canceled) has no + * continuation, and settlement must not defer on it (AgentSession.getQueueCutCutter + * names the cause instead). */ async hasOutstandingBashMonitorWake(workspaceId: string): Promise { const id = workspaceId.trim(); diff --git a/src/node/services/workspaceTurnManager.ts b/src/node/services/workspaceTurnManager.ts index b81bc3a842..36f9a12884 100644 --- a/src/node/services/workspaceTurnManager.ts +++ b/src/node/services/workspaceTurnManager.ts @@ -281,6 +281,17 @@ const WORKSPACE_TURN_STALE_RESTART_ERROR = "Workspace turn interrupted after res const WORKSPACE_TURN_SUPERSEDED_BY_NEW_INPUT_ERROR = "Workspace turn superseded by new input in the target workspace; the workspace continues under that input and this delegated turn will not report"; +/** + * Reason persisted when the target stream yielded at a tool boundary to a bash-monitor + * wake that was then retracted (monitor canceled) before the wake turn was sent. No + * continuation follows, so deferring would leave the owner's wait hanging; settling as a + * truncation would misreport the delegated work as failed output. Same supersede family + * as new-input cuts: self-heal eligible should a late correlated continuation prove the + * turn went on after all. + */ +const WORKSPACE_TURN_YIELDED_TO_RETRACTED_WAKE_ERROR = + "Workspace turn yielded at a tool boundary to a bash-monitor wake that was retracted before delivery; the target workspace is idle and this delegated turn did not complete"; + /** * Reason prefix persisted when the owner's OWN follow-up turn (task * kind="workspace", mode="existing", tool-end dispatch) cut its active @@ -331,7 +342,8 @@ function isSupersededWorkspaceTurnInterrupt( ): boolean { return ( (record.status === "interrupted" && - record.error === WORKSPACE_TURN_SUPERSEDED_BY_NEW_INPUT_ERROR) || + (record.error === WORKSPACE_TURN_SUPERSEDED_BY_NEW_INPUT_ERROR || + record.error === WORKSPACE_TURN_YIELDED_TO_RETRACTED_WAKE_ERROR)) || isOwnerFollowUpSupersededWorkspaceTurnInterrupt(record) ); } @@ -378,6 +390,7 @@ function ownerFollowUpSupersedeSkipsDirectParent( type QueueCutSupersedeEvidence = | { kind: "same_owner_follow_up"; successorHandleId: string } | { kind: "other_input" } + | { kind: "retracted_wake" } | { kind: "preserved"; error: string } | null; @@ -4091,7 +4104,9 @@ export class WorkspaceTurnManager { ? buildOwnerFollowUpSupersededError(evidence.successorHandleId) : evidence.kind === "preserved" ? evidence.error - : WORKSPACE_TURN_SUPERSEDED_BY_NEW_INPUT_ERROR; + : evidence.kind === "retracted_wake" + ? WORKSPACE_TURN_YIELDED_TO_RETRACTED_WAKE_ERROR + : WORKSPACE_TURN_SUPERSEDED_BY_NEW_INPUT_ERROR; return { ...baseRecord, status: "interrupted", @@ -4324,8 +4339,19 @@ export class WorkspaceTurnManager { } // A stream that ended with "tool-calls" while a wake is outstanding yielded to that // wake; the wake turn inherits this correlation (inheritOpenWorkspaceTurnMetadata). - if (await this.workspaceService.hasOutstandingBashMonitorWake(event.workspaceId)) { - return true; + // The probe is advisory: if its I/O fails, settle through the normal path rather + // than leave the handle running with its terminal stream-end already consumed (a + // late correlated continuation can still self-heal it) (Codex P2 PRRT_kwDOPxxmWM6fEQIr). + try { + if (await this.workspaceService.hasOutstandingBashMonitorWake(event.workspaceId)) { + return true; + } + } catch (error) { + log.warn("Bash monitor wake probe failed during workspace turn settlement", { + workspaceId: event.workspaceId, + taskHandleId: correlation.taskHandleId, + error, + }); } const activeStream = this.streamManager?.getStreamInfo(event.workspaceId); if (activeStream == null || activeStream.messageId === event.messageId) { @@ -4401,6 +4427,11 @@ export class WorkspaceTurnManager { ? classifyMetadata(cutter.muxMetadata) : { kind: "other_input" }; } + // The stream yielded to the wake level and the caller found no continuation + // (level low, no wake turn admitted): the wake was retracted after the cut. + if (cutter?.stage === "bash-monitor-wake") { + return { kind: "retracted_wake" }; + } // Residual legacy positives (e.g. hasPendingAutoRetry with an empty queue) // stay generic supersede evidence. return snapshot.hasPendingQueuedOrPreparingTurn ? { kind: "other_input" } : null; From b84c699fa32609072707b5b6e03e7b373a78374b Mon Sep 17 00:00:00 2001 From: Ethan Dickson Date: Thu, 3 Sep 2026 22:05:34 +0000 Subject: [PATCH 06/26] Codex round 4: freeze isCurrent at acceptance, keep requireIdle across wake compaction follow-up, gate background waits on effective yield flag --- .../agentSession.autoCompaction.test.ts | 35 ++++++++++++ src/node/services/agentSession.ts | 10 ++++ .../bashMonitorWakeReconciler.test.ts | 14 +++++ .../services/bashMonitorWakeReconciler.ts | 18 ++++-- src/node/services/workspaceService.test.ts | 56 ++++++++++++++++++- src/node/services/workspaceService.ts | 11 +++- 6 files changed, 137 insertions(+), 7 deletions(-) diff --git a/src/node/services/agentSession.autoCompaction.test.ts b/src/node/services/agentSession.autoCompaction.test.ts index 0161cb9d1e..053798a10e 100644 --- a/src/node/services/agentSession.autoCompaction.test.ts +++ b/src/node/services/agentSession.autoCompaction.test.ts @@ -6,6 +6,7 @@ import { createMuxMessage, type CompactionFollowUpRequest, type MuxMessage, + type MuxMessageMetadata, } from "@/common/types/message"; import { GOAL_CONTINUATION_KIND } from "@/constants/goals"; import { Ok, Err } from "@/common/types/result"; @@ -313,6 +314,40 @@ describe("AgentSession on-send auto-compaction snapshot deferral", () => { expect(unstamped).toBeUndefined(); }); + test("bash-monitor wake follow-ups keep their idle-only admission across compaction", async () => { + // The wake was sent with requireIdle; a manual message queued during the compaction + // stream must still win over the re-dispatched continuation (dispatchPendingFollowUp). + const { session } = await createSessionHarness({ + workspaceId: "ws-auto-compaction-wake-require-idle", + }); + const build = ( + session as unknown as { + buildAutoCompactionFollowUp: (params: { + messageText: string; + options: SendMessageOptions; + modelForStream: string; + muxMetadata?: MuxMessageMetadata; + }) => CompactionFollowUpRequest; + } + ).buildAutoCompactionFollowUp.bind(session); + + const wakeFollowUp = build({ + messageText: "READY", + options: { model: "openai:gpt-4o", agentId: "exec" }, + modelForStream: "openai:gpt-4o", + muxMetadata: { type: "bash-monitor-wake", records: [] }, + }); + expect(wakeFollowUp.dispatchOptions?.requireIdle).toBe(true); + + const plainFollowUp = build({ + messageText: "hello", + options: { model: "openai:gpt-4o", agentId: "exec" }, + modelForStream: "openai:gpt-4o", + }); + expect(plainFollowUp.dispatchOptions?.requireIdle).toBeUndefined(); + session.dispose(); + }); + test("preserves goal kind and goal identity on auto-compaction follow-up requests", async () => { const { session } = await createSessionHarness({ workspaceId: "ws-auto-compaction-goal-kind", diff --git a/src/node/services/agentSession.ts b/src/node/services/agentSession.ts index 2b6a2e71f6..afa0cb7347 100644 --- a/src/node/services/agentSession.ts +++ b/src/node/services/agentSession.ts @@ -4487,6 +4487,16 @@ export class AgentSession { if (params.muxMetadata) { followUp.muxMetadata = params.muxMetadata; + // A bash-monitor wake is an idle-only send (WorkspaceService.dispatchBashMonitorWake + // sends it with requireIdle). The compaction hand-off must keep that rule so a manual + // message queued or in preflight during the compaction stream still wins over the + // wake continuation (dispatchPendingFollowUp). A skipped wake follow-up is lost — its + // signals were consumed at acceptance — which is the bounded cost of letting the + // user's correction go first; the process output stays readable via task_await + // (Codex P2 PRRT_kwDOPxxmWM6fFJ4N). + if (params.muxMetadata.type === "bash-monitor-wake") { + followUp.dispatchOptions = { ...followUp.dispatchOptions, requireIdle: true }; + } } if (params.workspaceTurnMetadata) { diff --git a/src/node/services/bashMonitorWakeReconciler.test.ts b/src/node/services/bashMonitorWakeReconciler.test.ts index 4d2f673b42..6f1507ba16 100644 --- a/src/node/services/bashMonitorWakeReconciler.test.ts +++ b/src/node/services/bashMonitorWakeReconciler.test.ts @@ -222,6 +222,20 @@ describe("BashMonitorWakeReconciler", () => { expect(await reconciler.hasOutstandingWake(OWNER)).toBe(false); }); + test("an accepted wake stays current through a later cancel or shown advance", async () => { + // Acceptance runs before the owner's final send-admission gate; a cancel landing in + // between must not make that gate refuse a turn whose row is already durable. + live = [liveSnapshot()]; + await reconciler.reconcile(OWNER); + expect(dispatches).toHaveLength(1); + await dispatches[0].onAccepted(); + expect(dispatches[0].isCurrent()).toBe(true); + + await reconciler.discardProcess(OWNER, "proc", CREATED_AT); + await reconciler.outputShown(OWNER, "proc"); + expect(dispatches[0].isCurrent()).toBe(true); + }); + test("disposal lowers the published level and retires the in-flight wake", async () => { live = [liveSnapshot()]; await reconciler.reconcile(OWNER); diff --git a/src/node/services/bashMonitorWakeReconciler.ts b/src/node/services/bashMonitorWakeReconciler.ts index 927d0122e3..0c1dc5decf 100644 --- a/src/node/services/bashMonitorWakeReconciler.ts +++ b/src/node/services/bashMonitorWakeReconciler.ts @@ -112,10 +112,12 @@ export interface BashMonitorWakeDispatch { prompt: string; muxMetadata: Extract; /** - * False once a full-history clear or disposal retired the signals behind this wake - * while it was in the receiver's hands. The receiver re-checks it after taking its own - * locks (the clear runs under the same history lock) and before sending, so a stale - * prompt is never appended to freshly cleared history. + * False once a full-history clear, disposal, monitor cancel, or shown-frontier advance + * retired the signals behind this wake while it was in the receiver's hands — until + * `onAccepted` runs, after which it stays true (the prompt is durable and the signals + * consumed). The receiver re-checks it after taking its own locks (the clear runs under + * the same history lock), before sending, and at every send-admission gate, so a stale + * prompt is never appended to history and an accepted one always gets its stream. */ isCurrent(): boolean; onAccepted(): Promise; @@ -575,7 +577,13 @@ export class BashMonitorWakeReconciler { ownerWorkspaceId, prompt: buildPrompt(dispatch.signals), muxMetadata: buildMetadata(dispatch.signals), - isCurrent: () => this.states.get(ownerWorkspaceId)?.dispatch === dispatch, + // Validity freezes at acceptance: the prompt row is durable and the signals are + // consumed, so a cancel/shown/clear landing in the owner's remaining pre-stream + // awaits must let the turn finish admission (refusing there would leave the row + // in history with no stream, to be replayed by a later manual turn) (Codex P2 + // PRRT_kwDOPxxmWM6fFJ4K). + isCurrent: () => + dispatch.accepted || this.states.get(ownerWorkspaceId)?.dispatch === dispatch, onAccepted: async () => this.accept(ownerWorkspaceId, dispatch), onDeferred: async () => this.defer(ownerWorkspaceId, dispatch), }); diff --git a/src/node/services/workspaceService.test.ts b/src/node/services/workspaceService.test.ts index ce14751e53..4039a76aa0 100644 --- a/src/node/services/workspaceService.test.ts +++ b/src/node/services/workspaceService.test.ts @@ -231,6 +231,8 @@ describe("WorkspaceService bash monitor wake reconciler wiring", () => { async function createWakeWiringService() { const { config, historyService, cleanup } = await createTestHistoryService(); const events = new EventEmitter(); + // The yield flag is a real mirror so hasQueuedMessage reflects the session's arbitration. + const yieldFlags = new Map(); const backgroundProcessManager = Object.assign(events, { notifyMonitorWakeStateChanged: mock(() => undefined), getActiveMonitorCount: mock(() => 0), @@ -238,7 +240,10 @@ describe("WorkspaceService bash monitor wake reconciler wiring", () => { getMonitorWakeDeliveryState: mock(() => Promise.resolve(undefined)), acknowledgeMonitorWake: mock(() => undefined), dropRetiredMonitor: mock(() => undefined), - setMessageQueued: mock(() => undefined), + setMessageQueued: mock((workspaceId: string, queued: boolean) => { + yieldFlags.set(workspaceId, queued); + }), + hasQueuedMessage: (workspaceId: string) => yieldFlags.get(workspaceId) === true, cleanup: mock(() => Promise.resolve()), }) as unknown as BackgroundProcessManager; const service = createWorkspaceServiceForTest({ @@ -1142,6 +1147,55 @@ describe("WorkspaceService bash monitor wake reconciler wiring", () => { } }); + test("a high level backgrounds foreground waits only when it pulls the yield lever", async () => { + // A turn-end queue head suppresses the wake cut (hasPendingToolEndInput arbitration), so + // the same level must not end a foreground task_await early either: the stream would + // not cut and the agent would simply get another model step. + const { service, cleanup } = await createWakeWiringService(); + const workspaceId = "level-background-waits-owner"; + const backgroundForegroundWaitsForWorkspace = mock(() => 0); + service.setAgentTaskIntegration( + makeAgentTaskIntegrationFake({ backgroundForegroundWaitsForWorkspace }) + ); + const internal = service as unknown as { + backgroundProcessManager: { pullMonitorWakeSignals: ReturnType }; + bashMonitorWakeReconciler: { hasOutstandingWake: (owner: string) => Promise }; + }; + const session = service.getOrCreateSession(workspaceId); + try { + internal.backgroundProcessManager.pullMonitorWakeSignals.mockImplementation(() => + Promise.resolve([ + { + processId: "proc", + taskId: "bash:proc", + ownerWorkspaceId: workspaceId, + filter: "READY", + filterExclude: false, + script: "run", + createdAt: "2026-08-31T12:00:00.000Z", + match: { throughOffset: 12, lines: ["READY"], totalMatches: 1 }, + retired: false, + }, + ]) + ); + + session.queueMessage("later", { + model: "anthropic:claude-sonnet-4-5", + agentId: "exec", + queueDispatchMode: "turn-end", + }); + expect(await internal.bashMonitorWakeReconciler.hasOutstandingWake(workspaceId)).toBe(true); + expect(backgroundForegroundWaitsForWorkspace).not.toHaveBeenCalled(); + + session.clearQueue(); + expect(await internal.bashMonitorWakeReconciler.hasOutstandingWake(workspaceId)).toBe(true); + expect(backgroundForegroundWaitsForWorkspace).toHaveBeenCalledWith(workspaceId); + } finally { + session.dispose(); + await cleanup(); + } + }); + test("the stream yields on the wake level, not on a queued snapshot of it", async () => { // Regression: a monitored bash matched mid-step while the same step's task_await showed // the matched lines. The old queued tool-end wake still cut the stream (finishReason diff --git a/src/node/services/workspaceService.ts b/src/node/services/workspaceService.ts index cfdfd79407..6b165e15bd 100644 --- a/src/node/services/workspaceService.ts +++ b/src/node/services/workspaceService.ts @@ -2401,7 +2401,16 @@ export class WorkspaceService extends EventEmitter implements WorkspaceHost { // disposal after the session went away), so drop it directly. this.backgroundProcessManager.setMessageQueued(ownerWorkspaceId, false); } - if (outstanding) { + // Background foreground waits only when the level actually pulls the yield lever: + // the session's mirror applies the queue-head arbitration (a turn-end head means + // the stream will not cut for this wake, so ending the wait early would just hand + // the agent another model step) (Codex P2 PRRT_kwDOPxxmWM6fFJ4Q). + if ( + outstanding && + session != null && + typeof this.backgroundProcessManager.hasQueuedMessage === "function" && + this.backgroundProcessManager.hasQueuedMessage(ownerWorkspaceId) + ) { this.agentTaskIntegration?.backgroundForegroundWaitsForWorkspace(ownerWorkspaceId); } }, From fba966d76100643a664d6a9ab6a8318741802825 Mon Sep 17 00:00:00 2001 From: Ethan Dickson Date: Fri, 4 Sep 2026 08:21:45 +0000 Subject: [PATCH 07/26] Codex round 5: transitions own their consequences (GVw_/GVxB/GVxG) - AgentSession owns the tool-end yield edge (onToolEndYieldRequested); WorkspaceService backgrounds foreground waits from that hook instead of from enqueue/level-publish call sites (PRRT_kwDOPxxmWM6fGVw_). - Reconciler: accept() never throws; acknowledgment is attempted inline and retried by every reconcile pass while the accepted dispatch stays consumed (overlay in collect) (PRRT_kwDOPxxmWM6fGVxB). - Abandoned compaction follow-ups settle their delegated turn via onWorkspaceTurnContinuationAbandoned -> settleSupersededWorkspaceTurnContinuation (PRRT_kwDOPxxmWM6fGVxG). --- ...gentSession.continueMessageAgentId.test.ts | 61 +++++++- .../agentSession.queueDispatch.test.ts | 52 +++++++ src/node/services/agentSession.testHarness.ts | 2 + src/node/services/agentSession.ts | 46 ++++++- .../bashMonitorWakeReconciler.test.ts | 44 +++++- .../services/bashMonitorWakeReconciler.ts | 130 +++++++++++++----- src/node/services/taskService.test.ts | 51 +++++++ src/node/services/taskService.ts | 10 ++ .../services/taskWorkspaceSeam.testUtils.ts | 1 + src/node/services/taskWorkspaceSeam.ts | 9 ++ src/node/services/workspaceService.test.ts | 92 +++++-------- src/node/services/workspaceService.ts | 31 ++--- src/node/services/workspaceTurnManager.ts | 21 +++ 13 files changed, 436 insertions(+), 114 deletions(-) diff --git a/src/node/services/agentSession.continueMessageAgentId.test.ts b/src/node/services/agentSession.continueMessageAgentId.test.ts index a0e0db679a..220d16c634 100644 --- a/src/node/services/agentSession.continueMessageAgentId.test.ts +++ b/src/node/services/agentSession.continueMessageAgentId.test.ts @@ -150,7 +150,13 @@ describe("AgentSession continue-message agentId fallback", () => { historyCleanup = undefined; }); - const createSession = async (messages: MuxMessage[] = []) => { + const createSession = async ( + messages: MuxMessage[] = [], + hooks: Pick< + ConstructorParameters[0], + "onWorkspaceTurnContinuationAbandoned" + > = {} + ) => { const { historyService, cleanup } = await createTestHistoryService(); historyCleanup = cleanup; for (const message of messages) { @@ -164,6 +170,7 @@ describe("AgentSession continue-message agentId fallback", () => { aiService: createAiService(), initStateManager: createInitStateManager(), backgroundProcessManager: createBackgroundProcessManager(), + ...hooks, }); sessions.push(session); @@ -313,6 +320,58 @@ describe("AgentSession continue-message agentId fallback", () => { expect(lastMessages.data[0]?.metadata?.muxMetadata).toEqual({ type: "compaction-summary" }); }); + test("abandoning a follow-up that carries a delegated turn settles that turn", async () => { + // A bash-monitor wake cut a delegated turn; the wake's on-send compaction stamped the + // correlation on its follow-up. If a manual send wins the idle race, nothing else can + // settle the owner's waiter (the compaction stream-end is uncorrelated and no later send + // inherits the metadata), so the discard itself must (Codex P2 PRRT_kwDOPxxmWM6fGVxG). + const workspaceTurnMetadata = { + type: "workspace-turn-task", + taskHandleId: "wst_abandoned", + ownerWorkspaceId: "parent-abandoned", + turnId: "turn-abandoned", + } as const; + const wakeFollowUp: CompactionFollowUpRequest = { + text: "monitor matched", + model: "openai:gpt-4o", + agentId: "exec", + muxMetadata: { type: "bash-monitor-wake", records: [] }, + workspaceTurnMetadata, + dispatchOptions: { requireIdle: true }, + }; + const abandoned = mock( + (_metadata: NonNullable) => + Promise.resolve() + ); + const { session, historyService, internals } = await createSession( + [compactionSummaryMessage("summary-wake", wakeFollowUp)], + { onWorkspaceTurnContinuationAbandoned: abandoned } + ); + internals.sendMessage = mock(() => Promise.resolve({ success: true as const })); + (session as unknown as { hasExternalSendPreflight?: () => boolean }).hasExternalSendPreflight = + () => true; + + expect(await internals.dispatchPendingFollowUp()).toBe(false); + expect(internals.sendMessage).not.toHaveBeenCalled(); + expect(abandoned).toHaveBeenCalledTimes(1); + expect(abandoned).toHaveBeenCalledWith(workspaceTurnMetadata); + const lastMessages = await historyService.getLastMessages("ws", 1); + expect(lastMessages.success && lastMessages.data[0]?.metadata?.muxMetadata).toEqual({ + type: "compaction-summary", + }); + + // A follow-up that is dispatched is the continuation itself: nothing to settle. + (session as unknown as { hasExternalSendPreflight?: () => boolean }).hasExternalSendPreflight = + () => false; + await historyService.appendToHistory( + "ws", + compactionSummaryMessage("summary-wake-2", wakeFollowUp) + ); + expect(await internals.dispatchPendingFollowUp()).toBe(true); + expect(internals.sendMessage).toHaveBeenCalledTimes(1); + expect(abandoned).toHaveBeenCalledTimes(1); + }); + test("dispatchPendingFollowUp removes heartbeat reset boundaries when idle-only follow-ups are skipped", async () => { const earlierMessage = createMuxMessage("before-reset", "assistant", "Earlier context"); const { session, historyService, internals } = await createSession([ diff --git a/src/node/services/agentSession.queueDispatch.test.ts b/src/node/services/agentSession.queueDispatch.test.ts index e8adb0be04..d90e1b0f15 100644 --- a/src/node/services/agentSession.queueDispatch.test.ts +++ b/src/node/services/agentSession.queueDispatch.test.ts @@ -803,6 +803,58 @@ describe("AgentSession queued message tool-call dispatch", () => { } }); + test("the tool-end yield edge fires once per rising transition, whatever raises it", async () => { + // Foreground task waits are backgrounded on this edge, so it must track the *effective* + // lever (queue-head arbitration ∪ level), not the events that happen to feed it + // (Codex P2 PRRT_kwDOPxxmWM6fGVw_). + const workspaceId = "queue-dispatch-yield-edge"; + const edges = mock(() => undefined); + const { session, cleanup } = await createAgentSessionHarness({ + workspaceId, + onToolEndYieldRequested: edges, + }); + const turnEnd = { model: TEST_MODEL, agentId: "exec", queueDispatchMode: "turn-end" as const }; + const toolEnd = { model: TEST_MODEL, agentId: "exec", queueDispatchMode: "tool-end" as const }; + try { + // A turn-end head does not pull the lever; a tool-end enqueue does, once. + session.queueMessage("later", turnEnd); + expect(edges).not.toHaveBeenCalled(); + session.queueMessage("sooner", toolEnd); + expect(edges).toHaveBeenCalledTimes(1); + session.queueMessage("sooner still", toolEnd); + expect(edges).toHaveBeenCalledTimes(1); + session.clearQueue(); + + // The GVw_ case: the level is high behind a turn-end head (no edge), then the head is + // cleared with no enqueue and no level publish in between — the lever becomes + // effective and the edge must fire from that queue transition alone. + session.queueMessage("later", turnEnd); + session.setBashMonitorWakeOutstanding(true); + expect(edges).toHaveBeenCalledTimes(1); + session.clearQueue(); + expect(edges).toHaveBeenCalledTimes(2); + + // Republishing a high level is not an edge; lowering and raising it is. + session.setBashMonitorWakeOutstanding(true); + expect(edges).toHaveBeenCalledTimes(2); + session.setBashMonitorWakeOutstanding(false); + session.setBashMonitorWakeOutstanding(true); + expect(edges).toHaveBeenCalledTimes(3); + session.setBashMonitorWakeOutstanding(false); + + // tool-end is sticky within an entry: a turn-end queued behind it neither lowers the + // lever nor re-fires the edge. + session.queueMessage("sooner", toolEnd); + expect(edges).toHaveBeenCalledTimes(4); + session.queueMessage("later", turnEnd); + expect(edges).toHaveBeenCalledTimes(4); + expect(session.hasQueuedMessages()).toBe(true); + } finally { + session.dispose(); + await cleanup(); + } + }); + test("disposing a session lowers the mirrored wake level", async () => { // The flag lives in BackgroundProcessManager keyed by workspace id and outlives the // session; a stale true would make a re-created session's bash reads return early. diff --git a/src/node/services/agentSession.testHarness.ts b/src/node/services/agentSession.testHarness.ts index c38911886e..d0c42f418d 100644 --- a/src/node/services/agentSession.testHarness.ts +++ b/src/node/services/agentSession.testHarness.ts @@ -111,6 +111,7 @@ export interface AgentSessionHarnessOptions { mcpServerManager?: MCPServerManager; onCompactionComplete?: (metadata: CompactionCompletionMetadata) => void; hasOutstandingBashMonitorWake?: () => Promise; + onToolEndYieldRequested?: () => void; captureEvents?: boolean; } @@ -156,6 +157,7 @@ export async function createAgentSessionHarness( backgroundProcessManager, onCompactionComplete: options.onCompactionComplete, hasOutstandingBashMonitorWake: options.hasOutstandingBashMonitorWake, + onToolEndYieldRequested: options.onToolEndYieldRequested, }); const events: WorkspaceChatMessage[] = []; diff --git a/src/node/services/agentSession.ts b/src/node/services/agentSession.ts index afa0cb7347..efba9f0b34 100644 --- a/src/node/services/agentSession.ts +++ b/src/node/services/agentSession.ts @@ -629,6 +629,20 @@ interface AgentSessionOptions { * queued as messages, so this is the only way a stream learns one is pending. */ hasOutstandingBashMonitorWake?: () => Promise; + /** + * The tool-end yield lever rose (false → true): the next tool boundary will cut the + * stream for a tool-end queue head or the wake level. Fired on every rising edge, whatever + * caused it (enqueue, queue-head clear/promote, wake level), so foreground waits that + * would outlive the boundary can be backgrounded exactly when the yield becomes effective. + */ + onToolEndYieldRequested?: () => void; + /** + * A compaction follow-up carrying a delegated turn's correlation was abandoned instead of + * dispatched (idle-rule skip, inadmissible summary/goal). Nothing else settles that turn + * — the compaction stream-end is not correlated and no later send inherits the metadata — + * so the owner is told the turn was superseded. + */ + onWorkspaceTurnContinuationAbandoned?: (metadata: WorkspaceTurnMuxMetadata) => Promise; } enum TurnPhase { @@ -671,11 +685,17 @@ export class AgentSession { private readonly onPostCompactionStateChange?: () => void; private readonly hasExternalSendPreflight?: () => boolean; private readonly hasOutstandingBashMonitorWake?: () => Promise; + private readonly onToolEndYieldRequested?: () => void; + private readonly onWorkspaceTurnContinuationAbandoned?: ( + metadata: WorkspaceTurnMuxMetadata + ) => Promise; /** * Last published wake level (see setBashMonitorWakeOutstanding). Feeds the * tool-end yield flag together with the queue head. */ private bashMonitorWakeOutstanding = false; + /** Last value pushed to the tool-end yield lever; edge detection for onToolEndYieldRequested. */ + private toolEndYieldRequested = false; private readonly emitter = new EventEmitter(); private readonly aiListeners: Array<{ event: string; handler: (...args: unknown[]) => void }> = []; @@ -933,6 +953,8 @@ export class AgentSession { onPostCompactionStateChange, hasExternalSendPreflight, hasOutstandingBashMonitorWake, + onToolEndYieldRequested, + onWorkspaceTurnContinuationAbandoned, } = options; assert(typeof workspaceId === "string", "workspaceId must be a string"); @@ -962,6 +984,8 @@ export class AgentSession { this.onPostCompactionStateChange = onPostCompactionStateChange; this.hasExternalSendPreflight = hasExternalSendPreflight; this.hasOutstandingBashMonitorWake = hasOutstandingBashMonitorWake; + this.onToolEndYieldRequested = onToolEndYieldRequested; + this.onWorkspaceTurnContinuationAbandoned = onWorkspaceTurnContinuationAbandoned; this.compactionHandler = new CompactionHandler({ workspaceId: this.workspaceId, @@ -6620,14 +6644,22 @@ export class AgentSession { * Tool-end yield flag = queue head is tool-end ∪ (queue empty ∧ wake level), mirroring * hasPendingToolEndInput's arbitration. `queueHeadToolEnd` lets stream-end / clear paths * assert the queue contribution is gone before the queue itself is observed empty. + * + * This is the single place the effective flag is computed, so it also owns the rising + * edge: a turn-end head that is cleared, removed, or promoted while the wake level is high + * makes the lever effective without any enqueue or level publish, and the consequence + * (backgrounding foreground waits) must follow that edge, not those events (Codex P2 + * PRRT_kwDOPxxmWM6fGVw_). */ private syncToolEndYieldRequested( queueHeadToolEnd = this.messageQueue.getNextDispatchableMode() === "tool-end" ): void { - this.backgroundProcessManager.setMessageQueued( - this.workspaceId, - queueHeadToolEnd || (this.bashMonitorWakeOutstanding && this.messageQueue.isEmpty()) - ); + const requested = + queueHeadToolEnd || (this.bashMonitorWakeOutstanding && this.messageQueue.isEmpty()); + const rising = requested && !this.toolEndYieldRequested; + this.toolEndYieldRequested = requested; + this.backgroundProcessManager.setMessageQueued(this.workspaceId, requested); + if (rising) this.onToolEndYieldRequested?.(); } /** Queued intra-tree agent peer messages awaiting dispatch (peer-message queue cap input). */ @@ -7497,6 +7529,12 @@ export class AgentSession { if (!updateResult.success) { throw new Error(`Failed to clear skipped pending follow-up: ${updateResult.error}`); } + // Every discard path funnels here, so this is the one place that knows the delegated + // turn's continuation is gone for good (Codex P2 PRRT_kwDOPxxmWM6fGVxG). + const workspaceTurnMetadata = muxMeta.pendingFollowUp.workspaceTurnMetadata; + if (workspaceTurnMetadata != null) { + await this.onWorkspaceTurnContinuationAbandoned?.(workspaceTurnMetadata); + } } /** diff --git a/src/node/services/bashMonitorWakeReconciler.test.ts b/src/node/services/bashMonitorWakeReconciler.test.ts index 6f1507ba16..9e32fdb89d 100644 --- a/src/node/services/bashMonitorWakeReconciler.test.ts +++ b/src/node/services/bashMonitorWakeReconciler.test.ts @@ -56,6 +56,7 @@ describe("BashMonitorWakeReconciler", () => { let dispatches: BashMonitorWakeDispatch[]; let dispatchOutcome: "in-flight" | "deferred"; let acknowledged: Array<{ processId: string; matchedThroughOffset?: number }>; + let acknowledgeError: Error | undefined; let removed: string[]; let removedOwners: string[]; let dropped: string[]; @@ -70,16 +71,18 @@ describe("BashMonitorWakeReconciler", () => { dispatches = []; dispatchOutcome = "in-flight"; acknowledged = []; + acknowledgeError = undefined; removed = []; removedOwners = []; dropped = []; droppedGenerations = []; - reconciler = new BashMonitorWakeReconciler({ + const current = new BashMonitorWakeReconciler({ sessionsDir: root, processManager: { pullMonitorWakeSignals: () => live, getMonitorWakeDeliveryState: () => Promise.resolve(deliveryState), acknowledgeMonitorWake: (processId, _generation, matchedThroughOffset) => { + if (acknowledgeError != null) throw acknowledgeError; acknowledged.push({ processId, ...(matchedThroughOffset != null ? { matchedThroughOffset } : {}), @@ -108,10 +111,14 @@ describe("BashMonitorWakeReconciler", () => { recordTerminal: () => undefined, }, onWake: (dispatch) => { + // A pass scheduled by the previous test's acceptance may still be draining; keep its + // hand-outs from leaking into this test's `dispatches`. + if (reconciler !== current) return "deferred"; dispatches.push(dispatch); return dispatchOutcome; }, }); + reconciler = current; }); afterEach(async () => { @@ -236,6 +243,41 @@ describe("BashMonitorWakeReconciler", () => { expect(dispatches[0].isCurrent()).toBe(true); }); + test("a failed acknowledgment keeps the accepted wake consumed and retries it", async () => { + live = [liveSnapshot()]; + await reconciler.reconcile(OWNER); + expect(dispatches).toHaveLength(1); + + // The owner's row is durable, so acceptance must not fail the send: it resolves, and the + // wake reads as consumed (level low, no duplicate hand-out) while durability is pending. + acknowledgeError = new Error("disk full"); + await dispatches[0].onAccepted(); + expect(dispatches[0].isCurrent()).toBe(true); + expect(await reconciler.hasOutstandingWake(OWNER)).toBe(false); + await expect(reconciler.reconcile(OWNER)).rejects.toThrow("disk full"); + expect(dispatches).toHaveLength(1); + + // Withdrawals never apply to an accepted dispatch: dropping it here would re-derive the + // same signals into a second prompt once the store recovers. + await reconciler.outputShown(OWNER, "proc"); + await reconciler.discardProcess(OWNER, "proc", CREATED_AT); + await expect(reconciler.reconcile(OWNER)).rejects.toThrow("disk full"); + expect(dispatches).toHaveLength(1); + + acknowledgeError = undefined; + await reconciler.reconcile(OWNER); + expect(acknowledged).toEqual([{ processId: "proc", matchedThroughOffset: 12 }]); + expect(dispatches).toHaveLength(1); + + // The slot is free again: a newer match dispatches normally. + live = [ + liveSnapshot({ match: { throughOffset: 24, lines: ["READY again"], totalMatches: 2 } }), + ]; + await reconciler.reconcile(OWNER); + expect(dispatches).toHaveLength(2); + expect(dispatches[1].prompt).toContain("READY again"); + }); + test("disposal lowers the published level and retires the in-flight wake", async () => { live = [liveSnapshot()]; await reconciler.reconcile(OWNER); diff --git a/src/node/services/bashMonitorWakeReconciler.ts b/src/node/services/bashMonitorWakeReconciler.ts index 0c1dc5decf..55b76a9ae4 100644 --- a/src/node/services/bashMonitorWakeReconciler.ts +++ b/src/node/services/bashMonitorWakeReconciler.ts @@ -5,12 +5,14 @@ import * as path from "node:path"; import type { MuxMessageMetadata } from "@/common/types/message"; import assert from "@/common/utils/assert"; +import { getErrorMessage } from "@/common/utils/errors"; import { BASH_MONITOR_WAKE_HEADINGS } from "@/common/utils/machineTurnPrompts"; import type { BashMonitorLostSummary, BashMonitorRegistryRecord, BashMonitorTerminalSummary, } from "@/node/services/bashMonitorRegistryStore"; +import { log } from "@/node/services/log"; import { MutexMap } from "@/node/utils/concurrency/mutexMap"; import { stripAnsiControlChars } from "@/node/utils/ansi"; import { isErrnoWithCode } from "@/node/utils/fs"; @@ -163,7 +165,14 @@ interface DerivedSignal { retired: boolean; } -/** A wake handed to `onWake` whose acceptance/deferral has not settled yet. */ +/** + * The one wake handed to `onWake` for an owner at a time. Lifecycle: an *offered* dispatch leaves the slot + * either by withdrawal (defer / forgetDispatchFor / consumeCurrent) or by acceptance; an + * *accepted* dispatch leaves the slot only once its signals are durably consumed — by the + * reconcile pass that acknowledges them, by consumeCurrent, or by dispose. Withdrawal never + * applies to an accepted dispatch: its prompt row is already durable, so the signals are + * consumed whether or not the durability write has landed yet (Codex P2 PRRT_kwDOPxxmWM6fGVxB). + */ interface DispatchState { signals: readonly DerivedSignal[]; accepted: boolean; @@ -483,7 +492,13 @@ export class BashMonitorWakeReconciler { ): Promise { await this.locks.withLock(ownerWorkspaceId, () => { const state = this.states.get(ownerWorkspaceId); - if (state?.dispatch?.signals.some(covers) === true) state.dispatch = undefined; + if ( + state?.dispatch != null && + !state.dispatch.accepted && + state.dispatch.signals.some(covers) + ) { + state.dispatch = undefined; + } return Promise.resolve(); }); this.scheduleReconcile(ownerWorkspaceId); @@ -554,6 +569,10 @@ export class BashMonitorWakeReconciler { private async reconcileOnce(ownerWorkspaceId: string): Promise { const dispatch = await this.locks.withLock(ownerWorkspaceId, async () => { + // An acknowledgment that failed at acceptance is retried first: on a throw the slot + // stays occupied and accepted, the loop's catch schedules the backoff retry, and no + // second wake is handed out meanwhile. + await this.acknowledgeAccepted(ownerWorkspaceId); const collected = await this.collect(ownerWorkspaceId, true); for (const readSettled of collected.deferredReads) { void readSettled.finally(() => this.scheduleReconcile(ownerWorkspaceId)); @@ -562,9 +581,9 @@ export class BashMonitorWakeReconciler { await this.cleanup(collected.autoConsumed); const state = this.state(ownerWorkspaceId); - // A wake already handed to the owner settles on its own (accept → watermarks advance - // and a reconcile is scheduled; defer → the owner re-arms a reconcile). Handing out a - // second one meanwhile could only duplicate or supersede the first. + // A wake already handed to the owner settles on its own (accept → acknowledged, and a + // reconcile is scheduled; defer → the owner re-arms a reconcile). Handing out a second + // one meanwhile could only duplicate or supersede the first. if (collected.signals.length === 0 || state.dispatch != null) return undefined; const next: DispatchState = { signals: collected.signals, accepted: false }; state.dispatch = next; @@ -601,21 +620,50 @@ export class BashMonitorWakeReconciler { return Promise.resolve(); }); } + /** + * The prompt row is durable: the dispatch's signals are consumed from here on, even if a + * full-history clear or process discard forgot the dispatch meanwhile. The flag flips first + * (collect() overlays an accepted dispatch onto the watermarks, so the level reads low and + * no duplicate derives whether or not the acknowledgment has landed); the acknowledgment + * itself is attempted inline and, if it throws, retried by the reconcile passes until it + * lands. Never throws: the caller is the owner's send, whose row already landed (Codex P2 + * PRRT_kwDOPxxmWM6fGVxB). + */ private async accept(ownerWorkspaceId: string, dispatch: DispatchState): Promise { await this.locks.withLock(ownerWorkspaceId, async () => { - // The prompt reached the model, so its signals are consumed even if a full-history - // clear or process discard forgot this dispatch meanwhile. + // Second call (onAcceptedPreStreamFailure): already consumed; any pending retry is the + // reconcile pass's. if (dispatch.accepted) return; dispatch.accepted = true; - const watermarks = await this.readWatermarks(ownerWorkspaceId); - await this.advanceWatermarks(ownerWorkspaceId, watermarks, dispatch.signals); - await this.cleanup(dispatch.signals); const state = this.state(ownerWorkspaceId); - if (state.dispatch === dispatch) state.dispatch = undefined; + // Withdrawn between the send's last admission gate and its row becoming durable + // (forgetDispatchFor / consumeCurrent take only this lock): re-occupy the slot so the + // acknowledgment covers these signals instead of re-deriving them into a duplicate wake. + state.dispatch ??= dispatch; + await this.acknowledgeAccepted(ownerWorkspaceId).catch((error: unknown) => { + log.warn("Bash monitor wake acknowledgment failed; the reconcile pass retries it", { + ownerWorkspaceId, + error: getErrorMessage(error), + }); + }); }); this.scheduleReconcile(ownerWorkspaceId); } + /** + * Durably consume the accepted dispatch's signals and free the slot. Caller holds the owner + * lock. Throws when durability fails, leaving the slot occupied and accepted for a retry. + */ + private async acknowledgeAccepted(ownerWorkspaceId: string): Promise { + const state = this.states.get(ownerWorkspaceId); + const dispatch = state?.dispatch; + if (state == null || dispatch?.accepted !== true) return; + const watermarks = await this.readWatermarks(ownerWorkspaceId); + await this.advanceWatermarks(ownerWorkspaceId, watermarks, dispatch.signals); + await this.cleanup(dispatch.signals); + if (state.dispatch === dispatch) state.dispatch = undefined; + } + private async consumeCurrent(ownerWorkspaceId: string): Promise { await this.locks.withLock(ownerWorkspaceId, async () => { // A wake already handed to the owner describes signals this consume retires; @@ -685,6 +733,12 @@ export class BashMonitorWakeReconciler { } if (pruned) await this.writeWatermarks(ownerWorkspaceId, watermarks); + // An accepted dispatch is consumed whether or not its acknowledgment has been written yet + // (the write may have failed and be awaiting retry). Overlaying it makes derive() treat + // those signals as delivered, so level reads stay low and nothing re-derives a duplicate. + const accepted = this.states.get(ownerWorkspaceId)?.dispatch; + if (accepted?.accepted === true) applySignalsToWatermarks(watermarks, accepted.signals); + const signals: DerivedSignal[] = []; const autoConsumed: DerivedSignal[] = []; const deferredReads: Array> = []; @@ -948,29 +1002,7 @@ export class BashMonitorWakeReconciler { signals: readonly DerivedSignal[] ): Promise { if (signals.length === 0) return; - for (const signal of signals) { - const previous = watermarks.get(signal.key); - watermarks.set(signal.key, { - processId: signal.processId, - createdAt: signal.createdAt, - ...(signal.matchOffset != null - ? { - matchedThroughOffset: Math.max( - signal.matchOffset, - previous?.matchedThroughOffset ?? -1 - ), - } - : previous?.matchedThroughOffset != null - ? { matchedThroughOffset: previous.matchedThroughOffset } - : {}), - ...(signal.terminal != null - ? { terminalSettledAt: signal.terminal.settledAt } - : previous?.terminalSettledAt != null - ? { terminalSettledAt: previous.terminalSettledAt } - : {}), - ...(signal.kind === "monitor-lost" || previous?.lost === true ? { lost: true } : {}), - }); - } + applySignalsToWatermarks(watermarks, signals); await this.writeWatermarks(ownerWorkspaceId, watermarks); } @@ -1068,3 +1100,33 @@ export class BashMonitorWakeReconciler { } } } + +/** Fold delivered signals into the in-memory watermark map (idempotent per signal). */ +function applySignalsToWatermarks( + watermarks: Map, + signals: readonly DerivedSignal[] +): void { + for (const signal of signals) { + const previous = watermarks.get(signal.key); + watermarks.set(signal.key, { + processId: signal.processId, + createdAt: signal.createdAt, + ...(signal.matchOffset != null + ? { + matchedThroughOffset: Math.max( + signal.matchOffset, + previous?.matchedThroughOffset ?? -1 + ), + } + : previous?.matchedThroughOffset != null + ? { matchedThroughOffset: previous.matchedThroughOffset } + : {}), + ...(signal.terminal != null + ? { terminalSettledAt: signal.terminal.settledAt } + : previous?.terminalSettledAt != null + ? { terminalSettledAt: previous.terminalSettledAt } + : {}), + ...(signal.kind === "monitor-lost" || previous?.lost === true ? { lost: true } : {}), + }); + } +} diff --git a/src/node/services/taskService.test.ts b/src/node/services/taskService.test.ts index 1e3ad50ee5..9ffb4ef6e9 100644 --- a/src/node/services/taskService.test.ts +++ b/src/node/services/taskService.test.ts @@ -24181,6 +24181,57 @@ describe("TaskService", () => { }); }); + test("settleSupersededWorkspaceTurnContinuation settles the abandoned continuation and wakes the waiter", async () => { + // The target abandoned a compaction follow-up carrying this correlation (a manual send + // won the idle race): no stream-end will ever carry the correlation again, so the + // abandonment itself settles the handle (Codex P2 PRRT_kwDOPxxmWM6fGVxG). + const { parentId, taskService } = await startWorkspaceTurnForTest(); + const waited = workspaceTurnManagerFor(taskService) + .waitForWorkspaceTurn("wst_handle", { requestingWorkspaceId: parentId, timeoutMs: 5_000 }) + .then( + () => null, + (error: unknown) => error + ); + + await taskService.settleSupersededWorkspaceTurnContinuation( + "childworkspace", + workspaceTurnMuxMetadata(parentId) + ); + + const error = await waited; + expect(error).toBeInstanceOf(Error); + expect((error as Error).message).toContain("superseded by new input"); + expect(await workspaceTurnSnapshot(taskService, parentId)).toMatchObject({ + status: "interrupted", + error: + "Workspace turn superseded by new input in the target workspace; the workspace continues under that input and this delegated turn will not report", + }); + + // Idempotent on a settled record, and a no-op for a correlation that is not this turn. + await taskService.settleSupersededWorkspaceTurnContinuation( + "childworkspace", + workspaceTurnMuxMetadata(parentId) + ); + expect(await workspaceTurnSnapshot(taskService, parentId)).toMatchObject({ + status: "interrupted", + }); + }); + + test("settleSupersededWorkspaceTurnContinuation ignores a stale correlation", async () => { + const { parentId, taskService } = await startWorkspaceTurnForTest(); + + await taskService.settleSupersededWorkspaceTurnContinuation( + "childworkspace", + workspaceTurnMuxMetadata(parentId, "wst_handle", "some-other-turn") + ); + await taskService.settleSupersededWorkspaceTurnContinuation( + "someone-else", + workspaceTurnMuxMetadata(parentId) + ); + + expect(await workspaceTurnSnapshot(taskService, parentId)).toMatchObject({ status: "running" }); + }); + const OWNER_FOLLOW_UP_SUPERSEDE_PREFIX = "Workspace turn superseded by follow-up turn "; function ownerFollowUpCutter(ownerWorkspaceId: string, successorHandleId: string) { diff --git a/src/node/services/taskService.ts b/src/node/services/taskService.ts index 1f75ba0f45..a5f88f2d11 100644 --- a/src/node/services/taskService.ts +++ b/src/node/services/taskService.ts @@ -6135,6 +6135,16 @@ export class TaskService implements AgentTaskIntegration { } } + async settleSupersededWorkspaceTurnContinuation( + workspaceId: string, + muxMetadata: Extract + ): Promise { + await this.getWorkspaceTurnManager().settleSupersededWorkspaceTurnContinuation( + workspaceId, + muxMetadata + ); + } + /** * Reject all foreground task waiters for a workspace that opted into backgrounding * when a new message is queued. Returns the number of waiters signaled. diff --git a/src/node/services/taskWorkspaceSeam.testUtils.ts b/src/node/services/taskWorkspaceSeam.testUtils.ts index 84bad1689c..ed53f5f066 100644 --- a/src/node/services/taskWorkspaceSeam.testUtils.ts +++ b/src/node/services/taskWorkspaceSeam.testUtils.ts @@ -70,6 +70,7 @@ export function makeAgentTaskIntegrationFake( getAgentTaskStatus: () => undefined, resetAutoResumeCount: () => undefined, backgroundForegroundWaitsForWorkspace: () => 0, + settleSupersededWorkspaceTurnContinuation: () => Promise.resolve(), markInterruptedTaskRunning: () => Promise.resolve(false), restoreInterruptedTaskAfterResumeFailure: () => Promise.resolve(), markParentWorkspaceInterrupted: () => undefined, diff --git a/src/node/services/taskWorkspaceSeam.ts b/src/node/services/taskWorkspaceSeam.ts index 342e8de393..2245c9d06f 100644 --- a/src/node/services/taskWorkspaceSeam.ts +++ b/src/node/services/taskWorkspaceSeam.ts @@ -514,6 +514,15 @@ export interface AgentTaskIntegration { getAgentTaskStatus(workspaceId: string): AgentTaskStatus | null | undefined; resetAutoResumeCount(workspaceId: string): void; backgroundForegroundWaitsForWorkspace(workspaceId: string): number; + /** + * The workspace dropped the continuation of a delegated turn (a compaction follow-up that + * carried the correlation was abandoned instead of dispatched). No later send inherits the + * correlation, so the owner's waiter is settled as superseded. Idempotent. + */ + settleSupersededWorkspaceTurnContinuation( + workspaceId: string, + muxMetadata: Extract + ): Promise; markInterruptedTaskRunning(workspaceId: string): Promise; restoreInterruptedTaskAfterResumeFailure(workspaceId: string): Promise; markParentWorkspaceInterrupted(workspaceId: string): void; diff --git a/src/node/services/workspaceService.test.ts b/src/node/services/workspaceService.test.ts index 4039a76aa0..aafd6b3113 100644 --- a/src/node/services/workspaceService.test.ts +++ b/src/node/services/workspaceService.test.ts @@ -1147,6 +1147,35 @@ describe("WorkspaceService bash monitor wake reconciler wiring", () => { } }); + test("routes the session's tool-end yield edge to backgroundForegroundWaitsForWorkspace", async () => { + // Which transitions raise the edge is the session's business + // (agentSession.queueDispatch.test.ts); the service only routes it. + const { service, cleanup } = await createWakeWiringService(); + const workspaceId = "yield-edge-owner"; + const backgroundForegroundWaitsForWorkspace = mock(() => 0); + service.setAgentTaskIntegration( + makeAgentTaskIntegrationFake({ backgroundForegroundWaitsForWorkspace }) + ); + const session = service.getOrCreateSession(workspaceId); + try { + session.queueMessage("later", { + model: "anthropic:claude-sonnet-4-5", + agentId: "exec", + queueDispatchMode: "turn-end", + }); + expect(backgroundForegroundWaitsForWorkspace).not.toHaveBeenCalled(); + session.queueMessage("sooner", { + model: "anthropic:claude-sonnet-4-5", + agentId: "exec", + queueDispatchMode: "tool-end", + }); + expect(backgroundForegroundWaitsForWorkspace).toHaveBeenCalledWith(workspaceId); + } finally { + session.dispose(); + await cleanup(); + } + }); + test("a high level backgrounds foreground waits only when it pulls the yield lever", async () => { // A turn-end queue head suppresses the wake cut (hasPendingToolEndInput arbitration), so // the same level must not end a foreground task_await early either: the stream would @@ -1187,9 +1216,13 @@ describe("WorkspaceService bash monitor wake reconciler wiring", () => { expect(await internal.bashMonitorWakeReconciler.hasOutstandingWake(workspaceId)).toBe(true); expect(backgroundForegroundWaitsForWorkspace).not.toHaveBeenCalled(); + // Clearing the turn-end head makes the (still high) level effective with no enqueue + // and no level publish in between: the session's yield edge alone must background the + // waits (Codex P2 PRRT_kwDOPxxmWM6fGVw_). session.clearQueue(); - expect(await internal.bashMonitorWakeReconciler.hasOutstandingWake(workspaceId)).toBe(true); expect(backgroundForegroundWaitsForWorkspace).toHaveBeenCalledWith(workspaceId); + expect(await internal.bashMonitorWakeReconciler.hasOutstandingWake(workspaceId)).toBe(true); + expect(backgroundForegroundWaitsForWorkspace).toHaveBeenCalledTimes(1); } finally { session.dispose(); await cleanup(); @@ -9697,63 +9730,6 @@ describe("WorkspaceService sendMessage status clearing", () => { expect(await settled).toBeInstanceOf(Error); }); - // The sticky case: incoming mode is turn-end but the queue's effective mode is - // tool-end from a prior enqueue, so the wait still backgrounds. - test.each([ - [ - "backgrounds foreground task waits when queuing a tool-end message", - "tool-end", - "hello", - undefined, - true, - ], - [ - "does not background foreground task waits when queuing a turn-end message", - "turn-end", - "hello", - "turn-end", - false, - ], - [ - "does not background foreground task waits when queueMessage enqueues nothing", - null, - " ", - undefined, - false, - ], - [ - "backgrounds foreground task waits when effective queue mode is tool-end despite incoming turn-end", - "tool-end", - "hello", - "turn-end", - true, - ], - ] as const)( - "%s", - async (_name, effectiveQueueMode, message, queueDispatchMode, expectBackgrounded) => { - fakeSession.isBusy.mockReturnValue(true); - fakeSession.queueMessage.mockReturnValue(effectiveQueueMode); - - const backgroundForegroundWaitsForWorkspace = mock(() => 0); - workspaceService.setAgentTaskIntegration( - makeAgentTaskIntegrationFake({ backgroundForegroundWaitsForWorkspace }) - ); - - const result = await workspaceService.sendMessage("test-workspace", message, { - model: "openai:gpt-4o-mini", - agentId: "exec", - queueDispatchMode, - }); - - expect(result.success).toBe(true); - if (expectBackgrounded) { - expect(backgroundForegroundWaitsForWorkspace).toHaveBeenCalledWith("test-workspace"); - } else { - expect(backgroundForegroundWaitsForWorkspace).not.toHaveBeenCalled(); - } - } - ); - test("registerSession clears persisted agent status for accepted user chat events", () => { const updateAgentStatus = spyOn( workspaceService as unknown as { diff --git a/src/node/services/workspaceService.ts b/src/node/services/workspaceService.ts index 6b165e15bd..661fe1287c 100644 --- a/src/node/services/workspaceService.ts +++ b/src/node/services/workspaceService.ts @@ -2389,6 +2389,8 @@ export class WorkspaceService extends EventEmitter implements WorkspaceHost { // The level drives the same tool-boundary side effects a queued tool-end message // does: long-polling bash reads return early and foreground agent-task waits are // backgrounded so the stream can reach the boundary where it yields to the wake. + // The session arbitrates the level against its queue head and fires the yield + // edge (onToolEndYieldRequested) when the lever actually becomes effective. const session = this.sessions.get(ownerWorkspaceId); if (session != null) { session.setBashMonitorWakeOutstanding(outstanding); @@ -2401,18 +2403,6 @@ export class WorkspaceService extends EventEmitter implements WorkspaceHost { // disposal after the session went away), so drop it directly. this.backgroundProcessManager.setMessageQueued(ownerWorkspaceId, false); } - // Background foreground waits only when the level actually pulls the yield lever: - // the session's mirror applies the queue-head arbitration (a turn-end head means - // the stream will not cut for this wake, so ending the wait early would just hand - // the agent another model step) (Codex P2 PRRT_kwDOPxxmWM6fFJ4Q). - if ( - outstanding && - session != null && - typeof this.backgroundProcessManager.hasQueuedMessage === "function" && - this.backgroundProcessManager.hasQueuedMessage(ownerWorkspaceId) - ) { - this.agentTaskIntegration?.backgroundForegroundWaitsForWorkspace(ownerWorkspaceId); - } }, }); if (typeof this.backgroundProcessManager.on === "function") { @@ -4053,6 +4043,19 @@ export class WorkspaceService extends EventEmitter implements WorkspaceHost { hasExternalSendPreflight: () => this.hasSessionInvisiblePreflight(workspaceId), hasOutstandingBashMonitorWake: () => this.bashMonitorWakeReconciler.hasOutstandingWake(workspaceId), + // The stream will cut at its next tool boundary; a foreground task_await that would + // outlive it is backgrounded so the boundary is reached. Waits registered while the + // lever is already high are backgrounded at registration + // (TaskService.backgroundForegroundWaitIfQueued → isToolEndYieldRequested). + onToolEndYieldRequested: () => { + this.agentTaskIntegration?.backgroundForegroundWaitsForWorkspace(workspaceId); + }, + onWorkspaceTurnContinuationAbandoned: async (metadata) => { + await this.agentTaskIntegration?.settleSupersededWorkspaceTurnContinuation( + workspaceId, + metadata + ); + }, }); } @@ -10979,10 +10982,6 @@ export class WorkspaceService extends EventEmitter implements WorkspaceHost { this.agentTaskIntegration?.resetAutoResumeCount(workspaceId); } - if (effectiveQueueDispatchMode === "tool-end") { - this.agentTaskIntegration?.backgroundForegroundWaitsForWorkspace(workspaceId); - } - return Ok(undefined); } diff --git a/src/node/services/workspaceTurnManager.ts b/src/node/services/workspaceTurnManager.ts index 36f9a12884..0f1a0fbd95 100644 --- a/src/node/services/workspaceTurnManager.ts +++ b/src/node/services/workspaceTurnManager.ts @@ -4678,6 +4678,27 @@ export class WorkspaceTurnManager { }); } + /** + * The target workspace abandoned the continuation carrying this correlation (a wake's + * compaction follow-up skipped for a racing manual send, or an inadmissible summary). + * Nothing downstream can settle the turn: the compaction stream-end is ignored by + * finalizeWorkspaceTurnFromStreamEnd and no later send inherits the correlation, so the + * owner would wait until restart. Settle it as superseded now; if the manual turn does run, + * its uncorrelated stream-end finds the record already settled (Codex P2 + * PRRT_kwDOPxxmWM6fGVxG). + */ + async settleSupersededWorkspaceTurnContinuation( + workspaceId: string, + muxMetadata: WorkspaceTurnMuxMetadata + ): Promise { + await this.settleWorkspaceTurnContinuationFailure( + workspaceId, + muxMetadata, + "interrupted", + WORKSPACE_TURN_SUPERSEDED_BY_NEW_INPUT_ERROR + ); + } + // A queued report can defer the preceding stream-end. If dispatch then fails, settle that // exact turn here because no replacement stream-end can arrive. async settleWorkspaceTurnContinuationFailure( From 44c09f287cdc33a1d88d06ab0808ef209cfa879d Mon Sep 17 00:00:00 2001 From: Ethan Dickson Date: Fri, 4 Sep 2026 08:44:38 +0000 Subject: [PATCH 08/26] Codex round 6: cutter-gated wake deferral, wake goal-sync retry, settle-before-erase - WorkspaceTurnManager.hasSameTurnContinuation consults the wake level only when the event-time cutter is the wake itself; a manual tool-end head owns the cut and settles the handle (PRRT_kwDOPxxmWM6fOH50). - A goal-sync failure on a durable bash-monitor wake row arms the in-session auto-retry resume before consuming the signal and schedules it (PRRT_kwDOPxxmWM6fOH54). - clearPendingFollowUpFromSummary settles the abandoned delegated turn before erasing the durable follow-up so a settlement failure stays retryable (PRRT_kwDOPxxmWM6fOH59). --- ...gentSession.continueMessageAgentId.test.ts | 29 +++++++++++-- .../agentSession.queueDispatch.test.ts | 23 ++++++++++ src/node/services/agentSession.ts | 30 ++++++++++--- src/node/services/taskService.test.ts | 42 ++++++++++++++++++- src/node/services/workspaceTurnManager.ts | 40 +++++++++++------- 5 files changed, 137 insertions(+), 27 deletions(-) diff --git a/src/node/services/agentSession.continueMessageAgentId.test.ts b/src/node/services/agentSession.continueMessageAgentId.test.ts index 4cb81ac124..66c6901697 100644 --- a/src/node/services/agentSession.continueMessageAgentId.test.ts +++ b/src/node/services/agentSession.continueMessageAgentId.test.ts @@ -379,9 +379,10 @@ describe("AgentSession continue-message agentId fallback", () => { workspaceTurnMetadata, dispatchOptions: { requireIdle: true }, }; + let settlementError: Error | undefined = new Error("task handle store unavailable"); const abandoned = mock( (_metadata: NonNullable) => - Promise.resolve() + settlementError != null ? Promise.reject(settlementError) : Promise.resolve() ); const { session, historyService, internals } = await createSession( [compactionSummaryMessage("summary-wake", wakeFollowUp)], @@ -391,10 +392,30 @@ describe("AgentSession continue-message agentId fallback", () => { (session as unknown as { hasExternalSendPreflight?: () => boolean }).hasExternalSendPreflight = () => true; - expect(await internals.dispatchPendingFollowUp()).toBe(false); + // Settlement runs BEFORE the follow-up is erased: a failure keeps the durable record (the + // only carrier of the correlation) so the next attempt retries it (Codex P2 + // PRRT_kwDOPxxmWM6fOH59). + let dispatchError: unknown; + try { + await internals.dispatchPendingFollowUp(); + } catch (error) { + dispatchError = error; + } + expect(dispatchError).toBeInstanceOf(Error); + expect((dispatchError as Error).message).toContain("task handle store unavailable"); expect(internals.sendMessage).not.toHaveBeenCalled(); expect(abandoned).toHaveBeenCalledTimes(1); - expect(abandoned).toHaveBeenCalledWith(workspaceTurnMetadata); + const retained = await historyService.getLastMessages("ws", 1); + expect(retained.success && retained.data[0]?.metadata?.muxMetadata).toMatchObject({ + type: "compaction-summary", + pendingFollowUp: { workspaceTurnMetadata }, + }); + + settlementError = undefined; + expect(await internals.dispatchPendingFollowUp()).toBe(false); + expect(internals.sendMessage).not.toHaveBeenCalled(); + expect(abandoned).toHaveBeenCalledTimes(2); + expect(abandoned).toHaveBeenLastCalledWith(workspaceTurnMetadata); const lastMessages = await historyService.getLastMessages("ws", 1); expect(lastMessages.success && lastMessages.data[0]?.metadata?.muxMetadata).toEqual({ type: "compaction-summary", @@ -409,7 +430,7 @@ describe("AgentSession continue-message agentId fallback", () => { ); expect(await internals.dispatchPendingFollowUp()).toBe(true); expect(internals.sendMessage).toHaveBeenCalledTimes(1); - expect(abandoned).toHaveBeenCalledTimes(1); + expect(abandoned).toHaveBeenCalledTimes(2); }); test("dispatchPendingFollowUp removes heartbeat reset boundaries when idle-only follow-ups are skipped", async () => { diff --git a/src/node/services/agentSession.queueDispatch.test.ts b/src/node/services/agentSession.queueDispatch.test.ts index d90e1b0f15..784fbb965f 100644 --- a/src/node/services/agentSession.queueDispatch.test.ts +++ b/src/node/services/agentSession.queueDispatch.test.ts @@ -1025,6 +1025,20 @@ describe("AgentSession queued message tool-call dispatch", () => { try { const canceledReasons: string[] = []; let accepted = false; + // Acceptance consumes the reconciler signal, so an in-session resume must already be + // armed by then: nothing upstream can resend the durable row (Codex P2 + // PRRT_kwDOPxxmWM6fOH54). + let resumeArmedAtAcceptance = false; + const chatEventTypes: string[] = []; + const unsubscribe = session.onChatEvent((event) => { + chatEventTypes.push(event.message.type); + }); + const readResumeRequest = () => + ( + session as unknown as { + lastAutoRetryResumeRequest?: { options: { muxMetadata?: unknown } }; + } + ).lastAutoRetryResumeRequest; const sendPromise = session.sendMessage( "Background monitor wake", { @@ -1040,11 +1054,13 @@ describe("AgentSession queued message tool-call dispatch", () => { }, onAccepted: () => { accepted = true; + resumeArmedAtAcceptance = readResumeRequest() != null; }, } ); await syncStarted; + expect(readResumeRequest()).toBeUndefined(); releaseSync(); let syncError: unknown; try { @@ -1052,11 +1068,18 @@ describe("AgentSession queued message tool-call dispatch", () => { } catch (error) { syncError = error; } + unsubscribe(); expect(syncError).toBeInstanceOf(Error); expect((syncError as Error).message).toContain("injected goal sync failure"); expect(accepted).toBe(true); expect(canceledReasons).toEqual([]); + expect(resumeArmedAtAcceptance).toBe(true); + expect(readResumeRequest()?.options.muxMetadata).toEqual({ + type: "bash-monitor-wake", + records: [], + }); + expect(chatEventTypes).toContain("auto-retry-scheduled"); const history = await historyService.getHistoryFromLatestBoundary(workspaceId); expect(history.success).toBe(true); if (history.success) { diff --git a/src/node/services/agentSession.ts b/src/node/services/agentSession.ts index 0af4b23add..4618959e9a 100644 --- a/src/node/services/agentSession.ts +++ b/src/node/services/agentSession.ts @@ -3963,7 +3963,20 @@ export class AgentSession { await this.workspaceGoalService?.syncGoalModeWithChatTail(this.workspaceId); } catch (error) { if (finalizeDurableWakeOnFailure) { + // Consuming the signal is what stops the reconciler from re-deriving this wake, so + // from that moment the durable row is the only carrier of the turn and nothing + // upstream can resend it. Arm the same in-session resume a failed stream start uses + // BEFORE consuming, then schedule it: auto-retry resumes the durable row without + // appending a second one, so the row (and any delegated turn waiting on its + // continuation) no longer depends on an application restart. With auto-retry + // disabled by the user this stays a startup-recovery row like every other + // pre-stream failure (Codex P2 PRRT_kwDOPxxmWM6fOH54). + this.setAutoRetryResumeState(optionsForStream, agentInitiated, goalKind, internal?.goalId); await internal?.onAccepted?.(); + await this.handleStreamFailureForAutoRetry({ + type: "unknown", + message: getErrorMessage(error), + }); } throw error; } @@ -7589,6 +7602,17 @@ export class AgentSession { return; } + // Every discard path funnels here, so this is the one place that knows the delegated + // turn's continuation is gone for good (Codex P2 PRRT_kwDOPxxmWM6fGVxG). Settle BEFORE + // erasing the follow-up: the durable record is the only carrier of the correlation, so + // a settlement failure must leave it in place for the next dispatch attempt (or startup + // recovery) to retry — settlement is idempotent, a lost record is not recoverable + // (Codex P2 PRRT_kwDOPxxmWM6fOH59). + const workspaceTurnMetadata = muxMeta.pendingFollowUp.workspaceTurnMetadata; + if (workspaceTurnMetadata != null) { + await this.onWorkspaceTurnContinuationAbandoned?.(workspaceTurnMetadata); + } + const { pendingFollowUp: _pendingFollowUp, ...muxMetadataWithoutFollowUp } = muxMeta; const updateResult = await this.historyService.updateHistory(this.workspaceId, { ...summaryMessage, @@ -7600,12 +7624,6 @@ export class AgentSession { if (!updateResult.success) { throw new Error(`Failed to clear skipped pending follow-up: ${updateResult.error}`); } - // Every discard path funnels here, so this is the one place that knows the delegated - // turn's continuation is gone for good (Codex P2 PRRT_kwDOPxxmWM6fGVxG). - const workspaceTurnMetadata = muxMeta.pendingFollowUp.workspaceTurnMetadata; - if (workspaceTurnMetadata != null) { - await this.onWorkspaceTurnContinuationAbandoned?.(workspaceTurnMetadata); - } } /** diff --git a/src/node/services/taskService.test.ts b/src/node/services/taskService.test.ts index 55c0250734..9897415288 100644 --- a/src/node/services/taskService.test.ts +++ b/src/node/services/taskService.test.ts @@ -24370,9 +24370,12 @@ describe("TaskService", () => { const hasOutstandingBashMonitorWake = mock((workspaceId: string) => Promise.resolve(workspaceId === "childworkspace") ); - const { parentId, taskService } = await startWorkspaceTurnForTest({ + const { parentId, taskService, workspaceMocks } = await startWorkspaceTurnForTest({ hasOutstandingBashMonitorWake, }); + workspaceMocks.getQueueCutCutter.mockImplementation(() => ({ + stage: "bash-monitor-wake" as const, + })); const internal = taskService as unknown as { handleStreamEnd: (event: StreamEndEvent) => Promise; }; @@ -24419,6 +24422,43 @@ describe("TaskService", () => { }); }); + test("a manual tool-end head owns the cut even while the wake level is high", async () => { + // The session attributes the cut to the queued entry (hasPendingToolEndInput: a queue + // head arbitrates alone), which runs first and breaks correlation inheritance; the wake + // behind it is not this turn's continuation. Deferring on the level would leave the + // handle running with no correlated stream-end to come (Codex P2 PRRT_kwDOPxxmWM6fOH50). + const hasOutstandingBashMonitorWake = mock(() => Promise.resolve(true)); + const { parentId, taskService, workspaceMocks } = await startWorkspaceTurnForTest({ + hasOutstandingBashMonitorWake, + }); + workspaceMocks.getQueueCutCutter.mockImplementation(() => ({ + stage: "queued" as const, + muxMetadata: undefined, + dispatchMode: "tool-end" as const, + })); + const internal = taskService as unknown as { + handleStreamEnd: (event: StreamEndEvent) => Promise; + }; + + await internal.handleStreamEnd({ + type: "stream-end", + workspaceId: "childworkspace", + messageId: "msg_manual_cut", + metadata: { + model: "anthropic:claude-opus-4-6", + agentId: "exec", + finishReason: "tool-calls", + muxMetadata: workspaceTurnMuxMetadata(parentId), + }, + parts: [{ type: "text", text: "Kicked off verification" }], + }); + + expect(hasOutstandingBashMonitorWake).not.toHaveBeenCalled(); + const settled = await workspaceTurnSnapshot(taskService, parentId); + expect(settled?.status).not.toBe("running"); + expect(settled).toMatchObject({ messageId: "msg_manual_cut" }); + }); + test("a wake retracted after the cut settles the handle as a wake cut instead of deferring", async () => { // The stream yielded to the wake level, then the operator canceled the monitor before // this stream-end was processed: the level is low and no wake turn was admitted, so no diff --git a/src/node/services/workspaceTurnManager.ts b/src/node/services/workspaceTurnManager.ts index 0f1a0fbd95..bfb39a9d28 100644 --- a/src/node/services/workspaceTurnManager.ts +++ b/src/node/services/workspaceTurnManager.ts @@ -4327,7 +4327,8 @@ export class WorkspaceTurnManager { */ private async hasSameTurnContinuation( event: StreamEndEvent, - correlation: { taskHandleId: string; ownerWorkspaceId: string; turnId: string } + correlation: { taskHandleId: string; ownerWorkspaceId: string; turnId: string }, + queueCutSnapshot: QueueCutAttributionSnapshot ): Promise { if ( this.workspaceService.hasPendingWorkspaceTurnContinuation(event.workspaceId, { @@ -4337,21 +4338,28 @@ export class WorkspaceTurnManager { ) { return true; } - // A stream that ended with "tool-calls" while a wake is outstanding yielded to that - // wake; the wake turn inherits this correlation (inheritOpenWorkspaceTurnMetadata). - // The probe is advisory: if its I/O fails, settle through the normal path rather - // than leave the handle running with its terminal stream-end already consumed (a - // late correlated continuation can still self-heal it) (Codex P2 PRRT_kwDOPxxmWM6fEQIr). - try { - if (await this.workspaceService.hasOutstandingBashMonitorWake(event.workspaceId)) { - return true; + // A stream that yielded to the wake level continues through the wake turn, which + // inherits this correlation from history (inheritOpenWorkspaceTurnMetadata). Only the + // event-time attribution says whether the level was the cutter: a manual tool-end head + // arbitrates the cut even while the level is high, runs first and breaks inheritance, so + // the wake behind it is not this turn's continuation and the handle must settle here + // instead of waiting on a stream-end that may never correlate (Codex P2 + // PRRT_kwDOPxxmWM6fOH50). Whether the wake still arrives is then read live: the probe is + // advisory, so if its I/O fails settle through the normal path rather than leave the + // handle running with its terminal stream-end already consumed (a late correlated + // continuation can still self-heal it) (Codex P2 PRRT_kwDOPxxmWM6fEQIr). + if (queueCutSnapshot.cutter?.stage === "bash-monitor-wake") { + try { + if (await this.workspaceService.hasOutstandingBashMonitorWake(event.workspaceId)) { + return true; + } + } catch (error) { + log.warn("Bash monitor wake probe failed during workspace turn settlement", { + workspaceId: event.workspaceId, + taskHandleId: correlation.taskHandleId, + error, + }); } - } catch (error) { - log.warn("Bash monitor wake probe failed during workspace turn settlement", { - workspaceId: event.workspaceId, - taskHandleId: correlation.taskHandleId, - error, - }); } const activeStream = this.streamManager?.getStreamInfo(event.workspaceId); if (activeStream == null || activeStream.messageId === event.messageId) { @@ -4522,7 +4530,7 @@ export class WorkspaceTurnManager { // must settle the old outcome here. if ( event.metadata.finishReason === "tool-calls" && - (await this.hasSameTurnContinuation(event, metadata)) + (await this.hasSameTurnContinuation(event, metadata, queueCutSnapshot)) ) { await this.markWorkspaceTurnStreamEndDeferred(event); return true; From 81594ca3892d8aff4ba76a39f75f5c06ba678bd3 Mon Sep 17 00:00:00 2001 From: Ethan Dickson Date: Fri, 4 Sep 2026 09:05:10 +0000 Subject: [PATCH 09/26] Codex round 7: keep wake identity through on-send compaction turnCarriesBashMonitorWake (was preparingBashMonitorWake) is derived from the admitted metadata by looking through a compaction-request to its follow-up, and outlives a compaction stream (whose metadata cannot show the wake) until the follow-up dispatches in COMPLETING or the turn ends (PRRT_kwDOPxxmWM6fOf9G). --- .../agentSession.autoCompaction.test.ts | 64 +++++++++++++++++++ src/node/services/agentSession.ts | 48 ++++++++++---- src/node/services/workspaceService.ts | 5 +- 3 files changed, 102 insertions(+), 15 deletions(-) diff --git a/src/node/services/agentSession.autoCompaction.test.ts b/src/node/services/agentSession.autoCompaction.test.ts index 053798a10e..68eb0fa695 100644 --- a/src/node/services/agentSession.autoCompaction.test.ts +++ b/src/node/services/agentSession.autoCompaction.test.ts @@ -1378,6 +1378,8 @@ describe("AgentSession on-send auto-compaction for synthetic guidance sends", () async function createGuidanceHarness(args: { workspaceId: string; summaryText?: string; + /** Observes each stream request (1-based) just before and just after its stream-start. */ + onStreamRequest?: (index: number, phase: "preparing" | "streaming") => void; }): Promise { const workspaceId = args.workspaceId; const streamHistories: MuxMessage[][] = []; @@ -1390,6 +1392,7 @@ describe("AgentSession on-send auto-compaction for synthetic guidance sends", () : undefined; streamHistories.push(Array.isArray(requestMessages) ? (requestMessages as MuxMessage[]) : []); + args.onStreamRequest?.(streamHistories.length, "preparing"); aiEmitter.emit("stream-start", { type: "stream-start", workspaceId, @@ -1398,6 +1401,7 @@ describe("AgentSession on-send auto-compaction for synthetic guidance sends", () historySequence: streamHistories.length, startTime: Date.now(), }); + args.onStreamRequest?.(streamHistories.length, "streaming"); const usage = { inputTokens: 42, @@ -1529,6 +1533,66 @@ describe("AgentSession on-send auto-compaction for synthetic guidance sends", () fixture.session.dispose(); }); + test("a wake consumed by on-send compaction stays a pending wake turn until its follow-up streams", async () => { + // The wake's onAccepted lowers the reconciler level, and the compaction turn's metadata is + // the compaction request — so while the compaction prepares/streams and until the follow-up + // dispatches, this marker is the only thing that tells delegated-turn settlement the wake + // continuation is still coming (Codex P1 PRRT_kwDOPxxmWM6fOf9G). + const observed: Array<[number, "preparing" | "streaming", boolean]> = []; + const fixtureRef: { session?: AgentSession } = {}; + const fixture = await createGuidanceHarness({ + workspaceId: "ws-auto-compaction-wake-identity", + onStreamRequest: (index, phase) => { + observed.push([index, phase, fixtureRef.session?.hasPendingBashMonitorWakeTurn() ?? false]); + }, + }); + fixtureRef.session = fixture.session; + // Compact the wake once; its follow-up must then stream as the wake itself. + let compactionChecks = 0; + ( + fixture.session as unknown as { compactionMonitor: CompactionMonitor } + ).compactionMonitor.checkBeforeSend = () => { + compactionChecks += 1; + return { + shouldShowWarning: compactionChecks === 1, + shouldForceCompact: compactionChecks === 1, + usagePercentage: compactionChecks === 1 ? 95 : 10, + thresholdPercentage: 70, + }; + }; + + const result = await fixture.session.sendMessage( + "READY", + { + model: "openai:gpt-4o", + agentId: "exec", + muxMetadata: { type: "bash-monitor-wake", records: [] }, + }, + { synthetic: true, agentInitiated: true, startStreamInBackground: true } + ); + expect(result.success).toBe(true); + expect(await waitFor(() => fixture.streamHistories.length >= 2)).toBe(true); + + expect(fixture.streamHistories[0].at(-1)?.metadata?.muxMetadata?.type).toBe( + "compaction-request" + ); + expect(fixture.streamHistories[1].at(-1)?.metadata?.muxMetadata?.type).toBe( + "bash-monitor-wake" + ); + expect(observed).toEqual([ + // Compaction turn: preparing and streaming both still carry the wake. + [1, "preparing", true], + [1, "streaming", true], + // Follow-up wake turn: preparing carries it; its own stream shows it instead. + [2, "preparing", true], + [2, "streaming", false], + ]); + expect(await waitFor(() => !fixture.session.isBusy())).toBe(true); + expect(fixture.session.hasPendingBashMonitorWakeTurn()).toBe(false); + + fixture.session.dispose(); + }); + // Characterization: sends carrying preTurnMessages (family-message payloads) // intentionally skip on-send compaction. The trigger row references its // payload by message ID, so compacting the payload away would dangle that diff --git a/src/node/services/agentSession.ts b/src/node/services/agentSession.ts index 4618959e9a..de50bb9853 100644 --- a/src/node/services/agentSession.ts +++ b/src/node/services/agentSession.ts @@ -444,6 +444,21 @@ function isCompactionRequestMetadata(meta: unknown): meta is CompactionRequestMe return true; } +/** + * Whether admitted send metadata carries a bash-monitor wake: the wake itself, or an on-send + * compaction request whose follow-up is the wake. The compaction row is what the turn shows + * (optionsForStream.muxMetadata, the stream's metadata), so the wake identity has to be read + * through it (Codex P1 PRRT_kwDOPxxmWM6fOf9G). + */ +function carriesBashMonitorWake(muxMetadata: unknown): boolean { + const meta = muxMetadata as MuxMessageMetadata | undefined; + if (meta?.type === "bash-monitor-wake") return true; + if (!isCompactionRequestMetadata(meta)) return false; + const followUpMetadata = + meta.parsed.followUpContent?.muxMetadata ?? meta.parsed.continueMessage?.muxMetadata; + return followUpMetadata?.type === "bash-monitor-wake"; +} + const AUTO_RETRY_PREFERENCE_FILE = "auto-retry-preference.json"; /** @@ -890,11 +905,16 @@ export class AgentSession { /** Correlation of the direct send currently in the PREPARING phase, if any. */ private preparingWorkspaceTurnMetadata?: WorkspaceTurnMuxMetadata; /** - * The send in the PREPARING phase is a bash-monitor wake. Its `onAccepted` lowers the - * reconciler level once the user row is durable, before the stream is observable, so - * this marker is what keeps the wake continuation visible across that window. + * The turn in flight carries a bash-monitor wake: the admitted send is the wake itself, or + * an on-send compaction whose follow-up is (carriesBashMonitorWake). The wake's `onAccepted` + * lowers the reconciler level once its row is durable, so from admission on this marker is + * the only thing that keeps the continuation visible to delegated-turn settlement until a + * stream that shows it exists. A direct wake stream shows it through its inherited + * correlation, so the marker drops at that stream's start; a compaction stream does not + * (its metadata is the compaction request), so the marker outlives it until the follow-up + * dispatches in COMPLETING or the turn ends (Codex P1 PRRT_kwDOPxxmWM6fOf9G). */ - private preparingBashMonitorWake = false; + private turnCarriesBashMonitorWake = false; /** * The last stream cut itself for the wake level (hasPendingToolEndInput returned true * from the level) and no turn has been admitted since. This is cut *attribution* @@ -5836,7 +5856,9 @@ export class AgentSession { this.dispatchingQueuedEntry = false; this.dispatchingQueuedEntryMuxMetadata = undefined; this.preparingWorkspaceTurnMetadata = undefined; - this.preparingBashMonitorWake = false; + // A compaction stream's metadata does not show the wake its follow-up carries; keep + // the marker until the follow-up dispatches (see turnCarriesBashMonitorWake). + if (this.activeCompactionRequest == null) this.turnCarriesBashMonitorWake = false; // A live stream is the continuation (or a superseding turn) the cut waited for. this.streamYieldedToBashMonitorWake = false; this.activeStreamStartedAtMs = payload.startTime; @@ -6337,7 +6359,7 @@ export class AgentSession { this.dispatchingQueuedEntry = false; this.dispatchingQueuedEntryMuxMetadata = undefined; this.preparingWorkspaceTurnMetadata = undefined; - this.preparingBashMonitorWake = false; + this.turnCarriesBashMonitorWake = false; // Turn ended: expire any mid-turn thinking override. Safe unconditionally // because a replacement turn (e.g. an edit) only creates its holder after // the preempted turn has already been transitioned to IDLE. @@ -6781,21 +6803,21 @@ export class AgentSession { /** Claim PREPARING for a send and record what kind of input it carries. */ private enterPreparing(muxMetadata: unknown): void { this.preparingWorkspaceTurnMetadata = getWorkspaceTurnMuxMetadata(muxMetadata); - this.preparingBashMonitorWake = - (muxMetadata as MuxMessageMetadata | undefined)?.type === "bash-monitor-wake"; + this.turnCarriesBashMonitorWake = carriesBashMonitorWake(muxMetadata); // Whatever is admitted now (the wake turn, or input superseding it) settles the cut. this.streamYieldedToBashMonitorWake = false; this.setTurnPhase(TurnPhase.PREPARING); } /** - * A bash-monitor wake turn between admission and stream start (direct send in PREPARING, - * or a dequeued wake entry). The reconciler level is already low here — `onAccepted` - * ran when the user row became durable — but no replacement stream is observable yet, - * so delegated-turn settlement must still see the continuation. + * A bash-monitor wake turn admitted but not yet shown by a correlated stream: a direct wake + * send in PREPARING, a dequeued wake entry, or an on-send compaction turn (preparing, + * streaming, or dispatching its follow-up) that consumed the wake. The reconciler level is + * already low here — `onAccepted` ran when the durable row landed — so delegated-turn + * settlement must read the continuation from this marker. */ hasPendingBashMonitorWakeTurn(): boolean { - if (this.preparingBashMonitorWake) return true; + if (this.turnCarriesBashMonitorWake) return true; const dispatching = this.dispatchingQueuedEntryMuxMetadata as MuxMessageMetadata | undefined; return dispatching?.type === "bash-monitor-wake"; } diff --git a/src/node/services/workspaceService.ts b/src/node/services/workspaceService.ts index 948c2f0ead..f82e00ac8e 100644 --- a/src/node/services/workspaceService.ts +++ b/src/node/services/workspaceService.ts @@ -11969,8 +11969,9 @@ export class WorkspaceService extends EventEmitter implements WorkspaceHost { /** * The bash-monitor wake level: a wake the workspace has not seen yet, or a wake turn - * admitted but not yet streaming (AgentSession.hasPendingBashMonitorWakeTurn — the - * reconciler level is already consumed there). A stream that ended with "tool-calls" + * admitted but not yet shown by a correlated stream — including an on-send compaction + * that consumed it (AgentSession.hasPendingBashMonitorWakeTurn — the reconciler level is + * already consumed there). A stream that ended with "tool-calls" * while this is high yielded to the wake and will be continued by it. Deliberately not * the session's cut latch: a wake retracted after the cut (monitor canceled) has no * continuation, and settlement must not defer on it (AgentSession.getQueueCutCutter From ab1f49a5a4e3ef6c49e89e0f77a8c3e528aa56aa Mon Sep 17 00:00:00 2001 From: Ethan Dickson Date: Fri, 4 Sep 2026 10:18:41 +0000 Subject: [PATCH 10/26] Redesign bash-monitor wake continuation as a session debt + reconciler lease - AgentSession: wakeContinuationDebt/wakeTurnInFlight replace the scattered mirrors; debt taken at the cut, redeemed at the wake stream start, voided (retracted/superseded/abandoned) through one hook - BashMonitorWakeReconciler: offered/committed leases; commit retires the replacement offer and acknowledges by identity - Settlement reads the session debt synchronously; voids settle deferred records under the workspace event lock --- .../agentSession.autoCompaction.test.ts | 10 +- ...gentSession.continueMessageAgentId.test.ts | 17 +- .../agentSession.queueDispatch.test.ts | 342 ++++++++++++++---- src/node/services/agentSession.testHarness.ts | 8 +- src/node/services/agentSession.ts | 312 +++++++++++----- .../bashMonitorWakeReconciler.test.ts | 45 ++- .../services/bashMonitorWakeReconciler.ts | 202 ++++++----- src/node/services/taskService.test.ts | 146 +++++--- src/node/services/taskService.ts | 22 +- .../services/taskWorkspaceSeam.testUtils.ts | 4 +- src/node/services/taskWorkspaceSeam.ts | 23 +- src/node/services/workspaceService.test.ts | 20 +- src/node/services/workspaceService.ts | 27 +- .../services/workspaceTurnManager.test.ts | 2 +- src/node/services/workspaceTurnManager.ts | 66 ++-- 15 files changed, 851 insertions(+), 395 deletions(-) diff --git a/src/node/services/agentSession.autoCompaction.test.ts b/src/node/services/agentSession.autoCompaction.test.ts index 68eb0fa695..6648216961 100644 --- a/src/node/services/agentSession.autoCompaction.test.ts +++ b/src/node/services/agentSession.autoCompaction.test.ts @@ -1533,11 +1533,11 @@ describe("AgentSession on-send auto-compaction for synthetic guidance sends", () fixture.session.dispose(); }); - test("a wake consumed by on-send compaction stays a pending wake turn until its follow-up streams", async () => { - // The wake's onAccepted lowers the reconciler level, and the compaction turn's metadata is - // the compaction request — so while the compaction prepares/streams and until the follow-up - // dispatches, this marker is the only thing that tells delegated-turn settlement the wake - // continuation is still coming (Codex P1 PRRT_kwDOPxxmWM6fOf9G). + test("a wake consumed by on-send compaction stays in flight until its follow-up streams", async () => { + // The wake's onAccepted lowers the reconciler level, and the compaction stream's request is + // the compaction row, not the wake — so the compaction stream must not redeem the wake: + // the wake turn stays in flight through the compaction and is redeemed only by the + // follow-up's own stream (see AgentSession.wakeContinuationDebt). const observed: Array<[number, "preparing" | "streaming", boolean]> = []; const fixtureRef: { session?: AgentSession } = {}; const fixture = await createGuidanceHarness({ diff --git a/src/node/services/agentSession.continueMessageAgentId.test.ts b/src/node/services/agentSession.continueMessageAgentId.test.ts index 66c6901697..dedafe095e 100644 --- a/src/node/services/agentSession.continueMessageAgentId.test.ts +++ b/src/node/services/agentSession.continueMessageAgentId.test.ts @@ -154,7 +154,7 @@ describe("AgentSession continue-message agentId fallback", () => { messages: MuxMessage[] = [], hooks: Pick< ConstructorParameters[0], - "onWorkspaceTurnContinuationAbandoned" + "onWorkspaceTurnContinuationVoided" > = {}, config = createConfig() ) => { @@ -364,7 +364,7 @@ describe("AgentSession continue-message agentId fallback", () => { // A bash-monitor wake cut a delegated turn; the wake's on-send compaction stamped the // correlation on its follow-up. If a manual send wins the idle race, nothing else can // settle the owner's waiter (the compaction stream-end is uncorrelated and no later send - // inherits the metadata), so the discard itself must (Codex P2 PRRT_kwDOPxxmWM6fGVxG). + // inherits the metadata), so the discard itself voids the continuation. const workspaceTurnMetadata = { type: "workspace-turn-task", taskHandleId: "wst_abandoned", @@ -381,20 +381,21 @@ describe("AgentSession continue-message agentId fallback", () => { }; let settlementError: Error | undefined = new Error("task handle store unavailable"); const abandoned = mock( - (_metadata: NonNullable) => - settlementError != null ? Promise.reject(settlementError) : Promise.resolve() + ( + _metadata: NonNullable, + _reason: string + ) => (settlementError != null ? Promise.reject(settlementError) : Promise.resolve()) ); const { session, historyService, internals } = await createSession( [compactionSummaryMessage("summary-wake", wakeFollowUp)], - { onWorkspaceTurnContinuationAbandoned: abandoned } + { onWorkspaceTurnContinuationVoided: abandoned } ); internals.sendMessage = mock(() => Promise.resolve({ success: true as const })); (session as unknown as { hasExternalSendPreflight?: () => boolean }).hasExternalSendPreflight = () => true; // Settlement runs BEFORE the follow-up is erased: a failure keeps the durable record (the - // only carrier of the correlation) so the next attempt retries it (Codex P2 - // PRRT_kwDOPxxmWM6fOH59). + // only carrier of the correlation) so the next attempt retries it. let dispatchError: unknown; try { await internals.dispatchPendingFollowUp(); @@ -415,7 +416,7 @@ describe("AgentSession continue-message agentId fallback", () => { expect(await internals.dispatchPendingFollowUp()).toBe(false); expect(internals.sendMessage).not.toHaveBeenCalled(); expect(abandoned).toHaveBeenCalledTimes(2); - expect(abandoned).toHaveBeenLastCalledWith(workspaceTurnMetadata); + expect(abandoned).toHaveBeenLastCalledWith(workspaceTurnMetadata, "abandoned"); const lastMessages = await historyService.getLastMessages("ws", 1); expect(lastMessages.success && lastMessages.data[0]?.metadata?.muxMetadata).toEqual({ type: "compaction-summary", diff --git a/src/node/services/agentSession.queueDispatch.test.ts b/src/node/services/agentSession.queueDispatch.test.ts index 784fbb965f..e8ab31b330 100644 --- a/src/node/services/agentSession.queueDispatch.test.ts +++ b/src/node/services/agentSession.queueDispatch.test.ts @@ -5,6 +5,7 @@ import { Ok } from "@/common/types/result"; import type { WorkspaceGoalService } from "./workspaceGoalService"; import { createAgentSessionHarness, createStartedTurnHandle } from "./agentSession.testHarness"; import type { AIService } from "./aiService"; +import type { AgentSession } from "./agentSession"; const TEST_MODEL = "anthropic:claude-sonnet-4-5"; const WORKSPACE_TURN_CORRELATION = { @@ -36,6 +37,25 @@ function streamStartEvent(workspaceId: string): Record { }; } +const DELEGATED_TURN: Extract = { + type: "workspace-turn-task", + taskHandleId: "wst_delegated", + ownerWorkspaceId: "owner-workspace", + turnId: "turn-1", +}; + +/** The correlation the active stream runs under; a wake cut records it as the debt's owner. */ +function setActiveStreamCorrelation( + session: AgentSession, + workspaceTurnMetadata: typeof DELEGATED_TURN | undefined +): void { + ( + session as unknown as { + activeStreamContext?: { workspaceTurnMetadata?: typeof DELEGATED_TURN }; + } + ).activeStreamContext = { workspaceTurnMetadata }; +} + function streamAbortEvent( workspaceId: string, abortReason: "system" | "user" @@ -665,12 +685,11 @@ describe("AgentSession queued message tool-call dispatch", () => { } }); - test("a stream cut for the wake level is attributed as the cutter until a turn is admitted", async () => { - // Settlement of a delegated turn runs after the cut; an operator canceling the monitor in - // between lowers the level, so the cut itself must name its cause. It is attribution - // only: a continuation is read live (hasPendingBashMonitorWakeTurn stays false), so a - // retracted wake settles the handle instead of deferring it forever. - const workspaceId = "queue-dispatch-wake-cut-latch"; + test("a stream cut for the wake level takes a continuation debt until other input supersedes it", async () => { + // Settlement of a delegated turn runs after the cut and reads the debt (the cutter and + // hasBashMonitorWakeContinuation), never the level: an operator canceling the monitor in + // between must not be able to hide that the cut happened. + const workspaceId = "queue-dispatch-wake-cut-debt"; let level = false; let markStreamRequested: () => void = () => undefined; const streamRequested = new Promise((resolve) => { @@ -680,9 +699,14 @@ describe("AgentSession queued message tool-call dispatch", () => { const streamRelease = new Promise((resolve) => { releaseStream = resolve; }); + const voided: Array<[MuxMessageMetadata, string]> = []; const { session, cleanup } = await createAgentSessionHarness({ workspaceId, hasOutstandingBashMonitorWake: () => Promise.resolve(level), + onWorkspaceTurnContinuationVoided: (correlation, reason) => { + voided.push([correlation, reason]); + return Promise.resolve(); + }, aiServiceOverrides: { streamMessage: mock(async () => { markStreamRequested(); @@ -693,26 +717,211 @@ describe("AgentSession queued message tool-call dispatch", () => { }); let disposed = false; try { + // The cut stream's correlation is what the debt records. + setActiveStreamCorrelation(session, DELEGATED_TURN); expect(session.getQueueCutCutter()).toBeUndefined(); level = true; expect(await session.hasPendingToolEndInput()).toBe(true); level = false; expect(session.getQueueCutCutter()).toEqual({ stage: "bash-monitor-wake" }); + expect(session.hasBashMonitorWakeContinuation()).toBe(true); expect(session.hasPendingBashMonitorWakeTurn()).toBe(false); - // Reading a low level later does not retract the recorded cut. + // Reading a low level later neither retracts the debt nor takes a second one. expect(await session.hasPendingToolEndInput()).toBe(false); expect(session.getQueueCutCutter()).toEqual({ stage: "bash-monitor-wake" }); + expect(voided).toEqual([]); - // Whatever turn is admitted next settles the cut — here a manual send, which is not - // itself a wake turn. + // Input that is not the wake supersedes the continuation: the owner is told once, at + // admission, and the cutter is now the admitted input. const sendPromise = session.sendMessage("hello", { model: TEST_MODEL, agentId: "exec" }); await streamRequested; expect(session.isBusy()).toBe(true); + expect(voided).toEqual([[DELEGATED_TURN, "superseded"]]); + expect(session.hasBashMonitorWakeContinuation()).toBe(false); expect(session.getQueueCutCutter()).toEqual({ stage: "preparing", muxMetadata: undefined }); session.dispose(); disposed = true; releaseStream(); await sendPromise; + expect(voided).toHaveLength(1); + } finally { + releaseStream(); + if (!disposed) session.dispose(); + await cleanup(); + } + }); + + test("a correlated turn admitted after the cut continues the debt without settling it", async () => { + // The delegated turn's own continuation (e.g. a queued same-turn message) supersedes + // nothing: its stream-end settles the turn, so the owner is not told. + const workspaceId = "queue-dispatch-wake-cut-same-turn"; + let level = true; + let markStreamRequested: () => void = () => undefined; + const streamRequested = new Promise((resolve) => { + markStreamRequested = resolve; + }); + let releaseStream: () => void = () => undefined; + const streamRelease = new Promise((resolve) => { + releaseStream = resolve; + }); + const voided: unknown[] = []; + const { session, cleanup } = await createAgentSessionHarness({ + workspaceId, + hasOutstandingBashMonitorWake: () => Promise.resolve(level), + onWorkspaceTurnContinuationVoided: (...args) => { + voided.push(args); + return Promise.resolve(); + }, + aiServiceOverrides: { + streamMessage: mock(async () => { + markStreamRequested(); + await streamRelease; + return Ok(createStartedTurnHandle("test-assistant-message")); + }), + }, + }); + let disposed = false; + try { + setActiveStreamCorrelation(session, DELEGATED_TURN); + expect(await session.hasPendingToolEndInput()).toBe(true); + level = false; + const sendPromise = session.sendMessage("continue", { + model: TEST_MODEL, + agentId: "exec", + muxMetadata: DELEGATED_TURN, + }); + await streamRequested; + expect(session.hasBashMonitorWakeContinuation()).toBe(false); + expect(voided).toEqual([]); + session.dispose(); + disposed = true; + releaseStream(); + await sendPromise; + } finally { + releaseStream(); + if (!disposed) session.dispose(); + await cleanup(); + } + }); + + test("the level lowering with no wake turn in flight voids the debt as retracted", async () => { + const workspaceId = "queue-dispatch-wake-retracted"; + const voided: Array<[MuxMessageMetadata, string]> = []; + const { session, cleanup } = await createAgentSessionHarness({ + workspaceId, + hasOutstandingBashMonitorWake: () => Promise.resolve(true), + onWorkspaceTurnContinuationVoided: (correlation, reason) => { + voided.push([correlation, reason]); + return Promise.resolve(); + }, + }); + try { + setActiveStreamCorrelation(session, DELEGATED_TURN); + session.setBashMonitorWakeOutstanding(true); + expect(await session.hasPendingToolEndInput()).toBe(true); + expect(session.getQueueCutCutter()).toEqual({ stage: "bash-monitor-wake" }); + + // Republishing high changes nothing; lowering it (monitor canceled, output shown, + // history cleared) leaves no wake to continue the cut stream. + session.setBashMonitorWakeOutstanding(true); + expect(voided).toEqual([]); + session.setBashMonitorWakeOutstanding(false); + expect(voided).toEqual([[DELEGATED_TURN, "retracted"]]); + expect(session.getQueueCutCutter()).toBeUndefined(); + expect(session.hasBashMonitorWakeContinuation()).toBe(false); + + // A cut with no correlation (manual stream) still records the cutter but has no + // owner to tell. + setActiveStreamCorrelation(session, undefined); + session.setBashMonitorWakeOutstanding(true); + expect(await session.hasPendingToolEndInput()).toBe(true); + session.setBashMonitorWakeOutstanding(false); + expect(session.getQueueCutCutter()).toBeUndefined(); + expect(voided).toHaveLength(1); + } finally { + session.dispose(); + await cleanup(); + } + }); + + test("a wake send is in flight from its first synchronous step until its stream starts", async () => { + // The wake's onAccepted lowers the level as soon as its row is durable, which is before + // PREPARING and long before a stream exists. Only the in-flight marker keeps the debt + // from being voided in that window. + const workspaceId = "queue-dispatch-wake-in-flight"; + let markStreamRequested: () => void = () => undefined; + const streamRequested = new Promise((resolve) => { + markStreamRequested = resolve; + }); + let releaseStream: () => void = () => undefined; + const streamRelease = new Promise((resolve) => { + releaseStream = resolve; + }); + const voided: unknown[] = []; + const { session, aiEmitter, cleanup } = await createAgentSessionHarness({ + workspaceId, + hasOutstandingBashMonitorWake: () => Promise.resolve(true), + onWorkspaceTurnContinuationVoided: (...args) => { + voided.push(args); + return Promise.resolve(); + }, + aiServiceOverrides: { + streamMessage: mock(async () => { + markStreamRequested(); + await streamRelease; + return Ok(createStartedTurnHandle("test-assistant-message")); + }), + }, + }); + + let disposed = false; + try { + setActiveStreamCorrelation(session, DELEGATED_TURN); + session.setBashMonitorWakeOutstanding(true); + expect(await session.hasPendingToolEndInput()).toBe(true); + setActiveStreamCorrelation(session, undefined); + + expect(session.hasPendingBashMonitorWakeTurn()).toBe(false); + let accepted = false; + const sendPromise = session.sendMessage( + "Background monitor wake", + { + model: TEST_MODEL, + agentId: "exec", + muxMetadata: { type: "bash-monitor-wake", records: [] }, + }, + { + synthetic: true, + agentInitiated: true, + onAccepted: () => { + accepted = true; + // The reconciler consumes the signals and publishes low here. + session.setBashMonitorWakeOutstanding(false); + }, + } + ); + // Synchronously at entry, before any admission await. + expect(session.hasPendingBashMonitorWakeTurn()).toBe(true); + + await streamRequested; + expect(accepted).toBe(true); + expect(session.isBusy()).toBe(true); + expect(session.hasPendingBashMonitorWakeTurn()).toBe(true); + expect(session.hasBashMonitorWakeContinuation()).toBe(true); + expect(voided).toEqual([]); + + // The wake stream shows the wake: debt redeemed, nothing to tell the owner. + aiEmitter.emit("stream-start", streamStartEvent(workspaceId)); + expect(session.hasPendingBashMonitorWakeTurn()).toBe(false); + expect(session.hasBashMonitorWakeContinuation()).toBe(false); + expect(session.getQueueCutCutter()).toBeUndefined(); + expect(voided).toEqual([]); + + session.dispose(); + disposed = true; + releaseStream(); + await sendPromise; + expect(voided).toEqual([]); } finally { releaseStream(); if (!disposed) session.dispose(); @@ -720,6 +929,55 @@ describe("AgentSession queued message tool-call dispatch", () => { } }); + test("a wake send refused before its row is durable leaves the debt to the next dispatch", async () => { + // Pre-commit refusals (stale admission) do not lower the level, so the debt is still + // owed and the reconciler re-dispatches; only a level drop voids it. + const workspaceId = "queue-dispatch-wake-refused"; + const voided: Array<[MuxMessageMetadata, string]> = []; + const { session, cleanup } = await createAgentSessionHarness({ + workspaceId, + hasOutstandingBashMonitorWake: () => Promise.resolve(true), + onWorkspaceTurnContinuationVoided: (correlation, reason) => { + voided.push([correlation, reason]); + return Promise.resolve(); + }, + }); + try { + setActiveStreamCorrelation(session, DELEGATED_TURN); + session.setBashMonitorWakeOutstanding(true); + expect(await session.hasPendingToolEndInput()).toBe(true); + + let accepted = false; + const result = await session.sendMessage( + "Background monitor wake", + { + model: TEST_MODEL, + agentId: "exec", + muxMetadata: { type: "bash-monitor-wake", records: [] }, + }, + { + synthetic: true, + agentInitiated: true, + admissionStale: () => true, + onAccepted: () => { + accepted = true; + }, + } + ); + expect(result.success).toBe(false); + expect(accepted).toBe(false); + expect(session.hasPendingBashMonitorWakeTurn()).toBe(false); + expect(session.getQueueCutCutter()).toEqual({ stage: "bash-monitor-wake" }); + expect(voided).toEqual([]); + + session.setBashMonitorWakeOutstanding(false); + expect(voided).toEqual([[DELEGATED_TURN, "retracted"]]); + } finally { + session.dispose(); + await cleanup(); + } + }); + test("a queued head is not recorded as a wake cut", async () => { const workspaceId = "queue-dispatch-queue-cut-not-wake"; let releaseLevel: () => void = () => undefined; @@ -879,67 +1137,6 @@ describe("AgentSession queued message tool-call dispatch", () => { } }); - test("a wake send in PREPARING is a pending wake turn until it streams or ends", async () => { - const workspaceId = "queue-dispatch-preparing-wake-turn"; - let markStreamRequested: () => void = () => undefined; - const streamRequested = new Promise((resolve) => { - markStreamRequested = resolve; - }); - let releaseStream: () => void = () => undefined; - const streamRelease = new Promise((resolve) => { - releaseStream = resolve; - }); - const { session, cleanup } = await createAgentSessionHarness({ - workspaceId, - aiServiceOverrides: { - // Blocks at the provider call: the user row is durable (onAccepted has run, so the - // reconciler level is already low) but no stream is observable yet. - streamMessage: mock(async () => { - markStreamRequested(); - await streamRelease; - return Ok(createStartedTurnHandle("test-assistant-message")); - }), - }, - }); - - let disposed = false; - try { - expect(session.hasPendingBashMonitorWakeTurn()).toBe(false); - let accepted = false; - const sendPromise = session.sendMessage( - "Background monitor wake", - { - model: TEST_MODEL, - agentId: "exec", - muxMetadata: { type: "bash-monitor-wake", records: [] }, - }, - { - synthetic: true, - agentInitiated: true, - onAccepted: () => { - accepted = true; - }, - } - ); - - await streamRequested; - expect(accepted).toBe(true); - expect(session.isBusy()).toBe(true); - expect(session.hasPendingBashMonitorWakeTurn()).toBe(true); - - // Turn end (here: teardown to IDLE) clears it. - session.dispose(); - disposed = true; - expect(session.hasPendingBashMonitorWakeTurn()).toBe(false); - releaseStream(); - await sendPromise; - } finally { - releaseStream(); - if (!disposed) session.dispose(); - await cleanup(); - } - }); - test("disposed sessions finalize durable wakes after goal sync completes", async () => { const workspaceId = "queue-dispatch-disposed-after-goal-sync"; let markSyncStarted: () => void = () => undefined; @@ -1080,6 +1277,11 @@ describe("AgentSession queued message tool-call dispatch", () => { records: [], }); expect(chatEventTypes).toContain("auto-retry-scheduled"); + // The armed resume is what will bring the wake's stream, so the wake turn stays in + // flight (a debt it carries is not voided) until that retry is given up. + expect(session.hasPendingBashMonitorWakeTurn()).toBe(true); + await session.setAutoRetryEnabled(false, { persist: false }); + expect(session.hasPendingBashMonitorWakeTurn()).toBe(false); const history = await historyService.getHistoryFromLatestBoundary(workspaceId); expect(history.success).toBe(true); if (history.success) { diff --git a/src/node/services/agentSession.testHarness.ts b/src/node/services/agentSession.testHarness.ts index d0c42f418d..13fc3efabe 100644 --- a/src/node/services/agentSession.testHarness.ts +++ b/src/node/services/agentSession.testHarness.ts @@ -5,7 +5,11 @@ import type { WorkspaceChatMessage } from "@/common/orpc/types"; import { Err, Ok } from "@/common/types/result"; import type { Config } from "@/node/config"; import type { TurnStreamHandle } from "@/node/services/streamManager"; -import { AgentSession, type AgentSessionAIService } from "@/node/services/agentSession"; +import { + AgentSession, + type AgentSessionAIService, + type AgentSessionOptions, +} from "@/node/services/agentSession"; import type { CompactionCompletionMetadata } from "@/common/types/compaction"; import type { BackgroundProcessManager } from "@/node/services/backgroundProcessManager"; import type { WorkspaceGoalService } from "@/node/services/workspaceGoalService"; @@ -112,6 +116,7 @@ export interface AgentSessionHarnessOptions { onCompactionComplete?: (metadata: CompactionCompletionMetadata) => void; hasOutstandingBashMonitorWake?: () => Promise; onToolEndYieldRequested?: () => void; + onWorkspaceTurnContinuationVoided?: AgentSessionOptions["onWorkspaceTurnContinuationVoided"]; captureEvents?: boolean; } @@ -158,6 +163,7 @@ export async function createAgentSessionHarness( onCompactionComplete: options.onCompactionComplete, hasOutstandingBashMonitorWake: options.hasOutstandingBashMonitorWake, onToolEndYieldRequested: options.onToolEndYieldRequested, + onWorkspaceTurnContinuationVoided: options.onWorkspaceTurnContinuationVoided, }); const events: WorkspaceChatMessage[] = []; diff --git a/src/node/services/agentSession.ts b/src/node/services/agentSession.ts index de50bb9853..0af11d72db 100644 --- a/src/node/services/agentSession.ts +++ b/src/node/services/agentSession.ts @@ -445,10 +445,9 @@ function isCompactionRequestMetadata(meta: unknown): meta is CompactionRequestMe } /** - * Whether admitted send metadata carries a bash-monitor wake: the wake itself, or an on-send - * compaction request whose follow-up is the wake. The compaction row is what the turn shows - * (optionsForStream.muxMetadata, the stream's metadata), so the wake identity has to be read - * through it (Codex P1 PRRT_kwDOPxxmWM6fOf9G). + * Whether send metadata carries a bash-monitor wake: the wake itself, or an on-send + * compaction request whose follow-up is the wake (the compaction row is what that turn + * shows, so the wake identity has to be read through it). */ function carriesBashMonitorWake(muxMetadata: unknown): boolean { const meta = muxMetadata as MuxMessageMetadata | undefined; @@ -599,7 +598,7 @@ export interface AgentSessionAIService extends BranchSummaryAiService { ): XumToolScope; } -interface AgentSessionOptions { +export interface AgentSessionOptions { workspaceId: string; config: Config; historyService: HistoryService; @@ -642,9 +641,10 @@ interface AgentSessionOptions { */ hasExternalSendPreflight?: () => boolean; /** - * The bash-monitor wake level for this workspace (BashMonitorWakeReconciler - * .hasOutstandingWake). Read live at every tool boundary; wakes are never - * queued as messages, so this is the only way a stream learns one is pending. + * Live read of the bash-monitor wake level for this workspace + * (BashMonitorWakeReconciler.hasOutstandingWake). Consulted at tool boundaries only; the + * published mirror (setBashMonitorWakeOutstanding) lags a shown frontier, so cutting on + * the mirror alone would yield for lines the step just displayed. */ hasOutstandingBashMonitorWake?: () => Promise; /** @@ -655,14 +655,28 @@ interface AgentSessionOptions { */ onToolEndYieldRequested?: () => void; /** - * A compaction follow-up carrying a delegated turn's correlation was abandoned instead of - * dispatched (idle-rule skip, inadmissible summary/goal). Nothing else settles that turn - * — the compaction stream-end is not correlated and no later send inherits the metadata — - * so the owner is told the turn was superseded. + * This session will never continue the delegated turn identified by `correlation`: + * the continuation it owed (see wakeContinuationDebt) was voided, or a compaction + * follow-up carrying the correlation was abandoned. Nothing else settles that turn — no + * correlated stream-end follows — so the owner settles it now. Called as a tracked + * promise from synchronous transitions; must be idempotent. */ - onWorkspaceTurnContinuationAbandoned?: (metadata: WorkspaceTurnMuxMetadata) => Promise; + onWorkspaceTurnContinuationVoided?: ( + correlation: WorkspaceTurnMuxMetadata, + reason: WorkspaceTurnContinuationVoidReason + ) => Promise; } +/** + * Why a session stopped owing a delegated turn's continuation. + * - `retracted`: the stream yielded to a bash-monitor wake that then went away (monitor + * canceled, output shown, history cleared, send failed with no retry) before any wake turn + * could show it. + * - `superseded`: other input was admitted in its place. + * - `abandoned`: a compaction follow-up carrying the correlation was dropped undispatched. + */ +export type WorkspaceTurnContinuationVoidReason = "retracted" | "superseded" | "abandoned"; + enum TurnPhase { IDLE = "idle", PREPARING = "preparing", @@ -704,12 +718,11 @@ export class AgentSession { private readonly hasExternalSendPreflight?: () => boolean; private readonly hasOutstandingBashMonitorWake?: () => Promise; private readonly onToolEndYieldRequested?: () => void; - private readonly onWorkspaceTurnContinuationAbandoned?: ( - metadata: WorkspaceTurnMuxMetadata - ) => Promise; + private readonly onWorkspaceTurnContinuationVoided?: AgentSessionOptions["onWorkspaceTurnContinuationVoided"]; /** - * Last published wake level (see setBashMonitorWakeOutstanding). Feeds the - * tool-end yield flag together with the queue head. + * Mirror of the reconciler's wake level (setBashMonitorWakeOutstanding): a wake this + * workspace has not seen yet. Feeds the tool-end yield lever together with the queue head + * and is the sync input to hasPendingToolEndInput. */ private bashMonitorWakeOutstanding = false; /** Last value pushed to the tool-end yield lever; edge detection for onToolEndYieldRequested. */ @@ -905,27 +918,39 @@ export class AgentSession { /** Correlation of the direct send currently in the PREPARING phase, if any. */ private preparingWorkspaceTurnMetadata?: WorkspaceTurnMuxMetadata; /** - * The turn in flight carries a bash-monitor wake: the admitted send is the wake itself, or - * an on-send compaction whose follow-up is (carriesBashMonitorWake). The wake's `onAccepted` - * lowers the reconciler level once its row is durable, so from admission on this marker is - * the only thing that keeps the continuation visible to delegated-turn settlement until a - * stream that shows it exists. A direct wake stream shows it through its inherited - * correlation, so the marker drops at that stream's start; a compaction stream does not - * (its metadata is the compaction request), so the marker outlives it until the follow-up - * dispatches in COMPLETING or the turn ends (Codex P1 PRRT_kwDOPxxmWM6fOf9G). - */ - private turnCarriesBashMonitorWake = false; - /** - * The last stream cut itself for the wake level (hasPendingToolEndInput returned true - * from the level) and no turn has been admitted since. This is cut *attribution* - * (getQueueCutCutter), not a continuation marker: whether the wake still arrives is read - * live from the level / the admitted wake turn. An operator canceling the monitor between - * the cut and the stream-end handler lowers the level with no wake turn to follow, so - * settlement must not defer on it (the parent's wait would hang until timeout); the - * attribution lets it settle the "tool-calls" end as a wake cut instead of a truncation - * failure (Codex P2 PRRT_kwDOPxxmWM6fDmpR, PRRT_kwDOPxxmWM6fEQIf). + * Bash-monitor wake continuation model. + * + * A stream that yields to the wake level (hasPendingToolEndInput) takes out a DEBT: this + * session owes the delegated turn it cut a continuation. The debt records the cut stream's + * correlation and is settled exactly once — REDEEMED when a stream that shows the wake + * starts, or VOIDED when no wake turn can come, in which case the owner is told + * (onWorkspaceTurnContinuationVoided). Settlement never probes the wake level: the debt is + * the only thing delegated-turn settlement reads (getQueueCutCutter), and it is sync. + * + * `wakeTurnInFlight` says a wake turn is somewhere inside this session — from sendMessage / + * resumeStream entry (raised synchronously, before any await, because the wake's + * `onAccepted` lowers the level as soon as its row is durable) until the stream that shows + * it starts or the send returns with no stream and no auto-retry armed for its durable row. + * + * Transitions — each is the only place its consequence is computed: + * + * hasPendingToolEndInput yields for the level → debt = { active stream's correlation } + * wake send / resume / queue dispatch begins → wakeTurnInFlight = true + * stream-start whose request carries the wake → redeem (in-flight false, debt cleared); + * (direct wake, or a compaction FOLLOW-UP) a compaction stream itself does not + * wake send returns without a stream → in-flight stays true only while an + * auto-retry resume of its durable row is + * armed; otherwise false → maybeVoid + * level lowered (cancel / shown / clear) → maybeVoid + * maybeVoid: debt ∧ ¬inFlight ∧ ¬level → void "retracted" + * non-wake input admitted (enterPreparing) → void "superseded" + * compaction follow-up with the correlation → void "abandoned" (before the erase) + * dropped (clearPendingFollowUpFromSummary) + * dispose / IDLE → in-flight false (IDLE keeps the debt: + * the wake dispatcher needs an idle session) */ - private streamYieldedToBashMonitorWake = false; + private wakeContinuationDebt?: { correlation?: WorkspaceTurnMuxMetadata }; + private wakeTurnInFlight = false; /** Context needed to retry the current stream (cleared on stream end/abort/error). */ private activeStreamContext?: { @@ -978,7 +1003,7 @@ export class AgentSession { hasExternalSendPreflight, hasOutstandingBashMonitorWake, onToolEndYieldRequested, - onWorkspaceTurnContinuationAbandoned, + onWorkspaceTurnContinuationVoided, } = options; assert(typeof workspaceId === "string", "workspaceId must be a string"); @@ -1009,7 +1034,7 @@ export class AgentSession { this.hasExternalSendPreflight = hasExternalSendPreflight; this.hasOutstandingBashMonitorWake = hasOutstandingBashMonitorWake; this.onToolEndYieldRequested = onToolEndYieldRequested; - this.onWorkspaceTurnContinuationAbandoned = onWorkspaceTurnContinuationAbandoned; + this.onWorkspaceTurnContinuationVoided = onWorkspaceTurnContinuationVoided; this.compactionHandler = new CompactionHandler({ workspaceId: this.workspaceId, @@ -1076,6 +1101,10 @@ export class AgentSession { this.bashMonitorWakeOutstanding = false; this.syncToolEndYieldRequested(false); } + // No wake turn can come from a disposed session; the debt dies with it silently + // (workspace teardown settles delegated turns through its own path). + this.wakeTurnInFlight = false; + this.wakeContinuationDebt = undefined; // Ensure any callers blocked on waitForIdle() can continue during teardown. this.setTurnPhase(TurnPhase.IDLE); @@ -1295,6 +1324,9 @@ export class AgentSession { this.autoRetryStarting = false; } this.emitRetryEvent(event); + // A retry given up (exhausted, non-retryable, disabled by the user) was the last thing + // that could still bring a durable wake row's stream. + if (event.type === "auto-retry-abandoned") this.settleWakeTurnInFlight(); } private emitRetryEvent(event: RetryStatusEvent): void { @@ -1462,6 +1494,9 @@ export class AgentSession { ); } finally { this.autoRetryStarting = false; + // The resume either started its stream (redeeming a wake it carried), re-armed a + // retry, or gave up — only the last leaves a carried wake with no way to arrive. + this.settleWakeTurnInFlight(); } } @@ -3124,7 +3159,25 @@ export class AgentSession { this.emitMetadata(metadata); } + /** + * Public send entry. A send carrying a bash-monitor wake marks the wake turn in flight + * synchronously — before the first await, hence before its `onAccepted` can lower the + * level — and settles the marker when it returns without a stream (see + * wakeContinuationDebt). + */ async sendMessage( + ...args: Parameters + ): Promise> { + const wake = carriesBashMonitorWake(args[1]?.muxMetadata); + if (wake) this.wakeTurnInFlight = true; + try { + return await this.sendMessageInner(...args); + } finally { + if (wake) this.settleWakeTurnInFlight(); + } + } + + private async sendMessageInner( message: string, options?: SendMessageOptions & { fileParts?: FilePart[] }, internal?: { @@ -3990,7 +4043,7 @@ export class AgentSession { // appending a second one, so the row (and any delegated turn waiting on its // continuation) no longer depends on an application restart. With auto-retry // disabled by the user this stays a startup-recovery row like every other - // pre-stream failure (Codex P2 PRRT_kwDOPxxmWM6fOH54). + // pre-stream failure, and the wake turn leaves flight (see settleWakeTurnInFlight). this.setAutoRetryResumeState(optionsForStream, agentInitiated, goalKind, internal?.goalId); await internal?.onAccepted?.(); await this.handleStreamFailureForAutoRetry({ @@ -4226,7 +4279,20 @@ export class AgentSession { return await startPreparedStream(); } + /** Like sendMessage, marks a carried wake in flight for the duration of the resume. */ async resumeStream( + ...args: Parameters + ): Promise> { + const wake = carriesBashMonitorWake(args[0].muxMetadata); + if (wake) this.wakeTurnInFlight = true; + try { + return await this.resumeStreamInner(...args); + } finally { + if (wake) this.settleWakeTurnInFlight(); + } + } + + private async resumeStreamInner( options: SendMessageOptions, internal?: { agentInitiated?: boolean; goalKind?: GoalSyntheticMessageKind; goalId?: string } ): Promise> { @@ -4586,8 +4652,7 @@ export class AgentSession { // message queued or in preflight during the compaction stream still wins over the // wake continuation (dispatchPendingFollowUp). A skipped wake follow-up is lost — its // signals were consumed at acceptance — which is the bounded cost of letting the - // user's correction go first; the process output stays readable via task_await - // (Codex P2 PRRT_kwDOPxxmWM6fFJ4N). + // user's correction go first; the process output stays readable via task_await. if (params.muxMetadata.type === "bash-monitor-wake") { followUp.dispatchOptions = { ...followUp.dispatchOptions, requireIdle: true }; } @@ -5856,11 +5921,16 @@ export class AgentSession { this.dispatchingQueuedEntry = false; this.dispatchingQueuedEntryMuxMetadata = undefined; this.preparingWorkspaceTurnMetadata = undefined; - // A compaction stream's metadata does not show the wake its follow-up carries; keep - // the marker until the follow-up dispatches (see turnCarriesBashMonitorWake). - if (this.activeCompactionRequest == null) this.turnCarriesBashMonitorWake = false; - // A live stream is the continuation (or a superseding turn) the cut waited for. - this.streamYieldedToBashMonitorWake = false; + // A stream that shows the wake redeems the continuation debt (see + // wakeContinuationDebt). Only the wake row's own stream qualifies: an on-send + // compaction stream's request is the compaction row, and the wake follows it. + const streamMetadata = this.activeStreamContext?.options?.muxMetadata as + | MuxMessageMetadata + | undefined; + if (streamMetadata?.type === "bash-monitor-wake") { + this.wakeTurnInFlight = false; + this.wakeContinuationDebt = undefined; + } this.activeStreamStartedAtMs = payload.startTime; // Codex P1 (PRRT_kwDOPxxmWM6cClKS): a new live stream makes mid-stream // setGoal deferral meaningful again — clear the goal service's settled @@ -6359,7 +6429,7 @@ export class AgentSession { this.dispatchingQueuedEntry = false; this.dispatchingQueuedEntryMuxMetadata = undefined; this.preparingWorkspaceTurnMetadata = undefined; - this.turnCarriesBashMonitorWake = false; + this.settleWakeTurnInFlight(); // Turn ended: expire any mid-turn thinking override. Safe unconditionally // because a replacement turn (e.g. an edit) only creates its holder after // the preempted turn has already been transitioned to IDLE. @@ -6702,7 +6772,12 @@ export class AgentSession { * * A non-empty queue arbitrates alone: its head runs next whatever the level says (the * wake dispatcher waits for an empty queue), so cutting for the wake behind a turn-end - * head would only promote that entry to tool-end (Codex P2 PRRT_kwDOPxxmWM6fDmpV). + * head would only promote that entry to tool-end. A message queued during the level read + * arbitrates the same way. + * + * The SDK only asks when the loop would otherwise continue, so a true from the level IS + * the cut: the continuation debt is taken here, in the same synchronous step as the + * decision (see wakeContinuationDebt). */ async hasPendingToolEndInput(): Promise { const nextMode = this.messageQueue.getNextDispatchableMode(); @@ -6710,13 +6785,13 @@ export class AgentSession { if (this.hasOutstandingBashMonitorWake == null) return false; try { const outstanding = await this.hasOutstandingBashMonitorWake(); - // A message queued during the level read arbitrates the same way: the stream-end - // drain would dispatch it whatever the level says (Codex P2 PRRT_kwDOPxxmWM6fEQIk). const modeAfterRead = this.messageQueue.getNextDispatchableMode(); if (modeAfterRead != null) return modeAfterRead === "tool-end"; - // The SDK only asks when the loop would otherwise continue, so a true here IS the cut. - if (outstanding) this.streamYieldedToBashMonitorWake = true; - return outstanding; + if (!outstanding) return false; + this.wakeContinuationDebt ??= { + correlation: this.activeStreamContext?.workspaceTurnMetadata, + }; + return true; } catch (error) { log.debug("hasPendingToolEndInput: wake level read failed; not yielding", { workspaceId: this.workspaceId, @@ -6729,11 +6804,74 @@ export class AgentSession { /** * Mirror the reconciler's wake level. While high, long-polling bash reads return early * so the stream reaches a tool boundary (same lever a queued tool-end message pulls). + * Lowered with no wake turn in flight, an owed continuation can no longer arrive. */ setBashMonitorWakeOutstanding(outstanding: boolean): void { if (this.bashMonitorWakeOutstanding === outstanding) return; this.bashMonitorWakeOutstanding = outstanding; this.syncToolEndYieldRequested(); + if (!outstanding) this.maybeVoidWakeContinuation(); + } + + /** A wake turn is inside this session (see wakeContinuationDebt). */ + hasPendingBashMonitorWakeTurn(): boolean { + return this.wakeTurnInFlight; + } + + /** + * A stream that yielded to the wake level will still be continued by a wake turn: the + * debt is outstanding (its wake not yet dispatched) or the wake turn is already in flight. + * Delegated-turn settlement reads this instead of probing the reconciler. + */ + hasBashMonitorWakeContinuation(): boolean { + return this.wakeContinuationDebt != null || this.wakeTurnInFlight; + } + + /** + * A wake send / resume returned, the turn went idle, or a retry was given up: the wake + * turn is still in flight only while something inside this session can still start its + * stream — a turn in progress (its stream, or a compaction whose follow-up is the wake), + * or an auto-retry armed for its durable row. + */ + private settleWakeTurnInFlight(): void { + if (!this.wakeTurnInFlight) return; + if (this.turnPhase !== TurnPhase.IDLE) return; + if ( + this.hasPendingAutoRetry() && + carriesBashMonitorWake(this.lastAutoRetryResumeRequest?.options.muxMetadata) + ) { + return; + } + this.wakeTurnInFlight = false; + this.maybeVoidWakeContinuation(); + } + + private maybeVoidWakeContinuation(): void { + if ( + this.wakeContinuationDebt != null && + !this.wakeTurnInFlight && + !this.bashMonitorWakeOutstanding + ) { + this.voidWakeContinuation("retracted"); + } + } + + /** + * Sync transition; the owner hook runs as a tracked promise (never awaited here — callers + * sit inside admission and phase transitions). + */ + private voidWakeContinuation(reason: WorkspaceTurnContinuationVoidReason): void { + const debt = this.wakeContinuationDebt; + this.wakeContinuationDebt = undefined; + const correlation = debt?.correlation; + if (correlation == null || this.onWorkspaceTurnContinuationVoided == null) return; + this.onWorkspaceTurnContinuationVoided(correlation, reason).catch((error: unknown) => { + log.error("Voided bash-monitor wake continuation could not be settled", { + workspaceId: this.workspaceId, + reason, + error: getErrorMessage(error), + }); + }); } /** @@ -6744,8 +6882,7 @@ export class AgentSession { * This is the single place the effective flag is computed, so it also owns the rising * edge: a turn-end head that is cleared, removed, or promoted while the wake level is high * makes the lever effective without any enqueue or level publish, and the consequence - * (backgrounding foreground waits) must follow that edge, not those events (Codex P2 - * PRRT_kwDOPxxmWM6fGVw_). + * (backgrounding foreground waits) must follow that edge, not those events. */ private syncToolEndYieldRequested( queueHeadToolEnd = this.messageQueue.getNextDispatchableMode() === "tool-end" @@ -6800,28 +6937,29 @@ export class AgentSession { return false; } - /** Claim PREPARING for a send and record what kind of input it carries. */ + /** + * Claim PREPARING for a send and record what kind of input it carries. Admitting anything + * but the wake while a continuation debt is outstanding settles that debt: a turn with the + * same correlation continues the delegated turn itself (its stream-end settles it), any + * other input supersedes it. + */ private enterPreparing(muxMetadata: unknown): void { this.preparingWorkspaceTurnMetadata = getWorkspaceTurnMuxMetadata(muxMetadata); - this.turnCarriesBashMonitorWake = carriesBashMonitorWake(muxMetadata); - // Whatever is admitted now (the wake turn, or input superseding it) settles the cut. - this.streamYieldedToBashMonitorWake = false; + if (!carriesBashMonitorWake(muxMetadata) && this.wakeContinuationDebt != null) { + if ( + hasSameWorkspaceTurnCorrelation( + this.preparingWorkspaceTurnMetadata, + this.wakeContinuationDebt.correlation + ) + ) { + this.wakeContinuationDebt = undefined; + } else { + this.voidWakeContinuation("superseded"); + } + } this.setTurnPhase(TurnPhase.PREPARING); } - /** - * A bash-monitor wake turn admitted but not yet shown by a correlated stream: a direct wake - * send in PREPARING, a dequeued wake entry, or an on-send compaction turn (preparing, - * streaming, or dispatching its follow-up) that consumed the wake. The reconciler level is - * already low here — `onAccepted` ran when the durable row landed — so delegated-turn - * settlement must read the continuation from this marker. - */ - hasPendingBashMonitorWakeTurn(): boolean { - if (this.turnCarriesBashMonitorWake) return true; - const dispatching = this.dispatchingQueuedEntryMuxMetadata as MuxMessageMetadata | undefined; - return dispatching?.type === "bash-monitor-wake"; - } - /** * Whether a queued or dispatching entry continues the exact workspace-turn correlation. */ @@ -6873,9 +7011,9 @@ export class AgentSession { } const candidate = this.messageQueue.getNextQueueCutCandidate(); if (candidate != null) return { stage: "queued", ...candidate }; - // No input holds the session: the stream itself yielded to the wake level. Settlement - // reads whether the wake still arrives from the level; this only names the cause. - return this.streamYieldedToBashMonitorWake ? { stage: "bash-monitor-wake" } : undefined; + // No input holds the session: the stream itself yielded to the wake level and the + // continuation is still owed (see wakeContinuationDebt). + return this.wakeContinuationDebt != null ? { stage: "bash-monitor-wake" } : undefined; } /** Whether a message queued with this dedupe key is still pending (see MessageQueue.addOnce). */ @@ -7625,14 +7763,22 @@ export class AgentSession { } // Every discard path funnels here, so this is the one place that knows the delegated - // turn's continuation is gone for good (Codex P2 PRRT_kwDOPxxmWM6fGVxG). Settle BEFORE - // erasing the follow-up: the durable record is the only carrier of the correlation, so - // a settlement failure must leave it in place for the next dispatch attempt (or startup - // recovery) to retry — settlement is idempotent, a lost record is not recoverable - // (Codex P2 PRRT_kwDOPxxmWM6fOH59). + // turn's continuation is gone for good. Settle BEFORE erasing the follow-up: the durable + // record is the only carrier of the correlation, so a settlement failure must leave it in + // place for the next dispatch attempt (or startup recovery) to retry — settlement is + // idempotent, a lost record is not recoverable. A wake follow-up also carried the + // continuation debt of the stream it cut; that debt is settled by this same void. const workspaceTurnMetadata = muxMeta.pendingFollowUp.workspaceTurnMetadata; if (workspaceTurnMetadata != null) { - await this.onWorkspaceTurnContinuationAbandoned?.(workspaceTurnMetadata); + if ( + hasSameWorkspaceTurnCorrelation( + this.wakeContinuationDebt?.correlation, + workspaceTurnMetadata + ) + ) { + this.wakeContinuationDebt = undefined; + } + await this.onWorkspaceTurnContinuationVoided?.(workspaceTurnMetadata, "abandoned"); } const { pendingFollowUp: _pendingFollowUp, ...muxMetadataWithoutFollowUp } = muxMeta; diff --git a/src/node/services/bashMonitorWakeReconciler.test.ts b/src/node/services/bashMonitorWakeReconciler.test.ts index 9e32fdb89d..0757beccbe 100644 --- a/src/node/services/bashMonitorWakeReconciler.test.ts +++ b/src/node/services/bashMonitorWakeReconciler.test.ts @@ -243,7 +243,48 @@ describe("BashMonitorWakeReconciler", () => { expect(dispatches[0].isCurrent()).toBe(true); }); - test("a failed acknowledgment keeps the accepted wake consumed and retries it", async () => { + test("a wake withdrawn under the send still commits and retires its replacement", async () => { + // Two processes; only `proc`'s lines get shown (shownThroughOffset 12 < other's offset). + const other = liveSnapshot({ + processId: "other", + taskId: "bash:other", + match: { throughOffset: 40, lines: ["READY other"], totalMatches: 1 }, + }); + live = [liveSnapshot(), other]; + await reconciler.reconcile(OWNER); + expect(dispatches).toHaveLength(1); + const first = dispatches[0]; + + // The owner's send is past its last admission gate when a manual read shows `proc`'s + // output: the lease is released and a replacement (only `other`) is offered. + deliveryState = { status: "settled", shownThroughOffset: 12, terminalStatusShown: false }; + await reconciler.outputShown(OWNER, "proc"); + expect(first.isCurrent()).toBe(false); + await reconciler.reconcile(OWNER); + expect(dispatches).toHaveLength(2); + const replacement = dispatches[1]; + expect(replacement.muxMetadata.records.map((record) => record.processId)).toEqual(["other"]); + expect(replacement.isCurrent()).toBe(true); + + // The first send's row lands: its commit consumes exactly its own signals (both + // processes), and the replacement — which re-describes `other` — is released so the + // owner drops it at its next gate instead of sending a duplicate. + acknowledged = []; + await first.onAccepted(); + expect(first.isCurrent()).toBe(true); + expect(replacement.isCurrent()).toBe(false); + expect(acknowledged).toEqual([ + { processId: "other", matchedThroughOffset: 40 }, + { processId: "proc", matchedThroughOffset: 12 }, + ]); + + // Nothing is left to derive: no third wake, level low. + await reconciler.reconcile(OWNER); + expect(dispatches).toHaveLength(2); + expect(await reconciler.hasOutstandingWake(OWNER)).toBe(false); + }); + + test("a failed acknowledgment keeps the committed wake consumed and retries it", async () => { live = [liveSnapshot()]; await reconciler.reconcile(OWNER); expect(dispatches).toHaveLength(1); @@ -257,7 +298,7 @@ describe("BashMonitorWakeReconciler", () => { await expect(reconciler.reconcile(OWNER)).rejects.toThrow("disk full"); expect(dispatches).toHaveLength(1); - // Withdrawals never apply to an accepted dispatch: dropping it here would re-derive the + // Withdrawals never apply to a committed lease: dropping it here would re-derive the // same signals into a second prompt once the store recovers. await reconciler.outputShown(OWNER, "proc"); await reconciler.discardProcess(OWNER, "proc", CREATED_AT); diff --git a/src/node/services/bashMonitorWakeReconciler.ts b/src/node/services/bashMonitorWakeReconciler.ts index 55b76a9ae4..d717fca0e4 100644 --- a/src/node/services/bashMonitorWakeReconciler.ts +++ b/src/node/services/bashMonitorWakeReconciler.ts @@ -114,12 +114,13 @@ export interface BashMonitorWakeDispatch { prompt: string; muxMetadata: Extract; /** - * False once a full-history clear, disposal, monitor cancel, or shown-frontier advance - * retired the signals behind this wake while it was in the receiver's hands — until - * `onAccepted` runs, after which it stays true (the prompt is durable and the signals - * consumed). The receiver re-checks it after taking its own locks (the clear runs under - * the same history lock), before sending, and at every send-admission gate, so a stale - * prompt is never appended to history and an accepted one always gets its stream. + * False once the lease behind this wake was released while it was in the receiver's + * hands (full-history clear, disposal, monitor cancel, shown-frontier advance, or an + * earlier wake committing meanwhile). Stays true from `onAccepted` on: the prompt is + * durable and the signals consumed. The receiver re-checks it after taking its own locks + * (the clear runs under the same history lock), before sending, and at every + * send-admission gate, so a stale prompt is never appended to history and an accepted one + * always gets its stream. */ isCurrent(): boolean; onAccepted(): Promise; @@ -166,23 +167,42 @@ interface DerivedSignal { } /** - * The one wake handed to `onWake` for an owner at a time. Lifecycle: an *offered* dispatch leaves the slot - * either by withdrawal (defer / forgetDispatchFor / consumeCurrent) or by acceptance; an - * *accepted* dispatch leaves the slot only once its signals are durably consumed — by the - * reconcile pass that acknowledges them, by consumeCurrent, or by dispose. Withdrawal never - * applies to an accepted dispatch: its prompt row is already durable, so the signals are - * consumed whether or not the durability write has landed yet (Codex P2 PRRT_kwDOPxxmWM6fGVxB). + * A wake handed to the owner is a LEASE on the signal set it describes. + * + * Lifecycle (one transition each, all under the owner lock): + * + * offered ──release──▶ released the owner did not send it: `onDeferred`, `onWake` + * threw, or the signals were withdrawn under it (cancel, + * shown frontier, full-history clear). `isCurrent()` turns + * false so the owner drops it at its next admission gate; + * whatever still derives is re-leased by the next reconcile. + * offered ──commit───▶ committed the owner's prompt row is durable (`onAccepted`). The + * released ─commit───▶ committed signals are consumed from here on regardless of what + * happened to the offer meanwhile (a release can land in + * the send's last pre-durability await): withdrawal never + * applies to a committed lease, and a replacement offered + * into the emptied slot is released because it re-describes + * signals this row already delivered. + * committed ─acknowledge─▶ (gone) watermarks advanced + monitors cleaned up for exactly the + * leased signals, by identity. Attempted inline at commit + * and retried by every reconcile pass until it lands; while + * pending, `collect()` overlays the committed signals so the + * level reads low and nothing re-derives a duplicate. + * + * Invariant: at most one offered and at most one committed lease per owner, and nothing is + * offered while either exists — a second wake meanwhile could only duplicate or supersede it. */ -interface DispatchState { +interface Lease { signals: readonly DerivedSignal[]; - accepted: boolean; + status: "offered" | "committed" | "released"; } interface ReconcileState { requested: boolean; scheduled: boolean; promise?: Promise; - dispatch?: DispatchState; + offered?: Lease; + committed?: Lease; } function signalKey(processId: string, createdAt: string): string { @@ -402,7 +422,8 @@ export class BashMonitorWakeReconciler { /** * The wake level: whether the owner has a wake it has not seen yet. Same-process * blocking reads (deferredReads) are not outstanding — the read shows the lines itself. - * Consumers: the stream's tool-boundary stop condition and delegated-turn settlement. + * Consumer: the stream's tool-boundary stop condition (AgentSession.hasPendingToolEndInput); + * delegated-turn settlement reads the session's continuation debt instead. */ async hasOutstandingWake(ownerWorkspaceId: string): Promise { if (this.defunctWorkspaces.has(ownerWorkspaceId)) return false; @@ -475,12 +496,10 @@ export class BashMonitorWakeReconciler { /** * A model-visible read advanced this process's shown frontier (or showed its terminal - * status). A wake already handed to the owner may now describe lines the owner has seen: - * the owner could have run a manual turn and returned idle while the wake was still - * resolving send options, so the reconcile that would re-derive it is queued behind that - * very hand-off. Forget the dispatch so its isCurrent() turns false at every admission - * gate; whatever still derives is re-handed by the reconcile scheduled here (Codex P2 - * PRRT_kwDOPxxmWM6fEQIa). + * status). An offered wake may now describe lines the owner has seen (the owner could have + * run a manual turn and returned idle while the wake was still resolving send options, so + * the reconcile that would re-derive it is queued behind that very hand-off): release it + * and let the reconcile scheduled here re-lease whatever still derives. */ async outputShown(ownerWorkspaceId: string, processId: string): Promise { await this.forgetDispatchFor(ownerWorkspaceId, (signal) => signal.processId === processId); @@ -492,18 +511,19 @@ export class BashMonitorWakeReconciler { ): Promise { await this.locks.withLock(ownerWorkspaceId, () => { const state = this.states.get(ownerWorkspaceId); - if ( - state?.dispatch != null && - !state.dispatch.accepted && - state.dispatch.signals.some(covers) - ) { - state.dispatch = undefined; - } + if (state?.offered?.signals.some(covers)) this.releaseOffered(state); return Promise.resolve(); }); this.scheduleReconcile(ownerWorkspaceId); } + /** Caller holds the owner lock. */ + private releaseOffered(state: ReconcileState): void { + if (state.offered == null) return; + state.offered.status = "released"; + state.offered = undefined; + } + async beginFullHistoryClear(ownerWorkspaceId: string): Promise { await this.consumeCurrent(ownerWorkspaceId); return { ownerWorkspaceId }; @@ -517,6 +537,11 @@ export class BashMonitorWakeReconciler { this.defunctWorkspaces.add(ownerWorkspaceId); this.resetRetry(ownerWorkspaceId); await this.locks.withLock(ownerWorkspaceId, () => { + const state = this.states.get(ownerWorkspaceId); + if (state != null) { + this.releaseOffered(state); + if (state.committed != null) state.committed.status = "released"; + } this.states.delete(ownerWorkspaceId); return Promise.resolve(); }); @@ -568,11 +593,11 @@ export class BashMonitorWakeReconciler { } private async reconcileOnce(ownerWorkspaceId: string): Promise { - const dispatch = await this.locks.withLock(ownerWorkspaceId, async () => { - // An acknowledgment that failed at acceptance is retried first: on a throw the slot - // stays occupied and accepted, the loop's catch schedules the backoff retry, and no - // second wake is handed out meanwhile. - await this.acknowledgeAccepted(ownerWorkspaceId); + const lease = await this.locks.withLock(ownerWorkspaceId, async () => { + // A commit whose acknowledgment failed is retried first: on a throw the loop's catch + // schedules the backoff retry and nothing is leased meanwhile. + const committed = this.states.get(ownerWorkspaceId)?.committed; + if (committed != null) await this.acknowledge(ownerWorkspaceId, committed); const collected = await this.collect(ownerWorkspaceId, true); for (const readSettled of collected.deferredReads) { void readSettled.finally(() => this.scheduleReconcile(ownerWorkspaceId)); @@ -581,66 +606,57 @@ export class BashMonitorWakeReconciler { await this.cleanup(collected.autoConsumed); const state = this.state(ownerWorkspaceId); - // A wake already handed to the owner settles on its own (accept → acknowledged, and a - // reconcile is scheduled; defer → the owner re-arms a reconcile). Handing out a second - // one meanwhile could only duplicate or supersede the first. - if (collected.signals.length === 0 || state.dispatch != null) return undefined; - const next: DispatchState = { signals: collected.signals, accepted: false }; - state.dispatch = next; + if (collected.signals.length === 0 || state.offered != null || state.committed != null) { + return undefined; + } + const next: Lease = { signals: collected.signals, status: "offered" }; + state.offered = next; return next; }); - if (dispatch == null) return; + if (lease == null) return; try { const outcome = await this.args.onWake({ ownerWorkspaceId, - prompt: buildPrompt(dispatch.signals), - muxMetadata: buildMetadata(dispatch.signals), - // Validity freezes at acceptance: the prompt row is durable and the signals are - // consumed, so a cancel/shown/clear landing in the owner's remaining pre-stream - // awaits must let the turn finish admission (refusing there would leave the row - // in history with no stream, to be replayed by a later manual turn) (Codex P2 - // PRRT_kwDOPxxmWM6fFJ4K). - isCurrent: () => - dispatch.accepted || this.states.get(ownerWorkspaceId)?.dispatch === dispatch, - onAccepted: async () => this.accept(ownerWorkspaceId, dispatch), - onDeferred: async () => this.defer(ownerWorkspaceId, dispatch), + prompt: buildPrompt(lease.signals), + muxMetadata: buildMetadata(lease.signals), + isCurrent: () => lease.status !== "released", + onAccepted: async () => this.commit(ownerWorkspaceId, lease), + onDeferred: async () => this.release(ownerWorkspaceId, lease), }); - if (outcome === "deferred") await this.defer(ownerWorkspaceId, dispatch); + if (outcome === "deferred") await this.release(ownerWorkspaceId, lease); } catch (error) { - await this.defer(ownerWorkspaceId, dispatch); + await this.release(ownerWorkspaceId, lease); throw error; } } - private async defer(ownerWorkspaceId: string, dispatch: DispatchState): Promise { + private async release(ownerWorkspaceId: string, lease: Lease): Promise { await this.locks.withLock(ownerWorkspaceId, () => { const state = this.state(ownerWorkspaceId); - if (state.dispatch === dispatch && !dispatch.accepted) state.dispatch = undefined; + if (state.offered === lease) this.releaseOffered(state); return Promise.resolve(); }); } + /** - * The prompt row is durable: the dispatch's signals are consumed from here on, even if a - * full-history clear or process discard forgot the dispatch meanwhile. The flag flips first - * (collect() overlays an accepted dispatch onto the watermarks, so the level reads low and - * no duplicate derives whether or not the acknowledgment has landed); the acknowledgment - * itself is attempted inline and, if it throws, retried by the reconcile passes until it - * lands. Never throws: the caller is the owner's send, whose row already landed (Codex P2 - * PRRT_kwDOPxxmWM6fGVxB). + * The owner's prompt row is durable. Never throws: the caller is the owner's send, whose + * row already landed; a failed acknowledgment is retried by the reconcile passes. */ - private async accept(ownerWorkspaceId: string, dispatch: DispatchState): Promise { + private async commit(ownerWorkspaceId: string, lease: Lease): Promise { await this.locks.withLock(ownerWorkspaceId, async () => { - // Second call (onAcceptedPreStreamFailure): already consumed; any pending retry is the - // reconcile pass's. - if (dispatch.accepted) return; - dispatch.accepted = true; + // Second call (onAcceptedPreStreamFailure), or the owner is gone. + if (lease.status === "committed" || this.defunctWorkspaces.has(ownerWorkspaceId)) return; + // A lease released while the send was between its last admission gate and durability + // still commits: the row exists, so its signals are consumed either way. const state = this.state(ownerWorkspaceId); - // Withdrawn between the send's last admission gate and its row becoming durable - // (forgetDispatchFor / consumeCurrent take only this lock): re-occupy the slot so the - // acknowledgment covers these signals instead of re-deriving them into a duplicate wake. - state.dispatch ??= dispatch; - await this.acknowledgeAccepted(ownerWorkspaceId).catch((error: unknown) => { + // The offered slot holds either this lease or a replacement offered after the signals + // were withdrawn under it; either way it empties (see the Lease lifecycle). + if (state.offered === lease) state.offered = undefined; + else this.releaseOffered(state); + lease.status = "committed"; + state.committed = lease; + await this.acknowledge(ownerWorkspaceId, lease).catch((error: unknown) => { log.warn("Bash monitor wake acknowledgment failed; the reconcile pass retries it", { ownerWorkspaceId, error: getErrorMessage(error), @@ -651,28 +667,32 @@ export class BashMonitorWakeReconciler { } /** - * Durably consume the accepted dispatch's signals and free the slot. Caller holds the owner - * lock. Throws when durability fails, leaving the slot occupied and accepted for a retry. + * Durably consume exactly the committed lease's signals. Caller holds the owner lock. + * Throws when durability fails, leaving the lease committed for a retry. */ - private async acknowledgeAccepted(ownerWorkspaceId: string): Promise { - const state = this.states.get(ownerWorkspaceId); - const dispatch = state?.dispatch; - if (state == null || dispatch?.accepted !== true) return; + private async acknowledge(ownerWorkspaceId: string, lease: Lease): Promise { const watermarks = await this.readWatermarks(ownerWorkspaceId); - await this.advanceWatermarks(ownerWorkspaceId, watermarks, dispatch.signals); - await this.cleanup(dispatch.signals); - if (state.dispatch === dispatch) state.dispatch = undefined; + await this.advanceWatermarks(ownerWorkspaceId, watermarks, lease.signals); + await this.cleanup(lease.signals); + const state = this.states.get(ownerWorkspaceId); + if (state?.committed === lease) state.committed = undefined; } private async consumeCurrent(ownerWorkspaceId: string): Promise { await this.locks.withLock(ownerWorkspaceId, async () => { - // A wake already handed to the owner describes signals this consume retires; - // forgetting it flips its isCurrent() so the owner drops it instead of sending. - this.state(ownerWorkspaceId).dispatch = undefined; + const state = this.state(ownerWorkspaceId); + // An offered wake describes signals this consume retires: release it so the owner + // drops it instead of sending. A committed one is consumed along with everything else. + this.releaseOffered(state); const collected = await this.collect(ownerWorkspaceId, false); - const consumed = [...collected.signals, ...collected.autoConsumed]; + const consumed = [ + ...collected.signals, + ...collected.autoConsumed, + ...(state.committed?.signals ?? []), + ]; await this.advanceWatermarks(ownerWorkspaceId, collected.watermarks, consumed); await this.cleanup(consumed); + state.committed = undefined; // Everything collected is consumed, so the level is low by construction; publish it // here because no read follows a consume. this.args.onOutstandingChanged?.(ownerWorkspaceId, false); @@ -733,11 +753,11 @@ export class BashMonitorWakeReconciler { } if (pruned) await this.writeWatermarks(ownerWorkspaceId, watermarks); - // An accepted dispatch is consumed whether or not its acknowledgment has been written yet - // (the write may have failed and be awaiting retry). Overlaying it makes derive() treat - // those signals as delivered, so level reads stay low and nothing re-derives a duplicate. - const accepted = this.states.get(ownerWorkspaceId)?.dispatch; - if (accepted?.accepted === true) applySignalsToWatermarks(watermarks, accepted.signals); + // A committed lease is consumed whether or not its acknowledgment has landed yet. + // Overlaying it makes derive() treat those signals as delivered, so level reads stay low + // and nothing re-derives a duplicate. + const committed = this.states.get(ownerWorkspaceId)?.committed; + if (committed != null) applySignalsToWatermarks(watermarks, committed.signals); const signals: DerivedSignal[] = []; const autoConsumed: DerivedSignal[] = []; diff --git a/src/node/services/taskService.test.ts b/src/node/services/taskService.test.ts index 9897415288..474a164ade 100644 --- a/src/node/services/taskService.test.ts +++ b/src/node/services/taskService.test.ts @@ -347,7 +347,7 @@ describe("TaskService", () => { isStreaming?: ReturnType; hasQueuedMessages?: ReturnType; hasPendingQueuedOrPreparingTurn?: ReturnType; - hasOutstandingBashMonitorWake?: ReturnType; + hasBashMonitorWakeContinuation?: ReturnType; hasPendingWorkspaceTurnContinuation?: ReturnType; getQueueCutCutter?: ReturnType; hasPendingAutoRetry?: ReturnType; @@ -24363,15 +24363,15 @@ describe("TaskService", () => { expect(snapshot?.reportMarkdown).toBeUndefined(); }); - test("workspace-turn tool-calls stream-end defers to an outstanding wake", async () => { - // An outstanding bash-monitor wake makes the correlated stream yield at a tool - // boundary (finishReason "tool-calls"); the wake turn then continues the same - // turn — the handle must stay running. - const hasOutstandingBashMonitorWake = mock((workspaceId: string) => - Promise.resolve(workspaceId === "childworkspace") + test("workspace-turn tool-calls stream-end defers to an owed wake continuation", async () => { + // The correlated stream yielded at a tool boundary (finishReason "tool-calls") to a + // bash-monitor wake; the session still owes that continuation, so the wake turn will + // continue the same turn — the handle must stay running. + const hasBashMonitorWakeContinuation = mock( + (workspaceId: string) => workspaceId === "childworkspace" ); const { parentId, taskService, workspaceMocks } = await startWorkspaceTurnForTest({ - hasOutstandingBashMonitorWake, + hasBashMonitorWakeContinuation, }); workspaceMocks.getQueueCutCutter.mockImplementation(() => ({ stage: "bash-monitor-wake" as const, @@ -24425,11 +24425,11 @@ describe("TaskService", () => { test("a manual tool-end head owns the cut even while the wake level is high", async () => { // The session attributes the cut to the queued entry (hasPendingToolEndInput: a queue // head arbitrates alone), which runs first and breaks correlation inheritance; the wake - // behind it is not this turn's continuation. Deferring on the level would leave the - // handle running with no correlated stream-end to come (Codex P2 PRRT_kwDOPxxmWM6fOH50). - const hasOutstandingBashMonitorWake = mock(() => Promise.resolve(true)); + // behind it is not this turn's continuation. Deferring on the wake would leave the + // handle running with no correlated stream-end to come. + const hasBashMonitorWakeContinuation = mock(() => true); const { parentId, taskService, workspaceMocks } = await startWorkspaceTurnForTest({ - hasOutstandingBashMonitorWake, + hasBashMonitorWakeContinuation, }); workspaceMocks.getQueueCutCutter.mockImplementation(() => ({ stage: "queued" as const, @@ -24453,7 +24453,7 @@ describe("TaskService", () => { parts: [{ type: "text", text: "Kicked off verification" }], }); - expect(hasOutstandingBashMonitorWake).not.toHaveBeenCalled(); + expect(hasBashMonitorWakeContinuation).not.toHaveBeenCalled(); const settled = await workspaceTurnSnapshot(taskService, parentId); expect(settled?.status).not.toBe("running"); expect(settled).toMatchObject({ messageId: "msg_manual_cut" }); @@ -24461,11 +24461,12 @@ describe("TaskService", () => { test("a wake retracted after the cut settles the handle as a wake cut instead of deferring", async () => { // The stream yielded to the wake level, then the operator canceled the monitor before - // this stream-end was processed: the level is low and no wake turn was admitted, so no - // continuation will ever arrive. Deferring would hang the owner's wait; the session's - // cut attribution settles it as a wake cut (not a truncation failure). + // this stream-end was processed: the session voided its continuation debt (the void's + // own settlement found the record not yet deferred), so no continuation will ever + // arrive. The event-time cut attribution still names the wake, so the record settles as + // a wake cut (not a truncation failure) rather than deferring forever. const { config, parentId, taskService, workspaceMocks } = await startWorkspaceTurnForTest({ - hasOutstandingBashMonitorWake: mock(() => Promise.resolve(false)), + hasBashMonitorWakeContinuation: mock(() => false), }); workspaceMocks.getQueueCutCutter.mockImplementation(() => ({ stage: "bash-monitor-wake" as const, @@ -24497,34 +24498,6 @@ describe("TaskService", () => { }); }); - test("a failing wake probe settles the handle instead of leaving it running", async () => { - // The probe is advisory: its I/O failing must not escape finalization with the terminal - // stream-end already consumed (the handle would stay running until the waiter timed out). - const { parentId, taskService } = await startWorkspaceTurnForTest({ - hasOutstandingBashMonitorWake: mock(() => Promise.reject(new Error("watermark read failed"))), - }); - const internal = taskService as unknown as { - handleStreamEnd: (event: StreamEndEvent) => Promise; - }; - - await internal.handleStreamEnd({ - type: "stream-end", - workspaceId: "childworkspace", - messageId: "msg_probe_failed", - metadata: { - model: "anthropic:claude-opus-4-6", - agentId: "exec", - finishReason: "tool-calls", - muxMetadata: workspaceTurnMuxMetadata(parentId), - }, - parts: [{ type: "text", text: "Kicked off verification" }], - }); - - const settled = await workspaceTurnSnapshot(taskService, parentId); - expect(settled?.status).not.toBe("running"); - expect(settled).toMatchObject({ messageId: "msg_probe_failed" }); - }); - test("nested agent progress preserves workspace-turn correlation", async () => { const hasPendingWorkspaceTurnContinuation = mock( ( @@ -24732,10 +24705,10 @@ describe("TaskService", () => { }); }); - test("settleSupersededWorkspaceTurnContinuation settles the abandoned continuation and wakes the waiter", async () => { + test("settleVoidedWorkspaceTurnContinuation settles an abandoned continuation and wakes the waiter", async () => { // The target abandoned a compaction follow-up carrying this correlation (a manual send // won the idle race): no stream-end will ever carry the correlation again, so the - // abandonment itself settles the handle (Codex P2 PRRT_kwDOPxxmWM6fGVxG). + // abandonment itself settles the handle, deferred or not. const { parentId, taskService } = await startWorkspaceTurnForTest(); const waited = workspaceTurnManagerFor(taskService) .waitForWorkspaceTurn("wst_handle", { requestingWorkspaceId: parentId, timeoutMs: 5_000 }) @@ -24744,9 +24717,10 @@ describe("TaskService", () => { (error: unknown) => error ); - await taskService.settleSupersededWorkspaceTurnContinuation( + await taskService.settleVoidedWorkspaceTurnContinuation( "childworkspace", - workspaceTurnMuxMetadata(parentId) + workspaceTurnMuxMetadata(parentId), + "abandoned" ); const error = await waited; @@ -24758,31 +24732,89 @@ describe("TaskService", () => { "Workspace turn superseded by new input in the target workspace; the workspace continues under that input and this delegated turn will not report", }); - // Idempotent on a settled record, and a no-op for a correlation that is not this turn. - await taskService.settleSupersededWorkspaceTurnContinuation( + // Idempotent on a settled record. + await taskService.settleVoidedWorkspaceTurnContinuation( "childworkspace", - workspaceTurnMuxMetadata(parentId) + workspaceTurnMuxMetadata(parentId), + "abandoned" ); expect(await workspaceTurnSnapshot(taskService, parentId)).toMatchObject({ status: "interrupted", }); }); - test("settleSupersededWorkspaceTurnContinuation ignores a stale correlation", async () => { + test("settleVoidedWorkspaceTurnContinuation ignores a stale correlation", async () => { const { parentId, taskService } = await startWorkspaceTurnForTest(); - await taskService.settleSupersededWorkspaceTurnContinuation( + await taskService.settleVoidedWorkspaceTurnContinuation( "childworkspace", - workspaceTurnMuxMetadata(parentId, "wst_handle", "some-other-turn") + workspaceTurnMuxMetadata(parentId, "wst_handle", "some-other-turn"), + "abandoned" ); - await taskService.settleSupersededWorkspaceTurnContinuation( + await taskService.settleVoidedWorkspaceTurnContinuation( "someone-else", - workspaceTurnMuxMetadata(parentId) + workspaceTurnMuxMetadata(parentId), + "abandoned" ); expect(await workspaceTurnSnapshot(taskService, parentId)).toMatchObject({ status: "running" }); }); + test("a retracted or superseded void settles only a record the stream-end already deferred", async () => { + // A record still running has its stream-end handler queued behind the void on the same + // workspace lock; that handler reads the cleared debt and settles the turn itself, so + // the void must not pre-empt it with a wake-cut outcome the handler may not agree with. + const hasBashMonitorWakeContinuation = mock(() => true); + const { parentId, taskService, workspaceMocks } = await startWorkspaceTurnForTest({ + hasBashMonitorWakeContinuation, + }); + const correlation = workspaceTurnMuxMetadata(parentId); + + await taskService.settleVoidedWorkspaceTurnContinuation( + "childworkspace", + correlation, + "retracted" + ); + expect(await workspaceTurnSnapshot(taskService, parentId)).toMatchObject({ status: "running" }); + + // The stream-end defers on the owed continuation ... + workspaceMocks.getQueueCutCutter.mockImplementation(() => ({ + stage: "bash-monitor-wake" as const, + })); + const internal = taskService as unknown as { + handleStreamEnd: (event: StreamEndEvent) => Promise; + }; + await internal.handleStreamEnd({ + type: "stream-end", + workspaceId: "childworkspace", + messageId: "msg_deferred_cut", + metadata: { + model: "anthropic:claude-opus-4-6", + agentId: "exec", + finishReason: "tool-calls", + muxMetadata: correlation, + }, + parts: [{ type: "text", text: "Waiting on the build" }], + }); + expect(await workspaceTurnSnapshot(taskService, parentId)).toMatchObject({ + status: "running", + deferredMessageIds: ["msg_deferred_cut"], + }); + + // ... and the continuation is then retracted: the deferred record is the one this void + // exists for. + await taskService.settleVoidedWorkspaceTurnContinuation( + "childworkspace", + correlation, + "retracted" + ); + expect(await workspaceTurnSnapshot(taskService, parentId)).toMatchObject({ + status: "interrupted", + error: + "Workspace turn yielded at a tool boundary to a bash-monitor wake that was retracted before delivery; the target workspace is idle and this delegated turn did not complete", + }); + }); + const OWNER_FOLLOW_UP_SUPERSEDE_PREFIX = "Workspace turn superseded by follow-up turn "; function ownerFollowUpCutter(ownerWorkspaceId: string, successorHandleId: string) { diff --git a/src/node/services/taskService.ts b/src/node/services/taskService.ts index 98da2ff366..0aec1f172e 100644 --- a/src/node/services/taskService.ts +++ b/src/node/services/taskService.ts @@ -48,6 +48,7 @@ import { type WorkspaceLifecycleResult, } from "@/node/services/taskWorkspaceSeam"; export type { TaskCreateArgs, TaskKind } from "@/node/services/taskWorkspaceSeam"; +import type { WorkspaceTurnContinuationVoidReason } from "@/node/services/agentSession"; import type { HistoryService } from "@/node/services/historyService"; import type { InitStateManager } from "@/node/services/initStateManager"; import { STRUCTURED_WORKFLOW_REPORT_PLACEHOLDER_MARKDOWN } from "@/common/constants/workflowReports"; @@ -6209,14 +6210,23 @@ export class TaskService implements AgentTaskIntegration { } } - async settleSupersededWorkspaceTurnContinuation( + /** + * Under the same per-workspace lock as handleStreamEnd, so "the finalizer deferred on the + * debt" and "the debt was voided" are ordered: whichever runs second sees the other's + * result (see WorkspaceTurnManager.settleVoidedWorkspaceTurnContinuation). + */ + async settleVoidedWorkspaceTurnContinuation( workspaceId: string, - muxMetadata: Extract + muxMetadata: Extract, + reason: WorkspaceTurnContinuationVoidReason ): Promise { - await this.getWorkspaceTurnManager().settleSupersededWorkspaceTurnContinuation( - workspaceId, - muxMetadata - ); + await this.workspaceEventLocks.withLock(workspaceId, async () => { + await this.getWorkspaceTurnManager().settleVoidedWorkspaceTurnContinuation( + workspaceId, + muxMetadata, + reason + ); + }); } /** diff --git a/src/node/services/taskWorkspaceSeam.testUtils.ts b/src/node/services/taskWorkspaceSeam.testUtils.ts index ed53f5f066..ad8060837b 100644 --- a/src/node/services/taskWorkspaceSeam.testUtils.ts +++ b/src/node/services/taskWorkspaceSeam.testUtils.ts @@ -14,7 +14,7 @@ export function makeWorkspaceHostFake(overrides: Partial = {}): W hasQueuedMessages: () => false, hasPendingQueuedOrPreparingTurn: () => false, hasPendingAutoRetry: () => false, - hasOutstandingBashMonitorWake: () => Promise.resolve(false), + hasBashMonitorWakeContinuation: () => false, isToolEndYieldRequested: () => false, hasPendingWorkspaceTurnContinuation: () => false, hasQueuedWorkspaceTurn: () => false, @@ -70,7 +70,7 @@ export function makeAgentTaskIntegrationFake( getAgentTaskStatus: () => undefined, resetAutoResumeCount: () => undefined, backgroundForegroundWaitsForWorkspace: () => 0, - settleSupersededWorkspaceTurnContinuation: () => Promise.resolve(), + settleVoidedWorkspaceTurnContinuation: () => Promise.resolve(), markInterruptedTaskRunning: () => Promise.resolve(false), restoreInterruptedTaskAfterResumeFailure: () => Promise.resolve(), markParentWorkspaceInterrupted: () => undefined, diff --git a/src/node/services/taskWorkspaceSeam.ts b/src/node/services/taskWorkspaceSeam.ts index 0c3471015f..51dac86ebc 100644 --- a/src/node/services/taskWorkspaceSeam.ts +++ b/src/node/services/taskWorkspaceSeam.ts @@ -11,7 +11,10 @@ import type { WorkspaceTurnTaskCorrelation, } from "@/common/types/message"; import type { Result } from "@/common/types/result"; -import type { StreamErrorRecoveryOutcome } from "@/node/services/agentSession"; +import type { + StreamErrorRecoveryOutcome, + WorkspaceTurnContinuationVoidReason, +} from "@/node/services/agentSession"; import type { RuntimeConfig } from "@/common/types/runtime"; import type { FrontendWorkspaceMetadata, WorkspaceMetadata } from "@/common/types/workspace"; import type { AgentAiSettingsLayerValues } from "@/common/types/agentAiSettings"; @@ -396,8 +399,11 @@ export interface TurnAdmissionHost { hasQueuedMessages(workspaceId: string, dispatchMode?: "tool-end" | "turn-end"): boolean; hasPendingQueuedOrPreparingTurn(workspaceId: string): boolean; hasPendingAutoRetry(workspaceId: string): boolean; - /** Bash-monitor wake level (see WorkspaceService.hasOutstandingBashMonitorWake). */ - hasOutstandingBashMonitorWake(workspaceId: string): Promise; + /** + * The session still owes, or already holds, a bash-monitor wake continuation (see + * AgentSession.hasBashMonitorWakeContinuation). Sync, no I/O. + */ + hasBashMonitorWakeContinuation(workspaceId: string): boolean; /** Pending input (queued tool-end message or outstanding wake) wants a tool boundary. */ isToolEndYieldRequested(workspaceId: string): boolean; hasPendingWorkspaceTurnContinuation( @@ -519,13 +525,14 @@ export interface AgentTaskIntegration { resetAutoResumeCount(workspaceId: string): void; backgroundForegroundWaitsForWorkspace(workspaceId: string): number; /** - * The workspace dropped the continuation of a delegated turn (a compaction follow-up that - * carried the correlation was abandoned instead of dispatched). No later send inherits the - * correlation, so the owner's waiter is settled as superseded. Idempotent. + * The workspace will never continue the delegated turn `muxMetadata` identifies + * (AgentSession.onWorkspaceTurnContinuationVoided). Runs under the workspace event lock; + * idempotent. See WorkspaceTurnManager.settleVoidedWorkspaceTurnContinuation. */ - settleSupersededWorkspaceTurnContinuation( + settleVoidedWorkspaceTurnContinuation( workspaceId: string, - muxMetadata: Extract + muxMetadata: Extract, + reason: WorkspaceTurnContinuationVoidReason ): Promise; markInterruptedTaskRunning(workspaceId: string): Promise; restoreInterruptedTaskAfterResumeFailure(workspaceId: string): Promise; diff --git a/src/node/services/workspaceService.test.ts b/src/node/services/workspaceService.test.ts index 55e292e037..4b13c44d6c 100644 --- a/src/node/services/workspaceService.test.ts +++ b/src/node/services/workspaceService.test.ts @@ -1128,19 +1128,20 @@ describe("WorkspaceService bash monitor wake reconciler wiring", () => { } }); - test("a wake turn in PREPARING keeps the outstanding level visible to turn settlement", async () => { + test("a wake continuation owed or in flight in the session is visible to turn settlement", async () => { const { service, cleanup } = await createWakeWiringService(); const workspaceId = "preparing-wake-owner"; const session = service.getOrCreateSession(workspaceId); const sessionInternal = session as unknown as { - hasPendingBashMonitorWakeTurn(): boolean; + hasBashMonitorWakeContinuation(): boolean; }; try { - expect(await service.hasOutstandingBashMonitorWake(workspaceId)).toBe(false); + expect(service.hasBashMonitorWakeContinuation(workspaceId)).toBe(false); // The reconciler level is already low here (onAccepted ran at row persistence); - // the session marker is what keeps the continuation visible until stream start. - sessionInternal.hasPendingBashMonitorWakeTurn = () => true; - expect(await service.hasOutstandingBashMonitorWake(workspaceId)).toBe(true); + // the session's own debt / in-flight state is what settlement reads. + sessionInternal.hasBashMonitorWakeContinuation = () => true; + expect(service.hasBashMonitorWakeContinuation(workspaceId)).toBe(true); + expect(service.hasBashMonitorWakeContinuation("no-such-session")).toBe(false); } finally { session.dispose(); await cleanup(); @@ -1266,10 +1267,10 @@ describe("WorkspaceService bash monitor wake reconciler wiring", () => { Promise.resolve({ status: "settled", shownThroughOffset, terminalStatusShown: false }) ); - // task_await showed the lines before the SDK asked: no yield, no wake turn. + // task_await showed the lines before the SDK asked: no yield, no debt. expect(await session.hasPendingToolEndInput()).toBe(false); expect(processManager.setMessageQueued).not.toHaveBeenCalledWith(workspaceId, true); - expect(await service.hasOutstandingBashMonitorWake(workspaceId)).toBe(false); + expect(service.hasBashMonitorWakeContinuation(workspaceId)).toBe(false); expect(session.hasQueuedMessages()).toBe(false); // A queued tool-end message still yields on its own. @@ -1286,7 +1287,8 @@ describe("WorkspaceService bash monitor wake reconciler wiring", () => { shownThroughOffset = 0; expect(await session.hasPendingToolEndInput()).toBe(true); expect(processManager.setMessageQueued).toHaveBeenLastCalledWith(workspaceId, true); - expect(await service.hasOutstandingBashMonitorWake(workspaceId)).toBe(true); + expect(service.hasBashMonitorWakeContinuation(workspaceId)).toBe(true); + expect(service.getQueueCutCutter(workspaceId)).toEqual({ stage: "bash-monitor-wake" }); } finally { session.dispose(); await cleanup(); diff --git a/src/node/services/workspaceService.ts b/src/node/services/workspaceService.ts index f82e00ac8e..8fd610c469 100644 --- a/src/node/services/workspaceService.ts +++ b/src/node/services/workspaceService.ts @@ -2587,8 +2587,7 @@ export class WorkspaceService extends EventEmitter implements WorkspaceHost { // park this wake behind it, out of reach of the level (a later monitor cancel // could not retract it). requireIdle turns that race into a skip (Err), and the // admission probe re-validates the wake at every gate before the user row is - // durable; both fall through to the after-idle re-arm below (Codex P2 - // PRRT_kwDOPxxmWM6fDmpJ). + // durable; both fall through to the after-idle re-arm below. requireIdle: true, admissionStale: () => !dispatch.isCurrent(), onAccepted: async () => { @@ -4081,10 +4080,11 @@ export class WorkspaceService extends EventEmitter implements WorkspaceHost { onToolEndYieldRequested: () => { this.agentTaskIntegration?.backgroundForegroundWaitsForWorkspace(workspaceId); }, - onWorkspaceTurnContinuationAbandoned: async (metadata) => { - await this.agentTaskIntegration?.settleSupersededWorkspaceTurnContinuation( + onWorkspaceTurnContinuationVoided: async (correlation, reason) => { + await this.agentTaskIntegration?.settleVoidedWorkspaceTurnContinuation( workspaceId, - metadata + correlation, + reason ); }, }); @@ -11967,20 +11967,9 @@ export class WorkspaceService extends EventEmitter implements WorkspaceHost { ); } - /** - * The bash-monitor wake level: a wake the workspace has not seen yet, or a wake turn - * admitted but not yet shown by a correlated stream — including an on-send compaction - * that consumed it (AgentSession.hasPendingBashMonitorWakeTurn — the reconciler level is - * already consumed there). A stream that ended with "tool-calls" - * while this is high yielded to the wake and will be continued by it. Deliberately not - * the session's cut latch: a wake retracted after the cut (monitor canceled) has no - * continuation, and settlement must not defer on it (AgentSession.getQueueCutCutter - * names the cause instead). - */ - async hasOutstandingBashMonitorWake(workspaceId: string): Promise { - const id = workspaceId.trim(); - if (this.sessions.get(id)?.hasPendingBashMonitorWakeTurn() === true) return true; - return this.bashMonitorWakeReconciler.hasOutstandingWake(id); + /** See AgentSession.hasBashMonitorWakeContinuation. */ + hasBashMonitorWakeContinuation(workspaceId: string): boolean { + return this.sessions.get(workspaceId.trim())?.hasBashMonitorWakeContinuation() === true; } /** diff --git a/src/node/services/workspaceTurnManager.test.ts b/src/node/services/workspaceTurnManager.test.ts index 23e66a40bd..6509596a8c 100644 --- a/src/node/services/workspaceTurnManager.test.ts +++ b/src/node/services/workspaceTurnManager.test.ts @@ -317,7 +317,7 @@ describe("WorkspaceTurnManager", () => { isStreaming?: ReturnType; hasQueuedMessages?: ReturnType; hasPendingQueuedOrPreparingTurn?: ReturnType; - hasOutstandingBashMonitorWake?: ReturnType; + hasBashMonitorWakeContinuation?: ReturnType; hasPendingWorkspaceTurnContinuation?: ReturnType; getQueueCutCutter?: ReturnType; hasPendingAutoRetry?: ReturnType; diff --git a/src/node/services/workspaceTurnManager.ts b/src/node/services/workspaceTurnManager.ts index bfb39a9d28..22e2041b78 100644 --- a/src/node/services/workspaceTurnManager.ts +++ b/src/node/services/workspaceTurnManager.ts @@ -18,6 +18,7 @@ import { type WorkspaceLifecycleResult, type WorkspaceTurnManagerHost, } from "@/node/services/taskWorkspaceSeam"; +import type { WorkspaceTurnContinuationVoidReason } from "@/node/services/agentSession"; import type { HistoryService } from "@/node/services/historyService"; import type { InitStateManager } from "@/node/services/initStateManager"; import { @@ -4325,11 +4326,11 @@ export class WorkspaceTurnManager { * Whether a continuation of this exact delegated turn is pending or streaming. * Pending entries must carry the same correlation metadata as the ended stream. */ - private async hasSameTurnContinuation( + private hasSameTurnContinuation( event: StreamEndEvent, correlation: { taskHandleId: string; ownerWorkspaceId: string; turnId: string }, queueCutSnapshot: QueueCutAttributionSnapshot - ): Promise { + ): boolean { if ( this.workspaceService.hasPendingWorkspaceTurnContinuation(event.workspaceId, { type: "workspace-turn-task", @@ -4342,24 +4343,16 @@ export class WorkspaceTurnManager { // inherits this correlation from history (inheritOpenWorkspaceTurnMetadata). Only the // event-time attribution says whether the level was the cutter: a manual tool-end head // arbitrates the cut even while the level is high, runs first and breaks inheritance, so - // the wake behind it is not this turn's continuation and the handle must settle here - // instead of waiting on a stream-end that may never correlate (Codex P2 - // PRRT_kwDOPxxmWM6fOH50). Whether the wake still arrives is then read live: the probe is - // advisory, so if its I/O fails settle through the normal path rather than leave the - // handle running with its terminal stream-end already consumed (a late correlated - // continuation can still self-heal it) (Codex P2 PRRT_kwDOPxxmWM6fEQIr). - if (queueCutSnapshot.cutter?.stage === "bash-monitor-wake") { - try { - if (await this.workspaceService.hasOutstandingBashMonitorWake(event.workspaceId)) { - return true; - } - } catch (error) { - log.warn("Bash monitor wake probe failed during workspace turn settlement", { - workspaceId: event.workspaceId, - taskHandleId: correlation.taskHandleId, - error, - }); - } + // the wake behind it is not this turn's continuation and the handle must settle here. + // Whether the wake can still arrive is the session's continuation debt, read live and + // synchronously: a void that landed before this handler (same per-workspace lock) has + // already cleared it, and one landing after finds the record deferred and settles it + // itself (settleVoidedWorkspaceTurnContinuation). + if ( + queueCutSnapshot.cutter?.stage === "bash-monitor-wake" && + this.workspaceService.hasBashMonitorWakeContinuation(event.workspaceId) + ) { + return true; } const activeStream = this.streamManager?.getStreamInfo(event.workspaceId); if (activeStream == null || activeStream.messageId === event.messageId) { @@ -4530,7 +4523,7 @@ export class WorkspaceTurnManager { // must settle the old outcome here. if ( event.metadata.finishReason === "tool-calls" && - (await this.hasSameTurnContinuation(event, metadata, queueCutSnapshot)) + this.hasSameTurnContinuation(event, metadata, queueCutSnapshot) ) { await this.markWorkspaceTurnStreamEndDeferred(event); return true; @@ -4687,23 +4680,28 @@ export class WorkspaceTurnManager { } /** - * The target workspace abandoned the continuation carrying this correlation (a wake's - * compaction follow-up skipped for a racing manual send, or an inadmissible summary). - * Nothing downstream can settle the turn: the compaction stream-end is ignored by - * finalizeWorkspaceTurnFromStreamEnd and no later send inherits the correlation, so the - * owner would wait until restart. Settle it as superseded now; if the manual turn does run, - * its uncorrelated stream-end finds the record already settled (Codex P2 - * PRRT_kwDOPxxmWM6fGVxG). + * The target session will never continue the delegated turn identified by `muxMetadata` + * (AgentSession.onWorkspaceTurnContinuationVoided). `retracted` / `superseded` void a + * continuation debt: only a record the stream-end handler already DEFERRED on that debt + * needs settling here — a record still running has its handler queued behind this call on + * the workspace event lock, and that handler reads the (now cleared) debt live and settles + * the turn itself. `abandoned` drops a compaction follow-up carrying the correlation: the + * record may still be running behind the compaction stream (whose stream-end is + * uncorrelated and settles nothing), so any active record settles. */ - async settleSupersededWorkspaceTurnContinuation( + async settleVoidedWorkspaceTurnContinuation( workspaceId: string, - muxMetadata: WorkspaceTurnMuxMetadata + muxMetadata: WorkspaceTurnMuxMetadata, + reason: WorkspaceTurnContinuationVoidReason ): Promise { await this.settleWorkspaceTurnContinuationFailure( workspaceId, muxMetadata, "interrupted", - WORKSPACE_TURN_SUPERSEDED_BY_NEW_INPUT_ERROR + reason === "retracted" + ? WORKSPACE_TURN_YIELDED_TO_RETRACTED_WAKE_ERROR + : WORKSPACE_TURN_SUPERSEDED_BY_NEW_INPUT_ERROR, + { deferredOnly: reason !== "abandoned" } ); } @@ -4713,7 +4711,8 @@ export class WorkspaceTurnManager { workspaceId: string, muxMetadata: WorkspaceTurnMuxMetadata, status: "interrupted" | "error", - error: string + error: string, + options?: { deferredOnly: boolean } ): Promise { const record = await this.taskHandleStore.getWorkspaceTurn( muxMetadata.ownerWorkspaceId, @@ -4722,7 +4721,8 @@ export class WorkspaceTurnManager { if ( record?.workspaceId !== workspaceId || record?.turnId !== muxMetadata.turnId || - !isActiveWorkspaceTurnTaskStatus(record?.status) + !isActiveWorkspaceTurnTaskStatus(record?.status) || + (options?.deferredOnly === true && (record.deferredMessageIds?.length ?? 0) === 0) ) { return; } From 46488272249858ce4952180be3dc623ade11c509 Mon Sep 17 00:00:00 2001 From: Ethan Dickson Date: Fri, 4 Sep 2026 10:37:20 +0000 Subject: [PATCH 11/26] test: fix main-inherited failures surfaced by the merge - workspaceService flat-reorder mock (#3994) sets getSessionDir on Partial, which the split Config (#4017) no longer has; the test only needs sessionsDir - ProjectSidebar flat-list mocks (#3994) lack archivingWorkspaceIds (#4068) - MCP prompt snapshot test asserts the cancelSignal argument this PR removed --- .../components/ProjectSidebar/ProjectSidebar.test.tsx | 5 +++++ src/node/services/agentSession.mcpPromptSnapshot.test.ts | 8 +------- src/node/services/workspaceService.test.ts | 2 +- 3 files changed, 7 insertions(+), 8 deletions(-) diff --git a/src/browser/components/ProjectSidebar/ProjectSidebar.test.tsx b/src/browser/components/ProjectSidebar/ProjectSidebar.test.tsx index e5bf32afd5..ee96b62512 100644 --- a/src/browser/components/ProjectSidebar/ProjectSidebar.test.tsx +++ b/src/browser/components/ProjectSidebar/ProjectSidebar.test.tsx @@ -957,6 +957,7 @@ describe("ProjectSidebar flat chat list", () => { spyOn(WorkspaceContextModule, "useWorkspaceActions").mockImplementation( () => ({ + archivingWorkspaceIds: new Set(), selectedWorkspace: null, setSelectedWorkspace: () => undefined, preflightArchiveWorkspace: () => @@ -1008,6 +1009,7 @@ describe("ProjectSidebar flat chat list", () => { spyOn(WorkspaceContextModule, "useWorkspaceActions").mockImplementation( () => ({ + archivingWorkspaceIds: new Set(), selectedWorkspace: null, setSelectedWorkspace: () => undefined, preflightArchiveWorkspace: () => @@ -1098,6 +1100,7 @@ describe("ProjectSidebar flat chat list", () => { spyOn(WorkspaceContextModule, "useWorkspaceActions").mockImplementation( () => ({ + archivingWorkspaceIds: new Set(), selectedWorkspace: null, setSelectedWorkspace: () => undefined, preflightArchiveWorkspace: () => @@ -1278,6 +1281,7 @@ describe("ProjectSidebar flat chat list", () => { spyOn(WorkspaceContextModule, "useWorkspaceActions").mockImplementation( () => ({ + archivingWorkspaceIds: new Set(), selectedWorkspace: null, setSelectedWorkspace: () => undefined, preflightArchiveWorkspace: () => @@ -1333,6 +1337,7 @@ describe("ProjectSidebar flat chat list", () => { spyOn(WorkspaceContextModule, "useWorkspaceActions").mockImplementation( () => ({ + archivingWorkspaceIds: new Set(), selectedWorkspace: null, setSelectedWorkspace: () => undefined, preflightArchiveWorkspace: () => diff --git a/src/node/services/agentSession.mcpPromptSnapshot.test.ts b/src/node/services/agentSession.mcpPromptSnapshot.test.ts index 6dbae538c2..bd9f243768 100644 --- a/src/node/services/agentSession.mcpPromptSnapshot.test.ts +++ b/src/node/services/agentSession.mcpPromptSnapshot.test.ts @@ -58,13 +58,7 @@ describe("AgentSession MCP prompt snapshots", () => { expect(history.data[0]?.parts.find((part) => part.type === "text")?.text).toBe( "Expanded prompt" ); - expect(getPrompt).toHaveBeenCalledWith( - "workspace", - "coder", - "review", - { path: "src" }, - undefined - ); + expect(getPrompt).toHaveBeenCalledWith("workspace", "coder", "review", { path: "src" }); // The live transcript must also emit the snapshot before the user row. const emittedIds = harness.events diff --git a/src/node/services/workspaceService.test.ts b/src/node/services/workspaceService.test.ts index d447631fe6..96aebf773c 100644 --- a/src/node/services/workspaceService.test.ts +++ b/src/node/services/workspaceService.test.ts @@ -13998,7 +13998,7 @@ describe("WorkspaceService reorderPinned across projects", () => { const mockConfig: Partial = { srcDir: "/tmp/src", - getSessionDir: mock(() => "/tmp/test/sessions"), + sessionsDir: "/tmp/test/sessions", findWorkspace: mock((id: string) => { const found = findEntry(id); if (!found) return null; From ea9bc3c8731c35623926a1f3f3be2ddc6babc142 Mon Sep 17 00:00:00 2001 From: Ethan Dickson Date: Fri, 4 Sep 2026 10:45:22 +0000 Subject: [PATCH 12/26] Settle the wake debt at acceptance; retry failed voids; spare records with another continuation - enterPreparing is a reservation only; other input settles the continuation debt once its row is durable (a refused send leaves the wake owed) - a void whose owner hook fails is parked and retried at the next debt transition and on a timer - settleVoidedWorkspaceTurnContinuation skips records whose turn has another correlated continuation queued or streaming --- .../agentSession.queueDispatch.test.ts | 95 +++++++++++++++- src/node/services/agentSession.ts | 104 +++++++++++++----- src/node/services/taskService.test.ts | 64 +++++++++++ src/node/services/workspaceTurnManager.ts | 33 +++++- 4 files changed, 264 insertions(+), 32 deletions(-) diff --git a/src/node/services/agentSession.queueDispatch.test.ts b/src/node/services/agentSession.queueDispatch.test.ts index e8ab31b330..ea4f223511 100644 --- a/src/node/services/agentSession.queueDispatch.test.ts +++ b/src/node/services/agentSession.queueDispatch.test.ts @@ -731,8 +731,8 @@ describe("AgentSession queued message tool-call dispatch", () => { expect(session.getQueueCutCutter()).toEqual({ stage: "bash-monitor-wake" }); expect(voided).toEqual([]); - // Input that is not the wake supersedes the continuation: the owner is told once, at - // admission, and the cutter is now the admitted input. + // Input that is not the wake supersedes the continuation: the owner is told once, when + // that input's row is durable, and the cutter is now the admitted input. const sendPromise = session.sendMessage("hello", { model: TEST_MODEL, agentId: "exec" }); await streamRequested; expect(session.isBusy()).toBe(true); @@ -978,6 +978,97 @@ describe("AgentSession queued message tool-call dispatch", () => { } }); + test("other input supersedes the debt only once its own row is durable", async () => { + // A superseding send that is refused before anything is persisted changes nothing: the + // wake is still outstanding and will still continue the delegated turn. Admission + // (PREPARING) is a reservation, not acceptance. + const workspaceId = "queue-dispatch-supersede-at-acceptance"; + const voided: Array<[MuxMessageMetadata, string]> = []; + const { session, cleanup } = await createAgentSessionHarness({ + workspaceId, + hasOutstandingBashMonitorWake: () => Promise.resolve(true), + onWorkspaceTurnContinuationVoided: (correlation, reason) => { + voided.push([correlation, reason]); + return Promise.resolve(); + }, + }); + try { + setActiveStreamCorrelation(session, DELEGATED_TURN); + session.setBashMonitorWakeOutstanding(true); + expect(await session.hasPendingToolEndInput()).toBe(true); + + const refused = await session.sendMessage( + "peer message", + { model: TEST_MODEL, agentId: "exec" }, + { admissionStale: () => true } + ); + expect(refused.success).toBe(false); + expect(session.getQueueCutCutter()).toEqual({ stage: "bash-monitor-wake" }); + expect(session.hasBashMonitorWakeContinuation()).toBe(true); + expect(voided).toEqual([]); + + // The same input accepted (row durable) supersedes it exactly once. + const accepted = await session.sendMessage("peer message", { + model: TEST_MODEL, + agentId: "exec", + }); + expect(accepted.success).toBe(true); + expect(voided).toEqual([[DELEGATED_TURN, "superseded"]]); + expect(session.hasBashMonitorWakeContinuation()).toBe(false); + } finally { + session.dispose(); + await cleanup(); + } + }); + + test("a void whose owner-side settlement fails is retried at the next debt transition", async () => { + // The debt is cleared when it is voided; the settlement itself is I/O on the owner's side + // and may fail transiently. The void is kept and retried rather than logged away, so a + // delegated handle already deferred on the debt does not wait for a restart. + const workspaceId = "queue-dispatch-void-retry"; + const calls: Array<[MuxMessageMetadata, string]> = []; + let failNext = true; + const { session, cleanup } = await createAgentSessionHarness({ + workspaceId, + hasOutstandingBashMonitorWake: () => Promise.resolve(true), + onWorkspaceTurnContinuationVoided: (correlation, reason) => { + calls.push([correlation, reason]); + if (failNext) { + failNext = false; + return Promise.reject(new Error("handle store unavailable")); + } + return Promise.resolve(); + }, + }); + try { + setActiveStreamCorrelation(session, DELEGATED_TURN); + session.setBashMonitorWakeOutstanding(true); + expect(await session.hasPendingToolEndInput()).toBe(true); + + session.setBashMonitorWakeOutstanding(false); + expect(calls).toEqual([[DELEGATED_TURN, "retracted"]]); + // Let the rejection settle; the debt itself stays cleared (the cut is not re-attributed). + await Promise.resolve(); + await Promise.resolve(); + expect(session.hasBashMonitorWakeContinuation()).toBe(false); + + // Any later level transition retries the parked void with the original reason. + session.setBashMonitorWakeOutstanding(true); + session.setBashMonitorWakeOutstanding(false); + expect(calls).toEqual([ + [DELEGATED_TURN, "retracted"], + [DELEGATED_TURN, "retracted"], + ]); + await Promise.resolve(); + session.setBashMonitorWakeOutstanding(true); + session.setBashMonitorWakeOutstanding(false); + expect(calls).toHaveLength(2); + } finally { + session.dispose(); + await cleanup(); + } + }); + test("a queued head is not recorded as a wake cut", async () => { const workspaceId = "queue-dispatch-queue-cut-not-wake"; let releaseLevel: () => void = () => undefined; diff --git a/src/node/services/agentSession.ts b/src/node/services/agentSession.ts index 0af11d72db..797568f394 100644 --- a/src/node/services/agentSession.ts +++ b/src/node/services/agentSession.ts @@ -533,6 +533,8 @@ export const CONTEXT_MUTATION_SEND_BLOCKED_MESSAGE = const SESSION_SHUTDOWN_SEND_BLOCKED_MESSAGE = "Xum is shutting down; the message was not sent."; const STARTUP_AUTO_RETRY_HISTORY_FAILURE_BASE_DELAY_MS = 1_000; +/** Retry cadence for a voided wake continuation whose owner-side settlement failed. */ +const UNSETTLED_WAKE_VOID_RETRY_DELAY_MS = 5_000; const STARTUP_AUTO_RETRY_HISTORY_FAILURE_MAX_DELAY_MS = 30_000; const MAX_STARTUP_RECOVERY_DEFERRED_ATTEMPTS = 4; @@ -943,14 +945,25 @@ export class AgentSession { * armed; otherwise false → maybeVoid * level lowered (cancel / shown / clear) → maybeVoid * maybeVoid: debt ∧ ¬inFlight ∧ ¬level → void "retracted" - * non-wake input admitted (enterPreparing) → void "superseded" + * non-wake input accepted (its row durable) → void "superseded"; a turn with the + * (settleWakeDebtForAcceptedInput) same correlation continues the debt + * itself (its stream-end settles it) * compaction follow-up with the correlation → void "abandoned" (before the erase) * dropped (clearPendingFollowUpFromSummary) * dispose / IDLE → in-flight false (IDLE keeps the debt: * the wake dispatcher needs an idle session) + * + * Voiding clears the debt synchronously; the owner hook may fail (handle store, waiter, + * cleanup I/O). A failed void is kept in `unsettledWakeVoids` and retried at every later + * debt transition and on a timer, so a deferred delegated handle never waits for a restart. */ private wakeContinuationDebt?: { correlation?: WorkspaceTurnMuxMetadata }; private wakeTurnInFlight = false; + private unsettledWakeVoids: Array<{ + correlation: WorkspaceTurnMuxMetadata; + reason: WorkspaceTurnContinuationVoidReason; + }> = []; + private unsettledWakeVoidRetryTimer?: ReturnType; /** Context needed to retry the current stream (cleared on stream end/abort/error). */ private activeStreamContext?: { @@ -1105,6 +1118,11 @@ export class AgentSession { // (workspace teardown settles delegated turns through its own path). this.wakeTurnInFlight = false; this.wakeContinuationDebt = undefined; + this.unsettledWakeVoids = []; + if (this.unsettledWakeVoidRetryTimer != null) { + clearTimeout(this.unsettledWakeVoidRetryTimer); + this.unsettledWakeVoidRetryTimer = undefined; + } // Ensure any callers blocked on waitForIdle() can continue during teardown. this.setTurnPhase(TurnPhase.IDLE); @@ -4026,6 +4044,7 @@ export class AgentSession { // accepted even if a later step throws: otherwise the reconciler re-derives the same wake // and delivers it twice (startup recovery resumes the durable row without redelivery). const finalizeDurableWakeOnFailure = typedMuxMetadata?.type === "bash-monitor-wake"; + this.settleWakeDebtForAcceptedInput(typedMuxMetadata); // r54: the pre-turn batch is now irrevocable — rollbackPersistedTurnRows // is never invoked past this point, so even a failure in goal sync or // acceptance leaves the payload + trigger rows durable in the transcript. @@ -6847,6 +6866,7 @@ export class AgentSession { } private maybeVoidWakeContinuation(): void { + this.retryUnsettledWakeVoids(); if ( this.wakeContinuationDebt != null && !this.wakeTurnInFlight && @@ -6856,6 +6876,28 @@ export class AgentSession { } } + /** + * Input other than the wake has crossed its acceptance boundary (row durable, so the send + * can no longer fail without leaving a resumable turn). Only now can it settle the debt: a + * send refused earlier — pricing, staleness, persistence — leaves the wake outstanding and + * its continuation still owed. Synchronous with the PREPARING reservation this is not: + * enterPreparing only reserves the turn. + */ + private settleWakeDebtForAcceptedInput(muxMetadata: unknown): void { + if (this.wakeContinuationDebt == null || carriesBashMonitorWake(muxMetadata)) return; + if ( + hasSameWorkspaceTurnCorrelation( + getWorkspaceTurnMuxMetadata(muxMetadata), + this.wakeContinuationDebt.correlation + ) + ) { + this.wakeContinuationDebt = undefined; + this.retryUnsettledWakeVoids(); + } else { + this.voidWakeContinuation("superseded"); + } + } + /** * Sync transition; the owner hook runs as a tracked promise (never awaited here — callers * sit inside admission and phase transitions). @@ -6863,15 +6905,40 @@ export class AgentSession { private voidWakeContinuation(reason: WorkspaceTurnContinuationVoidReason): void { const debt = this.wakeContinuationDebt; this.wakeContinuationDebt = undefined; + this.retryUnsettledWakeVoids(); const correlation = debt?.correlation; - if (correlation == null || this.onWorkspaceTurnContinuationVoided == null) return; - this.onWorkspaceTurnContinuationVoided(correlation, reason).catch((error: unknown) => { - log.error("Voided bash-monitor wake continuation could not be settled", { - workspaceId: this.workspaceId, - reason, - error: getErrorMessage(error), - }); - }); + if (correlation == null) return; + this.settleVoidedWakeContinuation({ correlation, reason }); + } + + /** Runs the owner hook; a failure parks the void for retry instead of dropping it. */ + private settleVoidedWakeContinuation(voided: { + correlation: WorkspaceTurnMuxMetadata; + reason: WorkspaceTurnContinuationVoidReason; + }): void { + if (this.onWorkspaceTurnContinuationVoided == null) return; + this.onWorkspaceTurnContinuationVoided(voided.correlation, voided.reason).catch( + (error: unknown) => { + log.error("Voided bash-monitor wake continuation could not be settled; will retry", { + workspaceId: this.workspaceId, + reason: voided.reason, + error: getErrorMessage(error), + }); + if (this.disposed) return; + this.unsettledWakeVoids.push(voided); + this.unsettledWakeVoidRetryTimer ??= setTimeout(() => { + this.unsettledWakeVoidRetryTimer = undefined; + this.retryUnsettledWakeVoids(); + }, UNSETTLED_WAKE_VOID_RETRY_DELAY_MS); + } + ); + } + + private retryUnsettledWakeVoids(): void { + if (this.unsettledWakeVoids.length === 0) return; + const pending = this.unsettledWakeVoids; + this.unsettledWakeVoids = []; + for (const voided of pending) this.settleVoidedWakeContinuation(voided); } /** @@ -6938,25 +7005,12 @@ export class AgentSession { } /** - * Claim PREPARING for a send and record what kind of input it carries. Admitting anything - * but the wake while a continuation debt is outstanding settles that debt: a turn with the - * same correlation continues the delegated turn itself (its stream-end settles it), any - * other input supersedes it. + * Claim PREPARING for a send and record what kind of input it carries. A reservation only: + * an outstanding wake continuation debt is settled when the input is accepted + * (settleWakeDebtForAcceptedInput), not when it is admitted. */ private enterPreparing(muxMetadata: unknown): void { this.preparingWorkspaceTurnMetadata = getWorkspaceTurnMuxMetadata(muxMetadata); - if (!carriesBashMonitorWake(muxMetadata) && this.wakeContinuationDebt != null) { - if ( - hasSameWorkspaceTurnCorrelation( - this.preparingWorkspaceTurnMetadata, - this.wakeContinuationDebt.correlation - ) - ) { - this.wakeContinuationDebt = undefined; - } else { - this.voidWakeContinuation("superseded"); - } - } this.setTurnPhase(TurnPhase.PREPARING); } diff --git a/src/node/services/taskService.test.ts b/src/node/services/taskService.test.ts index 474a164ade..ba4268d72b 100644 --- a/src/node/services/taskService.test.ts +++ b/src/node/services/taskService.test.ts @@ -24815,6 +24815,70 @@ describe("TaskService", () => { }); }); + test("a void leaves a record whose turn has another correlated continuation queued", async () => { + // A correlated report queued after the wake cut also deferred the stream-end and will + // settle the record with its own stream-end. A retracted wake says nothing about that + // continuation, so the void must not interrupt the turn under it. + const hasBashMonitorWakeContinuation = mock(() => true); + let queuedContinuation = false; + const hasPendingWorkspaceTurnContinuation = mock( + (workspaceId: string, metadata: { taskHandleId: string; turnId: string }) => + queuedContinuation && + workspaceId === "childworkspace" && + metadata.taskHandleId === "wst_handle" && + metadata.turnId === "turn" + ); + const { parentId, taskService, workspaceMocks } = await startWorkspaceTurnForTest({ + hasBashMonitorWakeContinuation, + hasPendingWorkspaceTurnContinuation, + }); + const correlation = workspaceTurnMuxMetadata(parentId); + workspaceMocks.getQueueCutCutter.mockImplementation(() => ({ + stage: "bash-monitor-wake" as const, + })); + const internal = taskService as unknown as { + handleStreamEnd: (event: StreamEndEvent) => Promise; + }; + await internal.handleStreamEnd({ + type: "stream-end", + workspaceId: "childworkspace", + messageId: "msg_deferred_cut", + metadata: { + model: "anthropic:claude-opus-4-6", + agentId: "exec", + finishReason: "tool-calls", + muxMetadata: correlation, + }, + parts: [{ type: "text", text: "Waiting on the build" }], + }); + expect(await workspaceTurnSnapshot(taskService, parentId)).toMatchObject({ + status: "running", + deferredMessageIds: ["msg_deferred_cut"], + }); + + queuedContinuation = true; + await taskService.settleVoidedWorkspaceTurnContinuation( + "childworkspace", + correlation, + "retracted" + ); + expect(await workspaceTurnSnapshot(taskService, parentId)).toMatchObject({ + status: "running", + deferredMessageIds: ["msg_deferred_cut"], + }); + + // With no other continuation left, the same void settles the deferred record. + queuedContinuation = false; + await taskService.settleVoidedWorkspaceTurnContinuation( + "childworkspace", + correlation, + "superseded" + ); + expect(await workspaceTurnSnapshot(taskService, parentId)).toMatchObject({ + status: "interrupted", + }); + }); + const OWNER_FOLLOW_UP_SUPERSEDE_PREFIX = "Workspace turn superseded by follow-up turn "; function ownerFollowUpCutter(ownerWorkspaceId: string, successorHandleId: string) { diff --git a/src/node/services/workspaceTurnManager.ts b/src/node/services/workspaceTurnManager.ts index 22e2041b78..6f26702e19 100644 --- a/src/node/services/workspaceTurnManager.ts +++ b/src/node/services/workspaceTurnManager.ts @@ -4354,8 +4354,20 @@ export class WorkspaceTurnManager { ) { return true; } - const activeStream = this.streamManager?.getStreamInfo(event.workspaceId); - if (activeStream == null || activeStream.messageId === event.messageId) { + return this.hasCorrelatedActiveStream(event.workspaceId, correlation, [event.messageId]); + } + + /** + * Whether a stream other than `excludeMessageIds` (the ending stream, or streams whose + * stream-end this record already deferred) is currently streaming this exact correlation. + */ + private hasCorrelatedActiveStream( + workspaceId: string, + correlation: { taskHandleId: string; ownerWorkspaceId: string; turnId: string }, + excludeMessageIds: readonly string[] + ): boolean { + const activeStream = this.streamManager?.getStreamInfo(workspaceId); + if (activeStream == null || excludeMessageIds.includes(activeStream.messageId)) { return false; } const activeCorrelation = this.getWorkspaceTurnMetadataFromValue(activeStream.muxMetadata); @@ -4688,12 +4700,20 @@ export class WorkspaceTurnManager { * the turn itself. `abandoned` drops a compaction follow-up carrying the correlation: the * record may still be running behind the compaction stream (whose stream-end is * uncorrelated and settles nothing), so any active record settles. + * + * A void says nothing about OTHER continuations of the same turn: a correlated report + * queued after the wake cut (or already streaming) also deferred the stream-end and will + * settle the record with its own stream-end. Settling here would interrupt a turn that is + * about to continue — and a disposable turn would delete its workspace under that stream. */ async settleVoidedWorkspaceTurnContinuation( workspaceId: string, muxMetadata: WorkspaceTurnMuxMetadata, reason: WorkspaceTurnContinuationVoidReason ): Promise { + if (this.workspaceService.hasPendingWorkspaceTurnContinuation(workspaceId, muxMetadata)) { + return; + } await this.settleWorkspaceTurnContinuationFailure( workspaceId, muxMetadata, @@ -4701,7 +4721,7 @@ export class WorkspaceTurnManager { reason === "retracted" ? WORKSPACE_TURN_YIELDED_TO_RETRACTED_WAKE_ERROR : WORKSPACE_TURN_SUPERSEDED_BY_NEW_INPUT_ERROR, - { deferredOnly: reason !== "abandoned" } + { deferredOnly: reason !== "abandoned", unlessCorrelatedStreamActive: true } ); } @@ -4712,7 +4732,7 @@ export class WorkspaceTurnManager { muxMetadata: WorkspaceTurnMuxMetadata, status: "interrupted" | "error", error: string, - options?: { deferredOnly: boolean } + options?: { deferredOnly: boolean; unlessCorrelatedStreamActive: boolean } ): Promise { const record = await this.taskHandleStore.getWorkspaceTurn( muxMetadata.ownerWorkspaceId, @@ -4722,7 +4742,10 @@ export class WorkspaceTurnManager { record?.workspaceId !== workspaceId || record?.turnId !== muxMetadata.turnId || !isActiveWorkspaceTurnTaskStatus(record?.status) || - (options?.deferredOnly === true && (record.deferredMessageIds?.length ?? 0) === 0) + (options?.deferredOnly === true && (record.deferredMessageIds?.length ?? 0) === 0) || + (options?.unlessCorrelatedStreamActive === true && + // A correlated stream whose stream-end the record has not deferred settles the turn. + this.hasCorrelatedActiveStream(workspaceId, muxMetadata, record.deferredMessageIds ?? [])) ) { return; } From 6368d9423f6f39c1551f4f6575d4918b2e9edc8e Mon Sep 17 00:00:00 2001 From: Ethan Dickson Date: Fri, 4 Sep 2026 10:52:37 +0000 Subject: [PATCH 13/26] Dispatch bash-monitor wakes under the owner's workspace event lock The lock is FIFO with the stream-end handler of the stream that yielded to the wake, so the wake cannot start (and redeem the continuation debt) before that handler has read the debt to defer the delegated turn. --- src/node/services/taskService.ts | 5 ++ .../services/taskWorkspaceSeam.testUtils.ts | 2 + src/node/services/taskWorkspaceSeam.ts | 6 ++ src/node/services/workspaceService.test.ts | 79 +++++++++++++++++++ src/node/services/workspaceService.ts | 17 ++++ 5 files changed, 109 insertions(+) diff --git a/src/node/services/taskService.ts b/src/node/services/taskService.ts index 0aec1f172e..635154e4d5 100644 --- a/src/node/services/taskService.ts +++ b/src/node/services/taskService.ts @@ -6229,6 +6229,11 @@ export class TaskService implements AgentTaskIntegration { }); } + withWorkspaceEventLock(workspaceId: string, operation: () => Promise): Promise { + assert(workspaceId.length > 0, "withWorkspaceEventLock requires workspaceId"); + return this.workspaceEventLocks.withLock(workspaceId, operation); + } + /** * Reject all foreground task waiters for a workspace that opted into backgrounding * when a new message is queued. Returns the number of waiters signaled. diff --git a/src/node/services/taskWorkspaceSeam.testUtils.ts b/src/node/services/taskWorkspaceSeam.testUtils.ts index ad8060837b..054094a6a9 100644 --- a/src/node/services/taskWorkspaceSeam.testUtils.ts +++ b/src/node/services/taskWorkspaceSeam.testUtils.ts @@ -71,6 +71,8 @@ export function makeAgentTaskIntegrationFake( resetAutoResumeCount: () => undefined, backgroundForegroundWaitsForWorkspace: () => 0, settleVoidedWorkspaceTurnContinuation: () => Promise.resolve(), + withWorkspaceEventLock: (_workspaceId: string, operation: () => Promise): Promise => + operation(), markInterruptedTaskRunning: () => Promise.resolve(false), restoreInterruptedTaskAfterResumeFailure: () => Promise.resolve(), markParentWorkspaceInterrupted: () => undefined, diff --git a/src/node/services/taskWorkspaceSeam.ts b/src/node/services/taskWorkspaceSeam.ts index 51dac86ebc..5d53f661aa 100644 --- a/src/node/services/taskWorkspaceSeam.ts +++ b/src/node/services/taskWorkspaceSeam.ts @@ -534,6 +534,12 @@ export interface AgentTaskIntegration { muxMetadata: Extract, reason: WorkspaceTurnContinuationVoidReason ): Promise; + /** + * Run `operation` under the per-workspace event lock that serializes stream-end/abort/error + * handling for `workspaceId` (FIFO). A send made inside it is ordered after every handler + * already queued for that workspace's earlier events. + */ + withWorkspaceEventLock(workspaceId: string, operation: () => Promise): Promise; markInterruptedTaskRunning(workspaceId: string): Promise; restoreInterruptedTaskAfterResumeFailure(workspaceId: string): Promise; markParentWorkspaceInterrupted(workspaceId: string): void; diff --git a/src/node/services/workspaceService.test.ts b/src/node/services/workspaceService.test.ts index 96aebf773c..4cad2d4dc7 100644 --- a/src/node/services/workspaceService.test.ts +++ b/src/node/services/workspaceService.test.ts @@ -967,6 +967,85 @@ describe("WorkspaceService bash monitor wake reconciler wiring", () => { } }); + test("the wake send waits behind the owner's queued stream-end handling", async () => { + // The stream that yielded to the wake emits stream-end, and TaskService handles it under + // the workspace event lock. That handler reads the continuation debt to defer a delegated + // turn; a wake that streamed first would have redeemed the debt and the handler would + // read the cut as retracted. Dispatch therefore enters the same (FIFO) lock. + const { config, service, cleanup } = await createWakeWiringService(); + const workspaceId = "event-locked-wake-owner"; + await config.addWorkspace("/tmp/event-locked-wake-project", { + id: workspaceId, + name: workspaceId, + projectName: "event-locked-wake-project", + projectPath: "/tmp/event-locked-wake-project", + runtimeConfig: { type: "local" }, + }); + let releaseHandler: () => void = () => undefined; + const handlerDone = new Promise((resolve) => { + releaseHandler = resolve; + }); + const lockedWorkspaceIds: string[] = []; + service.setAgentTaskIntegration( + makeAgentTaskIntegrationFake({ + withWorkspaceEventLock: async (lockedWorkspaceId, operation) => { + lockedWorkspaceIds.push(lockedWorkspaceId); + await handlerDone; + return operation(); + }, + }) + ); + const sendMessage = mock( + ( + _workspaceId: string, + _prompt: string, + _options: unknown, + internal?: { onAccepted?: () => Promise } + ) => internal?.onAccepted?.().then(() => Ok(undefined)) ?? Promise.resolve(Ok(undefined)) + ); + const internal = service as unknown as { + aiService: { isStreaming(workspaceId: string): boolean }; + hasPendingQueuedOrPreparingTurn(workspaceId: string): boolean; + isBusyForMessage(workspaceId: string): boolean; + getDelegatedTurnContinuationSendOptions(workspaceId: string): Promise; + sendMessage: typeof sendMessage; + dispatchBashMonitorWake(dispatch: { + ownerWorkspaceId: string; + prompt: string; + muxMetadata: { type: "bash-monitor-wake"; records: [] }; + isCurrent(): boolean; + onAccepted(): Promise; + onDeferred(): Promise; + }): Promise<"in-flight" | "deferred">; + }; + try { + internal.aiService = { isStreaming: () => false }; + internal.hasPendingQueuedOrPreparingTurn = () => false; + internal.isBusyForMessage = () => false; + internal.getDelegatedTurnContinuationSendOptions = () => Promise.resolve({}); + internal.sendMessage = sendMessage; + + const outcome = internal.dispatchBashMonitorWake({ + ownerWorkspaceId: workspaceId, + prompt: "wake", + muxMetadata: { type: "bash-monitor-wake", records: [] }, + isCurrent: () => true, + onAccepted: () => Promise.resolve(), + onDeferred: () => Promise.resolve(), + }); + await Promise.resolve(); + await Promise.resolve(); + expect(lockedWorkspaceIds).toEqual([workspaceId]); + expect(sendMessage).not.toHaveBeenCalled(); + + releaseHandler(); + expect(await outcome).toBe("in-flight"); + expect(sendMessage).toHaveBeenCalledTimes(1); + } finally { + await cleanup(); + } + }); + test("a monitor canceled while send options resolve retires the wake before it is sent", async () => { const { config, service, cleanup } = await createWakeWiringService(); const workspaceId = "canceled-mid-dispatch-wake-owner"; diff --git a/src/node/services/workspaceService.ts b/src/node/services/workspaceService.ts index 24a9e905eb..0b02f689cb 100644 --- a/src/node/services/workspaceService.ts +++ b/src/node/services/workspaceService.ts @@ -2538,9 +2538,26 @@ export class WorkspaceService extends EventEmitter implements WorkspaceHost { * message: while the owner streams, the stream itself reads the level at each tool * boundary (AgentSession.hasPendingToolEndInput) and yields with finishReason * "tool-calls"; the after-idle reconcile then lands here again and sends directly. + * + * The send runs under the owner's workspace event lock (outermost — workspace removal + * takes the history lock while holding it, so this order is the only deadlock-free one). + * The lock is FIFO, so the wake cannot start — and redeem the continuation debt the cut + * stream took — before the stream-end handler of the stream that yielded to it has run: + * that handler is what reads the debt to defer, rather than settle, a delegated turn. */ private async dispatchBashMonitorWake( dispatch: BashMonitorWakeDispatch + ): Promise { + const ownerWorkspaceId = dispatch.ownerWorkspaceId; + const underEventLock = (operation: () => Promise): Promise => + this.agentTaskIntegration != null + ? this.agentTaskIntegration.withWorkspaceEventLock(ownerWorkspaceId, operation) + : operation(); + return underEventLock(() => this.dispatchBashMonitorWakeUnderEventLock(dispatch)); + } + + private async dispatchBashMonitorWakeUnderEventLock( + dispatch: BashMonitorWakeDispatch ): Promise { return this.bashMonitorHistoryLocks.withLock(dispatch.ownerWorkspaceId, async () => { const ownerWorkspaceId = dispatch.ownerWorkspaceId; From 6f97041a60535f32fd3dc37125e4b85c3b6927e6 Mon Sep 17 00:00:00 2001 From: Ethan Dickson Date: Fri, 4 Sep 2026 11:02:26 +0000 Subject: [PATCH 14/26] A correlated continuation assumes the wake debt until its stream starts; re-check pending continuations after the record read --- .../agentSession.queueDispatch.test.ts | 18 +++++- src/node/services/agentSession.ts | 58 +++++++++++++++---- src/node/services/taskService.test.ts | 25 ++++++-- src/node/services/workspaceTurnManager.ts | 22 ++++--- 4 files changed, 95 insertions(+), 28 deletions(-) diff --git a/src/node/services/agentSession.queueDispatch.test.ts b/src/node/services/agentSession.queueDispatch.test.ts index ea4f223511..4eb1b85956 100644 --- a/src/node/services/agentSession.queueDispatch.test.ts +++ b/src/node/services/agentSession.queueDispatch.test.ts @@ -751,9 +751,11 @@ describe("AgentSession queued message tool-call dispatch", () => { } }); - test("a correlated turn admitted after the cut continues the debt without settling it", async () => { + test("a correlated turn admitted after the cut assumes the debt until its stream starts", async () => { // The delegated turn's own continuation (e.g. a queued same-turn message) supersedes - // nothing: its stream-end settles the turn, so the owner is not told. + // nothing: its stream-end settles the turn, so the owner is not told. Until that stream + // starts the debt stays visible to settlement (a stream-end handler running in the gap + // must still defer) and cannot be retracted by the level lowering. const workspaceId = "queue-dispatch-wake-cut-same-turn"; let level = true; let markStreamRequested: () => void = () => undefined; @@ -765,7 +767,7 @@ describe("AgentSession queued message tool-call dispatch", () => { releaseStream = resolve; }); const voided: unknown[] = []; - const { session, cleanup } = await createAgentSessionHarness({ + const { session, cleanup, aiEmitter } = await createAgentSessionHarness({ workspaceId, hasOutstandingBashMonitorWake: () => Promise.resolve(level), onWorkspaceTurnContinuationVoided: (...args) => { @@ -791,7 +793,17 @@ describe("AgentSession queued message tool-call dispatch", () => { muxMetadata: DELEGATED_TURN, }); await streamRequested; + expect(session.hasBashMonitorWakeContinuation()).toBe(true); + // PREPARING attributes the cut to this continuation; either attribution defers. + expect(session.getQueueCutCutter()?.stage).toBe("preparing"); + session.setBashMonitorWakeOutstanding(true); + session.setBashMonitorWakeOutstanding(false); + expect(session.hasBashMonitorWakeContinuation()).toBe(true); + expect(voided).toEqual([]); + + aiEmitter.emit("stream-start", streamStartEvent(workspaceId)); expect(session.hasBashMonitorWakeContinuation()).toBe(false); + expect(session.getQueueCutCutter()).toBeUndefined(); expect(voided).toEqual([]); session.dispose(); disposed = true; diff --git a/src/node/services/agentSession.ts b/src/node/services/agentSession.ts index 797568f394..70a3273785 100644 --- a/src/node/services/agentSession.ts +++ b/src/node/services/agentSession.ts @@ -946,8 +946,13 @@ export class AgentSession { * level lowered (cancel / shown / clear) → maybeVoid * maybeVoid: debt ∧ ¬inFlight ∧ ¬level → void "retracted" * non-wake input accepted (its row durable) → void "superseded"; a turn with the - * (settleWakeDebtForAcceptedInput) same correlation continues the debt - * itself (its stream-end settles it) + * (settleWakeDebtForAcceptedInput) same correlation ASSUMES the debt: it + * stays visible to settlement, cannot be + * retracted under the continuation, and + * its stream-start discharges it (that + * stream's own stream-end settles the turn) + * assuming send returns without a stream → un-assume unless an auto-retry of its + * durable row is armed; then maybeVoid * compaction follow-up with the correlation → void "abandoned" (before the erase) * dropped (clearPendingFollowUpFromSummary) * dispose / IDLE → in-flight false (IDLE keeps the debt: @@ -957,7 +962,7 @@ export class AgentSession { * cleanup I/O). A failed void is kept in `unsettledWakeVoids` and retried at every later * debt transition and on a timer, so a deferred delegated handle never waits for a restart. */ - private wakeContinuationDebt?: { correlation?: WorkspaceTurnMuxMetadata }; + private wakeContinuationDebt?: { correlation?: WorkspaceTurnMuxMetadata; assumed?: true }; private wakeTurnInFlight = false; private unsettledWakeVoids: Array<{ correlation: WorkspaceTurnMuxMetadata; @@ -3191,7 +3196,7 @@ export class AgentSession { try { return await this.sendMessageInner(...args); } finally { - if (wake) this.settleWakeTurnInFlight(); + if (wake || this.wakeContinuationDebt?.assumed === true) this.settleWakeTurnInFlight(); } } @@ -4307,7 +4312,7 @@ export class AgentSession { try { return await this.resumeStreamInner(...args); } finally { - if (wake) this.settleWakeTurnInFlight(); + if (wake || this.wakeContinuationDebt?.assumed === true) this.settleWakeTurnInFlight(); } } @@ -5949,6 +5954,16 @@ export class AgentSession { if (streamMetadata?.type === "bash-monitor-wake") { this.wakeTurnInFlight = false; this.wakeContinuationDebt = undefined; + } else if ( + this.wakeContinuationDebt != null && + hasSameWorkspaceTurnCorrelation( + getWorkspaceTurnMuxMetadata(streamMetadata), + this.wakeContinuationDebt.correlation + ) + ) { + // The delegated turn's own continuation is streaming: its stream-end settles the + // turn, so the debt it assumed at acceptance is discharged. + this.wakeContinuationDebt = undefined; } this.activeStreamStartedAtMs = payload.startTime; // Codex P1 (PRRT_kwDOPxxmWM6cClKS): a new live stream makes mid-stream @@ -6853,15 +6868,29 @@ export class AgentSession { * or an auto-retry armed for its durable row. */ private settleWakeTurnInFlight(): void { - if (!this.wakeTurnInFlight) return; + const assumed = this.wakeContinuationDebt?.assumed === true; + if (!this.wakeTurnInFlight && !assumed) return; + // A turn in progress (the wake's / continuation's own stream, PREPARING, or a compaction + // stream whose follow-up is that turn) can still start the stream. if (this.turnPhase !== TurnPhase.IDLE) return; + const retryMetadata: unknown = this.hasPendingAutoRetry() + ? this.lastAutoRetryResumeRequest?.options.muxMetadata + : undefined; + if (this.wakeTurnInFlight && !carriesBashMonitorWake(retryMetadata)) { + this.wakeTurnInFlight = false; + } if ( - this.hasPendingAutoRetry() && - carriesBashMonitorWake(this.lastAutoRetryResumeRequest?.options.muxMetadata) + assumed && + this.wakeContinuationDebt != null && + !hasSameWorkspaceTurnCorrelation( + getWorkspaceTurnMuxMetadata(retryMetadata), + this.wakeContinuationDebt.correlation + ) ) { - return; + // The continuation that assumed the debt is not coming from this send; the debt is + // plain owed again (the wake, if still outstanding, will discharge it). + delete this.wakeContinuationDebt.assumed; } - this.wakeTurnInFlight = false; this.maybeVoidWakeContinuation(); } @@ -6869,6 +6898,7 @@ export class AgentSession { this.retryUnsettledWakeVoids(); if ( this.wakeContinuationDebt != null && + this.wakeContinuationDebt.assumed !== true && !this.wakeTurnInFlight && !this.bashMonitorWakeOutstanding ) { @@ -6882,6 +6912,12 @@ export class AgentSession { * send refused earlier — pricing, staleness, persistence — leaves the wake outstanding and * its continuation still owed. Synchronous with the PREPARING reservation this is not: * enterPreparing only reserves the turn. + * + * A continuation of the delegated turn itself does not clear the debt here: between this + * point and its PREPARING / stream-start nothing else would tell settlement that the turn + * continues, and a stream-end handler running in that gap would settle (and, for a + * disposable turn, remove the workspace under) the accepted continuation. It assumes the + * debt instead; its stream-start discharges it. */ private settleWakeDebtForAcceptedInput(muxMetadata: unknown): void { if (this.wakeContinuationDebt == null || carriesBashMonitorWake(muxMetadata)) return; @@ -6891,7 +6927,7 @@ export class AgentSession { this.wakeContinuationDebt.correlation ) ) { - this.wakeContinuationDebt = undefined; + this.wakeContinuationDebt.assumed = true; this.retryUnsettledWakeVoids(); } else { this.voidWakeContinuation("superseded"); diff --git a/src/node/services/taskService.test.ts b/src/node/services/taskService.test.ts index ba4268d72b..649f73715e 100644 --- a/src/node/services/taskService.test.ts +++ b/src/node/services/taskService.test.ts @@ -24856,12 +24856,25 @@ describe("TaskService", () => { deferredMessageIds: ["msg_deferred_cut"], }); - queuedContinuation = true; - await taskService.settleVoidedWorkspaceTurnContinuation( - "childworkspace", - correlation, - "retracted" - ); + // The continuation is queued while the void is reading the handle store: the check must + // run after that read, not once up front. + const store = (taskService as unknown as { taskHandleStore: TaskHandleStore }).taskHandleStore; + const getWorkspaceTurn = store.getWorkspaceTurn.bind(store); + const readSpy = spyOn(store, "getWorkspaceTurn").mockImplementationOnce(async (...args) => { + const record = await getWorkspaceTurn(...args); + queuedContinuation = true; + return record; + }); + try { + await taskService.settleVoidedWorkspaceTurnContinuation( + "childworkspace", + correlation, + "retracted" + ); + } finally { + readSpy.mockRestore(); + } + expect(queuedContinuation).toBe(true); expect(await workspaceTurnSnapshot(taskService, parentId)).toMatchObject({ status: "running", deferredMessageIds: ["msg_deferred_cut"], diff --git a/src/node/services/workspaceTurnManager.ts b/src/node/services/workspaceTurnManager.ts index 6f26702e19..ae2bbe559c 100644 --- a/src/node/services/workspaceTurnManager.ts +++ b/src/node/services/workspaceTurnManager.ts @@ -4705,15 +4705,14 @@ export class WorkspaceTurnManager { * queued after the wake cut (or already streaming) also deferred the stream-end and will * settle the record with its own stream-end. Settling here would interrupt a turn that is * about to continue — and a disposable turn would delete its workspace under that stream. + * That check runs after the record read (`unlessTurnContinues`): a continuation queued + * while the handle store was being read must be seen too. */ async settleVoidedWorkspaceTurnContinuation( workspaceId: string, muxMetadata: WorkspaceTurnMuxMetadata, reason: WorkspaceTurnContinuationVoidReason ): Promise { - if (this.workspaceService.hasPendingWorkspaceTurnContinuation(workspaceId, muxMetadata)) { - return; - } await this.settleWorkspaceTurnContinuationFailure( workspaceId, muxMetadata, @@ -4721,7 +4720,7 @@ export class WorkspaceTurnManager { reason === "retracted" ? WORKSPACE_TURN_YIELDED_TO_RETRACTED_WAKE_ERROR : WORKSPACE_TURN_SUPERSEDED_BY_NEW_INPUT_ERROR, - { deferredOnly: reason !== "abandoned", unlessCorrelatedStreamActive: true } + { deferredOnly: reason !== "abandoned", unlessTurnContinues: true } ); } @@ -4732,20 +4731,27 @@ export class WorkspaceTurnManager { muxMetadata: WorkspaceTurnMuxMetadata, status: "interrupted" | "error", error: string, - options?: { deferredOnly: boolean; unlessCorrelatedStreamActive: boolean } + options?: { deferredOnly: boolean; unlessTurnContinues: boolean } ): Promise { const record = await this.taskHandleStore.getWorkspaceTurn( muxMetadata.ownerWorkspaceId, muxMetadata.taskHandleId ); + // Both continuation reads are synchronous and sit after the last await before the + // settlement write, so nothing queued or started during the record read is missed. if ( record?.workspaceId !== workspaceId || record?.turnId !== muxMetadata.turnId || !isActiveWorkspaceTurnTaskStatus(record?.status) || (options?.deferredOnly === true && (record.deferredMessageIds?.length ?? 0) === 0) || - (options?.unlessCorrelatedStreamActive === true && - // A correlated stream whose stream-end the record has not deferred settles the turn. - this.hasCorrelatedActiveStream(workspaceId, muxMetadata, record.deferredMessageIds ?? [])) + (options?.unlessTurnContinues === true && + (this.workspaceService.hasPendingWorkspaceTurnContinuation(workspaceId, muxMetadata) || + // A correlated stream whose stream-end the record has not deferred settles the turn. + this.hasCorrelatedActiveStream( + workspaceId, + muxMetadata, + record.deferredMessageIds ?? [] + ))) ) { return; } From 501cce4ad18be07339cdd0b4aa3768992635247e Mon Sep 17 00:00:00 2001 From: Ethan Dickson Date: Fri, 4 Sep 2026 11:13:27 +0000 Subject: [PATCH 15/26] Never await the void hook from compaction completion; unref the void retry timer; let a retried void re-enter the terminal settlement branch --- ...gentSession.continueMessageAgentId.test.ts | 27 ++++----- src/node/services/agentSession.ts | 44 ++++++++++----- src/node/services/taskService.test.ts | 55 +++++++++++++++++++ src/node/services/workspaceTurnManager.ts | 19 ++++++- 4 files changed, 110 insertions(+), 35 deletions(-) diff --git a/src/node/services/agentSession.continueMessageAgentId.test.ts b/src/node/services/agentSession.continueMessageAgentId.test.ts index dedafe095e..48ed93c2d5 100644 --- a/src/node/services/agentSession.continueMessageAgentId.test.ts +++ b/src/node/services/agentSession.continueMessageAgentId.test.ts @@ -394,27 +394,20 @@ describe("AgentSession continue-message agentId fallback", () => { (session as unknown as { hasExternalSendPreflight?: () => boolean }).hasExternalSendPreflight = () => true; - // Settlement runs BEFORE the follow-up is erased: a failure keeps the durable record (the - // only carrier of the correlation) so the next attempt retries it. - let dispatchError: unknown; - try { - await internals.dispatchPendingFollowUp(); - } catch (error) { - dispatchError = error; - } - expect(dispatchError).toBeInstanceOf(Error); - expect((dispatchError as Error).message).toContain("task handle store unavailable"); + // The owner-side settlement is never awaited here: this path runs while the owner's + // stream-end listener holds the workspace event lock (waiting on the compaction decision) + // and the settlement needs that lock. The discard completes; a failed settlement is parked + // and retried, not lost. + expect(await internals.dispatchPendingFollowUp()).toBe(false); expect(internals.sendMessage).not.toHaveBeenCalled(); expect(abandoned).toHaveBeenCalledTimes(1); - const retained = await historyService.getLastMessages("ws", 1); - expect(retained.success && retained.data[0]?.metadata?.muxMetadata).toMatchObject({ - type: "compaction-summary", - pendingFollowUp: { workspaceTurnMetadata }, - }); + expect(abandoned).toHaveBeenLastCalledWith(workspaceTurnMetadata, "abandoned"); + await Promise.resolve(); + await Promise.resolve(); settlementError = undefined; - expect(await internals.dispatchPendingFollowUp()).toBe(false); - expect(internals.sendMessage).not.toHaveBeenCalled(); + session.setBashMonitorWakeOutstanding(true); + session.setBashMonitorWakeOutstanding(false); expect(abandoned).toHaveBeenCalledTimes(2); expect(abandoned).toHaveBeenLastCalledWith(workspaceTurnMetadata, "abandoned"); const lastMessages = await historyService.getLastMessages("ws", 1); diff --git a/src/node/services/agentSession.ts b/src/node/services/agentSession.ts index 70a3273785..4214614215 100644 --- a/src/node/services/agentSession.ts +++ b/src/node/services/agentSession.ts @@ -1099,6 +1099,13 @@ export class AgentSession { beginShutdown(): void { this.shuttingDown = true; this.retryManager.cancel(); + this.clearUnsettledWakeVoidRetryTimer(); + } + + private clearUnsettledWakeVoidRetryTimer(): void { + if (this.unsettledWakeVoidRetryTimer == null) return; + clearTimeout(this.unsettledWakeVoidRetryTimer); + this.unsettledWakeVoidRetryTimer = undefined; } dispose(): void { @@ -1124,10 +1131,7 @@ export class AgentSession { this.wakeTurnInFlight = false; this.wakeContinuationDebt = undefined; this.unsettledWakeVoids = []; - if (this.unsettledWakeVoidRetryTimer != null) { - clearTimeout(this.unsettledWakeVoidRetryTimer); - this.unsettledWakeVoidRetryTimer = undefined; - } + this.clearUnsettledWakeVoidRetryTimer(); // Ensure any callers blocked on waitForIdle() can continue during teardown. this.setTurnPhase(TurnPhase.IDLE); @@ -6960,12 +6964,16 @@ export class AgentSession { reason: voided.reason, error: getErrorMessage(error), }); - if (this.disposed) return; + if (this.disposed || this.shuttingDown) return; this.unsettledWakeVoids.push(voided); - this.unsettledWakeVoidRetryTimer ??= setTimeout(() => { - this.unsettledWakeVoidRetryTimer = undefined; - this.retryUnsettledWakeVoids(); - }, UNSETTLED_WAKE_VOID_RETRY_DELAY_MS); + if (this.unsettledWakeVoidRetryTimer == null) { + this.unsettledWakeVoidRetryTimer = setTimeout(() => { + this.unsettledWakeVoidRetryTimer = undefined; + this.retryUnsettledWakeVoids(); + }, UNSETTLED_WAKE_VOID_RETRY_DELAY_MS); + // Recovery must not keep a graceful process exit alive (beginShutdown also cancels). + this.unsettledWakeVoidRetryTimer.unref?.(); + } } ); } @@ -7853,11 +7861,14 @@ export class AgentSession { } // Every discard path funnels here, so this is the one place that knows the delegated - // turn's continuation is gone for good. Settle BEFORE erasing the follow-up: the durable - // record is the only carrier of the correlation, so a settlement failure must leave it in - // place for the next dispatch attempt (or startup recovery) to retry — settlement is - // idempotent, a lost record is not recoverable. A wake follow-up also carried the - // continuation debt of the stream it cut; that debt is settled by this same void. + // turn's continuation is gone for good. The void is a synchronous state transition here + // and the owner-side settlement runs as a tracked, retried promise (see + // settleVoidedWakeContinuation) — it is never awaited: this path runs while the + // TaskService stream-end listener holds the workspace event lock waiting for the + // compaction completion decision, and the owner settles under that same lock. A wake + // follow-up also carried the continuation debt of the stream it cut; the same void clears + // it. Whether the erase below succeeds does not affect the settlement: the void carries + // the correlation itself. const workspaceTurnMetadata = muxMeta.pendingFollowUp.workspaceTurnMetadata; if (workspaceTurnMetadata != null) { if ( @@ -7868,7 +7879,10 @@ export class AgentSession { ) { this.wakeContinuationDebt = undefined; } - await this.onWorkspaceTurnContinuationVoided?.(workspaceTurnMetadata, "abandoned"); + this.settleVoidedWakeContinuation({ + correlation: workspaceTurnMetadata, + reason: "abandoned", + }); } const { pendingFollowUp: _pendingFollowUp, ...muxMetadataWithoutFollowUp } = muxMeta; diff --git a/src/node/services/taskService.test.ts b/src/node/services/taskService.test.ts index 649f73715e..424e1f46ba 100644 --- a/src/node/services/taskService.test.ts +++ b/src/node/services/taskService.test.ts @@ -24743,6 +24743,61 @@ describe("TaskService", () => { }); }); + test("a void retried after a partially persisted settlement still wakes the waiter", async () => { + // The first attempt persists the terminal handle and then fails (the execution-state + // mirror write rejects) before resolving the waiter. The session retries the void; the + // record is now terminal, so the retry must re-enter settlement's idempotent terminal + // branch rather than treat "already settled" as nothing left to do. + const { parentId, taskService } = await startWorkspaceTurnForTest(); + const manager = workspaceTurnManagerFor(taskService) as unknown as { + updateAgentTaskExecutionState: (...args: unknown[]) => Promise; + }; + const mirrorSpy = spyOn(manager, "updateAgentTaskExecutionState").mockImplementationOnce(() => + Promise.reject(new Error("execution state mirror unavailable")) + ); + let settled = false; + const waited = workspaceTurnManagerFor(taskService) + .waitForWorkspaceTurn("wst_handle", { requestingWorkspaceId: parentId, timeoutMs: 5_000 }) + .then( + () => null, + (error: unknown) => error + ) + .finally(() => { + settled = true; + }); + try { + const firstAttempt = await taskService + .settleVoidedWorkspaceTurnContinuation( + "childworkspace", + workspaceTurnMuxMetadata(parentId), + "abandoned" + ) + .then( + () => null, + (error: unknown) => error + ); + expect(firstAttempt).toBeInstanceOf(Error); + expect((firstAttempt as Error).message).toContain("execution state mirror unavailable"); + expect(mirrorSpy).toHaveBeenCalledTimes(1); + expect(await workspaceTurnSnapshot(taskService, parentId)).toMatchObject({ + status: "interrupted", + }); + await Promise.resolve(); + expect(settled).toBe(false); + + await taskService.settleVoidedWorkspaceTurnContinuation( + "childworkspace", + workspaceTurnMuxMetadata(parentId), + "abandoned" + ); + const error = await waited; + expect(error).toBeInstanceOf(Error); + expect((error as Error).message).toContain("superseded by new input"); + } finally { + mirrorSpy.mockRestore(); + } + }); + test("settleVoidedWorkspaceTurnContinuation ignores a stale correlation", async () => { const { parentId, taskService } = await startWorkspaceTurnForTest(); diff --git a/src/node/services/workspaceTurnManager.ts b/src/node/services/workspaceTurnManager.ts index ae2bbe559c..b8eefff87d 100644 --- a/src/node/services/workspaceTurnManager.ts +++ b/src/node/services/workspaceTurnManager.ts @@ -4737,12 +4737,25 @@ export class WorkspaceTurnManager { muxMetadata.ownerWorkspaceId, muxMetadata.taskHandleId ); + if (record?.workspaceId !== workspaceId || record.turnId !== muxMetadata.turnId) { + return; + } + if (this.isTerminalWorkspaceTurnStatus(record.status)) { + // Already settled — possibly by an earlier attempt of this very settlement that + // persisted the terminal handle and then failed before resolving its waiters (the + // void is retried on failure). settleWorkspaceTurn's terminal branch is idempotent and + // resolves whatever is still waiting on the persisted outcome. + await this.settleWorkspaceTurn({ + record, + next: record, + waiterSettlement: { status: "error", error: new Error(error) }, + }); + return; + } // Both continuation reads are synchronous and sit after the last await before the // settlement write, so nothing queued or started during the record read is missed. if ( - record?.workspaceId !== workspaceId || - record?.turnId !== muxMetadata.turnId || - !isActiveWorkspaceTurnTaskStatus(record?.status) || + !isActiveWorkspaceTurnTaskStatus(record.status) || (options?.deferredOnly === true && (record.deferredMessageIds?.length ?? 0) === 0) || (options?.unlessTurnContinues === true && (this.workspaceService.hasPendingWorkspaceTurnContinuation(workspaceId, muxMetadata) || From 63bffd92a3ed43fe50cc43206bb204857772567b Mon Sep 17 00:00:00 2001 From: Ethan Dickson Date: Fri, 4 Sep 2026 11:29:58 +0000 Subject: [PATCH 16/26] Round 13: revalidate a void at the settlement commit point (abandonIf); recover a committed lease from the durable wake row after restart; carry the assumed correlation through compaction retries; remember recent stream starts so a fast continuation still defers its predecessor's stream-end --- .../agentSession.queueDispatch.test.ts | 44 ++++++++++++ src/node/services/agentSession.ts | 67 ++++++++++++++++++- .../bashMonitorWakeReconciler.test.ts | 63 +++++++++++++++++ .../services/bashMonitorWakeReconciler.ts | 54 +++++++++++++-- src/node/services/taskService.test.ts | 61 +++++++++++++++-- .../services/taskWorkspaceSeam.testUtils.ts | 1 + src/node/services/taskWorkspaceSeam.ts | 10 +++ src/node/services/workspaceService.ts | 42 ++++++++++++ src/node/services/workspaceTurnManager.ts | 57 ++++++++++++---- 9 files changed, 374 insertions(+), 25 deletions(-) diff --git a/src/node/services/agentSession.queueDispatch.test.ts b/src/node/services/agentSession.queueDispatch.test.ts index 4eb1b85956..f7fa2c94db 100644 --- a/src/node/services/agentSession.queueDispatch.test.ts +++ b/src/node/services/agentSession.queueDispatch.test.ts @@ -816,6 +816,50 @@ describe("AgentSession queued message tool-call dispatch", () => { } }); + test("the stream-start ledger remembers a correlated continuation after it ended", async () => { + // A stream-end handler can run after the turn's next stream already started and ended; + // it asks the ledger whether the turn continued after the stream it handles. + const workspaceId = "queue-dispatch-stream-ledger"; + const { session, cleanup, aiEmitter } = await createAgentSessionHarness({ workspaceId }); + try { + const startedAs = (messageId: string, correlation: typeof DELEGATED_TURN | undefined) => { + setActiveStreamCorrelation(session, correlation); + aiEmitter.emit("stream-start", { ...streamStartEvent(workspaceId), messageId }); + }; + startedAs("assistant-delegated-1", DELEGATED_TURN); + expect( + session.hasCorrelatedStreamStartedAfter(DELEGATED_TURN, ["assistant-delegated-1"]) + ).toBe(false); + + startedAs("assistant-manual", undefined); + expect( + session.hasCorrelatedStreamStartedAfter(DELEGATED_TURN, ["assistant-delegated-1"]) + ).toBe(false); + + startedAs("assistant-delegated-2", DELEGATED_TURN); + expect( + session.hasCorrelatedStreamStartedAfter(DELEGATED_TURN, ["assistant-delegated-1"]) + ).toBe(true); + // Relative to the continuation itself nothing followed. + expect( + session.hasCorrelatedStreamStartedAfter(DELEGATED_TURN, ["assistant-delegated-2"]) + ).toBe(false); + // A different turn's correlation never matches. + expect( + session.hasCorrelatedStreamStartedAfter({ ...DELEGATED_TURN, turnId: "turn-2" }, [ + "assistant-delegated-1", + ]) + ).toBe(false); + // A stream the ledger no longer holds predates everything remembered. + expect(session.hasCorrelatedStreamStartedAfter(DELEGATED_TURN, ["assistant-evicted"])).toBe( + true + ); + } finally { + session.dispose(); + await cleanup(); + } + }); + test("the level lowering with no wake turn in flight voids the debt as retracted", async () => { const workspaceId = "queue-dispatch-wake-retracted"; const voided: Array<[MuxMessageMetadata, string]> = []; diff --git a/src/node/services/agentSession.ts b/src/node/services/agentSession.ts index 4214614215..ca876bf8d4 100644 --- a/src/node/services/agentSession.ts +++ b/src/node/services/agentSession.ts @@ -458,6 +458,23 @@ function carriesBashMonitorWake(muxMetadata: unknown): boolean { return followUpMetadata?.type === "bash-monitor-wake"; } +/** + * The delegated-turn correlation a send / retry will stream under: its own metadata, or — + * for an on-send compaction request — the correlation stamped on the follow-up it carries + * (the follow-up stream inherits it from the summary). + */ +function getCarriedWorkspaceTurnCorrelation( + muxMetadata: unknown +): WorkspaceTurnMuxMetadata | undefined { + const meta = muxMetadata as MuxMessageMetadata | undefined; + if (!isCompactionRequestMetadata(meta)) return getWorkspaceTurnMuxMetadata(meta); + const followUp = meta.parsed.followUpContent; + return ( + followUp?.workspaceTurnMetadata ?? + getWorkspaceTurnMuxMetadata(followUp?.muxMetadata ?? meta.parsed.continueMessage?.muxMetadata) + ); +} + const AUTO_RETRY_PREFERENCE_FILE = "auto-retry-preference.json"; /** @@ -535,6 +552,8 @@ const SESSION_SHUTDOWN_SEND_BLOCKED_MESSAGE = "Xum is shutting down; the message const STARTUP_AUTO_RETRY_HISTORY_FAILURE_BASE_DELAY_MS = 1_000; /** Retry cadence for a voided wake continuation whose owner-side settlement failed. */ const UNSETTLED_WAKE_VOID_RETRY_DELAY_MS = 5_000; +/** Stream starts remembered for hasCorrelatedStreamStartedAfter; older ones are evicted. */ +const RECENT_STREAM_STARTS_LIMIT = 16; const STARTUP_AUTO_RETRY_HISTORY_FAILURE_MAX_DELAY_MS = 30_000; const MAX_STARTUP_RECOVERY_DEFERRED_ATTEMPTS = 4; @@ -964,6 +983,17 @@ export class AgentSession { */ private wakeContinuationDebt?: { correlation?: WorkspaceTurnMuxMetadata; assumed?: true }; private wakeTurnInFlight = false; + /** + * Recent stream starts, oldest first, with the delegated-turn correlation each streamed + * under. A stream-end handler runs behind the workspace event lock and may be late: a + * same-turn continuation can have started — and finished — after the stream it handles, + * leaving nothing pending, in flight, or owed to probe. The ledger answers "did the turn + * continue after this stream?" regardless (hasCorrelatedStreamStartedAfter). + */ + private recentStreamStarts: Array<{ + messageId: string; + correlation?: WorkspaceTurnMuxMetadata; + }> = []; private unsettledWakeVoids: Array<{ correlation: WorkspaceTurnMuxMetadata; reason: WorkspaceTurnContinuationVoidReason; @@ -5949,6 +5979,13 @@ export class AgentSession { this.dispatchingQueuedEntry = false; this.dispatchingQueuedEntryMuxMetadata = undefined; this.preparingWorkspaceTurnMetadata = undefined; + this.recentStreamStarts.push({ + messageId: payload.messageId, + correlation: this.activeStreamContext?.workspaceTurnMetadata, + }); + if (this.recentStreamStarts.length > RECENT_STREAM_STARTS_LIMIT) { + this.recentStreamStarts.shift(); + } // A stream that shows the wake redeems the continuation debt (see // wakeContinuationDebt). Only the wake row's own stream qualifies: an on-send // compaction stream's request is the compaction row, and the wake follows it. @@ -6887,7 +6924,8 @@ export class AgentSession { assumed && this.wakeContinuationDebt != null && !hasSameWorkspaceTurnCorrelation( - getWorkspaceTurnMuxMetadata(retryMetadata), + // A retry of an on-send compaction still starts the correlated follow-up behind it. + getCarriedWorkspaceTurnCorrelation(retryMetadata), this.wakeContinuationDebt.correlation ) ) { @@ -7061,6 +7099,33 @@ export class AgentSession { /** * Whether a queued or dispatching entry continues the exact workspace-turn correlation. */ + /** + * Whether a stream carrying `correlation` started after every stream in `messageIds` + * (running or already ended). A message id the ledger no longer holds was evicted by + * later starts (or predates this session instance), so every remembered start is after it. + */ + hasCorrelatedStreamStartedAfter( + correlation: { taskHandleId: string; ownerWorkspaceId: string; turnId: string }, + messageIds: readonly string[] + ): boolean { + let after = -1; + for (const messageId of messageIds) { + after = Math.max( + after, + this.recentStreamStarts.findIndex((entry) => entry.messageId === messageId) + ); + } + return this.recentStreamStarts + .slice(after + 1) + .some( + (entry) => + entry.correlation != null && + entry.correlation.taskHandleId === correlation.taskHandleId && + entry.correlation.ownerWorkspaceId === correlation.ownerWorkspaceId && + entry.correlation.turnId === correlation.turnId + ); + } + hasPendingWorkspaceTurnContinuation( metadata: Extract ): boolean { diff --git a/src/node/services/bashMonitorWakeReconciler.test.ts b/src/node/services/bashMonitorWakeReconciler.test.ts index 0757beccbe..e2768282bd 100644 --- a/src/node/services/bashMonitorWakeReconciler.test.ts +++ b/src/node/services/bashMonitorWakeReconciler.test.ts @@ -319,6 +319,69 @@ describe("BashMonitorWakeReconciler", () => { expect(dispatches[1].prompt).toContain("READY again"); }); + test("after a restart, the durable wake row acknowledges a commit whose watermark never landed", async () => { + // The owner's row landed, then the process died before the watermark write. The in-memory + // committed lease is gone; without the row as durable evidence the fresh reconciler would + // derive the same signal again and dispatch a duplicate next to the row's own recovery. + live = [liveSnapshot()]; + await reconciler.reconcile(OWNER); + expect(dispatches).toHaveLength(1); + const deliveredRecords = dispatches[0].muxMetadata.records; + + const restart = ( + readDeliveredWakeRecords: (() => Promise) | undefined + ) => { + const restarted: BashMonitorWakeDispatch[] = []; + const instance = new BashMonitorWakeReconciler({ + // A fresh sessions dir: no watermark was ever written. + sessionsDir: path.join(root, readDeliveredWakeRecords == null ? "control" : "recovered"), + processManager: { + pullMonitorWakeSignals: () => live, + getMonitorWakeDeliveryState: () => Promise.resolve(deliveryState), + acknowledgeMonitorWake: (processId, _generation, matchedThroughOffset) => { + acknowledged.push({ + processId, + ...(matchedThroughOffset != null ? { matchedThroughOffset } : {}), + }); + }, + dropRetiredMonitor: () => undefined, + }, + registry: { + listAll: () => Promise.resolve(rows), + remove: () => undefined, + recordTerminal: () => undefined, + }, + onWake: (dispatch) => { + restarted.push(dispatch); + return "in-flight"; + }, + ...(readDeliveredWakeRecords != null ? { readDeliveredWakeRecords } : {}), + }); + return { instance, restarted }; + }; + + // Control: the same restart without the row re-dispatches the delivered signal. + const control = restart(undefined); + await control.instance.reconcile(OWNER); + expect(control.restarted).toHaveLength(1); + + const recovered = restart(() => Promise.resolve(deliveredRecords)); + acknowledged = []; + await recovered.instance.reconcile(OWNER); + expect(recovered.restarted).toHaveLength(0); + expect(acknowledged).toEqual([{ processId: "proc", matchedThroughOffset: 12 }]); + expect(await recovered.instance.hasOutstandingWake(OWNER)).toBe(false); + + // The watermark is durable now: a later read does not consult the row again, and a + // newer match still wakes. + live = [ + liveSnapshot({ match: { throughOffset: 24, lines: ["READY again"], totalMatches: 2 } }), + ]; + await recovered.instance.reconcile(OWNER); + expect(recovered.restarted).toHaveLength(1); + expect(recovered.restarted[0].prompt).toContain("READY again"); + }); + test("disposal lowers the published level and retires the in-flight wake", async () => { live = [liveSnapshot()]; await reconciler.reconcile(OWNER); diff --git a/src/node/services/bashMonitorWakeReconciler.ts b/src/node/services/bashMonitorWakeReconciler.ts index d717fca0e4..eca93e9f6f 100644 --- a/src/node/services/bashMonitorWakeReconciler.ts +++ b/src/node/services/bashMonitorWakeReconciler.ts @@ -3,7 +3,7 @@ import { randomUUID } from "node:crypto"; import * as fsPromises from "node:fs/promises"; import * as path from "node:path"; -import type { MuxMessageMetadata } from "@/common/types/message"; +import type { BashMonitorWakeDisplayRecord, MuxMessageMetadata } from "@/common/types/message"; import assert from "@/common/utils/assert"; import { getErrorMessage } from "@/common/utils/errors"; import { BASH_MONITOR_WAKE_HEADINGS } from "@/common/utils/machineTurnPrompts"; @@ -362,6 +362,19 @@ function buildPrompt(signals: readonly DerivedSignal[]): string { return `${header}\n\n${sections.join("\n\n---\n\n")}\n\n${closingParts.join(" ")}`; } +/** + * Version of the signal as it appears in the durable wake row (`records[].wakeUpdatedAt`). + * Together with processId it identifies a delivered signal after a restart (see + * readDeliveredWakeRecords). + */ +function wakeUpdatedAtOf(signal: DerivedSignal): string { + return ( + signal.lost?.failedAt ?? + signal.terminal?.settledAt ?? + (signal.matchOffset != null ? signal.createdAt + ":" + signal.matchOffset : signal.createdAt) + ); +} + function buildMetadata( signals: readonly DerivedSignal[] ): Extract { @@ -369,12 +382,7 @@ function buildMetadata( type: "bash-monitor-wake", records: signals.map((signal) => ({ processId: signal.processId, - wakeUpdatedAt: - signal.lost?.failedAt ?? - signal.terminal?.settledAt ?? - (signal.matchOffset != null - ? signal.createdAt + ":" + signal.matchOffset - : signal.createdAt), + wakeUpdatedAt: wakeUpdatedAtOf(signal), kind: signal.kind === "monitor-lost" ? "monitor-lost" : "match", displayName: signal.displayName ?? signal.processId, filter: signal.filter, @@ -398,6 +406,8 @@ export class BashMonitorWakeReconciler { private readonly locks = new MutexMap(); private readonly states = new Map(); private readonly legacyCleanupAttempted = new Set(); + /** Owners whose durable wake row has been reconciled against derived signals (once per process). */ + private readonly deliveryRecovered = new Set(); private readonly retryTimers = new Map(); private readonly retryAttempts = new Map(); private readonly defunctWorkspaces = new Set(); @@ -416,6 +426,16 @@ export class BashMonitorWakeReconciler { * long-poll return, backgrounding foreground waits) from the level itself. */ onOutstandingChanged?(ownerWorkspaceId: string, outstanding: boolean): void; + /** + * Records of the owner's most recent durable wake row, if any. The wake row is the + * durable acknowledgment: a commit whose watermark write failed and was then lost to a + * restart (the in-memory committed lease dies with the process) would otherwise + * re-derive and re-dispatch the very signals that row already delivers. Consulted once + * per owner, the first time signals derive outstanding in this process. + */ + readDeliveredWakeRecords?( + ownerWorkspaceId: string + ): Promise; } ) {} @@ -774,6 +794,26 @@ export class BashMonitorWakeReconciler { else if (derived.outstanding) signals.push(derived.signal); else if (derived.consume) autoConsumed.push(derived.signal); } + if (signals.length > 0 && !this.deliveryRecovered.has(ownerWorkspaceId)) { + // Signals the durable wake row already delivers are consumed, not re-dispatched. The + // watermark advance is written here (not left to the caller): level reads do not + // persist autoConsumed, and a recovery that only held in memory would be lost again. + const delivered = await this.args.readDeliveredWakeRecords?.(ownerWorkspaceId); + const deliveredKeys = new Set( + (delivered ?? []).map((record) => record.processId + "\u0000" + record.wakeUpdatedAt) + ); + const recovered = signals.filter((signal) => + deliveredKeys.has(signal.processId + "\u0000" + wakeUpdatedAtOf(signal)) + ); + if (recovered.length > 0) { + await this.advanceWatermarks(ownerWorkspaceId, watermarks, recovered); + for (const signal of recovered) { + signals.splice(signals.indexOf(signal), 1); + autoConsumed.push(signal); + } + } + this.deliveryRecovered.add(ownerWorkspaceId); + } signals.sort( (a, b) => a.createdAt.localeCompare(b.createdAt) || a.processId.localeCompare(b.processId) ); diff --git a/src/node/services/taskService.test.ts b/src/node/services/taskService.test.ts index 424e1f46ba..8b9dfd8f55 100644 --- a/src/node/services/taskService.test.ts +++ b/src/node/services/taskService.test.ts @@ -348,6 +348,7 @@ describe("TaskService", () => { hasQueuedMessages?: ReturnType; hasPendingQueuedOrPreparingTurn?: ReturnType; hasBashMonitorWakeContinuation?: ReturnType; + hasCorrelatedStreamStartedAfter?: ReturnType; hasPendingWorkspaceTurnContinuation?: ReturnType; getQueueCutCutter?: ReturnType; hasPendingAutoRetry?: ReturnType; @@ -24459,6 +24460,54 @@ describe("TaskService", () => { expect(settled).toMatchObject({ messageId: "msg_manual_cut" }); }); + test("a same-turn continuation that already ended still defers the predecessor's stream-end", async () => { + // The continuation's stream started and finished before this (older) stream-end handler + // reached the workspace event lock: nothing is pending, in flight, or owed any more, and + // the debt it discharged is gone. Its own stream-end is queued right behind this one and + // settles the turn; settling here would pre-empt it (and delete a disposable workspace + // under the continuation's work). The session's stream-start ledger is the evidence. + const hasCorrelatedStreamStartedAfter = mock( + ( + workspaceId: string, + correlation: { taskHandleId: string; turnId: string }, + messageIds: readonly string[] + ) => + workspaceId === "childworkspace" && + correlation.taskHandleId === "wst_handle" && + correlation.turnId === "turn" && + messageIds.includes("msg_fast_continuation_cut") + ); + const { parentId, taskService, workspaceMocks } = await startWorkspaceTurnForTest({ + hasBashMonitorWakeContinuation: mock(() => false), + hasCorrelatedStreamStartedAfter, + }); + workspaceMocks.getQueueCutCutter.mockImplementation(() => ({ + stage: "bash-monitor-wake" as const, + })); + const internal = taskService as unknown as { + handleStreamEnd: (event: StreamEndEvent) => Promise; + }; + + await internal.handleStreamEnd({ + type: "stream-end", + workspaceId: "childworkspace", + messageId: "msg_fast_continuation_cut", + metadata: { + model: "anthropic:claude-opus-4-6", + agentId: "exec", + finishReason: "tool-calls", + muxMetadata: workspaceTurnMuxMetadata(parentId), + }, + parts: [{ type: "text", text: "Waiting on the build" }], + }); + + expect(hasCorrelatedStreamStartedAfter).toHaveBeenCalled(); + expect(await workspaceTurnSnapshot(taskService, parentId)).toMatchObject({ + status: "running", + deferredMessageIds: ["msg_fast_continuation_cut"], + }); + }); + test("a wake retracted after the cut settles the handle as a wake cut instead of deferring", async () => { // The stream yielded to the wake level, then the operator canceled the monitor before // this stream-end was processed: the session voided its continuation debt (the void's @@ -24911,13 +24960,16 @@ describe("TaskService", () => { deferredMessageIds: ["msg_deferred_cut"], }); - // The continuation is queued while the void is reading the handle store: the check must - // run after that read, not once up front. + // The continuation becomes visible only while the void is already inside the settlement + // (its handle reread under the settlement lock): the check must run at that commit + // point, not on a snapshot taken before the awaits. const store = (taskService as unknown as { taskHandleStore: TaskHandleStore }).taskHandleStore; const getWorkspaceTurn = store.getWorkspaceTurn.bind(store); - const readSpy = spyOn(store, "getWorkspaceTurn").mockImplementationOnce(async (...args) => { + let reads = 0; + const readSpy = spyOn(store, "getWorkspaceTurn").mockImplementation(async (...args) => { const record = await getWorkspaceTurn(...args); - queuedContinuation = true; + // First read: the void's own; second: settleWorkspaceTurn's reread under its lock. + if (++reads === 2) queuedContinuation = true; return record; }); try { @@ -24929,6 +24981,7 @@ describe("TaskService", () => { } finally { readSpy.mockRestore(); } + expect(reads).toBeGreaterThanOrEqual(2); expect(queuedContinuation).toBe(true); expect(await workspaceTurnSnapshot(taskService, parentId)).toMatchObject({ status: "running", diff --git a/src/node/services/taskWorkspaceSeam.testUtils.ts b/src/node/services/taskWorkspaceSeam.testUtils.ts index 054094a6a9..90744600c6 100644 --- a/src/node/services/taskWorkspaceSeam.testUtils.ts +++ b/src/node/services/taskWorkspaceSeam.testUtils.ts @@ -15,6 +15,7 @@ export function makeWorkspaceHostFake(overrides: Partial = {}): W hasPendingQueuedOrPreparingTurn: () => false, hasPendingAutoRetry: () => false, hasBashMonitorWakeContinuation: () => false, + hasCorrelatedStreamStartedAfter: () => false, isToolEndYieldRequested: () => false, hasPendingWorkspaceTurnContinuation: () => false, hasQueuedWorkspaceTurn: () => false, diff --git a/src/node/services/taskWorkspaceSeam.ts b/src/node/services/taskWorkspaceSeam.ts index 5d53f661aa..8bea018574 100644 --- a/src/node/services/taskWorkspaceSeam.ts +++ b/src/node/services/taskWorkspaceSeam.ts @@ -404,6 +404,16 @@ export interface TurnAdmissionHost { * AgentSession.hasBashMonitorWakeContinuation). Sync, no I/O. */ hasBashMonitorWakeContinuation(workspaceId: string): boolean; + /** + * A stream carrying `correlation` started after every stream in `messageIds` — whether it + * is still running or already ended (see AgentSession.hasCorrelatedStreamStartedAfter). + * Sync, no I/O. + */ + hasCorrelatedStreamStartedAfter( + workspaceId: string, + correlation: { taskHandleId: string; ownerWorkspaceId: string; turnId: string }, + messageIds: readonly string[] + ): boolean; /** Pending input (queued tool-end message or outstanding wake) wants a tool boundary. */ isToolEndYieldRequested(workspaceId: string): boolean; hasPendingWorkspaceTurnContinuation( diff --git a/src/node/services/workspaceService.ts b/src/node/services/workspaceService.ts index 0b02f689cb..e2ace011df 100644 --- a/src/node/services/workspaceService.ts +++ b/src/node/services/workspaceService.ts @@ -186,6 +186,7 @@ import { getCompactionFollowUpContent, parseWorkspaceTurnTaskCorrelation, pickPreservedSendOptions, + type BashMonitorWakeDisplayRecord, type CompactionFollowUpRequest, type MuxMessageMetadata, type MuxMessage, @@ -374,6 +375,12 @@ const ORPHAN_SESSION_DIR_GRACE_MS = 24 * 60 * 60 * 1000; // Upper bound on startup .code-workspace reconciliation (see initialize()). const STARTUP_CODE_WORKSPACE_SYNC_TIMEOUT_MS = 10_000; +/** + * How far back readLastBashMonitorWakeRecords looks for the last durable wake row. A commit + * lost to a restart leaves that row at the tail; anything older had a running process (and + * its in-memory acknowledgment retries) behind it. + */ +const LAST_BASH_MONITOR_WAKE_ROW_SCAN_DEPTH = 50; /** * Base name used when /new auto-generates a branch name. Numbered suffixes @@ -2390,6 +2397,8 @@ export class WorkspaceService extends EventEmitter implements WorkspaceHost { }, registry: this.bashMonitorRegistryStore, onWake: (dispatch) => this.dispatchBashMonitorWake(dispatch), + readDeliveredWakeRecords: (ownerWorkspaceId) => + this.readLastBashMonitorWakeRecords(ownerWorkspaceId), onOutstandingChanged: (ownerWorkspaceId, outstanding) => { // The level drives the same tool-boundary side effects a queued tool-end message // does: long-polling bash reads return early and foreground agent-task waits are @@ -12013,6 +12022,19 @@ export class WorkspaceService extends EventEmitter implements WorkspaceHost { ); } + /** See AgentSession.hasCorrelatedStreamStartedAfter. */ + hasCorrelatedStreamStartedAfter( + workspaceId: string, + correlation: { taskHandleId: string; ownerWorkspaceId: string; turnId: string }, + messageIds: readonly string[] + ): boolean { + return ( + this.sessions + .get(workspaceId.trim()) + ?.hasCorrelatedStreamStartedAfter(correlation, messageIds) === true + ); + } + /** See AgentSession.hasBashMonitorWakeContinuation. */ hasBashMonitorWakeContinuation(workspaceId: string): boolean { return this.sessions.get(workspaceId.trim())?.hasBashMonitorWakeContinuation() === true; @@ -12029,6 +12051,26 @@ export class WorkspaceService extends EventEmitter implements WorkspaceHost { /** * Whether a queued or dispatching entry continues the exact workspace-turn correlation. */ + /** + * Records of the most recent durable bash-monitor wake row (BashMonitorWakeReconciler + * readDeliveredWakeRecords). Only the history tail is scanned: the row this recovers was + * committed right before a restart, so it is at or near the end. + */ + private async readLastBashMonitorWakeRecords( + ownerWorkspaceId: string + ): Promise { + const tail = await this.historyService.getLastMessages( + ownerWorkspaceId, + LAST_BASH_MONITOR_WAKE_ROW_SCAN_DEPTH + ); + if (!tail.success) return undefined; + for (const message of tail.data.toReversed()) { + const muxMetadata = message.metadata?.muxMetadata; + if (muxMetadata?.type === "bash-monitor-wake") return muxMetadata.records; + } + return undefined; + } + hasPendingWorkspaceTurnContinuation( workspaceId: string, metadata: Extract diff --git a/src/node/services/workspaceTurnManager.ts b/src/node/services/workspaceTurnManager.ts index b8eefff87d..c83d4a8e03 100644 --- a/src/node/services/workspaceTurnManager.ts +++ b/src/node/services/workspaceTurnManager.ts @@ -2118,6 +2118,14 @@ export class WorkspaceTurnManager { * it would leak the disposable checkout with no owner left to clean it up. */ disposableOwnershipTransferred?: boolean; + /** + * Re-evaluated under the settlement lock, after the handle reread and before anything is + * written: when it returns true the settlement is abandoned. Callers whose reason to + * settle can be invalidated by concurrent admission (a continuation of the turn queued or + * started while this call awaited the lock / store) revalidate here, at the commit point, + * rather than trusting a snapshot taken before those awaits. + */ + abandonIf?: () => boolean; }): Promise { assert( params.next.handleId === params.record.handleId, @@ -2198,6 +2206,10 @@ export class WorkspaceTurnManager { return { pendingNotify: null, winningStatus: current.status }; } + if (params.abandonIf?.() === true) { + return null; + } + // Decide the terminal wake-up using persisted policy + the restart-safe dedupe marker. // A resettle corrects a previously reported outcome, so it re-arms the wake-up even if // the stale settlement was already notified/consumed. Owner-follow-up supersedes settle @@ -4354,13 +4366,31 @@ export class WorkspaceTurnManager { ) { return true; } - return this.hasCorrelatedActiveStream(event.workspaceId, correlation, [event.messageId]); + return this.hasCorrelatedStreamAfter(event.workspaceId, correlation, [event.messageId]); } /** * Whether a stream other than `excludeMessageIds` (the ending stream, or streams whose * stream-end this record already deferred) is currently streaming this exact correlation. */ + /** + * The turn continued after the given stream(s): a correlated stream started later — still + * running, or already ended with its own stream-end queued behind this handler on the + * workspace event lock. Settling here would pre-empt that stream-end (and, for a disposable + * turn, delete the workspace under its work). The session's start ledger is authoritative; + * the live stream check covers a session that is not in memory. + */ + private hasCorrelatedStreamAfter( + workspaceId: string, + correlation: { taskHandleId: string; ownerWorkspaceId: string; turnId: string }, + messageIds: readonly string[] + ): boolean { + return ( + this.workspaceService.hasCorrelatedStreamStartedAfter(workspaceId, correlation, messageIds) || + this.hasCorrelatedActiveStream(workspaceId, correlation, messageIds) + ); + } + private hasCorrelatedActiveStream( workspaceId: string, correlation: { taskHandleId: string; ownerWorkspaceId: string; turnId: string }, @@ -4705,8 +4735,9 @@ export class WorkspaceTurnManager { * queued after the wake cut (or already streaming) also deferred the stream-end and will * settle the record with its own stream-end. Settling here would interrupt a turn that is * about to continue — and a disposable turn would delete its workspace under that stream. - * That check runs after the record read (`unlessTurnContinues`): a continuation queued - * while the handle store was being read must be seen too. + * That check runs after the record read and again at the settlement's commit point + * (`unlessTurnContinues`): a continuation queued while the handle store or the settlement + * lock was being awaited must be seen too. */ async settleVoidedWorkspaceTurnContinuation( workspaceId: string, @@ -4752,19 +4783,18 @@ export class WorkspaceTurnManager { }); return; } - // Both continuation reads are synchronous and sit after the last await before the - // settlement write, so nothing queued or started during the record read is missed. + // The turn continues when a correlated continuation is pending, or a correlated stream + // whose stream-end the record has not deferred started (it settles the turn itself). + // Checked here and again at the commit point inside settleWorkspaceTurn (abandonIf): a + // correlated send is invisible during its preflight and can become queued while the + // settlement awaits the lock and the store. + const turnContinues = () => + this.workspaceService.hasPendingWorkspaceTurnContinuation(workspaceId, muxMetadata) || + this.hasCorrelatedStreamAfter(workspaceId, muxMetadata, record.deferredMessageIds ?? []); if ( !isActiveWorkspaceTurnTaskStatus(record.status) || (options?.deferredOnly === true && (record.deferredMessageIds?.length ?? 0) === 0) || - (options?.unlessTurnContinues === true && - (this.workspaceService.hasPendingWorkspaceTurnContinuation(workspaceId, muxMetadata) || - // A correlated stream whose stream-end the record has not deferred settles the turn. - this.hasCorrelatedActiveStream( - workspaceId, - muxMetadata, - record.deferredMessageIds ?? [] - ))) + (options?.unlessTurnContinues === true && turnContinues()) ) { return; } @@ -4780,6 +4810,7 @@ export class WorkspaceTurnManager { record, next, waiterSettlement: { status: "error", error: new Error(error) }, + ...(options?.unlessTurnContinues === true ? { abandonIf: turnContinues } : {}), }); } From 5294afb0105b3745994c07988a5b9e9499810fee Mon Sep 17 00:00:00 2001 From: Ethan Dickson Date: Fri, 4 Sep 2026 11:42:29 +0000 Subject: [PATCH 17/26] Round 14: roll back the on-send compaction row when a send is refused at the admission gates; publish the wake level to transient startup-recovery sessions; resume disposable cleanup from the terminal settlement branch --- .../agentSession.autoCompaction.test.ts | 47 ++++++++++++++ src/node/services/agentSession.ts | 11 +++- src/node/services/taskService.test.ts | 62 ++++++++++++++++++ src/node/services/workspaceService.test.ts | 31 +++++++++ src/node/services/workspaceService.ts | 65 ++++++++++++------- src/node/services/workspaceTurnManager.ts | 11 ++++ 6 files changed, 202 insertions(+), 25 deletions(-) diff --git a/src/node/services/agentSession.autoCompaction.test.ts b/src/node/services/agentSession.autoCompaction.test.ts index 6648216961..d0022175e1 100644 --- a/src/node/services/agentSession.autoCompaction.test.ts +++ b/src/node/services/agentSession.autoCompaction.test.ts @@ -422,6 +422,53 @@ describe("AgentSession on-send auto-compaction snapshot deferral", () => { session.dispose(); }); + test("a send refused after its on-send compaction row landed rolls that row back", async () => { + // The compaction row is the one durable write that precedes the admission gates. A send + // whose admission went stale in between (a bash-monitor wake whose monitor was cancelled, + // a peer send racing a Stop) is refused without a stream — leaving the row would let + // startup recovery resume a compaction whose follow-up nobody accepted. + const workspaceId = "ws-auto-compaction-stale-admission-rollback"; + const streamMessage = mock(() => Promise.resolve(Ok(createStartedTurnHandle()))); + const { session, historyService } = await createSessionHarness({ + workspaceId, + streamMessage: streamMessage as unknown as AIService["streamMessage"], + }); + + (session as unknown as { compactionMonitor: CompactionMonitor }).compactionMonitor = { + checkBeforeSend: mock(() => ({ + shouldShowWarning: true, + shouldForceCompact: false, + usagePercentage: 72, + thresholdPercentage: 70, + })), + checkMidStream: mock(() => false), + resetForNewStream: mock(() => undefined), + setThreshold: mock(() => undefined), + getThreshold: mock(() => 0.7), + } as unknown as CompactionMonitor; + + const result = await session.sendMessage( + "hello", + { model: "openai:gpt-4o", agentId: "exec" }, + { admissionStale: () => true } + ); + expect(result.success).toBe(false); + expect(streamMessage).not.toHaveBeenCalled(); + + const historyResult = await historyService.getHistoryFromLatestBoundary(workspaceId); + expect(historyResult.success).toBe(true); + if (!historyResult.success) { + throw new Error(`failed to load history: ${String(historyResult.error)}`); + } + expect( + historyResult.data.some( + (message) => message.metadata?.muxMetadata?.type === "compaction-request" + ) + ).toBe(false); + + session.dispose(); + }); + test("uses preferred compaction model for on-send auto-compaction requests", async () => { const workspaceId = "ws-auto-compaction-preferred-model"; diff --git a/src/node/services/agentSession.ts b/src/node/services/agentSession.ts index ca876bf8d4..c70fc7496a 100644 --- a/src/node/services/agentSession.ts +++ b/src/node/services/agentSession.ts @@ -3938,12 +3938,19 @@ export class AgentSession { // refuse while sends are in preflight (r42), so rows can no longer land // after a mutation commits; this check and the PREPARING gate remain // backstops for entry-accounting bypasses. + // + // "Pre-persist" has one exception: the on-send compaction row above is already durable and + // carries this send as its follow-up. Refusing without removing it would leave a compaction + // request that startup recovery later resumes — for a bash-monitor wake whose lease is + // released by this refusal, that resubmits output the reconciler has already re-derived. if (this.turnAdmissionBlocks > 0 || isAdmissionStale()) { + await rollbackPersistedTurnRows(); return Err(createUnknownSendMessageError(CONTEXT_MUTATION_SEND_BLOCKED_MESSAGE)); } - // Still pre-persist: a row appended now would read as a dispatched turn on the next startup - // while streamWithHistory's own latch check keeps its stream from ever running. + // A row appended now would read as a dispatched turn on the next startup while + // streamWithHistory's own latch check keeps its stream from ever running. if (this.shuttingDown) { + await rollbackPersistedTurnRows(); return Err(createUnknownSendMessageError(SESSION_SHUTDOWN_SEND_BLOCKED_MESSAGE)); } diff --git a/src/node/services/taskService.test.ts b/src/node/services/taskService.test.ts index 8b9dfd8f55..0b36f35a8e 100644 --- a/src/node/services/taskService.test.ts +++ b/src/node/services/taskService.test.ts @@ -344,6 +344,7 @@ describe("TaskService", () => { disposable?: boolean; sendMessage?: ReturnType; remove?: ReturnType; + getInfo?: ReturnType; isStreaming?: ReturnType; hasQueuedMessages?: ReturnType; hasPendingQueuedOrPreparingTurn?: ReturnType; @@ -24847,6 +24848,67 @@ describe("TaskService", () => { } }); + test("a void retried after a partially persisted settlement still removes the disposable workspace", async () => { + // Same partial settlement as above, for a disposable workspace: the first attempt persists + // the terminal handle and throws before cleanup. The retry re-enters the terminal branch, + // which must resume the skipped cleanup instead of only repairing waiter/mirror state — + // otherwise the checkout leaks with nothing left to own it. + const remove = mock( + (_workspaceId: string): Promise> => Promise.resolve(Ok(undefined)) + ); + const getInfo = mock( + (): Promise<{ id: string } | null> => Promise.resolve({ id: "childworkspace" }) + ); + const { parentId, taskService } = await startWorkspaceTurnForTest({ + disposable: true, + remove, + getInfo, + }); + const manager = workspaceTurnManagerFor(taskService) as unknown as { + updateAgentTaskExecutionState: (...args: unknown[]) => Promise; + }; + const mirrorSpy = spyOn(manager, "updateAgentTaskExecutionState").mockImplementationOnce(() => + Promise.reject(new Error("execution state mirror unavailable")) + ); + try { + const firstAttempt = await taskService + .settleVoidedWorkspaceTurnContinuation( + "childworkspace", + workspaceTurnMuxMetadata(parentId), + "abandoned" + ) + .then( + () => null, + (error: unknown) => error + ); + expect(firstAttempt).toBeInstanceOf(Error); + expect(await workspaceTurnSnapshot(taskService, parentId)).toMatchObject({ + status: "interrupted", + disposableWorkspace: true, + }); + expect(remove).not.toHaveBeenCalled(); + + await taskService.settleVoidedWorkspaceTurnContinuation( + "childworkspace", + workspaceTurnMuxMetadata(parentId), + "abandoned" + ); + expect(remove).toHaveBeenCalledTimes(1); + expect(remove.mock.calls[0]?.[0]).toBe("childworkspace"); + + // Once the workspace is gone, later replays into the terminal branch do not retry it. + getInfo.mockImplementation(() => Promise.resolve(null)); + await taskService.settleVoidedWorkspaceTurnContinuation( + "childworkspace", + workspaceTurnMuxMetadata(parentId), + "abandoned" + ); + expect(remove).toHaveBeenCalledTimes(1); + } finally { + mirrorSpy.mockRestore(); + } + }); + test("settleVoidedWorkspaceTurnContinuation ignores a stale correlation", async () => { const { parentId, taskService } = await startWorkspaceTurnForTest(); diff --git a/src/node/services/workspaceService.test.ts b/src/node/services/workspaceService.test.ts index 4cad2d4dc7..0c9ed2ce7f 100644 --- a/src/node/services/workspaceService.test.ts +++ b/src/node/services/workspaceService.test.ts @@ -1227,6 +1227,37 @@ describe("WorkspaceService bash monitor wake reconciler wiring", () => { } }); + test("the wake level reaches a session still running startup recovery", async () => { + // Startup recovery runs the recovered turn inside a transient session before promoting + // it. A wake published while that turn streams must reach it, or the stream's foreground + // waits are never backgrounded and the deferred wake waits for the stream to end. + const { service, cleanup } = await createWakeWiringService(); + const workspaceId = "transient-recovery-wake-owner"; + const internal = service as unknown as { + backgroundProcessManager: { setMessageQueued: ReturnType }; + createSession(workspaceId: string): AgentSession; + transientStartupRecoverySessions: Map; + publishBashMonitorWakeLevel(ownerWorkspaceId: string, outstanding: boolean): void; + }; + const setMessageQueued = internal.backgroundProcessManager.setMessageQueued; + const session = internal.createSession(workspaceId); + internal.transientStartupRecoverySessions.set(workspaceId, session); + try { + // The session's lever is the observable: with no queue head the level is effective at + // once, so long-polling bash reads return early and foreground waits background. + internal.publishBashMonitorWakeLevel(workspaceId, true); + expect(setMessageQueued).toHaveBeenLastCalledWith(workspaceId, true); + + // Promotion keeps the mirror: it lives on the session, not on the map it sits in. + expect(service.getOrCreateSession(workspaceId)).toBe(session); + internal.publishBashMonitorWakeLevel(workspaceId, false); + expect(setMessageQueued).toHaveBeenLastCalledWith(workspaceId, false); + } finally { + session.dispose(); + await cleanup(); + } + }); + test("routes the session's tool-end yield edge to backgroundForegroundWaitsForWorkspace", async () => { // Which transitions raise the edge is the session's business // (agentSession.queueDispatch.test.ts); the service only routes it. diff --git a/src/node/services/workspaceService.ts b/src/node/services/workspaceService.ts index e2ace011df..6ed1c10e17 100644 --- a/src/node/services/workspaceService.ts +++ b/src/node/services/workspaceService.ts @@ -2399,25 +2399,8 @@ export class WorkspaceService extends EventEmitter implements WorkspaceHost { onWake: (dispatch) => this.dispatchBashMonitorWake(dispatch), readDeliveredWakeRecords: (ownerWorkspaceId) => this.readLastBashMonitorWakeRecords(ownerWorkspaceId), - onOutstandingChanged: (ownerWorkspaceId, outstanding) => { - // The level drives the same tool-boundary side effects a queued tool-end message - // does: long-polling bash reads return early and foreground agent-task waits are - // backgrounded so the stream can reach the boundary where it yields to the wake. - // The session arbitrates the level against its queue head and fires the yield - // edge (onToolEndYieldRequested) when the lever actually becomes effective. - const session = this.sessions.get(ownerWorkspaceId); - if (session != null) { - session.setBashMonitorWakeOutstanding(outstanding); - } else if ( - !outstanding && - // Partial BackgroundProcessManager stubs in tests (see the constructor guards). - typeof this.backgroundProcessManager.setMessageQueued === "function" - ) { - // No session, no queue: the flag can only be a stale mirror (e.g. reconciler - // disposal after the session went away), so drop it directly. - this.backgroundProcessManager.setMessageQueued(ownerWorkspaceId, false); - } - }, + onOutstandingChanged: (ownerWorkspaceId, outstanding) => + this.publishBashMonitorWakeLevel(ownerWorkspaceId, outstanding), }); if (typeof this.backgroundProcessManager.on === "function") { this.backgroundProcessManager.on("output:shown", this.bashOutputShownListener); @@ -4138,6 +4121,43 @@ export class WorkspaceService extends EventEmitter implements WorkspaceHost { }); } + /** + * Reconciler level → session mirror. The level drives the same tool-boundary side effects a + * queued tool-end message does: long-polling bash reads return early and foreground + * agent-task waits are backgrounded so the stream can reach the boundary where it yields to + * the wake. The session arbitrates the level against its queue head and fires the yield edge + * (onToolEndYieldRequested) when the lever actually becomes effective. + * + * Published to the live session, including one still running startup recovery: a recovered + * stream must see the level too, or a long-running foreground wait inside it is never + * backgrounded and the deferred wake waits for that stream to end on its own. + */ + private publishBashMonitorWakeLevel(ownerWorkspaceId: string, outstanding: boolean): void { + const session = this.getLiveSession(ownerWorkspaceId); + if (session != null) { + session.setBashMonitorWakeOutstanding(outstanding); + } else if ( + !outstanding && + // Partial BackgroundProcessManager stubs in tests (see the constructor guards). + typeof this.backgroundProcessManager.setMessageQueued === "function" + ) { + // No session, no queue: the flag can only be a stale mirror (e.g. reconciler disposal + // after the session went away), so drop it directly. + this.backgroundProcessManager.setMessageQueued(ownerWorkspaceId, false); + } + } + + /** + * The session currently owning a workspace's runtime state, whether cached or still + * transient for startup recovery. Read-only lookups of per-session state (wake level, debt, + * stream ledger) must use this rather than `sessions` alone: a recovered turn runs inside the + * transient session before it is promoted. + */ + private getLiveSession(workspaceId: string): AgentSession | undefined { + const trimmed = workspaceId.trim(); + return this.sessions.get(trimmed) ?? this.transientStartupRecoverySessions.get(trimmed); + } + public getOrCreateSession(workspaceId: string): AgentSession { assert(typeof workspaceId === "string", "workspaceId must be a string"); const trimmed = workspaceId.trim(); @@ -12029,15 +12049,14 @@ export class WorkspaceService extends EventEmitter implements WorkspaceHost { messageIds: readonly string[] ): boolean { return ( - this.sessions - .get(workspaceId.trim()) - ?.hasCorrelatedStreamStartedAfter(correlation, messageIds) === true + this.getLiveSession(workspaceId)?.hasCorrelatedStreamStartedAfter(correlation, messageIds) === + true ); } /** See AgentSession.hasBashMonitorWakeContinuation. */ hasBashMonitorWakeContinuation(workspaceId: string): boolean { - return this.sessions.get(workspaceId.trim())?.hasBashMonitorWakeContinuation() === true; + return this.getLiveSession(workspaceId)?.hasBashMonitorWakeContinuation() === true; } /** diff --git a/src/node/services/workspaceTurnManager.ts b/src/node/services/workspaceTurnManager.ts index c83d4a8e03..4f2f543c68 100644 --- a/src/node/services/workspaceTurnManager.ts +++ b/src/node/services/workspaceTurnManager.ts @@ -2203,6 +2203,17 @@ export class WorkspaceTurnManager { current.status ); this.taskHost.markTaskForegroundRelevant(current.handleId); + // The terminal row is the first durable write of a settlement; the phases after it + // (mirror, waiters, disposable cleanup) can be skipped by a throw, and this branch is + // where the retry lands. Resume the cleanup phase here too — a still-registered + // disposable workspace on a terminal record means no settlement reached it (cleanup + // either removes the workspace or, when forwarding it, clears the flag). + if ( + current.disposableWorkspace && + (await this.workspaceService.getInfo(current.workspaceId)) != null + ) { + await this.cleanupDisposableWorkspaceTurn(current); + } return { pendingNotify: null, winningStatus: current.status }; } From 8eed56d10df13b95895fffb1193411b1d10fcc3a Mon Sep 17 00:00:00 2001 From: Ethan Dickson Date: Fri, 4 Sep 2026 11:46:38 +0000 Subject: [PATCH 18/26] Round 15: recover delivered wake records through an on-send compaction row (getCarriedBashMonitorWake shared with carriesBashMonitorWake) --- src/node/services/agentSession.ts | 21 ++++++--- src/node/services/workspaceService.test.ts | 54 +++++++++++++++++++++- src/node/services/workspaceService.ts | 7 ++- 3 files changed, 72 insertions(+), 10 deletions(-) diff --git a/src/node/services/agentSession.ts b/src/node/services/agentSession.ts index c70fc7496a..d596c9515d 100644 --- a/src/node/services/agentSession.ts +++ b/src/node/services/agentSession.ts @@ -445,17 +445,24 @@ function isCompactionRequestMetadata(meta: unknown): meta is CompactionRequestMe } /** - * Whether send metadata carries a bash-monitor wake: the wake itself, or an on-send - * compaction request whose follow-up is the wake (the compaction row is what that turn - * shows, so the wake identity has to be read through it). + * The bash-monitor wake a row or send carries: the wake itself, or — for an on-send compaction + * request — the wake nested in the follow-up it carries. The compaction row is what that turn + * shows and what stays durable, so wake identity (and the delivered records the reconciler + * recovers from after a restart) has to be read through it. */ -function carriesBashMonitorWake(muxMetadata: unknown): boolean { +export function getCarriedBashMonitorWake( + muxMetadata: unknown +): Extract | undefined { const meta = muxMetadata as MuxMessageMetadata | undefined; - if (meta?.type === "bash-monitor-wake") return true; - if (!isCompactionRequestMetadata(meta)) return false; + if (meta?.type === "bash-monitor-wake") return meta; + if (!isCompactionRequestMetadata(meta)) return undefined; const followUpMetadata = meta.parsed.followUpContent?.muxMetadata ?? meta.parsed.continueMessage?.muxMetadata; - return followUpMetadata?.type === "bash-monitor-wake"; + return followUpMetadata?.type === "bash-monitor-wake" ? followUpMetadata : undefined; +} + +function carriesBashMonitorWake(muxMetadata: unknown): boolean { + return getCarriedBashMonitorWake(muxMetadata) != null; } /** diff --git a/src/node/services/workspaceService.test.ts b/src/node/services/workspaceService.test.ts index 0c9ed2ce7f..84fe325fbc 100644 --- a/src/node/services/workspaceService.test.ts +++ b/src/node/services/workspaceService.test.ts @@ -255,9 +255,61 @@ describe("WorkspaceService bash monitor wake reconciler wiring", () => { ), backgroundProcessManager, }); - return { config, service, events, cleanup }; + return { config, service, events, historyService, cleanup }; } + test("delivery recovery reads a wake diverted through on-send compaction", async () => { + // The reconciler recovers "already delivered" from the owner's last durable wake row. A + // wake that crossed the compaction threshold is durable as the compaction request that + // carries it as follow-up, so the reader has to unwrap that row like carriesBashMonitorWake + // does — otherwise a restart after a failed acknowledgment re-dispatches the same output. + const { service, historyService, cleanup } = await createWakeWiringService(); + const workspaceId = "compaction-carried-wake-owner"; + const internal = service as unknown as { + readLastBashMonitorWakeRecords( + ownerWorkspaceId: string + ): Promise | undefined>; + }; + const wakeRecord = { + processId: "proc", + wakeUpdatedAt: "2026-08-31T12:00:00.000Z", + kind: "match" as const, + displayName: "run", + filter: "READY", + filterExclude: false, + }; + try { + expect(await internal.readLastBashMonitorWakeRecords(workspaceId)).toBeUndefined(); + await historyService.appendToHistory( + workspaceId, + createMuxMessage("wake-compaction", "user", "Compacting to continue", { + timestamp: 1_000, + synthetic: true, + muxMetadata: { + type: "compaction-request", + rawCommand: "/compact", + parsed: { + followUpContent: { + text: "Monitor output", + model: "openai:gpt-5.2", + agentId: "exec", + muxMetadata: { type: "bash-monitor-wake", records: [wakeRecord] }, + }, + }, + source: "auto-compaction", + }, + }) + ); + await historyService.appendToHistory( + workspaceId, + createMuxMessage("summary", "assistant", "summary", { timestamp: 1_100 }) + ); + expect(await internal.readLastBashMonitorWakeRecords(workspaceId)).toEqual([wakeRecord]); + } finally { + await cleanup(); + } + }); + test("monitor lifecycle and shown-output events poke the reconciler", async () => { const { service, events, cleanup } = await createWakeWiringService(); const scheduleReconcile = mock(() => undefined); diff --git a/src/node/services/workspaceService.ts b/src/node/services/workspaceService.ts index 6ed1c10e17..fe6022d948 100644 --- a/src/node/services/workspaceService.ts +++ b/src/node/services/workspaceService.ts @@ -39,6 +39,7 @@ import { AgentSession, clearProviderConfigFixableAbandonMarkers, CONTEXT_MUTATION_SEND_BLOCKED_MESSAGE, + getCarriedBashMonitorWake, inheritOpenWorkspaceTurnMetadata, type StreamErrorRecoveryOutcome, } from "@/node/services/agentSession"; @@ -12084,8 +12085,10 @@ export class WorkspaceService extends EventEmitter implements WorkspaceHost { ); if (!tail.success) return undefined; for (const message of tail.data.toReversed()) { - const muxMetadata = message.metadata?.muxMetadata; - if (muxMetadata?.type === "bash-monitor-wake") return muxMetadata.records; + // A wake diverted through on-send compaction is durable as the compaction row that + // carries it as follow-up; that row is the acknowledgment too. + const wake = getCarriedBashMonitorWake(message.metadata?.muxMetadata); + if (wake != null) return wake.records; } return undefined; } From 6594bca1b499de703f7f96f1d43dfdfa4b116715 Mon Sep 17 00:00:00 2001 From: Ethan Dickson Date: Fri, 4 Sep 2026 12:05:51 +0000 Subject: [PATCH 19/26] Round 16: resolve the queue-cut cutter and the other sync session predicates through the live session (transient recovery included); a failed wake-row read fails the reconcile instead of counting as no row; recheck the queue head when the level read fails --- .../agentSession.queueDispatch.test.ts | 10 +++++++- src/node/services/agentSession.ts | 6 +++-- .../bashMonitorWakeReconciler.test.ts | 17 ++++++++++++- .../services/bashMonitorWakeReconciler.ts | 4 ++- src/node/services/workspaceService.test.ts | 25 +++++++++++++++++++ src/node/services/workspaceService.ts | 23 ++++++++++------- 6 files changed, 71 insertions(+), 14 deletions(-) diff --git a/src/node/services/agentSession.queueDispatch.test.ts b/src/node/services/agentSession.queueDispatch.test.ts index f7fa2c94db..8eee1c9683 100644 --- a/src/node/services/agentSession.queueDispatch.test.ts +++ b/src/node/services/agentSession.queueDispatch.test.ts @@ -661,9 +661,17 @@ describe("AgentSession queued message tool-call dispatch", () => { level = () => Promise.resolve(false); expect(await session.hasPendingToolEndInput()).toBe(false); - // A failing level read must not cut the stream. + // A failing level read must not cut the stream on the level's account... level = () => Promise.reject(new Error("watermark read failed")); expect(await session.hasPendingToolEndInput()).toBe(false); + // ...but a tool-end message queued while that read was in flight still arbitrates the + // boundary: the failure says nothing about the queue. + level = () => { + session.queueMessage("correction", { model: TEST_MODEL, agentId: "exec" }); + return Promise.reject(new Error("watermark read failed")); + }; + expect(await session.hasPendingToolEndInput()).toBe(true); + session.clearQueue(); // A non-empty queue arbitrates alone: a turn-end head is not promoted to tool-end by // a high wake level (the wake dispatcher waits for the queue to drain anyway). diff --git a/src/node/services/agentSession.ts b/src/node/services/agentSession.ts index d596c9515d..417c641cbe 100644 --- a/src/node/services/agentSession.ts +++ b/src/node/services/agentSession.ts @@ -6882,11 +6882,13 @@ export class AgentSession { }; return true; } catch (error) { - log.debug("hasPendingToolEndInput: wake level read failed; not yielding", { + log.debug("hasPendingToolEndInput: wake level read failed; not yielding on the level", { workspaceId: this.workspaceId, error, }); - return false; + // A message queued while the level read was in flight still arbitrates this boundary: + // the read's failure says nothing about the queue. + return this.messageQueue.getNextDispatchableMode() === "tool-end"; } } diff --git a/src/node/services/bashMonitorWakeReconciler.test.ts b/src/node/services/bashMonitorWakeReconciler.test.ts index e2768282bd..70b045415c 100644 --- a/src/node/services/bashMonitorWakeReconciler.test.ts +++ b/src/node/services/bashMonitorWakeReconciler.test.ts @@ -365,8 +365,23 @@ describe("BashMonitorWakeReconciler", () => { await control.instance.reconcile(OWNER); expect(control.restarted).toHaveLength(1); - const recovered = restart(() => Promise.resolve(deliveredRecords)); + // A read that cannot answer fails the reconcile (retried) instead of counting as "no row": + // recovery is consulted once per owner, so a swallowed failure would dispatch a duplicate. + let readFails = true; + const recovered = restart(() => + readFails + ? Promise.reject(new Error("history unavailable")) + : Promise.resolve(deliveredRecords) + ); acknowledged = []; + const failedReconcile = await recovered.instance.reconcile(OWNER).then( + () => null, + (error: unknown) => error + ); + expect(failedReconcile).toBeInstanceOf(Error); + expect(recovered.restarted).toHaveLength(0); + expect(acknowledged).toEqual([]); + readFails = false; await recovered.instance.reconcile(OWNER); expect(recovered.restarted).toHaveLength(0); expect(acknowledged).toEqual([{ processId: "proc", matchedThroughOffset: 12 }]); diff --git a/src/node/services/bashMonitorWakeReconciler.ts b/src/node/services/bashMonitorWakeReconciler.ts index eca93e9f6f..0b58ea6233 100644 --- a/src/node/services/bashMonitorWakeReconciler.ts +++ b/src/node/services/bashMonitorWakeReconciler.ts @@ -431,7 +431,9 @@ export class BashMonitorWakeReconciler { * durable acknowledgment: a commit whose watermark write failed and was then lost to a * restart (the in-memory committed lease dies with the process) would otherwise * re-derive and re-dispatch the very signals that row already delivers. Consulted once - * per owner, the first time signals derive outstanding in this process. + * per owner, the first time signals derive outstanding in this process. A read that + * cannot answer must throw (not return undefined): the reconcile fails and retries, so + * "no row" is only ever concluded from a successful read. */ readDeliveredWakeRecords?( ownerWorkspaceId: string diff --git a/src/node/services/workspaceService.test.ts b/src/node/services/workspaceService.test.ts index 84fe325fbc..05ed3a6792 100644 --- a/src/node/services/workspaceService.test.ts +++ b/src/node/services/workspaceService.test.ts @@ -305,6 +305,20 @@ describe("WorkspaceService bash monitor wake reconciler wiring", () => { createMuxMessage("summary", "assistant", "summary", { timestamp: 1_100 }) ); expect(await internal.readLastBashMonitorWakeRecords(workspaceId)).toEqual([wakeRecord]); + + // "Could not read" is not "no row": the reconciler consults this once per owner. + const readSpy = spyOn(historyService, "getLastMessages").mockImplementationOnce(() => + Promise.resolve(Err("history unavailable")) + ); + try { + const failed = await internal.readLastBashMonitorWakeRecords(workspaceId).then( + () => null, + (error: unknown) => error + ); + expect(failed).toBeInstanceOf(Error); + } finally { + readSpy.mockRestore(); + } } finally { await cleanup(); } @@ -1300,6 +1314,17 @@ describe("WorkspaceService bash monitor wake reconciler wiring", () => { internal.publishBashMonitorWakeLevel(workspaceId, true); expect(setMessageQueued).toHaveBeenLastCalledWith(workspaceId, true); + // Settlement's reads of the cut resolve through the same live lookup: a recovered + // delegated stream that yielded to the wake must not settle for want of a cutter. + const sessionInternal = session as unknown as { + getQueueCutCutter(): unknown; + hasBashMonitorWakeContinuation(): boolean; + }; + sessionInternal.getQueueCutCutter = () => ({ stage: "bash-monitor-wake" }); + sessionInternal.hasBashMonitorWakeContinuation = () => true; + expect(service.getQueueCutCutter(workspaceId)).toEqual({ stage: "bash-monitor-wake" }); + expect(service.hasBashMonitorWakeContinuation(workspaceId)).toBe(true); + // Promotion keeps the mirror: it lives on the session, not on the map it sits in. expect(service.getOrCreateSession(workspaceId)).toBe(session); internal.publishBashMonitorWakeLevel(workspaceId, false); diff --git a/src/node/services/workspaceService.ts b/src/node/services/workspaceService.ts index fe6022d948..c8ff49dc7f 100644 --- a/src/node/services/workspaceService.ts +++ b/src/node/services/workspaceService.ts @@ -11931,11 +11931,11 @@ export class WorkspaceService extends EventEmitter implements WorkspaceHost { } isBusyForMessage(workspaceId: string): boolean { - return this.sessions.get(workspaceId.trim())?.isBusy() === true; + return this.getLiveSession(workspaceId)?.isBusy() === true; } hasQueuedWorkspaceTurn(workspaceId: string, handleId: string): boolean { - return this.sessions.get(workspaceId.trim())?.hasQueuedWorkspaceTurn(handleId) ?? false; + return this.getLiveSession(workspaceId)?.hasQueuedWorkspaceTurn(handleId) ?? false; } /** @@ -11961,11 +11961,11 @@ export class WorkspaceService extends EventEmitter implements WorkspaceHost { } hasQueuedOrDispatchingEntry(workspaceId: string): boolean { - return this.sessions.get(workspaceId.trim())?.hasQueuedOrDispatchingEntry() ?? false; + return this.getLiveSession(workspaceId)?.hasQueuedOrDispatchingEntry() ?? false; } hasQueuedMessages(workspaceId: string, dispatchMode?: "tool-end" | "turn-end"): boolean { - return this.sessions.get(workspaceId.trim())?.hasQueuedMessages(dispatchMode) ?? false; + return this.getLiveSession(workspaceId)?.hasQueuedMessages(dispatchMode) ?? false; } async waitForPendingCompactionCompletionDecision( @@ -12033,7 +12033,7 @@ export class WorkspaceService extends EventEmitter implements WorkspaceHost { } hasPendingQueuedOrPreparingTurn(workspaceId: string): boolean { - const session = this.sessions.get(workspaceId.trim()); + const session = this.getLiveSession(workspaceId); if (!session) { return false; } @@ -12083,7 +12083,12 @@ export class WorkspaceService extends EventEmitter implements WorkspaceHost { ownerWorkspaceId, LAST_BASH_MONITOR_WAKE_ROW_SCAN_DEPTH ); - if (!tail.success) return undefined; + // Distinguish "no row" from "could not read": the reconciler recovers once per owner, so a + // read failure swallowed here would let the reconcile dispatch a duplicate of a wake the + // row already delivered. Throwing fails this reconcile; its retry reads again. + if (!tail.success) { + throw new Error(`Failed to read the last bash-monitor wake row: ${tail.error}`); + } for (const message of tail.data.toReversed()) { // A wake diverted through on-send compaction is durable as the compaction row that // carries it as follow-up; that row is the acknowledgment too. @@ -12097,7 +12102,7 @@ export class WorkspaceService extends EventEmitter implements WorkspaceHost { workspaceId: string, metadata: Extract ): boolean { - const session = this.sessions.get(workspaceId.trim()); + const session = this.getLiveSession(workspaceId); return session?.hasPendingWorkspaceTurnContinuation(metadata) ?? false; } @@ -12106,7 +12111,7 @@ export class WorkspaceService extends EventEmitter implements WorkspaceHost { * See AgentSession.getQueueCutCutter for stage semantics. */ getQueueCutCutter(workspaceId: string): QueueCutCutter | undefined { - const session = this.sessions.get(workspaceId.trim()); + const session = this.getLiveSession(workspaceId); return session?.getQueueCutCutter(); } @@ -12118,7 +12123,7 @@ export class WorkspaceService extends EventEmitter implements WorkspaceHost { * hasPendingQueuedOrPreparingTurn. */ hasPendingAutoRetry(workspaceId: string): boolean { - const session = this.sessions.get(workspaceId.trim()); + const session = this.getLiveSession(workspaceId); return session?.hasPendingAutoRetry() ?? false; } From eae9eefc08792214abd615ab3d6b93ffaa1a60a6 Mon Sep 17 00:00:00 2001 From: Ethan Dickson Date: Fri, 4 Sep 2026 12:19:00 +0000 Subject: [PATCH 20/26] Resolve idle waits through the live session so a wake deferred on a busy startup-recovery session does not spin --- src/node/services/workspaceService.test.ts | 24 ++++++++++++++++++++++ src/node/services/workspaceService.ts | 9 ++++++-- 2 files changed, 31 insertions(+), 2 deletions(-) diff --git a/src/node/services/workspaceService.test.ts b/src/node/services/workspaceService.test.ts index 05ed3a6792..4ef50c0912 100644 --- a/src/node/services/workspaceService.test.ts +++ b/src/node/services/workspaceService.test.ts @@ -1325,6 +1325,30 @@ describe("WorkspaceService bash monitor wake reconciler wiring", () => { expect(service.getQueueCutCutter(workspaceId)).toEqual({ stage: "bash-monitor-wake" }); expect(service.hasBashMonitorWakeContinuation(workspaceId)).toBe(true); + // A wake deferred because this session is busy waits for *this* session: if the wait + // resolved through `sessions` alone it would return at once and re-defer in a loop. + let busy = true; + let releaseIdle: () => void = () => undefined; + const idle = new Promise((resolve) => { + releaseIdle = resolve; + }); + const busyInternal = session as unknown as { + isBusy(): boolean; + waitForIdle(): Promise; + }; + busyInternal.isBusy = () => busy; + busyInternal.waitForIdle = () => idle; + expect(service.isBusyForMessage(workspaceId)).toBe(true); + let idleWaitResolved = false; + const idleWait = service.waitForIdleAndNoQueuedMessages(workspaceId).then(() => { + idleWaitResolved = true; + }); + await new Promise((resolve) => setTimeout(resolve, 0)); + expect(idleWaitResolved).toBe(false); + busy = false; + releaseIdle(); + await idleWait; + // Promotion keeps the mirror: it lives on the session, not on the map it sits in. expect(service.getOrCreateSession(workspaceId)).toBe(session); internal.publishBashMonitorWakeLevel(workspaceId, false); diff --git a/src/node/services/workspaceService.ts b/src/node/services/workspaceService.ts index c8ff49dc7f..fde303f010 100644 --- a/src/node/services/workspaceService.ts +++ b/src/node/services/workspaceService.ts @@ -11984,13 +11984,18 @@ export class WorkspaceService extends EventEmitter implements WorkspaceHost { return session?.waitForPendingStreamErrorRecoveryDecision(messageId); } + // Both waits resolve through the live lookup, matching the busy predicates above: a wake + // deferred because a transient startup-recovery session is busy waits for *that* session to + // go idle. Waiting on `sessions` alone would resolve at once and re-defer in a tight loop of + // history/registry reads until recovery promoted it. A transient session either promotes as + // the same instance or is disposed, and dispose releases idle waiters. async waitForIdle(workspaceId: string): Promise { - const session = this.sessions.get(workspaceId.trim()); + const session = this.getLiveSession(workspaceId); await session?.waitForIdle(); } async waitForIdleAndNoQueuedMessages(workspaceId: string): Promise { - const session = this.sessions.get(workspaceId.trim()); + const session = this.getLiveSession(workspaceId); if (!session) { return; } From 551ca90cd78a97757591c22cf86a04b5ceb36b86 Mon Sep 17 00:00:00 2001 From: Ethan Dickson Date: Fri, 4 Sep 2026 13:03:12 +0000 Subject: [PATCH 21/26] Consume a refused wake's lease when its on-send compaction row cannot be rolled back --- .../agentSession.autoCompaction.test.ts | 67 ++++++++++++++++++- src/node/services/agentSession.ts | 19 ++++-- 2 files changed, 80 insertions(+), 6 deletions(-) diff --git a/src/node/services/agentSession.autoCompaction.test.ts b/src/node/services/agentSession.autoCompaction.test.ts index d0022175e1..51b0f680bd 100644 --- a/src/node/services/agentSession.autoCompaction.test.ts +++ b/src/node/services/agentSession.autoCompaction.test.ts @@ -447,13 +447,20 @@ describe("AgentSession on-send auto-compaction snapshot deferral", () => { getThreshold: mock(() => 0.7), } as unknown as CompactionMonitor; + const onAccepted = mock(() => Promise.resolve()); const result = await session.sendMessage( "hello", - { model: "openai:gpt-4o", agentId: "exec" }, - { admissionStale: () => true } + { + model: "openai:gpt-4o", + agentId: "exec", + muxMetadata: { type: "bash-monitor-wake", records: [] }, + }, + { admissionStale: () => true, onAccepted } ); expect(result.success).toBe(false); expect(streamMessage).not.toHaveBeenCalled(); + // The row is gone, so the wake lease stays released for the reconciler to re-derive. + expect(onAccepted).not.toHaveBeenCalled(); const historyResult = await historyService.getHistoryFromLatestBoundary(workspaceId); expect(historyResult.success).toBe(true); @@ -469,6 +476,62 @@ describe("AgentSession on-send auto-compaction snapshot deferral", () => { session.dispose(); }); + test("a refused wake whose compaction row cannot be rolled back consumes its lease", async () => { + // If the rollback fails the row stays durable and startup recovery will resume it with the + // wake as its follow-up. Releasing the lease too would have the reconciler deliver the same + // output a second time (possibly after the monitor retracted it), so the refusal must + // consume the wake and leave the durable row as its only carrier. + const workspaceId = "ws-auto-compaction-rollback-failure-consumes-wake"; + const streamMessage = mock(() => Promise.resolve(Ok(createStartedTurnHandle()))); + const { session, historyService } = await createSessionHarness({ + workspaceId, + streamMessage: streamMessage as unknown as AIService["streamMessage"], + }); + + (session as unknown as { compactionMonitor: CompactionMonitor }).compactionMonitor = { + checkBeforeSend: mock(() => ({ + shouldShowWarning: true, + shouldForceCompact: false, + usagePercentage: 72, + thresholdPercentage: 70, + })), + checkMidStream: mock(() => false), + resetForNewStream: mock(() => undefined), + setThreshold: mock(() => undefined), + getThreshold: mock(() => 0.7), + } as unknown as CompactionMonitor; + spyOn(historyService, "deleteMessages").mockImplementationOnce(() => + Promise.resolve(Err("disk unavailable")) + ); + + const onAccepted = mock(() => Promise.resolve()); + const result = await session.sendMessage( + "hello", + { + model: "openai:gpt-4o", + agentId: "exec", + muxMetadata: { type: "bash-monitor-wake", records: [] }, + }, + { admissionStale: () => true, onAccepted } + ); + expect(result.success).toBe(false); + expect(streamMessage).not.toHaveBeenCalled(); + expect(onAccepted).toHaveBeenCalledTimes(1); + + const historyResult = await historyService.getHistoryFromLatestBoundary(workspaceId); + expect(historyResult.success).toBe(true); + if (!historyResult.success) { + throw new Error(`failed to load history: ${String(historyResult.error)}`); + } + expect( + historyResult.data.some( + (message) => message.metadata?.muxMetadata?.type === "compaction-request" + ) + ).toBe(true); + + session.dispose(); + }); + test("uses preferred compaction model for on-send auto-compaction requests", async () => { const workspaceId = "ws-auto-compaction-preferred-model"; diff --git a/src/node/services/agentSession.ts b/src/node/services/agentSession.ts index 417c641cbe..2e532c8aa5 100644 --- a/src/node/services/agentSession.ts +++ b/src/node/services/agentSession.ts @@ -3950,15 +3950,26 @@ export class AgentSession { // carries this send as its follow-up. Refusing without removing it would leave a compaction // request that startup recovery later resumes — for a bash-monitor wake whose lease is // released by this refusal, that resubmits output the reconciler has already re-derived. + // + // When the rollback itself fails the row stays durable and startup recovery WILL resume it, + // follow-up wake included. A wake must then be consumed (same convention as the `disposed` + // path below): leaving the lease released would have the reconciler redeliver output that + // the durable row already carries — twice, one copy of which the monitor may since have + // retracted. The refusal still stands; the row is the wake's only carrier from here. + const refuseAfterCompactionRow = async (message: string): Promise> => { + const rolledBack = await rollbackPersistedTurnRows(); + if (!rolledBack && typedMuxMetadata?.type === "bash-monitor-wake") { + await internal?.onAccepted?.(); + } + return Err(createUnknownSendMessageError(message)); + }; if (this.turnAdmissionBlocks > 0 || isAdmissionStale()) { - await rollbackPersistedTurnRows(); - return Err(createUnknownSendMessageError(CONTEXT_MUTATION_SEND_BLOCKED_MESSAGE)); + return refuseAfterCompactionRow(CONTEXT_MUTATION_SEND_BLOCKED_MESSAGE); } // A row appended now would read as a dispatched turn on the next startup while // streamWithHistory's own latch check keeps its stream from ever running. if (this.shuttingDown) { - await rollbackPersistedTurnRows(); - return Err(createUnknownSendMessageError(SESSION_SHUTDOWN_SEND_BLOCKED_MESSAGE)); + return refuseAfterCompactionRow(SESSION_SHUTDOWN_SEND_BLOCKED_MESSAGE); } // Persist snapshots only when this turn will be sent immediately. From cd483a011e7ecd16354ed1a741d5b5e947152d5f Mon Sep 17 00:00:00 2001 From: Ethan Dickson Date: Fri, 4 Sep 2026 13:16:03 +0000 Subject: [PATCH 22/26] Consume a refused wake only when its compaction row verifiably remains, and arm the in-session resume when it does --- .../agentSession.autoCompaction.test.ts | 132 ++++++++++++++---- src/node/services/agentSession.ts | 58 +++++--- 2 files changed, 137 insertions(+), 53 deletions(-) diff --git a/src/node/services/agentSession.autoCompaction.test.ts b/src/node/services/agentSession.autoCompaction.test.ts index 51b0f680bd..5d9eb0c821 100644 --- a/src/node/services/agentSession.autoCompaction.test.ts +++ b/src/node/services/agentSession.autoCompaction.test.ts @@ -476,18 +476,14 @@ describe("AgentSession on-send auto-compaction snapshot deferral", () => { session.dispose(); }); - test("a refused wake whose compaction row cannot be rolled back consumes its lease", async () => { - // If the rollback fails the row stays durable and startup recovery will resume it with the - // wake as its follow-up. Releasing the lease too would have the reconciler deliver the same - // output a second time (possibly after the monitor retracted it), so the refusal must - // consume the wake and leave the durable row as its only carrier. - const workspaceId = "ws-auto-compaction-rollback-failure-consumes-wake"; + // Shared setup for the rollback-failure refusals below: on-send compaction lands its row, + // then the send is refused as stale and the row's deletion fails. + async function createRefusedWakeAfterFailedRollbackHarness(workspaceId: string) { const streamMessage = mock(() => Promise.resolve(Ok(createStartedTurnHandle()))); const { session, historyService } = await createSessionHarness({ workspaceId, streamMessage: streamMessage as unknown as AIService["streamMessage"], }); - (session as unknown as { compactionMonitor: CompactionMonitor }).compactionMonitor = { checkBeforeSend: mock(() => ({ shouldShowWarning: true, @@ -500,36 +496,110 @@ describe("AgentSession on-send auto-compaction snapshot deferral", () => { setThreshold: mock(() => undefined), getThreshold: mock(() => 0.7), } as unknown as CompactionMonitor; - spyOn(historyService, "deleteMessages").mockImplementationOnce(() => + const deleteMessages = spyOn(historyService, "deleteMessages").mockImplementationOnce(() => Promise.resolve(Err("disk unavailable")) ); - + const chatEventTypes: string[] = []; + session.onChatEvent((event) => { + chatEventTypes.push(event.message.type); + }); + const readResumeRequest = () => + ( + session as unknown as { + lastAutoRetryResumeRequest?: { options: { muxMetadata?: unknown } }; + } + ).lastAutoRetryResumeRequest; const onAccepted = mock(() => Promise.resolve()); - const result = await session.sendMessage( - "hello", - { - model: "openai:gpt-4o", - agentId: "exec", - muxMetadata: { type: "bash-monitor-wake", records: [] }, - }, - { admissionStale: () => true, onAccepted } - ); - expect(result.success).toBe(false); - expect(streamMessage).not.toHaveBeenCalled(); - expect(onAccepted).toHaveBeenCalledTimes(1); + const sendRefusedWake = () => + session.sendMessage( + "hello", + { + model: "openai:gpt-4o", + agentId: "exec", + muxMetadata: { type: "bash-monitor-wake", records: [] }, + }, + { admissionStale: () => true, onAccepted } + ); + const hasCompactionRow = async () => { + const historyResult = await historyService.getHistoryFromLatestBoundary(workspaceId); + if (!historyResult.success) { + throw new Error(`failed to load history: ${String(historyResult.error)}`); + } + return historyResult.data.some( + (message) => message.metadata?.muxMetadata?.type === "compaction-request" + ); + }; + return { + session, + historyService, + streamMessage, + onAccepted, + chatEventTypes, + readResumeRequest, + sendRefusedWake, + hasCompactionRow, + deleteMessagesCalls: () => deleteMessages.mock.calls.length, + }; + } - const historyResult = await historyService.getHistoryFromLatestBoundary(workspaceId); - expect(historyResult.success).toBe(true); - if (!historyResult.success) { - throw new Error(`failed to load history: ${String(historyResult.error)}`); + test("a refused wake whose compaction row verifiably remains consumes its lease and arms a resume", async () => { + // The durable row carries the wake as its follow-up. Releasing the lease too would have the + // reconciler deliver the same output a second time (possibly after the monitor retracted + // it), so the refusal consumes the wake — and arms the in-session resume like every other + // durable pre-stream failure, because startup recovery only resumes an interrupted history + // tail and a competing manual send could bury the request under a newer turn. + const h = await createRefusedWakeAfterFailedRollbackHarness( + "ws-auto-compaction-rollback-failure-consumes-wake" + ); + try { + let resumeArmedAtAcceptance = false; + h.onAccepted.mockImplementation(() => { + resumeArmedAtAcceptance = h.readResumeRequest() != null; + return Promise.resolve(); + }); + const result = await h.sendRefusedWake(); + expect(result.success).toBe(false); + expect(h.streamMessage).not.toHaveBeenCalled(); + expect(h.onAccepted).toHaveBeenCalledTimes(1); + expect(resumeArmedAtAcceptance).toBe(true); + // The resume replays the compaction request (which carries the wake), not a fresh row. + expect( + (h.readResumeRequest()?.options.muxMetadata as { type?: string } | undefined)?.type + ).toBe("compaction-request"); + expect(h.chatEventTypes).toContain("auto-retry-scheduled"); + expect(await h.hasCompactionRow()).toBe(true); + } finally { + await h.session.setAutoRetryEnabled(false, { persist: false }); + h.session.dispose(); } - expect( - historyResult.data.some( - (message) => message.metadata?.muxMetadata?.type === "compaction-request" - ) - ).toBe(true); + }); - session.dispose(); + test("a refused wake whose rollback outcome is unknown keeps its lease released", async () => { + // deleteMessages can fail after committing; if the readback fails too the row's fate is + // unknown. Consuming the lease then could leave neither carrier nor signal (the wake and any + // delegated continuation deferred behind it would be lost), so the refusal must release + // and let the reconciler re-derive — a duplicate delivery is the tolerable failure mode. + const h = await createRefusedWakeAfterFailedRollbackHarness( + "ws-auto-compaction-rollback-unknown-releases-wake" + ); + try { + // Fail only the readback that follows the failed delete; sendMessage reads history + // earlier (compaction check) and those reads must stay healthy for the row to land. + const readHistory = h.historyService.getHistoryFromLatestBoundary.bind(h.historyService); + spyOn(h.historyService, "getHistoryFromLatestBoundary").mockImplementation((...args) => + h.deleteMessagesCalls() > 0 + ? Promise.resolve(Err("disk unavailable")) + : readHistory(...args) + ); + const result = await h.sendRefusedWake(); + expect(result.success).toBe(false); + expect(h.streamMessage).not.toHaveBeenCalled(); + expect(h.onAccepted).not.toHaveBeenCalled(); + expect(h.readResumeRequest()).toBeUndefined(); + expect(h.chatEventTypes).not.toContain("auto-retry-scheduled"); + } finally { + h.session.dispose(); + } }); test("uses preferred compaction model for on-send auto-compaction requests", async () => { diff --git a/src/node/services/agentSession.ts b/src/node/services/agentSession.ts index 2e532c8aa5..a444e2ddf8 100644 --- a/src/node/services/agentSession.ts +++ b/src/node/services/agentSession.ts @@ -3328,19 +3328,22 @@ export class AgentSession { // Roll back synthetic snapshots if the invoking user row fails to persist, or // later provider requests could consume orphaned context. /** - * Returns whether the rows are verifiably gone. deleteMessages can fail AFTER its atomic + * "deleted" means the rows are verifiably gone. deleteMessages can fail AFTER its atomic * rewrite committed, so a reported failure re-reads the durable history before concluding — * callers that couple side effects to the rollback (peer budget refunds) must only act when * deletion actually committed, or a "canceled" payload would stay durable while no longer - * counting against the sender's budget. + * counting against the sender's budget. "remains" is the readback confirming a row is still + * durable; "unknown" is a readback that itself failed. Callers whose side effect is only + * safe against a row that truly exists (consuming a wake lease) must not treat "unknown" + * as "remains". */ - const rollbackPersistedTurnRows = async (): Promise => { - if (persistedTurnRowMessageIds.length === 0) return true; + const rollbackPersistedTurnRows = async (): Promise<"deleted" | "remains" | "unknown"> => { + if (persistedTurnRowMessageIds.length === 0) return "deleted"; const rollbackResult = await this.historyService.deleteMessages( this.workspaceId, persistedTurnRowMessageIds ); - if (rollbackResult.success) return true; + if (rollbackResult.success) return "deleted"; log.error("Failed to roll back partially persisted turn rows", { workspaceId: this.workspaceId, error: rollbackResult.error, @@ -3348,12 +3351,12 @@ export class AgentSession { const historyResult = await this.historyService.getHistoryFromLatestBoundary( this.workspaceId ); - return ( - historyResult.success && - persistedTurnRowMessageIds.every( - (messageId) => !historyResult.data.some((message) => message.id === messageId) - ) - ); + if (!historyResult.success) return "unknown"; + return persistedTurnRowMessageIds.every( + (messageId) => !historyResult.data.some((message) => message.id === messageId) + ) + ? "deleted" + : "remains"; }; // Last-line-of-defence pricing gate: every dispatch path (initial sends, @@ -3951,15 +3954,25 @@ export class AgentSession { // request that startup recovery later resumes — for a bash-monitor wake whose lease is // released by this refusal, that resubmits output the reconciler has already re-derived. // - // When the rollback itself fails the row stays durable and startup recovery WILL resume it, - // follow-up wake included. A wake must then be consumed (same convention as the `disposed` - // path below): leaving the lease released would have the reconciler redeliver output that - // the durable row already carries — twice, one copy of which the monitor may since have - // retracted. The refusal still stands; the row is the wake's only carrier from here. + // When the rollback fails and the readback CONFIRMS the row still durable, that row already + // carries the wake as its follow-up. Leaving the lease released would have the reconciler + // redeliver output the row carries — twice, one copy of which the monitor may since have + // retracted. So the wake takes the same handoff as every other durable pre-stream failure + // (see the goal-sync catch below): arm the in-session resume, consume the lease, schedule + // the retry. The resume is what keeps the row reachable — startup recovery only resumes an + // interrupted history *tail*, and a competing manual send (one cause of this refusal) could + // otherwise complete a newer turn on top of the buried request. The refusal still stands. + // + // An "unknown" readback must NOT consume: if the row was in fact deleted, consuming would + // leave neither carrier nor signal and the wake (and any delegated continuation deferred + // behind it) would be lost. Releasing risks at worst a duplicate delivery, which the + // reconciler's watermarks and the wake-debt settlement already tolerate. const refuseAfterCompactionRow = async (message: string): Promise> => { - const rolledBack = await rollbackPersistedTurnRows(); - if (!rolledBack && typedMuxMetadata?.type === "bash-monitor-wake") { + const outcome = await rollbackPersistedTurnRows(); + if (outcome === "remains" && typedMuxMetadata?.type === "bash-monitor-wake") { + this.setAutoRetryResumeState(optionsForStream, agentInitiated, goalKind, internal?.goalId); await internal?.onAccepted?.(); + await this.handleStreamFailureForAutoRetry({ type: "unknown", message }); } return Err(createUnknownSendMessageError(message)); }; @@ -4080,13 +4093,14 @@ export class AgentSession { // rollback is forbidden by design (goal sync observes the durable row), so a Stop landing in // the remaining pre-stream awaits refuses the turn at the PREPARING gate with rows retained. if (internal?.admissionStale?.() === true) { - const rolledBack = await rollbackPersistedTurnRows(); + const rollbackOutcome = await rollbackPersistedTurnRows(); // Probe-carrying sends are peer messages whose caller already returned success when the // entry was queued — the cancellation hook is their only way to observe this refusal and // release the budget reservation (the refund closure is idempotent). Fire it ONLY when the - // rollback verifiably committed: rows that remain durable can enter provider context after - // a resume, so their charge must stand (budget charged ⇔ rows durable). - if (rolledBack) { + // rollback verifiably committed: rows that remain durable (or whose fate is unknown) can + // enter provider context after a resume, so their charge must stand (budget charged ⇔ + // rows durable). + if (rollbackOutcome === "deleted") { await internal?.onCanceled?.( "Send refused: the caller's admission became stale before the turn was accepted." ); From 91c35e31eb11aade2d83f76fd13229a525448763 Mon Sep 17 00:00:00 2001 From: Ethan Dickson Date: Fri, 4 Sep 2026 13:26:57 +0000 Subject: [PATCH 23/26] Scan history backward without a depth cap when recovering the last delivered wake row --- src/node/services/workspaceService.test.ts | 21 +++++++++- src/node/services/workspaceService.ts | 45 ++++++++++++---------- 2 files changed, 45 insertions(+), 21 deletions(-) diff --git a/src/node/services/workspaceService.test.ts b/src/node/services/workspaceService.test.ts index 4ef50c0912..ef36a3d86d 100644 --- a/src/node/services/workspaceService.test.ts +++ b/src/node/services/workspaceService.test.ts @@ -306,8 +306,27 @@ describe("WorkspaceService bash monitor wake reconciler wiring", () => { ); expect(await internal.readLastBashMonitorWakeRecords(workspaceId)).toEqual([wakeRecord]); + // An acknowledgment that kept failing while the accepted wake turn ran leaves the row + // behind however many rows that turn produced; a fixed tail depth would miss it and the + // restarted reconciler would redeliver the output. Bury the row deep and cross a + // compaction boundary on the way. + await historyService.appendToHistory( + workspaceId, + createMuxMessage("boundary", "assistant", "compacted summary", { + timestamp: 1_200, + compacted: "user", + }) + ); + for (let i = 0; i < 300; i++) { + await historyService.appendToHistory( + workspaceId, + createMuxMessage(`tool-step-${i}`, "assistant", `step ${i}`, { timestamp: 2_000 + i }) + ); + } + expect(await internal.readLastBashMonitorWakeRecords(workspaceId)).toEqual([wakeRecord]); + // "Could not read" is not "no row": the reconciler consults this once per owner. - const readSpy = spyOn(historyService, "getLastMessages").mockImplementationOnce(() => + const readSpy = spyOn(historyService, "iterateFullHistory").mockImplementationOnce(() => Promise.resolve(Err("history unavailable")) ); try { diff --git a/src/node/services/workspaceService.ts b/src/node/services/workspaceService.ts index fde303f010..9727ee3c47 100644 --- a/src/node/services/workspaceService.ts +++ b/src/node/services/workspaceService.ts @@ -376,13 +376,6 @@ const ORPHAN_SESSION_DIR_GRACE_MS = 24 * 60 * 60 * 1000; // Upper bound on startup .code-workspace reconciliation (see initialize()). const STARTUP_CODE_WORKSPACE_SYNC_TIMEOUT_MS = 10_000; -/** - * How far back readLastBashMonitorWakeRecords looks for the last durable wake row. A commit - * lost to a restart leaves that row at the tail; anything older had a running process (and - * its in-memory acknowledgment retries) behind it. - */ -const LAST_BASH_MONITOR_WAKE_ROW_SCAN_DEPTH = 50; - /** * Base name used when /new auto-generates a branch name. Numbered suffixes * (`workspace-1`, `workspace-2`, ...) come from {@link generateForkBranchName} @@ -12078,29 +12071,41 @@ export class WorkspaceService extends EventEmitter implements WorkspaceHost { */ /** * Records of the most recent durable bash-monitor wake row (BashMonitorWakeReconciler - * readDeliveredWakeRecords). Only the history tail is scanned: the row this recovers was - * committed right before a restart, so it is at or near the end. + * readDeliveredWakeRecords). Scans backward from the tail and stops at the first wake row: + * usually that row is at or near the end (committed right before the restart), but the state + * this repairs — an acknowledgment that kept failing while the accepted wake turn ran — can + * push it behind an arbitrarily long tool-heavy turn, so no fixed depth is safe. The scan + * also crosses compaction boundaries: the wake stays acknowledged by its row wherever the + * row sits, and a summary over it must not turn into a redelivery. */ private async readLastBashMonitorWakeRecords( ownerWorkspaceId: string ): Promise { - const tail = await this.historyService.getLastMessages( + let records: readonly BashMonitorWakeDisplayRecord[] | undefined; + const iterateResult = await this.historyService.iterateFullHistory( ownerWorkspaceId, - LAST_BASH_MONITOR_WAKE_ROW_SCAN_DEPTH + "backward", + (messages) => { + // Chunks arrive newest-first, as do the rows within a chunk. + for (const message of messages) { + // A wake diverted through on-send compaction is durable as the compaction row that + // carries it as follow-up; that row is the acknowledgment too. + const wake = getCarriedBashMonitorWake(message.metadata?.muxMetadata); + if (wake != null) { + records = wake.records; + return false; + } + } + return true; + } ); // Distinguish "no row" from "could not read": the reconciler recovers once per owner, so a // read failure swallowed here would let the reconcile dispatch a duplicate of a wake the // row already delivered. Throwing fails this reconcile; its retry reads again. - if (!tail.success) { - throw new Error(`Failed to read the last bash-monitor wake row: ${tail.error}`); - } - for (const message of tail.data.toReversed()) { - // A wake diverted through on-send compaction is durable as the compaction row that - // carries it as follow-up; that row is the acknowledgment too. - const wake = getCarriedBashMonitorWake(message.metadata?.muxMetadata); - if (wake != null) return wake.records; + if (!iterateResult.success) { + throw new Error(`Failed to read the last bash-monitor wake row: ${iterateResult.error}`); } - return undefined; + return records; } hasPendingWorkspaceTurnContinuation( From 4f5a05df2fdddeb31b6648aa7499757d3e88818d Mon Sep 17 00:00:00 2001 From: Ethan Dickson Date: Fri, 4 Sep 2026 13:38:15 +0000 Subject: [PATCH 24/26] Bound the delivered-wake-row scan at the oldest outstanding monitor's arm time and skip rows without usable identities --- .../bashMonitorWakeReconciler.test.ts | 17 ++++-- .../services/bashMonitorWakeReconciler.ts | 13 ++++- src/node/services/workspaceService.test.ts | 57 +++++++++++++++++-- src/node/services/workspaceService.ts | 55 ++++++++++++++---- 4 files changed, 118 insertions(+), 24 deletions(-) diff --git a/src/node/services/bashMonitorWakeReconciler.test.ts b/src/node/services/bashMonitorWakeReconciler.test.ts index 70b045415c..a83e563d13 100644 --- a/src/node/services/bashMonitorWakeReconciler.test.ts +++ b/src/node/services/bashMonitorWakeReconciler.test.ts @@ -329,7 +329,9 @@ describe("BashMonitorWakeReconciler", () => { const deliveredRecords = dispatches[0].muxMetadata.records; const restart = ( - readDeliveredWakeRecords: (() => Promise) | undefined + readDeliveredWakeRecords: + | ((ownerWorkspaceId: string, notBefore: string) => Promise) + | undefined ) => { const restarted: BashMonitorWakeDispatch[] = []; const instance = new BashMonitorWakeReconciler({ @@ -368,11 +370,13 @@ describe("BashMonitorWakeReconciler", () => { // A read that cannot answer fails the reconcile (retried) instead of counting as "no row": // recovery is consulted once per owner, so a swallowed failure would dispatch a duplicate. let readFails = true; - const recovered = restart(() => - readFails + const readBounds: string[] = []; + const recovered = restart((_ownerWorkspaceId, notBefore) => { + readBounds.push(notBefore); + return readFails ? Promise.reject(new Error("history unavailable")) - : Promise.resolve(deliveredRecords) - ); + : Promise.resolve(deliveredRecords); + }); acknowledged = []; const failedReconcile = await recovered.instance.reconcile(OWNER).then( () => null, @@ -386,6 +390,9 @@ describe("BashMonitorWakeReconciler", () => { expect(recovered.restarted).toHaveLength(0); expect(acknowledged).toEqual([{ processId: "proc", matchedThroughOffset: 12 }]); expect(await recovered.instance.hasOutstandingWake(OWNER)).toBe(false); + // The reader is told how far back a row could possibly acknowledge these signals: the + // arm time of the oldest outstanding monitor. + expect(readBounds).toEqual([CREATED_AT, CREATED_AT]); // The watermark is durable now: a later read does not consult the row again, and a // newer match still wakes. diff --git a/src/node/services/bashMonitorWakeReconciler.ts b/src/node/services/bashMonitorWakeReconciler.ts index 0b58ea6233..09bb01e5fb 100644 --- a/src/node/services/bashMonitorWakeReconciler.ts +++ b/src/node/services/bashMonitorWakeReconciler.ts @@ -434,9 +434,14 @@ export class BashMonitorWakeReconciler { * per owner, the first time signals derive outstanding in this process. A read that * cannot answer must throw (not return undefined): the reconcile fails and retries, so * "no row" is only ever concluded from a successful read. + * + * `notBefore` (ISO) is the arm time of the oldest outstanding monitor: a row appended + * before any of these monitors existed cannot acknowledge them, so the reader may stop + * its backward scan there instead of parsing the whole transcript. */ readDeliveredWakeRecords?( - ownerWorkspaceId: string + ownerWorkspaceId: string, + notBefore: string ): Promise; } ) {} @@ -800,7 +805,11 @@ export class BashMonitorWakeReconciler { // Signals the durable wake row already delivers are consumed, not re-dispatched. The // watermark advance is written here (not left to the caller): level reads do not // persist autoConsumed, and a recovery that only held in memory would be lost again. - const delivered = await this.args.readDeliveredWakeRecords?.(ownerWorkspaceId); + // ISO timestamps order lexicographically. + const notBefore = signals + .map((signal) => signal.createdAt) + .reduce((oldest, createdAt) => (createdAt < oldest ? createdAt : oldest)); + const delivered = await this.args.readDeliveredWakeRecords?.(ownerWorkspaceId, notBefore); const deliveredKeys = new Set( (delivered ?? []).map((record) => record.processId + "\u0000" + record.wakeUpdatedAt) ); diff --git a/src/node/services/workspaceService.test.ts b/src/node/services/workspaceService.test.ts index ef36a3d86d..b5eb4bee7f 100644 --- a/src/node/services/workspaceService.test.ts +++ b/src/node/services/workspaceService.test.ts @@ -54,7 +54,7 @@ import type { DesktopSessionManager } from "@/node/services/desktop/DesktopSessi import type { WorktreeArchiveSnapshot } from "@/common/schemas/project"; import type { BashToolResult } from "@/common/types/tools"; import type { SendMessageOptions, WorkspaceChatMessage } from "@/common/orpc/types"; -import { createMuxMessage } from "@/common/types/message"; +import { createMuxMessage, type MuxMessageMetadata } from "@/common/types/message"; import { buildStagedAttachmentNotice } from "@/browser/features/ChatInput/stagedAttachments"; import { WORKFLOW_RESULT_METADATA_TYPE, @@ -267,9 +267,12 @@ describe("WorkspaceService bash monitor wake reconciler wiring", () => { const workspaceId = "compaction-carried-wake-owner"; const internal = service as unknown as { readLastBashMonitorWakeRecords( - ownerWorkspaceId: string + ownerWorkspaceId: string, + notBefore: string ): Promise | undefined>; }; + // History rows below are stamped 1_000..2_300 ms; a bound before them scans everything. + const beforeAll = new Date(0).toISOString(); const wakeRecord = { processId: "proc", wakeUpdatedAt: "2026-08-31T12:00:00.000Z", @@ -279,7 +282,7 @@ describe("WorkspaceService bash monitor wake reconciler wiring", () => { filterExclude: false, }; try { - expect(await internal.readLastBashMonitorWakeRecords(workspaceId)).toBeUndefined(); + expect(await internal.readLastBashMonitorWakeRecords(workspaceId, beforeAll)).toBeUndefined(); await historyService.appendToHistory( workspaceId, createMuxMessage("wake-compaction", "user", "Compacting to continue", { @@ -304,7 +307,9 @@ describe("WorkspaceService bash monitor wake reconciler wiring", () => { workspaceId, createMuxMessage("summary", "assistant", "summary", { timestamp: 1_100 }) ); - expect(await internal.readLastBashMonitorWakeRecords(workspaceId)).toEqual([wakeRecord]); + expect(await internal.readLastBashMonitorWakeRecords(workspaceId, beforeAll)).toEqual([ + wakeRecord, + ]); // An acknowledgment that kept failing while the accepted wake turn ran leaves the row // behind however many rows that turn produced; a fixed tail depth would miss it and the @@ -323,14 +328,54 @@ describe("WorkspaceService bash monitor wake reconciler wiring", () => { createMuxMessage(`tool-step-${i}`, "assistant", `step ${i}`, { timestamp: 2_000 + i }) ); } - expect(await internal.readLastBashMonitorWakeRecords(workspaceId)).toEqual([wakeRecord]); + expect(await internal.readLastBashMonitorWakeRecords(workspaceId, beforeAll)).toEqual([ + wakeRecord, + ]); + + // A newer wake row whose persisted `records` is not usable (corrupt shape, or a legacy + // row without identities) is skipped rather than returned: the reconciler maps over the + // result, so returning it would fail every reconcile retry and strand current wakes. + await historyService.appendToHistory( + workspaceId, + createMuxMessage("wake-corrupt", "user", "Background monitor wake", { + timestamp: 2_300, + synthetic: true, + muxMetadata: { type: "bash-monitor-wake", records: "corrupt" } as unknown as Extract< + MuxMessageMetadata, + { type: "bash-monitor-wake" } + >, + }) + ); + await historyService.appendToHistory( + workspaceId, + createMuxMessage("wake-legacy", "user", "Background monitor wake", { + timestamp: 2_301, + synthetic: true, + muxMetadata: { + type: "bash-monitor-wake", + records: [{ kind: "match", displayName: "run", filter: "READY", filterExclude: false }], + }, + }) + ); + expect(await internal.readLastBashMonitorWakeRecords(workspaceId, beforeAll)).toEqual([ + wakeRecord, + ]); + + // The scan is bounded by the oldest outstanding monitor's arm time: rows appended before + // that monitor existed cannot acknowledge it, so they are never parsed (an owner's first + // wake would otherwise read the entire transcript on the stream's tool-boundary path). + // The bound carries a clock-step margin, so place it well past the wake row. + const armedAfterWakeRow = new Date(1_000 + 60_000 + 1_000_000).toISOString(); + expect( + await internal.readLastBashMonitorWakeRecords(workspaceId, armedAfterWakeRow) + ).toBeUndefined(); // "Could not read" is not "no row": the reconciler consults this once per owner. const readSpy = spyOn(historyService, "iterateFullHistory").mockImplementationOnce(() => Promise.resolve(Err("history unavailable")) ); try { - const failed = await internal.readLastBashMonitorWakeRecords(workspaceId).then( + const failed = await internal.readLastBashMonitorWakeRecords(workspaceId, beforeAll).then( () => null, (error: unknown) => error ); diff --git a/src/node/services/workspaceService.ts b/src/node/services/workspaceService.ts index 9727ee3c47..c664b92018 100644 --- a/src/node/services/workspaceService.ts +++ b/src/node/services/workspaceService.ts @@ -376,6 +376,11 @@ const ORPHAN_SESSION_DIR_GRACE_MS = 24 * 60 * 60 * 1000; // Upper bound on startup .code-workspace reconciliation (see initialize()). const STARTUP_CODE_WORKSPACE_SYNC_TIMEOUT_MS = 10_000; +/** + * Slack subtracted from the "not before" bound of readLastBashMonitorWakeRecords so a wall-clock + * step between a monitor's arm stamp and its wake row's append stamp cannot hide the row. + */ +const BASH_MONITOR_WAKE_ROW_SCAN_CLOCK_MARGIN_MS = 60_000; /** * Base name used when /new auto-generates a branch name. Numbered suffixes * (`workspace-1`, `workspace-2`, ...) come from {@link generateForkBranchName} @@ -2391,8 +2396,8 @@ export class WorkspaceService extends EventEmitter implements WorkspaceHost { }, registry: this.bashMonitorRegistryStore, onWake: (dispatch) => this.dispatchBashMonitorWake(dispatch), - readDeliveredWakeRecords: (ownerWorkspaceId) => - this.readLastBashMonitorWakeRecords(ownerWorkspaceId), + readDeliveredWakeRecords: (ownerWorkspaceId, notBefore) => + this.readLastBashMonitorWakeRecords(ownerWorkspaceId, notBefore), onOutstandingChanged: (ownerWorkspaceId, outstanding) => this.publishBashMonitorWakeLevel(ownerWorkspaceId, outstanding), }); @@ -12071,16 +12076,29 @@ export class WorkspaceService extends EventEmitter implements WorkspaceHost { */ /** * Records of the most recent durable bash-monitor wake row (BashMonitorWakeReconciler - * readDeliveredWakeRecords). Scans backward from the tail and stops at the first wake row: - * usually that row is at or near the end (committed right before the restart), but the state - * this repairs — an acknowledgment that kept failing while the accepted wake turn ran — can - * push it behind an arbitrarily long tool-heavy turn, so no fixed depth is safe. The scan - * also crosses compaction boundaries: the wake stays acknowledged by its row wherever the - * row sits, and a summary over it must not turn into a redelivery. + * readDeliveredWakeRecords). Scans backward from the tail and stops at the first wake row + * with usable record identities, or at the first row older than `notBefore`. + * + * No fixed depth is safe: the state this repairs — an acknowledgment that kept failing while + * the accepted wake turn ran — can push the row behind an arbitrarily long tool-heavy turn + * (and behind a compaction boundary; the wake stays acknowledged by its row wherever it + * sits). No unbounded scan is acceptable either: an owner's first wake has no row to find, + * and this read sits on the stream's tool-boundary predicate under the history lock. The + * arm time of the oldest outstanding monitor is the durable exclusion point — a row appended + * before that monitor existed cannot acknowledge it — so the cost is the rows since arming, + * usually one tail chunk. */ private async readLastBashMonitorWakeRecords( - ownerWorkspaceId: string + ownerWorkspaceId: string, + notBefore: string ): Promise { + // Registry and history stamps come from the same wall clock; the margin absorbs a clock + // step between arming and the row's append. An unparseable bound (never produced by the + // registry) degrades to an unbounded scan rather than to a silent redelivery. + const parsedNotBefore = Date.parse(notBefore); + const cutoffMs = Number.isFinite(parsedNotBefore) + ? parsedNotBefore - BASH_MONITOR_WAKE_ROW_SCAN_CLOCK_MARGIN_MS + : Number.NEGATIVE_INFINITY; let records: readonly BashMonitorWakeDisplayRecord[] | undefined; const iterateResult = await this.historyService.iterateFullHistory( ownerWorkspaceId, @@ -12088,11 +12106,26 @@ export class WorkspaceService extends EventEmitter implements WorkspaceHost { (messages) => { // Chunks arrive newest-first, as do the rows within a chunk. for (const message of messages) { + const timestamp = message.metadata?.timestamp; + if (typeof timestamp === "number" && timestamp < cutoffMs) return false; // A wake diverted through on-send compaction is durable as the compaction row that // carries it as follow-up; that row is the acknowledgment too. const wake = getCarriedBashMonitorWake(message.metadata?.muxMetadata); - if (wake != null) { - records = wake.records; + if (wake == null) continue; + // Persisted rows are data, not trusted structure: one malformed `records` (or a legacy + // row without identities) must not stop the scan and brick recovery for every + // reconcile retry — the reconciler maps over what this returns. + const usable = Array.isArray(wake.records) + ? wake.records.filter( + (record): record is BashMonitorWakeDisplayRecord => + typeof record === "object" && + record !== null && + typeof record.processId === "string" && + typeof record.wakeUpdatedAt === "string" + ) + : []; + if (usable.length > 0) { + records = usable; return false; } } From a11a718fe9401cc6159c7b9276f6d9fa76d7713a Mon Sep 17 00:00:00 2001 From: Ethan Dickson Date: Fri, 4 Sep 2026 13:49:02 +0000 Subject: [PATCH 25/26] fix: skip RLM preserved-tail copies in the wake acknowledgment scan Keep-recent compaction re-appends copies of the pre-boundary tail after the boundary with their source timestamps, so an old-stamped copy at the tail tripped the notBefore cutoff before the scan reached the wake row, and a restart after a failed acknowledgment redelivered the wake. Copies carry nothing their originals lack; skip them before the cutoff and before reading wake metadata. Made-with: Xum --- src/node/services/workspaceService.test.ts | 18 ++++++++++++++++++ src/node/services/workspaceService.ts | 6 ++++++ 2 files changed, 24 insertions(+) diff --git a/src/node/services/workspaceService.test.ts b/src/node/services/workspaceService.test.ts index b5eb4bee7f..fdb862666a 100644 --- a/src/node/services/workspaceService.test.ts +++ b/src/node/services/workspaceService.test.ts @@ -361,6 +361,24 @@ describe("WorkspaceService bash monitor wake reconciler wiring", () => { wakeRecord, ]); + // RLM keep-recent compaction re-appends copies of the pre-boundary tail after the + // boundary with their *source* timestamps. Sitting at the tail, an old-stamped copy is + // reached before the wake row; if it tripped the cutoff the scan would stop short and + // the restarted reconciler would redeliver an acknowledged wake. Bound so the copy reads + // as too old while the wake row does not. + await historyService.appendToHistory( + workspaceId, + createMuxMessage("tail-copy", "user", "old turn copy", { + timestamp: 500, + synthetic: true, + rlmPreservedTailCopy: true, + }) + ); + const armedAtWakeRow = new Date(1_000 + 60_000).toISOString(); + expect(await internal.readLastBashMonitorWakeRecords(workspaceId, armedAtWakeRow)).toEqual([ + wakeRecord, + ]); + // The scan is bounded by the oldest outstanding monitor's arm time: rows appended before // that monitor existed cannot acknowledge it, so they are never parsed (an owner's first // wake would otherwise read the entire transcript on the stream's tool-boundary path). diff --git a/src/node/services/workspaceService.ts b/src/node/services/workspaceService.ts index c664b92018..af3e6b27cb 100644 --- a/src/node/services/workspaceService.ts +++ b/src/node/services/workspaceService.ts @@ -12106,6 +12106,12 @@ export class WorkspaceService extends EventEmitter implements WorkspaceHost { (messages) => { // Chunks arrive newest-first, as do the rows within a chunk. for (const message of messages) { + // RLM keep-recent compaction re-appends copies of the pre-boundary tail *after* the + // boundary while keeping their source timestamps, so a copy can read as older than + // the cutoff while sitting above rows that are newer. Copies carry nothing their + // originals (still in history) lack: skip them before the cutoff and before reading + // wake metadata, or a stale-stamped copy ends the scan short of the wake row. + if (message.metadata?.rlmPreservedTailCopy === true) continue; const timestamp = message.metadata?.timestamp; if (typeof timestamp === "number" && timestamp < cutoffMs) return false; // A wake diverted through on-send compaction is durable as the compaction row that From 1a71e62fc17ba9d6a082ca7ae0440063751ab401 Mon Sep 17 00:00:00 2001 From: Ethan Dickson Date: Fri, 4 Sep 2026 14:07:52 +0000 Subject: [PATCH 26/26] fix: withdraw stale wake offers and void follow-ups only after a durable erase Reconciler: an offer is a claim on signals that derive now. A pass that ran between a monitor's cancel and its registry-row removal offered a monitor-lost wake for the canceled monitor and kept it current after the row was gone; the pass now withdraws an offer whose signals no longer all derive outstanding, and discardProcess no longer schedules the mid-removal pass itself (the caller schedules after the removal). AgentSession: clearPendingFollowUpFromSummary settled the delegated turn before the follow-up erase was durable, so a failed rewrite left a live, dispatchable follow-up under an interrupted handle. The void now follows a successful rewrite, with a readback that recognises a rewrite that landed but reported failure. Pins the empty-queue drain keeping the wake's tool-end yield flag up. Made-with: Xum --- ...gentSession.continueMessageAgentId.test.ts | 71 ++++++++++++++++++- .../agentSession.queueDispatch.test.ts | 9 +++ src/node/services/agentSession.ts | 47 ++++++++---- .../bashMonitorWakeReconciler.test.ts | 28 ++++++++ .../services/bashMonitorWakeReconciler.ts | 32 ++++++--- 5 files changed, 165 insertions(+), 22 deletions(-) diff --git a/src/node/services/agentSession.continueMessageAgentId.test.ts b/src/node/services/agentSession.continueMessageAgentId.test.ts index 48ed93c2d5..19933cad5a 100644 --- a/src/node/services/agentSession.continueMessageAgentId.test.ts +++ b/src/node/services/agentSession.continueMessageAgentId.test.ts @@ -1,4 +1,4 @@ -import { afterEach, describe, expect, mock, test } from "bun:test"; +import { afterEach, describe, expect, mock, spyOn, test } from "bun:test"; import { createMuxMessage } from "@/common/types/message"; import type { CompactionFollowUpRequest, MuxMessage } from "@/common/types/message"; import type { FilePart, SendMessageOptions } from "@/common/orpc/types"; @@ -427,6 +427,75 @@ describe("AgentSession continue-message agentId fallback", () => { expect(abandoned).toHaveBeenCalledTimes(2); }); + test("abandoning a follow-up settles its delegated turn only once the erase is durable", async () => { + // While the summary still carries the follow-up, startup recovery or a retry can dispatch + // it; settling the delegated turn first would interrupt the handle under a continuation + // that is still live. A failed rewrite therefore keeps the turn unsettled — unless the + // rewrite actually landed and only its result was lost, which a readback recognises. + const workspaceTurnMetadata = { + type: "workspace-turn-task", + taskHandleId: "wst_durable", + ownerWorkspaceId: "parent-durable", + turnId: "turn-durable", + } as const; + const wakeFollowUp: CompactionFollowUpRequest = { + text: "monitor matched", + model: "openai:gpt-4o", + agentId: "exec", + muxMetadata: { type: "bash-monitor-wake", records: [] }, + workspaceTurnMetadata, + dispatchOptions: { requireIdle: true }, + }; + const abandoned = mock( + ( + _metadata: NonNullable, + _reason: string + ) => Promise.resolve() + ); + const { session, historyService, internals } = await createSession( + [compactionSummaryMessage("summary-durable", wakeFollowUp)], + { onWorkspaceTurnContinuationVoided: abandoned } + ); + internals.sendMessage = mock(() => Promise.resolve({ success: true as const })); + (session as unknown as { hasExternalSendPreflight?: () => boolean }).hasExternalSendPreflight = + () => true; + + // Rewrite genuinely fails: the follow-up stays durable and the turn stays unsettled. + const realUpdate = historyService.updateHistory.bind(historyService); + const updateSpy = spyOn(historyService, "updateHistory").mockImplementationOnce(() => + Promise.resolve({ success: false as const, error: "disk full" }) + ); + try { + const failed = await internals.dispatchPendingFollowUp().then( + () => null, + (error: unknown) => error + ); + expect(failed).toBeInstanceOf(Error); + expect(abandoned).not.toHaveBeenCalled(); + const stillPending = await historyService.getLastMessages("ws", 1); + expect(stillPending.success && stillPending.data[0]?.metadata?.muxMetadata).toMatchObject({ + type: "compaction-summary", + pendingFollowUp: { text: "monitor matched" }, + }); + + // Rewrite landed but reported failure: the readback finds the follow-up gone, so the + // delegated turn is settled rather than stranded behind a follow-up nothing can dispatch. + updateSpy.mockImplementationOnce(async (workspaceId, message) => { + await realUpdate(workspaceId, message); + return { success: false as const, error: "result lost" }; + }); + expect(await internals.dispatchPendingFollowUp()).toBe(false); + expect(abandoned).toHaveBeenCalledTimes(1); + expect(abandoned).toHaveBeenLastCalledWith(workspaceTurnMetadata, "abandoned"); + const cleared = await historyService.getLastMessages("ws", 1); + expect(cleared.success && cleared.data[0]?.metadata?.muxMetadata).toEqual({ + type: "compaction-summary", + }); + } finally { + updateSpy.mockRestore(); + } + }); + test("dispatchPendingFollowUp removes heartbeat reset boundaries when idle-only follow-ups are skipped", async () => { const earlierMessage = createMuxMessage("before-reset", "assistant", "Earlier context"); const { session, historyService, internals } = await createSession([ diff --git a/src/node/services/agentSession.queueDispatch.test.ts b/src/node/services/agentSession.queueDispatch.test.ts index 8eee1c9683..a1b38bef3f 100644 --- a/src/node/services/agentSession.queueDispatch.test.ts +++ b/src/node/services/agentSession.queueDispatch.test.ts @@ -1198,6 +1198,15 @@ describe("AgentSession queued message tool-call dispatch", () => { session.setBashMonitorWakeOutstanding(false); expect(lastFlag()).toBe(false); + // A stream ending onto an empty queue while the level is high (the drain + // sendQueuedMessages runs at stream end) keeps the flag up: the wake still asks the + // next stream — a racing manual send's — to yield at its first tool boundary. + session.setBashMonitorWakeOutstanding(true); + session.sendQueuedMessages(); + expect(lastFlag()).toBe(true); + session.setBashMonitorWakeOutstanding(false); + expect(lastFlag()).toBe(false); + // A turn-end head owns the next dispatch: the level must not pull the early-return // lever for it (mirrors hasPendingToolEndInput's arbitration). session.setBashMonitorWakeOutstanding(true); diff --git a/src/node/services/agentSession.ts b/src/node/services/agentSession.ts index a444e2ddf8..3f8aa53be5 100644 --- a/src/node/services/agentSession.ts +++ b/src/node/services/agentSession.ts @@ -7966,6 +7966,23 @@ export class AgentSession { return; } + const { pendingFollowUp: _pendingFollowUp, ...muxMetadataWithoutFollowUp } = muxMeta; + const updateResult = await this.historyService.updateHistory(this.workspaceId, { + ...summaryMessage, + metadata: { + ...(summaryMessage.metadata ?? {}), + muxMetadata: muxMetadataWithoutFollowUp, + }, + }); + // The erase must be durable before the delegated turn it carried is settled: while the + // summary still holds the follow-up, startup recovery or a retry can dispatch it, and a + // settlement issued now would interrupt the handle (and remove a disposable workspace) + // under a continuation that is still live. A failed write is re-checked on disk so a + // rewrite that landed but reported failure does not strand the turn unsettled. + if (!updateResult.success && !(await this.isPendingFollowUpCleared(summaryMessage))) { + throw new Error(`Failed to clear skipped pending follow-up: ${updateResult.error}`); + } + // Every discard path funnels here, so this is the one place that knows the delegated // turn's continuation is gone for good. The void is a synchronous state transition here // and the owner-side settlement runs as a tracked, retried promise (see @@ -7973,8 +7990,7 @@ export class AgentSession { // TaskService stream-end listener holds the workspace event lock waiting for the // compaction completion decision, and the owner settles under that same lock. A wake // follow-up also carried the continuation debt of the stream it cut; the same void clears - // it. Whether the erase below succeeds does not affect the settlement: the void carries - // the correlation itself. + // it. const workspaceTurnMetadata = muxMeta.pendingFollowUp.workspaceTurnMetadata; if (workspaceTurnMetadata != null) { if ( @@ -7990,18 +8006,23 @@ export class AgentSession { reason: "abandoned", }); } + } - const { pendingFollowUp: _pendingFollowUp, ...muxMetadataWithoutFollowUp } = muxMeta; - const updateResult = await this.historyService.updateHistory(this.workspaceId, { - ...summaryMessage, - metadata: { - ...(summaryMessage.metadata ?? {}), - muxMetadata: muxMetadataWithoutFollowUp, - }, - }); - if (!updateResult.success) { - throw new Error(`Failed to clear skipped pending follow-up: ${updateResult.error}`); - } + /** + * Whether the durable copy of a compaction summary no longer carries a pending follow-up. + * Only a successful read that finds the row without one counts; an unreadable or missing row + * is "unknown", which callers treat as not cleared. + */ + private async isPendingFollowUpCleared(summaryMessage: MuxMessage): Promise { + const historyResult = await this.historyService.getHistoryFromLatestBoundary(this.workspaceId); + if (!historyResult.success) return false; + const durable = historyResult.data.find((message) => message.id === summaryMessage.id); + const durableMuxMeta = durable?.metadata?.muxMetadata; + return ( + durable != null && + isCompactionSummaryMetadata(durableMuxMeta) && + durableMuxMeta.pendingFollowUp == null + ); } /** diff --git a/src/node/services/bashMonitorWakeReconciler.test.ts b/src/node/services/bashMonitorWakeReconciler.test.ts index a83e563d13..c52d81b979 100644 --- a/src/node/services/bashMonitorWakeReconciler.test.ts +++ b/src/node/services/bashMonitorWakeReconciler.test.ts @@ -207,6 +207,34 @@ describe("BashMonitorWakeReconciler", () => { expect(dispatches[1].isCurrent()).toBe(true); }); + test("a wake offered from a registry row mid-removal is withdrawn once the row is gone", async () => { + // Cancel path: the owner discards the process, then removes its registry row. A pass that + // lands between the two sees a row without a process and offers a monitor-lost wake for + // the canceled monitor. The post-removal pass derives nothing for it, and that must + // retire the offer — otherwise it stays current and the canceled monitor starts a turn. + rows = [registryRecord()]; + await reconciler.reconcile(OWNER); + expect(dispatches).toHaveLength(1); + expect(dispatches[0].muxMetadata.records[0]).toMatchObject({ + processId: "dead", + kind: "monitor-lost", + }); + expect(dispatches[0].isCurrent()).toBe(true); + + rows = []; + await reconciler.reconcile(OWNER); + expect(dispatches[0].isCurrent()).toBe(false); + expect(dispatches).toHaveLength(1); + expect(await reconciler.hasOutstandingWake(OWNER)).toBe(false); + + // The cancel itself schedules no pass: only the caller's post-removal schedule does, so + // no intermediate offer is manufactured for a row the caller is about to delete. + rows = [registryRecord()]; + await reconciler.discardProcess(OWNER, "dead", CREATED_AT); + await new Promise((resolve) => setTimeout(resolve, 0)); + expect(dispatches).toHaveLength(1); + }); + test("a shown-frontier advance retires a handed-out wake so a stale send is refused", async () => { // The owner ran a manual turn that task_await-ed the monitored process while this wake // was still resolving send options: the reconcile that would re-derive it is queued diff --git a/src/node/services/bashMonitorWakeReconciler.ts b/src/node/services/bashMonitorWakeReconciler.ts index 09bb01e5fb..f6446b0475 100644 --- a/src/node/services/bashMonitorWakeReconciler.ts +++ b/src/node/services/bashMonitorWakeReconciler.ts @@ -173,9 +173,11 @@ interface DerivedSignal { * * offered ──release──▶ released the owner did not send it: `onDeferred`, `onWake` * threw, or the signals were withdrawn under it (cancel, - * shown frontier, full-history clear). `isCurrent()` turns - * false so the owner drops it at its next admission gate; - * whatever still derives is re-leased by the next reconcile. + * shown frontier, full-history clear, or a reconcile pass + * under which they no longer all derive). `isCurrent()` + * turns false so the owner drops it at its next admission + * gate; whatever still derives is re-leased by the next + * reconcile. * offered ──commit───▶ committed the owner's prompt row is durable (`onAccepted`). The * released ─commit───▶ committed signals are consumed from here on regardless of what * happened to the offer meanwhile (a release can land in @@ -508,14 +510,16 @@ export class BashMonitorWakeReconciler { /** * The operator canceled a monitor: a wake already handed to the owner that carries this * process's output must not be sent (its isCurrent() turns false). Its other signals, if - * any, re-derive on the next reconcile. + * any, re-derive on the next reconcile — which the caller schedules after it removes the + * registry row, not here: a pass between the cancel and that removal would see a row + * without a process and offer a monitor-lost wake for the canceled monitor. */ async discardProcess( ownerWorkspaceId: string, processId: string, createdAt: string ): Promise { - await this.forgetDispatchFor( + await this.releaseOfferedCovering( ownerWorkspaceId, (signal) => signal.processId === processId && signal.createdAt === createdAt ); @@ -529,10 +533,11 @@ export class BashMonitorWakeReconciler { * and let the reconcile scheduled here re-lease whatever still derives. */ async outputShown(ownerWorkspaceId: string, processId: string): Promise { - await this.forgetDispatchFor(ownerWorkspaceId, (signal) => signal.processId === processId); + await this.releaseOfferedCovering(ownerWorkspaceId, (signal) => signal.processId === processId); + this.scheduleReconcile(ownerWorkspaceId); } - private async forgetDispatchFor( + private async releaseOfferedCovering( ownerWorkspaceId: string, covers: (signal: DerivedSignal) => boolean ): Promise { @@ -541,7 +546,6 @@ export class BashMonitorWakeReconciler { if (state?.offered?.signals.some(covers)) this.releaseOffered(state); return Promise.resolve(); }); - this.scheduleReconcile(ownerWorkspaceId); } /** Caller holds the owner lock. */ @@ -633,6 +637,18 @@ export class BashMonitorWakeReconciler { await this.cleanup(collected.autoConsumed); const state = this.state(ownerWorkspaceId); + // An offer is a claim on signals that derive *now*. A pass that ran between a monitor's + // cancel and its registry-row removal sees a row without a process and offers a + // monitor-lost wake for it; once the row is gone nothing derives, yet the offer would + // stay current and the canceled monitor could still start an agent turn. Withdraw an + // offer whose signals no longer all derive outstanding (also covers a frontier advance + // whose outputShown release lost the race with this pass). + if (state.offered != null) { + const outstandingKeys = new Set(collected.signals.map((signal) => signal.key)); + if (!state.offered.signals.every((signal) => outstandingKeys.has(signal.key))) { + this.releaseOffered(state); + } + } if (collected.signals.length === 0 || state.offered != null || state.committed != null) { return undefined; }