diff --git a/docs/research/claude-code-cross-session-messaging-comparison.md b/docs/research/claude-code-cross-session-messaging-comparison.md index 2395ea38fa..d07198ac96 100644 --- a/docs/research/claude-code-cross-session-messaging-comparison.md +++ b/docs/research/claude-code-cross-session-messaging-comparison.md @@ -54,7 +54,7 @@ Mux's unit is not a terminal session bound to a socket; it is a **workspace** (w Messages to a busy workspace enter its `MessageQueue` (`src/node/services/messageQueue.ts`) and dispatch at a boundary chosen by `queueDispatchMode`: -- `tool-end`: the stream's stop conditions include `hasQueuedMessages("tool-end")`, evaluated by the AI SDK only after every sibling tool result in the current step settles (`createStopWhenCondition`, `src/node/services/streamManager.ts`); `AgentSession` soft-stops only once `activeToolCallIds` is empty. **A running tool call is never interrupted** — same guarantee as Claude Code. +- `tool-end`: the stream's stop conditions include `hasPendingToolEndInput()` (a queued tool-end message or an outstanding bash-monitor wake, read live), evaluated by the AI SDK only after every sibling tool result in the current step settles (`createStopWhenCondition`, `src/node/services/streamManager.ts`); `AgentSession` soft-stops only once `activeToolCallIds` is empty. **A running tool call is never interrupted** — same guarantee as Claude Code. - `turn-end`: dispatches after the current turn completes. - Idle target: the message starts a new turn immediately. diff --git a/src/browser/components/ProjectSidebar/ProjectSidebar.test.tsx b/src/browser/components/ProjectSidebar/ProjectSidebar.test.tsx index e5bf32afd5..ee96b62512 100644 --- a/src/browser/components/ProjectSidebar/ProjectSidebar.test.tsx +++ b/src/browser/components/ProjectSidebar/ProjectSidebar.test.tsx @@ -957,6 +957,7 @@ describe("ProjectSidebar flat chat list", () => { spyOn(WorkspaceContextModule, "useWorkspaceActions").mockImplementation( () => ({ + archivingWorkspaceIds: new Set(), selectedWorkspace: null, setSelectedWorkspace: () => undefined, preflightArchiveWorkspace: () => @@ -1008,6 +1009,7 @@ describe("ProjectSidebar flat chat list", () => { spyOn(WorkspaceContextModule, "useWorkspaceActions").mockImplementation( () => ({ + archivingWorkspaceIds: new Set(), selectedWorkspace: null, setSelectedWorkspace: () => undefined, preflightArchiveWorkspace: () => @@ -1098,6 +1100,7 @@ describe("ProjectSidebar flat chat list", () => { spyOn(WorkspaceContextModule, "useWorkspaceActions").mockImplementation( () => ({ + archivingWorkspaceIds: new Set(), selectedWorkspace: null, setSelectedWorkspace: () => undefined, preflightArchiveWorkspace: () => @@ -1278,6 +1281,7 @@ describe("ProjectSidebar flat chat list", () => { spyOn(WorkspaceContextModule, "useWorkspaceActions").mockImplementation( () => ({ + archivingWorkspaceIds: new Set(), selectedWorkspace: null, setSelectedWorkspace: () => undefined, preflightArchiveWorkspace: () => @@ -1333,6 +1337,7 @@ describe("ProjectSidebar flat chat list", () => { spyOn(WorkspaceContextModule, "useWorkspaceActions").mockImplementation( () => ({ + archivingWorkspaceIds: new Set(), selectedWorkspace: null, setSelectedWorkspace: () => undefined, preflightArchiveWorkspace: () => diff --git a/src/node/services/agentSession.autoCompaction.test.ts b/src/node/services/agentSession.autoCompaction.test.ts index 0161cb9d1e..5d9eb0c821 100644 --- a/src/node/services/agentSession.autoCompaction.test.ts +++ b/src/node/services/agentSession.autoCompaction.test.ts @@ -6,6 +6,7 @@ import { createMuxMessage, type CompactionFollowUpRequest, type MuxMessage, + type MuxMessageMetadata, } from "@/common/types/message"; import { GOAL_CONTINUATION_KIND } from "@/constants/goals"; import { Ok, Err } from "@/common/types/result"; @@ -313,6 +314,40 @@ describe("AgentSession on-send auto-compaction snapshot deferral", () => { expect(unstamped).toBeUndefined(); }); + test("bash-monitor wake follow-ups keep their idle-only admission across compaction", async () => { + // The wake was sent with requireIdle; a manual message queued during the compaction + // stream must still win over the re-dispatched continuation (dispatchPendingFollowUp). + const { session } = await createSessionHarness({ + workspaceId: "ws-auto-compaction-wake-require-idle", + }); + const build = ( + session as unknown as { + buildAutoCompactionFollowUp: (params: { + messageText: string; + options: SendMessageOptions; + modelForStream: string; + muxMetadata?: MuxMessageMetadata; + }) => CompactionFollowUpRequest; + } + ).buildAutoCompactionFollowUp.bind(session); + + const wakeFollowUp = build({ + messageText: "READY", + options: { model: "openai:gpt-4o", agentId: "exec" }, + modelForStream: "openai:gpt-4o", + muxMetadata: { type: "bash-monitor-wake", records: [] }, + }); + expect(wakeFollowUp.dispatchOptions?.requireIdle).toBe(true); + + const plainFollowUp = build({ + messageText: "hello", + options: { model: "openai:gpt-4o", agentId: "exec" }, + modelForStream: "openai:gpt-4o", + }); + expect(plainFollowUp.dispatchOptions?.requireIdle).toBeUndefined(); + session.dispose(); + }); + test("preserves goal kind and goal identity on auto-compaction follow-up requests", async () => { const { session } = await createSessionHarness({ workspaceId: "ws-auto-compaction-goal-kind", @@ -387,6 +422,186 @@ describe("AgentSession on-send auto-compaction snapshot deferral", () => { session.dispose(); }); + test("a send refused after its on-send compaction row landed rolls that row back", async () => { + // The compaction row is the one durable write that precedes the admission gates. A send + // whose admission went stale in between (a bash-monitor wake whose monitor was cancelled, + // a peer send racing a Stop) is refused without a stream — leaving the row would let + // startup recovery resume a compaction whose follow-up nobody accepted. + const workspaceId = "ws-auto-compaction-stale-admission-rollback"; + const streamMessage = mock(() => Promise.resolve(Ok(createStartedTurnHandle()))); + const { session, historyService } = await createSessionHarness({ + workspaceId, + streamMessage: streamMessage as unknown as AIService["streamMessage"], + }); + + (session as unknown as { compactionMonitor: CompactionMonitor }).compactionMonitor = { + checkBeforeSend: mock(() => ({ + shouldShowWarning: true, + shouldForceCompact: false, + usagePercentage: 72, + thresholdPercentage: 70, + })), + checkMidStream: mock(() => false), + resetForNewStream: mock(() => undefined), + setThreshold: mock(() => undefined), + getThreshold: mock(() => 0.7), + } as unknown as CompactionMonitor; + + const onAccepted = mock(() => Promise.resolve()); + const result = await session.sendMessage( + "hello", + { + model: "openai:gpt-4o", + agentId: "exec", + muxMetadata: { type: "bash-monitor-wake", records: [] }, + }, + { admissionStale: () => true, onAccepted } + ); + expect(result.success).toBe(false); + expect(streamMessage).not.toHaveBeenCalled(); + // The row is gone, so the wake lease stays released for the reconciler to re-derive. + expect(onAccepted).not.toHaveBeenCalled(); + + const historyResult = await historyService.getHistoryFromLatestBoundary(workspaceId); + expect(historyResult.success).toBe(true); + if (!historyResult.success) { + throw new Error(`failed to load history: ${String(historyResult.error)}`); + } + expect( + historyResult.data.some( + (message) => message.metadata?.muxMetadata?.type === "compaction-request" + ) + ).toBe(false); + + session.dispose(); + }); + + // Shared setup for the rollback-failure refusals below: on-send compaction lands its row, + // then the send is refused as stale and the row's deletion fails. + async function createRefusedWakeAfterFailedRollbackHarness(workspaceId: string) { + const streamMessage = mock(() => Promise.resolve(Ok(createStartedTurnHandle()))); + const { session, historyService } = await createSessionHarness({ + workspaceId, + streamMessage: streamMessage as unknown as AIService["streamMessage"], + }); + (session as unknown as { compactionMonitor: CompactionMonitor }).compactionMonitor = { + checkBeforeSend: mock(() => ({ + shouldShowWarning: true, + shouldForceCompact: false, + usagePercentage: 72, + thresholdPercentage: 70, + })), + checkMidStream: mock(() => false), + resetForNewStream: mock(() => undefined), + setThreshold: mock(() => undefined), + getThreshold: mock(() => 0.7), + } as unknown as CompactionMonitor; + const deleteMessages = spyOn(historyService, "deleteMessages").mockImplementationOnce(() => + Promise.resolve(Err("disk unavailable")) + ); + const chatEventTypes: string[] = []; + session.onChatEvent((event) => { + chatEventTypes.push(event.message.type); + }); + const readResumeRequest = () => + ( + session as unknown as { + lastAutoRetryResumeRequest?: { options: { muxMetadata?: unknown } }; + } + ).lastAutoRetryResumeRequest; + const onAccepted = mock(() => Promise.resolve()); + const sendRefusedWake = () => + session.sendMessage( + "hello", + { + model: "openai:gpt-4o", + agentId: "exec", + muxMetadata: { type: "bash-monitor-wake", records: [] }, + }, + { admissionStale: () => true, onAccepted } + ); + const hasCompactionRow = async () => { + const historyResult = await historyService.getHistoryFromLatestBoundary(workspaceId); + if (!historyResult.success) { + throw new Error(`failed to load history: ${String(historyResult.error)}`); + } + return historyResult.data.some( + (message) => message.metadata?.muxMetadata?.type === "compaction-request" + ); + }; + return { + session, + historyService, + streamMessage, + onAccepted, + chatEventTypes, + readResumeRequest, + sendRefusedWake, + hasCompactionRow, + deleteMessagesCalls: () => deleteMessages.mock.calls.length, + }; + } + + test("a refused wake whose compaction row verifiably remains consumes its lease and arms a resume", async () => { + // The durable row carries the wake as its follow-up. Releasing the lease too would have the + // reconciler deliver the same output a second time (possibly after the monitor retracted + // it), so the refusal consumes the wake — and arms the in-session resume like every other + // durable pre-stream failure, because startup recovery only resumes an interrupted history + // tail and a competing manual send could bury the request under a newer turn. + const h = await createRefusedWakeAfterFailedRollbackHarness( + "ws-auto-compaction-rollback-failure-consumes-wake" + ); + try { + let resumeArmedAtAcceptance = false; + h.onAccepted.mockImplementation(() => { + resumeArmedAtAcceptance = h.readResumeRequest() != null; + return Promise.resolve(); + }); + const result = await h.sendRefusedWake(); + expect(result.success).toBe(false); + expect(h.streamMessage).not.toHaveBeenCalled(); + expect(h.onAccepted).toHaveBeenCalledTimes(1); + expect(resumeArmedAtAcceptance).toBe(true); + // The resume replays the compaction request (which carries the wake), not a fresh row. + expect( + (h.readResumeRequest()?.options.muxMetadata as { type?: string } | undefined)?.type + ).toBe("compaction-request"); + expect(h.chatEventTypes).toContain("auto-retry-scheduled"); + expect(await h.hasCompactionRow()).toBe(true); + } finally { + await h.session.setAutoRetryEnabled(false, { persist: false }); + h.session.dispose(); + } + }); + + test("a refused wake whose rollback outcome is unknown keeps its lease released", async () => { + // deleteMessages can fail after committing; if the readback fails too the row's fate is + // unknown. Consuming the lease then could leave neither carrier nor signal (the wake and any + // delegated continuation deferred behind it would be lost), so the refusal must release + // and let the reconciler re-derive — a duplicate delivery is the tolerable failure mode. + const h = await createRefusedWakeAfterFailedRollbackHarness( + "ws-auto-compaction-rollback-unknown-releases-wake" + ); + try { + // Fail only the readback that follows the failed delete; sendMessage reads history + // earlier (compaction check) and those reads must stay healthy for the row to land. + const readHistory = h.historyService.getHistoryFromLatestBoundary.bind(h.historyService); + spyOn(h.historyService, "getHistoryFromLatestBoundary").mockImplementation((...args) => + h.deleteMessagesCalls() > 0 + ? Promise.resolve(Err("disk unavailable")) + : readHistory(...args) + ); + const result = await h.sendRefusedWake(); + expect(result.success).toBe(false); + expect(h.streamMessage).not.toHaveBeenCalled(); + expect(h.onAccepted).not.toHaveBeenCalled(); + expect(h.readResumeRequest()).toBeUndefined(); + expect(h.chatEventTypes).not.toContain("auto-retry-scheduled"); + } finally { + h.session.dispose(); + } + }); + test("uses preferred compaction model for on-send auto-compaction requests", async () => { const workspaceId = "ws-auto-compaction-preferred-model"; @@ -1343,6 +1558,8 @@ describe("AgentSession on-send auto-compaction for synthetic guidance sends", () async function createGuidanceHarness(args: { workspaceId: string; summaryText?: string; + /** Observes each stream request (1-based) just before and just after its stream-start. */ + onStreamRequest?: (index: number, phase: "preparing" | "streaming") => void; }): Promise { const workspaceId = args.workspaceId; const streamHistories: MuxMessage[][] = []; @@ -1355,6 +1572,7 @@ describe("AgentSession on-send auto-compaction for synthetic guidance sends", () : undefined; streamHistories.push(Array.isArray(requestMessages) ? (requestMessages as MuxMessage[]) : []); + args.onStreamRequest?.(streamHistories.length, "preparing"); aiEmitter.emit("stream-start", { type: "stream-start", workspaceId, @@ -1363,6 +1581,7 @@ describe("AgentSession on-send auto-compaction for synthetic guidance sends", () historySequence: streamHistories.length, startTime: Date.now(), }); + args.onStreamRequest?.(streamHistories.length, "streaming"); const usage = { inputTokens: 42, @@ -1494,6 +1713,66 @@ describe("AgentSession on-send auto-compaction for synthetic guidance sends", () fixture.session.dispose(); }); + test("a wake consumed by on-send compaction stays in flight until its follow-up streams", async () => { + // The wake's onAccepted lowers the reconciler level, and the compaction stream's request is + // the compaction row, not the wake — so the compaction stream must not redeem the wake: + // the wake turn stays in flight through the compaction and is redeemed only by the + // follow-up's own stream (see AgentSession.wakeContinuationDebt). + const observed: Array<[number, "preparing" | "streaming", boolean]> = []; + const fixtureRef: { session?: AgentSession } = {}; + const fixture = await createGuidanceHarness({ + workspaceId: "ws-auto-compaction-wake-identity", + onStreamRequest: (index, phase) => { + observed.push([index, phase, fixtureRef.session?.hasPendingBashMonitorWakeTurn() ?? false]); + }, + }); + fixtureRef.session = fixture.session; + // Compact the wake once; its follow-up must then stream as the wake itself. + let compactionChecks = 0; + ( + fixture.session as unknown as { compactionMonitor: CompactionMonitor } + ).compactionMonitor.checkBeforeSend = () => { + compactionChecks += 1; + return { + shouldShowWarning: compactionChecks === 1, + shouldForceCompact: compactionChecks === 1, + usagePercentage: compactionChecks === 1 ? 95 : 10, + thresholdPercentage: 70, + }; + }; + + const result = await fixture.session.sendMessage( + "READY", + { + model: "openai:gpt-4o", + agentId: "exec", + muxMetadata: { type: "bash-monitor-wake", records: [] }, + }, + { synthetic: true, agentInitiated: true, startStreamInBackground: true } + ); + expect(result.success).toBe(true); + expect(await waitFor(() => fixture.streamHistories.length >= 2)).toBe(true); + + expect(fixture.streamHistories[0].at(-1)?.metadata?.muxMetadata?.type).toBe( + "compaction-request" + ); + expect(fixture.streamHistories[1].at(-1)?.metadata?.muxMetadata?.type).toBe( + "bash-monitor-wake" + ); + expect(observed).toEqual([ + // Compaction turn: preparing and streaming both still carry the wake. + [1, "preparing", true], + [1, "streaming", true], + // Follow-up wake turn: preparing carries it; its own stream shows it instead. + [2, "preparing", true], + [2, "streaming", false], + ]); + expect(await waitFor(() => !fixture.session.isBusy())).toBe(true); + expect(fixture.session.hasPendingBashMonitorWakeTurn()).toBe(false); + + fixture.session.dispose(); + }); + // Characterization: sends carrying preTurnMessages (family-message payloads) // intentionally skip on-send compaction. The trigger row references its // payload by message ID, so compacting the payload away would dangle that diff --git a/src/node/services/agentSession.continueMessageAgentId.test.ts b/src/node/services/agentSession.continueMessageAgentId.test.ts index a2786e8920..19933cad5a 100644 --- a/src/node/services/agentSession.continueMessageAgentId.test.ts +++ b/src/node/services/agentSession.continueMessageAgentId.test.ts @@ -1,4 +1,4 @@ -import { afterEach, describe, expect, mock, test } from "bun:test"; +import { afterEach, describe, expect, mock, spyOn, test } from "bun:test"; import { createMuxMessage } from "@/common/types/message"; import type { CompactionFollowUpRequest, MuxMessage } from "@/common/types/message"; import type { FilePart, SendMessageOptions } from "@/common/orpc/types"; @@ -150,7 +150,14 @@ describe("AgentSession continue-message agentId fallback", () => { historyCleanup = undefined; }); - const createSession = async (messages: MuxMessage[] = [], config = createConfig()) => { + const createSession = async ( + messages: MuxMessage[] = [], + hooks: Pick< + ConstructorParameters[0], + "onWorkspaceTurnContinuationVoided" + > = {}, + config = createConfig() + ) => { const { historyService, cleanup } = await createTestHistoryService(); historyCleanup = cleanup; for (const message of messages) { @@ -164,6 +171,7 @@ describe("AgentSession continue-message agentId fallback", () => { aiService: createAiService(), initStateManager: createInitStateManager(), backgroundProcessManager: createBackgroundProcessManager(), + ...hooks, }); sessions.push(session); @@ -311,6 +319,7 @@ describe("AgentSession continue-message agentId fallback", () => { agentId: "exec", }), ], + {}, archivedConfig ); internals.sendMessage = mock(() => Promise.resolve({ success: true as const })); @@ -351,6 +360,142 @@ describe("AgentSession continue-message agentId fallback", () => { expect(lastMessages.data[0]?.metadata?.muxMetadata).toEqual({ type: "compaction-summary" }); }); + test("abandoning a follow-up that carries a delegated turn settles that turn", async () => { + // A bash-monitor wake cut a delegated turn; the wake's on-send compaction stamped the + // correlation on its follow-up. If a manual send wins the idle race, nothing else can + // settle the owner's waiter (the compaction stream-end is uncorrelated and no later send + // inherits the metadata), so the discard itself voids the continuation. + const workspaceTurnMetadata = { + type: "workspace-turn-task", + taskHandleId: "wst_abandoned", + ownerWorkspaceId: "parent-abandoned", + turnId: "turn-abandoned", + } as const; + const wakeFollowUp: CompactionFollowUpRequest = { + text: "monitor matched", + model: "openai:gpt-4o", + agentId: "exec", + muxMetadata: { type: "bash-monitor-wake", records: [] }, + workspaceTurnMetadata, + dispatchOptions: { requireIdle: true }, + }; + let settlementError: Error | undefined = new Error("task handle store unavailable"); + const abandoned = mock( + ( + _metadata: NonNullable, + _reason: string + ) => (settlementError != null ? Promise.reject(settlementError) : Promise.resolve()) + ); + const { session, historyService, internals } = await createSession( + [compactionSummaryMessage("summary-wake", wakeFollowUp)], + { onWorkspaceTurnContinuationVoided: abandoned } + ); + internals.sendMessage = mock(() => Promise.resolve({ success: true as const })); + (session as unknown as { hasExternalSendPreflight?: () => boolean }).hasExternalSendPreflight = + () => true; + + // The owner-side settlement is never awaited here: this path runs while the owner's + // stream-end listener holds the workspace event lock (waiting on the compaction decision) + // and the settlement needs that lock. The discard completes; a failed settlement is parked + // and retried, not lost. + expect(await internals.dispatchPendingFollowUp()).toBe(false); + expect(internals.sendMessage).not.toHaveBeenCalled(); + expect(abandoned).toHaveBeenCalledTimes(1); + expect(abandoned).toHaveBeenLastCalledWith(workspaceTurnMetadata, "abandoned"); + await Promise.resolve(); + await Promise.resolve(); + + settlementError = undefined; + session.setBashMonitorWakeOutstanding(true); + session.setBashMonitorWakeOutstanding(false); + expect(abandoned).toHaveBeenCalledTimes(2); + expect(abandoned).toHaveBeenLastCalledWith(workspaceTurnMetadata, "abandoned"); + const lastMessages = await historyService.getLastMessages("ws", 1); + expect(lastMessages.success && lastMessages.data[0]?.metadata?.muxMetadata).toEqual({ + type: "compaction-summary", + }); + + // A follow-up that is dispatched is the continuation itself: nothing to settle. + (session as unknown as { hasExternalSendPreflight?: () => boolean }).hasExternalSendPreflight = + () => false; + await historyService.appendToHistory( + "ws", + compactionSummaryMessage("summary-wake-2", wakeFollowUp) + ); + expect(await internals.dispatchPendingFollowUp()).toBe(true); + expect(internals.sendMessage).toHaveBeenCalledTimes(1); + expect(abandoned).toHaveBeenCalledTimes(2); + }); + + test("abandoning a follow-up settles its delegated turn only once the erase is durable", async () => { + // While the summary still carries the follow-up, startup recovery or a retry can dispatch + // it; settling the delegated turn first would interrupt the handle under a continuation + // that is still live. A failed rewrite therefore keeps the turn unsettled — unless the + // rewrite actually landed and only its result was lost, which a readback recognises. + const workspaceTurnMetadata = { + type: "workspace-turn-task", + taskHandleId: "wst_durable", + ownerWorkspaceId: "parent-durable", + turnId: "turn-durable", + } as const; + const wakeFollowUp: CompactionFollowUpRequest = { + text: "monitor matched", + model: "openai:gpt-4o", + agentId: "exec", + muxMetadata: { type: "bash-monitor-wake", records: [] }, + workspaceTurnMetadata, + dispatchOptions: { requireIdle: true }, + }; + const abandoned = mock( + ( + _metadata: NonNullable, + _reason: string + ) => Promise.resolve() + ); + const { session, historyService, internals } = await createSession( + [compactionSummaryMessage("summary-durable", wakeFollowUp)], + { onWorkspaceTurnContinuationVoided: abandoned } + ); + internals.sendMessage = mock(() => Promise.resolve({ success: true as const })); + (session as unknown as { hasExternalSendPreflight?: () => boolean }).hasExternalSendPreflight = + () => true; + + // Rewrite genuinely fails: the follow-up stays durable and the turn stays unsettled. + const realUpdate = historyService.updateHistory.bind(historyService); + const updateSpy = spyOn(historyService, "updateHistory").mockImplementationOnce(() => + Promise.resolve({ success: false as const, error: "disk full" }) + ); + try { + const failed = await internals.dispatchPendingFollowUp().then( + () => null, + (error: unknown) => error + ); + expect(failed).toBeInstanceOf(Error); + expect(abandoned).not.toHaveBeenCalled(); + const stillPending = await historyService.getLastMessages("ws", 1); + expect(stillPending.success && stillPending.data[0]?.metadata?.muxMetadata).toMatchObject({ + type: "compaction-summary", + pendingFollowUp: { text: "monitor matched" }, + }); + + // Rewrite landed but reported failure: the readback finds the follow-up gone, so the + // delegated turn is settled rather than stranded behind a follow-up nothing can dispatch. + updateSpy.mockImplementationOnce(async (workspaceId, message) => { + await realUpdate(workspaceId, message); + return { success: false as const, error: "result lost" }; + }); + expect(await internals.dispatchPendingFollowUp()).toBe(false); + expect(abandoned).toHaveBeenCalledTimes(1); + expect(abandoned).toHaveBeenLastCalledWith(workspaceTurnMetadata, "abandoned"); + const cleared = await historyService.getLastMessages("ws", 1); + expect(cleared.success && cleared.data[0]?.metadata?.muxMetadata).toEqual({ + type: "compaction-summary", + }); + } finally { + updateSpy.mockRestore(); + } + }); + test("dispatchPendingFollowUp removes heartbeat reset boundaries when idle-only follow-ups are skipped", async () => { const earlierMessage = createMuxMessage("before-reset", "assistant", "Earlier context"); const { session, historyService, internals } = await createSession([ diff --git a/src/node/services/agentSession.mcpPromptSnapshot.test.ts b/src/node/services/agentSession.mcpPromptSnapshot.test.ts index 6dbae538c2..bd9f243768 100644 --- a/src/node/services/agentSession.mcpPromptSnapshot.test.ts +++ b/src/node/services/agentSession.mcpPromptSnapshot.test.ts @@ -58,13 +58,7 @@ describe("AgentSession MCP prompt snapshots", () => { expect(history.data[0]?.parts.find((part) => part.type === "text")?.text).toBe( "Expanded prompt" ); - expect(getPrompt).toHaveBeenCalledWith( - "workspace", - "coder", - "review", - { path: "src" }, - undefined - ); + expect(getPrompt).toHaveBeenCalledWith("workspace", "coder", "review", { path: "src" }); // The live transcript must also emit the snapshot before the user row. const emittedIds = harness.events diff --git a/src/node/services/agentSession.queueDispatch.test.ts b/src/node/services/agentSession.queueDispatch.test.ts index 08c6774a27..a1b38bef3f 100644 --- a/src/node/services/agentSession.queueDispatch.test.ts +++ b/src/node/services/agentSession.queueDispatch.test.ts @@ -1,10 +1,11 @@ import { describe, expect, mock, spyOn, test } from "bun:test"; import type { MuxMessageMetadata } from "@/common/types/message"; -import { Err, Ok } from "@/common/types/result"; +import { Ok } from "@/common/types/result"; import type { WorkspaceGoalService } from "./workspaceGoalService"; import { createAgentSessionHarness, createStartedTurnHandle } from "./agentSession.testHarness"; import type { AIService } from "./aiService"; +import type { AgentSession } from "./agentSession"; const TEST_MODEL = "anthropic:claude-sonnet-4-5"; const WORKSPACE_TURN_CORRELATION = { @@ -36,6 +37,25 @@ function streamStartEvent(workspaceId: string): Record { }; } +const DELEGATED_TURN: Extract = { + type: "workspace-turn-task", + taskHandleId: "wst_delegated", + ownerWorkspaceId: "owner-workspace", + turnId: "turn-1", +}; + +/** The correlation the active stream runs under; a wake cut records it as the debt's owner. */ +function setActiveStreamCorrelation( + session: AgentSession, + workspaceTurnMetadata: typeof DELEGATED_TURN | undefined +): void { + ( + session as unknown as { + activeStreamContext?: { workspaceTurnMetadata?: typeof DELEGATED_TURN }; + } + ).activeStreamContext = { workspaceTurnMetadata }; +} + function streamAbortEvent( workspaceId: string, abortReason: "system" | "user" @@ -372,87 +392,6 @@ describe("AgentSession queued message tool-call dispatch", () => { } }); - test("withdrawn tool-end entry neither soft-stops nor hides a later entry's mode", async () => { - const workspaceId = "queue-dispatch-withdrawn-head"; - const queuedSignals: boolean[] = []; - const { session, cleanup, aiEmitter, aiService } = await createAgentSessionHarness({ - workspaceId, - backgroundProcessManagerOverrides: { - setMessageQueued: mock((_workspaceId: string, queued: boolean) => { - queuedSignals.push(queued); - }), - }, - }); - const stopStream = spyOn(aiService, "stopStream").mockResolvedValue(Ok(undefined)); - - try { - aiEmitter.emit("stream-start", streamStartEvent(workspaceId)); - const controller = new AbortController(); - session.queueMessage( - "Background monitor wake", - { model: TEST_MODEL, agentId: "exec", queueDispatchMode: "tool-end" }, - { synthetic: true, agentInitiated: true, cancelSignal: controller.signal } - ); - expect(session.hasQueuedMessages("tool-end")).toBe(true); - - controller.abort("monitor withdrawn"); - expect(session.hasQueuedMessages("tool-end")).toBe(false); - expect(session.hasQueuedMessages()).toBe(false); - - aiEmitter.emit("tool-call-end", { - ...toolCallEndEvent(workspaceId), - toolName: "web_search", - providerExecuted: true, - }); - expect(stopStream).not.toHaveBeenCalled(); - - session.queueMessage("follow up", { - model: TEST_MODEL, - agentId: "exec", - queueDispatchMode: "turn-end", - }); - expect(session.hasQueuedMessages("tool-end")).toBe(false); - expect(session.hasQueuedMessages("turn-end")).toBe(true); - expect(queuedSignals).toEqual([true, false]); - } finally { - stopStream.mockRestore(); - session.dispose(); - await cleanup(); - } - }); - - test.each([ - ["turn-end", "tool-end"], - ["tool-end", "turn-end"], - ] as const)( - "queueMessage reports the live entry's mode behind a withdrawn %s head", - async (withdrawnMode, liveMode) => { - const { session, cleanup } = await createAgentSessionHarness({ - workspaceId: "queue-dispatch-withdrawn-" + withdrawnMode + "-head", - }); - try { - const controller = new AbortController(); - session.queueMessage( - "Background monitor wake", - { model: TEST_MODEL, agentId: "exec", queueDispatchMode: withdrawnMode }, - { synthetic: true, agentInitiated: true, cancelSignal: controller.signal } - ); - controller.abort("monitor withdrawn"); - - expect( - session.queueMessage("follow up", { - model: TEST_MODEL, - agentId: "exec", - queueDispatchMode: liveMode, - }) - ).toBe(liveMode); - } finally { - session.dispose(); - await cleanup(); - } - } - ); - test("waits for every known sibling before stopping after a provider-executed result", async () => { const workspaceId = "queue-dispatch-provider-siblings"; const { session, cleanup, aiEmitter, aiService } = await createAgentSessionHarness({ @@ -706,316 +645,657 @@ describe("AgentSession queued message tool-call dispatch", () => { } }); - test("cancel signal retracts a synthetic entry after dequeue while history append is preparing", async () => { - const workspaceId = "queue-dispatch-cancel-preparing"; - const { session, cleanup, historyService, events } = await createAgentSessionHarness({ + test("hasPendingToolEndInput unions the queued tool-end head with the live wake level", async () => { + const workspaceId = "queue-dispatch-pending-tool-end-input"; + let level: () => Promise = () => Promise.resolve(false); + const { session, cleanup } = await createAgentSessionHarness({ workspaceId, - captureEvents: true, + hasOutstandingBashMonitorWake: () => level(), }); - const originalAppend = historyService.appendToHistory.bind(historyService); - let markAppendStarted: () => void = () => undefined; - const appendStarted = new Promise((resolve) => { - markAppendStarted = resolve; + try { + expect(await session.hasPendingToolEndInput()).toBe(false); + + // The level is read live — no snapshot survives from one boundary to the next. + level = () => Promise.resolve(true); + expect(await session.hasPendingToolEndInput()).toBe(true); + level = () => Promise.resolve(false); + expect(await session.hasPendingToolEndInput()).toBe(false); + + // A failing level read must not cut the stream on the level's account... + level = () => Promise.reject(new Error("watermark read failed")); + expect(await session.hasPendingToolEndInput()).toBe(false); + // ...but a tool-end message queued while that read was in flight still arbitrates the + // boundary: the failure says nothing about the queue. + level = () => { + session.queueMessage("correction", { model: TEST_MODEL, agentId: "exec" }); + return Promise.reject(new Error("watermark read failed")); + }; + expect(await session.hasPendingToolEndInput()).toBe(true); + session.clearQueue(); + + // A non-empty queue arbitrates alone: a turn-end head is not promoted to tool-end by + // a high wake level (the wake dispatcher waits for the queue to drain anyway). + level = () => Promise.resolve(true); + session.queueMessage("later", { + model: TEST_MODEL, + agentId: "exec", + queueDispatchMode: "turn-end", + }); + expect(await session.hasPendingToolEndInput()).toBe(false); + session.clearQueue(); + expect(await session.hasPendingToolEndInput()).toBe(true); + level = () => Promise.resolve(false); + session.queueMessage("now", { model: TEST_MODEL, agentId: "exec" }); + expect(await session.hasPendingToolEndInput()).toBe(true); + } finally { + session.dispose(); + await cleanup(); + } + }); + + test("a stream cut for the wake level takes a continuation debt until other input supersedes it", async () => { + // Settlement of a delegated turn runs after the cut and reads the debt (the cutter and + // hasBashMonitorWakeContinuation), never the level: an operator canceling the monitor in + // between must not be able to hide that the cut happened. + const workspaceId = "queue-dispatch-wake-cut-debt"; + let level = false; + let markStreamRequested: () => void = () => undefined; + const streamRequested = new Promise((resolve) => { + markStreamRequested = resolve; }); - let releaseAppend: () => void = () => undefined; - const appendRelease = new Promise((resolve) => { - releaseAppend = resolve; + let releaseStream: () => void = () => undefined; + const streamRelease = new Promise((resolve) => { + releaseStream = resolve; }); - const appendSpy = spyOn(historyService, "appendToHistory").mockImplementation( - async (...args) => { - markAppendStarted(); - await appendRelease; - return originalAppend(...args); - } - ); + const voided: Array<[MuxMessageMetadata, string]> = []; + const { session, cleanup } = await createAgentSessionHarness({ + workspaceId, + hasOutstandingBashMonitorWake: () => Promise.resolve(level), + onWorkspaceTurnContinuationVoided: (correlation, reason) => { + voided.push([correlation, reason]); + return Promise.resolve(); + }, + aiServiceOverrides: { + streamMessage: mock(async () => { + markStreamRequested(); + await streamRelease; + return Ok(createStartedTurnHandle("test-assistant-message")); + }), + }, + }); + let disposed = false; + try { + // The cut stream's correlation is what the debt records. + setActiveStreamCorrelation(session, DELEGATED_TURN); + expect(session.getQueueCutCutter()).toBeUndefined(); + level = true; + expect(await session.hasPendingToolEndInput()).toBe(true); + level = false; + expect(session.getQueueCutCutter()).toEqual({ stage: "bash-monitor-wake" }); + expect(session.hasBashMonitorWakeContinuation()).toBe(true); + expect(session.hasPendingBashMonitorWakeTurn()).toBe(false); + // Reading a low level later neither retracts the debt nor takes a second one. + expect(await session.hasPendingToolEndInput()).toBe(false); + expect(session.getQueueCutCutter()).toEqual({ stage: "bash-monitor-wake" }); + expect(voided).toEqual([]); + + // Input that is not the wake supersedes the continuation: the owner is told once, when + // that input's row is durable, and the cutter is now the admitted input. + const sendPromise = session.sendMessage("hello", { model: TEST_MODEL, agentId: "exec" }); + await streamRequested; + expect(session.isBusy()).toBe(true); + expect(voided).toEqual([[DELEGATED_TURN, "superseded"]]); + expect(session.hasBashMonitorWakeContinuation()).toBe(false); + expect(session.getQueueCutCutter()).toEqual({ stage: "preparing", muxMetadata: undefined }); + session.dispose(); + disposed = true; + releaseStream(); + await sendPromise; + expect(voided).toHaveLength(1); + } finally { + releaseStream(); + if (!disposed) session.dispose(); + await cleanup(); + } + }); + test("a correlated turn admitted after the cut assumes the debt until its stream starts", async () => { + // The delegated turn's own continuation (e.g. a queued same-turn message) supersedes + // nothing: its stream-end settles the turn, so the owner is not told. Until that stream + // starts the debt stays visible to settlement (a stream-end handler running in the gap + // must still defer) and cannot be retracted by the level lowering. + const workspaceId = "queue-dispatch-wake-cut-same-turn"; + let level = true; + let markStreamRequested: () => void = () => undefined; + const streamRequested = new Promise((resolve) => { + markStreamRequested = resolve; + }); + let releaseStream: () => void = () => undefined; + const streamRelease = new Promise((resolve) => { + releaseStream = resolve; + }); + const voided: unknown[] = []; + const { session, cleanup, aiEmitter } = await createAgentSessionHarness({ + workspaceId, + hasOutstandingBashMonitorWake: () => Promise.resolve(level), + onWorkspaceTurnContinuationVoided: (...args) => { + voided.push(args); + return Promise.resolve(); + }, + aiServiceOverrides: { + streamMessage: mock(async () => { + markStreamRequested(); + await streamRelease; + return Ok(createStartedTurnHandle("test-assistant-message")); + }), + }, + }); + let disposed = false; try { - const controller = new AbortController(); - const cancelState = { canceledBeforeAcceptance: false }; - const canceledReasons: string[] = []; - session.queueMessage( - "Background monitor wake", - { model: TEST_MODEL, agentId: "exec" }, - { - synthetic: true, - agentInitiated: true, - cancelState, - cancelSignal: controller.signal, - onCanceled: (reason) => { - canceledReasons.push(reason); - }, - } - ); + setActiveStreamCorrelation(session, DELEGATED_TURN); + expect(await session.hasPendingToolEndInput()).toBe(true); + level = false; + const sendPromise = session.sendMessage("continue", { + model: TEST_MODEL, + agentId: "exec", + muxMetadata: DELEGATED_TURN, + }); + await streamRequested; + expect(session.hasBashMonitorWakeContinuation()).toBe(true); + // PREPARING attributes the cut to this continuation; either attribution defers. + expect(session.getQueueCutCutter()?.stage).toBe("preparing"); + session.setBashMonitorWakeOutstanding(true); + session.setBashMonitorWakeOutstanding(false); + expect(session.hasBashMonitorWakeContinuation()).toBe(true); + expect(voided).toEqual([]); - session.sendQueuedMessages(); - await appendStarted; - controller.abort("monitor canceled"); - releaseAppend(); + aiEmitter.emit("stream-start", streamStartEvent(workspaceId)); + expect(session.hasBashMonitorWakeContinuation()).toBe(false); + expect(session.getQueueCutCutter()).toBeUndefined(); + expect(voided).toEqual([]); + session.dispose(); + disposed = true; + releaseStream(); + await sendPromise; + } finally { + releaseStream(); + if (!disposed) session.dispose(); + await cleanup(); + } + }); - expect(await waitForCondition(() => canceledReasons.length === 1)).toBe(true); - expect(await waitForCondition(() => !session.isBusy())).toBe(true); - expect(canceledReasons).toEqual(["monitor canceled"]); + test("the stream-start ledger remembers a correlated continuation after it ended", async () => { + // A stream-end handler can run after the turn's next stream already started and ended; + // it asks the ledger whether the turn continued after the stream it handles. + const workspaceId = "queue-dispatch-stream-ledger"; + const { session, cleanup, aiEmitter } = await createAgentSessionHarness({ workspaceId }); + try { + const startedAs = (messageId: string, correlation: typeof DELEGATED_TURN | undefined) => { + setActiveStreamCorrelation(session, correlation); + aiEmitter.emit("stream-start", { ...streamStartEvent(workspaceId), messageId }); + }; + startedAs("assistant-delegated-1", DELEGATED_TURN); + expect( + session.hasCorrelatedStreamStartedAfter(DELEGATED_TURN, ["assistant-delegated-1"]) + ).toBe(false); - const history = await historyService.getHistoryFromLatestBoundary(workspaceId); - expect(history.success).toBe(true); - if (history.success) { - expect( - history.data.some((message) => - message.parts.some( - (part) => part.type === "text" && part.text === "Background monitor wake" - ) - ) - ).toBe(false); - } + startedAs("assistant-manual", undefined); expect( - events.some( - (event) => - event.type === "message" && - event.role === "user" && - event.parts.some( - (part) => part.type === "text" && part.text === "Background monitor wake" - ) - ) + session.hasCorrelatedStreamStartedAfter(DELEGATED_TURN, ["assistant-delegated-1"]) + ).toBe(false); + + startedAs("assistant-delegated-2", DELEGATED_TURN); + expect( + session.hasCorrelatedStreamStartedAfter(DELEGATED_TURN, ["assistant-delegated-1"]) + ).toBe(true); + // Relative to the continuation itself nothing followed. + expect( + session.hasCorrelatedStreamStartedAfter(DELEGATED_TURN, ["assistant-delegated-2"]) + ).toBe(false); + // A different turn's correlation never matches. + expect( + session.hasCorrelatedStreamStartedAfter({ ...DELEGATED_TURN, turnId: "turn-2" }, [ + "assistant-delegated-1", + ]) ).toBe(false); + // A stream the ledger no longer holds predates everything remembered. + expect(session.hasCorrelatedStreamStartedAfter(DELEGATED_TURN, ["assistant-evicted"])).toBe( + true + ); } finally { - releaseAppend(); - appendSpy.mockRestore(); session.dispose(); await cleanup(); } }); - test("rollback failure preserves the wake and continues acceptance", async () => { - const workspaceId = "queue-dispatch-cancel-rollback-failure"; - const { session, cleanup, historyService } = await createAgentSessionHarness({ workspaceId }); - const originalAppend = historyService.appendToHistory.bind(historyService); - let markAppendStarted: () => void = () => undefined; - const appendStarted = new Promise((resolve) => { - markAppendStarted = resolve; + test("the level lowering with no wake turn in flight voids the debt as retracted", async () => { + const workspaceId = "queue-dispatch-wake-retracted"; + const voided: Array<[MuxMessageMetadata, string]> = []; + const { session, cleanup } = await createAgentSessionHarness({ + workspaceId, + hasOutstandingBashMonitorWake: () => Promise.resolve(true), + onWorkspaceTurnContinuationVoided: (correlation, reason) => { + voided.push([correlation, reason]); + return Promise.resolve(); + }, }); - let releaseAppend: () => void = () => undefined; - const appendRelease = new Promise((resolve) => { - releaseAppend = resolve; + try { + setActiveStreamCorrelation(session, DELEGATED_TURN); + session.setBashMonitorWakeOutstanding(true); + expect(await session.hasPendingToolEndInput()).toBe(true); + expect(session.getQueueCutCutter()).toEqual({ stage: "bash-monitor-wake" }); + + // Republishing high changes nothing; lowering it (monitor canceled, output shown, + // history cleared) leaves no wake to continue the cut stream. + session.setBashMonitorWakeOutstanding(true); + expect(voided).toEqual([]); + session.setBashMonitorWakeOutstanding(false); + expect(voided).toEqual([[DELEGATED_TURN, "retracted"]]); + expect(session.getQueueCutCutter()).toBeUndefined(); + expect(session.hasBashMonitorWakeContinuation()).toBe(false); + + // A cut with no correlation (manual stream) still records the cutter but has no + // owner to tell. + setActiveStreamCorrelation(session, undefined); + session.setBashMonitorWakeOutstanding(true); + expect(await session.hasPendingToolEndInput()).toBe(true); + session.setBashMonitorWakeOutstanding(false); + expect(session.getQueueCutCutter()).toBeUndefined(); + expect(voided).toHaveLength(1); + } finally { + session.dispose(); + await cleanup(); + } + }); + + test("a wake send is in flight from its first synchronous step until its stream starts", async () => { + // The wake's onAccepted lowers the level as soon as its row is durable, which is before + // PREPARING and long before a stream exists. Only the in-flight marker keeps the debt + // from being voided in that window. + const workspaceId = "queue-dispatch-wake-in-flight"; + let markStreamRequested: () => void = () => undefined; + const streamRequested = new Promise((resolve) => { + markStreamRequested = resolve; + }); + let releaseStream: () => void = () => undefined; + const streamRelease = new Promise((resolve) => { + releaseStream = resolve; + }); + const voided: unknown[] = []; + const { session, aiEmitter, cleanup } = await createAgentSessionHarness({ + workspaceId, + hasOutstandingBashMonitorWake: () => Promise.resolve(true), + onWorkspaceTurnContinuationVoided: (...args) => { + voided.push(args); + return Promise.resolve(); + }, + aiServiceOverrides: { + streamMessage: mock(async () => { + markStreamRequested(); + await streamRelease; + return Ok(createStartedTurnHandle("test-assistant-message")); + }), + }, }); - const appendSpy = spyOn(historyService, "appendToHistory").mockImplementation( - async (...args) => { - markAppendStarted(); - await appendRelease; - return originalAppend(...args); - } - ); - const deleteMessagesSpy = spyOn(historyService, "deleteMessages").mockResolvedValue( - Err("injected rollback failure") - ); + let disposed = false; try { - const controller = new AbortController(); - const cancelState = { canceledBeforeAcceptance: false }; - const canceledReasons: string[] = []; + setActiveStreamCorrelation(session, DELEGATED_TURN); + session.setBashMonitorWakeOutstanding(true); + expect(await session.hasPendingToolEndInput()).toBe(true); + setActiveStreamCorrelation(session, undefined); + + expect(session.hasPendingBashMonitorWakeTurn()).toBe(false); let accepted = false; const sendPromise = session.sendMessage( "Background monitor wake", - { model: TEST_MODEL, agentId: "exec" }, + { + model: TEST_MODEL, + agentId: "exec", + muxMetadata: { type: "bash-monitor-wake", records: [] }, + }, { synthetic: true, agentInitiated: true, - cancelState, - cancelSignal: controller.signal, - onCanceled: (reason) => { - canceledReasons.push(reason); - }, onAccepted: () => { accepted = true; + // The reconciler consumes the signals and publishes low here. + session.setBashMonitorWakeOutstanding(false); }, } ); + // Synchronously at entry, before any admission await. + expect(session.hasPendingBashMonitorWakeTurn()).toBe(true); - await appendStarted; - controller.abort("monitor canceled"); - releaseAppend(); - const result = await sendPromise; - - expect(result.success).toBe(true); - expect(deleteMessagesSpy).toHaveBeenCalledTimes(1); - expect(canceledReasons).toEqual([]); - expect(cancelState.canceledBeforeAcceptance).toBe(false); + await streamRequested; expect(accepted).toBe(true); + expect(session.isBusy()).toBe(true); + expect(session.hasPendingBashMonitorWakeTurn()).toBe(true); + expect(session.hasBashMonitorWakeContinuation()).toBe(true); + expect(voided).toEqual([]); + + // The wake stream shows the wake: debt redeemed, nothing to tell the owner. + aiEmitter.emit("stream-start", streamStartEvent(workspaceId)); + expect(session.hasPendingBashMonitorWakeTurn()).toBe(false); + expect(session.hasBashMonitorWakeContinuation()).toBe(false); + expect(session.getQueueCutCutter()).toBeUndefined(); + expect(voided).toEqual([]); - const history = await historyService.getHistoryFromLatestBoundary(workspaceId); - expect(history.success).toBe(true); - if (history.success) { - expect( - history.data.some((message) => - message.parts.some( - (part) => part.type === "text" && part.text === "Background monitor wake" - ) - ) - ).toBe(true); - } - } finally { - releaseAppend(); - deleteMessagesSpy.mockRestore(); - appendSpy.mockRestore(); session.dispose(); + disposed = true; + releaseStream(); + await sendPromise; + expect(voided).toEqual([]); + } finally { + releaseStream(); + if (!disposed) session.dispose(); await cleanup(); } }); - test("verifies a committed rollback when batch deletion reports a post-write failure", async () => { - const workspaceId = "queue-dispatch-cancel-post-write-failure"; - const { session, cleanup, historyService } = await createAgentSessionHarness({ workspaceId }); - const originalAppend = historyService.appendToHistory.bind(historyService); - const originalDeleteMessages = historyService.deleteMessages.bind(historyService); - let markAppendStarted: () => void = () => undefined; - const appendStarted = new Promise((resolve) => { - markAppendStarted = resolve; - }); - let releaseAppend: () => void = () => undefined; - const appendRelease = new Promise((resolve) => { - releaseAppend = resolve; + test("a wake send refused before its row is durable leaves the debt to the next dispatch", async () => { + // Pre-commit refusals (stale admission) do not lower the level, so the debt is still + // owed and the reconciler re-dispatches; only a level drop voids it. + const workspaceId = "queue-dispatch-wake-refused"; + const voided: Array<[MuxMessageMetadata, string]> = []; + const { session, cleanup } = await createAgentSessionHarness({ + workspaceId, + hasOutstandingBashMonitorWake: () => Promise.resolve(true), + onWorkspaceTurnContinuationVoided: (correlation, reason) => { + voided.push([correlation, reason]); + return Promise.resolve(); + }, }); - const appendSpy = spyOn(historyService, "appendToHistory").mockImplementation( - async (...args) => { - markAppendStarted(); - await appendRelease; - return originalAppend(...args); - } - ); - const deleteMessagesSpy = spyOn(historyService, "deleteMessages").mockImplementation( - async (...args) => { - const result = await originalDeleteMessages(...args); - expect(result.success).toBe(true); - return Err("injected post-write failure"); - } - ); - try { - const controller = new AbortController(); - const cancelState = { canceledBeforeAcceptance: false }; - const canceledReasons: string[] = []; + setActiveStreamCorrelation(session, DELEGATED_TURN); + session.setBashMonitorWakeOutstanding(true); + expect(await session.hasPendingToolEndInput()).toBe(true); + let accepted = false; - const sendPromise = session.sendMessage( + const result = await session.sendMessage( "Background monitor wake", - { model: TEST_MODEL, agentId: "exec" }, + { + model: TEST_MODEL, + agentId: "exec", + muxMetadata: { type: "bash-monitor-wake", records: [] }, + }, { synthetic: true, agentInitiated: true, - cancelState, - cancelSignal: controller.signal, - onCanceled: (reason) => { - canceledReasons.push(reason); - }, + admissionStale: () => true, onAccepted: () => { accepted = true; }, } ); - - await appendStarted; - controller.abort("monitor canceled"); - releaseAppend(); - const result = await sendPromise; - - expect(result.success).toBe(true); - expect(deleteMessagesSpy).toHaveBeenCalledTimes(1); - expect(canceledReasons).toEqual(["monitor canceled"]); - expect(cancelState.canceledBeforeAcceptance).toBe(true); + expect(result.success).toBe(false); expect(accepted).toBe(false); + expect(session.hasPendingBashMonitorWakeTurn()).toBe(false); + expect(session.getQueueCutCutter()).toEqual({ stage: "bash-monitor-wake" }); + expect(voided).toEqual([]); - const history = await historyService.getHistoryFromLatestBoundary(workspaceId); - expect(history.success).toBe(true); - if (history.success) { - expect( - history.data.some((message) => - message.parts.some( - (part) => part.type === "text" && part.text === "Background monitor wake" - ) - ) - ).toBe(false); - } + session.setBashMonitorWakeOutstanding(false); + expect(voided).toEqual([[DELEGATED_TURN, "retracted"]]); } finally { - releaseAppend(); - deleteMessagesSpy.mockRestore(); - appendSpy.mockRestore(); session.dispose(); await cleanup(); } }); - test("cancellation during goal sync crosses the acceptance point of no return", async () => { - const workspaceId = "queue-dispatch-cancel-goal-reconcile"; - let markInitialSyncStarted: () => void = () => undefined; - const initialSyncStarted = new Promise((resolve) => { - markInitialSyncStarted = resolve; - }); - let releaseInitialSync: () => void = () => undefined; - const initialSyncRelease = new Promise((resolve) => { - releaseInitialSync = resolve; + test("other input supersedes the debt only once its own row is durable", async () => { + // A superseding send that is refused before anything is persisted changes nothing: the + // wake is still outstanding and will still continue the delegated turn. Admission + // (PREPARING) is a reservation, not acceptance. + const workspaceId = "queue-dispatch-supersede-at-acceptance"; + const voided: Array<[MuxMessageMetadata, string]> = []; + const { session, cleanup } = await createAgentSessionHarness({ + workspaceId, + hasOutstandingBashMonitorWake: () => Promise.resolve(true), + onWorkspaceTurnContinuationVoided: (correlation, reason) => { + voided.push([correlation, reason]); + return Promise.resolve(); + }, }); - let syncCalls = 0; - const syncGoalModeWithChatTail = mock(async () => { - syncCalls += 1; - if (syncCalls === 1) { - markInitialSyncStarted(); - await initialSyncRelease; - } - return null; + try { + setActiveStreamCorrelation(session, DELEGATED_TURN); + session.setBashMonitorWakeOutstanding(true); + expect(await session.hasPendingToolEndInput()).toBe(true); + + const refused = await session.sendMessage( + "peer message", + { model: TEST_MODEL, agentId: "exec" }, + { admissionStale: () => true } + ); + expect(refused.success).toBe(false); + expect(session.getQueueCutCutter()).toEqual({ stage: "bash-monitor-wake" }); + expect(session.hasBashMonitorWakeContinuation()).toBe(true); + expect(voided).toEqual([]); + + // The same input accepted (row durable) supersedes it exactly once. + const accepted = await session.sendMessage("peer message", { + model: TEST_MODEL, + agentId: "exec", + }); + expect(accepted.success).toBe(true); + expect(voided).toEqual([[DELEGATED_TURN, "superseded"]]); + expect(session.hasBashMonitorWakeContinuation()).toBe(false); + } finally { + session.dispose(); + await cleanup(); + } + }); + + test("a void whose owner-side settlement fails is retried at the next debt transition", async () => { + // The debt is cleared when it is voided; the settlement itself is I/O on the owner's side + // and may fail transiently. The void is kept and retried rather than logged away, so a + // delegated handle already deferred on the debt does not wait for a restart. + const workspaceId = "queue-dispatch-void-retry"; + const calls: Array<[MuxMessageMetadata, string]> = []; + let failNext = true; + const { session, cleanup } = await createAgentSessionHarness({ + workspaceId, + hasOutstandingBashMonitorWake: () => Promise.resolve(true), + onWorkspaceTurnContinuationVoided: (correlation, reason) => { + calls.push([correlation, reason]); + if (failNext) { + failNext = false; + return Promise.reject(new Error("handle store unavailable")); + } + return Promise.resolve(); + }, }); - const workspaceGoalService = { - assertPricedModelForBudgetedGoal: mock(() => Promise.resolve(Ok(undefined))), - syncGoalModeWithChatTail, - } as unknown as WorkspaceGoalService; - const { session, cleanup, historyService } = await createAgentSessionHarness({ + try { + setActiveStreamCorrelation(session, DELEGATED_TURN); + session.setBashMonitorWakeOutstanding(true); + expect(await session.hasPendingToolEndInput()).toBe(true); + + session.setBashMonitorWakeOutstanding(false); + expect(calls).toEqual([[DELEGATED_TURN, "retracted"]]); + // Let the rejection settle; the debt itself stays cleared (the cut is not re-attributed). + await Promise.resolve(); + await Promise.resolve(); + expect(session.hasBashMonitorWakeContinuation()).toBe(false); + + // Any later level transition retries the parked void with the original reason. + session.setBashMonitorWakeOutstanding(true); + session.setBashMonitorWakeOutstanding(false); + expect(calls).toEqual([ + [DELEGATED_TURN, "retracted"], + [DELEGATED_TURN, "retracted"], + ]); + await Promise.resolve(); + session.setBashMonitorWakeOutstanding(true); + session.setBashMonitorWakeOutstanding(false); + expect(calls).toHaveLength(2); + } finally { + session.dispose(); + await cleanup(); + } + }); + + test("a queued head is not recorded as a wake cut", async () => { + const workspaceId = "queue-dispatch-queue-cut-not-wake"; + let releaseLevel: () => void = () => undefined; + const { session, cleanup } = await createAgentSessionHarness({ workspaceId, - workspaceGoalService, + hasOutstandingBashMonitorWake: () => + new Promise((resolve) => { + releaseLevel = () => resolve(true); + }), }); + try { + // A message queued while the level is being read arbitrates like one queued before: + // a turn-end head means no cut (and no wake attribution), a tool-end head cuts as + // queued input. + const pendingTurnEnd = session.hasPendingToolEndInput(); + session.queueMessage("later", { + model: TEST_MODEL, + agentId: "exec", + queueDispatchMode: "turn-end", + }); + releaseLevel(); + expect(await pendingTurnEnd).toBe(false); + expect(session.getQueueCutCutter()).toMatchObject({ stage: "queued" }); + session.clearQueue(); + + const pendingToolEnd = session.hasPendingToolEndInput(); + session.queueMessage("now", { model: TEST_MODEL, agentId: "exec" }); + releaseLevel(); + expect(await pendingToolEnd).toBe(true); + expect(session.getQueueCutCutter()).toMatchObject({ stage: "queued" }); + session.clearQueue(); + expect(session.getQueueCutCutter()).toBeUndefined(); + } finally { + session.dispose(); + await cleanup(); + } + }); + test("the wake level and the queue head jointly drive the bash early-return flag", async () => { + const workspaceId = "queue-dispatch-yield-flag"; + const flags: boolean[] = []; + const { session, cleanup } = await createAgentSessionHarness({ + workspaceId, + backgroundProcessManagerOverrides: { + setMessageQueued: (_workspaceId: string, queued: boolean) => { + flags.push(queued); + }, + }, + }); + const lastFlag = () => flags.at(-1); try { - const controller = new AbortController(); - const cancelState = { canceledBeforeAcceptance: false }; - const canceledReasons: string[] = []; - let accepted = false; - const sendPromise = session.sendMessage( - "Background monitor wake", - { model: TEST_MODEL, agentId: "exec" }, - { - synthetic: true, - agentInitiated: true, - cancelState, - cancelSignal: controller.signal, - onCanceled: (reason) => { - canceledReasons.push(reason); - }, - onAccepted: () => { - accepted = true; - }, - } - ); + session.setBashMonitorWakeOutstanding(true); + expect(lastFlag()).toBe(true); + session.setBashMonitorWakeOutstanding(false); + expect(lastFlag()).toBe(false); - await initialSyncStarted; - controller.abort("monitor canceled"); - releaseInitialSync(); - const result = await sendPromise; + // Clearing the queue while the level is high must not drop the flag. + session.queueMessage("follow up", { model: TEST_MODEL, agentId: "exec" }); + expect(lastFlag()).toBe(true); + session.setBashMonitorWakeOutstanding(true); + session.clearQueue(); + expect(lastFlag()).toBe(true); + session.setBashMonitorWakeOutstanding(false); + expect(lastFlag()).toBe(false); + + // A stream ending onto an empty queue while the level is high (the drain + // sendQueuedMessages runs at stream end) keeps the flag up: the wake still asks the + // next stream — a racing manual send's — to yield at its first tool boundary. + session.setBashMonitorWakeOutstanding(true); + session.sendQueuedMessages(); + expect(lastFlag()).toBe(true); + session.setBashMonitorWakeOutstanding(false); + expect(lastFlag()).toBe(false); + + // A turn-end head owns the next dispatch: the level must not pull the early-return + // lever for it (mirrors hasPendingToolEndInput's arbitration). + session.setBashMonitorWakeOutstanding(true); + session.queueMessage("later", { + model: TEST_MODEL, + agentId: "exec", + queueDispatchMode: "turn-end", + }); + expect(lastFlag()).toBe(false); + session.clearQueue(); + expect(lastFlag()).toBe(true); + session.setBashMonitorWakeOutstanding(false); + } finally { + session.dispose(); + await cleanup(); + } + }); - expect(result.success).toBe(true); - expect(syncGoalModeWithChatTail).toHaveBeenCalledTimes(1); - expect(canceledReasons).toEqual([]); - expect(cancelState.canceledBeforeAcceptance).toBe(false); - expect(accepted).toBe(true); + test("the tool-end yield edge fires once per rising transition, whatever raises it", async () => { + // Foreground task waits are backgrounded on this edge, so it must track the *effective* + // lever (queue-head arbitration ∪ level), not the events that happen to feed it + // (Codex P2 PRRT_kwDOPxxmWM6fGVw_). + const workspaceId = "queue-dispatch-yield-edge"; + const edges = mock(() => undefined); + const { session, cleanup } = await createAgentSessionHarness({ + workspaceId, + onToolEndYieldRequested: edges, + }); + const turnEnd = { model: TEST_MODEL, agentId: "exec", queueDispatchMode: "turn-end" as const }; + const toolEnd = { model: TEST_MODEL, agentId: "exec", queueDispatchMode: "tool-end" as const }; + try { + // A turn-end head does not pull the lever; a tool-end enqueue does, once. + session.queueMessage("later", turnEnd); + expect(edges).not.toHaveBeenCalled(); + session.queueMessage("sooner", toolEnd); + expect(edges).toHaveBeenCalledTimes(1); + session.queueMessage("sooner still", toolEnd); + expect(edges).toHaveBeenCalledTimes(1); + session.clearQueue(); + + // The GVw_ case: the level is high behind a turn-end head (no edge), then the head is + // cleared with no enqueue and no level publish in between — the lever becomes + // effective and the edge must fire from that queue transition alone. + session.queueMessage("later", turnEnd); + session.setBashMonitorWakeOutstanding(true); + expect(edges).toHaveBeenCalledTimes(1); + session.clearQueue(); + expect(edges).toHaveBeenCalledTimes(2); + + // Republishing a high level is not an edge; lowering and raising it is. + session.setBashMonitorWakeOutstanding(true); + expect(edges).toHaveBeenCalledTimes(2); + session.setBashMonitorWakeOutstanding(false); + session.setBashMonitorWakeOutstanding(true); + expect(edges).toHaveBeenCalledTimes(3); + session.setBashMonitorWakeOutstanding(false); + + // tool-end is sticky within an entry: a turn-end queued behind it neither lowers the + // lever nor re-fires the edge. + session.queueMessage("sooner", toolEnd); + expect(edges).toHaveBeenCalledTimes(4); + session.queueMessage("later", turnEnd); + expect(edges).toHaveBeenCalledTimes(4); + expect(session.hasQueuedMessages()).toBe(true); + } finally { + session.dispose(); + await cleanup(); + } + }); - const history = await historyService.getHistoryFromLatestBoundary(workspaceId); - expect(history.success).toBe(true); - if (history.success) { - expect( - history.data.some((message) => - message.parts.some( - (part) => part.type === "text" && part.text === "Background monitor wake" - ) - ) - ).toBe(true); - } + test("disposing a session lowers the mirrored wake level", async () => { + // The flag lives in BackgroundProcessManager keyed by workspace id and outlives the + // session; a stale true would make a re-created session's bash reads return early. + const workspaceId = "queue-dispatch-dispose-clears-level"; + const flags: boolean[] = []; + const { session, cleanup } = await createAgentSessionHarness({ + workspaceId, + backgroundProcessManagerOverrides: { + setMessageQueued: (_workspaceId: string, queued: boolean) => { + flags.push(queued); + }, + }, + }); + try { + session.setBashMonitorWakeOutstanding(true); + expect(flags.at(-1)).toBe(true); + session.dispose(); + expect(flags.at(-1)).toBe(false); } finally { - releaseInitialSync(); session.dispose(); await cleanup(); } @@ -1047,17 +1327,17 @@ describe("AgentSession queued message tool-call dispatch", () => { let disposed = false; try { - const controller = new AbortController(); - const cancelState = { canceledBeforeAcceptance: false }; let accepted = false; const sendPromise = session.sendMessage( "Background monitor wake", - { model: TEST_MODEL, agentId: "exec" }, + { + model: TEST_MODEL, + agentId: "exec", + muxMetadata: { type: "bash-monitor-wake", records: [] }, + }, { synthetic: true, agentInitiated: true, - cancelState, - cancelSignal: controller.signal, onAccepted: () => { accepted = true; }, @@ -1072,7 +1352,6 @@ describe("AgentSession queued message tool-call dispatch", () => { expect(result.success).toBe(true); expect(accepted).toBe(true); - expect(cancelState.canceledBeforeAcceptance).toBe(false); } finally { releaseSync(); if (!disposed) session.dispose(); @@ -1105,28 +1384,44 @@ describe("AgentSession queued message tool-call dispatch", () => { }); try { - const controller = new AbortController(); - const cancelState = { canceledBeforeAcceptance: false }; const canceledReasons: string[] = []; let accepted = false; + // Acceptance consumes the reconciler signal, so an in-session resume must already be + // armed by then: nothing upstream can resend the durable row (Codex P2 + // PRRT_kwDOPxxmWM6fOH54). + let resumeArmedAtAcceptance = false; + const chatEventTypes: string[] = []; + const unsubscribe = session.onChatEvent((event) => { + chatEventTypes.push(event.message.type); + }); + const readResumeRequest = () => + ( + session as unknown as { + lastAutoRetryResumeRequest?: { options: { muxMetadata?: unknown } }; + } + ).lastAutoRetryResumeRequest; const sendPromise = session.sendMessage( "Background monitor wake", - { model: TEST_MODEL, agentId: "exec" }, + { + model: TEST_MODEL, + agentId: "exec", + muxMetadata: { type: "bash-monitor-wake", records: [] }, + }, { synthetic: true, agentInitiated: true, - cancelState, - cancelSignal: controller.signal, onCanceled: (reason) => { canceledReasons.push(reason); }, onAccepted: () => { accepted = true; + resumeArmedAtAcceptance = readResumeRequest() != null; }, } ); await syncStarted; + expect(readResumeRequest()).toBeUndefined(); releaseSync(); let syncError: unknown; try { @@ -1134,12 +1429,23 @@ describe("AgentSession queued message tool-call dispatch", () => { } catch (error) { syncError = error; } + unsubscribe(); expect(syncError).toBeInstanceOf(Error); expect((syncError as Error).message).toContain("injected goal sync failure"); expect(accepted).toBe(true); expect(canceledReasons).toEqual([]); - expect(cancelState.canceledBeforeAcceptance).toBe(false); + expect(resumeArmedAtAcceptance).toBe(true); + expect(readResumeRequest()?.options.muxMetadata).toEqual({ + type: "bash-monitor-wake", + records: [], + }); + expect(chatEventTypes).toContain("auto-retry-scheduled"); + // The armed resume is what will bring the wake's stream, so the wake turn stays in + // flight (a debt it carries is not voided) until that retry is given up. + expect(session.hasPendingBashMonitorWakeTurn()).toBe(true); + await session.setAutoRetryEnabled(false, { persist: false }); + expect(session.hasPendingBashMonitorWakeTurn()).toBe(false); const history = await historyService.getHistoryFromLatestBoundary(workspaceId); expect(history.success).toBe(true); if (history.success) { diff --git a/src/node/services/agentSession.testHarness.ts b/src/node/services/agentSession.testHarness.ts index fac756ca28..13fc3efabe 100644 --- a/src/node/services/agentSession.testHarness.ts +++ b/src/node/services/agentSession.testHarness.ts @@ -5,7 +5,11 @@ import type { WorkspaceChatMessage } from "@/common/orpc/types"; import { Err, Ok } from "@/common/types/result"; import type { Config } from "@/node/config"; import type { TurnStreamHandle } from "@/node/services/streamManager"; -import { AgentSession, type AgentSessionAIService } from "@/node/services/agentSession"; +import { + AgentSession, + type AgentSessionAIService, + type AgentSessionOptions, +} from "@/node/services/agentSession"; import type { CompactionCompletionMetadata } from "@/common/types/compaction"; import type { BackgroundProcessManager } from "@/node/services/backgroundProcessManager"; import type { WorkspaceGoalService } from "@/node/services/workspaceGoalService"; @@ -110,6 +114,9 @@ export interface AgentSessionHarnessOptions { workspaceGoalService?: WorkspaceGoalService; mcpServerManager?: MCPServerManager; onCompactionComplete?: (metadata: CompactionCompletionMetadata) => void; + hasOutstandingBashMonitorWake?: () => Promise; + onToolEndYieldRequested?: () => void; + onWorkspaceTurnContinuationVoided?: AgentSessionOptions["onWorkspaceTurnContinuationVoided"]; captureEvents?: boolean; } @@ -154,6 +161,9 @@ export async function createAgentSessionHarness( workspaceGoalService: options.workspaceGoalService, backgroundProcessManager, onCompactionComplete: options.onCompactionComplete, + hasOutstandingBashMonitorWake: options.hasOutstandingBashMonitorWake, + onToolEndYieldRequested: options.onToolEndYieldRequested, + onWorkspaceTurnContinuationVoided: options.onWorkspaceTurnContinuationVoided, }); const events: WorkspaceChatMessage[] = []; diff --git a/src/node/services/agentSession.ts b/src/node/services/agentSession.ts index 4ea8bac14a..3f8aa53be5 100644 --- a/src/node/services/agentSession.ts +++ b/src/node/services/agentSession.ts @@ -115,7 +115,7 @@ import { createRuntimeContextForWorkspace, createRuntimeForWorkspace, } from "@/node/runtime/runtimeHelpers"; -import { MessageQueue, cancelReasonBeforeAcceptance } from "./messageQueue"; +import { MessageQueue } from "./messageQueue"; import type { QueueCutCutter } from "./messageQueue"; import { copyStreamLifecycleSnapshot, @@ -384,9 +384,9 @@ function hasSameWorkspaceTurnCorrelation( * Find the still-open workspace-turn correlation for a bash-monitor-wake * continuation stream. * - * A queued monitor wake dispatched at a tool boundary cuts the in-flight - * stream (finishReason "tool-calls") and immediately continues the same - * delegated work in a new stream. That continuation must inherit the cut + * An outstanding monitor wake makes the in-flight stream yield at a tool + * boundary (finishReason "tool-calls"); the wake turn sent once the owner is + * idle continues the same delegated work in a new stream. That continuation must inherit the cut * stream's workspace-turn metadata — otherwise the delegating parent sees the * cut as a premature turn failure ("Workspace turn ended before completion") * and the turn's real outcome can never settle the task handle (see @@ -444,6 +444,44 @@ function isCompactionRequestMetadata(meta: unknown): meta is CompactionRequestMe return true; } +/** + * The bash-monitor wake a row or send carries: the wake itself, or — for an on-send compaction + * request — the wake nested in the follow-up it carries. The compaction row is what that turn + * shows and what stays durable, so wake identity (and the delivered records the reconciler + * recovers from after a restart) has to be read through it. + */ +export function getCarriedBashMonitorWake( + muxMetadata: unknown +): Extract | undefined { + const meta = muxMetadata as MuxMessageMetadata | undefined; + if (meta?.type === "bash-monitor-wake") return meta; + if (!isCompactionRequestMetadata(meta)) return undefined; + const followUpMetadata = + meta.parsed.followUpContent?.muxMetadata ?? meta.parsed.continueMessage?.muxMetadata; + return followUpMetadata?.type === "bash-monitor-wake" ? followUpMetadata : undefined; +} + +function carriesBashMonitorWake(muxMetadata: unknown): boolean { + return getCarriedBashMonitorWake(muxMetadata) != null; +} + +/** + * The delegated-turn correlation a send / retry will stream under: its own metadata, or — + * for an on-send compaction request — the correlation stamped on the follow-up it carries + * (the follow-up stream inherits it from the summary). + */ +function getCarriedWorkspaceTurnCorrelation( + muxMetadata: unknown +): WorkspaceTurnMuxMetadata | undefined { + const meta = muxMetadata as MuxMessageMetadata | undefined; + if (!isCompactionRequestMetadata(meta)) return getWorkspaceTurnMuxMetadata(meta); + const followUp = meta.parsed.followUpContent; + return ( + followUp?.workspaceTurnMetadata ?? + getWorkspaceTurnMuxMetadata(followUp?.muxMetadata ?? meta.parsed.continueMessage?.muxMetadata) + ); +} + const AUTO_RETRY_PREFERENCE_FILE = "auto-retry-preference.json"; /** @@ -519,6 +557,10 @@ export const CONTEXT_MUTATION_SEND_BLOCKED_MESSAGE = const SESSION_SHUTDOWN_SEND_BLOCKED_MESSAGE = "Xum is shutting down; the message was not sent."; const STARTUP_AUTO_RETRY_HISTORY_FAILURE_BASE_DELAY_MS = 1_000; +/** Retry cadence for a voided wake continuation whose owner-side settlement failed. */ +const UNSETTLED_WAKE_VOID_RETRY_DELAY_MS = 5_000; +/** Stream starts remembered for hasCorrelatedStreamStartedAfter; older ones are evicted. */ +const RECENT_STREAM_STARTS_LIMIT = 16; const STARTUP_AUTO_RETRY_HISTORY_FAILURE_MAX_DELAY_MS = 30_000; const MAX_STARTUP_RECOVERY_DEFERRED_ATTEMPTS = 4; @@ -584,7 +626,7 @@ export interface AgentSessionAIService extends BranchSummaryAiService { ): XumToolScope; } -interface AgentSessionOptions { +export interface AgentSessionOptions { workspaceId: string; config: Config; historyService: HistoryService; @@ -626,8 +668,43 @@ interface AgentSessionOptions { * to yield to a manual send that is still awaiting pricing/settings. */ hasExternalSendPreflight?: () => boolean; + /** + * Live read of the bash-monitor wake level for this workspace + * (BashMonitorWakeReconciler.hasOutstandingWake). Consulted at tool boundaries only; the + * published mirror (setBashMonitorWakeOutstanding) lags a shown frontier, so cutting on + * the mirror alone would yield for lines the step just displayed. + */ + hasOutstandingBashMonitorWake?: () => Promise; + /** + * The tool-end yield lever rose (false → true): the next tool boundary will cut the + * stream for a tool-end queue head or the wake level. Fired on every rising edge, whatever + * caused it (enqueue, queue-head clear/promote, wake level), so foreground waits that + * would outlive the boundary can be backgrounded exactly when the yield becomes effective. + */ + onToolEndYieldRequested?: () => void; + /** + * This session will never continue the delegated turn identified by `correlation`: + * the continuation it owed (see wakeContinuationDebt) was voided, or a compaction + * follow-up carrying the correlation was abandoned. Nothing else settles that turn — no + * correlated stream-end follows — so the owner settles it now. Called as a tracked + * promise from synchronous transitions; must be idempotent. + */ + onWorkspaceTurnContinuationVoided?: ( + correlation: WorkspaceTurnMuxMetadata, + reason: WorkspaceTurnContinuationVoidReason + ) => Promise; } +/** + * Why a session stopped owing a delegated turn's continuation. + * - `retracted`: the stream yielded to a bash-monitor wake that then went away (monitor + * canceled, output shown, history cleared, send failed with no retry) before any wake turn + * could show it. + * - `superseded`: other input was admitted in its place. + * - `abandoned`: a compaction follow-up carrying the correlation was dropped undispatched. + */ +export type WorkspaceTurnContinuationVoidReason = "retracted" | "superseded" | "abandoned"; + enum TurnPhase { IDLE = "idle", PREPARING = "preparing", @@ -667,6 +744,17 @@ export class AgentSession { private readonly sanitizeCliWorkspaceRegistration?: AgentSessionOptions["sanitizeCliWorkspaceRegistration"]; private readonly onPostCompactionStateChange?: () => void; private readonly hasExternalSendPreflight?: () => boolean; + private readonly hasOutstandingBashMonitorWake?: () => Promise; + private readonly onToolEndYieldRequested?: () => void; + private readonly onWorkspaceTurnContinuationVoided?: AgentSessionOptions["onWorkspaceTurnContinuationVoided"]; + /** + * Mirror of the reconciler's wake level (setBashMonitorWakeOutstanding): a wake this + * workspace has not seen yet. Feeds the tool-end yield lever together with the queue head + * and is the sync input to hasPendingToolEndInput. + */ + private bashMonitorWakeOutstanding = false; + /** Last value pushed to the tool-end yield lever; edge detection for onToolEndYieldRequested. */ + private toolEndYieldRequested = false; private readonly emitter = new EventEmitter(); private readonly aiListeners: Array<{ event: string; handler: (...args: unknown[]) => void }> = []; @@ -849,14 +937,75 @@ export class AgentSession { /** * muxMetadata of the queued entry currently being dispatched, held from * dequeue until its sendMessage settles (the stream has started or failed). - * Lets hasPendingBashMonitorWakeContinuation see a wake continuation during - * the dequeue→stream-start window without consulting stale stream context. + * Lets hasPendingWorkspaceTurnContinuation / hasQueuedOrDispatchingEntry see the + * dequeue→stream-start window without consulting stale stream context. */ private dispatchingQueuedEntry = false; private dispatchingQueuedEntryMuxMetadata?: unknown; /** Correlation of the direct send currently in the PREPARING phase, if any. */ private preparingWorkspaceTurnMetadata?: WorkspaceTurnMuxMetadata; + /** + * Bash-monitor wake continuation model. + * + * A stream that yields to the wake level (hasPendingToolEndInput) takes out a DEBT: this + * session owes the delegated turn it cut a continuation. The debt records the cut stream's + * correlation and is settled exactly once — REDEEMED when a stream that shows the wake + * starts, or VOIDED when no wake turn can come, in which case the owner is told + * (onWorkspaceTurnContinuationVoided). Settlement never probes the wake level: the debt is + * the only thing delegated-turn settlement reads (getQueueCutCutter), and it is sync. + * + * `wakeTurnInFlight` says a wake turn is somewhere inside this session — from sendMessage / + * resumeStream entry (raised synchronously, before any await, because the wake's + * `onAccepted` lowers the level as soon as its row is durable) until the stream that shows + * it starts or the send returns with no stream and no auto-retry armed for its durable row. + * + * Transitions — each is the only place its consequence is computed: + * + * hasPendingToolEndInput yields for the level → debt = { active stream's correlation } + * wake send / resume / queue dispatch begins → wakeTurnInFlight = true + * stream-start whose request carries the wake → redeem (in-flight false, debt cleared); + * (direct wake, or a compaction FOLLOW-UP) a compaction stream itself does not + * wake send returns without a stream → in-flight stays true only while an + * auto-retry resume of its durable row is + * armed; otherwise false → maybeVoid + * level lowered (cancel / shown / clear) → maybeVoid + * maybeVoid: debt ∧ ¬inFlight ∧ ¬level → void "retracted" + * non-wake input accepted (its row durable) → void "superseded"; a turn with the + * (settleWakeDebtForAcceptedInput) same correlation ASSUMES the debt: it + * stays visible to settlement, cannot be + * retracted under the continuation, and + * its stream-start discharges it (that + * stream's own stream-end settles the turn) + * assuming send returns without a stream → un-assume unless an auto-retry of its + * durable row is armed; then maybeVoid + * compaction follow-up with the correlation → void "abandoned" (before the erase) + * dropped (clearPendingFollowUpFromSummary) + * dispose / IDLE → in-flight false (IDLE keeps the debt: + * the wake dispatcher needs an idle session) + * + * Voiding clears the debt synchronously; the owner hook may fail (handle store, waiter, + * cleanup I/O). A failed void is kept in `unsettledWakeVoids` and retried at every later + * debt transition and on a timer, so a deferred delegated handle never waits for a restart. + */ + private wakeContinuationDebt?: { correlation?: WorkspaceTurnMuxMetadata; assumed?: true }; + private wakeTurnInFlight = false; + /** + * Recent stream starts, oldest first, with the delegated-turn correlation each streamed + * under. A stream-end handler runs behind the workspace event lock and may be late: a + * same-turn continuation can have started — and finished — after the stream it handles, + * leaving nothing pending, in flight, or owed to probe. The ledger answers "did the turn + * continue after this stream?" regardless (hasCorrelatedStreamStartedAfter). + */ + private recentStreamStarts: Array<{ + messageId: string; + correlation?: WorkspaceTurnMuxMetadata; + }> = []; + private unsettledWakeVoids: Array<{ + correlation: WorkspaceTurnMuxMetadata; + reason: WorkspaceTurnContinuationVoidReason; + }> = []; + private unsettledWakeVoidRetryTimer?: ReturnType; /** Context needed to retry the current stream (cleared on stream end/abort/error). */ private activeStreamContext?: { @@ -907,6 +1056,9 @@ export class AgentSession { onIdleCompactionOutcome, onPostCompactionStateChange, hasExternalSendPreflight, + hasOutstandingBashMonitorWake, + onToolEndYieldRequested, + onWorkspaceTurnContinuationVoided, } = options; assert(typeof workspaceId === "string", "workspaceId must be a string"); @@ -935,6 +1087,9 @@ export class AgentSession { this.sanitizeCliWorkspaceRegistration = sanitizeCliWorkspaceRegistration; this.onPostCompactionStateChange = onPostCompactionStateChange; this.hasExternalSendPreflight = hasExternalSendPreflight; + this.hasOutstandingBashMonitorWake = hasOutstandingBashMonitorWake; + this.onToolEndYieldRequested = onToolEndYieldRequested; + this.onWorkspaceTurnContinuationVoided = onWorkspaceTurnContinuationVoided; this.compactionHandler = new CompactionHandler({ workspaceId: this.workspaceId, @@ -981,6 +1136,13 @@ export class AgentSession { beginShutdown(): void { this.shuttingDown = true; this.retryManager.cancel(); + this.clearUnsettledWakeVoidRetryTimer(); + } + + private clearUnsettledWakeVoidRetryTimer(): void { + if (this.unsettledWakeVoidRetryTimer == null) return; + clearTimeout(this.unsettledWakeVoidRetryTimer); + this.unsettledWakeVoidRetryTimer = undefined; } dispose(): void { @@ -992,6 +1154,22 @@ export class AgentSession { this.activePreparedTurnAbortController?.abort(); this.activePreparedTurnAbortController = null; + // The bash early-return flag is keyed by workspace id and outlives this session; a + // stale wake level would otherwise make a re-created session's long-polling reads + // return early forever. (The reconciler's own dispose lowers the level too.) Only + // touched when this session raised it: tests dispose sessions built on partial + // BackgroundProcessManager stubs. + if (this.bashMonitorWakeOutstanding) { + this.bashMonitorWakeOutstanding = false; + this.syncToolEndYieldRequested(false); + } + // No wake turn can come from a disposed session; the debt dies with it silently + // (workspace teardown settles delegated turns through its own path). + this.wakeTurnInFlight = false; + this.wakeContinuationDebt = undefined; + this.unsettledWakeVoids = []; + this.clearUnsettledWakeVoidRetryTimer(); + // Ensure any callers blocked on waitForIdle() can continue during teardown. this.setTurnPhase(TurnPhase.IDLE); @@ -1210,6 +1388,9 @@ export class AgentSession { this.autoRetryStarting = false; } this.emitRetryEvent(event); + // A retry given up (exhausted, non-retryable, disabled by the user) was the last thing + // that could still bring a durable wake row's stream. + if (event.type === "auto-retry-abandoned") this.settleWakeTurnInFlight(); } private emitRetryEvent(event: RetryStatusEvent): void { @@ -1377,6 +1558,9 @@ export class AgentSession { ); } finally { this.autoRetryStarting = false; + // The resume either started its stream (redeeming a wake it carried), re-armed a + // retry, or gave up — only the last leaves a carried wake with no way to arrive. + this.settleWakeTurnInFlight(); } } @@ -3039,7 +3223,25 @@ export class AgentSession { this.emitMetadata(metadata); } + /** + * Public send entry. A send carrying a bash-monitor wake marks the wake turn in flight + * synchronously — before the first await, hence before its `onAccepted` can lower the + * level — and settles the marker when it returns without a stream (see + * wakeContinuationDebt). + */ async sendMessage( + ...args: Parameters + ): Promise> { + const wake = carriesBashMonitorWake(args[1]?.muxMetadata); + if (wake) this.wakeTurnInFlight = true; + try { + return await this.sendMessageInner(...args); + } finally { + if (wake || this.wakeContinuationDebt?.assumed === true) this.settleWakeTurnInFlight(); + } + } + + private async sendMessageInner( message: string, options?: SendMessageOptions & { fileParts?: FilePart[] }, internal?: { @@ -3053,8 +3255,6 @@ export class AgentSession { onAccepted?: () => Promise | void; onAcceptedPreStreamFailure?: (error: SendMessageError) => Promise | void; onCanceled?: (reason: string) => Promise | void; - cancelState?: { canceledBeforeAcceptance: boolean }; - cancelSignal?: AbortSignal; /** * For queue-dispatched sends: when the user last added to the queued * entry. Goal safety compares it against the goal's explicit @@ -3069,8 +3269,8 @@ export class AgentSession { * turn claims PREPARING (isBusy() becomes true). WorkspaceService keeps * its session-invisible preflight reservation armed until this fires so * follow-up recovery cannot observe the idle gap between the service - * handoff and the busy claim (cancelBeforeAcceptance and the other - * admission awaits yield) and admit a synthetic turn ahead of the + * handoff and the busy claim (the admission awaits yield) and admit a + * synthetic turn ahead of the * accepted manual send. Refusal paths never fire it — the service's * scoped disposal releases the reservation when the call returns. */ @@ -3124,24 +3324,26 @@ export class AgentSession { const isAdmissionStale = () => internal?.admissionEpochStale?.() === true || internal?.admissionStale?.() === true; - const cancelSignal = internal?.cancelSignal; - const persistedCancelableMessageIds: string[] = []; + const persistedTurnRowMessageIds: string[] = []; // Roll back synthetic snapshots if the invoking user row fails to persist, or // later provider requests could consume orphaned context. /** - * Returns whether the rows are verifiably gone. deleteMessages can fail AFTER its atomic + * "deleted" means the rows are verifiably gone. deleteMessages can fail AFTER its atomic * rewrite committed, so a reported failure re-reads the durable history before concluding — * callers that couple side effects to the rollback (peer budget refunds) must only act when * deletion actually committed, or a "canceled" payload would stay durable while no longer - * counting against the sender's budget. + * counting against the sender's budget. "remains" is the readback confirming a row is still + * durable; "unknown" is a readback that itself failed. Callers whose side effect is only + * safe against a row that truly exists (consuming a wake lease) must not treat "unknown" + * as "remains". */ - const rollbackPersistedTurnRows = async (): Promise => { - if (persistedCancelableMessageIds.length === 0) return true; + const rollbackPersistedTurnRows = async (): Promise<"deleted" | "remains" | "unknown"> => { + if (persistedTurnRowMessageIds.length === 0) return "deleted"; const rollbackResult = await this.historyService.deleteMessages( this.workspaceId, - persistedCancelableMessageIds + persistedTurnRowMessageIds ); - if (rollbackResult.success) return true; + if (rollbackResult.success) return "deleted"; log.error("Failed to roll back partially persisted turn rows", { workspaceId: this.workspaceId, error: rollbackResult.error, @@ -3149,68 +3351,14 @@ export class AgentSession { const historyResult = await this.historyService.getHistoryFromLatestBoundary( this.workspaceId ); - return ( - historyResult.success && - persistedCancelableMessageIds.every( - (messageId) => !historyResult.data.some((message) => message.id === messageId) - ) - ); - }; - let cancellationHandled = false; - let cancellationDisabled = false; - const cancelBeforeAcceptance = async (): Promise => { - if (cancelSignal?.aborted !== true || cancellationDisabled) return false; - if (cancellationHandled) return true; - - if (persistedCancelableMessageIds.length > 0) { - // History also has non-session writers (for example goal pause boundaries). Delete exactly - // this preparing turn's rows in one atomic rewrite so later concurrent rows are preserved. - const rollbackResult = await this.historyService.deleteMessages( - this.workspaceId, - persistedCancelableMessageIds - ); - if (!rollbackResult.success) { - // deleteMessages can fail after its atomic rewrite (for example while refreshing - // sequence metadata). Verify the durable result before deciding whether cancellation won. - const historyResult = await this.historyService.getHistoryFromLatestBoundary( - this.workspaceId - ); - const rollbackCommitted = - historyResult.success && - persistedCancelableMessageIds.every( - (messageId) => !historyResult.data.some((message) => message.id === messageId) - ); - if (!rollbackCommitted) { - // Do not report cancellation (which would supersede the durable monitor wake) unless the - // not-yet-accepted row is actually gone. Continue accepting this wake instead of leaving - // a hidden synthetic row that can leak into a later provider request. - cancellationDisabled = true; - log.error("Failed to roll back canceled preparing turn; continuing acceptance", { - workspaceId: this.workspaceId, - error: rollbackResult.error, - verificationError: historyResult.success ? undefined : historyResult.error, - }); - return false; - } - log.warn("Preparing-turn rollback reported failure after its rewrite committed", { - workspaceId: this.workspaceId, - error: rollbackResult.error, - }); - } - } - - cancellationHandled = true; - await internal?.onCanceled?.(cancelReasonBeforeAcceptance(cancelSignal)); - if (internal?.cancelState != null) { - internal.cancelState.canceledBeforeAcceptance = true; - } - return true; + if (!historyResult.success) return "unknown"; + return persistedTurnRowMessageIds.every( + (messageId) => !historyResult.data.some((message) => message.id === messageId) + ) + ? "deleted" + : "remains"; }; - if (await cancelBeforeAcceptance()) { - return Ok(undefined); - } - // Last-line-of-defence pricing gate: every dispatch path (initial sends, // sendQueuedMessages, dispatchPendingFollowUp, // post-compaction follow-ups) lands here, so a budgeted goal that became @@ -3232,9 +3380,6 @@ export class AgentSession { this.workspaceId, options?.model ); - if (await cancelBeforeAcceptance()) { - return Ok(undefined); - } if (!pricingGate.success) { if (isManualUserMessage) { const persisted = await this.preserveRejectedManualSend( @@ -3649,10 +3794,6 @@ export class AgentSession { // File changes after this point are surfaced via diffs instead. const snapshotResult = await this.materializeFileAtMentionsSnapshot(trimmedMessage); - if (await cancelBeforeAcceptance()) { - return Ok(undefined); - } - // Check compaction threshold BEFORE persisting the user message. // Skill snapshots are materialized AFTER this decision (below): when on-send // compaction defers the turn, the follow-up re-enters sendMessage with the same @@ -3677,9 +3818,6 @@ export class AgentSession { // so the compaction monitor can detect context limits even before any live // stream events have populated lastUsageState. await this.seedUsageStateFromHistory(); - if (await cancelBeforeAcceptance()) { - return Ok(undefined); - } const providersConfigForCompaction = this.getProvidersConfigSafe(); const compactionResult = this.compactionMonitor.checkBeforeSend({ @@ -3784,10 +3922,7 @@ export class AgentSession { if (!appendCompactionResult.success) { return Err(createUnknownSendMessageError(appendCompactionResult.error)); } - persistedCancelableMessageIds.push(autoCompactionMessage.id); - if (await cancelBeforeAcceptance()) { - return Ok(undefined); - } + persistedTurnRowMessageIds.push(autoCompactionMessage.id); this.emitChatEvent({ type: "auto-compaction-triggered", @@ -3813,13 +3948,41 @@ export class AgentSession { // refuse while sends are in preflight (r42), so rows can no longer land // after a mutation commits; this check and the PREPARING gate remain // backstops for entry-accounting bypasses. + // + // "Pre-persist" has one exception: the on-send compaction row above is already durable and + // carries this send as its follow-up. Refusing without removing it would leave a compaction + // request that startup recovery later resumes — for a bash-monitor wake whose lease is + // released by this refusal, that resubmits output the reconciler has already re-derived. + // + // When the rollback fails and the readback CONFIRMS the row still durable, that row already + // carries the wake as its follow-up. Leaving the lease released would have the reconciler + // redeliver output the row carries — twice, one copy of which the monitor may since have + // retracted. So the wake takes the same handoff as every other durable pre-stream failure + // (see the goal-sync catch below): arm the in-session resume, consume the lease, schedule + // the retry. The resume is what keeps the row reachable — startup recovery only resumes an + // interrupted history *tail*, and a competing manual send (one cause of this refusal) could + // otherwise complete a newer turn on top of the buried request. The refusal still stands. + // + // An "unknown" readback must NOT consume: if the row was in fact deleted, consuming would + // leave neither carrier nor signal and the wake (and any delegated continuation deferred + // behind it) would be lost. Releasing risks at worst a duplicate delivery, which the + // reconciler's watermarks and the wake-debt settlement already tolerate. + const refuseAfterCompactionRow = async (message: string): Promise> => { + const outcome = await rollbackPersistedTurnRows(); + if (outcome === "remains" && typedMuxMetadata?.type === "bash-monitor-wake") { + this.setAutoRetryResumeState(optionsForStream, agentInitiated, goalKind, internal?.goalId); + await internal?.onAccepted?.(); + await this.handleStreamFailureForAutoRetry({ type: "unknown", message }); + } + return Err(createUnknownSendMessageError(message)); + }; if (this.turnAdmissionBlocks > 0 || isAdmissionStale()) { - return Err(createUnknownSendMessageError(CONTEXT_MUTATION_SEND_BLOCKED_MESSAGE)); + return refuseAfterCompactionRow(CONTEXT_MUTATION_SEND_BLOCKED_MESSAGE); } - // Still pre-persist: a row appended now would read as a dispatched turn on the next startup - // while streamWithHistory's own latch check keeps its stream from ever running. + // A row appended now would read as a dispatched turn on the next startup while + // streamWithHistory's own latch check keeps its stream from ever running. if (this.shuttingDown) { - return Err(createUnknownSendMessageError(SESSION_SHUTDOWN_SEND_BLOCKED_MESSAGE)); + return refuseAfterCompactionRow(SESSION_SHUTDOWN_SEND_BLOCKED_MESSAGE); } // Persist snapshots only when this turn will be sent immediately. @@ -3836,15 +3999,11 @@ export class AgentSession { ); mcpPromptSnapshotMessages = await this.materializeMcpPromptSnapshots( typedMuxMetadata, - userMessage.id, - cancelSignal + userMessage.id ); } catch (error) { return Err(createUnknownSendMessageError(getErrorMessage(error))); } - if (await cancelBeforeAcceptance()) { - return Ok(undefined); - } } if (shouldPersistTurnSnapshots && snapshotResult?.snapshotMessage) { @@ -3855,10 +4014,7 @@ export class AgentSession { if (!snapshotAppendResult.success) { return Err(createUnknownSendMessageError(snapshotAppendResult.error)); } - persistedCancelableMessageIds.push(snapshotResult.snapshotMessage.id); - if (await cancelBeforeAcceptance()) { - return Ok(undefined); - } + persistedTurnRowMessageIds.push(snapshotResult.snapshotMessage.id); } if (shouldPersistTurnSnapshots && skillSnapshotMessages.length > 0) { @@ -3871,10 +4027,7 @@ export class AgentSession { await rollbackPersistedTurnRows(); return Err(createUnknownSendMessageError(skillSnapshotAppendResult.error)); } - persistedCancelableMessageIds.push(snapshotMessage.id); - if (await cancelBeforeAcceptance()) { - return Ok(undefined); - } + persistedTurnRowMessageIds.push(snapshotMessage.id); } } @@ -3888,10 +4041,7 @@ export class AgentSession { await rollbackPersistedTurnRows(); return Err(createUnknownSendMessageError(appendResult.error)); } - persistedCancelableMessageIds.push(snapshotMessage.id); - if (await cancelBeforeAcceptance()) { - return Ok(undefined); - } + persistedTurnRowMessageIds.push(snapshotMessage.id); } } @@ -3920,13 +4070,10 @@ export class AgentSession { await rollbackPersistedTurnRows(); return Err(createUnknownSendMessageError(batchAppendResult.error)); } - persistedCancelableMessageIds.push( + persistedTurnRowMessageIds.push( ...internal.preTurnMessages.map((message) => message.id), userMessage.id ); - if (await cancelBeforeAcceptance()) { - return Ok(undefined); - } } else if (!autoCompactionMessage) { // When on-send compaction triggers, the user message is NOT persisted to // history (it's sent as follow-up after compaction). Otherwise, persist @@ -3936,10 +4083,7 @@ export class AgentSession { await rollbackPersistedTurnRows(); return Err(createUnknownSendMessageError(appendResult.error)); } - persistedCancelableMessageIds.push(userMessage.id); - if (await cancelBeforeAcceptance()) { - return Ok(undefined); - } + persistedTurnRowMessageIds.push(userMessage.id); } // Caller-probe staleness (peer sends racing a Stop) must resolve BEFORE the pre-turn batch @@ -3949,13 +4093,14 @@ export class AgentSession { // rollback is forbidden by design (goal sync observes the durable row), so a Stop landing in // the remaining pre-stream awaits refuses the turn at the PREPARING gate with rows retained. if (internal?.admissionStale?.() === true) { - const rolledBack = await rollbackPersistedTurnRows(); + const rollbackOutcome = await rollbackPersistedTurnRows(); // Probe-carrying sends are peer messages whose caller already returned success when the // entry was queued — the cancellation hook is their only way to observe this refusal and // release the budget reservation (the refund closure is idempotent). Fire it ONLY when the - // rollback verifiably committed: rows that remain durable can enter provider context after - // a resume, so their charge must stand (budget charged ⇔ rows durable). - if (rolledBack) { + // rollback verifiably committed: rows that remain durable (or whose fate is unknown) can + // enter provider context after a resume, so their charge must stand (budget charged ⇔ + // rows durable). + if (rollbackOutcome === "deleted") { await internal?.onCanceled?.( "Send refused: the caller's admission became stale before the turn was accepted." ); @@ -3973,12 +4118,11 @@ export class AgentSession { ); } - // Goal synchronization can mutate goal.json based on this durable user row. Once it begins, the - // turn has crossed the cancellation point-of-no-return: a concurrent monitor stop must let this - // wake finish acceptance rather than delete the row after goal state has already observed it. - if (cancelSignal != null) { - cancellationDisabled = true; - } + // The user row is durable from here on. A bash-monitor wake whose row is durable must be + // accepted even if a later step throws: otherwise the reconciler re-derives the same wake + // and delivers it twice (startup recovery resumes the durable row without redelivery). + const finalizeDurableWakeOnFailure = typedMuxMetadata?.type === "bash-monitor-wake"; + this.settleWakeDebtForAcceptedInput(typedMuxMetadata); // r54: the pre-turn batch is now irrevocable — rollbackPersistedTurnRows // is never invoked past this point, so even a failure in goal sync or // acceptance leaves the payload + trigger rows durable in the transcript. @@ -3988,10 +4132,21 @@ export class AgentSession { try { await this.workspaceGoalService?.syncGoalModeWithChatTail(this.workspaceId); } catch (error) { - if (cancelSignal != null) { - // The durable row crossed the point of no return, so every later goal-sync failure must still - // finalize this monitor wake. Startup recovery can resume the row without redelivering it. + if (finalizeDurableWakeOnFailure) { + // Consuming the signal is what stops the reconciler from re-deriving this wake, so + // from that moment the durable row is the only carrier of the turn and nothing + // upstream can resend it. Arm the same in-session resume a failed stream start uses + // BEFORE consuming, then schedule it: auto-retry resumes the durable row without + // appending a second one, so the row (and any delegated turn waiting on its + // continuation) no longer depends on an application restart. With auto-retry + // disabled by the user this stays a startup-recovery row like every other + // pre-stream failure, and the wake turn leaves flight (see settleWakeTurnInFlight). + this.setAutoRetryResumeState(optionsForStream, agentInitiated, goalKind, internal?.goalId); await internal?.onAccepted?.(); + await this.handleStreamFailureForAutoRetry({ + type: "unknown", + message: getErrorMessage(error), + }); } throw error; } @@ -4004,10 +4159,10 @@ export class AgentSession { } // Workspace may be tearing down while we await filesystem IO. - // If so, skip event emission + streaming to avoid races with dispose(). A cancelable monitor - // wake past the point of no return is already durable, so finalize it before leaving. + // If so, skip event emission + streaming to avoid races with dispose(). A monitor wake's + // row is already durable, so finalize it before leaving. if (this.disposed) { - if (cancelSignal != null && cancellationDisabled) { + if (finalizeDurableWakeOnFailure) { await internal?.onAccepted?.(); } return Ok(undefined); @@ -4118,8 +4273,7 @@ export class AgentSession { const preparedTurnAbortController = new AbortController(); this.activePreparedTurnAbortController = preparedTurnAbortController; - this.preparingWorkspaceTurnMetadata = getWorkspaceTurnMuxMetadata(optionsForStream.muxMetadata); - this.setTurnPhase(TurnPhase.PREPARING); + this.enterPreparing(optionsForStream.muxMetadata); // From this synchronous point isBusy() reports the turn — release the // service-side preflight reservation (see onTurnAdmissionCommitted doc). internal?.onTurnAdmissionCommitted?.(); @@ -4133,7 +4287,7 @@ export class AgentSession { return Ok(undefined); } // Background processes are workspace-scoped, not context-scoped. Compaction must preserve - // processes, monitors, and queued wakes so a waiting agent is not stranded. + // processes, monitors, and pending wakes so a waiting agent is not stranded. // Note: Follow-up content for compaction is now stored on the summary message // and dispatched via dispatchPendingFollowUp() after compaction completes. // This provides crash safety - the follow-up survives app restarts. @@ -4222,7 +4376,20 @@ export class AgentSession { return await startPreparedStream(); } + /** Like sendMessage, marks a carried wake in flight for the duration of the resume. */ async resumeStream( + ...args: Parameters + ): Promise> { + const wake = carriesBashMonitorWake(args[0].muxMetadata); + if (wake) this.wakeTurnInFlight = true; + try { + return await this.resumeStreamInner(...args); + } finally { + if (wake || this.wakeContinuationDebt?.assumed === true) this.settleWakeTurnInFlight(); + } + } + + private async resumeStreamInner( options: SendMessageOptions, internal?: { agentInitiated?: boolean; goalKind?: GoalSyntheticMessageKind; goalId?: string } ): Promise> { @@ -4271,8 +4438,7 @@ export class AgentSession { internal?.goalKind, internal?.goalId ); - this.preparingWorkspaceTurnMetadata = getWorkspaceTurnMuxMetadata(optionsForStream.muxMetadata); - this.setTurnPhase(TurnPhase.PREPARING); + this.enterPreparing(optionsForStream.muxMetadata); // Open the mid-turn thinking override window for the resumed turn (after // setTurnPhase(PREPARING), which clears the holder on the IDLE transition). const turnThinkingOverride: ActiveTurnThinkingOverride = {}; @@ -4578,6 +4744,15 @@ export class AgentSession { if (params.muxMetadata) { followUp.muxMetadata = params.muxMetadata; + // A bash-monitor wake is an idle-only send (WorkspaceService.dispatchBashMonitorWake + // sends it with requireIdle). The compaction hand-off must keep that rule so a manual + // message queued or in preflight during the compaction stream still wins over the + // wake continuation (dispatchPendingFollowUp). A skipped wake follow-up is lost — its + // signals were consumed at acceptance — which is the bounded cost of letting the + // user's correction go first; the process output stays readable via task_await. + if (params.muxMetadata.type === "bash-monitor-wake") { + followUp.dispatchOptions = { ...followUp.dispatchOptions, requireIdle: true }; + } } if (params.workspaceTurnMetadata) { @@ -5210,7 +5385,7 @@ export class AgentSession { const optionsMuxMetadata = options?.muxMetadata as MuxMessageMetadata | undefined; const retryMuxMetadata = lastUserMessage?.metadata?.muxMetadata; // Bash-monitor-wake continuations inherit the correlation of a delegated - // workspace turn that was cut mid-work by the wake's queued dispatch, so + // workspace turn whose stream yielded mid-work to the wake, so // the turn's eventual terminal stream-end can settle the parent's handle. const streamMuxMetadata = optionsMuxMetadata?.type === "workspace-turn-task" @@ -5267,7 +5442,7 @@ export class AgentSession { experiments: options?.experiments, disableWorkspaceAgents: options?.disableWorkspaceAgents, strictAgentResolution: options?.strictAgentResolution, - hasQueuedMessages: this.hasQueuedMessages.bind(this), + hasPendingToolEndInput: () => this.hasPendingToolEndInput(), openaiTruncationModeOverride, // Mid-turn thinking overrides clamp against the same floor as the // send-time level above (single source of truth for the floor). @@ -5516,10 +5691,7 @@ export class AgentSession { retryGoalKind, retryGoalId ); - this.preparingWorkspaceTurnMetadata = getWorkspaceTurnMuxMetadata( - retryOptionsForResume.muxMetadata - ); - this.setTurnPhase(TurnPhase.PREPARING); + this.enterPreparing(retryOptionsForResume.muxMetadata); let retryResult: Result; try { retryResult = await this.streamWithHistory( @@ -5625,8 +5797,7 @@ export class AgentSession { } // Retry the same request, but without post-compaction injection. - this.preparingWorkspaceTurnMetadata = getWorkspaceTurnMuxMetadata(context.options?.muxMetadata); - this.setTurnPhase(TurnPhase.PREPARING); + this.enterPreparing(context.options?.muxMetadata); let retryResult: Result; try { retryResult = await this.streamWithHistory( @@ -5847,6 +6018,33 @@ export class AgentSession { this.dispatchingQueuedEntry = false; this.dispatchingQueuedEntryMuxMetadata = undefined; this.preparingWorkspaceTurnMetadata = undefined; + this.recentStreamStarts.push({ + messageId: payload.messageId, + correlation: this.activeStreamContext?.workspaceTurnMetadata, + }); + if (this.recentStreamStarts.length > RECENT_STREAM_STARTS_LIMIT) { + this.recentStreamStarts.shift(); + } + // A stream that shows the wake redeems the continuation debt (see + // wakeContinuationDebt). Only the wake row's own stream qualifies: an on-send + // compaction stream's request is the compaction row, and the wake follows it. + const streamMetadata = this.activeStreamContext?.options?.muxMetadata as + | MuxMessageMetadata + | undefined; + if (streamMetadata?.type === "bash-monitor-wake") { + this.wakeTurnInFlight = false; + this.wakeContinuationDebt = undefined; + } else if ( + this.wakeContinuationDebt != null && + hasSameWorkspaceTurnCorrelation( + getWorkspaceTurnMuxMetadata(streamMetadata), + this.wakeContinuationDebt.correlation + ) + ) { + // The delegated turn's own continuation is streaming: its stream-end settles the + // turn, so the debt it assumed at acceptance is discharged. + this.wakeContinuationDebt = undefined; + } this.activeStreamStartedAtMs = payload.startTime; // Codex P1 (PRRT_kwDOPxxmWM6cClKS): a new live stream makes mid-stream // setGoal deferral meaningful again — clear the goal service's settled @@ -6195,7 +6393,7 @@ export class AgentSession { if (this.deferQueuedFlushUntilAfterEdit) { this.queuedProviderToolEndAbortInFlight = false; // Clear the queued-message signal while the edit flow owns the next dispatch. - this.backgroundProcessManager.setMessageQueued(this.workspaceId, false); + this.syncToolEndYieldRequested(false); // Do not dispatch stream-end follow-ups while the edit flow is waiting // for IDLE; truncation must run before any synthetic turn resumes. } else { @@ -6345,6 +6543,7 @@ export class AgentSession { this.dispatchingQueuedEntry = false; this.dispatchingQueuedEntryMuxMetadata = undefined; this.preparingWorkspaceTurnMetadata = undefined; + this.settleWakeTurnInFlight(); // Turn ended: expire any mid-turn thinking override. Safe unconditionally // because a replacement turn (e.g. an edit) only creates its holder after // the preempted turn has already been transitioned to IDLE. @@ -6562,8 +6761,6 @@ export class AgentSession { onAccepted?: () => Promise | void; onAcceptedPreStreamFailure?: (error: SendMessageError) => Promise | void; onCanceled?: (reason: string) => Promise | void; - cancelState?: { canceledBeforeAcceptance: boolean }; - cancelSignal?: AbortSignal; /** Synthetic assistant rows persisted just before the dispatched turn's user row. */ preTurnMessages?: MuxMessage[]; /** r54: fired once pre-turn rows cross the rollback horizon at dispatch. */ @@ -6582,15 +6779,11 @@ export class AgentSession { } this.emitQueuedMessageChanged(); // Signal to bash_output that it should return early to process queued messages - // only for tool-end dispatches. Return the same mode so the caller's foreground - // task waits follow the entry that will actually run, not a withdrawn FIFO head. + // only for tool-end dispatches. Return the FIFO head's mode so the caller's foreground + // task waits follow the entry that will actually run next, not the one just added. const nextDispatchableMode = this.messageQueue.getNextDispatchableMode(); - this.backgroundProcessManager.setMessageQueued( - this.workspaceId, - nextDispatchableMode === "tool-end" - ); - // Undefined only if the entry just added is itself withdrawn; WorkspaceService.sendMessage - // refuses those before enqueue, so null keeps its "nothing pending was queued" meaning. + this.syncToolEndYieldRequested(nextDispatchableMode === "tool-end"); + // The queue is non-empty right after a successful add, so this is never null here. return nextDispatchableMode ?? null; } @@ -6599,7 +6792,7 @@ export class AgentSession { const callbackSets = this.messageQueue.getClearCallbacks(); this.messageQueue.clear(); this.emitQueuedMessageChanged(); - this.backgroundProcessManager.setMessageQueued(this.workspaceId, false); + this.syncToolEndYieldRequested(false); for (const callbacks of callbackSets) { this.notifyQueuedMessageCleared(callbacks, cancelReason); } @@ -6615,10 +6808,7 @@ export class AgentSession { this.emitQueuedMessageChanged(); // Only the FIFO head can dispatch next; later hidden entries must not pull an earlier // user-authored turn-end entry forward to a step boundary. - this.backgroundProcessManager.setMessageQueued( - this.workspaceId, - this.messageQueue.getNextDispatchableMode() === "tool-end" - ); + this.syncToolEndYieldRequested(); return true; } @@ -6652,10 +6842,7 @@ export class AgentSession { return 0; } this.emitQueuedMessageChanged(); - this.backgroundProcessManager.setMessageQueued( - this.workspaceId, - this.messageQueue.getNextDispatchableMode() === "tool-end" - ); + this.syncToolEndYieldRequested(); for (const callbacks of removal.callbacks) { this.notifyQueuedMessageCleared(callbacks, cancelReason); } @@ -6680,20 +6867,224 @@ export class AgentSession { return false; } this.emitQueuedMessageChanged(); - this.backgroundProcessManager.setMessageQueued( - this.workspaceId, - this.messageQueue.getNextDispatchableMode() === "tool-end" - ); + this.syncToolEndYieldRequested(); this.notifyQueuedMessageCleared(callbacks, cancelReason); return true; } - /** Pending work only: withdrawn (aborted) entries still occupy the queue but never start a turn. */ + /** Whether the FIFO head (the only entry the next drain can send) matches `dispatchMode`. */ hasQueuedMessages(dispatchMode?: "tool-end" | "turn-end"): boolean { const nextMode = this.messageQueue.getNextDispatchableMode(); return nextMode != null && (dispatchMode == null || nextMode === dispatchMode); } + /** + * Stream stop condition: input that must run at a tool boundary is pending. A queued + * tool-end message is an edge we hold; a bash-monitor wake is a level read from the + * reconciler, so a wake whose lines this very step showed (task_await on the monitored + * process) is already gone by the time the SDK asks — the stream keeps going. + * + * A non-empty queue arbitrates alone: its head runs next whatever the level says (the + * wake dispatcher waits for an empty queue), so cutting for the wake behind a turn-end + * head would only promote that entry to tool-end. A message queued during the level read + * arbitrates the same way. + * + * The SDK only asks when the loop would otherwise continue, so a true from the level IS + * the cut: the continuation debt is taken here, in the same synchronous step as the + * decision (see wakeContinuationDebt). + */ + async hasPendingToolEndInput(): Promise { + const nextMode = this.messageQueue.getNextDispatchableMode(); + if (nextMode != null) return nextMode === "tool-end"; + if (this.hasOutstandingBashMonitorWake == null) return false; + try { + const outstanding = await this.hasOutstandingBashMonitorWake(); + const modeAfterRead = this.messageQueue.getNextDispatchableMode(); + if (modeAfterRead != null) return modeAfterRead === "tool-end"; + if (!outstanding) return false; + this.wakeContinuationDebt ??= { + correlation: this.activeStreamContext?.workspaceTurnMetadata, + }; + return true; + } catch (error) { + log.debug("hasPendingToolEndInput: wake level read failed; not yielding on the level", { + workspaceId: this.workspaceId, + error, + }); + // A message queued while the level read was in flight still arbitrates this boundary: + // the read's failure says nothing about the queue. + return this.messageQueue.getNextDispatchableMode() === "tool-end"; + } + } + + /** + * Mirror the reconciler's wake level. While high, long-polling bash reads return early + * so the stream reaches a tool boundary (same lever a queued tool-end message pulls). + * Lowered with no wake turn in flight, an owed continuation can no longer arrive. + */ + setBashMonitorWakeOutstanding(outstanding: boolean): void { + if (this.bashMonitorWakeOutstanding === outstanding) return; + this.bashMonitorWakeOutstanding = outstanding; + this.syncToolEndYieldRequested(); + if (!outstanding) this.maybeVoidWakeContinuation(); + } + + /** A wake turn is inside this session (see wakeContinuationDebt). */ + hasPendingBashMonitorWakeTurn(): boolean { + return this.wakeTurnInFlight; + } + + /** + * A stream that yielded to the wake level will still be continued by a wake turn: the + * debt is outstanding (its wake not yet dispatched) or the wake turn is already in flight. + * Delegated-turn settlement reads this instead of probing the reconciler. + */ + hasBashMonitorWakeContinuation(): boolean { + return this.wakeContinuationDebt != null || this.wakeTurnInFlight; + } + + /** + * A wake send / resume returned, the turn went idle, or a retry was given up: the wake + * turn is still in flight only while something inside this session can still start its + * stream — a turn in progress (its stream, or a compaction whose follow-up is the wake), + * or an auto-retry armed for its durable row. + */ + private settleWakeTurnInFlight(): void { + const assumed = this.wakeContinuationDebt?.assumed === true; + if (!this.wakeTurnInFlight && !assumed) return; + // A turn in progress (the wake's / continuation's own stream, PREPARING, or a compaction + // stream whose follow-up is that turn) can still start the stream. + if (this.turnPhase !== TurnPhase.IDLE) return; + const retryMetadata: unknown = this.hasPendingAutoRetry() + ? this.lastAutoRetryResumeRequest?.options.muxMetadata + : undefined; + if (this.wakeTurnInFlight && !carriesBashMonitorWake(retryMetadata)) { + this.wakeTurnInFlight = false; + } + if ( + assumed && + this.wakeContinuationDebt != null && + !hasSameWorkspaceTurnCorrelation( + // A retry of an on-send compaction still starts the correlated follow-up behind it. + getCarriedWorkspaceTurnCorrelation(retryMetadata), + this.wakeContinuationDebt.correlation + ) + ) { + // The continuation that assumed the debt is not coming from this send; the debt is + // plain owed again (the wake, if still outstanding, will discharge it). + delete this.wakeContinuationDebt.assumed; + } + this.maybeVoidWakeContinuation(); + } + + private maybeVoidWakeContinuation(): void { + this.retryUnsettledWakeVoids(); + if ( + this.wakeContinuationDebt != null && + this.wakeContinuationDebt.assumed !== true && + !this.wakeTurnInFlight && + !this.bashMonitorWakeOutstanding + ) { + this.voidWakeContinuation("retracted"); + } + } + + /** + * Input other than the wake has crossed its acceptance boundary (row durable, so the send + * can no longer fail without leaving a resumable turn). Only now can it settle the debt: a + * send refused earlier — pricing, staleness, persistence — leaves the wake outstanding and + * its continuation still owed. Synchronous with the PREPARING reservation this is not: + * enterPreparing only reserves the turn. + * + * A continuation of the delegated turn itself does not clear the debt here: between this + * point and its PREPARING / stream-start nothing else would tell settlement that the turn + * continues, and a stream-end handler running in that gap would settle (and, for a + * disposable turn, remove the workspace under) the accepted continuation. It assumes the + * debt instead; its stream-start discharges it. + */ + private settleWakeDebtForAcceptedInput(muxMetadata: unknown): void { + if (this.wakeContinuationDebt == null || carriesBashMonitorWake(muxMetadata)) return; + if ( + hasSameWorkspaceTurnCorrelation( + getWorkspaceTurnMuxMetadata(muxMetadata), + this.wakeContinuationDebt.correlation + ) + ) { + this.wakeContinuationDebt.assumed = true; + this.retryUnsettledWakeVoids(); + } else { + this.voidWakeContinuation("superseded"); + } + } + + /** + * Sync transition; the owner hook runs as a tracked promise (never awaited here — callers + * sit inside admission and phase transitions). + */ + private voidWakeContinuation(reason: WorkspaceTurnContinuationVoidReason): void { + const debt = this.wakeContinuationDebt; + this.wakeContinuationDebt = undefined; + this.retryUnsettledWakeVoids(); + const correlation = debt?.correlation; + if (correlation == null) return; + this.settleVoidedWakeContinuation({ correlation, reason }); + } + + /** Runs the owner hook; a failure parks the void for retry instead of dropping it. */ + private settleVoidedWakeContinuation(voided: { + correlation: WorkspaceTurnMuxMetadata; + reason: WorkspaceTurnContinuationVoidReason; + }): void { + if (this.onWorkspaceTurnContinuationVoided == null) return; + this.onWorkspaceTurnContinuationVoided(voided.correlation, voided.reason).catch( + (error: unknown) => { + log.error("Voided bash-monitor wake continuation could not be settled; will retry", { + workspaceId: this.workspaceId, + reason: voided.reason, + error: getErrorMessage(error), + }); + if (this.disposed || this.shuttingDown) return; + this.unsettledWakeVoids.push(voided); + if (this.unsettledWakeVoidRetryTimer == null) { + this.unsettledWakeVoidRetryTimer = setTimeout(() => { + this.unsettledWakeVoidRetryTimer = undefined; + this.retryUnsettledWakeVoids(); + }, UNSETTLED_WAKE_VOID_RETRY_DELAY_MS); + // Recovery must not keep a graceful process exit alive (beginShutdown also cancels). + this.unsettledWakeVoidRetryTimer.unref?.(); + } + } + ); + } + + private retryUnsettledWakeVoids(): void { + if (this.unsettledWakeVoids.length === 0) return; + const pending = this.unsettledWakeVoids; + this.unsettledWakeVoids = []; + for (const voided of pending) this.settleVoidedWakeContinuation(voided); + } + + /** + * Tool-end yield flag = queue head is tool-end ∪ (queue empty ∧ wake level), mirroring + * hasPendingToolEndInput's arbitration. `queueHeadToolEnd` lets stream-end / clear paths + * assert the queue contribution is gone before the queue itself is observed empty. + * + * This is the single place the effective flag is computed, so it also owns the rising + * edge: a turn-end head that is cleared, removed, or promoted while the wake level is high + * makes the lever effective without any enqueue or level publish, and the consequence + * (backgrounding foreground waits) must follow that edge, not those events. + */ + private syncToolEndYieldRequested( + queueHeadToolEnd = this.messageQueue.getNextDispatchableMode() === "tool-end" + ): void { + const requested = + queueHeadToolEnd || (this.bashMonitorWakeOutstanding && this.messageQueue.isEmpty()); + const rising = requested && !this.toolEndYieldRequested; + this.toolEndYieldRequested = requested; + this.backgroundProcessManager.setMessageQueued(this.workspaceId, requested); + if (rising) this.onToolEndYieldRequested?.(); + } + /** Queued intra-tree agent peer messages awaiting dispatch (peer-message queue cap input). */ countQueuedAgentPeerMessages(): number { return this.messageQueue.countAgentPeerMessageEntries(); @@ -6737,26 +7128,45 @@ export class AgentSession { } /** - * Whether a bash-monitor-wake continuation is pending dispatch: the next - * queued entry is a wake, or a dequeued wake is mid-dispatch (dequeue → - * stream start). Wake sends are the only input that inherits an open - * delegated workspace turn's correlation, so TaskService uses this — not - * generic queued/preparing state — to decide whether a correlated - * "tool-calls" queue cut will be continued rather than superseded. Once the - * wake's stream starts, TaskService matches the active stream's inherited - * correlation instead (see hasSameTurnWakeContinuation). + * Claim PREPARING for a send and record what kind of input it carries. A reservation only: + * an outstanding wake continuation debt is settled when the input is accepted + * (settleWakeDebtForAcceptedInput), not when it is admitted. */ - hasPendingBashMonitorWakeContinuation(): boolean { - if (this.messageQueue.isNextEntryBashMonitorWake()) { - return true; - } - const dispatching = this.dispatchingQueuedEntryMuxMetadata as MuxMessageMetadata | undefined; - return dispatching?.type === "bash-monitor-wake"; + private enterPreparing(muxMetadata: unknown): void { + this.preparingWorkspaceTurnMetadata = getWorkspaceTurnMuxMetadata(muxMetadata); + this.setTurnPhase(TurnPhase.PREPARING); } /** * Whether a queued or dispatching entry continues the exact workspace-turn correlation. */ + /** + * Whether a stream carrying `correlation` started after every stream in `messageIds` + * (running or already ended). A message id the ledger no longer holds was evicted by + * later starts (or predates this session instance), so every remembered start is after it. + */ + hasCorrelatedStreamStartedAfter( + correlation: { taskHandleId: string; ownerWorkspaceId: string; turnId: string }, + messageIds: readonly string[] + ): boolean { + let after = -1; + for (const messageId of messageIds) { + after = Math.max( + after, + this.recentStreamStarts.findIndex((entry) => entry.messageId === messageId) + ); + } + return this.recentStreamStarts + .slice(after + 1) + .some( + (entry) => + entry.correlation != null && + entry.correlation.taskHandleId === correlation.taskHandleId && + entry.correlation.ownerWorkspaceId === correlation.ownerWorkspaceId && + entry.correlation.turnId === correlation.turnId + ); + } + hasPendingWorkspaceTurnContinuation( metadata: Extract ): boolean { @@ -6804,7 +7214,10 @@ export class AgentSession { return { stage: "dispatching", muxMetadata: this.dispatchingQueuedEntryMuxMetadata }; } const candidate = this.messageQueue.getNextQueueCutCandidate(); - return candidate != null ? { stage: "queued", ...candidate } : undefined; + if (candidate != null) return { stage: "queued", ...candidate }; + // No input holds the session: the stream itself yielded to the wake level and the + // continuation is still owed (see wakeContinuationDebt). + return this.wakeContinuationDebt != null ? { stage: "bash-monitor-wake" } : undefined; } /** Whether a message queued with this dedupe key is still pending (see MessageQueue.addOnce). */ @@ -6860,7 +7273,6 @@ export class AgentSession { return false; } - // Physical check: withdrawn entries must still drain so their onCanceled fires. const shouldDispatch = abortReason !== "user" && !this.deferQueuedFlushUntilAfterEdit && @@ -7014,7 +7426,7 @@ export class AgentSession { this.queuedProviderToolEndAbortInFlight = false; // Clear the queued message flag (even if queue is empty, to handle race conditions) - this.backgroundProcessManager.setMessageQueued(this.workspaceId, false); + this.syncToolEndYieldRequested(false); if (!this.messageQueue.isEmpty()) { // Entries dispatch one at a time (FIFO): special sends (compaction, agent @@ -7027,15 +7439,11 @@ export class AgentSession { // Re-arm dispatch signals for the remaining entries so the stream we are // about to start drains them at its next tool end (or stream end). - this.backgroundProcessManager.setMessageQueued( - this.workspaceId, - this.messageQueue.getNextDispatchableMode() === "tool-end" - ); + this.syncToolEndYieldRequested(); // Set PREPARING synchronously before the async sendMessage to prevent // incoming messages from bypassing the queue during the await gap. - this.preparingWorkspaceTurnMetadata = getWorkspaceTurnMuxMetadata(options?.muxMetadata); - this.setTurnPhase(TurnPhase.PREPARING); + this.enterPreparing(options?.muxMetadata); void this.sendMessage(message, options, { ...internal, enqueuedAtMs }) .then(async (result) => { @@ -7055,14 +7463,6 @@ export class AgentSession { this.sendQueuedMessages(); return; } - if (internal?.cancelState?.canceledBeforeAcceptance === true) { - // Cancellation can arrive after dequeue while sendMessage is validating or writing - // history. No stream will start, so release PREPARING and continue with later entries. - if (this.turnPhase === TurnPhase.PREPARING) { - this.setTurnPhase(TurnPhase.IDLE); - } - this.sendQueuedMessages(); - } }) .catch(async (error: unknown) => { // A REJECTED sendMessage (thrown, not returned Err — e.g. an awaited history or goal @@ -7574,9 +7974,55 @@ export class AgentSession { muxMetadata: muxMetadataWithoutFollowUp, }, }); - if (!updateResult.success) { + // The erase must be durable before the delegated turn it carried is settled: while the + // summary still holds the follow-up, startup recovery or a retry can dispatch it, and a + // settlement issued now would interrupt the handle (and remove a disposable workspace) + // under a continuation that is still live. A failed write is re-checked on disk so a + // rewrite that landed but reported failure does not strand the turn unsettled. + if (!updateResult.success && !(await this.isPendingFollowUpCleared(summaryMessage))) { throw new Error(`Failed to clear skipped pending follow-up: ${updateResult.error}`); } + + // Every discard path funnels here, so this is the one place that knows the delegated + // turn's continuation is gone for good. The void is a synchronous state transition here + // and the owner-side settlement runs as a tracked, retried promise (see + // settleVoidedWakeContinuation) — it is never awaited: this path runs while the + // TaskService stream-end listener holds the workspace event lock waiting for the + // compaction completion decision, and the owner settles under that same lock. A wake + // follow-up also carried the continuation debt of the stream it cut; the same void clears + // it. + const workspaceTurnMetadata = muxMeta.pendingFollowUp.workspaceTurnMetadata; + if (workspaceTurnMetadata != null) { + if ( + hasSameWorkspaceTurnCorrelation( + this.wakeContinuationDebt?.correlation, + workspaceTurnMetadata + ) + ) { + this.wakeContinuationDebt = undefined; + } + this.settleVoidedWakeContinuation({ + correlation: workspaceTurnMetadata, + reason: "abandoned", + }); + } + } + + /** + * Whether the durable copy of a compaction summary no longer carries a pending follow-up. + * Only a successful read that finds the row without one counts; an unreadable or missing row + * is "unknown", which callers treat as not cleared. + */ + private async isPendingFollowUpCleared(summaryMessage: MuxMessage): Promise { + const historyResult = await this.historyService.getHistoryFromLatestBoundary(this.workspaceId); + if (!historyResult.success) return false; + const durable = historyResult.data.find((message) => message.id === summaryMessage.id); + const durableMuxMeta = durable?.metadata?.muxMetadata; + return ( + durable != null && + isCompactionSummaryMetadata(durableMuxMeta) && + durableMuxMeta.pendingFollowUp == null + ); } /** @@ -7917,8 +8363,7 @@ export class AgentSession { private async materializeMcpPromptSnapshots( muxMetadata: MuxMessageMetadata | undefined, - invokingMessageId: string, - cancelSignal: AbortSignal | undefined + invokingMessageId: string ): Promise { const mcpServerManager = this.mcpServerManager; if (!mcpServerManager) return []; @@ -7931,8 +8376,7 @@ export class AgentSession { this.workspaceId, ref.serverName, ref.promptName, - ref.arguments ?? {}, - cancelSignal !== undefined ? { signal: cancelSignal } : undefined + ref.arguments ?? {} ); return createMuxMessage(createMcpPromptSnapshotMessageId(), "user", prompt.text, { timestamp: Date.now(), @@ -7946,8 +8390,6 @@ export class AgentSession { }, }); } catch (error) { - // Cancellation is handled by cancelBeforeAcceptance after this returns. - if (cancelSignal?.aborted) return null; // A slash-invoked prompt was explicitly selected; sending the turn // without its expansion would silently change what the user asked // for. Inline references degrade to the authored text instead. diff --git a/src/node/services/agentSkills/builtInSkillContent.generated.ts b/src/node/services/agentSkills/builtInSkillContent.generated.ts index f6ca94d410..fb6bbd0bde 100644 --- a/src/node/services/agentSkills/builtInSkillContent.generated.ts +++ b/src/node/services/agentSkills/builtInSkillContent.generated.ts @@ -7725,7 +7725,7 @@ export const BUILTIN_SKILL_FILES: Record> = { "", "Messages to a busy workspace enter its `MessageQueue` (`src/node/services/messageQueue.ts`) and dispatch at a boundary chosen by `queueDispatchMode`:", "", - '- `tool-end`: the stream\'s stop conditions include `hasQueuedMessages("tool-end")`, evaluated by the AI SDK only after every sibling tool result in the current step settles (`createStopWhenCondition`, `src/node/services/streamManager.ts`); `AgentSession` soft-stops only once `activeToolCallIds` is empty. **A running tool call is never interrupted** — same guarantee as Claude Code.', + "- `tool-end`: the stream's stop conditions include `hasPendingToolEndInput()` (a queued tool-end message or an outstanding bash-monitor wake, read live), evaluated by the AI SDK only after every sibling tool result in the current step settles (`createStopWhenCondition`, `src/node/services/streamManager.ts`); `AgentSession` soft-stops only once `activeToolCallIds` is empty. **A running tool call is never interrupted** — same guarantee as Claude Code.", "- `turn-end`: dispatches after the current turn completes.", "- Idle target: the message starts a new turn immediately.", "", diff --git a/src/node/services/bashMonitorWakeReconciler.test.ts b/src/node/services/bashMonitorWakeReconciler.test.ts index 4348bd3c33..c52d81b979 100644 --- a/src/node/services/bashMonitorWakeReconciler.test.ts +++ b/src/node/services/bashMonitorWakeReconciler.test.ts @@ -56,6 +56,7 @@ describe("BashMonitorWakeReconciler", () => { let dispatches: BashMonitorWakeDispatch[]; let dispatchOutcome: "in-flight" | "deferred"; let acknowledged: Array<{ processId: string; matchedThroughOffset?: number }>; + let acknowledgeError: Error | undefined; let removed: string[]; let removedOwners: string[]; let dropped: string[]; @@ -70,16 +71,18 @@ describe("BashMonitorWakeReconciler", () => { dispatches = []; dispatchOutcome = "in-flight"; acknowledged = []; + acknowledgeError = undefined; removed = []; removedOwners = []; dropped = []; droppedGenerations = []; - reconciler = new BashMonitorWakeReconciler({ + const current = new BashMonitorWakeReconciler({ sessionsDir: root, processManager: { pullMonitorWakeSignals: () => live, getMonitorWakeDeliveryState: () => Promise.resolve(deliveryState), acknowledgeMonitorWake: (processId, _generation, matchedThroughOffset) => { + if (acknowledgeError != null) throw acknowledgeError; acknowledged.push({ processId, ...(matchedThroughOffset != null ? { matchedThroughOffset } : {}), @@ -108,10 +111,14 @@ describe("BashMonitorWakeReconciler", () => { recordTerminal: () => undefined, }, onWake: (dispatch) => { + // A pass scheduled by the previous test's acceptance may still be draining; keep its + // hand-outs from leaking into this test's `dispatches`. + if (reconciler !== current) return "deferred"; dispatches.push(dispatch); return dispatchOutcome; }, }); + reconciler = current; }); afterEach(async () => { @@ -130,51 +137,309 @@ describe("BashMonitorWakeReconciler", () => { expect(dispatches).toHaveLength(2); }); - test("re-dispatches unchanged signals after a queued delivery is canceled", async () => { + test("re-dispatches unchanged signals after the owner defers an in-flight wake", async () => { live = [liveSnapshot()]; await reconciler.reconcile(OWNER); - const queued = dispatches[0]; + const inFlight = dispatches[0]; - await queued.onDeferred(); + await inFlight.onDeferred(); await reconciler.reconcile(OWNER); expect(dispatches).toHaveLength(2); }); - test("superseding a queued wake uses a distinct queue key", async () => { - const queuedKeys = new Set(); - const queuedDispatches: BashMonitorWakeDispatch[] = []; - const queueing = new BashMonitorWakeReconciler({ - sessionsDir: root, - processManager: { - pullMonitorWakeSignals: () => live, - getMonitorWakeDeliveryState: () => Promise.resolve(deliveryState), - acknowledgeMonitorWake: () => undefined, - dropRetiredMonitor: () => undefined, - }, - registry: { - listAll: () => Promise.resolve([]), - remove: () => undefined, - recordTerminal: () => undefined, - }, - onWake: (dispatch) => { - if (queuedKeys.has(dispatch.dedupeKey)) return "deferred"; - queuedKeys.add(dispatch.dedupeKey); - queuedDispatches.push(dispatch); - return "in-flight"; - }, + test("hands out one wake at a time: a newer match waits for the in-flight acceptance", async () => { + live = [liveSnapshot()]; + await reconciler.reconcile(OWNER); + expect(dispatches).toHaveLength(1); + + live = [ + liveSnapshot({ match: { throughOffset: 24, lines: ["READY again"], totalMatches: 2 } }), + ]; + await reconciler.reconcile(OWNER); + expect(dispatches).toHaveLength(1); + + await dispatches[0].onAccepted(); + await reconciler.reconcile(OWNER); + expect(dispatches).toHaveLength(2); + expect(dispatches[1].prompt).toContain("READY again"); + }); + + test("a full-history clear retires a wake already in the owner's hands", async () => { + live = [liveSnapshot()]; + await reconciler.reconcile(OWNER); + expect(dispatches).toHaveLength(1); + expect(dispatches[0].isCurrent()).toBe(true); + + // The owner has not sent yet (it is waiting on its own history lock); the clear + // consumes the signals, so the receiver must find the wake stale and drop it. + const token = await reconciler.beginFullHistoryClear(OWNER); + expect(dispatches[0].isCurrent()).toBe(false); + expect(acknowledged).toEqual([{ processId: "proc", matchedThroughOffset: 12 }]); + await reconciler.finishFullHistoryClear(token); + + // Deferring the stale wake is a no-op, and the level stays low. + await dispatches[0].onDeferred(); + expect(await reconciler.hasOutstandingWake(OWNER)).toBe(false); + }); + + test("canceling a monitor retires a wake carrying its output and re-derives the rest", async () => { + live = [liveSnapshot()]; + rows = [registryRecord()]; + await reconciler.reconcile(OWNER); + expect(dispatches).toHaveLength(1); + expect(dispatches[0].isCurrent()).toBe(true); + + // The wake combines the live match and the dead registry row. Canceling the live + // monitor must retire the whole handed-out wake (its prompt embeds that output)... + await reconciler.discardProcess(OWNER, "proc", CREATED_AT); + expect(dispatches[0].isCurrent()).toBe(false); + + // ...and the unrelated dead-process signal comes back on its own in a fresh wake. + live = []; + await reconciler.reconcile(OWNER); + expect(dispatches).toHaveLength(2); + expect(dispatches[1].prompt).not.toContain("READY"); + expect(dispatches[1].isCurrent()).toBe(true); + + // Canceling an unrelated process leaves the current wake alone. + await reconciler.discardProcess(OWNER, "someone-else", CREATED_AT); + expect(dispatches[1].isCurrent()).toBe(true); + }); + + test("a wake offered from a registry row mid-removal is withdrawn once the row is gone", async () => { + // Cancel path: the owner discards the process, then removes its registry row. A pass that + // lands between the two sees a row without a process and offers a monitor-lost wake for + // the canceled monitor. The post-removal pass derives nothing for it, and that must + // retire the offer — otherwise it stays current and the canceled monitor starts a turn. + rows = [registryRecord()]; + await reconciler.reconcile(OWNER); + expect(dispatches).toHaveLength(1); + expect(dispatches[0].muxMetadata.records[0]).toMatchObject({ + processId: "dead", + kind: "monitor-lost", }); + expect(dispatches[0].isCurrent()).toBe(true); + + rows = []; + await reconciler.reconcile(OWNER); + expect(dispatches[0].isCurrent()).toBe(false); + expect(dispatches).toHaveLength(1); + expect(await reconciler.hasOutstandingWake(OWNER)).toBe(false); + + // The cancel itself schedules no pass: only the caller's post-removal schedule does, so + // no intermediate offer is manufactured for a row the caller is about to delete. + rows = [registryRecord()]; + await reconciler.discardProcess(OWNER, "dead", CREATED_AT); + await new Promise((resolve) => setTimeout(resolve, 0)); + expect(dispatches).toHaveLength(1); + }); + + test("a shown-frontier advance retires a handed-out wake so a stale send is refused", async () => { + // The owner ran a manual turn that task_await-ed the monitored process while this wake + // was still resolving send options: the reconcile that would re-derive it is queued + // behind the hand-off, so the frontier transition must invalidate the dispatch itself. + live = [liveSnapshot()]; + await reconciler.reconcile(OWNER); + expect(dispatches).toHaveLength(1); + expect(dispatches[0].isCurrent()).toBe(true); + + await reconciler.outputShown(OWNER, "someone-else"); + expect(dispatches[0].isCurrent()).toBe(true); + + deliveryState = { status: "settled", shownThroughOffset: 12, terminalStatusShown: false }; + await reconciler.outputShown(OWNER, "proc"); + expect(dispatches[0].isCurrent()).toBe(false); + + // Nothing derives any more: the lines were shown, so no replacement wake is handed out. + await reconciler.reconcile(OWNER); + expect(dispatches).toHaveLength(1); + expect(await reconciler.hasOutstandingWake(OWNER)).toBe(false); + }); + + test("an accepted wake stays current through a later cancel or shown advance", async () => { + // Acceptance runs before the owner's final send-admission gate; a cancel landing in + // between must not make that gate refuse a turn whose row is already durable. live = [liveSnapshot()]; - await queueing.reconcile(OWNER); + await reconciler.reconcile(OWNER); + expect(dispatches).toHaveLength(1); + await dispatches[0].onAccepted(); + expect(dispatches[0].isCurrent()).toBe(true); + + await reconciler.discardProcess(OWNER, "proc", CREATED_AT); + await reconciler.outputShown(OWNER, "proc"); + expect(dispatches[0].isCurrent()).toBe(true); + }); + + test("a wake withdrawn under the send still commits and retires its replacement", async () => { + // Two processes; only `proc`'s lines get shown (shownThroughOffset 12 < other's offset). + const other = liveSnapshot({ + processId: "other", + taskId: "bash:other", + match: { throughOffset: 40, lines: ["READY other"], totalMatches: 1 }, + }); + live = [liveSnapshot(), other]; + await reconciler.reconcile(OWNER); + expect(dispatches).toHaveLength(1); + const first = dispatches[0]; + + // The owner's send is past its last admission gate when a manual read shows `proc`'s + // output: the lease is released and a replacement (only `other`) is offered. + deliveryState = { status: "settled", shownThroughOffset: 12, terminalStatusShown: false }; + await reconciler.outputShown(OWNER, "proc"); + expect(first.isCurrent()).toBe(false); + await reconciler.reconcile(OWNER); + expect(dispatches).toHaveLength(2); + const replacement = dispatches[1]; + expect(replacement.muxMetadata.records.map((record) => record.processId)).toEqual(["other"]); + expect(replacement.isCurrent()).toBe(true); + + // The first send's row lands: its commit consumes exactly its own signals (both + // processes), and the replacement — which re-describes `other` — is released so the + // owner drops it at its next gate instead of sending a duplicate. + acknowledged = []; + await first.onAccepted(); + expect(first.isCurrent()).toBe(true); + expect(replacement.isCurrent()).toBe(false); + expect(acknowledged).toEqual([ + { processId: "other", matchedThroughOffset: 40 }, + { processId: "proc", matchedThroughOffset: 12 }, + ]); + + // Nothing is left to derive: no third wake, level low. + await reconciler.reconcile(OWNER); + expect(dispatches).toHaveLength(2); + expect(await reconciler.hasOutstandingWake(OWNER)).toBe(false); + }); + + test("a failed acknowledgment keeps the committed wake consumed and retries it", async () => { + live = [liveSnapshot()]; + await reconciler.reconcile(OWNER); + expect(dispatches).toHaveLength(1); + + // The owner's row is durable, so acceptance must not fail the send: it resolves, and the + // wake reads as consumed (level low, no duplicate hand-out) while durability is pending. + acknowledgeError = new Error("disk full"); + await dispatches[0].onAccepted(); + expect(dispatches[0].isCurrent()).toBe(true); + expect(await reconciler.hasOutstandingWake(OWNER)).toBe(false); + await expect(reconciler.reconcile(OWNER)).rejects.toThrow("disk full"); + expect(dispatches).toHaveLength(1); + + // Withdrawals never apply to a committed lease: dropping it here would re-derive the + // same signals into a second prompt once the store recovers. + await reconciler.outputShown(OWNER, "proc"); + await reconciler.discardProcess(OWNER, "proc", CREATED_AT); + await expect(reconciler.reconcile(OWNER)).rejects.toThrow("disk full"); + expect(dispatches).toHaveLength(1); + + acknowledgeError = undefined; + await reconciler.reconcile(OWNER); + expect(acknowledged).toEqual([{ processId: "proc", matchedThroughOffset: 12 }]); + expect(dispatches).toHaveLength(1); + + // The slot is free again: a newer match dispatches normally. + live = [ + liveSnapshot({ match: { throughOffset: 24, lines: ["READY again"], totalMatches: 2 } }), + ]; + await reconciler.reconcile(OWNER); + expect(dispatches).toHaveLength(2); + expect(dispatches[1].prompt).toContain("READY again"); + }); + + test("after a restart, the durable wake row acknowledges a commit whose watermark never landed", async () => { + // The owner's row landed, then the process died before the watermark write. The in-memory + // committed lease is gone; without the row as durable evidence the fresh reconciler would + // derive the same signal again and dispatch a duplicate next to the row's own recovery. + live = [liveSnapshot()]; + await reconciler.reconcile(OWNER); + expect(dispatches).toHaveLength(1); + const deliveredRecords = dispatches[0].muxMetadata.records; + + const restart = ( + readDeliveredWakeRecords: + | ((ownerWorkspaceId: string, notBefore: string) => Promise) + | undefined + ) => { + const restarted: BashMonitorWakeDispatch[] = []; + const instance = new BashMonitorWakeReconciler({ + // A fresh sessions dir: no watermark was ever written. + sessionsDir: path.join(root, readDeliveredWakeRecords == null ? "control" : "recovered"), + processManager: { + pullMonitorWakeSignals: () => live, + getMonitorWakeDeliveryState: () => Promise.resolve(deliveryState), + acknowledgeMonitorWake: (processId, _generation, matchedThroughOffset) => { + acknowledged.push({ + processId, + ...(matchedThroughOffset != null ? { matchedThroughOffset } : {}), + }); + }, + dropRetiredMonitor: () => undefined, + }, + registry: { + listAll: () => Promise.resolve(rows), + remove: () => undefined, + recordTerminal: () => undefined, + }, + onWake: (dispatch) => { + restarted.push(dispatch); + return "in-flight"; + }, + ...(readDeliveredWakeRecords != null ? { readDeliveredWakeRecords } : {}), + }); + return { instance, restarted }; + }; + + // Control: the same restart without the row re-dispatches the delivered signal. + const control = restart(undefined); + await control.instance.reconcile(OWNER); + expect(control.restarted).toHaveLength(1); + + // A read that cannot answer fails the reconcile (retried) instead of counting as "no row": + // recovery is consulted once per owner, so a swallowed failure would dispatch a duplicate. + let readFails = true; + const readBounds: string[] = []; + const recovered = restart((_ownerWorkspaceId, notBefore) => { + readBounds.push(notBefore); + return readFails + ? Promise.reject(new Error("history unavailable")) + : Promise.resolve(deliveredRecords); + }); + acknowledged = []; + const failedReconcile = await recovered.instance.reconcile(OWNER).then( + () => null, + (error: unknown) => error + ); + expect(failedReconcile).toBeInstanceOf(Error); + expect(recovered.restarted).toHaveLength(0); + expect(acknowledged).toEqual([]); + readFails = false; + await recovered.instance.reconcile(OWNER); + expect(recovered.restarted).toHaveLength(0); + expect(acknowledged).toEqual([{ processId: "proc", matchedThroughOffset: 12 }]); + expect(await recovered.instance.hasOutstandingWake(OWNER)).toBe(false); + // The reader is told how far back a row could possibly acknowledge these signals: the + // arm time of the oldest outstanding monitor. + expect(readBounds).toEqual([CREATED_AT, CREATED_AT]); + + // The watermark is durable now: a later read does not consult the row again, and a + // newer match still wakes. live = [ liveSnapshot({ match: { throughOffset: 24, lines: ["READY again"], totalMatches: 2 } }), ]; + await recovered.instance.reconcile(OWNER); + expect(recovered.restarted).toHaveLength(1); + expect(recovered.restarted[0].prompt).toContain("READY again"); + }); - await queueing.reconcile(OWNER); + test("disposal lowers the published level and retires the in-flight wake", async () => { + live = [liveSnapshot()]; + await reconciler.reconcile(OWNER); + expect(dispatches).toHaveLength(1); - expect(queuedDispatches).toHaveLength(2); - expect(queuedKeys.size).toBe(2); - expect(queuedDispatches[0].cancelSignal.aborted).toBe(true); + await reconciler.dispose(OWNER); + expect(dispatches[0].isCurrent()).toBe(false); + expect(await reconciler.hasOutstandingWake(OWNER)).toBe(false); }); test("keeps dead registry evidence until the queued wake is accepted", async () => { @@ -198,25 +463,83 @@ describe("BashMonitorWakeReconciler", () => { expect(dispatches).toHaveLength(1); }); - test("cancels a queued wake when the level no longer has an outstanding signal", async () => { + test("hasOutstandingWake reads the level without dispatching", async () => { live = [liveSnapshot()]; - await reconciler.reconcile(OWNER); - const queued = dispatches[0]; + expect(await reconciler.hasOutstandingWake(OWNER)).toBe(true); + expect(dispatches).toHaveLength(0); - deliveryState = { status: "settled", shownThroughOffset: 12, terminalStatusShown: false }; - await reconciler.reconcile(OWNER); + // A same-process blocking read will show the lines itself: not outstanding. + deliveryState = { status: "blocked", readSettled: new Promise(() => undefined) }; + expect(await reconciler.hasOutstandingWake(OWNER)).toBe(false); - expect(queued.cancelSignal.aborted).toBe(true); - await queued.onAccepted(); - deliveryState = { status: "settled", shownThroughOffset: 0, terminalStatusShown: false }; + // Shown frontier past the match: the wake is gone, nothing to deliver later either. + deliveryState = { status: "settled", shownThroughOffset: 12, terminalStatusShown: false }; + expect(await reconciler.hasOutstandingWake(OWNER)).toBe(false); await reconciler.reconcile(OWNER); - expect(dispatches).toHaveLength(1); + expect(dispatches).toHaveLength(0); live = [ liveSnapshot({ match: { throughOffset: 24, lines: ["READY again"], totalMatches: 2 } }), ]; - await reconciler.reconcile(OWNER); - expect(dispatches).toHaveLength(2); + expect(await reconciler.hasOutstandingWake(OWNER)).toBe(true); + }); + + test("publishes the level on every read", async () => { + const levels: boolean[] = []; + const publishing = new BashMonitorWakeReconciler({ + sessionsDir: root, + processManager: { + pullMonitorWakeSignals: () => live, + getMonitorWakeDeliveryState: () => Promise.resolve(deliveryState), + acknowledgeMonitorWake: () => undefined, + dropRetiredMonitor: () => undefined, + }, + registry: { + listAll: () => Promise.resolve([]), + remove: () => undefined, + recordTerminal: () => undefined, + }, + onWake: () => "deferred", + onOutstandingChanged: (_owner, outstanding) => { + levels.push(outstanding); + }, + }); + live = [liveSnapshot()]; + await publishing.reconcile(OWNER); + deliveryState = { status: "settled", shownThroughOffset: 12, terminalStatusShown: false }; + await publishing.hasOutstandingWake(OWNER); + + expect(levels).toEqual([true, false]); + }); + + test("a full-history clear lowers the level without a follow-up read", async () => { + const levels: boolean[] = []; + const publishing = new BashMonitorWakeReconciler({ + sessionsDir: root, + processManager: { + pullMonitorWakeSignals: () => live, + getMonitorWakeDeliveryState: () => Promise.resolve(deliveryState), + acknowledgeMonitorWake: () => undefined, + dropRetiredMonitor: () => undefined, + }, + registry: { + listAll: () => Promise.resolve([]), + remove: () => undefined, + recordTerminal: () => undefined, + }, + onWake: () => "deferred", + onOutstandingChanged: (_owner, outstanding) => { + levels.push(outstanding); + }, + }); + live = [liveSnapshot()]; + await publishing.reconcile(OWNER); + expect(levels).toEqual([true]); + + // The consume path retires the signals it collects, so it must not republish them as + // outstanding (that would leave the owner's tool-end yield flag stuck high). + await publishing.beginFullHistoryClear(OWNER); + expect(levels).toEqual([true, false]); }); test("advances the watermark only on acceptance and later delivers a newer match", async () => { @@ -329,18 +652,18 @@ describe("BashMonitorWakeReconciler", () => { expect(restartedDispatches).toHaveLength(1); }); - test("explicit cancellation retracts a queued wake without consuming a later generation", async () => { + test("accepting a wake whose processes vanished does not consume a later generation", async () => { live = [liveSnapshot()]; rows = [registryRecord()]; await reconciler.reconcile(OWNER); - const queued = dispatches[0]; + const inFlight = dispatches[0]; live = []; rows = []; await reconciler.reconcile(OWNER); + expect(dispatches).toHaveLength(1); - expect(queued.cancelSignal.aborted).toBe(true); - await queued.onAccepted(); + await inFlight.onAccepted(); live = [ liveSnapshot({ createdAt: "2026-08-31T12:03:00.000Z", diff --git a/src/node/services/bashMonitorWakeReconciler.ts b/src/node/services/bashMonitorWakeReconciler.ts index 46728fc4be..f6446b0475 100644 --- a/src/node/services/bashMonitorWakeReconciler.ts +++ b/src/node/services/bashMonitorWakeReconciler.ts @@ -3,14 +3,16 @@ import { randomUUID } from "node:crypto"; import * as fsPromises from "node:fs/promises"; import * as path from "node:path"; -import type { MuxMessageMetadata } from "@/common/types/message"; +import type { BashMonitorWakeDisplayRecord, MuxMessageMetadata } from "@/common/types/message"; import assert from "@/common/utils/assert"; +import { getErrorMessage } from "@/common/utils/errors"; import { BASH_MONITOR_WAKE_HEADINGS } from "@/common/utils/machineTurnPrompts"; import type { BashMonitorLostSummary, BashMonitorRegistryRecord, BashMonitorTerminalSummary, } from "@/node/services/bashMonitorRegistryStore"; +import { log } from "@/node/services/log"; import { MutexMap } from "@/node/utils/concurrency/mutexMap"; import { stripAnsiControlChars } from "@/node/utils/ansi"; import { isErrnoWithCode } from "@/node/utils/fs"; @@ -101,12 +103,26 @@ export interface BashMonitorWakeReconcilerRegistry { export type BashMonitorWakeDispatchOutcome = "in-flight" | "deferred"; +/** + * A wake handed to the owner. Wakes are a LEVEL derived from process state, never a + * queued edge: the receiver either starts a turn from it now (`onAccepted`) or leaves it + * pending (`onDeferred` / "deferred") and re-reconciles later. There is nothing to cancel — + * a wake whose matched output is shown meanwhile simply stops deriving on the next read. + */ export interface BashMonitorWakeDispatch { ownerWorkspaceId: string; prompt: string; muxMetadata: Extract; - dedupeKey: string; - cancelSignal: AbortSignal; + /** + * False once the lease behind this wake was released while it was in the receiver's + * hands (full-history clear, disposal, monitor cancel, shown-frontier advance, or an + * earlier wake committing meanwhile). Stays true from `onAccepted` on: the prompt is + * durable and the signals consumed. The receiver re-checks it after taking its own locks + * (the clear runs under the same history lock), before sending, and at every + * send-admission gate, so a stale prompt is never appended to history and an accepted one + * always gets its stream. + */ + isCurrent(): boolean; onAccepted(): Promise; onDeferred(): Promise; } @@ -150,19 +166,45 @@ interface DerivedSignal { retired: boolean; } -interface DispatchState { - id: string; - signature: string; - controller: AbortController; +/** + * A wake handed to the owner is a LEASE on the signal set it describes. + * + * Lifecycle (one transition each, all under the owner lock): + * + * offered ──release──▶ released the owner did not send it: `onDeferred`, `onWake` + * threw, or the signals were withdrawn under it (cancel, + * shown frontier, full-history clear, or a reconcile pass + * under which they no longer all derive). `isCurrent()` + * turns false so the owner drops it at its next admission + * gate; whatever still derives is re-leased by the next + * reconcile. + * offered ──commit───▶ committed the owner's prompt row is durable (`onAccepted`). The + * released ─commit───▶ committed signals are consumed from here on regardless of what + * happened to the offer meanwhile (a release can land in + * the send's last pre-durability await): withdrawal never + * applies to a committed lease, and a replacement offered + * into the emptied slot is released because it re-describes + * signals this row already delivered. + * committed ─acknowledge─▶ (gone) watermarks advanced + monitors cleaned up for exactly the + * leased signals, by identity. Attempted inline at commit + * and retried by every reconcile pass until it lands; while + * pending, `collect()` overlays the committed signals so the + * level reads low and nothing re-derives a duplicate. + * + * Invariant: at most one offered and at most one committed lease per owner, and nothing is + * offered while either exists — a second wake meanwhile could only duplicate or supersede it. + */ +interface Lease { signals: readonly DerivedSignal[]; - accepted: boolean; + status: "offered" | "committed" | "released"; } interface ReconcileState { requested: boolean; scheduled: boolean; promise?: Promise; - dispatch?: DispatchState; + offered?: Lease; + committed?: Lease; } function signalKey(processId: string, createdAt: string): string { @@ -322,6 +364,19 @@ function buildPrompt(signals: readonly DerivedSignal[]): string { return `${header}\n\n${sections.join("\n\n---\n\n")}\n\n${closingParts.join(" ")}`; } +/** + * Version of the signal as it appears in the durable wake row (`records[].wakeUpdatedAt`). + * Together with processId it identifies a delivered signal after a restart (see + * readDeliveredWakeRecords). + */ +function wakeUpdatedAtOf(signal: DerivedSignal): string { + return ( + signal.lost?.failedAt ?? + signal.terminal?.settledAt ?? + (signal.matchOffset != null ? signal.createdAt + ":" + signal.matchOffset : signal.createdAt) + ); +} + function buildMetadata( signals: readonly DerivedSignal[] ): Extract { @@ -329,12 +384,7 @@ function buildMetadata( type: "bash-monitor-wake", records: signals.map((signal) => ({ processId: signal.processId, - wakeUpdatedAt: - signal.lost?.failedAt ?? - signal.terminal?.settledAt ?? - (signal.matchOffset != null - ? signal.createdAt + ":" + signal.matchOffset - : signal.createdAt), + wakeUpdatedAt: wakeUpdatedAtOf(signal), kind: signal.kind === "monitor-lost" ? "monitor-lost" : "match", displayName: signal.displayName ?? signal.processId, filter: signal.filter, @@ -358,6 +408,8 @@ export class BashMonitorWakeReconciler { private readonly locks = new MutexMap(); private readonly states = new Map(); private readonly legacyCleanupAttempted = new Set(); + /** Owners whose durable wake row has been reconciled against derived signals (once per process). */ + private readonly deliveryRecovered = new Set(); private readonly retryTimers = new Map(); private readonly retryAttempts = new Map(); private readonly defunctWorkspaces = new Set(); @@ -370,9 +422,43 @@ export class BashMonitorWakeReconciler { onWake( dispatch: BashMonitorWakeDispatch ): Promise | BashMonitorWakeDispatchOutcome; + /** + * Published on every level read (reconcile, snapshot, hasOutstandingWake) so the + * owner can mirror "pending input wants a tool boundary" side effects (early + * long-poll return, backgrounding foreground waits) from the level itself. + */ + onOutstandingChanged?(ownerWorkspaceId: string, outstanding: boolean): void; + /** + * Records of the owner's most recent durable wake row, if any. The wake row is the + * durable acknowledgment: a commit whose watermark write failed and was then lost to a + * restart (the in-memory committed lease dies with the process) would otherwise + * re-derive and re-dispatch the very signals that row already delivers. Consulted once + * per owner, the first time signals derive outstanding in this process. A read that + * cannot answer must throw (not return undefined): the reconcile fails and retries, so + * "no row" is only ever concluded from a successful read. + * + * `notBefore` (ISO) is the arm time of the oldest outstanding monitor: a row appended + * before any of these monitors existed cannot acknowledge them, so the reader may stop + * its backward scan there instead of parsing the whole transcript. + */ + readDeliveredWakeRecords?( + ownerWorkspaceId: string, + notBefore: string + ): Promise; } ) {} + /** + * The wake level: whether the owner has a wake it has not seen yet. Same-process + * blocking reads (deferredReads) are not outstanding — the read shows the lines itself. + * Consumer: the stream's tool-boundary stop condition (AgentSession.hasPendingToolEndInput); + * delegated-turn settlement reads the session's continuation debt instead. + */ + async hasOutstandingWake(ownerWorkspaceId: string): Promise { + if (this.defunctWorkspaces.has(ownerWorkspaceId)) return false; + return (await this.snapshot(ownerWorkspaceId)).pendingWakeKinds.size > 0; + } + scheduleReconcile(ownerWorkspaceId: string): void { if (this.defunctWorkspaces.has(ownerWorkspaceId)) return; const state = this.state(ownerWorkspaceId); @@ -421,26 +507,55 @@ export class BashMonitorWakeReconciler { return snapshot.pendingWakeKinds.get(processId); } + /** + * The operator canceled a monitor: a wake already handed to the owner that carries this + * process's output must not be sent (its isCurrent() turns false). Its other signals, if + * any, re-derive on the next reconcile — which the caller schedules after it removes the + * registry row, not here: a pass between the cancel and that removal would see a row + * without a process and offer a monitor-lost wake for the canceled monitor. + */ async discardProcess( ownerWorkspaceId: string, processId: string, createdAt: string + ): Promise { + await this.releaseOfferedCovering( + ownerWorkspaceId, + (signal) => signal.processId === processId && signal.createdAt === createdAt + ); + } + + /** + * A model-visible read advanced this process's shown frontier (or showed its terminal + * status). An offered wake may now describe lines the owner has seen (the owner could have + * run a manual turn and returned idle while the wake was still resolving send options, so + * the reconcile that would re-derive it is queued behind that very hand-off): release it + * and let the reconcile scheduled here re-lease whatever still derives. + */ + async outputShown(ownerWorkspaceId: string, processId: string): Promise { + await this.releaseOfferedCovering(ownerWorkspaceId, (signal) => signal.processId === processId); + this.scheduleReconcile(ownerWorkspaceId); + } + + private async releaseOfferedCovering( + ownerWorkspaceId: string, + covers: (signal: DerivedSignal) => boolean ): Promise { await this.locks.withLock(ownerWorkspaceId, () => { - const state = this.state(ownerWorkspaceId); - if ( - state.dispatch?.signals.some( - (signal) => signal.processId === processId && signal.createdAt === createdAt - ) === true - ) { - state.dispatch.controller.abort(); - state.dispatch = undefined; - } + const state = this.states.get(ownerWorkspaceId); + if (state?.offered?.signals.some(covers)) this.releaseOffered(state); return Promise.resolve(); }); } + + /** Caller holds the owner lock. */ + private releaseOffered(state: ReconcileState): void { + if (state.offered == null) return; + state.offered.status = "released"; + state.offered = undefined; + } + async beginFullHistoryClear(ownerWorkspaceId: string): Promise { - this.abortDispatch(ownerWorkspaceId); await this.consumeCurrent(ownerWorkspaceId); return { ownerWorkspaceId }; } @@ -454,10 +569,14 @@ export class BashMonitorWakeReconciler { this.resetRetry(ownerWorkspaceId); await this.locks.withLock(ownerWorkspaceId, () => { const state = this.states.get(ownerWorkspaceId); - state?.dispatch?.controller.abort(); + if (state != null) { + this.releaseOffered(state); + if (state.committed != null) state.committed.status = "released"; + } this.states.delete(ownerWorkspaceId); return Promise.resolve(); }); + this.args.onOutstandingChanged?.(ownerWorkspaceId, false); } revive(ownerWorkspaceId: string): void { @@ -505,7 +624,11 @@ export class BashMonitorWakeReconciler { } private async reconcileOnce(ownerWorkspaceId: string): Promise { - const dispatch = await this.locks.withLock(ownerWorkspaceId, async () => { + const lease = await this.locks.withLock(ownerWorkspaceId, async () => { + // A commit whose acknowledgment failed is retried first: on a throw the loop's catch + // schedules the backoff retry and nothing is leased meanwhile. + const committed = this.states.get(ownerWorkspaceId)?.committed; + if (committed != null) await this.acknowledge(ownerWorkspaceId, committed); const collected = await this.collect(ownerWorkspaceId, true); for (const readSettled of collected.deferredReads) { void readSettled.finally(() => this.scheduleReconcile(ownerWorkspaceId)); @@ -514,91 +637,108 @@ export class BashMonitorWakeReconciler { await this.cleanup(collected.autoConsumed); const state = this.state(ownerWorkspaceId); - if (collected.signals.length === 0) { - state.dispatch?.controller.abort(); - state.dispatch = undefined; - return undefined; + // An offer is a claim on signals that derive *now*. A pass that ran between a monitor's + // cancel and its registry-row removal sees a row without a process and offers a + // monitor-lost wake for it; once the row is gone nothing derives, yet the offer would + // stay current and the canceled monitor could still start an agent turn. Withdraw an + // offer whose signals no longer all derive outstanding (also covers a frontier advance + // whose outputShown release lost the race with this pass). + if (state.offered != null) { + const outstandingKeys = new Set(collected.signals.map((signal) => signal.key)); + if (!state.offered.signals.every((signal) => outstandingKeys.has(signal.key))) { + this.releaseOffered(state); + } } - - const signature = JSON.stringify( - collected.signals.map((signal) => [ - signal.key, - signal.kind, - signal.matchOffset, - signal.terminal?.settledAt, - signal.matchedOutputAlreadyShown, - ]) - ); - if (state.dispatch?.signature === signature && !state.dispatch.controller.signal.aborted) { + if (collected.signals.length === 0 || state.offered != null || state.committed != null) { return undefined; } - state.dispatch?.controller.abort(); - const next: DispatchState = { - id: randomUUID(), - signature, - controller: new AbortController(), - signals: collected.signals, - accepted: false, - }; - state.dispatch = next; + const next: Lease = { signals: collected.signals, status: "offered" }; + state.offered = next; return next; }); - if (dispatch == null) return; + if (lease == null) return; try { const outcome = await this.args.onWake({ ownerWorkspaceId, - prompt: buildPrompt(dispatch.signals), - muxMetadata: buildMetadata(dispatch.signals), - dedupeKey: "bash-monitor-wake:" + ownerWorkspaceId + ":" + dispatch.id, - cancelSignal: dispatch.controller.signal, - onAccepted: async () => this.accept(ownerWorkspaceId, dispatch), - onDeferred: async () => this.defer(ownerWorkspaceId, dispatch), + prompt: buildPrompt(lease.signals), + muxMetadata: buildMetadata(lease.signals), + isCurrent: () => lease.status !== "released", + onAccepted: async () => this.commit(ownerWorkspaceId, lease), + onDeferred: async () => this.release(ownerWorkspaceId, lease), }); - if (outcome === "deferred") await this.defer(ownerWorkspaceId, dispatch); + if (outcome === "deferred") await this.release(ownerWorkspaceId, lease); } catch (error) { - await this.locks.withLock(ownerWorkspaceId, () => { - const state = this.state(ownerWorkspaceId); - if (state.dispatch === dispatch) state.dispatch = undefined; - return Promise.resolve(); - }); + await this.release(ownerWorkspaceId, lease); throw error; } } - private async defer(ownerWorkspaceId: string, dispatch: DispatchState): Promise { + private async release(ownerWorkspaceId: string, lease: Lease): Promise { await this.locks.withLock(ownerWorkspaceId, () => { const state = this.state(ownerWorkspaceId); - if (state.dispatch === dispatch && !dispatch.accepted) state.dispatch = undefined; + if (state.offered === lease) this.releaseOffered(state); return Promise.resolve(); }); } - private async accept(ownerWorkspaceId: string, dispatch: DispatchState): Promise { + + /** + * The owner's prompt row is durable. Never throws: the caller is the owner's send, whose + * row already landed; a failed acknowledgment is retried by the reconcile passes. + */ + private async commit(ownerWorkspaceId: string, lease: Lease): Promise { await this.locks.withLock(ownerWorkspaceId, async () => { - if (dispatch.accepted || dispatch.controller.signal.aborted) return; - dispatch.accepted = true; - const watermarks = await this.readWatermarks(ownerWorkspaceId); - await this.advanceWatermarks(ownerWorkspaceId, watermarks, dispatch.signals); - await this.cleanup(dispatch.signals); + // Second call (onAcceptedPreStreamFailure), or the owner is gone. + if (lease.status === "committed" || this.defunctWorkspaces.has(ownerWorkspaceId)) return; + // A lease released while the send was between its last admission gate and durability + // still commits: the row exists, so its signals are consumed either way. const state = this.state(ownerWorkspaceId); - if (state.dispatch === dispatch) state.dispatch = undefined; + // The offered slot holds either this lease or a replacement offered after the signals + // were withdrawn under it; either way it empties (see the Lease lifecycle). + if (state.offered === lease) state.offered = undefined; + else this.releaseOffered(state); + lease.status = "committed"; + state.committed = lease; + await this.acknowledge(ownerWorkspaceId, lease).catch((error: unknown) => { + log.warn("Bash monitor wake acknowledgment failed; the reconcile pass retries it", { + ownerWorkspaceId, + error: getErrorMessage(error), + }); + }); }); this.scheduleReconcile(ownerWorkspaceId); } - private abortDispatch(ownerWorkspaceId: string): void { - const state = this.state(ownerWorkspaceId); - state.dispatch?.controller.abort(); - state.dispatch = undefined; + /** + * Durably consume exactly the committed lease's signals. Caller holds the owner lock. + * Throws when durability fails, leaving the lease committed for a retry. + */ + private async acknowledge(ownerWorkspaceId: string, lease: Lease): Promise { + const watermarks = await this.readWatermarks(ownerWorkspaceId); + await this.advanceWatermarks(ownerWorkspaceId, watermarks, lease.signals); + await this.cleanup(lease.signals); + const state = this.states.get(ownerWorkspaceId); + if (state?.committed === lease) state.committed = undefined; } private async consumeCurrent(ownerWorkspaceId: string): Promise { await this.locks.withLock(ownerWorkspaceId, async () => { - this.abortDispatch(ownerWorkspaceId); + const state = this.state(ownerWorkspaceId); + // An offered wake describes signals this consume retires: release it so the owner + // drops it instead of sending. A committed one is consumed along with everything else. + this.releaseOffered(state); const collected = await this.collect(ownerWorkspaceId, false); - const consumed = [...collected.signals, ...collected.autoConsumed]; + const consumed = [ + ...collected.signals, + ...collected.autoConsumed, + ...(state.committed?.signals ?? []), + ]; await this.advanceWatermarks(ownerWorkspaceId, collected.watermarks, consumed); await this.cleanup(consumed); + state.committed = undefined; + // Everything collected is consumed, so the level is low by construction; publish it + // here because no read follows a consume. + this.args.onOutstandingChanged?.(ownerWorkspaceId, false); }); } @@ -656,6 +796,12 @@ export class BashMonitorWakeReconciler { } if (pruned) await this.writeWatermarks(ownerWorkspaceId, watermarks); + // A committed lease is consumed whether or not its acknowledgment has landed yet. + // Overlaying it makes derive() treat those signals as delivered, so level reads stay low + // and nothing re-derives a duplicate. + const committed = this.states.get(ownerWorkspaceId)?.committed; + if (committed != null) applySignalsToWatermarks(watermarks, committed.signals); + const signals: DerivedSignal[] = []; const autoConsumed: DerivedSignal[] = []; const deferredReads: Array> = []; @@ -671,9 +817,36 @@ export class BashMonitorWakeReconciler { else if (derived.outstanding) signals.push(derived.signal); else if (derived.consume) autoConsumed.push(derived.signal); } + if (signals.length > 0 && !this.deliveryRecovered.has(ownerWorkspaceId)) { + // Signals the durable wake row already delivers are consumed, not re-dispatched. The + // watermark advance is written here (not left to the caller): level reads do not + // persist autoConsumed, and a recovery that only held in memory would be lost again. + // ISO timestamps order lexicographically. + const notBefore = signals + .map((signal) => signal.createdAt) + .reduce((oldest, createdAt) => (createdAt < oldest ? createdAt : oldest)); + const delivered = await this.args.readDeliveredWakeRecords?.(ownerWorkspaceId, notBefore); + const deliveredKeys = new Set( + (delivered ?? []).map((record) => record.processId + "\u0000" + record.wakeUpdatedAt) + ); + const recovered = signals.filter((signal) => + deliveredKeys.has(signal.processId + "\u0000" + wakeUpdatedAtOf(signal)) + ); + if (recovered.length > 0) { + await this.advanceWatermarks(ownerWorkspaceId, watermarks, recovered); + for (const signal of recovered) { + signals.splice(signals.indexOf(signal), 1); + autoConsumed.push(signal); + } + } + this.deliveryRecovered.add(ownerWorkspaceId); + } signals.sort( (a, b) => a.createdAt.localeCompare(b.createdAt) || a.processId.localeCompare(b.processId) ); + // Only level reads publish; the consume path (applyFrontier=false) is about to retire + // these very signals and publishes low itself. + if (applyFrontier) this.args.onOutstandingChanged?.(ownerWorkspaceId, signals.length > 0); return { signals, autoConsumed, deferredReads, watermarks }; } @@ -916,29 +1089,7 @@ export class BashMonitorWakeReconciler { signals: readonly DerivedSignal[] ): Promise { if (signals.length === 0) return; - for (const signal of signals) { - const previous = watermarks.get(signal.key); - watermarks.set(signal.key, { - processId: signal.processId, - createdAt: signal.createdAt, - ...(signal.matchOffset != null - ? { - matchedThroughOffset: Math.max( - signal.matchOffset, - previous?.matchedThroughOffset ?? -1 - ), - } - : previous?.matchedThroughOffset != null - ? { matchedThroughOffset: previous.matchedThroughOffset } - : {}), - ...(signal.terminal != null - ? { terminalSettledAt: signal.terminal.settledAt } - : previous?.terminalSettledAt != null - ? { terminalSettledAt: previous.terminalSettledAt } - : {}), - ...(signal.kind === "monitor-lost" || previous?.lost === true ? { lost: true } : {}), - }); - } + applySignalsToWatermarks(watermarks, signals); await this.writeWatermarks(ownerWorkspaceId, watermarks); } @@ -1036,3 +1187,33 @@ export class BashMonitorWakeReconciler { } } } + +/** Fold delivered signals into the in-memory watermark map (idempotent per signal). */ +function applySignalsToWatermarks( + watermarks: Map, + signals: readonly DerivedSignal[] +): void { + for (const signal of signals) { + const previous = watermarks.get(signal.key); + watermarks.set(signal.key, { + processId: signal.processId, + createdAt: signal.createdAt, + ...(signal.matchOffset != null + ? { + matchedThroughOffset: Math.max( + signal.matchOffset, + previous?.matchedThroughOffset ?? -1 + ), + } + : previous?.matchedThroughOffset != null + ? { matchedThroughOffset: previous.matchedThroughOffset } + : {}), + ...(signal.terminal != null + ? { terminalSettledAt: signal.terminal.settledAt } + : previous?.terminalSettledAt != null + ? { terminalSettledAt: previous.terminalSettledAt } + : {}), + ...(signal.kind === "monitor-lost" || previous?.lost === true ? { lost: true } : {}), + }); + } +} diff --git a/src/node/services/messageQueue.test.ts b/src/node/services/messageQueue.test.ts index 82106e398c..7755dace1f 100644 --- a/src/node/services/messageQueue.test.ts +++ b/src/node/services/messageQueue.test.ts @@ -488,25 +488,6 @@ describe("MessageQueue", () => { expect(queue.getQueueDispatchMode()).toBe("tool-end"); }); - it("skips withdrawn entries when reporting the next dispatchable mode", () => { - const validOptions: SendMessageOptions = { model: "gpt-4", agentId: "exec" }; - const withdrawn = new AbortController(); - queue.add( - "withdrawn wake", - { ...validOptions, queueDispatchMode: "tool-end" }, - { synthetic: true, agentInitiated: true, cancelSignal: withdrawn.signal } - ); - expect(queue.getNextDispatchableMode()).toBe("tool-end"); - - withdrawn.abort(); - expect(queue.getNextDispatchableMode()).toBeUndefined(); - expect(queue.isEmpty()).toBe(false); - - queue.add("follow up", { ...validOptions, queueDispatchMode: "turn-end" }); - expect(queue.getNextDispatchableMode()).toBe("turn-end"); - expect(queue.getNextQueueDispatchMode()).toBe("tool-end"); - }); - it("should reset mode to tool-end when cleared", () => { queue.add("Follow up", { model: "gpt-4", diff --git a/src/node/services/messageQueue.ts b/src/node/services/messageQueue.ts index 93593b9f5d..d8fde86e13 100644 --- a/src/node/services/messageQueue.ts +++ b/src/node/services/messageQueue.ts @@ -93,24 +93,20 @@ type GoalInterventionPolicy = NonNullable; -/** onCanceled text for a send whose cancel signal fired before the turn was accepted. */ -export function cancelReasonBeforeAcceptance(signal: AbortSignal): string { - return typeof signal.reason === "string" - ? signal.reason - : "Queued message canceled before acceptance."; -} - /** * Input poised to take over a session at a queue cut (see * AgentSession.getQueueCutCutter). Engaged stages win over the queue head; an * engaged stage is reported even when its metadata is undefined (manual * message) so callers cannot misattribute the cut to an entry queued behind - * the engaged one. + * the engaged one. "bash-monitor-wake" is the stream yielding to the wake level + * with no input holding the session: the wake turn, if it still arrives, is a + * separate idle-only send, so this stage carries no metadata. */ export type QueueCutCutter = | { stage: "preparing"; muxMetadata: unknown } | { stage: "dispatching"; muxMetadata: unknown } - | { stage: "queued"; muxMetadata: unknown; dispatchMode: QueueDispatchMode }; + | { stage: "queued"; muxMetadata: unknown; dispatchMode: QueueDispatchMode } + | { stage: "bash-monitor-wake"; muxMetadata?: undefined }; interface QueuedMessageInternalOptions { synthetic?: boolean; @@ -132,10 +128,6 @@ interface QueuedMessageInternalOptions { onAccepted?: () => Promise | void; onAcceptedPreStreamFailure?: (error: SendMessageError) => Promise | void; onCanceled?: (reason: string) => Promise | void; - /** Mutable dispatch outcome shared with sendQueuedMessages. */ - cancelState?: { canceledBeforeAcceptance: boolean }; - /** Cancels a queued entry even after it has been dequeued into PREPARING. */ - cancelSignal?: AbortSignal; /** * Synthetic rows persisted by AgentSession.sendMessage immediately before the * turn's user row (family-message payloads). Deferring them with the trigger @@ -198,8 +190,6 @@ interface QueueEntry { onCanceled?: (reason: string) => Promise | void; onAccepted?: () => Promise | void; onAcceptedPreStreamFailure?: (error: SendMessageError) => Promise | void; - cancelState?: { canceledBeforeAcceptance: boolean }; - cancelSignal?: AbortSignal; /** Pre-turn rows delivered with this entry (entries carrying them are sealed). */ preTurnMessages?: MuxMessage[]; /** r54: fired once this entry's pre-turn rows cross the rollback horizon. */ @@ -271,13 +261,9 @@ export class MessageQueue { return this.entries[0]?.dispatchMode ?? "tool-end"; } - /** - * Dispatch mode of the first entry whose cancel signal has not fired, or undefined - * when none remains. Aborted entries still drain FIFO (as no-ops that fire - * onCanceled), but they are not pending work and must not arm a tool-end stop. - */ + /** Dispatch mode of the FIFO head, or undefined when the queue is empty. */ getNextDispatchableMode(): QueueDispatchMode | undefined { - return this.entries.find((entry) => entry.cancelSignal?.aborted !== true)?.dispatchMode; + return this.entries[0]?.dispatchMode; } /** @@ -341,18 +327,6 @@ export class MessageQueue { return { muxMetadata: head.muxMetadata, dispatchMode: head.dispatchMode }; } - /** - * Whether the next entry to dispatch is a bash-monitor wake. Wake sends are - * the only queued input that continues an open delegated workspace turn - * (see AgentSession.inheritOpenWorkspaceTurnMetadata); any other head entry - * supersedes the turn when it dispatches. - */ - isNextEntryBashMonitorWake(): boolean { - const muxMetadata = this.entries[0]?.muxMetadata; - if (typeof muxMetadata !== "object" || muxMetadata === null) return false; - return (muxMetadata as Record).type === "bash-monitor-wake"; - } - /** * Effective dispatch mode across pending entries: any entry queued for tool-end * makes the whole queue dispatch at tool-end (sticky, matching pre-entry behavior), @@ -512,8 +486,7 @@ export class MessageQueue { const incomingHasAcceptedCallbacks = internal?.onAccepted != null || internal?.onAcceptedPreStreamFailure != null || - internal?.onCanceled != null || - internal?.cancelSignal != null; + internal?.onCanceled != null; const incomingIsUserAuthored = internal?.synthetic !== true && internal?.agentInitiated !== true; // Sealed entries must own their turn end-to-end: workspace-turn metadata and @@ -626,12 +599,6 @@ export class MessageQueue { }; } - if (internal?.cancelState != null) { - entry.cancelState = internal.cancelState; - } - if (internal?.cancelSignal != null) { - entry.cancelSignal = internal.cancelSignal; - } if (internal?.admissionStale != null) { entry.admissionStale = internal.admissionStale; } @@ -891,7 +858,6 @@ export class MessageQueue { entry.onAccepted != null || entry.onAcceptedPreStreamFailure != null || entry.onCanceled != null || - entry.cancelSignal != null || entry.admissionStale != null || (entry.preTurnMessages?.length ?? 0) > 0; const internal = hasInternalOptions @@ -899,8 +865,6 @@ export class MessageQueue { ...(allAddsAreSynthetic ? { synthetic: true } : {}), ...(allAddsAreAgentInitiated ? { agentInitiated: true } : {}), ...(entry.onCanceled != null ? { onCanceled: entry.onCanceled } : {}), - ...(entry.cancelState != null ? { cancelState: entry.cancelState } : {}), - ...(entry.cancelSignal != null ? { cancelSignal: entry.cancelSignal } : {}), ...(entry.onAccepted != null ? { onAccepted: entry.onAccepted } : {}), ...(entry.onAcceptedPreStreamFailure != null ? { onAcceptedPreStreamFailure: entry.onAcceptedPreStreamFailure } diff --git a/src/node/services/streamManager.test.ts b/src/node/services/streamManager.test.ts index 000488057a..523664a0f5 100644 --- a/src/node/services/streamManager.test.ts +++ b/src/node/services/streamManager.test.ts @@ -1563,9 +1563,9 @@ describe("StreamManager - engine supervision (AppFiberScope occupant)", () => { }); describe("StreamManager - stopWhen configuration", () => { - type StopWhenCondition = (options: { steps: unknown[] }) => boolean; + type StopWhenCondition = (options: { steps: unknown[] }) => boolean | Promise; type BuildStopWhenCondition = (request: { - hasQueuedMessages?: (dispatchMode?: "tool-end" | "turn-end") => boolean; + hasPendingToolEndInput?: () => Promise | boolean; toolPolicy?: ToolPolicy; }) => StopWhenCondition[]; @@ -1578,7 +1578,7 @@ describe("StreamManager - stopWhen configuration", () => { function requiredToolConditionForTests(toolPolicy: ToolPolicy): StopWhenCondition { const [, , requiredToolCondition] = buildStopWhenForTests()({ - hasQueuedMessages: () => false, + hasPendingToolEndInput: () => false, toolPolicy, }); return requiredToolCondition; @@ -1588,23 +1588,30 @@ describe("StreamManager - stopWhen configuration", () => { return { steps: [{ toolResults: [{ toolName, output }] }] }; } - test("returns step-cap and queued-message conditions with no policy", () => { - let queued = false; - const stopWhen = buildStopWhenForTests()({ hasQueuedMessages: () => queued }); + test("returns step-cap and pending tool-end input conditions with no policy", async () => { + // The hook is a live level read (queued tool-end message or outstanding bash-monitor + // wake), evaluated after the step's tool results settle — so it is awaited per step. + let pending: Promise | boolean = false; + const stopWhen = buildStopWhenForTests()({ hasPendingToolEndInput: () => pending }); expect(stopWhen).toHaveLength(3); - const [maxStepCondition, queuedMessageCondition, requiredToolCondition] = stopWhen; + const [maxStepCondition, pendingInputCondition, requiredToolCondition] = stopWhen; expect(maxStepCondition({ steps: new Array(99999) })).toBe(false); expect(maxStepCondition({ steps: new Array(100000) })).toBe(true); - expect(queuedMessageCondition({ steps: [] })).toBe(false); - queued = true; - expect(queuedMessageCondition({ steps: [] })).toBe(true); + expect(await pendingInputCondition({ steps: [] })).toBe(false); + pending = Promise.resolve(true); + expect(await pendingInputCondition({ steps: [] })).toBe(true); expect(requiredToolCondition(stepsWithToolResult("agent_report", { success: true }))).toBe( false ); }); + test("omitting the pending-input hook never stops the step loop", async () => { + const [, pendingInputCondition] = buildStopWhenForTests()({}); + expect(await pendingInputCondition({ steps: [] })).toBe(false); + }); + const requiredToolCases: Array<{ name: string; toolPolicy: ToolPolicy; @@ -2136,7 +2143,7 @@ describe("StreamManager - sequential tool execution", () => { headers?: Record; maxOutputTokens?: number; streamCallSettings?: Record; - hasQueuedMessages?: (dispatchMode?: "tool-end" | "turn-end") => boolean; + hasPendingToolEndInput?: () => Promise | boolean; toolPolicy?: ToolPolicy; toolChoice?: { type: "tool"; toolName: string }; } @@ -2245,7 +2252,7 @@ describe("StreamManager - sequential tool execution", () => { messages: [{ role: "user", content: "hello" }], system: "system", tools, - hasQueuedMessages: () => false, + hasPendingToolEndInput: () => false, }); createStreamResult(request, new AbortController()); diff --git a/src/node/services/streamManager.ts b/src/node/services/streamManager.ts index 645bd3e11a..58180d91c4 100644 --- a/src/node/services/streamManager.ts +++ b/src/node/services/streamManager.ts @@ -244,7 +244,12 @@ interface StreamRequestOptions { maxOutputTokens?: number; callSettingsOverrides?: ResolvedCallSettingsOverrides; toolPolicy?: ToolPolicy; - hasQueuedMessages?: (dispatchMode?: "tool-end" | "turn-end") => boolean; + /** + * Whether input that must run at a tool boundary is pending: a queued tool-end message + * or an outstanding bash-monitor wake. Read (not snapshotted) after every step's tool + * results settle, so a wake whose lines the step itself just showed no longer counts. + */ + hasPendingToolEndInput?: () => Promise | boolean; headers?: Record; onChunk?: StreamTextOnChunk; onStepMessages?: (messages: ModelMessage[]) => void; @@ -289,7 +294,7 @@ interface StreamRequestConfig { headers?: Record; maxOutputTokens?: number; streamCallSettings?: Omit; - hasQueuedMessages?: (dispatchMode?: "tool-end" | "turn-end") => boolean; + hasPendingToolEndInput?: () => Promise | boolean; /** Optional hook for callers that need chunk-level visibility during streaming. */ onChunk?: StreamTextOnChunk; /** Optional hook for callers that need the live prepared step transcript. */ @@ -2132,7 +2137,7 @@ export class StreamManager { maxOutputTokens, callSettingsOverrides, toolPolicy, - hasQueuedMessages, + hasPendingToolEndInput, headers, onChunk, onStepMessages, @@ -2182,7 +2187,7 @@ export class StreamManager { maxOutputTokens: effectiveMaxOutputTokens, streamCallSettings: Object.keys(streamCallSettings).length > 0 ? streamCallSettings : undefined, - hasQueuedMessages, + hasPendingToolEndInput, onChunk, onStepMessages, toolPolicy, @@ -2195,7 +2200,7 @@ export class StreamManager { } private createStopWhenCondition( - request: Pick + request: Pick ): Array> { // Completion-tool stop check: completion/routing tools use explicit // success/ok markers (agent_report, propose_plan). @@ -2241,7 +2246,7 @@ export class StreamManager { // The SDK evaluates stop conditions only after every sibling tool result in the // model's current step settles. Do not move this to individual tool-call-end events: // that would abort the remaining calls the model emitted in the same batch. - () => request.hasQueuedMessages?.("tool-end") ?? false, + async () => (await request.hasPendingToolEndInput?.()) ?? false, hasSuccessfulRequiredToolResult, ]; } @@ -3219,7 +3224,7 @@ export class StreamManager { maxOutputTokens: fallbackState.original.maxOutputTokens, callSettingsOverrides: prepared.data.callSettingsOverrides, toolPolicy: streamInfo.request.toolPolicy, - hasQueuedMessages: streamInfo.request.hasQueuedMessages, + hasPendingToolEndInput: streamInfo.request.hasPendingToolEndInput, headers: prepared.data.headers, onChunk: streamInfo.request.onChunk, onStepMessages: streamInfo.request.onStepMessages, diff --git a/src/node/services/taskService.test.ts b/src/node/services/taskService.test.ts index 29715309dd..0b36f35a8e 100644 --- a/src/node/services/taskService.test.ts +++ b/src/node/services/taskService.test.ts @@ -344,10 +344,12 @@ describe("TaskService", () => { disposable?: boolean; sendMessage?: ReturnType; remove?: ReturnType; + getInfo?: ReturnType; isStreaming?: ReturnType; hasQueuedMessages?: ReturnType; hasPendingQueuedOrPreparingTurn?: ReturnType; - hasPendingBashMonitorWakeContinuation?: ReturnType; + hasBashMonitorWakeContinuation?: ReturnType; + hasCorrelatedStreamStartedAfter?: ReturnType; hasPendingWorkspaceTurnContinuation?: ReturnType; getQueueCutCutter?: ReturnType; hasPendingAutoRetry?: ReturnType; @@ -14400,7 +14402,7 @@ describe("TaskService", () => { expect(count2).toBe(0); }); - test("backgrounds waiters when tool-end message was already queued", async () => { + test("backgrounds waiters when tool-end input was already pending", async () => { const config = await createTestConfig(rootDir); const parentId = "parent-ws"; @@ -14424,8 +14426,9 @@ describe("TaskService", () => { testTaskSettings(2, 3) ); - const hasQueuedMessages = mock(() => true); - const { workspaceService } = createWorkspaceServiceMocks({ hasQueuedMessages }); + // The union flag: a queued tool-end message or an outstanding bash-monitor wake. + const isToolEndYieldRequested = mock(() => true); + const { workspaceService } = createWorkspaceServiceMocks({ isToolEndYieldRequested }); const { taskService } = createTaskServiceHarness(config, { workspaceService }); const internal = taskService as unknown as { backgroundableForegroundWaitersByWorkspaceId: Map>; @@ -14441,7 +14444,7 @@ describe("TaskService", () => { .catch((error: unknown) => error); expect(waitError).toBeInstanceOf(ForegroundWaitBackgroundedError); - expect(hasQueuedMessages).toHaveBeenCalledWith(parentId, "tool-end"); + expect(isToolEndYieldRequested).toHaveBeenCalledWith(parentId); expect(taskService.backgroundForegroundWaitsForWorkspace(parentId)).toBe(0); expect(internal.backgroundableForegroundWaitersByWorkspaceId.has(parentId)).toBe(false); expect(internal.pendingStartWaitersByTaskId.has(childId)).toBe(false); @@ -24362,16 +24365,19 @@ describe("TaskService", () => { expect(snapshot?.reportMarkdown).toBeUndefined(); }); - test("workspace-turn tool-calls stream-end defers to a queued wake continuation", async () => { - // A queued bash-monitor wake cuts the correlated stream at a tool boundary - // (finishReason "tool-calls") while the child seamlessly continues the - // same turn — the handle must stay running. - const hasPendingBashMonitorWakeContinuation = mock( + test("workspace-turn tool-calls stream-end defers to an owed wake continuation", async () => { + // The correlated stream yielded at a tool boundary (finishReason "tool-calls") to a + // bash-monitor wake; the session still owes that continuation, so the wake turn will + // continue the same turn — the handle must stay running. + const hasBashMonitorWakeContinuation = mock( (workspaceId: string) => workspaceId === "childworkspace" ); - const { parentId, taskService } = await startWorkspaceTurnForTest({ - hasPendingBashMonitorWakeContinuation, + const { parentId, taskService, workspaceMocks } = await startWorkspaceTurnForTest({ + hasBashMonitorWakeContinuation, }); + workspaceMocks.getQueueCutCutter.mockImplementation(() => ({ + stage: "bash-monitor-wake" as const, + })); const internal = taskService as unknown as { handleStreamEnd: (event: StreamEndEvent) => Promise; }; @@ -24418,6 +24424,130 @@ describe("TaskService", () => { }); }); + test("a manual tool-end head owns the cut even while the wake level is high", async () => { + // The session attributes the cut to the queued entry (hasPendingToolEndInput: a queue + // head arbitrates alone), which runs first and breaks correlation inheritance; the wake + // behind it is not this turn's continuation. Deferring on the wake would leave the + // handle running with no correlated stream-end to come. + const hasBashMonitorWakeContinuation = mock(() => true); + const { parentId, taskService, workspaceMocks } = await startWorkspaceTurnForTest({ + hasBashMonitorWakeContinuation, + }); + workspaceMocks.getQueueCutCutter.mockImplementation(() => ({ + stage: "queued" as const, + muxMetadata: undefined, + dispatchMode: "tool-end" as const, + })); + const internal = taskService as unknown as { + handleStreamEnd: (event: StreamEndEvent) => Promise; + }; + + await internal.handleStreamEnd({ + type: "stream-end", + workspaceId: "childworkspace", + messageId: "msg_manual_cut", + metadata: { + model: "anthropic:claude-opus-4-6", + agentId: "exec", + finishReason: "tool-calls", + muxMetadata: workspaceTurnMuxMetadata(parentId), + }, + parts: [{ type: "text", text: "Kicked off verification" }], + }); + + expect(hasBashMonitorWakeContinuation).not.toHaveBeenCalled(); + const settled = await workspaceTurnSnapshot(taskService, parentId); + expect(settled?.status).not.toBe("running"); + expect(settled).toMatchObject({ messageId: "msg_manual_cut" }); + }); + + test("a same-turn continuation that already ended still defers the predecessor's stream-end", async () => { + // The continuation's stream started and finished before this (older) stream-end handler + // reached the workspace event lock: nothing is pending, in flight, or owed any more, and + // the debt it discharged is gone. Its own stream-end is queued right behind this one and + // settles the turn; settling here would pre-empt it (and delete a disposable workspace + // under the continuation's work). The session's stream-start ledger is the evidence. + const hasCorrelatedStreamStartedAfter = mock( + ( + workspaceId: string, + correlation: { taskHandleId: string; turnId: string }, + messageIds: readonly string[] + ) => + workspaceId === "childworkspace" && + correlation.taskHandleId === "wst_handle" && + correlation.turnId === "turn" && + messageIds.includes("msg_fast_continuation_cut") + ); + const { parentId, taskService, workspaceMocks } = await startWorkspaceTurnForTest({ + hasBashMonitorWakeContinuation: mock(() => false), + hasCorrelatedStreamStartedAfter, + }); + workspaceMocks.getQueueCutCutter.mockImplementation(() => ({ + stage: "bash-monitor-wake" as const, + })); + const internal = taskService as unknown as { + handleStreamEnd: (event: StreamEndEvent) => Promise; + }; + + await internal.handleStreamEnd({ + type: "stream-end", + workspaceId: "childworkspace", + messageId: "msg_fast_continuation_cut", + metadata: { + model: "anthropic:claude-opus-4-6", + agentId: "exec", + finishReason: "tool-calls", + muxMetadata: workspaceTurnMuxMetadata(parentId), + }, + parts: [{ type: "text", text: "Waiting on the build" }], + }); + + expect(hasCorrelatedStreamStartedAfter).toHaveBeenCalled(); + expect(await workspaceTurnSnapshot(taskService, parentId)).toMatchObject({ + status: "running", + deferredMessageIds: ["msg_fast_continuation_cut"], + }); + }); + + test("a wake retracted after the cut settles the handle as a wake cut instead of deferring", async () => { + // The stream yielded to the wake level, then the operator canceled the monitor before + // this stream-end was processed: the session voided its continuation debt (the void's + // own settlement found the record not yet deferred), so no continuation will ever + // arrive. The event-time cut attribution still names the wake, so the record settles as + // a wake cut (not a truncation failure) rather than deferring forever. + const { config, parentId, taskService, workspaceMocks } = await startWorkspaceTurnForTest({ + hasBashMonitorWakeContinuation: mock(() => false), + }); + workspaceMocks.getQueueCutCutter.mockImplementation(() => ({ + stage: "bash-monitor-wake" as const, + })); + const internal = taskService as unknown as { + handleStreamEnd: (event: StreamEndEvent) => Promise; + }; + + await internal.handleStreamEnd({ + type: "stream-end", + workspaceId: "childworkspace", + messageId: "msg_retracted_wake_cut", + metadata: { + model: "anthropic:claude-opus-4-6", + agentId: "exec", + finishReason: "tool-calls", + muxMetadata: workspaceTurnMuxMetadata(parentId), + }, + parts: [{ type: "text", text: "Waiting on the build" }], + }); + + expect( + await new TaskHandleStore(config).getWorkspaceTurn(parentId, "wst_handle") + ).toMatchObject({ + status: "interrupted", + messageId: "msg_retracted_wake_cut", + error: + "Workspace turn yielded at a tool boundary to a bash-monitor wake that was retracted before delivery; the target workspace is idle and this delegated turn did not complete", + }); + }); + test("nested agent progress preserves workspace-turn correlation", async () => { const hasPendingWorkspaceTurnContinuation = mock( ( @@ -24625,6 +24755,313 @@ describe("TaskService", () => { }); }); + test("settleVoidedWorkspaceTurnContinuation settles an abandoned continuation and wakes the waiter", async () => { + // The target abandoned a compaction follow-up carrying this correlation (a manual send + // won the idle race): no stream-end will ever carry the correlation again, so the + // abandonment itself settles the handle, deferred or not. + const { parentId, taskService } = await startWorkspaceTurnForTest(); + const waited = workspaceTurnManagerFor(taskService) + .waitForWorkspaceTurn("wst_handle", { requestingWorkspaceId: parentId, timeoutMs: 5_000 }) + .then( + () => null, + (error: unknown) => error + ); + + await taskService.settleVoidedWorkspaceTurnContinuation( + "childworkspace", + workspaceTurnMuxMetadata(parentId), + "abandoned" + ); + + const error = await waited; + expect(error).toBeInstanceOf(Error); + expect((error as Error).message).toContain("superseded by new input"); + expect(await workspaceTurnSnapshot(taskService, parentId)).toMatchObject({ + status: "interrupted", + error: + "Workspace turn superseded by new input in the target workspace; the workspace continues under that input and this delegated turn will not report", + }); + + // Idempotent on a settled record. + await taskService.settleVoidedWorkspaceTurnContinuation( + "childworkspace", + workspaceTurnMuxMetadata(parentId), + "abandoned" + ); + expect(await workspaceTurnSnapshot(taskService, parentId)).toMatchObject({ + status: "interrupted", + }); + }); + + test("a void retried after a partially persisted settlement still wakes the waiter", async () => { + // The first attempt persists the terminal handle and then fails (the execution-state + // mirror write rejects) before resolving the waiter. The session retries the void; the + // record is now terminal, so the retry must re-enter settlement's idempotent terminal + // branch rather than treat "already settled" as nothing left to do. + const { parentId, taskService } = await startWorkspaceTurnForTest(); + const manager = workspaceTurnManagerFor(taskService) as unknown as { + updateAgentTaskExecutionState: (...args: unknown[]) => Promise; + }; + const mirrorSpy = spyOn(manager, "updateAgentTaskExecutionState").mockImplementationOnce(() => + Promise.reject(new Error("execution state mirror unavailable")) + ); + let settled = false; + const waited = workspaceTurnManagerFor(taskService) + .waitForWorkspaceTurn("wst_handle", { requestingWorkspaceId: parentId, timeoutMs: 5_000 }) + .then( + () => null, + (error: unknown) => error + ) + .finally(() => { + settled = true; + }); + try { + const firstAttempt = await taskService + .settleVoidedWorkspaceTurnContinuation( + "childworkspace", + workspaceTurnMuxMetadata(parentId), + "abandoned" + ) + .then( + () => null, + (error: unknown) => error + ); + expect(firstAttempt).toBeInstanceOf(Error); + expect((firstAttempt as Error).message).toContain("execution state mirror unavailable"); + expect(mirrorSpy).toHaveBeenCalledTimes(1); + expect(await workspaceTurnSnapshot(taskService, parentId)).toMatchObject({ + status: "interrupted", + }); + await Promise.resolve(); + expect(settled).toBe(false); + + await taskService.settleVoidedWorkspaceTurnContinuation( + "childworkspace", + workspaceTurnMuxMetadata(parentId), + "abandoned" + ); + const error = await waited; + expect(error).toBeInstanceOf(Error); + expect((error as Error).message).toContain("superseded by new input"); + } finally { + mirrorSpy.mockRestore(); + } + }); + + test("a void retried after a partially persisted settlement still removes the disposable workspace", async () => { + // Same partial settlement as above, for a disposable workspace: the first attempt persists + // the terminal handle and throws before cleanup. The retry re-enters the terminal branch, + // which must resume the skipped cleanup instead of only repairing waiter/mirror state — + // otherwise the checkout leaks with nothing left to own it. + const remove = mock( + (_workspaceId: string): Promise> => Promise.resolve(Ok(undefined)) + ); + const getInfo = mock( + (): Promise<{ id: string } | null> => Promise.resolve({ id: "childworkspace" }) + ); + const { parentId, taskService } = await startWorkspaceTurnForTest({ + disposable: true, + remove, + getInfo, + }); + const manager = workspaceTurnManagerFor(taskService) as unknown as { + updateAgentTaskExecutionState: (...args: unknown[]) => Promise; + }; + const mirrorSpy = spyOn(manager, "updateAgentTaskExecutionState").mockImplementationOnce(() => + Promise.reject(new Error("execution state mirror unavailable")) + ); + try { + const firstAttempt = await taskService + .settleVoidedWorkspaceTurnContinuation( + "childworkspace", + workspaceTurnMuxMetadata(parentId), + "abandoned" + ) + .then( + () => null, + (error: unknown) => error + ); + expect(firstAttempt).toBeInstanceOf(Error); + expect(await workspaceTurnSnapshot(taskService, parentId)).toMatchObject({ + status: "interrupted", + disposableWorkspace: true, + }); + expect(remove).not.toHaveBeenCalled(); + + await taskService.settleVoidedWorkspaceTurnContinuation( + "childworkspace", + workspaceTurnMuxMetadata(parentId), + "abandoned" + ); + expect(remove).toHaveBeenCalledTimes(1); + expect(remove.mock.calls[0]?.[0]).toBe("childworkspace"); + + // Once the workspace is gone, later replays into the terminal branch do not retry it. + getInfo.mockImplementation(() => Promise.resolve(null)); + await taskService.settleVoidedWorkspaceTurnContinuation( + "childworkspace", + workspaceTurnMuxMetadata(parentId), + "abandoned" + ); + expect(remove).toHaveBeenCalledTimes(1); + } finally { + mirrorSpy.mockRestore(); + } + }); + + test("settleVoidedWorkspaceTurnContinuation ignores a stale correlation", async () => { + const { parentId, taskService } = await startWorkspaceTurnForTest(); + + await taskService.settleVoidedWorkspaceTurnContinuation( + "childworkspace", + workspaceTurnMuxMetadata(parentId, "wst_handle", "some-other-turn"), + "abandoned" + ); + await taskService.settleVoidedWorkspaceTurnContinuation( + "someone-else", + workspaceTurnMuxMetadata(parentId), + "abandoned" + ); + + expect(await workspaceTurnSnapshot(taskService, parentId)).toMatchObject({ status: "running" }); + }); + + test("a retracted or superseded void settles only a record the stream-end already deferred", async () => { + // A record still running has its stream-end handler queued behind the void on the same + // workspace lock; that handler reads the cleared debt and settles the turn itself, so + // the void must not pre-empt it with a wake-cut outcome the handler may not agree with. + const hasBashMonitorWakeContinuation = mock(() => true); + const { parentId, taskService, workspaceMocks } = await startWorkspaceTurnForTest({ + hasBashMonitorWakeContinuation, + }); + const correlation = workspaceTurnMuxMetadata(parentId); + + await taskService.settleVoidedWorkspaceTurnContinuation( + "childworkspace", + correlation, + "retracted" + ); + expect(await workspaceTurnSnapshot(taskService, parentId)).toMatchObject({ status: "running" }); + + // The stream-end defers on the owed continuation ... + workspaceMocks.getQueueCutCutter.mockImplementation(() => ({ + stage: "bash-monitor-wake" as const, + })); + const internal = taskService as unknown as { + handleStreamEnd: (event: StreamEndEvent) => Promise; + }; + await internal.handleStreamEnd({ + type: "stream-end", + workspaceId: "childworkspace", + messageId: "msg_deferred_cut", + metadata: { + model: "anthropic:claude-opus-4-6", + agentId: "exec", + finishReason: "tool-calls", + muxMetadata: correlation, + }, + parts: [{ type: "text", text: "Waiting on the build" }], + }); + expect(await workspaceTurnSnapshot(taskService, parentId)).toMatchObject({ + status: "running", + deferredMessageIds: ["msg_deferred_cut"], + }); + + // ... and the continuation is then retracted: the deferred record is the one this void + // exists for. + await taskService.settleVoidedWorkspaceTurnContinuation( + "childworkspace", + correlation, + "retracted" + ); + expect(await workspaceTurnSnapshot(taskService, parentId)).toMatchObject({ + status: "interrupted", + error: + "Workspace turn yielded at a tool boundary to a bash-monitor wake that was retracted before delivery; the target workspace is idle and this delegated turn did not complete", + }); + }); + + test("a void leaves a record whose turn has another correlated continuation queued", async () => { + // A correlated report queued after the wake cut also deferred the stream-end and will + // settle the record with its own stream-end. A retracted wake says nothing about that + // continuation, so the void must not interrupt the turn under it. + const hasBashMonitorWakeContinuation = mock(() => true); + let queuedContinuation = false; + const hasPendingWorkspaceTurnContinuation = mock( + (workspaceId: string, metadata: { taskHandleId: string; turnId: string }) => + queuedContinuation && + workspaceId === "childworkspace" && + metadata.taskHandleId === "wst_handle" && + metadata.turnId === "turn" + ); + const { parentId, taskService, workspaceMocks } = await startWorkspaceTurnForTest({ + hasBashMonitorWakeContinuation, + hasPendingWorkspaceTurnContinuation, + }); + const correlation = workspaceTurnMuxMetadata(parentId); + workspaceMocks.getQueueCutCutter.mockImplementation(() => ({ + stage: "bash-monitor-wake" as const, + })); + const internal = taskService as unknown as { + handleStreamEnd: (event: StreamEndEvent) => Promise; + }; + await internal.handleStreamEnd({ + type: "stream-end", + workspaceId: "childworkspace", + messageId: "msg_deferred_cut", + metadata: { + model: "anthropic:claude-opus-4-6", + agentId: "exec", + finishReason: "tool-calls", + muxMetadata: correlation, + }, + parts: [{ type: "text", text: "Waiting on the build" }], + }); + expect(await workspaceTurnSnapshot(taskService, parentId)).toMatchObject({ + status: "running", + deferredMessageIds: ["msg_deferred_cut"], + }); + + // The continuation becomes visible only while the void is already inside the settlement + // (its handle reread under the settlement lock): the check must run at that commit + // point, not on a snapshot taken before the awaits. + const store = (taskService as unknown as { taskHandleStore: TaskHandleStore }).taskHandleStore; + const getWorkspaceTurn = store.getWorkspaceTurn.bind(store); + let reads = 0; + const readSpy = spyOn(store, "getWorkspaceTurn").mockImplementation(async (...args) => { + const record = await getWorkspaceTurn(...args); + // First read: the void's own; second: settleWorkspaceTurn's reread under its lock. + if (++reads === 2) queuedContinuation = true; + return record; + }); + try { + await taskService.settleVoidedWorkspaceTurnContinuation( + "childworkspace", + correlation, + "retracted" + ); + } finally { + readSpy.mockRestore(); + } + expect(reads).toBeGreaterThanOrEqual(2); + expect(queuedContinuation).toBe(true); + expect(await workspaceTurnSnapshot(taskService, parentId)).toMatchObject({ + status: "running", + deferredMessageIds: ["msg_deferred_cut"], + }); + + // With no other continuation left, the same void settles the deferred record. + queuedContinuation = false; + await taskService.settleVoidedWorkspaceTurnContinuation( + "childworkspace", + correlation, + "superseded" + ); + expect(await workspaceTurnSnapshot(taskService, parentId)).toMatchObject({ + status: "interrupted", + }); + }); + const OWNER_FOLLOW_UP_SUPERSEDE_PREFIX = "Workspace turn superseded by follow-up turn "; function ownerFollowUpCutter(ownerWorkspaceId: string, successorHandleId: string) { diff --git a/src/node/services/taskService.ts b/src/node/services/taskService.ts index b4d5ab7706..635154e4d5 100644 --- a/src/node/services/taskService.ts +++ b/src/node/services/taskService.ts @@ -48,6 +48,7 @@ import { type WorkspaceLifecycleResult, } from "@/node/services/taskWorkspaceSeam"; export type { TaskCreateArgs, TaskKind } from "@/node/services/taskWorkspaceSeam"; +import type { WorkspaceTurnContinuationVoidReason } from "@/node/services/agentSession"; import type { HistoryService } from "@/node/services/historyService"; import type { InitStateManager } from "@/node/services/initStateManager"; import { STRUCTURED_WORKFLOW_REPORT_PLACEHOLDER_MARKDOWN } from "@/common/constants/workflowReports"; @@ -6209,6 +6210,30 @@ export class TaskService implements AgentTaskIntegration { } } + /** + * Under the same per-workspace lock as handleStreamEnd, so "the finalizer deferred on the + * debt" and "the debt was voided" are ordered: whichever runs second sees the other's + * result (see WorkspaceTurnManager.settleVoidedWorkspaceTurnContinuation). + */ + async settleVoidedWorkspaceTurnContinuation( + workspaceId: string, + muxMetadata: Extract, + reason: WorkspaceTurnContinuationVoidReason + ): Promise { + await this.workspaceEventLocks.withLock(workspaceId, async () => { + await this.getWorkspaceTurnManager().settleVoidedWorkspaceTurnContinuation( + workspaceId, + muxMetadata, + reason + ); + }); + } + + withWorkspaceEventLock(workspaceId: string, operation: () => Promise): Promise { + assert(workspaceId.length > 0, "withWorkspaceEventLock requires workspaceId"); + return this.workspaceEventLocks.withLock(workspaceId, operation); + } + /** * Reject all foreground task waiters for a workspace that opted into backgrounding * when a new message is queued. Returns the number of waiters signaled. @@ -7842,7 +7867,7 @@ export class TaskService implements AgentTaskIntegration { /** * Background any registered foreground waits for the requesting workspace when a - * tool-end message is already queued. Shared by both wait-registration paths + * tool-end message is already queued or a bash-monitor wake is outstanding. Shared by both wait-registration paths * (workspace-turn and task await): the auto-backgrounding signal is edge-triggered * on enqueue, so a message queued before the waiter registered must be re-checked * here. No-op when backgrounding is disabled or no requesting workspace is set. @@ -7854,7 +7879,7 @@ export class TaskService implements AgentTaskIntegration { if ( shouldBackgroundOnQueuedMessage && requestingWorkspaceId && - this.workspaceService.hasQueuedMessages(requestingWorkspaceId, "tool-end") + this.workspaceService.isToolEndYieldRequested(requestingWorkspaceId) ) { this.backgroundForegroundWaitsForWorkspace(requestingWorkspaceId); } diff --git a/src/node/services/taskWorkspaceSeam.testUtils.ts b/src/node/services/taskWorkspaceSeam.testUtils.ts index 5b81af0977..90744600c6 100644 --- a/src/node/services/taskWorkspaceSeam.testUtils.ts +++ b/src/node/services/taskWorkspaceSeam.testUtils.ts @@ -14,7 +14,9 @@ export function makeWorkspaceHostFake(overrides: Partial = {}): W hasQueuedMessages: () => false, hasPendingQueuedOrPreparingTurn: () => false, hasPendingAutoRetry: () => false, - hasPendingBashMonitorWakeContinuation: () => false, + hasBashMonitorWakeContinuation: () => false, + hasCorrelatedStreamStartedAfter: () => false, + isToolEndYieldRequested: () => false, hasPendingWorkspaceTurnContinuation: () => false, hasQueuedWorkspaceTurn: () => false, removeQueuedWorkspaceTurn: () => Ok(true), @@ -69,6 +71,9 @@ export function makeAgentTaskIntegrationFake( getAgentTaskStatus: () => undefined, resetAutoResumeCount: () => undefined, backgroundForegroundWaitsForWorkspace: () => 0, + settleVoidedWorkspaceTurnContinuation: () => Promise.resolve(), + withWorkspaceEventLock: (_workspaceId: string, operation: () => Promise): Promise => + operation(), markInterruptedTaskRunning: () => Promise.resolve(false), restoreInterruptedTaskAfterResumeFailure: () => Promise.resolve(), markParentWorkspaceInterrupted: () => undefined, diff --git a/src/node/services/taskWorkspaceSeam.ts b/src/node/services/taskWorkspaceSeam.ts index ecd8810187..8bea018574 100644 --- a/src/node/services/taskWorkspaceSeam.ts +++ b/src/node/services/taskWorkspaceSeam.ts @@ -11,7 +11,10 @@ import type { WorkspaceTurnTaskCorrelation, } from "@/common/types/message"; import type { Result } from "@/common/types/result"; -import type { StreamErrorRecoveryOutcome } from "@/node/services/agentSession"; +import type { + StreamErrorRecoveryOutcome, + WorkspaceTurnContinuationVoidReason, +} from "@/node/services/agentSession"; import type { RuntimeConfig } from "@/common/types/runtime"; import type { FrontendWorkspaceMetadata, WorkspaceMetadata } from "@/common/types/workspace"; import type { AgentAiSettingsLayerValues } from "@/common/types/agentAiSettings"; @@ -322,9 +325,6 @@ export interface SendMessageInternalOptions { onAccepted?: () => Promise | void; onCanceled?: (reason: string) => Promise | void; onAcceptedPreStreamFailure?: (error: SendMessageError) => Promise | void; - cancelState?: { canceledBeforeAcceptance: boolean }; - /** Cancels a synthetic send even after it has left MessageQueue for PREPARING. */ - cancelSignal?: AbortSignal; /** * Synchronous staleness probe from the caller, re-evaluated at the real admission points * (the enqueue block and the session's turn-admission gates) in addition to the @@ -399,7 +399,23 @@ export interface TurnAdmissionHost { hasQueuedMessages(workspaceId: string, dispatchMode?: "tool-end" | "turn-end"): boolean; hasPendingQueuedOrPreparingTurn(workspaceId: string): boolean; hasPendingAutoRetry(workspaceId: string): boolean; - hasPendingBashMonitorWakeContinuation(workspaceId: string): boolean; + /** + * The session still owes, or already holds, a bash-monitor wake continuation (see + * AgentSession.hasBashMonitorWakeContinuation). Sync, no I/O. + */ + hasBashMonitorWakeContinuation(workspaceId: string): boolean; + /** + * A stream carrying `correlation` started after every stream in `messageIds` — whether it + * is still running or already ended (see AgentSession.hasCorrelatedStreamStartedAfter). + * Sync, no I/O. + */ + hasCorrelatedStreamStartedAfter( + workspaceId: string, + correlation: { taskHandleId: string; ownerWorkspaceId: string; turnId: string }, + messageIds: readonly string[] + ): boolean; + /** Pending input (queued tool-end message or outstanding wake) wants a tool boundary. */ + isToolEndYieldRequested(workspaceId: string): boolean; hasPendingWorkspaceTurnContinuation( workspaceId: string, metadata: Extract @@ -518,6 +534,22 @@ export interface AgentTaskIntegration { getAgentTaskStatus(workspaceId: string): AgentTaskStatus | null | undefined; resetAutoResumeCount(workspaceId: string): void; backgroundForegroundWaitsForWorkspace(workspaceId: string): number; + /** + * The workspace will never continue the delegated turn `muxMetadata` identifies + * (AgentSession.onWorkspaceTurnContinuationVoided). Runs under the workspace event lock; + * idempotent. See WorkspaceTurnManager.settleVoidedWorkspaceTurnContinuation. + */ + settleVoidedWorkspaceTurnContinuation( + workspaceId: string, + muxMetadata: Extract, + reason: WorkspaceTurnContinuationVoidReason + ): Promise; + /** + * Run `operation` under the per-workspace event lock that serializes stream-end/abort/error + * handling for `workspaceId` (FIFO). A send made inside it is ordered after every handler + * already queued for that workspace's earlier events. + */ + withWorkspaceEventLock(workspaceId: string, operation: () => Promise): Promise; markInterruptedTaskRunning(workspaceId: string): Promise; restoreInterruptedTaskAfterResumeFailure(workspaceId: string): Promise; markParentWorkspaceInterrupted(workspaceId: string): void; diff --git a/src/node/services/turnRequestBuilder.ts b/src/node/services/turnRequestBuilder.ts index e851b13662..1f3f8aef16 100644 --- a/src/node/services/turnRequestBuilder.ts +++ b/src/node/services/turnRequestBuilder.ts @@ -272,7 +272,8 @@ export interface StreamMessageOptions { allowAgentSetGoal?: boolean; workspaceGoalService?: WorkspaceGoalService; disableWorkspaceAgents?: boolean; - hasQueuedMessages?: (dispatchMode?: "tool-end" | "turn-end") => boolean; + /** Whether input that must run at a tool boundary is pending (see StreamRequestInput). */ + hasPendingToolEndInput?: () => Promise | boolean; muxMetadata?: MuxMessageMetadata; openaiTruncationModeOverride?: "auto" | "disabled"; /** @@ -736,7 +737,7 @@ export class TurnRequestBuilder { allowAgentSetGoal, workspaceGoalService, disableWorkspaceAgents, - hasQueuedMessages, + hasPendingToolEndInput, openaiTruncationModeOverride, muxMetadata, minThinkingLevel: providedMinThinkingLevel, @@ -2856,7 +2857,7 @@ export class TurnRequestBuilder { maxOutputTokens, toolPolicy: effectiveToolPolicy, providedStreamToken: streamToken, - hasQueuedMessages, + hasPendingToolEndInput, workspaceName: metadata.name, thinkingLevel: streamThinkingLevel, headers: requestHeaders, diff --git a/src/node/services/workspaceService.test.ts b/src/node/services/workspaceService.test.ts index 12e71b83f3..fdb862666a 100644 --- a/src/node/services/workspaceService.test.ts +++ b/src/node/services/workspaceService.test.ts @@ -54,7 +54,7 @@ import type { DesktopSessionManager } from "@/node/services/desktop/DesktopSessi import type { WorktreeArchiveSnapshot } from "@/common/schemas/project"; import type { BashToolResult } from "@/common/types/tools"; import type { SendMessageOptions, WorkspaceChatMessage } from "@/common/orpc/types"; -import { createMuxMessage } from "@/common/types/message"; +import { createMuxMessage, type MuxMessageMetadata } from "@/common/types/message"; import { buildStagedAttachmentNotice } from "@/browser/features/ChatInput/stagedAttachments"; import { WORKFLOW_RESULT_METADATA_TYPE, @@ -231,6 +231,8 @@ describe("WorkspaceService bash monitor wake reconciler wiring", () => { async function createWakeWiringService() { const { config, historyService, cleanup } = await createTestHistoryService(); const events = new EventEmitter(); + // The yield flag is a real mirror so hasQueuedMessage reflects the session's arbitration. + const yieldFlags = new Map(); const backgroundProcessManager = Object.assign(events, { notifyMonitorWakeStateChanged: mock(() => undefined), getActiveMonitorCount: mock(() => 0), @@ -238,7 +240,11 @@ describe("WorkspaceService bash monitor wake reconciler wiring", () => { getMonitorWakeDeliveryState: mock(() => Promise.resolve(undefined)), acknowledgeMonitorWake: mock(() => undefined), dropRetiredMonitor: mock(() => undefined), - setMessageQueued: mock(() => undefined), + setMessageQueued: mock((workspaceId: string, queued: boolean) => { + yieldFlags.set(workspaceId, queued); + }), + hasQueuedMessage: (workspaceId: string) => yieldFlags.get(workspaceId) === true, + cleanup: mock(() => Promise.resolve()), }) as unknown as BackgroundProcessManager; const service = createWorkspaceServiceForTest({ config, @@ -249,13 +255,162 @@ describe("WorkspaceService bash monitor wake reconciler wiring", () => { ), backgroundProcessManager, }); - return { config, service, events, cleanup }; + return { config, service, events, historyService, cleanup }; } + test("delivery recovery reads a wake diverted through on-send compaction", async () => { + // The reconciler recovers "already delivered" from the owner's last durable wake row. A + // wake that crossed the compaction threshold is durable as the compaction request that + // carries it as follow-up, so the reader has to unwrap that row like carriesBashMonitorWake + // does — otherwise a restart after a failed acknowledgment re-dispatches the same output. + const { service, historyService, cleanup } = await createWakeWiringService(); + const workspaceId = "compaction-carried-wake-owner"; + const internal = service as unknown as { + readLastBashMonitorWakeRecords( + ownerWorkspaceId: string, + notBefore: string + ): Promise | undefined>; + }; + // History rows below are stamped 1_000..2_300 ms; a bound before them scans everything. + const beforeAll = new Date(0).toISOString(); + const wakeRecord = { + processId: "proc", + wakeUpdatedAt: "2026-08-31T12:00:00.000Z", + kind: "match" as const, + displayName: "run", + filter: "READY", + filterExclude: false, + }; + try { + expect(await internal.readLastBashMonitorWakeRecords(workspaceId, beforeAll)).toBeUndefined(); + await historyService.appendToHistory( + workspaceId, + createMuxMessage("wake-compaction", "user", "Compacting to continue", { + timestamp: 1_000, + synthetic: true, + muxMetadata: { + type: "compaction-request", + rawCommand: "/compact", + parsed: { + followUpContent: { + text: "Monitor output", + model: "openai:gpt-5.2", + agentId: "exec", + muxMetadata: { type: "bash-monitor-wake", records: [wakeRecord] }, + }, + }, + source: "auto-compaction", + }, + }) + ); + await historyService.appendToHistory( + workspaceId, + createMuxMessage("summary", "assistant", "summary", { timestamp: 1_100 }) + ); + expect(await internal.readLastBashMonitorWakeRecords(workspaceId, beforeAll)).toEqual([ + wakeRecord, + ]); + + // An acknowledgment that kept failing while the accepted wake turn ran leaves the row + // behind however many rows that turn produced; a fixed tail depth would miss it and the + // restarted reconciler would redeliver the output. Bury the row deep and cross a + // compaction boundary on the way. + await historyService.appendToHistory( + workspaceId, + createMuxMessage("boundary", "assistant", "compacted summary", { + timestamp: 1_200, + compacted: "user", + }) + ); + for (let i = 0; i < 300; i++) { + await historyService.appendToHistory( + workspaceId, + createMuxMessage(`tool-step-${i}`, "assistant", `step ${i}`, { timestamp: 2_000 + i }) + ); + } + expect(await internal.readLastBashMonitorWakeRecords(workspaceId, beforeAll)).toEqual([ + wakeRecord, + ]); + + // A newer wake row whose persisted `records` is not usable (corrupt shape, or a legacy + // row without identities) is skipped rather than returned: the reconciler maps over the + // result, so returning it would fail every reconcile retry and strand current wakes. + await historyService.appendToHistory( + workspaceId, + createMuxMessage("wake-corrupt", "user", "Background monitor wake", { + timestamp: 2_300, + synthetic: true, + muxMetadata: { type: "bash-monitor-wake", records: "corrupt" } as unknown as Extract< + MuxMessageMetadata, + { type: "bash-monitor-wake" } + >, + }) + ); + await historyService.appendToHistory( + workspaceId, + createMuxMessage("wake-legacy", "user", "Background monitor wake", { + timestamp: 2_301, + synthetic: true, + muxMetadata: { + type: "bash-monitor-wake", + records: [{ kind: "match", displayName: "run", filter: "READY", filterExclude: false }], + }, + }) + ); + expect(await internal.readLastBashMonitorWakeRecords(workspaceId, beforeAll)).toEqual([ + wakeRecord, + ]); + + // RLM keep-recent compaction re-appends copies of the pre-boundary tail after the + // boundary with their *source* timestamps. Sitting at the tail, an old-stamped copy is + // reached before the wake row; if it tripped the cutoff the scan would stop short and + // the restarted reconciler would redeliver an acknowledged wake. Bound so the copy reads + // as too old while the wake row does not. + await historyService.appendToHistory( + workspaceId, + createMuxMessage("tail-copy", "user", "old turn copy", { + timestamp: 500, + synthetic: true, + rlmPreservedTailCopy: true, + }) + ); + const armedAtWakeRow = new Date(1_000 + 60_000).toISOString(); + expect(await internal.readLastBashMonitorWakeRecords(workspaceId, armedAtWakeRow)).toEqual([ + wakeRecord, + ]); + + // The scan is bounded by the oldest outstanding monitor's arm time: rows appended before + // that monitor existed cannot acknowledge it, so they are never parsed (an owner's first + // wake would otherwise read the entire transcript on the stream's tool-boundary path). + // The bound carries a clock-step margin, so place it well past the wake row. + const armedAfterWakeRow = new Date(1_000 + 60_000 + 1_000_000).toISOString(); + expect( + await internal.readLastBashMonitorWakeRecords(workspaceId, armedAfterWakeRow) + ).toBeUndefined(); + + // "Could not read" is not "no row": the reconciler consults this once per owner. + const readSpy = spyOn(historyService, "iterateFullHistory").mockImplementationOnce(() => + Promise.resolve(Err("history unavailable")) + ); + try { + const failed = await internal.readLastBashMonitorWakeRecords(workspaceId, beforeAll).then( + () => null, + (error: unknown) => error + ); + expect(failed).toBeInstanceOf(Error); + } finally { + readSpy.mockRestore(); + } + } finally { + await cleanup(); + } + }); + test("monitor lifecycle and shown-output events poke the reconciler", async () => { const { service, events, cleanup } = await createWakeWiringService(); const scheduleReconcile = mock(() => undefined); const discardProcess = mock(() => Promise.resolve()); + const outputShown = mock(() => Promise.resolve()); const upsert = mock(() => Promise.resolve()); const remove = mock(() => Promise.resolve()); const recordTerminal = mock(() => Promise.resolve()); @@ -264,6 +419,7 @@ describe("WorkspaceService bash monitor wake reconciler wiring", () => { bashMonitorWakeReconciler: { scheduleReconcile: typeof scheduleReconcile; discardProcess: typeof discardProcess; + outputShown: typeof outputShown; }; bashMonitorRegistryStore: { upsert: typeof upsert; @@ -273,7 +429,7 @@ describe("WorkspaceService bash monitor wake reconciler wiring", () => { }; try { await internal.bashMonitorRecoveryPromise; - internal.bashMonitorWakeReconciler = { scheduleReconcile, discardProcess }; + internal.bashMonitorWakeReconciler = { scheduleReconcile, discardProcess, outputShown }; internal.bashMonitorRegistryStore = { upsert, remove, recordTerminal }; const armed = { processId: "proc", @@ -285,7 +441,7 @@ describe("WorkspaceService bash monitor wake reconciler wiring", () => { createdAt: "2026-08-31T12:00:00.000Z", }; events.emit("monitor:match", "owner", {}); - events.emit("output:shown", "owner", {}); + events.emit("output:shown", "owner", { processId: "proc", shownThroughOffset: 12 }); events.emit("monitor:armed", "owner", armed); events.emit("monitor:stopped", "owner", { processId: "proc", @@ -297,8 +453,12 @@ describe("WorkspaceService bash monitor wake reconciler wiring", () => { } expect(upsert).toHaveBeenCalledWith(armed); + // Cancellation retires any wake already handed out for this process before the + // registry row goes, so its captured output is never sent as a turn. expect(discardProcess).toHaveBeenCalledWith("owner", "proc", armed.createdAt); expect(remove).toHaveBeenCalledWith("owner", "proc", armed.createdAt); + // A shown-frontier advance revalidates any wake already handed out for the process. + expect(outputShown).toHaveBeenCalledWith("owner", "proc"); expect(scheduleReconcile).toHaveBeenCalledTimes(4); } finally { await cleanup(); @@ -505,7 +665,6 @@ describe("WorkspaceService bash monitor wake reconciler wiring", () => { test("cancellation invalidates a scheduled runtime failure persistence retry", async () => { const { service, events, cleanup } = await createWakeWiringService(); const scheduleReconcile = mock(() => undefined); - const discardProcess = mock(() => Promise.resolve()); const upsert = mock(() => Promise.resolve()); const remove = mock(() => Promise.resolve()); const recordLost = mock(() => Promise.reject(new Error("transient registry write failure"))); @@ -513,7 +672,7 @@ describe("WorkspaceService bash monitor wake reconciler wiring", () => { bashMonitorRecoveryPromise: Promise; bashMonitorWakeReconciler: { scheduleReconcile: typeof scheduleReconcile; - discardProcess: typeof discardProcess; + discardProcess(workspaceId: string, processId: string, createdAt: string): Promise; }; bashMonitorRegistryStore: { upsert: typeof upsert; @@ -524,7 +683,10 @@ describe("WorkspaceService bash monitor wake reconciler wiring", () => { }; try { await internal.bashMonitorRecoveryPromise; - internal.bashMonitorWakeReconciler = { scheduleReconcile, discardProcess }; + internal.bashMonitorWakeReconciler = { + scheduleReconcile, + discardProcess: () => Promise.resolve(), + }; internal.bashMonitorRegistryStore = { upsert, remove, @@ -560,11 +722,6 @@ describe("WorkspaceService bash monitor wake reconciler wiring", () => { expect(recordLost).toHaveBeenCalledTimes(1); expect(upsert).toHaveBeenCalledTimes(1); expect(remove).toHaveBeenCalledWith("owner", armMetadata.processId, armMetadata.createdAt); - expect(discardProcess).toHaveBeenCalledWith( - "owner", - armMetadata.processId, - armMetadata.createdAt - ); // The invalidated chain must also release its tracking entry so the // per-process failure-persist map stays bounded by in-flight chains. const tracking = ( @@ -748,8 +905,7 @@ describe("WorkspaceService bash monitor wake reconciler wiring", () => { ownerWorkspaceId: string; prompt: string; muxMetadata: { type: "bash-monitor-wake"; records: [] }; - dedupeKey: string; - cancelSignal: AbortSignal; + isCurrent(): boolean; onAccepted(): Promise; onDeferred(): Promise; }): Promise<"in-flight" | "deferred">; @@ -761,8 +917,7 @@ describe("WorkspaceService bash monitor wake reconciler wiring", () => { ownerWorkspaceId: workspaceId, prompt: "wake", muxMetadata: { type: "bash-monitor-wake", records: [] }, - dedupeKey: "wake", - cancelSignal: new AbortController().signal, + isCurrent: () => true, onAccepted, onDeferred, }); @@ -774,7 +929,7 @@ describe("WorkspaceService bash monitor wake reconciler wiring", () => { } }); - test("active session-backed streams queue monitor wakes at tool end", async () => { + test("streaming owners never queue a wake: the stream yields on the level instead", async () => { const { config, service, cleanup } = await createWakeWiringService(); const workspaceId = "streaming-wake-owner"; await config.addWorkspace("/tmp/streaming-wake-project", { @@ -784,34 +939,19 @@ describe("WorkspaceService bash monitor wake reconciler wiring", () => { projectPath: "/tmp/streaming-wake-project", runtimeConfig: { type: "local" }, }); - let queuedMode: string | undefined; - let queuedCancelState: { canceledBeforeAcceptance: boolean } | undefined; - const sendMessage = mock( - ( - _workspaceId: string, - _prompt: string, - options: { queueDispatchMode?: string }, - internal?: { cancelState?: { canceledBeforeAcceptance: boolean } } - ) => { - queuedMode = options.queueDispatchMode; - queuedCancelState = internal?.cancelState; - return Promise.resolve(Ok(undefined)); - } - ); + const sendMessage = mock(() => Promise.resolve(Ok(undefined))); const afterIdle = mock(() => undefined); const internal = service as unknown as { aiService: { isStreaming(workspaceId: string): boolean }; hasPendingQueuedOrPreparingTurn(workspaceId: string): boolean; isBusyForMessage(workspaceId: string): boolean; scheduleBashMonitorWakeReconcileAfterIdle(workspaceId: string): void; - getDelegatedTurnContinuationSendOptions(workspaceId: string): Promise; sendMessage: typeof sendMessage; dispatchBashMonitorWake(dispatch: { ownerWorkspaceId: string; prompt: string; muxMetadata: { type: "bash-monitor-wake"; records: [] }; - dedupeKey: string; - cancelSignal: AbortSignal; + isCurrent(): boolean; onAccepted(): Promise; onDeferred(): Promise; }): Promise<"in-flight" | "deferred">; @@ -821,71 +961,111 @@ describe("WorkspaceService bash monitor wake reconciler wiring", () => { internal.hasPendingQueuedOrPreparingTurn = () => false; internal.isBusyForMessage = () => true; internal.scheduleBashMonitorWakeReconcileAfterIdle = afterIdle; - internal.getDelegatedTurnContinuationSendOptions = () => Promise.resolve({}); internal.sendMessage = sendMessage; const outcome = await internal.dispatchBashMonitorWake({ ownerWorkspaceId: workspaceId, prompt: "wake", muxMetadata: { type: "bash-monitor-wake", records: [] }, - dedupeKey: "wake", - cancelSignal: new AbortController().signal, + isCurrent: () => true, onAccepted: () => Promise.resolve(), onDeferred: () => Promise.resolve(), }); - expect(outcome).toBe("in-flight"); - expect(sendMessage).toHaveBeenCalledTimes(1); - expect(queuedMode).toBe("tool-end"); - expect(queuedCancelState).toEqual({ canceledBeforeAcceptance: false }); + expect(outcome).toBe("deferred"); + expect(sendMessage).not.toHaveBeenCalled(); + expect(afterIdle).toHaveBeenCalledWith(workspaceId); + } finally { + await cleanup(); + } + }); + + test("a stream without a busy session defers without re-arming the idle wait", async () => { + // The idle wait resolves immediately when no session is busy, so re-arming here would + // spin reconcile → defer → re-arm until the stream ends; stream-end schedules instead. + const { config, service, cleanup } = await createWakeWiringService(); + const workspaceId = "orphan-stream-wake-owner"; + await config.addWorkspace("/tmp/orphan-stream-wake-project", { + id: workspaceId, + name: workspaceId, + projectName: "orphan-stream-wake-project", + projectPath: "/tmp/orphan-stream-wake-project", + runtimeConfig: { type: "local" }, + }); + const sendMessage = mock(() => Promise.resolve(Ok(undefined))); + const afterIdle = mock(() => undefined); + const internal = service as unknown as { + aiService: { isStreaming(workspaceId: string): boolean }; + hasPendingQueuedOrPreparingTurn(workspaceId: string): boolean; + isBusyForMessage(workspaceId: string): boolean; + scheduleBashMonitorWakeReconcileAfterIdle(workspaceId: string): void; + sendMessage: typeof sendMessage; + dispatchBashMonitorWake(dispatch: { + ownerWorkspaceId: string; + prompt: string; + muxMetadata: { type: "bash-monitor-wake"; records: [] }; + isCurrent(): boolean; + onAccepted(): Promise; + onDeferred(): Promise; + }): Promise<"in-flight" | "deferred">; + }; + try { + internal.aiService = { isStreaming: () => true }; + internal.hasPendingQueuedOrPreparingTurn = () => false; + internal.isBusyForMessage = () => false; + internal.scheduleBashMonitorWakeReconcileAfterIdle = afterIdle; + internal.sendMessage = sendMessage; + + const outcome = await internal.dispatchBashMonitorWake({ + ownerWorkspaceId: workspaceId, + prompt: "wake", + muxMetadata: { type: "bash-monitor-wake", records: [] }, + isCurrent: () => true, + onAccepted: () => Promise.resolve(), + onDeferred: () => Promise.resolve(), + }); + + expect(outcome).toBe("deferred"); + expect(sendMessage).not.toHaveBeenCalled(); expect(afterIdle).not.toHaveBeenCalled(); } finally { await cleanup(); } }); - test("withdrawing a queued monitor wake removes it and releases its dedupe key", async () => { + test("idle owners receive the wake as a direct synthetic turn", async () => { const { config, service, cleanup } = await createWakeWiringService(); - const workspaceId = "withdrawn-wake-owner"; - await config.addWorkspace("/tmp/withdrawn-wake-project", { + const workspaceId = "idle-wake-owner"; + await config.addWorkspace("/tmp/idle-wake-project", { id: workspaceId, name: workspaceId, - projectName: "withdrawn-wake-project", - projectPath: "/tmp/withdrawn-wake-project", + projectName: "idle-wake-project", + projectPath: "/tmp/idle-wake-project", runtimeConfig: { type: "local" }, }); - const session = service.getOrCreateSession(workspaceId); - const queuedModes: Array<"tool-end" | "turn-end" | null> = []; - // The real sendMessage queues behind a busy session; mirror only that branch. + let sentOptions: { queueDispatchMode?: string; muxMetadata?: unknown } | undefined; + let sentInternal: + | { requireIdle?: boolean; admissionStale?: () => boolean; onCanceled?: unknown } + | undefined; const sendMessage = mock( ( _workspaceId: string, - prompt: string, - options: SendMessageOptions, + _prompt: string, + options: { queueDispatchMode?: string; muxMetadata?: unknown }, internal?: { - synthetic?: boolean; - agentInitiated?: boolean; - queueDedupeKey?: string; - removableQueueDedupeKey?: boolean; - cancelState?: { canceledBeforeAcceptance: boolean }; - cancelSignal?: AbortSignal; - onCanceled?: (reason: string) => Promise | void; + onAccepted?: () => Promise; + requireIdle?: boolean; + admissionStale?: () => boolean; + onCanceled?: unknown; } ) => { - queuedModes.push( - session.queueMessage(prompt, options, { - synthetic: internal?.synthetic, - agentInitiated: internal?.agentInitiated, - dedupeKey: internal?.queueDedupeKey, - removableDedupeKey: internal?.removableQueueDedupeKey, - cancelState: internal?.cancelState, - cancelSignal: internal?.cancelSignal, - onCanceled: internal?.onCanceled, - }) - ); - return Promise.resolve(Ok(undefined)); + sentOptions = options; + sentInternal = internal; + return internal?.onAccepted?.().then(() => Ok(undefined)) ?? Promise.resolve(Ok(undefined)); } ); + const onAccepted = mock(() => Promise.resolve()); + let current = true; const internal = service as unknown as { aiService: { isStreaming(workspaceId: string): boolean }; hasPendingQueuedOrPreparingTurn(workspaceId: string): boolean; @@ -896,50 +1076,514 @@ describe("WorkspaceService bash monitor wake reconciler wiring", () => { ownerWorkspaceId: string; prompt: string; muxMetadata: { type: "bash-monitor-wake"; records: [] }; - dedupeKey: string; - cancelSignal: AbortSignal; + isCurrent(): boolean; onAccepted(): Promise; onDeferred(): Promise; }): Promise<"in-flight" | "deferred">; }; - const dedupeKey = "bash-monitor-wake:" + workspaceId + ":dispatch-1"; - const onDeferred = mock(() => Promise.resolve()); - const dispatch = (cancelSignal: AbortSignal) => - internal.dispatchBashMonitorWake({ + try { + internal.aiService = { isStreaming: () => false }; + internal.hasPendingQueuedOrPreparingTurn = () => false; + internal.isBusyForMessage = () => false; + internal.getDelegatedTurnContinuationSendOptions = () => Promise.resolve({}); + internal.sendMessage = sendMessage; + + const outcome = await internal.dispatchBashMonitorWake({ + ownerWorkspaceId: workspaceId, + prompt: "wake", + muxMetadata: { type: "bash-monitor-wake", records: [] }, + isCurrent: () => current, + onAccepted, + onDeferred: () => Promise.resolve(), + }); + + expect(outcome).toBe("in-flight"); + expect(sendMessage).toHaveBeenCalledTimes(1); + expect(sentOptions?.queueDispatchMode).toBeUndefined(); + expect(sentOptions?.muxMetadata).toEqual({ type: "bash-monitor-wake", records: [] }); + expect(onAccepted).toHaveBeenCalledTimes(1); + // The wake is never queued (a racing manual send makes it a skip instead), and the + // admission probe tracks the wake's validity through every pre-durability gate so a + // monitor canceled mid-admission refuses the send. + expect(sentInternal?.requireIdle).toBe(true); + expect(sentInternal?.onCanceled).toBeUndefined(); + expect(sentInternal?.admissionStale?.()).toBe(false); + current = false; + expect(sentInternal?.admissionStale?.()).toBe(true); + } finally { + await cleanup(); + } + }); + + test("the wake send waits behind the owner's queued stream-end handling", async () => { + // The stream that yielded to the wake emits stream-end, and TaskService handles it under + // the workspace event lock. That handler reads the continuation debt to defer a delegated + // turn; a wake that streamed first would have redeemed the debt and the handler would + // read the cut as retracted. Dispatch therefore enters the same (FIFO) lock. + const { config, service, cleanup } = await createWakeWiringService(); + const workspaceId = "event-locked-wake-owner"; + await config.addWorkspace("/tmp/event-locked-wake-project", { + id: workspaceId, + name: workspaceId, + projectName: "event-locked-wake-project", + projectPath: "/tmp/event-locked-wake-project", + runtimeConfig: { type: "local" }, + }); + let releaseHandler: () => void = () => undefined; + const handlerDone = new Promise((resolve) => { + releaseHandler = resolve; + }); + const lockedWorkspaceIds: string[] = []; + service.setAgentTaskIntegration( + makeAgentTaskIntegrationFake({ + withWorkspaceEventLock: async (lockedWorkspaceId, operation) => { + lockedWorkspaceIds.push(lockedWorkspaceId); + await handlerDone; + return operation(); + }, + }) + ); + const sendMessage = mock( + ( + _workspaceId: string, + _prompt: string, + _options: unknown, + internal?: { onAccepted?: () => Promise } + ) => internal?.onAccepted?.().then(() => Ok(undefined)) ?? Promise.resolve(Ok(undefined)) + ); + const internal = service as unknown as { + aiService: { isStreaming(workspaceId: string): boolean }; + hasPendingQueuedOrPreparingTurn(workspaceId: string): boolean; + isBusyForMessage(workspaceId: string): boolean; + getDelegatedTurnContinuationSendOptions(workspaceId: string): Promise; + sendMessage: typeof sendMessage; + dispatchBashMonitorWake(dispatch: { + ownerWorkspaceId: string; + prompt: string; + muxMetadata: { type: "bash-monitor-wake"; records: [] }; + isCurrent(): boolean; + onAccepted(): Promise; + onDeferred(): Promise; + }): Promise<"in-flight" | "deferred">; + }; + try { + internal.aiService = { isStreaming: () => false }; + internal.hasPendingQueuedOrPreparingTurn = () => false; + internal.isBusyForMessage = () => false; + internal.getDelegatedTurnContinuationSendOptions = () => Promise.resolve({}); + internal.sendMessage = sendMessage; + + const outcome = internal.dispatchBashMonitorWake({ ownerWorkspaceId: workspaceId, prompt: "wake", muxMetadata: { type: "bash-monitor-wake", records: [] }, - dedupeKey, - cancelSignal, + isCurrent: () => true, onAccepted: () => Promise.resolve(), - onDeferred, + onDeferred: () => Promise.resolve(), + }); + await Promise.resolve(); + await Promise.resolve(); + expect(lockedWorkspaceIds).toEqual([workspaceId]); + expect(sendMessage).not.toHaveBeenCalled(); + + releaseHandler(); + expect(await outcome).toBe("in-flight"); + expect(sendMessage).toHaveBeenCalledTimes(1); + } finally { + await cleanup(); + } + }); + + test("a monitor canceled while send options resolve retires the wake before it is sent", async () => { + const { config, service, cleanup } = await createWakeWiringService(); + const workspaceId = "canceled-mid-dispatch-wake-owner"; + await config.addWorkspace("/tmp/canceled-mid-dispatch-project", { + id: workspaceId, + name: workspaceId, + projectName: "canceled-mid-dispatch-project", + projectPath: "/tmp/canceled-mid-dispatch-project", + runtimeConfig: { type: "local" }, + }); + const sendMessage = mock(() => Promise.resolve(Ok(undefined))); + let current = true; + const internal = service as unknown as { + aiService: { isStreaming(workspaceId: string): boolean }; + hasPendingQueuedOrPreparingTurn(workspaceId: string): boolean; + isBusyForMessage(workspaceId: string): boolean; + getDelegatedTurnContinuationSendOptions(workspaceId: string): Promise; + sendMessage: typeof sendMessage; + dispatchBashMonitorWake(dispatch: { + ownerWorkspaceId: string; + prompt: string; + muxMetadata: { type: "bash-monitor-wake"; records: [] }; + isCurrent(): boolean; + onAccepted(): Promise; + onDeferred(): Promise; + }): Promise<"in-flight" | "deferred">; + }; + try { + internal.aiService = { isStreaming: () => false }; + internal.hasPendingQueuedOrPreparingTurn = () => false; + internal.isBusyForMessage = () => false; + // The operator cancels the monitor (discardProcess) while the continuation options + // are being resolved: the wake passed the entry check but must not reach sendMessage. + internal.getDelegatedTurnContinuationSendOptions = () => { + current = false; + return Promise.resolve({}); + }; + internal.sendMessage = sendMessage; + + const outcome = await internal.dispatchBashMonitorWake({ + ownerWorkspaceId: workspaceId, + prompt: "wake", + muxMetadata: { type: "bash-monitor-wake", records: [] }, + isCurrent: () => current, + onAccepted: () => Promise.resolve(), + onDeferred: () => Promise.resolve(), }); + + expect(outcome).toBe("deferred"); + expect(sendMessage).not.toHaveBeenCalled(); + } finally { + await cleanup(); + } + }); + + test("a wake retired by a history clear while waiting for the lock is dropped, not sent", async () => { + const { config, service, cleanup } = await createWakeWiringService(); + const workspaceId = "stale-wake-owner"; + await config.addWorkspace("/tmp/stale-wake-project", { + id: workspaceId, + name: workspaceId, + projectName: "stale-wake-project", + projectPath: "/tmp/stale-wake-project", + runtimeConfig: { type: "local" }, + }); + const sendMessage = mock(() => Promise.resolve(Ok(undefined))); + const internal = service as unknown as { + aiService: { isStreaming(workspaceId: string): boolean }; + hasPendingQueuedOrPreparingTurn(workspaceId: string): boolean; + isBusyForMessage(workspaceId: string): boolean; + getDelegatedTurnContinuationSendOptions(workspaceId: string): Promise; + sendMessage: typeof sendMessage; + dispatchBashMonitorWake(dispatch: { + ownerWorkspaceId: string; + prompt: string; + muxMetadata: { type: "bash-monitor-wake"; records: [] }; + isCurrent(): boolean; + onAccepted(): Promise; + onDeferred(): Promise; + }): Promise<"in-flight" | "deferred">; + }; try { - internal.aiService = { isStreaming: () => true }; + internal.aiService = { isStreaming: () => false }; internal.hasPendingQueuedOrPreparingTurn = () => false; - internal.isBusyForMessage = () => true; + internal.isBusyForMessage = () => false; + internal.getDelegatedTurnContinuationSendOptions = () => Promise.resolve({}); + internal.sendMessage = sendMessage; + + const outcome = await internal.dispatchBashMonitorWake({ + ownerWorkspaceId: workspaceId, + prompt: "wake", + muxMetadata: { type: "bash-monitor-wake", records: [] }, + isCurrent: () => false, + onAccepted: () => Promise.resolve(), + onDeferred: () => Promise.resolve(), + }); + + expect(outcome).toBe("deferred"); + expect(sendMessage).not.toHaveBeenCalled(); + } finally { + await cleanup(); + } + }); + + test("a wake skipped by a manual send that won the race re-arms instead of queuing", async () => { + const { config, service, cleanup } = await createWakeWiringService(); + const workspaceId = "skipped-wake-owner"; + await config.addWorkspace("/tmp/skipped-wake-project", { + id: workspaceId, + name: workspaceId, + projectName: "skipped-wake-project", + projectPath: "/tmp/skipped-wake-project", + runtimeConfig: { type: "local" }, + }); + // requireIdle skip: the send never queued and onAccepted never fired. + const sendMessage = mock(() => + Promise.resolve( + Err({ type: "unknown", raw: "Workspace is busy; idle-only send was skipped." }) + ) + ); + const afterIdle = mock(() => undefined); + const internal = service as unknown as { + aiService: { isStreaming(workspaceId: string): boolean }; + hasPendingQueuedOrPreparingTurn(workspaceId: string): boolean; + isBusyForMessage(workspaceId: string): boolean; + getDelegatedTurnContinuationSendOptions(workspaceId: string): Promise; + scheduleBashMonitorWakeReconcileAfterIdle(workspaceId: string): void; + sendMessage: typeof sendMessage; + dispatchBashMonitorWake(dispatch: { + ownerWorkspaceId: string; + prompt: string; + muxMetadata: { type: "bash-monitor-wake"; records: [] }; + isCurrent(): boolean; + onAccepted(): Promise; + onDeferred(): Promise; + }): Promise<"in-flight" | "deferred">; + }; + try { + internal.aiService = { isStreaming: () => false }; + internal.hasPendingQueuedOrPreparingTurn = () => false; + internal.isBusyForMessage = () => false; internal.getDelegatedTurnContinuationSendOptions = () => Promise.resolve({}); + internal.scheduleBashMonitorWakeReconcileAfterIdle = afterIdle; internal.sendMessage = sendMessage; - const controller = new AbortController(); - expect(await dispatch(controller.signal)).toBe("in-flight"); - expect(session.hasQueuedMessages("tool-end")).toBe(true); + const outcome = await internal.dispatchBashMonitorWake({ + ownerWorkspaceId: workspaceId, + prompt: "wake", + muxMetadata: { type: "bash-monitor-wake", records: [] }, + isCurrent: () => true, + onAccepted: () => Promise.resolve(), + onDeferred: () => Promise.resolve(), + }); - controller.abort("output already shown"); - expect(session.hasQueuedMessages()).toBe(false); - expect(service.removeQueuedMessagesByDedupeKeyPrefix(workspaceId, dedupeKey)).toEqual(Ok(0)); + expect(outcome).toBe("deferred"); + expect(afterIdle).toHaveBeenCalledTimes(1); + } finally { + await cleanup(); + } + }); + + test("a wake continuation owed or in flight in the session is visible to turn settlement", async () => { + const { service, cleanup } = await createWakeWiringService(); + const workspaceId = "preparing-wake-owner"; + const session = service.getOrCreateSession(workspaceId); + const sessionInternal = session as unknown as { + hasBashMonitorWakeContinuation(): boolean; + }; + try { + expect(service.hasBashMonitorWakeContinuation(workspaceId)).toBe(false); + // The reconciler level is already low here (onAccepted ran at row persistence); + // the session's own debt / in-flight state is what settlement reads. + sessionInternal.hasBashMonitorWakeContinuation = () => true; + expect(service.hasBashMonitorWakeContinuation(workspaceId)).toBe(true); + expect(service.hasBashMonitorWakeContinuation("no-such-session")).toBe(false); + } finally { + session.dispose(); + await cleanup(); + } + }); + + test("the wake level reaches a session still running startup recovery", async () => { + // Startup recovery runs the recovered turn inside a transient session before promoting + // it. A wake published while that turn streams must reach it, or the stream's foreground + // waits are never backgrounded and the deferred wake waits for the stream to end. + const { service, cleanup } = await createWakeWiringService(); + const workspaceId = "transient-recovery-wake-owner"; + const internal = service as unknown as { + backgroundProcessManager: { setMessageQueued: ReturnType }; + createSession(workspaceId: string): AgentSession; + transientStartupRecoverySessions: Map; + publishBashMonitorWakeLevel(ownerWorkspaceId: string, outstanding: boolean): void; + }; + const setMessageQueued = internal.backgroundProcessManager.setMessageQueued; + const session = internal.createSession(workspaceId); + internal.transientStartupRecoverySessions.set(workspaceId, session); + try { + // The session's lever is the observable: with no queue head the level is effective at + // once, so long-polling bash reads return early and foreground waits background. + internal.publishBashMonitorWakeLevel(workspaceId, true); + expect(setMessageQueued).toHaveBeenLastCalledWith(workspaceId, true); + + // Settlement's reads of the cut resolve through the same live lookup: a recovered + // delegated stream that yielded to the wake must not settle for want of a cutter. + const sessionInternal = session as unknown as { + getQueueCutCutter(): unknown; + hasBashMonitorWakeContinuation(): boolean; + }; + sessionInternal.getQueueCutCutter = () => ({ stage: "bash-monitor-wake" }); + sessionInternal.hasBashMonitorWakeContinuation = () => true; + expect(service.getQueueCutCutter(workspaceId)).toEqual({ stage: "bash-monitor-wake" }); + expect(service.hasBashMonitorWakeContinuation(workspaceId)).toBe(true); + + // A wake deferred because this session is busy waits for *this* session: if the wait + // resolved through `sessions` alone it would return at once and re-defer in a loop. + let busy = true; + let releaseIdle: () => void = () => undefined; + const idle = new Promise((resolve) => { + releaseIdle = resolve; + }); + const busyInternal = session as unknown as { + isBusy(): boolean; + waitForIdle(): Promise; + }; + busyInternal.isBusy = () => busy; + busyInternal.waitForIdle = () => idle; + expect(service.isBusyForMessage(workspaceId)).toBe(true); + let idleWaitResolved = false; + const idleWait = service.waitForIdleAndNoQueuedMessages(workspaceId).then(() => { + idleWaitResolved = true; + }); await new Promise((resolve) => setTimeout(resolve, 0)); - expect(onDeferred).toHaveBeenCalledTimes(1); + expect(idleWaitResolved).toBe(false); + busy = false; + releaseIdle(); + await idleWait; - // Already withdrawn at dispatch: never reaches the send, so nothing can be enqueued. - expect(await dispatch(controller.signal)).toBe("deferred"); - expect(sendMessage).toHaveBeenCalledTimes(1); + // Promotion keeps the mirror: it lives on the session, not on the map it sits in. + expect(service.getOrCreateSession(workspaceId)).toBe(session); + internal.publishBashMonitorWakeLevel(workspaceId, false); + expect(setMessageQueued).toHaveBeenLastCalledWith(workspaceId, false); + } finally { + session.dispose(); + await cleanup(); + } + }); + + test("routes the session's tool-end yield edge to backgroundForegroundWaitsForWorkspace", async () => { + // Which transitions raise the edge is the session's business + // (agentSession.queueDispatch.test.ts); the service only routes it. + const { service, cleanup } = await createWakeWiringService(); + const workspaceId = "yield-edge-owner"; + const backgroundForegroundWaitsForWorkspace = mock(() => 0); + service.setAgentTaskIntegration( + makeAgentTaskIntegrationFake({ backgroundForegroundWaitsForWorkspace }) + ); + const session = service.getOrCreateSession(workspaceId); + try { + session.queueMessage("later", { + model: "anthropic:claude-sonnet-4-5", + agentId: "exec", + queueDispatchMode: "turn-end", + }); + expect(backgroundForegroundWaitsForWorkspace).not.toHaveBeenCalled(); + session.queueMessage("sooner", { + model: "anthropic:claude-sonnet-4-5", + agentId: "exec", + queueDispatchMode: "tool-end", + }); + expect(backgroundForegroundWaitsForWorkspace).toHaveBeenCalledWith(workspaceId); + } finally { + session.dispose(); + await cleanup(); + } + }); + + test("a high level backgrounds foreground waits only when it pulls the yield lever", async () => { + // A turn-end queue head suppresses the wake cut (hasPendingToolEndInput arbitration), so + // the same level must not end a foreground task_await early either: the stream would + // not cut and the agent would simply get another model step. + const { service, cleanup } = await createWakeWiringService(); + const workspaceId = "level-background-waits-owner"; + const backgroundForegroundWaitsForWorkspace = mock(() => 0); + service.setAgentTaskIntegration( + makeAgentTaskIntegrationFake({ backgroundForegroundWaitsForWorkspace }) + ); + const internal = service as unknown as { + backgroundProcessManager: { pullMonitorWakeSignals: ReturnType }; + bashMonitorWakeReconciler: { hasOutstandingWake: (owner: string) => Promise }; + }; + const session = service.getOrCreateSession(workspaceId); + try { + internal.backgroundProcessManager.pullMonitorWakeSignals.mockImplementation(() => + Promise.resolve([ + { + processId: "proc", + taskId: "bash:proc", + ownerWorkspaceId: workspaceId, + filter: "READY", + filterExclude: false, + script: "run", + createdAt: "2026-08-31T12:00:00.000Z", + match: { throughOffset: 12, lines: ["READY"], totalMatches: 1 }, + retired: false, + }, + ]) + ); + + session.queueMessage("later", { + model: "anthropic:claude-sonnet-4-5", + agentId: "exec", + queueDispatchMode: "turn-end", + }); + expect(await internal.bashMonitorWakeReconciler.hasOutstandingWake(workspaceId)).toBe(true); + expect(backgroundForegroundWaitsForWorkspace).not.toHaveBeenCalled(); + + // Clearing the turn-end head makes the (still high) level effective with no enqueue + // and no level publish in between: the session's yield edge alone must background the + // waits (Codex P2 PRRT_kwDOPxxmWM6fGVw_). + session.clearQueue(); + expect(backgroundForegroundWaitsForWorkspace).toHaveBeenCalledWith(workspaceId); + expect(await internal.bashMonitorWakeReconciler.hasOutstandingWake(workspaceId)).toBe(true); + expect(backgroundForegroundWaitsForWorkspace).toHaveBeenCalledTimes(1); + } finally { + session.dispose(); + await cleanup(); + } + }); + + test("the stream yields on the wake level, not on a queued snapshot of it", async () => { + // Regression: a monitored bash matched mid-step while the same step's task_await showed + // the matched lines. The old queued tool-end wake still cut the stream (finishReason + // "tool-calls") and was then withdrawn, ending the turn with no assistant text. + const { service, cleanup } = await createWakeWiringService(); + const workspaceId = "level-yield-owner"; + const createdAt = "2026-08-31T12:00:00.000Z"; + const internal = service as unknown as { + backgroundProcessManager: { + pullMonitorWakeSignals: ReturnType; + getMonitorWakeDeliveryState: ReturnType; + setMessageQueued: ReturnType; + }; + }; + const processManager = internal.backgroundProcessManager; + const session = service.getOrCreateSession(workspaceId); + try { + processManager.pullMonitorWakeSignals.mockImplementation(() => + Promise.resolve([ + { + processId: "proc", + taskId: "bash:proc", + ownerWorkspaceId: workspaceId, + filter: "READY", + filterExclude: false, + script: "run", + createdAt, + match: { throughOffset: 12, lines: ["READY"], totalMatches: 1 }, + retired: false, + }, + ]) + ); + let shownThroughOffset = 12; + processManager.getMonitorWakeDeliveryState.mockImplementation(() => + Promise.resolve({ status: "settled", shownThroughOffset, terminalStatusShown: false }) + ); + + // task_await showed the lines before the SDK asked: no yield, no debt. + expect(await session.hasPendingToolEndInput()).toBe(false); + expect(processManager.setMessageQueued).not.toHaveBeenCalledWith(workspaceId, true); + expect(service.hasBashMonitorWakeContinuation(workspaceId)).toBe(false); expect(session.hasQueuedMessages()).toBe(false); - expect(onDeferred).toHaveBeenCalledTimes(1); - expect(await dispatch(new AbortController().signal)).toBe("in-flight"); - expect(queuedModes).toEqual(["tool-end", "tool-end"]); + // A queued tool-end message still yields on its own. + session.queueMessage("follow up", { + model: "anthropic:claude-sonnet-4-5", + agentId: "exec", + queueDispatchMode: "tool-end", + }); + expect(await session.hasPendingToolEndInput()).toBe(true); + session.clearQueue(); + + // Match not shown (a different process, or a filtered read): the boundary yields, + // bash long-polls return early, and the cut stays visible to turn settlement. + shownThroughOffset = 0; + expect(await session.hasPendingToolEndInput()).toBe(true); + expect(processManager.setMessageQueued).toHaveBeenLastCalledWith(workspaceId, true); + expect(service.hasBashMonitorWakeContinuation(workspaceId)).toBe(true); + expect(service.getQueueCutCutter(workspaceId)).toEqual({ stage: "bash-monitor-wake" }); } finally { + session.dispose(); await cleanup(); } }); @@ -9390,35 +10034,6 @@ describe("WorkspaceService sendMessage status clearing", () => { } }); - test("refuses to queue a send whose cancel signal already fired", async () => { - fakeSession.isBusy.mockReturnValue(true); - const controller = new AbortController(); - controller.abort("monitor withdrawn"); - const onCanceled = mock(() => undefined); - const cancelState = { canceledBeforeAcceptance: false }; - - const result = await workspaceService.sendMessage( - "test-workspace", - "wake", - { model: "openai:gpt-4o-mini", agentId: "exec" }, - { - synthetic: true, - agentInitiated: true, - cancelSignal: controller.signal, - cancelState, - onCanceled, - queueDedupeKey: "bash-monitor-wake:test-workspace:1", - removableQueueDedupeKey: true, - } - ); - - expect(result.success).toBe(true); - expect(fakeSession.queueMessage).not.toHaveBeenCalled(); - expect(onCanceled).toHaveBeenCalledTimes(1); - expect(onCanceled).toHaveBeenCalledWith("monitor withdrawn"); - expect(cancelState.canceledBeforeAcceptance).toBe(true); - }); - test("strips stale workspace-turn correlation behind an earlier queued entry", async () => { fakeSession.hasQueuedOrDispatchingEntry.mockReturnValue(true); const onCanceled = mock(() => undefined); @@ -9623,63 +10238,6 @@ describe("WorkspaceService sendMessage status clearing", () => { expect(await settled).toBeInstanceOf(Error); }); - // The sticky case: incoming mode is turn-end but the queue's effective mode is - // tool-end from a prior enqueue, so the wait still backgrounds. - test.each([ - [ - "backgrounds foreground task waits when queuing a tool-end message", - "tool-end", - "hello", - undefined, - true, - ], - [ - "does not background foreground task waits when queuing a turn-end message", - "turn-end", - "hello", - "turn-end", - false, - ], - [ - "does not background foreground task waits when queueMessage enqueues nothing", - null, - " ", - undefined, - false, - ], - [ - "backgrounds foreground task waits when effective queue mode is tool-end despite incoming turn-end", - "tool-end", - "hello", - "turn-end", - true, - ], - ] as const)( - "%s", - async (_name, effectiveQueueMode, message, queueDispatchMode, expectBackgrounded) => { - fakeSession.isBusy.mockReturnValue(true); - fakeSession.queueMessage.mockReturnValue(effectiveQueueMode); - - const backgroundForegroundWaitsForWorkspace = mock(() => 0); - workspaceService.setAgentTaskIntegration( - makeAgentTaskIntegrationFake({ backgroundForegroundWaitsForWorkspace }) - ); - - const result = await workspaceService.sendMessage("test-workspace", message, { - model: "openai:gpt-4o-mini", - agentId: "exec", - queueDispatchMode, - }); - - expect(result.success).toBe(true); - if (expectBackgrounded) { - expect(backgroundForegroundWaitsForWorkspace).toHaveBeenCalledWith("test-workspace"); - } else { - expect(backgroundForegroundWaitsForWorkspace).not.toHaveBeenCalled(); - } - } - ); - test("registerSession clears persisted agent status for accepted user chat events", () => { const updateAgentStatus = spyOn( workspaceService as unknown as { @@ -13733,7 +14291,7 @@ describe("WorkspaceService reorderPinned across projects", () => { const mockConfig: Partial = { srcDir: "/tmp/src", - getSessionDir: mock(() => "/tmp/test/sessions"), + sessionsDir: "/tmp/test/sessions", findWorkspace: mock((id: string) => { const found = findEntry(id); if (!found) return null; diff --git a/src/node/services/workspaceService.ts b/src/node/services/workspaceService.ts index c39d187e7d..af3e6b27cb 100644 --- a/src/node/services/workspaceService.ts +++ b/src/node/services/workspaceService.ts @@ -39,11 +39,11 @@ import { AgentSession, clearProviderConfigFixableAbandonMarkers, CONTEXT_MUTATION_SEND_BLOCKED_MESSAGE, + getCarriedBashMonitorWake, inheritOpenWorkspaceTurnMetadata, type StreamErrorRecoveryOutcome, } from "@/node/services/agentSession"; import type { QueueCutCutter } from "@/node/services/messageQueue"; -import { cancelReasonBeforeAcceptance } from "@/node/services/messageQueue"; import type { HistoryService } from "@/node/services/historyService"; import type { AIService } from "@/node/services/aiService"; import type { StreamManager } from "@/node/services/streamManager"; @@ -187,6 +187,7 @@ import { getCompactionFollowUpContent, parseWorkspaceTurnTaskCorrelation, pickPreservedSendOptions, + type BashMonitorWakeDisplayRecord, type CompactionFollowUpRequest, type MuxMessageMetadata, type MuxMessage, @@ -375,7 +376,11 @@ const ORPHAN_SESSION_DIR_GRACE_MS = 24 * 60 * 60 * 1000; // Upper bound on startup .code-workspace reconciliation (see initialize()). const STARTUP_CODE_WORKSPACE_SYNC_TIMEOUT_MS = 10_000; - +/** + * Slack subtracted from the "not before" bound of readLastBashMonitorWakeRecords so a wall-clock + * step between a monitor's arm stamp and its wake row's append stamp cannot hide the row. + */ +const BASH_MONITOR_WAKE_ROW_SCAN_CLOCK_MARGIN_MS = 60_000; /** * Base name used when /new auto-generates a branch name. Numbered suffixes * (`workspace-1`, `workspace-2`, ...) come from {@link generateForkBranchName} @@ -1876,8 +1881,14 @@ export class WorkspaceService extends EventEmitter implements WorkspaceHost { } private readonly bashOutputShownListener = ( workspaceId: string, - _payload: OutputShownPayload + payload: OutputShownPayload ): void => { + if (this.removingWorkspaces.has(workspaceId)) return; + this.bashMonitorWakeReconciler + .outputShown(workspaceId, payload.processId) + .catch((error: unknown) => { + log.debug("Bash monitor output-shown revalidation failed", { workspaceId, error }); + }); this.scheduleBashMonitorWakeReconcile(workspaceId); }; private readonly bashMonitorMatchListener = ( @@ -2385,6 +2396,10 @@ export class WorkspaceService extends EventEmitter implements WorkspaceHost { }, registry: this.bashMonitorRegistryStore, onWake: (dispatch) => this.dispatchBashMonitorWake(dispatch), + readDeliveredWakeRecords: (ownerWorkspaceId, notBefore) => + this.readLastBashMonitorWakeRecords(ownerWorkspaceId, notBefore), + onOutstandingChanged: (ownerWorkspaceId, outstanding) => + this.publishBashMonitorWakeLevel(ownerWorkspaceId, outstanding), }); if (typeof this.backgroundProcessManager.on === "function") { this.backgroundProcessManager.on("output:shown", this.bashOutputShownListener); @@ -2509,27 +2524,54 @@ export class WorkspaceService extends EventEmitter implements WorkspaceHost { this.pendingBashMonitorWakeIdleWaitsByOwner.set(ownerWorkspaceId, promise); } + /** + * Start a wake turn from the level, or leave it pending. A wake is never queued as a + * message: while the owner streams, the stream itself reads the level at each tool + * boundary (AgentSession.hasPendingToolEndInput) and yields with finishReason + * "tool-calls"; the after-idle reconcile then lands here again and sends directly. + * + * The send runs under the owner's workspace event lock (outermost — workspace removal + * takes the history lock while holding it, so this order is the only deadlock-free one). + * The lock is FIFO, so the wake cannot start — and redeem the continuation debt the cut + * stream took — before the stream-end handler of the stream that yielded to it has run: + * that handler is what reads the debt to defer, rather than settle, a delegated turn. + */ private async dispatchBashMonitorWake( dispatch: BashMonitorWakeDispatch + ): Promise { + const ownerWorkspaceId = dispatch.ownerWorkspaceId; + const underEventLock = (operation: () => Promise): Promise => + this.agentTaskIntegration != null + ? this.agentTaskIntegration.withWorkspaceEventLock(ownerWorkspaceId, operation) + : operation(); + return underEventLock(() => this.dispatchBashMonitorWakeUnderEventLock(dispatch)); + } + + private async dispatchBashMonitorWakeUnderEventLock( + dispatch: BashMonitorWakeDispatch ): Promise { return this.bashMonitorHistoryLocks.withLock(dispatch.ownerWorkspaceId, async () => { const ownerWorkspaceId = dispatch.ownerWorkspaceId; + // A full-history clear (same lock) may have retired these signals while this wake + // waited for the lock; sending now would append a stale prompt to cleared history. + if (!dispatch.isCurrent()) return "deferred"; const entry = findWorkspaceEntry(this.config.loadConfigOrDefault(), ownerWorkspaceId); if (entry == null) { await dispatch.onAccepted(); this.notifyBashMonitorWakeStateChanged(ownerWorkspaceId); return "in-flight"; } - const hasPendingTurn = this.hasPendingQueuedOrPreparingTurn(ownerWorkspaceId); - const hasSessionBackedBusyState = this.isBusyForMessage(ownerWorkspaceId); - const hasAiServiceStream = this.aiService.isStreaming(ownerWorkspaceId); - if (hasPendingTurn || (hasSessionBackedBusyState && !hasAiServiceStream)) { + if ( + this.hasPendingQueuedOrPreparingTurn(ownerWorkspaceId) || + this.isBusyForMessage(ownerWorkspaceId) + ) { this.scheduleBashMonitorWakeReconcileAfterIdle(ownerWorkspaceId); return "deferred"; } - if (hasAiServiceStream && !hasSessionBackedBusyState) { - return "deferred"; - } + // Streaming without a busy session (teardown window, or no session at all): the + // after-idle wait would resolve immediately and spin. The stream-end/abort/error + // listeners schedule the next reconcile instead. + if (this.aiService.isStreaming(ownerWorkspaceId)) return "deferred"; const sendOptions = (await this.getDelegatedTurnContinuationSendOptions(ownerWorkspaceId)) ?? (await this.getWorkflowContinuationSendOptions(ownerWorkspaceId)); @@ -2537,44 +2579,26 @@ export class WorkspaceService extends EventEmitter implements WorkspaceHost { log.debug("Bash monitor wake has no send options; leaving pending", { ownerWorkspaceId }); return "deferred"; } - - // Withdrawn while awaiting send options above: the abort listener below would never - // fire, and send preflight (which persists AI settings) has nothing left to admit. - if (dispatch.cancelSignal.aborted) return "deferred"; + // Re-checked after the awaits above: a monitor canceled meanwhile (discardProcess) must + // not have its captured stdout submitted as an agent-initiated turn. + if (!dispatch.isCurrent()) return "deferred"; let accepted = false; - // A queued wake can be superseded after dequeue. Share cancellation state so - // AgentSession can release PREPARING when cancellation wins before acceptance. - const cancelState = { canceledBeforeAcceptance: false }; - // Withdrawal (output already shown, process discarded, history cleared) must - // free the queue slot now, not at stream end: a lingering entry keeps the - // workspace reported busy and its dedupe key held. The key is unique per - // dispatch, so this cannot drop a newer wake's entry. - dispatch.cancelSignal.addEventListener( - "abort", - () => { - this.removeQueuedMessagesByDedupeKeyPrefix(ownerWorkspaceId, dispatch.dedupeKey, { - cancelReason: "Bash monitor wake withdrawn before dispatch.", - }); - }, - { once: true } - ); const sendResult = await this.sendMessage( ownerWorkspaceId, dispatch.prompt, - { - ...sendOptions, - queueDispatchMode: "tool-end", - muxMetadata: dispatch.muxMetadata, - }, + { ...sendOptions, muxMetadata: dispatch.muxMetadata }, { skipAutoResumeReset: true, synthetic: true, agentInitiated: true, - cancelState, - cancelSignal: dispatch.cancelSignal, - queueDedupeKey: dispatch.dedupeKey, - removableQueueDedupeKey: true, + // Never queue: a manual send racing past the idle checks above would otherwise + // park this wake behind it, out of reach of the level (a later monitor cancel + // could not retract it). requireIdle turns that race into a skip (Err), and the + // admission probe re-validates the wake at every gate before the user row is + // durable; both fall through to the after-idle re-arm below. + requireIdle: true, + admissionStale: () => !dispatch.isCurrent(), onAccepted: async () => { accepted = true; await dispatch.onAccepted(); @@ -2583,12 +2607,6 @@ export class WorkspaceService extends EventEmitter implements WorkspaceHost { onAcceptedPreStreamFailure: async () => { if (accepted) await dispatch.onAccepted(); }, - onCanceled: async () => { - if (!accepted) { - await dispatch.onDeferred(); - this.scheduleBashMonitorWakeReconcile(ownerWorkspaceId); - } - }, } ); if (!sendResult.success && !accepted) { @@ -4062,6 +4080,22 @@ export class WorkspaceService extends EventEmitter implements WorkspaceHost { // is released at its queue/session handoff so a follow-up dispatched // from within that turn does not veto itself. hasExternalSendPreflight: () => this.hasSessionInvisiblePreflight(workspaceId), + hasOutstandingBashMonitorWake: () => + this.bashMonitorWakeReconciler.hasOutstandingWake(workspaceId), + // The stream will cut at its next tool boundary; a foreground task_await that would + // outlive it is backgrounded so the boundary is reached. Waits registered while the + // lever is already high are backgrounded at registration + // (TaskService.backgroundForegroundWaitIfQueued → isToolEndYieldRequested). + onToolEndYieldRequested: () => { + this.agentTaskIntegration?.backgroundForegroundWaitsForWorkspace(workspaceId); + }, + onWorkspaceTurnContinuationVoided: async (correlation, reason) => { + await this.agentTaskIntegration?.settleVoidedWorkspaceTurnContinuation( + workspaceId, + correlation, + reason + ); + }, }); } @@ -4086,6 +4120,43 @@ export class WorkspaceService extends EventEmitter implements WorkspaceHost { }); } + /** + * Reconciler level → session mirror. The level drives the same tool-boundary side effects a + * queued tool-end message does: long-polling bash reads return early and foreground + * agent-task waits are backgrounded so the stream can reach the boundary where it yields to + * the wake. The session arbitrates the level against its queue head and fires the yield edge + * (onToolEndYieldRequested) when the lever actually becomes effective. + * + * Published to the live session, including one still running startup recovery: a recovered + * stream must see the level too, or a long-running foreground wait inside it is never + * backgrounded and the deferred wake waits for that stream to end on its own. + */ + private publishBashMonitorWakeLevel(ownerWorkspaceId: string, outstanding: boolean): void { + const session = this.getLiveSession(ownerWorkspaceId); + if (session != null) { + session.setBashMonitorWakeOutstanding(outstanding); + } else if ( + !outstanding && + // Partial BackgroundProcessManager stubs in tests (see the constructor guards). + typeof this.backgroundProcessManager.setMessageQueued === "function" + ) { + // No session, no queue: the flag can only be a stale mirror (e.g. reconciler disposal + // after the session went away), so drop it directly. + this.backgroundProcessManager.setMessageQueued(ownerWorkspaceId, false); + } + } + + /** + * The session currently owning a workspace's runtime state, whether cached or still + * transient for startup recovery. Read-only lookups of per-session state (wake level, debt, + * stream ledger) must use this rather than `sessions` alone: a recovered turn runs inside the + * transient session before it is promoted. + */ + private getLiveSession(workspaceId: string): AgentSession | undefined { + const trimmed = workspaceId.trim(); + return this.sessions.get(trimmed) ?? this.transientStartupRecoverySessions.get(trimmed); + } + public getOrCreateSession(workspaceId: string): AgentSession { assert(typeof workspaceId === "string", "workspaceId must be a string"); const trimmed = workspaceId.trim(); @@ -10892,8 +10963,6 @@ export class WorkspaceService extends EventEmitter implements WorkspaceHost { agentInitiated: internal?.agentInitiated, goalKind: internal?.goalKind, goalId: internal?.goalId, - cancelState: internal?.cancelState, - cancelSignal: internal?.cancelSignal, onCanceled: internal?.onCanceled, onAccepted: internal?.onAccepted, onAcceptedPreStreamFailure: internal?.onAcceptedPreStreamFailure, @@ -10971,18 +11040,6 @@ export class WorkspaceService extends EventEmitter implements WorkspaceHost { } if (shouldQueue) { - // Mirrors AgentSession's cancelBeforeAcceptance for the queue path: a send withdrawn - // during the preflight awaits above must not occupy a queue slot (and hold its dedupe - // key) until the stream drains it. Nothing is persisted yet, so only the handshake runs. - if (internal?.cancelSignal?.aborted === true) { - await getContinuationSendState().onCanceled?.( - cancelReasonBeforeAcceptance(internal.cancelSignal) - ); - if (internal.cancelState != null) { - internal.cancelState.canceledBeforeAcceptance = true; - } - return Ok(undefined); - } // Everything from here to queueMessage is synchronous, so a probe pass here cannot go // stale before the entry is enqueued. if (internal?.admissionStale?.() === true) { @@ -11071,8 +11128,6 @@ export class WorkspaceService extends EventEmitter implements WorkspaceHost { workspaceTurnContinuation: internal?.workspaceTurnContinuation, dedupeKey: internal?.queueDedupeKey, removableDedupeKey: internal?.removableQueueDedupeKey, - cancelState: internal?.cancelState, - cancelSignal: internal?.cancelSignal, onCanceled: continuationSendState.onCanceled, onAccepted: internal?.onAccepted, onAcceptedPreStreamFailure: continuationSendState.onAcceptedPreStreamFailure, @@ -11098,10 +11153,6 @@ export class WorkspaceService extends EventEmitter implements WorkspaceHost { this.agentTaskIntegration?.resetAutoResumeCount(workspaceId); } - if (effectiveQueueDispatchMode === "tool-end") { - this.agentTaskIntegration?.backgroundForegroundWaitsForWorkspace(workspaceId); - } - return Ok(undefined); } @@ -11171,9 +11222,9 @@ export class WorkspaceService extends EventEmitter implements WorkspaceHost { // follow-up redispatched from within this very turn (on-send compaction // completion) does not veto itself — while the admission awaits between // here and the busy claim stay covered. Codex P2 (PRRT_kwDOPxxmWM6cSRkH): - // releasing at the handoff itself left AgentSession's - // cancelBeforeAcceptance yield observable as idle, letting follow-up - // recovery admit an exec turn ahead of the accepted manual send. Refusal + // releasing at the handoff itself left AgentSession's pre-acceptance + // yield observable as idle, letting follow-up recovery admit an exec + // turn ahead of the accepted manual send. Refusal // paths never fire the callback; the scoped disposal releases on return. const result = await session.sendMessage(message, continuationSendState.options, { onTurnAdmissionCommitted: () => sessionInvisiblePreflight.release(), @@ -11183,8 +11234,6 @@ export class WorkspaceService extends EventEmitter implements WorkspaceHost { goalId: internal?.goalId, goalContinuation: internal?.goalContinuation, startStreamInBackground: internal?.startStreamInBackground, - cancelState: internal?.cancelState, - cancelSignal: internal?.cancelSignal, // Same authoring-time race as the queued path: the goal-creating // stream can end during the preflight awaits above, making a fresh // goal visible after the user hit enter but before this dispatch. @@ -11880,11 +11929,11 @@ export class WorkspaceService extends EventEmitter implements WorkspaceHost { } isBusyForMessage(workspaceId: string): boolean { - return this.sessions.get(workspaceId.trim())?.isBusy() === true; + return this.getLiveSession(workspaceId)?.isBusy() === true; } hasQueuedWorkspaceTurn(workspaceId: string, handleId: string): boolean { - return this.sessions.get(workspaceId.trim())?.hasQueuedWorkspaceTurn(handleId) ?? false; + return this.getLiveSession(workspaceId)?.hasQueuedWorkspaceTurn(handleId) ?? false; } /** @@ -11910,11 +11959,11 @@ export class WorkspaceService extends EventEmitter implements WorkspaceHost { } hasQueuedOrDispatchingEntry(workspaceId: string): boolean { - return this.sessions.get(workspaceId.trim())?.hasQueuedOrDispatchingEntry() ?? false; + return this.getLiveSession(workspaceId)?.hasQueuedOrDispatchingEntry() ?? false; } hasQueuedMessages(workspaceId: string, dispatchMode?: "tool-end" | "turn-end"): boolean { - return this.sessions.get(workspaceId.trim())?.hasQueuedMessages(dispatchMode) ?? false; + return this.getLiveSession(workspaceId)?.hasQueuedMessages(dispatchMode) ?? false; } async waitForPendingCompactionCompletionDecision( @@ -11933,13 +11982,18 @@ export class WorkspaceService extends EventEmitter implements WorkspaceHost { return session?.waitForPendingStreamErrorRecoveryDecision(messageId); } + // Both waits resolve through the live lookup, matching the busy predicates above: a wake + // deferred because a transient startup-recovery session is busy waits for *that* session to + // go idle. Waiting on `sessions` alone would resolve at once and re-defer in a tight loop of + // history/registry reads until recovery promoted it. A transient session either promotes as + // the same instance or is disposed, and dispose releases idle waiters. async waitForIdle(workspaceId: string): Promise { - const session = this.sessions.get(workspaceId.trim()); + const session = this.getLiveSession(workspaceId); await session?.waitForIdle(); } async waitForIdleAndNoQueuedMessages(workspaceId: string): Promise { - const session = this.sessions.get(workspaceId.trim()); + const session = this.getLiveSession(workspaceId); if (!session) { return; } @@ -11982,7 +12036,7 @@ export class WorkspaceService extends EventEmitter implements WorkspaceHost { } hasPendingQueuedOrPreparingTurn(workspaceId: string): boolean { - const session = this.sessions.get(workspaceId.trim()); + const session = this.getLiveSession(workspaceId); if (!session) { return false; } @@ -11992,23 +12046,112 @@ export class WorkspaceService extends EventEmitter implements WorkspaceHost { ); } + /** See AgentSession.hasCorrelatedStreamStartedAfter. */ + hasCorrelatedStreamStartedAfter( + workspaceId: string, + correlation: { taskHandleId: string; ownerWorkspaceId: string; turnId: string }, + messageIds: readonly string[] + ): boolean { + return ( + this.getLiveSession(workspaceId)?.hasCorrelatedStreamStartedAfter(correlation, messageIds) === + true + ); + } + + /** See AgentSession.hasBashMonitorWakeContinuation. */ + hasBashMonitorWakeContinuation(workspaceId: string): boolean { + return this.getLiveSession(workspaceId)?.hasBashMonitorWakeContinuation() === true; + } + /** - * Whether a bash-monitor-wake continuation is queued next or mid-dispatch. - * See AgentSession.hasPendingBashMonitorWakeContinuation for semantics. + * Whether pending input (queued tool-end message or outstanding wake) wants the active + * stream to reach a tool boundary — the union flag long-polling bash reads consult. */ - hasPendingBashMonitorWakeContinuation(workspaceId: string): boolean { - const session = this.sessions.get(workspaceId.trim()); - return session?.hasPendingBashMonitorWakeContinuation() ?? false; + isToolEndYieldRequested(workspaceId: string): boolean { + return this.backgroundProcessManager.hasQueuedMessage(workspaceId.trim()); } /** * Whether a queued or dispatching entry continues the exact workspace-turn correlation. */ + /** + * Records of the most recent durable bash-monitor wake row (BashMonitorWakeReconciler + * readDeliveredWakeRecords). Scans backward from the tail and stops at the first wake row + * with usable record identities, or at the first row older than `notBefore`. + * + * No fixed depth is safe: the state this repairs — an acknowledgment that kept failing while + * the accepted wake turn ran — can push the row behind an arbitrarily long tool-heavy turn + * (and behind a compaction boundary; the wake stays acknowledged by its row wherever it + * sits). No unbounded scan is acceptable either: an owner's first wake has no row to find, + * and this read sits on the stream's tool-boundary predicate under the history lock. The + * arm time of the oldest outstanding monitor is the durable exclusion point — a row appended + * before that monitor existed cannot acknowledge it — so the cost is the rows since arming, + * usually one tail chunk. + */ + private async readLastBashMonitorWakeRecords( + ownerWorkspaceId: string, + notBefore: string + ): Promise { + // Registry and history stamps come from the same wall clock; the margin absorbs a clock + // step between arming and the row's append. An unparseable bound (never produced by the + // registry) degrades to an unbounded scan rather than to a silent redelivery. + const parsedNotBefore = Date.parse(notBefore); + const cutoffMs = Number.isFinite(parsedNotBefore) + ? parsedNotBefore - BASH_MONITOR_WAKE_ROW_SCAN_CLOCK_MARGIN_MS + : Number.NEGATIVE_INFINITY; + let records: readonly BashMonitorWakeDisplayRecord[] | undefined; + const iterateResult = await this.historyService.iterateFullHistory( + ownerWorkspaceId, + "backward", + (messages) => { + // Chunks arrive newest-first, as do the rows within a chunk. + for (const message of messages) { + // RLM keep-recent compaction re-appends copies of the pre-boundary tail *after* the + // boundary while keeping their source timestamps, so a copy can read as older than + // the cutoff while sitting above rows that are newer. Copies carry nothing their + // originals (still in history) lack: skip them before the cutoff and before reading + // wake metadata, or a stale-stamped copy ends the scan short of the wake row. + if (message.metadata?.rlmPreservedTailCopy === true) continue; + const timestamp = message.metadata?.timestamp; + if (typeof timestamp === "number" && timestamp < cutoffMs) return false; + // A wake diverted through on-send compaction is durable as the compaction row that + // carries it as follow-up; that row is the acknowledgment too. + const wake = getCarriedBashMonitorWake(message.metadata?.muxMetadata); + if (wake == null) continue; + // Persisted rows are data, not trusted structure: one malformed `records` (or a legacy + // row without identities) must not stop the scan and brick recovery for every + // reconcile retry — the reconciler maps over what this returns. + const usable = Array.isArray(wake.records) + ? wake.records.filter( + (record): record is BashMonitorWakeDisplayRecord => + typeof record === "object" && + record !== null && + typeof record.processId === "string" && + typeof record.wakeUpdatedAt === "string" + ) + : []; + if (usable.length > 0) { + records = usable; + return false; + } + } + return true; + } + ); + // Distinguish "no row" from "could not read": the reconciler recovers once per owner, so a + // read failure swallowed here would let the reconcile dispatch a duplicate of a wake the + // row already delivered. Throwing fails this reconcile; its retry reads again. + if (!iterateResult.success) { + throw new Error(`Failed to read the last bash-monitor wake row: ${iterateResult.error}`); + } + return records; + } + hasPendingWorkspaceTurnContinuation( workspaceId: string, metadata: Extract ): boolean { - const session = this.sessions.get(workspaceId.trim()); + const session = this.getLiveSession(workspaceId); return session?.hasPendingWorkspaceTurnContinuation(metadata) ?? false; } @@ -12017,7 +12160,7 @@ export class WorkspaceService extends EventEmitter implements WorkspaceHost { * See AgentSession.getQueueCutCutter for stage semantics. */ getQueueCutCutter(workspaceId: string): QueueCutCutter | undefined { - const session = this.sessions.get(workspaceId.trim()); + const session = this.getLiveSession(workspaceId); return session?.getQueueCutCutter(); } @@ -12029,7 +12172,7 @@ export class WorkspaceService extends EventEmitter implements WorkspaceHost { * hasPendingQueuedOrPreparingTurn. */ hasPendingAutoRetry(workspaceId: string): boolean { - const session = this.sessions.get(workspaceId.trim()); + const session = this.getLiveSession(workspaceId); return session?.hasPendingAutoRetry() ?? false; } diff --git a/src/node/services/workspaceTurnManager.test.ts b/src/node/services/workspaceTurnManager.test.ts index 20588945ad..6509596a8c 100644 --- a/src/node/services/workspaceTurnManager.test.ts +++ b/src/node/services/workspaceTurnManager.test.ts @@ -317,7 +317,7 @@ describe("WorkspaceTurnManager", () => { isStreaming?: ReturnType; hasQueuedMessages?: ReturnType; hasPendingQueuedOrPreparingTurn?: ReturnType; - hasPendingBashMonitorWakeContinuation?: ReturnType; + hasBashMonitorWakeContinuation?: ReturnType; hasPendingWorkspaceTurnContinuation?: ReturnType; getQueueCutCutter?: ReturnType; hasPendingAutoRetry?: ReturnType; diff --git a/src/node/services/workspaceTurnManager.ts b/src/node/services/workspaceTurnManager.ts index b74f3b9d6e..4f2f543c68 100644 --- a/src/node/services/workspaceTurnManager.ts +++ b/src/node/services/workspaceTurnManager.ts @@ -18,6 +18,7 @@ import { type WorkspaceLifecycleResult, type WorkspaceTurnManagerHost, } from "@/node/services/taskWorkspaceSeam"; +import type { WorkspaceTurnContinuationVoidReason } from "@/node/services/agentSession"; import type { HistoryService } from "@/node/services/historyService"; import type { InitStateManager } from "@/node/services/initStateManager"; import { @@ -281,6 +282,17 @@ const WORKSPACE_TURN_STALE_RESTART_ERROR = "Workspace turn interrupted after res const WORKSPACE_TURN_SUPERSEDED_BY_NEW_INPUT_ERROR = "Workspace turn superseded by new input in the target workspace; the workspace continues under that input and this delegated turn will not report"; +/** + * Reason persisted when the target stream yielded at a tool boundary to a bash-monitor + * wake that was then retracted (monitor canceled) before the wake turn was sent. No + * continuation follows, so deferring would leave the owner's wait hanging; settling as a + * truncation would misreport the delegated work as failed output. Same supersede family + * as new-input cuts: self-heal eligible should a late correlated continuation prove the + * turn went on after all. + */ +const WORKSPACE_TURN_YIELDED_TO_RETRACTED_WAKE_ERROR = + "Workspace turn yielded at a tool boundary to a bash-monitor wake that was retracted before delivery; the target workspace is idle and this delegated turn did not complete"; + /** * Reason prefix persisted when the owner's OWN follow-up turn (task * kind="workspace", mode="existing", tool-end dispatch) cut its active @@ -331,7 +343,8 @@ function isSupersededWorkspaceTurnInterrupt( ): boolean { return ( (record.status === "interrupted" && - record.error === WORKSPACE_TURN_SUPERSEDED_BY_NEW_INPUT_ERROR) || + (record.error === WORKSPACE_TURN_SUPERSEDED_BY_NEW_INPUT_ERROR || + record.error === WORKSPACE_TURN_YIELDED_TO_RETRACTED_WAKE_ERROR)) || isOwnerFollowUpSupersededWorkspaceTurnInterrupt(record) ); } @@ -378,6 +391,7 @@ function ownerFollowUpSupersedeSkipsDirectParent( type QueueCutSupersedeEvidence = | { kind: "same_owner_follow_up"; successorHandleId: string } | { kind: "other_input" } + | { kind: "retracted_wake" } | { kind: "preserved"; error: string } | null; @@ -2104,6 +2118,14 @@ export class WorkspaceTurnManager { * it would leak the disposable checkout with no owner left to clean it up. */ disposableOwnershipTransferred?: boolean; + /** + * Re-evaluated under the settlement lock, after the handle reread and before anything is + * written: when it returns true the settlement is abandoned. Callers whose reason to + * settle can be invalidated by concurrent admission (a continuation of the turn queued or + * started while this call awaited the lock / store) revalidate here, at the commit point, + * rather than trusting a snapshot taken before those awaits. + */ + abandonIf?: () => boolean; }): Promise { assert( params.next.handleId === params.record.handleId, @@ -2181,9 +2203,24 @@ export class WorkspaceTurnManager { current.status ); this.taskHost.markTaskForegroundRelevant(current.handleId); + // The terminal row is the first durable write of a settlement; the phases after it + // (mirror, waiters, disposable cleanup) can be skipped by a throw, and this branch is + // where the retry lands. Resume the cleanup phase here too — a still-registered + // disposable workspace on a terminal record means no settlement reached it (cleanup + // either removes the workspace or, when forwarding it, clears the flag). + if ( + current.disposableWorkspace && + (await this.workspaceService.getInfo(current.workspaceId)) != null + ) { + await this.cleanupDisposableWorkspaceTurn(current); + } return { pendingNotify: null, winningStatus: current.status }; } + if (params.abandonIf?.() === true) { + return null; + } + // Decide the terminal wake-up using persisted policy + the restart-safe dedupe marker. // A resettle corrects a previously reported outcome, so it re-arms the wake-up even if // the stale settlement was already notified/consumed. Owner-follow-up supersedes settle @@ -4091,7 +4128,9 @@ export class WorkspaceTurnManager { ? buildOwnerFollowUpSupersededError(evidence.successorHandleId) : evidence.kind === "preserved" ? evidence.error - : WORKSPACE_TURN_SUPERSEDED_BY_NEW_INPUT_ERROR; + : evidence.kind === "retracted_wake" + ? WORKSPACE_TURN_YIELDED_TO_RETRACTED_WAKE_ERROR + : WORKSPACE_TURN_SUPERSEDED_BY_NEW_INPUT_ERROR; return { ...baseRecord, status: "interrupted", @@ -4312,7 +4351,8 @@ export class WorkspaceTurnManager { */ private hasSameTurnContinuation( event: StreamEndEvent, - correlation: { taskHandleId: string; ownerWorkspaceId: string; turnId: string } + correlation: { taskHandleId: string; ownerWorkspaceId: string; turnId: string }, + queueCutSnapshot: QueueCutAttributionSnapshot ): boolean { if ( this.workspaceService.hasPendingWorkspaceTurnContinuation(event.workspaceId, { @@ -4322,11 +4362,53 @@ export class WorkspaceTurnManager { ) { return true; } - if (this.workspaceService.hasPendingBashMonitorWakeContinuation(event.workspaceId)) { + // A stream that yielded to the wake level continues through the wake turn, which + // inherits this correlation from history (inheritOpenWorkspaceTurnMetadata). Only the + // event-time attribution says whether the level was the cutter: a manual tool-end head + // arbitrates the cut even while the level is high, runs first and breaks inheritance, so + // the wake behind it is not this turn's continuation and the handle must settle here. + // Whether the wake can still arrive is the session's continuation debt, read live and + // synchronously: a void that landed before this handler (same per-workspace lock) has + // already cleared it, and one landing after finds the record deferred and settles it + // itself (settleVoidedWorkspaceTurnContinuation). + if ( + queueCutSnapshot.cutter?.stage === "bash-monitor-wake" && + this.workspaceService.hasBashMonitorWakeContinuation(event.workspaceId) + ) { return true; } - const activeStream = this.streamManager?.getStreamInfo(event.workspaceId); - if (activeStream == null || activeStream.messageId === event.messageId) { + return this.hasCorrelatedStreamAfter(event.workspaceId, correlation, [event.messageId]); + } + + /** + * Whether a stream other than `excludeMessageIds` (the ending stream, or streams whose + * stream-end this record already deferred) is currently streaming this exact correlation. + */ + /** + * The turn continued after the given stream(s): a correlated stream started later — still + * running, or already ended with its own stream-end queued behind this handler on the + * workspace event lock. Settling here would pre-empt that stream-end (and, for a disposable + * turn, delete the workspace under its work). The session's start ledger is authoritative; + * the live stream check covers a session that is not in memory. + */ + private hasCorrelatedStreamAfter( + workspaceId: string, + correlation: { taskHandleId: string; ownerWorkspaceId: string; turnId: string }, + messageIds: readonly string[] + ): boolean { + return ( + this.workspaceService.hasCorrelatedStreamStartedAfter(workspaceId, correlation, messageIds) || + this.hasCorrelatedActiveStream(workspaceId, correlation, messageIds) + ); + } + + private hasCorrelatedActiveStream( + workspaceId: string, + correlation: { taskHandleId: string; ownerWorkspaceId: string; turnId: string }, + excludeMessageIds: readonly string[] + ): boolean { + const activeStream = this.streamManager?.getStreamInfo(workspaceId); + if (activeStream == null || excludeMessageIds.includes(activeStream.messageId)) { return false; } const activeCorrelation = this.getWorkspaceTurnMetadataFromValue(activeStream.muxMetadata); @@ -4399,6 +4481,11 @@ export class WorkspaceTurnManager { ? classifyMetadata(cutter.muxMetadata) : { kind: "other_input" }; } + // The stream yielded to the wake level and the caller found no continuation + // (level low, no wake turn admitted): the wake was retracted after the cut. + if (cutter?.stage === "bash-monitor-wake") { + return { kind: "retracted_wake" }; + } // Residual legacy positives (e.g. hasPendingAutoRetry with an empty queue) // stay generic supersede evidence. return snapshot.hasPendingQueuedOrPreparingTurn ? { kind: "other_input" } : null; @@ -4480,16 +4567,16 @@ export class WorkspaceTurnManager { return true; } - // A queued continuation can stop the in-flight stream at a tool boundary with - // finishReason "tool-calls" and continue the same delegated turn. Report - // wake-ups carry the exact correlation explicitly; bash-monitor wakes inherit - // it from history. Defer settlement until the continuation's terminal + // A queued continuation (or an outstanding bash-monitor wake) can stop the + // in-flight stream at a tool boundary with finishReason "tool-calls" and + // continue the same delegated turn. Report wake-ups carry the exact + // correlation explicitly; bash-monitor wakes inherit it from history. Defer settlement until the continuation's terminal // stream-end instead of reporting a false completion failure to the owner. // Any other queued input (manual message, /compact) supersedes the turn and // must settle the old outcome here. if ( event.metadata.finishReason === "tool-calls" && - this.hasSameTurnContinuation(event, metadata) + this.hasSameTurnContinuation(event, metadata, queueCutSnapshot) ) { await this.markWorkspaceTurnStreamEndDeferred(event); return true; @@ -4645,22 +4732,80 @@ export class WorkspaceTurnManager { }); } + /** + * The target session will never continue the delegated turn identified by `muxMetadata` + * (AgentSession.onWorkspaceTurnContinuationVoided). `retracted` / `superseded` void a + * continuation debt: only a record the stream-end handler already DEFERRED on that debt + * needs settling here — a record still running has its handler queued behind this call on + * the workspace event lock, and that handler reads the (now cleared) debt live and settles + * the turn itself. `abandoned` drops a compaction follow-up carrying the correlation: the + * record may still be running behind the compaction stream (whose stream-end is + * uncorrelated and settles nothing), so any active record settles. + * + * A void says nothing about OTHER continuations of the same turn: a correlated report + * queued after the wake cut (or already streaming) also deferred the stream-end and will + * settle the record with its own stream-end. Settling here would interrupt a turn that is + * about to continue — and a disposable turn would delete its workspace under that stream. + * That check runs after the record read and again at the settlement's commit point + * (`unlessTurnContinues`): a continuation queued while the handle store or the settlement + * lock was being awaited must be seen too. + */ + async settleVoidedWorkspaceTurnContinuation( + workspaceId: string, + muxMetadata: WorkspaceTurnMuxMetadata, + reason: WorkspaceTurnContinuationVoidReason + ): Promise { + await this.settleWorkspaceTurnContinuationFailure( + workspaceId, + muxMetadata, + "interrupted", + reason === "retracted" + ? WORKSPACE_TURN_YIELDED_TO_RETRACTED_WAKE_ERROR + : WORKSPACE_TURN_SUPERSEDED_BY_NEW_INPUT_ERROR, + { deferredOnly: reason !== "abandoned", unlessTurnContinues: true } + ); + } + // A queued report can defer the preceding stream-end. If dispatch then fails, settle that // exact turn here because no replacement stream-end can arrive. async settleWorkspaceTurnContinuationFailure( workspaceId: string, muxMetadata: WorkspaceTurnMuxMetadata, status: "interrupted" | "error", - error: string + error: string, + options?: { deferredOnly: boolean; unlessTurnContinues: boolean } ): Promise { const record = await this.taskHandleStore.getWorkspaceTurn( muxMetadata.ownerWorkspaceId, muxMetadata.taskHandleId ); + if (record?.workspaceId !== workspaceId || record.turnId !== muxMetadata.turnId) { + return; + } + if (this.isTerminalWorkspaceTurnStatus(record.status)) { + // Already settled — possibly by an earlier attempt of this very settlement that + // persisted the terminal handle and then failed before resolving its waiters (the + // void is retried on failure). settleWorkspaceTurn's terminal branch is idempotent and + // resolves whatever is still waiting on the persisted outcome. + await this.settleWorkspaceTurn({ + record, + next: record, + waiterSettlement: { status: "error", error: new Error(error) }, + }); + return; + } + // The turn continues when a correlated continuation is pending, or a correlated stream + // whose stream-end the record has not deferred started (it settles the turn itself). + // Checked here and again at the commit point inside settleWorkspaceTurn (abandonIf): a + // correlated send is invisible during its preflight and can become queued while the + // settlement awaits the lock and the store. + const turnContinues = () => + this.workspaceService.hasPendingWorkspaceTurnContinuation(workspaceId, muxMetadata) || + this.hasCorrelatedStreamAfter(workspaceId, muxMetadata, record.deferredMessageIds ?? []); if ( - record?.workspaceId !== workspaceId || - record?.turnId !== muxMetadata.turnId || - !isActiveWorkspaceTurnTaskStatus(record?.status) + !isActiveWorkspaceTurnTaskStatus(record.status) || + (options?.deferredOnly === true && (record.deferredMessageIds?.length ?? 0) === 0) || + (options?.unlessTurnContinues === true && turnContinues()) ) { return; } @@ -4676,6 +4821,7 @@ export class WorkspaceTurnManager { record, next, waiterSettlement: { status: "error", error: new Error(error) }, + ...(options?.unlessTurnContinues === true ? { abandonIf: turnContinues } : {}), }); }