diff --git a/src/browser/stores/WorkspaceStore.test.ts b/src/browser/stores/WorkspaceStore.test.ts index f49bed623c..ca7f0629fe 100644 --- a/src/browser/stores/WorkspaceStore.test.ts +++ b/src/browser/stores/WorkspaceStore.test.ts @@ -2230,6 +2230,75 @@ describe("WorkspaceStore", () => { expect(store.getStreamingMessage(workspaceId, secondRow.id, secondMessageId)).not.toBeNull(); }); + it("keeps the replacement channel for a cleanup-only abort", async () => { + const workspaceId = "cleanup-only-abort-channel"; + const oldMessageId = "old-stream"; + const replacementMessageId = "replacement-stream"; + createAndAddWorkspace(store, workspaceId); + const rawStore = getInternal<{ + streamingMessageStore: { has: (key: string) => boolean }; + handleChatMessage: (id: string, event: WorkspaceChatMessage) => void; + processStreamEvent: ( + id: string, + aggregator: ReturnType, + event: WorkspaceChatMessage + ) => void; + }>(store); + const dispatch = (event: WorkspaceChatMessage) => + rawStore.processStreamEvent(workspaceId, store.getAggregator(workspaceId), event); + + rawStore.handleChatMessage(workspaceId, { + type: "stream-start", + workspaceId, + messageId: oldMessageId, + historySequence: 1, + model: TEST_MODEL, + startTime: 1, + }); + rawStore.handleChatMessage(workspaceId, caughtUpEvent()); + dispatch({ + type: "stream-start", + workspaceId, + messageId: replacementMessageId, + historySequence: 2, + model: TEST_MODEL, + startTime: 2, + }); + for (const [delta, timestamp] of [ + ["hello", 3], + [" world", 4], + ] as const) { + dispatch({ + type: "stream-delta", + workspaceId, + messageId: replacementMessageId, + delta, + tokens: 1, + timestamp, + }); + } + await new Promise((resolve) => queueMicrotask(resolve)); + + const replacementRow = store + .getAggregator(workspaceId)! + .getDisplayedMessages() + .find((message) => "historyId" in message && message.historyId === replacementMessageId)!; + const replacementKey = `${workspaceId}\u0000${replacementRow.id}`; + expect(rawStore.streamingMessageStore.has(replacementKey)).toBe(true); + + dispatch({ + type: "stream-abort", + workspaceId, + messageId: oldMessageId, + abortReason: "user", + rendererCleanupOnly: true, + }); + + expect(rawStore.streamingMessageStore.has(replacementKey)).toBe(true); + expect(store.getAggregator(workspaceId)!.isStreamActive(oldMessageId)).toBe(false); + expect(store.getAggregator(workspaceId)!.isStreamActive(replacementMessageId)).toBe(true); + }); + it("releases the keyed channel when a background activity stop clears the stream", async () => { const workspaceId = "keyed-channel-background-stop"; createAndAddWorkspace(store, workspaceId); diff --git a/src/browser/stores/WorkspaceStore.ts b/src/browser/stores/WorkspaceStore.ts index 4198349201..d8b25aba97 100644 --- a/src/browser/stores/WorkspaceStore.ts +++ b/src/browser/stores/WorkspaceStore.ts @@ -979,6 +979,13 @@ export class WorkspaceStore { "stream-abort": (workspaceId, aggregator, data) => { const streamAbortData = data as StreamAbortEvent; applyWorkspaceChatEventToAggregator(aggregator, streamAbortData); + if (streamAbortData.rendererCleanupOnly === true) { + // This delayed event only closes its old message. Keep replacement stream state intact. + this.cancelPendingStreamingBump(workspaceId, streamAbortData.messageId); + this.states.bump(workspaceId); + this.streamingStatsStore.bump(workspaceId); + return; + } this.releaseStreamingMessageChannel(workspaceId); // Track stream interruption telemetry (get model from aggregator) @@ -1735,7 +1742,13 @@ export class WorkspaceStore { }); } - private cancelPendingStreamingBump(workspaceId: string): void { + private cancelPendingStreamingBump(workspaceId: string, messageId?: string): void { + if ( + messageId !== undefined && + this.pendingStreamingMessageBump.get(workspaceId) !== messageId + ) { + return; + } this.pendingStreamingMessageBump.delete(workspaceId); } diff --git a/src/browser/utils/messages/StreamingMessageAggregator.test.ts b/src/browser/utils/messages/StreamingMessageAggregator.test.ts index dcaa407081..6ac20d79da 100644 --- a/src/browser/utils/messages/StreamingMessageAggregator.test.ts +++ b/src/browser/utils/messages/StreamingMessageAggregator.test.ts @@ -3932,6 +3932,47 @@ describe("StreamingMessageAggregator", () => { }); describe("abort reason tracking", () => { + test("cleanup-only abort preserves a replacement stream", () => { + const aggregator = new StreamingMessageAggregator(TEST_CREATED_AT); + aggregator.handleStreamStart({ + type: "stream-start", + workspaceId: "test-workspace", + messageId: "old-stream", + historySequence: 1, + model: "claude-3-5-sonnet-20241022", + startTime: 1, + }); + aggregator.setInterrupting(); + aggregator.handleStreamStart({ + type: "stream-start", + workspaceId: "test-workspace", + messageId: "replacement-stream", + historySequence: 2, + model: "claude-3-5-sonnet-20241022", + startTime: 2, + }); + aggregator.handleStreamLifecycle({ + type: "stream-lifecycle", + workspaceId: "test-workspace", + phase: "streaming", + hadAnyOutput: false, + }); + + aggregator.handleStreamAbort({ + type: "stream-abort", + workspaceId: "test-workspace", + messageId: "old-stream", + abortReason: "user", + rendererCleanupOnly: true, + }); + + expect(aggregator.isStreamActive("old-stream")).toBe(false); + expect(aggregator.isStreamActive("replacement-stream")).toBe(true); + expect(aggregator.hasInterruptingStream()).toBe(false); + expect(aggregator.getStreamLifecycle()?.phase).toBe("streaming"); + expect(aggregator.getLastAbortReason()).toBeNull(); + }); + test("stores last abort reason and clears on stream-start", () => { const aggregator = new StreamingMessageAggregator(TEST_CREATED_AT); diff --git a/src/browser/utils/messages/StreamingMessageAggregator.ts b/src/browser/utils/messages/StreamingMessageAggregator.ts index 37d1120360..367ffb2ce8 100644 --- a/src/browser/utils/messages/StreamingMessageAggregator.ts +++ b/src/browser/utils/messages/StreamingMessageAggregator.ts @@ -2402,13 +2402,16 @@ export class StreamingMessageAggregator { } handleStreamAbort(data: StreamAbortEvent): void { - // Abort can arrive before stream-start. Clear pending lifecycle UI immediately. - this.clearPendingStreamLifecycleState(); - this.clearInFlightStreamLifecycle(); - this.lastAbortReason = { - reason: data.abortReason ?? "system", - at: Date.now(), - }; + const rendererCleanupOnly = data.rendererCleanupOnly === true; + if (!rendererCleanupOnly) { + // Abort can arrive before stream-start. Clear pending lifecycle UI immediately. + this.clearPendingStreamLifecycleState(); + this.clearInFlightStreamLifecycle(); + this.lastAbortReason = { + reason: data.abortReason ?? "system", + at: Date.now(), + }; + } // Clear "interrupting" state - stream is now fully "interrupted" if (this.interruptingMessageId === data.messageId) { diff --git a/src/common/constants/paths.ts b/src/common/constants/paths.ts index c65b317f72..32b22618c4 100644 --- a/src/common/constants/paths.ts +++ b/src/common/constants/paths.ts @@ -37,6 +37,12 @@ export const TIMELINE_FILE_NAME = "timeline.jsonl"; */ export const CHAT_ARCHIVE_FILE_NAME = "chat-archive.jsonl"; +/** + * Per-workspace tombstones for synthetic history rows that must stay out of + * provider requests when the primary history rewrite fails. + */ +export const PROVIDER_EXCLUDED_MESSAGE_IDS_FILE_NAME = "provider-excluded-message-ids.jsonl"; + /** * Per-workspace sidecar recording headless AI usage (status generation, * memory consolidation/harvest) that produces no chat.jsonl assistant row. diff --git a/src/common/orpc/schemas/stream.ts b/src/common/orpc/schemas/stream.ts index 6108159fcb..18ef6cabc6 100644 --- a/src/common/orpc/schemas/stream.ts +++ b/src/common/orpc/schemas/stream.ts @@ -336,12 +336,14 @@ export const StreamAbortEventSchema = z.object({ // Last step's provider metadata (for context window cache display) contextProviderMetadata: z.record(z.string(), z.unknown()).optional(), duration: z.number().optional(), + abortTurnGeneration: z.number().int().nonnegative().optional(), }) .optional() .meta({ description: "Metadata may contain usage if abort occurred after stream completed processing", }), abandonPartial: z.boolean().optional(), + rendererCleanupOnly: z.boolean().optional(), acpPromptId: z .string() .optional() diff --git a/src/common/types/message.ts b/src/common/types/message.ts index 128d63f0b0..2100154f7f 100644 --- a/src/common/types/message.ts +++ b/src/common/types/message.ts @@ -478,6 +478,24 @@ export function filterOrphanedMcpPromptSnapshots(messages: MuxMessage[]): MuxMes }); } +/** Identify a synthetic row that a user Stop canceled after its rollback boundary. */ +export function isProviderExcludedMessage(message: MuxMessage): boolean { + return ( + message.metadata?.providerExcluded === true && + message.metadata.synthetic === true && + message.metadata.contextBoundaryKind == null && + message.metadata.compactionBoundary !== true + ); +} + +/** Remove durable rows that a user Stop canceled after their rollback boundary. */ +export function filterProviderExcludedMessages(messages: MuxMessage[]): MuxMessage[] { + if (!messages.some(isProviderExcludedMessage)) { + return messages; + } + return messages.filter((message) => !isProviderExcludedMessage(message)); +} + export function dedupeMcpPromptRefs(refs: MCPPromptReference[]): MCPPromptReference[] { const deduped = new Map(); for (const ref of refs) { @@ -929,6 +947,8 @@ export interface MuxMetadata { systemMessageTokens?: number; // Token count for system message sent with this request (calculated by AIService) partial?: boolean; // Whether this message was interrupted and is incomplete synthetic?: boolean; // Whether this message was synthetically generated (e.g., [CONTINUE] sentinel) + /** Keep a canceled durable admission in history, but exclude it from every provider request. */ + providerExcluded?: boolean; /** * For queue-dispatched user turns: when the user last added to the queued * entry. The row `timestamp` is stamped at dispatch (after the blocking turn diff --git a/src/common/utils/messages/compactionBoundary.ts b/src/common/utils/messages/compactionBoundary.ts index 6043cf14e6..52e216daeb 100644 --- a/src/common/utils/messages/compactionBoundary.ts +++ b/src/common/utils/messages/compactionBoundary.ts @@ -6,7 +6,7 @@ import { import { isPositiveInteger } from "@/common/utils/numbers"; import { hasProviderReplayableContent } from "@/common/utils/messages/providerEligibility"; -import type { MuxMessage } from "@/common/types/message"; +import { isProviderExcludedMessage, type MuxMessage } from "@/common/types/message"; export { CONTEXT_BOUNDARY_KINDS, type ContextBoundaryKind }; @@ -129,7 +129,7 @@ export function sliceMessagesFromLatestCompactionBoundary(messages: MuxMessage[] } export function isProviderEligibleMessage(message: MuxMessage): boolean { - if (isDurableContextResetBoundaryMarker(message)) { + if (isProviderExcludedMessage(message) || isDurableContextResetBoundaryMarker(message)) { return false; } diff --git a/src/node/services/agentSession.queueDispatch.test.ts b/src/node/services/agentSession.queueDispatch.test.ts index a0396703a1..a20b230c2f 100644 --- a/src/node/services/agentSession.queueDispatch.test.ts +++ b/src/node/services/agentSession.queueDispatch.test.ts @@ -1,10 +1,11 @@ import { describe, expect, mock, spyOn, test } from "bun:test"; -import type { MuxMessageMetadata } from "@/common/types/message"; +import { createMuxMessage, type MuxMessageMetadata } from "@/common/types/message"; import { Err, Ok } from "@/common/types/result"; import type { WorkspaceGoalService } from "./workspaceGoalService"; import { createAgentSessionHarness, createStartedTurnHandle } from "./agentSession.testHarness"; import type { AIService } from "./aiService"; +import { MessageQueue, type ToolEndQueueClaim } from "./messageQueue"; const TEST_MODEL = "anthropic:claude-sonnet-4-5"; const WORKSPACE_TURN_CORRELATION = { @@ -38,14 +39,18 @@ function streamStartEvent(workspaceId: string): Record { function streamAbortEvent( workspaceId: string, - abortReason: "system" | "user" + abortReason: "system" | "user", + abortTurnGeneration?: number ): Record { return { type: "stream-abort", workspaceId, messageId: "assistant-1", abortReason, - metadata: { duration: 1 }, + metadata: { + duration: 1, + ...(abortTurnGeneration != null ? { abortTurnGeneration } : {}), + }, }; } @@ -132,240 +137,1663 @@ describe("AgentSession queued message tool-call dispatch", () => { } }); + test("does not treat a canceled-only queue as a continuation predecessor", async () => { + const { session, cleanup } = await createAgentSessionHarness({ + workspaceId: "queue-dispatch-canceled-only-predecessor", + }); + + try { + const controller = new AbortController(); + const cancelState = { canceledBeforeAcceptance: false }; + const onCanceled = mock(() => undefined); + session.queueMessage( + "Canceled monitor wake", + { + model: TEST_MODEL, + agentId: "exec", + muxMetadata: { type: "bash-monitor-wake", records: [] }, + }, + { cancelSignal: controller.signal, cancelState, onCanceled } + ); + controller.abort("monitor wake became stale"); + + expect(session.hasQueuedMessages()).toBe(false); + expect(session.hasQueuedOrDispatchingEntry(WORKSPACE_TURN_CORRELATION)).toBe(false); + session.sendQueuedMessages(); + expect(session.isPreparingTurn()).toBe(false); + expect(cancelState.canceledBeforeAcceptance).toBe(true); + expect(await waitForCondition(() => onCanceled.mock.calls.length === 1)).toBe(true); + } finally { + session.dispose(); + await cleanup(); + } + }); + + test("clears the bash-output queue signal when the only live entry is canceled", async () => { + const workspaceId = "queue-dispatch-canceled-signal"; + const { session, cleanup, backgroundProcessManager } = await createAgentSessionHarness({ + workspaceId, + }); + const setMessageQueued = spyOn(backgroundProcessManager, "setMessageQueued"); + + try { + const controller = new AbortController(); + session.queueMessage( + "Canceled monitor wake", + { model: TEST_MODEL, agentId: "exec", queueDispatchMode: "tool-end" }, + { synthetic: true, cancelSignal: controller.signal } + ); + expect(setMessageQueued).toHaveBeenLastCalledWith(workspaceId, true); + + controller.abort("monitor wake became stale"); + + expect(setMessageQueued).toHaveBeenLastCalledWith(workspaceId, false); + } finally { + setMessageQueued.mockRestore(); + session.dispose(); + await cleanup(); + } + }); + test("preserves correlation for same-turn queued and dequeued predecessors", async () => { const { session, cleanup } = await createAgentSessionHarness({ workspaceId: "queue-dispatch-same-turn-predecessor", }); - const differentCorrelation = { - ...WORKSPACE_TURN_CORRELATION, - turnId: "turn-different", - }; + const differentCorrelation = { + ...WORKSPACE_TURN_CORRELATION, + turnId: "turn-different", + }; + + try { + session.queueMessage( + "queued continuation", + { model: TEST_MODEL, agentId: "exec", muxMetadata: WORKSPACE_TURN_CORRELATION }, + { synthetic: true } + ); + expect(session.hasQueuedOrDispatchingEntry(WORKSPACE_TURN_CORRELATION)).toBe(false); + expect(session.hasQueuedOrDispatchingEntry(differentCorrelation)).toBe(true); + + session.queueMessage( + "second queued continuation", + { model: TEST_MODEL, agentId: "exec", muxMetadata: WORKSPACE_TURN_CORRELATION }, + { synthetic: true } + ); + expect(session.hasQueuedOrDispatchingEntry(WORKSPACE_TURN_CORRELATION)).toBe(false); + + session.queueMessage( + "unrelated predecessor", + { model: TEST_MODEL, agentId: "exec" }, + { + synthetic: true, + } + ); + expect(session.hasQueuedOrDispatchingEntry(WORKSPACE_TURN_CORRELATION)).toBe(true); + + const sendMessage = spyOn(session, "sendMessage").mockResolvedValue(Ok(undefined)); + session.sendQueuedMessages(); + expect(session.hasQueuedOrDispatchingEntry(WORKSPACE_TURN_CORRELATION)).toBe(true); + expect(session.hasQueuedOrDispatchingEntry(differentCorrelation)).toBe(true); + sendMessage.mockRestore(); + } finally { + session.dispose(); + await cleanup(); + } + }); + + test("getQueueCutCutter reports an engaged no-metadata dispatch over a queued follow-up", async () => { + // Queue-cut attribution must never blame an entry queued BEHIND the input + // actually taking over the session: a manual message being dispatched wins + // over a workspace-turn follow-up waiting behind it, even though its + // metadata is undefined. + const { session, cleanup } = await createAgentSessionHarness({ + workspaceId: "queue-cut-cutter-preparing", + }); + + try { + expect(session.getQueueCutCutter()).toBeUndefined(); + + session.queueMessage( + "manual message", + { model: TEST_MODEL, agentId: "exec" }, + { synthetic: true } + ); + session.queueMessage( + "workspace-turn follow-up", + { model: TEST_MODEL, agentId: "exec", muxMetadata: WORKSPACE_TURN_CORRELATION }, + { synthetic: true } + ); + + // Queued stage: the manual head entry is the candidate (no metadata). + const queued = session.getQueueCutCutter(); + expect(queued?.stage).toBe("queued"); + expect(queued?.muxMetadata).toBeUndefined(); + + // Dispatch the manual entry: it becomes the engaged PREPARING cutter and + // keeps winning over the follow-up still queued behind it. + const sendMessage = spyOn(session, "sendMessage").mockResolvedValue(Ok(undefined)); + session.sendQueuedMessages(); + const engaged = session.getQueueCutCutter(); + expect(engaged?.stage).toBe("preparing"); + expect(engaged?.muxMetadata).toBeUndefined(); + sendMessage.mockRestore(); + } finally { + session.dispose(); + await cleanup(); + } + }); + + test("getQueueCutCutter reports a no-metadata mid-dispatch entry over a queued follow-up", async () => { + const { session, cleanup } = await createAgentSessionHarness({ + workspaceId: "queue-cut-cutter-dispatching", + }); + + try { + session.queueMessage( + "workspace-turn follow-up", + { model: TEST_MODEL, agentId: "exec", muxMetadata: WORKSPACE_TURN_CORRELATION }, + { synthetic: true } + ); + // Force the dequeue-to-stream-start window with PREPARING already + // released (a background send can resolve before stream-start): the + // dispatched entry stays the engaged cutter. + const internal = session as unknown as { + dispatchingQueuedEntry: boolean; + dispatchingQueuedEntryMuxMetadata?: unknown; + }; + internal.dispatchingQueuedEntry = true; + internal.dispatchingQueuedEntryMuxMetadata = undefined; + + const cutter = session.getQueueCutCutter(); + expect(cutter?.stage).toBe("dispatching"); + expect(cutter?.muxMetadata).toBeUndefined(); + } finally { + session.dispose(); + await cleanup(); + } + }); + + test("getQueueCutCutter exposes the queued head's dispatch mode and correlation", async () => { + const { session, cleanup } = await createAgentSessionHarness({ + workspaceId: "queue-cut-cutter-queued", + }); + + try { + session.queueMessage( + "workspace-turn follow-up", + { + model: TEST_MODEL, + agentId: "exec", + muxMetadata: WORKSPACE_TURN_CORRELATION, + queueDispatchMode: "turn-end", + }, + { synthetic: true } + ); + + const cutter = session.getQueueCutCutter(); + expect(cutter?.stage).toBe("queued"); + expect(cutter?.stage === "queued" ? cutter.dispatchMode : undefined).toBe("turn-end"); + expect((cutter?.muxMetadata as MuxMessageMetadata | undefined)?.type).toBe( + "workspace-turn-task" + ); + + // Once dispatched, the follow-up's correlation rides through PREPARING. + const sendMessage = spyOn(session, "sendMessage").mockResolvedValue(Ok(undefined)); + session.sendQueuedMessages(); + const engaged = session.getQueueCutCutter(); + expect(engaged?.stage).toBe("preparing"); + expect((engaged?.muxMetadata as MuxMessageMetadata | undefined)?.type).toBe( + "workspace-turn-task" + ); + sendMessage.mockRestore(); + } finally { + session.dispose(); + await cleanup(); + } + }); + + test("waits for stream-end instead of interrupting between sibling tool results", async () => { + const workspaceId = "queue-dispatch-full-step"; + const { session, cleanup, aiEmitter, aiService } = await createAgentSessionHarness({ + workspaceId, + }); + const stopStream = spyOn(aiService, "stopStream").mockResolvedValue(Ok(undefined)); + const sendQueuedMessages = spyOn(session, "sendQueuedMessages").mockImplementation( + () => undefined + ); + + try { + aiEmitter.emit("stream-start", streamStartEvent(workspaceId)); + session.queueMessage("follow up", { model: TEST_MODEL, agentId: "exec" }); + + aiEmitter.emit("tool-call-end", toolCallEndEvent(workspaceId)); + aiEmitter.emit("tool-call-end", { + ...toolCallEndEvent(workspaceId), + toolCallId: "tool-call-2", + }); + + expect(stopStream).not.toHaveBeenCalled(); + expect(sendQueuedMessages).not.toHaveBeenCalled(); + + aiEmitter.emit("stream-end", { + type: "stream-end", + workspaceId, + messageId: "assistant-1", + parts: [], + metadata: { + model: TEST_MODEL, + contextUsage: { inputTokens: 5, outputTokens: 5, totalTokens: 10 }, + providerMetadata: {}, + finishReason: "tool-calls", + }, + }); + + const didDispatch = await waitForCondition(() => sendQueuedMessages.mock.calls.length > 0); + expect(didDispatch).toBe(true); + expect(sendQueuedMessages).toHaveBeenCalledTimes(1); + } finally { + sendQueuedMessages.mockRestore(); + stopStream.mockRestore(); + session.dispose(); + await cleanup(); + } + }); + + test("dispatches a claimed monitor wake after task output cancels its signal", async () => { + const workspaceId = "queue-dispatch-claimed-monitor-wake"; + let claimQueuedToolEndMessage: (() => boolean) | undefined; + const streamMessage = mock((options: Parameters[0]) => { + claimQueuedToolEndMessage ??= options.claimQueuedToolEndMessage; + return Promise.resolve(Ok(createStartedTurnHandle())); + }); + const { session, cleanup, aiEmitter } = await createAgentSessionHarness({ + workspaceId, + aiServiceOverrides: { + streamMessage: streamMessage as unknown as AIService["streamMessage"], + }, + }); + + try { + const initialSend = await session.sendMessage("Start work", { + model: TEST_MODEL, + agentId: "exec", + }); + expect(initialSend.success).toBe(true); + aiEmitter.emit("stream-start", streamStartEvent(workspaceId)); + + const controller = new AbortController(); + const cancelState = { canceledBeforeAcceptance: false }; + const onCanceled = mock(() => undefined); + session.queueMessage( + "Background monitor wake", + { + model: TEST_MODEL, + agentId: "exec", + queueDispatchMode: "tool-end", + muxMetadata: { type: "bash-monitor-wake", records: [] }, + }, + { + synthetic: true, + agentInitiated: true, + cancelState, + cancelSignal: controller.signal, + onCanceled, + } + ); + + expect(claimQueuedToolEndMessage?.()).toBe(true); + controller.abort("task_await returned the terminal output"); + aiEmitter.emit("stream-end", { + type: "stream-end", + workspaceId, + messageId: "assistant-1", + parts: [], + metadata: { + model: TEST_MODEL, + contextUsage: { inputTokens: 5, outputTokens: 5, totalTokens: 10 }, + providerMetadata: {}, + finishReason: "tool-calls", + }, + }); + + expect(await waitForCondition(() => streamMessage.mock.calls.length === 2)).toBe(true); + expect(cancelState.canceledBeforeAcceptance).toBe(false); + expect(onCanceled).not.toHaveBeenCalled(); + } finally { + session.dispose(); + await cleanup(); + } + }); + + test("restores an SDK queue claim before a hard user interrupt", async () => { + const workspaceId = "queue-dispatch-sdk-claim-hard-interrupt"; + let claimQueuedToolEndMessage: (() => boolean) | undefined; + const streamMessage = mock((options: Parameters[0]) => { + claimQueuedToolEndMessage ??= options.claimQueuedToolEndMessage; + return Promise.resolve(Ok(createStartedTurnHandle())); + }); + const { session, cleanup, aiEmitter, aiService } = await createAgentSessionHarness({ + workspaceId, + aiServiceOverrides: { + streamMessage: streamMessage as unknown as AIService["streamMessage"], + }, + }); + const stopStream = spyOn(aiService, "stopStream").mockResolvedValue(Ok(undefined)); + + try { + const initialSend = await session.sendMessage("Start work", { + model: TEST_MODEL, + agentId: "exec", + }); + expect(initialSend.success).toBe(true); + aiEmitter.emit("stream-start", streamStartEvent(workspaceId)); + + const controller = new AbortController(); + const cancelState = { canceledBeforeAcceptance: false }; + const onCanceled = mock(() => undefined); + session.queueMessage( + "Background monitor wake", + { model: TEST_MODEL, agentId: "exec", queueDispatchMode: "tool-end" }, + { synthetic: true, cancelSignal: controller.signal, cancelState, onCanceled } + ); + + expect(claimQueuedToolEndMessage?.()).toBe(true); + expect((await session.interruptStream()).success).toBe(true); + controller.abort("hard interrupt canceled the monitor wake"); + session.sendQueuedMessages(); + + expect(await waitForCondition(() => onCanceled.mock.calls.length === 1)).toBe(true); + expect(cancelState.canceledBeforeAcceptance).toBe(true); + expect(streamMessage).toHaveBeenCalledTimes(1); + } finally { + stopStream.mockRestore(); + session.dispose(); + await cleanup(); + } + }); + + test("rejects an SDK queue claim after a hard interrupt starts", async () => { + const workspaceId = "queue-dispatch-sdk-claim-during-hard-interrupt"; + let claimQueuedToolEndMessage: (() => boolean) | undefined; + const streamMessage = mock((options: Parameters[0]) => { + claimQueuedToolEndMessage ??= options.claimQueuedToolEndMessage; + return Promise.resolve(Ok(createStartedTurnHandle())); + }); + const { session, cleanup, aiEmitter, aiService, historyService } = + await createAgentSessionHarness({ + workspaceId, + aiServiceOverrides: { + streamMessage: streamMessage as unknown as AIService["streamMessage"], + }, + }); + const stopStream = spyOn(aiService, "stopStream").mockResolvedValue(Ok(undefined)); + let markDeleteStarted: () => void = () => undefined; + const deleteStarted = new Promise((resolve) => { + markDeleteStarted = resolve; + }); + let releaseDelete: () => void = () => undefined; + const deleteRelease = new Promise((resolve) => { + releaseDelete = resolve; + }); + const deletePartial = spyOn(historyService, "deletePartial").mockImplementationOnce( + async () => { + markDeleteStarted(); + await deleteRelease; + return Ok(undefined); + } + ); + + try { + expect( + ( + await session.sendMessage("Start work", { + model: TEST_MODEL, + agentId: "exec", + }) + ).success + ).toBe(true); + aiEmitter.emit("stream-start", streamStartEvent(workspaceId)); + session.queueMessage("Follow up", { + model: TEST_MODEL, + agentId: "exec", + queueDispatchMode: "tool-end", + }); + + const interruptPromise = session.interruptStream({ abandonPartial: true }); + await deleteStarted; + + expect(claimQueuedToolEndMessage?.()).toBe(false); + releaseDelete(); + expect((await interruptPromise).success).toBe(true); + } finally { + releaseDelete(); + deletePartial.mockRestore(); + stopStream.mockRestore(); + session.dispose(); + await cleanup(); + } + }); + + test("restores an SDK queue claim when the hard interrupt fails", async () => { + const workspaceId = "queue-dispatch-sdk-claim-hard-interrupt-failure"; + let claimQueuedToolEndMessage: (() => boolean) | undefined; + const streamMessage = mock((options: Parameters[0]) => { + claimQueuedToolEndMessage ??= options.claimQueuedToolEndMessage; + return Promise.resolve(Ok(createStartedTurnHandle())); + }); + const { session, cleanup, aiEmitter, aiService } = await createAgentSessionHarness({ + workspaceId, + aiServiceOverrides: { + streamMessage: streamMessage as unknown as AIService["streamMessage"], + }, + }); + const stopStream = spyOn(aiService, "stopStream").mockResolvedValue( + Err("injected hard-interrupt failure") + ); + + try { + expect( + ( + await session.sendMessage("Start work", { + model: TEST_MODEL, + agentId: "exec", + }) + ).success + ).toBe(true); + aiEmitter.emit("stream-start", streamStartEvent(workspaceId)); + session.queueMessage("User follow-up", { + model: TEST_MODEL, + agentId: "exec", + queueDispatchMode: "tool-end", + }); + + expect(claimQueuedToolEndMessage?.()).toBe(true); + expect((await session.interruptStream()).success).toBe(false); + + expect(session.hasQueuedMessages()).toBe(true); + expect(claimQueuedToolEndMessage?.()).toBe(true); + } finally { + stopStream.mockRestore(); + session.dispose(); + await cleanup(); + } + }); + + test("preserves a claimed user entry for Send now", async () => { + const workspaceId = "queue-dispatch-claimed-user-send-now"; + let claimQueuedToolEndMessage: (() => boolean) | undefined; + const streamMessage = mock((options: Parameters[0]) => { + claimQueuedToolEndMessage ??= options.claimQueuedToolEndMessage; + return Promise.resolve(Ok(createStartedTurnHandle())); + }); + const { session, cleanup, aiEmitter, aiService } = await createAgentSessionHarness({ + workspaceId, + aiServiceOverrides: { + streamMessage: streamMessage as unknown as AIService["streamMessage"], + }, + }); + const stopStream = spyOn(aiService, "stopStream").mockImplementation(async () => { + aiEmitter.emit("stream-abort", streamAbortEvent(workspaceId, "user")); + await new Promise((resolve) => setTimeout(resolve, 10)); + return Ok(undefined); + }); + + try { + expect( + ( + await session.sendMessage("Start work", { + model: TEST_MODEL, + agentId: "exec", + }) + ).success + ).toBe(true); + aiEmitter.emit("stream-start", streamStartEvent(workspaceId)); + session.queueMessage("User send now", { + model: TEST_MODEL, + agentId: "exec", + queueDispatchMode: "tool-end", + }); + + expect(claimQueuedToolEndMessage?.()).toBe(true); + expect((await session.interruptStream({ sendQueuedImmediately: true })).success).toBe(true); + expect(session.sendNextUserQueuedMessage()).toBe(true); + + expect(await waitForCondition(() => streamMessage.mock.calls.length === 2)).toBe(true); + } finally { + stopStream.mockRestore(); + session.dispose(); + await cleanup(); + } + }); + + test("lets an admitted user claim finish during Send now", async () => { + const workspaceId = "queue-dispatch-admitted-user-send-now"; + let claimQueuedToolEndMessage: (() => boolean) | undefined; + const streamMessage = mock((options: Parameters[0]) => { + claimQueuedToolEndMessage ??= options.claimQueuedToolEndMessage; + return Promise.resolve(Ok(createStartedTurnHandle())); + }); + const { session, cleanup, aiEmitter, aiService, historyService } = + await createAgentSessionHarness({ + workspaceId, + aiServiceOverrides: { + streamMessage: streamMessage as unknown as AIService["streamMessage"], + }, + }); + const stopStream = spyOn(aiService, "stopStream").mockResolvedValue(Ok(undefined)); + let markAppendStarted: () => void = () => undefined; + const appendStarted = new Promise((resolve) => { + markAppendStarted = resolve; + }); + let releaseAppend: () => void = () => undefined; + const appendRelease = new Promise((resolve) => { + releaseAppend = resolve; + }); + + try { + expect( + ( + await session.sendMessage("Start work", { + model: TEST_MODEL, + agentId: "exec", + }) + ).success + ).toBe(true); + aiEmitter.emit("stream-start", streamStartEvent(workspaceId)); + const originalAppend = historyService.appendToHistory.bind(historyService); + const appendToHistory = spyOn(historyService, "appendToHistory").mockImplementationOnce( + async (...args) => { + markAppendStarted(); + await appendRelease; + return originalAppend(...args); + } + ); + try { + session.queueMessage("User send now", { + model: TEST_MODEL, + agentId: "exec", + queueDispatchMode: "tool-end", + }); + + expect(claimQueuedToolEndMessage?.()).toBe(true); + session.sendQueuedMessages(); + await appendStarted; + + expect((await session.interruptStream({ sendQueuedImmediately: true })).success).toBe(true); + releaseAppend(); + await new Promise((resolve) => setTimeout(resolve, 25)); + expect(streamMessage).toHaveBeenCalledTimes(1); + expect(session.sendNextUserQueuedMessage()).toBe(true); + + expect(await waitForCondition(() => streamMessage.mock.calls.length === 2)).toBe(true); + const history = await historyService.getHistoryFromLatestBoundary(workspaceId); + expect(history.success).toBe(true); + if (history.success) { + const text = history.data + .flatMap((message) => message.parts) + .map((part) => (part.type === "text" ? part.text : "")); + expect(text).toContain("User send now"); + } + } finally { + releaseAppend(); + appendToHistory.mockRestore(); + } + } finally { + releaseAppend(); + stopStream.mockRestore(); + session.dispose(); + await cleanup(); + } + }); + + test("restores an admitted user claim when Stop follows Send now", async () => { + const workspaceId = "queue-dispatch-admitted-user-second-stop"; + let claimQueuedToolEndMessage: (() => boolean) | undefined; + const streamMessage = mock((options: Parameters[0]) => { + claimQueuedToolEndMessage ??= options.claimQueuedToolEndMessage; + return Promise.resolve(Ok(createStartedTurnHandle())); + }); + const { session, cleanup, aiEmitter, aiService, historyService } = + await createAgentSessionHarness({ + workspaceId, + aiServiceOverrides: { + streamMessage: streamMessage as unknown as AIService["streamMessage"], + }, + }); + const stopStream = spyOn(aiService, "stopStream").mockResolvedValue(Ok(undefined)); + let markAppendStarted: () => void = () => undefined; + const appendStarted = new Promise((resolve) => { + markAppendStarted = resolve; + }); + let releaseAppend: () => void = () => undefined; + const appendRelease = new Promise((resolve) => { + releaseAppend = resolve; + }); + + try { + expect( + ( + await session.sendMessage("Start work", { + model: TEST_MODEL, + agentId: "exec", + }) + ).success + ).toBe(true); + aiEmitter.emit("stream-start", streamStartEvent(workspaceId)); + const originalAppend = historyService.appendToHistory.bind(historyService); + const appendToHistory = spyOn(historyService, "appendToHistory").mockImplementationOnce( + async (...args) => { + markAppendStarted(); + await appendRelease; + return originalAppend(...args); + } + ); + const restoredTexts: string[] = []; + const unsubscribe = session.onChatEvent((event) => { + if (event.message.type === "restore-to-input") { + restoredTexts.push(event.message.text); + } + }); + try { + session.queueMessage("User send now", { + model: TEST_MODEL, + agentId: "exec", + queueDispatchMode: "tool-end", + }); + + expect(claimQueuedToolEndMessage?.()).toBe(true); + session.sendQueuedMessages(); + await appendStarted; + + expect((await session.interruptStream({ sendQueuedImmediately: true })).success).toBe(true); + expect((await session.interruptStream()).success).toBe(true); + releaseAppend(); + + await new Promise((resolve) => setTimeout(resolve, 25)); + expect(streamMessage).toHaveBeenCalledTimes(1); + expect(restoredTexts).toEqual(["User send now"]); + expect(session.sendNextUserQueuedMessage()).toBe(false); + const history = await historyService.getHistoryFromLatestBoundary(workspaceId); + expect(history.success).toBe(true); + if (history.success) { + const text = history.data + .flatMap((message) => message.parts) + .map((part) => (part.type === "text" ? part.text : "")); + expect(text).not.toContain("User send now"); + } + } finally { + unsubscribe(); + releaseAppend(); + appendToHistory.mockRestore(); + } + } finally { + releaseAppend(); + stopStream.mockRestore(); + session.dispose(); + await cleanup(); + } + }); + + test("soft-stops after a provider-executed tool result and dispatches after abort", async () => { + const workspaceId = "queue-dispatch-provider-tool"; + const { session, cleanup, aiEmitter, aiService } = await createAgentSessionHarness({ + workspaceId, + }); + const stopStream = spyOn(aiService, "stopStream").mockResolvedValue(Ok(undefined)); + const sendQueuedMessages = spyOn(session, "sendQueuedMessages").mockImplementation( + () => undefined + ); + + try { + aiEmitter.emit("stream-start", streamStartEvent(workspaceId)); + session.queueMessage("follow up", { model: TEST_MODEL, agentId: "exec" }); + + aiEmitter.emit("tool-call-end", { + ...toolCallEndEvent(workspaceId), + toolName: "web_search", + providerExecuted: true, + }); + + expect(stopStream).toHaveBeenCalledWith(workspaceId, { + soft: true, + abortReason: "system", + }); + + aiEmitter.emit("stream-abort", streamAbortEvent(workspaceId, "system")); + const didDispatch = await waitForCondition(() => sendQueuedMessages.mock.calls.length > 0); + expect(didDispatch).toBe(true); + expect(sendQueuedMessages).toHaveBeenCalledTimes(1); + } finally { + sendQueuedMessages.mockRestore(); + stopStream.mockRestore(); + session.dispose(); + await cleanup(); + } + }); + + test("keeps a provider-tool queue cut after monitor cancellation", async () => { + const workspaceId = "queue-dispatch-provider-tool-monitor-wake"; + const streamMessage = mock(() => Promise.resolve(Ok(createStartedTurnHandle()))); + const { session, cleanup, aiEmitter, aiService } = await createAgentSessionHarness({ + workspaceId, + aiServiceOverrides: { + streamMessage: streamMessage as unknown as AIService["streamMessage"], + }, + }); + const stopStream = spyOn(aiService, "stopStream").mockResolvedValue(Ok(undefined)); + + try { + const initialSend = await session.sendMessage("Start work", { + model: TEST_MODEL, + agentId: "exec", + }); + expect(initialSend.success).toBe(true); + aiEmitter.emit("stream-start", streamStartEvent(workspaceId)); + + const controller = new AbortController(); + const cancelState = { canceledBeforeAcceptance: false }; + const onCanceled = mock(() => undefined); + session.queueMessage( + "Background monitor wake", + { + model: TEST_MODEL, + agentId: "exec", + queueDispatchMode: "tool-end", + muxMetadata: { type: "bash-monitor-wake", records: [] }, + }, + { + synthetic: true, + agentInitiated: true, + cancelState, + cancelSignal: controller.signal, + onCanceled, + } + ); + + aiEmitter.emit("tool-call-end", { + ...toolCallEndEvent(workspaceId), + toolName: "web_search", + providerExecuted: true, + }); + expect(await waitForCondition(() => stopStream.mock.calls.length === 1)).toBe(true); + + controller.abort("task_await returned the terminal output"); + aiEmitter.emit("stream-abort", streamAbortEvent(workspaceId, "system")); + + expect(await waitForCondition(() => streamMessage.mock.calls.length === 2)).toBe(true); + expect(cancelState.canceledBeforeAcceptance).toBe(false); + expect(onCanceled).not.toHaveBeenCalled(); + } finally { + stopStream.mockRestore(); + session.dispose(); + await cleanup(); + } + }); + + test("drains the next live entry when the claimed provider entry is removed", async () => { + const workspaceId = "queue-dispatch-removed-provider-claim"; + const streamMessage = mock(() => Promise.resolve(Ok(createStartedTurnHandle()))); + const { session, cleanup, aiEmitter, aiService } = await createAgentSessionHarness({ + workspaceId, + aiServiceOverrides: { + streamMessage: streamMessage as unknown as AIService["streamMessage"], + }, + }); + const stopStream = spyOn(aiService, "stopStream").mockResolvedValue(Ok(undefined)); + + try { + expect( + ( + await session.sendMessage("Start work", { + model: TEST_MODEL, + agentId: "exec", + }) + ).success + ).toBe(true); + aiEmitter.emit("stream-start", streamStartEvent(workspaceId)); + session.queueMessage( + "Incremental child report", + { model: TEST_MODEL, agentId: "exec", queueDispatchMode: "tool-end" }, + { + synthetic: true, + agentInitiated: true, + dedupeKey: "agent-report:child:progress", + removableDedupeKey: true, + } + ); + session.queueMessage("User follow-up", { + model: TEST_MODEL, + agentId: "exec", + queueDispatchMode: "tool-end", + }); + + aiEmitter.emit("tool-call-end", { + ...toolCallEndEvent(workspaceId), + toolName: "web_search", + providerExecuted: true, + }); + expect(await waitForCondition(() => stopStream.mock.calls.length === 1)).toBe(true); + expect( + session.removeQueuedMessagesByDedupeKeyPrefix( + "agent-report:child:", + "Terminal report superseded the progress report." + ) + ).toBe(1); + + aiEmitter.emit("stream-abort", streamAbortEvent(workspaceId, "system")); + + expect(await waitForCondition(() => streamMessage.mock.calls.length === 2)).toBe(true); + } finally { + stopStream.mockRestore(); + session.dispose(); + await cleanup(); + } + }); + + test("excludes a provider dispatch when hard Stop lands during acceptance", async () => { + const workspaceId = "queue-dispatch-provider-claim-preparing-stop"; + const streamMessage = mock((_options: Parameters[0]) => + Promise.resolve(Ok(createStartedTurnHandle())) + ); + const { session, cleanup, aiEmitter, aiService } = await createAgentSessionHarness({ + workspaceId, + aiServiceOverrides: { + streamMessage: streamMessage as unknown as AIService["streamMessage"], + }, + }); + let stopCallCount = 0; + let releaseHardStop: () => void = () => undefined; + const hardStopRelease = new Promise((resolve) => { + releaseHardStop = resolve; + }); + const stopStream = spyOn(aiService, "stopStream").mockImplementation(async () => { + stopCallCount += 1; + if (stopCallCount === 2) { + await hardStopRelease; + } + return Ok(undefined); + }); + let markAcceptanceStarted: () => void = () => undefined; + const acceptanceStarted = new Promise((resolve) => { + markAcceptanceStarted = resolve; + }); + let markAcceptanceFinished: () => void = () => undefined; + const acceptanceFinished = new Promise((resolve) => { + markAcceptanceFinished = resolve; + }); + let releaseAcceptance: () => void = () => undefined; + const acceptanceRelease = new Promise((resolve) => { + releaseAcceptance = resolve; + }); + + try { + expect( + ( + await session.sendMessage("Start work", { + model: TEST_MODEL, + agentId: "exec", + }) + ).success + ).toBe(true); + aiEmitter.emit("stream-start", streamStartEvent(workspaceId)); + session.queueMessage( + "Background monitor wake", + { + model: TEST_MODEL, + agentId: "exec", + queueDispatchMode: "tool-end", + muxMetadata: { type: "bash-monitor-wake", records: [] }, + }, + { + synthetic: true, + agentInitiated: true, + onAccepted: async () => { + markAcceptanceStarted(); + await acceptanceRelease; + markAcceptanceFinished(); + }, + } + ); + session.queueMessage("User send now", { + model: TEST_MODEL, + agentId: "exec", + queueDispatchMode: "tool-end", + }); + + aiEmitter.emit("tool-call-end", { + ...toolCallEndEvent(workspaceId), + toolName: "web_search", + providerExecuted: true, + }); + expect(await waitForCondition(() => stopStream.mock.calls.length === 1)).toBe(true); + aiEmitter.emit("stream-abort", streamAbortEvent(workspaceId, "system")); + await acceptanceStarted; + + const interruptPromise = session.interruptStream({ + sendQueuedImmediately: true, + deferQueueSettlement: true, + }); + expect(await waitForCondition(() => stopCallCount === 2)).toBe(true); + releaseAcceptance(); + await acceptanceFinished; + await Promise.resolve(); + expect(streamMessage).toHaveBeenCalledTimes(1); + expect(session.hasQueuedMessages()).toBe(true); + + releaseHardStop(); + expect((await interruptPromise).success).toBe(true); + expect(streamMessage).toHaveBeenCalledTimes(1); + + expect(session.sendNextUserQueuedMessage()).toBe(true); + + expect(await waitForCondition(() => streamMessage.mock.calls.length === 2)).toBe(true); + const requestMessages = streamMessage.mock.calls[1]?.[0].messages ?? []; + const requestText = requestMessages + .flatMap((message) => message.parts) + .map((part) => (part.type === "text" ? part.text : "")); + expect(requestText).toContain("User send now"); + expect(requestText).not.toContain("Background monitor wake"); + expect(session.hasQueuedMessages()).toBe(false); + expect(session.isPreparingTurn()).toBe(false); + } finally { + releaseAcceptance(); + releaseHardStop(); + stopStream.mockRestore(); + session.dispose(); + await cleanup(); + } + }); + + test("retains a synthetic claim until startup accepts cancellation", async () => { + const workspaceId = "queue-dispatch-provider-claim-startup-stop"; + const streamMessage = mock((_options: Parameters[0]) => + Promise.resolve(Ok(createStartedTurnHandle())) + ); + const { session, cleanup, aiEmitter, aiService, historyService } = + await createAgentSessionHarness({ + workspaceId, + aiServiceOverrides: { + streamMessage: streamMessage as unknown as AIService["streamMessage"], + }, + }); + const stopStream = spyOn(aiService, "stopStream").mockResolvedValue(Ok(undefined)); + let markCommitStarted: () => void = () => undefined; + const commitStarted = new Promise((resolve) => { + markCommitStarted = resolve; + }); + let releaseCommit: () => void = () => undefined; + const commitRelease = new Promise((resolve) => { + releaseCommit = resolve; + }); + const accepted = mock(() => undefined); + + try { + expect( + ( + await session.sendMessage("Start work", { + model: TEST_MODEL, + agentId: "exec", + }) + ).success + ).toBe(true); + aiEmitter.emit("stream-start", streamStartEvent(workspaceId)); + const originalCommit = historyService.commitPartial.bind(historyService); + const commitPartial = spyOn(historyService, "commitPartial").mockImplementationOnce( + async (...args) => { + markCommitStarted(); + await commitRelease; + return originalCommit(...args); + } + ); + try { + session.queueMessage( + "Background monitor wake", + { + model: TEST_MODEL, + agentId: "exec", + queueDispatchMode: "tool-end", + muxMetadata: { type: "bash-monitor-wake", records: [] }, + }, + { synthetic: true, agentInitiated: true, onAccepted: accepted } + ); + session.queueMessage("User send now", { + model: TEST_MODEL, + agentId: "exec", + queueDispatchMode: "tool-end", + }); + + aiEmitter.emit("tool-call-end", { + ...toolCallEndEvent(workspaceId), + toolName: "web_search", + providerExecuted: true, + }); + expect(await waitForCondition(() => stopStream.mock.calls.length === 1)).toBe(true); + aiEmitter.emit("stream-abort", streamAbortEvent(workspaceId, "system")); + await commitStarted; + expect(accepted).toHaveBeenCalledTimes(1); + expect(streamMessage).toHaveBeenCalledTimes(1); + + expect( + ( + await session.interruptStream({ + sendQueuedImmediately: true, + deferQueueSettlement: true, + }) + ).success + ).toBe(true); + expect(session.sendNextUserQueuedMessage()).toBe(true); + releaseCommit(); + + expect(await waitForCondition(() => streamMessage.mock.calls.length === 2)).toBe(true); + const requestMessages = streamMessage.mock.calls[1]?.[0].messages ?? []; + const requestText = requestMessages + .flatMap((message) => message.parts) + .map((part) => (part.type === "text" ? part.text : "")); + expect(requestText).toContain("User send now"); + expect(requestText).not.toContain("Background monitor wake"); + } finally { + releaseCommit(); + commitPartial.mockRestore(); + } + } finally { + releaseCommit(); + stopStream.mockRestore(); + session.dispose(); + await cleanup(); + } + }); + + test("cancels a claimed synthetic dispatch before Send now", async () => { + const workspaceId = "queue-dispatch-provider-claim-pre-acceptance-send-now"; + const streamMessage = mock(() => Promise.resolve(Ok(createStartedTurnHandle()))); + const { session, cleanup, aiEmitter, aiService, historyService } = + await createAgentSessionHarness({ + workspaceId, + aiServiceOverrides: { + streamMessage: streamMessage as unknown as AIService["streamMessage"], + }, + }); + const stopStream = spyOn(aiService, "stopStream").mockResolvedValue(Ok(undefined)); + + try { + expect( + ( + await session.sendMessage("Start work", { + model: TEST_MODEL, + agentId: "exec", + }) + ).success + ).toBe(true); + aiEmitter.emit("stream-start", streamStartEvent(workspaceId)); + + let markAppendStarted: () => void = () => undefined; + const appendStarted = new Promise((resolve) => { + markAppendStarted = resolve; + }); + let releaseAppend: () => void = () => undefined; + const appendRelease = new Promise((resolve) => { + releaseAppend = resolve; + }); + const originalAppend = historyService.appendToHistory.bind(historyService); + const appendToHistory = spyOn(historyService, "appendToHistory").mockImplementationOnce( + async (...args) => { + markAppendStarted(); + await appendRelease; + return originalAppend(...args); + } + ); + try { + const cancelState = { canceledBeforeAcceptance: false }; + const onAcceptedPreStreamFailure = mock(() => undefined); + session.queueMessage( + "Background monitor wake", + { + model: TEST_MODEL, + agentId: "exec", + queueDispatchMode: "tool-end", + muxMetadata: { type: "bash-monitor-wake", records: [] }, + }, + { + synthetic: true, + agentInitiated: true, + cancelState, + onAcceptedPreStreamFailure, + } + ); + session.queueMessage("User send now", { + model: TEST_MODEL, + agentId: "exec", + queueDispatchMode: "tool-end", + }); + + aiEmitter.emit("tool-call-end", { + ...toolCallEndEvent(workspaceId), + toolName: "web_search", + providerExecuted: true, + }); + expect(await waitForCondition(() => stopStream.mock.calls.length === 1)).toBe(true); + aiEmitter.emit("stream-abort", streamAbortEvent(workspaceId, "system")); + await appendStarted; + + expect((await session.interruptStream({ sendQueuedImmediately: true })).success).toBe(true); + expect(session.sendNextUserQueuedMessage()).toBe(true); + releaseAppend(); + + expect( + await waitForCondition(() => onAcceptedPreStreamFailure.mock.calls.length === 1) + ).toBe(true); + expect(await waitForCondition(() => streamMessage.mock.calls.length === 2)).toBe(true); + expect(cancelState.canceledBeforeAcceptance).toBe(true); + expect(session.hasPendingBashMonitorWakeContinuation()).toBe(false); + const history = await historyService.getHistoryFromLatestBoundary(workspaceId); + expect(history.success).toBe(true); + if (history.success) { + const text = history.data + .flatMap((message) => message.parts) + .map((part) => (part.type === "text" ? part.text : "")); + expect(text).toContain("User send now"); + expect(text).not.toContain("Background monitor wake"); + } + } finally { + releaseAppend(); + appendToHistory.mockRestore(); + } + } finally { + stopStream.mockRestore(); + session.dispose(); + await cleanup(); + } + }); + + test("excludes an irreversible synthetic claim before a Send now user", async () => { + const workspaceId = "queue-dispatch-irrevocable-synthetic-send-now"; + let claimQueuedToolEndMessage: (() => boolean) | undefined; + const streamMessage = mock((options: Parameters[0]) => { + claimQueuedToolEndMessage ??= options.claimQueuedToolEndMessage; + return Promise.resolve(Ok(createStartedTurnHandle())); + }); + let markSyncStarted: () => void = () => undefined; + const syncStarted = new Promise((resolve) => { + markSyncStarted = resolve; + }); + let releaseSync: () => void = () => undefined; + const syncRelease = new Promise((resolve) => { + releaseSync = resolve; + }); + let syncCalls = 0; + const syncGoalModeWithChatTail = mock(async () => { + syncCalls += 1; + if (syncCalls === 2) { + markSyncStarted(); + await syncRelease; + } + return null; + }); + const workspaceGoalService = { + assertPricedModelForBudgetedGoal: mock(() => Promise.resolve(Ok(undefined))), + syncGoalModeWithChatTail, + recordStreamStarted: mock(() => undefined), + takePendingContinuationCandidateForManualUserMessage: mock(() => undefined), + acknowledgeUser: mock(() => Promise.resolve(null)), + clearPendingContinuationForManualUserMessage: mock(() => undefined), + } as unknown as WorkspaceGoalService; + const { session, cleanup, aiEmitter, aiService, historyService } = + await createAgentSessionHarness({ + workspaceId, + workspaceGoalService, + aiServiceOverrides: { + streamMessage: streamMessage as unknown as AIService["streamMessage"], + }, + }); + const stopStream = spyOn(aiService, "stopStream").mockResolvedValue(Ok(undefined)); + let admissionStale = false; try { + expect( + ( + await session.sendMessage( + "Start work", + { + model: TEST_MODEL, + agentId: "exec", + }, + { synthetic: true, agentInitiated: true } + ) + ).success + ).toBe(true); + aiEmitter.emit("stream-start", streamStartEvent(workspaceId)); + let accepted = false; session.queueMessage( - "queued continuation", - { model: TEST_MODEL, agentId: "exec", muxMetadata: WORKSPACE_TURN_CORRELATION }, - { synthetic: true } - ); - expect(session.hasQueuedOrDispatchingEntry(WORKSPACE_TURN_CORRELATION)).toBe(false); - expect(session.hasQueuedOrDispatchingEntry(differentCorrelation)).toBe(true); - - session.queueMessage( - "second queued continuation", - { model: TEST_MODEL, agentId: "exec", muxMetadata: WORKSPACE_TURN_CORRELATION }, - { synthetic: true } - ); - expect(session.hasQueuedOrDispatchingEntry(WORKSPACE_TURN_CORRELATION)).toBe(false); - - session.queueMessage( - "unrelated predecessor", - { model: TEST_MODEL, agentId: "exec" }, + "Background monitor wake", + { + model: TEST_MODEL, + agentId: "exec", + queueDispatchMode: "tool-end", + muxMetadata: { type: "bash-monitor-wake", records: [] }, + }, { synthetic: true, + agentInitiated: true, + onAccepted: () => { + accepted = true; + }, + admissionStale: () => admissionStale, } ); - expect(session.hasQueuedOrDispatchingEntry(WORKSPACE_TURN_CORRELATION)).toBe(true); + session.queueMessage("User send now", { + model: TEST_MODEL, + agentId: "exec", + queueDispatchMode: "tool-end", + }); - const sendMessage = spyOn(session, "sendMessage").mockResolvedValue(Ok(undefined)); + expect(claimQueuedToolEndMessage?.()).toBe(true); session.sendQueuedMessages(); - expect(session.hasQueuedOrDispatchingEntry(WORKSPACE_TURN_CORRELATION)).toBe(true); - expect(session.hasQueuedOrDispatchingEntry(differentCorrelation)).toBe(true); - sendMessage.mockRestore(); + await syncStarted; + + expect((await session.interruptStream({ sendQueuedImmediately: true })).success).toBe(true); + // Descendant cleanup can stale the producer after its durable row crosses the rollback + // boundary. The exact synthetic turn must finish before the prioritized user turn. + admissionStale = true; + expect(session.sendNextUserQueuedMessage()).toBe(true); + releaseSync(); + + expect(await waitForCondition(() => accepted)).toBe(true); + expect(await waitForCondition(() => streamMessage.mock.calls.length === 2)).toBe(true); + const requestMessages = streamMessage.mock.calls[1]?.[0].messages ?? []; + const requestText = requestMessages + .flatMap((message) => message.parts) + .map((part) => (part.type === "text" ? part.text : "")); + expect(requestText).toContain("User send now"); + expect(requestText).not.toContain("Background monitor wake"); + expect(session.hasQueuedMessages()).toBe(false); + const history = await historyService.getHistoryFromLatestBoundary(workspaceId); + expect(history.success).toBe(true); + if (history.success) { + const wake = history.data.find((message) => + message.parts.some( + (part) => part.type === "text" && part.text === "Background monitor wake" + ) + ); + expect(wake?.metadata?.providerExcluded).toBe(true); + } } finally { + releaseSync(); + stopStream.mockRestore(); session.dispose(); await cleanup(); } }); - test("getQueueCutCutter reports an engaged no-metadata dispatch over a queued follow-up", async () => { - // Queue-cut attribution must never blame an entry queued BEHIND the input - // actually taking over the session: a manual message being dispatched wins - // over a workspace-turn follow-up waiting behind it, even though its - // metadata is undefined. - const { session, cleanup } = await createAgentSessionHarness({ - workspaceId: "queue-cut-cutter-preparing", + test("binds a hard interrupt to the claim present before stopStream yields", async () => { + const workspaceId = "queue-dispatch-delayed-abort-claim"; + const { session, cleanup, aiService } = await createAgentSessionHarness({ workspaceId }); + const makeQueueClaim = (): ToolEndQueueClaim => { + const controller = new AbortController(); + return { + userAuthored: true, + admissionSignal: controller.signal, + commit: mock(() => true), + restoreCancellation: mock(() => undefined), + cancelAdmission: mock((reason: string) => controller.abort(reason)), + requeueAdmission: mock(() => false), + release: mock(() => undefined), + }; + }; + const interruptedQueueClaim = makeQueueClaim(); + const replacementQueueClaim = makeQueueClaim(); + const claimNextToolEndEntry = spyOn( + MessageQueue.prototype, + "claimNextToolEndEntry" + ).mockReturnValueOnce(interruptedQueueClaim); + interface TestQueuedClaim { + queueClaim: ToolEndQueueClaim; + source: "sdk"; + dispatchStarted: false; + dispatchDeferredByHardInterrupt: false; + admissionIrreversible: false; + persistedMessageIds: string[]; + retryAfterCancellation: false; + } + const internalSession = session as unknown as { + queuedToolEndClaim?: TestQueuedClaim; + setTurnPhase(phase: "preparing"): void; + claimQueuedToolEndMessage(source: "sdk" | "provider"): TestQueuedClaim | undefined; + }; + const replacementClaim: TestQueuedClaim = { + queueClaim: replacementQueueClaim, + source: "sdk", + dispatchStarted: false, + dispatchDeferredByHardInterrupt: false, + admissionIrreversible: false, + persistedMessageIds: [], + retryAfterCancellation: false, + }; + const stopStream = spyOn(aiService, "stopStream").mockImplementation(() => { + interruptedQueueClaim.cancelAdmission("old Stop completed"); + internalSession.queuedToolEndClaim = replacementClaim; + return Promise.resolve(Ok(undefined)); }); try { - expect(session.getQueueCutCutter()).toBeUndefined(); - - session.queueMessage( - "manual message", - { model: TEST_MODEL, agentId: "exec" }, - { synthetic: true } - ); - session.queueMessage( - "workspace-turn follow-up", - { model: TEST_MODEL, agentId: "exec", muxMetadata: WORKSPACE_TURN_CORRELATION }, - { synthetic: true } - ); + internalSession.setTurnPhase("preparing"); + const interruptedClaim = internalSession.claimQueuedToolEndMessage("sdk"); + expect(interruptedClaim).toBeDefined(); + expect((await session.interruptStream()).success).toBe(true); + expect(interruptedQueueClaim.admissionSignal.aborted).toBe(true); + expect(internalSession.queuedToolEndClaim).toBe(replacementClaim); + expect(replacementQueueClaim.admissionSignal.aborted).toBe(false); + } finally { + stopStream.mockRestore(); + claimNextToolEndEntry.mockRestore(); + session.dispose(); + await cleanup(); + } + }); - // Queued stage: the manual head entry is the candidate (no metadata). - const queued = session.getQueueCutCutter(); - expect(queued?.stage).toBe("queued"); - expect(queued?.muxMetadata).toBeUndefined(); + test("ignores a stale pre-stream abort before replacement state changes", async () => { + const workspaceId = "queue-dispatch-stale-abort-generation"; + const recordUserStoppedStream = mock(() => Promise.resolve()); + const workspaceGoalService = { + recordUserStoppedStream, + } as unknown as WorkspaceGoalService; + const { session, cleanup, aiEmitter, events } = await createAgentSessionHarness({ + workspaceId, + workspaceGoalService, + captureEvents: true, + }); + const updateStartupAutoRetryAbandonFromAbort = mock(() => Promise.resolve()); + const internalSession = session as unknown as { + preparingTurnGeneration: number; + setTurnPhase(phase: "preparing"): void; + updateStartupAutoRetryAbandonFromAbort: typeof updateStartupAutoRetryAbandonFromAbort; + }; + internalSession.updateStartupAutoRetryAbandonFromAbort = updateStartupAutoRetryAbandonFromAbort; - // Dispatch the manual entry: it becomes the engaged PREPARING cutter and - // keeps winning over the follow-up still queued behind it. - const sendMessage = spyOn(session, "sendMessage").mockResolvedValue(Ok(undefined)); - session.sendQueuedMessages(); - const engaged = session.getQueueCutCutter(); - expect(engaged?.stage).toBe("preparing"); - expect(engaged?.muxMetadata).toBeUndefined(); - sendMessage.mockRestore(); + try { + internalSession.preparingTurnGeneration = 2; + internalSession.setTurnPhase("preparing"); + aiEmitter.emit("stream-abort", streamAbortEvent(workspaceId, "user", 1)); + await Promise.resolve(); + + expect(recordUserStoppedStream).not.toHaveBeenCalled(); + expect(updateStartupAutoRetryAbandonFromAbort).not.toHaveBeenCalled(); + const abortEvent = events.find((event) => event.type === "stream-abort"); + expect(abortEvent?.type).toBe("stream-abort"); + if (abortEvent?.type === "stream-abort") { + expect(abortEvent.rendererCleanupOnly).toBe(true); + } } finally { session.dispose(); await cleanup(); } }); - test("getQueueCutCutter reports a no-metadata mid-dispatch entry over a queued follow-up", async () => { - const { session, cleanup } = await createAgentSessionHarness({ - workspaceId: "queue-cut-cutter-dispatching", + test("settles a wake before provider exclusion falls back to deletion", async () => { + const workspaceId = "queue-dispatch-exclusion-delete-settlement"; + const records = [ + { + processId: "proc", + wakeUpdatedAt: "2026-08-31T12:00:00.000Z:12", + kind: "match" as const, + displayName: "CI watcher", + filter: "READY", + filterExclude: false, + }, + ]; + let markSettlementStarted: () => void = () => undefined; + const settlementStarted = new Promise((resolve) => { + markSettlementStarted = resolve; + }); + let releaseSettlement: () => void = () => undefined; + const settlementRelease = new Promise((resolve) => { + releaseSettlement = resolve; + }); + const settleProviderExcludedWakeRecords = mock(async () => { + markSettlementStarted(); + await settlementRelease; + }); + const { session, cleanup, historyService } = await createAgentSessionHarness({ + workspaceId, + settleProviderExcludedWakeRecords, }); + const messageId = "excluded-wake"; + const appendResult = await historyService.appendToHistory( + workspaceId, + createMuxMessage(messageId, "user", "monitor wake", { + synthetic: true, + muxMetadata: { type: "bash-monitor-wake", records }, + }) + ); + expect(appendResult.success).toBe(true); + const markMessagesProviderExcluded = spyOn( + historyService, + "markMessagesProviderExcluded" + ).mockResolvedValueOnce(Err("injected marker failure")); + const deleteMessages = spyOn(historyService, "deleteMessages"); + const admissionController = new AbortController(); + admissionController.abort("user Stop"); + const queueClaim: ToolEndQueueClaim = { + userAuthored: false, + admissionSignal: admissionController.signal, + commit: mock(() => true), + restoreCancellation: mock(() => undefined), + cancelAdmission: mock(() => undefined), + requeueAdmission: mock(() => false), + release: mock(() => undefined), + }; + const activeClaim = { + queueClaim, + admissionIrreversible: true, + persistedMessageIds: [messageId], + providerExcludedWakeRecords: records, + }; + const internalSession = session as unknown as { + excludeCanceledIrreversibleClaim(claim: typeof activeClaim): Promise; + }; try { - session.queueMessage( - "workspace-turn follow-up", - { model: TEST_MODEL, agentId: "exec", muxMetadata: WORKSPACE_TURN_CORRELATION }, - { synthetic: true } - ); - // Force the dequeue-to-stream-start window with PREPARING already - // released (a background send can resolve before stream-start): the - // dispatched entry stays the engaged cutter. - const internal = session as unknown as { - dispatchingQueuedEntry: boolean; - dispatchingQueuedEntryMuxMetadata?: unknown; - }; - internal.dispatchingQueuedEntry = true; - internal.dispatchingQueuedEntryMuxMetadata = undefined; + const exclusion = internalSession.excludeCanceledIrreversibleClaim(activeClaim); + await settlementStarted; + expect(deleteMessages).not.toHaveBeenCalled(); + const durableHistory = await historyService.getHistoryFromLatestBoundary(workspaceId); + expect(durableHistory.success).toBe(true); + if (durableHistory.success) { + expect(durableHistory.data[0]?.metadata?.providerExcluded).toBe(true); + } - const cutter = session.getQueueCutCutter(); - expect(cutter?.stage).toBe("dispatching"); - expect(cutter?.muxMetadata).toBeUndefined(); + releaseSettlement(); + await exclusion; + + expect(settleProviderExcludedWakeRecords).toHaveBeenCalledWith(records); + expect(deleteMessages).toHaveBeenCalledWith(workspaceId, [messageId]); + const history = await historyService.getHistoryFromLatestBoundary(workspaceId); + expect(history.success).toBe(true); + if (history.success) expect(history.data).toEqual([]); } finally { + releaseSettlement(); + deleteMessages.mockRestore(); + markMessagesProviderExcluded.mockRestore(); session.dispose(); await cleanup(); } }); - test("getQueueCutCutter exposes the queued head's dispatch mode and correlation", async () => { - const { session, cleanup } = await createAgentSessionHarness({ - workspaceId: "queue-cut-cutter-queued", + test("keeps a stopped wake held when both exclusion writes fail", async () => { + const workspaceId = "queue-dispatch-exclusion-write-failure"; + const records = [ + { + processId: "proc", + wakeUpdatedAt: "2026-08-31T12:00:00.000Z:12", + kind: "match" as const, + displayName: "CI watcher", + filter: "READY", + filterExclude: false, + }, + ]; + const settleProviderExcludedWakeRecords = mock(() => Promise.resolve()); + const { session, cleanup, aiService, historyService } = await createAgentSessionHarness({ + workspaceId, + settleProviderExcludedWakeRecords, }); + const messageId = "unexcluded-wake"; + const appendResult = await historyService.appendToHistory( + workspaceId, + createMuxMessage(messageId, "user", "monitor wake", { + synthetic: true, + muxMetadata: { type: "bash-monitor-wake", records }, + }) + ); + expect(appendResult.success).toBe(true); + const markMessagesProviderExcluded = spyOn( + historyService, + "markMessagesProviderExcluded" + ).mockResolvedValueOnce(Err("injected marker failure")); + const addProviderExclusionTombstones = spyOn( + historyService, + "addProviderExclusionTombstones" + ).mockResolvedValueOnce(Err("injected tombstone failure")); + const stopStream = spyOn(aiService, "stopStream").mockResolvedValue(Ok(undefined)); + const admissionController = new AbortController(); + const release = mock(() => undefined); + const queueClaim: ToolEndQueueClaim = { + userAuthored: false, + admissionSignal: admissionController.signal, + commit: mock(() => true), + restoreCancellation: mock(() => undefined), + cancelAdmission: mock((reason: string) => admissionController.abort(reason)), + requeueAdmission: mock(() => false), + release, + }; + const activeClaim = { + queueClaim, + source: "sdk" as const, + dispatchStarted: true, + dispatchDeferredByHardInterrupt: false, + admissionIrreversible: true, + persistedMessageIds: [messageId], + providerExcludedWakeRecords: records, + retryAfterCancellation: false, + admissionHold: undefined as { promise: Promise; release: () => void } | undefined, + }; + const internalSession = session as unknown as { + queuedToolEndClaim: typeof activeClaim; + }; + internalSession.queuedToolEndClaim = activeClaim; try { - session.queueMessage( - "workspace-turn follow-up", - { - model: TEST_MODEL, - agentId: "exec", - muxMetadata: WORKSPACE_TURN_CORRELATION, - queueDispatchMode: "turn-end", - }, - { synthetic: true } - ); + const result = await session.interruptStream({ deferQueueSettlement: true }); - const cutter = session.getQueueCutCutter(); - expect(cutter?.stage).toBe("queued"); - expect(cutter?.stage === "queued" ? cutter.dispatchMode : undefined).toBe("turn-end"); - expect((cutter?.muxMetadata as MuxMessageMetadata | undefined)?.type).toBe( - "workspace-turn-task" - ); - - // Once dispatched, the follow-up's correlation rides through PREPARING. - const sendMessage = spyOn(session, "sendMessage").mockResolvedValue(Ok(undefined)); - session.sendQueuedMessages(); - const engaged = session.getQueueCutCutter(); - expect(engaged?.stage).toBe("preparing"); - expect((engaged?.muxMetadata as MuxMessageMetadata | undefined)?.type).toBe( - "workspace-turn-task" - ); - sendMessage.mockRestore(); + expect(result.success).toBe(false); + expect(settleProviderExcludedWakeRecords).not.toHaveBeenCalled(); + expect(release).not.toHaveBeenCalled(); + expect(activeClaim.admissionHold).toBeDefined(); + const history = await historyService.getHistoryFromLatestBoundary(workspaceId); + expect(history.success).toBe(true); + if (history.success) { + expect(history.data[0]?.metadata?.providerExcluded).not.toBe(true); + } } finally { + activeClaim.admissionHold?.release(); + stopStream.mockRestore(); + addProviderExclusionTombstones.mockRestore(); + markMessagesProviderExcluded.mockRestore(); session.dispose(); await cleanup(); } }); - test("waits for stream-end instead of interrupting between sibling tool results", async () => { - const workspaceId = "queue-dispatch-full-step"; + test("restores the provider-tool queue claim when the soft stop fails", async () => { + const workspaceId = "queue-dispatch-provider-tool-stop-failure"; const { session, cleanup, aiEmitter, aiService } = await createAgentSessionHarness({ workspaceId, }); - const stopStream = spyOn(aiService, "stopStream").mockResolvedValue(Ok(undefined)); - const sendQueuedMessages = spyOn(session, "sendQueuedMessages").mockImplementation( - () => undefined + const commit = mock(() => true); + const restoreCancellation = mock(() => undefined); + const cancelAdmission = mock((_reason: string) => undefined); + const requeueAdmission = mock((_reason: string) => true); + const release = mock(() => undefined); + const admissionSignal = new AbortController().signal; + const claimNextToolEndEntry = spyOn( + MessageQueue.prototype, + "claimNextToolEndEntry" + ).mockReturnValue({ + userAuthored: true, + admissionSignal, + commit, + restoreCancellation, + cancelAdmission, + requeueAdmission, + release, + }); + const stopStream = spyOn(aiService, "stopStream").mockResolvedValue( + Err("injected provider-tool soft-stop failure") ); try { aiEmitter.emit("stream-start", streamStartEvent(workspaceId)); session.queueMessage("follow up", { model: TEST_MODEL, agentId: "exec" }); - aiEmitter.emit("tool-call-end", toolCallEndEvent(workspaceId)); aiEmitter.emit("tool-call-end", { ...toolCallEndEvent(workspaceId), - toolCallId: "tool-call-2", - }); - - expect(stopStream).not.toHaveBeenCalled(); - expect(sendQueuedMessages).not.toHaveBeenCalled(); - - aiEmitter.emit("stream-end", { - type: "stream-end", - workspaceId, - messageId: "assistant-1", - parts: [], - metadata: { - model: TEST_MODEL, - contextUsage: { inputTokens: 5, outputTokens: 5, totalTokens: 10 }, - providerMetadata: {}, - finishReason: "tool-calls", - }, + toolName: "web_search", + providerExecuted: true, }); - const didDispatch = await waitForCondition(() => sendQueuedMessages.mock.calls.length > 0); - expect(didDispatch).toBe(true); - expect(sendQueuedMessages).toHaveBeenCalledTimes(1); + expect(await waitForCondition(() => restoreCancellation.mock.calls.length === 1)).toBe(true); + expect(stopStream).toHaveBeenCalledTimes(1); } finally { - sendQueuedMessages.mockRestore(); stopStream.mockRestore(); + claimNextToolEndEntry.mockRestore(); session.dispose(); await cleanup(); } }); - test("soft-stops after a provider-executed tool result and dispatches after abort", async () => { - const workspaceId = "queue-dispatch-provider-tool"; - const { session, cleanup, aiEmitter, aiService } = await createAgentSessionHarness({ + test("emits the restored canceled state when a provider soft stop fails", async () => { + const workspaceId = "queue-dispatch-provider-stop-failure-snapshot"; + const { session, cleanup, aiEmitter, aiService, events } = await createAgentSessionHarness({ workspaceId, + captureEvents: true, + }); + let markStopStarted: () => void = () => undefined; + const stopStarted = new Promise((resolve) => { + markStopStarted = resolve; + }); + let releaseStop: () => void = () => undefined; + const stopRelease = new Promise((resolve) => { + releaseStop = resolve; + }); + const stopStream = spyOn(aiService, "stopStream").mockImplementationOnce(async () => { + markStopStarted(); + await stopRelease; + return Err("injected provider-tool soft-stop failure"); }); - const stopStream = spyOn(aiService, "stopStream").mockResolvedValue(Ok(undefined)); - const sendQueuedMessages = spyOn(session, "sendQueuedMessages").mockImplementation( - () => undefined - ); try { aiEmitter.emit("stream-start", streamStartEvent(workspaceId)); - session.queueMessage("follow up", { model: TEST_MODEL, agentId: "exec" }); - + const controller = new AbortController(); + session.queueMessage( + "Background monitor wake", + { model: TEST_MODEL, agentId: "exec", queueDispatchMode: "tool-end" }, + { synthetic: true, cancelSignal: controller.signal } + ); aiEmitter.emit("tool-call-end", { ...toolCallEndEvent(workspaceId), toolName: "web_search", providerExecuted: true, }); + await stopStarted; - expect(stopStream).toHaveBeenCalledWith(workspaceId, { - soft: true, - abortReason: "system", - }); + controller.abort("monitor wake became stale"); + releaseStop(); - aiEmitter.emit("stream-abort", streamAbortEvent(workspaceId, "system")); - const didDispatch = await waitForCondition(() => sendQueuedMessages.mock.calls.length > 0); - expect(didDispatch).toBe(true); - expect(sendQueuedMessages).toHaveBeenCalledTimes(1); + expect( + await waitForCondition(() => { + const queueEvents = events.filter((event) => event.type === "queued-message-changed"); + return queueEvents.at(-1)?.hasQueuedMessages === false; + }) + ).toBe(true); } finally { - sendQueuedMessages.mockRestore(); + releaseStop(); stopStream.mockRestore(); session.dispose(); await cleanup(); @@ -1083,6 +2511,24 @@ describe("AgentSession queued message tool-call dispatch", () => { workspaceId, }); const stopStream = spyOn(aiService, "stopStream").mockResolvedValue(Ok(undefined)); + const commit = mock(() => true); + const restoreCancellation = mock(() => undefined); + const cancelAdmission = mock((_reason: string) => undefined); + const requeueAdmission = mock((_reason: string) => true); + const release = mock(() => undefined); + const admissionSignal = new AbortController().signal; + const claimNextToolEndEntry = spyOn( + MessageQueue.prototype, + "claimNextToolEndEntry" + ).mockReturnValue({ + userAuthored: true, + admissionSignal, + commit, + restoreCancellation, + cancelAdmission, + requeueAdmission, + release, + }); const sendQueuedMessages = spyOn(session, "sendQueuedMessages").mockImplementation( () => undefined ); @@ -1099,6 +2545,7 @@ describe("AgentSession queued message tool-call dispatch", () => { const interruptResult = await session.interruptStream(); expect(interruptResult.success).toBe(true); + expect(cancelAdmission).toHaveBeenCalledTimes(1); // The native soft-stop can still win the event race after the hard user interrupt. aiEmitter.emit("stream-abort", streamAbortEvent(workspaceId, "system")); @@ -1106,6 +2553,7 @@ describe("AgentSession queued message tool-call dispatch", () => { expect(sendQueuedMessages).not.toHaveBeenCalled(); } finally { sendQueuedMessages.mockRestore(); + claimNextToolEndEntry.mockRestore(); stopStream.mockRestore(); session.dispose(); await cleanup(); diff --git a/src/node/services/agentSession.testHarness.ts b/src/node/services/agentSession.testHarness.ts index fac756ca28..3ca97efa9e 100644 --- a/src/node/services/agentSession.testHarness.ts +++ b/src/node/services/agentSession.testHarness.ts @@ -2,6 +2,7 @@ import { mock } from "bun:test"; import { EventEmitter } from "events"; import type { WorkspaceChatMessage } from "@/common/orpc/types"; +import type { BashMonitorWakeDisplayRecord } from "@/common/types/message"; import { Err, Ok } from "@/common/types/result"; import type { Config } from "@/node/config"; import type { TurnStreamHandle } from "@/node/services/streamManager"; @@ -110,6 +111,10 @@ export interface AgentSessionHarnessOptions { workspaceGoalService?: WorkspaceGoalService; mcpServerManager?: MCPServerManager; onCompactionComplete?: (metadata: CompactionCompletionMetadata) => void; + onProviderExcludedHistoryChange?: () => void; + settleProviderExcludedWakeRecords?: ( + records: readonly BashMonitorWakeDisplayRecord[] + ) => Promise; captureEvents?: boolean; } @@ -154,6 +159,8 @@ export async function createAgentSessionHarness( workspaceGoalService: options.workspaceGoalService, backgroundProcessManager, onCompactionComplete: options.onCompactionComplete, + onProviderExcludedHistoryChange: options.onProviderExcludedHistoryChange, + settleProviderExcludedWakeRecords: options.settleProviderExcludedWakeRecords, }); const events: WorkspaceChatMessage[] = []; diff --git a/src/node/services/agentSession.ts b/src/node/services/agentSession.ts index 7c360101ac..fc6d01d0ee 100644 --- a/src/node/services/agentSession.ts +++ b/src/node/services/agentSession.ts @@ -89,6 +89,8 @@ import { dedupeAgentSkillRefs, dedupeMcpPromptRefs, filterOrphanedMcpPromptSnapshots, + filterProviderExcludedMessages, + isProviderExcludedMessage, sanitizeAgentSkillRefs, sanitizeMcpPromptRefs, isCompactionSummaryMetadata, @@ -98,6 +100,7 @@ import { type AgentSkillReference, isSyntheticSnapshotUserMessage, type CompactionFollowUpRequest, + type BashMonitorWakeDisplayRecord, type MuxMessageMetadata, type MuxFilePart, type MuxMessage, @@ -114,7 +117,7 @@ import { createRuntimeForWorkspace, } from "@/node/runtime/runtimeHelpers"; import { MessageQueue } from "./messageQueue"; -import type { QueueCutCutter } from "./messageQueue"; +import type { QueueAdmissionCancelState, QueueCutCutter, ToolEndQueueClaim } from "./messageQueue"; import { copyStreamLifecycleSnapshot, type RuntimeStatusEvent, @@ -539,7 +542,12 @@ interface AgentSessionActiveStreamInfo { export interface AgentSessionStreamManager { stopStream( workspaceId: string, - options?: { soft?: boolean; abandonPartial?: boolean; abortReason?: StreamAbortReason } + options?: { + soft?: boolean; + abandonPartial?: boolean; + abortReason?: StreamAbortReason; + abortTurnGeneration?: number; + } ): Promise>; isStreaming(workspaceId: string): boolean; getStreamInfo(workspaceId: string): AgentSessionActiveStreamInfo | undefined; @@ -615,6 +623,12 @@ interface AgentSessionOptions { onIdleCompactionOutcome?: (success: boolean) => void; /** Called when post-compaction context state may have changed (plan/file edits) */ onPostCompactionStateChange?: () => void; + /** Called after a canceled synthetic turn changes its durable provider exclusion. */ + onProviderExcludedHistoryChange?: () => void; + /** Persists a stopped wake settlement before provider exclusion falls back to deletion. */ + settleProviderExcludedWakeRecords?: ( + records: readonly BashMonitorWakeDisplayRecord[] + ) => Promise; /** * Codex P1 (PRRT_kwDOPxxmWM6cRJD-): true while a service-level send is in * its preflight (counted in WorkspaceService.preflightSendCounts but not @@ -649,6 +663,21 @@ interface CachedMemoryContext { includesHotMemories: boolean; } +interface QueuedToolEndClaim { + queueClaim: ToolEndQueueClaim; + source: "sdk" | "provider"; + dispatchStarted: boolean; + dispatchDeferredByHardInterrupt: boolean; + admissionIrreversible: boolean; + persistedMessageIds: string[]; + providerExcludedWakeRecords: readonly BashMonitorWakeDisplayRecord[]; + retryAfterCancellation: boolean; + admissionHold?: { + promise: Promise; + release: () => void; + }; +} + export class AgentSession { private readonly workspaceId: string; private readonly config: Config; @@ -663,6 +692,8 @@ export class AgentSession { private readonly keepBackgroundProcesses: boolean; private readonly sanitizeCliWorkspaceRegistration?: AgentSessionOptions["sanitizeCliWorkspaceRegistration"]; private readonly onPostCompactionStateChange?: () => void; + private readonly onProviderExcludedHistoryChange?: () => void; + private readonly settleProviderExcludedWakeRecords?: AgentSessionOptions["settleProviderExcludedWakeRecords"]; private readonly hasExternalSendPreflight?: () => boolean; private readonly emitter = new EventEmitter(); private readonly aiListeners: Array<{ event: string; handler: (...args: unknown[]) => void }> = @@ -671,6 +702,7 @@ export class AgentSession { []; private disposed = false; private turnPhase: TurnPhase = TurnPhase.IDLE; + private preparingTurnGeneration = 0; /** Edit-flow admission reservations currently holding busy-ness (see isBusy, r32). */ private editAdmissionDepth = 0; /** @@ -693,7 +725,15 @@ export class AgentSession { // Provider-executed tools (for example native web_search/web_fetch) complete inside one // provider response, so the SDK's between-step stopWhen hook cannot preempt after them. // Track known siblings and reserve soft interruption for that native-only boundary. - private queuedProviderToolEndAbortInFlight = false; + private queuedToolEndClaim: QueuedToolEndClaim | undefined; + // Fail closed in this process if the durable provider-exclusion rewrite cannot complete. + private readonly providerExcludedHistoryMessageIds = new Set(); + // A hard Stop blocks late SDK and provider callbacks until a new stream starts. + private hardInterruptPendingStreamStart = false; + // Hold a claimed admission while a hard interrupt waits for its stop result. + private hardInterruptClaimSettlementPending = false; + // Send now preserves a claimed user entry for the immediate queue drain. + private preserveQueuedToolEndClaimForImmediateSend = false; private readonly activeToolCallIds = new Set(); private idleWaiters: Array<() => void> = []; @@ -902,6 +942,8 @@ export class AgentSession { onCompactionComplete, onIdleCompactionOutcome, onPostCompactionStateChange, + onProviderExcludedHistoryChange, + settleProviderExcludedWakeRecords, hasExternalSendPreflight, } = options; @@ -930,6 +972,8 @@ export class AgentSession { this.keepBackgroundProcesses = keepBackgroundProcesses ?? false; this.sanitizeCliWorkspaceRegistration = sanitizeCliWorkspaceRegistration; this.onPostCompactionStateChange = onPostCompactionStateChange; + this.onProviderExcludedHistoryChange = onProviderExcludedHistoryChange; + this.settleProviderExcludedWakeRecords = settleProviderExcludedWakeRecords; this.hasExternalSendPreflight = hasExternalSendPreflight; this.compactionHandler = new CompactionHandler({ @@ -1677,6 +1721,9 @@ export class AgentSession { if (candidate.role === "system") { continue; } + if (isProviderExcludedMessage(candidate)) { + continue; + } if (this.isSyntheticGoalPauseBoundaryMessage(candidate)) { continue; } @@ -1893,6 +1940,9 @@ export class AgentSession { if (message.role !== "user") { return false; } + if (isProviderExcludedMessage(message)) { + return false; + } if (this.isVisibleCompletedSubagentReportMessage(message)) { return false; @@ -2876,7 +2926,7 @@ export class AgentSession { message: { type: "queued-message-changed", workspaceId: this.workspaceId, - hasQueuedMessages: !this.messageQueue.isEmpty(), + hasQueuedMessages: this.messageQueue.hasLiveEntries(), queuedMessages: this.messageQueue.getVisibleMessages(), displayText: this.messageQueue.getVisibleDisplayText(), fileParts: this.messageQueue.getVisibleFileParts(), @@ -3024,7 +3074,7 @@ export class AgentSession { onAccepted?: () => Promise | void; onAcceptedPreStreamFailure?: (error: SendMessageError) => Promise | void; onCanceled?: (reason: string) => Promise | void; - cancelState?: { canceledBeforeAcceptance: boolean }; + cancelState?: QueueAdmissionCancelState; cancelSignal?: AbortSignal; /** * For queue-dispatched sends: when the user last added to the queued @@ -3083,6 +3133,12 @@ export class AgentSession { * these admission gates instead of starting a privileged turn on a stopped target. */ admissionStale?: () => boolean; + /** Wait for a hard-stop cascade before this queued admission can become irreversible. */ + waitForAdmissionRelease?: () => Promise; + /** Record that persisted turn rows have crossed their rollback boundary. */ + onAdmissionIrreversible?: (persistedMessageIds: readonly string[]) => void; + /** Exclude a claimed synthetic turn when a hard Stop cancels it after the boundary. */ + excludeIrreversibleAdmissionOnCancel?: boolean; } ): Promise> { this.assertNotDisposed("sendMessage"); @@ -3940,11 +3996,17 @@ export class AgentSession { ); } + await internal?.waitForAdmissionRelease?.(); + if (await cancelBeforeAcceptance()) { + return Ok(undefined); + } + // Goal synchronization can mutate goal.json based on this durable user row. Once it begins, the // turn has crossed the cancellation point-of-no-return: a concurrent monitor stop must let this // wake finish acceptance rather than delete the row after goal state has already observed it. if (cancelSignal != null) { cancellationDisabled = true; + internal?.onAdmissionIrreversible?.(persistedCancelableMessageIds); } // r54: the pre-turn batch is now irrevocable — rollbackPersistedTurnRows // is never invoked past this point, so even a failure in goal sync or @@ -4050,6 +4112,20 @@ export class AgentSession { return Err(createUnknownSendMessageError(getErrorMessage(error))); } + const isIrreversibleAdmissionCanceled = (): boolean => + cancellationDisabled && + cancelSignal?.aborted === true && + internal?.excludeIrreversibleAdmissionOnCancel === true; + const markProviderExcludedAfterAcceptance = (): void => { + if (internal?.cancelState != null) { + internal.cancelState.providerExcludedAfterAcceptance = true; + } + }; + if (isIrreversibleAdmissionCanceled()) { + markProviderExcludedAfterAcceptance(); + return Ok(undefined); + } + let acceptedPreStreamFailureNotified = false; const notifyAcceptedPreStreamFailure = async (error: SendMessageError): Promise => { if (acceptedPreStreamFailureNotified) { @@ -4086,7 +4162,11 @@ export class AgentSession { const preparedTurnAbortController = new AbortController(); this.activePreparedTurnAbortController = preparedTurnAbortController; this.preparingWorkspaceTurnMetadata = getWorkspaceTurnMuxMetadata(optionsForStream.muxMetadata); - this.setTurnPhase(TurnPhase.PREPARING); + if (this.dispatchingQueuedEntry && this.turnPhase === TurnPhase.PREPARING) { + this.setTurnPhase(TurnPhase.PREPARING); + } else { + this.beginPreparingTurn(); + } // From this synchronous point isBusy() reports the turn — release the // service-side preflight reservation (see onTurnAdmissionCommitted doc). internal?.onTurnAdmissionCommitted?.(); @@ -4122,9 +4202,17 @@ export class AgentSession { preparedTurnAbortController.signal, goalKind, internal?.goalId, - turnThinkingOverride + turnThinkingOverride, + async () => { + await internal?.waitForAdmissionRelease?.(); + return !isIrreversibleAdmissionCanceled(); + } ); - if (streamResult.success && preparedTurnAbortController.signal.aborted) { + if ( + streamResult.success && + preparedTurnAbortController.signal.aborted && + !isIrreversibleAdmissionCanceled() + ) { await notifyAcceptedPreStreamFailure( createUnknownSendMessageError( "Accepted stream startup was canceled during preparation." @@ -4186,7 +4274,12 @@ export class AgentSession { // Non-edit sends preserve the old behavior so pre-stream startup failures still propagate to // synchronous callers (draft restore, interrupted-task rollback, etc.). - return await startPreparedStream(); + const streamResult = await startPreparedStream(); + if (isIrreversibleAdmissionCanceled()) { + markProviderExcludedAfterAcceptance(); + return Ok(undefined); + } + return streamResult; } async resumeStream( @@ -4239,7 +4332,7 @@ export class AgentSession { internal?.goalId ); this.preparingWorkspaceTurnMetadata = getWorkspaceTurnMuxMetadata(optionsForStream.muxMetadata); - this.setTurnPhase(TurnPhase.PREPARING); + this.beginPreparingTurn(); // Open the mid-turn thinking override window for the resumed turn (after // setTurnPhase(PREPARING), which clears the holder on the IDLE transition). const turnThinkingOverride: ActiveTurnThinkingOverride = {}; @@ -4863,14 +4956,24 @@ export class AgentSession { async interruptStream(options?: { soft?: boolean; abandonPartial?: boolean; + sendQueuedImmediately?: boolean; + /** Keep queue drains blocked until WorkspaceService completes descendant cleanup. */ + deferQueueSettlement?: boolean; }): Promise> { this.assertNotDisposed("interruptStream"); // Explicit user interruption should immediately stop any pending auto-retry loop. this.retryManager.cancel(); - if (options?.soft !== true) { - this.queuedProviderToolEndAbortInFlight = false; + const isHardInterrupt = options?.soft !== true; + // Bind Stop to the current claim before partial cleanup or abort delivery can yield. + const interruptedClaim = isHardInterrupt ? this.queuedToolEndClaim : undefined; + const interruptedTurnGeneration = this.preparingTurnGeneration; + if (isHardInterrupt) { + this.hardInterruptPendingStreamStart = true; + this.hardInterruptClaimSettlementPending = true; + this.preserveQueuedToolEndClaimForImmediateSend = options?.sendQueuedImmediately === true; + this.pauseQueuedToolEndClaimAdmission(); this.activeToolCallIds.clear(); } @@ -4878,20 +4981,66 @@ export class AgentSession { // from committing it. For soft interrupts, defer to stream-abort handler since // the stream continues running and would recreate the partial. if (options?.abandonPartial && !options?.soft) { - const deleteResult = await this.historyService.deletePartial(this.workspaceId); + let deleteResult: Result; + try { + deleteResult = await this.historyService.deletePartial(this.workspaceId); + } catch (error: unknown) { + this.resumeQueuedToolEndClaimAfterFailedInterrupt(); + throw error; + } if (!deleteResult.success) { + this.resumeQueuedToolEndClaimAfterFailedInterrupt(); return Err(deleteResult.error); } } - const stopResult = await this.streamManager.stopStream(this.workspaceId, { - ...options, - abortReason: "user", - }); + let stopResult: Result; + try { + stopResult = await this.streamManager.stopStream(this.workspaceId, { + ...options, + abortReason: "user", + ...(isHardInterrupt ? { abortTurnGeneration: interruptedTurnGeneration } : {}), + }); + } catch (error: unknown) { + if (isHardInterrupt) { + this.resumeQueuedToolEndClaimAfterFailedInterrupt(); + } + throw error; + } if (!stopResult.success) { + if (isHardInterrupt) { + this.resumeQueuedToolEndClaimAfterFailedInterrupt(); + } return Err(stopResult.error); } + if (isHardInterrupt) { + this.hardInterruptClaimSettlementPending = false; + this.settleQueuedToolEndClaimAfterUserInterrupt(interruptedClaim); + if ( + interruptedClaim?.admissionIrreversible === true && + !interruptedClaim.queueClaim.userAuthored + ) { + this.activePreparedTurnAbortController?.abort(); + } + // A synthetic acceptance callback can already be running and cannot wait on the new hold. + // Keep its completion from draining later entries before the workspace-level policy runs. + this.hardInterruptClaimSettlementPending = options?.deferQueueSettlement === true; + const exclusionResult = await this.excludeCanceledIrreversibleClaim(interruptedClaim); + if (!exclusionResult.success) { + this.preserveQueuedToolEndClaimForImmediateSend = false; + return exclusionResult; + } + if ( + options?.deferQueueSettlement !== true && + interruptedClaim?.admissionIrreversible === true && + !interruptedClaim.queueClaim.userAuthored + ) { + this.resumeQueuedToolEndClaimAdmission(interruptedClaim); + } + this.preserveQueuedToolEndClaimForImmediateSend = false; + } + return Ok(undefined); } @@ -4977,7 +5126,9 @@ export class AgentSession { // Session-owned per-turn holder for mid-turn thinking changes. Passed // explicitly (not read from the field) so a preempted turn can never pick // up its replacement's holder. Absent for internal retry paths. - activeTurnThinkingOverride?: ActiveTurnThinkingOverride + activeTurnThinkingOverride?: ActiveTurnThinkingOverride, + /** Wait for a claimed turn to retain provider-start permission. */ + waitForProviderStartAdmission?: () => Promise ): Promise> { const isStartupAbortRequested = (): boolean => abortSignal?.aborted === true; @@ -5050,9 +5201,15 @@ export class AgentSession { ); } - // A crash between snapshot and user-row appends can leave orphaned prompt - // expansions on disk; exclude them from every provider request. - let requestMessages = filterOrphanedMcpPromptSnapshots(historyResult.data); + // A crash between snapshot and user-row appends can leave orphaned prompt expansions on disk. + // A failed exclusion rewrite also stays blocked in memory for this process. + const filterRequestMessages = (messages: MuxMessage[]): MuxMessage[] => + filterOrphanedMcpPromptSnapshots( + filterProviderExcludedMessages(messages).filter( + (message) => !this.providerExcludedHistoryMessageIds.has(message.id) + ) + ); + let requestMessages = filterRequestMessages(historyResult.data); if (requestMessages.length === 0) { return await this.handleStreamWithHistoryFailure( @@ -5080,7 +5237,7 @@ export class AgentSession { await this.historyService.appendToHistory(this.workspaceId, sentinelMessage); const refreshed = await this.historyService.getHistoryFromLatestBoundary(this.workspaceId); if (refreshed.success) { - requestMessages = filterOrphanedMcpPromptSnapshots(refreshed.data); + requestMessages = filterRequestMessages(refreshed.data); } } @@ -5179,6 +5336,13 @@ export class AgentSession { // emit an error event for fire-and-forget senders and then return Err; // collect them so the Err path resolves each exactly once. const preStartErrors: StreamErrorPayload[] = []; + const providerStartAdmitted = await waitForProviderStartAdmission?.(); + if (isStartupAbortRequested() || providerStartAdmitted === false) { + this.activeCompactionRequest = undefined; + this.resetActiveStreamState(); + return Ok(undefined); + } + const streamResult = await this.aiService.streamMessage({ messages: requestMessages, workspaceId: this.workspaceId, @@ -5210,7 +5374,7 @@ export class AgentSession { experiments: options?.experiments, disableWorkspaceAgents: options?.disableWorkspaceAgents, strictAgentResolution: options?.strictAgentResolution, - hasQueuedMessages: this.hasQueuedMessages.bind(this), + claimQueuedToolEndMessage: () => this.claimQueuedToolEndMessage("sdk") != null, openaiTruncationModeOverride, // Mid-turn thinking overrides clamp against the same floor as the // send-time level above (single source of truth for the floor). @@ -5462,7 +5626,7 @@ export class AgentSession { this.preparingWorkspaceTurnMetadata = getWorkspaceTurnMuxMetadata( retryOptionsForResume.muxMetadata ); - this.setTurnPhase(TurnPhase.PREPARING); + this.beginPreparingTurn(); let retryResult: Result; try { retryResult = await this.streamWithHistory( @@ -5569,7 +5733,7 @@ export class AgentSession { // Retry the same request, but without post-compaction injection. this.preparingWorkspaceTurnMetadata = getWorkspaceTurnMuxMetadata(context.options?.muxMetadata); - this.setTurnPhase(TurnPhase.PREPARING); + this.beginPreparingTurn(); let retryResult: Result; try { retryResult = await this.streamWithHistory( @@ -5718,7 +5882,7 @@ export class AgentSession { private async handleStreamError(data: StreamErrorPayload): Promise { this.setTurnPhase(TurnPhase.COMPLETING); - this.queuedProviderToolEndAbortInFlight = false; + this.restoreQueuedToolEndClaim(); this.clearLiveUsageState(); const hadCompactionRequest = this.activeCompactionRequest !== undefined; if ( @@ -5787,6 +5951,8 @@ export class AgentSession { forward("stream-start", (payload) => { if (payload.type === "stream-start") { + this.hardInterruptPendingStreamStart = false; + this.preserveQueuedToolEndClaimForImmediateSend = false; this.dispatchingQueuedEntry = false; this.dispatchingQueuedEntryMuxMetadata = undefined; this.preparingWorkspaceTurnMetadata = undefined; @@ -5796,7 +5962,7 @@ export class AgentSession { // fast-path synchronously so a model set_goal in THIS stream queues // for its stream-end drain instead of writing goal.json mid-stream. this.workspaceGoalService?.recordStreamStarted(this.workspaceId); - this.queuedProviderToolEndAbortInFlight = false; + this.restoreQueuedToolEndClaim(); this.activeToolCallIds.clear(); } this.setTurnPhase(TurnPhase.STREAMING); @@ -5928,6 +6094,25 @@ export class AgentSession { this.emitChatEvent(payload); return; } + const abortReason = "abortReason" in payload ? payload.abortReason : undefined; + const abortTurnGeneration = payload.metadata?.abortTurnGeneration; + if ( + abortReason === "user" && + abortTurnGeneration != null && + abortTurnGeneration !== this.preparingTurnGeneration + ) { + log.debug("Ignoring a stale user abort", { + workspaceId: this.workspaceId, + abortTurnGeneration, + preparingTurnGeneration: this.preparingTurnGeneration, + }); + // The renderer still needs the old terminal event to clear that exact stream. + this.emitChatEvent({ + ...payload, + rendererCleanupOnly: true, + }); + return; + } // stopStream() emits synthetic aborts even when no real stream is active // (e.g., during PREPARING or after COMPLETING). We must still forward the @@ -5942,7 +6127,7 @@ export class AgentSession { turnPhase: this.turnPhase, }); - const preStreamAbortReason = "abortReason" in payload ? payload.abortReason : undefined; + const preStreamAbortReason = abortReason; if (this.turnPhase === TurnPhase.PREPARING) { this.clearPreparingRuntimeStatus(); this.setTerminalStreamLifecycle("interrupted", { @@ -5958,7 +6143,11 @@ export class AgentSession { this.activeStreamUserMessageId ); - this.queuedProviderToolEndAbortInFlight = false; + // interruptStream owns user claim settlement. StreamManager can deliver this event + // after a replacement admission starts, so the event must not inspect queue state. + if (preStreamAbortReason !== "user") { + this.restoreQueuedToolEndClaim(); + } this.activeToolCallIds.clear(); this.emitChatEvent(payload); return; @@ -5979,9 +6168,8 @@ export class AgentSession { const failedUserMessageId = this.activeStreamUserMessageId; const hadCompactionRequest = this.activeCompactionRequest !== undefined; - const abortReason = "abortReason" in payload ? payload.abortReason : undefined; const isQueuedProviderToolEndAbort = - this.queuedProviderToolEndAbortInFlight && abortReason !== "user"; + this.queuedToolEndClaim?.source === "provider" && abortReason !== "user"; if (abortReason === "user") { await this.workspaceGoalService?.recordUserStoppedStream(this.workspaceId); } @@ -6136,7 +6324,7 @@ export class AgentSession { // P2: if an edit is waiting, skip the queue flush so the edit truncates first. const hadQueuedMessages = this.hasPendingManualFollowUp(); if (this.deferQueuedFlushUntilAfterEdit) { - this.queuedProviderToolEndAbortInFlight = false; + this.restoreQueuedToolEndClaim(); // Clear the queued-message signal while the edit flow owns the next dispatch. this.backgroundProcessManager.setMessageQueued(this.workspaceId, false); // Do not dispatch stream-end follow-ups while the edit flow is waiting @@ -6273,6 +6461,11 @@ export class AgentSession { } satisfies AgentSessionChatEvent); } + private beginPreparingTurn(): void { + this.preparingTurnGeneration += 1; + this.setTurnPhase(TurnPhase.PREPARING); + } + private setTurnPhase(next: TurnPhase): void { this.turnPhase = next; this.clearPreparingRuntimeStatus(); @@ -6505,7 +6698,7 @@ export class AgentSession { onAccepted?: () => Promise | void; onAcceptedPreStreamFailure?: (error: SendMessageError) => Promise | void; onCanceled?: (reason: string) => Promise | void; - cancelState?: { canceledBeforeAcceptance: boolean }; + cancelState?: QueueAdmissionCancelState; cancelSignal?: AbortSignal; /** Synthetic assistant rows persisted just before the dispatched turn's user row. */ preTurnMessages?: MuxMessage[]; @@ -6523,15 +6716,33 @@ export class AgentSession { if (!didEnqueue) { return null; } + if (internal?.cancelSignal != null) { + internal.cancelSignal.addEventListener( + "abort", + () => { + if (this.disposed) { + return; + } + this.emitQueuedMessageChanged(); + this.syncBackgroundQueuedMessageSignal(); + }, + { once: true } + ); + } this.emitQueuedMessageChanged(); // Signal to bash_output that it should return early to process queued messages // only for tool-end dispatches. const effectiveDispatchMode = this.messageQueue.getNextQueueDispatchMode(); + this.syncBackgroundQueuedMessageSignal(); + return effectiveDispatchMode; + } + + private syncBackgroundQueuedMessageSignal(): void { this.backgroundProcessManager.setMessageQueued( this.workspaceId, - effectiveDispatchMode === "tool-end" + this.messageQueue.hasLiveEntries() && + this.messageQueue.getNextQueueDispatchMode() === "tool-end" ); - return effectiveDispatchMode; } clearQueue(cancelReason = "Queued message cleared before dispatch."): void { @@ -6557,7 +6768,8 @@ export class AgentSession { // user-authored turn-end entry forward to a step boundary. this.backgroundProcessManager.setMessageQueued( this.workspaceId, - !this.messageQueue.isEmpty() && this.messageQueue.getNextQueueDispatchMode() === "tool-end" + this.messageQueue.hasLiveEntries() && + this.messageQueue.getNextQueueDispatchMode() === "tool-end" ); return true; } @@ -6594,7 +6806,8 @@ export class AgentSession { this.emitQueuedMessageChanged(); this.backgroundProcessManager.setMessageQueued( this.workspaceId, - !this.messageQueue.isEmpty() && this.messageQueue.getNextQueueDispatchMode() === "tool-end" + this.messageQueue.hasLiveEntries() && + this.messageQueue.getNextQueueDispatchMode() === "tool-end" ); for (const callbacks of removal.callbacks) { this.notifyQueuedMessageCleared(callbacks, cancelReason); @@ -6622,7 +6835,8 @@ export class AgentSession { this.emitQueuedMessageChanged(); this.backgroundProcessManager.setMessageQueued( this.workspaceId, - !this.messageQueue.isEmpty() && this.messageQueue.getNextQueueDispatchMode() === "tool-end" + this.messageQueue.hasLiveEntries() && + this.messageQueue.getNextQueueDispatchMode() === "tool-end" ); this.notifyQueuedMessageCleared(callbacks, cancelReason); return true; @@ -6630,7 +6844,7 @@ export class AgentSession { hasQueuedMessages(dispatchMode?: "tool-end" | "turn-end"): boolean { return ( - !this.messageQueue.isEmpty() && + this.messageQueue.hasLiveEntries() && (dispatchMode == null || this.messageQueue.getNextQueueDispatchMode() === dispatchMode) ); } @@ -6663,7 +6877,7 @@ export class AgentSession { } } - if (!this.messageQueue.isEmpty()) { + if (this.messageQueue.hasLiveEntries()) { if (continuationMetadata == null) { return true; } @@ -6773,20 +6987,25 @@ export class AgentSession { private async requestQueuedProviderToolEndDispatch(): Promise { if ( this.turnPhase !== TurnPhase.STREAMING || - this.queuedProviderToolEndAbortInFlight || - this.activeToolCallIds.size > 0 || - !this.hasQueuedMessages("tool-end") + this.queuedToolEndClaim != null || + this.activeToolCallIds.size > 0 ) { return; } - this.queuedProviderToolEndAbortInFlight = true; + const activeClaim = this.claimQueuedToolEndMessage("provider"); + if (activeClaim == null) { + return; + } + const result = await this.streamManager.stopStream(this.workspaceId, { soft: true, abortReason: "system", }); if (!result.success) { - this.queuedProviderToolEndAbortInFlight = false; + if (this.queuedToolEndClaim === activeClaim) { + this.restoreQueuedToolEndClaim(); + } log.warn("Failed to stop stream after provider-executed tool result", { workspaceId: this.workspaceId, error: result.error, @@ -6797,15 +7016,29 @@ export class AgentSession { private dispatchQueuedProviderToolEndMessageAfterAbort( abortReason: StreamAbortReason | undefined ): boolean { - if (!this.queuedProviderToolEndAbortInFlight) { + if (this.hardInterruptClaimSettlementPending) { + if (abortReason !== "user" && this.queuedToolEndClaim?.source === "provider") { + this.queuedToolEndClaim.dispatchDeferredByHardInterrupt = true; + } + return false; + } + if (this.queuedToolEndClaim == null) { + return false; + } + if (this.queuedToolEndClaim.source !== "provider") { + this.restoreQueuedToolEndClaim(); return false; } const shouldDispatch = abortReason !== "user" && !this.deferQueuedFlushUntilAfterEdit && this.hasQueuedMessages(); - this.queuedProviderToolEndAbortInFlight = false; if (!shouldDispatch) { + if (abortReason === "user") { + this.settleQueuedToolEndClaimAfterUserInterrupt(); + } else { + this.restoreQueuedToolEndClaim(); + } return false; } @@ -6813,6 +7046,244 @@ export class AgentSession { return true; } + private claimQueuedToolEndMessage(source: "sdk" | "provider"): QueuedToolEndClaim | undefined { + if (this.hardInterruptPendingStreamStart || this.queuedToolEndClaim != null) { + return undefined; + } + const queueClaim = this.messageQueue.claimNextToolEndEntry(); + if (queueClaim == null) { + return undefined; + } + const queueMuxMetadata = queueClaim.muxMetadata as MuxMessageMetadata | undefined; + const activeClaim = { + queueClaim, + source, + dispatchStarted: false, + dispatchDeferredByHardInterrupt: false, + admissionIrreversible: false, + persistedMessageIds: [], + providerExcludedWakeRecords: + queueMuxMetadata?.type === "bash-monitor-wake" ? queueMuxMetadata.records : [], + retryAfterCancellation: false, + }; + this.queuedToolEndClaim = activeClaim; + return activeClaim; + } + + private commitQueuedToolEndClaim(): QueuedToolEndClaim | undefined { + const activeClaim = this.queuedToolEndClaim; + if (activeClaim == null || activeClaim.dispatchStarted) { + return undefined; + } + if (!activeClaim.queueClaim.commit()) { + activeClaim.queueClaim.release(); + this.queuedToolEndClaim = undefined; + this.syncBackgroundQueuedMessageSignal(); + return undefined; + } + activeClaim.dispatchStarted = true; + return activeClaim; + } + + private restoreQueuedToolEndClaim(): void { + const activeClaim = this.queuedToolEndClaim; + this.queuedToolEndClaim = undefined; + this.resumeQueuedToolEndClaimAdmission(activeClaim); + activeClaim?.queueClaim.restoreCancellation(); + if (activeClaim != null) { + this.emitQueuedMessageChanged(); + } + this.syncBackgroundQueuedMessageSignal(); + } + + private cancelQueuedToolEndClaim(reason: string): void { + const activeClaim = this.queuedToolEndClaim; + this.queuedToolEndClaim = undefined; + this.resumeQueuedToolEndClaimAdmission(activeClaim); + activeClaim?.queueClaim.cancelAdmission(reason); + if (activeClaim != null) { + this.emitQueuedMessageChanged(); + } + this.syncBackgroundQueuedMessageSignal(); + } + + private settleQueuedToolEndClaimAfterUserInterrupt(activeClaim = this.queuedToolEndClaim): void { + if (this.hardInterruptClaimSettlementPending) { + return; + } + if (activeClaim == null || this.queuedToolEndClaim !== activeClaim) { + return; + } + if (activeClaim?.admissionIrreversible) { + // A hard Stop must not start a synthetic provider turn. Keep its durable row for + // producer settlement, but abort admission so the request path excludes it. + if (!activeClaim.queueClaim.userAuthored) { + activeClaim.queueClaim.cancelAdmission("Queue dispatch canceled by user interrupt."); + } + return; + } + if ( + this.preserveQueuedToolEndClaimForImmediateSend && + activeClaim?.queueClaim.userAuthored === true + ) { + if (!activeClaim.dispatchStarted) { + this.restoreQueuedToolEndClaim(); + } + return; + } + if ( + activeClaim?.dispatchStarted === true && + activeClaim.queueClaim.userAuthored && + !activeClaim.admissionIrreversible + ) { + activeClaim.retryAfterCancellation = true; + const didRequeue = activeClaim.queueClaim.requeueAdmission( + "Queue dispatch canceled by user interrupt." + ); + if (didRequeue) { + this.queuedToolEndClaim = undefined; + this.resumeQueuedToolEndClaimAdmission(activeClaim); + this.emitQueuedMessageChanged(); + this.syncBackgroundQueuedMessageSignal(); + // A later service callback cannot recover a dequeued entry. Restore it now. + this.restoreQueueToInput(); + return; + } + activeClaim.retryAfterCancellation = false; + } + this.cancelQueuedToolEndClaim("Queue dispatch canceled by user interrupt."); + } + + private async excludeCanceledIrreversibleClaim( + activeClaim: QueuedToolEndClaim | undefined + ): Promise> { + if ( + activeClaim?.admissionIrreversible !== true || + activeClaim.queueClaim.userAuthored || + !activeClaim.queueClaim.admissionSignal.aborted + ) { + return Ok(undefined); + } + + const messageIds = activeClaim.persistedMessageIds; + for (const messageId of messageIds) { + this.providerExcludedHistoryMessageIds.add(messageId); + } + if (messageIds.length === 0) { + return Ok(undefined); + } + + const exclusionResult = await this.historyService.markMessagesProviderExcluded( + this.workspaceId, + messageIds + ); + if (exclusionResult.success) { + this.onProviderExcludedHistoryChange?.(); + return Ok(undefined); + } + + // Persist the exclusion before producer settlement. A crash must never expose + // the synthetic row after its producer watermark says that the wake was consumed. + const tombstoneResult = await this.historyService.addProviderExclusionTombstones( + this.workspaceId, + messageIds + ); + if (!tombstoneResult.success) { + // Keep the admission hold closed. A later Stop retries both durable writes. + return Err( + `Cannot stop the synthetic wake safely: ${exclusionResult.error}; ${tombstoneResult.error}` + ); + } + this.onProviderExcludedHistoryChange?.(); + + const wakeRecords = activeClaim.providerExcludedWakeRecords; + if (wakeRecords.length > 0) { + if (this.settleProviderExcludedWakeRecords == null) { + log.error("Cannot delete a canceled wake without durable producer settlement", { + workspaceId: this.workspaceId, + messageIds, + exclusionError: exclusionResult.error, + }); + return Ok(undefined); + } + try { + // The tombstone keeps the recovery record excluded while this watermark advances. + await this.settleProviderExcludedWakeRecords(wakeRecords); + } catch (error) { + log.error("Failed to settle a canceled wake before provider-exclusion deletion", { + workspaceId: this.workspaceId, + messageIds, + exclusionError: exclusionResult.error, + error: getErrorMessage(error), + }); + return Ok(undefined); + } + } + + // Keep the hard Stop fail-closed. Deletion is a safe fallback when the marker rewrite fails. + const deleteResult = await this.historyService.deleteMessages(this.workspaceId, messageIds); + if (deleteResult.success) { + this.onProviderExcludedHistoryChange?.(); + } else { + log.error("Failed to persist provider exclusion for a canceled synthetic admission", { + workspaceId: this.workspaceId, + messageIds, + exclusionError: exclusionResult.error, + deleteError: deleteResult.error, + }); + } + return Ok(undefined); + } + + private pauseQueuedToolEndClaimAdmission(): void { + const activeClaim = this.queuedToolEndClaim; + if (activeClaim?.dispatchStarted !== true || activeClaim.admissionHold != null) { + return; + } + let release: () => void = () => undefined; + const promise = new Promise((resolve) => { + release = resolve; + }); + activeClaim.admissionHold = { promise, release }; + } + + private resumeQueuedToolEndClaimAdmission(activeClaim?: QueuedToolEndClaim): void { + const admissionHold = activeClaim?.admissionHold; + if (activeClaim != null) { + activeClaim.admissionHold = undefined; + } + admissionHold?.release(); + } + + private async waitForQueuedToolEndClaimAdmission(activeClaim: QueuedToolEndClaim): Promise { + await activeClaim.admissionHold?.promise; + } + + private resumeQueuedToolEndClaimAfterFailedInterrupt(): void { + this.hardInterruptClaimSettlementPending = false; + this.hardInterruptPendingStreamStart = false; + this.preserveQueuedToolEndClaimForImmediateSend = false; + const activeClaim = this.queuedToolEndClaim; + this.resumeQueuedToolEndClaimAdmission(activeClaim); + if ( + activeClaim?.dispatchStarted !== true && + activeClaim?.dispatchDeferredByHardInterrupt !== true + ) { + this.restoreQueuedToolEndClaim(); + } + if (this.turnPhase === TurnPhase.IDLE) { + this.sendQueuedMessages(); + } + } + + private releaseQueuedToolEndClaim(activeClaim: QueuedToolEndClaim): void { + this.resumeQueuedToolEndClaimAdmission(activeClaim); + activeClaim.queueClaim.release(); + if (this.queuedToolEndClaim === activeClaim) { + this.queuedToolEndClaim = undefined; + } + } + async waitForPendingCompactionCompletionDecision(messageId: string): Promise { if (!this.compactionCompletionDecisions.has(messageId)) { if (this.activeCompactionRequest == null) return false; @@ -6846,7 +7317,7 @@ export class AgentSession { } hasPendingManualFollowUp(): boolean { - return !this.messageQueue.isEmpty() || this.pendingExternalManualFollowUps > 0; + return this.messageQueue.hasLiveEntries() || this.pendingExternalManualFollowUps > 0; } /** @@ -6856,6 +7327,12 @@ export class AgentSession { */ restoreQueueToInput(): void { this.assertNotDisposed("restoreQueueToInput"); + this.hardInterruptClaimSettlementPending = false; + const activeClaim = this.queuedToolEndClaim; + if (activeClaim?.admissionIrreversible === true && activeClaim.admissionHold != null) { + // WorkspaceService calls this after descendant cleanup, even when no queue entry remains. + this.resumeQueuedToolEndClaimAdmission(activeClaim); + } if (this.messageQueue.isEmpty()) { return; } @@ -6886,7 +7363,7 @@ export class AgentSession { this.emitChatEvent({ type: "queued-message-changed", workspaceId: this.workspaceId, - hasQueuedMessages: !this.messageQueue.isEmpty(), + hasQueuedMessages: this.messageQueue.hasLiveEntries(), queuedMessages: this.messageQueue.getVisibleMessages(), displayText: this.messageQueue.getVisibleDisplayText(), fileParts: this.messageQueue.getVisibleFileParts(), @@ -6902,7 +7379,20 @@ export class AgentSession { */ sendNextUserQueuedMessage(): boolean { this.assertNotDisposed("sendNextUserQueuedMessage"); - if (!this.messageQueue.prioritizeNextUserEntry()) { + this.hardInterruptClaimSettlementPending = false; + const activeClaim = this.queuedToolEndClaim; + const didPrioritizeUserEntry = this.messageQueue.prioritizeNextUserEntry(); + if ( + activeClaim?.dispatchStarted === true && + activeClaim.admissionHold != null && + (activeClaim.queueClaim.userAuthored || activeClaim.admissionIrreversible) + ) { + // WorkspaceService calls this after descendant cleanup. An irreversible synthetic turn + // must finish before the prioritized user entry can start, so release either held turn. + this.resumeQueuedToolEndClaimAdmission(activeClaim); + return activeClaim.queueClaim.userAuthored || didPrioritizeUserEntry; + } + if (!didPrioritizeUserEntry) { return false; } this.sendQueuedMessages(); @@ -6950,15 +7440,90 @@ export class AgentSession { return; } - this.queuedProviderToolEndAbortInFlight = false; + // A hard Stop owns queue admission until WorkspaceService finishes descendant cleanup. + if (this.hardInterruptClaimSettlementPending) { + if (this.queuedToolEndClaim != null) { + this.queuedToolEndClaim.dispatchDeferredByHardInterrupt = true; + } + return; + } + + // A dequeued entry owns admission until stream start or explicit failure cleanup. + if (this.dispatchingQueuedEntry || this.queuedToolEndClaim?.dispatchStarted === true) { + return; + } + const dispatchClaim = this.commitQueuedToolEndClaim(); // Clear the queued message flag (even if queue is empty, to handle race conditions) this.backgroundProcessManager.setMessageQueued(this.workspaceId, false); + const canceledEntries = this.messageQueue.discardCanceledEntries(); + if (canceledEntries.length > 0) { + this.emitQueuedMessageChanged(); + for (const canceledEntry of canceledEntries) { + this.notifyQueuedMessageCleared(canceledEntry, canceledEntry.cancelReason); + } + } + if (!this.messageQueue.isEmpty()) { // Entries dispatch one at a time (FIFO): special sends (compaction, agent // skills, workspace-turn follow-ups) own their turn, and anything queued // behind them dispatches on a later drain instead of batching into them. const { message, options, internal, enqueuedAtMs } = this.messageQueue.dequeueNext(); + const dispatchCancelState = + dispatchClaim != null + ? (internal?.cancelState ?? { canceledBeforeAcceptance: false }) + : internal?.cancelState; + const effectiveInternal = + dispatchClaim == null + ? internal + : { + ...internal, + cancelState: dispatchCancelState, + cancelSignal: dispatchClaim.queueClaim.admissionSignal, + admissionStale: () => + !dispatchClaim.admissionIrreversible && + (dispatchClaim.queueClaim.admissionSignal.aborted || + internal?.admissionStale?.() === true), + waitForAdmissionRelease: () => this.waitForQueuedToolEndClaimAdmission(dispatchClaim), + onAdmissionIrreversible: (persistedMessageIds: readonly string[]) => { + dispatchClaim.admissionIrreversible = true; + dispatchClaim.persistedMessageIds = [...persistedMessageIds]; + }, + excludeIrreversibleAdmissionOnCancel: !dispatchClaim.queueClaim.userAuthored, + onCanceled: async (reason: string) => { + if (!dispatchClaim.retryAfterCancellation) { + await internal?.onCanceled?.(reason); + } + }, + onAccepted: async () => { + await this.waitForQueuedToolEndClaimAdmission(dispatchClaim); + if ( + dispatchClaim.queueClaim.admissionSignal.aborted && + !dispatchClaim.admissionIrreversible + ) { + const reason: unknown = dispatchClaim.queueClaim.admissionSignal.reason; + this.releaseQueuedToolEndClaim(dispatchClaim); + throw new Error( + typeof reason === "string" + ? reason + : "Queue dispatch canceled by user interrupt." + ); + } + if (dispatchClaim.queueClaim.userAuthored) { + this.releaseQueuedToolEndClaim(dispatchClaim); + await internal?.onAccepted?.(); + return; + } + // Keep a synthetic claim visible through its acceptance callback. A concurrent + // hard Stop can then mark its durable rows provider-excluded before startup. + try { + await internal?.onAccepted?.(); + } finally { + // A hard Stop can install the hold while the producer callback runs. + await this.waitForQueuedToolEndClaimAdmission(dispatchClaim); + } + }, + }; this.dispatchingQueuedEntry = true; this.dispatchingQueuedEntryMuxMetadata = options?.muxMetadata; this.emitQueuedMessageChanged(); @@ -6975,9 +7540,9 @@ export class AgentSession { // Set PREPARING synchronously before the async sendMessage to prevent // incoming messages from bypassing the queue during the await gap. this.preparingWorkspaceTurnMetadata = getWorkspaceTurnMuxMetadata(options?.muxMetadata); - this.setTurnPhase(TurnPhase.PREPARING); + this.beginPreparingTurn(); - void this.sendMessage(message, options, { ...internal, enqueuedAtMs }) + void this.sendMessage(message, options, { ...effectiveInternal, enqueuedAtMs }) .then(async (result) => { // Keep the dispatch marker through the dequeue-to-stream-start window. A background // send can resolve before startup emits stream-start, and later reports must not claim @@ -6985,6 +7550,11 @@ export class AgentSession { // If sendMessage fails before it can start streaming, ensure we don't // leave the session stuck in PREPARING and notify correlated internal callers. if (!result.success) { + if (dispatchClaim != null) { + this.releaseQueuedToolEndClaim(dispatchClaim); + } + this.dispatchingQueuedEntry = false; + this.dispatchingQueuedEntryMuxMetadata = undefined; await internal?.onAcceptedPreStreamFailure?.(result.error); if (this.turnPhase === TurnPhase.PREPARING) { this.setTurnPhase(TurnPhase.IDLE); @@ -6995,16 +7565,54 @@ export class AgentSession { this.sendQueuedMessages(); return; } - if (internal?.cancelState?.canceledBeforeAcceptance === true) { + if ( + dispatchCancelState?.canceledBeforeAcceptance === true || + dispatchCancelState?.providerExcludedAfterAcceptance === true + ) { + if (dispatchClaim != null) { + this.releaseQueuedToolEndClaim(dispatchClaim); + } + if ( + dispatchCancelState.canceledBeforeAcceptance === true && + !dispatchClaim?.retryAfterCancellation && + internal?.onCanceled == null && + internal?.onAcceptedPreStreamFailure != null + ) { + const reason: unknown = effectiveInternal?.cancelSignal?.reason; + try { + await internal.onAcceptedPreStreamFailure( + createUnknownSendMessageError( + typeof reason === "string" + ? reason + : "Queued message canceled before acceptance." + ) + ); + } catch (error: unknown) { + log.error("Canceled queue admission callback failed", { + workspaceId: this.workspaceId, + error: getErrorMessage(error), + }); + } + } + this.dispatchingQueuedEntry = false; + this.dispatchingQueuedEntryMuxMetadata = undefined; // Cancellation can arrive after dequeue while sendMessage is validating or writing // history. No stream will start, so release PREPARING and continue with later entries. if (this.turnPhase === TurnPhase.PREPARING) { this.setTurnPhase(TurnPhase.IDLE); } this.sendQueuedMessages(); + return; + } + if (dispatchClaim != null) { + // Keep synthetic ownership until StreamManager accepts the abort signal. + this.releaseQueuedToolEndClaim(dispatchClaim); } }) .catch(async (error: unknown) => { + if (dispatchClaim != null) { + this.releaseQueuedToolEndClaim(dispatchClaim); + } // A REJECTED sendMessage (thrown, not returned Err — e.g. an awaited history or goal // service throwing pre-persistence) must reach the same failure hook as the // returned-error branch above: peer sends refund their family-message reservation diff --git a/src/node/services/agentStatusService.test.ts b/src/node/services/agentStatusService.test.ts index 966bbed4b5..e06bffa613 100644 --- a/src/node/services/agentStatusService.test.ts +++ b/src/node/services/agentStatusService.test.ts @@ -207,6 +207,27 @@ describe("AgentStatusService", () => { expect(persistedStatus).toEqual({ emoji: "🛠️", message: "Editing source" }); }); + test("excludes stopped synthetic rows from the status transcript", async () => { + await historyHandle.historyService.appendToHistory( + workspaceId, + createMuxMessage("stopped-wake", "user", "Untrusted stopped process output", { + synthetic: true, + providerExcluded: true, + }) + ); + await historyHandle.historyService.appendToHistory( + workspaceId, + createMuxMessage("u1", "user", "Continue with the requested work") + ); + + const service = createService(); + await getInternals(service).runForWorkspace(workspaceId); + + const transcript = generateSpy.mock.calls[0][0]; + expect(transcript).toContain("Continue with the requested work"); + expect(transcript).not.toContain("Untrusted stopped process output"); + }); + test("skips regeneration when the trailing transcript is unchanged (dedup)", async () => { // "Frozen chat" behavior: identical hash → no further LLM calls. await historyHandle.historyService.appendToHistory( diff --git a/src/node/services/agentStatusService.ts b/src/node/services/agentStatusService.ts index 287a21778d..6a830c5cae 100644 --- a/src/node/services/agentStatusService.ts +++ b/src/node/services/agentStatusService.ts @@ -16,7 +16,7 @@ import { AGENT_STATUS_TICK_INTERVAL_MS, } from "@/constants/agentStatus"; import type { Config } from "@/node/config"; -import type { MuxMessage } from "@/common/types/message"; +import { filterProviderExcludedMessages, type MuxMessage } from "@/common/types/message"; import { isWorkspaceArchived } from "@/common/utils/archive"; import type { AIService } from "./aiService"; import type { ExtensionMetadataService } from "./ExtensionMetadataService"; @@ -559,7 +559,7 @@ export class AgentStatusService { ); if (!result.success) return ""; - const committedMessages: MuxMessage[] = [...result.data]; + const committedMessages: MuxMessage[] = filterProviderExcludedMessages(result.data); const partial = await this.historyService.readPartial(workspaceId); // Partial messages get an "(in progress)" role suffix so the model sees diff --git a/src/node/services/aiService.ts b/src/node/services/aiService.ts index 5ce958eca2..6bd6152edf 100644 --- a/src/node/services/aiService.ts +++ b/src/node/services/aiService.ts @@ -940,7 +940,12 @@ export class AIService extends EventEmitter { async stopStream( workspaceId: string, - options?: { soft?: boolean; abandonPartial?: boolean; abortReason?: StreamAbortReason } + options?: { + soft?: boolean; + abandonPartial?: boolean; + abortReason?: StreamAbortReason; + abortTurnGeneration?: number; + } ): Promise> { return this.streamManager.stopStream(workspaceId, options); } diff --git a/src/node/services/bashMonitorWakeReconciler.test.ts b/src/node/services/bashMonitorWakeReconciler.test.ts index 4348bd3c33..62b46af4d6 100644 --- a/src/node/services/bashMonitorWakeReconciler.test.ts +++ b/src/node/services/bashMonitorWakeReconciler.test.ts @@ -2,6 +2,7 @@ import * as fsPromises from "node:fs/promises"; import * as os from "node:os"; import * as path from "node:path"; +import type { BashMonitorWakeDisplayRecord } from "@/common/types/message"; import { classifyMachineTurnPromptKind } from "@/common/utils/machineTurnPrompts"; import type { BashMonitorRegistryRecord, @@ -60,6 +61,7 @@ describe("BashMonitorWakeReconciler", () => { let removedOwners: string[]; let dropped: string[]; let droppedGenerations: Array; + let providerExcludedWakeRecords: BashMonitorWakeDisplayRecord[]; let reconciler: BashMonitorWakeReconciler; beforeEach(async () => { @@ -74,6 +76,7 @@ describe("BashMonitorWakeReconciler", () => { removedOwners = []; dropped = []; droppedGenerations = []; + providerExcludedWakeRecords = []; reconciler = new BashMonitorWakeReconciler({ sessionsDir: root, processManager: { @@ -107,6 +110,7 @@ describe("BashMonitorWakeReconciler", () => { }, recordTerminal: () => undefined, }, + listProviderExcludedWakeRecords: () => Promise.resolve(providerExcludedWakeRecords), onWake: (dispatch) => { dispatches.push(dispatch); return dispatchOutcome; @@ -115,6 +119,7 @@ describe("BashMonitorWakeReconciler", () => { }); afterEach(async () => { + await reconciler.dispose(OWNER); await fsPromises.rm(root, { recursive: true, force: true }); }); @@ -141,6 +146,93 @@ describe("BashMonitorWakeReconciler", () => { expect(dispatches).toHaveLength(2); }); + test("consumes a provider-excluded wake after restart", async () => { + live = [liveSnapshot()]; + providerExcludedWakeRecords = [ + { + processId: "proc", + wakeUpdatedAt: CREATED_AT + ":12", + kind: "match", + displayName: "CI watcher", + filter: "READY", + filterExclude: false, + }, + ]; + + await reconciler.reconcile(OWNER); + + expect(dispatches).toEqual([]); + expect(acknowledged).toEqual([{ processId: "proc", matchedThroughOffset: 12 }]); + + providerExcludedWakeRecords = []; + await reconciler.reconcile(OWNER); + expect(dispatches).toEqual([]); + }); + + test("dispatches new records beside a provider-excluded wake", async () => { + live = [ + liveSnapshot(), + liveSnapshot({ + processId: "fresh", + taskId: "bash:fresh", + createdAt: "2026-08-31T12:01:00.000Z", + match: { throughOffset: 5, lines: ["FRESH"], totalMatches: 1 }, + }), + ]; + providerExcludedWakeRecords = [ + { + processId: "proc", + wakeUpdatedAt: CREATED_AT + ":12", + kind: "match", + displayName: "CI watcher", + filter: "READY", + filterExclude: false, + }, + ]; + + await reconciler.reconcile(OWNER); + + expect(dispatches).toHaveLength(1); + expect(dispatches[0].muxMetadata.records).toHaveLength(1); + expect(dispatches[0].muxMetadata.records[0]?.processId).toBe("fresh"); + expect(dispatches[0].prompt).toContain("FRESH"); + expect(dispatches[0].prompt).not.toContain("> READY"); + }); + + test("settles an excluded match offset before deriving a newer batch", async () => { + live = [ + liveSnapshot({ + match: { + throughOffset: 20, + lines: ["READY old", "READY new"], + totalMatches: 2, + batches: [ + { throughOffset: 12, lines: ["READY old"], totalMatches: 1, droppedLines: 0 }, + { throughOffset: 20, lines: ["READY new"], totalMatches: 1, droppedLines: 0 }, + ], + }, + }), + ]; + providerExcludedWakeRecords = [ + { + processId: "proc", + wakeUpdatedAt: CREATED_AT + ":12", + kind: "match", + displayName: "CI watcher", + filter: "READY", + filterExclude: false, + }, + ]; + + await reconciler.reconcile(OWNER); + + expect(acknowledged).toEqual([{ processId: "proc", matchedThroughOffset: 12 }]); + expect(dispatches).toHaveLength(1); + expect(dispatches[0].muxMetadata.records[0]?.wakeUpdatedAt).toBe(CREATED_AT + ":20"); + expect(dispatches[0].prompt).toContain("READY new"); + expect(dispatches[0].prompt).not.toContain("READY old"); + }); + test("superseding a queued wake uses a distinct queue key", async () => { const queuedKeys = new Set(); const queuedDispatches: BashMonitorWakeDispatch[] = []; @@ -157,6 +249,7 @@ describe("BashMonitorWakeReconciler", () => { remove: () => undefined, recordTerminal: () => undefined, }, + listProviderExcludedWakeRecords: () => Promise.resolve(providerExcludedWakeRecords), onWake: (dispatch) => { if (queuedKeys.has(dispatch.dedupeKey)) return "deferred"; queuedKeys.add(dispatch.dedupeKey); @@ -313,6 +406,7 @@ describe("BashMonitorWakeReconciler", () => { }, recordTerminal: () => undefined, }, + listProviderExcludedWakeRecords: () => Promise.resolve(providerExcludedWakeRecords), onWake: (dispatch) => { restartedDispatches.push(dispatch); return "in-flight"; @@ -395,6 +489,7 @@ describe("BashMonitorWakeReconciler", () => { remove: () => undefined, recordTerminal: () => undefined, }, + listProviderExcludedWakeRecords: () => Promise.resolve(providerExcludedWakeRecords), onWake: (dispatch) => { afterRestart.push(dispatch); return "in-flight"; @@ -661,6 +756,7 @@ describe("BashMonitorWakeReconciler", () => { }, recordTerminal: () => undefined, }, + listProviderExcludedWakeRecords: () => Promise.resolve(providerExcludedWakeRecords), onWake: (dispatch) => { afterRestart.push(dispatch); return "in-flight"; @@ -800,6 +896,7 @@ describe("BashMonitorWakeReconciler", () => { remove: () => undefined, recordTerminal: () => undefined, }, + listProviderExcludedWakeRecords: () => Promise.resolve(providerExcludedWakeRecords), onWake: (dispatch) => { retryDispatches.push(dispatch); return "in-flight"; @@ -899,6 +996,35 @@ describe("BashMonitorWakeReconciler", () => { expect(dispatches[0].muxMetadata.records[0]).not.toHaveProperty("terminal"); }); + test("consumes a provider-excluded opted-out lost wake", async () => { + rows = [ + registryRecord({ + status: "exited", + exitCode: 0, + settledAt: "2026-09-01T00:02:00.000Z", + wakeOnExit: false, + terminalStatusShown: false, + matchedThroughOffset: 12, + }), + ]; + providerExcludedWakeRecords = [ + { + processId: "dead", + wakeUpdatedAt: CREATED_AT + ":12", + kind: "monitor-lost", + lostReason: "restart", + displayName: "dead", + filter: "DONE", + filterExclude: false, + }, + ]; + + await reconciler.reconcile(OWNER); + + expect(dispatches).toEqual([]); + expect(removed).toEqual(["dead"]); + }); + test("snapshot supplies pending kinds without dispatching and removes the legacy wake directory", async () => { rows = [ registryRecord({ diff --git a/src/node/services/bashMonitorWakeReconciler.ts b/src/node/services/bashMonitorWakeReconciler.ts index 46728fc4be..4e84930ad9 100644 --- a/src/node/services/bashMonitorWakeReconciler.ts +++ b/src/node/services/bashMonitorWakeReconciler.ts @@ -3,7 +3,7 @@ import { randomUUID } from "node:crypto"; import * as fsPromises from "node:fs/promises"; import * as path from "node:path"; -import type { MuxMessageMetadata } from "@/common/types/message"; +import type { BashMonitorWakeDisplayRecord, MuxMessageMetadata } from "@/common/types/message"; import assert from "@/common/utils/assert"; import { BASH_MONITOR_WAKE_HEADINGS } from "@/common/utils/machineTurnPrompts"; import type { @@ -128,6 +128,12 @@ interface WatermarkEntry { lost?: true; } +interface ProviderExcludedWakeSettlement { + matchedThroughOffset?: number; + terminalSettledAt?: string; + lost?: true; +} + interface DerivedSignal { key: string; ownerWorkspaceId: string; @@ -169,6 +175,64 @@ function signalKey(processId: string, createdAt: string): string { return processId + "\u0000" + createdAt; } +function getProviderExcludedWakeSettlement( + snapshot: BashMonitorProcessSnapshot, + records: readonly BashMonitorWakeDisplayRecord[] +): ProviderExcludedWakeSettlement | undefined { + const maximumKnownMatchOffset = Math.max( + snapshot.match?.throughOffset ?? -1, + snapshot.terminal?.matchedThroughOffset ?? -1, + snapshot.lost?.failedMatch?.matchedThroughOffset ?? -1 + ); + let matchedThroughOffset = -1; + let terminalSettledAt: string | undefined; + let lost = false; + let matchedRecord = false; + + for (const record of records) { + if (record.wakeUpdatedAt == null) continue; + if ( + record.terminal != null && + snapshot.terminal != null && + record.wakeUpdatedAt === snapshot.terminal.settledAt + ) { + terminalSettledAt = snapshot.terminal.settledAt; + matchedThroughOffset = maximumKnownMatchOffset; + matchedRecord = true; + continue; + } + if ( + record.kind === "monitor-lost" && + (record.wakeUpdatedAt === snapshot.createdAt || + record.wakeUpdatedAt === snapshot.lost?.failedAt) + ) { + lost = true; + matchedThroughOffset = maximumKnownMatchOffset; + matchedRecord = true; + continue; + } + + const matchPrefix = snapshot.createdAt + ":"; + if (!record.wakeUpdatedAt.startsWith(matchPrefix)) continue; + const excludedOffset = Number(record.wakeUpdatedAt.slice(matchPrefix.length)); + if ( + Number.isSafeInteger(excludedOffset) && + excludedOffset >= 0 && + excludedOffset <= maximumKnownMatchOffset + ) { + matchedThroughOffset = Math.max(matchedThroughOffset, excludedOffset); + if (record.kind === "monitor-lost") lost = true; + matchedRecord = true; + } + } + if (!matchedRecord) return undefined; + return { + ...(matchedThroughOffset >= 0 ? { matchedThroughOffset } : {}), + ...(terminalSettledAt != null ? { terminalSettledAt } : {}), + ...(lost ? { lost: true as const } : {}), + }; +} + function normalizedTerminalStatus( terminal: BashMonitorTerminalSummary ): "exited" | "killed" | "failed" { @@ -367,6 +431,9 @@ export class BashMonitorWakeReconciler { sessionsDir: string; processManager: BashMonitorWakeReconcilerProcessManager; registry: BashMonitorWakeReconcilerRegistry; + listProviderExcludedWakeRecords( + ownerWorkspaceId: string + ): Promise; onWake( dispatch: BashMonitorWakeDispatch ): Promise | BashMonitorWakeDispatchOutcome; @@ -414,6 +481,19 @@ export class BashMonitorWakeReconciler { }); } + async settleProviderExcludedWakeRecords( + ownerWorkspaceId: string, + records: readonly BashMonitorWakeDisplayRecord[] + ): Promise { + if (records.length === 0) return; + await this.locks.withLock(ownerWorkspaceId, async () => { + const collected = await this.collect(ownerWorkspaceId, true, records); + await this.advanceWatermarks(ownerWorkspaceId, collected.watermarks, collected.autoConsumed); + await this.cleanup(collected.autoConsumed); + }); + this.scheduleReconcile(ownerWorkspaceId); + } + pendingWakeKind( snapshot: BashMonitorWakeReconcilerSnapshot, processId: string @@ -604,7 +684,8 @@ export class BashMonitorWakeReconciler { private async collect( ownerWorkspaceId: string, - applyFrontier: boolean + applyFrontier: boolean, + providerExcludedRecords?: readonly BashMonitorWakeDisplayRecord[] ): Promise<{ signals: DerivedSignal[]; autoConsumed: DerivedSignal[]; @@ -655,6 +736,17 @@ export class BashMonitorWakeReconciler { } } if (pruned) await this.writeWatermarks(ownerWorkspaceId, watermarks); + if (applyFrontier) { + const records = + providerExcludedRecords ?? + (await this.args.listProviderExcludedWakeRecords(ownerWorkspaceId)); + await this.applyProviderExcludedWakeRecords( + ownerWorkspaceId, + candidates, + watermarks, + records + ); + } const signals: DerivedSignal[] = []; const autoConsumed: DerivedSignal[] = []; @@ -677,6 +769,88 @@ export class BashMonitorWakeReconciler { return { signals, autoConsumed, deferredReads, watermarks }; } + private async applyProviderExcludedWakeRecords( + ownerWorkspaceId: string, + candidates: ReadonlyArray<{ + snapshot: BashMonitorProcessSnapshot; + deadRegistryRow: boolean; + }>, + watermarks: Map, + records: readonly BashMonitorWakeDisplayRecord[] + ): Promise { + if (candidates.length === 0 || records.length === 0) return; + const recordsByProcess = new Map(); + for (const record of records) { + if (record.processId == null) continue; + const processRecords = recordsByProcess.get(record.processId) ?? []; + processRecords.push(record); + recordsByProcess.set(record.processId, processRecords); + } + + let watermarksChanged = false; + const acknowledgements: Array<{ + snapshot: BashMonitorProcessSnapshot; + matchedThroughOffset?: number; + terminalSettledAt?: string; + }> = []; + for (const candidate of candidates) { + const snapshot = candidate.snapshot; + const excluded = getProviderExcludedWakeSettlement( + snapshot, + recordsByProcess.get(snapshot.processId) ?? [] + ); + if (excluded == null) continue; + + const key = signalKey(snapshot.processId, snapshot.createdAt); + const previous = watermarks.get(key); + const next: WatermarkEntry = { + processId: snapshot.processId, + createdAt: snapshot.createdAt, + ...(excluded.matchedThroughOffset != null || previous?.matchedThroughOffset != null + ? { + matchedThroughOffset: Math.max( + excluded.matchedThroughOffset ?? -1, + previous?.matchedThroughOffset ?? -1 + ), + } + : {}), + ...(excluded.terminalSettledAt != null || previous?.terminalSettledAt != null + ? { terminalSettledAt: excluded.terminalSettledAt ?? previous?.terminalSettledAt } + : {}), + ...(excluded.lost === true || previous?.lost === true ? { lost: true } : {}), + }; + if ( + next.matchedThroughOffset !== previous?.matchedThroughOffset || + next.terminalSettledAt !== previous?.terminalSettledAt || + next.lost !== previous?.lost + ) { + watermarks.set(key, next); + watermarksChanged = true; + } + if (!candidate.deadRegistryRow) { + acknowledgements.push({ + snapshot, + ...(next.matchedThroughOffset != null + ? { matchedThroughOffset: next.matchedThroughOffset } + : {}), + ...(next.terminalSettledAt != null ? { terminalSettledAt: next.terminalSettledAt } : {}), + }); + } + } + + if (watermarksChanged) { + await this.writeWatermarks(ownerWorkspaceId, watermarks); + } + for (const acknowledgement of acknowledgements) { + await this.args.processManager.acknowledgeMonitorWake( + acknowledgement.snapshot.processId, + Date.parse(acknowledgement.snapshot.createdAt), + acknowledgement.matchedThroughOffset, + acknowledgement.terminalSettledAt + ); + } + } + private async derive( snapshot: BashMonitorProcessSnapshot, deadRegistryRow: boolean, @@ -822,8 +996,9 @@ export class BashMonitorWakeReconciler { lines: readonly string[]; droppedLines: number; } { + const visibleAfterOffset = Math.max(deliveredMatchedThroughOffset, shownThroughOffset); const visibleMatchBatches = snapshot.match?.batches?.filter( - (batch) => batch.throughOffset > shownThroughOffset + (batch) => batch.throughOffset > visibleAfterOffset ); const retained = visibleMatchBatches != null diff --git a/src/node/services/branchSummary.test.ts b/src/node/services/branchSummary.test.ts index 4de7143658..bf08d469ac 100644 --- a/src/node/services/branchSummary.test.ts +++ b/src/node/services/branchSummary.test.ts @@ -216,6 +216,19 @@ describe("buildAbandonedBranchTranscript", () => { // Clamped from the end: the newest content survives. expect(transcript.endsWith("TAIL-MARKER")).toBe(true); }); + + test("excludes stopped synthetic rows", () => { + const stoppedWake = createMuxMessage("stopped-wake", "user", "untrusted stopped output", { + synthetic: true, + providerExcluded: true, + }); + const retained = createMuxMessage("retained", "user", "continue the requested work"); + + const transcript = buildAbandonedBranchTranscript([stoppedWake, retained]); + + expect(transcript).toContain("continue the requested work"); + expect(transcript).not.toContain("untrusted stopped output"); + }); }); describe("getSideChannelModelCandidates (r23: provider confinement)", () => { diff --git a/src/node/services/branchSummary.ts b/src/node/services/branchSummary.ts index 13c8236439..524b57de0b 100644 --- a/src/node/services/branchSummary.ts +++ b/src/node/services/branchSummary.ts @@ -20,7 +20,11 @@ import type { LanguageModelV2Usage } from "@ai-sdk/provider"; import { EXPERIMENT_IDS, type ExperimentId } from "@/common/constants/experiments"; import { buildCompactionPrompt } from "@/common/constants/ui"; -import { createMuxMessage, type MuxMessage } from "@/common/types/message"; +import { + createMuxMessage, + filterProviderExcludedMessages, + type MuxMessage, +} from "@/common/types/message"; import type { WorkspaceMetadata } from "@/common/types/workspace"; import * as fs from "node:fs/promises"; import * as path from "node:path"; @@ -147,7 +151,9 @@ function formatMessageForBranchTranscript(message: MuxMessage): string { */ export function buildAbandonedBranchTranscript(messages: MuxMessage[]): string { assert(Array.isArray(messages), "buildAbandonedBranchTranscript requires a message array"); - const formatted = messages.map(formatMessageForBranchTranscript).filter((s) => s.length > 0); + const formatted = filterProviderExcludedMessages(messages) + .map(formatMessageForBranchTranscript) + .filter((s) => s.length > 0); let totalChars = formatted.reduce((sum, s) => sum + s.length, 0); let drop = 0; diff --git a/src/node/services/historyService.test.ts b/src/node/services/historyService.test.ts index 12b419f469..00eb0dd40e 100644 --- a/src/node/services/historyService.test.ts +++ b/src/node/services/historyService.test.ts @@ -5,7 +5,11 @@ import { HistoryService } from "./historyService"; import type { Config } from "@/node/config"; import { createTestHistoryService } from "./testHistoryService"; import { updateSubagentTranscriptArtifactsFile } from "./subagentTranscriptArtifacts"; -import { createMuxMessage, type MuxMessage } from "@/common/types/message"; +import { + createMuxMessage, + filterProviderExcludedMessages, + type MuxMessage, +} from "@/common/types/message"; import assert from "node:assert"; import { createHash } from "node:crypto"; import * as fs from "fs/promises"; @@ -104,6 +108,31 @@ describe("HistoryService", () => { expect(messages[1].id).toBe("msg2"); }); + it("applies provider exclusion tombstones after restart", async () => { + const workspaceId = "provider-exclusion-tombstone"; + const messageId = "stopped-wake"; + const appendResult = await service.appendToHistory( + workspaceId, + createMuxMessage(messageId, "user", "monitor wake", { synthetic: true }) + ); + expect(appendResult.success).toBe(true); + + const tombstoneResult = await service.addProviderExclusionTombstones(workspaceId, [ + messageId, + ]); + expect(tombstoneResult.success).toBe(true); + + const restarted = new HistoryService(config); + const history = await restarted.getHistoryFromLatestBoundary(workspaceId); + expect(history.success).toBe(true); + if (!history.success) return; + expect(history.data[0]?.metadata?.providerExcluded).toBe(true); + expect(filterProviderExcludedMessages(history.data)).toEqual([]); + + const fullHistory = await collectFullHistory(restarted, workspaceId); + expect(fullHistory[0]?.metadata?.providerExcluded).toBe(true); + }); + it("hydrates legacy cmuxMetadata entries", async () => { const workspaceId = "workspace-legacy"; const legacyMessage = createMuxMessage("msg-legacy", "user", "legacy", { diff --git a/src/node/services/historyService.ts b/src/node/services/historyService.ts index bdc9a09973..893fe378c1 100644 --- a/src/node/services/historyService.ts +++ b/src/node/services/historyService.ts @@ -31,7 +31,11 @@ import { isDurableContextBoundaryMarker, } from "@/common/utils/messages/compactionBoundary"; import { filterWorkflowDisplayOnlyMessages } from "@/common/utils/workflowRunMessages"; -import { CHAT_FILE_NAME, CHAT_ARCHIVE_FILE_NAME } from "@/common/constants/paths"; +import { + CHAT_FILE_NAME, + CHAT_ARCHIVE_FILE_NAME, + PROVIDER_EXCLUDED_MESSAGE_IDS_FILE_NAME, +} from "@/common/constants/paths"; import { coerceThinkingLevel, type ThinkingLevel } from "@/common/types/thinking"; import { readSubagentTranscriptArtifactsFile, @@ -499,6 +503,115 @@ export class HistoryService { return path.join(this.getSessionDir(workspaceId), this.CHAT_ARCHIVE_FILE); } + private getProviderExcludedMessageIdsPath(workspaceId: string): string { + return path.join(this.getSessionDir(workspaceId), PROVIDER_EXCLUDED_MESSAGE_IDS_FILE_NAME); + } + + private async readProviderExcludedMessageIds(workspaceId: string): Promise> { + const contents = await this.readExistingFile( + this.getProviderExcludedMessageIdsPath(workspaceId) + ); + const messageIds = new Set(); + if (contents === null) { + return messageIds; + } + + for (const line of contents.split("\n")) { + if (line.trim().length === 0) continue; + try { + const messageId: unknown = JSON.parse(line); + if (typeof messageId === "string" && messageId.length > 0) { + messageIds.add(messageId); + } + } catch { + // A malformed tombstone must not hide the remaining valid exclusions. + } + } + return messageIds; + } + + private applyProviderExclusionTombstones( + messages: MuxMessage[], + messageIds: ReadonlySet + ): MuxMessage[] { + if (messageIds.size === 0) return messages; + + return messages.map((message) => { + if ( + !messageIds.has(message.id) || + message.metadata?.synthetic !== true || + message.metadata.contextBoundaryKind != null || + message.metadata.compactionBoundary === true || + message.metadata.providerExcluded === true + ) { + return message; + } + return { + ...message, + metadata: { ...message.metadata, providerExcluded: true }, + }; + }); + } + + /** + * Return a cross-process token for files that can contain provider exclusions. + * Callers use this token to invalidate caches after another backend writes history. + */ + async getProviderExclusionChangeToken(workspaceId: string): Promise { + const paths = [ + this.getChatArchivePath(workspaceId), + this.getChatHistoryPath(workspaceId), + this.getProviderExcludedMessageIdsPath(workspaceId), + ]; + const versions = await Promise.all( + paths.map(async (filePath) => { + try { + const stat = await fs.stat(filePath, { bigint: true }); + return `${stat.ino}:${stat.size}:${stat.mtimeNs}`; + } catch (error) { + if (isErrnoWithCode(error, "ENOENT")) return "missing"; + throw error; + } + }) + ); + return versions.join("|"); + } + + /** + * Durably exclude synthetic rows before their producer watermark advances. + * Tombstones remain after row deletion because removal would create a cross-file read race. + */ + async addProviderExclusionTombstones( + workspaceId: string, + messageIds: readonly string[] + ): Promise> { + assert(messageIds.length > 0, "addProviderExclusionTombstones requires message IDs"); + const ids = new Set(messageIds); + assert(ids.size === messageIds.length, "addProviderExclusionTombstones requires unique IDs"); + + return this.withRecoveredHistoryWriteResultLock( + workspaceId, + "Failed to add provider exclusion tombstones", + async () => { + try { + const persistedIds = await this.readProviderExcludedMessageIds(workspaceId); + for (const messageId of ids) persistedIds.add(messageId); + const contents = [...persistedIds] + .sort() + .map((messageId) => JSON.stringify(messageId)) + .join("\n"); + await writeFileAtomic( + this.getProviderExcludedMessageIdsPath(workspaceId), + `${contents}\n` + ); + return Ok(undefined); + } catch (error) { + return Err(`Failed to add provider exclusion tombstones: ${getErrorMessage(error)}`); + } + } + ); + } + private getTruncateTransactionPath(workspaceId: string): string { return `${this.getChatArchivePath(workspaceId)}.truncate.json`; } @@ -1301,6 +1414,9 @@ export class HistoryService { Ok({ archive: await this.readExistingFile(this.getChatArchivePath(sourceWorkspaceId)), chat: await this.readExistingFile(this.getChatHistoryPath(sourceWorkspaceId)), + providerExclusions: await this.readExistingFile( + this.getProviderExcludedMessageIdsPath(sourceWorkspaceId) + ), }) ); if (!snapshot.success) { @@ -1312,6 +1428,10 @@ export class HistoryService { for (const [targetPath, contents] of [ [this.getChatArchivePath(targetWorkspaceId), snapshot.data.archive], [this.getChatHistoryPath(targetWorkspaceId), snapshot.data.chat], + [ + this.getProviderExcludedMessageIdsPath(targetWorkspaceId), + snapshot.data.providerExclusions, + ], ] as const) { if (contents === null) { await fs.rm(targetPath, { force: true }); @@ -1333,16 +1453,19 @@ export class HistoryService { const chatPath = this.getChatHistoryPath(workspaceId); const archivePath = this.getChatArchivePath(workspaceId); try { + const providerExcludedMessageIds = await this.readProviderExcludedMessageIds(workspaceId); + const visitWithExclusions = (messages: MuxMessage[]) => + visitor(this.applyProviderExclusionTombstones(messages, providerExcludedMessageIds)); if (direction === "forward") { // Archived rows are strictly older than active rows. - const completed = await this.iterateForward(archivePath, visitor); + const completed = await this.iterateForward(archivePath, visitWithExclusions); if (completed) { - await this.iterateForward(chatPath, visitor); + await this.iterateForward(chatPath, visitWithExclusions); } } else { - const completed = await this.iterateBackward(chatPath, visitor); + const completed = await this.iterateBackward(chatPath, visitWithExclusions); if (completed) { - await this.iterateBackward(archivePath, visitor); + await this.iterateBackward(archivePath, visitWithExclusions); } } return Ok(undefined); @@ -1644,6 +1767,9 @@ export class HistoryService { const chatPath = this.getChatHistoryPath(workspaceId); const archivePath = this.getChatArchivePath(workspaceId); + const providerExcludedMessageIds = await this.readProviderExcludedMessageIds(workspaceId); + const complete = (messages: MuxMessage[]): Result => + Ok(this.applyProviderExclusionTombstones(messages, providerExcludedMessageIds)); // Try the requested boundary in chat.jsonl, falling back to less-skipped boundaries. let chatBoundaryCount = 0; @@ -1652,7 +1778,7 @@ export class HistoryService { const offset = await this.findLastBoundaryByteOffset(chatPath, s); if (offset !== null) { if (s === skip) { - return Ok(await this.readHistoryFromOffset(chatPath, offset)); + return complete(await this.readHistoryFromOffset(chatPath, offset)); } // chat.jsonl has fewer boundaries than requested; remember its oldest // boundary as a fallback and keep counting into the archive. @@ -1669,18 +1795,18 @@ export class HistoryService { if (offset !== null) { const archived = await this.readHistoryFromOffset(archivePath, offset); const active = await this.readChatHistory(workspaceId); - return Ok([...archived, ...active]); + return complete([...archived, ...active]); } } if (chatFallbackOffset !== null) { - return Ok(await this.readHistoryFromOffset(chatPath, chatFallbackOffset)); + return complete(await this.readHistoryFromOffset(chatPath, chatFallbackOffset)); } // No boundaries at all — workspace is uncompacted, full read is the only option const archived = await this.readArchivedHistory(workspaceId); const active = await this.readChatHistory(workspaceId); - return Ok([...archived, ...active]); + return complete([...archived, ...active]); }; try { @@ -1811,9 +1937,17 @@ export class HistoryService { this.getChatArchivePath(workspaceId), n - messages.length ); - return Ok([...archived, ...messages]); + const providerExcludedMessageIds = + await this.readProviderExcludedMessageIds(workspaceId); + return Ok( + this.applyProviderExclusionTombstones( + [...archived, ...messages], + providerExcludedMessageIds + ) + ); } - return Ok(messages); + const providerExcludedMessageIds = await this.readProviderExcludedMessageIds(workspaceId); + return Ok(this.applyProviderExclusionTombstones(messages, providerExcludedMessageIds)); } catch (error) { const message = getErrorMessage(error); return Err(`Failed to read last ${n} messages: ${message}`); @@ -2691,6 +2825,64 @@ export class HistoryService { ); } + /** + * Atomically exclude recent active-history rows from all provider requests. + * A hard Stop uses this after a synthetic turn crosses its rollback boundary. + */ + async markMessagesProviderExcluded( + workspaceId: string, + messageIds: readonly string[] + ): Promise> { + assert(messageIds.length > 0, "markMessagesProviderExcluded requires message IDs"); + const ids = new Set(messageIds); + assert(ids.size === messageIds.length, "markMessagesProviderExcluded requires unique IDs"); + + return this.withRecoveredHistoryWriteResultLock( + workspaceId, + "Failed to exclude messages from provider requests", + async () => { + try { + const messages = await this.readChatHistory(workspaceId); + const foundIds = new Set( + messages.filter((message) => ids.has(message.id)).map((message) => message.id) + ); + const missingIds = messageIds.filter((messageId) => !foundIds.has(messageId)); + if (missingIds.length > 0) { + return Err(`Messages not found in active history: ${missingIds.join(", ")}`); + } + const invalidIds = messages + .filter( + (message) => + ids.has(message.id) && + (message.metadata?.synthetic !== true || + message.metadata.contextBoundaryKind != null || + message.metadata.compactionBoundary === true) + ) + .map((message) => message.id); + if (invalidIds.length > 0) { + return Err(`Messages are not excludable synthetic rows: ${invalidIds.join(", ")}`); + } + + const updatedMessages = messages.map((message) => + ids.has(message.id) + ? { + ...message, + metadata: { ...message.metadata, providerExcluded: true }, + } + : message + ); + await writeFileAtomic( + this.getChatHistoryPath(workspaceId), + this.serializeHistoryEntries(updatedMessages, workspaceId) + ); + return Ok(undefined); + } catch (error) { + return Err(`Failed to exclude messages: ${getErrorMessage(error)}`); + } + } + ); + } + /** * Delete a single message by ID while preserving the rest of the history. * diff --git a/src/node/services/memoryHarvest.test.ts b/src/node/services/memoryHarvest.test.ts index 68bda5606f..dd36b644dc 100644 --- a/src/node/services/memoryHarvest.test.ts +++ b/src/node/services/memoryHarvest.test.ts @@ -322,6 +322,34 @@ describe("runMemoryHarvest", () => { expect(prompt).toContain("</message> { + using fixture = createFixture(); + fixture.messages = [ + createMuxMessage("stopped-wake", "user", "Untrusted stopped process output", { + historySequence: 0, + synthetic: true, + providerExcluded: true, + }), + createMuxMessage("retained", "user", "Continue with the requested work", { + historySequence: 1, + }), + ]; + let prompt = ""; + + await runHarvest( + fixture, + new MockLanguageModelV3({ + doStream: (options) => { + prompt = userPromptText(options); + return Promise.resolve({ stream: simulateReadableStream({ chunks: [finishChunk()] }) }); + }, + }) + ); + + expect(prompt).toContain("Continue with the requested work"); + expect(prompt).not.toContain("Untrusted stopped process output"); + }); + it("chunks oversized epochs before calling the model", async () => { using fixture = createFixture(); fixture.messages = Array.from({ length: 12 }, (_, index) => diff --git a/src/node/services/memoryHarvest.ts b/src/node/services/memoryHarvest.ts index a8851bfb86..cbe6e015f8 100644 --- a/src/node/services/memoryHarvest.ts +++ b/src/node/services/memoryHarvest.ts @@ -3,7 +3,7 @@ import type { LanguageModelV2Usage } from "@ai-sdk/provider"; import { z } from "zod"; import type { CompactionCompletionMetadata } from "@/common/types/compaction"; -import type { MuxMessage } from "@/common/types/message"; +import { filterProviderExcludedMessages, type MuxMessage } from "@/common/types/message"; import { getErrorMessage } from "@/common/utils/errors"; import { accumulateStepsProviderMetadata } from "@/common/utils/tokens/usageHelpers"; import assert from "@/common/utils/assert"; @@ -261,7 +261,7 @@ export async function runMemoryHarvest(args: { }, }); - const chunks = buildHarvestChunks(args.messages); + const chunks = buildHarvestChunks(filterProviderExcludedMessages(args.messages)); const streamErrors: string[] = []; let usage: MemoryHarvestResult["usage"]; for (const [index, chunk] of chunks.entries()) { diff --git a/src/node/services/messageQueue.test.ts b/src/node/services/messageQueue.test.ts index 7755dace1f..0d5359f9ca 100644 --- a/src/node/services/messageQueue.test.ts +++ b/src/node/services/messageQueue.test.ts @@ -600,6 +600,167 @@ describe("MessageQueue", () => { expect((candidate?.muxMetadata as MuxMessageMetadata).type).toBe("workspace-turn-task"); }); + it("keeps a claimed tool-end entry dispatchable after its cancel signal aborts", () => { + const controller = new AbortController(); + queue.add( + "Monitor wake", + { model: "gpt-4", agentId: "exec", queueDispatchMode: "tool-end" }, + { cancelSignal: controller.signal } + ); + + const claim = queue.claimNextToolEndEntry(); + controller.abort("task output consumed the wake"); + + expect(claim).toBeDefined(); + expect(queue.dequeueNext().internal?.cancelSignal).toBe(claim?.admissionSignal); + }); + + it("restores cancellation when a claimed tool-end queue cut fails", () => { + const controller = new AbortController(); + queue.add( + "Monitor wake", + { model: "gpt-4", agentId: "exec", queueDispatchMode: "tool-end" }, + { cancelSignal: controller.signal } + ); + + const claim = queue.claimNextToolEndEntry(); + controller.abort("task output consumed the wake"); + claim?.restoreCancellation(); + + expect(queue.dequeueNext().internal?.cancelSignal).toBe(controller.signal); + }); + + it("cancels the claimed entry through its admission signal after commit", () => { + queue.add("Monitor wake", { + model: "gpt-4", + agentId: "exec", + queueDispatchMode: "tool-end", + }); + + const claim = queue.claimNextToolEndEntry(); + expect(claim?.commit()).toBe(true); + const cancelSignal = queue.dequeueNext().internal?.cancelSignal; + + claim?.cancelAdmission("user stopped the queue dispatch"); + + expect(cancelSignal?.aborted).toBe(true); + expect(cancelSignal?.reason).toBe("user stopped the queue dispatch"); + }); + + it("requeues a claimed user entry after its admission is canceled", () => { + const controller = new AbortController(); + const cancelState = { canceledBeforeAcceptance: false }; + queue.add( + "User follow-up", + { model: "gpt-4", agentId: "exec", queueDispatchMode: "tool-end" }, + { cancelSignal: controller.signal, cancelState } + ); + + const claim = queue.claimNextToolEndEntry(); + expect(claim?.userAuthored).toBe(true); + expect(claim?.commit()).toBe(true); + const firstAdmission = queue.dequeueNext(); + expect(firstAdmission.internal?.cancelSignal).toBe(claim?.admissionSignal); + + expect(claim?.requeueAdmission("user stopped admission")).toBe(true); + expect(claim?.admissionSignal.aborted).toBe(true); + const retriedAdmission = queue.dequeueNext(); + expect(retriedAdmission.message).toBe("User follow-up"); + expect(retriedAdmission.internal?.cancelSignal).toBe(controller.signal); + expect(retriedAdmission.internal?.cancelState).toEqual({ canceledBeforeAcceptance: false }); + expect(retriedAdmission.internal?.cancelState).not.toBe(cancelState); + }); + + it("does not claim a tool-end entry that cancellation already retracted", () => { + const controller = new AbortController(); + queue.add( + "Monitor wake", + { model: "gpt-4", agentId: "exec", queueDispatchMode: "tool-end" }, + { cancelSignal: controller.signal } + ); + controller.abort("task output consumed the wake"); + + expect(queue.hasLiveEntries()).toBe(false); + expect(queue.isEmpty()).toBe(false); + expect(queue.entryCount()).toBe(0); + expect(queue.claimNextToolEndEntry()).toBeUndefined(); + }); + + it("commits the claimed entry ahead of a later user reordering", () => { + const controller = new AbortController(); + queue.add( + "Monitor wake", + { model: "gpt-4", agentId: "exec", queueDispatchMode: "tool-end" }, + { synthetic: true, cancelSignal: controller.signal } + ); + queue.add("User follow-up", { + model: "gpt-4", + agentId: "exec", + queueDispatchMode: "turn-end", + }); + + const claim = queue.claimNextToolEndEntry(); + expect(queue.setVisibleQueueDispatchMode("tool-end")).toBe(true); + expect(queue.getMessages()).toEqual(["User follow-up", "Monitor wake"]); + expect(claim?.commit()).toBe(true); + + expect(queue.dequeueNext().message).toBe("Monitor wake"); + expect(queue.dequeueNext().message).toBe("User follow-up"); + }); + + it("claims the next live tool-end entry after a canceled head", () => { + const canceledController = new AbortController(); + const liveController = new AbortController(); + queue.add( + "Canceled monitor wake", + { + model: "gpt-4", + agentId: "exec", + queueDispatchMode: "turn-end", + muxMetadata: { type: "bash-monitor-wake", records: [] }, + }, + { cancelSignal: canceledController.signal } + ); + queue.add( + "Live follow-up", + { model: "gpt-4", agentId: "exec", queueDispatchMode: "tool-end" }, + { cancelSignal: liveController.signal } + ); + canceledController.abort("task output consumed the first wake"); + + expect(queue.getNextQueueDispatchMode()).toBe("tool-end"); + expect(queue.isNextEntryBashMonitorWake()).toBe(false); + expect(queue.getNextQueueCutCandidate()?.muxMetadata).toBeUndefined(); + const claim = queue.claimNextToolEndEntry(); + expect(queue.dequeueNext().internal?.cancelSignal).toBe(canceledController.signal); + expect(queue.dequeueNext().internal?.cancelSignal).toBe(claim?.admissionSignal); + }); + + it("finds a live workspace-turn continuation after a canceled predecessor", () => { + const canceledController = new AbortController(); + queue.add( + "Canceled predecessor", + { model: "gpt-4", agentId: "exec" }, + { cancelSignal: canceledController.signal } + ); + queue.add("Workspace-turn continuation", { + model: "gpt-4", + agentId: "exec", + muxMetadata: metadata, + }); + canceledController.abort("predecessor became stale"); + + expect( + queue.hasNextWorkspaceTurnContinuation("wst_followup", "parent-workspace", "turn-1") + ).toBe(true); + expect( + queue.hasAllWorkspaceTurnContinuations("wst_followup", "parent-workspace", "turn-1") + ).toBe(true); + expect((queue.getNextQueueCutCandidate()?.muxMetadata as MuxMessageMetadata).type).toBe( + "workspace-turn-task" + ); + }); + it("never batches a user message into a sealed workspace-turn entry", () => { // Cut attribution reads the head entry's muxMetadata; the sealing // invariant guarantees a manual user message queued after a diff --git a/src/node/services/messageQueue.ts b/src/node/services/messageQueue.ts index 2bc02977d6..b481521a94 100644 --- a/src/node/services/messageQueue.ts +++ b/src/node/services/messageQueue.ts @@ -93,6 +93,26 @@ type GoalInterventionPolicy = NonNullable; +/** Cancellation handoff for a claimed tool-end queue cut. */ +export interface ToolEndQueueClaim { + /** Whether the claimed entry contains user-authored input. */ + readonly userAuthored: boolean; + /** Metadata for the exact claimed entry. */ + readonly muxMetadata?: unknown; + /** Signal used to stop this exact entry during dispatch admission. */ + readonly admissionSignal: AbortSignal; + /** Move the claimed entry to the dispatch head. */ + commit(): boolean; + /** Restore the entry's cancellation when the requested queue cut does not occur. */ + restoreCancellation(): void; + /** Stop the claimed entry during queue or preparing admission. */ + cancelAdmission(reason: string): void; + /** Put a dequeued entry back at the queue head and stop its current admission. */ + requeueAdmission(reason: string): boolean; + /** Release claim ownership after the entry crosses admission. */ + release(): void; +} + /** * Input poised to take over a session at a queue cut (see * AgentSession.getQueueCutCutter). Engaged stages win over the queue head; an @@ -105,6 +125,11 @@ export type QueueCutCutter = | { stage: "dispatching"; muxMetadata: unknown } | { stage: "queued"; muxMetadata: unknown; dispatchMode: QueueDispatchMode }; +export interface QueueAdmissionCancelState { + canceledBeforeAcceptance: boolean; + providerExcludedAfterAcceptance?: boolean; +} + interface QueuedMessageInternalOptions { synthetic?: boolean; agentInitiated?: boolean; @@ -126,7 +151,7 @@ interface QueuedMessageInternalOptions { onAcceptedPreStreamFailure?: (error: SendMessageError) => Promise | void; onCanceled?: (reason: string) => Promise | void; /** Mutable dispatch outcome shared with sendQueuedMessages. */ - cancelState?: { canceledBeforeAcceptance: boolean }; + cancelState?: QueueAdmissionCancelState; /** Cancels a queued entry even after it has been dequeued into PREPARING. */ cancelSignal?: AbortSignal; /** @@ -191,7 +216,7 @@ interface QueueEntry { onCanceled?: (reason: string) => Promise | void; onAccepted?: () => Promise | void; onAcceptedPreStreamFailure?: (error: SendMessageError) => Promise | void; - cancelState?: { canceledBeforeAcceptance: boolean }; + cancelState?: QueueAdmissionCancelState; cancelSignal?: AbortSignal; /** Pre-turn rows delivered with this entry (entries carrying them are sealed). */ preTurnMessages?: MuxMessage[]; @@ -259,9 +284,87 @@ export class MessageQueue { return entries.some((entry) => entry.dispatchMode === "tool-end") ? "tool-end" : "turn-end"; } - /** Dispatch boundary for the FIFO head entry — the only entry the next drain can send. */ + private getLiveEntries(): QueueEntry[] { + return this.entries.filter((entry) => entry.cancelSignal?.aborted !== true); + } + + private getNextLiveEntry(): QueueEntry | undefined { + return this.entries.find((entry) => entry.cancelSignal?.aborted !== true); + } + + /** Dispatch boundary for the next live FIFO entry. */ getNextQueueDispatchMode(): QueueDispatchMode { - return this.entries[0]?.dispatchMode ?? "tool-end"; + return this.getNextLiveEntry()?.dispatchMode ?? "tool-end"; + } + + /** + * Claim the next live entry as the continuation for a tool-end queue cut. + * + * Once this claim stops the current model step, the original cancellation must not + * retract the continuation. A separate admission signal lets a later user Stop + * cancel the exact claimed entry through its PREPARING gates. + */ + claimNextToolEndEntry(): ToolEndQueueClaim | undefined { + const entry = this.getNextLiveEntry(); + if (entry?.dispatchMode !== "tool-end") { + return undefined; + } + + const cancelSignal = entry.cancelSignal; + const admissionController = new AbortController(); + entry.cancelSignal = admissionController.signal; + let settled = false; + let committed = false; + return { + userAuthored: entry.userAuthored, + muxMetadata: entry.muxMetadata, + admissionSignal: admissionController.signal, + commit: () => { + if (settled || committed) { + return false; + } + committed = true; + const index = this.entries.indexOf(entry); + if (index === -1) { + return false; + } + if (index > 0) { + this.entries.splice(index, 1); + this.entries.unshift(entry); + } + return true; + }, + restoreCancellation: () => { + if (settled) { + return; + } + settled = true; + entry.cancelSignal = cancelSignal; + }, + cancelAdmission: (reason) => { + if (settled) { + return; + } + settled = true; + admissionController.abort(reason); + }, + requeueAdmission: (reason) => { + if (settled || !committed || this.entries.includes(entry)) { + return false; + } + settled = true; + admissionController.abort(reason); + entry.cancelSignal = cancelSignal; + if (entry.cancelState != null) { + entry.cancelState = { canceledBeforeAcceptance: false }; + } + this.entries.unshift(entry); + return true; + }, + release: () => { + settled = true; + }, + }; } /** @@ -275,9 +378,10 @@ export class MessageQueue { ownerWorkspaceId: string, turnId: string ): boolean { + const liveEntries = this.getLiveEntries(); return ( - this.entries.length > 0 && - this.entries.every((entry) => { + liveEntries.length > 0 && + liveEntries.every((entry) => { const metadata = entry.muxMetadata; return ( isWorkspaceTurnMetadata(metadata) && @@ -297,7 +401,7 @@ export class MessageQueue { ownerWorkspaceId: string, turnId: string ): boolean { - const metadata = this.entries[0]?.muxMetadata; + const metadata = this.getNextLiveEntry()?.muxMetadata; return ( isWorkspaceTurnMetadata(metadata) && metadata.taskHandleId === taskHandleId && @@ -307,7 +411,7 @@ export class MessageQueue { } /** - * FIFO head entry's cut-attribution view: its first muxMetadata plus dispatch mode. + * Next live FIFO entry's cut-attribution view. * * Soundness of metadata-based cut attribution rests on the sealing invariant * (see class docblock): workspace-turn entries are sealed at add time and @@ -318,11 +422,11 @@ export class MessageQueue { getNextQueueCutCandidate(): | { muxMetadata: unknown; dispatchMode: QueueDispatchMode } | undefined { - const head = this.entries[0]; - if (head == null) { + const entry = this.getNextLiveEntry(); + if (entry == null) { return undefined; } - return { muxMetadata: head.muxMetadata, dispatchMode: head.dispatchMode }; + return { muxMetadata: entry.muxMetadata, dispatchMode: entry.dispatchMode }; } /** @@ -332,7 +436,7 @@ export class MessageQueue { * supersedes the turn when it dispatches. */ isNextEntryBashMonitorWake(): boolean { - const muxMetadata = this.entries[0]?.muxMetadata; + const muxMetadata = this.getNextLiveEntry()?.muxMetadata; if (typeof muxMetadata !== "object" || muxMetadata === null) return false; return (muxMetadata as Record).type === "bash-monitor-wake"; } @@ -343,7 +447,7 @@ export class MessageQueue { * otherwise turn-end. Empty queue reports the tool-end default. */ getQueueDispatchMode(): QueueDispatchMode { - return this.getDispatchMode(this.entries); + return this.getDispatchMode(this.getLiveEntries()); } /** @@ -910,6 +1014,32 @@ export class MessageQueue { this.entries = []; } + /** Remove entries whose cancellation fired before acceptance. */ + discardCanceledEntries(): Array { + const canceledEntries: Array = []; + this.entries = this.entries.filter((entry) => { + if (entry.cancelSignal?.aborted !== true) { + return true; + } + + if (entry.cancelState != null) { + entry.cancelState.canceledBeforeAcceptance = true; + } + canceledEntries.push({ + ...(entry.onCanceled != null ? { onCanceled: entry.onCanceled } : {}), + ...(entry.onAcceptedPreStreamFailure != null + ? { onAcceptedPreStreamFailure: entry.onAcceptedPreStreamFailure } + : {}), + cancelReason: + typeof entry.cancelSignal.reason === "string" + ? entry.cancelSignal.reason + : "Queued message canceled before acceptance.", + }); + return false; + }); + return canceledEntries; + } + /** * Check if queue is empty (no pending entries). */ @@ -917,13 +1047,18 @@ export class MessageQueue { return this.entries.length === 0; } + /** Whether the queue contains an entry that cancellation did not retract. */ + hasLiveEntries(): boolean { + return this.getNextLiveEntry() != null; + } + /** - * Number of pending entries, including synthetic/internal ones. Archive admission uses + * Number of live pending entries, including synthetic/internal ones. Archive admission uses * this to compare the queue against the delegated turns it is about to interrupt, so it * must count every entry — a "visible" count could hide user work behind synthetic * entries. */ entryCount(): number { - return this.entries.length; + return this.getLiveEntries().length; } } diff --git a/src/node/services/refinement/refineService.test.ts b/src/node/services/refinement/refineService.test.ts index b542a08ae9..b8500acc22 100644 --- a/src/node/services/refinement/refineService.test.ts +++ b/src/node/services/refinement/refineService.test.ts @@ -378,6 +378,27 @@ describe("RefineService", () => { expect(prompts[0]).toContain("[/workspace_trajectory]"); }); + it("excludes stopped synthetic rows from the refine transcript", async () => { + const prompts: string[] = []; + using fixture = await createFixture({ + modelFactory: () => noOpModel((prompt) => prompts.push(prompt)), + }); + await fixture.historyService.appendToHistory( + WORKSPACE_ID, + createMuxMessage("stopped-wake", "user", "Untrusted stopped process output", { + synthetic: true, + providerExcluded: true, + }) + ); + await fixture.seedTrajectory(["Continue with the requested work."]); + + const result = await fixture.service.run(WORKSPACE_ID); + + expect(result.success).toBe(true); + expect(prompts[0]).toContain("Continue with the requested work."); + expect(prompts[0]).not.toContain("Untrusted stopped process output"); + }); + it("rejects apply while another process holds the cross-process apply lock (r32)", async () => { // A second backend over the same root (XUM_ALLOW_MULTIPLE_INSTANCES=1) // shares no in-process inFlight map; the durable lockfile must reject it. diff --git a/src/node/services/refinement/refineService.ts b/src/node/services/refinement/refineService.ts index 85403777b8..c27e69a7bd 100644 --- a/src/node/services/refinement/refineService.ts +++ b/src/node/services/refinement/refineService.ts @@ -27,7 +27,11 @@ import type { LanguageModel, Tool } from "ai"; import { EXPERIMENT_IDS, type ExperimentId } from "@/common/constants/experiments"; import type { RefineAppliedEditPayload, RefineRecordPayload } from "@/common/orpc/schemas/api"; -import { createMuxMessage, type MuxMessage } from "@/common/types/message"; +import { + createMuxMessage, + filterProviderExcludedMessages, + type MuxMessage, +} from "@/common/types/message"; import { MemoryRefinementActionSchema, RefinementEvidenceSchema, @@ -935,7 +939,9 @@ export class RefineService { if (!messagesResult.success) { return Err(`could not read workspace history: ${messagesResult.error}`); } - const activeSegment = sliceMessagesForProviderFromLatestContextBoundary(messagesResult.data); + const activeSegment = filterProviderExcludedMessages( + sliceMessagesForProviderFromLatestContextBoundary(messagesResult.data) + ); // r47: fingerprint the snapshot rows for the pre-publication recheck. // Row IDs alone cannot detect same-ID rewrites: StreamManager finalizes // a streaming assistant row through updateHistory() PRESERVING its ID @@ -1172,7 +1178,9 @@ export class RefineService { const recheckBoundaryIndex = findLatestContextBoundaryIndex(recheckResult.data); const recheckBoundaryId = recheckBoundaryIndex >= 0 ? recheckResult.data[recheckBoundaryIndex].id : null; - const recheckSegment = sliceMessagesForProviderFromLatestContextBoundary(recheckResult.data); + const recheckSegment = filterProviderExcludedMessages( + sliceMessagesForProviderFromLatestContextBoundary(recheckResult.data) + ); const snapshotIsUnchangedPrefix = activeSegment.length <= recheckSegment.length && snapshotRowFingerprints.every( diff --git a/src/node/services/replay/replayRequestBuilder.ts b/src/node/services/replay/replayRequestBuilder.ts index 2259d44e25..14c7b12de4 100644 --- a/src/node/services/replay/replayRequestBuilder.ts +++ b/src/node/services/replay/replayRequestBuilder.ts @@ -28,7 +28,11 @@ import type { } from "@ai-sdk/provider"; import type { ProvidersConfigMap } from "@/common/orpc/types"; import type { PostCompactionAttachment } from "@/common/types/attachment"; -import { filterOrphanedMcpPromptSnapshots, type MuxMessage } from "@/common/types/message"; +import { + filterOrphanedMcpPromptSnapshots, + filterProviderExcludedMessages, + type MuxMessage, +} from "@/common/types/message"; import type { ThinkingLevel } from "@/common/types/thinking"; import type { AnthropicCacheTtl } from "@/common/utils/ai/cacheStrategy"; import { normalizeToCanonical } from "@/common/utils/ai/models"; @@ -194,7 +198,9 @@ export async function buildReplayRequest(inputs: ReplayRequestInputs): Promise { describe("StreamManager - stopWhen configuration", () => { type StopWhenCondition = (options: { steps: unknown[] }) => boolean; type BuildStopWhenCondition = (request: { - hasQueuedMessages?: (dispatchMode?: "tool-end" | "turn-end") => boolean; + claimQueuedToolEndMessage?: () => boolean; toolPolicy?: ToolPolicy; }) => StopWhenCondition[]; @@ -1239,7 +1239,7 @@ describe("StreamManager - stopWhen configuration", () => { function requiredToolConditionForTests(toolPolicy: ToolPolicy): StopWhenCondition { const [, , requiredToolCondition] = buildStopWhenForTests()({ - hasQueuedMessages: () => false, + claimQueuedToolEndMessage: () => false, toolPolicy, }); return requiredToolCondition; @@ -1251,7 +1251,7 @@ describe("StreamManager - stopWhen configuration", () => { test("returns step-cap and queued-message conditions with no policy", () => { let queued = false; - const stopWhen = buildStopWhenForTests()({ hasQueuedMessages: () => queued }); + const stopWhen = buildStopWhenForTests()({ claimQueuedToolEndMessage: () => queued }); expect(stopWhen).toHaveLength(3); const [maxStepCondition, queuedMessageCondition, requiredToolCondition] = stopWhen; @@ -1797,7 +1797,7 @@ describe("StreamManager - sequential tool execution", () => { headers?: Record; maxOutputTokens?: number; streamCallSettings?: Record; - hasQueuedMessages?: (dispatchMode?: "tool-end" | "turn-end") => boolean; + claimQueuedToolEndMessage?: () => boolean; toolPolicy?: ToolPolicy; toolChoice?: { type: "tool"; toolName: string }; } @@ -1906,7 +1906,7 @@ describe("StreamManager - sequential tool execution", () => { messages: [{ role: "user", content: "hello" }], system: "system", tools, - hasQueuedMessages: () => false, + claimQueuedToolEndMessage: () => false, }); createStreamResult(request, new AbortController()); @@ -5724,23 +5724,35 @@ describe("StreamManager - stopStream", () => { const streamManager = new StreamManager(historyService); // Track emitted events - const abortEvents: Array<{ workspaceId: string; messageId: string }> = []; + const abortEvents: Array<{ + workspaceId: string; + messageId: string; + metadata?: { abortTurnGeneration?: number }; + }> = []; onTurnEngineEvent( streamManager, "stream-abort", - (data: { workspaceId: string; messageId: string }) => { + (data: { + workspaceId: string; + messageId: string; + metadata?: { abortTurnGeneration?: number }; + }) => { abortEvents.push(data); } ); // Stop a stream that doesn't exist (simulates interrupt before stream-start) - const result = await streamManager.stopStream("test-workspace"); + const result = await streamManager.stopStream("test-workspace", { + abortReason: "user", + abortTurnGeneration: 7, + }); expect(result.success).toBe(true); expect(abortEvents).toHaveLength(1); expect(abortEvents[0].workspaceId).toBe("test-workspace"); // messageId is empty for synthetic abort (no actual stream existed) expect(abortEvents[0].messageId).toBe(""); + expect(abortEvents[0].metadata?.abortTurnGeneration).toBe(7); }); }); diff --git a/src/node/services/streamManager.ts b/src/node/services/streamManager.ts index 8284b05f5e..1abeb6361e 100644 --- a/src/node/services/streamManager.ts +++ b/src/node/services/streamManager.ts @@ -244,7 +244,7 @@ interface StreamRequestOptions { maxOutputTokens?: number; callSettingsOverrides?: ResolvedCallSettingsOverrides; toolPolicy?: ToolPolicy; - hasQueuedMessages?: (dispatchMode?: "tool-end" | "turn-end") => boolean; + claimQueuedToolEndMessage?: () => boolean; headers?: Record; onChunk?: StreamTextOnChunk; onStepMessages?: (messages: ModelMessage[]) => void; @@ -289,7 +289,7 @@ interface StreamRequestConfig { headers?: Record; maxOutputTokens?: number; streamCallSettings?: Omit; - hasQueuedMessages?: (dispatchMode?: "tool-end" | "turn-end") => boolean; + claimQueuedToolEndMessage?: () => boolean; /** Optional hook for callers that need chunk-level visibility during streaming. */ onChunk?: StreamTextOnChunk; /** Optional hook for callers that need the live prepared step transcript. */ @@ -1767,7 +1767,8 @@ export class StreamManager { workspaceId: WorkspaceId, streamInfo: WorkspaceStreamInfo, abortReason: StreamAbortReason, - abandonPartial?: boolean + abandonPartial?: boolean, + abortTurnGeneration?: number ): Promise { // If stream already completed normally (emitted stream-end), wait for its // finally block to finish before returning. This happens when ensureStreamSafety @@ -1791,7 +1792,13 @@ export class StreamManager { streamInfo.abortController.abort(); // Unlike checkSoftCancelStream, await cleanup (blocking) - await this.cleanupAbortedStream(workspaceId, streamInfo, abortReason, abandonPartial); + await this.cleanupAbortedStream( + workspaceId, + streamInfo, + abortReason, + abandonPartial, + abortTurnGeneration + ); } catch (error) { log.error("Error during stream cancellation:", error); // Force cleanup even if cancellation fails @@ -1829,7 +1836,8 @@ export class StreamManager { workspaceId: WorkspaceId, streamInfo: WorkspaceStreamInfo, abortReason: StreamAbortReason, - abandonPartial?: boolean + abandonPartial?: boolean, + abortTurnGeneration?: number ): Promise { // CRITICAL: Wait for processing to fully complete before cleanup // This prevents race conditions where the old stream is still running @@ -1931,7 +1939,14 @@ export class StreamManager { const abortDelivery = this.emitStreamAbort( workspaceId, streamInfo.messageId, - { usage, contextUsage, duration, providerMetadata, contextProviderMetadata }, + { + usage, + contextUsage, + duration, + providerMetadata, + contextProviderMetadata, + ...(abortTurnGeneration != null ? { abortTurnGeneration } : {}), + }, abortReason, abandonPartial, streamInfo.initialMetadata?.acpPromptId @@ -2068,7 +2083,7 @@ export class StreamManager { maxOutputTokens, callSettingsOverrides, toolPolicy, - hasQueuedMessages, + claimQueuedToolEndMessage, headers, onChunk, onStepMessages, @@ -2118,7 +2133,7 @@ export class StreamManager { maxOutputTokens: effectiveMaxOutputTokens, streamCallSettings: Object.keys(streamCallSettings).length > 0 ? streamCallSettings : undefined, - hasQueuedMessages, + claimQueuedToolEndMessage, onChunk, onStepMessages, toolPolicy, @@ -2131,7 +2146,7 @@ export class StreamManager { } private createStopWhenCondition( - request: Pick + request: Pick ): Array> { // Completion-tool stop check: completion/routing tools use explicit // success/ok markers (agent_report, propose_plan). @@ -2177,7 +2192,7 @@ export class StreamManager { // The SDK evaluates stop conditions only after every sibling tool result in the // model's current step settles. Do not move this to individual tool-call-end events: // that would abort the remaining calls the model emitted in the same batch. - () => request.hasQueuedMessages?.("tool-end") ?? false, + () => request.claimQueuedToolEndMessage?.() ?? false, hasSuccessfulRequiredToolResult, ]; } @@ -3155,7 +3170,7 @@ export class StreamManager { maxOutputTokens: fallbackState.original.maxOutputTokens, callSettingsOverrides: prepared.data.callSettingsOverrides, toolPolicy: streamInfo.request.toolPolicy, - hasQueuedMessages: streamInfo.request.hasQueuedMessages, + claimQueuedToolEndMessage: streamInfo.request.claimQueuedToolEndMessage, headers: prepared.data.headers, onChunk: streamInfo.request.onChunk, onStepMessages: streamInfo.request.onStepMessages, @@ -5043,7 +5058,12 @@ export class StreamManager { */ async stopStream( workspaceId: string, - options?: { soft?: boolean; abandonPartial?: boolean; abortReason?: StreamAbortReason } + options?: { + soft?: boolean; + abandonPartial?: boolean; + abortReason?: StreamAbortReason; + abortTurnGeneration?: number; + } ): Promise> { const typedWorkspaceId = workspaceId as WorkspaceId; const pending = this.pendingStreamStarts.get(workspaceId); @@ -5057,7 +5077,12 @@ export class StreamManager { await this.emitStreamAbort( typedWorkspaceId, pending.syntheticMessageId, - { duration: Date.now() - pending.startTime }, + { + duration: Date.now() - pending.startTime, + ...(options?.abortTurnGeneration != null + ? { abortTurnGeneration: options.abortTurnGeneration } + : {}), + }, options?.abortReason ?? "startup", options?.abandonPartial, pending.acpPromptId @@ -5077,7 +5102,9 @@ export class StreamManager { void this.emitStreamAbort( typedWorkspaceId, "", - {}, + options?.abortTurnGeneration != null + ? { abortTurnGeneration: options.abortTurnGeneration } + : {}, options?.abortReason ?? "startup", options?.abandonPartial ).catch((error) => { @@ -5101,7 +5128,8 @@ export class StreamManager { typedWorkspaceId, streamInfo, abortReason, - options?.abandonPartial + options?.abandonPartial, + options?.abortTurnGeneration ); } return Ok(undefined); diff --git a/src/node/services/turnContextAssembler.test.ts b/src/node/services/turnContextAssembler.test.ts index 189e8fb65e..e0ac13a0a5 100644 --- a/src/node/services/turnContextAssembler.test.ts +++ b/src/node/services/turnContextAssembler.test.ts @@ -112,6 +112,22 @@ async function buildSystemContextForTest(args: { } describe("prepareProviderRequestMessages", () => { + test("excludes durable rows canceled after admission", () => { + const canceledWake = createMuxMessage("canceled-wake", "user", "untrusted wake output", { + historySequence: 1, + synthetic: true, + providerExcluded: true, + }); + const userMessage = createMuxMessage("user", "user", "continue", { + historySequence: 2, + }); + + const result = prepareProviderRequestMessages([canceledWake, userMessage], "openai", "off"); + + expect(result.activeContextMessages.map((message) => message.id)).toEqual(["user"]); + expect(result.providerRequestMessages.map((message) => message.id)).toEqual(["user"]); + }); + test("slices at reset boundaries before filtering empty assistant messages", () => { const oldMessage = createMuxMessage("old-user", "user", "old context", { historySequence: 1, diff --git a/src/node/services/turnContextAssembler.ts b/src/node/services/turnContextAssembler.ts index dbcfb4bf0e..e9cfcec721 100644 --- a/src/node/services/turnContextAssembler.ts +++ b/src/node/services/turnContextAssembler.ts @@ -7,7 +7,7 @@ import * as path from "node:path"; import assert from "@/common/utils/assert"; import { ADVISOR_USAGE_GUIDANCE } from "@/common/constants/advisor"; -import type { MuxMessage } from "@/common/types/message"; +import { filterProviderExcludedMessages, type MuxMessage } from "@/common/types/message"; import type { ThinkingLevel } from "@/common/types/thinking"; import type { PostCompactionAttachment } from "@/common/types/attachment"; import { @@ -66,7 +66,9 @@ export function prepareProviderRequestMessages( contextBoundarySlicedCount: number; } { // Workflow display rows are durable UI history, not main-agent context. - const messagesWithoutWorkflowDisplay = filterWorkflowDisplayOnlyMessages(messages); + const messagesWithoutWorkflowDisplay = filterWorkflowDisplayOnlyMessages( + filterProviderExcludedMessages(messages) + ); // RLM keep-recent floor: a stamped compaction request summarizes only the older head. const activeContextMessages = excludeKeepRecentTailForCompactionRequest( sliceMessagesForProviderFromLatestContextBoundary(messagesWithoutWorkflowDisplay) diff --git a/src/node/services/turnRequestBuilder.ts b/src/node/services/turnRequestBuilder.ts index e851b13662..7719acc1d5 100644 --- a/src/node/services/turnRequestBuilder.ts +++ b/src/node/services/turnRequestBuilder.ts @@ -272,7 +272,8 @@ export interface StreamMessageOptions { allowAgentSetGoal?: boolean; workspaceGoalService?: WorkspaceGoalService; disableWorkspaceAgents?: boolean; - hasQueuedMessages?: (dispatchMode?: "tool-end" | "turn-end") => boolean; + /** Atomically claim the queued continuation before a tool-end queue cut. */ + claimQueuedToolEndMessage?: () => boolean; muxMetadata?: MuxMessageMetadata; openaiTruncationModeOverride?: "auto" | "disabled"; /** @@ -736,7 +737,7 @@ export class TurnRequestBuilder { allowAgentSetGoal, workspaceGoalService, disableWorkspaceAgents, - hasQueuedMessages, + claimQueuedToolEndMessage, openaiTruncationModeOverride, muxMetadata, minThinkingLevel: providedMinThinkingLevel, @@ -2856,7 +2857,7 @@ export class TurnRequestBuilder { maxOutputTokens, toolPolicy: effectiveToolPolicy, providedStreamToken: streamToken, - hasQueuedMessages, + claimQueuedToolEndMessage, workspaceName: metadata.name, thinkingLevel: streamThinkingLevel, headers: requestHeaders, diff --git a/src/node/services/workspaceGoalService.test.ts b/src/node/services/workspaceGoalService.test.ts index a2ad52e592..0693ed93a6 100644 --- a/src/node/services/workspaceGoalService.test.ts +++ b/src/node/services/workspaceGoalService.test.ts @@ -665,6 +665,26 @@ describe("WorkspaceGoalService", () => { expect(reconciled).toMatchObject({ status: "active" }); }); + test("chat-tail reconciliation ignores provider-excluded goal continuations", async () => { + const created = await setGoalOk(service, { + workspaceId, + objective: "Keep the stopped goal paused", + status: "paused", + }); + await appendUserHistoryMessage(historyService, workspaceId, "Canceled continuation", { + timestamp: Date.now(), + synthetic: true, + uiVisible: true, + kind: GOAL_CONTINUATION_KIND, + goalId: created.goalId, + providerExcluded: true, + }); + + const reconciled = await service.getGoal(workspaceId); + + expect(reconciled).toMatchObject({ status: "paused" }); + }); + test("pause appends a hidden user boundary so the chat tail no longer marks the goal active", async () => { await setGoalOk(service, { workspaceId, objective: "Pause from continuation" }); await appendUserHistoryMessage(historyService, workspaceId, "Continue goal", { diff --git a/src/node/services/workspaceGoalService.ts b/src/node/services/workspaceGoalService.ts index 8bff915a8f..7c853cb33b 100644 --- a/src/node/services/workspaceGoalService.ts +++ b/src/node/services/workspaceGoalService.ts @@ -25,6 +25,7 @@ import { import type { GoalBoardEntry, GoalBoardSnapshot, GoalBoardV1 } from "@/common/types/goal"; import { createMuxMessage, + isProviderExcludedMessage, isSyntheticSnapshotUserMessage, pickStartupRetrySendOptions, } from "@/common/types/message"; @@ -745,7 +746,11 @@ export class WorkspaceGoalService { let crossedOtherGoalHistory = false; for (let index = historyResult.data.length - 1; index >= 0; index -= 1) { const message = historyResult.data[index]; - if (message.role !== "user" || isSyntheticSnapshotUserMessage(message)) { + if ( + message.role !== "user" || + isProviderExcludedMessage(message) || + isSyntheticSnapshotUserMessage(message) + ) { continue; } diff --git a/src/node/services/workspaceService.test.ts b/src/node/services/workspaceService.test.ts index a73f238509..418c009d3a 100644 --- a/src/node/services/workspaceService.test.ts +++ b/src/node/services/workspaceService.test.ts @@ -23,7 +23,7 @@ import { SCRATCH_PROJECT_CONFIG_KEY } from "@/common/constants/scratch"; import type { SendMessageError } from "@/common/types/errors"; import type { ProjectsConfig } from "@/common/types/project"; import type { Config, SecretsStore } from "@/node/config"; -import type { HistoryService } from "./historyService"; +import { HistoryService } from "./historyService"; import { createTestHistoryService } from "./testHistoryService"; import type { SessionTimingService } from "./sessionTimingService"; import { SessionUsageService } from "./sessionUsageService"; @@ -248,7 +248,7 @@ describe("WorkspaceService bash monitor wake reconciler wiring", () => { ), backgroundProcessManager, }); - return { config, service, events, cleanup }; + return { config, service, events, historyService, cleanup }; } test("monitor lifecycle and shown-output events poke the reconciler", async () => { @@ -304,6 +304,101 @@ describe("WorkspaceService bash monitor wake reconciler wiring", () => { } }); + test("provider-excluded wake reads reuse unchanged shared history", async () => { + const { service, historyService, cleanup } = await createWakeWiringService(); + const ownerWorkspaceId = "cache-owner"; + const internal = service as unknown as { + providerExcludedBashMonitorWakeRecordsByWorkspace: Map; + listProviderExcludedBashMonitorWakeRecords( + workspaceId: string + ): Promise>; + }; + await historyService.appendToHistory( + ownerWorkspaceId, + createMuxMessage("excluded-wake", "user", "monitor wake", { + synthetic: true, + providerExcluded: true, + muxMetadata: { + type: "bash-monitor-wake", + records: [ + { + processId: "proc", + wakeUpdatedAt: "2026-08-31T12:00:00.000Z:12", + kind: "match", + displayName: "CI watcher", + filter: "READY", + filterExclude: false, + }, + ], + }, + }) + ); + const iterateFullHistory = spyOn(historyService, "iterateFullHistory"); + + try { + const first = await internal.listProviderExcludedBashMonitorWakeRecords(ownerWorkspaceId); + const second = await internal.listProviderExcludedBashMonitorWakeRecords(ownerWorkspaceId); + + expect(first).toEqual(second); + expect(first[0]?.processId).toBe("proc"); + expect(iterateFullHistory).toHaveBeenCalledTimes(1); + + internal.providerExcludedBashMonitorWakeRecordsByWorkspace.delete(ownerWorkspaceId); + await internal.listProviderExcludedBashMonitorWakeRecords(ownerWorkspaceId); + expect(iterateFullHistory).toHaveBeenCalledTimes(2); + } finally { + iterateFullHistory.mockRestore(); + await cleanup(); + } + }); + + test("provider-excluded wake reads refresh after a foreign history write", async () => { + const { config, service, historyService, cleanup } = await createWakeWiringService(); + const ownerWorkspaceId = "foreign-cache-owner"; + const internal = service as unknown as { + listProviderExcludedBashMonitorWakeRecords( + workspaceId: string + ): Promise>; + }; + const iterateFullHistory = spyOn(historyService, "iterateFullHistory"); + + try { + expect(await internal.listProviderExcludedBashMonitorWakeRecords(ownerWorkspaceId)).toEqual( + [] + ); + + const foreignHistoryService = new HistoryService(config); + const appendResult = await foreignHistoryService.appendToHistory( + ownerWorkspaceId, + createMuxMessage("foreign-excluded-wake", "user", "monitor wake", { + synthetic: true, + providerExcluded: true, + muxMetadata: { + type: "bash-monitor-wake", + records: [ + { + processId: "foreign-proc", + wakeUpdatedAt: "2026-08-31T12:00:00.000Z:12", + kind: "match", + displayName: "CI watcher", + filter: "READY", + filterExclude: false, + }, + ], + }, + }) + ); + expect(appendResult.success).toBe(true); + + const records = await internal.listProviderExcludedBashMonitorWakeRecords(ownerWorkspaceId); + expect(records[0]?.processId).toBe("foreign-proc"); + expect(iterateFullHistory).toHaveBeenCalledTimes(2); + } finally { + iterateFullHistory.mockRestore(); + await cleanup(); + } + }); + test("workspace removal drain waits for armed and failed-monitor registry writes", async () => { const { service, events, cleanup } = await createWakeWiringService(); let releaseArmed: (() => void) | undefined; @@ -16652,6 +16747,13 @@ describe("WorkspaceService regenerateTitle", () => { compactionEpoch: 1, }) ); + await historyService.appendToHistory( + workspaceId, + createMuxMessage("canceled-wake", "user", "Canceled process output", { + synthetic: true, + providerExcluded: true, + }) + ); await historyService.appendToHistory( workspaceId, createMuxMessage("assistant-after-boundary", "assistant", "No new user messages yet") @@ -16689,6 +16791,7 @@ describe("WorkspaceService regenerateTitle", () => { expect(context).toContain("Refactor sidebar loading"); expect(context).toContain("Compacted summary"); expect(context).toContain("No new user messages yet"); + expect(context).not.toContain("Canceled process output"); expect(context).not.toContain("omitted for brevity"); } expect(call?.[4]).toBe("Refactor sidebar loading"); @@ -17626,6 +17729,58 @@ describe("WorkspaceService interruptStream", () => { getOrCreateSessionSpy.mockRestore(); } }); + + test("restores the queue when hard-interrupt cleanup fails", async () => { + const workspaceId = "ws-interrupt-cleanup-failure-111"; + const mockConfig: MockWorkspaceConfig = { + srcDir: "/tmp/test", + sessionsDir: "/tmp/test/sessions", + generateStableId: mock(() => "test-id"), + findWorkspace: mock(() => null), + }; + const mockAIService = { + ...createStreamLifecycleMocks(), + isStreaming: mock(() => false), + getWorkspaceMetadata: mock(() => Promise.resolve({ success: false, error: "not found" })), + // eslint-disable-next-line @typescript-eslint/no-empty-function + on: mock(() => {}), + // eslint-disable-next-line @typescript-eslint/no-empty-function + off: mock(() => {}), + } as unknown as AIService; + const workspaceService = createWorkspaceServiceForTest({ + config: mockConfig, + historyService, + aiService: mockAIService, + initStateManager: mockInitStateManager as InitStateManager, + }); + const restoreQueueToInput = mock(() => undefined); + const fakeSession = { + interruptStream: mock(() => Promise.resolve(Ok(undefined))), + sendNextUserQueuedMessage: mock(() => true), + restoreQueueToInput, + }; + const getOrCreateSessionSpy = spyOn(workspaceService, "getOrCreateSession").mockReturnValue( + fakeSession as unknown as AgentSession + ); + const deletePartialSpy = spyOn(historyService, "deletePartial").mockRejectedValueOnce( + new Error("disk unavailable") + ); + + try { + const result = await workspaceService.interruptStream(workspaceId, { + abandonPartial: true, + }); + + expect(result).toEqual({ + success: false, + error: "Failed to interrupt stream: disk unavailable", + }); + expect(restoreQueueToInput).toHaveBeenCalledTimes(1); + } finally { + deletePartialSpy.mockRestore(); + getOrCreateSessionSpy.mockRestore(); + } + }); }); // --- Pure helper tests (no mocks needed) --- diff --git a/src/node/services/workspaceService.ts b/src/node/services/workspaceService.ts index 7b8db4706c..2176ad790a 100644 --- a/src/node/services/workspaceService.ts +++ b/src/node/services/workspaceService.ts @@ -182,9 +182,11 @@ import { import { UIModeSchema, type UIMode } from "@/common/types/mode"; import { createMuxMessage, + filterProviderExcludedMessages, getCompactionFollowUpContent, parseWorkspaceTurnTaskCorrelation, pickPreservedSendOptions, + type BashMonitorWakeDisplayRecord, type CompactionFollowUpRequest, type MuxMessageMetadata, type MuxMessage, @@ -782,7 +784,8 @@ function collectWorkspaceTitleContextTurns( ): WorkspaceTitleContextTurn[] { const turns: WorkspaceTitleContextTurn[] = []; - for (const message of messages) { + // A hard Stop excludes its synthetic row from every later provider request. + for (const message of filterProviderExcludedMessages([...messages])) { if (message.role !== "user" && message.role !== "assistant") { continue; } @@ -1840,6 +1843,13 @@ export class WorkspaceService extends EventEmitter implements WorkspaceHost { private readonly constructedAtMs = Date.now(); private readonly pendingBashMonitorWakeIdleWaitsByOwner = new Map>(); private readonly bashMonitorHistoryLocks = new MutexMap(); + private readonly providerExcludedBashMonitorWakeRecordsByWorkspace = new Map< + string, + { + changeToken: string; + records: Promise; + } + >(); private readonly bashMonitorRecoveryPromise: Promise; private readonly pendingBashMonitorPersistenceByWorkspace = new Map>>(); // Failed-persistence chains active per process, so a later cancellation (task_stop after a @@ -2378,6 +2388,8 @@ export class WorkspaceService extends EventEmitter implements WorkspaceHost { : undefined, }, registry: this.bashMonitorRegistryStore, + listProviderExcludedWakeRecords: (ownerWorkspaceId) => + this.listProviderExcludedBashMonitorWakeRecords(ownerWorkspaceId), onWake: (dispatch) => this.dispatchBashMonitorWake(dispatch), }); if (typeof this.backgroundProcessManager.on === "function") { @@ -2503,6 +2515,64 @@ export class WorkspaceService extends EventEmitter implements WorkspaceHost { this.pendingBashMonitorWakeIdleWaitsByOwner.set(ownerWorkspaceId, promise); } + private async readProviderExcludedBashMonitorWakeRecords( + ownerWorkspaceId: string + ): Promise { + const records: BashMonitorWakeDisplayRecord[] = []; + const result = await this.historyService.iterateFullHistory( + ownerWorkspaceId, + "backward", + (messages) => { + for (const message of messages) { + const muxMetadata = message.metadata?.muxMetadata; + if ( + message.metadata?.providerExcluded === true && + muxMetadata?.type === "bash-monitor-wake" + ) { + records.push(...muxMetadata.records); + } + } + } + ); + if (!result.success) { + throw new Error( + `Failed to read provider-excluded bash monitor wakes for ${ownerWorkspaceId}: ${result.error}` + ); + } + return records; + } + + private async listProviderExcludedBashMonitorWakeRecords( + ownerWorkspaceId: string + ): Promise { + const changeToken = await this.historyService.getProviderExclusionChangeToken(ownerWorkspaceId); + const cached = this.providerExcludedBashMonitorWakeRecordsByWorkspace.get(ownerWorkspaceId); + if (cached?.changeToken === changeToken) return cached.records; + + const records = this.readProviderExcludedBashMonitorWakeRecords(ownerWorkspaceId).then( + async (result) => { + const finalChangeToken = + await this.historyService.getProviderExclusionChangeToken(ownerWorkspaceId); + if (finalChangeToken !== changeToken) { + throw new Error( + `Provider exclusion history changed while reading ${ownerWorkspaceId}; retry reconciliation.` + ); + } + return result; + } + ); + const entry = { changeToken, records }; + this.providerExcludedBashMonitorWakeRecordsByWorkspace.set(ownerWorkspaceId, entry); + try { + return await records; + } catch (error) { + if (this.providerExcludedBashMonitorWakeRecordsByWorkspace.get(ownerWorkspaceId) === entry) { + this.providerExcludedBashMonitorWakeRecordsByWorkspace.delete(ownerWorkspaceId); + } + throw error; + } + } + private async dispatchBashMonitorWake( dispatch: BashMonitorWakeDispatch ): Promise { @@ -4004,6 +4074,13 @@ export class WorkspaceService extends EventEmitter implements WorkspaceHost { onPostCompactionStateChange: () => { this.schedulePostCompactionMetadataRefresh(workspaceId); }, + onProviderExcludedHistoryChange: () => { + // The next reconciliation reads the new durable exclusion once, then reuses it. + this.providerExcludedBashMonitorWakeRecordsByWorkspace.delete(workspaceId); + this.scheduleBashMonitorWakeReconcile(workspaceId); + }, + settleProviderExcludedWakeRecords: (records) => + this.bashMonitorWakeReconciler.settleProviderExcludedWakeRecords(workspaceId, records), // Codex P1 (PRRT_kwDOPxxmWM6cRJD-): expose service-level send // preflights (manual sends counted but not yet queued or busy) to the // session's follow-up idle probes so redispatched synthetic turns yield @@ -5947,6 +6024,7 @@ export class WorkspaceService extends EventEmitter implements WorkspaceHost { await this.bashMonitorHistoryLocks.withLock(workspaceId, () => this.bashMonitorWakeReconciler.dispose(workspaceId) ); + this.providerExcludedBashMonitorWakeRecordsByWorkspace.delete(workspaceId); // Remove session data const sessionDir = path.join(this.config.sessionsDir, workspaceId); @@ -11399,6 +11477,7 @@ export class WorkspaceService extends EventEmitter implements WorkspaceHost { options?: { soft?: boolean; abandonPartial?: boolean; sendQueuedImmediately?: boolean } ): Promise> { let releaseHardStopLatch: (() => void) | undefined; + let deferredQueueSettlementSession: AgentSession | undefined; try { this.agentTaskIntegration?.resetAutoResumeCount(workspaceId); if (!options?.soft) { @@ -11416,7 +11495,10 @@ export class WorkspaceService extends EventEmitter implements WorkspaceHost { } const session = this.getOrCreateSession(workspaceId); - const stopResult = await session.interruptStream(options); + const stopResult = await session.interruptStream({ + ...options, + deferQueueSettlement: options?.soft !== true, + }); if (!stopResult.success) { // Interrupt failed, so clear hard-interrupt suppression we set above. if (!options?.soft) { @@ -11425,6 +11507,9 @@ export class WorkspaceService extends EventEmitter implements WorkspaceHost { log.error("Failed to stop stream:", stopResult.error); return Err(stopResult.error); } + if (!options?.soft) { + deferredQueueSettlementSession = session; + } // For hard interrupts, delete partial immediately. For soft interrupts, // defer to stream-abort handler (stream is still running and may recreate partial). @@ -11465,9 +11550,18 @@ export class WorkspaceService extends EventEmitter implements WorkspaceHost { // Restore queued messages to input box for user-initiated interrupts session.restoreQueueToInput(); } + deferredQueueSettlementSession = undefined; return Ok(undefined); } catch (error) { + try { + deferredQueueSettlementSession?.restoreQueueToInput(); + } catch (settlementError) { + log.error("Failed to release deferred queue settlement after interrupt failure", { + workspaceId, + error: settlementError, + }); + } if (!options?.soft) { // Keep suppression state consistent if interrupt setup/stop throws. this.agentTaskIntegration?.resetAutoResumeCount(workspaceId); @@ -11950,7 +12044,14 @@ export class WorkspaceService extends EventEmitter implements WorkspaceHost { clear: () => Promise>, options?: { discardUnacceptedOnSuccess?: boolean } ): Promise> { - if (options?.discardUnacceptedOnSuccess !== true) return clear(); + if (options?.discardUnacceptedOnSuccess !== true) { + return clear().then((result) => { + if (result.success) { + this.providerExcludedBashMonitorWakeRecordsByWorkspace.delete(workspaceId); + } + return result; + }); + } return this.bashMonitorRecoveryPromise.then(() => this.bashMonitorHistoryLocks.withLock(workspaceId, async () => { if (this.removingWorkspaces.has(workspaceId)) { @@ -11960,6 +12061,7 @@ export class WorkspaceService extends EventEmitter implements WorkspaceHost { this.notifyBashMonitorWakeStateChanged(workspaceId); const result = await clear(); if (result.success) { + this.providerExcludedBashMonitorWakeRecordsByWorkspace.delete(workspaceId); await this.bashMonitorWakeReconciler.finishFullHistoryClear(clearToken); this.notifyBashMonitorWakeStateChanged(workspaceId); }