From d35f042459575996411b6359cdadee18aa62e4be Mon Sep 17 00:00:00 2001 From: Michael Suchacz <203725896+ibetitsmike@users.noreply.github.com> Date: Wed, 2 Sep 2026 22:33:41 +0000 Subject: [PATCH 01/28] fix(agentSession): resume turns stranded by a withdrawn queued tool-end message StreamManager's stopWhen ends the model loop at a tool boundary whenever a tool-end message is queued. When that queued entry is then withdrawn before it starts a turn (a bash-monitor wake canceled by the reconciler after the model already consumed the output, a cleared queue, or a pre-stream failure), the stream-end drain finds nothing to dispatch and the session goes idle with a tool result the model never answered. Record why the loop stopped (onQueuedMessageStop, skipped when a required tool completed the turn) and have AgentSession owe a continuation for that stop. Any stream that actually starts consumes the mark; otherwise one idempotent sweep (resumeStrandedTurnIfIdle) resumes from history at every idle transition and queue removal. The provider-executed soft-stop path owes the same continuation. Consecutive stranded resumes are capped at 3. --- .../agentSession.queueDispatch.test.ts | 237 +++++++++++++++++- src/node/services/agentSession.ts | 162 ++++++++++-- src/node/services/streamManager.test.ts | 31 +++ src/node/services/streamManager.ts | 36 ++- src/node/services/turnRequestBuilder.ts | 4 + 5 files changed, 443 insertions(+), 27 deletions(-) diff --git a/src/node/services/agentSession.queueDispatch.test.ts b/src/node/services/agentSession.queueDispatch.test.ts index a0396703a1..9c44a6d380 100644 --- a/src/node/services/agentSession.queueDispatch.test.ts +++ b/src/node/services/agentSession.queueDispatch.test.ts @@ -1,10 +1,12 @@ import { describe, expect, mock, spyOn, test } from "bun:test"; -import type { MuxMessageMetadata } from "@/common/types/message"; +import { EventEmitter } from "node:events"; + +import { createMuxMessage, type MuxMessageMetadata } from "@/common/types/message"; import { Err, Ok } from "@/common/types/result"; import type { WorkspaceGoalService } from "./workspaceGoalService"; import { createAgentSessionHarness, createStartedTurnHandle } from "./agentSession.testHarness"; -import type { AIService } from "./aiService"; +import type { AIService, StreamMessageOptions } from "./aiService"; const TEST_MODEL = "anthropic:claude-sonnet-4-5"; const WORKSPACE_TURN_CORRELATION = { @@ -49,6 +51,77 @@ function streamAbortEvent( }; } +function streamEndEvent(workspaceId: string): Record { + return { + type: "stream-end", + workspaceId, + messageId: "assistant-1", + parts: [], + metadata: { + model: TEST_MODEL, + contextUsage: { inputTokens: 5, outputTokens: 5, totalTokens: 10 }, + providerMetadata: {}, + finishReason: "tool-calls", + }, + }; +} + +/** + * Session whose engine double behaves like the real one for turn phases: every + * streamMessage call emits stream-start before resolving, so the session is STREAMING + * (not back to IDLE) once a send or resume returns. + */ +async function createStreamingTurnHarness(workspaceId: string) { + const aiEmitter = new EventEmitter(); + const streamMessage = mock((_options: StreamMessageOptions) => { + aiEmitter.emit("stream-start", streamStartEvent(workspaceId)); + return Promise.resolve(Ok(createStartedTurnHandle("assistant-1"))); + }); + const harness = await createAgentSessionHarness({ + workspaceId, + aiEmitter, + aiServiceOverrides: { streamMessage: streamMessage as unknown as AIService["streamMessage"] }, + }); + const sent = await harness.session.sendMessage("run the checks", { + model: TEST_MODEL, + agentId: "exec", + }); + expect(sent.success).toBe(true); + expect(streamMessage).toHaveBeenCalledTimes(1); + expect(harness.session.isBusy()).toBe(true); + // The tool step's committed assistant row: the row the model never answered when stranded. + await harness.historyService.appendToHistory( + workspaceId, + createMuxMessage("assistant-1", "assistant", "ran the checks", { timestamp: Date.now() }) + ); + + /** Queue a synthetic wake whose cancel signal the caller controls. */ + const queueCancelableWake = (): AbortController => { + const controller = new AbortController(); + harness.session.queueMessage( + "Background monitor wake", + { model: TEST_MODEL, agentId: "exec" }, + { + synthetic: true, + agentInitiated: true, + cancelState: { canceledBeforeAcceptance: false }, + cancelSignal: controller.signal, + onCanceled: () => undefined, + } + ); + return controller; + }; + const latestRequest = (): StreamMessageOptions => { + const call = streamMessage.mock.calls[streamMessage.mock.calls.length - 1]; + if (call == null) { + throw new Error("no streamMessage call recorded"); + } + return call[0]; + }; + + return { ...harness, streamMessage, queueCancelableWake, latestRequest }; +} + async function waitForCondition(condition: () => boolean, timeoutMs = 500): Promise { const deadline = Date.now() + timeoutMs; while (Date.now() < deadline) { @@ -704,6 +777,166 @@ describe("AgentSession queued message tool-call dispatch", () => { } }); + test("resumes a turn whose queued tool-end stop message is withdrawn before acceptance", async () => { + const workspaceId = "queue-dispatch-stranded-resume"; + const harness = await createStreamingTurnHarness(workspaceId); + const { session, cleanup, aiEmitter, historyService, streamMessage } = harness; + + try { + const wake = harness.queueCancelableWake(); + const request = harness.latestRequest(); + expect(request.hasQueuedMessages?.("tool-end")).toBe(true); + // StreamManager stopped the loop for the queued wake; the wake is then withdrawn + // (its output was consumed another way) before the stream-end drain dispatches it. + request.onQueuedMessageStop?.(); + wake.abort("monitor consumed"); + aiEmitter.emit("stream-end", streamEndEvent(workspaceId)); + + expect(await waitForCondition(() => streamMessage.mock.calls.length === 2)).toBe(true); + const resumed = harness.latestRequest(); + expect(resumed.agentInitiated).toBe(true); + expect(resumed.modelString).toBe(TEST_MODEL); + expect(resumed.agentId).toBe("exec"); + // The resumed request ends with a user turn so the model has something to answer. + expect(resumed.messages[resumed.messages.length - 1]?.role).toBe("user"); + expect(session.isBusy()).toBe(true); + expect(session.hasQueuedMessages()).toBe(false); + + await new Promise((resolve) => setTimeout(resolve, 25)); + expect(streamMessage).toHaveBeenCalledTimes(2); + const history = await historyService.getHistoryFromLatestBoundary(workspaceId); + expect(history.success).toBe(true); + if (history.success) { + expect( + history.data.some((message) => + message.parts.some( + (part) => part.type === "text" && part.text === "Background monitor wake" + ) + ) + ).toBe(false); + } + } finally { + session.dispose(); + await cleanup(); + } + }); + + test("a queued tool-end message that dispatches normally is the continuation", async () => { + const workspaceId = "queue-dispatch-stranded-dispatched"; + const harness = await createStreamingTurnHarness(workspaceId); + const { session, cleanup, aiEmitter, streamMessage } = harness; + + try { + harness.queueCancelableWake(); + harness.latestRequest().onQueuedMessageStop?.(); + aiEmitter.emit("stream-end", streamEndEvent(workspaceId)); + + expect(await waitForCondition(() => streamMessage.mock.calls.length === 2)).toBe(true); + const dispatched = harness.latestRequest(); + const lastMessage = dispatched.messages[dispatched.messages.length - 1]; + expect(lastMessage?.role).toBe("user"); + expect( + lastMessage?.parts.some( + (part) => part.type === "text" && part.text === "Background monitor wake" + ) + ).toBe(true); + expect(session.isBusy()).toBe(true); + + await new Promise((resolve) => setTimeout(resolve, 25)); + expect(streamMessage).toHaveBeenCalledTimes(2); + } finally { + session.dispose(); + await cleanup(); + } + }); + + test("does not resume when the loop ended without stopping for a queued message", async () => { + const workspaceId = "queue-dispatch-stranded-not-stopped"; + const harness = await createStreamingTurnHarness(workspaceId); + const { session, cleanup, aiEmitter, streamMessage } = harness; + + try { + // A required tool (agent_report) ended the turn while a wake happened to be queued. + const wake = harness.queueCancelableWake(); + wake.abort("monitor consumed"); + aiEmitter.emit("stream-end", streamEndEvent(workspaceId)); + + expect(await waitForCondition(() => !session.isBusy())).toBe(true); + await new Promise((resolve) => setTimeout(resolve, 25)); + expect(streamMessage).toHaveBeenCalledTimes(1); + expect(session.isBusy()).toBe(false); + } finally { + session.dispose(); + await cleanup(); + } + }); + + test("caps consecutive stranded resumes", async () => { + const workspaceId = "queue-dispatch-stranded-cap"; + const harness = await createStreamingTurnHarness(workspaceId); + const { session, cleanup, aiEmitter, streamMessage } = harness; + + try { + const strandTurn = () => { + harness.latestRequest().onQueuedMessageStop?.(); + harness.queueCancelableWake().abort("monitor consumed"); + aiEmitter.emit("stream-end", streamEndEvent(workspaceId)); + }; + + for (let resumes = 1; resumes <= 3; resumes += 1) { + strandTurn(); + expect(await waitForCondition(() => streamMessage.mock.calls.length === resumes + 1)).toBe( + true + ); + expect(session.isBusy()).toBe(true); + } + + strandTurn(); + expect(await waitForCondition(() => !session.isBusy())).toBe(true); + await new Promise((resolve) => setTimeout(resolve, 25)); + expect(streamMessage).toHaveBeenCalledTimes(4); + expect(session.isBusy()).toBe(false); + } finally { + session.dispose(); + await cleanup(); + } + }); + + test("resumes after a provider-tool soft stop whose queued message was withdrawn", async () => { + const workspaceId = "queue-dispatch-stranded-provider-tool"; + const harness = await createStreamingTurnHarness(workspaceId); + const { session, cleanup, aiEmitter, aiService, streamMessage } = harness; + const stopStream = spyOn(aiService, "stopStream").mockResolvedValue(Ok(undefined)); + + try { + session.queueMessage( + "follow up", + { model: TEST_MODEL, agentId: "exec" }, + { synthetic: true, dedupeKey: "wake:1" } + ); + aiEmitter.emit("tool-call-end", { + ...toolCallEndEvent(workspaceId), + toolName: "web_search", + providerExecuted: true, + }); + expect(stopStream).toHaveBeenCalledTimes(1); + + // Withdrawn between the soft stop request and the abort it produces. + expect(session.removeQueuedMessagesByDedupeKeyPrefix("wake:", "superseded")).toBe(1); + aiEmitter.emit("stream-abort", streamAbortEvent(workspaceId, "system")); + + expect(await waitForCondition(() => streamMessage.mock.calls.length === 2)).toBe(true); + expect(harness.latestRequest().agentInitiated).toBe(true); + expect(session.isBusy()).toBe(true); + await new Promise((resolve) => setTimeout(resolve, 25)); + expect(streamMessage).toHaveBeenCalledTimes(2); + } finally { + stopStream.mockRestore(); + session.dispose(); + await cleanup(); + } + }); + test("rollback failure preserves the wake and continues acceptance", async () => { const workspaceId = "queue-dispatch-cancel-rollback-failure"; const { session, cleanup, historyService } = await createAgentSessionHarness({ workspaceId }); diff --git a/src/node/services/agentSession.ts b/src/node/services/agentSession.ts index 7c360101ac..fa1903a2f8 100644 --- a/src/node/services/agentSession.ts +++ b/src/node/services/agentSession.ts @@ -365,6 +365,21 @@ function getWorkspaceTurnMuxMetadata(muxMetadata: unknown): WorkspaceTurnMuxMeta return metadata?.type === "workspace-turn-task" ? metadata : undefined; } +/** + * Options for resuming a turn stranded by a withdrawn queued message: the ended stream's + * settings minus anything that would replay its dispatch (edit target, queue dispatch mode) + * or re-stamp per-message metadata (compaction, skill, wake). Only the workspace-turn + * correlation survives because the delegating owner still waits on this continuation. + */ +function buildStrandedTurnResumeOptions(options: SendMessageOptions): SendMessageOptions { + const { + editMessageId: _editMessageId, + queueDispatchMode: _queueDispatchMode, + ...resumeOptions + } = options; + return { ...resumeOptions, muxMetadata: getWorkspaceTurnMuxMetadata(options.muxMetadata) }; +} + function hasSameWorkspaceTurnCorrelation( first: WorkspaceTurnMuxMetadata | undefined, second: WorkspaceTurnMuxMetadata | undefined @@ -518,6 +533,11 @@ export const CONTEXT_MUTATION_SEND_BLOCKED_MESSAGE = const STARTUP_AUTO_RETRY_HISTORY_FAILURE_BASE_DELAY_MS = 1_000; const STARTUP_AUTO_RETRY_HISTORY_FAILURE_MAX_DELAY_MS = 30_000; const MAX_STARTUP_RECOVERY_DEFERRED_ATTEMPTS = 4; +/** + * Runaway guard for pathological queue flapping (stop for a queued message, withdraw it, + * repeat): after this many back-to-back stranded resumes the turn is left idle. + */ +const MAX_CONSECUTIVE_STRANDED_TURN_RESUMES = 3; export interface AgentSessionChatEvent { workspaceId: string; @@ -695,6 +715,13 @@ export class AgentSession { // Track known siblings and reserve soft interruption for that native-only boundary. private queuedProviderToolEndAbortInFlight = false; private readonly activeToolCallIds = new Set(); + // The model loop stops at a tool boundary on behalf of a queued tool-end message. If that + // message never starts a turn (canceled wake, cleared queue, pre-stream failure), the stop + // would strand the turn on an unanswered tool result, so the continuation is owed here until + // a stream actually starts (setTurnPhase STREAMING) or the sweep resumes it. + private strandedTurnResume?: SendMessageOptions; + private activeStreamStoppedForQueuedMessage = false; + private consecutiveStrandedResumes = 0; private idleWaiters: Array<() => void> = []; private pendingExternalManualFollowUps = 0; @@ -4028,6 +4055,9 @@ export class AgentSession { // Synthetic/system sends (mid-stream compaction, task recovery prompts, etc.) // must not silently opt users back into auto-retry after they've disabled it. if (isManualUserMessage) { + // The user's own message supersedes any continuation owed to a stranded turn. + this.strandedTurnResume = undefined; + this.consecutiveStrandedResumes = 0; // A fresh accepted user send supersedes any persisted startup-abandon // classification from previous turns. await this.clearStartupAutoRetryAbandon(); @@ -5002,6 +5032,7 @@ export class AgentSession { providersConfig, }; this.activeStreamUserMessageId = undefined; + this.activeStreamStoppedForQueuedMessage = false; const commitResult = await this.historyService.commitPartial(this.workspaceId); if (!commitResult.success) { @@ -5211,6 +5242,9 @@ export class AgentSession { disableWorkspaceAgents: options?.disableWorkspaceAgents, strictAgentResolution: options?.strictAgentResolution, hasQueuedMessages: this.hasQueuedMessages.bind(this), + onQueuedMessageStop: () => { + this.activeStreamStoppedForQueuedMessage = true; + }, openaiTruncationModeOverride, // Mid-turn thinking overrides clamp against the same floor as the // send-time level above (single source of truth for the floor). @@ -5979,6 +6013,7 @@ export class AgentSession { const failedUserMessageId = this.activeStreamUserMessageId; const hadCompactionRequest = this.activeCompactionRequest !== undefined; + const abortedStreamOptions = this.activeStreamContext?.options; const abortReason = "abortReason" in payload ? payload.abortReason : undefined; const isQueuedProviderToolEndAbort = this.queuedProviderToolEndAbortInFlight && abortReason !== "user"; @@ -6018,10 +6053,13 @@ export class AgentSession { } await this.updateStartupAutoRetryAbandonFromAbort(abortReason, failedUserMessageId); this.emitChatEvent(payload); - const dispatchedQueuedMessage = - this.dispatchQueuedProviderToolEndMessageAfterAbort(abortReason); + const dispatchedQueuedMessage = this.dispatchQueuedProviderToolEndMessageAfterAbort( + abortReason, + abortedStreamOptions + ); if (!dispatchedQueuedMessage) { this.setTurnPhase(TurnPhase.IDLE); + this.resumeStrandedTurnIfIdle(); } }); forward("runtime-status", (payload) => { @@ -6044,6 +6082,7 @@ export class AgentSession { const streamEndPayload = payload; const activeStreamGoalKind = this.activeStreamContext?.goalKind; const activeStreamOptions = this.activeStreamContext?.options; + const stoppedForQueuedMessage = this.activeStreamStoppedForQueuedMessage; let goalContinuationRequest: { sendOptions: SendMessageOptions; @@ -6142,10 +6181,29 @@ export class AgentSession { // Do not dispatch stream-end follow-ups while the edit flow is waiting // for IDLE; truncation must run before any synthetic turn resumes. } else { + if (!handled) { + if (stoppedForQueuedMessage) { + // The queued dispatch below normally consumes this; it stays owed only when + // that message does not start a turn. + this.strandedTurnResume = buildStrandedTurnResumeOptions( + activeStreamOptions ?? { + model: streamEndPayload.metadata.model, + agentId: WORKSPACE_DEFAULTS.agentId, + } + ); + } else { + this.consecutiveStrandedResumes = 0; + } + } this.sendQueuedMessages(); } - if (!handled && !this.deferQueuedFlushUntilAfterEdit && !hadQueuedMessages) { + if ( + !handled && + !this.deferQueuedFlushUntilAfterEdit && + !hadQueuedMessages && + this.strandedTurnResume == null + ) { const sendOptions = activeStreamOptions ?? { model: streamEndPayload.metadata.model, agentId: WORKSPACE_DEFAULTS.agentId, @@ -6201,6 +6259,7 @@ export class AgentSession { if (this.turnPhase === TurnPhase.COMPLETING) { this.resetActiveStreamState(); this.setTurnPhase(TurnPhase.IDLE); + this.resumeStrandedTurnIfIdle(); if (goalContinuationRequest != null) { await this.workspaceGoalService?.requestContinuationAfterStreamEnd({ workspaceId: this.workspaceId, @@ -6284,6 +6343,12 @@ export class AgentSession { this.emitStreamLifecycleIfChanged(); + if (next === TurnPhase.STREAMING) { + // Any stream that actually starts is the continuation the stranded turn was waiting + // for. PREPARING is not enough: a dequeued entry can still be canceled before acceptance. + this.strandedTurnResume = undefined; + } + if (next === TurnPhase.IDLE) { this.dispatchingQueuedEntry = false; this.dispatchingQueuedEntryMuxMetadata = undefined; @@ -6344,6 +6409,7 @@ export class AgentSession { async discardAutoRetryForContextMutation(): Promise> { this.retryManager.cancel(); this.setAutoRetryResumeState(undefined); + this.strandedTurnResume = undefined; const deleteResult = await this.historyService.deletePartial(this.workspaceId); if (!deleteResult.success) { return Err(deleteResult.error); @@ -6381,14 +6447,14 @@ export class AgentSession { // Entries left queued while the block was held have no stream-end // drain to dispatch them (the session stayed idle throughout) — // drain now, mirroring the edit-admission release. Only when entries - // exist: releases from a session that never queued must stay - // side-effect free. - if ( - this.turnAdmissionBlocks === 0 && - this.turnPhase === TurnPhase.IDLE && - !this.messageQueue.isEmpty() - ) { - this.sendQueuedMessages(); + // (or an owed stranded resume) exist: releases from a session that + // never queued must stay side-effect free. + if (this.turnAdmissionBlocks === 0 && this.turnPhase === TurnPhase.IDLE) { + if (!this.messageQueue.isEmpty()) { + this.sendQueuedMessages(); + } else { + this.resumeStrandedTurnIfIdle(); + } } }, }; @@ -6543,6 +6609,7 @@ export class AgentSession { for (const callbacks of callbackSets) { this.notifyQueuedMessageCleared(callbacks, cancelReason); } + this.resumeStrandedTurnIfIdle(); } setQueuedMessageDispatchMode(mode: "tool-end" | "turn-end"): boolean { @@ -6599,6 +6666,7 @@ export class AgentSession { for (const callbacks of removal.callbacks) { this.notifyQueuedMessageCleared(callbacks, cancelReason); } + this.resumeStrandedTurnIfIdle(); return removal.removedCount; } @@ -6625,6 +6693,7 @@ export class AgentSession { !this.messageQueue.isEmpty() && this.messageQueue.getNextQueueDispatchMode() === "tool-end" ); this.notifyQueuedMessageCleared(callbacks, cancelReason); + this.resumeStrandedTurnIfIdle(); return true; } @@ -6795,17 +6864,24 @@ export class AgentSession { } private dispatchQueuedProviderToolEndMessageAfterAbort( - abortReason: StreamAbortReason | undefined + abortReason: StreamAbortReason | undefined, + abortedStreamOptions: SendMessageOptions | undefined ): boolean { if (!this.queuedProviderToolEndAbortInFlight) { return false; } - - const shouldDispatch = - abortReason !== "user" && !this.deferQueuedFlushUntilAfterEdit && this.hasQueuedMessages(); this.queuedProviderToolEndAbortInFlight = false; - if (!shouldDispatch) { + if (abortReason === "user" || this.deferQueuedFlushUntilAfterEdit) { + return false; + } + + // The soft stop was made on behalf of the queued message; if that message has been + // withdrawn (or is withdrawn after dequeue), the interrupted turn must resume instead. + if (abortedStreamOptions != null) { + this.strandedTurnResume = buildStrandedTurnResumeOptions(abortedStreamOptions); + } + if (!this.hasQueuedMessages()) { return false; } @@ -6813,6 +6889,57 @@ export class AgentSession { return true; } + /** + * One idempotent sweep for the owed continuation (see strandedTurnResume). Cheap enough to + * run from every idle transition and queue removal; only the first eligible call acts. + */ + private resumeStrandedTurnIfIdle(): void { + const options = this.strandedTurnResume; + if ( + options == null || + this.disposed || + this.turnAdmissionBlocks > 0 || + this.hasActiveOrPendingTurnWork() || + !this.messageQueue.isEmpty() || + this.hasPendingAutoRetry() || + this.deferQueuedFlushUntilAfterEdit + ) { + return; + } + this.strandedTurnResume = undefined; + + if (this.consecutiveStrandedResumes >= MAX_CONSECUTIVE_STRANDED_TURN_RESUMES) { + log.warn("Leaving stranded turn idle: consecutive resume cap reached", { + workspaceId: this.workspaceId, + cap: MAX_CONSECUTIVE_STRANDED_TURN_RESUMES, + }); + return; + } + this.consecutiveStrandedResumes += 1; + log.info("Resuming turn stranded by a withdrawn queued message", { + workspaceId: this.workspaceId, + attempt: this.consecutiveStrandedResumes, + }); + + this.resumeStream(options, { agentInitiated: true }) + .then((result) => { + if (!result.success) { + log.warn("Stranded turn resume failed", { + workspaceId: this.workspaceId, + error: result.error, + }); + } else if (!result.data.started) { + log.warn("Stranded turn resume did not start", { workspaceId: this.workspaceId }); + } + }) + .catch((error: unknown) => { + log.warn("Stranded turn resume threw", { + workspaceId: this.workspaceId, + error: getErrorMessage(error), + }); + }); + } + async waitForPendingCompactionCompletionDecision(messageId: string): Promise { if (!this.compactionCompletionDecisions.has(messageId)) { if (this.activeCompactionRequest == null) return false; @@ -6921,6 +7048,7 @@ export class AgentSession { * failed-startup drains elsewhere in this file. */ drainQueuedMessagesIfIdle(): void { + this.resumeStrandedTurnIfIdle(); if ( this.hasActiveOrPendingTurnWork() || this.deferQueuedFlushUntilAfterEdit || @@ -7027,6 +7155,8 @@ export class AgentSession { } this.sendQueuedMessages(); }); + } else { + this.resumeStrandedTurnIfIdle(); } } diff --git a/src/node/services/streamManager.test.ts b/src/node/services/streamManager.test.ts index 4f5337ffc0..3ab85ecdd1 100644 --- a/src/node/services/streamManager.test.ts +++ b/src/node/services/streamManager.test.ts @@ -1227,6 +1227,7 @@ describe("StreamManager - stopWhen configuration", () => { type StopWhenCondition = (options: { steps: unknown[] }) => boolean; type BuildStopWhenCondition = (request: { hasQueuedMessages?: (dispatchMode?: "tool-end" | "turn-end") => boolean; + onQueuedMessageStop?: () => void; toolPolicy?: ToolPolicy; }) => StopWhenCondition[]; @@ -1266,6 +1267,36 @@ describe("StreamManager - stopWhen configuration", () => { ); }); + test("queued-message stop reports itself only when no required tool completed", () => { + let queued = false; + let stopsForQueuedMessage = 0; + const [, queuedMessageCondition] = buildStopWhenForTests()({ + hasQueuedMessages: () => queued, + onQueuedMessageStop: () => { + stopsForQueuedMessage += 1; + }, + toolPolicy: [{ regex_match: "agent_report", action: "require" }], + }); + const bashStep = stepsWithToolResult("bash", { success: true }); + + expect(queuedMessageCondition(bashStep)).toBe(false); + expect(stopsForQueuedMessage).toBe(0); + + queued = true; + expect(queuedMessageCondition(bashStep)).toBe(true); + expect(stopsForQueuedMessage).toBe(1); + + expect(queuedMessageCondition(stepsWithToolResult("agent_report", { success: true }))).toBe( + true + ); + expect(stopsForQueuedMessage).toBe(1); + + expect(queuedMessageCondition(stepsWithToolResult("agent_report", { success: false }))).toBe( + true + ); + expect(stopsForQueuedMessage).toBe(2); + }); + const requiredToolCases: Array<{ name: string; toolPolicy: ToolPolicy; diff --git a/src/node/services/streamManager.ts b/src/node/services/streamManager.ts index 8284b05f5e..6f9fbee0a7 100644 --- a/src/node/services/streamManager.ts +++ b/src/node/services/streamManager.ts @@ -245,6 +245,7 @@ interface StreamRequestOptions { callSettingsOverrides?: ResolvedCallSettingsOverrides; toolPolicy?: ToolPolicy; hasQueuedMessages?: (dispatchMode?: "tool-end" | "turn-end") => boolean; + onQueuedMessageStop?: () => void; headers?: Record; onChunk?: StreamTextOnChunk; onStepMessages?: (messages: ModelMessage[]) => void; @@ -290,6 +291,12 @@ interface StreamRequestConfig { maxOutputTokens?: number; streamCallSettings?: Omit; hasQueuedMessages?: (dispatchMode?: "tool-end" | "turn-end") => boolean; + /** + * Invoked when the loop stops on behalf of a queued tool-end message (and not + * because a required tool completed). The session uses it to resume the turn + * if that queued message is later withdrawn instead of starting a turn. + */ + onQueuedMessageStop?: () => void; /** Optional hook for callers that need chunk-level visibility during streaming. */ onChunk?: StreamTextOnChunk; /** Optional hook for callers that need the live prepared step transcript. */ @@ -2069,6 +2076,7 @@ export class StreamManager { callSettingsOverrides, toolPolicy, hasQueuedMessages, + onQueuedMessageStop, headers, onChunk, onStepMessages, @@ -2119,6 +2127,7 @@ export class StreamManager { streamCallSettings: Object.keys(streamCallSettings).length > 0 ? streamCallSettings : undefined, hasQueuedMessages, + onQueuedMessageStop, onChunk, onStepMessages, toolPolicy, @@ -2131,7 +2140,7 @@ export class StreamManager { } private createStopWhenCondition( - request: Pick + request: Pick ): Array> { // Completion-tool stop check: completion/routing tools use explicit // success/ok markers (agent_report, propose_plan). @@ -2172,14 +2181,22 @@ export class StreamManager { ); }; - return [ - stepCountIs(100000), - // The SDK evaluates stop conditions only after every sibling tool result in the - // model's current step settles. Do not move this to individual tool-call-end events: - // that would abort the remaining calls the model emitted in the same batch. - () => request.hasQueuedMessages?.("tool-end") ?? false, - hasSuccessfulRequiredToolResult, - ]; + // The SDK evaluates stop conditions only after every sibling tool result in the + // model's current step settles. Do not move this to individual tool-call-end events: + // that would abort the remaining calls the model emitted in the same batch. + const hasQueuedToolEndMessage: ReturnType = (state) => { + if (!(request.hasQueuedMessages?.("tool-end") ?? false)) { + return false; + } + // A successful required tool result is a legitimate end of turn on its own; + // only a stop made purely for the queued message may need resuming later. + if (!hasSuccessfulRequiredToolResult(state)) { + request.onQueuedMessageStop?.(); + } + return true; + }; + + return [stepCountIs(100000), hasQueuedToolEndMessage, hasSuccessfulRequiredToolResult]; } /** @@ -3156,6 +3173,7 @@ export class StreamManager { callSettingsOverrides: prepared.data.callSettingsOverrides, toolPolicy: streamInfo.request.toolPolicy, hasQueuedMessages: streamInfo.request.hasQueuedMessages, + onQueuedMessageStop: streamInfo.request.onQueuedMessageStop, headers: prepared.data.headers, onChunk: streamInfo.request.onChunk, onStepMessages: streamInfo.request.onStepMessages, diff --git a/src/node/services/turnRequestBuilder.ts b/src/node/services/turnRequestBuilder.ts index e851b13662..d8eee22052 100644 --- a/src/node/services/turnRequestBuilder.ts +++ b/src/node/services/turnRequestBuilder.ts @@ -273,6 +273,8 @@ export interface StreamMessageOptions { workspaceGoalService?: WorkspaceGoalService; disableWorkspaceAgents?: boolean; hasQueuedMessages?: (dispatchMode?: "tool-end" | "turn-end") => boolean; + /** Fires when the model loop stops solely on behalf of a queued tool-end message. */ + onQueuedMessageStop?: () => void; muxMetadata?: MuxMessageMetadata; openaiTruncationModeOverride?: "auto" | "disabled"; /** @@ -737,6 +739,7 @@ export class TurnRequestBuilder { workspaceGoalService, disableWorkspaceAgents, hasQueuedMessages, + onQueuedMessageStop, openaiTruncationModeOverride, muxMetadata, minThinkingLevel: providedMinThinkingLevel, @@ -2857,6 +2860,7 @@ export class TurnRequestBuilder { toolPolicy: effectiveToolPolicy, providedStreamToken: streamToken, hasQueuedMessages, + onQueuedMessageStop, workspaceName: metadata.name, thinkingLevel: streamThinkingLevel, headers: requestHeaders, From 91ca0eecc8b36cc61a0c9dbcc807ae8cc5743e0d Mon Sep 17 00:00:00 2001 From: Michael Suchacz <203725896+ibetitsmike@users.noreply.github.com> Date: Wed, 2 Sep 2026 23:29:47 +0000 Subject: [PATCH 02/28] fix(agentSession): owe the stranded continuation from the stop decision, keep it until a stream starts Codex round 1: record the owed continuation synchronously so delegated-turn settlement sees it, resume with the resolved workspace-turn correlation and goal attribution, yield to manual sends in preflight, keep the continuation owed when the resume fails pre-start, and do not report queue stops that coincide with the step cap. --- .../agentSession.queueDispatch.test.ts | 291 +++++++++++++++++- src/node/services/agentSession.testHarness.ts | 2 + src/node/services/agentSession.ts | 147 ++++++--- src/node/services/streamManager.test.ts | 18 ++ src/node/services/streamManager.ts | 14 +- 5 files changed, 420 insertions(+), 52 deletions(-) diff --git a/src/node/services/agentSession.queueDispatch.test.ts b/src/node/services/agentSession.queueDispatch.test.ts index 9c44a6d380..0955d721cd 100644 --- a/src/node/services/agentSession.queueDispatch.test.ts +++ b/src/node/services/agentSession.queueDispatch.test.ts @@ -2,11 +2,18 @@ import { describe, expect, mock, spyOn, test } from "bun:test"; import { EventEmitter } from "node:events"; +import type { SendMessageOptions } from "@/common/orpc/types"; import { createMuxMessage, type MuxMessageMetadata } from "@/common/types/message"; import { Err, Ok } from "@/common/types/result"; +import { GOAL_CONTINUATION_KIND } from "@/constants/goals"; import type { WorkspaceGoalService } from "./workspaceGoalService"; -import { createAgentSessionHarness, createStartedTurnHandle } from "./agentSession.testHarness"; +import { + createAgentSessionHarness, + createStartedTurnHandle, + type AgentSessionHarnessOptions, +} from "./agentSession.testHarness"; import type { AIService, StreamMessageOptions } from "./aiService"; +import type { HistoryService } from "./historyService"; const TEST_MODEL = "anthropic:claude-sonnet-4-5"; const WORKSPACE_TURN_CORRELATION = { @@ -71,21 +78,32 @@ function streamEndEvent(workspaceId: string): Record { * streamMessage call emits stream-start before resolving, so the session is STREAMING * (not back to IDLE) once a send or resume returns. */ -async function createStreamingTurnHarness(workspaceId: string) { +async function createStreamingTurnHarness( + workspaceId: string, + setup?: { + harness?: Partial>; + seedHistory?: (historyService: HistoryService) => Promise; + sendOptions?: Partial; + sendInternal?: { synthetic?: boolean; agentInitiated?: boolean }; + } +) { const aiEmitter = new EventEmitter(); const streamMessage = mock((_options: StreamMessageOptions) => { aiEmitter.emit("stream-start", streamStartEvent(workspaceId)); return Promise.resolve(Ok(createStartedTurnHandle("assistant-1"))); }); const harness = await createAgentSessionHarness({ + ...setup?.harness, workspaceId, aiEmitter, aiServiceOverrides: { streamMessage: streamMessage as unknown as AIService["streamMessage"] }, }); - const sent = await harness.session.sendMessage("run the checks", { - model: TEST_MODEL, - agentId: "exec", - }); + await setup?.seedHistory?.(harness.historyService); + const sent = await harness.session.sendMessage( + "run the checks", + { model: TEST_MODEL, agentId: "exec", ...setup?.sendOptions }, + setup?.sendInternal + ); expect(sent.success).toBe(true); expect(streamMessage).toHaveBeenCalledTimes(1); expect(harness.session.isBusy()).toBe(true); @@ -794,7 +812,8 @@ describe("AgentSession queued message tool-call dispatch", () => { expect(await waitForCondition(() => streamMessage.mock.calls.length === 2)).toBe(true); const resumed = harness.latestRequest(); - expect(resumed.agentInitiated).toBe(true); + // The continuation keeps the interrupted (user-started) turn's attribution. + expect(resumed.agentInitiated).toBe(streamMessage.mock.calls[0]?.[0].agentInitiated); expect(resumed.modelString).toBe(TEST_MODEL); expect(resumed.agentId).toBe("exec"); // The resumed request ends with a user turn so the model has something to answer. @@ -926,7 +945,9 @@ describe("AgentSession queued message tool-call dispatch", () => { aiEmitter.emit("stream-abort", streamAbortEvent(workspaceId, "system")); expect(await waitForCondition(() => streamMessage.mock.calls.length === 2)).toBe(true); - expect(harness.latestRequest().agentInitiated).toBe(true); + expect(harness.latestRequest().agentInitiated).toBe( + streamMessage.mock.calls[0]?.[0].agentInitiated + ); expect(session.isBusy()).toBe(true); await new Promise((resolve) => setTimeout(resolve, 25)); expect(streamMessage).toHaveBeenCalledTimes(2); @@ -937,6 +958,260 @@ describe("AgentSession queued message tool-call dispatch", () => { } }); + test("owes the delegated turn its continuation from the stop decision onward", async () => { + const workspaceId = "queue-dispatch-stranded-delegated"; + const harness = await createStreamingTurnHarness(workspaceId, { + sendOptions: { muxMetadata: WORKSPACE_TURN_CORRELATION }, + }); + const { session, cleanup, aiEmitter, streamMessage } = harness; + + try { + // Nothing owed yet: a tool-calls cut with an empty queue is a plain interruption. + expect(session.hasPendingWorkspaceTurnContinuation(WORKSPACE_TURN_CORRELATION)).toBe(false); + + const wake = harness.queueCancelableWake(); + harness.latestRequest().onQueuedMessageStop?.(); + // The entry is gone before the stream ends (dedupe removal / clearQueue shape). + wake.abort("monitor consumed"); + session.clearQueue("monitor consumed"); + + // The owner's settlement runs synchronously with stream-end; it must already see the + // continuation, and only for this correlation. + expect(session.hasPendingWorkspaceTurnContinuation(WORKSPACE_TURN_CORRELATION)).toBe(true); + expect( + session.hasPendingWorkspaceTurnContinuation({ + ...WORKSPACE_TURN_CORRELATION, + turnId: "another-turn", + }) + ).toBe(false); + + aiEmitter.emit("stream-end", streamEndEvent(workspaceId)); + expect(await waitForCondition(() => streamMessage.mock.calls.length === 2)).toBe(true); + expect(harness.latestRequest().muxMetadata).toEqual(WORKSPACE_TURN_CORRELATION); + // Consumed by the resumed stream. + expect(session.hasPendingWorkspaceTurnContinuation(WORKSPACE_TURN_CORRELATION)).toBe(false); + } finally { + session.dispose(); + await cleanup(); + } + }); + + test("a queued unrelated entry supersedes the delegated turn despite the owed continuation", async () => { + const workspaceId = "queue-dispatch-stranded-delegated-superseded"; + const harness = await createStreamingTurnHarness(workspaceId, { + sendOptions: { muxMetadata: WORKSPACE_TURN_CORRELATION }, + }); + const { session, cleanup } = harness; + + try { + harness.latestRequest().onQueuedMessageStop?.(); + session.queueMessage("user follow-up", { model: TEST_MODEL, agentId: "exec" }); + expect(session.hasPendingWorkspaceTurnContinuation(WORKSPACE_TURN_CORRELATION)).toBe(false); + } finally { + session.dispose(); + await cleanup(); + } + }); + + test("a wake-started continuation resumes with the correlation it inherited from history", async () => { + const workspaceId = "queue-dispatch-stranded-inherited-correlation"; + const wakeMetadata: MuxMessageMetadata = { + type: "bash-monitor-wake", + records: [ + { + processId: "proc-1", + wakeUpdatedAt: "2026-01-01T00:00:00.000Z", + kind: "match", + displayName: "marker", + filter: "MARKER", + filterExclude: false, + }, + ], + }; + const harness = await createStreamingTurnHarness(workspaceId, { + seedHistory: async (historyService) => { + // The delegated turn's stream was cut at a tool boundary by the first wake. + await historyService.appendToHistory( + workspaceId, + createMuxMessage("user-0", "user", "delegated prompt", { + timestamp: Date.now(), + muxMetadata: WORKSPACE_TURN_CORRELATION, + }) + ); + await historyService.appendToHistory( + workspaceId, + createMuxMessage("assistant-0", "assistant", "working", { + timestamp: Date.now(), + finishReason: "tool-calls", + muxMetadata: WORKSPACE_TURN_CORRELATION, + }) + ); + }, + sendOptions: { muxMetadata: wakeMetadata }, + sendInternal: { synthetic: true, agentInitiated: true }, + }); + const { session, cleanup, aiEmitter, streamMessage } = harness; + + try { + const wake = harness.queueCancelableWake(); + harness.latestRequest().onQueuedMessageStop?.(); + wake.abort("monitor consumed"); + expect(session.hasPendingWorkspaceTurnContinuation(WORKSPACE_TURN_CORRELATION)).toBe(false); + session.clearQueue("monitor consumed"); + expect(session.hasPendingWorkspaceTurnContinuation(WORKSPACE_TURN_CORRELATION)).toBe(true); + + aiEmitter.emit("stream-end", streamEndEvent(workspaceId)); + expect(await waitForCondition(() => streamMessage.mock.calls.length === 2)).toBe(true); + expect(harness.latestRequest().muxMetadata).toEqual(WORKSPACE_TURN_CORRELATION); + } finally { + session.dispose(); + await cleanup(); + } + }); + + test("yields the stranded resume to a manual send in preflight", async () => { + const workspaceId = "queue-dispatch-stranded-preflight"; + let preflightInFlight = true; + const harness = await createStreamingTurnHarness(workspaceId, { + harness: { hasExternalSendPreflight: () => preflightInFlight }, + }); + const { session, cleanup, aiEmitter, streamMessage } = harness; + + try { + const wake = harness.queueCancelableWake(); + harness.latestRequest().onQueuedMessageStop?.(); + wake.abort("monitor consumed"); + aiEmitter.emit("stream-end", streamEndEvent(workspaceId)); + + expect(await waitForCondition(() => !session.isBusy())).toBe(true); + await new Promise((resolve) => setTimeout(resolve, 25)); + expect(streamMessage).toHaveBeenCalledTimes(1); + + // The preflight settled without a turn; its idle drain delivers the owed continuation. + preflightInFlight = false; + session.drainQueuedMessagesIfIdle(); + expect(await waitForCondition(() => streamMessage.mock.calls.length === 2)).toBe(true); + } finally { + session.dispose(); + await cleanup(); + } + }); + + test("keeps the continuation owed when the resume fails before its stream starts", async () => { + const workspaceId = "queue-dispatch-stranded-retry"; + let gateOpen = false; + const workspaceGoalService = { + assertPricedModelForBudgetedGoal: mock(() => + Promise.resolve(gateOpen ? Ok(undefined) : Err({ type: "unknown", raw: "gate closed" })) + ), + recordStreamAccounting: mock(() => Promise.resolve()), + applyPendingAfterStreamEnd: mock(() => Promise.resolve()), + requestContinuationAfterStreamEnd: mock(() => Promise.resolve()), + recordStreamStarted: mock(() => Promise.resolve()), + syncGoalModeWithChatTail: mock(() => Promise.resolve(null)), + } as unknown as WorkspaceGoalService; + // The gate is closed for the resume only: the initial send passes while it is open. + gateOpen = true; + const harness = await createStreamingTurnHarness(workspaceId, { + harness: { workspaceGoalService }, + sendInternal: { synthetic: true, agentInitiated: true }, + }); + const { session, cleanup, aiEmitter, streamMessage } = harness; + + try { + gateOpen = false; + const wake = harness.queueCancelableWake(); + harness.latestRequest().onQueuedMessageStop?.(); + wake.abort("monitor consumed"); + aiEmitter.emit("stream-end", streamEndEvent(workspaceId)); + + expect(await waitForCondition(() => !session.isBusy())).toBe(true); + await new Promise((resolve) => setTimeout(resolve, 25)); + expect(streamMessage).toHaveBeenCalledTimes(1); + + gateOpen = true; + session.drainQueuedMessagesIfIdle(); + expect(await waitForCondition(() => streamMessage.mock.calls.length === 2)).toBe(true); + } finally { + session.dispose(); + await cleanup(); + } + }); + + test("a stranded goal continuation resumes under the same goal attribution", async () => { + const workspaceId = "queue-dispatch-stranded-goal"; + const recordStreamAccounting = mock((_input: { streamOriginKind: string }) => + Promise.resolve() + ); + const workspaceGoalService = { + assertPricedModelForBudgetedGoal: mock(() => Promise.resolve(Ok(undefined))), + recordStreamAccounting, + applyPendingAfterStreamEnd: mock(() => Promise.resolve()), + requestContinuationAfterStreamEnd: mock(() => Promise.resolve()), + recordStreamStarted: mock(() => Promise.resolve()), + syncGoalModeWithChatTail: mock(() => Promise.resolve(null)), + completeGoalFromSilentContinuation: mock(() => Promise.resolve(false)), + } as unknown as WorkspaceGoalService; + const aiEmitter = new EventEmitter(); + const streamMessage = mock((_options: StreamMessageOptions) => { + aiEmitter.emit("stream-start", streamStartEvent(workspaceId)); + return Promise.resolve(Ok(createStartedTurnHandle("assistant-1"))); + }); + const { session, cleanup, historyService } = await createAgentSessionHarness({ + workspaceId, + aiEmitter, + aiServiceOverrides: { streamMessage: streamMessage as unknown as AIService["streamMessage"] }, + workspaceGoalService, + }); + + try { + await historyService.appendToHistory( + workspaceId, + createMuxMessage("user-0", "user", "keep going", { timestamp: Date.now() }) + ); + const resumed = await session.resumeStream( + { model: TEST_MODEL, agentId: "exec" }, + { agentInitiated: true, goalKind: GOAL_CONTINUATION_KIND, goalId: "goal-1" } + ); + expect(resumed.success).toBe(true); + expect(streamMessage).toHaveBeenCalledTimes(1); + await historyService.appendToHistory( + workspaceId, + createMuxMessage("assistant-1", "assistant", "ran the checks", { timestamp: Date.now() }) + ); + + const controller = new AbortController(); + session.queueMessage( + "Background monitor wake", + { model: TEST_MODEL, agentId: "exec" }, + { + synthetic: true, + agentInitiated: true, + cancelState: { canceledBeforeAcceptance: false }, + cancelSignal: controller.signal, + onCanceled: () => undefined, + } + ); + streamMessage.mock.calls[0]?.[0].onQueuedMessageStop?.(); + controller.abort("monitor consumed"); + aiEmitter.emit("stream-end", streamEndEvent(workspaceId)); + expect(await waitForCondition(() => streamMessage.mock.calls.length === 2)).toBe(true); + + // The resumed stream's own end is accounted as the same goal continuation. + aiEmitter.emit("stream-end", { ...streamEndEvent(workspaceId), messageId: "assistant-2" }); + expect(await waitForCondition(() => recordStreamAccounting.mock.calls.length === 2)).toBe( + true + ); + expect(recordStreamAccounting.mock.calls.map((call) => call[0].streamOriginKind)).toEqual([ + "goal_continuation", + "goal_continuation", + ]); + } finally { + session.dispose(); + await cleanup(); + } + }); + test("rollback failure preserves the wake and continues acceptance", async () => { const workspaceId = "queue-dispatch-cancel-rollback-failure"; const { session, cleanup, historyService } = await createAgentSessionHarness({ workspaceId }); diff --git a/src/node/services/agentSession.testHarness.ts b/src/node/services/agentSession.testHarness.ts index fac756ca28..f89c93b4e9 100644 --- a/src/node/services/agentSession.testHarness.ts +++ b/src/node/services/agentSession.testHarness.ts @@ -110,6 +110,7 @@ export interface AgentSessionHarnessOptions { workspaceGoalService?: WorkspaceGoalService; mcpServerManager?: MCPServerManager; onCompactionComplete?: (metadata: CompactionCompletionMetadata) => void; + hasExternalSendPreflight?: () => boolean; captureEvents?: boolean; } @@ -154,6 +155,7 @@ export async function createAgentSessionHarness( workspaceGoalService: options.workspaceGoalService, backgroundProcessManager, onCompactionComplete: options.onCompactionComplete, + hasExternalSendPreflight: options.hasExternalSendPreflight, }); const events: WorkspaceChatMessage[] = []; diff --git a/src/node/services/agentSession.ts b/src/node/services/agentSession.ts index fa1903a2f8..0701cc210e 100644 --- a/src/node/services/agentSession.ts +++ b/src/node/services/agentSession.ts @@ -365,19 +365,43 @@ function getWorkspaceTurnMuxMetadata(muxMetadata: unknown): WorkspaceTurnMuxMeta return metadata?.type === "workspace-turn-task" ? metadata : undefined; } +interface StrandedTurnResume { + options: SendMessageOptions; + agentInitiated?: boolean; + goalKind?: GoalSyntheticMessageKind; + goalId?: string; +} + /** - * Options for resuming a turn stranded by a withdrawn queued message: the ended stream's - * settings minus anything that would replay its dispatch (edit target, queue dispatch mode) - * or re-stamp per-message metadata (compaction, skill, wake). Only the workspace-turn - * correlation survives because the delegating owner still waits on this continuation. + * Continuation for a turn stranded by a withdrawn queued message: the interrupted stream's + * settings and goal attribution minus anything that would replay its dispatch (edit target, + * queue dispatch mode) or re-stamp per-message metadata (compaction, skill, wake). Only the + * workspace-turn correlation survives, taken from the resolved stream context because a + * wake-started continuation inherits it from history rather than from its own send options. */ -function buildStrandedTurnResumeOptions(options: SendMessageOptions): SendMessageOptions { +function buildStrandedTurnResume(context: { + modelString: string; + options?: SendMessageOptions; + agentInitiated?: boolean; + goalKind?: GoalSyntheticMessageKind; + goalId?: string; + workspaceTurnMetadata?: WorkspaceTurnMuxMetadata; +}): StrandedTurnResume { const { editMessageId: _editMessageId, queueDispatchMode: _queueDispatchMode, ...resumeOptions - } = options; - return { ...resumeOptions, muxMetadata: getWorkspaceTurnMuxMetadata(options.muxMetadata) }; + } = context.options ?? { model: context.modelString, agentId: WORKSPACE_DEFAULTS.agentId }; + return { + options: { + ...resumeOptions, + muxMetadata: + context.workspaceTurnMetadata ?? getWorkspaceTurnMuxMetadata(context.options?.muxMetadata), + }, + ...(context.agentInitiated != null ? { agentInitiated: context.agentInitiated } : {}), + ...(context.goalKind != null ? { goalKind: context.goalKind } : {}), + ...(context.goalId != null ? { goalId: context.goalId } : {}), + }; } function hasSameWorkspaceTurnCorrelation( @@ -717,10 +741,11 @@ export class AgentSession { private readonly activeToolCallIds = new Set(); // The model loop stops at a tool boundary on behalf of a queued tool-end message. If that // message never starts a turn (canceled wake, cleared queue, pre-stream failure), the stop - // would strand the turn on an unanswered tool result, so the continuation is owed here until - // a stream actually starts (setTurnPhase STREAMING) or the sweep resumes it. - private strandedTurnResume?: SendMessageOptions; - private activeStreamStoppedForQueuedMessage = false; + // would strand the turn on an unanswered tool result, so the continuation is owed here from + // the moment the stop is decided until a stream actually starts (setTurnPhase STREAMING). + // Recorded synchronously so the delegating owner's stream-end settlement can see it. + private strandedTurnResume?: StrandedTurnResume; + private strandedTurnResumeInFlight = false; private consecutiveStrandedResumes = 0; private idleWaiters: Array<() => void> = []; @@ -5032,7 +5057,6 @@ export class AgentSession { providersConfig, }; this.activeStreamUserMessageId = undefined; - this.activeStreamStoppedForQueuedMessage = false; const commitResult = await this.historyService.commitPartial(this.workspaceId); if (!commitResult.success) { @@ -5243,7 +5267,9 @@ export class AgentSession { strictAgentResolution: options?.strictAgentResolution, hasQueuedMessages: this.hasQueuedMessages.bind(this), onQueuedMessageStop: () => { - this.activeStreamStoppedForQueuedMessage = true; + if (this.activeStreamContext != null) { + this.strandedTurnResume = buildStrandedTurnResume(this.activeStreamContext); + } }, openaiTruncationModeOverride, // Mid-turn thinking overrides clamp against the same floor as the @@ -6013,7 +6039,7 @@ export class AgentSession { const failedUserMessageId = this.activeStreamUserMessageId; const hadCompactionRequest = this.activeCompactionRequest !== undefined; - const abortedStreamOptions = this.activeStreamContext?.options; + const abortedStreamContext = this.activeStreamContext; const abortReason = "abortReason" in payload ? payload.abortReason : undefined; const isQueuedProviderToolEndAbort = this.queuedProviderToolEndAbortInFlight && abortReason !== "user"; @@ -6055,7 +6081,7 @@ export class AgentSession { this.emitChatEvent(payload); const dispatchedQueuedMessage = this.dispatchQueuedProviderToolEndMessageAfterAbort( abortReason, - abortedStreamOptions + abortedStreamContext ); if (!dispatchedQueuedMessage) { this.setTurnPhase(TurnPhase.IDLE); @@ -6082,7 +6108,6 @@ export class AgentSession { const streamEndPayload = payload; const activeStreamGoalKind = this.activeStreamContext?.goalKind; const activeStreamOptions = this.activeStreamContext?.options; - const stoppedForQueuedMessage = this.activeStreamStoppedForQueuedMessage; let goalContinuationRequest: { sendOptions: SendMessageOptions; @@ -6179,22 +6204,17 @@ export class AgentSession { // Clear the queued-message signal while the edit flow owns the next dispatch. this.backgroundProcessManager.setMessageQueued(this.workspaceId, false); // Do not dispatch stream-end follow-ups while the edit flow is waiting - // for IDLE; truncation must run before any synthetic turn resumes. + // for IDLE; truncation must run before any synthetic turn resumes. The edit + // rewrites the interrupted turn, so nothing is owed to it either. + this.strandedTurnResume = undefined; } else { - if (!handled) { - if (stoppedForQueuedMessage) { - // The queued dispatch below normally consumes this; it stays owed only when - // that message does not start a turn. - this.strandedTurnResume = buildStrandedTurnResumeOptions( - activeStreamOptions ?? { - model: streamEndPayload.metadata.model, - agentId: WORKSPACE_DEFAULTS.agentId, - } - ); - } else { - this.consecutiveStrandedResumes = 0; - } + if (handled) { + this.strandedTurnResume = undefined; + } else if (this.strandedTurnResume == null) { + this.consecutiveStrandedResumes = 0; } + // The queued dispatch below normally consumes the owed continuation (STREAMING + // transition); it stays owed only when that message does not start a turn. this.sendQueuedMessages(); } @@ -6785,11 +6805,25 @@ export class AgentSession { } const dispatching = this.dispatchingQueuedEntryMuxMetadata as MuxMessageMetadata | undefined; - return ( + if ( dispatching?.type === "workspace-turn-task" && dispatching.taskHandleId === metadata.taskHandleId && dispatching.ownerWorkspaceId === metadata.ownerWorkspaceId && dispatching.turnId === metadata.turnId + ) { + return true; + } + + // A stop owed a continuation with nothing else queued to take the turn: the stranded + // resume will carry this correlation, so the owner must not settle the turn at the cut. + // A queued or dispatching entry that failed the checks above supersedes it instead. + return ( + this.messageQueue.isEmpty() && + !this.dispatchingQueuedEntry && + hasSameWorkspaceTurnCorrelation( + getWorkspaceTurnMuxMetadata(this.strandedTurnResume?.options.muxMetadata), + metadata + ) ); } @@ -6865,7 +6899,7 @@ export class AgentSession { private dispatchQueuedProviderToolEndMessageAfterAbort( abortReason: StreamAbortReason | undefined, - abortedStreamOptions: SendMessageOptions | undefined + abortedStreamContext: AgentSession["activeStreamContext"] ): boolean { if (!this.queuedProviderToolEndAbortInFlight) { return false; @@ -6878,8 +6912,8 @@ export class AgentSession { // The soft stop was made on behalf of the queued message; if that message has been // withdrawn (or is withdrawn after dequeue), the interrupted turn must resume instead. - if (abortedStreamOptions != null) { - this.strandedTurnResume = buildStrandedTurnResumeOptions(abortedStreamOptions); + if (abortedStreamContext != null) { + this.strandedTurnResume = buildStrandedTurnResume(abortedStreamContext); } if (!this.hasQueuedMessages()) { return false; @@ -6894,21 +6928,26 @@ export class AgentSession { * run from every idle transition and queue removal; only the first eligible call acts. */ private resumeStrandedTurnIfIdle(): void { - const options = this.strandedTurnResume; + const resume = this.strandedTurnResume; if ( - options == null || + resume == null || + this.strandedTurnResumeInFlight || this.disposed || this.turnAdmissionBlocks > 0 || this.hasActiveOrPendingTurnWork() || !this.messageQueue.isEmpty() || this.hasPendingAutoRetry() || - this.deferQueuedFlushUntilAfterEdit + this.deferQueuedFlushUntilAfterEdit || + // A manual send inside WorkspaceService preflight is not queued yet and the session + // reads idle; the user's message supersedes the owed continuation (it drains here + // through drainQueuedMessagesIfIdle if it settles without a turn). + this.hasExternalSendPreflight?.() === true ) { return; } - this.strandedTurnResume = undefined; if (this.consecutiveStrandedResumes >= MAX_CONSECUTIVE_STRANDED_TURN_RESUMES) { + this.strandedTurnResume = undefined; log.warn("Leaving stranded turn idle: consecutive resume cap reached", { workspaceId: this.workspaceId, cap: MAX_CONSECUTIVE_STRANDED_TURN_RESUMES, @@ -6921,22 +6960,50 @@ export class AgentSession { attempt: this.consecutiveStrandedResumes, }); - this.resumeStream(options, { agentInitiated: true }) + // The owed continuation is cleared by the STREAMING transition, not here: a resume that + // fails before its stream starts (pricing gate, history read) stays owed for the next + // sweep, bounded by the cap above. + this.strandedTurnResumeInFlight = true; + this.resumeStream(resume.options, { + agentInitiated: resume.agentInitiated, + goalKind: resume.goalKind, + goalId: resume.goalId, + }) .then((result) => { if (!result.success) { log.warn("Stranded turn resume failed", { workspaceId: this.workspaceId, error: result.error, }); - } else if (!result.data.started) { + return false; + } + if (!result.data.started) { log.warn("Stranded turn resume did not start", { workspaceId: this.workspaceId }); + return false; } + return true; }) .catch((error: unknown) => { log.warn("Stranded turn resume threw", { workspaceId: this.workspaceId, error: getErrorMessage(error), }); + return false; + }) + .then((started) => { + this.strandedTurnResumeInFlight = false; + // resumeStream settles only after its stream ends, so a stranding of that resumed + // stream can find the flag still set; sweep again once it clears. A resume that never + // started stays owed for the next natural poke rather than retrying in a tight loop. + if (started) { + this.resumeStrandedTurnIfIdle(); + } + }) + .catch((error: unknown) => { + log.error("Stranded turn resume sweep failed", { + workspaceId: this.workspaceId, + error: getErrorMessage(error), + }); }); } diff --git a/src/node/services/streamManager.test.ts b/src/node/services/streamManager.test.ts index 3ab85ecdd1..365746cc9b 100644 --- a/src/node/services/streamManager.test.ts +++ b/src/node/services/streamManager.test.ts @@ -1297,6 +1297,24 @@ describe("StreamManager - stopWhen configuration", () => { expect(stopsForQueuedMessage).toBe(2); }); + test("queued-message stop does not report itself once the step cap is reached", () => { + let stopsForQueuedMessage = 0; + const [maxStepCondition, queuedMessageCondition] = buildStopWhenForTests()({ + hasQueuedMessages: () => true, + onQueuedMessageStop: () => { + stopsForQueuedMessage += 1; + }, + }); + const cappedSteps = { steps: new Array(100000).fill({}) }; + + expect(maxStepCondition(cappedSteps)).toBe(true); + expect(queuedMessageCondition(cappedSteps)).toBe(true); + expect(stopsForQueuedMessage).toBe(0); + + expect(queuedMessageCondition({ steps: new Array(99999).fill({}) })).toBe(true); + expect(stopsForQueuedMessage).toBe(1); + }); + const requiredToolCases: Array<{ name: string; toolPolicy: ToolPolicy; diff --git a/src/node/services/streamManager.ts b/src/node/services/streamManager.ts index 6f9fbee0a7..c7c1812bfd 100644 --- a/src/node/services/streamManager.ts +++ b/src/node/services/streamManager.ts @@ -114,6 +114,8 @@ const EMPTY_STREAM_OUTPUT_ERROR_MESSAGE = "The model ended the stream before producing any assistant-visible output. This usually means the upstream stream was dropped rather than completed normally. Xum will retry automatically when possible, and if retries keep failing you should try again or switch models."; const MAX_EMPTY_STREAM_RECOVERY_ATTEMPTS = 1; +/** Hard per-stream step cap; the practical limit is the model's own finish. */ +const MAX_STREAM_STEPS = 100_000; /** Drop reason for a partial that never reaches chat.jsonl. */ type DroppedStreamSource = "aborted_stream" | "errored_stream"; @@ -2188,15 +2190,19 @@ export class StreamManager { if (!(request.hasQueuedMessages?.("tool-end") ?? false)) { return false; } - // A successful required tool result is a legitimate end of turn on its own; - // only a stop made purely for the queued message may need resuming later. - if (!hasSuccessfulRequiredToolResult(state)) { + // The step cap and a successful required tool result each end the turn on their + // own; only a stop made purely for the queued message may need resuming later. + if (state.steps.length < MAX_STREAM_STEPS && !hasSuccessfulRequiredToolResult(state)) { request.onQueuedMessageStop?.(); } return true; }; - return [stepCountIs(100000), hasQueuedToolEndMessage, hasSuccessfulRequiredToolResult]; + return [ + stepCountIs(MAX_STREAM_STEPS), + hasQueuedToolEndMessage, + hasSuccessfulRequiredToolResult, + ]; } /** From 2077d46d56c94040c1197589530afb440202a06b Mon Sep 17 00:00:00 2001 From: Michael Suchacz <203725896+ibetitsmike@users.noreply.github.com> Date: Thu, 3 Sep 2026 00:23:43 +0000 Subject: [PATCH 03/28] fix(agentSession): forfeit the stranded continuation on user Stop, goal veto, and cap; claim the turn before resume gates - User Stop (interruptStream + user/system/startup aborts) withdraws the owed continuation so restoreQueueToInput's clearQueue sweep cannot restart the model. - resumeStream claims PREPARING before its async admission gates (pricing, goal), closing the idle window a manual send could slip through. - Stranded goal turns revalidate against buildGoalRedispatchAdmission once the turn is claimed; a paused or transitioned goal forfeits the continuation. - The resume options come from the startup-retry whitelist, dropping ACP-only fields (acpPromptId, delegatedToolNames) and other per-dispatch options. - Past the consecutive-resume cap the marker is forfeited on the first sweep and never advertised to hasPendingWorkspaceTurnContinuation; "consecutive" now resets whenever a non-resume stream starts. --- .../agentSession.queueDispatch.test.ts | 220 ++++++++++++++++++ src/node/services/agentSession.ts | 136 ++++++++--- 2 files changed, 319 insertions(+), 37 deletions(-) diff --git a/src/node/services/agentSession.queueDispatch.test.ts b/src/node/services/agentSession.queueDispatch.test.ts index 0955d721cd..3f0d7d5ea8 100644 --- a/src/node/services/agentSession.queueDispatch.test.ts +++ b/src/node/services/agentSession.queueDispatch.test.ts @@ -1143,8 +1143,12 @@ describe("AgentSession queued message tool-call dispatch", () => { const recordStreamAccounting = mock((_input: { streamOriginKind: string }) => Promise.resolve() ); + const buildGoalRedispatchAdmission = mock(() => + Promise.resolve({ admissible: true as const, admissionStale: () => false }) + ); const workspaceGoalService = { assertPricedModelForBudgetedGoal: mock(() => Promise.resolve(Ok(undefined))), + buildGoalRedispatchAdmission, recordStreamAccounting, applyPendingAfterStreamEnd: mock(() => Promise.resolve()), requestContinuationAfterStreamEnd: mock(() => Promise.resolve()), @@ -1206,6 +1210,222 @@ describe("AgentSession queued message tool-call dispatch", () => { "goal_continuation", "goal_continuation", ]); + // Resumed under the goal's own admission, like any redispatched goal turn. + expect(buildGoalRedispatchAdmission).toHaveBeenCalledWith( + workspaceId, + "goal-1", + GOAL_CONTINUATION_KIND + ); + } finally { + session.dispose(); + await cleanup(); + } + }); + + test("drops a stranded goal continuation the goal no longer admits", async () => { + const workspaceId = "queue-dispatch-stranded-goal-paused"; + const workspaceGoalService = { + assertPricedModelForBudgetedGoal: mock(() => Promise.resolve(Ok(undefined))), + // The user paused the goal while the turn ran; the pause landed at stream end. + buildGoalRedispatchAdmission: mock(() => Promise.resolve({ admissible: false as const })), + recordStreamAccounting: mock(() => Promise.resolve()), + applyPendingAfterStreamEnd: mock(() => Promise.resolve()), + requestContinuationAfterStreamEnd: mock(() => Promise.resolve()), + recordStreamStarted: mock(() => Promise.resolve()), + syncGoalModeWithChatTail: mock(() => Promise.resolve(null)), + completeGoalFromSilentContinuation: mock(() => Promise.resolve(false)), + } as unknown as WorkspaceGoalService; + const aiEmitter = new EventEmitter(); + const streamMessage = mock((_options: StreamMessageOptions) => { + aiEmitter.emit("stream-start", streamStartEvent(workspaceId)); + return Promise.resolve(Ok(createStartedTurnHandle("assistant-1"))); + }); + const { session, cleanup, historyService } = await createAgentSessionHarness({ + workspaceId, + aiEmitter, + aiServiceOverrides: { streamMessage: streamMessage as unknown as AIService["streamMessage"] }, + workspaceGoalService, + }); + + try { + await historyService.appendToHistory( + workspaceId, + createMuxMessage("user-0", "user", "keep going", { timestamp: Date.now() }) + ); + const resumed = await session.resumeStream( + { model: TEST_MODEL, agentId: "exec" }, + { agentInitiated: true, goalKind: GOAL_CONTINUATION_KIND, goalId: "goal-1" } + ); + expect(resumed.success).toBe(true); + await historyService.appendToHistory( + workspaceId, + createMuxMessage("assistant-1", "assistant", "ran the checks", { timestamp: Date.now() }) + ); + + const controller = new AbortController(); + session.queueMessage( + "Background monitor wake", + { model: TEST_MODEL, agentId: "exec" }, + { + synthetic: true, + agentInitiated: true, + cancelState: { canceledBeforeAcceptance: false }, + cancelSignal: controller.signal, + onCanceled: () => undefined, + } + ); + streamMessage.mock.calls[0]?.[0].onQueuedMessageStop?.(); + controller.abort("monitor consumed"); + aiEmitter.emit("stream-end", streamEndEvent(workspaceId)); + + expect(await waitForCondition(() => !session.isBusy())).toBe(true); + await new Promise((resolve) => setTimeout(resolve, 25)); + expect(streamMessage).toHaveBeenCalledTimes(1); + // Forfeited, not deferred: a later idle poke must not revive the paused goal's turn. + session.drainQueuedMessagesIfIdle(); + await new Promise((resolve) => setTimeout(resolve, 25)); + expect(streamMessage).toHaveBeenCalledTimes(1); + expect(session.isBusy()).toBe(false); + } finally { + session.dispose(); + await cleanup(); + } + }); + + test("a user Stop withdraws the owed continuation", async () => { + const workspaceId = "queue-dispatch-stranded-user-stop"; + const harness = await createStreamingTurnHarness(workspaceId); + const { session, cleanup, aiEmitter, streamMessage } = harness; + + try { + harness.queueCancelableWake(); + harness.latestRequest().onQueuedMessageStop?.(); + // WorkspaceService.interruptStream: hard stop, then restore the queue to the composer. + expect((await session.interruptStream()).success).toBe(true); + aiEmitter.emit("stream-abort", streamAbortEvent(workspaceId, "user")); + expect(await waitForCondition(() => !session.isBusy())).toBe(true); + session.restoreQueueToInput(); + + await new Promise((resolve) => setTimeout(resolve, 25)); + expect(streamMessage).toHaveBeenCalledTimes(1); + expect(session.isBusy()).toBe(false); + expect(session.hasQueuedMessages()).toBe(false); + } finally { + session.dispose(); + await cleanup(); + } + }); + + test("the resume holds the turn while its admission gates run", async () => { + const workspaceId = "queue-dispatch-stranded-claims-turn"; + let releaseGate: () => void = () => undefined; + let gateArmed = false; + let gateReached = false; + const workspaceGoalService = { + assertPricedModelForBudgetedGoal: mock(() => { + if (!gateArmed) { + return Promise.resolve(Ok(undefined)); + } + gateReached = true; + return new Promise((resolve) => { + releaseGate = () => resolve(Ok(undefined)); + }); + }), + recordStreamAccounting: mock(() => Promise.resolve()), + applyPendingAfterStreamEnd: mock(() => Promise.resolve()), + requestContinuationAfterStreamEnd: mock(() => Promise.resolve()), + recordStreamStarted: mock(() => Promise.resolve()), + syncGoalModeWithChatTail: mock(() => Promise.resolve(null)), + } as unknown as WorkspaceGoalService; + const harness = await createStreamingTurnHarness(workspaceId, { + harness: { workspaceGoalService }, + sendInternal: { synthetic: true, agentInitiated: true }, + }); + const { session, cleanup, aiEmitter, streamMessage } = harness; + + try { + gateArmed = true; + const wake = harness.queueCancelableWake(); + harness.latestRequest().onQueuedMessageStop?.(); + wake.abort("monitor consumed"); + aiEmitter.emit("stream-end", streamEndEvent(workspaceId)); + + // The resume is parked on the pricing gate: the session must already read busy so a + // manual send arriving now queues behind it instead of starting a colliding stream. + expect(await waitForCondition(() => gateReached)).toBe(true); + await new Promise((resolve) => setTimeout(resolve, 25)); + expect(streamMessage).toHaveBeenCalledTimes(1); + expect(session.isBusy()).toBe(true); + + releaseGate(); + expect(await waitForCondition(() => streamMessage.mock.calls.length === 2)).toBe(true); + } finally { + session.dispose(); + await cleanup(); + } + }); + + test("the resume drops the interrupted turn's ACP prompt binding", async () => { + const workspaceId = "queue-dispatch-stranded-acp"; + const harness = await createStreamingTurnHarness(workspaceId, { + sendOptions: { acpPromptId: "prompt-1", delegatedToolNames: ["bash"] }, + }); + const { session, cleanup, aiEmitter, streamMessage } = harness; + + try { + const original = harness.latestRequest(); + expect(original.acpPromptId).toBe("prompt-1"); + expect(original.delegatedToolNames).toEqual(["bash"]); + + const wake = harness.queueCancelableWake(); + original.onQueuedMessageStop?.(); + wake.abort("monitor consumed"); + aiEmitter.emit("stream-end", streamEndEvent(workspaceId)); + + expect(await waitForCondition(() => streamMessage.mock.calls.length === 2)).toBe(true); + const resumed = harness.latestRequest(); + expect(resumed.modelString).toBe(TEST_MODEL); + // The ACP turn completed at the first stream-end; a delegated tool call on the resumed + // stream would otherwise wait on a prompt nobody answers. + expect(resumed.acpPromptId).toBeUndefined(); + expect(resumed.delegatedToolNames).toBeUndefined(); + } finally { + session.dispose(); + await cleanup(); + } + }); + + test("stops advertising the delegated continuation once the resume cap is exhausted", async () => { + const workspaceId = "queue-dispatch-stranded-delegated-cap"; + const harness = await createStreamingTurnHarness(workspaceId, { + sendOptions: { muxMetadata: WORKSPACE_TURN_CORRELATION }, + }); + const { session, cleanup, aiEmitter, streamMessage } = harness; + + try { + const strandTurn = () => { + harness.latestRequest().onQueuedMessageStop?.(); + harness.queueCancelableWake().abort("monitor consumed"); + session.clearQueue("monitor consumed"); + }; + + for (let resumes = 1; resumes <= 3; resumes += 1) { + strandTurn(); + expect(session.hasPendingWorkspaceTurnContinuation(WORKSPACE_TURN_CORRELATION)).toBe(true); + aiEmitter.emit("stream-end", streamEndEvent(workspaceId)); + expect(await waitForCondition(() => streamMessage.mock.calls.length === resumes + 1)).toBe( + true + ); + } + + // No resume will follow this cut, so the owner must settle the turn here instead of + // deferring to a continuation that never starts. + strandTurn(); + expect(session.hasPendingWorkspaceTurnContinuation(WORKSPACE_TURN_CORRELATION)).toBe(false); + aiEmitter.emit("stream-end", streamEndEvent(workspaceId)); + expect(await waitForCondition(() => !session.isBusy())).toBe(true); + await new Promise((resolve) => setTimeout(resolve, 25)); + expect(streamMessage).toHaveBeenCalledTimes(4); } finally { session.dispose(); await cleanup(); diff --git a/src/node/services/agentSession.ts b/src/node/services/agentSession.ts index 0701cc210e..859753189d 100644 --- a/src/node/services/agentSession.ts +++ b/src/node/services/agentSession.ts @@ -374,10 +374,13 @@ interface StrandedTurnResume { /** * Continuation for a turn stranded by a withdrawn queued message: the interrupted stream's - * settings and goal attribution minus anything that would replay its dispatch (edit target, - * queue dispatch mode) or re-stamp per-message metadata (compaction, skill, wake). Only the - * workspace-turn correlation survives, taken from the resolved stream context because a - * wake-started continuation inherits it from history rather than from its own send options. + * request configuration (the same retry-safe whitelist startup recovery resumes with) and goal + * attribution. Anything that replays the original dispatch is dropped: edit target, queue + * dispatch mode, per-message metadata (compaction, skill, wake), and the ACP prompt binding, + * whose turn completion the first stream-end already resolved, so a delegated tool call on the + * resumed stream would wait on a prompt nobody answers. Only the workspace-turn correlation + * survives, taken from the resolved stream context because a wake-started continuation + * inherits it from history rather than from its own send options. */ function buildStrandedTurnResume(context: { modelString: string; @@ -387,11 +390,9 @@ function buildStrandedTurnResume(context: { goalId?: string; workspaceTurnMetadata?: WorkspaceTurnMuxMetadata; }): StrandedTurnResume { - const { - editMessageId: _editMessageId, - queueDispatchMode: _queueDispatchMode, - ...resumeOptions - } = context.options ?? { model: context.modelString, agentId: WORKSPACE_DEFAULTS.agentId }; + const resumeOptions = pickStartupRetrySendOptions( + context.options ?? { model: context.modelString, agentId: WORKSPACE_DEFAULTS.agentId } + ); return { options: { ...resumeOptions, @@ -4082,7 +4083,6 @@ export class AgentSession { if (isManualUserMessage) { // The user's own message supersedes any continuation owed to a stranded turn. this.strandedTurnResume = undefined; - this.consecutiveStrandedResumes = 0; // A fresh accepted user send supersedes any persisted startup-abandon // classification from previous turns. await this.clearStartupAutoRetryAbandon(); @@ -4246,7 +4246,16 @@ export class AgentSession { async resumeStream( options: SendMessageOptions, - internal?: { agentInitiated?: boolean; goalKind?: GoalSyntheticMessageKind; goalId?: string } + internal?: { + agentInitiated?: boolean; + goalKind?: GoalSyntheticMessageKind; + goalId?: string; + /** + * Caller admission gate awaited once the turn is claimed, so an async check (durable goal + * state) cannot race a manual send that would otherwise see an idle session. + */ + admit?: () => Promise; + } ): Promise> { this.assertNotDisposed("resumeStream"); @@ -4264,16 +4273,6 @@ export class AgentSession { return Ok({ started: false }); } - if (this.workspaceGoalService) { - const pricingGate = await this.workspaceGoalService.assertPricedModelForBudgetedGoal( - this.workspaceId, - modelForStream - ); - if (!pricingGate.success) { - return Err(pricingGate.error); - } - } - // r40: refuse resume admission while a context-discarding mutation is // mid-flight (see holdTurnAdmission) — checked in the same synchronous // block that sets PREPARING. A non-started resume reads as retriable to @@ -4293,6 +4292,9 @@ export class AgentSession { internal?.goalKind, internal?.goalId ); + // Claim the turn before any await: the admission gates below do I/O, and a manual send + // entering meanwhile must see a busy session rather than start a stream this resume + // would then collide with. this.preparingWorkspaceTurnMetadata = getWorkspaceTurnMuxMetadata(optionsForStream.muxMetadata); this.setTurnPhase(TurnPhase.PREPARING); // Open the mid-turn thinking override window for the resumed turn (after @@ -4300,6 +4302,19 @@ export class AgentSession { const turnThinkingOverride: ActiveTurnThinkingOverride = {}; this.activeTurnThinkingOverride = turnThinkingOverride; try { + if (this.workspaceGoalService) { + const pricingGate = await this.workspaceGoalService.assertPricedModelForBudgetedGoal( + this.workspaceId, + modelForStream + ); + if (!pricingGate.success) { + return Err(pricingGate.error); + } + } + if (internal?.admit != null && !(await internal.admit())) { + return Ok({ started: false }); + } + // Must await here so the finally block runs after streaming completes, // not immediately when the Promise is returned. const result = await this.streamWithHistory( @@ -4921,8 +4936,11 @@ export class AgentSession { }): Promise> { this.assertNotDisposed("interruptStream"); - // Explicit user interruption should immediately stop any pending auto-retry loop. + // Explicit user interruption should immediately stop any pending auto-retry loop and + // withdraw any continuation owed to a stranded turn (the stream-abort handler repeats this + // for a stop that lands mid-step). this.retryManager.cancel(); + this.strandedTurnResume = undefined; if (options?.soft !== true) { this.queuedProviderToolEndAbortInFlight = false; @@ -6043,6 +6061,12 @@ export class AgentSession { const abortReason = "abortReason" in payload ? payload.abortReason : undefined; const isQueuedProviderToolEndAbort = this.queuedProviderToolEndAbortInFlight && abortReason !== "user"; + // Only the queued-message soft stop owes the cut turn a continuation (rebuilt below from + // the aborted context). A user Stop must leave the session idle even though the queue it + // restores to the composer pokes the resume sweep; other aborts belong to auto-retry. + if (!isQueuedProviderToolEndAbort) { + this.strandedTurnResume = undefined; + } if (abortReason === "user") { await this.workspaceGoalService?.recordUserStoppedStream(this.workspaceId); } @@ -6210,8 +6234,6 @@ export class AgentSession { } else { if (handled) { this.strandedTurnResume = undefined; - } else if (this.strandedTurnResume == null) { - this.consecutiveStrandedResumes = 0; } // The queued dispatch below normally consumes the owed continuation (STREAMING // transition); it stays owed only when that message does not start a turn. @@ -6367,6 +6389,11 @@ export class AgentSession { // Any stream that actually starts is the continuation the stranded turn was waiting // for. PREPARING is not enough: a dequeued entry can still be canceled before acceptance. this.strandedTurnResume = undefined; + // "Consecutive" means uninterrupted by any other stream: a wake, user, or goal turn + // starting in between restores the runaway budget. + if (!this.strandedTurnResumeInFlight) { + this.consecutiveStrandedResumes = 0; + } } if (next === TurnPhase.IDLE) { @@ -6821,12 +6848,23 @@ export class AgentSession { this.messageQueue.isEmpty() && !this.dispatchingQueuedEntry && hasSameWorkspaceTurnCorrelation( - getWorkspaceTurnMuxMetadata(this.strandedTurnResume?.options.muxMetadata), + getWorkspaceTurnMuxMetadata(this.owedStrandedTurnResume()?.options.muxMetadata), metadata ) ); } + /** + * The continuation still owed to a stranded turn. Past the runaway cap the marker is + * forfeited: nothing may advertise it (the owner would defer settlement for a resume that + * never starts) and the next sweep drops it. + */ + private owedStrandedTurnResume(): StrandedTurnResume | undefined { + return this.consecutiveStrandedResumes < MAX_CONSECUTIVE_STRANDED_TURN_RESUMES + ? this.strandedTurnResume + : undefined; + } + /** * Input poised to take over this session at a queue cut. Engaged stages win * over the queue head; an engaged stage is reported even when its metadata is @@ -6928,9 +6966,18 @@ export class AgentSession { * run from every idle transition and queue removal; only the first eligible call acts. */ private resumeStrandedTurnIfIdle(): void { - const resume = this.strandedTurnResume; + const resume = this.owedStrandedTurnResume(); + if (resume == null) { + if (this.strandedTurnResume != null) { + this.strandedTurnResume = undefined; + log.warn("Leaving stranded turn idle: consecutive resume cap reached", { + workspaceId: this.workspaceId, + cap: MAX_CONSECUTIVE_STRANDED_TURN_RESUMES, + }); + } + return; + } if ( - resume == null || this.strandedTurnResumeInFlight || this.disposed || this.turnAdmissionBlocks > 0 || @@ -6946,14 +6993,6 @@ export class AgentSession { return; } - if (this.consecutiveStrandedResumes >= MAX_CONSECUTIVE_STRANDED_TURN_RESUMES) { - this.strandedTurnResume = undefined; - log.warn("Leaving stranded turn idle: consecutive resume cap reached", { - workspaceId: this.workspaceId, - cap: MAX_CONSECUTIVE_STRANDED_TURN_RESUMES, - }); - return; - } this.consecutiveStrandedResumes += 1; log.info("Resuming turn stranded by a withdrawn queued message", { workspaceId: this.workspaceId, @@ -6964,10 +7003,33 @@ export class AgentSession { // fails before its stream starts (pricing gate, history read) stays owed for the next // sweep, bounded by the cap above. this.strandedTurnResumeInFlight = true; + const goalService = this.workspaceGoalService; + const { goalKind, goalId } = resume; this.resumeStream(resume.options, { agentInitiated: resume.agentInitiated, - goalKind: resume.goalKind, - goalId: resume.goalId, + goalKind, + goalId, + // A goal turn resumes only if the goal still admits it: a Pause or terminal transition + // that landed while the stream ran (applied at its end) forfeits the continuation, the + // same veto durable goal redispatches apply. + admit: + goalService != null && goalKind != null && goalId != null + ? async () => { + const admission = await goalService.buildGoalRedispatchAdmission( + this.workspaceId, + goalId, + goalKind + ); + if (!admission.admissible) { + this.strandedTurnResume = undefined; + log.info("Dropping stranded goal turn: goal no longer admits it", { + workspaceId: this.workspaceId, + goalKind, + }); + } + return admission.admissible; + } + : undefined, }) .then((result) => { if (!result.success) { From 31427863177c4264e64996f9162181380d056fe1 Mon Sep 17 00:00:00 2001 From: Michael Suchacz <203725896+ibetitsmike@users.noreply.github.com> Date: Thu, 3 Sep 2026 01:11:40 +0000 Subject: [PATCH 04/28] fix(agentSession): cancel a withdrawn resume in its pre-stream window, keep goal and hard-stop vetoes honest - The sweep's resume carries an AbortController; withdrawStrandedTurnResume (user Stop, superseding send, edit, context discard, non-soft aborts) clears the marker and aborts it, and resumeStream threads the signal through its gates and streamWithHistory so a Stop during admission starts nothing. - resumeStream revalidates goal turns itself (revalidateGoal) and rechecks buildGoalRedispatchAdmission's staleness probe before launch; a refusal reports goalRefused so the sweep drops the marker. - A correlated marker is forfeited when sendQueuedMessages dequeues an entry that is not that turn's continuation (the owner settled the turn at the cut). - Messages queued behind a rejected resume drain from the sweep's settlement. - onQueuedMessageStop carries the request's modelString and stream-abort metadata carries the active model, so a resume continues on the fallback model that reached the cut; a mid-turn applied thinking level is kept. - The provider-tool soft stop uses a dedicated "queued-message" abort reason; only that reason (with the in-flight flag) rebuilds the marker, so a hard "system" stop from task_stop or an interrupt cascade cannot revive the turn. --- src/common/orpc/schemas/stream.ts | 7 +- .../agentSession.queueDispatch.test.ts | 324 +++++++++++++++++- src/node/services/agentSession.ts | 197 +++++++---- src/node/services/streamManager.test.ts | 18 +- src/node/services/streamManager.ts | 26 +- src/node/services/turnRequestBuilder.ts | 2 +- 6 files changed, 482 insertions(+), 92 deletions(-) diff --git a/src/common/orpc/schemas/stream.ts b/src/common/orpc/schemas/stream.ts index 6108159fcb..0135b2cbff 100644 --- a/src/common/orpc/schemas/stream.ts +++ b/src/common/orpc/schemas/stream.ts @@ -292,7 +292,10 @@ export const StreamEndEventSchema = z.object({ }), }); -export const StreamAbortReasonSchema = z.enum(["user", "startup", "system"]); +// "queued-message": the backend's own soft stop at a provider-executed tool boundary so a queued +// tool-end message can dispatch; distinct from "system" so a concurrent hard stop cannot be +// mistaken for it. +export const StreamAbortReasonSchema = z.enum(["user", "startup", "system", "queued-message"]); export const StreamLifecyclePhaseSchema = z.enum([ "idle", @@ -336,6 +339,8 @@ export const StreamAbortEventSchema = z.object({ // Last step's provider metadata (for context window cache display) contextProviderMetadata: z.record(z.string(), z.unknown()).optional(), duration: z.number().optional(), + // Model active at the abort (a configured fallback may differ from the requested model) + model: z.string().optional(), }) .optional() .meta({ diff --git a/src/node/services/agentSession.queueDispatch.test.ts b/src/node/services/agentSession.queueDispatch.test.ts index 3f0d7d5ea8..60c3f7d200 100644 --- a/src/node/services/agentSession.queueDispatch.test.ts +++ b/src/node/services/agentSession.queueDispatch.test.ts @@ -47,7 +47,7 @@ function streamStartEvent(workspaceId: string): Record { function streamAbortEvent( workspaceId: string, - abortReason: "system" | "user" + abortReason: "system" | "user" | "queued-message" ): Record { return { type: "stream-abort", @@ -448,10 +448,10 @@ describe("AgentSession queued message tool-call dispatch", () => { expect(stopStream).toHaveBeenCalledWith(workspaceId, { soft: true, - abortReason: "system", + abortReason: "queued-message", }); - aiEmitter.emit("stream-abort", streamAbortEvent(workspaceId, "system")); + aiEmitter.emit("stream-abort", streamAbortEvent(workspaceId, "queued-message")); const didDispatch = await waitForCondition(() => sendQueuedMessages.mock.calls.length > 0); expect(didDispatch).toBe(true); expect(sendQueuedMessages).toHaveBeenCalledTimes(1); @@ -806,7 +806,7 @@ describe("AgentSession queued message tool-call dispatch", () => { expect(request.hasQueuedMessages?.("tool-end")).toBe(true); // StreamManager stopped the loop for the queued wake; the wake is then withdrawn // (its output was consumed another way) before the stream-end drain dispatches it. - request.onQueuedMessageStop?.(); + request.onQueuedMessageStop?.({ modelString: TEST_MODEL }); wake.abort("monitor consumed"); aiEmitter.emit("stream-end", streamEndEvent(workspaceId)); @@ -847,7 +847,7 @@ describe("AgentSession queued message tool-call dispatch", () => { try { harness.queueCancelableWake(); - harness.latestRequest().onQueuedMessageStop?.(); + harness.latestRequest().onQueuedMessageStop?.({ modelString: TEST_MODEL }); aiEmitter.emit("stream-end", streamEndEvent(workspaceId)); expect(await waitForCondition(() => streamMessage.mock.calls.length === 2)).toBe(true); @@ -897,7 +897,7 @@ describe("AgentSession queued message tool-call dispatch", () => { try { const strandTurn = () => { - harness.latestRequest().onQueuedMessageStop?.(); + harness.latestRequest().onQueuedMessageStop?.({ modelString: TEST_MODEL }); harness.queueCancelableWake().abort("monitor consumed"); aiEmitter.emit("stream-end", streamEndEvent(workspaceId)); }; @@ -942,7 +942,7 @@ describe("AgentSession queued message tool-call dispatch", () => { // Withdrawn between the soft stop request and the abort it produces. expect(session.removeQueuedMessagesByDedupeKeyPrefix("wake:", "superseded")).toBe(1); - aiEmitter.emit("stream-abort", streamAbortEvent(workspaceId, "system")); + aiEmitter.emit("stream-abort", streamAbortEvent(workspaceId, "queued-message")); expect(await waitForCondition(() => streamMessage.mock.calls.length === 2)).toBe(true); expect(harness.latestRequest().agentInitiated).toBe( @@ -970,7 +970,7 @@ describe("AgentSession queued message tool-call dispatch", () => { expect(session.hasPendingWorkspaceTurnContinuation(WORKSPACE_TURN_CORRELATION)).toBe(false); const wake = harness.queueCancelableWake(); - harness.latestRequest().onQueuedMessageStop?.(); + harness.latestRequest().onQueuedMessageStop?.({ modelString: TEST_MODEL }); // The entry is gone before the stream ends (dedupe removal / clearQueue shape). wake.abort("monitor consumed"); session.clearQueue("monitor consumed"); @@ -1004,7 +1004,7 @@ describe("AgentSession queued message tool-call dispatch", () => { const { session, cleanup } = harness; try { - harness.latestRequest().onQueuedMessageStop?.(); + harness.latestRequest().onQueuedMessageStop?.({ modelString: TEST_MODEL }); session.queueMessage("user follow-up", { model: TEST_MODEL, agentId: "exec" }); expect(session.hasPendingWorkspaceTurnContinuation(WORKSPACE_TURN_CORRELATION)).toBe(false); } finally { @@ -1054,7 +1054,7 @@ describe("AgentSession queued message tool-call dispatch", () => { try { const wake = harness.queueCancelableWake(); - harness.latestRequest().onQueuedMessageStop?.(); + harness.latestRequest().onQueuedMessageStop?.({ modelString: TEST_MODEL }); wake.abort("monitor consumed"); expect(session.hasPendingWorkspaceTurnContinuation(WORKSPACE_TURN_CORRELATION)).toBe(false); session.clearQueue("monitor consumed"); @@ -1079,7 +1079,7 @@ describe("AgentSession queued message tool-call dispatch", () => { try { const wake = harness.queueCancelableWake(); - harness.latestRequest().onQueuedMessageStop?.(); + harness.latestRequest().onQueuedMessageStop?.({ modelString: TEST_MODEL }); wake.abort("monitor consumed"); aiEmitter.emit("stream-end", streamEndEvent(workspaceId)); @@ -1121,7 +1121,7 @@ describe("AgentSession queued message tool-call dispatch", () => { try { gateOpen = false; const wake = harness.queueCancelableWake(); - harness.latestRequest().onQueuedMessageStop?.(); + harness.latestRequest().onQueuedMessageStop?.({ modelString: TEST_MODEL }); wake.abort("monitor consumed"); aiEmitter.emit("stream-end", streamEndEvent(workspaceId)); @@ -1196,7 +1196,7 @@ describe("AgentSession queued message tool-call dispatch", () => { onCanceled: () => undefined, } ); - streamMessage.mock.calls[0]?.[0].onQueuedMessageStop?.(); + streamMessage.mock.calls[0]?.[0].onQueuedMessageStop?.({ modelString: TEST_MODEL }); controller.abort("monitor consumed"); aiEmitter.emit("stream-end", streamEndEvent(workspaceId)); expect(await waitForCondition(() => streamMessage.mock.calls.length === 2)).toBe(true); @@ -1274,7 +1274,7 @@ describe("AgentSession queued message tool-call dispatch", () => { onCanceled: () => undefined, } ); - streamMessage.mock.calls[0]?.[0].onQueuedMessageStop?.(); + streamMessage.mock.calls[0]?.[0].onQueuedMessageStop?.({ modelString: TEST_MODEL }); controller.abort("monitor consumed"); aiEmitter.emit("stream-end", streamEndEvent(workspaceId)); @@ -1299,7 +1299,7 @@ describe("AgentSession queued message tool-call dispatch", () => { try { harness.queueCancelableWake(); - harness.latestRequest().onQueuedMessageStop?.(); + harness.latestRequest().onQueuedMessageStop?.({ modelString: TEST_MODEL }); // WorkspaceService.interruptStream: hard stop, then restore the queue to the composer. expect((await session.interruptStream()).success).toBe(true); aiEmitter.emit("stream-abort", streamAbortEvent(workspaceId, "user")); @@ -1346,7 +1346,7 @@ describe("AgentSession queued message tool-call dispatch", () => { try { gateArmed = true; const wake = harness.queueCancelableWake(); - harness.latestRequest().onQueuedMessageStop?.(); + harness.latestRequest().onQueuedMessageStop?.({ modelString: TEST_MODEL }); wake.abort("monitor consumed"); aiEmitter.emit("stream-end", streamEndEvent(workspaceId)); @@ -1378,7 +1378,7 @@ describe("AgentSession queued message tool-call dispatch", () => { expect(original.delegatedToolNames).toEqual(["bash"]); const wake = harness.queueCancelableWake(); - original.onQueuedMessageStop?.(); + original.onQueuedMessageStop?.({ modelString: TEST_MODEL }); wake.abort("monitor consumed"); aiEmitter.emit("stream-end", streamEndEvent(workspaceId)); @@ -1404,7 +1404,7 @@ describe("AgentSession queued message tool-call dispatch", () => { try { const strandTurn = () => { - harness.latestRequest().onQueuedMessageStop?.(); + harness.latestRequest().onQueuedMessageStop?.({ modelString: TEST_MODEL }); harness.queueCancelableWake().abort("monitor consumed"); session.clearQueue("monitor consumed"); }; @@ -1432,6 +1432,292 @@ describe("AgentSession queued message tool-call dispatch", () => { } }); + test("a user Stop during the resume's admission gate cancels it", async () => { + const workspaceId = "queue-dispatch-stranded-stop-in-admission"; + let releaseGate: () => void = () => undefined; + let gateArmed = false; + let gateReached = false; + const workspaceGoalService = { + assertPricedModelForBudgetedGoal: mock(() => { + if (!gateArmed) { + return Promise.resolve(Ok(undefined)); + } + gateReached = true; + return new Promise((resolve) => { + releaseGate = () => resolve(Ok(undefined)); + }); + }), + recordStreamAccounting: mock(() => Promise.resolve()), + applyPendingAfterStreamEnd: mock(() => Promise.resolve()), + requestContinuationAfterStreamEnd: mock(() => Promise.resolve()), + recordStreamStarted: mock(() => Promise.resolve()), + syncGoalModeWithChatTail: mock(() => Promise.resolve(null)), + } as unknown as WorkspaceGoalService; + const harness = await createStreamingTurnHarness(workspaceId, { + harness: { workspaceGoalService }, + sendInternal: { synthetic: true, agentInitiated: true }, + }); + const { session, cleanup, aiEmitter, streamMessage } = harness; + + try { + gateArmed = true; + const wake = harness.queueCancelableWake(); + harness.latestRequest().onQueuedMessageStop?.({ modelString: TEST_MODEL }); + wake.abort("monitor consumed"); + aiEmitter.emit("stream-end", streamEndEvent(workspaceId)); + expect(await waitForCondition(() => gateReached)).toBe(true); + + // Stop lands while the claimed resume is parked on its gate: StreamManager has no stream + // to abort, so the resume itself must not proceed once the gate opens. + expect((await session.interruptStream()).success).toBe(true); + releaseGate(); + + expect(await waitForCondition(() => !session.isBusy())).toBe(true); + await new Promise((resolve) => setTimeout(resolve, 25)); + expect(streamMessage).toHaveBeenCalledTimes(1); + expect(session.isBusy()).toBe(false); + } finally { + session.dispose(); + await cleanup(); + } + }); + + test("a goal transition between admission and launch drops the resume", async () => { + const workspaceId = "queue-dispatch-stranded-goal-stale"; + const workspaceGoalService = { + assertPricedModelForBudgetedGoal: mock(() => Promise.resolve(Ok(undefined))), + // Admitted on read, but the pause generation moved before the stream could launch. + buildGoalRedispatchAdmission: mock(() => + Promise.resolve({ admissible: true as const, admissionStale: () => true }) + ), + recordStreamAccounting: mock(() => Promise.resolve()), + applyPendingAfterStreamEnd: mock(() => Promise.resolve()), + requestContinuationAfterStreamEnd: mock(() => Promise.resolve()), + recordStreamStarted: mock(() => Promise.resolve()), + syncGoalModeWithChatTail: mock(() => Promise.resolve(null)), + completeGoalFromSilentContinuation: mock(() => Promise.resolve(false)), + } as unknown as WorkspaceGoalService; + const aiEmitter = new EventEmitter(); + const streamMessage = mock((_options: StreamMessageOptions) => { + aiEmitter.emit("stream-start", streamStartEvent(workspaceId)); + return Promise.resolve(Ok(createStartedTurnHandle("assistant-1"))); + }); + const { session, cleanup, historyService } = await createAgentSessionHarness({ + workspaceId, + aiEmitter, + aiServiceOverrides: { streamMessage: streamMessage as unknown as AIService["streamMessage"] }, + workspaceGoalService, + }); + + try { + await historyService.appendToHistory( + workspaceId, + createMuxMessage("user-0", "user", "keep going", { timestamp: Date.now() }) + ); + const resumed = await session.resumeStream( + { model: TEST_MODEL, agentId: "exec" }, + { agentInitiated: true, goalKind: GOAL_CONTINUATION_KIND, goalId: "goal-1" } + ); + expect(resumed.success).toBe(true); + await historyService.appendToHistory( + workspaceId, + createMuxMessage("assistant-1", "assistant", "ran the checks", { timestamp: Date.now() }) + ); + + const controller = new AbortController(); + session.queueMessage( + "Background monitor wake", + { model: TEST_MODEL, agentId: "exec" }, + { + synthetic: true, + agentInitiated: true, + cancelState: { canceledBeforeAcceptance: false }, + cancelSignal: controller.signal, + onCanceled: () => undefined, + } + ); + streamMessage.mock.calls[0]?.[0].onQueuedMessageStop?.({ modelString: TEST_MODEL }); + controller.abort("monitor consumed"); + aiEmitter.emit("stream-end", streamEndEvent(workspaceId)); + + expect(await waitForCondition(() => !session.isBusy())).toBe(true); + await new Promise((resolve) => setTimeout(resolve, 25)); + expect(streamMessage).toHaveBeenCalledTimes(1); + session.drainQueuedMessagesIfIdle(); + await new Promise((resolve) => setTimeout(resolve, 25)); + expect(streamMessage).toHaveBeenCalledTimes(1); + } finally { + session.dispose(); + await cleanup(); + } + }); + + test("an unrelated queued entry that never starts does not revive the superseded delegated turn", async () => { + const workspaceId = "queue-dispatch-stranded-delegated-orphan"; + const harness = await createStreamingTurnHarness(workspaceId, { + sendOptions: { muxMetadata: WORKSPACE_TURN_CORRELATION }, + }); + const { session, cleanup, aiEmitter, streamMessage } = harness; + + try { + const wake = harness.queueCancelableWake(); + harness.latestRequest().onQueuedMessageStop?.({ modelString: TEST_MODEL }); + // The queued wake is not this turn's continuation: the owner settles the delegated turn. + expect(session.hasPendingWorkspaceTurnContinuation(WORKSPACE_TURN_CORRELATION)).toBe(false); + aiEmitter.emit("stream-end", streamEndEvent(workspaceId)); + // The wake is withdrawn after dequeue, before acceptance; the settled turn must stay cut. + wake.abort("monitor consumed"); + + expect(await waitForCondition(() => !session.isBusy())).toBe(true); + await new Promise((resolve) => setTimeout(resolve, 25)); + expect(streamMessage).toHaveBeenCalledTimes(1); + expect(session.hasPendingWorkspaceTurnContinuation(WORKSPACE_TURN_CORRELATION)).toBe(false); + } finally { + session.dispose(); + await cleanup(); + } + }); + + test("a message queued behind a rejected resume drains", async () => { + const workspaceId = "queue-dispatch-stranded-rejected-drain"; + let rejectGate: () => void = () => undefined; + let gateArmed = false; + let gateReached = false; + const workspaceGoalService = { + assertPricedModelForBudgetedGoal: mock(() => { + if (!gateArmed) { + return Promise.resolve(Ok(undefined)); + } + gateReached = true; + return new Promise((resolve) => { + rejectGate = () => resolve(Err({ type: "unknown", raw: "gate closed" })); + }); + }), + recordStreamAccounting: mock(() => Promise.resolve()), + applyPendingAfterStreamEnd: mock(() => Promise.resolve()), + requestContinuationAfterStreamEnd: mock(() => Promise.resolve()), + recordStreamStarted: mock(() => Promise.resolve()), + syncGoalModeWithChatTail: mock(() => Promise.resolve(null)), + } as unknown as WorkspaceGoalService; + const harness = await createStreamingTurnHarness(workspaceId, { + harness: { workspaceGoalService }, + sendInternal: { synthetic: true, agentInitiated: true }, + }); + const { session, cleanup, aiEmitter, streamMessage } = harness; + + try { + gateArmed = true; + const wake = harness.queueCancelableWake(); + harness.latestRequest().onQueuedMessageStop?.({ modelString: TEST_MODEL }); + wake.abort("monitor consumed"); + aiEmitter.emit("stream-end", streamEndEvent(workspaceId)); + expect(await waitForCondition(() => gateReached)).toBe(true); + + // WorkspaceService queues a send behind the busy (PREPARING) resume. + session.queueMessage("hello", { model: TEST_MODEL, agentId: "exec" }, { synthetic: true }); + gateArmed = false; + rejectGate(); + + // The rejected resume has no stream end to drain the queue at; the message must not wait + // for an unrelated later poke. + expect(await waitForCondition(() => streamMessage.mock.calls.length === 2)).toBe(true); + const dispatched = harness.latestRequest(); + const lastMessage = dispatched.messages[dispatched.messages.length - 1]; + expect(lastMessage?.parts.some((part) => part.type === "text" && part.text === "hello")).toBe( + true + ); + } finally { + session.dispose(); + await cleanup(); + } + }); + + test("the resume continues on the model that reached the cut", async () => { + const workspaceId = "queue-dispatch-stranded-fallback-model"; + const harness = await createStreamingTurnHarness(workspaceId); + const { session, cleanup, aiEmitter, streamMessage } = harness; + + try { + const wake = harness.queueCancelableWake(); + // StreamManager reports the request that was running at the stop: a configured fallback + // model, not the refused primary this stream was sent with. + harness.latestRequest().onQueuedMessageStop?.({ modelString: "anthropic:claude-opus-4-8" }); + wake.abort("monitor consumed"); + aiEmitter.emit("stream-end", streamEndEvent(workspaceId)); + + expect(await waitForCondition(() => streamMessage.mock.calls.length === 2)).toBe(true); + expect(harness.latestRequest().modelString).toBe("anthropic:claude-opus-4-8"); + } finally { + session.dispose(); + await cleanup(); + } + }); + + test("a mid-turn thinking change carries into the resume", async () => { + const workspaceId = "queue-dispatch-stranded-thinking"; + const harness = await createStreamingTurnHarness(workspaceId, { + sendOptions: { thinkingLevel: "low" }, + }); + const { session, cleanup, aiEmitter, streamMessage } = harness; + + try { + const original = harness.latestRequest(); + expect(original.thinkingLevel).toBe("low"); + // The user raised the level mid-turn and the loop applied it at a step boundary. + const holder = original.activeTurnThinkingOverride; + expect(holder).toBeDefined(); + if (holder != null) { + holder.applied = "high"; + } + + const wake = harness.queueCancelableWake(); + original.onQueuedMessageStop?.({ modelString: TEST_MODEL }); + wake.abort("monitor consumed"); + aiEmitter.emit("stream-end", streamEndEvent(workspaceId)); + + expect(await waitForCondition(() => streamMessage.mock.calls.length === 2)).toBe(true); + expect(harness.latestRequest().thinkingLevel).toBe("high"); + } finally { + session.dispose(); + await cleanup(); + } + }); + + test("a system hard stop during a pending provider-tool soft stop does not resume", async () => { + const workspaceId = "queue-dispatch-stranded-hard-system-stop"; + const harness = await createStreamingTurnHarness(workspaceId); + const { session, cleanup, aiEmitter, aiService, streamMessage } = harness; + const stopStream = spyOn(aiService, "stopStream").mockResolvedValue(Ok(undefined)); + + try { + session.queueMessage( + "follow up", + { model: TEST_MODEL, agentId: "exec" }, + { synthetic: true, dedupeKey: "wake:1" } + ); + aiEmitter.emit("tool-call-end", { + ...toolCallEndEvent(workspaceId), + toolName: "web_search", + providerExecuted: true, + }); + expect(stopStream).toHaveBeenCalledTimes(1); + + // task_stop / interrupt cascade: clears the queue and hard-stops through aiService directly, + // bypassing interruptStream, while the soft stop is still pending. + session.clearQueue("task stopped"); + aiEmitter.emit("stream-abort", streamAbortEvent(workspaceId, "system")); + + expect(await waitForCondition(() => !session.isBusy())).toBe(true); + await new Promise((resolve) => setTimeout(resolve, 25)); + expect(streamMessage).toHaveBeenCalledTimes(1); + expect(session.isBusy()).toBe(false); + } finally { + stopStream.mockRestore(); + session.dispose(); + await cleanup(); + } + }); + test("rollback failure preserves the wake and continues acceptance", async () => { const workspaceId = "queue-dispatch-cancel-rollback-failure"; const { session, cleanup, historyService } = await createAgentSessionHarness({ workspaceId }); @@ -1828,7 +2114,7 @@ describe("AgentSession queued message tool-call dispatch", () => { const interruptResult = await session.interruptStream(); expect(interruptResult.success).toBe(true); // The native soft-stop can still win the event race after the hard user interrupt. - aiEmitter.emit("stream-abort", streamAbortEvent(workspaceId, "system")); + aiEmitter.emit("stream-abort", streamAbortEvent(workspaceId, "queued-message")); await new Promise((resolve) => setTimeout(resolve, 25)); expect(sendQueuedMessages).not.toHaveBeenCalled(); diff --git a/src/node/services/agentSession.ts b/src/node/services/agentSession.ts index 859753189d..6fffafd5fa 100644 --- a/src/node/services/agentSession.ts +++ b/src/node/services/agentSession.ts @@ -383,12 +383,15 @@ interface StrandedTurnResume { * inherits it from history rather than from its own send options. */ function buildStrandedTurnResume(context: { + /** Model that reached the cut: a configured fallback may differ from the requested one. */ modelString: string; options?: SendMessageOptions; agentInitiated?: boolean; goalKind?: GoalSyntheticMessageKind; goalId?: string; workspaceTurnMetadata?: WorkspaceTurnMuxMetadata; + /** Effective level after a mid-turn thinking change; the request's own level is stale then. */ + appliedThinkingLevel?: ThinkingLevel; }): StrandedTurnResume { const resumeOptions = pickStartupRetrySendOptions( context.options ?? { model: context.modelString, agentId: WORKSPACE_DEFAULTS.agentId } @@ -396,6 +399,10 @@ function buildStrandedTurnResume(context: { return { options: { ...resumeOptions, + model: context.modelString, + ...(context.appliedThinkingLevel != null + ? { thinkingLevel: context.appliedThinkingLevel } + : {}), muxMetadata: context.workspaceTurnMetadata ?? getWorkspaceTurnMuxMetadata(context.options?.muxMetadata), }, @@ -746,7 +753,9 @@ export class AgentSession { // the moment the stop is decided until a stream actually starts (setTurnPhase STREAMING). // Recorded synchronously so the delegating owner's stream-end settlement can see it. private strandedTurnResume?: StrandedTurnResume; - private strandedTurnResumeInFlight = false; + // Set while the sweep's resume is running (claim through stream end); aborting it cancels only + // the pre-stream window, since StreamManager unlinks the signal once a stream registers. + private strandedTurnResumeInFlight: AbortController | null = null; private consecutiveStrandedResumes = 0; private idleWaiters: Array<() => void> = []; @@ -1603,11 +1612,11 @@ export class AgentSession { abortReason: StreamAbortReason | undefined, userMessageId?: string ): Promise { - // "system" and "startup" aborts come from backend-orchestrated flows - // (for example, mid-stream auto-compaction or canceling a pending startup). - // They are not user intent and must not poison startup recovery with a - // persisted non-retryable "aborted" marker. - if (abortReason === "system" || abortReason === "startup") { + // "system", "startup", and "queued-message" aborts come from backend-orchestrated flows + // (for example, mid-stream auto-compaction, canceling a pending startup, or the soft stop + // for a queued message). They are not user intent and must not poison startup recovery + // with a persisted non-retryable "aborted" marker. + if (abortReason === "system" || abortReason === "startup" || abortReason === "queued-message") { return; } @@ -4082,7 +4091,7 @@ export class AgentSession { // must not silently opt users back into auto-retry after they've disabled it. if (isManualUserMessage) { // The user's own message supersedes any continuation owed to a stranded turn. - this.strandedTurnResume = undefined; + this.withdrawStrandedTurnResume(); // A fresh accepted user send supersedes any persisted startup-abandon // classification from previous turns. await this.clearStartupAutoRetryAbandon(); @@ -4250,13 +4259,15 @@ export class AgentSession { agentInitiated?: boolean; goalKind?: GoalSyntheticMessageKind; goalId?: string; + /** Cancels the pre-stream window (admission gates, history reads) once the resume is withdrawn. */ + abortSignal?: AbortSignal; /** - * Caller admission gate awaited once the turn is claimed, so an async check (durable goal - * state) cannot race a manual send that would otherwise see an idle session. + * Revalidate a goal turn against durable goal state once the turn is claimed, the same + * veto durable goal redispatches apply; a refusal reports `goalRefused`. */ - admit?: () => Promise; + revalidateGoal?: boolean; } - ): Promise> { + ): Promise> { this.assertNotDisposed("resumeStream"); assert(options, "resumeStream requires options"); @@ -4302,6 +4313,7 @@ export class AgentSession { const turnThinkingOverride: ActiveTurnThinkingOverride = {}; this.activeTurnThinkingOverride = turnThinkingOverride; try { + const withdrawn = (): boolean => internal?.abortSignal?.aborted === true; if (this.workspaceGoalService) { const pricingGate = await this.workspaceGoalService.assertPricedModelForBudgetedGoal( this.workspaceId, @@ -4311,9 +4323,35 @@ export class AgentSession { return Err(pricingGate.error); } } - if (internal?.admit != null && !(await internal.admit())) { + if (withdrawn()) { + return Ok({ started: false }); + } + let goalAdmissionStale: (() => boolean) | undefined; + if ( + internal?.revalidateGoal === true && + internal.goalKind != null && + internal.goalId != null && + this.workspaceGoalService + ) { + const admission = await this.workspaceGoalService.buildGoalRedispatchAdmission( + this.workspaceId, + internal.goalId, + internal.goalKind + ); + if (!admission.admissible) { + return Ok({ started: false, goalRefused: true }); + } + goalAdmissionStale = admission.admissionStale; + } + // Last admission check before the stream's own pre-start I/O, like sendMessage's PREPARING + // gate: a withdrawal or goal transition that landed during the awaits above refuses the + // turn here; the abort signal covers the history reads that follow. + if (withdrawn()) { return Ok({ started: false }); } + if (goalAdmissionStale?.() === true) { + return Ok({ started: false, goalRefused: true }); + } // Must await here so the finally block runs after streaming completes, // not immediately when the Promise is returned. @@ -4323,7 +4361,7 @@ export class AgentSession { undefined, undefined, internal?.agentInitiated, - undefined, + internal?.abortSignal, internal?.goalKind, internal?.goalId, turnThinkingOverride @@ -4332,7 +4370,9 @@ export class AgentSession { return result; } - return Ok({ started: true }); + // A withdrawal inside streamWithHistory returns Ok before any stream registers, so the + // turn is still PREPARING here; after a real stream it has moved on. + return Ok({ started: !(withdrawn() && this.turnPhase === TurnPhase.PREPARING) }); } finally { if (this.turnPhase === TurnPhase.PREPARING) { this.setTurnPhase(TurnPhase.IDLE); @@ -4937,10 +4977,11 @@ export class AgentSession { this.assertNotDisposed("interruptStream"); // Explicit user interruption should immediately stop any pending auto-retry loop and - // withdraw any continuation owed to a stranded turn (the stream-abort handler repeats this - // for a stop that lands mid-step). + // withdraw any continuation owed to a stranded turn, including a resume still in its + // pre-stream window, which StreamManager has no stream to abort for (the stream-abort + // handler repeats this for a stop that lands mid-step). this.retryManager.cancel(); - this.strandedTurnResume = undefined; + this.withdrawStrandedTurnResume(); if (options?.soft !== true) { this.queuedProviderToolEndAbortInFlight = false; @@ -5284,9 +5325,13 @@ export class AgentSession { disableWorkspaceAgents: options?.disableWorkspaceAgents, strictAgentResolution: options?.strictAgentResolution, hasQueuedMessages: this.hasQueuedMessages.bind(this), - onQueuedMessageStop: () => { + onQueuedMessageStop: ({ modelString: stoppedModelString }) => { if (this.activeStreamContext != null) { - this.strandedTurnResume = buildStrandedTurnResume(this.activeStreamContext); + this.strandedTurnResume = buildStrandedTurnResume({ + ...this.activeStreamContext, + modelString: stoppedModelString, + appliedThinkingLevel: activeTurnThinkingOverride?.applied, + }); } }, openaiTruncationModeOverride, @@ -6059,13 +6104,17 @@ export class AgentSession { const hadCompactionRequest = this.activeCompactionRequest !== undefined; const abortedStreamContext = this.activeStreamContext; const abortReason = "abortReason" in payload ? payload.abortReason : undefined; + // The soft stop is recognized by its own reason, not by the in-flight flag alone: a hard + // "system" stop (task_stop, interrupt cascade, workflow timeout) can land while the soft + // stop is pending. The flag still gates it so a hard user interrupt that reset it cancels + // the dispatch even when the soft stop wins the event race. const isQueuedProviderToolEndAbort = - this.queuedProviderToolEndAbortInFlight && abortReason !== "user"; + abortReason === "queued-message" && this.queuedProviderToolEndAbortInFlight; // Only the queued-message soft stop owes the cut turn a continuation (rebuilt below from // the aborted context). A user Stop must leave the session idle even though the queue it // restores to the composer pokes the resume sweep; other aborts belong to auto-retry. if (!isQueuedProviderToolEndAbort) { - this.strandedTurnResume = undefined; + this.withdrawStrandedTurnResume(); } if (abortReason === "user") { await this.workspaceGoalService?.recordUserStoppedStream(this.workspaceId); @@ -6104,8 +6153,9 @@ export class AgentSession { await this.updateStartupAutoRetryAbandonFromAbort(abortReason, failedUserMessageId); this.emitChatEvent(payload); const dispatchedQueuedMessage = this.dispatchQueuedProviderToolEndMessageAfterAbort( - abortReason, - abortedStreamContext + isQueuedProviderToolEndAbort, + abortedStreamContext, + payload.metadata?.model ); if (!dispatchedQueuedMessage) { this.setTurnPhase(TurnPhase.IDLE); @@ -6230,7 +6280,7 @@ export class AgentSession { // Do not dispatch stream-end follow-ups while the edit flow is waiting // for IDLE; truncation must run before any synthetic turn resumes. The edit // rewrites the interrupted turn, so nothing is owed to it either. - this.strandedTurnResume = undefined; + this.withdrawStrandedTurnResume(); } else { if (handled) { this.strandedTurnResume = undefined; @@ -6456,7 +6506,7 @@ export class AgentSession { async discardAutoRetryForContextMutation(): Promise> { this.retryManager.cancel(); this.setAutoRetryResumeState(undefined); - this.strandedTurnResume = undefined; + this.withdrawStrandedTurnResume(); const deleteResult = await this.historyService.deletePartial(this.workspaceId); if (!deleteResult.success) { return Err(deleteResult.error); @@ -6865,6 +6915,15 @@ export class AgentSession { : undefined; } + /** + * Nothing is owed anymore (user Stop, superseding input, context discard): drop the marker and + * cancel a resume still in its pre-stream window, which already copied the marker. + */ + private withdrawStrandedTurnResume(): void { + this.strandedTurnResume = undefined; + this.strandedTurnResumeInFlight?.abort(); + } + /** * Input poised to take over this session at a queue cut. Engaged stages win * over the queue head; an engaged stage is reported even when its metadata is @@ -6924,7 +6983,7 @@ export class AgentSession { this.queuedProviderToolEndAbortInFlight = true; const result = await this.streamManager.stopStream(this.workspaceId, { soft: true, - abortReason: "system", + abortReason: "queued-message", }); if (!result.success) { this.queuedProviderToolEndAbortInFlight = false; @@ -6936,22 +6995,23 @@ export class AgentSession { } private dispatchQueuedProviderToolEndMessageAfterAbort( - abortReason: StreamAbortReason | undefined, - abortedStreamContext: AgentSession["activeStreamContext"] + isQueuedProviderToolEndAbort: boolean, + abortedStreamContext: AgentSession["activeStreamContext"], + abortedModelString: string | undefined ): boolean { - if (!this.queuedProviderToolEndAbortInFlight) { - return false; - } this.queuedProviderToolEndAbortInFlight = false; - - if (abortReason === "user" || this.deferQueuedFlushUntilAfterEdit) { + if (!isQueuedProviderToolEndAbort || this.deferQueuedFlushUntilAfterEdit) { return false; } // The soft stop was made on behalf of the queued message; if that message has been // withdrawn (or is withdrawn after dequeue), the interrupted turn must resume instead. if (abortedStreamContext != null) { - this.strandedTurnResume = buildStrandedTurnResume(abortedStreamContext); + this.strandedTurnResume = buildStrandedTurnResume({ + ...abortedStreamContext, + modelString: abortedModelString ?? abortedStreamContext.modelString, + appliedThinkingLevel: this.activeTurnThinkingOverride?.applied, + }); } if (!this.hasQueuedMessages()) { return false; @@ -7002,34 +7062,14 @@ export class AgentSession { // The owed continuation is cleared by the STREAMING transition, not here: a resume that // fails before its stream starts (pricing gate, history read) stays owed for the next // sweep, bounded by the cap above. - this.strandedTurnResumeInFlight = true; - const goalService = this.workspaceGoalService; - const { goalKind, goalId } = resume; + const inFlight = new AbortController(); + this.strandedTurnResumeInFlight = inFlight; this.resumeStream(resume.options, { agentInitiated: resume.agentInitiated, - goalKind, - goalId, - // A goal turn resumes only if the goal still admits it: a Pause or terminal transition - // that landed while the stream ran (applied at its end) forfeits the continuation, the - // same veto durable goal redispatches apply. - admit: - goalService != null && goalKind != null && goalId != null - ? async () => { - const admission = await goalService.buildGoalRedispatchAdmission( - this.workspaceId, - goalId, - goalKind - ); - if (!admission.admissible) { - this.strandedTurnResume = undefined; - log.info("Dropping stranded goal turn: goal no longer admits it", { - workspaceId: this.workspaceId, - goalKind, - }); - } - return admission.admissible; - } - : undefined, + goalKind: resume.goalKind, + goalId: resume.goalId, + abortSignal: inFlight.signal, + revalidateGoal: true, }) .then((result) => { if (!result.success) { @@ -7039,6 +7079,15 @@ export class AgentSession { }); return false; } + if (result.data.goalRefused === true) { + // A Pause or terminal transition landed while the goal turn ran: nothing is owed to it. + this.strandedTurnResume = undefined; + log.info("Dropping stranded goal turn: goal no longer admits it", { + workspaceId: this.workspaceId, + goalKind: resume.goalKind, + }); + return false; + } if (!result.data.started) { log.warn("Stranded turn resume did not start", { workspaceId: this.workspaceId }); return false; @@ -7053,12 +7102,15 @@ export class AgentSession { return false; }) .then((started) => { - this.strandedTurnResumeInFlight = false; + this.strandedTurnResumeInFlight = null; // resumeStream settles only after its stream ends, so a stranding of that resumed // stream can find the flag still set; sweep again once it clears. A resume that never - // started stays owed for the next natural poke rather than retrying in a tight loop. + // started stays owed for the next natural poke rather than retrying in a tight loop, + // but anything queued behind its PREPARING claim has no stream end to wait for. if (started) { this.resumeStrandedTurnIfIdle(); + } else { + this.dispatchQueuedMessagesIfIdle(); } }) .catch((error: unknown) => { @@ -7178,6 +7230,10 @@ export class AgentSession { */ drainQueuedMessagesIfIdle(): void { this.resumeStrandedTurnIfIdle(); + this.dispatchQueuedMessagesIfIdle(); + } + + private dispatchQueuedMessagesIfIdle(): void { if ( this.hasActiveOrPendingTurnWork() || this.deferQueuedFlushUntilAfterEdit || @@ -7212,6 +7268,23 @@ export class AgentSession { this.backgroundProcessManager.setMessageQueued(this.workspaceId, false); if (!this.messageQueue.isEmpty()) { + // A queued entry that is not the stranded delegated turn's own continuation takes the + // turn from it: the owner settles that turn at the cut (hasPendingWorkspaceTurnContinuation + // reports no continuation), so it must not resume later as orphaned work even if this + // entry never starts a stream. + const owedCorrelation = getWorkspaceTurnMuxMetadata( + this.strandedTurnResume?.options.muxMetadata + ); + if ( + owedCorrelation != null && + !this.messageQueue.hasNextWorkspaceTurnContinuation( + owedCorrelation.taskHandleId, + owedCorrelation.ownerWorkspaceId, + owedCorrelation.turnId + ) + ) { + this.strandedTurnResume = undefined; + } // Entries dispatch one at a time (FIFO): special sends (compaction, agent // skills, workspace-turn follow-ups) own their turn, and anything queued // behind them dispatches on a later drain instead of batching into them. diff --git a/src/node/services/streamManager.test.ts b/src/node/services/streamManager.test.ts index 365746cc9b..dcf5de122e 100644 --- a/src/node/services/streamManager.test.ts +++ b/src/node/services/streamManager.test.ts @@ -1226,10 +1226,12 @@ describe("StreamManager - stream resource scope", () => { describe("StreamManager - stopWhen configuration", () => { type StopWhenCondition = (options: { steps: unknown[] }) => boolean; type BuildStopWhenCondition = (request: { + modelString: string; hasQueuedMessages?: (dispatchMode?: "tool-end" | "turn-end") => boolean; - onQueuedMessageStop?: () => void; + onQueuedMessageStop?: (stop: { modelString: string }) => void; toolPolicy?: ToolPolicy; }) => StopWhenCondition[]; + const TEST_MODEL_STRING = "anthropic:claude-sonnet-4-5"; function buildStopWhenForTests(streamManager = new StreamManager(historyService)) { return getPrivateMethodForTests( @@ -1240,6 +1242,7 @@ describe("StreamManager - stopWhen configuration", () => { function requiredToolConditionForTests(toolPolicy: ToolPolicy): StopWhenCondition { const [, , requiredToolCondition] = buildStopWhenForTests()({ + modelString: TEST_MODEL_STRING, hasQueuedMessages: () => false, toolPolicy, }); @@ -1252,7 +1255,10 @@ describe("StreamManager - stopWhen configuration", () => { test("returns step-cap and queued-message conditions with no policy", () => { let queued = false; - const stopWhen = buildStopWhenForTests()({ hasQueuedMessages: () => queued }); + const stopWhen = buildStopWhenForTests()({ + modelString: TEST_MODEL_STRING, + hasQueuedMessages: () => queued, + }); expect(stopWhen).toHaveLength(3); const [maxStepCondition, queuedMessageCondition, requiredToolCondition] = stopWhen; @@ -1270,10 +1276,13 @@ describe("StreamManager - stopWhen configuration", () => { test("queued-message stop reports itself only when no required tool completed", () => { let queued = false; let stopsForQueuedMessage = 0; + let stoppedModel: string | undefined; const [, queuedMessageCondition] = buildStopWhenForTests()({ + modelString: "openai:gpt-5-fallback", hasQueuedMessages: () => queued, - onQueuedMessageStop: () => { + onQueuedMessageStop: ({ modelString }) => { stopsForQueuedMessage += 1; + stoppedModel = modelString; }, toolPolicy: [{ regex_match: "agent_report", action: "require" }], }); @@ -1285,6 +1294,8 @@ describe("StreamManager - stopWhen configuration", () => { queued = true; expect(queuedMessageCondition(bashStep)).toBe(true); expect(stopsForQueuedMessage).toBe(1); + // The stop names the request's own model, which is the fallback's after a model swap. + expect(stoppedModel).toBe("openai:gpt-5-fallback"); expect(queuedMessageCondition(stepsWithToolResult("agent_report", { success: true }))).toBe( true @@ -1300,6 +1311,7 @@ describe("StreamManager - stopWhen configuration", () => { test("queued-message stop does not report itself once the step cap is reached", () => { let stopsForQueuedMessage = 0; const [maxStepCondition, queuedMessageCondition] = buildStopWhenForTests()({ + modelString: TEST_MODEL_STRING, hasQueuedMessages: () => true, onQueuedMessageStop: () => { stopsForQueuedMessage += 1; diff --git a/src/node/services/streamManager.ts b/src/node/services/streamManager.ts index c7c1812bfd..4c2d8d3e43 100644 --- a/src/node/services/streamManager.ts +++ b/src/node/services/streamManager.ts @@ -247,7 +247,7 @@ interface StreamRequestOptions { callSettingsOverrides?: ResolvedCallSettingsOverrides; toolPolicy?: ToolPolicy; hasQueuedMessages?: (dispatchMode?: "tool-end" | "turn-end") => boolean; - onQueuedMessageStop?: () => void; + onQueuedMessageStop?: (stop: { modelString: string }) => void; headers?: Record; onChunk?: StreamTextOnChunk; onStepMessages?: (messages: ModelMessage[]) => void; @@ -283,6 +283,8 @@ interface StepMessageTracker { } interface StreamRequestConfig { model: LanguageModel; + /** Canonical model string of `model` (the fallback's once a fallback request replaces this). */ + modelString: string; messages: ModelMessage[]; /** Provider-ready system instructions from TurnContextAssembler. */ system?: string | SystemModelMessage; @@ -296,9 +298,10 @@ interface StreamRequestConfig { /** * Invoked when the loop stops on behalf of a queued tool-end message (and not * because a required tool completed). The session uses it to resume the turn - * if that queued message is later withdrawn instead of starting a turn. + * if that queued message is later withdrawn instead of starting a turn. Carries the + * model that reached the cut, which a configured fallback may have swapped mid-turn. */ - onQueuedMessageStop?: () => void; + onQueuedMessageStop?: (stop: { modelString: string }) => void; /** Optional hook for callers that need chunk-level visibility during streaming. */ onChunk?: StreamTextOnChunk; /** Optional hook for callers that need the live prepared step transcript. */ @@ -1940,7 +1943,14 @@ export class StreamManager { const abortDelivery = this.emitStreamAbort( workspaceId, streamInfo.messageId, - { usage, contextUsage, duration, providerMetadata, contextProviderMetadata }, + { + usage, + contextUsage, + duration, + providerMetadata, + contextProviderMetadata, + model: streamInfo.model, + }, abortReason, abandonPartial, streamInfo.initialMetadata?.acpPromptId @@ -2118,6 +2128,7 @@ export class StreamManager { return { model, + modelString, messages, system, // Keep provider-level parallel tool planning enabled, but serialize sibling @@ -2142,7 +2153,10 @@ export class StreamManager { } private createStopWhenCondition( - request: Pick + request: Pick< + StreamRequestConfig, + "hasQueuedMessages" | "onQueuedMessageStop" | "toolPolicy" | "modelString" + > ): Array> { // Completion-tool stop check: completion/routing tools use explicit // success/ok markers (agent_report, propose_plan). @@ -2193,7 +2207,7 @@ export class StreamManager { // The step cap and a successful required tool result each end the turn on their // own; only a stop made purely for the queued message may need resuming later. if (state.steps.length < MAX_STREAM_STEPS && !hasSuccessfulRequiredToolResult(state)) { - request.onQueuedMessageStop?.(); + request.onQueuedMessageStop?.({ modelString: request.modelString }); } return true; }; diff --git a/src/node/services/turnRequestBuilder.ts b/src/node/services/turnRequestBuilder.ts index d8eee22052..4d0f3ffdb8 100644 --- a/src/node/services/turnRequestBuilder.ts +++ b/src/node/services/turnRequestBuilder.ts @@ -274,7 +274,7 @@ export interface StreamMessageOptions { disableWorkspaceAgents?: boolean; hasQueuedMessages?: (dispatchMode?: "tool-end" | "turn-end") => boolean; /** Fires when the model loop stops solely on behalf of a queued tool-end message. */ - onQueuedMessageStop?: () => void; + onQueuedMessageStop?: (stop: { modelString: string }) => void; muxMetadata?: MuxMessageMetadata; openaiTruncationModeOverride?: "auto" | "disabled"; /** From e5cf350e9ed6301f28e14668c84481906e8cd678 Mon Sep 17 00:00:00 2001 From: Michael Suchacz <203725896+ibetitsmike@users.noreply.github.com> Date: Thu, 3 Sep 2026 01:45:52 +0000 Subject: [PATCH 05/28] Hold the stranded resume's admission to the launch boundary Codex round 4 on #4065: - Thread the goal admission probe from resumeStream through streamWithHistory and aiService.streamMessage into TurnExecutionOptions.refuseStreamStart, so a Pause or goal replacement landing during the pre-stream history reads or request construction refuses the launch; StreamManager rechecks it right before the stream registers. - dispose() withdraws an in-flight stranded resume: past streamWithHistory's disposed check only its abort signal can stop it registering a stream after teardown. - A synthetic pre-stream abort (task_stop / interrupt cascade through aiService.stopStream with no registered stream) withdraws a preparing resume; only the queued-message soft stop keeps its obligation. - The resume snapshots a mid-turn thinking change still pending at the cut, not only an already applied one. --- .../agentSession.queueDispatch.test.ts | 185 ++++++++++++++++++ src/node/services/agentSession.ts | 63 ++++-- src/node/services/streamManager.test.ts | 39 ++++ src/node/services/streamManager.ts | 7 +- src/node/services/turnRequestBuilder.ts | 7 + 5 files changed, 287 insertions(+), 14 deletions(-) diff --git a/src/node/services/agentSession.queueDispatch.test.ts b/src/node/services/agentSession.queueDispatch.test.ts index 60c3f7d200..7692a37f1f 100644 --- a/src/node/services/agentSession.queueDispatch.test.ts +++ b/src/node/services/agentSession.queueDispatch.test.ts @@ -1718,6 +1718,191 @@ describe("AgentSession queued message tool-call dispatch", () => { } }); + /** + * Strand a turn whose resume parks on its first pre-stream I/O (commitPartial) until the + * caller releases it, so an event can land while the resume is PREPARING with no stream + * registered for StreamManager to abort. + */ + async function strandWithResumeParkedInPreStreamIo(workspaceId: string) { + const harness = await createStreamingTurnHarness(workspaceId, { + sendInternal: { synthetic: true, agentInitiated: true }, + }); + const { historyService } = harness; + const originalCommitPartial = historyService.commitPartial.bind(historyService); + let releaseIo: () => void = () => undefined; + let ioReached = false; + const commitPartial = spyOn(historyService, "commitPartial").mockImplementation( + async (...args) => { + ioReached = true; + await new Promise((resolve) => { + releaseIo = resolve; + }); + return originalCommitPartial(...args); + } + ); + const wake = harness.queueCancelableWake(); + harness.latestRequest().onQueuedMessageStop?.({ modelString: TEST_MODEL }); + wake.abort("monitor consumed"); + harness.aiEmitter.emit("stream-end", streamEndEvent(workspaceId)); + expect(await waitForCondition(() => ioReached)).toBe(true); + expect(harness.session.isBusy()).toBe(true); + return { + ...harness, + releaseIo: () => releaseIo(), + restore: () => commitPartial.mockRestore(), + }; + } + + test("a goal transition during the resume's pre-stream I/O drops the resume", async () => { + const workspaceId = "queue-dispatch-stranded-goal-stale-in-io"; + let stale = false; + const workspaceGoalService = { + assertPricedModelForBudgetedGoal: mock(() => Promise.resolve(Ok(undefined))), + buildGoalRedispatchAdmission: mock(() => + Promise.resolve({ admissible: true as const, admissionStale: () => stale }) + ), + recordStreamAccounting: mock(() => Promise.resolve()), + applyPendingAfterStreamEnd: mock(() => Promise.resolve()), + requestContinuationAfterStreamEnd: mock(() => Promise.resolve()), + recordStreamStarted: mock(() => Promise.resolve()), + syncGoalModeWithChatTail: mock(() => Promise.resolve(null)), + completeGoalFromSilentContinuation: mock(() => Promise.resolve(false)), + } as unknown as WorkspaceGoalService; + const aiEmitter = new EventEmitter(); + const streamMessage = mock((_options: StreamMessageOptions) => { + aiEmitter.emit("stream-start", streamStartEvent(workspaceId)); + return Promise.resolve(Ok(createStartedTurnHandle("assistant-1"))); + }); + const { session, cleanup, historyService } = await createAgentSessionHarness({ + workspaceId, + aiEmitter, + aiServiceOverrides: { streamMessage: streamMessage as unknown as AIService["streamMessage"] }, + workspaceGoalService, + }); + const originalCommitPartial = historyService.commitPartial.bind(historyService); + let armed = false; + const commitPartial = spyOn(historyService, "commitPartial").mockImplementation((...args) => { + // The Pause lands after the resume's admission read, inside the stream's own pre-start I/O. + if (armed) { + stale = true; + } + return originalCommitPartial(...args); + }); + + try { + await historyService.appendToHistory( + workspaceId, + createMuxMessage("user-0", "user", "keep going", { timestamp: Date.now() }) + ); + const resumed = await session.resumeStream( + { model: TEST_MODEL, agentId: "exec" }, + { agentInitiated: true, goalKind: GOAL_CONTINUATION_KIND, goalId: "goal-1" } + ); + expect(resumed.success).toBe(true); + await historyService.appendToHistory( + workspaceId, + createMuxMessage("assistant-1", "assistant", "ran the checks", { timestamp: Date.now() }) + ); + + const controller = new AbortController(); + session.queueMessage( + "Background monitor wake", + { model: TEST_MODEL, agentId: "exec" }, + { + synthetic: true, + agentInitiated: true, + cancelState: { canceledBeforeAcceptance: false }, + cancelSignal: controller.signal, + onCanceled: () => undefined, + } + ); + armed = true; + streamMessage.mock.calls[0]?.[0].onQueuedMessageStop?.({ modelString: TEST_MODEL }); + controller.abort("monitor consumed"); + aiEmitter.emit("stream-end", streamEndEvent(workspaceId)); + + expect(await waitForCondition(() => !session.isBusy())).toBe(true); + await new Promise((resolve) => setTimeout(resolve, 25)); + expect(streamMessage).toHaveBeenCalledTimes(1); + } finally { + commitPartial.mockRestore(); + session.dispose(); + await cleanup(); + } + }); + + test("disposing the session during the resume's pre-stream I/O cancels it", async () => { + const workspaceId = "queue-dispatch-stranded-dispose-in-io"; + const harness = await strandWithResumeParkedInPreStreamIo(workspaceId); + const { session, cleanup, streamMessage } = harness; + + try { + // Workspace removal tears the session down while the resume is past streamWithHistory's + // disposed check and StreamManager has no stream to stop. + session.dispose(); + harness.releaseIo(); + + await new Promise((resolve) => setTimeout(resolve, 50)); + expect(streamMessage).toHaveBeenCalledTimes(1); + } finally { + harness.restore(); + await cleanup(); + } + }); + + test("a system hard stop with no registered stream cancels a preparing resume", async () => { + const workspaceId = "queue-dispatch-stranded-system-stop-in-io"; + const harness = await strandWithResumeParkedInPreStreamIo(workspaceId); + const { session, cleanup, aiEmitter, streamMessage } = harness; + + try { + // task_stop / interrupt cascade: clears the queue and hard-stops through aiService while + // the resume is still preparing, so StreamManager emits a synthetic pre-stream abort. + session.clearQueue("task stopped"); + aiEmitter.emit("stream-abort", streamAbortEvent(workspaceId, "system")); + harness.releaseIo(); + + expect(await waitForCondition(() => !session.isBusy())).toBe(true); + await new Promise((resolve) => setTimeout(resolve, 25)); + expect(streamMessage).toHaveBeenCalledTimes(1); + session.drainQueuedMessagesIfIdle(); + await new Promise((resolve) => setTimeout(resolve, 25)); + expect(streamMessage).toHaveBeenCalledTimes(1); + } finally { + harness.restore(); + session.dispose(); + await cleanup(); + } + }); + + test("a thinking change still pending at the cut carries into the resume", async () => { + const workspaceId = "queue-dispatch-stranded-pending-thinking"; + const harness = await createStreamingTurnHarness(workspaceId, { + sendOptions: { thinkingLevel: "low" }, + }); + const { session, cleanup, aiEmitter, streamMessage } = harness; + + try { + const original = harness.latestRequest(); + expect(original.thinkingLevel).toBe("low"); + // The user raised the level while a tool was running; the boundary was cut before any + // prepareStep could apply it. + expect(session.setActiveTurnThinkingLevel("high").accepted).toBe(true); + expect(original.activeTurnThinkingOverride?.applied).toBeUndefined(); + + const wake = harness.queueCancelableWake(); + original.onQueuedMessageStop?.({ modelString: TEST_MODEL }); + wake.abort("monitor consumed"); + aiEmitter.emit("stream-end", streamEndEvent(workspaceId)); + + expect(await waitForCondition(() => streamMessage.mock.calls.length === 2)).toBe(true); + expect(harness.latestRequest().thinkingLevel).toBe("high"); + } finally { + session.dispose(); + await cleanup(); + } + }); + test("rollback failure preserves the wake and continues acceptance", async () => { const workspaceId = "queue-dispatch-cancel-rollback-failure"; const { session, cleanup, historyService } = await createAgentSessionHarness({ workspaceId }); diff --git a/src/node/services/agentSession.ts b/src/node/services/agentSession.ts index 6fffafd5fa..da2e1578dd 100644 --- a/src/node/services/agentSession.ts +++ b/src/node/services/agentSession.ts @@ -390,8 +390,12 @@ function buildStrandedTurnResume(context: { goalKind?: GoalSyntheticMessageKind; goalId?: string; workspaceTurnMetadata?: WorkspaceTurnMuxMetadata; - /** Effective level after a mid-turn thinking change; the request's own level is stale then. */ - appliedThinkingLevel?: ThinkingLevel; + /** + * Level a mid-turn thinking change left in effect at the cut: still pending when the boundary + * was cut during a tool call (no prepareStep consumed it), else the last applied one. The + * request's own level is stale then; streamWithHistory re-clamps against the model floor. + */ + thinkingLevelAtCut?: ThinkingLevel; }): StrandedTurnResume { const resumeOptions = pickStartupRetrySendOptions( context.options ?? { model: context.modelString, agentId: WORKSPACE_DEFAULTS.agentId } @@ -400,9 +404,7 @@ function buildStrandedTurnResume(context: { options: { ...resumeOptions, model: context.modelString, - ...(context.appliedThinkingLevel != null - ? { thinkingLevel: context.appliedThinkingLevel } - : {}), + ...(context.thinkingLevelAtCut != null ? { thinkingLevel: context.thinkingLevelAtCut } : {}), muxMetadata: context.workspaceTurnMetadata ?? getWorkspaceTurnMuxMetadata(context.options?.muxMetadata), }, @@ -1039,6 +1041,9 @@ export class AgentSession { this.activePreparedTurnAbortController?.abort(); this.activePreparedTurnAbortController = null; + // A resume parked in its pre-stream I/O passed streamWithHistory's disposed check already; + // its abort signal is the only thing that stops it registering a stream after teardown. + this.withdrawStrandedTurnResume(); // Ensure any callers blocked on waitForIdle() can continue during teardown. this.setTurnPhase(TurnPhase.IDLE); @@ -4344,12 +4349,24 @@ export class AgentSession { goalAdmissionStale = admission.admissionStale; } // Last admission check before the stream's own pre-start I/O, like sendMessage's PREPARING - // gate: a withdrawal or goal transition that landed during the awaits above refuses the - // turn here; the abort signal covers the history reads that follow. + // gate: a withdrawal that landed during the awaits above refuses the turn here; the abort + // signal covers the history reads that follow. if (withdrawn()) { return Ok({ started: false }); } - if (goalAdmissionStale?.() === true) { + // The goal probe rides along to the stream-admission boundary: a Pause or goal replacement + // landing during the history reads and request construction below has no stream to + // interrupt, so the launch itself rechecks it (StreamManager last, right before + // registration). Sticky so the return value matches what refused the launch. + let goalRefused = false; + const refuseStreamStart = + goalAdmissionStale != null + ? (): boolean => { + goalRefused ||= goalAdmissionStale(); + return goalRefused; + } + : undefined; + if (refuseStreamStart?.() === true) { return Ok({ started: false, goalRefused: true }); } @@ -4364,11 +4381,15 @@ export class AgentSession { internal?.abortSignal, internal?.goalKind, internal?.goalId, - turnThinkingOverride + turnThinkingOverride, + refuseStreamStart ); if (!result.success) { return result; } + if (goalRefused) { + return Ok({ started: false, goalRefused: true }); + } // A withdrawal inside streamWithHistory returns Ok before any stream registers, so the // turn is still PREPARING here; after a real stream it has moved on. @@ -5091,9 +5112,13 @@ export class AgentSession { // Session-owned per-turn holder for mid-turn thinking changes. Passed // explicitly (not read from the field) so a preempted turn can never pick // up its replacement's holder. Absent for internal retry paths. - activeTurnThinkingOverride?: ActiveTurnThinkingOverride + activeTurnThinkingOverride?: ActiveTurnThinkingOverride, + // Pull-based admission probe (goal state) with no push into abortSignal; checked wherever + // the signal is, and by StreamManager right before the stream registers. + refuseStreamStart?: () => boolean ): Promise> { - const isStartupAbortRequested = (): boolean => abortSignal?.aborted === true; + const isStartupAbortRequested = (): boolean => + abortSignal?.aborted === true || refuseStreamStart?.() === true; if (this.disposed || isStartupAbortRequested()) { return Ok(undefined); @@ -5298,6 +5323,7 @@ export class AgentSession { workspaceId: this.workspaceId, modelString, abortSignal, + refuseStreamStart, thinkingLevel: effectiveThinkingLevel, // Orthogonal to thinking level; buildRequestHeaders gates it per model. reasoningMode: options?.reasoningMode, @@ -5330,7 +5356,8 @@ export class AgentSession { this.strandedTurnResume = buildStrandedTurnResume({ ...this.activeStreamContext, modelString: stoppedModelString, - appliedThinkingLevel: activeTurnThinkingOverride?.applied, + thinkingLevelAtCut: + activeTurnThinkingOverride?.pending ?? activeTurnThinkingOverride?.applied, }); } }, @@ -6066,6 +6093,15 @@ export class AgentSession { }); const preStreamAbortReason = "abortReason" in payload ? payload.abortReason : undefined; + // A hard stop with no registered stream (task_stop / interrupt cascade through + // aiService.stopStream while a stranded resume is still in its pre-stream I/O) must + // withdraw that resume here: StreamManager had nothing to abort, and the STREAMING + // branch below never runs for it. Only the queued-message soft stop keeps its obligation. + if ( + !(preStreamAbortReason === "queued-message" && this.queuedProviderToolEndAbortInFlight) + ) { + this.withdrawStrandedTurnResume(); + } if (this.turnPhase === TurnPhase.PREPARING) { this.clearPreparingRuntimeStatus(); this.setTerminalStreamLifecycle("interrupted", { @@ -7010,7 +7046,8 @@ export class AgentSession { this.strandedTurnResume = buildStrandedTurnResume({ ...abortedStreamContext, modelString: abortedModelString ?? abortedStreamContext.modelString, - appliedThinkingLevel: this.activeTurnThinkingOverride?.applied, + thinkingLevelAtCut: + this.activeTurnThinkingOverride?.pending ?? this.activeTurnThinkingOverride?.applied, }); } if (!this.hasQueuedMessages()) { diff --git a/src/node/services/streamManager.test.ts b/src/node/services/streamManager.test.ts index dcf5de122e..fe3287df77 100644 --- a/src/node/services/streamManager.test.ts +++ b/src/node/services/streamManager.test.ts @@ -2811,6 +2811,45 @@ describe("StreamManager - Concurrent Stream Prevention", () => { } }); + test("refuses registration when the caller's admission probe turns stale during setup", async () => { + const workspaceId = "test-workspace-refuse-before-create"; + + let createCalled = false; + let streamStartEmitted = false; + let refused = false; + + onTurnEngineEvent(streamManager, "stream-start", () => { + streamStartEmitted = true; + }); + Reflect.set(streamManager, "createTempDirForStream", (): Promise => { + // A goal Pause lands during startup I/O: nothing aborts the signal, only the probe knows. + refused = true; + return Promise.resolve("/tmp/mock-stream-temp"); + }); + Reflect.set(streamManager, "cleanupStreamTempDir", (): void => undefined); + Reflect.set(streamManager, "createStreamAtomically", (): never => { + createCalled = true; + throw new Error("createStreamAtomically should not be called"); + }); + + const result = await streamManager.startStream( + testStartOptions({ + workspaceId, + messageId: "test-msg-refuse", + model: createTestLanguageModel(), + runtime, + refuseStreamStart: () => refused, + tools: {}, + }) + ); + expect(result.success).toBe(true); + if (!result.success) throw new Error("Expected aborted startup handle"); + expect(await result.data.completion).toEqual({ status: "aborted", abortReason: "startup" }); + expect(createCalled).toBe(false); + expect(streamStartEmitted).toBe(false); + expect(streamManager.isStreaming(workspaceId)).toBe(false); + }); + test("should honor abortSignal before atomic stream creation", async () => { const workspaceId = "test-workspace-abort-before-create"; diff --git a/src/node/services/streamManager.ts b/src/node/services/streamManager.ts index 4c2d8d3e43..5dfe7585c8 100644 --- a/src/node/services/streamManager.ts +++ b/src/node/services/streamManager.ts @@ -265,6 +265,8 @@ export interface TurnExecutionOptions extends StreamRequestOptions { runtime: Runtime; messageId: string; abortSignal?: AbortSignal; + /** Startup-only admission probe; a true answer at registration time refuses the stream like an abort. */ + refuseStreamStart?: () => boolean; initialMetadata?: Partial; providedStreamToken?: StreamToken; workspaceName?: string; @@ -4771,6 +4773,7 @@ export class StreamManager { runtime, messageId, abortSignal, + refuseStreamStart, providedStreamToken, providedRuntimeTempDir, onStreamConstructed, @@ -4862,7 +4865,9 @@ export class StreamManager { ) ); - if (streamAbortController.signal.aborted) { + // The caller's pull-based admission probe (goal state) has no signal to abort; this is + // its last read before the stream becomes real. + if (streamAbortController.signal.aborted || refuseStreamStart?.() === true) { return settleStartupAbort(); } diff --git a/src/node/services/turnRequestBuilder.ts b/src/node/services/turnRequestBuilder.ts index 4d0f3ffdb8..7428fca977 100644 --- a/src/node/services/turnRequestBuilder.ts +++ b/src/node/services/turnRequestBuilder.ts @@ -275,6 +275,11 @@ export interface StreamMessageOptions { hasQueuedMessages?: (dispatchMode?: "tool-end" | "turn-end") => boolean; /** Fires when the model loop stops solely on behalf of a queued tool-end message. */ onQueuedMessageStop?: (stop: { modelString: string }) => void; + /** + * Pull-based startup refusal (a goal admission probe with no push into abortSignal), rechecked + * by StreamManager right before the stream registers. + */ + refuseStreamStart?: () => boolean; muxMetadata?: MuxMessageMetadata; openaiTruncationModeOverride?: "auto" | "disabled"; /** @@ -740,6 +745,7 @@ export class TurnRequestBuilder { disableWorkspaceAgents, hasQueuedMessages, onQueuedMessageStop, + refuseStreamStart, openaiTruncationModeOverride, muxMetadata, minThinkingLevel: providedMinThinkingLevel, @@ -2861,6 +2867,7 @@ export class TurnRequestBuilder { providedStreamToken: streamToken, hasQueuedMessages, onQueuedMessageStop, + refuseStreamStart, workspaceName: metadata.name, thinkingLevel: streamThinkingLevel, headers: requestHeaders, From 4a784323f17386bbd08de67bc4903e948b0b2de9 Mon Sep 17 00:00:00 2001 From: Michael Suchacz <203725896+ibetitsmike@users.noreply.github.com> Date: Thu, 3 Sep 2026 02:03:08 +0000 Subject: [PATCH 06/28] Count only never-started resumes toward the stranded resume cap Dogfood UAT: a prompt that awaits four monitored background processes in a row (each task_await consuming the wake) strands four times; the old cap counted every resume and dropped the fourth, leaving the turn on a text-less tool row with no answer, the original symptom. Every stranding follows a completed model step, so a resume that starts a stream is real progress and resets the counter; the cap now bounds only resume attempts that fail before their stream starts (pricing gate, history read, refused admission). --- .../agentSession.queueDispatch.test.ts | 106 +++++++++++++----- src/node/services/agentSession.ts | 17 +-- 2 files changed, 89 insertions(+), 34 deletions(-) diff --git a/src/node/services/agentSession.queueDispatch.test.ts b/src/node/services/agentSession.queueDispatch.test.ts index 7692a37f1f..22d17bf3c4 100644 --- a/src/node/services/agentSession.queueDispatch.test.ts +++ b/src/node/services/agentSession.queueDispatch.test.ts @@ -890,31 +890,76 @@ describe("AgentSession queued message tool-call dispatch", () => { } }); - test("caps consecutive stranded resumes", async () => { - const workspaceId = "queue-dispatch-stranded-cap"; + test("a turn stranded after each of several awaited monitors resumes every time", async () => { + const workspaceId = "queue-dispatch-stranded-chain"; const harness = await createStreamingTurnHarness(workspaceId); const { session, cleanup, aiEmitter, streamMessage } = harness; try { + // Each cut follows a completed step whose task_await consumed the monitor's wake (dogfood + // UAT: four sequential background+await calls in one prompt); the cap must not end it. const strandTurn = () => { harness.latestRequest().onQueuedMessageStop?.({ modelString: TEST_MODEL }); harness.queueCancelableWake().abort("monitor consumed"); aiEmitter.emit("stream-end", streamEndEvent(workspaceId)); }; - for (let resumes = 1; resumes <= 3; resumes += 1) { + for (let resumes = 1; resumes <= 4; resumes += 1) { strandTurn(); expect(await waitForCondition(() => streamMessage.mock.calls.length === resumes + 1)).toBe( true ); expect(session.isBusy()).toBe(true); } + } finally { + session.dispose(); + await cleanup(); + } + }); - strandTurn(); + test("caps resume attempts that never start a stream", async () => { + const workspaceId = "queue-dispatch-stranded-cap"; + let gateOpen = true; + const pricingGate = mock(() => + Promise.resolve(gateOpen ? Ok(undefined) : Err({ type: "unknown", raw: "gate closed" })) + ); + const workspaceGoalService = { + assertPricedModelForBudgetedGoal: pricingGate, + recordStreamAccounting: mock(() => Promise.resolve()), + applyPendingAfterStreamEnd: mock(() => Promise.resolve()), + requestContinuationAfterStreamEnd: mock(() => Promise.resolve()), + recordStreamStarted: mock(() => Promise.resolve()), + syncGoalModeWithChatTail: mock(() => Promise.resolve(null)), + } as unknown as WorkspaceGoalService; + const harness = await createStreamingTurnHarness(workspaceId, { + harness: { workspaceGoalService }, + sendInternal: { synthetic: true, agentInitiated: true }, + }); + const { session, cleanup, aiEmitter, streamMessage } = harness; + + try { + gateOpen = false; + const wake = harness.queueCancelableWake(); + harness.latestRequest().onQueuedMessageStop?.({ modelString: TEST_MODEL }); + wake.abort("monitor consumed"); + aiEmitter.emit("stream-end", streamEndEvent(workspaceId)); expect(await waitForCondition(() => !session.isBusy())).toBe(true); + const attemptsBefore = pricingGate.mock.calls.length; + + // Each idle poke retries the failing resume until the cap, then the marker is dropped and + // further pokes do nothing. + for (let poke = 1; poke <= 4; poke += 1) { + session.drainQueuedMessagesIfIdle(); + expect(await waitForCondition(() => !session.isBusy())).toBe(true); + } await new Promise((resolve) => setTimeout(resolve, 25)); - expect(streamMessage).toHaveBeenCalledTimes(4); - expect(session.isBusy()).toBe(false); + expect(pricingGate.mock.calls.length - attemptsBefore).toBe(2); + expect(streamMessage).toHaveBeenCalledTimes(1); + + gateOpen = true; + session.drainQueuedMessagesIfIdle(); + await new Promise((resolve) => setTimeout(resolve, 25)); + expect(streamMessage).toHaveBeenCalledTimes(1); } finally { session.dispose(); await cleanup(); @@ -1397,35 +1442,44 @@ describe("AgentSession queued message tool-call dispatch", () => { test("stops advertising the delegated continuation once the resume cap is exhausted", async () => { const workspaceId = "queue-dispatch-stranded-delegated-cap"; + let gateOpen = true; + const workspaceGoalService = { + assertPricedModelForBudgetedGoal: mock(() => + Promise.resolve(gateOpen ? Ok(undefined) : Err({ type: "unknown", raw: "gate closed" })) + ), + recordStreamAccounting: mock(() => Promise.resolve()), + applyPendingAfterStreamEnd: mock(() => Promise.resolve()), + requestContinuationAfterStreamEnd: mock(() => Promise.resolve()), + recordStreamStarted: mock(() => Promise.resolve()), + syncGoalModeWithChatTail: mock(() => Promise.resolve(null)), + } as unknown as WorkspaceGoalService; const harness = await createStreamingTurnHarness(workspaceId, { + harness: { workspaceGoalService }, sendOptions: { muxMetadata: WORKSPACE_TURN_CORRELATION }, + sendInternal: { synthetic: true, agentInitiated: true }, }); const { session, cleanup, aiEmitter, streamMessage } = harness; try { - const strandTurn = () => { - harness.latestRequest().onQueuedMessageStop?.({ modelString: TEST_MODEL }); - harness.queueCancelableWake().abort("monitor consumed"); - session.clearQueue("monitor consumed"); - }; - - for (let resumes = 1; resumes <= 3; resumes += 1) { - strandTurn(); - expect(session.hasPendingWorkspaceTurnContinuation(WORKSPACE_TURN_CORRELATION)).toBe(true); - aiEmitter.emit("stream-end", streamEndEvent(workspaceId)); - expect(await waitForCondition(() => streamMessage.mock.calls.length === resumes + 1)).toBe( - true - ); - } - - // No resume will follow this cut, so the owner must settle the turn here instead of - // deferring to a continuation that never starts. - strandTurn(); - expect(session.hasPendingWorkspaceTurnContinuation(WORKSPACE_TURN_CORRELATION)).toBe(false); + gateOpen = false; + harness.latestRequest().onQueuedMessageStop?.({ modelString: TEST_MODEL }); + harness.queueCancelableWake().abort("monitor consumed"); + session.clearQueue("monitor consumed"); + expect(session.hasPendingWorkspaceTurnContinuation(WORKSPACE_TURN_CORRELATION)).toBe(true); aiEmitter.emit("stream-end", streamEndEvent(workspaceId)); expect(await waitForCondition(() => !session.isBusy())).toBe(true); + + // The resume keeps failing before its stream starts; while attempts remain the owner + // defers settlement, and once the cap is exhausted the continuation is no longer + // advertised so the owner settles the turn at the cut. + session.drainQueuedMessagesIfIdle(); + expect(await waitForCondition(() => !session.isBusy())).toBe(true); + expect(session.hasPendingWorkspaceTurnContinuation(WORKSPACE_TURN_CORRELATION)).toBe(true); + session.drainQueuedMessagesIfIdle(); + expect(await waitForCondition(() => !session.isBusy())).toBe(true); + expect(session.hasPendingWorkspaceTurnContinuation(WORKSPACE_TURN_CORRELATION)).toBe(false); await new Promise((resolve) => setTimeout(resolve, 25)); - expect(streamMessage).toHaveBeenCalledTimes(4); + expect(streamMessage).toHaveBeenCalledTimes(1); } finally { session.dispose(); await cleanup(); diff --git a/src/node/services/agentSession.ts b/src/node/services/agentSession.ts index da2e1578dd..529afe7784 100644 --- a/src/node/services/agentSession.ts +++ b/src/node/services/agentSession.ts @@ -568,8 +568,11 @@ const STARTUP_AUTO_RETRY_HISTORY_FAILURE_BASE_DELAY_MS = 1_000; const STARTUP_AUTO_RETRY_HISTORY_FAILURE_MAX_DELAY_MS = 30_000; const MAX_STARTUP_RECOVERY_DEFERRED_ATTEMPTS = 4; /** - * Runaway guard for pathological queue flapping (stop for a queued message, withdraw it, - * repeat): after this many back-to-back stranded resumes the turn is left idle. + * Retry bound for a stranded resume that keeps failing before its stream starts (pricing gate, + * history read, refused admission): after this many attempts with no stream the marker is + * dropped. Resumes that do start are not counted; each later stranding follows a completed + * model step of real work and is a fresh obligation, so a legitimate turn that awaits several + * monitored processes in a row resumes every time. */ const MAX_CONSECUTIVE_STRANDED_TURN_RESUMES = 3; @@ -6475,11 +6478,9 @@ export class AgentSession { // Any stream that actually starts is the continuation the stranded turn was waiting // for. PREPARING is not enough: a dequeued entry can still be canceled before acceptance. this.strandedTurnResume = undefined; - // "Consecutive" means uninterrupted by any other stream: a wake, user, or goal turn - // starting in between restores the runaway budget. - if (!this.strandedTurnResumeInFlight) { - this.consecutiveStrandedResumes = 0; - } + // "Consecutive" counts only resume attempts that never got this far: a stream that starts + // (the resume's own included) consumed the marker, so a later stranding is new work. + this.consecutiveStrandedResumes = 0; } if (next === TurnPhase.IDLE) { @@ -6941,7 +6942,7 @@ export class AgentSession { } /** - * The continuation still owed to a stranded turn. Past the runaway cap the marker is + * The continuation still owed to a stranded turn. Past the retry cap the marker is * forfeited: nothing may advertise it (the owner would defer settlement for a resume that * never starts) and the next sweep drops it. */ From 12e627afbe691a0db9d0c8e14306617ef1899e44 Mon Sep 17 00:00:00 2001 From: Michael Suchacz <203725896+ibetitsmike@users.noreply.github.com> Date: Thu, 3 Sep 2026 02:25:19 +0000 Subject: [PATCH 07/28] Forfeit the stranded resume at every hard-stop and failure boundary Codex round 5 on #4065: - The soft-stop abort handler re-reads queuedProviderToolEndAbortInFlight before rebuilding the marker; a hard stop landing during its awaits reset it. - clearQueue gains a hardStop option. TaskService hard stops (task_stop, descendant cascade, workflow hard timeout) clear the queue and then stop the stream through StreamManager directly, which emits no abort when the stream has completed or has not registered, so the queue clear is the boundary that forfeits the owed continuation and a pending provider-tool soft stop. - Forfeiting a correlated marker without a successor stream (goal admission refused, retry cap) settles the delegated turn whose stream-end the owner deferred, through a new session hook wired to AgentTaskIntegration.settleWorkspaceTurnContinuationFailure. - handleStreamError withdraws the owed continuation; the error path owns what happens next. --- .../agentSession.queueDispatch.test.ts | 264 +++++++++++++++++- src/node/services/agentSession.testHarness.ts | 8 +- src/node/services/agentSession.ts | 61 +++- src/node/services/taskService.ts | 19 +- .../services/taskWorkspaceSeam.testUtils.ts | 1 + src/node/services/taskWorkspaceSeam.ts | 11 +- src/node/services/workspaceService.ts | 15 +- 7 files changed, 365 insertions(+), 14 deletions(-) diff --git a/src/node/services/agentSession.queueDispatch.test.ts b/src/node/services/agentSession.queueDispatch.test.ts index 22d17bf3c4..531dbd6853 100644 --- a/src/node/services/agentSession.queueDispatch.test.ts +++ b/src/node/services/agentSession.queueDispatch.test.ts @@ -14,6 +14,7 @@ import { } from "./agentSession.testHarness"; import type { AIService, StreamMessageOptions } from "./aiService"; import type { HistoryService } from "./historyService"; +import type { TurnStreamHandle } from "./streamManager"; const TEST_MODEL = "anthropic:claude-sonnet-4-5"; const WORKSPACE_TURN_CORRELATION = { @@ -1453,8 +1454,12 @@ describe("AgentSession queued message tool-call dispatch", () => { recordStreamStarted: mock(() => Promise.resolve()), syncGoalModeWithChatTail: mock(() => Promise.resolve(null)), } as unknown as WorkspaceGoalService; + const settleForfeited = mock((_metadata: unknown, _reason: string) => Promise.resolve()); const harness = await createStreamingTurnHarness(workspaceId, { - harness: { workspaceGoalService }, + harness: { + workspaceGoalService, + settleForfeitedWorkspaceTurnContinuation: settleForfeited, + }, sendOptions: { muxMetadata: WORKSPACE_TURN_CORRELATION }, sendInternal: { synthetic: true, agentInitiated: true }, }); @@ -1471,15 +1476,19 @@ describe("AgentSession queued message tool-call dispatch", () => { // The resume keeps failing before its stream starts; while attempts remain the owner // defers settlement, and once the cap is exhausted the continuation is no longer - // advertised so the owner settles the turn at the cut. + // advertised and the owner is told to settle the turn it deferred at the cut. session.drainQueuedMessagesIfIdle(); expect(await waitForCondition(() => !session.isBusy())).toBe(true); expect(session.hasPendingWorkspaceTurnContinuation(WORKSPACE_TURN_CORRELATION)).toBe(true); + expect(settleForfeited).not.toHaveBeenCalled(); session.drainQueuedMessagesIfIdle(); expect(await waitForCondition(() => !session.isBusy())).toBe(true); expect(session.hasPendingWorkspaceTurnContinuation(WORKSPACE_TURN_CORRELATION)).toBe(false); + session.drainQueuedMessagesIfIdle(); await new Promise((resolve) => setTimeout(resolve, 25)); expect(streamMessage).toHaveBeenCalledTimes(1); + expect(settleForfeited).toHaveBeenCalledTimes(1); + expect(settleForfeited.mock.calls[0]?.[0]).toEqual(WORKSPACE_TURN_CORRELATION); } finally { session.dispose(); await cleanup(); @@ -1957,6 +1966,257 @@ describe("AgentSession queued message tool-call dispatch", () => { } }); + /** Goal service double whose stream-end/abort drain can be parked by the test. */ + function createParkableGoalService() { + let releaseDrain: () => void = () => undefined; + let drainArmed = false; + let drainReached = false; + const workspaceGoalService = { + assertPricedModelForBudgetedGoal: mock(() => Promise.resolve(Ok(undefined))), + recordStreamAccounting: mock(() => Promise.resolve()), + applyPendingAfterStreamEnd: mock(() => { + if (!drainArmed) { + return Promise.resolve(); + } + drainReached = true; + return new Promise((resolve) => { + releaseDrain = resolve; + }); + }), + requestContinuationAfterStreamEnd: mock(() => Promise.resolve()), + recordStreamStarted: mock(() => Promise.resolve()), + syncGoalModeWithChatTail: mock(() => Promise.resolve(null)), + completeGoalFromSilentContinuation: mock(() => Promise.resolve(false)), + } as unknown as WorkspaceGoalService; + return { + workspaceGoalService, + armDrain: () => { + drainArmed = true; + }, + drainReached: () => drainReached, + releaseDrain: () => releaseDrain(), + }; + } + + test("a hard stop landing during the soft-stop abort's cleanup does not resume", async () => { + const workspaceId = "queue-dispatch-stranded-soft-stop-cleanup-race"; + const goal = createParkableGoalService(); + const harness = await createStreamingTurnHarness(workspaceId, { + harness: { workspaceGoalService: goal.workspaceGoalService }, + sendInternal: { synthetic: true, agentInitiated: true }, + }); + const { session, cleanup, aiEmitter, aiService, streamMessage } = harness; + const stopStream = spyOn(aiService, "stopStream").mockResolvedValue(Ok(undefined)); + + try { + session.queueMessage( + "follow up", + { model: TEST_MODEL, agentId: "exec" }, + { synthetic: true, dedupeKey: "wake:1" } + ); + aiEmitter.emit("tool-call-end", { + ...toolCallEndEvent(workspaceId), + toolName: "web_search", + providerExecuted: true, + }); + expect(stopStream).toHaveBeenCalledTimes(1); + + // The soft stop's abort handler samples its claim, then parks in its accounting awaits. + goal.armDrain(); + aiEmitter.emit("stream-abort", streamAbortEvent(workspaceId, "queued-message")); + expect(await waitForCondition(() => goal.drainReached())).toBe(true); + + // task_stop lands in that window: queue cleared, synthetic system abort with no stream. + session.clearQueue("task stopped"); + aiEmitter.emit("stream-abort", streamAbortEvent(workspaceId, "system")); + await new Promise((resolve) => setTimeout(resolve, 10)); + goal.releaseDrain(); + + expect(await waitForCondition(() => !session.isBusy())).toBe(true); + session.drainQueuedMessagesIfIdle(); + await new Promise((resolve) => setTimeout(resolve, 25)); + expect(streamMessage).toHaveBeenCalledTimes(1); + expect(session.isBusy()).toBe(false); + } finally { + stopStream.mockRestore(); + session.dispose(); + await cleanup(); + } + }); + + test("a task hard stop that finds the stream already completed withdraws the continuation", async () => { + const workspaceId = "queue-dispatch-stranded-hard-stop-completed"; + const goal = createParkableGoalService(); + const harness = await createStreamingTurnHarness(workspaceId, { + harness: { workspaceGoalService: goal.workspaceGoalService }, + sendInternal: { synthetic: true, agentInitiated: true }, + }); + const { session, cleanup, aiEmitter, streamMessage } = harness; + + try { + const wake = harness.queueCancelableWake(); + harness.latestRequest().onQueuedMessageStop?.({ modelString: TEST_MODEL }); + wake.abort("monitor consumed"); + // The loop ended normally; stream-end cleanup is parked in COMPLETING. + goal.armDrain(); + aiEmitter.emit("stream-end", streamEndEvent(workspaceId)); + expect(await waitForCondition(() => goal.drainReached())).toBe(true); + + // TaskService's hard stop: the queue clear is its only session-visible step, because + // stopStream finds a completed stream and emits no abort at all. + session.clearQueue("task stopped", { hardStop: true }); + goal.releaseDrain(); + + expect(await waitForCondition(() => !session.isBusy())).toBe(true); + session.drainQueuedMessagesIfIdle(); + await new Promise((resolve) => setTimeout(resolve, 25)); + expect(streamMessage).toHaveBeenCalledTimes(1); + expect(session.isBusy()).toBe(false); + } finally { + session.dispose(); + await cleanup(); + } + }); + + test("a goal-refused resume settles the delegated turn it had advertised", async () => { + const workspaceId = "queue-dispatch-stranded-goal-refused-delegated"; + const workspaceGoalService = { + assertPricedModelForBudgetedGoal: mock(() => Promise.resolve(Ok(undefined))), + buildGoalRedispatchAdmission: mock(() => Promise.resolve({ admissible: false as const })), + recordStreamAccounting: mock(() => Promise.resolve()), + applyPendingAfterStreamEnd: mock(() => Promise.resolve()), + requestContinuationAfterStreamEnd: mock(() => Promise.resolve()), + recordStreamStarted: mock(() => Promise.resolve()), + syncGoalModeWithChatTail: mock(() => Promise.resolve(null)), + completeGoalFromSilentContinuation: mock(() => Promise.resolve(false)), + } as unknown as WorkspaceGoalService; + const settleForfeited = mock((_metadata: unknown, _reason: string) => Promise.resolve()); + const aiEmitter = new EventEmitter(); + const streamMessage = mock((_options: StreamMessageOptions) => { + aiEmitter.emit("stream-start", streamStartEvent(workspaceId)); + return Promise.resolve(Ok(createStartedTurnHandle("assistant-1"))); + }); + const { session, cleanup, historyService } = await createAgentSessionHarness({ + workspaceId, + aiEmitter, + aiServiceOverrides: { streamMessage: streamMessage as unknown as AIService["streamMessage"] }, + workspaceGoalService, + settleForfeitedWorkspaceTurnContinuation: settleForfeited, + }); + + try { + await historyService.appendToHistory( + workspaceId, + createMuxMessage("user-0", "user", "keep going", { timestamp: Date.now() }) + ); + const resumed = await session.resumeStream( + { model: TEST_MODEL, agentId: "exec", muxMetadata: WORKSPACE_TURN_CORRELATION }, + { agentInitiated: true, goalKind: GOAL_CONTINUATION_KIND, goalId: "goal-1" } + ); + expect(resumed.success).toBe(true); + await historyService.appendToHistory( + workspaceId, + createMuxMessage("assistant-1", "assistant", "ran the checks", { timestamp: Date.now() }) + ); + + const controller = new AbortController(); + session.queueMessage( + "Background monitor wake", + { model: TEST_MODEL, agentId: "exec" }, + { + synthetic: true, + agentInitiated: true, + cancelState: { canceledBeforeAcceptance: false }, + cancelSignal: controller.signal, + onCanceled: () => undefined, + } + ); + streamMessage.mock.calls[0]?.[0].onQueuedMessageStop?.({ modelString: TEST_MODEL }); + controller.abort("monitor consumed"); + session.clearQueue("monitor consumed"); + // The owner defers this stream-end on the strength of the advertised continuation. + expect(session.hasPendingWorkspaceTurnContinuation(WORKSPACE_TURN_CORRELATION)).toBe(true); + aiEmitter.emit("stream-end", streamEndEvent(workspaceId)); + + // The Pause refuses the resume: no successor stream will ever settle that deferral. + expect(await waitForCondition(() => settleForfeited.mock.calls.length === 1)).toBe(true); + expect(settleForfeited.mock.calls[0]?.[0]).toEqual(WORKSPACE_TURN_CORRELATION); + expect(session.hasPendingWorkspaceTurnContinuation(WORKSPACE_TURN_CORRELATION)).toBe(false); + expect(streamMessage).toHaveBeenCalledTimes(1); + } finally { + session.dispose(); + await cleanup(); + } + }); + + test("a stream error after the stop decision discards the owed continuation", async () => { + const workspaceId = "queue-dispatch-stranded-stream-error"; + // The first stream's completion is settled by the test so its terminal processing can fail + // after the stop decision instead of ending the stream. + let failFirstStream: () => void = () => undefined; + const aiEmitter = new EventEmitter(); + const streamMessage = mock((_options: StreamMessageOptions) => { + aiEmitter.emit("stream-start", streamStartEvent(workspaceId)); + const completion = new Promise< + TurnStreamHandle["completion"] extends Promise ? T : never + >((resolve) => { + failFirstStream = () => + resolve({ + status: "failed", + streamError: { + messageId: "assistant-1", + error: "provider closed the connection", + errorType: "api", + }, + }); + }); + return Promise.resolve(Ok({ messageId: "assistant-1", completion })); + }); + const { session, cleanup, historyService } = await createAgentSessionHarness({ + workspaceId, + aiEmitter, + aiServiceOverrides: { streamMessage: streamMessage as unknown as AIService["streamMessage"] }, + }); + + try { + const sent = await session.sendMessage("run the checks", { + model: TEST_MODEL, + agentId: "exec", + }); + expect(sent.success).toBe(true); + // Disabled after the send (a manual send re-enables it): the error is terminal, not retried. + await session.setAutoRetryEnabled(false, { persist: false }); + await historyService.appendToHistory( + workspaceId, + createMuxMessage("assistant-1", "assistant", "ran the checks", { timestamp: Date.now() }) + ); + const controller = new AbortController(); + session.queueMessage( + "Background monitor wake", + { model: TEST_MODEL, agentId: "exec" }, + { + synthetic: true, + agentInitiated: true, + cancelState: { canceledBeforeAcceptance: false }, + cancelSignal: controller.signal, + onCanceled: () => undefined, + } + ); + streamMessage.mock.calls[0]?.[0].onQueuedMessageStop?.({ modelString: TEST_MODEL }); + controller.abort("monitor consumed"); + failFirstStream(); + + expect(await waitForCondition(() => !session.isBusy())).toBe(true); + // A later idle poke (queue cleared, admission block released) must not restart the failed turn. + session.drainQueuedMessagesIfIdle(); + await new Promise((resolve) => setTimeout(resolve, 25)); + expect(streamMessage).toHaveBeenCalledTimes(1); + expect(session.isBusy()).toBe(false); + } finally { + session.dispose(); + await cleanup(); + } + }); test("rollback failure preserves the wake and continues acceptance", async () => { const workspaceId = "queue-dispatch-cancel-rollback-failure"; const { session, cleanup, historyService } = await createAgentSessionHarness({ workspaceId }); diff --git a/src/node/services/agentSession.testHarness.ts b/src/node/services/agentSession.testHarness.ts index f89c93b4e9..0ffbeb6b1c 100644 --- a/src/node/services/agentSession.testHarness.ts +++ b/src/node/services/agentSession.testHarness.ts @@ -5,7 +5,11 @@ import type { WorkspaceChatMessage } from "@/common/orpc/types"; import { Err, Ok } from "@/common/types/result"; import type { Config } from "@/node/config"; import type { TurnStreamHandle } from "@/node/services/streamManager"; -import { AgentSession, type AgentSessionAIService } from "@/node/services/agentSession"; +import { + AgentSession, + type AgentSessionAIService, + type AgentSessionOptions, +} from "@/node/services/agentSession"; import type { CompactionCompletionMetadata } from "@/common/types/compaction"; import type { BackgroundProcessManager } from "@/node/services/backgroundProcessManager"; import type { WorkspaceGoalService } from "@/node/services/workspaceGoalService"; @@ -111,6 +115,7 @@ export interface AgentSessionHarnessOptions { mcpServerManager?: MCPServerManager; onCompactionComplete?: (metadata: CompactionCompletionMetadata) => void; hasExternalSendPreflight?: () => boolean; + settleForfeitedWorkspaceTurnContinuation?: AgentSessionOptions["settleForfeitedWorkspaceTurnContinuation"]; captureEvents?: boolean; } @@ -156,6 +161,7 @@ export async function createAgentSessionHarness( backgroundProcessManager, onCompactionComplete: options.onCompactionComplete, hasExternalSendPreflight: options.hasExternalSendPreflight, + settleForfeitedWorkspaceTurnContinuation: options.settleForfeitedWorkspaceTurnContinuation, }); const events: WorkspaceChatMessage[] = []; diff --git a/src/node/services/agentSession.ts b/src/node/services/agentSession.ts index 529afe7784..d94abe6acb 100644 --- a/src/node/services/agentSession.ts +++ b/src/node/services/agentSession.ts @@ -638,7 +638,7 @@ export interface AgentSessionAIService extends BranchSummaryAiService { ): XumToolScope; } -interface AgentSessionOptions { +export interface AgentSessionOptions { workspaceId: string; config: Config; historyService: HistoryService; @@ -680,6 +680,15 @@ interface AgentSessionOptions { * to yield to a manual send that is still awaiting pricing/settings. */ hasExternalSendPreflight?: () => boolean; + /** + * Settles a delegated workspace turn whose owner deferred its stream-end because this session + * advertised an owed continuation that is now forfeited without a successor stream (goal no + * longer admits it, resume retry cap): no later stream-end will arrive for that turn. + */ + settleForfeitedWorkspaceTurnContinuation?: ( + metadata: WorkspaceTurnMuxMetadata, + reason: string + ) => Promise; } enum TurnPhase { @@ -721,6 +730,7 @@ export class AgentSession { private readonly sanitizeCliWorkspaceRegistration?: AgentSessionOptions["sanitizeCliWorkspaceRegistration"]; private readonly onPostCompactionStateChange?: () => void; private readonly hasExternalSendPreflight?: () => boolean; + private readonly settleForfeitedWorkspaceTurnContinuation?: AgentSessionOptions["settleForfeitedWorkspaceTurnContinuation"]; private readonly emitter = new EventEmitter(); private readonly aiListeners: Array<{ event: string; handler: (...args: unknown[]) => void }> = []; @@ -970,6 +980,7 @@ export class AgentSession { onIdleCompactionOutcome, onPostCompactionStateChange, hasExternalSendPreflight, + settleForfeitedWorkspaceTurnContinuation, } = options; assert(typeof workspaceId === "string", "workspaceId must be a string"); @@ -998,6 +1009,7 @@ export class AgentSession { this.sanitizeCliWorkspaceRegistration = sanitizeCliWorkspaceRegistration; this.onPostCompactionStateChange = onPostCompactionStateChange; this.hasExternalSendPreflight = hasExternalSendPreflight; + this.settleForfeitedWorkspaceTurnContinuation = settleForfeitedWorkspaceTurnContinuation; this.compactionHandler = new CompactionHandler({ workspaceId: this.workspaceId, @@ -5872,6 +5884,9 @@ export class AgentSession { this.setTurnPhase(TurnPhase.COMPLETING); this.queuedProviderToolEndAbortInFlight = false; + // A stop decided for a queued message can still end as a stream error; the error path + // (auto-retry or a terminal error row) owns what happens next, not the stranded resume. + this.withdrawStrandedTurnResume(); this.clearLiveUsageState(); const hadCompactionRequest = this.activeCompactionRequest !== undefined; if ( @@ -6191,8 +6206,11 @@ export class AgentSession { } await this.updateStartupAutoRetryAbandonFromAbort(abortReason, failedUserMessageId); this.emitChatEvent(payload); + // Re-read the in-flight flag: a hard stop that landed during the awaits above (synthetic + // system abort while COMPLETING, or a task hard stop clearing the queue) reset it and + // withdrew the marker, so the soft stop sampled at entry no longer owes a continuation. const dispatchedQueuedMessage = this.dispatchQueuedProviderToolEndMessageAfterAbort( - isQueuedProviderToolEndAbort, + isQueuedProviderToolEndAbort && this.queuedProviderToolEndAbortInFlight, abortedStreamContext, payload.metadata?.model ); @@ -6734,8 +6752,20 @@ export class AgentSession { return effectiveDispatchMode; } - clearQueue(cancelReason = "Queued message cleared before dispatch."): void { + clearQueue( + cancelReason = "Queued message cleared before dispatch.", + options?: { hardStop?: boolean } + ): void { this.assertNotDisposed("clearQueue"); + // A task hard stop (task_stop, interrupt cascade, workflow timeout) clears the queue and + // then stops the stream through StreamManager directly. That stop emits nothing when the + // stream has already completed or has not registered yet, so the queue clear is the only + // session-visible boundary at which the owed continuation and a pending provider-tool soft + // stop can be forfeited. A user clearing the queue keeps them: the cut turn still resumes. + if (options?.hardStop === true) { + this.queuedProviderToolEndAbortInFlight = false; + this.withdrawStrandedTurnResume(); + } const callbackSets = this.messageQueue.getClearCallbacks(); this.messageQueue.clear(); this.emitQueuedMessageChanged(); @@ -6952,6 +6982,27 @@ export class AgentSession { : undefined; } + /** + * The sweep gives the continuation up with no successor stream. A delegated turn's owner may + * have deferred its stream-end on the strength of this marker (hasPendingWorkspaceTurnContinuation), + * and no later stream-end will arrive for it, so the owner settles that turn here. + */ + private forfeitStrandedTurnResume(reason: string): void { + const correlation = getWorkspaceTurnMuxMetadata(this.strandedTurnResume?.options.muxMetadata); + this.strandedTurnResume = undefined; + if (correlation == null || this.settleForfeitedWorkspaceTurnContinuation == null) { + return; + } + void this.settleForfeitedWorkspaceTurnContinuation(correlation, reason).catch( + (error: unknown) => { + log.warn("Failed to settle forfeited workspace turn continuation", { + workspaceId: this.workspaceId, + error: getErrorMessage(error), + }); + } + ); + } + /** * Nothing is owed anymore (user Stop, superseding input, context discard): drop the marker and * cancel a resume still in its pre-stream window, which already copied the marker. @@ -7067,11 +7118,11 @@ export class AgentSession { const resume = this.owedStrandedTurnResume(); if (resume == null) { if (this.strandedTurnResume != null) { - this.strandedTurnResume = undefined; log.warn("Leaving stranded turn idle: consecutive resume cap reached", { workspaceId: this.workspaceId, cap: MAX_CONSECUTIVE_STRANDED_TURN_RESUMES, }); + this.forfeitStrandedTurnResume("Stranded turn resume gave up: retry cap reached."); } return; } @@ -7119,11 +7170,11 @@ export class AgentSession { } if (result.data.goalRefused === true) { // A Pause or terminal transition landed while the goal turn ran: nothing is owed to it. - this.strandedTurnResume = undefined; log.info("Dropping stranded goal turn: goal no longer admits it", { workspaceId: this.workspaceId, goalKind: resume.goalKind, }); + this.forfeitStrandedTurnResume("Stranded turn resume dropped: goal no longer admits it."); return false; } if (!result.data.started) { diff --git a/src/node/services/taskService.ts b/src/node/services/taskService.ts index 4fbf97844f..0365933560 100644 --- a/src/node/services/taskService.ts +++ b/src/node/services/taskService.ts @@ -5378,7 +5378,7 @@ export class TaskService implements AgentTaskIntegration { }); } - const clearQueueResult = this.workspaceService.clearQueue(id); + const clearQueueResult = this.workspaceService.clearQueue(id, { hardStop: true }); if (!clearQueueResult.success) { log.debug("stopDescendantAgentTask: clearQueue failed", { taskId: id, @@ -5857,7 +5857,7 @@ export class TaskService implements AgentTaskIntegration { // Best-effort: clear queue first. AgentSession stream-end cleanup auto-flushes // queued messages, so descendants must not keep pending input after a hard interrupt. try { - const clearQueueResult = this.workspaceService.clearQueue(id); + const clearQueueResult = this.workspaceService.clearQueue(id, { hardStop: true }); if (!clearQueueResult.success) { log.debug("terminateAllDescendantAgentTasks: clearQueue failed", { taskId: id, @@ -8207,7 +8207,7 @@ export class TaskService implements AgentTaskIntegration { return; } try { - const clearQueueResult = this.workspaceService.clearQueue(taskId); + const clearQueueResult = this.workspaceService.clearQueue(taskId, { hardStop: true }); if (!clearQueueResult.success) { log.debug("failAgentTaskForHardTimeout: clearQueue failed", { taskId, @@ -9562,6 +9562,19 @@ export class TaskService implements AgentTaskIntegration { return blocking; } + async settleWorkspaceTurnContinuationFailure( + workspaceId: string, + muxMetadata: Extract, + status: "interrupted" | "error", + error: string + ): Promise { + await this.getWorkspaceTurnManager().settleWorkspaceTurnContinuationFailure( + workspaceId, + muxMetadata, + status, + error + ); + } async noteWorkspaceUnarchived(workspaceId: string): Promise { assert(workspaceId.length > 0, "noteWorkspaceUnarchived requires workspaceId"); // Archived owners park workflow terminal wakes unsettled (the drain drops the in-memory diff --git a/src/node/services/taskWorkspaceSeam.testUtils.ts b/src/node/services/taskWorkspaceSeam.testUtils.ts index 5b81af0977..777c2f090e 100644 --- a/src/node/services/taskWorkspaceSeam.testUtils.ts +++ b/src/node/services/taskWorkspaceSeam.testUtils.ts @@ -75,6 +75,7 @@ export function makeAgentTaskIntegrationFake( latchHardInterruptCascade: () => undefined, terminateAllDescendantAgentTasks: () => Promise.resolve([]), noteWorkspaceUnarchived: () => Promise.resolve(), + settleWorkspaceTurnContinuationFailure: () => Promise.resolve(), ...overrides, }; } diff --git a/src/node/services/taskWorkspaceSeam.ts b/src/node/services/taskWorkspaceSeam.ts index f156763a52..e12c042ef9 100644 --- a/src/node/services/taskWorkspaceSeam.ts +++ b/src/node/services/taskWorkspaceSeam.ts @@ -374,7 +374,10 @@ export interface WorkspaceTurnHost { options: SendMessageOptions, internal?: { allowQueuedAgentTask?: boolean; agentInitiated?: boolean } ): Promise>; - clearQueue(workspaceId: string, options?: { cancelReason?: string }): Result; + clearQueue( + workspaceId: string, + options?: { cancelReason?: string; hardStop?: boolean } + ): Result; replaceHistory( workspaceId: string, summaryMessage: MuxMessage, @@ -523,6 +526,12 @@ export interface AgentTaskIntegration { options?: { workflowRunId?: string } ): Promise; noteWorkspaceUnarchived(workspaceId: string): Promise; + settleWorkspaceTurnContinuationFailure( + workspaceId: string, + muxMetadata: Extract, + status: "interrupted" | "error", + error: string + ): Promise; } export interface WorkspaceTurnTaskHost { diff --git a/src/node/services/workspaceService.ts b/src/node/services/workspaceService.ts index 7b8db4706c..94889ea6e2 100644 --- a/src/node/services/workspaceService.ts +++ b/src/node/services/workspaceService.ts @@ -4012,6 +4012,14 @@ export class WorkspaceService extends EventEmitter implements WorkspaceHost { // is released at its queue/session handoff so a follow-up dispatched // from within that turn does not veto itself. hasExternalSendPreflight: () => this.hasSessionInvisiblePreflight(workspaceId), + settleForfeitedWorkspaceTurnContinuation: async (metadata, reason) => { + await this.agentTaskIntegration?.settleWorkspaceTurnContinuationFailure( + workspaceId, + metadata, + "interrupted", + reason + ); + }, }); } @@ -11674,10 +11682,13 @@ export class WorkspaceService extends EventEmitter implements WorkspaceHost { } } - clearQueue(workspaceId: string, options?: { cancelReason?: string }): Result { + clearQueue( + workspaceId: string, + options?: { cancelReason?: string; hardStop?: boolean } + ): Result { try { const session = this.getOrCreateSession(workspaceId); - session.clearQueue(options?.cancelReason); + session.clearQueue(options?.cancelReason, { hardStop: options?.hardStop }); return Ok(undefined); } catch (error) { const errorMessage = getErrorMessage(error); From 8a8e25608563ca158a2e584e4e08bb7e88b5a6ae Mon Sep 17 00:00:00 2001 From: Michael Suchacz <203725896+ibetitsmike@users.noreply.github.com> Date: Thu, 3 Sep 2026 02:30:32 +0000 Subject: [PATCH 08/28] Keep the live scratchpad snapshot in the stranded resume pickStartupRetrySendOptions omits additionalSystemContext because it is not durable retry state, but the stranded resume is the same turn continued in memory: without it the resumed stream falls back to the persisted scratchpad, which can be stale or empty while the renderer's save is still in flight. --- .../agentSession.queueDispatch.test.ts | 23 +++++++++++++++++++ src/node/services/agentSession.ts | 6 +++++ 2 files changed, 29 insertions(+) diff --git a/src/node/services/agentSession.queueDispatch.test.ts b/src/node/services/agentSession.queueDispatch.test.ts index 531dbd6853..90b87f7843 100644 --- a/src/node/services/agentSession.queueDispatch.test.ts +++ b/src/node/services/agentSession.queueDispatch.test.ts @@ -1938,6 +1938,29 @@ describe("AgentSession queued message tool-call dispatch", () => { } }); + test("the resume keeps the live scratchpad snapshot the cut stream was sent with", async () => { + const workspaceId = "queue-dispatch-stranded-scratchpad"; + const harness = await createStreamingTurnHarness(workspaceId, { + sendOptions: { additionalSystemContext: "live scratchpad" }, + }); + const { session, cleanup, aiEmitter, streamMessage } = harness; + + try { + expect(harness.latestRequest().additionalSystemContext).toBe("live scratchpad"); + const wake = harness.queueCancelableWake(); + harness.latestRequest().onQueuedMessageStop?.({ modelString: TEST_MODEL }); + wake.abort("monitor consumed"); + aiEmitter.emit("stream-end", streamEndEvent(workspaceId)); + + expect(await waitForCondition(() => streamMessage.mock.calls.length === 2)).toBe(true); + // Falling back to the persisted scratchpad mid-turn would change the model's instructions. + expect(harness.latestRequest().additionalSystemContext).toBe("live scratchpad"); + } finally { + session.dispose(); + await cleanup(); + } + }); + test("a thinking change still pending at the cut carries into the resume", async () => { const workspaceId = "queue-dispatch-stranded-pending-thinking"; const harness = await createStreamingTurnHarness(workspaceId, { diff --git a/src/node/services/agentSession.ts b/src/node/services/agentSession.ts index d94abe6acb..6930a0e829 100644 --- a/src/node/services/agentSession.ts +++ b/src/node/services/agentSession.ts @@ -403,6 +403,12 @@ function buildStrandedTurnResume(context: { return { options: { ...resumeOptions, + // Not durable retry state (the picker omits it), but this continuation is the same turn in + // memory: the resumed stream must see the live scratchpad snapshot the cut stream did, not + // whatever the renderer's save had persisted by then. + ...(context.options?.additionalSystemContext != null + ? { additionalSystemContext: context.options.additionalSystemContext } + : {}), model: context.modelString, ...(context.thinkingLevelAtCut != null ? { thinkingLevel: context.thinkingLevelAtCut } : {}), muxMetadata: From 8a8494f84ed6813b7a916e9e8f8a3ceb6ed6134f Mon Sep 17 00:00:00 2001 From: Michael Suchacz <203725896+ibetitsmike@users.noreply.github.com> Date: Thu, 3 Sep 2026 02:56:28 +0000 Subject: [PATCH 09/28] Settle a deferred delegated turn when its owed continuation is discarded A context-discarding mutation or session disposal drops the owed continuation while the session is idle, with no successor stream or terminal event to settle the delegated turn whose stream-end the owner deferred on it. Forfeit (withdraw plus settle) at both boundaries instead of withdrawing silently. --- .../agentSession.queueDispatch.test.ts | 70 ++++++++++++++++--- src/node/services/agentSession.ts | 23 +++--- 2 files changed, 74 insertions(+), 19 deletions(-) diff --git a/src/node/services/agentSession.queueDispatch.test.ts b/src/node/services/agentSession.queueDispatch.test.ts index 90b87f7843..2f605b5c6d 100644 --- a/src/node/services/agentSession.queueDispatch.test.ts +++ b/src/node/services/agentSession.queueDispatch.test.ts @@ -1441,8 +1441,12 @@ describe("AgentSession queued message tool-call dispatch", () => { } }); - test("stops advertising the delegated continuation once the resume cap is exhausted", async () => { - const workspaceId = "queue-dispatch-stranded-delegated-cap"; + /** + * Strand a delegated turn whose resume fails at the pricing gate before its stream starts: + * the owner deferred the cut stream-end on the advertised continuation and the session sits + * idle with it still owed. + */ + async function strandDelegatedTurnBehindClosedGate(workspaceId: string) { let gateOpen = true; const workspaceGoalService = { assertPricedModelForBudgetedGoal: mock(() => @@ -1463,17 +1467,23 @@ describe("AgentSession queued message tool-call dispatch", () => { sendOptions: { muxMetadata: WORKSPACE_TURN_CORRELATION }, sendInternal: { synthetic: true, agentInitiated: true }, }); - const { session, cleanup, aiEmitter, streamMessage } = harness; + const { session, aiEmitter } = harness; + gateOpen = false; + harness.latestRequest().onQueuedMessageStop?.({ modelString: TEST_MODEL }); + harness.queueCancelableWake().abort("monitor consumed"); + session.clearQueue("monitor consumed"); + expect(session.hasPendingWorkspaceTurnContinuation(WORKSPACE_TURN_CORRELATION)).toBe(true); + aiEmitter.emit("stream-end", streamEndEvent(workspaceId)); + expect(await waitForCondition(() => !session.isBusy())).toBe(true); + return { ...harness, settleForfeited }; + } - try { - gateOpen = false; - harness.latestRequest().onQueuedMessageStop?.({ modelString: TEST_MODEL }); - harness.queueCancelableWake().abort("monitor consumed"); - session.clearQueue("monitor consumed"); - expect(session.hasPendingWorkspaceTurnContinuation(WORKSPACE_TURN_CORRELATION)).toBe(true); - aiEmitter.emit("stream-end", streamEndEvent(workspaceId)); - expect(await waitForCondition(() => !session.isBusy())).toBe(true); + test("stops advertising the delegated continuation once the resume cap is exhausted", async () => { + const workspaceId = "queue-dispatch-stranded-delegated-cap"; + const harness = await strandDelegatedTurnBehindClosedGate(workspaceId); + const { session, cleanup, streamMessage, settleForfeited } = harness; + try { // The resume keeps failing before its stream starts; while attempts remain the owner // defers settlement, and once the cap is exhausted the continuation is no longer // advertised and the owner is told to settle the turn it deferred at the cut. @@ -1495,6 +1505,44 @@ describe("AgentSession queued message tool-call dispatch", () => { } }); + test("a context-discarding mutation settles the delegated turn it had advertised", async () => { + const workspaceId = "queue-dispatch-stranded-delegated-context-discard"; + const harness = await strandDelegatedTurnBehindClosedGate(workspaceId); + const { session, cleanup, streamMessage, settleForfeited } = harness; + + try { + // A history clear admitted on the idle session discards the transcript the continuation + // would resume from; no stream follows it to settle the turn the owner deferred. + const discarded = await session.discardAutoRetryForContextMutation(); + expect(discarded.success).toBe(true); + expect(settleForfeited).toHaveBeenCalledTimes(1); + expect(settleForfeited.mock.calls[0]?.[0]).toEqual(WORKSPACE_TURN_CORRELATION); + expect(session.hasPendingWorkspaceTurnContinuation(WORKSPACE_TURN_CORRELATION)).toBe(false); + session.drainQueuedMessagesIfIdle(); + await new Promise((resolve) => setTimeout(resolve, 25)); + expect(streamMessage).toHaveBeenCalledTimes(1); + } finally { + session.dispose(); + await cleanup(); + } + }); + + test("disposing the session settles the delegated turn it had advertised", async () => { + const workspaceId = "queue-dispatch-stranded-delegated-dispose"; + const harness = await strandDelegatedTurnBehindClosedGate(workspaceId); + const { session, cleanup, settleForfeited } = harness; + + try { + // Workspace removal tears the idle session down with the continuation still owed. + session.dispose(); + expect(settleForfeited).toHaveBeenCalledTimes(1); + expect(settleForfeited.mock.calls[0]?.[0]).toEqual(WORKSPACE_TURN_CORRELATION); + } finally { + session.dispose(); + await cleanup(); + } + }); + test("a user Stop during the resume's admission gate cancels it", async () => { const workspaceId = "queue-dispatch-stranded-stop-in-admission"; let releaseGate: () => void = () => undefined; diff --git a/src/node/services/agentSession.ts b/src/node/services/agentSession.ts index 6930a0e829..e04afcb2ce 100644 --- a/src/node/services/agentSession.ts +++ b/src/node/services/agentSession.ts @@ -1064,7 +1064,8 @@ export class AgentSession { this.activePreparedTurnAbortController = null; // A resume parked in its pre-stream I/O passed streamWithHistory's disposed check already; // its abort signal is the only thing that stops it registering a stream after teardown. - this.withdrawStrandedTurnResume(); + // Disposal (workspace removal) also produces no successor stream for a deferred delegated turn. + this.forfeitStrandedTurnResume("Stranded turn resume discarded: workspace session disposed."); // Ensure any callers blocked on waitForIdle() can continue during teardown. this.setTurnPhase(TurnPhase.IDLE); @@ -6567,7 +6568,11 @@ export class AgentSession { async discardAutoRetryForContextMutation(): Promise> { this.retryManager.cancel(); this.setAutoRetryResumeState(undefined); - this.withdrawStrandedTurnResume(); + // The discarded transcript yields no successor stream, so a delegated turn whose stream-end + // the owner deferred on the owed continuation is settled here rather than left running. + this.forfeitStrandedTurnResume( + "Stranded turn resume discarded: workspace context was mutated." + ); const deleteResult = await this.historyService.deletePartial(this.workspaceId); if (!deleteResult.success) { return Err(deleteResult.error); @@ -6989,13 +6994,14 @@ export class AgentSession { } /** - * The sweep gives the continuation up with no successor stream. A delegated turn's owner may - * have deferred its stream-end on the strength of this marker (hasPendingWorkspaceTurnContinuation), - * and no later stream-end will arrive for it, so the owner settles that turn here. + * The continuation is given up with no successor stream (sweep cap or goal refusal, context + * discard, session disposal). A delegated turn's owner may have deferred its stream-end on the + * strength of this marker (hasPendingWorkspaceTurnContinuation), and no later stream-end will + * arrive for it, so the owner settles that turn here. */ private forfeitStrandedTurnResume(reason: string): void { const correlation = getWorkspaceTurnMuxMetadata(this.strandedTurnResume?.options.muxMetadata); - this.strandedTurnResume = undefined; + this.withdrawStrandedTurnResume(); if (correlation == null || this.settleForfeitedWorkspaceTurnContinuation == null) { return; } @@ -7010,8 +7016,9 @@ export class AgentSession { } /** - * Nothing is owed anymore (user Stop, superseding input, context discard): drop the marker and - * cancel a resume still in its pre-stream window, which already copied the marker. + * Nothing is owed anymore (user Stop, superseding input): drop the marker and cancel a resume + * still in its pre-stream window, which already copied the marker. The owner of a delegated + * turn learns of it from the stream event or hard stop that caused the withdrawal. */ private withdrawStrandedTurnResume(): void { this.strandedTurnResume = undefined; From 55fa4024ad5eaeaeec67097be399f0450d7fe936 Mon Sep 17 00:00:00 2001 From: Michael Suchacz <203725896+ibetitsmike@users.noreply.github.com> Date: Thu, 3 Sep 2026 09:52:05 +0000 Subject: [PATCH 10/28] Bind the owed continuation to the owner's settlement decision The delegated turn owner decided DEFER vs SETTLE from two session reads (workspace-turn continuation, pending bash wake) while the session dropped a correlated marker at dequeue whenever the queue head was not a same-turn entry. The two views diverged both ways: a real bash-monitor wake at the head (no correlation on the entry) made the owner defer while the session dropped the marker, so a wake withdrawn after dequeue left the delegated turn stranded and the owner hanging; and an unrelated head removed before dequeue left the marker alive to resume a turn the owner had already settled. Replace both reads with one claimWorkspaceTurnContinuation(metadata, streamEndMessageId) that is the owner's decision: a pending wake counts as the turn's continuation, and a false answer voids the marker for that exact cut, so the marker cannot outlive the settlement regardless of how the superseding entry leaves the queue. The marker records the cut stream's message id so an owner still settling an older stream-end of the same turn defers instead of voiding a newer cut's continuation. The dequeue-time drop is removed. --- .../agentSession.queueDispatch.test.ts | 219 +++++++++++++++--- src/node/services/agentSession.ts | 101 ++++---- src/node/services/taskService.test.ts | 17 +- .../services/taskWorkspaceSeam.testUtils.ts | 3 +- src/node/services/taskWorkspaceSeam.ts | 6 +- src/node/services/workspaceService.ts | 20 +- .../services/workspaceTurnManager.test.ts | 3 +- src/node/services/workspaceTurnManager.ts | 17 +- 8 files changed, 276 insertions(+), 110 deletions(-) diff --git a/src/node/services/agentSession.queueDispatch.test.ts b/src/node/services/agentSession.queueDispatch.test.ts index 2f605b5c6d..af23849747 100644 --- a/src/node/services/agentSession.queueDispatch.test.ts +++ b/src/node/services/agentSession.queueDispatch.test.ts @@ -114,12 +114,12 @@ async function createStreamingTurnHarness( createMuxMessage("assistant-1", "assistant", "ran the checks", { timestamp: Date.now() }) ); - /** Queue a synthetic wake whose cancel signal the caller controls. */ - const queueCancelableWake = (): AbortController => { + /** Queue a synthetic tool-end entry whose cancel signal the caller controls. */ + const queueCancelable = (message: string, muxMetadata?: MuxMessageMetadata): AbortController => { const controller = new AbortController(); harness.session.queueMessage( - "Background monitor wake", - { model: TEST_MODEL, agentId: "exec" }, + message, + { model: TEST_MODEL, agentId: "exec", muxMetadata }, { synthetic: true, agentInitiated: true, @@ -130,6 +130,10 @@ async function createStreamingTurnHarness( ); return controller; }; + // A real wake is typed so a delegated turn's owner recognizes it as the turn's continuation. + const queueCancelableWake = (): AbortController => + queueCancelable("Background monitor wake", { type: "bash-monitor-wake", records: [] }); + const queueCancelableUnrelatedEntry = (): AbortController => queueCancelable("peer follow-up"); const latestRequest = (): StreamMessageOptions => { const call = streamMessage.mock.calls[streamMessage.mock.calls.length - 1]; if (call == null) { @@ -138,7 +142,13 @@ async function createStreamingTurnHarness( return call[0]; }; - return { ...harness, streamMessage, queueCancelableWake, latestRequest }; + return { + ...harness, + streamMessage, + queueCancelableWake, + queueCancelableUnrelatedEntry, + latestRequest, + }; } async function waitForCondition(condition: () => boolean, timeoutMs = 500): Promise { @@ -159,8 +169,9 @@ describe("AgentSession queued message tool-call dispatch", () => { hasQueuedOrDispatchingEntry( continuationMetadata?: Extract ): boolean; - hasPendingWorkspaceTurnContinuation( - continuationMetadata: Extract + claimWorkspaceTurnContinuation( + continuationMetadata: Extract, + streamEndMessageId: string ): boolean; }; } = {}; @@ -184,12 +195,13 @@ describe("AgentSession queued message tool-call dispatch", () => { }) === true, uncorrelated: session?.hasQueuedOrDispatchingEntry() === true, pendingSameTurn: - session?.hasPendingWorkspaceTurnContinuation(WORKSPACE_TURN_CORRELATION) === true, + session?.claimWorkspaceTurnContinuation(WORKSPACE_TURN_CORRELATION, "assistant-1") === + true, pendingDifferentTurn: - session?.hasPendingWorkspaceTurnContinuation({ - ...WORKSPACE_TURN_CORRELATION, - turnId: "turn-different", - }) === true, + session?.claimWorkspaceTurnContinuation( + { ...WORKSPACE_TURN_CORRELATION, turnId: "turn-different" }, + "assistant-1" + ) === true, }; return Promise.resolve(Ok(createStartedTurnHandle())); }); @@ -1013,7 +1025,9 @@ describe("AgentSession queued message tool-call dispatch", () => { try { // Nothing owed yet: a tool-calls cut with an empty queue is a plain interruption. - expect(session.hasPendingWorkspaceTurnContinuation(WORKSPACE_TURN_CORRELATION)).toBe(false); + expect( + session.claimWorkspaceTurnContinuation(WORKSPACE_TURN_CORRELATION, "assistant-1") + ).toBe(false); const wake = harness.queueCancelableWake(); harness.latestRequest().onQueuedMessageStop?.({ modelString: TEST_MODEL }); @@ -1023,19 +1037,23 @@ describe("AgentSession queued message tool-call dispatch", () => { // The owner's settlement runs synchronously with stream-end; it must already see the // continuation, and only for this correlation. - expect(session.hasPendingWorkspaceTurnContinuation(WORKSPACE_TURN_CORRELATION)).toBe(true); expect( - session.hasPendingWorkspaceTurnContinuation({ - ...WORKSPACE_TURN_CORRELATION, - turnId: "another-turn", - }) + session.claimWorkspaceTurnContinuation(WORKSPACE_TURN_CORRELATION, "assistant-1") + ).toBe(true); + expect( + session.claimWorkspaceTurnContinuation( + { ...WORKSPACE_TURN_CORRELATION, turnId: "another-turn" }, + "assistant-1" + ) ).toBe(false); aiEmitter.emit("stream-end", streamEndEvent(workspaceId)); expect(await waitForCondition(() => streamMessage.mock.calls.length === 2)).toBe(true); expect(harness.latestRequest().muxMetadata).toEqual(WORKSPACE_TURN_CORRELATION); // Consumed by the resumed stream. - expect(session.hasPendingWorkspaceTurnContinuation(WORKSPACE_TURN_CORRELATION)).toBe(false); + expect( + session.claimWorkspaceTurnContinuation(WORKSPACE_TURN_CORRELATION, "assistant-1") + ).toBe(false); } finally { session.dispose(); await cleanup(); @@ -1052,7 +1070,9 @@ describe("AgentSession queued message tool-call dispatch", () => { try { harness.latestRequest().onQueuedMessageStop?.({ modelString: TEST_MODEL }); session.queueMessage("user follow-up", { model: TEST_MODEL, agentId: "exec" }); - expect(session.hasPendingWorkspaceTurnContinuation(WORKSPACE_TURN_CORRELATION)).toBe(false); + expect( + session.claimWorkspaceTurnContinuation(WORKSPACE_TURN_CORRELATION, "assistant-1") + ).toBe(false); } finally { session.dispose(); await cleanup(); @@ -1102,9 +1122,14 @@ describe("AgentSession queued message tool-call dispatch", () => { const wake = harness.queueCancelableWake(); harness.latestRequest().onQueuedMessageStop?.({ modelString: TEST_MODEL }); wake.abort("monitor consumed"); - expect(session.hasPendingWorkspaceTurnContinuation(WORKSPACE_TURN_CORRELATION)).toBe(false); + // Owed whether the owner asks while the wake is still queued or after it is cleared. + expect( + session.claimWorkspaceTurnContinuation(WORKSPACE_TURN_CORRELATION, "assistant-1") + ).toBe(true); session.clearQueue("monitor consumed"); - expect(session.hasPendingWorkspaceTurnContinuation(WORKSPACE_TURN_CORRELATION)).toBe(true); + expect( + session.claimWorkspaceTurnContinuation(WORKSPACE_TURN_CORRELATION, "assistant-1") + ).toBe(true); aiEmitter.emit("stream-end", streamEndEvent(workspaceId)); expect(await waitForCondition(() => streamMessage.mock.calls.length === 2)).toBe(true); @@ -1472,7 +1497,9 @@ describe("AgentSession queued message tool-call dispatch", () => { harness.latestRequest().onQueuedMessageStop?.({ modelString: TEST_MODEL }); harness.queueCancelableWake().abort("monitor consumed"); session.clearQueue("monitor consumed"); - expect(session.hasPendingWorkspaceTurnContinuation(WORKSPACE_TURN_CORRELATION)).toBe(true); + expect(session.claimWorkspaceTurnContinuation(WORKSPACE_TURN_CORRELATION, "assistant-1")).toBe( + true + ); aiEmitter.emit("stream-end", streamEndEvent(workspaceId)); expect(await waitForCondition(() => !session.isBusy())).toBe(true); return { ...harness, settleForfeited }; @@ -1489,11 +1516,15 @@ describe("AgentSession queued message tool-call dispatch", () => { // advertised and the owner is told to settle the turn it deferred at the cut. session.drainQueuedMessagesIfIdle(); expect(await waitForCondition(() => !session.isBusy())).toBe(true); - expect(session.hasPendingWorkspaceTurnContinuation(WORKSPACE_TURN_CORRELATION)).toBe(true); + expect( + session.claimWorkspaceTurnContinuation(WORKSPACE_TURN_CORRELATION, "assistant-1") + ).toBe(true); expect(settleForfeited).not.toHaveBeenCalled(); session.drainQueuedMessagesIfIdle(); expect(await waitForCondition(() => !session.isBusy())).toBe(true); - expect(session.hasPendingWorkspaceTurnContinuation(WORKSPACE_TURN_CORRELATION)).toBe(false); + expect( + session.claimWorkspaceTurnContinuation(WORKSPACE_TURN_CORRELATION, "assistant-1") + ).toBe(false); session.drainQueuedMessagesIfIdle(); await new Promise((resolve) => setTimeout(resolve, 25)); expect(streamMessage).toHaveBeenCalledTimes(1); @@ -1517,7 +1548,9 @@ describe("AgentSession queued message tool-call dispatch", () => { expect(discarded.success).toBe(true); expect(settleForfeited).toHaveBeenCalledTimes(1); expect(settleForfeited.mock.calls[0]?.[0]).toEqual(WORKSPACE_TURN_CORRELATION); - expect(session.hasPendingWorkspaceTurnContinuation(WORKSPACE_TURN_CORRELATION)).toBe(false); + expect( + session.claimWorkspaceTurnContinuation(WORKSPACE_TURN_CORRELATION, "assistant-1") + ).toBe(false); session.drainQueuedMessagesIfIdle(); await new Promise((resolve) => setTimeout(resolve, 25)); expect(streamMessage).toHaveBeenCalledTimes(1); @@ -1671,18 +1704,132 @@ describe("AgentSession queued message tool-call dispatch", () => { const { session, cleanup, aiEmitter, streamMessage } = harness; try { - const wake = harness.queueCancelableWake(); + const entry = harness.queueCancelableUnrelatedEntry(); harness.latestRequest().onQueuedMessageStop?.({ modelString: TEST_MODEL }); - // The queued wake is not this turn's continuation: the owner settles the delegated turn. - expect(session.hasPendingWorkspaceTurnContinuation(WORKSPACE_TURN_CORRELATION)).toBe(false); + // The queued entry is not this turn's continuation: the owner settles the delegated turn. + expect( + session.claimWorkspaceTurnContinuation(WORKSPACE_TURN_CORRELATION, "assistant-1") + ).toBe(false); aiEmitter.emit("stream-end", streamEndEvent(workspaceId)); - // The wake is withdrawn after dequeue, before acceptance; the settled turn must stay cut. - wake.abort("monitor consumed"); + // The entry is withdrawn after dequeue, before acceptance; the settled turn must stay cut. + entry.abort("superseded"); expect(await waitForCondition(() => !session.isBusy())).toBe(true); await new Promise((resolve) => setTimeout(resolve, 25)); expect(streamMessage).toHaveBeenCalledTimes(1); - expect(session.hasPendingWorkspaceTurnContinuation(WORKSPACE_TURN_CORRELATION)).toBe(false); + expect( + session.claimWorkspaceTurnContinuation(WORKSPACE_TURN_CORRELATION, "assistant-1") + ).toBe(false); + } finally { + session.dispose(); + await cleanup(); + } + }); + + test("an unrelated entry cleared before dispatch does not revive the superseded delegated turn", async () => { + const workspaceId = "queue-dispatch-stranded-delegated-orphan-cleared"; + const harness = await createStreamingTurnHarness(workspaceId, { + sendOptions: { muxMetadata: WORKSPACE_TURN_CORRELATION }, + }); + const { session, cleanup, aiEmitter, streamMessage } = harness; + + try { + harness.queueCancelableUnrelatedEntry(); + harness.latestRequest().onQueuedMessageStop?.({ modelString: TEST_MODEL }); + // The owner settles the turn on this answer. The entry then leaves the queue without ever + // dispatching (user clears the queue), which must not bring the settled turn back. + expect( + session.claimWorkspaceTurnContinuation(WORKSPACE_TURN_CORRELATION, "assistant-1") + ).toBe(false); + session.clearQueue("queue cleared by user"); + aiEmitter.emit("stream-end", streamEndEvent(workspaceId)); + + expect(await waitForCondition(() => !session.isBusy())).toBe(true); + await new Promise((resolve) => setTimeout(resolve, 25)); + expect(streamMessage).toHaveBeenCalledTimes(1); + // With the queue empty, only a surviving marker could answer true here. + expect( + session.claimWorkspaceTurnContinuation(WORKSPACE_TURN_CORRELATION, "assistant-1") + ).toBe(false); + } finally { + session.dispose(); + await cleanup(); + } + }); + + test("an unrelated entry removed before the owner's claim leaves the continuation owed", async () => { + const workspaceId = "queue-dispatch-stranded-delegated-removed-before-claim"; + const harness = await createStreamingTurnHarness(workspaceId, { + sendOptions: { muxMetadata: WORKSPACE_TURN_CORRELATION }, + }); + const { session, cleanup, aiEmitter, streamMessage } = harness; + + try { + harness.queueCancelableUnrelatedEntry(); + harness.latestRequest().onQueuedMessageStop?.({ modelString: TEST_MODEL }); + session.clearQueue("queue cleared by user"); + // Nothing supersedes the turn by the time the owner asks: it defers, and the resume runs. + expect( + session.claimWorkspaceTurnContinuation(WORKSPACE_TURN_CORRELATION, "assistant-1") + ).toBe(true); + aiEmitter.emit("stream-end", streamEndEvent(workspaceId)); + + expect(await waitForCondition(() => streamMessage.mock.calls.length === 2)).toBe(true); + expect(harness.latestRequest().muxMetadata).toEqual(WORKSPACE_TURN_CORRELATION); + } finally { + session.dispose(); + await cleanup(); + } + }); + + test("a delegated turn cut by a wake withdrawn after dequeue resumes", async () => { + const workspaceId = "queue-dispatch-stranded-delegated-wake-after-dequeue"; + const harness = await createStreamingTurnHarness(workspaceId, { + sendOptions: { muxMetadata: WORKSPACE_TURN_CORRELATION }, + }); + const { session, cleanup, aiEmitter, streamMessage } = harness; + + try { + const wake = harness.queueCancelableWake(); + harness.latestRequest().onQueuedMessageStop?.({ modelString: TEST_MODEL }); + // A wake inherits the turn's correlation when it sends, so the owner defers on it even + // though the queued entry carries none; the continuation stays owed behind it. + expect( + session.claimWorkspaceTurnContinuation(WORKSPACE_TURN_CORRELATION, "assistant-1") + ).toBe(true); + aiEmitter.emit("stream-end", streamEndEvent(workspaceId)); + // Withdrawn after dequeue, before acceptance: the deferred turn must resume, not hang. + wake.abort("monitor consumed"); + + expect(await waitForCondition(() => streamMessage.mock.calls.length === 2)).toBe(true); + expect(harness.latestRequest().muxMetadata).toEqual(WORKSPACE_TURN_CORRELATION); + } finally { + session.dispose(); + await cleanup(); + } + }); + + test("an older stream-end's claim does not void a newer cut's continuation", async () => { + const workspaceId = "queue-dispatch-stranded-delegated-stale-claim"; + const harness = await createStreamingTurnHarness(workspaceId, { + sendOptions: { muxMetadata: WORKSPACE_TURN_CORRELATION }, + }); + const { session, cleanup, aiEmitter, streamMessage } = harness; + + try { + harness.queueCancelableUnrelatedEntry(); + harness.latestRequest().onQueuedMessageStop?.({ modelString: TEST_MODEL }); + // The owner is still settling an earlier stream of this turn: that a later cut owes a + // continuation proves the turn went on, so it defers without touching the marker. + expect( + session.claimWorkspaceTurnContinuation(WORKSPACE_TURN_CORRELATION, "assistant-0") + ).toBe(true); + session.clearQueue("queue cleared by user"); + expect( + session.claimWorkspaceTurnContinuation(WORKSPACE_TURN_CORRELATION, "assistant-1") + ).toBe(true); + aiEmitter.emit("stream-end", streamEndEvent(workspaceId)); + expect(await waitForCondition(() => streamMessage.mock.calls.length === 2)).toBe(true); } finally { session.dispose(); await cleanup(); @@ -2206,13 +2353,17 @@ describe("AgentSession queued message tool-call dispatch", () => { controller.abort("monitor consumed"); session.clearQueue("monitor consumed"); // The owner defers this stream-end on the strength of the advertised continuation. - expect(session.hasPendingWorkspaceTurnContinuation(WORKSPACE_TURN_CORRELATION)).toBe(true); + expect( + session.claimWorkspaceTurnContinuation(WORKSPACE_TURN_CORRELATION, "assistant-1") + ).toBe(true); aiEmitter.emit("stream-end", streamEndEvent(workspaceId)); // The Pause refuses the resume: no successor stream will ever settle that deferral. expect(await waitForCondition(() => settleForfeited.mock.calls.length === 1)).toBe(true); expect(settleForfeited.mock.calls[0]?.[0]).toEqual(WORKSPACE_TURN_CORRELATION); - expect(session.hasPendingWorkspaceTurnContinuation(WORKSPACE_TURN_CORRELATION)).toBe(false); + expect( + session.claimWorkspaceTurnContinuation(WORKSPACE_TURN_CORRELATION, "assistant-1") + ).toBe(false); expect(streamMessage).toHaveBeenCalledTimes(1); } finally { session.dispose(); diff --git a/src/node/services/agentSession.ts b/src/node/services/agentSession.ts index e04afcb2ce..05274e93e6 100644 --- a/src/node/services/agentSession.ts +++ b/src/node/services/agentSession.ts @@ -367,6 +367,8 @@ function getWorkspaceTurnMuxMetadata(muxMetadata: unknown): WorkspaceTurnMuxMeta interface StrandedTurnResume { options: SendMessageOptions; + /** Assistant message id of the stream the cut ended (claimWorkspaceTurnContinuation). */ + cutMessageId: string; agentInitiated?: boolean; goalKind?: GoalSyntheticMessageKind; goalId?: string; @@ -385,6 +387,7 @@ interface StrandedTurnResume { function buildStrandedTurnResume(context: { /** Model that reached the cut: a configured fallback may differ from the requested one. */ modelString: string; + cutMessageId: string; options?: SendMessageOptions; agentInitiated?: boolean; goalKind?: GoalSyntheticMessageKind; @@ -414,6 +417,7 @@ function buildStrandedTurnResume(context: { muxMetadata: context.workspaceTurnMetadata ?? getWorkspaceTurnMuxMetadata(context.options?.muxMetadata), }, + cutMessageId: context.cutMessageId, ...(context.agentInitiated != null ? { agentInitiated: context.agentInitiated } : {}), ...(context.goalKind != null ? { goalKind: context.goalKind } : {}), ...(context.goalId != null ? { goalId: context.goalId } : {}), @@ -861,6 +865,8 @@ export class AgentSession { /** Tracks the user message id that initiated the currently active stream (for retry guards). */ private activeStreamUserMessageId?: string; + /** Assistant message id of the active stream, from its stream-start event. */ + private activeStreamMessageId?: string; /** Track user message ids that already retried without post-compaction injection. */ private readonly postCompactionRetryAttempts = new Set(); @@ -5374,10 +5380,11 @@ export class AgentSession { strictAgentResolution: options?.strictAgentResolution, hasQueuedMessages: this.hasQueuedMessages.bind(this), onQueuedMessageStop: ({ modelString: stoppedModelString }) => { - if (this.activeStreamContext != null) { + if (this.activeStreamContext != null && this.activeStreamMessageId != null) { this.strandedTurnResume = buildStrandedTurnResume({ ...this.activeStreamContext, modelString: stoppedModelString, + cutMessageId: this.activeStreamMessageId, thinkingLevelAtCut: activeTurnThinkingOverride?.pending ?? activeTurnThinkingOverride?.applied, }); @@ -5881,6 +5888,7 @@ export class AgentSession { this.activeToolCallIds.clear(); this.activeStreamContext = undefined; this.activeStreamUserMessageId = undefined; + this.activeStreamMessageId = undefined; this.activeStreamStartedAtMs = undefined; this.activeStreamHadPostCompactionInjection = false; this.activeStreamHadAnyDelta = false; @@ -5965,6 +5973,7 @@ export class AgentSession { this.dispatchingQueuedEntry = false; this.dispatchingQueuedEntryMuxMetadata = undefined; this.preparingWorkspaceTurnMetadata = undefined; + this.activeStreamMessageId = payload.messageId; this.activeStreamStartedAtMs = payload.startTime; // Codex P1 (PRRT_kwDOPxxmWM6cClKS): a new live stream makes mid-stream // setGoal deferral meaningful again — clear the goal service's settled @@ -6219,7 +6228,8 @@ export class AgentSession { const dispatchedQueuedMessage = this.dispatchQueuedProviderToolEndMessageAfterAbort( isQueuedProviderToolEndAbort && this.queuedProviderToolEndAbortInFlight, abortedStreamContext, - payload.metadata?.model + payload.metadata?.model, + payload.messageId ); if (!dispatchedQueuedMessage) { this.setTurnPhase(TurnPhase.IDLE); @@ -6925,13 +6935,12 @@ export class AgentSession { * Whether a bash-monitor-wake continuation is pending dispatch: the next * queued entry is a wake, or a dequeued wake is mid-dispatch (dequeue → * stream start). Wake sends are the only input that inherits an open - * delegated workspace turn's correlation, so TaskService uses this — not - * generic queued/preparing state — to decide whether a correlated - * "tool-calls" queue cut will be continued rather than superseded. Once the - * wake's stream starts, TaskService matches the active stream's inherited - * correlation instead (see hasSameTurnWakeContinuation). + * delegated workspace turn's correlation, so claimWorkspaceTurnContinuation + * treats one as the turn's continuation rather than a superseding entry. Once + * the wake's stream starts, the owner matches the active stream's inherited + * correlation instead. */ - hasPendingBashMonitorWakeContinuation(): boolean { + private hasPendingBashMonitorWakeContinuation(): boolean { if (this.messageQueue.isNextEntryBashMonitorWake()) { return true; } @@ -6940,10 +6949,16 @@ export class AgentSession { } /** - * Whether a queued or dispatching entry continues the exact workspace-turn correlation. + * The delegated turn owner's settlement decision for a correlated "tool-calls" stream-end: + * true when a continuation of that exact turn is pending (the owner defers), false when the + * cut superseded it (the owner settles the turn now). A false answer binds the owed + * continuation: the marker for that cut is voided here, so the turn cannot resume as orphaned + * work no matter how the superseding entry later leaves the queue. The owner reads under its + * own event lock, so this is the only point where its view and the marker are the same. */ - hasPendingWorkspaceTurnContinuation( - metadata: Extract + claimWorkspaceTurnContinuation( + metadata: Extract, + streamEndMessageId: string ): boolean { if (hasSameWorkspaceTurnCorrelation(this.preparingWorkspaceTurnMetadata, metadata)) { return true; @@ -6969,17 +6984,40 @@ export class AgentSession { return true; } - // A stop owed a continuation with nothing else queued to take the turn: the stranded - // resume will carry this correlation, so the owner must not settle the turn at the cut. - // A queued or dispatching entry that failed the checks above supersedes it instead. - return ( - this.messageQueue.isEmpty() && - !this.dispatchingQueuedEntry && - hasSameWorkspaceTurnCorrelation( - getWorkspaceTurnMuxMetadata(this.owedStrandedTurnResume()?.options.muxMetadata), + // A wake send inherits the open delegated turn's correlation from history, so a wake at + // the queue head continues the turn even though the entry carries no correlation itself. + if (this.hasPendingBashMonitorWakeContinuation()) { + return true; + } + + const owed = this.strandedTurnResume; + if ( + owed == null || + !hasSameWorkspaceTurnCorrelation( + getWorkspaceTurnMuxMetadata(owed.options.muxMetadata), metadata ) - ); + ) { + return false; + } + // A marker from a later cut of this turn proves a continuation already ran after the + // stream-end being settled (the owner is processing an older event): defer, and leave the + // newer cut's continuation to its own stream-end. + if (owed.cutMessageId !== streamEndMessageId) { + return true; + } + // Past the retry cap the marker is no longer advertised; the sweep forfeits it. + if (this.owedStrandedTurnResume() == null) { + return false; + } + // Nothing else queued to take the turn: the stranded resume will carry this correlation. + if (this.messageQueue.isEmpty() && !this.dispatchingQueuedEntry) { + return true; + } + // A queued or dispatching entry that failed the checks above supersedes the turn; the + // owner settles it on this answer, so the continuation must not outlive that entry. + this.withdrawStrandedTurnResume(); + return false; } /** @@ -6996,7 +7034,7 @@ export class AgentSession { /** * The continuation is given up with no successor stream (sweep cap or goal refusal, context * discard, session disposal). A delegated turn's owner may have deferred its stream-end on the - * strength of this marker (hasPendingWorkspaceTurnContinuation), and no later stream-end will + * strength of this marker (claimWorkspaceTurnContinuation), and no later stream-end will * arrive for it, so the owner settles that turn here. */ private forfeitStrandedTurnResume(reason: string): void { @@ -7098,7 +7136,8 @@ export class AgentSession { private dispatchQueuedProviderToolEndMessageAfterAbort( isQueuedProviderToolEndAbort: boolean, abortedStreamContext: AgentSession["activeStreamContext"], - abortedModelString: string | undefined + abortedModelString: string | undefined, + abortedMessageId: string ): boolean { this.queuedProviderToolEndAbortInFlight = false; if (!isQueuedProviderToolEndAbort || this.deferQueuedFlushUntilAfterEdit) { @@ -7111,6 +7150,7 @@ export class AgentSession { this.strandedTurnResume = buildStrandedTurnResume({ ...abortedStreamContext, modelString: abortedModelString ?? abortedStreamContext.modelString, + cutMessageId: abortedMessageId, thinkingLevelAtCut: this.activeTurnThinkingOverride?.pending ?? this.activeTurnThinkingOverride?.applied, }); @@ -7370,23 +7410,6 @@ export class AgentSession { this.backgroundProcessManager.setMessageQueued(this.workspaceId, false); if (!this.messageQueue.isEmpty()) { - // A queued entry that is not the stranded delegated turn's own continuation takes the - // turn from it: the owner settles that turn at the cut (hasPendingWorkspaceTurnContinuation - // reports no continuation), so it must not resume later as orphaned work even if this - // entry never starts a stream. - const owedCorrelation = getWorkspaceTurnMuxMetadata( - this.strandedTurnResume?.options.muxMetadata - ); - if ( - owedCorrelation != null && - !this.messageQueue.hasNextWorkspaceTurnContinuation( - owedCorrelation.taskHandleId, - owedCorrelation.ownerWorkspaceId, - owedCorrelation.turnId - ) - ) { - this.strandedTurnResume = undefined; - } // Entries dispatch one at a time (FIFO): special sends (compaction, agent // skills, workspace-turn follow-ups) own their turn, and anything queued // behind them dispatches on a later drain instead of batching into them. diff --git a/src/node/services/taskService.test.ts b/src/node/services/taskService.test.ts index 4dc2b3b77f..e43280d5f4 100644 --- a/src/node/services/taskService.test.ts +++ b/src/node/services/taskService.test.ts @@ -347,8 +347,7 @@ describe("TaskService", () => { isStreaming?: ReturnType; hasQueuedMessages?: ReturnType; hasPendingQueuedOrPreparingTurn?: ReturnType; - hasPendingBashMonitorWakeContinuation?: ReturnType; - hasPendingWorkspaceTurnContinuation?: ReturnType; + claimWorkspaceTurnContinuation?: ReturnType; getQueueCutCutter?: ReturnType; hasPendingAutoRetry?: ReturnType; waitForPendingStreamErrorRecoveryDecision?: ReturnType; @@ -23855,11 +23854,11 @@ describe("TaskService", () => { // A queued bash-monitor wake cuts the correlated stream at a tool boundary // (finishReason "tool-calls") while the child seamlessly continues the // same turn — the handle must stay running. - const hasPendingBashMonitorWakeContinuation = mock( + const claimWorkspaceTurnContinuation = mock( (workspaceId: string) => workspaceId === "childworkspace" ); const { parentId, taskService } = await startWorkspaceTurnForTest({ - hasPendingBashMonitorWakeContinuation, + claimWorkspaceTurnContinuation, }); const internal = taskService as unknown as { handleStreamEnd: (event: StreamEndEvent) => Promise; @@ -23882,6 +23881,12 @@ describe("TaskService", () => { const running = await workspaceTurnSnapshot(taskService, parentId); expect(running).toMatchObject({ status: "running", workspaceId: "childworkspace" }); expect(running?.error).toBeUndefined(); + // The claim is bound to the exact cut it settles. + expect(claimWorkspaceTurnContinuation).toHaveBeenCalledWith( + "childworkspace", + correlation, + "msg_queue_cut" + ); // The continuation stream inherits the correlation metadata (see // AgentSession.inheritOpenWorkspaceTurnMetadata); its terminal stream-end @@ -23908,7 +23913,7 @@ describe("TaskService", () => { }); test("nested agent progress preserves workspace-turn correlation", async () => { - const hasPendingWorkspaceTurnContinuation = mock( + const claimWorkspaceTurnContinuation = mock( ( workspaceId: string, metadata: { taskHandleId: string; ownerWorkspaceId: string; turnId: string } @@ -23918,7 +23923,7 @@ describe("TaskService", () => { metadata.turnId === "turn" ); const { config, parentId, taskService, workspaceMocks } = await startWorkspaceTurnForTest({ - hasPendingWorkspaceTurnContinuation, + claimWorkspaceTurnContinuation, }); const correlation = workspaceTurnMuxMetadata(parentId); diff --git a/src/node/services/taskWorkspaceSeam.testUtils.ts b/src/node/services/taskWorkspaceSeam.testUtils.ts index 777c2f090e..f1e507d296 100644 --- a/src/node/services/taskWorkspaceSeam.testUtils.ts +++ b/src/node/services/taskWorkspaceSeam.testUtils.ts @@ -14,8 +14,7 @@ export function makeWorkspaceHostFake(overrides: Partial = {}): W hasQueuedMessages: () => false, hasPendingQueuedOrPreparingTurn: () => false, hasPendingAutoRetry: () => false, - hasPendingBashMonitorWakeContinuation: () => false, - hasPendingWorkspaceTurnContinuation: () => false, + claimWorkspaceTurnContinuation: () => false, hasQueuedWorkspaceTurn: () => false, removeQueuedWorkspaceTurn: () => Ok(true), removeQueuedMessagesByDedupeKeyPrefix: () => Ok(0), diff --git a/src/node/services/taskWorkspaceSeam.ts b/src/node/services/taskWorkspaceSeam.ts index e12c042ef9..853136eea5 100644 --- a/src/node/services/taskWorkspaceSeam.ts +++ b/src/node/services/taskWorkspaceSeam.ts @@ -402,10 +402,10 @@ export interface TurnAdmissionHost { hasQueuedMessages(workspaceId: string, dispatchMode?: "tool-end" | "turn-end"): boolean; hasPendingQueuedOrPreparingTurn(workspaceId: string): boolean; hasPendingAutoRetry(workspaceId: string): boolean; - hasPendingBashMonitorWakeContinuation(workspaceId: string): boolean; - hasPendingWorkspaceTurnContinuation( + claimWorkspaceTurnContinuation( workspaceId: string, - metadata: Extract + metadata: Extract, + streamEndMessageId: string ): boolean; hasQueuedWorkspaceTurn(workspaceId: string, handleId: string): boolean; removeQueuedWorkspaceTurn( diff --git a/src/node/services/workspaceService.ts b/src/node/services/workspaceService.ts index 94889ea6e2..ab3d38f187 100644 --- a/src/node/services/workspaceService.ts +++ b/src/node/services/workspaceService.ts @@ -11847,24 +11847,14 @@ export class WorkspaceService extends EventEmitter implements WorkspaceHost { ); } - /** - * Whether a bash-monitor-wake continuation is queued next or mid-dispatch. - * See AgentSession.hasPendingBashMonitorWakeContinuation for semantics. - */ - hasPendingBashMonitorWakeContinuation(workspaceId: string): boolean { - const session = this.sessions.get(workspaceId.trim()); - return session?.hasPendingBashMonitorWakeContinuation() ?? false; - } - - /** - * Whether a queued or dispatching entry continues the exact workspace-turn correlation. - */ - hasPendingWorkspaceTurnContinuation( + /** See AgentSession.claimWorkspaceTurnContinuation for semantics. */ + claimWorkspaceTurnContinuation( workspaceId: string, - metadata: Extract + metadata: Extract, + streamEndMessageId: string ): boolean { const session = this.sessions.get(workspaceId.trim()); - return session?.hasPendingWorkspaceTurnContinuation(metadata) ?? false; + return session?.claimWorkspaceTurnContinuation(metadata, streamEndMessageId) ?? false; } /** diff --git a/src/node/services/workspaceTurnManager.test.ts b/src/node/services/workspaceTurnManager.test.ts index 20588945ad..6e6e7bbe52 100644 --- a/src/node/services/workspaceTurnManager.test.ts +++ b/src/node/services/workspaceTurnManager.test.ts @@ -317,8 +317,7 @@ describe("WorkspaceTurnManager", () => { isStreaming?: ReturnType; hasQueuedMessages?: ReturnType; hasPendingQueuedOrPreparingTurn?: ReturnType; - hasPendingBashMonitorWakeContinuation?: ReturnType; - hasPendingWorkspaceTurnContinuation?: ReturnType; + claimWorkspaceTurnContinuation?: ReturnType; getQueueCutCutter?: ReturnType; hasPendingAutoRetry?: ReturnType; waitForPendingStreamErrorRecoveryDecision?: ReturnType; diff --git a/src/node/services/workspaceTurnManager.ts b/src/node/services/workspaceTurnManager.ts index b74f3b9d6e..745263e070 100644 --- a/src/node/services/workspaceTurnManager.ts +++ b/src/node/services/workspaceTurnManager.ts @@ -4307,24 +4307,23 @@ export class WorkspaceTurnManager { } /** - * Whether a continuation of this exact delegated turn is pending or streaming. - * Pending entries must carry the same correlation metadata as the ended stream. + * Whether a continuation of this exact delegated turn is pending or streaming. The + * session's answer is binding: a false claim voids the continuation it owed to this cut, so + * the settlement made here cannot be followed by an orphaned resume of the same turn. */ private hasSameTurnContinuation( event: StreamEndEvent, correlation: { taskHandleId: string; ownerWorkspaceId: string; turnId: string } ): boolean { if ( - this.workspaceService.hasPendingWorkspaceTurnContinuation(event.workspaceId, { - type: "workspace-turn-task", - ...correlation, - }) + this.workspaceService.claimWorkspaceTurnContinuation( + event.workspaceId, + { type: "workspace-turn-task", ...correlation }, + event.messageId + ) ) { return true; } - if (this.workspaceService.hasPendingBashMonitorWakeContinuation(event.workspaceId)) { - return true; - } const activeStream = this.streamManager?.getStreamInfo(event.workspaceId); if (activeStream == null || activeStream.messageId === event.messageId) { return false; From 23e9c8727ac5b7a45a7b534da605d0922574a39c Mon Sep 17 00:00:00 2001 From: Michael Suchacz <203725896+ibetitsmike@users.noreply.github.com> Date: Thu, 3 Sep 2026 12:38:42 +0000 Subject: [PATCH 11/28] Expect the hard-stop flag in TaskService clearQueue assertions Task hard stops (hard timeout, descendant terminate cascade) now clear the queue with { hardStop: true } so the session forfeits an owed stranded continuation; the two existing call-shape assertions still expected the bare call. --- src/node/services/taskService.test.ts | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/src/node/services/taskService.test.ts b/src/node/services/taskService.test.ts index e43280d5f4..7fa571abf7 100644 --- a/src/node/services/taskService.test.ts +++ b/src/node/services/taskService.test.ts @@ -12189,7 +12189,7 @@ describe("TaskService", () => { reason: "timed out", }); - expect(clearQueue).toHaveBeenCalledWith(childTaskId); + expect(clearQueue).toHaveBeenCalledWith(childTaskId, { hardStop: true }); expect(stopStream).toHaveBeenCalledWith(childTaskId, { abandonPartial: true, abortReason: "system", @@ -12507,8 +12507,8 @@ describe("TaskService", () => { const interruptedTaskIds = await taskService.terminateAllDescendantAgentTasks(rootWorkspaceId); expect(interruptedTaskIds).toEqual([childTaskId, parentTaskId]); - expect(clearQueue).toHaveBeenNthCalledWith(1, childTaskId); - expect(clearQueue).toHaveBeenNthCalledWith(2, parentTaskId); + expect(clearQueue).toHaveBeenNthCalledWith(1, childTaskId, { hardStop: true }); + expect(clearQueue).toHaveBeenNthCalledWith(2, parentTaskId, { hardStop: true }); expect(stopStream).toHaveBeenNthCalledWith( 1, childTaskId, From ba2a9da29762c02594336306b11674a5f1b28584 Mon Sep 17 00:00:00 2001 From: Michael Suchacz <203725896+ibetitsmike@users.noreply.github.com> Date: Thu, 3 Sep 2026 14:48:55 +0000 Subject: [PATCH 12/28] Close stranded-resume terminal races Retain and autonomously retry the delegated-turn settlement owed when a stranded continuation is forfeited, including after AgentSession disposal. Recheck pull-based admission after the durable turn-envelope write. Before an internal stranded resume starts, reapply workspace archive/removal admission and verify a correlated workspace-turn handle is still active, carrying the workspace stop epoch through StreamManager's pre-start probes. Add red-green coverage for settlement retry through disposal, stopped-turn refusal at read and after admission, post-envelope goal refusal, and the TaskService handle/epoch admission contract. --- .../agentSession.queueDispatch.test.ts | 131 +++++++++++++++- src/node/services/agentSession.testHarness.ts | 2 + src/node/services/agentSession.ts | 143 ++++++++++++++---- src/node/services/streamManager.test.ts | 44 ++++++ src/node/services/streamManager.ts | 6 +- src/node/services/taskService.test.ts | 33 ++++ src/node/services/taskService.ts | 24 +++ .../services/taskWorkspaceSeam.testUtils.ts | 2 + src/node/services/taskWorkspaceSeam.ts | 8 + src/node/services/workspaceService.ts | 34 +++++ 10 files changed, 392 insertions(+), 35 deletions(-) diff --git a/src/node/services/agentSession.queueDispatch.test.ts b/src/node/services/agentSession.queueDispatch.test.ts index af23849747..20b3101adb 100644 --- a/src/node/services/agentSession.queueDispatch.test.ts +++ b/src/node/services/agentSession.queueDispatch.test.ts @@ -1471,7 +1471,15 @@ describe("AgentSession queued message tool-call dispatch", () => { * the owner deferred the cut stream-end on the advertised continuation and the session sits * idle with it still owed. */ - async function strandDelegatedTurnBehindClosedGate(workspaceId: string) { + async function strandDelegatedTurnBehindClosedGate( + workspaceId: string, + extra?: { + harness?: Partial>; + settleForfeited?: ReturnType< + typeof mock<(metadata: unknown, reason: string) => Promise> + >; + } + ) { let gateOpen = true; const workspaceGoalService = { assertPricedModelForBudgetedGoal: mock(() => @@ -1483,11 +1491,13 @@ describe("AgentSession queued message tool-call dispatch", () => { recordStreamStarted: mock(() => Promise.resolve()), syncGoalModeWithChatTail: mock(() => Promise.resolve(null)), } as unknown as WorkspaceGoalService; - const settleForfeited = mock((_metadata: unknown, _reason: string) => Promise.resolve()); + const settleForfeited = + extra?.settleForfeited ?? mock((_metadata: unknown, _reason: string) => Promise.resolve()); const harness = await createStreamingTurnHarness(workspaceId, { harness: { workspaceGoalService, settleForfeitedWorkspaceTurnContinuation: settleForfeited, + ...extra?.harness, }, sendOptions: { muxMetadata: WORKSPACE_TURN_CORRELATION }, sendInternal: { synthetic: true, agentInitiated: true }, @@ -1502,7 +1512,13 @@ describe("AgentSession queued message tool-call dispatch", () => { ); aiEmitter.emit("stream-end", streamEndEvent(workspaceId)); expect(await waitForCondition(() => !session.isBusy())).toBe(true); - return { ...harness, settleForfeited }; + return { + ...harness, + settleForfeited, + openGate: () => { + gateOpen = true; + }, + }; } test("stops advertising the delegated continuation once the resume cap is exhausted", async () => { @@ -1576,6 +1592,115 @@ describe("AgentSession queued message tool-call dispatch", () => { } }); + test("a forfeited turn's failed settlement stays owed and retries through disposal", async () => { + const workspaceId = "queue-dispatch-stranded-delegated-settle-retry"; + let settleAttempts = 0; + const settleForfeited = mock((_metadata: unknown, _reason: string) => { + settleAttempts += 1; + return settleAttempts === 1 + ? Promise.reject(new Error("task store unavailable")) + : Promise.resolve(); + }); + const harness = await strandDelegatedTurnBehindClosedGate(workspaceId, { settleForfeited }); + const { session, cleanup } = harness; + + try { + // A history clear forfeits the continuation; the owner's store rejects the first settlement. + expect((await session.discardAutoRetryForContextMutation()).success).toBe(true); + expect(await waitForCondition(() => settleAttempts === 1)).toBe(true); + await new Promise((resolve) => setTimeout(resolve, 10)); + expect(settleAttempts).toBe(1); + + // The obligation survives the failure and session teardown: its own retry lands even when + // no future idle poke can occur after workspace removal. + session.dispose(); + expect(await waitForCondition(() => settleAttempts === 2, 1_500)).toBe(true); + expect(settleForfeited.mock.calls[1]?.[0]).toEqual(WORKSPACE_TURN_CORRELATION); + await new Promise((resolve) => setTimeout(resolve, 20)); + expect(settleAttempts).toBe(2); + } finally { + session.dispose(); + await cleanup(); + } + }); + + test("a resume whose workspace or delegated turn no longer admits it settles the turn", async () => { + const workspaceId = "queue-dispatch-stranded-delegated-turn-stopped"; + let turnActive = true; + const admitStrandedTurnResume = mock((_correlation: unknown) => + Promise.resolve({ admissible: turnActive, admissionStale: () => !turnActive }) + ); + const settleForfeited = mock((_metadata: unknown, _reason: string) => Promise.resolve()); + const harness = await strandDelegatedTurnBehindClosedGate(workspaceId, { + harness: { admitStrandedTurnResume }, + settleForfeited, + }); + const { session, cleanup, streamMessage } = harness; + + try { + // The pricing gate failed the first resume; the continuation is still advertised. + expect( + session.claimWorkspaceTurnContinuation(WORKSPACE_TURN_CORRELATION, "assistant-1") + ).toBe(true); + + // task_stop / interrupt_active settled the handle while the cut stream was already + // complete: no abort reached the session, only the owner's record changed. + turnActive = false; + harness.openGate(); + session.drainQueuedMessagesIfIdle(); + expect(await waitForCondition(() => settleForfeited.mock.calls.length === 1)).toBe(true); + expect(admitStrandedTurnResume.mock.calls[0]?.[0]).toEqual(WORKSPACE_TURN_CORRELATION); + expect( + session.claimWorkspaceTurnContinuation(WORKSPACE_TURN_CORRELATION, "assistant-1") + ).toBe(false); + session.drainQueuedMessagesIfIdle(); + await new Promise((resolve) => setTimeout(resolve, 25)); + expect(streamMessage).toHaveBeenCalledTimes(1); + } finally { + session.dispose(); + await cleanup(); + } + }); + + test("a stop landing between admission and launch refuses the resume", async () => { + const workspaceId = "queue-dispatch-stranded-delegated-stop-in-flight"; + // Admitted on read, but the workspace's stop epoch moved before the stream could launch. + const admitStrandedTurnResume = mock((_correlation: unknown) => + Promise.resolve({ admissible: true, admissionStale: () => true }) + ); + const aiEmitter = new EventEmitter(); + const streamMessage = mock((_options: StreamMessageOptions) => { + aiEmitter.emit("stream-start", streamStartEvent(workspaceId)); + return Promise.resolve(Ok(createStartedTurnHandle("assistant-1"))); + }); + const { session, cleanup, historyService } = await createAgentSessionHarness({ + workspaceId, + aiEmitter, + aiServiceOverrides: { streamMessage: streamMessage as unknown as AIService["streamMessage"] }, + admitStrandedTurnResume, + }); + + try { + await historyService.appendToHistory( + workspaceId, + createMuxMessage("user-0", "user", "keep going", { + timestamp: Date.now(), + muxMetadata: WORKSPACE_TURN_CORRELATION, + }) + ); + const resumed = await session.resumeStream( + { model: TEST_MODEL, agentId: "exec", muxMetadata: WORKSPACE_TURN_CORRELATION }, + { agentInitiated: true, revalidateAdmission: true } + ); + expect(resumed).toEqual(Ok({ started: false, refusedBy: "workspace-turn" })); + expect(streamMessage).not.toHaveBeenCalled(); + expect(session.isBusy()).toBe(false); + } finally { + session.dispose(); + await cleanup(); + } + }); + test("a user Stop during the resume's admission gate cancels it", async () => { const workspaceId = "queue-dispatch-stranded-stop-in-admission"; let releaseGate: () => void = () => undefined; diff --git a/src/node/services/agentSession.testHarness.ts b/src/node/services/agentSession.testHarness.ts index 0ffbeb6b1c..86ad42d1a4 100644 --- a/src/node/services/agentSession.testHarness.ts +++ b/src/node/services/agentSession.testHarness.ts @@ -116,6 +116,7 @@ export interface AgentSessionHarnessOptions { onCompactionComplete?: (metadata: CompactionCompletionMetadata) => void; hasExternalSendPreflight?: () => boolean; settleForfeitedWorkspaceTurnContinuation?: AgentSessionOptions["settleForfeitedWorkspaceTurnContinuation"]; + admitStrandedTurnResume?: AgentSessionOptions["admitStrandedTurnResume"]; captureEvents?: boolean; } @@ -162,6 +163,7 @@ export async function createAgentSessionHarness( onCompactionComplete: options.onCompactionComplete, hasExternalSendPreflight: options.hasExternalSendPreflight, settleForfeitedWorkspaceTurnContinuation: options.settleForfeitedWorkspaceTurnContinuation, + admitStrandedTurnResume: options.admitStrandedTurnResume, }); const events: WorkspaceChatMessage[] = []; diff --git a/src/node/services/agentSession.ts b/src/node/services/agentSession.ts index 05274e93e6..84db0f6689 100644 --- a/src/node/services/agentSession.ts +++ b/src/node/services/agentSession.ts @@ -585,6 +585,7 @@ const MAX_STARTUP_RECOVERY_DEFERRED_ATTEMPTS = 4; * monitored processes in a row resumes every time. */ const MAX_CONSECUTIVE_STRANDED_TURN_RESUMES = 3; +const FORFEIT_SETTLEMENT_RETRY_DELAY_MS = 1_000; export interface AgentSessionChatEvent { workspaceId: string; @@ -699,6 +700,14 @@ export interface AgentSessionOptions { metadata: WorkspaceTurnMuxMetadata, reason: string ) => Promise; + /** + * Admission for a stranded resume, read once the turn is claimed: the workspace still accepts + * streams (not being removed or archived) and, for a delegated turn, its owner still has the + * turn running. The probe reports a stop that lands after the read (workspace stop epoch). + */ + admitStrandedTurnResume?: ( + correlation: WorkspaceTurnMuxMetadata | undefined + ) => Promise<{ admissible: boolean; admissionStale?: () => boolean }>; } enum TurnPhase { @@ -741,6 +750,7 @@ export class AgentSession { private readonly onPostCompactionStateChange?: () => void; private readonly hasExternalSendPreflight?: () => boolean; private readonly settleForfeitedWorkspaceTurnContinuation?: AgentSessionOptions["settleForfeitedWorkspaceTurnContinuation"]; + private readonly admitStrandedTurnResume?: AgentSessionOptions["admitStrandedTurnResume"]; private readonly emitter = new EventEmitter(); private readonly aiListeners: Array<{ event: string; handler: (...args: unknown[]) => void }> = []; @@ -781,6 +791,12 @@ export class AgentSession { // Set while the sweep's resume is running (claim through stream end); aborting it cancels only // the pre-stream window, since StreamManager unlinks the signal once a stream registers. private strandedTurnResumeInFlight: AbortController | null = null; + /** Owner settlements for forfeited continuations that have not landed yet (settleOwedForfeits). */ + private readonly owedForfeitSettlements = new Map< + string, + { correlation: WorkspaceTurnMuxMetadata; reason: string; inFlight: boolean } + >(); + private owedForfeitSettlementRetryTimer: ReturnType | null = null; private consecutiveStrandedResumes = 0; private idleWaiters: Array<() => void> = []; @@ -993,6 +1009,7 @@ export class AgentSession { onPostCompactionStateChange, hasExternalSendPreflight, settleForfeitedWorkspaceTurnContinuation, + admitStrandedTurnResume, } = options; assert(typeof workspaceId === "string", "workspaceId must be a string"); @@ -1022,6 +1039,7 @@ export class AgentSession { this.onPostCompactionStateChange = onPostCompactionStateChange; this.hasExternalSendPreflight = hasExternalSendPreflight; this.settleForfeitedWorkspaceTurnContinuation = settleForfeitedWorkspaceTurnContinuation; + this.admitStrandedTurnResume = admitStrandedTurnResume; this.compactionHandler = new CompactionHandler({ workspaceId: this.workspaceId, @@ -4295,12 +4313,13 @@ export class AgentSession { /** Cancels the pre-stream window (admission gates, history reads) once the resume is withdrawn. */ abortSignal?: AbortSignal; /** - * Revalidate a goal turn against durable goal state once the turn is claimed, the same - * veto durable goal redispatches apply; a refusal reports `goalRefused`. + * Revalidate the resume once the turn is claimed: a goal turn against durable goal state + * (the same veto durable goal redispatches apply) and the workspace plus delegated turn + * through admitStrandedTurnResume; a refusal reports `refusedBy`. */ - revalidateGoal?: boolean; + revalidateAdmission?: boolean; } - ): Promise> { + ): Promise> { this.assertNotDisposed("resumeStream"); assert(options, "resumeStream requires options"); @@ -4361,7 +4380,7 @@ export class AgentSession { } let goalAdmissionStale: (() => boolean) | undefined; if ( - internal?.revalidateGoal === true && + internal?.revalidateAdmission === true && internal.goalKind != null && internal.goalId != null && this.workspaceGoalService @@ -4372,30 +4391,41 @@ export class AgentSession { internal.goalKind ); if (!admission.admissible) { - return Ok({ started: false, goalRefused: true }); + return Ok({ started: false, refusedBy: "goal" }); } goalAdmissionStale = admission.admissionStale; } + let turnAdmissionStale: (() => boolean) | undefined; + if (internal?.revalidateAdmission === true && this.admitStrandedTurnResume) { + const admission = await this.admitStrandedTurnResume( + getWorkspaceTurnMuxMetadata(optionsForStream.muxMetadata) + ); + if (!admission.admissible) { + return Ok({ started: false, refusedBy: "workspace-turn" }); + } + turnAdmissionStale = admission.admissionStale; + } // Last admission check before the stream's own pre-start I/O, like sendMessage's PREPARING // gate: a withdrawal that landed during the awaits above refuses the turn here; the abort // signal covers the history reads that follow. if (withdrawn()) { return Ok({ started: false }); } - // The goal probe rides along to the stream-admission boundary: a Pause or goal replacement - // landing during the history reads and request construction below has no stream to - // interrupt, so the launch itself rechecks it (StreamManager last, right before - // registration). Sticky so the return value matches what refused the launch. - let goalRefused = false; + // The admission probes ride along to the stream-admission boundary: a Pause, goal + // replacement, or workspace stop landing during the history reads and request construction + // below has no stream to interrupt, so the launch itself rechecks them (StreamManager last, + // right before registration). Sticky so the return value matches what refused the launch. + let refusedBy: "goal" | "workspace-turn" | undefined; const refuseStreamStart = - goalAdmissionStale != null + goalAdmissionStale != null || turnAdmissionStale != null ? (): boolean => { - goalRefused ||= goalAdmissionStale(); - return goalRefused; + refusedBy ??= goalAdmissionStale?.() === true ? "goal" : undefined; + refusedBy ??= turnAdmissionStale?.() === true ? "workspace-turn" : undefined; + return refusedBy != null; } : undefined; if (refuseStreamStart?.() === true) { - return Ok({ started: false, goalRefused: true }); + return Ok({ started: false, refusedBy }); } // Must await here so the finally block runs after streaming completes, @@ -4415,8 +4445,8 @@ export class AgentSession { if (!result.success) { return result; } - if (goalRefused) { - return Ok({ started: false, goalRefused: true }); + if (refusedBy != null) { + return Ok({ started: false, refusedBy }); } // A withdrawal inside streamWithHistory returns Ok before any stream registers, so the @@ -7039,18 +7069,64 @@ export class AgentSession { */ private forfeitStrandedTurnResume(reason: string): void { const correlation = getWorkspaceTurnMuxMetadata(this.strandedTurnResume?.options.muxMetadata); + const settlementOwed = + correlation != null && this.settleForfeitedWorkspaceTurnContinuation != null; + if (settlementOwed) { + // Retain the owner's only terminal path before dropping the marker that advertised it. + this.owedForfeitSettlements.set( + `${correlation.ownerWorkspaceId}/${correlation.taskHandleId}/${correlation.turnId}`, + { correlation, reason, inFlight: false } + ); + } this.withdrawStrandedTurnResume(); - if (correlation == null || this.settleForfeitedWorkspaceTurnContinuation == null) { + if (settlementOwed) { + this.settleOwedForfeits(); + } + } + + /** + * The settlement is the owner's only remaining path to a terminal record for that turn, so a + * failed attempt (task store I/O) stays owed and retries on its own (including after session + * disposal) as well as from idle sweeps. Settlement is idempotent on the owner's side. + */ + private settleOwedForfeits(): void { + const settle = this.settleForfeitedWorkspaceTurnContinuation; + if (settle == null) { return; } - void this.settleForfeitedWorkspaceTurnContinuation(correlation, reason).catch( - (error: unknown) => { - log.warn("Failed to settle forfeited workspace turn continuation", { - workspaceId: this.workspaceId, - error: getErrorMessage(error), - }); + for (const [key, owed] of this.owedForfeitSettlements) { + if (owed.inFlight) { + continue; } - ); + owed.inFlight = true; + void settle(owed.correlation, owed.reason) + .then(() => { + if (this.owedForfeitSettlements.get(key) === owed) { + this.owedForfeitSettlements.delete(key); + } + }) + .catch((error: unknown) => { + owed.inFlight = false; + log.warn("Failed to settle forfeited workspace turn continuation; retrying", { + workspaceId: this.workspaceId, + error: getErrorMessage(error), + }); + this.scheduleOwedForfeitSettlementRetry(); + }); + } + } + + private scheduleOwedForfeitSettlementRetry(): void { + if (this.owedForfeitSettlementRetryTimer != null) { + return; + } + // Do not cancel this on dispose: workspace removal is itself one of the boundaries that can + // forfeit a delegated turn, and its owner still requires the terminal task-store write. + this.owedForfeitSettlementRetryTimer = setTimeout(() => { + this.owedForfeitSettlementRetryTimer = null; + this.settleOwedForfeits(); + }, FORFEIT_SETTLEMENT_RETRY_DELAY_MS); + this.owedForfeitSettlementRetryTimer.unref(); } /** @@ -7168,6 +7244,7 @@ export class AgentSession { * run from every idle transition and queue removal; only the first eligible call acts. */ private resumeStrandedTurnIfIdle(): void { + this.settleOwedForfeits(); const resume = this.owedStrandedTurnResume(); if (resume == null) { if (this.strandedTurnResume != null) { @@ -7211,7 +7288,7 @@ export class AgentSession { goalKind: resume.goalKind, goalId: resume.goalId, abortSignal: inFlight.signal, - revalidateGoal: true, + revalidateAdmission: true, }) .then((result) => { if (!result.success) { @@ -7221,13 +7298,19 @@ export class AgentSession { }); return false; } - if (result.data.goalRefused === true) { - // A Pause or terminal transition landed while the goal turn ran: nothing is owed to it. - log.info("Dropping stranded goal turn: goal no longer admits it", { + if (result.data.refusedBy != null) { + // A Pause or terminal goal transition, or a stop on the workspace or delegated turn, + // landed while the cut turn waited: nothing is owed to it anymore. + log.info("Dropping stranded turn: no longer admitted", { workspaceId: this.workspaceId, + refusedBy: result.data.refusedBy, goalKind: resume.goalKind, }); - this.forfeitStrandedTurnResume("Stranded turn resume dropped: goal no longer admits it."); + this.forfeitStrandedTurnResume( + result.data.refusedBy === "goal" + ? "Stranded turn resume dropped: goal no longer admits it." + : "Stranded turn resume dropped: workspace or delegated turn no longer admits it." + ); return false; } if (!result.data.started) { diff --git a/src/node/services/streamManager.test.ts b/src/node/services/streamManager.test.ts index fe3287df77..c2c29cd06a 100644 --- a/src/node/services/streamManager.test.ts +++ b/src/node/services/streamManager.test.ts @@ -2850,6 +2850,50 @@ describe("StreamManager - Concurrent Stream Prevention", () => { expect(streamManager.isStreaming(workspaceId)).toBe(false); }); + test("refuses processing when the admission probe turns stale during the envelope write", async () => { + const workspaceId = "test-workspace-refuse-after-construct"; + + let processCalled = false; + let streamStartEmitted = false; + let refused = false; + + onTurnEngineEvent(streamManager, "stream-start", () => { + streamStartEmitted = true; + }); + Reflect.set( + streamManager, + "createTempDirForStream", + (): Promise => Promise.resolve("/tmp/mock-stream-temp") + ); + Reflect.set(streamManager, "cleanupStreamTempDir", (): void => undefined); + Reflect.set(streamManager, "processStreamWithCleanup", (): Promise => { + processCalled = true; + return Promise.resolve(); + }); + + const result = await streamManager.startStream( + testStartOptions({ + workspaceId, + messageId: "test-msg-refuse-after-construct", + model: createTestLanguageModel(), + runtime, + refuseStreamStart: () => refused, + // The stream is registered by now; a goal Pause lands while the envelope is written. + onStreamConstructed: () => { + refused = true; + return Promise.resolve(); + }, + tools: {}, + }) + ); + expect(result.success).toBe(true); + if (!result.success) throw new Error("Expected aborted startup handle"); + expect(await result.data.completion).toEqual({ status: "aborted", abortReason: "startup" }); + expect(processCalled).toBe(false); + expect(streamStartEmitted).toBe(false); + expect(streamManager.isStreaming(workspaceId)).toBe(false); + }); + test("should honor abortSignal before atomic stream creation", async () => { const workspaceId = "test-workspace-abort-before-create"; diff --git a/src/node/services/streamManager.ts b/src/node/services/streamManager.ts index 5dfe7585c8..e2d6f0107b 100644 --- a/src/node/services/streamManager.ts +++ b/src/node/services/streamManager.ts @@ -4903,10 +4903,12 @@ export class StreamManager { // stream may already occupy this workspace's slot. Launching // processing now would emit stream-start after the abort and its // cleanup would later delete that replacement. Bail out; the finally - // block releases this never-processed stream's resources. + // block releases this never-processed stream's resources. The caller's + // admission probe has no signal to abort with, so it is re-read here too. if ( streamAbortController.signal.aborted || - this.workspaceStreams.get(typedWorkspaceId) !== streamInfo + this.workspaceStreams.get(typedWorkspaceId) !== streamInfo || + refuseStreamStart?.() === true ) { if (this.workspaceStreams.get(typedWorkspaceId) === streamInfo) { this.workspaceStreams.delete(typedWorkspaceId); diff --git a/src/node/services/taskService.test.ts b/src/node/services/taskService.test.ts index 7fa571abf7..719ac71ce8 100644 --- a/src/node/services/taskService.test.ts +++ b/src/node/services/taskService.test.ts @@ -23912,6 +23912,39 @@ describe("TaskService", () => { }); }); + test("workspace-turn continuation admission tracks the handle and later stops", async () => { + const { parentId, taskService } = await startWorkspaceTurnForTest(); + const correlation = workspaceTurnMuxMetadata(parentId); + + // Running handle on this workspace: admitted, and the probe stays fresh until a stop lands. + const admitted = await taskService.getWorkspaceTurnContinuationAdmission( + "childworkspace", + correlation + ); + expect(admitted.admissible).toBe(true); + expect(admitted.admissionStale()).toBe(false); + + // A stop on the workspace after the read (interruptWorkspaceTurn bumps the stop epoch inside + // its settlement boundary) turns the earlier probe stale and refuses a fresh read. + const stopped = await workspaceTurnManagerFor(taskService).interruptWorkspaceTurn( + parentId, + correlation.taskHandleId + ); + expect(stopped.success).toBe(true); + expect(admitted.admissionStale()).toBe(true); + const refused = await taskService.getWorkspaceTurnContinuationAdmission( + "childworkspace", + correlation + ); + expect(refused.admissible).toBe(false); + + // A different workspace or turn never matches the handle. + expect( + (await taskService.getWorkspaceTurnContinuationAdmission("otherworkspace", correlation)) + .admissible + ).toBe(false); + }); + test("nested agent progress preserves workspace-turn correlation", async () => { const claimWorkspaceTurnContinuation = mock( ( diff --git a/src/node/services/taskService.ts b/src/node/services/taskService.ts index 0365933560..43ae22f089 100644 --- a/src/node/services/taskService.ts +++ b/src/node/services/taskService.ts @@ -9575,6 +9575,30 @@ export class TaskService implements AgentTaskIntegration { error ); } + + async getWorkspaceTurnContinuationAdmission( + workspaceId: string, + muxMetadata: Extract + ): Promise<{ admissible: boolean; admissionStale: () => boolean }> { + // Every stop on the workspace (interruptWorkspaceTurn, task hard-stop cascades) bumps the + // epoch synchronously inside its settlement boundary, before the store write lands. + const stopEpoch = this.getWorkspaceStopEpoch(workspaceId); + const record = await this.getWorkspaceTurnManager().getWorkspaceTurnRecord( + muxMetadata.ownerWorkspaceId, + muxMetadata.taskHandleId + ); + const admissible = + record?.workspaceId === workspaceId && + record.turnId === muxMetadata.turnId && + isActiveWorkspaceTurnTaskStatus(record.status) && + !this.isWorkspaceStopInProgress(workspaceId); + return { + admissible, + admissionStale: () => + this.getWorkspaceStopEpoch(workspaceId) !== stopEpoch || + this.isWorkspaceStopInProgress(workspaceId), + }; + } async noteWorkspaceUnarchived(workspaceId: string): Promise { assert(workspaceId.length > 0, "noteWorkspaceUnarchived requires workspaceId"); // Archived owners park workflow terminal wakes unsettled (the drain drops the in-memory diff --git a/src/node/services/taskWorkspaceSeam.testUtils.ts b/src/node/services/taskWorkspaceSeam.testUtils.ts index f1e507d296..48a408caed 100644 --- a/src/node/services/taskWorkspaceSeam.testUtils.ts +++ b/src/node/services/taskWorkspaceSeam.testUtils.ts @@ -75,6 +75,8 @@ export function makeAgentTaskIntegrationFake( terminateAllDescendantAgentTasks: () => Promise.resolve([]), noteWorkspaceUnarchived: () => Promise.resolve(), settleWorkspaceTurnContinuationFailure: () => Promise.resolve(), + getWorkspaceTurnContinuationAdmission: () => + Promise.resolve({ admissible: true, admissionStale: () => false }), ...overrides, }; } diff --git a/src/node/services/taskWorkspaceSeam.ts b/src/node/services/taskWorkspaceSeam.ts index 853136eea5..dd308be4da 100644 --- a/src/node/services/taskWorkspaceSeam.ts +++ b/src/node/services/taskWorkspaceSeam.ts @@ -532,6 +532,14 @@ export interface AgentTaskIntegration { status: "interrupted" | "error", error: string ): Promise; + /** + * Whether the delegated turn is still active on this workspace, with a probe that turns stale + * once a stop lands on the workspace after the read. + */ + getWorkspaceTurnContinuationAdmission( + workspaceId: string, + muxMetadata: Extract + ): Promise<{ admissible: boolean; admissionStale: () => boolean }>; } export interface WorkspaceTurnTaskHost { diff --git a/src/node/services/workspaceService.ts b/src/node/services/workspaceService.ts index ab3d38f187..5a5b884007 100644 --- a/src/node/services/workspaceService.ts +++ b/src/node/services/workspaceService.ts @@ -4020,6 +4020,33 @@ export class WorkspaceService extends EventEmitter implements WorkspaceHost { reason ); }, + // The stranded resume starts a stream from inside the session, so it re-applies the + // stream-start guards WorkspaceService.resumeStream enforces (removal, archive) and, for a + // delegated turn, checks the owner still has it running: a task_stop or lifecycle interrupt + // that found the cut stream already completed had no abort to withdraw the marker with. + admitStrandedTurnResume: async (correlation) => { + const workspaceRefused = (): boolean => + this.removingWorkspaces.has(workspaceId) || + this.archivingWorkspaces.has(workspaceId) || + this.isWorkspaceArchivedInConfig(workspaceId); + if (workspaceRefused()) { + return { admissible: false }; + } + if (correlation == null || this.agentTaskIntegration == null) { + return { admissible: true, admissionStale: workspaceRefused }; + } + const turn = await this.agentTaskIntegration.getWorkspaceTurnContinuationAdmission( + workspaceId, + correlation + ); + if (!turn.admissible) { + return { admissible: false }; + } + return { + admissible: true, + admissionStale: () => workspaceRefused() || turn.admissionStale(), + }; + }, }); } @@ -11847,6 +11874,13 @@ export class WorkspaceService extends EventEmitter implements WorkspaceHost { ); } + private isWorkspaceArchivedInConfig(workspaceId: string): boolean { + const entry = findWorkspaceEntry(this.config.loadConfigOrDefault(), workspaceId); + return ( + entry != null && isWorkspaceArchived(entry.workspace.archivedAt, entry.workspace.unarchivedAt) + ); + } + /** See AgentSession.claimWorkspaceTurnContinuation for semantics. */ claimWorkspaceTurnContinuation( workspaceId: string, From 9d9fa8d8f432ad5d8ba54001ed3f565d9e7ba6bd Mon Sep 17 00:00:00 2001 From: Michael Suchacz <203725896+ibetitsmike@users.noreply.github.com> Date: Thu, 3 Sep 2026 17:30:50 +0000 Subject: [PATCH 13/28] fix: bound stranded-turn resume chains by the turn's step budget A turn cut for a queued tool-end message and resumed after the message was withdrawn started every resume under the full MAX_STREAM_STEPS ceiling; only failed starts were capped. Each cut now reports the steps the stream had left, the resume runs under that remainder, and at zero the ceiling ends the turn, so cut plus resumes share one turn's budget. --- src/common/orpc/schemas/stream.ts | 3 + .../agentSession.queueDispatch.test.ts | 141 +++++++++++++----- src/node/services/agentSession.ts | 52 +++++-- src/node/services/streamManager.test.ts | 24 ++- src/node/services/streamManager.ts | 47 ++++-- src/node/services/turnRequestBuilder.ts | 6 +- 6 files changed, 213 insertions(+), 60 deletions(-) diff --git a/src/common/orpc/schemas/stream.ts b/src/common/orpc/schemas/stream.ts index 0135b2cbff..d2ac08ce2e 100644 --- a/src/common/orpc/schemas/stream.ts +++ b/src/common/orpc/schemas/stream.ts @@ -341,6 +341,9 @@ export const StreamAbortEventSchema = z.object({ duration: z.number().optional(), // Model active at the abort (a configured fallback may differ from the requested model) model: z.string().optional(), + // Steps left under the stream's ceiling at the abort; a turn cut for a queued message + // resumes under this budget rather than a fresh one. + stepsRemaining: z.number().int().nonnegative().optional(), }) .optional() .meta({ diff --git a/src/node/services/agentSession.queueDispatch.test.ts b/src/node/services/agentSession.queueDispatch.test.ts index 20b3101adb..557ab9d87f 100644 --- a/src/node/services/agentSession.queueDispatch.test.ts +++ b/src/node/services/agentSession.queueDispatch.test.ts @@ -17,6 +17,12 @@ import type { HistoryService } from "./historyService"; import type { TurnStreamHandle } from "./streamManager"; const TEST_MODEL = "anthropic:claude-sonnet-4-5"; +/** Steps a cut stream reports as left; tests about the budget itself pass their own value. */ +const CUT_STEPS_REMAINING = 1_000; +const queuedStop = (modelString = TEST_MODEL) => ({ + modelString, + stepsRemaining: CUT_STEPS_REMAINING, +}); const WORKSPACE_TURN_CORRELATION = { type: "workspace-turn-task", taskHandleId: "wst_preparing", @@ -48,14 +54,15 @@ function streamStartEvent(workspaceId: string): Record { function streamAbortEvent( workspaceId: string, - abortReason: "system" | "user" | "queued-message" + abortReason: "system" | "user" | "queued-message", + stepsRemaining = CUT_STEPS_REMAINING ): Record { return { type: "stream-abort", workspaceId, messageId: "assistant-1", abortReason, - metadata: { duration: 1 }, + metadata: { duration: 1, stepsRemaining }, }; } @@ -819,7 +826,7 @@ describe("AgentSession queued message tool-call dispatch", () => { expect(request.hasQueuedMessages?.("tool-end")).toBe(true); // StreamManager stopped the loop for the queued wake; the wake is then withdrawn // (its output was consumed another way) before the stream-end drain dispatches it. - request.onQueuedMessageStop?.({ modelString: TEST_MODEL }); + request.onQueuedMessageStop?.(queuedStop()); wake.abort("monitor consumed"); aiEmitter.emit("stream-end", streamEndEvent(workspaceId)); @@ -860,7 +867,7 @@ describe("AgentSession queued message tool-call dispatch", () => { try { harness.queueCancelableWake(); - harness.latestRequest().onQueuedMessageStop?.({ modelString: TEST_MODEL }); + harness.latestRequest().onQueuedMessageStop?.(queuedStop()); aiEmitter.emit("stream-end", streamEndEvent(workspaceId)); expect(await waitForCondition(() => streamMessage.mock.calls.length === 2)).toBe(true); @@ -910,19 +917,22 @@ describe("AgentSession queued message tool-call dispatch", () => { try { // Each cut follows a completed step whose task_await consumed the monitor's wake (dogfood - // UAT: four sequential background+await calls in one prompt); the cap must not end it. - const strandTurn = () => { - harness.latestRequest().onQueuedMessageStop?.({ modelString: TEST_MODEL }); + // UAT: four sequential background+await calls in one prompt); the failed-start cap must + // not end it. What bounds the chain is the turn's step budget, run down by every cut. + const strandTurn = (stepsRemaining: number) => { + harness.latestRequest().onQueuedMessageStop?.({ modelString: TEST_MODEL, stepsRemaining }); harness.queueCancelableWake().abort("monitor consumed"); aiEmitter.emit("stream-end", streamEndEvent(workspaceId)); }; for (let resumes = 1; resumes <= 4; resumes += 1) { - strandTurn(); + const stepsRemaining = 5 - resumes; + strandTurn(stepsRemaining); expect(await waitForCondition(() => streamMessage.mock.calls.length === resumes + 1)).toBe( true ); expect(session.isBusy()).toBe(true); + expect(harness.latestRequest().stepBudget).toBe(stepsRemaining); } } finally { session.dispose(); @@ -930,6 +940,63 @@ describe("AgentSession queued message tool-call dispatch", () => { } }); + test("a resumed turn runs under the cut stream's remaining step budget", async () => { + const workspaceId = "queue-dispatch-stranded-step-budget"; + const harness = await createStreamingTurnHarness(workspaceId); + const { session, cleanup, aiEmitter, aiService, streamMessage } = harness; + const stopStream = spyOn(aiService, "stopStream").mockResolvedValue(Ok(undefined)); + + try { + expect(harness.latestRequest().stepBudget).toBeUndefined(); + + // Cut by the loop's stop condition: the resume inherits what that stream had left. + harness.latestRequest().onQueuedMessageStop?.({ modelString: TEST_MODEL, stepsRemaining: 7 }); + harness.queueCancelableWake().abort("monitor consumed"); + aiEmitter.emit("stream-end", streamEndEvent(workspaceId)); + expect(await waitForCondition(() => streamMessage.mock.calls.length === 2)).toBe(true); + expect(harness.latestRequest().stepBudget).toBe(7); + + // Cut by a provider-tool soft stop: the abort reports the budget the next stream gets. + session.queueMessage( + "follow up", + { model: TEST_MODEL, agentId: "exec" }, + { synthetic: true, dedupeKey: "wake:1" } + ); + aiEmitter.emit("tool-call-end", { + ...toolCallEndEvent(workspaceId), + toolName: "web_search", + providerExecuted: true, + }); + expect(stopStream).toHaveBeenCalledTimes(1); + expect(session.removeQueuedMessagesByDedupeKeyPrefix("wake:", "superseded")).toBe(1); + aiEmitter.emit("stream-abort", streamAbortEvent(workspaceId, "queued-message", 3)); + expect(await waitForCondition(() => streamMessage.mock.calls.length === 3)).toBe(true); + expect(harness.latestRequest().stepBudget).toBe(3); + + // A cut stream that spent its whole ceiling ended the turn: nothing is owed. + session.queueMessage( + "follow up", + { model: TEST_MODEL, agentId: "exec" }, + { synthetic: true, dedupeKey: "wake:2" } + ); + aiEmitter.emit("tool-call-end", { + ...toolCallEndEvent(workspaceId), + toolName: "web_search", + providerExecuted: true, + }); + expect(stopStream).toHaveBeenCalledTimes(2); + expect(session.removeQueuedMessagesByDedupeKeyPrefix("wake:", "superseded")).toBe(1); + aiEmitter.emit("stream-abort", streamAbortEvent(workspaceId, "queued-message", 0)); + expect(await waitForCondition(() => !session.isBusy())).toBe(true); + await new Promise((resolve) => setTimeout(resolve, 25)); + expect(streamMessage).toHaveBeenCalledTimes(3); + } finally { + stopStream.mockRestore(); + session.dispose(); + await cleanup(); + } + }); + test("caps resume attempts that never start a stream", async () => { const workspaceId = "queue-dispatch-stranded-cap"; let gateOpen = true; @@ -953,7 +1020,7 @@ describe("AgentSession queued message tool-call dispatch", () => { try { gateOpen = false; const wake = harness.queueCancelableWake(); - harness.latestRequest().onQueuedMessageStop?.({ modelString: TEST_MODEL }); + harness.latestRequest().onQueuedMessageStop?.(queuedStop()); wake.abort("monitor consumed"); aiEmitter.emit("stream-end", streamEndEvent(workspaceId)); expect(await waitForCondition(() => !session.isBusy())).toBe(true); @@ -1030,7 +1097,7 @@ describe("AgentSession queued message tool-call dispatch", () => { ).toBe(false); const wake = harness.queueCancelableWake(); - harness.latestRequest().onQueuedMessageStop?.({ modelString: TEST_MODEL }); + harness.latestRequest().onQueuedMessageStop?.(queuedStop()); // The entry is gone before the stream ends (dedupe removal / clearQueue shape). wake.abort("monitor consumed"); session.clearQueue("monitor consumed"); @@ -1068,7 +1135,7 @@ describe("AgentSession queued message tool-call dispatch", () => { const { session, cleanup } = harness; try { - harness.latestRequest().onQueuedMessageStop?.({ modelString: TEST_MODEL }); + harness.latestRequest().onQueuedMessageStop?.(queuedStop()); session.queueMessage("user follow-up", { model: TEST_MODEL, agentId: "exec" }); expect( session.claimWorkspaceTurnContinuation(WORKSPACE_TURN_CORRELATION, "assistant-1") @@ -1120,7 +1187,7 @@ describe("AgentSession queued message tool-call dispatch", () => { try { const wake = harness.queueCancelableWake(); - harness.latestRequest().onQueuedMessageStop?.({ modelString: TEST_MODEL }); + harness.latestRequest().onQueuedMessageStop?.(queuedStop()); wake.abort("monitor consumed"); // Owed whether the owner asks while the wake is still queued or after it is cleared. expect( @@ -1150,7 +1217,7 @@ describe("AgentSession queued message tool-call dispatch", () => { try { const wake = harness.queueCancelableWake(); - harness.latestRequest().onQueuedMessageStop?.({ modelString: TEST_MODEL }); + harness.latestRequest().onQueuedMessageStop?.(queuedStop()); wake.abort("monitor consumed"); aiEmitter.emit("stream-end", streamEndEvent(workspaceId)); @@ -1192,7 +1259,7 @@ describe("AgentSession queued message tool-call dispatch", () => { try { gateOpen = false; const wake = harness.queueCancelableWake(); - harness.latestRequest().onQueuedMessageStop?.({ modelString: TEST_MODEL }); + harness.latestRequest().onQueuedMessageStop?.(queuedStop()); wake.abort("monitor consumed"); aiEmitter.emit("stream-end", streamEndEvent(workspaceId)); @@ -1267,7 +1334,7 @@ describe("AgentSession queued message tool-call dispatch", () => { onCanceled: () => undefined, } ); - streamMessage.mock.calls[0]?.[0].onQueuedMessageStop?.({ modelString: TEST_MODEL }); + streamMessage.mock.calls[0]?.[0].onQueuedMessageStop?.(queuedStop()); controller.abort("monitor consumed"); aiEmitter.emit("stream-end", streamEndEvent(workspaceId)); expect(await waitForCondition(() => streamMessage.mock.calls.length === 2)).toBe(true); @@ -1345,7 +1412,7 @@ describe("AgentSession queued message tool-call dispatch", () => { onCanceled: () => undefined, } ); - streamMessage.mock.calls[0]?.[0].onQueuedMessageStop?.({ modelString: TEST_MODEL }); + streamMessage.mock.calls[0]?.[0].onQueuedMessageStop?.(queuedStop()); controller.abort("monitor consumed"); aiEmitter.emit("stream-end", streamEndEvent(workspaceId)); @@ -1370,7 +1437,7 @@ describe("AgentSession queued message tool-call dispatch", () => { try { harness.queueCancelableWake(); - harness.latestRequest().onQueuedMessageStop?.({ modelString: TEST_MODEL }); + harness.latestRequest().onQueuedMessageStop?.(queuedStop()); // WorkspaceService.interruptStream: hard stop, then restore the queue to the composer. expect((await session.interruptStream()).success).toBe(true); aiEmitter.emit("stream-abort", streamAbortEvent(workspaceId, "user")); @@ -1417,7 +1484,7 @@ describe("AgentSession queued message tool-call dispatch", () => { try { gateArmed = true; const wake = harness.queueCancelableWake(); - harness.latestRequest().onQueuedMessageStop?.({ modelString: TEST_MODEL }); + harness.latestRequest().onQueuedMessageStop?.(queuedStop()); wake.abort("monitor consumed"); aiEmitter.emit("stream-end", streamEndEvent(workspaceId)); @@ -1449,7 +1516,7 @@ describe("AgentSession queued message tool-call dispatch", () => { expect(original.delegatedToolNames).toEqual(["bash"]); const wake = harness.queueCancelableWake(); - original.onQueuedMessageStop?.({ modelString: TEST_MODEL }); + original.onQueuedMessageStop?.(queuedStop()); wake.abort("monitor consumed"); aiEmitter.emit("stream-end", streamEndEvent(workspaceId)); @@ -1504,7 +1571,7 @@ describe("AgentSession queued message tool-call dispatch", () => { }); const { session, aiEmitter } = harness; gateOpen = false; - harness.latestRequest().onQueuedMessageStop?.({ modelString: TEST_MODEL }); + harness.latestRequest().onQueuedMessageStop?.(queuedStop()); harness.queueCancelableWake().abort("monitor consumed"); session.clearQueue("monitor consumed"); expect(session.claimWorkspaceTurnContinuation(WORKSPACE_TURN_CORRELATION, "assistant-1")).toBe( @@ -1731,7 +1798,7 @@ describe("AgentSession queued message tool-call dispatch", () => { try { gateArmed = true; const wake = harness.queueCancelableWake(); - harness.latestRequest().onQueuedMessageStop?.({ modelString: TEST_MODEL }); + harness.latestRequest().onQueuedMessageStop?.(queuedStop()); wake.abort("monitor consumed"); aiEmitter.emit("stream-end", streamEndEvent(workspaceId)); expect(await waitForCondition(() => gateReached)).toBe(true); @@ -1805,7 +1872,7 @@ describe("AgentSession queued message tool-call dispatch", () => { onCanceled: () => undefined, } ); - streamMessage.mock.calls[0]?.[0].onQueuedMessageStop?.({ modelString: TEST_MODEL }); + streamMessage.mock.calls[0]?.[0].onQueuedMessageStop?.(queuedStop()); controller.abort("monitor consumed"); aiEmitter.emit("stream-end", streamEndEvent(workspaceId)); @@ -1830,7 +1897,7 @@ describe("AgentSession queued message tool-call dispatch", () => { try { const entry = harness.queueCancelableUnrelatedEntry(); - harness.latestRequest().onQueuedMessageStop?.({ modelString: TEST_MODEL }); + harness.latestRequest().onQueuedMessageStop?.(queuedStop()); // The queued entry is not this turn's continuation: the owner settles the delegated turn. expect( session.claimWorkspaceTurnContinuation(WORKSPACE_TURN_CORRELATION, "assistant-1") @@ -1860,7 +1927,7 @@ describe("AgentSession queued message tool-call dispatch", () => { try { harness.queueCancelableUnrelatedEntry(); - harness.latestRequest().onQueuedMessageStop?.({ modelString: TEST_MODEL }); + harness.latestRequest().onQueuedMessageStop?.(queuedStop()); // The owner settles the turn on this answer. The entry then leaves the queue without ever // dispatching (user clears the queue), which must not bring the settled turn back. expect( @@ -1891,7 +1958,7 @@ describe("AgentSession queued message tool-call dispatch", () => { try { harness.queueCancelableUnrelatedEntry(); - harness.latestRequest().onQueuedMessageStop?.({ modelString: TEST_MODEL }); + harness.latestRequest().onQueuedMessageStop?.(queuedStop()); session.clearQueue("queue cleared by user"); // Nothing supersedes the turn by the time the owner asks: it defers, and the resume runs. expect( @@ -1916,7 +1983,7 @@ describe("AgentSession queued message tool-call dispatch", () => { try { const wake = harness.queueCancelableWake(); - harness.latestRequest().onQueuedMessageStop?.({ modelString: TEST_MODEL }); + harness.latestRequest().onQueuedMessageStop?.(queuedStop()); // A wake inherits the turn's correlation when it sends, so the owner defers on it even // though the queued entry carries none; the continuation stays owed behind it. expect( @@ -1943,7 +2010,7 @@ describe("AgentSession queued message tool-call dispatch", () => { try { harness.queueCancelableUnrelatedEntry(); - harness.latestRequest().onQueuedMessageStop?.({ modelString: TEST_MODEL }); + harness.latestRequest().onQueuedMessageStop?.(queuedStop()); // The owner is still settling an earlier stream of this turn: that a later cut owes a // continuation proves the turn went on, so it defers without touching the marker. expect( @@ -1991,7 +2058,7 @@ describe("AgentSession queued message tool-call dispatch", () => { try { gateArmed = true; const wake = harness.queueCancelableWake(); - harness.latestRequest().onQueuedMessageStop?.({ modelString: TEST_MODEL }); + harness.latestRequest().onQueuedMessageStop?.(queuedStop()); wake.abort("monitor consumed"); aiEmitter.emit("stream-end", streamEndEvent(workspaceId)); expect(await waitForCondition(() => gateReached)).toBe(true); @@ -2024,7 +2091,7 @@ describe("AgentSession queued message tool-call dispatch", () => { const wake = harness.queueCancelableWake(); // StreamManager reports the request that was running at the stop: a configured fallback // model, not the refused primary this stream was sent with. - harness.latestRequest().onQueuedMessageStop?.({ modelString: "anthropic:claude-opus-4-8" }); + harness.latestRequest().onQueuedMessageStop?.(queuedStop("anthropic:claude-opus-4-8")); wake.abort("monitor consumed"); aiEmitter.emit("stream-end", streamEndEvent(workspaceId)); @@ -2054,7 +2121,7 @@ describe("AgentSession queued message tool-call dispatch", () => { } const wake = harness.queueCancelableWake(); - original.onQueuedMessageStop?.({ modelString: TEST_MODEL }); + original.onQueuedMessageStop?.(queuedStop()); wake.abort("monitor consumed"); aiEmitter.emit("stream-end", streamEndEvent(workspaceId)); @@ -2124,7 +2191,7 @@ describe("AgentSession queued message tool-call dispatch", () => { } ); const wake = harness.queueCancelableWake(); - harness.latestRequest().onQueuedMessageStop?.({ modelString: TEST_MODEL }); + harness.latestRequest().onQueuedMessageStop?.(queuedStop()); wake.abort("monitor consumed"); harness.aiEmitter.emit("stream-end", streamEndEvent(workspaceId)); expect(await waitForCondition(() => ioReached)).toBe(true); @@ -2200,7 +2267,7 @@ describe("AgentSession queued message tool-call dispatch", () => { } ); armed = true; - streamMessage.mock.calls[0]?.[0].onQueuedMessageStop?.({ modelString: TEST_MODEL }); + streamMessage.mock.calls[0]?.[0].onQueuedMessageStop?.(queuedStop()); controller.abort("monitor consumed"); aiEmitter.emit("stream-end", streamEndEvent(workspaceId)); @@ -2268,7 +2335,7 @@ describe("AgentSession queued message tool-call dispatch", () => { try { expect(harness.latestRequest().additionalSystemContext).toBe("live scratchpad"); const wake = harness.queueCancelableWake(); - harness.latestRequest().onQueuedMessageStop?.({ modelString: TEST_MODEL }); + harness.latestRequest().onQueuedMessageStop?.(queuedStop()); wake.abort("monitor consumed"); aiEmitter.emit("stream-end", streamEndEvent(workspaceId)); @@ -2297,7 +2364,7 @@ describe("AgentSession queued message tool-call dispatch", () => { expect(original.activeTurnThinkingOverride?.applied).toBeUndefined(); const wake = harness.queueCancelableWake(); - original.onQueuedMessageStop?.({ modelString: TEST_MODEL }); + original.onQueuedMessageStop?.(queuedStop()); wake.abort("monitor consumed"); aiEmitter.emit("stream-end", streamEndEvent(workspaceId)); @@ -2398,7 +2465,7 @@ describe("AgentSession queued message tool-call dispatch", () => { try { const wake = harness.queueCancelableWake(); - harness.latestRequest().onQueuedMessageStop?.({ modelString: TEST_MODEL }); + harness.latestRequest().onQueuedMessageStop?.(queuedStop()); wake.abort("monitor consumed"); // The loop ended normally; stream-end cleanup is parked in COMPLETING. goal.armDrain(); @@ -2474,7 +2541,7 @@ describe("AgentSession queued message tool-call dispatch", () => { onCanceled: () => undefined, } ); - streamMessage.mock.calls[0]?.[0].onQueuedMessageStop?.({ modelString: TEST_MODEL }); + streamMessage.mock.calls[0]?.[0].onQueuedMessageStop?.(queuedStop()); controller.abort("monitor consumed"); session.clearQueue("monitor consumed"); // The owner defers this stream-end on the strength of the advertised continuation. @@ -2549,7 +2616,7 @@ describe("AgentSession queued message tool-call dispatch", () => { onCanceled: () => undefined, } ); - streamMessage.mock.calls[0]?.[0].onQueuedMessageStop?.({ modelString: TEST_MODEL }); + streamMessage.mock.calls[0]?.[0].onQueuedMessageStop?.(queuedStop()); controller.abort("monitor consumed"); failFirstStream(); diff --git a/src/node/services/agentSession.ts b/src/node/services/agentSession.ts index 84db0f6689..1755c1be6d 100644 --- a/src/node/services/agentSession.ts +++ b/src/node/services/agentSession.ts @@ -244,6 +244,7 @@ interface AutoRetryResumeRequest { goalKind?: GoalSyntheticMessageKind; /** Goal identity matching goalKind; keeps retried streams goal-scoped. */ goalId?: string; + stepBudget?: number; } function stripGoalInterventionPolicy(options: SendMessageOptions): SendMessageOptions { @@ -369,6 +370,11 @@ interface StrandedTurnResume { options: SendMessageOptions; /** Assistant message id of the stream the cut ended (claimWorkspaceTurnContinuation). */ cutMessageId: string; + /** + * Steps the cut stream had left under its ceiling; the resumed stream runs under this budget + * so a chain of cuts and resumes spends one turn's steps, not a fresh cap per resume. + */ + stepBudget?: number; agentInitiated?: boolean; goalKind?: GoalSyntheticMessageKind; goalId?: string; @@ -388,6 +394,7 @@ function buildStrandedTurnResume(context: { /** Model that reached the cut: a configured fallback may differ from the requested one. */ modelString: string; cutMessageId: string; + stepBudget?: number; options?: SendMessageOptions; agentInitiated?: boolean; goalKind?: GoalSyntheticMessageKind; @@ -418,6 +425,7 @@ function buildStrandedTurnResume(context: { context.workspaceTurnMetadata ?? getWorkspaceTurnMuxMetadata(context.options?.muxMetadata), }, cutMessageId: context.cutMessageId, + ...(context.stepBudget != null ? { stepBudget: context.stepBudget } : {}), ...(context.agentInitiated != null ? { agentInitiated: context.agentInitiated } : {}), ...(context.goalKind != null ? { goalKind: context.goalKind } : {}), ...(context.goalId != null ? { goalId: context.goalId } : {}), @@ -969,6 +977,8 @@ export class AgentSession { goalKind?: GoalSyntheticMessageKind; /** Goal identity matching goalKind, so mid-stream compaction follow-ups stay goal-scoped. */ goalId?: string; + /** Step ceiling this stream runs under when it continues a cut turn (see StrandedTurnResume). */ + stepBudget?: number; workspaceTurnMetadata?: Extract; }; @@ -1391,7 +1401,8 @@ export class AgentSession { options: SendMessageOptions | undefined, agentInitiated?: boolean, goalKind?: GoalSyntheticMessageKind, - goalId?: string + goalId?: string, + stepBudget?: number ): void { if (!options) { this.lastAutoRetryResumeRequest = undefined; @@ -1403,6 +1414,7 @@ export class AgentSession { ...(agentInitiated === true ? { agentInitiated: true } : {}), ...(goalKind != null ? { goalKind } : {}), ...(goalId != null ? { goalId } : {}), + ...(stepBudget != null ? { stepBudget } : {}), }; } @@ -1431,6 +1443,7 @@ export class AgentSession { agentInitiated: request.agentInitiated === true ? true : undefined, goalKind: request.goalKind, goalId: request.goalId, + stepBudget: request.stepBudget, }); if (result.success) { if (!result.data.started) { @@ -4318,6 +4331,8 @@ export class AgentSession { * through admitStrandedTurnResume; a refusal reports `refusedBy`. */ revalidateAdmission?: boolean; + /** Step ceiling for the resumed stream when it continues a cut turn (StrandedTurnResume). */ + stepBudget?: number; } ): Promise> { this.assertNotDisposed("resumeStream"); @@ -4353,7 +4368,8 @@ export class AgentSession { optionsForStream, internal?.agentInitiated, internal?.goalKind, - internal?.goalId + internal?.goalId, + internal?.stepBudget ); // Claim the turn before any await: the admission gates below do I/O, and a manual send // entering meanwhile must see a busy session rather than start a stream this resume @@ -4440,7 +4456,8 @@ export class AgentSession { internal?.goalKind, internal?.goalId, turnThinkingOverride, - refuseStreamStart + refuseStreamStart, + internal?.stepBudget ); if (!result.success) { return result; @@ -5173,7 +5190,8 @@ export class AgentSession { activeTurnThinkingOverride?: ActiveTurnThinkingOverride, // Pull-based admission probe (goal state) with no push into abortSignal; checked wherever // the signal is, and by StreamManager right before the stream registers. - refuseStreamStart?: () => boolean + refuseStreamStart?: () => boolean, + stepBudget?: number ): Promise> { const isStartupAbortRequested = (): boolean => abortSignal?.aborted === true || refuseStreamStart?.() === true; @@ -5196,6 +5214,7 @@ export class AgentSession { openaiTruncationModeOverride, ...(goalKind != null ? { goalKind } : {}), ...(goalId != null ? { goalId } : {}), + ...(stepBudget != null ? { stepBudget } : {}), providersConfig, }; this.activeStreamUserMessageId = undefined; @@ -5382,6 +5401,7 @@ export class AgentSession { modelString, abortSignal, refuseStreamStart, + stepBudget, thinkingLevel: effectiveThinkingLevel, // Orthogonal to thinking level; buildRequestHeaders gates it per model. reasoningMode: options?.reasoningMode, @@ -5409,12 +5429,13 @@ export class AgentSession { disableWorkspaceAgents: options?.disableWorkspaceAgents, strictAgentResolution: options?.strictAgentResolution, hasQueuedMessages: this.hasQueuedMessages.bind(this), - onQueuedMessageStop: ({ modelString: stoppedModelString }) => { + onQueuedMessageStop: ({ modelString: stoppedModelString, stepsRemaining }) => { if (this.activeStreamContext != null && this.activeStreamMessageId != null) { this.strandedTurnResume = buildStrandedTurnResume({ ...this.activeStreamContext, modelString: stoppedModelString, cutMessageId: this.activeStreamMessageId, + stepBudget: stepsRemaining, thinkingLevelAtCut: activeTurnThinkingOverride?.pending ?? activeTurnThinkingOverride?.applied, }); @@ -5789,7 +5810,10 @@ export class AgentSession { context.agentInitiated, undefined, context.goalKind, - context.goalId + context.goalId, + undefined, + undefined, + context.stepBudget ); } finally { if (this.turnPhase === TurnPhase.PREPARING) { @@ -6259,7 +6283,8 @@ export class AgentSession { isQueuedProviderToolEndAbort && this.queuedProviderToolEndAbortInFlight, abortedStreamContext, payload.metadata?.model, - payload.messageId + payload.messageId, + payload.metadata?.stepsRemaining ); if (!dispatchedQueuedMessage) { this.setTurnPhase(TurnPhase.IDLE); @@ -7213,7 +7238,8 @@ export class AgentSession { isQueuedProviderToolEndAbort: boolean, abortedStreamContext: AgentSession["activeStreamContext"], abortedModelString: string | undefined, - abortedMessageId: string + abortedMessageId: string, + abortedStepsRemaining: number | undefined ): boolean { this.queuedProviderToolEndAbortInFlight = false; if (!isQueuedProviderToolEndAbort || this.deferQueuedFlushUntilAfterEdit) { @@ -7222,11 +7248,18 @@ export class AgentSession { // The soft stop was made on behalf of the queued message; if that message has been // withdrawn (or is withdrawn after dequeue), the interrupted turn must resume instead. - if (abortedStreamContext != null) { + // Only under the steps the cut stream had left: at zero the ceiling ended the turn, and an + // abort that reports no budget must not hand the resume a fresh one. + if ( + abortedStreamContext != null && + abortedStepsRemaining != null && + abortedStepsRemaining > 0 + ) { this.strandedTurnResume = buildStrandedTurnResume({ ...abortedStreamContext, modelString: abortedModelString ?? abortedStreamContext.modelString, cutMessageId: abortedMessageId, + stepBudget: abortedStepsRemaining, thinkingLevelAtCut: this.activeTurnThinkingOverride?.pending ?? this.activeTurnThinkingOverride?.applied, }); @@ -7289,6 +7322,7 @@ export class AgentSession { goalId: resume.goalId, abortSignal: inFlight.signal, revalidateAdmission: true, + stepBudget: resume.stepBudget, }) .then((result) => { if (!result.success) { diff --git a/src/node/services/streamManager.test.ts b/src/node/services/streamManager.test.ts index c2c29cd06a..4f7a0d1bd1 100644 --- a/src/node/services/streamManager.test.ts +++ b/src/node/services/streamManager.test.ts @@ -1228,7 +1228,8 @@ describe("StreamManager - stopWhen configuration", () => { type BuildStopWhenCondition = (request: { modelString: string; hasQueuedMessages?: (dispatchMode?: "tool-end" | "turn-end") => boolean; - onQueuedMessageStop?: (stop: { modelString: string }) => void; + onQueuedMessageStop?: (stop: { modelString: string; stepsRemaining: number }) => void; + stepBudget?: number; toolPolicy?: ToolPolicy; }) => StopWhenCondition[]; const TEST_MODEL_STRING = "anthropic:claude-sonnet-4-5"; @@ -1327,6 +1328,27 @@ describe("StreamManager - stopWhen configuration", () => { expect(stopsForQueuedMessage).toBe(1); }); + test("a step budget replaces the default ceiling and the cut reports what is left", () => { + const stops: number[] = []; + const [maxStepCondition, queuedMessageCondition] = buildStopWhenForTests()({ + modelString: TEST_MODEL_STRING, + hasQueuedMessages: () => true, + onQueuedMessageStop: ({ stepsRemaining }) => { + stops.push(stepsRemaining); + }, + stepBudget: 5, + }); + + expect(maxStepCondition({ steps: new Array(4).fill({}) })).toBe(false); + expect(maxStepCondition({ steps: new Array(5).fill({}) })).toBe(true); + + expect(queuedMessageCondition({ steps: new Array(2).fill({}) })).toBe(true); + expect(stops).toEqual([3]); + // At the budget the ceiling ends the turn; the cut owes nothing. + expect(queuedMessageCondition({ steps: new Array(5).fill({}) })).toBe(true); + expect(stops).toEqual([3]); + }); + const requiredToolCases: Array<{ name: string; toolPolicy: ToolPolicy; diff --git a/src/node/services/streamManager.ts b/src/node/services/streamManager.ts index e2d6f0107b..d9b7663589 100644 --- a/src/node/services/streamManager.ts +++ b/src/node/services/streamManager.ts @@ -247,7 +247,8 @@ interface StreamRequestOptions { callSettingsOverrides?: ResolvedCallSettingsOverrides; toolPolicy?: ToolPolicy; hasQueuedMessages?: (dispatchMode?: "tool-end" | "turn-end") => boolean; - onQueuedMessageStop?: (stop: { modelString: string }) => void; + onQueuedMessageStop?: (stop: { modelString: string; stepsRemaining: number }) => void; + stepBudget?: number; headers?: Record; onChunk?: StreamTextOnChunk; onStepMessages?: (messages: ModelMessage[]) => void; @@ -301,9 +302,16 @@ interface StreamRequestConfig { * Invoked when the loop stops on behalf of a queued tool-end message (and not * because a required tool completed). The session uses it to resume the turn * if that queued message is later withdrawn instead of starting a turn. Carries the - * model that reached the cut, which a configured fallback may have swapped mid-turn. + * model that reached the cut, which a configured fallback may have swapped mid-turn, and + * the steps left under this request's budget for the resumed stream to run under. */ - onQueuedMessageStop?: (stop: { modelString: string }) => void; + onQueuedMessageStop?: (stop: { modelString: string; stepsRemaining: number }) => void; + /** + * Step ceiling for this stream instead of MAX_STREAM_STEPS. A stream resuming a turn cut + * for a queued message inherits the cut stream's remaining steps, so cut plus resumes + * share one turn's ceiling; without it, every resume would restart the full cap. + */ + stepBudget?: number; /** Optional hook for callers that need chunk-level visibility during streaming. */ onChunk?: StreamTextOnChunk; /** Optional hook for callers that need the live prepared step transcript. */ @@ -637,6 +645,10 @@ interface WorkspaceStreamInfo { // original start timestamp even after they gain output. toolCompletionTimestamps: Map; + // Steps started so far, the in-progress one included: the budget left at an abort counts a + // step the model already began. + stepCount: number; + // Workflow tools can create the durable run before their stream part is stored. Keep the exact // attachment and apply it as soon as the matching dynamic-tool part lands. pendingWorkflowRunAttachments: Map; @@ -1952,6 +1964,10 @@ export class StreamManager { providerMetadata, contextProviderMetadata, model: streamInfo.model, + stepsRemaining: Math.max( + 0, + (streamInfo.request.stepBudget ?? MAX_STREAM_STEPS) - Math.max(1, streamInfo.stepCount) + ), }, abortReason, abandonPartial, @@ -2091,6 +2107,7 @@ export class StreamManager { toolPolicy, hasQueuedMessages, onQueuedMessageStop, + stepBudget, headers, onChunk, onStepMessages, @@ -2143,6 +2160,7 @@ export class StreamManager { Object.keys(streamCallSettings).length > 0 ? streamCallSettings : undefined, hasQueuedMessages, onQueuedMessageStop, + stepBudget, onChunk, onStepMessages, toolPolicy, @@ -2157,9 +2175,10 @@ export class StreamManager { private createStopWhenCondition( request: Pick< StreamRequestConfig, - "hasQueuedMessages" | "onQueuedMessageStop" | "toolPolicy" | "modelString" + "hasQueuedMessages" | "onQueuedMessageStop" | "toolPolicy" | "modelString" | "stepBudget" > ): Array> { + const stepBudget = request.stepBudget ?? MAX_STREAM_STEPS; // Completion-tool stop check: completion/routing tools use explicit // success/ok markers (agent_report, propose_plan). // When a marker is present, respect it — success:false means the tool @@ -2207,18 +2226,19 @@ export class StreamManager { return false; } // The step cap and a successful required tool result each end the turn on their - // own; only a stop made purely for the queued message may need resuming later. - if (state.steps.length < MAX_STREAM_STEPS && !hasSuccessfulRequiredToolResult(state)) { - request.onQueuedMessageStop?.({ modelString: request.modelString }); + // own; only a stop made purely for the queued message may need resuming later. A cut + // spends at least one step, so a chain of cuts and resumes always runs the budget down. + const stepsSpent = Math.max(1, state.steps.length); + if (stepsSpent < stepBudget && !hasSuccessfulRequiredToolResult(state)) { + request.onQueuedMessageStop?.({ + modelString: request.modelString, + stepsRemaining: stepBudget - stepsSpent, + }); } return true; }; - return [ - stepCountIs(MAX_STREAM_STEPS), - hasQueuedToolEndMessage, - hasSuccessfulRequiredToolResult, - ]; + return [stepCountIs(stepBudget), hasQueuedToolEndMessage, hasSuccessfulRequiredToolResult]; } /** @@ -2444,6 +2464,7 @@ export class StreamManager { startTime, lastPartTimestamp: startTime, toolCompletionTimestamps: new Map(), + stepCount: 0, pendingWorkflowRunAttachments: new Map(), pendingNestedCalls: new Map(), pendingToolExecutionStarts: new Map(), @@ -3196,6 +3217,7 @@ export class StreamManager { toolPolicy: streamInfo.request.toolPolicy, hasQueuedMessages: streamInfo.request.hasQueuedMessages, onQueuedMessageStop: streamInfo.request.onQueuedMessageStop, + stepBudget: streamInfo.request.stepBudget, headers: prepared.data.headers, onChunk: streamInfo.request.onChunk, onStepMessages: streamInfo.request.onStepMessages, @@ -3412,6 +3434,7 @@ export class StreamManager { switch (part.type) { case "start-step": { streamInfo.currentStepStartIndex = streamInfo.parts.length; + streamInfo.stepCount += 1; break; } diff --git a/src/node/services/turnRequestBuilder.ts b/src/node/services/turnRequestBuilder.ts index 7428fca977..9489975507 100644 --- a/src/node/services/turnRequestBuilder.ts +++ b/src/node/services/turnRequestBuilder.ts @@ -274,7 +274,9 @@ export interface StreamMessageOptions { disableWorkspaceAgents?: boolean; hasQueuedMessages?: (dispatchMode?: "tool-end" | "turn-end") => boolean; /** Fires when the model loop stops solely on behalf of a queued tool-end message. */ - onQueuedMessageStop?: (stop: { modelString: string }) => void; + onQueuedMessageStop?: (stop: { modelString: string; stepsRemaining: number }) => void; + /** Step ceiling for this stream; a stream resuming a cut turn runs under the cut's remainder. */ + stepBudget?: number; /** * Pull-based startup refusal (a goal admission probe with no push into abortSignal), rechecked * by StreamManager right before the stream registers. @@ -745,6 +747,7 @@ export class TurnRequestBuilder { disableWorkspaceAgents, hasQueuedMessages, onQueuedMessageStop, + stepBudget, refuseStreamStart, openaiTruncationModeOverride, muxMetadata, @@ -2867,6 +2870,7 @@ export class TurnRequestBuilder { providedStreamToken: streamToken, hasQueuedMessages, onQueuedMessageStop, + stepBudget, refuseStreamStart, workspaceName: metadata.name, thinkingLevel: streamThinkingLevel, From d56f87d7d460cb526e608ae87bc33b2957bc6355 Mon Sep 17 00:00:00 2001 From: Michael Suchacz <203725896+ibetitsmike@users.noreply.github.com> Date: Thu, 3 Sep 2026 18:55:32 +0000 Subject: [PATCH 14/28] Hold the step budget across in-stream loop restarts and skip withdrawn queue entries A fallback swap, empty-output retry, and previous_response_id retry each start a new SDK loop that counts steps from zero; they now inherit the stream's remaining budget (restartStepBudget) and none runs once it is spent. Auto-retry of a resume that was admitted under revalidation repeats the goal and delegated-turn admission and abandons when refused. MessageQueue's next-entry readers (wake, workspace-turn continuation, cut candidate) look past withdrawn entries like the dispatch mode already did, and the continuation claim treats a queue holding only withdrawn entries as empty. --- .../agentSession.queueDispatch.test.ts | 130 ++++++++++++++++++ src/node/services/agentSession.ts | 23 +++- src/node/services/messageQueue.test.ts | 31 +++++ src/node/services/messageQueue.ts | 25 ++-- src/node/services/streamManager.test.ts | 96 +++++++++++++ src/node/services/streamManager.ts | 40 +++++- 6 files changed, 328 insertions(+), 17 deletions(-) diff --git a/src/node/services/agentSession.queueDispatch.test.ts b/src/node/services/agentSession.queueDispatch.test.ts index 1d0d7914b0..b57ca2e03b 100644 --- a/src/node/services/agentSession.queueDispatch.test.ts +++ b/src/node/services/agentSession.queueDispatch.test.ts @@ -9,6 +9,7 @@ import { GOAL_CONTINUATION_KIND } from "@/constants/goals"; import type { WorkspaceGoalService } from "./workspaceGoalService"; import { createAgentSessionHarness, + createFailedTurnHandle, createStartedTurnHandle, type AgentSessionHarnessOptions, } from "./agentSession.testHarness"; @@ -1471,6 +1472,101 @@ describe("AgentSession queued message tool-call dispatch", () => { } }); + test("an auto-retry of a stranded goal resume is admitted by the goal again", async () => { + const workspaceId = "queue-dispatch-stranded-goal-retry-admission"; + const buildGoalRedispatchAdmission = mock( + (): ReturnType => + Promise.resolve({ admissible: true, admissionStale: () => false }) + ); + const workspaceGoalService = { + assertPricedModelForBudgetedGoal: mock(() => Promise.resolve(Ok(undefined))), + buildGoalRedispatchAdmission, + recordStreamAccounting: mock(() => Promise.resolve()), + applyPendingAfterStreamEnd: mock(() => Promise.resolve()), + requestContinuationAfterStreamEnd: mock(() => Promise.resolve()), + recordStreamStarted: mock(() => Promise.resolve()), + syncGoalModeWithChatTail: mock(() => Promise.resolve(null)), + completeGoalFromSilentContinuation: mock(() => Promise.resolve(false)), + } as unknown as WorkspaceGoalService; + const aiEmitter = new EventEmitter(); + let streams = 0; + const streamMessage = mock((_options: StreamMessageOptions) => { + streams += 1; + aiEmitter.emit("stream-start", streamStartEvent(workspaceId)); + // The resumed stream (the second) fails with a retryable error. + return Promise.resolve( + Ok( + streams === 2 + ? createFailedTurnHandle("assistant-2", { + error: "provider closed the connection", + errorType: "api", + }) + : createStartedTurnHandle(`assistant-${streams}`) + ) + ); + }); + const { session, cleanup, historyService } = await createAgentSessionHarness({ + workspaceId, + aiEmitter, + aiServiceOverrides: { streamMessage: streamMessage as unknown as AIService["streamMessage"] }, + workspaceGoalService, + }); + const retryEvents: string[] = []; + const unsubscribe = session.onChatEvent((event) => { + if (event.message.type === "auto-retry-abandoned") { + retryEvents.push(event.message.type); + } + }); + + try { + await historyService.appendToHistory( + workspaceId, + createMuxMessage("user-0", "user", "keep going", { timestamp: Date.now() }) + ); + const resumed = await session.resumeStream( + { model: TEST_MODEL, agentId: "exec" }, + { agentInitiated: true, goalKind: GOAL_CONTINUATION_KIND, goalId: "goal-1" } + ); + expect(resumed.success).toBe(true); + await historyService.appendToHistory( + workspaceId, + createMuxMessage("assistant-1", "assistant", "ran the checks", { timestamp: Date.now() }) + ); + + const controller = new AbortController(); + session.queueMessage( + "Background monitor wake", + { model: TEST_MODEL, agentId: "exec" }, + { + synthetic: true, + agentInitiated: true, + cancelState: { canceledBeforeAcceptance: false }, + cancelSignal: controller.signal, + onCanceled: () => undefined, + } + ); + streamMessage.mock.calls[0]?.[0].onQueuedMessageStop?.(queuedStop()); + controller.abort("monitor consumed"); + aiEmitter.emit("stream-end", streamEndEvent(workspaceId)); + expect(await waitForCondition(() => streamMessage.mock.calls.length === 2)).toBe(true); + expect(buildGoalRedispatchAdmission).toHaveBeenCalledTimes(1); + + // The goal is paused during the retry backoff: the retry asks the goal again and, refused, + // starts nothing instead of restarting tool-enabled work after the pause. + buildGoalRedispatchAdmission.mockImplementation(() => Promise.resolve({ admissible: false })); + expect( + await waitForCondition(() => buildGoalRedispatchAdmission.mock.calls.length === 2, 4_000) + ).toBe(true); + expect(await waitForCondition(() => retryEvents.length === 1)).toBe(true); + expect(streamMessage).toHaveBeenCalledTimes(2); + expect(session.isBusy()).toBe(false); + } finally { + unsubscribe(); + session.dispose(); + await cleanup(); + } + }); + test("drops a stranded goal continuation the goal no longer admits", async () => { const workspaceId = "queue-dispatch-stranded-goal-paused"; const workspaceGoalService = { @@ -2112,6 +2208,40 @@ describe("AgentSession queued message tool-call dispatch", () => { } }); + test("a withdrawn wake ahead of an unrelated entry is not the delegated turn's continuation", async () => { + const workspaceId = "queue-dispatch-stranded-delegated-withdrawn-wake-ahead"; + const harness = await createStreamingTurnHarness(workspaceId, { + sendOptions: { muxMetadata: WORKSPACE_TURN_CORRELATION }, + }); + const { session, cleanup, aiEmitter, streamMessage } = harness; + + try { + const wake = harness.queueCancelableWake(); + const entry = harness.queueCancelableUnrelatedEntry(); + harness.latestRequest().onQueuedMessageStop?.(queuedStop()); + // The wake's monitor was consumed before the owner asks; the live entry behind it is the + // cutter, and it does not continue this turn. + wake.abort("monitor consumed"); + expect( + session.claimWorkspaceTurnContinuation(WORKSPACE_TURN_CORRELATION, "assistant-1") + ).toBe(false); + aiEmitter.emit("stream-end", streamEndEvent(workspaceId)); + // The owner settled the turn on that answer: the entry leaving before acceptance must not + // bring it back. + entry.abort("superseded"); + + expect(await waitForCondition(() => !session.isBusy())).toBe(true); + await new Promise((resolve) => setTimeout(resolve, 25)); + expect(streamMessage).toHaveBeenCalledTimes(1); + expect( + session.claimWorkspaceTurnContinuation(WORKSPACE_TURN_CORRELATION, "assistant-1") + ).toBe(false); + } finally { + session.dispose(); + await cleanup(); + } + }); + test("an older stream-end's claim does not void a newer cut's continuation", async () => { const workspaceId = "queue-dispatch-stranded-delegated-stale-claim"; const harness = await createStreamingTurnHarness(workspaceId, { diff --git a/src/node/services/agentSession.ts b/src/node/services/agentSession.ts index 989b68dbdd..6d62a0dee8 100644 --- a/src/node/services/agentSession.ts +++ b/src/node/services/agentSession.ts @@ -245,6 +245,8 @@ interface AutoRetryResumeRequest { /** Goal identity matching goalKind; keeps retried streams goal-scoped. */ goalId?: string; stepBudget?: number; + /** The retried stream was admitted under resumeStream's revalidation; the retry repeats it. */ + revalidateAdmission?: boolean; } function stripGoalInterventionPolicy(options: SendMessageOptions): SendMessageOptions { @@ -1402,7 +1404,8 @@ export class AgentSession { agentInitiated?: boolean, goalKind?: GoalSyntheticMessageKind, goalId?: string, - stepBudget?: number + stepBudget?: number, + revalidateAdmission?: boolean ): void { if (!options) { this.lastAutoRetryResumeRequest = undefined; @@ -1415,6 +1418,7 @@ export class AgentSession { ...(goalKind != null ? { goalKind } : {}), ...(goalId != null ? { goalId } : {}), ...(stepBudget != null ? { stepBudget } : {}), + ...(revalidateAdmission === true ? { revalidateAdmission: true } : {}), }; } @@ -1444,8 +1448,17 @@ export class AgentSession { goalKind: request.goalKind, goalId: request.goalId, stepBudget: request.stepBudget, + revalidateAdmission: request.revalidateAdmission, }); if (result.success) { + if (result.data.refusedBy != null) { + // The goal or delegated turn ended during the backoff; no retry can readmit it. + this.emitRetryEvent({ + type: "auto-retry-abandoned", + reason: `${result.data.refusedBy}_admission_refused`, + }); + return; + } if (!result.data.started) { // resumeStream can defer when a turn is still PREPARING/COMPLETING. // Treat this as retriable so auto-retry keeps progressing instead of @@ -4365,7 +4378,8 @@ export class AgentSession { internal?.agentInitiated, internal?.goalKind, internal?.goalId, - internal?.stepBudget + internal?.stepBudget, + internal?.revalidateAdmission ); // Claim the turn before any await: the admission gates below do I/O, and a manual send // entering meanwhile must see a busy session rather than start a stream this resume @@ -7063,8 +7077,9 @@ export class AgentSession { if (this.owedStrandedTurnResume() == null) { return false; } - // Nothing else queued to take the turn: the stranded resume will carry this correlation. - if (this.messageQueue.isEmpty() && !this.dispatchingQueuedEntry) { + // Nothing dispatchable queued to take the turn (withdrawn entries drain as no-ops): the + // stranded resume will carry this correlation. + if (this.messageQueue.getNextDispatchableMode() == null && !this.dispatchingQueuedEntry) { return true; } // A queued or dispatching entry that failed the checks above supersedes the turn; the diff --git a/src/node/services/messageQueue.test.ts b/src/node/services/messageQueue.test.ts index 82106e398c..5729761ff6 100644 --- a/src/node/services/messageQueue.test.ts +++ b/src/node/services/messageQueue.test.ts @@ -604,6 +604,37 @@ describe("MessageQueue", () => { ).toBe(false); }); + it("reads the next entry past withdrawn ones for wake, correlation, and cut candidate", () => { + const withdrawn = new AbortController(); + queue.add( + "withdrawn wake", + { + model: "gpt-4", + agentId: "exec", + muxMetadata: { type: "bash-monitor-wake", records: [] }, + queueDispatchMode: "tool-end", + }, + { synthetic: true, agentInitiated: true, cancelSignal: withdrawn.signal } + ); + expect(queue.isNextEntryBashMonitorWake()).toBe(true); + + withdrawn.abort(); + expect(queue.isNextEntryBashMonitorWake()).toBe(false); + expect(queue.getNextQueueCutCandidate()).toBeUndefined(); + + queue.add("Follow up", { + model: "gpt-4", + agentId: "exec", + muxMetadata: metadata, + queueDispatchMode: "turn-end", + }); + expect(queue.isNextEntryBashMonitorWake()).toBe(false); + expect( + queue.hasNextWorkspaceTurnContinuation("wst_followup", "parent-workspace", "turn-1") + ).toBe(true); + expect(queue.getNextQueueCutCandidate()?.dispatchMode).toBe("turn-end"); + }); + it("exposes the head entry's metadata and dispatch mode as the queue-cut candidate", () => { expect(queue.getNextQueueCutCandidate()).toBeUndefined(); diff --git a/src/node/services/messageQueue.ts b/src/node/services/messageQueue.ts index 93593b9f5d..c3b29379c3 100644 --- a/src/node/services/messageQueue.ts +++ b/src/node/services/messageQueue.ts @@ -272,12 +272,17 @@ export class MessageQueue { } /** - * Dispatch mode of the first entry whose cancel signal has not fired, or undefined - * when none remains. Aborted entries still drain FIFO (as no-ops that fire - * onCanceled), but they are not pending work and must not arm a tool-end stop. + * The first entry whose cancel signal has not fired. Aborted entries still drain FIFO (as + * no-ops that fire onCanceled), but they are not pending work: they must not arm a tool-end + * stop, count as a turn's continuation, or be attributed a cut. */ + private nextDispatchableEntry(): QueueEntry | undefined { + return this.entries.find((entry) => entry.cancelSignal?.aborted !== true); + } + + /** Dispatch mode of the next dispatchable entry, or undefined when none remains. */ getNextDispatchableMode(): QueueDispatchMode | undefined { - return this.entries.find((entry) => entry.cancelSignal?.aborted !== true)?.dispatchMode; + return this.nextDispatchableEntry()?.dispatchMode; } /** @@ -306,14 +311,14 @@ export class MessageQueue { } /** - * Whether the next entry continues the exact workspace turn correlation. + * Whether the next dispatchable entry continues the exact workspace turn correlation. */ hasNextWorkspaceTurnContinuation( taskHandleId: string, ownerWorkspaceId: string, turnId: string ): boolean { - const metadata = this.entries[0]?.muxMetadata; + const metadata = this.nextDispatchableEntry()?.muxMetadata; return ( isWorkspaceTurnMetadata(metadata) && metadata.taskHandleId === taskHandleId && @@ -323,7 +328,7 @@ export class MessageQueue { } /** - * FIFO head entry's cut-attribution view: its first muxMetadata plus dispatch mode. + * Next dispatchable entry's cut-attribution view: its first muxMetadata plus dispatch mode. * * Soundness of metadata-based cut attribution rests on the sealing invariant * (see class docblock): workspace-turn entries are sealed at add time and @@ -334,7 +339,7 @@ export class MessageQueue { getNextQueueCutCandidate(): | { muxMetadata: unknown; dispatchMode: QueueDispatchMode } | undefined { - const head = this.entries[0]; + const head = this.nextDispatchableEntry(); if (head == null) { return undefined; } @@ -342,13 +347,13 @@ export class MessageQueue { } /** - * Whether the next entry to dispatch is a bash-monitor wake. Wake sends are + * Whether the next dispatchable entry is a bash-monitor wake. Wake sends are * the only queued input that continues an open delegated workspace turn * (see AgentSession.inheritOpenWorkspaceTurnMetadata); any other head entry * supersedes the turn when it dispatches. */ isNextEntryBashMonitorWake(): boolean { - const muxMetadata = this.entries[0]?.muxMetadata; + const muxMetadata = this.nextDispatchableEntry()?.muxMetadata; if (typeof muxMetadata !== "object" || muxMetadata === null) return false; return (muxMetadata as Record).type === "bash-monitor-wake"; } diff --git a/src/node/services/streamManager.test.ts b/src/node/services/streamManager.test.ts index 4fd1a275ce..f5f09727a8 100644 --- a/src/node/services/streamManager.test.ts +++ b/src/node/services/streamManager.test.ts @@ -256,6 +256,7 @@ function createStreamInfoForTests( didRetryPreviousResponseIdAtStep: false, receivedTerminalEvent: false, currentStepStartIndex: 0, + stepCount: 0, stepTracker: {}, ...overrides, }; @@ -3732,6 +3733,100 @@ describe("StreamManager - empty stream completions", () => { expect(swappedRequest.system).toBe("fallback system"); }); + test("a fallback hop runs under the refused stream's remaining step budget, none once spent", async () => { + const runRefusalWithFallback = async (stepBudget: number) => { + const streamManager = new StreamManager(historyService); + const errorEvents: unknown[] = []; + onTurnEngineEvent(streamManager, "error", (data) => errorEvents.push(data)); + Reflect.set(streamManager, "tokenTracker", { + setModel: () => Promise.resolve(undefined), + countTokens: () => Promise.resolve(0), + }); + + const workspaceId = `fallback-step-budget-${stepBudget}-workspace`; + const messageId = "fallback-step-budget-message"; + await appendPartialAssistantForTests(workspaceId, messageId, 1); + const processStreamWithCleanup = getProcessStreamWithCleanupForTests(streamManager); + + const createStreamResult = mock((_request: { stepBudget?: number }) => + createStreamResultForTests( + (async function* () { + await Promise.resolve(); + yield { type: "text-delta", text: "fallback answer" }; + yield { type: "finish", finishReason: "stop" }; + })(), + { inputTokens: 5, outputTokens: 3, totalTokens: 8 } + ) + ); + expect(Reflect.set(streamManager, "createStreamResult", createStreamResult)).toBe(true); + const prepare = mock((nextModelString: string) => + Promise.resolve( + Ok({ + model: createTestLanguageModel("fallback-model"), + modelString: nextModelString, + messages: [], + system: "fallback system", + tools: {}, + thinkingLevel: "off", + }) + ) + ); + + const startTime = Date.now() - 250; + const streamInfo = createStreamInfoForTests({ + streamResult: createStreamResultForTests( + (async function* () { + await Promise.resolve(); + // The refused step is a step the turn spent. + yield { type: "start-step" }; + yield { + type: "finish-step", + usage: { inputTokens: 30, outputTokens: 0, totalTokens: 30 }, + }; + yield { type: "finish", finishReason: "content-filter", rawFinishReason: "refusal" }; + })(), + { inputTokens: 30, outputTokens: 0, totalTokens: 30 } + ), + messageId, + startTime, + lastPartTimestamp: startTime, + model: KNOWN_MODELS.SONNET.id, + metadataModel: KNOWN_MODELS.SONNET.id, + historySequence: 1, + initialMetadata: { agentId: "plan" }, + runtime, + request: { + model: createTestLanguageModel("refused-model"), + messages: [], + providerOptions: undefined, + stepBudget, + }, + modelFallback: { + options: { chain: [KNOWN_MODELS.GPT.id], prepare }, + requestedModel: KNOWN_MODELS.SONNET.id, + refusedModels: [], + original: { maxOutputTokens: undefined }, + }, + }); + + await processStreamWithCleanup.call(streamManager, workspaceId, streamInfo, 1); + return { errorEvents, prepare, createStreamResult }; + }; + + // Three steps allowed and one spent on the refusal: the hop's own loop gets the other two. + const hop = await runRefusalWithFallback(3); + expect(hop.errorEvents).toHaveLength(0); + expect(hop.createStreamResult).toHaveBeenCalledTimes(1); + expect(hop.createStreamResult.mock.calls[0]?.[0].stepBudget).toBe(2); + + // The refusal spent the last step: the ceiling ended the turn, so no hop is bought. + const spent = await runRefusalWithFallback(1); + expect(spent.prepare).not.toHaveBeenCalled(); + expect(spent.createStreamResult).not.toHaveBeenCalled(); + expect(spent.errorEvents).toHaveLength(1); + expect(spent.errorEvents[0]).toMatchObject({ errorType: "model_refusal" }); + }); + test("partial refusal with a configured fallback continues from cloned partial output", async () => { const streamManager = new StreamManager(historyService); const errorEvents: unknown[] = []; @@ -5297,6 +5392,7 @@ describe("StreamManager - previousResponseId recovery", () => { stepTracker: { latestMessages: stepMessages }, didRetryPreviousResponseIdAtStep: false, currentStepStartIndex: 1, + stepCount: 1, request: { model, messages: [{ role: "user", content: "original" }], diff --git a/src/node/services/streamManager.ts b/src/node/services/streamManager.ts index 44b5c10a96..6ecce1199b 100644 --- a/src/node/services/streamManager.ts +++ b/src/node/services/streamManager.ts @@ -645,8 +645,9 @@ interface WorkspaceStreamInfo { // original start timestamp even after they gain output. toolCompletionTimestamps: Map; - // Steps started so far, the in-progress one included: the budget left at an abort counts a - // step the model already began. + // Steps started by the current SDK loop, the in-progress one included: the budget left at + // an abort counts a step the model already began. Reset when the loop restarts under + // request.stepBudget (restartStepBudget). stepCount: number; // Workflow tools can create the durable run before their stream part is stored. Keep the exact @@ -3155,6 +3156,14 @@ export class StreamManager { }; } + const stepBudget = this.restartStepBudget(streamInfo); + if (stepBudget == null) { + return { + kind: "terminal", + terminalNote: "Model fallback was skipped because the turn's step budget is spent.", + }; + } + const workspaceLog = this.getWorkspaceLogger(workspaceId, streamInfo); // A throw out of prepare() must not escape to the generic stream-error path: // it would be categorized as a retryable api/unknown error and re-enter the @@ -3217,7 +3226,7 @@ export class StreamManager { toolPolicy: streamInfo.request.toolPolicy, hasQueuedMessages: streamInfo.request.hasQueuedMessages, onQueuedMessageStop: streamInfo.request.onQueuedMessageStop, - stepBudget: streamInfo.request.stepBudget, + stepBudget, headers: prepared.data.headers, onChunk: streamInfo.request.onChunk, onStepMessages: streamInfo.request.onStepMessages, @@ -3308,12 +3317,23 @@ export class StreamManager { // refused model (e.g. an OpenAI WS transport socket) would leak per hop. runLanguageModelCleanup(streamInfo.request.model); streamInfo.request = nextRequest; + streamInfo.stepCount = 0; streamInfo.streamResult = nextStreamResult; await this.tokenTracker.setModel(streamInfo.model, streamInfo.metadataModel); return { kind: "swapped" }; } + /** + * Step ceiling for an SDK loop restarted under this stream (fallback swap, same-model + * retry): the new loop counts its steps from zero, so it inherits what the stream has + * left rather than a fresh ceiling. Undefined when the budget is spent. + */ + private restartStepBudget(streamInfo: WorkspaceStreamInfo): number | undefined { + const remaining = (streamInfo.request.stepBudget ?? MAX_STREAM_STEPS) - streamInfo.stepCount; + return remaining > 0 ? remaining : undefined; + } + private async handleTruncatedStreamCompletion( workspaceId: WorkspaceId, streamInfo: WorkspaceStreamInfo @@ -3364,6 +3384,11 @@ export class StreamManager { return false; } + const stepBudget = this.restartStepBudget(streamInfo); + if (stepBudget == null) { + return false; + } + const workspaceLog = this.getWorkspaceLogger(workspaceId, streamInfo); workspaceLog.warn("Retrying stream after empty-output completion", { messageId: streamInfo.messageId, @@ -3379,6 +3404,8 @@ export class StreamManager { workspaceLog, }); streamInfo.currentStepStartIndex = 0; + streamInfo.request = { ...streamInfo.request, stepBudget }; + streamInfo.stepCount = 0; streamInfo.streamResult = this.createStreamResult( streamInfo.request, streamInfo.abortController, @@ -4542,6 +4569,11 @@ export class StreamManager { return false; } + const stepBudget = this.restartStepBudget(streamInfo); + if (stepBudget == null) { + return false; + } + const workspaceLog = this.getWorkspaceLogger(workspaceId, streamInfo); this.recordLostResponseIdIfApplicable(workspaceId, error, streamInfo, workspaceLog); @@ -4571,7 +4603,9 @@ export class StreamManager { ...streamInfo.request, ...(stepMessages ? { messages: stepMessages } : {}), providerOptions, + stepBudget, }; + streamInfo.stepCount = 0; streamInfo.streamResult = this.createStreamResult( streamInfo.request, streamInfo.abortController, From 8848517e85e3ef8fd5a55ba963e1a061489c53c9 Mon Sep 17 00:00:00 2001 From: Michael Suchacz <203725896+ibetitsmike@users.noreply.github.com> Date: Thu, 3 Sep 2026 19:19:30 +0000 Subject: [PATCH 15/28] Forfeit the stranded continuation on task hard stops and refuse resumes during a rename A hard stop that finds the cut stream already completed gives the delegated turn's owner no stream event, so the queue clear now forfeits the marker (settling the turn the owner deferred on) instead of only withdrawing it. The stranded resume's workspace admission also refuses while the workspace is being renamed, matching the public send and resume entry points. --- .../agentSession.queueDispatch.test.ts | 24 +++++++++++++++ src/node/services/agentSession.ts | 5 +++- src/node/services/workspaceService.test.ts | 29 +++++++++++++++++++ src/node/services/workspaceService.ts | 8 +++-- 4 files changed, 62 insertions(+), 4 deletions(-) diff --git a/src/node/services/agentSession.queueDispatch.test.ts b/src/node/services/agentSession.queueDispatch.test.ts index b57ca2e03b..27355bda8d 100644 --- a/src/node/services/agentSession.queueDispatch.test.ts +++ b/src/node/services/agentSession.queueDispatch.test.ts @@ -1850,6 +1850,30 @@ describe("AgentSession queued message tool-call dispatch", () => { } }); + test("a task hard stop on the stranded delegated turn settles the turn it had advertised", async () => { + const workspaceId = "queue-dispatch-stranded-delegated-hard-stop"; + const harness = await strandDelegatedTurnBehindClosedGate(workspaceId); + const { session, cleanup, streamMessage, settleForfeited } = harness; + + try { + // The stop found the cut stream completed, so the queue clear is the only boundary the + // session sees; the owner deferred on the marker and no stream event will reach it. + session.clearQueue("task stopped", { hardStop: true }); + expect(settleForfeited).toHaveBeenCalledTimes(1); + expect(settleForfeited.mock.calls[0]?.[0]).toEqual(WORKSPACE_TURN_CORRELATION); + expect( + session.claimWorkspaceTurnContinuation(WORKSPACE_TURN_CORRELATION, "assistant-1") + ).toBe(false); + harness.openGate(); + session.drainQueuedMessagesIfIdle(); + await new Promise((resolve) => setTimeout(resolve, 25)); + expect(streamMessage).toHaveBeenCalledTimes(1); + } finally { + session.dispose(); + await cleanup(); + } + }); + test("disposing the session settles the delegated turn it had advertised", async () => { const workspaceId = "queue-dispatch-stranded-delegated-dispose"; const harness = await strandDelegatedTurnBehindClosedGate(workspaceId); diff --git a/src/node/services/agentSession.ts b/src/node/services/agentSession.ts index 6d62a0dee8..86a2c97b7c 100644 --- a/src/node/services/agentSession.ts +++ b/src/node/services/agentSession.ts @@ -6851,9 +6851,12 @@ export class AgentSession { // stream has already completed or has not registered yet, so the queue clear is the only // session-visible boundary at which the owed continuation and a pending provider-tool soft // stop can be forfeited. A user clearing the queue keeps them: the cut turn still resumes. + // Forfeit rather than withdraw: with the stream already completed, the owner that deferred on + // the marker gets no stream event, so this settles the delegated turn (a no-op when the stop + // already did). if (options?.hardStop === true) { this.queuedProviderToolEndAbortInFlight = false; - this.withdrawStrandedTurnResume(); + this.forfeitStrandedTurnResume("Stranded turn resume dropped: task hard stop."); } const callbackSets = this.messageQueue.getClearCallbacks(); this.messageQueue.clear(); diff --git a/src/node/services/workspaceService.test.ts b/src/node/services/workspaceService.test.ts index c885ff3dec..c8a75159a5 100644 --- a/src/node/services/workspaceService.test.ts +++ b/src/node/services/workspaceService.test.ts @@ -252,6 +252,35 @@ describe("WorkspaceService bash monitor wake reconciler wiring", () => { return { config, service, events, cleanup }; } + test("a rename refuses the stranded resume at read and at the launch boundary", async () => { + const { config, service, cleanup } = await createWakeWiringService(); + const workspaceId = "renaming-resume-owner"; + await config.addWorkspace("/tmp/renaming-resume-project", { + id: workspaceId, + name: workspaceId, + projectName: "renaming-resume-project", + projectPath: "/tmp/renaming-resume-project", + runtimeConfig: { type: "local" }, + }); + try { + const session = service.getOrCreateSession(workspaceId); + const admit = Reflect.get(session, "admitStrandedTurnResume") as ( + correlation: undefined + ) => Promise<{ admissible: boolean; admissionStale?: () => boolean }>; + const admitted = await admit(undefined); + expect(admitted.admissible).toBe(true); + expect(admitted.admissionStale?.()).toBe(false); + + // rename() sees no registered stream while the resume is still in its pre-stream window + // and proceeds; the resume must not launch against paths being moved. + addToRenamingWorkspaces(service, workspaceId); + expect(admitted.admissionStale?.()).toBe(true); + expect((await admit(undefined)).admissible).toBe(false); + } finally { + await cleanup(); + } + }); + test("monitor lifecycle and shown-output events poke the reconciler", async () => { const { service, events, cleanup } = await createWakeWiringService(); const scheduleReconcile = mock(() => undefined); diff --git a/src/node/services/workspaceService.ts b/src/node/services/workspaceService.ts index 0e6e0c3569..377042bbb0 100644 --- a/src/node/services/workspaceService.ts +++ b/src/node/services/workspaceService.ts @@ -4039,11 +4039,13 @@ export class WorkspaceService extends EventEmitter implements WorkspaceHost { ); }, // The stranded resume starts a stream from inside the session, so it re-applies the - // stream-start guards WorkspaceService.resumeStream enforces (removal, archive) and, for a - // delegated turn, checks the owner still has it running: a task_stop or lifecycle interrupt - // that found the cut stream already completed had no abort to withdraw the marker with. + // stream-start guards WorkspaceService.resumeStream enforces (rename, removal, archive) + // and, for a delegated turn, checks the owner still has it running: a task_stop or + // lifecycle interrupt that found the cut stream already completed had no abort to + // withdraw the marker with. admitStrandedTurnResume: async (correlation) => { const workspaceRefused = (): boolean => + this.renamingWorkspaces.has(workspaceId) || this.removingWorkspaces.has(workspaceId) || this.archivingWorkspaces.has(workspaceId) || this.isWorkspaceArchivedInConfig(workspaceId); From 9bfb89840444d1457f966201f009716cacbcba0b Mon Sep 17 00:00:00 2001 From: Michael Suchacz <203725896+ibetitsmike@users.noreply.github.com> Date: Thu, 3 Sep 2026 19:54:49 +0000 Subject: [PATCH 16/28] Settle a cleared queued continuation, carry the spent budget into retries, roll back an orphaned sentinel A queued entry continuing the owed delegated turn is that turn's terminal path when it is cleared: the continuation is forfeited before its onCanceled runs, and every queue removal now sweeps only after the cancellation callbacks settle, as the dequeue path already did. Failed turn completions report the steps left under the stream's ceiling so an auto-retry of a stranded resume runs under what the failed attempt left (and is abandoned at zero) instead of the budget the attempt started with. A launch refused or withdrawn after streamWithHistory appended its [CONTINUE] sentinel removes that row instead of leaving it for the next turn's provider request. --- .../agentSession.queueDispatch.test.ts | 270 ++++++++++++++++++ src/node/services/agentSession.ts | 93 +++++- src/node/services/messageQueue.ts | 2 +- src/node/services/streamManager.ts | 35 ++- 4 files changed, 377 insertions(+), 23 deletions(-) diff --git a/src/node/services/agentSession.queueDispatch.test.ts b/src/node/services/agentSession.queueDispatch.test.ts index 27355bda8d..34c1aa39b4 100644 --- a/src/node/services/agentSession.queueDispatch.test.ts +++ b/src/node/services/agentSession.queueDispatch.test.ts @@ -1079,6 +1079,159 @@ describe("AgentSession queued message tool-call dispatch", () => { } }); + test("an auto-retry runs under what the failed resumed attempt left of the step budget", async () => { + const workspaceId = "queue-dispatch-stranded-retry-step-budget"; + const aiEmitter = new EventEmitter(); + let streams = 0; + const streamMessage = mock((_options: StreamMessageOptions) => { + streams += 1; + aiEmitter.emit("stream-start", streamStartEvent(workspaceId)); + // The resumed stream spends steps, then fails with a retryable error. + return Promise.resolve( + Ok( + streams === 2 + ? { + messageId: "assistant-2", + completion: Promise.resolve({ + status: "failed" as const, + streamError: { + messageId: "assistant-2", + error: "provider closed the connection", + errorType: "api" as const, + }, + stepsRemaining: 2, + }), + } + : createStartedTurnHandle(`assistant-${streams}`) + ) + ); + }); + const { session, cleanup, historyService } = await createAgentSessionHarness({ + workspaceId, + aiEmitter, + aiServiceOverrides: { streamMessage: streamMessage as unknown as AIService["streamMessage"] }, + }); + + try { + const sent = await session.sendMessage("run the checks", { + model: TEST_MODEL, + agentId: "exec", + }); + expect(sent.success).toBe(true); + await historyService.appendToHistory( + workspaceId, + createMuxMessage("assistant-1", "assistant", "ran the checks", { timestamp: Date.now() }) + ); + const controller = new AbortController(); + session.queueMessage( + "Background monitor wake", + { model: TEST_MODEL, agentId: "exec" }, + { + synthetic: true, + agentInitiated: true, + cancelState: { canceledBeforeAcceptance: false }, + cancelSignal: controller.signal, + onCanceled: () => undefined, + } + ); + streamMessage.mock.calls[0]?.[0].onQueuedMessageStop?.({ + modelString: TEST_MODEL, + stepsRemaining: 5, + }); + controller.abort("monitor consumed"); + aiEmitter.emit("stream-end", streamEndEvent(workspaceId)); + expect(await waitForCondition(() => streamMessage.mock.calls.length === 2)).toBe(true); + expect(streamMessage.mock.calls[1]?.[0].stepBudget).toBe(5); + + // The retry continues the same logical turn under the 2 steps the failed attempt left, not + // the 5 it started with. + expect(await waitForCondition(() => streamMessage.mock.calls.length === 3, 4_000)).toBe(true); + expect(streamMessage.mock.calls[2]?.[0].stepBudget).toBe(2); + } finally { + session.dispose(); + await cleanup(); + } + }); + + test("an auto-retry is abandoned when the failed resumed attempt spent the step budget", async () => { + const workspaceId = "queue-dispatch-stranded-retry-step-budget-spent"; + const aiEmitter = new EventEmitter(); + let streams = 0; + const streamMessage = mock((_options: StreamMessageOptions) => { + streams += 1; + aiEmitter.emit("stream-start", streamStartEvent(workspaceId)); + return Promise.resolve( + Ok( + streams === 2 + ? { + messageId: "assistant-2", + completion: Promise.resolve({ + status: "failed" as const, + streamError: { + messageId: "assistant-2", + error: "provider closed the connection", + errorType: "api" as const, + }, + stepsRemaining: 0, + }), + } + : createStartedTurnHandle(`assistant-${streams}`) + ) + ); + }); + const { session, cleanup, historyService } = await createAgentSessionHarness({ + workspaceId, + aiEmitter, + aiServiceOverrides: { streamMessage: streamMessage as unknown as AIService["streamMessage"] }, + }); + const abandoned: string[] = []; + const unsubscribe = session.onChatEvent((event) => { + if (event.message.type === "auto-retry-abandoned") { + abandoned.push(event.message.reason); + } + }); + + try { + const sent = await session.sendMessage("run the checks", { + model: TEST_MODEL, + agentId: "exec", + }); + expect(sent.success).toBe(true); + await historyService.appendToHistory( + workspaceId, + createMuxMessage("assistant-1", "assistant", "ran the checks", { timestamp: Date.now() }) + ); + const controller = new AbortController(); + session.queueMessage( + "Background monitor wake", + { model: TEST_MODEL, agentId: "exec" }, + { + synthetic: true, + agentInitiated: true, + cancelState: { canceledBeforeAcceptance: false }, + cancelSignal: controller.signal, + onCanceled: () => undefined, + } + ); + streamMessage.mock.calls[0]?.[0].onQueuedMessageStop?.({ + modelString: TEST_MODEL, + stepsRemaining: 1, + }); + controller.abort("monitor consumed"); + aiEmitter.emit("stream-end", streamEndEvent(workspaceId)); + expect(await waitForCondition(() => streamMessage.mock.calls.length === 2)).toBe(true); + + // The failed attempt spent the turn's last step: the ceiling ended the turn, no retry runs. + expect(await waitForCondition(() => abandoned.length === 1, 4_000)).toBe(true); + expect(streamMessage).toHaveBeenCalledTimes(2); + expect(session.isBusy()).toBe(false); + } finally { + unsubscribe(); + session.dispose(); + await cleanup(); + } + }); + test("caps resume attempts that never start a stream", async () => { const workspaceId = "queue-dispatch-stranded-cap"; let gateOpen = true; @@ -1999,6 +2152,75 @@ describe("AgentSession queued message tool-call dispatch", () => { } }); + test("a launch refused after the [CONTINUE] sentinel landed removes the sentinel", async () => { + const workspaceId = "queue-dispatch-stranded-refused-sentinel"; + // Admitted on read; the stop lands while the resume appends its sentinel. + let stale = false; + const admitStrandedTurnResume = mock((_correlation: unknown) => + Promise.resolve({ admissible: true, admissionStale: () => stale }) + ); + const aiEmitter = new EventEmitter(); + const streamMessage = mock((_options: StreamMessageOptions) => { + aiEmitter.emit("stream-start", streamStartEvent(workspaceId)); + return Promise.resolve(Ok(createStartedTurnHandle("assistant-1"))); + }); + const { session, cleanup, historyService } = await createAgentSessionHarness({ + workspaceId, + aiEmitter, + aiServiceOverrides: { streamMessage: streamMessage as unknown as AIService["streamMessage"] }, + admitStrandedTurnResume, + }); + const append = historyService.appendToHistory.bind(historyService); + const appendSpy = spyOn(historyService, "appendToHistory").mockImplementation( + async (targetWorkspaceId, message) => { + const result = await append(targetWorkspaceId, message); + if ( + message.role === "user" && + message.parts.some((part) => part.type === "text" && part.text === "[CONTINUE]") + ) { + stale = true; + } + return result; + } + ); + + try { + await historyService.appendToHistory( + workspaceId, + createMuxMessage("user-0", "user", "delegated prompt", { + timestamp: Date.now(), + muxMetadata: WORKSPACE_TURN_CORRELATION, + }) + ); + await historyService.appendToHistory( + workspaceId, + createMuxMessage("assistant-0", "assistant", "working", { + timestamp: Date.now(), + finishReason: "tool-calls", + muxMetadata: WORKSPACE_TURN_CORRELATION, + }) + ); + const resumed = await session.resumeStream( + { model: TEST_MODEL, agentId: "exec", muxMetadata: WORKSPACE_TURN_CORRELATION }, + { agentInitiated: true, revalidateAdmission: true } + ); + expect(resumed).toEqual(Ok({ started: false, refusedBy: "workspace-turn" })); + expect(streamMessage).not.toHaveBeenCalled(); + + // The sentinel was appended for a launch that never happened; a later unrelated turn must + // not send it to the provider. + const history = await historyService.getHistoryFromLatestBoundary(workspaceId); + expect(history.success).toBe(true); + if (history.success) { + expect(history.data.map((message) => message.id)).toEqual(["user-0", "assistant-0"]); + } + } finally { + appendSpy.mockRestore(); + session.dispose(); + await cleanup(); + } + }); + test("a user Stop during the resume's admission gate cancels it", async () => { const workspaceId = "queue-dispatch-stranded-stop-in-admission"; let releaseGate: () => void = () => undefined; @@ -2266,6 +2488,54 @@ describe("AgentSession queued message tool-call dispatch", () => { } }); + test("clearing the queued continuation of the delegated turn settles it before any sweep", async () => { + const workspaceId = "queue-dispatch-stranded-delegated-continuation-cleared"; + const settleForfeited = mock((_metadata: unknown, _reason: string) => Promise.resolve()); + const harness = await createStreamingTurnHarness(workspaceId, { + harness: { settleForfeitedWorkspaceTurnContinuation: settleForfeited }, + sendOptions: { muxMetadata: WORKSPACE_TURN_CORRELATION }, + }); + const { session, cleanup, aiEmitter, streamMessage } = harness; + + try { + // The owner queued a continuation of the same turn; its onCanceled is the owner's settlement + // of that turn and takes its time (task-store I/O). + let finishCancel: () => void = () => undefined; + const canceled = new Promise((resolve) => { + finishCancel = resolve; + }); + session.queueMessage( + "continue the turn", + { model: TEST_MODEL, agentId: "exec", muxMetadata: WORKSPACE_TURN_CORRELATION }, + { agentInitiated: true, onCanceled: () => canceled } + ); + harness.latestRequest().onQueuedMessageStop?.(queuedStop()); + expect( + session.claimWorkspaceTurnContinuation(WORKSPACE_TURN_CORRELATION, "assistant-1") + ).toBe(true); + // Admission is held across the stream end (a history mutation), so the entry stays queued + // on an idle session instead of dispatching. + const hold = session.holdTurnAdmission(); + aiEmitter.emit("stream-end", streamEndEvent(workspaceId)); + expect(await waitForCondition(() => !session.isBusy())).toBe(true); + + // The user clears the queue: the continuation is that turn's terminal path, so nothing is + // owed to the cut anymore, whatever the sweep that runs next reads from the handle. + session.clearQueue("queue cleared by user"); + expect(settleForfeited).toHaveBeenCalledTimes(1); + expect(settleForfeited.mock.calls[0]?.[0]).toEqual(WORKSPACE_TURN_CORRELATION); + hold[Symbol.dispose](); + await new Promise((resolve) => setTimeout(resolve, 25)); + finishCancel(); + await new Promise((resolve) => setTimeout(resolve, 25)); + expect(streamMessage).toHaveBeenCalledTimes(1); + expect(session.isBusy()).toBe(false); + } finally { + session.dispose(); + await cleanup(); + } + }); + test("an older stream-end's claim does not void a newer cut's continuation", async () => { const workspaceId = "queue-dispatch-stranded-delegated-stale-claim"; const harness = await createStreamingTurnHarness(workspaceId, { diff --git a/src/node/services/agentSession.ts b/src/node/services/agentSession.ts index 86a2c97b7c..18c5b84524 100644 --- a/src/node/services/agentSession.ts +++ b/src/node/services/agentSession.ts @@ -114,7 +114,7 @@ import { createRuntimeForWorkspace, } from "@/node/runtime/runtimeHelpers"; import { MessageQueue, cancelReasonBeforeAcceptance } from "./messageQueue"; -import type { QueueCutCutter } from "./messageQueue"; +import type { QueueClearCallbacks, QueueCutCutter } from "./messageQueue"; import { copyStreamLifecycleSnapshot, type RuntimeStatusEvent, @@ -1442,6 +1442,11 @@ export class AgentSession { this.emitRetryEvent({ type: "auto-retry-abandoned", reason: "missing_retry_options" }); return; } + if (request.stepBudget != null && request.stepBudget <= 0) { + // The failed attempt spent the turn's last step; the ceiling ends the turn here. + this.emitRetryEvent({ type: "auto-retry-abandoned", reason: "step_budget_spent" }); + return; + } const result = await this.resumeStream(request.options, { agentInitiated: request.agentInitiated === true ? true : undefined, @@ -5171,6 +5176,12 @@ export class AgentSession { // A disposed session must not persist retry/goal state post-teardown. if (outcome.status !== "failed" || this.disposed) return; + // A retry continues the same logical turn: it runs under what the failed attempt left + // of the ceiling, not the budget the attempt started with. + const retryRequest = this.lastAutoRetryResumeRequest; + if (retryRequest?.stepBudget != null && outcome.stepsRemaining != null) { + this.lastAutoRetryResumeRequest = { ...retryRequest, stepBudget: outcome.stepsRemaining }; + } try { await this.handleStreamError(outcome.streamError); } finally { @@ -5294,6 +5305,7 @@ export class AgentSession { // [CONTINUE] sentinel so the model has a valid conversation to respond to. This is // defense-in-depth; callers should prefer sendMessage() which persists a real user message. const lastMsg = requestMessages[requestMessages.length - 1]; + let sentinelMessageId: string | undefined; if (lastMsg?.role === "assistant" && !lastMsg.metadata?.partial) { log.warn("streamWithHistory: trailing non-partial assistant detected, injecting [CONTINUE]", { workspaceId: this.workspaceId, @@ -5304,11 +5316,29 @@ export class AgentSession { synthetic: true, }); await this.historyService.appendToHistory(this.workspaceId, sentinelMessage); + sentinelMessageId = sentinelMessage.id; const refreshed = await this.historyService.getHistoryFromLatestBoundary(this.workspaceId); if (refreshed.success) { requestMessages = filterOrphanedMcpPromptSnapshots(refreshed.data); } } + // A launch refused or withdrawn after the sentinel landed would leave it as an orphan that the + // next unrelated turn sends to the provider; it goes with the launch it was appended for. + const abortStartup = async (): Promise> => { + if (sentinelMessageId != null) { + const removed = await this.historyService.deleteMessage( + this.workspaceId, + sentinelMessageId + ); + if (!removed.success) { + log.warn("Failed to remove the [CONTINUE] sentinel of a refused launch", { + workspaceId: this.workspaceId, + error: removed.error, + }); + } + } + return Ok(undefined); + }; // Capture the current user message id so retries are stable across assistant message ids. // Retry-eligible rows only: startup recovery matches this persisted ID @@ -5328,7 +5358,7 @@ export class AgentSession { ); if (isStartupAbortRequested()) { - return Ok(undefined); + return await abortStartup(); } // Check if post-compaction attachments should be injected. @@ -5337,7 +5367,7 @@ export class AgentSession { ? null : await this.getPostCompactionAttachmentsIfNeeded(this.isRlmCompactionEnabled(options)); if (isStartupAbortRequested()) { - return Ok(undefined); + return await abortStartup(); } this.activeStreamHadPostCompactionInjection = @@ -5467,6 +5497,11 @@ export class AgentSession { ); } + // stream-start moves the turn to STREAMING synchronously inside startStream, so a turn still + // PREPARING here was refused or withdrawn at the launch boundary and registered nothing. + if (this.turnPhase === TurnPhase.PREPARING && isStartupAbortRequested()) { + return await abortStartup(); + } this.consumeTurnCompletion(streamResult.data); return Ok(undefined); } @@ -6858,14 +6893,12 @@ export class AgentSession { this.queuedProviderToolEndAbortInFlight = false; this.forfeitStrandedTurnResume("Stranded turn resume dropped: task hard stop."); } + const heldOwedContinuation = this.queueHoldsOwedTurnContinuation(); const callbackSets = this.messageQueue.getClearCallbacks(); this.messageQueue.clear(); this.emitQueuedMessageChanged(); this.backgroundProcessManager.setMessageQueued(this.workspaceId, false); - for (const callbacks of callbackSets) { - this.notifyQueuedMessageCleared(callbacks, cancelReason); - } - this.resumeStrandedTurnIfIdle(); + this.cancelRemovedEntries(callbackSets, cancelReason, heldOwedContinuation); } setQueuedMessageDispatchMode(mode: "tool-end" | "turn-end"): boolean { @@ -6891,7 +6924,7 @@ export class AgentSession { onAcceptedPreStreamFailure?: (error: SendMessageError) => Promise | void; }, cancelReason: string - ): void { + ): Promise { const notify = async () => { if (callbacks.onCanceled != null) { await callbacks.onCanceled(cancelReason); @@ -6899,7 +6932,7 @@ export class AgentSession { } await callbacks.onAcceptedPreStreamFailure?.(createUnknownSendMessageError(cancelReason)); }; - notify().catch((error: unknown) => { + return notify().catch((error: unknown) => { log.error("Queued message clear callback failed", { workspaceId: this.workspaceId, error: getErrorMessage(error), @@ -6907,9 +6940,42 @@ export class AgentSession { }); } + /** + * Whether the queue holds an entry continuing the owed marker's delegated turn. The owner + * defers that turn's settlement on such an entry (claimWorkspaceTurnContinuation), and the + * entry's onCanceled settles the handle if it is removed unstarted. + */ + private queueHoldsOwedTurnContinuation(): boolean { + const correlation = getWorkspaceTurnMuxMetadata(this.strandedTurnResume?.options.muxMetadata); + return correlation != null && this.messageQueue.hasWorkspaceTurn(correlation.taskHandleId); + } + + /** + * Runs removed entries' cancellation and sweeps for the owed continuation only once it has + * settled, as the dequeue path does: a canceled workspace-turn entry settles its handle in + * onCanceled, and a sweep admitted before that lands would resume against an interrupted + * handle. An entry that continued the owed turn was that turn's terminal path, so the + * continuation is forfeited before its cancellation runs. + */ + private cancelRemovedEntries( + callbackSets: QueueClearCallbacks[], + cancelReason: string, + heldOwedContinuation: boolean + ): void { + if (heldOwedContinuation && !this.queueHoldsOwedTurnContinuation()) { + this.forfeitStrandedTurnResume( + "Stranded turn resume dropped: its queued continuation was canceled." + ); + } + void Promise.all( + callbackSets.map((callbacks) => this.notifyQueuedMessageCleared(callbacks, cancelReason)) + ).then(() => this.resumeStrandedTurnIfIdle()); + } + removeQueuedMessagesByDedupeKeyPrefix(prefix: string, cancelReason: string): number { this.assertNotDisposed("removeQueuedMessagesByDedupeKeyPrefix"); assert(prefix.length > 0, "removeQueuedMessagesByDedupeKeyPrefix requires prefix"); + const heldOwedContinuation = this.queueHoldsOwedTurnContinuation(); const removal = this.messageQueue.removeByDedupeKeyPrefix(prefix); if (removal.removedCount === 0) { return 0; @@ -6919,10 +6985,7 @@ export class AgentSession { this.workspaceId, this.messageQueue.getNextDispatchableMode() === "tool-end" ); - for (const callbacks of removal.callbacks) { - this.notifyQueuedMessageCleared(callbacks, cancelReason); - } - this.resumeStrandedTurnIfIdle(); + this.cancelRemovedEntries(removal.callbacks, cancelReason, heldOwedContinuation); return removal.removedCount; } @@ -6939,6 +7002,7 @@ export class AgentSession { removeQueuedWorkspaceTurn(handleId: string, cancelReason: string): boolean { this.assertNotDisposed("removeQueuedWorkspaceTurn"); assert(handleId.length > 0, "removeQueuedWorkspaceTurn requires handleId"); + const heldOwedContinuation = this.queueHoldsOwedTurnContinuation(); const callbacks = this.messageQueue.removeWorkspaceTurn(handleId); if (callbacks == null) { return false; @@ -6948,8 +7012,7 @@ export class AgentSession { this.workspaceId, this.messageQueue.getNextDispatchableMode() === "tool-end" ); - this.notifyQueuedMessageCleared(callbacks, cancelReason); - this.resumeStrandedTurnIfIdle(); + this.cancelRemovedEntries([callbacks], cancelReason, heldOwedContinuation); return true; } diff --git a/src/node/services/messageQueue.ts b/src/node/services/messageQueue.ts index c3b29379c3..ab11950ca6 100644 --- a/src/node/services/messageQueue.ts +++ b/src/node/services/messageQueue.ts @@ -153,7 +153,7 @@ interface QueuedMessageInternalOptions { admissionStale?: () => boolean; } -type QueueClearCallbacks = Pick< +export type QueueClearCallbacks = Pick< QueuedMessageInternalOptions, "onCanceled" | "onAcceptedPreStreamFailure" >; diff --git a/src/node/services/streamManager.ts b/src/node/services/streamManager.ts index 6ecce1199b..9b28678b3c 100644 --- a/src/node/services/streamManager.ts +++ b/src/node/services/streamManager.ts @@ -206,7 +206,15 @@ export type TurnEngineEventSink = (event: TurnEngineEvent) => void | Promise Date: Thu, 3 Sep 2026 20:20:00 +0000 Subject: [PATCH 17/28] Gate the in-session context_exceeded retries like a resume The compaction and post-compaction retries relaunch through streamWithHistory directly. A stream admitted under resumeStream's revalidation now records that on its context, and both retries revalidate the goal and delegated turn again (admitResumeLaunch, shared with resumeStream), carry the launch-boundary probe, and run under what the failed attempt left of the step budget, skipping the retry when it is spent. --- .../agentSession.queueDispatch.test.ts | 149 ++++++++++++- src/node/services/agentSession.ts | 204 +++++++++++++----- 2 files changed, 304 insertions(+), 49 deletions(-) diff --git a/src/node/services/agentSession.queueDispatch.test.ts b/src/node/services/agentSession.queueDispatch.test.ts index 34c1aa39b4..34d32ce8cc 100644 --- a/src/node/services/agentSession.queueDispatch.test.ts +++ b/src/node/services/agentSession.queueDispatch.test.ts @@ -15,7 +15,7 @@ import { } from "./agentSession.testHarness"; import type { AIService, StreamMessageOptions } from "./aiService"; import type { HistoryService } from "./historyService"; -import type { TurnStreamHandle } from "./streamManager"; +import type { TurnCompletion, TurnStreamHandle } from "./streamManager"; const TEST_MODEL = "anthropic:claude-sonnet-4-5"; /** Steps a cut stream reports as left; tests about the budget itself pass their own value. */ @@ -1720,6 +1720,153 @@ describe("AgentSession queued message tool-call dispatch", () => { } }); + /** + * A stranded goal resume (admitted under revalidation, running under a step budget) whose + * stream fails with context_exceeded after post-compaction context was injected, so the + * in-session retry without that context is the recovery path; the caller settles the failure. + */ + async function strandGoalResumeIntoPostCompactionRetry( + workspaceId: string, + buildGoalRedispatchAdmission: ReturnType< + typeof mock + >, + stepsRemainingAfterFailure: number + ) { + const workspaceGoalService = { + assertPricedModelForBudgetedGoal: mock(() => Promise.resolve(Ok(undefined))), + buildGoalRedispatchAdmission, + recordStreamAccounting: mock(() => Promise.resolve()), + applyPendingAfterStreamEnd: mock(() => Promise.resolve()), + requestContinuationAfterStreamEnd: mock(() => Promise.resolve()), + recordStreamStarted: mock(() => Promise.resolve()), + syncGoalModeWithChatTail: mock(() => Promise.resolve(null)), + completeGoalFromSilentContinuation: mock(() => Promise.resolve(false)), + } as unknown as WorkspaceGoalService; + const aiEmitter = new EventEmitter(); + let streams = 0; + let failResumed: () => void = () => undefined; + const streamMessage = mock((_options: StreamMessageOptions) => { + streams += 1; + aiEmitter.emit("stream-start", streamStartEvent(workspaceId)); + if (streams !== 2) { + return Promise.resolve(Ok(createStartedTurnHandle(`assistant-${streams}`))); + } + const completion = new Promise((resolve) => { + failResumed = () => + resolve({ + status: "failed", + streamError: { + messageId: "assistant-2", + error: "context window exceeded", + errorType: "context_exceeded", + }, + stepsRemaining: stepsRemainingAfterFailure, + }); + }); + return Promise.resolve(Ok({ messageId: "assistant-2", completion })); + }); + const harness = await createAgentSessionHarness({ + workspaceId, + aiEmitter, + aiServiceOverrides: { streamMessage: streamMessage as unknown as AIService["streamMessage"] }, + workspaceGoalService, + }); + const { session, historyService } = harness; + await historyService.appendToHistory( + workspaceId, + createMuxMessage("user-0", "user", "keep going", { timestamp: Date.now() }) + ); + const resumed = await session.resumeStream( + { model: TEST_MODEL, agentId: "exec" }, + { agentInitiated: true, goalKind: GOAL_CONTINUATION_KIND, goalId: "goal-1" } + ); + expect(resumed.success).toBe(true); + await historyService.appendToHistory( + workspaceId, + createMuxMessage("assistant-1", "assistant", "ran the checks", { timestamp: Date.now() }) + ); + const controller = new AbortController(); + session.queueMessage( + "Background monitor wake", + { model: TEST_MODEL, agentId: "exec" }, + { + synthetic: true, + agentInitiated: true, + cancelState: { canceledBeforeAcceptance: false }, + cancelSignal: controller.signal, + onCanceled: () => undefined, + } + ); + streamMessage.mock.calls[0]?.[0].onQueuedMessageStop?.({ + modelString: TEST_MODEL, + stepsRemaining: 5, + }); + controller.abort("monitor consumed"); + aiEmitter.emit("stream-end", streamEndEvent(workspaceId)); + expect(await waitForCondition(() => streamMessage.mock.calls.length === 2)).toBe(true); + expect(streamMessage.mock.calls[1]?.[0].stepBudget).toBe(5); + expect(buildGoalRedispatchAdmission).toHaveBeenCalledTimes(1); + // The resumed stream carried post-compaction context (the retry's precondition). + Reflect.set(session, "activeStreamHadPostCompactionInjection", true); + return { ...harness, streamMessage, failResumed: () => failResumed() }; + } + + test("the post-compaction retry of a stranded goal resume is admitted by the goal again", async () => { + const workspaceId = "queue-dispatch-stranded-goal-post-compaction-retry-admission"; + const buildGoalRedispatchAdmission = mock( + (): ReturnType => + Promise.resolve({ admissible: true, admissionStale: () => false }) + ); + const harness = await strandGoalResumeIntoPostCompactionRetry( + workspaceId, + buildGoalRedispatchAdmission, + 3 + ); + const { session, cleanup, streamMessage } = harness; + + try { + // The goal is paused before the failure's cleanup finishes: the retry asks the goal again + // and, refused, starts nothing. + buildGoalRedispatchAdmission.mockImplementation(() => Promise.resolve({ admissible: false })); + harness.failResumed(); + expect( + await waitForCondition(() => buildGoalRedispatchAdmission.mock.calls.length === 2) + ).toBe(true); + expect(await waitForCondition(() => !session.isBusy())).toBe(true); + await new Promise((resolve) => setTimeout(resolve, 25)); + expect(streamMessage).toHaveBeenCalledTimes(2); + } finally { + session.dispose(); + await cleanup(); + } + }); + + test("the post-compaction retry of a stranded resume runs under what the failed attempt left", async () => { + const workspaceId = "queue-dispatch-stranded-goal-post-compaction-retry-budget"; + const buildGoalRedispatchAdmission = mock( + (): ReturnType => + Promise.resolve({ admissible: true, admissionStale: () => false }) + ); + const harness = await strandGoalResumeIntoPostCompactionRetry( + workspaceId, + buildGoalRedispatchAdmission, + 3 + ); + const { session, cleanup, streamMessage } = harness; + + try { + harness.failResumed(); + expect(await waitForCondition(() => streamMessage.mock.calls.length === 3)).toBe(true); + // Same logical turn: the retry runs under the 3 steps the failed attempt left, not the 5 it + // started with, and the goal admitted it again. + expect(streamMessage.mock.calls[2]?.[0].stepBudget).toBe(3); + expect(buildGoalRedispatchAdmission).toHaveBeenCalledTimes(2); + } finally { + session.dispose(); + await cleanup(); + } + }); + test("drops a stranded goal continuation the goal no longer admits", async () => { const workspaceId = "queue-dispatch-stranded-goal-paused"; const workspaceGoalService = { diff --git a/src/node/services/agentSession.ts b/src/node/services/agentSession.ts index 18c5b84524..5fd5075ecc 100644 --- a/src/node/services/agentSession.ts +++ b/src/node/services/agentSession.ts @@ -981,6 +981,8 @@ export class AgentSession { goalId?: string; /** Step ceiling this stream runs under when it continues a cut turn (see StrandedTurnResume). */ stepBudget?: number; + /** Admitted under resumeStream's revalidation; in-session retries of this stream repeat it. */ + revalidateAdmission?: boolean; workspaceTurnMetadata?: Extract; }; @@ -4409,32 +4411,19 @@ export class AgentSession { if (withdrawn()) { return Ok({ started: false }); } - let goalAdmissionStale: (() => boolean) | undefined; - if ( - internal?.revalidateAdmission === true && - internal.goalKind != null && - internal.goalId != null && - this.workspaceGoalService - ) { - const admission = await this.workspaceGoalService.buildGoalRedispatchAdmission( - this.workspaceId, - internal.goalId, - internal.goalKind - ); - if (!admission.admissible) { - return Ok({ started: false, refusedBy: "goal" }); - } - goalAdmissionStale = admission.admissionStale; - } - let turnAdmissionStale: (() => boolean) | undefined; - if (internal?.revalidateAdmission === true && this.admitStrandedTurnResume) { - const admission = await this.admitStrandedTurnResume( - getWorkspaceTurnMuxMetadata(optionsForStream.muxMetadata) - ); + let refuseStreamStart: (() => boolean) | undefined; + let launchRefusedBy: () => "goal" | "workspace-turn" | undefined = () => undefined; + if (internal?.revalidateAdmission === true) { + const admission = await this.admitResumeLaunch({ + goalKind: internal.goalKind, + goalId: internal.goalId, + muxMetadata: optionsForStream.muxMetadata, + }); if (!admission.admissible) { - return Ok({ started: false, refusedBy: "workspace-turn" }); + return Ok({ started: false, refusedBy: admission.refusedBy }); } - turnAdmissionStale = admission.admissionStale; + refuseStreamStart = admission.refuseStreamStart; + launchRefusedBy = admission.refusedBy; } // Last admission check before the stream's own pre-start I/O, like sendMessage's PREPARING // gate: a withdrawal that landed during the awaits above refuses the turn here; the abort @@ -4442,21 +4431,8 @@ export class AgentSession { if (withdrawn()) { return Ok({ started: false }); } - // The admission probes ride along to the stream-admission boundary: a Pause, goal - // replacement, or workspace stop landing during the history reads and request construction - // below has no stream to interrupt, so the launch itself rechecks them (StreamManager last, - // right before registration). Sticky so the return value matches what refused the launch. - let refusedBy: "goal" | "workspace-turn" | undefined; - const refuseStreamStart = - goalAdmissionStale != null || turnAdmissionStale != null - ? (): boolean => { - refusedBy ??= goalAdmissionStale?.() === true ? "goal" : undefined; - refusedBy ??= turnAdmissionStale?.() === true ? "workspace-turn" : undefined; - return refusedBy != null; - } - : undefined; if (refuseStreamStart?.() === true) { - return Ok({ started: false, refusedBy }); + return Ok({ started: false, refusedBy: launchRefusedBy() }); } // Must await here so the finally block runs after streaming completes, @@ -4472,11 +4448,13 @@ export class AgentSession { internal?.goalId, turnThinkingOverride, refuseStreamStart, - internal?.stepBudget + internal?.stepBudget, + internal?.revalidateAdmission ); if (!result.success) { return result; } + const refusedBy = launchRefusedBy(); if (refusedBy != null) { return Ok({ started: false, refusedBy }); } @@ -4491,6 +4469,59 @@ export class AgentSession { } } + /** + * Revalidates a resume against durable goal state (the veto durable goal redispatches apply) + * and the workspace plus delegated turn (admitStrandedTurnResume). The probes ride along to + * the stream-admission boundary: a Pause, goal replacement, or workspace stop landing during + * the history reads and request construction has no stream to interrupt, so the launch itself + * rechecks them (StreamManager last, right before registration). Sticky about what refused it. + */ + private async admitResumeLaunch(input: { + goalKind?: GoalSyntheticMessageKind; + goalId?: string; + muxMetadata: unknown; + }): Promise< + | { admissible: false; refusedBy: "goal" | "workspace-turn" } + | { + admissible: true; + refuseStreamStart?: () => boolean; + refusedBy: () => "goal" | "workspace-turn" | undefined; + } + > { + let goalAdmissionStale: (() => boolean) | undefined; + if (input.goalKind != null && input.goalId != null && this.workspaceGoalService) { + const admission = await this.workspaceGoalService.buildGoalRedispatchAdmission( + this.workspaceId, + input.goalId, + input.goalKind + ); + if (!admission.admissible) { + return { admissible: false, refusedBy: "goal" }; + } + goalAdmissionStale = admission.admissionStale; + } + let turnAdmissionStale: (() => boolean) | undefined; + if (this.admitStrandedTurnResume) { + const admission = await this.admitStrandedTurnResume( + getWorkspaceTurnMuxMetadata(input.muxMetadata) + ); + if (!admission.admissible) { + return { admissible: false, refusedBy: "workspace-turn" }; + } + turnAdmissionStale = admission.admissionStale; + } + let refusedBy: "goal" | "workspace-turn" | undefined; + const refuseStreamStart = + goalAdmissionStale != null || turnAdmissionStale != null + ? (): boolean => { + refusedBy ??= goalAdmissionStale?.() === true ? "goal" : undefined; + refusedBy ??= turnAdmissionStale?.() === true ? "workspace-turn" : undefined; + return refusedBy != null; + } + : undefined; + return { admissible: true, refuseStreamStart, refusedBy: () => refusedBy }; + } + async setAutoRetryEnabled( enabled: boolean, options?: { persist?: boolean } @@ -5177,10 +5208,19 @@ export class AgentSession { if (outcome.status !== "failed" || this.disposed) return; // A retry continues the same logical turn: it runs under what the failed attempt left - // of the ceiling, not the budget the attempt started with. - const retryRequest = this.lastAutoRetryResumeRequest; - if (retryRequest?.stepBudget != null && outcome.stepsRemaining != null) { - this.lastAutoRetryResumeRequest = { ...retryRequest, stepBudget: outcome.stepsRemaining }; + // of the ceiling, not the budget the attempt started with (auto-retry and the in-session + // context_exceeded retries alike). + if (outcome.stepsRemaining != null) { + const retryRequest = this.lastAutoRetryResumeRequest; + if (retryRequest?.stepBudget != null) { + this.lastAutoRetryResumeRequest = { + ...retryRequest, + stepBudget: outcome.stepsRemaining, + }; + } + if (this.activeStreamContext?.stepBudget != null) { + this.activeStreamContext.stepBudget = outcome.stepsRemaining; + } } try { await this.handleStreamError(outcome.streamError); @@ -5212,7 +5252,8 @@ export class AgentSession { // Pull-based admission probe (goal state) with no push into abortSignal; checked wherever // the signal is, and by StreamManager right before the stream registers. refuseStreamStart?: () => boolean, - stepBudget?: number + stepBudget?: number, + revalidateAdmission?: boolean ): Promise> { const isStartupAbortRequested = (): boolean => abortSignal?.aborted === true || refuseStreamStart?.() === true; @@ -5236,6 +5277,7 @@ export class AgentSession { ...(goalKind != null ? { goalKind } : {}), ...(goalId != null ? { goalId } : {}), ...(stepBudget != null ? { stepBudget } : {}), + ...(revalidateAdmission === true ? { revalidateAdmission: true } : {}), providersConfig, }; this.activeStreamUserMessageId = undefined; @@ -5653,6 +5695,41 @@ export class AgentSession { return provider === "openai" && modelName?.toLowerCase().startsWith("gpt-"); } + /** + * Gate for the in-session context_exceeded retries, which relaunch through streamWithHistory + * directly: a stream admitted under resumeStream's revalidation is revalidated again (a Pause, + * Stop, or delegated-turn interrupt that landed during the cleanup awaits ends the turn rather + * than the retry restarting tool-enabled work), and a spent step budget ends it too. Undefined + * when the retry must not run; otherwise the launch-boundary probe to pass along. + */ + private async admitInSessionRetry(input: { + stepBudget?: number; + revalidateAdmission?: boolean; + goalKind?: GoalSyntheticMessageKind; + goalId?: string; + muxMetadata: unknown; + retryLabel: string; + }): Promise<{ refuseStreamStart?: () => boolean } | undefined> { + if (input.stepBudget != null && input.stepBudget <= 0) { + log.info(`Skipping ${input.retryLabel}: the turn's step budget is spent`, { + workspaceId: this.workspaceId, + }); + return undefined; + } + if (input.revalidateAdmission !== true) { + return {}; + } + const admission = await this.admitResumeLaunch(input); + if (!admission.admissible) { + log.info(`Skipping ${input.retryLabel}: no longer admitted`, { + workspaceId: this.workspaceId, + refusedBy: admission.refusedBy, + }); + return undefined; + } + return { refuseStreamStart: admission.refuseStreamStart }; + } + private async maybeRetryCompactionOnContextExceeded(data: { messageId: string; errorType?: string; @@ -5709,6 +5786,8 @@ export class AgentSession { const retryAgentInitiated = this.activeStreamContext?.agentInitiated; const retryGoalKind = this.activeStreamContext?.goalKind; const retryGoalId = this.activeStreamContext?.goalId; + const retryStepBudget = this.activeStreamContext?.stepBudget; + const retryRevalidateAdmission = this.activeStreamContext?.revalidateAdmission; const retryOptionsForResume = retryOptions ?? { model: context.modelString, agentId: WORKSPACE_DEFAULTS.agentId, @@ -5727,12 +5806,25 @@ export class AgentSession { }); return false; } + const retryAdmission = await this.admitInSessionRetry({ + stepBudget: retryStepBudget, + revalidateAdmission: retryRevalidateAdmission, + goalKind: retryGoalKind, + goalId: retryGoalId, + muxMetadata: retryOptionsForResume.muxMetadata, + retryLabel: "compaction retry", + }); + if (retryAdmission == null) { + return false; + } this.setAutoRetryResumeState( retryOptionsForResume, retryAgentInitiated, retryGoalKind, - retryGoalId + retryGoalId, + retryStepBudget, + retryRevalidateAdmission ); this.preparingWorkspaceTurnMetadata = getWorkspaceTurnMuxMetadata( retryOptionsForResume.muxMetadata @@ -5748,7 +5840,11 @@ export class AgentSession { retryAgentInitiated, undefined, retryGoalKind, - retryGoalId + retryGoalId, + undefined, + retryAdmission.refuseStreamStart, + retryStepBudget, + retryRevalidateAdmission ); } finally { if (this.turnPhase === TurnPhase.PREPARING) { @@ -5841,6 +5937,17 @@ export class AgentSession { ); return false; } + const retryAdmission = await this.admitInSessionRetry({ + stepBudget: context.stepBudget, + revalidateAdmission: context.revalidateAdmission, + goalKind: context.goalKind, + goalId: context.goalId, + muxMetadata: context.options?.muxMetadata, + retryLabel: "post-compaction retry", + }); + if (retryAdmission == null) { + return false; + } // Retry the same request, but without post-compaction injection. this.preparingWorkspaceTurnMetadata = getWorkspaceTurnMuxMetadata(context.options?.muxMetadata); @@ -5857,8 +5964,9 @@ export class AgentSession { context.goalKind, context.goalId, undefined, - undefined, - context.stepBudget + retryAdmission.refuseStreamStart, + context.stepBudget, + context.revalidateAdmission ); } finally { if (this.turnPhase === TurnPhase.PREPARING) { From 59ca94a6bffb8c351df95e5c7419ed27f483ea3b Mon Sep 17 00:00:00 2001 From: Michael Suchacz <203725896+ibetitsmike@users.noreply.github.com> Date: Thu, 3 Sep 2026 21:38:49 +0000 Subject: [PATCH 18/28] Retain consumed cut evidence for a late owner claim, drop the resume once a later row is durable, carry the fallback chain across the resume --- src/common/orpc/schemas.ts | 1 + src/common/orpc/schemas/stream.ts | 8 ++ src/common/types/stream.ts | 2 + .../agentSession.queueDispatch.test.ts | 136 ++++++++++++++++++ src/node/services/agentSession.ts | 127 ++++++++++++++-- src/node/services/aiService.test.ts | 40 ++++++ src/node/services/streamManager.test.ts | 132 +++++++++++++++++ src/node/services/streamManager.ts | 92 +++++++++--- src/node/services/turnRequestBuilder.ts | 29 ++-- 9 files changed, 525 insertions(+), 42 deletions(-) diff --git a/src/common/orpc/schemas.ts b/src/common/orpc/schemas.ts index 17e054d6d6..6640e17f9a 100644 --- a/src/common/orpc/schemas.ts +++ b/src/common/orpc/schemas.ts @@ -253,6 +253,7 @@ export { ErrorEventSchema, GoalBudgetLimitedEventSchema, LanguageModelV2UsageSchema, + ModelFallbackProgressSchema, OnChatDowngradeReasonSchema, QueuedMessageChangedEventSchema, ReasoningDeltaEventSchema, diff --git a/src/common/orpc/schemas/stream.ts b/src/common/orpc/schemas/stream.ts index d2ac08ce2e..6d28002360 100644 --- a/src/common/orpc/schemas/stream.ts +++ b/src/common/orpc/schemas/stream.ts @@ -323,6 +323,12 @@ export const StreamLifecycleEventSchema = StreamLifecycleSnapshotSchema.extend({ workspaceId: z.string(), }); +// Refusal-fallback chain a turn runs under and how far along it is. A stream that resumes a cut +// turn continues this chain instead of resolving one from the model it resumes on. +export const ModelFallbackProgressSchema = ModelFallbackRecordSchema.extend({ + chain: z.array(z.string()), +}); + export const StreamAbortEventSchema = z.object({ type: z.literal("stream-abort"), workspaceId: z.string(), @@ -344,6 +350,8 @@ export const StreamAbortEventSchema = z.object({ // Steps left under the stream's ceiling at the abort; a turn cut for a queued message // resumes under this budget rather than a fresh one. stepsRemaining: z.number().int().nonnegative().optional(), + // Fallback chain state at the abort, carried into the resumed stream for the same reason. + modelFallbackProgress: ModelFallbackProgressSchema.optional(), }) .optional() .meta({ diff --git a/src/common/types/stream.ts b/src/common/types/stream.ts index c1979b0bd4..3b16f114f8 100644 --- a/src/common/types/stream.ts +++ b/src/common/types/stream.ts @@ -11,6 +11,7 @@ import type { AutoRetryScheduledEventSchema, AutoRetryStartingEventSchema, ErrorEventSchema, + ModelFallbackProgressSchema, ReasoningDeltaEventSchema, ReasoningEndEventSchema, StreamAbortReasonSchema, @@ -45,6 +46,7 @@ export type StreamStartEvent = z.infer; export type StreamDeltaEvent = z.infer; export type StreamEndEvent = z.infer; export type StreamAbortReason = z.infer; +export type ModelFallbackProgress = z.infer; export type StreamLifecyclePhase = z.infer; export type StreamLifecycleSnapshot = z.infer; export type StreamLifecycleEvent = z.infer; diff --git a/src/node/services/agentSession.queueDispatch.test.ts b/src/node/services/agentSession.queueDispatch.test.ts index 34d32ce8cc..de1e876457 100644 --- a/src/node/services/agentSession.queueDispatch.test.ts +++ b/src/node/services/agentSession.queueDispatch.test.ts @@ -1079,6 +1079,67 @@ describe("AgentSession queued message tool-call dispatch", () => { } }); + test("a resumed turn continues the cut stream's fallback chain", async () => { + const workspaceId = "queue-dispatch-stranded-fallback-chain"; + const harness = await createStreamingTurnHarness(workspaceId); + const { session, cleanup, aiEmitter, aiService, streamMessage } = harness; + const stopStream = spyOn(aiService, "stopStream").mockResolvedValue(Ok(undefined)); + // The requested model refused and the cut reached its first fallback. + const progress = { + requestedModel: TEST_MODEL, + refusedModels: [TEST_MODEL], + chain: ["openai:gpt-5-fallback", "google:gemini-fallback"], + }; + + try { + expect(harness.latestRequest().modelFallbackProgress).toBeUndefined(); + + // Cut by the loop's stop condition: the resume runs on the fallback under the cut turn's + // chain, not a chain of the fallback's own. + harness.latestRequest().onQueuedMessageStop?.({ + modelString: "openai:gpt-5-fallback", + stepsRemaining: 7, + modelFallbackProgress: progress, + }); + harness.queueCancelableWake().abort("monitor consumed"); + aiEmitter.emit("stream-end", streamEndEvent(workspaceId)); + expect(await waitForCondition(() => streamMessage.mock.calls.length === 2)).toBe(true); + expect(harness.latestRequest().modelString).toBe("openai:gpt-5-fallback"); + expect(harness.latestRequest().modelFallbackProgress).toEqual(progress); + + // Cut by a provider-tool soft stop: the abort reports the chain state the next stream gets. + session.queueMessage( + "follow up", + { model: TEST_MODEL, agentId: "exec" }, + { synthetic: true, dedupeKey: "wake:1" } + ); + aiEmitter.emit("tool-call-end", { + ...toolCallEndEvent(workspaceId), + toolName: "web_search", + providerExecuted: true, + }); + expect(stopStream).toHaveBeenCalledTimes(1); + expect(session.removeQueuedMessagesByDedupeKeyPrefix("wake:", "superseded")).toBe(1); + const abortedProgress = { ...progress, refusedModels: [TEST_MODEL, "openai:gpt-5-fallback"] }; + aiEmitter.emit("stream-abort", { + ...streamAbortEvent(workspaceId, "queued-message", 3), + metadata: { + duration: 1, + stepsRemaining: 3, + model: "google:gemini-fallback", + modelFallbackProgress: abortedProgress, + }, + }); + expect(await waitForCondition(() => streamMessage.mock.calls.length === 3)).toBe(true); + expect(harness.latestRequest().modelString).toBe("google:gemini-fallback"); + expect(harness.latestRequest().modelFallbackProgress).toEqual(abortedProgress); + } finally { + stopStream.mockRestore(); + session.dispose(); + await cleanup(); + } + }); + test("an auto-retry runs under what the failed resumed attempt left of the step budget", async () => { const workspaceId = "queue-dispatch-stranded-retry-step-budget"; const aiEmitter = new EventEmitter(); @@ -2683,6 +2744,81 @@ describe("AgentSession queued message tool-call dispatch", () => { } }); + test("a late owner claim after the continuation already ran and ended still defers", async () => { + const workspaceId = "queue-dispatch-stranded-delegated-late-claim"; + const harness = await createStreamingTurnHarness(workspaceId, { + sendOptions: { muxMetadata: WORKSPACE_TURN_CORRELATION }, + }); + const { session, cleanup, aiEmitter, streamMessage } = harness; + + try { + const wake = harness.queueCancelableWake(); + harness.latestRequest().onQueuedMessageStop?.(queuedStop()); + wake.abort("monitor consumed"); + // The owner is behind its event lock and has not asked about assistant-1 when the resume + // starts, runs, and ends. + aiEmitter.emit("stream-end", streamEndEvent(workspaceId)); + expect(await waitForCondition(() => streamMessage.mock.calls.length === 2)).toBe(true); + expect(harness.latestRequest().muxMetadata).toEqual(WORKSPACE_TURN_CORRELATION); + aiEmitter.emit("stream-end", { ...streamEndEvent(workspaceId), messageId: "assistant-2" }); + expect(await waitForCondition(() => !session.isBusy())).toBe(true); + + // The successor's own events settle the turn; the late claim on the cut must not settle it + // as failed first. The evidence is spent by that one claim. + expect( + session.claimWorkspaceTurnContinuation(WORKSPACE_TURN_CORRELATION, "assistant-1") + ).toBe(true); + expect( + session.claimWorkspaceTurnContinuation(WORKSPACE_TURN_CORRELATION, "assistant-1") + ).toBe(false); + } finally { + session.dispose(); + await cleanup(); + } + }); + + test("a queued entry whose durable row outlives its failed startup supersedes the continuation", async () => { + const workspaceId = "queue-dispatch-stranded-durable-cutter"; + const harness = await createStreamingTurnHarness(workspaceId); + const { session, cleanup, aiEmitter, streamMessage, historyService } = harness; + + try { + harness.latestRequest().onQueuedMessageStop?.(queuedStop()); + // An unrelated synthetic entry cuts the turn; its acceptance hook fails after its row is + // durable, so no stream of its own starts. + session.queueMessage( + "peer follow-up", + { model: TEST_MODEL, agentId: "exec" }, + { + synthetic: true, + agentInitiated: true, + onAccepted: () => { + throw new Error("acceptance exploded"); + }, + } + ); + aiEmitter.emit("stream-end", streamEndEvent(workspaceId)); + expect(await waitForCondition(() => !session.isBusy())).toBe(true); + await new Promise((resolve) => setTimeout(resolve, 50)); + + // The cut turn must not run the follow-up's row as its own continuation. + expect(streamMessage).toHaveBeenCalledTimes(1); + expect(session.isBusy()).toBe(false); + const history = await historyService.getHistoryFromLatestBoundary(workspaceId); + expect(history.success).toBe(true); + if (history.success) { + const last = history.data[history.data.length - 1]; + expect(last?.role).toBe("user"); + expect( + last?.parts.some((part) => part.type === "text" && part.text === "peer follow-up") + ).toBe(true); + } + } finally { + session.dispose(); + await cleanup(); + } + }); + test("an older stream-end's claim does not void a newer cut's continuation", async () => { const workspaceId = "queue-dispatch-stranded-delegated-stale-claim"; const harness = await createStreamingTurnHarness(workspaceId, { diff --git a/src/node/services/agentSession.ts b/src/node/services/agentSession.ts index 5fd5075ecc..2361a165d2 100644 --- a/src/node/services/agentSession.ts +++ b/src/node/services/agentSession.ts @@ -117,6 +117,7 @@ import { MessageQueue, cancelReasonBeforeAcceptance } from "./messageQueue"; import type { QueueClearCallbacks, QueueCutCutter } from "./messageQueue"; import { copyStreamLifecycleSnapshot, + type ModelFallbackProgress, type RuntimeStatusEvent, type StreamAbortReason, type StreamEndEvent, @@ -235,6 +236,9 @@ interface CompactionRequestMetadata { type GoalInterventionPolicy = NonNullable; +/** Consumed continuation cuts kept for a late owner claim (see consumedContinuationCuts). */ +const MAX_RETAINED_CONTINUATION_CUTS = 8; + interface AutoRetryResumeRequest { // Same-session auto-retry must preserve the full normalized request because // ACP correlation/delegation lives in transient send options that are @@ -247,6 +251,7 @@ interface AutoRetryResumeRequest { stepBudget?: number; /** The retried stream was admitted under resumeStream's revalidation; the retry repeats it. */ revalidateAdmission?: boolean; + modelFallbackProgress?: ModelFallbackProgress; } function stripGoalInterventionPolicy(options: SendMessageOptions): SendMessageOptions { @@ -372,11 +377,18 @@ interface StrandedTurnResume { options: SendMessageOptions; /** Assistant message id of the stream the cut ended (claimWorkspaceTurnContinuation). */ cutMessageId: string; + /** The owner already deferred on this cut's stream-end; no late claim is coming for it. */ + claimed?: boolean; /** * Steps the cut stream had left under its ceiling; the resumed stream runs under this budget * so a chain of cuts and resumes spends one turn's steps, not a fresh cap per resume. */ stepBudget?: number; + /** + * Fallback chain the cut stream ran under, with the refusals so far: the resumed stream continues + * it, rather than the chain its own (possibly fallback) model would resolve. + */ + modelFallbackProgress?: ModelFallbackProgress; agentInitiated?: boolean; goalKind?: GoalSyntheticMessageKind; goalId?: string; @@ -397,6 +409,7 @@ function buildStrandedTurnResume(context: { modelString: string; cutMessageId: string; stepBudget?: number; + modelFallbackProgress?: ModelFallbackProgress; options?: SendMessageOptions; agentInitiated?: boolean; goalKind?: GoalSyntheticMessageKind; @@ -428,6 +441,9 @@ function buildStrandedTurnResume(context: { }, cutMessageId: context.cutMessageId, ...(context.stepBudget != null ? { stepBudget: context.stepBudget } : {}), + ...(context.modelFallbackProgress != null + ? { modelFallbackProgress: context.modelFallbackProgress } + : {}), ...(context.agentInitiated != null ? { agentInitiated: context.agentInitiated } : {}), ...(context.goalKind != null ? { goalKind: context.goalKind } : {}), ...(context.goalId != null ? { goalId: context.goalId } : {}), @@ -801,6 +817,13 @@ export class AgentSession { // Set while the sweep's resume is running (claim through stream end); aborting it cancels only // the pre-stream window, since StreamManager unlinks the signal once a stream registers. private strandedTurnResumeInFlight: AbortController | null = null; + /** + * Cuts of delegated turns whose continuation already ran, keyed by the cut stream's message id. + * The owner settles a correlated tool-calls stream-end under its own event lock, so it can ask + * about a cut after the successor stream has come and gone; the successor's own events settle + * the turn, and this evidence keeps the late claim from settling it as failed first. + */ + private readonly consumedContinuationCuts = new Map(); /** Owner settlements for forfeited continuations that have not landed yet (settleOwedForfeits). */ private readonly owedForfeitSettlements = new Map< string, @@ -981,6 +1004,8 @@ export class AgentSession { goalId?: string; /** Step ceiling this stream runs under when it continues a cut turn (see StrandedTurnResume). */ stepBudget?: number; + /** Fallback chain this stream continues when it continues a cut turn (see StrandedTurnResume). */ + modelFallbackProgress?: ModelFallbackProgress; /** Admitted under resumeStream's revalidation; in-session retries of this stream repeat it. */ revalidateAdmission?: boolean; workspaceTurnMetadata?: Extract; @@ -1407,7 +1432,8 @@ export class AgentSession { goalKind?: GoalSyntheticMessageKind, goalId?: string, stepBudget?: number, - revalidateAdmission?: boolean + revalidateAdmission?: boolean, + modelFallbackProgress?: ModelFallbackProgress ): void { if (!options) { this.lastAutoRetryResumeRequest = undefined; @@ -1421,6 +1447,7 @@ export class AgentSession { ...(goalId != null ? { goalId } : {}), ...(stepBudget != null ? { stepBudget } : {}), ...(revalidateAdmission === true ? { revalidateAdmission: true } : {}), + ...(modelFallbackProgress != null ? { modelFallbackProgress } : {}), }; } @@ -1456,6 +1483,7 @@ export class AgentSession { goalId: request.goalId, stepBudget: request.stepBudget, revalidateAdmission: request.revalidateAdmission, + modelFallbackProgress: request.modelFallbackProgress, }); if (result.success) { if (result.data.refusedBy != null) { @@ -4088,6 +4116,14 @@ export class AgentSession { if (cancelSignal != null) { cancellationDisabled = true; } + // The durable row of anything but a continuation of the cut turn (a wake, or the same + // delegated turn) supersedes the owed continuation from here on: a failure below leaves the + // row in place, and the cut turn must not resume over it with its own identity. + if (!this.continuesOwedTurn(typedMuxMetadata)) { + this.forfeitStrandedTurnResume( + "Stranded turn resume dropped: superseded by a later durable message." + ); + } // r54: the pre-turn batch is now irrevocable — rollbackPersistedTurnRows // is never invoked past this point, so even a failure in goal sync or // acceptance leaves the payload + trigger rows durable in the transcript. @@ -4170,8 +4206,6 @@ export class AgentSession { // Synthetic/system sends (mid-stream compaction, task recovery prompts, etc.) // must not silently opt users back into auto-retry after they've disabled it. if (isManualUserMessage) { - // The user's own message supersedes any continuation owed to a stranded turn. - this.withdrawStrandedTurnResume(); // A fresh accepted user send supersedes any persisted startup-abandon // classification from previous turns. await this.clearStartupAutoRetryAbandon(); @@ -4349,6 +4383,8 @@ export class AgentSession { revalidateAdmission?: boolean; /** Step ceiling for the resumed stream when it continues a cut turn (StrandedTurnResume). */ stepBudget?: number; + /** Fallback chain the resumed stream continues when it continues a cut turn. */ + modelFallbackProgress?: ModelFallbackProgress; } ): Promise> { this.assertNotDisposed("resumeStream"); @@ -4386,7 +4422,8 @@ export class AgentSession { internal?.goalKind, internal?.goalId, internal?.stepBudget, - internal?.revalidateAdmission + internal?.revalidateAdmission, + internal?.modelFallbackProgress ); // Claim the turn before any await: the admission gates below do I/O, and a manual send // entering meanwhile must see a busy session rather than start a stream this resume @@ -4449,7 +4486,8 @@ export class AgentSession { turnThinkingOverride, refuseStreamStart, internal?.stepBudget, - internal?.revalidateAdmission + internal?.revalidateAdmission, + internal?.modelFallbackProgress ); if (!result.success) { return result; @@ -5253,7 +5291,8 @@ export class AgentSession { // the signal is, and by StreamManager right before the stream registers. refuseStreamStart?: () => boolean, stepBudget?: number, - revalidateAdmission?: boolean + revalidateAdmission?: boolean, + modelFallbackProgress?: ModelFallbackProgress ): Promise> { const isStartupAbortRequested = (): boolean => abortSignal?.aborted === true || refuseStreamStart?.() === true; @@ -5277,6 +5316,7 @@ export class AgentSession { ...(goalKind != null ? { goalKind } : {}), ...(goalId != null ? { goalId } : {}), ...(stepBudget != null ? { stepBudget } : {}), + ...(modelFallbackProgress != null ? { modelFallbackProgress } : {}), ...(revalidateAdmission === true ? { revalidateAdmission: true } : {}), providersConfig, }; @@ -5484,6 +5524,7 @@ export class AgentSession { abortSignal, refuseStreamStart, stepBudget, + modelFallbackProgress, thinkingLevel: effectiveThinkingLevel, // Orthogonal to thinking level; buildRequestHeaders gates it per model. reasoningMode: options?.reasoningMode, @@ -5511,13 +5552,14 @@ export class AgentSession { disableWorkspaceAgents: options?.disableWorkspaceAgents, strictAgentResolution: options?.strictAgentResolution, hasQueuedMessages: this.hasQueuedMessages.bind(this), - onQueuedMessageStop: ({ modelString: stoppedModelString, stepsRemaining }) => { + onQueuedMessageStop: (stop) => { if (this.activeStreamContext != null && this.activeStreamMessageId != null) { this.strandedTurnResume = buildStrandedTurnResume({ ...this.activeStreamContext, - modelString: stoppedModelString, + modelString: stop.modelString, cutMessageId: this.activeStreamMessageId, - stepBudget: stepsRemaining, + stepBudget: stop.stepsRemaining, + modelFallbackProgress: stop.modelFallbackProgress, thinkingLevelAtCut: activeTurnThinkingOverride?.pending ?? activeTurnThinkingOverride?.applied, }); @@ -5787,6 +5829,7 @@ export class AgentSession { const retryGoalKind = this.activeStreamContext?.goalKind; const retryGoalId = this.activeStreamContext?.goalId; const retryStepBudget = this.activeStreamContext?.stepBudget; + const retryModelFallbackProgress = this.activeStreamContext?.modelFallbackProgress; const retryRevalidateAdmission = this.activeStreamContext?.revalidateAdmission; const retryOptionsForResume = retryOptions ?? { model: context.modelString, @@ -5824,7 +5867,8 @@ export class AgentSession { retryGoalKind, retryGoalId, retryStepBudget, - retryRevalidateAdmission + retryRevalidateAdmission, + retryModelFallbackProgress ); this.preparingWorkspaceTurnMetadata = getWorkspaceTurnMuxMetadata( retryOptionsForResume.muxMetadata @@ -5844,7 +5888,8 @@ export class AgentSession { undefined, retryAdmission.refuseStreamStart, retryStepBudget, - retryRevalidateAdmission + retryRevalidateAdmission, + retryModelFallbackProgress ); } finally { if (this.turnPhase === TurnPhase.PREPARING) { @@ -5966,7 +6011,8 @@ export class AgentSession { undefined, retryAdmission.refuseStreamStart, context.stepBudget, - context.revalidateAdmission + context.revalidateAdmission, + context.modelFallbackProgress ); } finally { if (this.turnPhase === TurnPhase.PREPARING) { @@ -6437,7 +6483,8 @@ export class AgentSession { abortedStreamContext, payload.metadata?.model, payload.messageId, - payload.metadata?.stepsRemaining + payload.metadata?.stepsRemaining, + payload.metadata?.modelFallbackProgress ); if (!dispatchedQueuedMessage) { this.setTurnPhase(TurnPhase.IDLE); @@ -6720,6 +6767,16 @@ export class AgentSession { if (next === TurnPhase.STREAMING) { // Any stream that actually starts is the continuation the stranded turn was waiting // for. PREPARING is not enough: a dequeued entry can still be canceled before acceptance. + const consumed = this.strandedTurnResume; + const consumedCorrelation = getWorkspaceTurnMuxMetadata(consumed?.options.muxMetadata); + if (consumed != null && consumed.claimed !== true && consumedCorrelation != null) { + this.consumedContinuationCuts.set(consumed.cutMessageId, consumedCorrelation); + // Bounded like the recovery decisions: unclaimed cuts (owner gone) must not accumulate. + for (const key of this.consumedContinuationCuts.keys()) { + if (this.consumedContinuationCuts.size <= MAX_RETAINED_CONTINUATION_CUTS) break; + this.consumedContinuationCuts.delete(key); + } + } this.strandedTurnResume = undefined; // "Consecutive" counts only resume attempts that never got this far: a stream that starts // (the resume's own included) consumed the marker, so a later stranding is new work. @@ -7058,6 +7115,22 @@ export class AgentSession { return correlation != null && this.messageQueue.hasWorkspaceTurn(correlation.taskHandleId); } + /** + * Whether a send with this metadata continues the owed cut turn (a wake inherits its + * correlation from history; a workspace-turn entry must carry the same correlation) rather than + * supersede it. Vacuously true when nothing is owed. + */ + private continuesOwedTurn(muxMetadata: MuxMessageMetadata | undefined): boolean { + const owed = this.strandedTurnResume; + if (owed == null || muxMetadata?.type === "bash-monitor-wake") { + return true; + } + return hasSameWorkspaceTurnCorrelation( + getWorkspaceTurnMuxMetadata(muxMetadata), + getWorkspaceTurnMuxMetadata(owed.options.muxMetadata) + ); + } + /** * Runs removed entries' cancellation and sweeps for the owed continuation only once it has * settled, as the dequeue path does: a canceled workspace-turn entry settles its handle in @@ -7200,6 +7273,19 @@ export class AgentSession { claimWorkspaceTurnContinuation( metadata: Extract, streamEndMessageId: string + ): boolean { + const deferred = this.answerWorkspaceTurnContinuationClaim(metadata, streamEndMessageId); + // The owner claims each stream-end once, so a cut it deferred on here needs no evidence + // retained for a late claim (consumedContinuationCuts). + if (deferred && this.strandedTurnResume?.cutMessageId === streamEndMessageId) { + this.strandedTurnResume.claimed = true; + } + return deferred; + } + + private answerWorkspaceTurnContinuationClaim( + metadata: Extract, + streamEndMessageId: string ): boolean { if (hasSameWorkspaceTurnCorrelation(this.preparingWorkspaceTurnMetadata, metadata)) { return true; @@ -7231,6 +7317,16 @@ export class AgentSession { return true; } + // The continuation of this exact cut already ran (and may already have ended): its own + // stream events settle the turn, so the late claim defers rather than settling it as failed. + const consumedCorrelation = this.consumedContinuationCuts.get(streamEndMessageId); + if (consumedCorrelation != null) { + this.consumedContinuationCuts.delete(streamEndMessageId); + if (hasSameWorkspaceTurnCorrelation(consumedCorrelation, metadata)) { + return true; + } + } + const owed = this.strandedTurnResume; if ( owed == null || @@ -7426,7 +7522,8 @@ export class AgentSession { abortedStreamContext: AgentSession["activeStreamContext"], abortedModelString: string | undefined, abortedMessageId: string, - abortedStepsRemaining: number | undefined + abortedStepsRemaining: number | undefined, + abortedModelFallbackProgress: ModelFallbackProgress | undefined ): boolean { this.queuedProviderToolEndAbortInFlight = false; if (!isQueuedProviderToolEndAbort || this.deferQueuedFlushUntilAfterEdit) { @@ -7447,6 +7544,7 @@ export class AgentSession { modelString: abortedModelString ?? abortedStreamContext.modelString, cutMessageId: abortedMessageId, stepBudget: abortedStepsRemaining, + modelFallbackProgress: abortedModelFallbackProgress, thinkingLevelAtCut: this.activeTurnThinkingOverride?.pending ?? this.activeTurnThinkingOverride?.applied, }); @@ -7512,6 +7610,7 @@ export class AgentSession { abortSignal: inFlight.signal, revalidateAdmission: true, stepBudget: resume.stepBudget, + modelFallbackProgress: resume.modelFallbackProgress, }) .then((result) => { if (!result.success) { diff --git a/src/node/services/aiService.test.ts b/src/node/services/aiService.test.ts index f11e5f3c4e..2db61e8647 100644 --- a/src/node/services/aiService.test.ts +++ b/src/node/services/aiService.test.ts @@ -1421,6 +1421,46 @@ describe("AIService.streamMessage compaction boundary slicing", () => { ); }); + it("a stream continuing a cut turn keeps that turn's fallback chain, not its model's", async () => { + using xumHome = new DisposableTempDir("ai-service-fallback-continuation"); + const projectPath = path.join(xumHome.path, "project"); + await fs.mkdir(projectPath, { recursive: true }); + + const workspaceId = "workspace-fallback-continuation"; + const requestedModel = KNOWN_MODELS.SONNET.id; + const cutModel = KNOWN_MODELS.GPT.id; + const nextModel = KNOWN_MODELS.GEMINI_FLASH.id; + // The resumed model has a chain of its own that would lead back to the model that refused. + await writeMainConfig(xumHome.path, { + modelFallbacks: { + [requestedModel]: { models: [cutModel, nextModel] }, + [cutModel]: { models: [requestedModel] }, + }, + }); + const harness = createHarness( + xumHome.path, + createLocalWorkspaceMetadata(workspaceId, projectPath), + { useRequestedModelString: true } + ); + const progress = { + requestedModel, + refusedModels: [requestedModel], + chain: [cutModel, nextModel], + }; + + const result = await harness.service.streamMessage({ + messages: [createMuxMessage("latest-user", "user", "fix the issue")], + workspaceId, + modelString: cutModel, + thinkingLevel: "off", + modelFallbackProgress: progress, + }); + expect(result.success).toBe(true); + + expect(harness.startStreamCalls[0]?.modelFallback?.chain).toEqual([cutModel, nextModel]); + expect(harness.startStreamCalls[0]?.modelFallbackProgress).toEqual(progress); + }); + it("emits startup breadcrumbs as runtime-status events before stream start", async () => { using xumHome = new DisposableTempDir("ai-service-startup-breadcrumbs"); const projectPath = path.join(xumHome.path, "project"); diff --git a/src/node/services/streamManager.test.ts b/src/node/services/streamManager.test.ts index f5f09727a8..49416a5811 100644 --- a/src/node/services/streamManager.test.ts +++ b/src/node/services/streamManager.test.ts @@ -8,6 +8,7 @@ import type { ProvidersConfigMap } from "@/common/orpc/types"; import { StreamEndEventSchema, ToolCallStartEventSchema } from "@/common/orpc/schemas/stream"; import type { CompletedMessagePart, + ModelFallbackProgress, ToolCallEndEvent, ToolCallExecutionStartEvent, ToolCallStartEvent, @@ -1865,6 +1866,137 @@ describe("StreamManager - fallback construction callbacks", () => { }); }); +describe("StreamManager - fallback chain continuation", () => { + const requestedModel = KNOWN_MODELS.SONNET.id; + const cutModel = KNOWN_MODELS.GPT.id; + const nextModel = KNOWN_MODELS.GEMINI_FLASH.id; + // The requested model refused and the turn was cut while running on the first fallback. + const progress: ModelFallbackProgress = { + requestedModel, + refusedModels: [requestedModel], + chain: [cutModel, nextModel], + }; + + async function runContinuationForTests( + workspaceId: string, + streams: Array<() => AsyncGenerator> + ) { + const streamManager = new StreamManager(historyService); + Reflect.set(streamManager, "tokenTracker", { + setModel: () => Promise.resolve(undefined), + countTokens: () => Promise.resolve(0), + }); + Reflect.set(streamManager, "createTempDirForStream", () => + Promise.resolve("/tmp/fallback-continuation-tempdir") + ); + Reflect.set(streamManager, "cleanupStreamTempDir", (): void => undefined); + const errorEvents: unknown[] = []; + const streamEndEvents: Array<{ + metadata?: { + model?: string; + modelFallback?: { requestedModel: string; refusedModels: string[] }; + }; + }> = []; + onTurnEngineEvent(streamManager, "error", (data) => errorEvents.push(data)); + onTurnEngineEvent(streamManager, "stream-end", (data) => { + streamEndEvents.push(data as (typeof streamEndEvents)[number]); + }); + + const messageId = `${workspaceId}-message`; + await appendPartialAssistantForTests(workspaceId, messageId, 1); + const createStreamResult = mock( + (_request: { modelFallbackProgress?: ModelFallbackProgress }) => { + const nextStream = streams.shift(); + if (nextStream == null) { + throw new Error("createStreamResult called more often than the test provided streams"); + } + return createStreamResultForTests(nextStream(), { + inputTokens: 5, + outputTokens: 3, + totalTokens: 8, + }); + } + ); + expect(Reflect.set(streamManager, "createStreamResult", createStreamResult)).toBe(true); + const prepare = mock((nextModelString: string) => + Promise.resolve( + Ok({ + model: createTestLanguageModel(`fallback-${nextModelString}`), + modelString: nextModelString, + messages: [], + system: "fallback system", + tools: undefined, + }) + ) + ); + + const result = await streamManager.startStream( + testStartOptions({ + workspaceId, + messageId, + model: createTestLanguageModel("cut-model"), + modelString: cutModel, + tools: {}, + modelFallback: { chain: progress.chain, prepare }, + modelFallbackProgress: progress, + }) + ); + expect(result.success).toBe(true); + if (!result.success) { + throw new Error("Expected the continuation stream to start"); + } + await result.data.completion; + return { errorEvents, streamEndEvents, createStreamResult, prepare }; + } + + const refusal = () => + (async function* () { + await Promise.resolve(); + yield { type: "finish", finishReason: "content-filter", rawFinishReason: "refusal" }; + })(); + const answer = () => + (async function* () { + await Promise.resolve(); + yield { type: "text-delta", text: "answer" }; + yield { type: "finish", finishReason: "stop" }; + })(); + + test("a refusal on the resumed stream moves on to the entry after the resumed model", async () => { + const run = await runContinuationForTests("fallback-continuation-refusal-workspace", [ + refusal, + answer, + ]); + + expect(run.errorEvents).toHaveLength(0); + // Not back to the chain's first entry (the resumed model itself) or a chain of its own. + expect(run.prepare.mock.calls.map((call) => call[0])).toEqual([nextModel]); + // Each request carries the chain state a further cut would report: the cut turn's at + // first, then the hop's. + expect(run.createStreamResult.mock.calls[0]?.[0].modelFallbackProgress).toEqual(progress); + expect(run.createStreamResult.mock.calls[1]?.[0].modelFallbackProgress).toEqual({ + ...progress, + refusedModels: [requestedModel, cutModel], + }); + expect(run.streamEndEvents[0]?.metadata?.model).toBe(nextModel); + expect(run.streamEndEvents[0]?.metadata?.modelFallback).toEqual({ + requestedModel, + refusedModels: [requestedModel, cutModel], + }); + }); + + test("a resumed stream that answers records the cut turn's fallback", async () => { + const run = await runContinuationForTests("fallback-continuation-answer-workspace", [answer]); + + expect(run.errorEvents).toHaveLength(0); + expect(run.prepare).not.toHaveBeenCalled(); + expect(run.streamEndEvents[0]?.metadata?.model).toBe(cutModel); + expect(run.streamEndEvents[0]?.metadata?.modelFallback).toEqual({ + requestedModel, + refusedModels: [requestedModel], + }); + }); +}); + describe("StreamManager - sequential tool execution", () => { interface Deferred { promise: Promise; diff --git a/src/node/services/streamManager.ts b/src/node/services/streamManager.ts index 9b28678b3c..4b21397f7b 100644 --- a/src/node/services/streamManager.ts +++ b/src/node/services/streamManager.ts @@ -35,6 +35,7 @@ import type { ReasoningDeltaEvent, ReasoningEndEvent, CompletedMessagePart, + ModelFallbackProgress, WorkflowRunAttachedEvent, } from "@/common/types/stream"; @@ -242,6 +243,17 @@ export function createTurnCompletionController(): TurnCompletionController { }; } +/** + * What a stream reports when its loop stops on behalf of a queued tool-end message: the model + * that reached the cut (a configured fallback may have swapped mid-turn), the steps left under + * its ceiling, and the fallback chain it was running under, for the resumed stream to continue. + */ +export interface QueuedMessageStop { + modelString: string; + stepsRemaining: number; + modelFallbackProgress?: ModelFallbackProgress; +} + // Request-construction options shared by the primary turn and model-fallback // hops (fallbacks rebuild these from the prepared fallback request). interface StreamRequestOptions { @@ -255,8 +267,9 @@ interface StreamRequestOptions { callSettingsOverrides?: ResolvedCallSettingsOverrides; toolPolicy?: ToolPolicy; hasQueuedMessages?: (dispatchMode?: "tool-end" | "turn-end") => boolean; - onQueuedMessageStop?: (stop: { modelString: string; stepsRemaining: number }) => void; + onQueuedMessageStop?: (stop: QueuedMessageStop) => void; stepBudget?: number; + modelFallbackProgress?: ModelFallbackProgress; headers?: Record; onChunk?: StreamTextOnChunk; onStepMessages?: (messages: ModelMessage[]) => void; @@ -309,17 +322,20 @@ interface StreamRequestConfig { /** * Invoked when the loop stops on behalf of a queued tool-end message (and not * because a required tool completed). The session uses it to resume the turn - * if that queued message is later withdrawn instead of starting a turn. Carries the - * model that reached the cut, which a configured fallback may have swapped mid-turn, and - * the steps left under this request's budget for the resumed stream to run under. + * if that queued message is later withdrawn instead of starting a turn. */ - onQueuedMessageStop?: (stop: { modelString: string; stepsRemaining: number }) => void; + onQueuedMessageStop?: (stop: QueuedMessageStop) => void; /** * Step ceiling for this stream instead of MAX_STREAM_STEPS. A stream resuming a turn cut * for a queued message inherits the cut stream's remaining steps, so cut plus resumes * share one turn's ceiling; without it, every resume would restart the full cap. */ stepBudget?: number; + /** + * Fallback chain state this request runs under (the fallback's request replaces the + * original's), reported at a queued-message cut so the resumed stream continues the chain. + */ + modelFallbackProgress?: ModelFallbackProgress; /** Optional hook for callers that need chunk-level visibility during streaming. */ onChunk?: StreamTextOnChunk; /** Optional hook for callers that need the live prepared step transcript. */ @@ -439,6 +455,19 @@ export interface ModelFallbackOptions { ) => Promise>; } +/** Snapshot of a stream's fallback chain state for a resumed stream to continue from. */ +function modelFallbackProgressOf( + state: WorkspaceStreamInfo["modelFallback"] +): ModelFallbackProgress | undefined { + return state == null + ? undefined + : { + requestedModel: state.requestedModel, + refusedModels: [...state.refusedModels], + chain: state.options.chain, + }; +} + function isKnownProviderName(provider: string): provider is keyof typeof PROVIDER_DEFINITIONS { return Object.hasOwn(PROVIDER_DEFINITIONS, provider); } @@ -1974,6 +2003,7 @@ export class StreamManager { contextProviderMetadata, model: streamInfo.model, stepsRemaining: this.remainingStepBudget(streamInfo), + modelFallbackProgress: modelFallbackProgressOf(streamInfo.modelFallback), }, abortReason, abandonPartial, @@ -2114,6 +2144,7 @@ export class StreamManager { hasQueuedMessages, onQueuedMessageStop, stepBudget, + modelFallbackProgress, headers, onChunk, onStepMessages, @@ -2167,6 +2198,7 @@ export class StreamManager { hasQueuedMessages, onQueuedMessageStop, stepBudget, + modelFallbackProgress, onChunk, onStepMessages, toolPolicy, @@ -2181,7 +2213,12 @@ export class StreamManager { private createStopWhenCondition( request: Pick< StreamRequestConfig, - "hasQueuedMessages" | "onQueuedMessageStop" | "toolPolicy" | "modelString" | "stepBudget" + | "hasQueuedMessages" + | "onQueuedMessageStop" + | "toolPolicy" + | "modelString" + | "stepBudget" + | "modelFallbackProgress" > ): Array> { const stepBudget = request.stepBudget ?? MAX_STREAM_STEPS; @@ -2239,6 +2276,7 @@ export class StreamManager { request.onQueuedMessageStop?.({ modelString: request.modelString, stepsRemaining: stepBudget - stepsSpent, + modelFallbackProgress: request.modelFallbackProgress, }); } return true; @@ -2442,8 +2480,23 @@ export class StreamManager { } = options; const stepTracker: StepMessageTracker = {}; const metadataModel = this.resolveMetadataModel(modelString, options.providersConfigSnapshot); + // A stream continuing a cut turn picks the chain up where the cut left it: the requested + // model and refusals are the cut turn's, and a refusal here moves on to the next entry. + const carried = options.modelFallbackProgress; + const modelFallbackState: WorkspaceStreamInfo["modelFallback"] = + modelFallback && modelFallback.chain.length > 0 + ? { + options: modelFallback, + requestedModel: carried?.requestedModel ?? normalizeToCanonical(modelString), + refusedModels: [...(carried?.refusedModels ?? [])], + // Pre-wrap inputs (NOT request.maxOutputTokens, which may already + // carry call-settings overrides for the original model). + original: { maxOutputTokens }, + } + : undefined; const request = this.buildStreamRequestConfig({ ...options, + modelFallbackProgress: modelFallbackProgressOf(modelFallbackState), onToolExecutionStart: (toolCallId) => this.handleToolExecutionStart(workspaceId, messageId, toolCallId), }); @@ -2477,22 +2530,22 @@ export class StreamManager { model: modelString, metadataModel, thinkingLevel, - initialMetadata, + // The resumed message answers on a fallback because the cut turn's requested model + // refused; record that as the swap would have. + initialMetadata: + modelFallbackState != null && modelFallbackState.refusedModels.length > 0 + ? { + ...initialMetadata, + modelFallback: { + requestedModel: modelFallbackState.requestedModel, + refusedModels: [...modelFallbackState.refusedModels], + }, + } + : initialMetadata, toolModelUsages: [], didRetryPreviousResponseIdAtStep: false, didRetryAfterEmptyOutput: false, - ...(modelFallback && modelFallback.chain.length > 0 - ? { - modelFallback: { - options: modelFallback, - requestedModel: normalizeToCanonical(modelString), - refusedModels: [], - // Pre-wrap inputs (NOT request.maxOutputTokens, which may already - // carry call-settings overrides for the original model). - original: { maxOutputTokens }, - }, - } - : {}), + ...(modelFallbackState != null ? { modelFallback: modelFallbackState } : {}), stepTracker, receivedTerminalEvent: false, currentStepStartIndex: 0, @@ -3232,6 +3285,7 @@ export class StreamManager { hasQueuedMessages: streamInfo.request.hasQueuedMessages, onQueuedMessageStop: streamInfo.request.onQueuedMessageStop, stepBudget, + modelFallbackProgress: modelFallbackProgressOf(fallbackState), headers: prepared.data.headers, onChunk: streamInfo.request.onChunk, onStepMessages: streamInfo.request.onStepMessages, diff --git a/src/node/services/turnRequestBuilder.ts b/src/node/services/turnRequestBuilder.ts index 9489975507..604c08cbbf 100644 --- a/src/node/services/turnRequestBuilder.ts +++ b/src/node/services/turnRequestBuilder.ts @@ -50,6 +50,7 @@ import type { StreamManager } from "./streamManager"; import { markProviderMetadataCostsIncluded, type ModelFallbackOptions, + type QueuedMessageStop, type StreamTextOnChunk, type TurnCompletion, type TurnExecutionOptions, @@ -220,7 +221,7 @@ export function resolveXumToolScope( } import type { PostCompactionAttachment } from "@/common/types/attachment"; -import type { ErrorEvent } from "@/common/types/stream"; +import type { ErrorEvent, ModelFallbackProgress } from "@/common/types/stream"; import type { ToolPolicy } from "@/common/utils/tools/toolPolicy"; import type { FileState } from "@/node/services/agentSession"; import type { ActiveTurnThinkingOverride } from "@/node/services/thinkingOverride"; @@ -274,9 +275,14 @@ export interface StreamMessageOptions { disableWorkspaceAgents?: boolean; hasQueuedMessages?: (dispatchMode?: "tool-end" | "turn-end") => boolean; /** Fires when the model loop stops solely on behalf of a queued tool-end message. */ - onQueuedMessageStop?: (stop: { modelString: string; stepsRemaining: number }) => void; + onQueuedMessageStop?: (stop: QueuedMessageStop) => void; /** Step ceiling for this stream; a stream resuming a cut turn runs under the cut's remainder. */ stepBudget?: number; + /** + * Fallback chain of the cut turn this stream continues, used in place of the chain `model` + * would resolve: the resumed model may be a fallback whose own chain is unrelated. + */ + modelFallbackProgress?: ModelFallbackProgress; /** * Pull-based startup refusal (a goal admission probe with no push into abortSignal), rechecked * by StreamManager right before the stream registers. @@ -748,6 +754,7 @@ export class TurnRequestBuilder { hasQueuedMessages, onQueuedMessageStop, stepBudget, + modelFallbackProgress, refuseStreamStart, openaiTruncationModeOverride, muxMetadata, @@ -2717,13 +2724,16 @@ export class TurnRequestBuilder { // a cross-typed Coder instance (coder:openai/x, type anthropic) must use // its own gateway-scoped chain, never the direct provider's. Task // children can opt out via taskOnRefusal: "fail" (see - // resolveWorkspaceModelFallbackChain). - const modelFallbackChain = resolveWorkspaceModelFallbackChain( - this.dependencies.config.loadConfigOrDefault(), - workspaceId, - modelString, - this.dependencies.providerService.getConfig() - ); + // resolveWorkspaceModelFallbackChain). A stream continuing a cut turn keeps that turn's + // chain instead. + const modelFallbackChain = + modelFallbackProgress?.chain ?? + resolveWorkspaceModelFallbackChain( + this.dependencies.config.loadConfigOrDefault(), + workspaceId, + modelString, + this.dependencies.providerService.getConfig() + ); // Lazily rebuilds the per-model slice of this pipeline (model creation, // provider-specific message prep, provider options, headers, parameter @@ -2871,6 +2881,7 @@ export class TurnRequestBuilder { hasQueuedMessages, onQueuedMessageStop, stepBudget, + modelFallbackProgress, refuseStreamStart, workspaceName: metadata.name, thinkingLevel: streamThinkingLevel, From 442a7ee7ea28b9dca11c2eba778e9e469e56382f Mon Sep 17 00:00:00 2001 From: Michael Suchacz <203725896+ibetitsmike@users.noreply.github.com> Date: Thu, 3 Sep 2026 22:12:37 +0000 Subject: [PATCH 19/28] Hand a failed attempt's model and chain state to its retry, carry the interrupted turn's remainder through mid-stream compaction --- src/common/types/message.ts | 7 ++ .../agentSession.autoCompaction.test.ts | 20 ++++- ...gentSession.continueMessageAgentId.test.ts | 68 +++++++++++++- .../agentSession.queueDispatch.test.ts | 88 +++++++++++++++++++ src/node/services/agentSession.ts | 85 +++++++++++++++++- src/node/services/streamManager.ts | 10 +++ 6 files changed, 272 insertions(+), 6 deletions(-) diff --git a/src/common/types/message.ts b/src/common/types/message.ts index 128d63f0b0..4315641070 100644 --- a/src/common/types/message.ts +++ b/src/common/types/message.ts @@ -9,6 +9,7 @@ import type { } from "@/common/constants/contextBoundary"; import type { GoalSyntheticMessageKind } from "@/constants/goals"; import type { SendMessageOptions } from "@/common/orpc/types"; +import type { ModelFallbackProgress } from "./stream"; import { withLegacyPtcExclusiveMirror } from "@/common/constants/experiments"; import type { z } from "zod"; import type { AgentMode } from "./mode"; @@ -220,6 +221,12 @@ export interface CompactionFollowUpRequest extends CompactionFollowUpInput, Pres goalId?: string; /** Internal dispatch guardrails for crash-safe follow-up recovery. */ dispatchOptions?: CompactionFollowUpDispatchOptions; + /** + * What the turn interrupted for mid-stream compaction had left of its step ceiling, and the + * fallback chain state it reached: the follow-up continues that turn, not a fresh one. + */ + stepBudget?: number; + modelFallbackProgress?: ModelFallbackProgress; /** * Open delegated workspace-turn correlation captured before on-send * compaction consumed this follow-up (e.g. a bash-monitor wake continuing a diff --git a/src/node/services/agentSession.autoCompaction.test.ts b/src/node/services/agentSession.autoCompaction.test.ts index 0161cb9d1e..f3740f7bb9 100644 --- a/src/node/services/agentSession.autoCompaction.test.ts +++ b/src/node/services/agentSession.autoCompaction.test.ts @@ -1160,8 +1160,14 @@ describe("AgentSession on-send auto-compaction snapshot deferral", () => { session.dispose(); }); - test("hides default follow-up sentinel in mid-stream auto-compaction prompts", async () => { + test("mid-stream auto-compaction hides the default follow-up sentinel and hands over the interrupted turn's remainder", async () => { const workspaceId = "ws-auto-compaction-mid-stream-sentinel"; + // The interrupted stream had already moved down its fallback chain and spent steps. + const interruptedProgress = { + requestedModel: "openai:gpt-4o", + refusedModels: ["openai:gpt-4o"], + chain: ["openai:gpt-4o-fallback"], + }; const { historyService, cleanup } = await createTestHistoryService(); historyCleanup = cleanup; @@ -1211,6 +1217,11 @@ describe("AgentSession on-send auto-compaction snapshot deferral", () => { workspaceId, messageId: "assistant-mid-stream", abortReason: "system", + metadata: { + model: "openai:gpt-4o-fallback", + stepsRemaining: 7, + modelFallbackProgress: interruptedProgress, + }, }); return Promise.resolve(Ok(undefined)); @@ -1309,6 +1320,13 @@ describe("AgentSession on-send auto-compaction snapshot deferral", () => { workspaceTurnMetadata ); expect(compactionRequestMetadata.parsed.followUpContent?.agentInitiated).toBe(true); + // The follow-up continues the interrupted turn: on the model it reached, under what it had + // left of the ceiling, with the refusals so far. + expect(compactionRequestMetadata.parsed.followUpContent).toMatchObject({ + model: "openai:gpt-4o-fallback", + stepBudget: 7, + modelFallbackProgress: interruptedProgress, + }); const compactionRequestText = compactionRequestMessage?.parts.find((part) => part.type === "text")?.text ?? ""; diff --git a/src/node/services/agentSession.continueMessageAgentId.test.ts b/src/node/services/agentSession.continueMessageAgentId.test.ts index a0e0db679a..434f8298c9 100644 --- a/src/node/services/agentSession.continueMessageAgentId.test.ts +++ b/src/node/services/agentSession.continueMessageAgentId.test.ts @@ -22,6 +22,15 @@ type SendMessageResult = interface AutoRetryResumeRequest { options: SendMessageOptions; agentInitiated?: boolean; + stepBudget?: number; + modelFallbackProgress?: unknown; +} + +interface SendInternal { + synthetic?: boolean; + agentInitiated?: boolean; + stepBudget?: number; + modelFallbackProgress?: unknown; } interface SessionInternals { @@ -29,7 +38,7 @@ interface SessionInternals { sendMessage: ( message: string, options?: SendOptions, - internal?: { synthetic?: boolean; agentInitiated?: boolean } + internal?: SendInternal ) => Promise; scheduleStartupRecovery: () => void; startupRecoveryPromise: Promise | null; @@ -266,6 +275,63 @@ describe("AgentSession continue-message agentId fallback", () => { expect(internals.lastAutoRetryResumeRequest?.agentInitiated).toBe(true); }); + test("dispatchPendingFollowUp continues the interrupted turn's step budget and fallback chain", async () => { + const progress = { + requestedModel: "anthropic:claude-sonnet-4-5", + refusedModels: ["anthropic:claude-sonnet-4-5"], + chain: ["openai:gpt-4o", "google:gemini-fallback"], + }; + const dispatched: SendInternal[] = []; + const { internals } = await createSession([ + compactionSummaryMessage("summary-remainder", { + text: "Continue", + model: "openai:gpt-4o", + agentId: "exec", + stepBudget: 7, + modelFallbackProgress: progress, + }), + ]); + internals.sendMessage = mock( + (_message: string, _options?: SendOptions, internal?: SendInternal) => { + dispatched.push(internal ?? {}); + return Promise.resolve({ success: true as const }); + } + ); + + await internals.dispatchPendingFollowUp(); + + expect(dispatched[0]).toMatchObject({ stepBudget: 7, modelFallbackProgress: progress }); + expect(internals.lastAutoRetryResumeRequest?.stepBudget).toBe(7); + expect(internals.lastAutoRetryResumeRequest?.modelFallbackProgress).toEqual(progress); + }); + + test("dispatchPendingFollowUp drops a malformed persisted remainder", async () => { + const dispatched: SendInternal[] = []; + const { internals } = await createSession([ + compactionSummaryMessage("summary-malformed-remainder", { + text: "Continue", + model: "openai:gpt-4o", + agentId: "exec", + stepBudget: "seven" as unknown as number, + modelFallbackProgress: { + requestedModel: 1, + } as unknown as CompactionFollowUpRequest["modelFallbackProgress"], + }), + ]); + internals.sendMessage = mock( + (_message: string, _options?: SendOptions, internal?: SendInternal) => { + dispatched.push(internal ?? {}); + return Promise.resolve({ success: true as const }); + } + ); + + await internals.dispatchPendingFollowUp(); + + expect(dispatched).toHaveLength(1); + expect(dispatched[0]?.stepBudget).toBeUndefined(); + expect(dispatched[0]?.modelFallbackProgress).toBeUndefined(); + }); + test("dispatchPendingFollowUp forwards strictAgentResolution to the resumed turn", async () => { let dispatchedOptions: SendOptions | undefined; const { internals } = await createSession([ diff --git a/src/node/services/agentSession.queueDispatch.test.ts b/src/node/services/agentSession.queueDispatch.test.ts index de1e876457..7039245881 100644 --- a/src/node/services/agentSession.queueDispatch.test.ts +++ b/src/node/services/agentSession.queueDispatch.test.ts @@ -1214,6 +1214,94 @@ describe("AgentSession queued message tool-call dispatch", () => { } }); + test("an auto-retry continues from the model the failed resumed attempt reached", async () => { + const workspaceId = "queue-dispatch-stranded-retry-fallback-chain"; + const aiEmitter = new EventEmitter(); + const progressAtCut = { + requestedModel: TEST_MODEL, + refusedModels: [TEST_MODEL], + chain: ["openai:gpt-5-fallback", "google:gemini-fallback"], + }; + // The resumed attempt started on the first fallback, which refused too; the second fallback + // then failed with a retryable error. + const progressAtFailure = { + ...progressAtCut, + refusedModels: [TEST_MODEL, "openai:gpt-5-fallback"], + }; + let streams = 0; + const streamMessage = mock((_options: StreamMessageOptions) => { + streams += 1; + aiEmitter.emit("stream-start", streamStartEvent(workspaceId)); + return Promise.resolve( + Ok( + streams === 2 + ? { + messageId: "assistant-2", + completion: Promise.resolve({ + status: "failed" as const, + streamError: { + messageId: "assistant-2", + error: "provider closed the connection", + errorType: "api" as const, + }, + stepsRemaining: 2, + modelString: "google:gemini-fallback", + modelFallbackProgress: progressAtFailure, + }), + } + : createStartedTurnHandle(`assistant-${streams}`) + ) + ); + }); + const { session, cleanup, historyService } = await createAgentSessionHarness({ + workspaceId, + aiEmitter, + aiServiceOverrides: { streamMessage: streamMessage as unknown as AIService["streamMessage"] }, + }); + + try { + const sent = await session.sendMessage("run the checks", { + model: TEST_MODEL, + agentId: "exec", + }); + expect(sent.success).toBe(true); + await historyService.appendToHistory( + workspaceId, + createMuxMessage("assistant-1", "assistant", "ran the checks", { timestamp: Date.now() }) + ); + const controller = new AbortController(); + session.queueMessage( + "Background monitor wake", + { model: TEST_MODEL, agentId: "exec" }, + { + synthetic: true, + agentInitiated: true, + cancelState: { canceledBeforeAcceptance: false }, + cancelSignal: controller.signal, + onCanceled: () => undefined, + } + ); + streamMessage.mock.calls[0]?.[0].onQueuedMessageStop?.({ + modelString: "openai:gpt-5-fallback", + stepsRemaining: 5, + modelFallbackProgress: progressAtCut, + }); + controller.abort("monitor consumed"); + aiEmitter.emit("stream-end", streamEndEvent(workspaceId)); + expect(await waitForCondition(() => streamMessage.mock.calls.length === 2)).toBe(true); + expect(streamMessage.mock.calls[1]?.[0].modelString).toBe("openai:gpt-5-fallback"); + + // The retry picks the chain up where the failed attempt left it instead of re-running the + // first fallback's refusal. + expect(await waitForCondition(() => streamMessage.mock.calls.length === 3, 4_000)).toBe(true); + expect(streamMessage.mock.calls[2]?.[0].modelString).toBe("google:gemini-fallback"); + expect(streamMessage.mock.calls[2]?.[0].modelFallbackProgress).toEqual(progressAtFailure); + } finally { + session.dispose(); + await cleanup(); + } + }); + test("an auto-retry is abandoned when the failed resumed attempt spent the step budget", async () => { const workspaceId = "queue-dispatch-stranded-retry-step-budget-spent"; const aiEmitter = new EventEmitter(); diff --git a/src/node/services/agentSession.ts b/src/node/services/agentSession.ts index 2361a165d2..9890cdfa66 100644 --- a/src/node/services/agentSession.ts +++ b/src/node/services/agentSession.ts @@ -45,7 +45,7 @@ import { SendMessageOptionsSchema, SkillNameSchema, } from "@/common/orpc/schemas"; -import { ToolPolicySchema } from "@/common/orpc/schemas/stream"; +import { ModelFallbackProgressSchema, ToolPolicySchema } from "@/common/orpc/schemas/stream"; import { normalizeAgentId, resolvePersistedAgentIdCandidates } from "@/common/utils/agentIds"; import { buildStreamErrorEventData, @@ -3195,6 +3195,10 @@ export class AgentSession { /** Goal identity persisted alongside goalKind so chat-tail reconciliation can scope the row. */ goalId?: string; startStreamInBackground?: boolean; + /** For a send continuing an interrupted turn: what that turn had left of its ceiling. */ + stepBudget?: number; + /** For a send continuing an interrupted turn: the fallback chain state it reached. */ + modelFallbackProgress?: ModelFallbackProgress; onAccepted?: () => Promise | void; onAcceptedPreStreamFailure?: (error: SendMessageError) => Promise | void; onCanceled?: (reason: string) => Promise | void; @@ -4216,7 +4220,15 @@ export class AgentSession { // Same-session retry should resume the exact accepted request we just finalized // in history, even if runtime warmup fails before streamWithHistory() starts. - this.setAutoRetryResumeState(optionsForStream, agentInitiated, goalKind, internal?.goalId); + this.setAutoRetryResumeState( + optionsForStream, + agentInitiated, + goalKind, + internal?.goalId, + internal?.stepBudget, + undefined, + internal?.modelFallbackProgress + ); try { await internal?.onAccepted?.(); } catch (error) { @@ -4300,7 +4312,11 @@ export class AgentSession { preparedTurnAbortController.signal, goalKind, internal?.goalId, - turnThinkingOverride + turnThinkingOverride, + undefined, + internal?.stepBudget, + undefined, + internal?.modelFallbackProgress ); if (streamResult.success && preparedTurnAbortController.signal.aborted) { await notifyAcceptedPreStreamFailure( @@ -4809,12 +4825,18 @@ export class AgentSession { goalId?: string; muxMetadata?: MuxMessageMetadata; workspaceTurnMetadata?: Extract; + stepBudget?: number; + modelFallbackProgress?: ModelFallbackProgress; }): CompactionFollowUpRequest { const followUp: CompactionFollowUpRequest = { text: params.messageText, model: params.modelForStream, agentId: params.options.agentId, ...pickPreservedSendOptions(params.options), + ...(params.stepBudget != null ? { stepBudget: params.stepBudget } : {}), + ...(params.modelFallbackProgress != null + ? { modelFallbackProgress: params.modelFallbackProgress } + : {}), }; if (params.agentInitiated === true) { @@ -5075,6 +5097,9 @@ export class AgentSession { goalId: streamContext.goalId, modelForStream: streamContext.modelString, muxMetadata: streamContext.workspaceTurnMetadata, + // The abort handler left the interrupted stream's remainder on this context. + stepBudget: streamContext.stepBudget, + modelFallbackProgress: streamContext.modelFallbackProgress, }); // Waterfall hook point: see the on-send compaction.prepare run above. await eventSpine.run("compaction.prepare", { @@ -5260,6 +5285,27 @@ export class AgentSession { this.activeStreamContext.stepBudget = outcome.stepsRemaining; } } + // Likewise the chain: an attempt that moved down its fallback chain hands the retry the + // model it reached and the refusals so far, so no refused hop is attempted again. + const progress = outcome.modelFallbackProgress; + if (progress != null && progress.refusedModels.length > 0 && outcome.modelString != null) { + const retryRequest = this.lastAutoRetryResumeRequest; + if (retryRequest != null) { + this.lastAutoRetryResumeRequest = { + ...retryRequest, + options: { ...retryRequest.options, model: outcome.modelString }, + modelFallbackProgress: progress, + }; + } + const context = this.activeStreamContext; + if (context != null) { + context.modelString = outcome.modelString; + context.modelFallbackProgress = progress; + if (context.options != null) { + context.options = { ...context.options, model: outcome.modelString }; + } + } + } try { await this.handleStreamError(outcome.streamError); } finally { @@ -6426,6 +6472,19 @@ export class AgentSession { const failedUserMessageId = this.activeStreamUserMessageId; const hadCompactionRequest = this.activeCompactionRequest !== undefined; const abortedStreamContext = this.activeStreamContext; + // Whoever continues the interrupted turn from this context (mid-stream compaction holds a + // reference to it) continues under what the aborted stream left of the ceiling and from + // the chain state it reached, as a resume or retry would. + if (abortedStreamContext != null) { + if (payload.metadata?.stepsRemaining != null) { + abortedStreamContext.stepBudget = payload.metadata.stepsRemaining; + } + const abortedProgress = payload.metadata?.modelFallbackProgress; + if (abortedProgress != null && payload.metadata?.model != null) { + abortedStreamContext.modelFallbackProgress = abortedProgress; + abortedStreamContext.modelString = payload.metadata.model; + } + } const abortReason = "abortReason" in payload ? payload.abortReason : undefined; // The soft stop is recognized by its own reason, not by the in-flight flag alone: a hard // "system" stop (task_stop, interrupt cascade, workflow timeout) can land while the soft @@ -8237,6 +8296,17 @@ export class AgentSession { options.muxMetadata = metadata; } + // Same raw JSON boundary: the interrupted turn's remainder is optional and dropped if malformed + // (the follow-up then runs under the default ceiling and its model's own chain). + const persistedStepBudget = + typeof followUp.stepBudget === "number" && Number.isInteger(followUp.stepBudget) + ? Math.max(0, followUp.stepBudget) + : undefined; + const persistedFallbackProgress = + followUp.modelFallbackProgress != null + ? ModelFallbackProgressSchema.safeParse(followUp.modelFallbackProgress) + : undefined; + // The compaction summary is now the source of truth for the next live resume // request. Pre-arm retry state from the reconstructed follow-up so failures // before stream startup do not fall back to the already-completed compact turn. @@ -8244,7 +8314,10 @@ export class AgentSession { options, followUp.agentInitiated, persistedGoalKind, - persistedGoalId + persistedGoalId, + persistedStepBudget, + undefined, + persistedFallbackProgress?.success ? persistedFallbackProgress.data : undefined ); // Await sendMessage to ensure the follow-up is persisted before returning. @@ -8264,6 +8337,10 @@ export class AgentSession { // Codex P1 (PRRT_kwDOPxxmWM6cPuMw): re-derived admission guard for the // redispatched goal turn (see buildGoalRedispatchAdmission above). admissionStale: followUpAdmissionStale, + stepBudget: persistedStepBudget, + modelFallbackProgress: persistedFallbackProgress?.success + ? persistedFallbackProgress.data + : undefined, }); if (!sendResult.success) { // A stale-admission refusal is the idle rule (or a goal transition) diff --git a/src/node/services/streamManager.ts b/src/node/services/streamManager.ts index 4b21397f7b..e8565b18a8 100644 --- a/src/node/services/streamManager.ts +++ b/src/node/services/streamManager.ts @@ -215,6 +215,12 @@ export type TurnCompletion = * Absent when the attempt failed before its loop ran a step. */ stepsRemaining?: number; + /** + * Model the failed attempt ran on and the fallback chain state it reached (a refusal may + * have moved it down the chain); a retry continues from there rather than repeating hops. + */ + modelString?: string; + modelFallbackProgress?: ModelFallbackProgress; }; export interface TurnStreamHandle { @@ -4261,6 +4267,8 @@ export class StreamManager { status: "failed", streamError: persistedPayload, stepsRemaining: this.remainingStepBudget(streamInfo), + modelString: streamInfo.model, + modelFallbackProgress: modelFallbackProgressOf(streamInfo.modelFallback), }; } @@ -5546,6 +5554,8 @@ export class StreamManager { status: "failed", streamError: persistedPayload, stepsRemaining: this.remainingStepBudget(streamInfo), + modelString: streamInfo.model, + modelFallbackProgress: modelFallbackProgressOf(streamInfo.modelFallback), }; // Wait for the stream processing to complete (cleanup) From 3e52e2672051c640902ffa409abb7478c446711e Mon Sep 17 00:00:00 2001 From: Michael Suchacz <203725896+ibetitsmike@users.noreply.github.com> Date: Thu, 3 Sep 2026 22:41:31 +0000 Subject: [PATCH 20/28] Carry admission revalidation through the compaction handoff and drop a follow-up whose turn spent its budget --- src/common/types/message.ts | 6 ++- .../agentSession.autoCompaction.test.ts | 11 +++-- ...gentSession.continueMessageAgentId.test.ts | 41 +++++++++++++++++-- src/node/services/agentSession.ts | 23 +++++++++-- 4 files changed, 69 insertions(+), 12 deletions(-) diff --git a/src/common/types/message.ts b/src/common/types/message.ts index 4315641070..ec18f52883 100644 --- a/src/common/types/message.ts +++ b/src/common/types/message.ts @@ -222,11 +222,13 @@ export interface CompactionFollowUpRequest extends CompactionFollowUpInput, Pres /** Internal dispatch guardrails for crash-safe follow-up recovery. */ dispatchOptions?: CompactionFollowUpDispatchOptions; /** - * What the turn interrupted for mid-stream compaction had left of its step ceiling, and the - * fallback chain state it reached: the follow-up continues that turn, not a fresh one. + * What the turn interrupted for mid-stream compaction had left of its step ceiling, the + * fallback chain state it reached, and whether it ran under admission revalidation: the + * follow-up continues that turn, not a fresh one. */ stepBudget?: number; modelFallbackProgress?: ModelFallbackProgress; + revalidateAdmission?: boolean; /** * Open delegated workspace-turn correlation captured before on-send * compaction consumed this follow-up (e.g. a bash-monitor wake continuing a diff --git a/src/node/services/agentSession.autoCompaction.test.ts b/src/node/services/agentSession.autoCompaction.test.ts index f3740f7bb9..243a4baf64 100644 --- a/src/node/services/agentSession.autoCompaction.test.ts +++ b/src/node/services/agentSession.autoCompaction.test.ts @@ -1286,14 +1286,18 @@ describe("AgentSession on-send auto-compaction snapshot deferral", () => { ownerWorkspaceId: "parent-mid-stream-compaction", turnId: "turn-mid-stream-compaction", } as const; - const result = await session.sendMessage( - "hello", + // The interrupted turn is a revalidated resume (a stranded delegated turn's continuation). + await historyService.appendToHistory( + workspaceId, + createMuxMessage("user-hello", "user", "hello", { timestamp: Date.now() }) + ); + const result = await session.resumeStream( { model: "openai:gpt-4o", agentId: "exec", muxMetadata: workspaceTurnMetadata, }, - { agentInitiated: true } + { agentInitiated: true, revalidateAdmission: true } ); expect(result.success).toBe(true); @@ -1326,6 +1330,7 @@ describe("AgentSession on-send auto-compaction snapshot deferral", () => { model: "openai:gpt-4o-fallback", stepBudget: 7, modelFallbackProgress: interruptedProgress, + revalidateAdmission: true, }); const compactionRequestText = diff --git a/src/node/services/agentSession.continueMessageAgentId.test.ts b/src/node/services/agentSession.continueMessageAgentId.test.ts index 434f8298c9..e481238c00 100644 --- a/src/node/services/agentSession.continueMessageAgentId.test.ts +++ b/src/node/services/agentSession.continueMessageAgentId.test.ts @@ -24,6 +24,7 @@ interface AutoRetryResumeRequest { agentInitiated?: boolean; stepBudget?: number; modelFallbackProgress?: unknown; + revalidateAdmission?: boolean; } interface SendInternal { @@ -31,6 +32,7 @@ interface SendInternal { agentInitiated?: boolean; stepBudget?: number; modelFallbackProgress?: unknown; + revalidateAdmission?: boolean; } interface SessionInternals { @@ -275,7 +277,7 @@ describe("AgentSession continue-message agentId fallback", () => { expect(internals.lastAutoRetryResumeRequest?.agentInitiated).toBe(true); }); - test("dispatchPendingFollowUp continues the interrupted turn's step budget and fallback chain", async () => { + test("dispatchPendingFollowUp continues the interrupted turn's step budget, fallback chain, and admission revalidation", async () => { const progress = { requestedModel: "anthropic:claude-sonnet-4-5", refusedModels: ["anthropic:claude-sonnet-4-5"], @@ -289,6 +291,7 @@ describe("AgentSession continue-message agentId fallback", () => { agentId: "exec", stepBudget: 7, modelFallbackProgress: progress, + revalidateAdmission: true, }), ]); internals.sendMessage = mock( @@ -300,9 +303,39 @@ describe("AgentSession continue-message agentId fallback", () => { await internals.dispatchPendingFollowUp(); - expect(dispatched[0]).toMatchObject({ stepBudget: 7, modelFallbackProgress: progress }); - expect(internals.lastAutoRetryResumeRequest?.stepBudget).toBe(7); - expect(internals.lastAutoRetryResumeRequest?.modelFallbackProgress).toEqual(progress); + expect(dispatched[0]).toMatchObject({ + stepBudget: 7, + modelFallbackProgress: progress, + revalidateAdmission: true, + }); + expect(internals.lastAutoRetryResumeRequest).toMatchObject({ + stepBudget: 7, + modelFallbackProgress: progress, + revalidateAdmission: true, + }); + }); + + test("dispatchPendingFollowUp discards a follow-up whose interrupted turn spent its step budget", async () => { + const { internals, historyService } = await createSession([ + compactionSummaryMessage("summary-spent", { + text: "Continue", + model: "openai:gpt-4o", + agentId: "exec", + stepBudget: 0, + }), + ]); + const sendMessage = mock(() => Promise.resolve({ success: true as const })); + internals.sendMessage = sendMessage; + + expect(await internals.dispatchPendingFollowUp()).toBe(false); + + // The ceiling ended the turn; the follow-up is dropped rather than left to redispatch later. + expect(sendMessage).not.toHaveBeenCalled(); + const tail = await historyService.getLastMessages("ws", 1); + expect(tail.success).toBe(true); + const summary = tail.success ? tail.data[0] : undefined; + expect(summary?.id).toBe("summary-spent"); + expect(summary?.metadata?.muxMetadata).toEqual({ type: "compaction-summary" }); }); test("dispatchPendingFollowUp drops a malformed persisted remainder", async () => { diff --git a/src/node/services/agentSession.ts b/src/node/services/agentSession.ts index 9890cdfa66..16fb4df1e0 100644 --- a/src/node/services/agentSession.ts +++ b/src/node/services/agentSession.ts @@ -3199,6 +3199,8 @@ export class AgentSession { stepBudget?: number; /** For a send continuing an interrupted turn: the fallback chain state it reached. */ modelFallbackProgress?: ModelFallbackProgress; + /** For a send continuing an interrupted turn: it ran under resumeStream's revalidation. */ + revalidateAdmission?: boolean; onAccepted?: () => Promise | void; onAcceptedPreStreamFailure?: (error: SendMessageError) => Promise | void; onCanceled?: (reason: string) => Promise | void; @@ -4226,7 +4228,7 @@ export class AgentSession { goalKind, internal?.goalId, internal?.stepBudget, - undefined, + internal?.revalidateAdmission, internal?.modelFallbackProgress ); try { @@ -4315,7 +4317,7 @@ export class AgentSession { turnThinkingOverride, undefined, internal?.stepBudget, - undefined, + internal?.revalidateAdmission, internal?.modelFallbackProgress ); if (streamResult.success && preparedTurnAbortController.signal.aborted) { @@ -4827,6 +4829,7 @@ export class AgentSession { workspaceTurnMetadata?: Extract; stepBudget?: number; modelFallbackProgress?: ModelFallbackProgress; + revalidateAdmission?: boolean; }): CompactionFollowUpRequest { const followUp: CompactionFollowUpRequest = { text: params.messageText, @@ -4837,6 +4840,7 @@ export class AgentSession { ...(params.modelFallbackProgress != null ? { modelFallbackProgress: params.modelFallbackProgress } : {}), + ...(params.revalidateAdmission === true ? { revalidateAdmission: true } : {}), }; if (params.agentInitiated === true) { @@ -5100,6 +5104,7 @@ export class AgentSession { // The abort handler left the interrupted stream's remainder on this context. stepBudget: streamContext.stepBudget, modelFallbackProgress: streamContext.modelFallbackProgress, + revalidateAdmission: streamContext.revalidateAdmission, }); // Waterfall hook point: see the on-send compaction.prepare run above. await eventSpine.run("compaction.prepare", { @@ -8306,6 +8311,17 @@ export class AgentSession { followUp.modelFallbackProgress != null ? ModelFallbackProgressSchema.safeParse(followUp.modelFallbackProgress) : undefined; + const persistedRevalidateAdmission = followUp.revalidateAdmission === true; + // The interrupted turn spent its last step before compaction: the ceiling ended it, and the + // loop's stop condition is only evaluated after a step, so a follow-up would run one more. + if (persistedStepBudget === 0) { + log.info("Discarding pending follow-up: the interrupted turn's step budget is spent", { + workspaceId: this.workspaceId, + summaryMessageId: lastMessage.id, + }); + await this.clearPendingFollowUpFromSummary(lastMessage); + return false; + } // The compaction summary is now the source of truth for the next live resume // request. Pre-arm retry state from the reconstructed follow-up so failures @@ -8316,7 +8332,7 @@ export class AgentSession { persistedGoalKind, persistedGoalId, persistedStepBudget, - undefined, + persistedRevalidateAdmission, persistedFallbackProgress?.success ? persistedFallbackProgress.data : undefined ); @@ -8341,6 +8357,7 @@ export class AgentSession { modelFallbackProgress: persistedFallbackProgress?.success ? persistedFallbackProgress.data : undefined, + revalidateAdmission: persistedRevalidateAdmission, }); if (!sendResult.success) { // A stale-admission refusal is the idle rule (or a goal transition) From c86e0ee10c4eb58f00023b1297113a5c05ee86e9 Mon Sep 17 00:00:00 2001 From: Michael Suchacz <203725896+ibetitsmike@users.noreply.github.com> Date: Thu, 3 Sep 2026 23:10:51 +0000 Subject: [PATCH 21/28] Admit a delegated turn's compaction follow-up like a stranded resume and settle it when dropped --- ...gentSession.continueMessageAgentId.test.ts | 107 ++++++++++-- .../agentSession.queueDispatch.test.ts | 29 ++++ src/node/services/agentSession.ts | 152 ++++++++++++++---- 3 files changed, 244 insertions(+), 44 deletions(-) diff --git a/src/node/services/agentSession.continueMessageAgentId.test.ts b/src/node/services/agentSession.continueMessageAgentId.test.ts index e481238c00..0d5eb21f5f 100644 --- a/src/node/services/agentSession.continueMessageAgentId.test.ts +++ b/src/node/services/agentSession.continueMessageAgentId.test.ts @@ -33,6 +33,7 @@ interface SendInternal { stepBudget?: number; modelFallbackProgress?: unknown; revalidateAdmission?: boolean; + refuseStreamStart?: () => boolean; } interface SessionInternals { @@ -161,7 +162,13 @@ describe("AgentSession continue-message agentId fallback", () => { historyCleanup = undefined; }); - const createSession = async (messages: MuxMessage[] = []) => { + const createSession = async ( + messages: MuxMessage[] = [], + turnOptions?: Pick< + ConstructorParameters[0], + "admitStrandedTurnResume" | "settleForfeitedWorkspaceTurnContinuation" + > + ) => { const { historyService, cleanup } = await createTestHistoryService(); historyCleanup = cleanup; for (const message of messages) { @@ -175,6 +182,7 @@ describe("AgentSession continue-message agentId fallback", () => { aiService: createAiService(), initStateManager: createInitStateManager(), backgroundProcessManager: createBackgroundProcessManager(), + ...turnOptions, }); sessions.push(session); @@ -315,22 +323,37 @@ describe("AgentSession continue-message agentId fallback", () => { }); }); + const DELEGATED_TURN = { + type: "workspace-turn-task", + taskHandleId: "wst_follow_up", + ownerWorkspaceId: "owner-ws", + turnId: "turn-follow-up", + } as const; + test("dispatchPendingFollowUp discards a follow-up whose interrupted turn spent its step budget", async () => { - const { internals, historyService } = await createSession([ - compactionSummaryMessage("summary-spent", { - text: "Continue", - model: "openai:gpt-4o", - agentId: "exec", - stepBudget: 0, - }), - ]); + const settle = mock((_correlation: unknown, _reason: string) => Promise.resolve()); + const { internals, historyService } = await createSession( + [ + compactionSummaryMessage("summary-spent", { + text: "Continue", + model: "openai:gpt-4o", + agentId: "exec", + stepBudget: 0, + muxMetadata: DELEGATED_TURN, + }), + ], + { settleForfeitedWorkspaceTurnContinuation: settle } + ); const sendMessage = mock(() => Promise.resolve({ success: true as const })); internals.sendMessage = sendMessage; expect(await internals.dispatchPendingFollowUp()).toBe(false); - // The ceiling ended the turn; the follow-up is dropped rather than left to redispatch later. + // The ceiling ended the turn; the follow-up is dropped rather than left to redispatch later, + // and the delegated turn it continued is settled since no successor stream will end it. expect(sendMessage).not.toHaveBeenCalled(); + expect(settle).toHaveBeenCalledTimes(1); + expect(settle.mock.calls[0]?.[0]).toEqual(DELEGATED_TURN); const tail = await historyService.getLastMessages("ws", 1); expect(tail.success).toBe(true); const summary = tail.success ? tail.data[0] : undefined; @@ -338,6 +361,70 @@ describe("AgentSession continue-message agentId fallback", () => { expect(summary?.metadata?.muxMetadata).toEqual({ type: "compaction-summary" }); }); + test("dispatchPendingFollowUp admits a delegated turn's follow-up like a stranded resume", async () => { + const settle = mock((_correlation: unknown, _reason: string) => Promise.resolve()); + let stale = false; + const admit = mock((_correlation: unknown) => + Promise.resolve({ admissible: true, admissionStale: () => stale }) + ); + const dispatched: SendInternal[] = []; + const { internals } = await createSession( + [ + compactionSummaryMessage("summary-delegated", { + text: "Continue", + model: "openai:gpt-4o", + agentId: "exec", + muxMetadata: DELEGATED_TURN, + }), + ], + { admitStrandedTurnResume: admit, settleForfeitedWorkspaceTurnContinuation: settle } + ); + internals.sendMessage = mock( + (_message: string, _options?: SendOptions, internal?: SendInternal) => { + dispatched.push(internal ?? {}); + return Promise.resolve({ success: true as const }); + } + ); + + expect(await internals.dispatchPendingFollowUp()).toBe(true); + + // Admitted against the delegated turn, with the handle probe carried to the launch boundary. + expect(admit.mock.calls[0]?.[0]).toEqual(DELEGATED_TURN); + expect(dispatched[0]?.refuseStreamStart?.()).toBe(false); + stale = true; + expect(dispatched[0]?.refuseStreamStart?.()).toBe(true); + expect(settle).not.toHaveBeenCalled(); + }); + + test("dispatchPendingFollowUp settles and drops a delegated turn's follow-up its owner no longer admits", async () => { + const settle = mock((_correlation: unknown, _reason: string) => Promise.resolve()); + const { internals, historyService } = await createSession( + [ + compactionSummaryMessage("summary-refused", { + text: "Continue", + model: "openai:gpt-4o", + agentId: "exec", + muxMetadata: DELEGATED_TURN, + }), + ], + { + admitStrandedTurnResume: mock(() => Promise.resolve({ admissible: false })), + settleForfeitedWorkspaceTurnContinuation: settle, + } + ); + const sendMessage = mock(() => Promise.resolve({ success: true as const })); + internals.sendMessage = sendMessage; + + expect(await internals.dispatchPendingFollowUp()).toBe(false); + + expect(sendMessage).not.toHaveBeenCalled(); + expect(settle).toHaveBeenCalledTimes(1); + expect(settle.mock.calls[0]?.[0]).toEqual(DELEGATED_TURN); + const tail = await historyService.getLastMessages("ws", 1); + const summary = tail.success ? tail.data[0] : undefined; + expect(summary?.metadata?.muxMetadata).toEqual({ type: "compaction-summary" }); + }); + test("dispatchPendingFollowUp drops a malformed persisted remainder", async () => { const dispatched: SendInternal[] = []; const { internals } = await createSession([ diff --git a/src/node/services/agentSession.queueDispatch.test.ts b/src/node/services/agentSession.queueDispatch.test.ts index 7039245881..c610adba38 100644 --- a/src/node/services/agentSession.queueDispatch.test.ts +++ b/src/node/services/agentSession.queueDispatch.test.ts @@ -1140,6 +1140,35 @@ describe("AgentSession queued message tool-call dispatch", () => { } }); + test("a send's launch-boundary admission probe reaches the stream request", async () => { + const workspaceId = "queue-dispatch-send-launch-probe"; + const aiEmitter = new EventEmitter(); + const streamMessage = mock((_options: StreamMessageOptions) => { + aiEmitter.emit("stream-start", streamStartEvent(workspaceId)); + return Promise.resolve(Ok(createStartedTurnHandle("assistant-1"))); + }); + const { session, cleanup } = await createAgentSessionHarness({ + workspaceId, + aiEmitter, + aiServiceOverrides: { streamMessage: streamMessage as unknown as AIService["streamMessage"] }, + }); + + try { + const refuseStreamStart = () => false; + const sent = await session.sendMessage( + "Continue", + { model: TEST_MODEL, agentId: "exec" }, + { synthetic: true, refuseStreamStart } + ); + expect(sent.success).toBe(true); + // StreamManager rechecks this probe right before the stream registers (see resumeStream). + expect(streamMessage.mock.calls[0]?.[0].refuseStreamStart).toBe(refuseStreamStart); + } finally { + session.dispose(); + await cleanup(); + } + }); + test("an auto-retry runs under what the failed resumed attempt left of the step budget", async () => { const workspaceId = "queue-dispatch-stranded-retry-step-budget"; const aiEmitter = new EventEmitter(); diff --git a/src/node/services/agentSession.ts b/src/node/services/agentSession.ts index 16fb4df1e0..15d755fd54 100644 --- a/src/node/services/agentSession.ts +++ b/src/node/services/agentSession.ts @@ -3201,6 +3201,12 @@ export class AgentSession { modelFallbackProgress?: ModelFallbackProgress; /** For a send continuing an interrupted turn: it ran under resumeStream's revalidation. */ revalidateAdmission?: boolean; + /** + * Launch-boundary admission probe (goal and delegated-turn state), rechecked by + * StreamManager right before the stream registers; unlike admissionStale it must not + * observe this send's own turn. + */ + refuseStreamStart?: () => boolean; onAccepted?: () => Promise | void; onAcceptedPreStreamFailure?: (error: SendMessageError) => Promise | void; onCanceled?: (reason: string) => Promise | void; @@ -4315,7 +4321,7 @@ export class AgentSession { goalKind, internal?.goalId, turnThinkingOverride, - undefined, + internal?.refuseStreamStart, internal?.stepBudget, internal?.revalidateAdmission, internal?.modelFallbackProgress @@ -7441,21 +7447,41 @@ export class AgentSession { */ private forfeitStrandedTurnResume(reason: string): void { const correlation = getWorkspaceTurnMuxMetadata(this.strandedTurnResume?.options.muxMetadata); - const settlementOwed = - correlation != null && this.settleForfeitedWorkspaceTurnContinuation != null; - if (settlementOwed) { - // Retain the owner's only terminal path before dropping the marker that advertised it. - this.owedForfeitSettlements.set( - `${correlation.ownerWorkspaceId}/${correlation.taskHandleId}/${correlation.turnId}`, - { correlation, reason, inFlight: false } - ); - } + // Retain the owner's only terminal path before dropping the marker that advertised it. + const settlementOwed = this.recordOwedForfeit(correlation, reason); this.withdrawStrandedTurnResume(); if (settlementOwed) { this.settleOwedForfeits(); } } + /** + * A delegated turn given up outside the marker (a dropped compaction follow-up that continued + * it): the same settlement, since nothing else will end the turn for its owner. + */ + private forfeitWorkspaceTurnContinuation( + correlation: WorkspaceTurnMuxMetadata | undefined, + reason: string + ): void { + if (this.recordOwedForfeit(correlation, reason)) { + this.settleOwedForfeits(); + } + } + + private recordOwedForfeit( + correlation: WorkspaceTurnMuxMetadata | undefined, + reason: string + ): boolean { + if (correlation == null || this.settleForfeitedWorkspaceTurnContinuation == null) { + return false; + } + this.owedForfeitSettlements.set( + `${correlation.ownerWorkspaceId}/${correlation.taskHandleId}/${correlation.turnId}`, + { correlation, reason, inFlight: false } + ); + return true; + } + /** * The settlement is the owner's only remaining path to a terminal record for that turn, so a * failed attempt (task store I/O) stays owed and retries on its own (including after session @@ -8150,6 +8176,37 @@ export class AgentSession { return false; } + // The delegated turn this follow-up continues, if any: stamped on the follow-up itself by + // mid-stream compaction, or beside a wake follow-up by on-send compaction. + const continuedTurn = + getWorkspaceTurnMuxMetadata(followUp.muxMetadata) ?? followUp.workspaceTurnMetadata; + // Same raw JSON boundary: the interrupted turn's remainder is optional and dropped if malformed + // (the follow-up then runs under the default ceiling and its model's own chain). + const persistedStepBudget = + typeof followUp.stepBudget === "number" && Number.isInteger(followUp.stepBudget) + ? Math.max(0, followUp.stepBudget) + : undefined; + const persistedFallbackProgress = + followUp.modelFallbackProgress != null + ? ModelFallbackProgressSchema.safeParse(followUp.modelFallbackProgress) + : undefined; + const persistedRevalidateAdmission = followUp.revalidateAdmission === true; + // The interrupted turn spent its last step before compaction: the ceiling ended it, and the + // loop's stop condition is only evaluated after a step, so a follow-up would run one more. A + // delegated turn ends here with no successor stream, so its owner settles it. + if (persistedStepBudget === 0) { + log.info("Discarding pending follow-up: the interrupted turn's step budget is spent", { + workspaceId: this.workspaceId, + summaryMessageId: lastMessage.id, + }); + this.forfeitWorkspaceTurnContinuation( + continuedTurn, + "Compaction follow-up dropped: the interrupted turn's step budget is spent." + ); + await this.clearPendingFollowUpFromSummary(lastMessage); + return false; + } + // Codex P1 (PRRT_kwDOPxxmWM6cPuMw): goal-loop follow-ups were originally // requireIdle sends — enforce the idle rule for them unconditionally so a // user message queued during the compaction stream wins the race instead @@ -8210,12 +8267,41 @@ export class AgentSession { workspaceId: this.workspaceId, goalKind: persistedGoalKind, }); + this.forfeitWorkspaceTurnContinuation( + continuedTurn, + "Compaction follow-up dropped: goal no longer admits it." + ); await this.clearPendingFollowUpFromSummary(lastMessage); return false; } goalAdmissionStale = admission.admissionStale; } + // A follow-up continuing a delegated turn, or a turn that ran under a stranded resume's + // revalidation, is admitted like that resume (admitResumeLaunch): the workspace must still + // accept streams and the turn's owner must still have it running. The probe rides along to + // the launch boundary below; a refusal ends the delegated turn with no successor stream. + let turnAdmissionStale: (() => boolean) | undefined; + if ((persistedRevalidateAdmission || continuedTurn != null) && this.admitStrandedTurnResume) { + const admission = await this.admitStrandedTurnResume(continuedTurn); + if (!admission.admissible) { + log.info( + "Skipping pending follow-up: the workspace or delegated turn no longer admits it", + { + workspaceId: this.workspaceId, + summaryMessageId: lastMessage.id, + } + ); + this.forfeitWorkspaceTurnContinuation( + continuedTurn, + "Compaction follow-up dropped: the workspace or delegated turn no longer admits it." + ); + await this.clearPendingFollowUpFromSummary(lastMessage); + return false; + } + turnAdmissionStale = admission.admissionStale; + } + // Codex P1 (PRRT_kwDOPxxmWM6cQt3j): the queue/busy sample above ages // across the awaited goal read and the send's own preflight. Re-evaluate // the idle rule through the send-admission gates — all of them run before @@ -8228,9 +8314,14 @@ export class AgentSession { this.hasExternalSendPreflight?.() === true || (this.isBusy() && this.turnPhase !== TurnPhase.COMPLETING) : undefined; + // Launch-safe probes only (the idle rule would trip on this send's own PREPARING turn). + const launchAdmissionStale = + goalAdmissionStale != null || turnAdmissionStale != null + ? () => goalAdmissionStale?.() === true || turnAdmissionStale?.() === true + : undefined; const followUpAdmissionStale = - idleRuleStale != null || goalAdmissionStale != null - ? () => idleRuleStale?.() === true || goalAdmissionStale?.() === true + idleRuleStale != null || launchAdmissionStale != null + ? () => idleRuleStale?.() === true || launchAdmissionStale?.() === true : undefined; log.debug("Dispatching pending follow-up from compaction summary", { @@ -8301,28 +8392,6 @@ export class AgentSession { options.muxMetadata = metadata; } - // Same raw JSON boundary: the interrupted turn's remainder is optional and dropped if malformed - // (the follow-up then runs under the default ceiling and its model's own chain). - const persistedStepBudget = - typeof followUp.stepBudget === "number" && Number.isInteger(followUp.stepBudget) - ? Math.max(0, followUp.stepBudget) - : undefined; - const persistedFallbackProgress = - followUp.modelFallbackProgress != null - ? ModelFallbackProgressSchema.safeParse(followUp.modelFallbackProgress) - : undefined; - const persistedRevalidateAdmission = followUp.revalidateAdmission === true; - // The interrupted turn spent its last step before compaction: the ceiling ended it, and the - // loop's stop condition is only evaluated after a step, so a follow-up would run one more. - if (persistedStepBudget === 0) { - log.info("Discarding pending follow-up: the interrupted turn's step budget is spent", { - workspaceId: this.workspaceId, - summaryMessageId: lastMessage.id, - }); - await this.clearPendingFollowUpFromSummary(lastMessage); - return false; - } - // The compaction summary is now the source of truth for the next live resume // request. Pre-arm retry state from the reconstructed follow-up so failures // before stream startup do not fall back to the already-completed compact turn. @@ -8353,6 +8422,7 @@ export class AgentSession { // Codex P1 (PRRT_kwDOPxxmWM6cPuMw): re-derived admission guard for the // redispatched goal turn (see buildGoalRedispatchAdmission above). admissionStale: followUpAdmissionStale, + refuseStreamStart: launchAdmissionStale, stepBudget: persistedStepBudget, modelFallbackProgress: persistedFallbackProgress?.success ? persistedFallbackProgress.data @@ -8360,6 +8430,20 @@ export class AgentSession { revalidateAdmission: persistedRevalidateAdmission, }); if (!sendResult.success) { + // The workspace or the delegated turn stopped admitting the follow-up during its + // preflight: no successor stream, so the turn is settled and the follow-up dropped. + if (turnAdmissionStale?.() === true) { + log.info("Pending follow-up refused at send admission: workspace or delegated turn", { + workspaceId: this.workspaceId, + summaryMessageId: lastMessage.id, + }); + this.forfeitWorkspaceTurnContinuation( + continuedTurn, + "Compaction follow-up dropped: the workspace or delegated turn no longer admits it." + ); + await this.clearPendingFollowUpFromSummary(lastMessage); + return false; + } // A stale-admission refusal is the idle rule (or a goal transition) // working as intended, not a recovery failure: route it through the // same skip path as the pre-send check instead of throwing. From 3ebfba3e1b2097485d53eccdc485c846ba02b9cd Mon Sep 17 00:00:00 2001 From: Michael Suchacz <203725896+ibetitsmike@users.noreply.github.com> Date: Thu, 3 Sep 2026 23:37:49 +0000 Subject: [PATCH 22/28] Settle a delegated turn's follow-up refused at the launch boundary or rejected as malformed, revalidate its retries, validate persisted correlations --- ...gentSession.continueMessageAgentId.test.ts | 87 +++++++++++++++- src/node/services/agentSession.ts | 98 +++++++++---------- 2 files changed, 134 insertions(+), 51 deletions(-) diff --git a/src/node/services/agentSession.continueMessageAgentId.test.ts b/src/node/services/agentSession.continueMessageAgentId.test.ts index 0d5eb21f5f..0b9171f9c8 100644 --- a/src/node/services/agentSession.continueMessageAgentId.test.ts +++ b/src/node/services/agentSession.continueMessageAgentId.test.ts @@ -388,14 +388,99 @@ describe("AgentSession continue-message agentId fallback", () => { expect(await internals.dispatchPendingFollowUp()).toBe(true); - // Admitted against the delegated turn, with the handle probe carried to the launch boundary. + // Admitted against the delegated turn, with the handle probe carried to the launch boundary, + // and retried under revalidation even though the interrupted turn was not a stranded resume. expect(admit.mock.calls[0]?.[0]).toEqual(DELEGATED_TURN); expect(dispatched[0]?.refuseStreamStart?.()).toBe(false); stale = true; expect(dispatched[0]?.refuseStreamStart?.()).toBe(true); + expect(dispatched[0]?.revalidateAdmission).toBe(true); + expect(internals.lastAutoRetryResumeRequest?.revalidateAdmission).toBe(true); expect(settle).not.toHaveBeenCalled(); }); + test("dispatchPendingFollowUp settles a delegated turn's follow-up refused at the launch boundary", async () => { + const settle = mock((_correlation: unknown, _reason: string) => Promise.resolve()); + let stale = false; + const { internals, historyService } = await createSession( + [ + compactionSummaryMessage("summary-launch-refused", { + text: "Continue", + model: "openai:gpt-4o", + agentId: "exec", + muxMetadata: DELEGATED_TURN, + }), + ], + { + admitStrandedTurnResume: mock(() => + Promise.resolve({ admissible: true, admissionStale: () => stale }) + ), + settleForfeitedWorkspaceTurnContinuation: settle, + } + ); + // The handle is interrupted while the send prepares: StreamManager refuses the launch and the + // send still resolves Ok (a startup-aborted handle), so the dispatch must read the probe. + internals.sendMessage = mock(() => { + stale = true; + return Promise.resolve({ success: true as const }); + }); + + expect(await internals.dispatchPendingFollowUp()).toBe(false); + + expect(settle).toHaveBeenCalledTimes(1); + expect(settle.mock.calls[0]?.[0]).toEqual(DELEGATED_TURN); + const tail = await historyService.getLastMessages("ws", 1); + const summary = tail.success ? tail.data[0] : undefined; + expect(summary?.metadata?.muxMetadata).toEqual({ type: "compaction-summary" }); + }); + + test("dispatchPendingFollowUp ignores a malformed persisted correlation", async () => { + const admit = mock(() => Promise.resolve({ admissible: false })); + const sendMessage = mock(() => Promise.resolve({ success: true as const })); + const { internals } = await createSession( + [ + compactionSummaryMessage("summary-malformed-correlation", { + text: "Continue", + model: "openai:gpt-4o", + agentId: "exec", + muxMetadata: { + type: "workspace-turn-task", + } as unknown as CompactionFollowUpRequest["muxMetadata"], + }), + ], + { admitStrandedTurnResume: admit } + ); + internals.sendMessage = sendMessage; + + // Not a delegated turn to admit or settle: the follow-up dispatches as an ordinary one. + expect(await internals.dispatchPendingFollowUp()).toBe(true); + expect(admit).not.toHaveBeenCalled(); + expect(sendMessage).toHaveBeenCalledTimes(1); + }); + + test("dispatchPendingFollowUp settles a delegated turn's follow-up rejected for malformed goal attribution", async () => { + const settle = mock((_correlation: unknown, _reason: string) => Promise.resolve()); + const sendMessage = mock(() => Promise.resolve({ success: true as const })); + const { internals } = await createSession( + [ + compactionSummaryMessage("summary-malformed-goal", { + text: "Continue", + model: "openai:gpt-4o", + agentId: "exec", + goalKind: "not-a-goal-kind" as unknown as CompactionFollowUpRequest["goalKind"], + muxMetadata: DELEGATED_TURN, + }), + ], + { settleForfeitedWorkspaceTurnContinuation: settle } + ); + internals.sendMessage = sendMessage; + + expect(await internals.dispatchPendingFollowUp()).toBe(false); + expect(sendMessage).not.toHaveBeenCalled(); + expect(settle).toHaveBeenCalledTimes(1); + expect(settle.mock.calls[0]?.[0]).toEqual(DELEGATED_TURN); + }); + test("dispatchPendingFollowUp settles and drops a delegated turn's follow-up its owner no longer admits", async () => { const settle = mock((_correlation: unknown, _reason: string) => Promise.resolve()); const { internals, historyService } = await createSession( diff --git a/src/node/services/agentSession.ts b/src/node/services/agentSession.ts index 15d755fd54..6eddc31ec1 100644 --- a/src/node/services/agentSession.ts +++ b/src/node/services/agentSession.ts @@ -101,6 +101,7 @@ import { type MuxMessageMetadata, type MuxFilePart, type MuxMessage, + parseWorkspaceTurnTaskCorrelation, type ReviewNoteDataForDisplay, type StartupRetrySendOptions, } from "@/common/types/message"; @@ -450,6 +451,12 @@ function buildStrandedTurnResume(context: { }; } +/** A persisted correlation (unchecked chat.jsonl) is used only when well formed. */ +function parsePersistedWorkspaceTurnMetadata(value: unknown): WorkspaceTurnMuxMetadata | undefined { + const correlation = parseWorkspaceTurnTaskCorrelation(value); + return correlation == null ? undefined : { type: "workspace-turn-task", ...correlation }; +} + function hasSameWorkspaceTurnCorrelation( first: WorkspaceTurnMuxMetadata | undefined, second: WorkspaceTurnMuxMetadata | undefined @@ -8142,6 +8149,21 @@ export class AgentSession { imageParts?: FilePart[]; }; + // The delegated turn this follow-up continues, if any: stamped on the follow-up itself by + // mid-stream compaction, or beside a wake follow-up by on-send compaction. Every drop below + // settles it, since the compaction abort and the compact stream end it for nobody. + const continuedTurn = + parsePersistedWorkspaceTurnMetadata(followUp.muxMetadata) ?? + parsePersistedWorkspaceTurnMetadata(followUp.workspaceTurnMetadata); + const dropFollowUp = async (reason: string): Promise => { + this.forfeitWorkspaceTurnContinuation( + continuedTurn, + `Compaction follow-up dropped: ${reason}` + ); + await this.clearPendingFollowUpFromSummary(lastMessage); + return false; + }; + // Compaction summaries are unchecked chat.jsonl. Reject malformed persisted // goal attribution instead of forwarding it into goal-service assertions or // repeatedly crashing startup recovery on the same row. @@ -8155,8 +8177,7 @@ export class AgentSession { workspaceId: this.workspaceId, summaryMessageId: lastMessage.id, }); - await this.clearPendingFollowUpFromSummary(lastMessage); - return false; + return dropFollowUp("malformed goal attribution."); } // Codex P1 (PRRT_kwDOPxxmWM6cS8Bq): pre-upgrade summaries persisted @@ -8172,14 +8193,9 @@ export class AgentSession { summaryMessageId: lastMessage.id, goalKind: persistedGoalKind, }); - await this.clearPendingFollowUpFromSummary(lastMessage); - return false; + return dropFollowUp("legacy goal follow-up without goal identity."); } - // The delegated turn this follow-up continues, if any: stamped on the follow-up itself by - // mid-stream compaction, or beside a wake follow-up by on-send compaction. - const continuedTurn = - getWorkspaceTurnMuxMetadata(followUp.muxMetadata) ?? followUp.workspaceTurnMetadata; // Same raw JSON boundary: the interrupted turn's remainder is optional and dropped if malformed // (the follow-up then runs under the default ceiling and its model's own chain). const persistedStepBudget = @@ -8190,21 +8206,17 @@ export class AgentSession { followUp.modelFallbackProgress != null ? ModelFallbackProgressSchema.safeParse(followUp.modelFallbackProgress) : undefined; - const persistedRevalidateAdmission = followUp.revalidateAdmission === true; + // A follow-up continuing a delegated turn is admitted and retried like a stranded resume even + // when the interrupted turn itself was not one (a delegated turn's first compaction). + const revalidateAdmission = followUp.revalidateAdmission === true || continuedTurn != null; // The interrupted turn spent its last step before compaction: the ceiling ended it, and the - // loop's stop condition is only evaluated after a step, so a follow-up would run one more. A - // delegated turn ends here with no successor stream, so its owner settles it. + // loop's stop condition is only evaluated after a step, so a follow-up would run one more. if (persistedStepBudget === 0) { log.info("Discarding pending follow-up: the interrupted turn's step budget is spent", { workspaceId: this.workspaceId, summaryMessageId: lastMessage.id, }); - this.forfeitWorkspaceTurnContinuation( - continuedTurn, - "Compaction follow-up dropped: the interrupted turn's step budget is spent." - ); - await this.clearPendingFollowUpFromSummary(lastMessage); - return false; + return dropFollowUp("the interrupted turn's step budget is spent."); } // Codex P1 (PRRT_kwDOPxxmWM6cPuMw): goal-loop follow-ups were originally @@ -8267,22 +8279,16 @@ export class AgentSession { workspaceId: this.workspaceId, goalKind: persistedGoalKind, }); - this.forfeitWorkspaceTurnContinuation( - continuedTurn, - "Compaction follow-up dropped: goal no longer admits it." - ); - await this.clearPendingFollowUpFromSummary(lastMessage); - return false; + return dropFollowUp("goal no longer admits it."); } goalAdmissionStale = admission.admissionStale; } - // A follow-up continuing a delegated turn, or a turn that ran under a stranded resume's - // revalidation, is admitted like that resume (admitResumeLaunch): the workspace must still - // accept streams and the turn's owner must still have it running. The probe rides along to - // the launch boundary below; a refusal ends the delegated turn with no successor stream. + // Admitted like a stranded resume (admitResumeLaunch): the workspace must still accept + // streams and a delegated turn's owner must still have it running. The probe rides along to + // the launch boundary below. let turnAdmissionStale: (() => boolean) | undefined; - if ((persistedRevalidateAdmission || continuedTurn != null) && this.admitStrandedTurnResume) { + if (revalidateAdmission && this.admitStrandedTurnResume) { const admission = await this.admitStrandedTurnResume(continuedTurn); if (!admission.admissible) { log.info( @@ -8292,12 +8298,7 @@ export class AgentSession { summaryMessageId: lastMessage.id, } ); - this.forfeitWorkspaceTurnContinuation( - continuedTurn, - "Compaction follow-up dropped: the workspace or delegated turn no longer admits it." - ); - await this.clearPendingFollowUpFromSummary(lastMessage); - return false; + return dropFollowUp("the workspace or delegated turn no longer admits it."); } turnAdmissionStale = admission.admissionStale; } @@ -8401,7 +8402,7 @@ export class AgentSession { persistedGoalKind, persistedGoalId, persistedStepBudget, - persistedRevalidateAdmission, + revalidateAdmission, persistedFallbackProgress?.success ? persistedFallbackProgress.data : undefined ); @@ -8427,23 +8428,20 @@ export class AgentSession { modelFallbackProgress: persistedFallbackProgress?.success ? persistedFallbackProgress.data : undefined, - revalidateAdmission: persistedRevalidateAdmission, + revalidateAdmission, }); + // The workspace or the delegated turn stopped admitting the follow-up during its preflight + // (an Err) or at the launch boundary (StreamManager refuses as a startup-aborted Ok handle, + // like a refused stranded resume): no successor stream, so the turn is settled and the + // follow-up dropped. + if (turnAdmissionStale?.() === true) { + log.info("Pending follow-up refused at admission: workspace or delegated turn", { + workspaceId: this.workspaceId, + summaryMessageId: lastMessage.id, + }); + return dropFollowUp("the workspace or delegated turn no longer admits it."); + } if (!sendResult.success) { - // The workspace or the delegated turn stopped admitting the follow-up during its - // preflight: no successor stream, so the turn is settled and the follow-up dropped. - if (turnAdmissionStale?.() === true) { - log.info("Pending follow-up refused at send admission: workspace or delegated turn", { - workspaceId: this.workspaceId, - summaryMessageId: lastMessage.id, - }); - this.forfeitWorkspaceTurnContinuation( - continuedTurn, - "Compaction follow-up dropped: the workspace or delegated turn no longer admits it." - ); - await this.clearPendingFollowUpFromSummary(lastMessage); - return false; - } // A stale-admission refusal is the idle rule (or a goal transition) // working as intended, not a recovery failure: route it through the // same skip path as the pre-send check instead of throwing. From 19302d57cf7f427260af03b779bc356711b7c9f0 Mon Sep 17 00:00:00 2001 From: Michael Suchacz <203725896+ibetitsmike@users.noreply.github.com> Date: Fri, 4 Sep 2026 00:44:53 +0000 Subject: [PATCH 23/28] Drop a follow-up refused by its goal at the launch boundary, settle its delegated turn before clearing it, treat a refused in-session retry as not started, and revalidate a wake's retries against its delegated turn - dispatchPendingFollowUp reads the combined launch probe after the send: a goal Pause or replacement during launch drops and settles the follow-up like a workspace or handle refusal. - dropFollowUp awaits the owner's settlement before clearing the durable follow-up; a failed attempt leaves it pending for the next startup to re-drop and re-settle. - The in-session context_exceeded retries launch through launchInSessionRetry, which treats an Ok with the turn still PREPARING under a tripped probe as no started retry, so the recovery decision settles terminal instead of publishing retry-started. - AutoRetryResumeRequest, sendMessage, resumeStream, admitResumeLaunch, and the in-session retries carry workspaceTurnMetadata, the delegated turn a wake's own metadata does not name, so retry admission still reaches the owner's handle. --- ...gentSession.continueMessageAgentId.test.ts | 164 ++++++++- .../agentSession.postCompactionRetry.test.ts | 150 +++++++++ src/node/services/agentSession.ts | 312 +++++++++++------- 3 files changed, 502 insertions(+), 124 deletions(-) diff --git a/src/node/services/agentSession.continueMessageAgentId.test.ts b/src/node/services/agentSession.continueMessageAgentId.test.ts index a75947df1d..39b1215739 100644 --- a/src/node/services/agentSession.continueMessageAgentId.test.ts +++ b/src/node/services/agentSession.continueMessageAgentId.test.ts @@ -3,7 +3,9 @@ import { createMuxMessage } from "@/common/types/message"; import type { CompactionFollowUpRequest, MuxMessage } from "@/common/types/message"; import type { FilePart, SendMessageOptions } from "@/common/orpc/types"; import type { Config } from "@/node/config"; +import { GOAL_CONTINUATION_KIND } from "@/constants/goals"; import { AgentSession } from "./agentSession"; +import type { WorkspaceGoalService } from "./workspaceGoalService"; import { createStreamLifecycleMocks } from "./agentSession.testHarness"; import type { AIService } from "./aiService"; import type { BackgroundProcessManager } from "./backgroundProcessManager"; @@ -25,6 +27,7 @@ interface AutoRetryResumeRequest { stepBudget?: number; modelFallbackProgress?: unknown; revalidateAdmission?: boolean; + workspaceTurnMetadata?: unknown; } interface SendInternal { @@ -33,11 +36,13 @@ interface SendInternal { stepBudget?: number; modelFallbackProgress?: unknown; revalidateAdmission?: boolean; + workspaceTurnMetadata?: unknown; refuseStreamStart?: () => boolean; } interface SessionInternals { dispatchPendingFollowUp: () => Promise; + retryActiveStream: () => Promise; sendMessage: ( message: string, options?: SendOptions, @@ -169,7 +174,9 @@ describe("AgentSession continue-message agentId fallback", () => { ...turnOptions }: Pick< ConstructorParameters[0], - "admitStrandedTurnResume" | "settleForfeitedWorkspaceTurnContinuation" + | "admitStrandedTurnResume" + | "settleForfeitedWorkspaceTurnContinuation" + | "workspaceGoalService" > & { config?: Config } = {} ) => { const { historyService, cleanup } = await createTestHistoryService(); @@ -437,6 +444,161 @@ describe("AgentSession continue-message agentId fallback", () => { expect(summary?.metadata?.muxMetadata).toEqual({ type: "compaction-summary" }); }); + test("dispatchPendingFollowUp settles a delegated turn's goal follow-up its goal refuses at the launch boundary", async () => { + const settle = mock((_correlation: unknown, _reason: string) => Promise.resolve()); + let goalStale = false; + const { internals, historyService } = await createSession( + [ + compactionSummaryMessage("summary-goal-launch-refused", { + text: "Continue", + model: "openai:gpt-4o", + agentId: "exec", + goalKind: GOAL_CONTINUATION_KIND, + goalId: "2f5a4c9e-3b7d-4e1f-9a6b-8c0d1e2f3a4b", + muxMetadata: DELEGATED_TURN, + }), + ], + { + workspaceGoalService: { + buildGoalRedispatchAdmission: mock(() => + Promise.resolve({ admissible: true, admissionStale: () => goalStale }) + ), + } as unknown as WorkspaceGoalService, + admitStrandedTurnResume: mock(() => + Promise.resolve({ admissible: true, admissionStale: () => false }) + ), + settleForfeitedWorkspaceTurnContinuation: settle, + } + ); + // The goal is paused while the send prepares: only the goal probe trips, and the send still + // resolves Ok (a startup-aborted handle). + internals.sendMessage = mock(() => { + goalStale = true; + return Promise.resolve({ success: true as const }); + }); + + expect(await internals.dispatchPendingFollowUp()).toBe(false); + + expect(settle).toHaveBeenCalledTimes(1); + expect(settle.mock.calls[0]?.[0]).toEqual(DELEGATED_TURN); + const tail = await historyService.getLastMessages("ws", 1); + const summary = tail.success ? tail.data[0] : undefined; + expect(summary?.metadata?.muxMetadata).toEqual({ type: "compaction-summary" }); + }); + + test("dispatchPendingFollowUp clears a dropped follow-up only once its delegated turn is settled", async () => { + let settled!: () => void; + const settle = mock( + (_correlation: unknown, _reason: string) => + new Promise((resolve) => { + settled = resolve; + }) + ); + const { internals, historyService } = await createSession( + [ + compactionSummaryMessage("summary-settle-first", { + text: "Continue", + model: "openai:gpt-4o", + agentId: "exec", + muxMetadata: DELEGATED_TURN, + }), + ], + { + admitStrandedTurnResume: mock(() => Promise.resolve({ admissible: false })), + settleForfeitedWorkspaceTurnContinuation: settle, + } + ); + const pendingFollowUp = async () => { + const tail = await historyService.getLastMessages("ws", 1); + const summary = tail.success ? tail.data[0] : undefined; + const muxMeta = summary?.metadata?.muxMetadata; + return muxMeta?.type === "compaction-summary" ? muxMeta.pendingFollowUp : undefined; + }; + + const dispatch = internals.dispatchPendingFollowUp(); + await new Promise((resolve) => setTimeout(resolve, 20)); + // Settlement outstanding: the follow-up stays durable so a crash here can still rediscover + // the correlation (the owed settlement itself lives only in memory). + expect(settle).toHaveBeenCalledTimes(1); + expect(await pendingFollowUp()).toBeDefined(); + + settled(); + expect(await dispatch).toBe(false); + expect(await pendingFollowUp()).toBeUndefined(); + }); + + test("dispatchPendingFollowUp keeps a dropped follow-up pending when its delegated turn fails to settle", async () => { + const settle = mock((_correlation: unknown, _reason: string) => + Promise.reject(new Error("task store unavailable")) + ); + const { internals, historyService } = await createSession( + [ + compactionSummaryMessage("summary-settle-failed", { + text: "Continue", + model: "openai:gpt-4o", + agentId: "exec", + muxMetadata: DELEGATED_TURN, + }), + ], + { + admitStrandedTurnResume: mock(() => Promise.resolve({ admissible: false })), + settleForfeitedWorkspaceTurnContinuation: settle, + } + ); + + expect(await internals.dispatchPendingFollowUp()).toBe(false); + + // Left for the next startup to re-drop and re-settle. + expect(settle).toHaveBeenCalledTimes(1); + const tail = await historyService.getLastMessages("ws", 1); + const summary = tail.success ? tail.data[0] : undefined; + expect(summary?.metadata?.muxMetadata).toMatchObject({ + type: "compaction-summary", + pendingFollowUp: { text: "Continue" }, + }); + }); + + test("dispatchPendingFollowUp revalidates an on-send-compacted wake's retries against its delegated turn", async () => { + let admissible = true; + const admit = mock((_correlation: unknown) => + Promise.resolve( + admissible ? { admissible: true, admissionStale: () => false } : { admissible: false } + ) + ); + const dispatched: SendInternal[] = []; + const { internals } = await createSession( + [ + compactionSummaryMessage("summary-wake", { + text: "Background monitor wake", + model: "openai:gpt-4o", + agentId: "exec", + // The wake's own metadata carries no correlation; on-send compaction stamped the + // delegated turn it continued beside it. + muxMetadata: { type: "bash-monitor-wake", records: [] }, + workspaceTurnMetadata: DELEGATED_TURN, + }), + ], + { admitStrandedTurnResume: admit } + ); + internals.sendMessage = mock( + (_message: string, _options?: SendOptions, internal?: SendInternal) => { + dispatched.push(internal ?? {}); + return Promise.resolve({ success: true as const }); + } + ); + + expect(await internals.dispatchPendingFollowUp()).toBe(true); + expect(admit.mock.calls[0]?.[0]).toEqual(DELEGATED_TURN); + expect(dispatched[0]?.workspaceTurnMetadata).toEqual(DELEGATED_TURN); + + // The handle is interrupted during the backoff: the retry's admission sees the delegated + // turn, not just the wake, and refuses. + admissible = false; + await internals.retryActiveStream(); + expect(admit).toHaveBeenCalledTimes(2); + expect(admit.mock.calls[1]?.[0]).toEqual(DELEGATED_TURN); + }); + test("dispatchPendingFollowUp ignores a malformed persisted correlation", async () => { const admit = mock(() => Promise.resolve({ admissible: false })); const sendMessage = mock(() => Promise.resolve({ success: true as const })); diff --git a/src/node/services/agentSession.postCompactionRetry.test.ts b/src/node/services/agentSession.postCompactionRetry.test.ts index 5307678945..d53732c1eb 100644 --- a/src/node/services/agentSession.postCompactionRetry.test.ts +++ b/src/node/services/agentSession.postCompactionRetry.test.ts @@ -504,4 +504,154 @@ describe("AgentSession post-compaction context retry", () => { session.dispose(); }); + + // A revalidated turn's retry is admitted, then the delegated handle is interrupted while the + // retry's launch prepares: StreamManager refuses the launch as a startup-aborted Ok. That is + // not a started retry; the episode must settle terminal so the owner's waiter is released. + test("a revalidated retry refused at the launch boundary settles terminal, not retry-started", async () => { + const workspaceId = "ws-launch-refused"; + const sessionsDir = await fsPromises.mkdtemp(path.join(os.tmpdir(), "mux-agentSession-")); + const sessionDir = path.join(sessionsDir, workspaceId); + await fsPromises.mkdir(sessionDir); + await createPersistedPostCompactionState({ + filePath: path.join(sessionDir, "post-compaction.json"), + diffs: [{ path: "/tmp/foo.ts", diff: "@@ -1 +1 @@\n-foo\n+bar\n", truncated: false }], + }); + const delegatedTurn = { + type: "workspace-turn-task", + taskHandleId: "wst_wake", + ownerWorkspaceId: "owner-ws", + turnId: "turn-wake", + } as const; + + const { historyService, cleanup } = await createTestHistoryService(); + historyCleanup = cleanup; + // An on-send-compacted wake continuation: the stream inherits its delegated turn from the + // summary, not from the wake row or the resume options. + await historyService.appendToHistory(workspaceId, { + id: "compaction-summary", + role: "assistant", + parts: [{ type: "text", text: "Summary" }], + metadata: { + timestamp: 1000, + compacted: "user", + muxMetadata: { + type: "compaction-summary", + pendingFollowUp: { + text: "Background monitor wake", + model: "openai:gpt-4o", + agentId: "exec", + workspaceTurnMetadata: delegatedTurn, + }, + }, + }, + }); + await historyService.appendToHistory(workspaceId, { + id: "user-1", + role: "user", + parts: [{ type: "text", text: "Background monitor wake" }], + metadata: { timestamp: 1100, muxMetadata: { type: "bash-monitor-wake", records: [] } }, + }); + + const aiEmitter = new EventEmitter(); + let retryLaunched!: () => void; + const retryLaunch = new Promise((resolve) => { + retryLaunched = resolve; + }); + let stale = false; + let callCount = 0; + const streamMessage = mock((..._args: unknown[]) => { + callCount += 1; + if (callCount === 1) { + aiEmitter.emit("error", { + workspaceId, + messageId: "assistant-ctx-exceeded", + error: "Context length exceeded", + errorType: "context_exceeded", + }); + return Promise.resolve(contextExceededResult("assistant-ctx-exceeded")); + } + // The handle is interrupted right before registration: no stream-start, a startup-aborted + // handle. + stale = true; + retryLaunched(); + return Promise.resolve({ + success: true as const, + data: { + messageId: "assistant-retry", + completion: Promise.resolve({ status: "aborted" as const, abortReason: "startup" }), + }, + }); + }); + const admit = mock((_correlation: unknown) => + Promise.resolve({ admissible: true, admissionStale: () => stale }) + ); + + const session = new AgentSession({ + workspaceId, + config: { + rootDir: sessionsDir, + sessionsDir, + srcDir: "/tmp", + loadConfigOrDefault: mock(() => ({})), + } as unknown as Config, + historyService, + aiService: { + ...createStreamLifecycleMocks(), + on(eventName: string | symbol, listener: (...args: unknown[]) => void) { + aiEmitter.on(String(eventName), listener); + return this; + }, + off(eventName: string | symbol, listener: (...args: unknown[]) => void) { + aiEmitter.off(String(eventName), listener); + return this; + }, + streamMessage, + getWorkspaceMetadata: mock(() => + Promise.resolve({ success: false as const, error: "nope" }) + ), + } as unknown as AIService, + initStateManager: { + on() { + return this; + }, + off() { + return this; + }, + } as unknown as InitStateManager, + backgroundProcessManager: { + setMessageQueued: mock(() => undefined), + cleanup: mock(() => Promise.resolve()), + } as unknown as BackgroundProcessManager, + admitStrandedTurnResume: admit, + }); + + const resumed = await session.resumeStream( + { model: "openai:gpt-4o", agentId: "exec" }, + { revalidateAdmission: true } + ); + expect(resumed.success).toBe(true); + + const withTimeout = (promise: Promise, label: string): Promise => + Promise.race([ + promise, + new Promise((_, reject) => + setTimeout(() => reject(new Error(`${label} timeout`)), 1000) + ), + ]); + await withTimeout(retryLaunch, "retry launch"); + expect( + await withTimeout( + session.waitForPendingStreamErrorRecoveryDecision("assistant-ctx-exceeded"), + "decision" + ) + ).toBe("terminal"); + expect(session.isPreparingTurn()).toBe(false); + expect(callCount).toBe(2); + // The retry revalidated against the inherited delegated turn, not just the wake. + expect(admit).toHaveBeenCalledTimes(2); + expect(admit.mock.calls[1]?.[0]).toEqual(delegatedTurn); + + session.dispose(); + }); }); diff --git a/src/node/services/agentSession.ts b/src/node/services/agentSession.ts index d31021c96f..96c42fdbc0 100644 --- a/src/node/services/agentSession.ts +++ b/src/node/services/agentSession.ts @@ -242,6 +242,14 @@ type GoalInterventionPolicy = NonNullable; +} + interface AutoRetryResumeRequest { // Same-session auto-retry must preserve the full normalized request because // ACP correlation/delegation lives in transient send options that are @@ -255,6 +263,12 @@ interface AutoRetryResumeRequest { /** The retried stream was admitted under resumeStream's revalidation; the retry repeats it. */ revalidateAdmission?: boolean; modelFallbackProgress?: ModelFallbackProgress; + /** + * Delegated turn the retried stream continues when `options.muxMetadata` does not carry it (a + * bash-monitor wake inherits its correlation from history), so revalidation still reaches the + * owner's handle. + */ + workspaceTurnMetadata?: WorkspaceTurnMuxMetadata; } function stripGoalInterventionPolicy(options: SendMessageOptions): SendMessageOptions { @@ -835,11 +849,8 @@ export class AgentSession { * the turn, and this evidence keeps the late claim from settling it as failed first. */ private readonly consumedContinuationCuts = new Map(); - /** Owner settlements for forfeited continuations that have not landed yet (settleOwedForfeits). */ - private readonly owedForfeitSettlements = new Map< - string, - { correlation: WorkspaceTurnMuxMetadata; reason: string; inFlight: boolean } - >(); + /** Owner settlements for forfeited continuations that have not landed yet (settleOwedForfeit). */ + private readonly owedForfeitSettlements = new Map(); private owedForfeitSettlementRetryTimer: ReturnType | null = null; private consecutiveStrandedResumes = 0; @@ -1457,7 +1468,8 @@ export class AgentSession { goalId?: string, stepBudget?: number, revalidateAdmission?: boolean, - modelFallbackProgress?: ModelFallbackProgress + modelFallbackProgress?: ModelFallbackProgress, + workspaceTurnMetadata?: WorkspaceTurnMuxMetadata ): void { if (!options) { this.lastAutoRetryResumeRequest = undefined; @@ -1472,6 +1484,7 @@ export class AgentSession { ...(stepBudget != null ? { stepBudget } : {}), ...(revalidateAdmission === true ? { revalidateAdmission: true } : {}), ...(modelFallbackProgress != null ? { modelFallbackProgress } : {}), + ...(workspaceTurnMetadata != null ? { workspaceTurnMetadata } : {}), }; } @@ -1514,6 +1527,7 @@ export class AgentSession { stepBudget: request.stepBudget, revalidateAdmission: request.revalidateAdmission, modelFallbackProgress: request.modelFallbackProgress, + workspaceTurnMetadata: request.workspaceTurnMetadata, }); if (result.success) { if (result.data.refusedBy != null) { @@ -3237,6 +3251,8 @@ export class AgentSession { modelFallbackProgress?: ModelFallbackProgress; /** For a send continuing an interrupted turn: it ran under resumeStream's revalidation. */ revalidateAdmission?: boolean; + /** For a send continuing a delegated turn its `muxMetadata` does not carry (a wake). */ + workspaceTurnMetadata?: WorkspaceTurnMuxMetadata; /** * Launch-boundary admission probe (goal and delegated-turn state), rechecked by * StreamManager right before the stream registers; unlike admissionStale it must not @@ -4279,7 +4295,8 @@ export class AgentSession { internal?.goalId, internal?.stepBudget, internal?.revalidateAdmission, - internal?.modelFallbackProgress + internal?.modelFallbackProgress, + internal?.workspaceTurnMetadata ); try { await internal?.onAccepted?.(); @@ -4453,6 +4470,8 @@ export class AgentSession { stepBudget?: number; /** Fallback chain the resumed stream continues when it continues a cut turn. */ modelFallbackProgress?: ModelFallbackProgress; + /** Delegated turn to revalidate against when `options.muxMetadata` does not carry it. */ + workspaceTurnMetadata?: WorkspaceTurnMuxMetadata; } ): Promise> { this.assertNotDisposed("resumeStream"); @@ -4491,7 +4510,8 @@ export class AgentSession { internal?.goalId, internal?.stepBudget, internal?.revalidateAdmission, - internal?.modelFallbackProgress + internal?.modelFallbackProgress, + internal?.workspaceTurnMetadata ); // Claim the turn before any await: the admission gates below do I/O, and a manual send // entering meanwhile must see a busy session rather than start a stream this resume @@ -4523,6 +4543,7 @@ export class AgentSession { goalKind: internal.goalKind, goalId: internal.goalId, muxMetadata: optionsForStream.muxMetadata, + workspaceTurnMetadata: internal.workspaceTurnMetadata, }); if (!admission.admissible) { return Ok({ started: false, refusedBy: admission.refusedBy }); @@ -4586,6 +4607,8 @@ export class AgentSession { goalKind?: GoalSyntheticMessageKind; goalId?: string; muxMetadata: unknown; + /** The delegated turn when `muxMetadata` does not carry it (an inherited correlation). */ + workspaceTurnMetadata?: WorkspaceTurnMuxMetadata; }): Promise< | { admissible: false; refusedBy: "goal" | "workspace-turn" } | { @@ -4609,7 +4632,7 @@ export class AgentSession { let turnAdmissionStale: (() => boolean) | undefined; if (this.admitStrandedTurnResume) { const admission = await this.admitStrandedTurnResume( - getWorkspaceTurnMuxMetadata(input.muxMetadata) + getWorkspaceTurnMuxMetadata(input.muxMetadata) ?? input.workspaceTurnMetadata ); if (!admission.admissible) { return { admissible: false, refusedBy: "workspace-turn" }; @@ -5877,6 +5900,7 @@ export class AgentSession { goalKind?: GoalSyntheticMessageKind; goalId?: string; muxMetadata: unknown; + workspaceTurnMetadata?: WorkspaceTurnMuxMetadata; retryLabel: string; }): Promise<{ refuseStreamStart?: () => boolean } | undefined> { if (input.stepBudget != null && input.stepBudget <= 0) { @@ -5899,6 +5923,46 @@ export class AgentSession { return { refuseStreamStart: admission.refuseStreamStart }; } + /** + * Runs an in-session retry's launch under PREPARING. False when no stream registered: an Err, + * or a launch refused at the boundary, which returns Ok before any stream registers (like a + * refused resumeStream). Either way the recovery decision stays pending: the terminal path in + * handleStreamError resolves it once settlement state is final, so waiters (task/workspace-turn + * settlement) never observe a transient "retry preparing" that already ended before startup. + */ + private async launchInSessionRetry( + retryLabel: string, + refuseStreamStart: (() => boolean) | undefined, + launch: () => Promise> + ): Promise { + try { + const result = await launch(); + if (!result.success) { + log.error(`${retryLabel} failed to start`, { + workspaceId: this.workspaceId, + error: result.error, + }); + return false; + } + // stream-start moves the turn to STREAMING synchronously inside startStream, so a turn + // still PREPARING here registered nothing. + if ( + this.turnPhase === TurnPhase.PREPARING && + (refuseStreamStart?.() === true || this.disposed || this.shuttingDown) + ) { + log.info(`${retryLabel} refused at the launch boundary`, { + workspaceId: this.workspaceId, + }); + return false; + } + return true; + } finally { + if (this.turnPhase === TurnPhase.PREPARING) { + this.setTurnPhase(TurnPhase.IDLE); + } + } + } + private async maybeRetryCompactionOnContextExceeded(data: { messageId: string; errorType?: string; @@ -5958,6 +6022,7 @@ export class AgentSession { const retryStepBudget = this.activeStreamContext?.stepBudget; const retryModelFallbackProgress = this.activeStreamContext?.modelFallbackProgress; const retryRevalidateAdmission = this.activeStreamContext?.revalidateAdmission; + const retryWorkspaceTurnMetadata = this.activeStreamContext?.workspaceTurnMetadata; const retryOptionsForResume = retryOptions ?? { model: context.modelString, agentId: WORKSPACE_DEFAULTS.agentId, @@ -5982,6 +6047,7 @@ export class AgentSession { goalKind: retryGoalKind, goalId: retryGoalId, muxMetadata: retryOptionsForResume.muxMetadata, + workspaceTurnMetadata: retryWorkspaceTurnMetadata, retryLabel: "compaction retry", }); if (retryAdmission == null) { @@ -5995,43 +6061,34 @@ export class AgentSession { retryGoalId, retryStepBudget, retryRevalidateAdmission, - retryModelFallbackProgress + retryModelFallbackProgress, + retryWorkspaceTurnMetadata ); this.preparingWorkspaceTurnMetadata = getWorkspaceTurnMuxMetadata( retryOptionsForResume.muxMetadata ); this.setTurnPhase(TurnPhase.PREPARING); - let retryResult: Result; - try { - retryResult = await this.streamWithHistory( - context.modelString, - retryOptions, - isGptClass ? "auto" : undefined, - undefined, - retryAgentInitiated, - undefined, - retryGoalKind, - retryGoalId, - undefined, - retryAdmission.refuseStreamStart, - retryStepBudget, - retryRevalidateAdmission, - retryModelFallbackProgress - ); - } finally { - if (this.turnPhase === TurnPhase.PREPARING) { - this.setTurnPhase(TurnPhase.IDLE); - } - } - if (!retryResult.success) { - // Leave the recovery decision pending: the terminal path in - // handleStreamError resolves it once settlement state is final, so - // waiters (task/workspace-turn settlement) never observe a transient - // "retry preparing" that already failed before stream startup. - log.error("Compaction retry failed to start", { - workspaceId: this.workspaceId, - error: retryResult.error, - }); + const retryStarted = await this.launchInSessionRetry( + "Compaction retry", + retryAdmission.refuseStreamStart, + () => + this.streamWithHistory( + context.modelString, + retryOptions, + isGptClass ? "auto" : undefined, + undefined, + retryAgentInitiated, + undefined, + retryGoalKind, + retryGoalId, + undefined, + retryAdmission.refuseStreamStart, + retryStepBudget, + retryRevalidateAdmission, + retryModelFallbackProgress + ) + ); + if (!retryStarted) { return false; } @@ -6115,6 +6172,7 @@ export class AgentSession { goalKind: context.goalKind, goalId: context.goalId, muxMetadata: context.options?.muxMetadata, + workspaceTurnMetadata: context.workspaceTurnMetadata, retryLabel: "post-compaction retry", }); if (retryAdmission == null) { @@ -6124,37 +6182,27 @@ export class AgentSession { // Retry the same request, but without post-compaction injection. this.preparingWorkspaceTurnMetadata = getWorkspaceTurnMuxMetadata(context.options?.muxMetadata); this.setTurnPhase(TurnPhase.PREPARING); - let retryResult: Result; - try { - retryResult = await this.streamWithHistory( - context.modelString, - context.options, - context.openaiTruncationModeOverride, - true, - context.agentInitiated, - undefined, - context.goalKind, - context.goalId, - undefined, - retryAdmission.refuseStreamStart, - context.stepBudget, - context.revalidateAdmission, - context.modelFallbackProgress - ); - } finally { - if (this.turnPhase === TurnPhase.PREPARING) { - this.setTurnPhase(TurnPhase.IDLE); - } - } - - if (!retryResult.success) { - // Leave the recovery decision pending: the terminal path in - // handleStreamError resolves it once settlement state is final (see - // maybeRetryCompactionOnContextExceeded). - log.error("Post-compaction retry failed to start", { - workspaceId: this.workspaceId, - error: retryResult.error, - }); + const retryStarted = await this.launchInSessionRetry( + "Post-compaction retry", + retryAdmission.refuseStreamStart, + () => + this.streamWithHistory( + context.modelString, + context.options, + context.openaiTruncationModeOverride, + true, + context.agentInitiated, + undefined, + context.goalKind, + context.goalId, + undefined, + retryAdmission.refuseStreamStart, + context.stepBudget, + context.revalidateAdmission, + context.modelFallbackProgress + ) + ); + if (!retryStarted) { return false; } @@ -7518,38 +7566,47 @@ export class AgentSession { private forfeitStrandedTurnResume(reason: string): void { const correlation = getWorkspaceTurnMuxMetadata(this.strandedTurnResume?.options.muxMetadata); // Retain the owner's only terminal path before dropping the marker that advertised it. - const settlementOwed = this.recordOwedForfeit(correlation, reason); + const owed = this.recordOwedForfeit(correlation, reason); this.withdrawStrandedTurnResume(); - if (settlementOwed) { - this.settleOwedForfeits(); + if (owed != null) { + void this.settleOwedForfeit(owed); } } /** * A delegated turn given up outside the marker (a dropped compaction follow-up that continued - * it): the same settlement, since nothing else will end the turn for its owner. + * it): the same settlement, since nothing else will end the turn for its owner. Resolves true + * once the owner has the terminal record (or nothing was owed), false when the attempt failed + * and the settlement stays owed. */ private forfeitWorkspaceTurnContinuation( correlation: WorkspaceTurnMuxMetadata | undefined, reason: string - ): void { - if (this.recordOwedForfeit(correlation, reason)) { - this.settleOwedForfeits(); - } + ): Promise { + const owed = this.recordOwedForfeit(correlation, reason); + return owed == null ? Promise.resolve(true) : this.settleOwedForfeit(owed); } private recordOwedForfeit( correlation: WorkspaceTurnMuxMetadata | undefined, reason: string - ): boolean { + ): OwedForfeitSettlement | undefined { if (correlation == null || this.settleForfeitedWorkspaceTurnContinuation == null) { - return false; + return undefined; + } + const owed: OwedForfeitSettlement = { + key: `${correlation.ownerWorkspaceId}/${correlation.taskHandleId}/${correlation.turnId}`, + correlation, + reason, + }; + this.owedForfeitSettlements.set(owed.key, owed); + return owed; + } + + private settleOwedForfeits(): void { + for (const owed of this.owedForfeitSettlements.values()) { + void this.settleOwedForfeit(owed); } - this.owedForfeitSettlements.set( - `${correlation.ownerWorkspaceId}/${correlation.taskHandleId}/${correlation.turnId}`, - { correlation, reason, inFlight: false } - ); - return true; } /** @@ -7557,31 +7614,30 @@ export class AgentSession { * failed attempt (task store I/O) stays owed and retries on its own (including after session * disposal) as well as from idle sweeps. Settlement is idempotent on the owner's side. */ - private settleOwedForfeits(): void { - const settle = this.settleForfeitedWorkspaceTurnContinuation; - if (settle == null) { - return; + private settleOwedForfeit(owed: OwedForfeitSettlement): Promise { + if (owed.inFlight != null) { + return owed.inFlight; } - for (const [key, owed] of this.owedForfeitSettlements) { - if (owed.inFlight) { - continue; - } - owed.inFlight = true; - void settle(owed.correlation, owed.reason) - .then(() => { - if (this.owedForfeitSettlements.get(key) === owed) { - this.owedForfeitSettlements.delete(key); - } - }) - .catch((error: unknown) => { - owed.inFlight = false; - log.warn("Failed to settle forfeited workspace turn continuation; retrying", { - workspaceId: this.workspaceId, - error: getErrorMessage(error), - }); - this.scheduleOwedForfeitSettlementRetry(); + const settle = this.settleForfeitedWorkspaceTurnContinuation; + assert(settle != null, "an owed forfeit settlement requires a settler"); + owed.inFlight = settle(owed.correlation, owed.reason).then( + () => { + if (this.owedForfeitSettlements.get(owed.key) === owed) { + this.owedForfeitSettlements.delete(owed.key); + } + return true; + }, + (error: unknown) => { + owed.inFlight = undefined; + log.warn("Failed to settle forfeited workspace turn continuation; retrying", { + workspaceId: this.workspaceId, + error: getErrorMessage(error), }); - } + this.scheduleOwedForfeitSettlementRetry(); + return false; + } + ); + return owed.inFlight; } private scheduleOwedForfeitSettlementRetry(): void { @@ -8219,11 +8275,16 @@ export class AgentSession { parsePersistedWorkspaceTurnMetadata(followUp.muxMetadata) ?? parsePersistedWorkspaceTurnMetadata(followUp.workspaceTurnMetadata); const dropFollowUp = async (reason: string): Promise => { - this.forfeitWorkspaceTurnContinuation( + // The owner's terminal record lands before the follow-up leaves history: the owed + // settlement is memory, so a crash between the two would strand the handle for good. A + // failed attempt keeps the follow-up pending for the next startup to re-drop and re-settle. + const settled = await this.forfeitWorkspaceTurnContinuation( continuedTurn, `Compaction follow-up dropped: ${reason}` ); - await this.clearPendingFollowUpFromSummary(lastMessage); + if (settled) { + await this.clearPendingFollowUpFromSummary(lastMessage); + } return false; }; @@ -8476,7 +8537,8 @@ export class AgentSession { persistedGoalId, persistedStepBudget, revalidateAdmission, - persistedFallbackProgress?.success ? persistedFallbackProgress.data : undefined + persistedFallbackProgress?.success ? persistedFallbackProgress.data : undefined, + continuedTurn ); // Await sendMessage to ensure the follow-up is persisted before returning. @@ -8502,22 +8564,26 @@ export class AgentSession { ? persistedFallbackProgress.data : undefined, revalidateAdmission, + // The wake's own metadata does not carry the delegated turn; its retries must still + // revalidate against that turn's handle. + workspaceTurnMetadata: continuedTurn, }); - // The workspace or the delegated turn stopped admitting the follow-up during its preflight - // (an Err) or at the launch boundary (StreamManager refuses as a startup-aborted Ok handle, - // like a refused stranded resume): no successor stream, so the turn is settled and the - // follow-up dropped. - if (turnAdmissionStale?.() === true) { - log.info("Pending follow-up refused at admission: workspace or delegated turn", { + // The goal, the workspace, or the delegated turn stopped admitting the follow-up during its + // preflight (an Err) or at the launch boundary (StreamManager refuses as a startup-aborted Ok + // handle, like a refused stranded resume): no successor stream, so the turn is settled and + // the follow-up dropped. + if (launchAdmissionStale?.() === true) { + log.info("Pending follow-up refused at admission: goal, workspace, or delegated turn", { workspaceId: this.workspaceId, summaryMessageId: lastMessage.id, + goalRefused: goalAdmissionStale?.() === true, }); - return dropFollowUp("the workspace or delegated turn no longer admits it."); + return dropFollowUp("no longer admitted by its goal, workspace, or delegated turn."); } if (!sendResult.success) { - // A stale-admission refusal is the idle rule (or a goal transition) - // working as intended, not a recovery failure: route it through the - // same skip path as the pre-send check instead of throwing. + // A stale-admission refusal is the idle rule working as intended, not a + // recovery failure: route it through the same skip path as the pre-send + // check instead of throwing. if (followUpAdmissionStale?.() === true) { log.info("Pending follow-up refused at send admission; skipping it", { workspaceId: this.workspaceId, From 7f6de853d30b1d7af82a57041a6989654c3bf6e4 Mon Sep 17 00:00:00 2001 From: Michael Suchacz <203725896+ibetitsmike@users.noreply.github.com> Date: Fri, 4 Sep 2026 01:31:41 +0000 Subject: [PATCH 24/28] Owe no continuation for a provider-tool soft stop whose step completed a required tool StreamManager records a successful required completion tool (toolPolicy) per step and reports it on the queued-message abort (requiredToolSatisfied); the session then treats the cut turn as complete instead of registering a stranded resume, matching the loop's own queued-message stop condition. --- src/common/orpc/schemas/stream.ts | 3 + .../agentSession.queueDispatch.test.ts | 35 +++++++++ src/node/services/agentSession.ts | 13 +++- src/node/services/streamManager.test.ts | 75 +++++++++++++++++++ src/node/services/streamManager.ts | 64 ++++++++++------ 5 files changed, 162 insertions(+), 28 deletions(-) diff --git a/src/common/orpc/schemas/stream.ts b/src/common/orpc/schemas/stream.ts index 6d28002360..0b24ddfe5f 100644 --- a/src/common/orpc/schemas/stream.ts +++ b/src/common/orpc/schemas/stream.ts @@ -350,6 +350,9 @@ export const StreamAbortEventSchema = z.object({ // Steps left under the stream's ceiling at the abort; a turn cut for a queued message // resumes under this budget rather than a fresh one. stepsRemaining: z.number().int().nonnegative().optional(), + // A required completion tool succeeded in the interrupted step: the turn was complete, so a + // queued-message soft stop owes it no continuation. + requiredToolSatisfied: z.boolean().optional(), // Fallback chain state at the abort, carried into the resumed stream for the same reason. modelFallbackProgress: ModelFallbackProgressSchema.optional(), }) diff --git a/src/node/services/agentSession.queueDispatch.test.ts b/src/node/services/agentSession.queueDispatch.test.ts index c610adba38..0da5a3cb74 100644 --- a/src/node/services/agentSession.queueDispatch.test.ts +++ b/src/node/services/agentSession.queueDispatch.test.ts @@ -1079,6 +1079,41 @@ describe("AgentSession queued message tool-call dispatch", () => { } }); + test("a provider-tool soft stop after a successful required tool owes no continuation", async () => { + const workspaceId = "queue-dispatch-soft-stop-required-tool"; + const harness = await createStreamingTurnHarness(workspaceId); + const { session, cleanup, aiEmitter, aiService, streamMessage } = harness; + const stopStream = spyOn(aiService, "stopStream").mockResolvedValue(Ok(undefined)); + + try { + // The provider-executed tool that ended the batch was the turn's required completion tool: + // the loop would have stopped on it one result later, so the withdrawn wake strands nothing. + session.queueMessage( + "follow up", + { model: TEST_MODEL, agentId: "exec" }, + { synthetic: true, dedupeKey: "wake:1" } + ); + aiEmitter.emit("tool-call-end", { + ...toolCallEndEvent(workspaceId), + toolName: "web_search", + providerExecuted: true, + }); + expect(stopStream).toHaveBeenCalledTimes(1); + expect(session.removeQueuedMessagesByDedupeKeyPrefix("wake:", "superseded")).toBe(1); + aiEmitter.emit("stream-abort", { + ...streamAbortEvent(workspaceId, "queued-message", 3), + metadata: { duration: 1, stepsRemaining: 3, requiredToolSatisfied: true }, + }); + expect(await waitForCondition(() => !session.isBusy())).toBe(true); + await new Promise((resolve) => setTimeout(resolve, 25)); + expect(streamMessage).toHaveBeenCalledTimes(1); + } finally { + stopStream.mockRestore(); + session.dispose(); + await cleanup(); + } + }); + test("a resumed turn continues the cut stream's fallback chain", async () => { const workspaceId = "queue-dispatch-stranded-fallback-chain"; const harness = await createStreamingTurnHarness(workspaceId); diff --git a/src/node/services/agentSession.ts b/src/node/services/agentSession.ts index 96c42fdbc0..cd98e5f7d8 100644 --- a/src/node/services/agentSession.ts +++ b/src/node/services/agentSession.ts @@ -6672,7 +6672,8 @@ export class AgentSession { payload.metadata?.model, payload.messageId, payload.metadata?.stepsRemaining, - payload.metadata?.modelFallbackProgress + payload.metadata?.modelFallbackProgress, + payload.metadata?.requiredToolSatisfied ); if (!dispatchedQueuedMessage) { this.setTurnPhase(TurnPhase.IDLE); @@ -7739,7 +7740,8 @@ export class AgentSession { abortedModelString: string | undefined, abortedMessageId: string, abortedStepsRemaining: number | undefined, - abortedModelFallbackProgress: ModelFallbackProgress | undefined + abortedModelFallbackProgress: ModelFallbackProgress | undefined, + abortedRequiredToolSatisfied: boolean | undefined ): boolean { this.queuedProviderToolEndAbortInFlight = false; if (!isQueuedProviderToolEndAbort || this.deferQueuedFlushUntilAfterEdit) { @@ -7749,11 +7751,14 @@ export class AgentSession { // The soft stop was made on behalf of the queued message; if that message has been // withdrawn (or is withdrawn after dequeue), the interrupted turn must resume instead. // Only under the steps the cut stream had left: at zero the ceiling ended the turn, and an - // abort that reports no budget must not hand the resume a fresh one. + // abort that reports no budget must not hand the resume a fresh one. A successful required + // tool in the cut step ended the turn too (the loop's stop condition would have, one tool + // result later), as in the loop's own queued-message stop. if ( abortedStreamContext != null && abortedStepsRemaining != null && - abortedStepsRemaining > 0 + abortedStepsRemaining > 0 && + abortedRequiredToolSatisfied !== true ) { this.strandedTurnResume = buildStrandedTurnResume({ ...abortedStreamContext, diff --git a/src/node/services/streamManager.test.ts b/src/node/services/streamManager.test.ts index 49416a5811..abf3bc8cb8 100644 --- a/src/node/services/streamManager.test.ts +++ b/src/node/services/streamManager.test.ts @@ -2592,6 +2592,7 @@ describe("StreamManager - turn completion", () => { createStreamResult?: (request: unknown, abortController: AbortController) => unknown; sink?: (event: TurnEngineEvent) => void | Promise; events?: TurnEngineEvent[]; + toolPolicy?: ToolPolicy; }) { const streamManager = new StreamManager( historyService, @@ -2616,6 +2617,7 @@ describe("StreamManager - turn completion", () => { messageId: input.messageId, model: createTestLanguageModel(), providedRuntimeTempDir: "", + toolPolicy: input.toolPolicy, }) ); expect(result.success).toBe(true); @@ -2623,6 +2625,79 @@ describe("StreamManager - turn completion", () => { return { streamManager, handle: result.data }; } + // The soft stop for a queued tool-end message lands right after a provider-executed tool + // result, before the loop's step-end stop conditions run; the abort reports whether that step + // already completed a required tool so the session knows the turn was over anyway. + for (const requiredToolCase of [ + { requiredTool: "web_search", output: { ok: true }, satisfied: true }, + { requiredTool: "web_search", output: { ok: false }, satisfied: false }, + { requiredTool: "agent_report", output: { ok: true }, satisfied: false }, + ]) { + test(`a queued-message soft stop reports a satisfied required tool: ${requiredToolCase.requiredTool} -> ${requiredToolCase.output.ok} is ${requiredToolCase.satisfied}`, async () => { + const workspaceId = `soft-stop-required-${requiredToolCase.requiredTool}-${requiredToolCase.output.ok}`; + const events: TurnEngineEvent[] = []; + const managerRef: { current?: StreamManager } = {}; + let releaseToolResult!: () => void; + const toolResultGate = new Promise((resolve) => { + releaseToolResult = resolve; + }); + const started = await startWithStreamResult({ + workspaceId, + messageId: "soft-stop-required-message", + toolPolicy: [{ regex_match: requiredToolCase.requiredTool, action: "require" }], + sink: (event) => { + events.push(event); + if (event.type === "tool-call-end") { + // AgentSession asks for the soft stop from this event, synchronously. + void managerRef.current?.stopStream(workspaceId, { + soft: true, + abortReason: "queued-message", + }); + } + }, + createStreamResult: (_request, abortController) => + createStreamResultForTests( + (async function* () { + // Armed before the tool result: the soft stop aborts while that result is handled. + const aborted = new Promise((resolve) => { + abortController.signal.addEventListener("abort", () => resolve(), { once: true }); + }); + yield { type: "start-step" }; + yield { + type: "tool-call", + toolCallId: "call-1", + toolName: "web_search", + input: { query: "x" }, + providerExecuted: true, + }; + await toolResultGate; + yield { + type: "tool-result", + toolCallId: "call-1", + toolName: "web_search", + output: requiredToolCase.output, + providerExecuted: true, + }; + await aborted; + })() + ), + }); + managerRef.current = started.streamManager; + releaseToolResult(); + + expect(await started.handle.completion).toEqual({ + status: "aborted", + abortReason: "queued-message", + }); + const abort = events.find((event) => event.type === "stream-abort"); + expect(abort?.type).toBe("stream-abort"); + if (abort?.type !== "stream-abort") throw new Error("Expected a stream-abort event"); + expect(abort.metadata?.requiredToolSatisfied).toBe( + requiredToolCase.satisfied ? true : undefined + ); + }); + } + test("pre-start failures return Err while successful startup owns an aborted completion", async () => { const streamManager = new StreamManager(historyService); const model = createTestLanguageModel(); diff --git a/src/node/services/streamManager.ts b/src/node/services/streamManager.ts index e8565b18a8..da94787ce7 100644 --- a/src/node/services/streamManager.ts +++ b/src/node/services/streamManager.ts @@ -660,6 +660,29 @@ function zeroTokenUsage(): LanguageModelV2Usage { return { inputTokens: 0, outputTokens: 0, totalTokens: 0 }; } +/** + * Completion-tool success check: completion/routing tools use explicit success/ok markers + * (agent_report, propose_plan). When a marker is present, respect it (success:false means the tool + * should be retried, so the turn goes on). When no marker is present (MCP tools, arbitrary + * required tools), treat non-null object results as successful completion unless error-shaped. + */ +function isSuccessfulRequiredToolOutput(output: unknown): boolean { + if (typeof output !== "object" || output === null) { + return false; + } + const parsedOutput = output as Record; + if ("success" in parsedOutput) { + return parsedOutput.success === true; + } + if ("ok" in parsedOutput) { + return parsedOutput.ok === true; + } + if (parsedOutput.error != null || parsedOutput.isError === true) { + return false; + } + return true; +} + function hasIncompleteToolCallPart(parts: CompletedMessagePart[]): boolean { return parts.some((part) => part.type === "dynamic-tool" && part.state !== "output-available"); } @@ -693,6 +716,11 @@ interface WorkspaceStreamInfo { // request.stepBudget (restartStepBudget). stepCount: number; + // A required completion tool (request.toolPolicy) succeeded in the step in progress. The + // step-end stop condition would end the turn on it, so a queued-message soft stop that lands + // first (after a provider-executed tool result) owes the turn no continuation. + requiredToolSatisfied?: boolean; + // Workflow tools can create the durable run before their stream part is stored. Keep the exact // attachment and apply it as soon as the matching dynamic-tool part lands. pendingWorkflowRunAttachments: Map; @@ -2010,6 +2038,7 @@ export class StreamManager { model: streamInfo.model, stepsRemaining: this.remainingStepBudget(streamInfo), modelFallbackProgress: modelFallbackProgressOf(streamInfo.modelFallback), + ...(streamInfo.requiredToolSatisfied === true ? { requiredToolSatisfied: true } : {}), }, abortReason, abandonPartial, @@ -2228,29 +2257,6 @@ export class StreamManager { > ): Array> { const stepBudget = request.stepBudget ?? MAX_STREAM_STEPS; - // Completion-tool stop check: completion/routing tools use explicit - // success/ok markers (agent_report, propose_plan). - // When a marker is present, respect it — success:false means the tool - // should be retried, so don't stop. When no marker is present (e.g., - // MCP tools, arbitrary required tools), treat non-null object results - // as successful completion unless the result is error-shaped. - const isSuccessfulOutput = (output: unknown): boolean => { - if (typeof output !== "object" || output === null) { - return false; - } - const parsedOutput = output as Record; - if ("success" in parsedOutput) { - return parsedOutput.success === true; - } - if ("ok" in parsedOutput) { - return parsedOutput.ok === true; - } - if (parsedOutput.error != null || parsedOutput.isError === true) { - return false; - } - return true; - }; - const requiredPatterns = buildRequiredToolPatterns(request.toolPolicy); const hasSuccessfulRequiredToolResult: ReturnType = ({ steps }) => { @@ -2262,7 +2268,7 @@ export class StreamManager { lastStep?.toolResults?.some( (toolResult) => requiredPatterns.some((pattern) => pattern.test(toolResult.toolName)) && - isSuccessfulOutput(toolResult.output) + isSuccessfulRequiredToolOutput(toolResult.output) ) ?? false ); }; @@ -2684,6 +2690,15 @@ export class StreamManager { output, providerExecuted ); + if ( + streamInfo.requiredToolSatisfied !== true && + isSuccessfulRequiredToolOutput(output) && + buildRequiredToolPatterns(streamInfo.request.toolPolicy).some((pattern) => + pattern.test(toolName) + ) + ) { + streamInfo.requiredToolSatisfied = true; + } await this.checkSoftCancelStream(workspaceId, streamInfo); } @@ -3527,6 +3542,7 @@ export class StreamManager { case "start-step": { streamInfo.currentStepStartIndex = streamInfo.parts.length; streamInfo.stepCount += 1; + streamInfo.requiredToolSatisfied = false; break; } From a7f2394dc423369f0eb699cb41172c8a78112239 Mon Sep 17 00:00:00 2001 From: Michael Suchacz <203725896+ibetitsmike@users.noreply.github.com> Date: Fri, 4 Sep 2026 02:22:04 +0000 Subject: [PATCH 25/28] Roll back a launch-refused continuation's rows, fail closed on a malformed persisted step budget, persist a queued cut's remainder for startup retry, and skip withdrawn entries in the all-predecessors scan - sendMessage: a synthetic continuation refused by its own launch probe at the boundary (Ok, turn still PREPARING) rolls its persisted rows back, emits their deletion, and re-derives goal state from the tail it leaves; the one rollback past goal sync. - dispatchPendingFollowUp: a present but malformed stepBudget drops and settles the follow-up instead of reading as absent and running under the default ceiling. - StreamManager stamps stepsRemaining on the partial committed by a queued-message abort; startup auto-retry runs the interrupted row under that remainder (abandoned at zero). - MessageQueue.hasAllWorkspaceTurnContinuations ignores withdrawn entries, so a draining canceled wake no longer strips a same-turn report of its correlation. --- src/common/orpc/schemas/message.ts | 4 + src/common/types/message.ts | 2 + ...gentSession.continueMessageAgentId.test.ts | 35 ++++++++- .../agentSession.queueDispatch.test.ts | 51 ++++++++++++ .../agentSession.startupAutoRetry.test.ts | 33 ++++++++ src/node/services/agentSession.ts | 78 +++++++++++++++++-- src/node/services/messageQueue.test.ts | 24 ++++++ src/node/services/messageQueue.ts | 29 +++---- src/node/services/streamManager.test.ts | 6 ++ src/node/services/streamManager.ts | 12 ++- 10 files changed, 248 insertions(+), 26 deletions(-) diff --git a/src/common/orpc/schemas/message.ts b/src/common/orpc/schemas/message.ts index 56b9da49c4..657e8f5856 100644 --- a/src/common/orpc/schemas/message.ts +++ b/src/common/orpc/schemas/message.ts @@ -190,6 +190,10 @@ export const MuxMessageSchema = z.object({ retrySendOptions: z.any().optional(), agentId: AgentIdSchema.optional().catch(undefined), partial: z.boolean().optional(), + // Steps the cut turn had left under its ceiling when a queued message interrupted it + // (stamped on the committed partial): a startup retry of that turn runs under this budget + // instead of a fresh ceiling. Self-healing read path: a malformed value reads as absent. + stepsRemaining: z.number().int().nonnegative().optional().catch(undefined), synthetic: z.boolean().optional(), uiVisible: z.boolean().optional(), // RLM keep-recent floor: sanitized post-boundary copy of a pre-compaction row. diff --git a/src/common/types/message.ts b/src/common/types/message.ts index ec18f52883..2d45680687 100644 --- a/src/common/types/message.ts +++ b/src/common/types/message.ts @@ -937,6 +937,8 @@ export interface MuxMetadata { contextProviderMetadata?: Record; systemMessageTokens?: number; // Token count for system message sent with this request (calculated by AIService) partial?: boolean; // Whether this message was interrupted and is incomplete + /** Steps a queued-message cut left under the turn's ceiling; a startup retry runs under it. */ + stepsRemaining?: number; synthetic?: boolean; // Whether this message was synthetically generated (e.g., [CONTINUE] sentinel) /** * For queue-dispatched user turns: when the user last added to the queued diff --git a/src/node/services/agentSession.continueMessageAgentId.test.ts b/src/node/services/agentSession.continueMessageAgentId.test.ts index 39b1215739..612d8bef8a 100644 --- a/src/node/services/agentSession.continueMessageAgentId.test.ts +++ b/src/node/services/agentSession.continueMessageAgentId.test.ts @@ -675,14 +675,13 @@ describe("AgentSession continue-message agentId fallback", () => { expect(summary?.metadata?.muxMetadata).toEqual({ type: "compaction-summary" }); }); - test("dispatchPendingFollowUp drops a malformed persisted remainder", async () => { + test("dispatchPendingFollowUp drops a malformed persisted chain state", async () => { const dispatched: SendInternal[] = []; const { internals } = await createSession([ - compactionSummaryMessage("summary-malformed-remainder", { + compactionSummaryMessage("summary-malformed-chain", { text: "Continue", model: "openai:gpt-4o", agentId: "exec", - stepBudget: "seven" as unknown as number, modelFallbackProgress: { requestedModel: 1, } as unknown as CompactionFollowUpRequest["modelFallbackProgress"], @@ -697,11 +696,39 @@ describe("AgentSession continue-message agentId fallback", () => { await internals.dispatchPendingFollowUp(); + // The chain state is only a preference order: the follow-up runs on its model's own chain. expect(dispatched).toHaveLength(1); - expect(dispatched[0]?.stepBudget).toBeUndefined(); expect(dispatched[0]?.modelFallbackProgress).toBeUndefined(); }); + test("dispatchPendingFollowUp settles and drops a follow-up whose persisted step budget is malformed", async () => { + const settle = mock((_correlation: unknown, _reason: string) => Promise.resolve()); + const sendMessage = mock(() => Promise.resolve({ success: true as const })); + const { internals, historyService } = await createSession( + [ + compactionSummaryMessage("summary-malformed-budget", { + text: "Continue", + model: "openai:gpt-4o", + agentId: "exec", + stepBudget: "seven" as unknown as number, + muxMetadata: DELEGATED_TURN, + }), + ], + { settleForfeitedWorkspaceTurnContinuation: settle } + ); + internals.sendMessage = sendMessage; + + expect(await internals.dispatchPendingFollowUp()).toBe(false); + + // The interrupted turn's ceiling is unknowable from this row; it must not get the default one. + expect(sendMessage).not.toHaveBeenCalled(); + expect(settle).toHaveBeenCalledTimes(1); + expect(settle.mock.calls[0]?.[0]).toEqual(DELEGATED_TURN); + const tail = await historyService.getLastMessages("ws", 1); + const summary = tail.success ? tail.data[0] : undefined; + expect(summary?.metadata?.muxMetadata).toEqual({ type: "compaction-summary" }); + }); + test("dispatchPendingFollowUp forwards strictAgentResolution to the resumed turn", async () => { let dispatchedOptions: SendOptions | undefined; const { internals } = await createSession([ diff --git a/src/node/services/agentSession.queueDispatch.test.ts b/src/node/services/agentSession.queueDispatch.test.ts index 0da5a3cb74..464b33aeae 100644 --- a/src/node/services/agentSession.queueDispatch.test.ts +++ b/src/node/services/agentSession.queueDispatch.test.ts @@ -1204,6 +1204,57 @@ describe("AgentSession queued message tool-call dispatch", () => { } }); + test("a send refused by its launch probe at the boundary leaves no durable row", async () => { + const workspaceId = "queue-dispatch-send-launch-refused"; + const aiEmitter = new EventEmitter(); + let refused = false; + const streamMessage = mock((_options: StreamMessageOptions) => { + // The goal is paused right before registration: StreamManager returns a startup-aborted + // handle without a stream-start. + refused = true; + return Promise.resolve( + Ok({ + messageId: "assistant-1", + completion: Promise.resolve({ status: "aborted" as const, abortReason: "startup" }), + }) + ); + }); + const { session, historyService, events, cleanup } = await createAgentSessionHarness({ + workspaceId, + aiEmitter, + aiServiceOverrides: { streamMessage: streamMessage as unknown as AIService["streamMessage"] }, + captureEvents: true, + }); + + try { + const sent = await session.sendMessage( + "Continue", + { model: TEST_MODEL, agentId: "exec" }, + { synthetic: true, refuseStreamStart: () => refused } + ); + expect(sent.success).toBe(true); + expect(streamMessage).toHaveBeenCalledTimes(1); + + // The row was persisted and shown, then withdrawn with the launch it was appended for. + const persisted = events.find((event) => event.type === "message" && event.role === "user"); + const persistedSequence = + persisted?.type === "message" ? persisted.metadata?.historySequence : undefined; + expect(persistedSequence).toBeDefined(); + const history = await historyService.getHistoryFromLatestBoundary(workspaceId); + expect(history.success && history.data.some((message) => message.role === "user")).toBe( + false + ); + const deleted = events.find((event) => event.type === "delete"); + expect(deleted?.type === "delete" ? deleted.historySequences : undefined).toEqual([ + persistedSequence ?? -1, + ]); + expect(session.isBusy()).toBe(false); + } finally { + session.dispose(); + await cleanup(); + } + }); + test("an auto-retry runs under what the failed resumed attempt left of the step budget", async () => { const workspaceId = "queue-dispatch-stranded-retry-step-budget"; const aiEmitter = new EventEmitter(); diff --git a/src/node/services/agentSession.startupAutoRetry.test.ts b/src/node/services/agentSession.startupAutoRetry.test.ts index 42ee1ea7b5..b2d99e2fad 100644 --- a/src/node/services/agentSession.startupAutoRetry.test.ts +++ b/src/node/services/agentSession.startupAutoRetry.test.ts @@ -376,6 +376,39 @@ describe("AgentSession startup auto-retry recovery", () => { session.dispose(); }); + test("startup auto-retry runs a queued-cut row under the remainder it persisted", async () => { + const workspaceId = "startup-retry-cut-step-budget"; + const { session, historyService, cleanup } = await createSessionBundle(workspaceId); + cleanups.push(cleanup); + + await historyService.appendToHistory( + workspaceId, + createMuxMessage("original-user", "user", "Continue the original task", { + timestamp: Date.now(), + }) + ); + // The soft-aborted partial was committed with the cut turn's remainder, then the process + // exited before the in-memory resume started. + await historyService.appendToHistory( + workspaceId, + createMuxMessage("assistant-cut", "assistant", "Interrupted response", { + timestamp: Date.now(), + partial: true, + stepsRemaining: 3, + }) + ); + + session.ensureStartupAutoRetryCheck(); + await (session as unknown as { startupAutoRetryCheckPromise: Promise | null }) + .startupAutoRetryCheckPromise; + + const retryRequest = ( + session as unknown as { lastAutoRetryResumeRequest?: { stepBudget?: number } } + ).lastAutoRetryResumeRequest; + expect(retryRequest?.stepBudget).toBe(3); + session.dispose(); + }); + test("hidden completed subagent reports preserve the existing startup retry fallback", async () => { const workspaceId = "startup-retry-hidden-subagent-report"; const { session, historyService, events, cleanup } = await createSessionBundle(workspaceId); diff --git a/src/node/services/agentSession.ts b/src/node/services/agentSession.ts index cd98e5f7d8..454002bb03 100644 --- a/src/node/services/agentSession.ts +++ b/src/node/services/agentSession.ts @@ -2521,7 +2521,22 @@ export class AgentSession { } const { agentInitiated, goalKind, goalId, ...resumeOptions } = retryRequest; - this.setAutoRetryResumeState(resumeOptions, agentInitiated, goalKind, goalId); + // A row cut for a queued message carries what the turn had left of its ceiling; the retry + // continues that turn, so it runs under the remainder (and is abandoned at zero) rather + // than a fresh ceiling. + const interruptedAssistant = + partial?.role === "assistant" + ? partial + : lastHistoryMessage?.role === "assistant" + ? lastHistoryMessage + : undefined; + this.setAutoRetryResumeState( + resumeOptions, + agentInitiated, + goalKind, + goalId, + interruptedAssistant?.metadata?.stepsRemaining + ); } // Disk reads above may race with user actions; retry once the current work settles @@ -3365,6 +3380,31 @@ export class AgentSession { ) ); }; + const rollbackRefusedLaunchRows = async (): Promise => { + const historyResult = await this.historyService.getHistoryFromLatestBoundary( + this.workspaceId + ); + const historySequences = historyResult.success + ? historyResult.data + .filter((message) => persistedCancelableMessageIds.includes(message.id)) + .map((message) => message.metadata?.historySequence) + .filter((sequence): sequence is number => isNonNegativeInteger(sequence)) + : []; + if (!(await rollbackPersistedTurnRows())) { + return; + } + if (historySequences.length > 0) { + this.emitChatEvent({ type: "delete", historySequences }); + } + try { + await this.workspaceGoalService?.syncGoalModeWithChatTail(this.workspaceId); + } catch (error) { + log.warn("Failed to resync goal state after a refused launch", { + workspaceId: this.workspaceId, + error: getErrorMessage(error), + }); + } + }; let cancellationHandled = false; let cancellationDisabled = false; const cancelBeforeAcceptance = async (): Promise => { @@ -4157,6 +4197,8 @@ export class AgentSession { // leaves no trace for a later human resume to replay into provider context. Past this point // rollback is forbidden by design (goal sync observes the durable row), so a Stop landing in // the remaining pre-stream awaits refuses the turn at the PREPARING gate with rows retained. + // The one exception is a synthetic continuation refused by its own launch probe + // (rollbackRefusedLaunchRows), which re-derives goal state after removing its rows. if (internal?.admissionStale?.() === true) { const rolledBack = await rollbackPersistedTurnRows(); // Probe-carrying sends are peer messages whose caller already returned success when the @@ -4394,6 +4436,19 @@ export class AgentSession { ) ); } + // The send's own launch probe refused the turn at the boundary (Ok with the turn still + // PREPARING, nothing registered): the goal or delegated turn this synthetic continuation + // served ended during startup. Its rows would otherwise sit at the tail as a prompt the + // next unrelated request replays, so they go with the launch; goal state re-derives from + // the tail they leave (the one rollback past goal sync, see the horizon note above). + if ( + streamResult.success && + this.turnPhase === TurnPhase.PREPARING && + !hasPreTurnMessages && + internal?.refuseStreamStart?.() === true + ) { + await rollbackRefusedLaunchRows(); + } return streamResult; } finally { // Success should advance via stream events; if startup never emitted any, don't leave the @@ -8325,12 +8380,21 @@ export class AgentSession { return dropFollowUp("legacy goal follow-up without goal identity."); } - // Same raw JSON boundary: the interrupted turn's remainder is optional and dropped if malformed - // (the follow-up then runs under the default ceiling and its model's own chain). - const persistedStepBudget = - typeof followUp.stepBudget === "number" && Number.isInteger(followUp.stepBudget) - ? Math.max(0, followUp.stepBudget) - : undefined; + // Same raw JSON boundary. A present but malformed remainder fails closed: the interrupted turn + // ran under a ceiling this row can no longer state, and an absent-legacy reading would hand a + // nearly spent autonomous turn the default ceiling instead. The chain state is only a + // preference order, so a malformed one falls back to the model's own chain. + const persistedStepBudget = followUp.stepBudget; + if ( + persistedStepBudget !== undefined && + !(Number.isInteger(persistedStepBudget) && persistedStepBudget >= 0) + ) { + log.warn("Discarding pending follow-up with a malformed persisted step budget", { + workspaceId: this.workspaceId, + summaryMessageId: lastMessage.id, + }); + return dropFollowUp("malformed persisted step budget."); + } const persistedFallbackProgress = followUp.modelFallbackProgress != null ? ModelFallbackProgressSchema.safeParse(followUp.modelFallbackProgress) diff --git a/src/node/services/messageQueue.test.ts b/src/node/services/messageQueue.test.ts index 5729761ff6..6822743cc1 100644 --- a/src/node/services/messageQueue.test.ts +++ b/src/node/services/messageQueue.test.ts @@ -604,6 +604,30 @@ describe("MessageQueue", () => { ).toBe(false); }); + it("ignores withdrawn predecessors when checking that every entry continues the turn", () => { + const withdrawn = new AbortController(); + queue.add( + "withdrawn wake", + { + model: "gpt-4", + agentId: "exec", + muxMetadata: { type: "bash-monitor-wake", records: [] }, + }, + { synthetic: true, agentInitiated: true, cancelSignal: withdrawn.signal } + ); + queue.add("Follow up", { model: "gpt-4", agentId: "exec", muxMetadata: metadata }); + + expect( + queue.hasAllWorkspaceTurnContinuations("wst_followup", "parent-workspace", "turn-1") + ).toBe(false); + + // Withdrawn but not yet drained: no longer pending work, so it supersedes nothing. + withdrawn.abort(); + expect( + queue.hasAllWorkspaceTurnContinuations("wst_followup", "parent-workspace", "turn-1") + ).toBe(true); + }); + it("reads the next entry past withdrawn ones for wake, correlation, and cut candidate", () => { const withdrawn = new AbortController(); queue.add( diff --git a/src/node/services/messageQueue.ts b/src/node/services/messageQueue.ts index ab11950ca6..c6b4912be6 100644 --- a/src/node/services/messageQueue.ts +++ b/src/node/services/messageQueue.ts @@ -286,28 +286,29 @@ export class MessageQueue { } /** - * Whether every queued entry continues the exact workspace turn correlation. + * Whether every pending queued entry continues the exact workspace turn correlation. * * The caller uses this for a new continuation that has not entered the queue. - * An unrelated entry anywhere ahead of it supersedes the correlation. + * An unrelated pending entry anywhere ahead of it supersedes the correlation; a withdrawn + * entry still draining is not pending work (see nextDispatchableEntry) and supersedes nothing. */ hasAllWorkspaceTurnContinuations( taskHandleId: string, ownerWorkspaceId: string, turnId: string ): boolean { - return ( - this.entries.length > 0 && - this.entries.every((entry) => { - const metadata = entry.muxMetadata; - return ( - isWorkspaceTurnMetadata(metadata) && - metadata.taskHandleId === taskHandleId && - metadata.ownerWorkspaceId === ownerWorkspaceId && - metadata.turnId === turnId - ); - }) - ); + return this.entries.every((entry) => { + if (entry.cancelSignal?.aborted === true) { + return true; + } + const metadata = entry.muxMetadata; + return ( + isWorkspaceTurnMetadata(metadata) && + metadata.taskHandleId === taskHandleId && + metadata.ownerWorkspaceId === ownerWorkspaceId && + metadata.turnId === turnId + ); + }); } /** diff --git a/src/node/services/streamManager.test.ts b/src/node/services/streamManager.test.ts index abf3bc8cb8..0024b319dc 100644 --- a/src/node/services/streamManager.test.ts +++ b/src/node/services/streamManager.test.ts @@ -2593,6 +2593,7 @@ describe("StreamManager - turn completion", () => { sink?: (event: TurnEngineEvent) => void | Promise; events?: TurnEngineEvent[]; toolPolicy?: ToolPolicy; + stepBudget?: number; }) { const streamManager = new StreamManager( historyService, @@ -2618,6 +2619,7 @@ describe("StreamManager - turn completion", () => { model: createTestLanguageModel(), providedRuntimeTempDir: "", toolPolicy: input.toolPolicy, + stepBudget: input.stepBudget, }) ); expect(result.success).toBe(true); @@ -2644,6 +2646,7 @@ describe("StreamManager - turn completion", () => { const started = await startWithStreamResult({ workspaceId, messageId: "soft-stop-required-message", + stepBudget: 5, toolPolicy: [{ regex_match: requiredToolCase.requiredTool, action: "require" }], sink: (event) => { events.push(event); @@ -2695,6 +2698,9 @@ describe("StreamManager - turn completion", () => { expect(abort.metadata?.requiredToolSatisfied).toBe( requiredToolCase.satisfied ? true : undefined ); + // The cut's remainder rides on the committed partial for a startup retry after a crash. + const partial = await historyService.readPartial(workspaceId); + expect(partial?.metadata?.stepsRemaining).toBe(4); }); } diff --git a/src/node/services/streamManager.ts b/src/node/services/streamManager.ts index da94787ce7..c31eea5f8a 100644 --- a/src/node/services/streamManager.ts +++ b/src/node/services/streamManager.ts @@ -1966,12 +1966,21 @@ export class StreamManager { streamInfo ); + // A queued-message cut owes the turn a continuation under what it left of the ceiling. The + // committed partial is that remainder's only durable carrier: a process exit before the + // in-memory resume starts leaves startup recovery to retry the row from history. + const stepsRemaining = + abortReason === "queued-message" ? this.remainingStepBudget(streamInfo) : undefined; + // Stamp the aborted turn's usage onto the partial message BEFORE emitting // stream-abort (whose handler commits the partial to chat.jsonl). Analytics // prices history rows from metadata.usage, so without this every // interrupted turn — user Esc, queued tool-end preemption, monitor wakes — // would ingest as $0 even though the provider billed all completed steps. - if (!abandonPartial && (usage !== undefined || streamInfo.toolModelUsages.length > 0)) { + if ( + !abandonPartial && + (usage !== undefined || streamInfo.toolModelUsages.length > 0 || stepsRemaining !== undefined) + ) { try { await this.awaitPendingPartialWrite(streamInfo); const partialMessage = this.buildPartialAssistantMessage(streamInfo, { @@ -1984,6 +1993,7 @@ export class StreamManager { ...(streamInfo.toolModelUsages.length > 0 ? { toolModelUsages: streamInfo.toolModelUsages.map(clonePersistedToolModelUsage) } : {}), + ...(stepsRemaining !== undefined ? { stepsRemaining } : {}), }, }); await this.historyService.writePartial(workspaceId as string, partialMessage); From 25197940e3243e75f06487d9670f88708cfa1a59 Mon Sep 17 00:00:00 2001 From: Michael Suchacz <203725896+ibetitsmike@users.noreply.github.com> Date: Fri, 4 Sep 2026 02:29:08 +0000 Subject: [PATCH 26/28] Give the model-only notification stream fixtures the request field the tool-result path now reads --- .../services/streamManager.modelOnlyNotifications.test.ts | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/src/node/services/streamManager.modelOnlyNotifications.test.ts b/src/node/services/streamManager.modelOnlyNotifications.test.ts index 93369e4db4..3d8f688f79 100644 --- a/src/node/services/streamManager.modelOnlyNotifications.test.ts +++ b/src/node/services/streamManager.modelOnlyNotifications.test.ts @@ -88,6 +88,8 @@ describe("StreamManager - model-only tool notifications", () => { lastStepUsage: undefined, lastStepProviderMetadata: undefined, toolModelUsages: [], + request: { messages: [], providerOptions: undefined }, + stepCount: 0, }; const method = Reflect.get(streamManager, "processStreamWithCleanup") as unknown; @@ -182,6 +184,8 @@ describe("StreamManager - model-only tool notifications", () => { lastStepUsage: undefined, lastStepProviderMetadata: undefined, toolModelUsages: [], + request: { messages: [], providerOptions: undefined }, + stepCount: 0, }; const method = Reflect.get(streamManager, "processStreamWithCleanup") as unknown; From 6a8263f3243e3e260b411fc7e3bc12bd9a73bd73 Mon Sep 17 00:00:00 2001 From: Michael Suchacz <203725896+ibetitsmike@users.noreply.github.com> Date: Fri, 4 Sep 2026 03:11:54 +0000 Subject: [PATCH 27/28] Sweep a failed stranded resume to the cap; fail closed on a malformed persisted remainder Codex round 25: a stranded resume that failed before its stream stayed owed with no later poke on an idle session, leaving a delegated owner running forever; the sweep now runs again after the failure, bounded by the consecutive cap whose forfeit settles the owner. Startup auto-retry validates the partial row's raw stepsRemaining as a nonnegative integer and abandons the retry otherwise instead of running under the default ceiling. --- .../agentSession.queueDispatch.test.ts | 115 +++++++++--------- .../agentSession.startupAutoRetry.test.ts | 36 ++++++ src/node/services/agentSession.ts | 29 +++-- 3 files changed, 118 insertions(+), 62 deletions(-) diff --git a/src/node/services/agentSession.queueDispatch.test.ts b/src/node/services/agentSession.queueDispatch.test.ts index 464b33aeae..01795dbd77 100644 --- a/src/node/services/agentSession.queueDispatch.test.ts +++ b/src/node/services/agentSession.queueDispatch.test.ts @@ -1521,23 +1521,20 @@ describe("AgentSession queued message tool-call dispatch", () => { const wake = harness.queueCancelableWake(); harness.latestRequest().onQueuedMessageStop?.(queuedStop()); wake.abort("monitor consumed"); + const attemptsBefore = pricingGate.mock.calls.length; aiEmitter.emit("stream-end", streamEndEvent(workspaceId)); expect(await waitForCondition(() => !session.isBusy())).toBe(true); - const attemptsBefore = pricingGate.mock.calls.length; - // Each idle poke retries the failing resume until the cap, then the marker is dropped and - // further pokes do nothing. - for (let poke = 1; poke <= 4; poke += 1) { - session.drainQueuedMessagesIfIdle(); - expect(await waitForCondition(() => !session.isBusy())).toBe(true); - } + // The failing resume is swept again on its own until the cap, with no poke; then the + // marker is dropped and later pokes do nothing, even once the gate opens. await new Promise((resolve) => setTimeout(resolve, 25)); - expect(pricingGate.mock.calls.length - attemptsBefore).toBe(2); + expect(pricingGate.mock.calls.length - attemptsBefore).toBe(3); expect(streamMessage).toHaveBeenCalledTimes(1); gateOpen = true; session.drainQueuedMessagesIfIdle(); await new Promise((resolve) => setTimeout(resolve, 25)); + expect(pricingGate.mock.calls.length - attemptsBefore).toBe(3); expect(streamMessage).toHaveBeenCalledTimes(1); } finally { session.dispose(); @@ -1764,21 +1761,24 @@ describe("AgentSession queued message tool-call dispatch", () => { } }); - test("keeps the continuation owed when the resume fails before its stream starts", async () => { + test("a resume that fails before its stream starts is swept again without a poke", async () => { const workspaceId = "queue-dispatch-stranded-retry"; - let gateOpen = false; + // The initial send passes the gate; the gate then fails this many calls (the resume's). + let failingGateCalls = 0; const workspaceGoalService = { - assertPricedModelForBudgetedGoal: mock(() => - Promise.resolve(gateOpen ? Ok(undefined) : Err({ type: "unknown", raw: "gate closed" })) - ), + assertPricedModelForBudgetedGoal: mock(() => { + if (failingGateCalls > 0) { + failingGateCalls -= 1; + return Promise.resolve(Err({ type: "unknown", raw: "gate closed" })); + } + return Promise.resolve(Ok(undefined)); + }), recordStreamAccounting: mock(() => Promise.resolve()), applyPendingAfterStreamEnd: mock(() => Promise.resolve()), requestContinuationAfterStreamEnd: mock(() => Promise.resolve()), recordStreamStarted: mock(() => Promise.resolve()), syncGoalModeWithChatTail: mock(() => Promise.resolve(null)), } as unknown as WorkspaceGoalService; - // The gate is closed for the resume only: the initial send passes while it is open. - gateOpen = true; const harness = await createStreamingTurnHarness(workspaceId, { harness: { workspaceGoalService }, sendInternal: { synthetic: true, agentInitiated: true }, @@ -1786,19 +1786,17 @@ describe("AgentSession queued message tool-call dispatch", () => { const { session, cleanup, aiEmitter, streamMessage } = harness; try { - gateOpen = false; + failingGateCalls = 1; const wake = harness.queueCancelableWake(); harness.latestRequest().onQueuedMessageStop?.(queuedStop()); wake.abort("monitor consumed"); aiEmitter.emit("stream-end", streamEndEvent(workspaceId)); - expect(await waitForCondition(() => !session.isBusy())).toBe(true); - await new Promise((resolve) => setTimeout(resolve, 25)); - expect(streamMessage).toHaveBeenCalledTimes(1); - - gateOpen = true; - session.drainQueuedMessagesIfIdle(); + // A transient failure before the resume's stream leaves the continuation owed on an idle + // session that nothing else pokes; the sweep tries again on its own and the turn resumes. expect(await waitForCondition(() => streamMessage.mock.calls.length === 2)).toBe(true); + expect(failingGateCalls).toBe(0); + expect(session.isBusy()).toBe(true); } finally { session.dispose(); await cleanup(); @@ -2305,24 +2303,27 @@ describe("AgentSession queued message tool-call dispatch", () => { }); /** - * Strand a delegated turn whose resume fails at the pricing gate before its stream starts: - * the owner deferred the cut stream-end on the advertised continuation and the session sits - * idle with it still owed. + * Strand a delegated turn whose owner deferred the cut stream-end on the advertised + * continuation. With `holdAdmission`, a history mutation holds turn admission across the + * stream end, so the session sits idle with the continuation still owed and no resume run. + * Without it, the resume fails at the pricing gate before its stream starts. */ - async function strandDelegatedTurnBehindClosedGate( + async function strandDelegatedTurn( workspaceId: string, extra?: { harness?: Partial>; settleForfeited?: ReturnType< typeof mock<(metadata: unknown, reason: string) => Promise> >; + holdAdmission?: boolean; } ) { let gateOpen = true; + const pricingGate = mock(() => + Promise.resolve(gateOpen ? Ok(undefined) : Err({ type: "unknown", raw: "gate closed" })) + ); const workspaceGoalService = { - assertPricedModelForBudgetedGoal: mock(() => - Promise.resolve(gateOpen ? Ok(undefined) : Err({ type: "unknown", raw: "gate closed" })) - ), + assertPricedModelForBudgetedGoal: pricingGate, recordStreamAccounting: mock(() => Promise.resolve()), applyPendingAfterStreamEnd: mock(() => Promise.resolve()), requestContinuationAfterStreamEnd: mock(() => Promise.resolve()), @@ -2341,49 +2342,48 @@ describe("AgentSession queued message tool-call dispatch", () => { sendInternal: { synthetic: true, agentInitiated: true }, }); const { session, aiEmitter } = harness; - gateOpen = false; + const hold = extra?.holdAdmission === true ? session.holdTurnAdmission() : undefined; + gateOpen = hold != null; harness.latestRequest().onQueuedMessageStop?.(queuedStop()); harness.queueCancelableWake().abort("monitor consumed"); session.clearQueue("monitor consumed"); expect(session.claimWorkspaceTurnContinuation(WORKSPACE_TURN_CORRELATION, "assistant-1")).toBe( true ); + const gateCallsBeforeCut = pricingGate.mock.calls.length; aiEmitter.emit("stream-end", streamEndEvent(workspaceId)); expect(await waitForCondition(() => !session.isBusy())).toBe(true); return { ...harness, settleForfeited, + resumeAttempts: () => pricingGate.mock.calls.length - gateCallsBeforeCut, openGate: () => { gateOpen = true; }, + releaseAdmission: () => hold?.[Symbol.dispose](), }; } test("stops advertising the delegated continuation once the resume cap is exhausted", async () => { const workspaceId = "queue-dispatch-stranded-delegated-cap"; - const harness = await strandDelegatedTurnBehindClosedGate(workspaceId); + const harness = await strandDelegatedTurn(workspaceId); const { session, cleanup, streamMessage, settleForfeited } = harness; try { - // The resume keeps failing before its stream starts; while attempts remain the owner - // defers settlement, and once the cap is exhausted the continuation is no longer - // advertised and the owner is told to settle the turn it deferred at the cut. - session.drainQueuedMessagesIfIdle(); - expect(await waitForCondition(() => !session.isBusy())).toBe(true); - expect( - session.claimWorkspaceTurnContinuation(WORKSPACE_TURN_CORRELATION, "assistant-1") - ).toBe(true); - expect(settleForfeited).not.toHaveBeenCalled(); - session.drainQueuedMessagesIfIdle(); - expect(await waitForCondition(() => !session.isBusy())).toBe(true); + // The resume keeps failing before its stream starts and is swept again on its own until + // the cap; then the continuation is no longer advertised and the owner is told to settle + // the turn it deferred at the cut, with no poke from anyone. + expect(await waitForCondition(() => settleForfeited.mock.calls.length === 1)).toBe(true); + expect(harness.resumeAttempts()).toBe(3); + expect(settleForfeited.mock.calls[0]?.[0]).toEqual(WORKSPACE_TURN_CORRELATION); expect( session.claimWorkspaceTurnContinuation(WORKSPACE_TURN_CORRELATION, "assistant-1") ).toBe(false); + harness.openGate(); session.drainQueuedMessagesIfIdle(); await new Promise((resolve) => setTimeout(resolve, 25)); expect(streamMessage).toHaveBeenCalledTimes(1); - expect(settleForfeited).toHaveBeenCalledTimes(1); - expect(settleForfeited.mock.calls[0]?.[0]).toEqual(WORKSPACE_TURN_CORRELATION); + expect(harness.resumeAttempts()).toBe(3); } finally { session.dispose(); await cleanup(); @@ -2392,12 +2392,13 @@ describe("AgentSession queued message tool-call dispatch", () => { test("a context-discarding mutation settles the delegated turn it had advertised", async () => { const workspaceId = "queue-dispatch-stranded-delegated-context-discard"; - const harness = await strandDelegatedTurnBehindClosedGate(workspaceId); + const harness = await strandDelegatedTurn(workspaceId, { holdAdmission: true }); const { session, cleanup, streamMessage, settleForfeited } = harness; try { - // A history clear admitted on the idle session discards the transcript the continuation - // would resume from; no stream follows it to settle the turn the owner deferred. + // A history clear admitted on the idle session (under the admission hold) discards the + // transcript the continuation would resume from; no stream follows it to settle the turn + // the owner deferred. const discarded = await session.discardAutoRetryForContextMutation(); expect(discarded.success).toBe(true); expect(settleForfeited).toHaveBeenCalledTimes(1); @@ -2405,6 +2406,7 @@ describe("AgentSession queued message tool-call dispatch", () => { expect( session.claimWorkspaceTurnContinuation(WORKSPACE_TURN_CORRELATION, "assistant-1") ).toBe(false); + harness.releaseAdmission(); session.drainQueuedMessagesIfIdle(); await new Promise((resolve) => setTimeout(resolve, 25)); expect(streamMessage).toHaveBeenCalledTimes(1); @@ -2416,7 +2418,7 @@ describe("AgentSession queued message tool-call dispatch", () => { test("a task hard stop on the stranded delegated turn settles the turn it had advertised", async () => { const workspaceId = "queue-dispatch-stranded-delegated-hard-stop"; - const harness = await strandDelegatedTurnBehindClosedGate(workspaceId); + const harness = await strandDelegatedTurn(workspaceId, { holdAdmission: true }); const { session, cleanup, streamMessage, settleForfeited } = harness; try { @@ -2428,7 +2430,7 @@ describe("AgentSession queued message tool-call dispatch", () => { expect( session.claimWorkspaceTurnContinuation(WORKSPACE_TURN_CORRELATION, "assistant-1") ).toBe(false); - harness.openGate(); + harness.releaseAdmission(); session.drainQueuedMessagesIfIdle(); await new Promise((resolve) => setTimeout(resolve, 25)); expect(streamMessage).toHaveBeenCalledTimes(1); @@ -2440,7 +2442,7 @@ describe("AgentSession queued message tool-call dispatch", () => { test("disposing the session settles the delegated turn it had advertised", async () => { const workspaceId = "queue-dispatch-stranded-delegated-dispose"; - const harness = await strandDelegatedTurnBehindClosedGate(workspaceId); + const harness = await strandDelegatedTurn(workspaceId, { holdAdmission: true }); const { session, cleanup, settleForfeited } = harness; try { @@ -2463,7 +2465,10 @@ describe("AgentSession queued message tool-call dispatch", () => { ? Promise.reject(new Error("task store unavailable")) : Promise.resolve(); }); - const harness = await strandDelegatedTurnBehindClosedGate(workspaceId, { settleForfeited }); + const harness = await strandDelegatedTurn(workspaceId, { + settleForfeited, + holdAdmission: true, + }); const { session, cleanup } = harness; try { @@ -2493,14 +2498,15 @@ describe("AgentSession queued message tool-call dispatch", () => { Promise.resolve({ admissible: turnActive, admissionStale: () => !turnActive }) ); const settleForfeited = mock((_metadata: unknown, _reason: string) => Promise.resolve()); - const harness = await strandDelegatedTurnBehindClosedGate(workspaceId, { + const harness = await strandDelegatedTurn(workspaceId, { harness: { admitStrandedTurnResume }, settleForfeited, + holdAdmission: true, }); const { session, cleanup, streamMessage } = harness; try { - // The pricing gate failed the first resume; the continuation is still advertised. + // No resume has run under the admission hold; the continuation is still advertised. expect( session.claimWorkspaceTurnContinuation(WORKSPACE_TURN_CORRELATION, "assistant-1") ).toBe(true); @@ -2508,8 +2514,7 @@ describe("AgentSession queued message tool-call dispatch", () => { // task_stop / interrupt_active settled the handle while the cut stream was already // complete: no abort reached the session, only the owner's record changed. turnActive = false; - harness.openGate(); - session.drainQueuedMessagesIfIdle(); + harness.releaseAdmission(); expect(await waitForCondition(() => settleForfeited.mock.calls.length === 1)).toBe(true); expect(admitStrandedTurnResume.mock.calls[0]?.[0]).toEqual(WORKSPACE_TURN_CORRELATION); expect( diff --git a/src/node/services/agentSession.startupAutoRetry.test.ts b/src/node/services/agentSession.startupAutoRetry.test.ts index b2d99e2fad..7c09b128e3 100644 --- a/src/node/services/agentSession.startupAutoRetry.test.ts +++ b/src/node/services/agentSession.startupAutoRetry.test.ts @@ -409,6 +409,42 @@ describe("AgentSession startup auto-retry recovery", () => { session.dispose(); }); + test("startup auto-retry fails closed on a malformed persisted remainder", async () => { + const workspaceId = "startup-retry-cut-step-budget-malformed"; + const { session, historyService, events, cleanup } = await createSessionBundle(workspaceId); + cleanups.push(cleanup); + + await historyService.appendToHistory( + workspaceId, + createMuxMessage("original-user", "user", "Continue the original task", { + timestamp: Date.now(), + }) + ); + // Raw chat.jsonl is not schema-checked on this path; a corrupt remainder must not read as + // absent and hand the cut turn the default ceiling. + await historyService.appendToHistory( + workspaceId, + createMuxMessage("assistant-cut", "assistant", "Interrupted response", { + timestamp: Date.now(), + partial: true, + stepsRemaining: { steps: 3 } as unknown as number, + }) + ); + + session.ensureStartupAutoRetryCheck(); + await (session as unknown as { startupAutoRetryCheckPromise: Promise | null }) + .startupAutoRetryCheckPromise; + + expect(events.find((event) => event.type === "auto-retry-abandoned")).toMatchObject({ + reason: "malformed_step_budget", + }); + expect(events.some((event) => event.type === "auto-retry-scheduled")).toBe(false); + expect( + (session as unknown as { lastAutoRetryResumeRequest?: unknown }).lastAutoRetryResumeRequest + ).toBeUndefined(); + session.dispose(); + }); + test("hidden completed subagent reports preserve the existing startup retry fallback", async () => { const workspaceId = "startup-retry-hidden-subagent-report"; const { session, historyService, events, cleanup } = await createSessionBundle(workspaceId); diff --git a/src/node/services/agentSession.ts b/src/node/services/agentSession.ts index 454002bb03..3e52d655ce 100644 --- a/src/node/services/agentSession.ts +++ b/src/node/services/agentSession.ts @@ -2530,12 +2530,27 @@ export class AgentSession { : lastHistoryMessage?.role === "assistant" ? lastHistoryMessage : undefined; + // Raw JSON boundary, like the pending follow-up's persisted budget: a present but malformed + // remainder fails closed. The row can no longer state the ceiling its turn ran under, and + // reading it as absent would hand the turn the default one. + const persistedStepsRemaining = interruptedAssistant?.metadata?.stepsRemaining; + if ( + persistedStepsRemaining !== undefined && + !(Number.isInteger(persistedStepsRemaining) && persistedStepsRemaining >= 0) + ) { + log.warn("Startup auto-retry abandoned: malformed persisted step budget", { + workspaceId: this.workspaceId, + messageId: interruptedAssistant?.id, + }); + this.emitRetryEvent({ type: "auto-retry-abandoned", reason: "malformed_step_budget" }); + return "completed"; + } this.setAutoRetryResumeState( resumeOptions, agentInitiated, goalKind, goalId, - interruptedAssistant?.metadata?.stepsRemaining + persistedStepsRemaining ); } @@ -7927,12 +7942,12 @@ export class AgentSession { .then((started) => { this.strandedTurnResumeInFlight = null; // resumeStream settles only after its stream ends, so a stranding of that resumed - // stream can find the flag still set; sweep again once it clears. A resume that never - // started stays owed for the next natural poke rather than retrying in a tight loop, - // but anything queued behind its PREPARING claim has no stream end to wait for. - if (started) { - this.resumeStrandedTurnIfIdle(); - } else { + // stream can find the flag still set; sweep again once it clears. A resume that failed + // before its stream is swept again too: an idle session gets no later poke, and the cap's + // forfeit settles a delegated owner instead of leaving it waiting on a resume that never + // runs. Anything queued behind the failed PREPARING claim has no stream end to wait for. + this.resumeStrandedTurnIfIdle(); + if (!started) { this.dispatchQueuedMessagesIfIdle(); } }) From 206328207c43a56f95475be156bfe56ad09117f5 Mon Sep 17 00:00:00 2001 From: Michael Suchacz <203725896+ibetitsmike@users.noreply.github.com> Date: Fri, 4 Sep 2026 03:34:28 +0000 Subject: [PATCH 28/28] Price an aborted stream against the model it ran on Codex round 26: the stream-abort accounting used the requested model from the active context; a configured fallback that ran the stream reports its usage under the effective model, which the abort event now carries. Mirror stream-end and price against payload.metadata.model when present. --- .../agentSession.queueDispatch.test.ts | 43 +++++++++++++++++++ src/node/services/agentSession.ts | 4 +- 2 files changed, 46 insertions(+), 1 deletion(-) diff --git a/src/node/services/agentSession.queueDispatch.test.ts b/src/node/services/agentSession.queueDispatch.test.ts index 01795dbd77..82e198b6b6 100644 --- a/src/node/services/agentSession.queueDispatch.test.ts +++ b/src/node/services/agentSession.queueDispatch.test.ts @@ -5,6 +5,8 @@ import { EventEmitter } from "node:events"; import type { SendMessageOptions } from "@/common/orpc/types"; import { createMuxMessage, type MuxMessageMetadata } from "@/common/types/message"; import { Err, Ok } from "@/common/types/result"; +import { createDisplayUsage } from "@/common/utils/tokens/displayUsage"; +import { getTotalCost } from "@/common/utils/tokens/usageAggregator"; import { GOAL_CONTINUATION_KIND } from "@/constants/goals"; import type { WorkspaceGoalService } from "./workspaceGoalService"; import { @@ -1175,6 +1177,47 @@ describe("AgentSession queued message tool-call dispatch", () => { } }); + test("an aborted stream is accounted under the model it ran on, not the one requested", async () => { + const workspaceId = "queue-dispatch-abort-accounting-effective-model"; + const recordStreamAccounting = mock((_input: { costUsd: number }) => Promise.resolve()); + const workspaceGoalService = { + assertPricedModelForBudgetedGoal: mock(() => Promise.resolve(Ok(undefined))), + recordStreamAccounting, + applyPendingAfterStreamEnd: mock(() => Promise.resolve()), + requestContinuationAfterStreamEnd: mock(() => Promise.resolve()), + recordStreamStarted: mock(() => Promise.resolve()), + syncGoalModeWithChatTail: mock(() => Promise.resolve(null)), + } as unknown as WorkspaceGoalService; + const harness = await createStreamingTurnHarness(workspaceId, { + harness: { workspaceGoalService }, + sendInternal: { synthetic: true, agentInitiated: true }, + }); + const { session, cleanup, aiEmitter } = harness; + + try { + // The request named TEST_MODEL; a configured fallback ran the stream on a differently + // priced model and reported the usage for it. + const effectiveModel = "anthropic:claude-opus-4-1"; + const usage = { inputTokens: 1_000_000, outputTokens: 0, totalTokens: 1_000_000 }; + const effectiveCost = getTotalCost(createDisplayUsage(usage, effectiveModel)); + const requestedCost = getTotalCost(createDisplayUsage(usage, TEST_MODEL)); + expect(effectiveCost).toBeGreaterThan(0); + expect(effectiveCost).not.toBe(requestedCost); + + aiEmitter.emit("stream-abort", { + ...streamAbortEvent(workspaceId, "system"), + metadata: { duration: 1, usage, model: effectiveModel }, + }); + expect(await waitForCondition(() => recordStreamAccounting.mock.calls.length === 1)).toBe( + true + ); + expect(recordStreamAccounting.mock.calls[0]?.[0].costUsd).toBeCloseTo(effectiveCost ?? -1); + } finally { + session.dispose(); + await cleanup(); + } + }); + test("a send's launch-boundary admission probe reaches the stream request", async () => { const workspaceId = "queue-dispatch-send-launch-probe"; const aiEmitter = new EventEmitter(); diff --git a/src/node/services/agentSession.ts b/src/node/services/agentSession.ts index 3e52d655ce..26c70ce3ce 100644 --- a/src/node/services/agentSession.ts +++ b/src/node/services/agentSession.ts @@ -6656,7 +6656,9 @@ export class AgentSession { } this.setTurnPhase(TurnPhase.COMPLETING); - const activeModelForAbort = this.activeStreamContext?.modelString; + // Price the abort against the model that produced the usage (a configured fallback may + // differ from the requested one), as stream-end does; the context still names the request. + const activeModelForAbort = payload.metadata?.model ?? this.activeStreamContext?.modelString; if (activeModelForAbort) { this.updateUsageStateFromModelUsage({ model: activeModelForAbort,