fix: preserve adaptive interruption across tool calls

.changeset/adaptive-interruption-tool-calls.md

@@ -0,0 +1,5 @@
+---
+'@livekit/agents': patch
+---
+
+Preserve adaptive interruption boundaries when agent playout pauses or enters a tool-call thinking gap.

agents/src/inference/interruption/interruption_stream.test.ts

@@ -0,0 +1,44 @@
+// SPDX-FileCopyrightText: 2026 LiveKit, Inc.
+//
+// SPDX-License-Identifier: Apache-2.0
+import { describe, expect, it, vi } from 'vitest';
+import type { InterruptionMetrics } from '../../metrics/base.js';
+import { MockWebSocket } from './_mock_ws.js';
+import { AdaptiveInterruptionDetector } from './interruption_detector.js';
+import { InterruptionStreamBase, InterruptionStreamSentinel } from './interruption_stream.js';
+
+vi.mock('ws', async () => {
+  const { MockWebSocket } = await import('./_mock_ws.js');
+  return { default: MockWebSocket, WebSocket: MockWebSocket };
+});
+
+describe('InterruptionStreamBase metrics', () => {
+  it('does not count agent-ended overlap as a backchannel', async () => {
+    MockWebSocket.instances.length = 0;
+    const detector = new AdaptiveInterruptionDetector({
+      apiKey: 'test-key',
+      apiSecret: 'test-secret',
+      baseUrl: 'http://localhost:9999',
+    });
+    const stream = new InterruptionStreamBase(detector, {});
+    const metrics: InterruptionMetrics[] = [];
+    detector.on('metrics_collected', (event) => metrics.push(event));
+    const reader = stream.stream().getReader();
+
+    try {
+      await vi.waitFor(() => expect(MockWebSocket.instances).toHaveLength(1));
+      MockWebSocket.instances[0]!.simulateOpen();
+      const event = reader.read();
+      await stream.pushFrame(InterruptionStreamSentinel.agentSpeechStarted());
+      await stream.pushFrame(InterruptionStreamSentinel.overlapSpeechStarted(0, Date.now()));
+      await stream.pushFrame(InterruptionStreamSentinel.overlapSpeechEnded(Date.now(), true));
+      await event;
+
+      expect(metrics).toHaveLength(1);
+      expect(metrics[0]).toMatchObject({ numInterruptions: 0, numBackchannels: 0 });
+    } finally {
+      await reader.cancel();
+      await stream.close();
+    }
+  });
+});

agents/src/inference/interruption/interruption_stream.ts

@@ -337,7 +337,7 @@ export class InterruptionStreamBase {
           predictionDuration: chunk.predictionDurationInS * 1000,
           detectionDelay: chunk.detectionDelayInS * 1000,
           numInterruptions: chunk.isInterruption ? 1 : 0,
-          numBackchannels: chunk.isInterruption ? 0 : 1,
+          numBackchannels: !chunk.isInterruption && !chunk.agentEnded ? 1 : 0,
           numRequests: chunk.numRequests,
           metadata: {
             modelProvider: this.model.provider,

agents/src/voice/agent_activity.test.ts

@@ -452,7 +452,7 @@ describe('AgentActivity - mainTask', () => {
     }
   });
 
-  it('does not deadlock cancelling a paused speech whose generation never finishes', async () => {
+  it('does not end paused agent speech twice when cancelling it', async () => {
     const handle = SpeechHandle.create({ allowInterruptions: true });
     handle._authorizeGeneration();
 
@@ -501,7 +501,7 @@ describe('AgentActivity - mainTask', () => {
     expect(result).toBe('resolved');
     expect(handle.interrupted).toBe(true);
     expect(fakeActivity.pausedSpeech).toBeUndefined();
-    expect(fakeActivity.audioRecognition.onEndOfAgentSpeech).toHaveBeenCalledOnce();
+    expect(fakeActivity.audioRecognition.onEndOfAgentSpeech).not.toHaveBeenCalled();
   });
 });
 

agents/src/voice/agent_activity.ts

@@ -1680,7 +1680,12 @@ export class AgentActivity implements RecognitionHooks {
         // spams the interruption stream with duplicate `agent-speech-ended` sentinels.
         const wasAgentSpeaking = this.agentSession.agentState === 'speaking';
 
-        if (wasAgentSpeaking && this.isInterruptionDetectionEnabled && this.audioRecognition) {
+        if (
+          wasAgentSpeaking &&
+          this.isInterruptionDetectionEnabled &&
+          this.audioRecognition &&
+          !this.audioRecognition.endpointingOverlapping
+        ) {
           this.audioRecognition.onStartOfOverlapSpeech(
             0,
             Date.now(),
@@ -1695,7 +1700,6 @@ export class AgentActivity implements RecognitionHooks {
           if (this.audioRecognition) {
             this.audioRecognition.onEndOfAgentSpeech(
               options?.ignoreUserTranscriptUntil ?? Date.now(),
-              { paused: true },
             );
           }
           if (this.isInterruptionDetectionEnabled) {
@@ -1718,10 +1722,8 @@ export class AgentActivity implements RecognitionHooks {
     this.interruptByAudioActivity({
       ignoreUserTranscriptUntil: ev.overlapStartedAt || ev.detectedAt,
     });
-    if (this.audioRecognition) {
-      this.audioRecognition.onEndOfAgentSpeech(ev.overlapStartedAt || ev.detectedAt, {
-        paused: this.pausedSpeech !== undefined,
-      });
+    if (this.audioRecognition && this.pausedSpeech === undefined) {
+      this.audioRecognition.onEndOfAgentSpeech(ev.overlapStartedAt || ev.detectedAt);
     }
   }
 
@@ -3485,6 +3487,12 @@ export class AgentActivity implements RecognitionHooks {
 
     if (!speechHandle.interrupted && toolOutput.output.length > 0) {
       this.agentSession._updateAgentState('thinking');
+      if (this.audioRecognition) {
+        this.audioRecognition.onEndOfAgentSpeech(Date.now());
+      }
+      if (this.isInterruptionDetectionEnabled) {
+        this.restoreInterruptionByAudioActivity();
+      }
     } else if (this.agentSession.agentState === 'speaking') {
       this.agentSession._updateAgentState('listening');
       if (this.audioRecognition) {
@@ -4056,6 +4064,19 @@ export class AgentActivity implements RecognitionHooks {
 
     addRealtimeMessageOutputs(messageOutputs);
 
+    let endedAgentSpeechBeforeTool = false;
+    if (this.agentSession.agentState === 'speaking') {
+      const toolBusy = !executeToolsTask.done || toolOutput.output.length > 0;
+      this.agentSession._updateAgentState(toolBusy ? 'thinking' : 'listening');
+      if (this.audioRecognition) {
+        this.audioRecognition.onEndOfAgentSpeech(Date.now());
+      }
+      if (this.isInterruptionDetectionEnabled) {
+        this.restoreInterruptionByAudioActivity();
+      }
+      endedAgentSpeechBeforeTool = true;
+    }
+
     // mark the playout done before waiting for the tool execution
     speechHandle._markGenerationDone();
     // TODO(brian): close tees
@@ -4069,11 +4090,21 @@ export class AgentActivity implements RecognitionHooks {
 
     if (toolOutput.output.length > 0) {
       this.agentSession._updateAgentState('thinking');
+      if (!endedAgentSpeechBeforeTool) {
+        if (this.audioRecognition) {
+          this.audioRecognition.onEndOfAgentSpeech(Date.now());
+        }
+        if (this.isInterruptionDetectionEnabled) {
+          this.restoreInterruptionByAudioActivity();
+        }
+      }
     } else if (this.agentSession.agentState === 'speaking') {
       this.agentSession._updateAgentState('listening');
       if (this.audioRecognition) {
         this.audioRecognition.onEndOfAgentSpeech(Date.now());
       }
+    } else if (endedAgentSpeechBeforeTool && this.agentSession.agentState === 'thinking') {
+      this.agentSession._updateAgentState('listening');
     }
 
     if (toolOutput.output.length === 0) {
@@ -4850,7 +4881,7 @@ export class AgentActivity implements RecognitionHooks {
           otelContext: this.pausedSpeech.handle._agentTurnContext,
         });
         if (this.audioRecognition && this.pausedSpeech.agentState === 'speaking') {
-          this.audioRecognition.onStartOfAgentSpeech(Date.now(), { resumed: true });
+          this.audioRecognition.onStartOfAgentSpeech(Date.now());
         }
         if (this.isInterruptionDetectionEnabled) {
           this.disableVadInterruptionSoon();
@@ -4932,16 +4963,6 @@ export class AgentActivity implements RecognitionHooks {
       return;
     }
 
-    // The pause withheld end-of-agent-speech for a resume. Interrupting ends the turn instead;
-    // audio stopped when it was paused, so no playout is left to wait for.
-    if (interrupt && this.audioRecognition) {
-      void this.audioRecognition
-        .onEndOfAgentSpeech(Date.now())
-        .catch((error) =>
-          this.logger.warn({ error }, 'failed to report end of agent speech on pause cancel'),
-        );
-    }
-
     if (
       interrupt &&
       !this.pausedSpeech.handle.interrupted &&

agents/src/voice/agent_activity_tool_output_commit.test.ts

@@ -1,13 +1,18 @@
 // SPDX-FileCopyrightText: 2026 LiveKit, Inc.
 //
 // SPDX-License-Identifier: Apache-2.0
+import { AudioFrame } from '@livekit/rtc-node';
+import { ReadableStream } from 'node:stream/web';
 import { describe, expect, it, vi } from 'vitest';
 import type { ChatContext } from '../llm/chat_context.js';
 import { tool } from '../llm/tool_context.js';
+import { FakeSTT } from '../stt/testing/fake_stt.js';
 import { Future } from '../utils.js';
 import { Agent } from './agent.js';
 import { AgentSession } from './agent_session.js';
+import { AudioRecognition } from './audio_recognition.js';
 import { RUNNING_TOOL_PLACEHOLDER } from './generation.js';
+import { AudioOutput } from './io.js';
 import { FakeLLM } from './testing/fake_llm.js';
 
 type PostToolContextObservation = {
@@ -49,7 +54,90 @@ class ContextInspectingLLM extends FakeLLM {
   }
 }
 
+class ImmediateOutput extends AudioOutput {
+  constructor() {
+    super(24_000);
+  }
+
+  override async captureFrame(frame: AudioFrame): Promise<void> {
+    const segmentCount = this.capturedPlayoutSegments;
+    await super.captureFrame(frame);
+    if (this.capturedPlayoutSegments > segmentCount) {
+      this.onPlaybackStarted(Date.now());
+    }
+  }
+
+  override flush(): void {
+    super.flush();
+    if (this.pendingPlayoutSegments > 0) {
+      this.onPlaybackFinished({ playbackPosition: 0.02, interrupted: false });
+    }
+  }
+
+  override clearBuffer(): void {
+    if (this.pendingPlayoutSegments > 0) {
+      this.onPlaybackFinished({ playbackPosition: 0, interrupted: true });
+    }
+  }
+}
+
+class FrameAgent extends Agent {
+  constructor() {
+    super({
+      instructions: 'test',
+      tools: {
+        lookup: tool({
+          description: 'Look up a value',
+          execute: async () => 'forecast',
+        }),
+      },
+    });
+  }
+
+  override async ttsNode(): Promise<ReadableStream<AudioFrame>> {
+    return new ReadableStream<AudioFrame>({
+      start(controller) {
+        controller.enqueue(new AudioFrame(new Int16Array(480), 24_000, 1, 480));
+        controller.close();
+      },
+    });
+  }
+}
+
 describe('AgentActivity tool output commit ordering', () => {
+  it('ends active speech after entering the tool-call thinking state', async () => {
+    const llm = new FakeLLM([
+      {
+        input: 'look it up',
+        content: 'Let me check.',
+        toolCalls: [{ name: 'lookup', args: {} }],
+      },
+      { input: '"forecast"', content: 'The forecast is clear.' },
+    ]);
+    const session = new AgentSession({ llm, stt: new FakeSTT() });
+    session.output.audio = new ImmediateOutput();
+    const speechEndStates: string[] = [];
+    const onEndOfAgentSpeech = AudioRecognition.prototype.onEndOfAgentSpeech;
+    const speechEndSpy = vi
+      .spyOn(AudioRecognition.prototype, 'onEndOfAgentSpeech')
+      .mockImplementation(async function (this: AudioRecognition, ignoreUntil: number) {
+        speechEndStates.push(session.agentState);
+        await onEndOfAgentSpeech.call(this, ignoreUntil);
+      });
+
+    await session.start({ agent: new FrameAgent() });
+    try {
+      const speech = session.generateReply({ userInput: 'look it up' });
+      await speech.waitForPlayout();
+
+      expect(speechEndStates[0]).toBe('thinking');
+      expect(speechEndStates.slice(1).every((state) => state === 'listening')).toBe(true);
+    } finally {
+      speechEndSpy.mockRestore();
+      await session.close();
+    }
+  });
+
   it('invalidates a stale preemptive generation when late EOU interrupts the post-tool reply', async () => {
     const llm = new ContextInspectingLLM([
       {