Fix realtime audio engine failures and teardown stalls on macOS

Sources/OpenAI/Private/Audio/AudioPCMPlayer.swift

@@ -7,12 +7,38 @@
 //
 
 #if canImport(AVFoundation)
-import AVFoundation
+@preconcurrency import AVFoundation
 import Foundation
 import OSLog
 
 private let logger = Logger(subsystem: "com.swiftopenai", category: "Audio")
 
+private final class AudioPlayerNodeStopper: @unchecked Sendable {
+  init(playerNode: AVAudioPlayerNode) {
+    self.playerNode = playerNode
+  }
+
+  func stop() async {
+    await withCheckedContinuation { continuation in
+      queue.async {
+        self.playerNode.stop()
+        continuation.resume()
+      }
+    }
+  }
+
+  func stopWithoutWaiting() {
+    queue.async {
+      self.playerNode.stop()
+    }
+  }
+
+  private let playerNode: AVAudioPlayerNode
+  private let queue = DispatchQueue(
+    label: "com.swiftopenai.audio-player-stop",
+    qos: .default)
+}
+
 // MARK: - AudioPCMPlayer
 
 /// Playback shares its `AVAudioEngine` with microphone capture. Keeping both directions in this
@@ -47,9 +73,13 @@ final class AudioPCMPlayer {
     let node = AVAudioPlayerNode()
 
     audioEngine.attach(node)
-    audioEngine.connect(node, to: audioEngine.outputNode, format: playableFormat)
+    // Route through the main mixer: connecting a 24 kHz mono format straight into the output
+    // node fails AUGraph initialization (-10875) on devices whose hardware format differs;
+    // the mixer performs the sample-rate conversion to the hardware format.
+    audioEngine.connect(node, to: audioEngine.mainMixerNode, format: playableFormat)
 
     playerNode = node
+    playerNodeStopper = AudioPlayerNodeStopper(playerNode: node)
     self.inputFormat = inputFormat
     self.playableFormat = playableFormat
   }
@@ -58,6 +88,10 @@ final class AudioPCMPlayer {
     logger.debug("AudioPCMPlayer is being freed")
   }
 
+  public var isPlaybackActive: Bool {
+    hasActivePlayback
+  }
+
   public func playPCM16Audio(from base64String: String, itemID: String?) {
     guard let audioData = Data(base64Encoded: base64String) else {
       logger.error("Could not decode base64 string for audio playback")
@@ -129,24 +163,35 @@ final class AudioPCMPlayer {
     }
   }
 
-  public func interruptPlayback() -> Int? {
+  public func interruptPlayback() async -> Int? {
     guard hasActivePlayback else {
-      playerNode.stop()
+      await playerNodeStopper.stop()
       return nil
     }
     logger.debug("Interrupting playback")
     let playedMilliseconds = Int((Double(playedFrameCount) / playableFormat.sampleRate) * 1000)
-    playerNode.stop()
     playbackGeneration += 1
     pendingBufferCount = 0
     resumePlaybackWaiters()
     activeItemID = nil
     hasActivePlayback = false
     playbackStartSampleTime = nil
     scheduledFrameCount = 0
+    await playerNodeStopper.stop()
     return playedMilliseconds
   }
 
+  public func stop() {
+    playbackGeneration += 1
+    pendingBufferCount = 0
+    resumePlaybackWaiters()
+    activeItemID = nil
+    hasActivePlayback = false
+    playbackStartSampleTime = nil
+    scheduledFrameCount = 0
+    playerNodeStopper.stopWithoutWaiting()
+  }
+
   public func waitUntilPlaybackFinishes() async {
     guard pendingBufferCount > 0 else { return }
     await withCheckedContinuation { continuation in
@@ -159,6 +204,7 @@ final class AudioPCMPlayer {
   private let inputFormat: AVAudioFormat
   private let playableFormat: AVAudioFormat
   private let playerNode: AVAudioPlayerNode
+  private let playerNodeStopper: AudioPlayerNodeStopper
   private var activeItemID: String?
   private var hasActivePlayback = false
   private var playbackStartSampleTime: AVAudioFramePosition?

Sources/OpenAI/Private/Audio/MicrophonePCMSampleVendorAE.swift

@@ -65,11 +65,23 @@ class MicrophonePCMSampleVendorAE: MicrophonePCMSampleVendor {
     logger.debug("MicrophonePCMSampleVendorAE is being freed")
   }
 
+  nonisolated static func makeMonoTapFormat(sampleRate: Double) -> AVAudioFormat? {
+    AVAudioFormat(
+      commonFormat: .pcmFormatInt16,
+      sampleRate: sampleRate,
+      channels: 1,
+      interleaved: false)
+  }
+
   func start() throws -> AsyncStream<AVAudioPCMBuffer> {
-    let tapFormat = inputNode.outputFormat(forBus: 0)
-    guard tapFormat.sampleRate > 0, tapFormat.channelCount == 1 else {
+    let outputFormat = inputNode.outputFormat(forBus: 0)
+    guard outputFormat.sampleRate > 0 else {
+      throw OpenAIError.audioConfigurationError(
+        "Realtime microphone input must have a valid sample rate")
+    }
+    guard let tapFormat = Self.makeMonoTapFormat(sampleRate: outputFormat.sampleRate) else {
       throw OpenAIError.audioConfigurationError(
-        "Realtime microphone input must have a valid mono output format")
+        "Could not create the mono tap format for realtime microphone input")
     }
 
     // The buffer size argument specifies the target number of audio frames.
@@ -97,7 +109,10 @@ class MicrophonePCMSampleVendorAE: MicrophonePCMSampleVendor {
     continuation?.finish()
     continuation = nil
     inputNode.removeTap(onBus: 0)
-    try? inputNode.setVoiceProcessingEnabled(false)
+    // Deliberately leave voice processing enabled: each session uses a
+    // throwaway engine, so the AU is torn down when the engine deallocates.
+    // Disabling it here blocks on default-QoS CoreAudio reconfiguration and
+    // trips the priority-inversion diagnostic whenever stop is boosted.
     microphonePCMSampleVendorCommon.audioConverter = nil
   }
 

Sources/OpenAI/Public/Shared/AudioController.swift

@@ -49,6 +49,13 @@ public final class AudioController {
 
     audioEngine = AVAudioEngine()
 
+    // The playback graph must be wired before the capture vendor enables voice processing on
+    // the input node. Enabling voice processing first and then touching the mixer/player nodes
+    // leaves the engine unable to initialize (kAudioUnitErr_FailedInitialization, -10875).
+    if modes.contains(.playback) {
+      audioPCMPlayer = try await AudioPCMPlayer(audioEngine: audioEngine)
+    }
+
     if modes.contains(.record) {
       #if os(macOS) || os(iOS)
       let needsSharedPlaybackReference = modes.contains(.playback)
@@ -64,10 +71,6 @@ public final class AudioController {
       #endif
     }
 
-    if modes.contains(.playback) {
-      audioPCMPlayer = try await AudioPCMPlayer(audioEngine: audioEngine)
-    }
-
     // Capture installs its input tap in `micStream()`. Starting the engine before that tap exists
     // can leave voice-processing input silent, particularly in Simulator. Playback-only controllers
     // have no capture tap to wait for and can start immediately.
@@ -83,6 +86,11 @@ public final class AudioController {
 
   public let modes: [Mode]
 
+  /// Whether queued assistant audio is still audibly playing.
+  public var isPlaybackActive: Bool {
+    audioPCMPlayer?.isPlaybackActive ?? false
+  }
+
   /// Installs the microphone tap and starts the shared audio engine. Call this once before expecting
   /// playback from a controller configured with both record and playback modes.
   public func micStream() throws -> AsyncStream<AVAudioPCMBuffer> {
@@ -110,7 +118,7 @@ public final class AudioController {
   }
 
   public func stop() {
-    _ = audioPCMPlayer?.interruptPlayback()
+    audioPCMPlayer?.stop()
     audioEngine.stop()
     microphonePCMSampleVendor?.stop()
   }
@@ -132,15 +140,15 @@ public final class AudioController {
 
   /// Stops queued playback and returns how many milliseconds of the current item were heard.
   @discardableResult
-  public func interruptPlayback() -> Int? {
+  public func interruptPlayback() async -> Int? {
     guard
       modes.contains(.playback),
       let audioPCMPlayer
     else {
       logger.error("Please pass [.playback] to the AudioController initializer")
       return nil
     }
-    return audioPCMPlayer.interruptPlayback()
+    return await audioPCMPlayer.interruptPlayback()
   }
 
   /// Suspends until all currently queued audio buffers have played.

Tests/OpenAITests/RealtimeAudioConversionTests.swift

@@ -5,6 +5,16 @@ import XCTest
 @testable import SwiftOpenAI
 
 final class RealtimeAudioConversionTests: XCTestCase {
+  func testVoiceProcessingOutputIsTappedAsMono() throws {
+    let tapFormat = try XCTUnwrap(
+      MicrophonePCMSampleVendorAE.makeMonoTapFormat(sampleRate: 48_000))
+
+    XCTAssertEqual(tapFormat.commonFormat, .pcmFormatInt16)
+    XCTAssertEqual(tapFormat.sampleRate, 48_000)
+    XCTAssertEqual(tapFormat.channelCount, 1)
+    XCTAssertFalse(tapFormat.isInterleaved)
+  }
+
   func testNativeFloatMicrophoneBuffersConvertToRealtimePCM16() throws {
     let inputFormat = try XCTUnwrap(AVAudioFormat(
       commonFormat: .pcmFormatFloat32,