From 9bac840febf58a294afd2761d012e793c3490c75 Mon Sep 17 00:00:00 2001 From: jamesrochabrun Date: Mon, 10 Aug 2026 23:49:30 -0700 Subject: [PATCH 1/2] Fix realtime audio engine failures and teardown stalls on macOS - Build the playback graph before enabling voice processing: enabling VP first leaves AVAudioEngine unable to initialize (-10875). - Route the player through mainMixerNode: connecting 24 kHz mono straight into the output node fails AUGraph init on mismatched hardware formats. - Expose isPlaybackActive so clients can gate their microphone while assistant audio is audibly playing (half-duplex echo suppression). - Stop disabling voice processing in the capture vendor's stop(): the engine is single-use, and the call blocks on default-QoS CoreAudio work, tripping priority-inversion diagnostics. --- .../OpenAI/Private/Audio/AudioPCMPlayer.swift | 56 +++++++++++++++++-- .../Audio/MicrophonePCMSampleVendorAE.swift | 23 ++++++-- .../Public/Shared/AudioController.swift | 22 +++++--- .../RealtimeAudioConversionTests.swift | 10 ++++ 4 files changed, 95 insertions(+), 16 deletions(-) diff --git a/Sources/OpenAI/Private/Audio/AudioPCMPlayer.swift b/Sources/OpenAI/Private/Audio/AudioPCMPlayer.swift index 22eb53c..df08340 100644 --- a/Sources/OpenAI/Private/Audio/AudioPCMPlayer.swift +++ b/Sources/OpenAI/Private/Audio/AudioPCMPlayer.swift @@ -7,12 +7,38 @@ // #if canImport(AVFoundation) -import AVFoundation +@preconcurrency import AVFoundation import Foundation import OSLog private let logger = Logger(subsystem: "com.swiftopenai", category: "Audio") +private final class AudioPlayerNodeStopper: @unchecked Sendable { + init(playerNode: AVAudioPlayerNode) { + self.playerNode = playerNode + } + + func stop() async { + await withCheckedContinuation { continuation in + queue.async { + self.playerNode.stop() + continuation.resume() + } + } + } + + func stopWithoutWaiting() { + queue.async { + self.playerNode.stop() + } + } + + private let playerNode: AVAudioPlayerNode + private let queue = DispatchQueue( + label: "com.swiftopenai.audio-player-stop", + qos: .default) +} + // MARK: - AudioPCMPlayer /// Playback shares its `AVAudioEngine` with microphone capture. Keeping both directions in this @@ -47,9 +73,13 @@ final class AudioPCMPlayer { let node = AVAudioPlayerNode() audioEngine.attach(node) - audioEngine.connect(node, to: audioEngine.outputNode, format: playableFormat) + // Route through the main mixer: connecting a 24 kHz mono format straight into the output + // node fails AUGraph initialization (-10875) on devices whose hardware format differs; + // the mixer performs the sample-rate conversion to the hardware format. + audioEngine.connect(node, to: audioEngine.mainMixerNode, format: playableFormat) playerNode = node + playerNodeStopper = AudioPlayerNodeStopper(playerNode: node) self.inputFormat = inputFormat self.playableFormat = playableFormat } @@ -129,14 +159,13 @@ final class AudioPCMPlayer { } } - public func interruptPlayback() -> Int? { + public func interruptPlayback() async -> Int? { guard hasActivePlayback else { - playerNode.stop() + await playerNodeStopper.stop() return nil } logger.debug("Interrupting playback") let playedMilliseconds = Int((Double(playedFrameCount) / playableFormat.sampleRate) * 1000) - playerNode.stop() playbackGeneration += 1 pendingBufferCount = 0 resumePlaybackWaiters() @@ -144,9 +173,25 @@ final class AudioPCMPlayer { hasActivePlayback = false playbackStartSampleTime = nil scheduledFrameCount = 0 + await playerNodeStopper.stop() return playedMilliseconds } + public func stop() { + playbackGeneration += 1 + pendingBufferCount = 0 + resumePlaybackWaiters() + activeItemID = nil + hasActivePlayback = false + playbackStartSampleTime = nil + scheduledFrameCount = 0 + playerNodeStopper.stopWithoutWaiting() + } + + public var isPlaybackActive: Bool { + hasActivePlayback + } + public func waitUntilPlaybackFinishes() async { guard pendingBufferCount > 0 else { return } await withCheckedContinuation { continuation in @@ -159,6 +204,7 @@ final class AudioPCMPlayer { private let inputFormat: AVAudioFormat private let playableFormat: AVAudioFormat private let playerNode: AVAudioPlayerNode + private let playerNodeStopper: AudioPlayerNodeStopper private var activeItemID: String? private var hasActivePlayback = false private var playbackStartSampleTime: AVAudioFramePosition? diff --git a/Sources/OpenAI/Private/Audio/MicrophonePCMSampleVendorAE.swift b/Sources/OpenAI/Private/Audio/MicrophonePCMSampleVendorAE.swift index 8899eea..6536573 100644 --- a/Sources/OpenAI/Private/Audio/MicrophonePCMSampleVendorAE.swift +++ b/Sources/OpenAI/Private/Audio/MicrophonePCMSampleVendorAE.swift @@ -66,10 +66,14 @@ class MicrophonePCMSampleVendorAE: MicrophonePCMSampleVendor { } func start() throws -> AsyncStream { - let tapFormat = inputNode.outputFormat(forBus: 0) - guard tapFormat.sampleRate > 0, tapFormat.channelCount == 1 else { + let outputFormat = inputNode.outputFormat(forBus: 0) + guard outputFormat.sampleRate > 0 else { throw OpenAIError.audioConfigurationError( - "Realtime microphone input must have a valid mono output format") + "Realtime microphone input must have a valid sample rate") + } + guard let tapFormat = Self.makeMonoTapFormat(sampleRate: outputFormat.sampleRate) else { + throw OpenAIError.audioConfigurationError( + "Could not create the mono tap format for realtime microphone input") } // The buffer size argument specifies the target number of audio frames. @@ -97,7 +101,10 @@ class MicrophonePCMSampleVendorAE: MicrophonePCMSampleVendor { continuation?.finish() continuation = nil inputNode.removeTap(onBus: 0) - try? inputNode.setVoiceProcessingEnabled(false) + // Deliberately leave voice processing enabled: each session uses a + // throwaway engine, so the AU is torn down when the engine deallocates. + // Disabling it here blocks on default-QoS CoreAudio reconfiguration and + // trips the priority-inversion diagnostic whenever stop is boosted. microphonePCMSampleVendorCommon.audioConverter = nil } @@ -107,6 +114,14 @@ class MicrophonePCMSampleVendorAE: MicrophonePCMSampleVendor { private var continuation: AsyncStream.Continuation? private var hasLoggedFirstBuffer = false + nonisolated static func makeMonoTapFormat(sampleRate: Double) -> AVAudioFormat? { + AVAudioFormat( + commonFormat: .pcmFormatInt16, + sampleRate: sampleRate, + channels: 1, + interleaved: false) + } + private nonisolated func installTapNonIsolated( inputNode: AVAudioInputNode, bufferSize: AVAudioFrameCount, diff --git a/Sources/OpenAI/Public/Shared/AudioController.swift b/Sources/OpenAI/Public/Shared/AudioController.swift index ef4114a..167489d 100644 --- a/Sources/OpenAI/Public/Shared/AudioController.swift +++ b/Sources/OpenAI/Public/Shared/AudioController.swift @@ -49,6 +49,13 @@ public final class AudioController { audioEngine = AVAudioEngine() + // The playback graph must be wired before the capture vendor enables voice processing on + // the input node. Enabling voice processing first and then touching the mixer/player nodes + // leaves the engine unable to initialize (kAudioUnitErr_FailedInitialization, -10875). + if modes.contains(.playback) { + audioPCMPlayer = try await AudioPCMPlayer(audioEngine: audioEngine) + } + if modes.contains(.record) { #if os(macOS) || os(iOS) let needsSharedPlaybackReference = modes.contains(.playback) @@ -64,10 +71,6 @@ public final class AudioController { #endif } - if modes.contains(.playback) { - audioPCMPlayer = try await AudioPCMPlayer(audioEngine: audioEngine) - } - // Capture installs its input tap in `micStream()`. Starting the engine before that tap exists // can leave voice-processing input silent, particularly in Simulator. Playback-only controllers // have no capture tap to wait for and can start immediately. @@ -110,7 +113,7 @@ public final class AudioController { } public func stop() { - _ = audioPCMPlayer?.interruptPlayback() + audioPCMPlayer?.stop() audioEngine.stop() microphonePCMSampleVendor?.stop() } @@ -132,7 +135,7 @@ public final class AudioController { /// Stops queued playback and returns how many milliseconds of the current item were heard. @discardableResult - public func interruptPlayback() -> Int? { + public func interruptPlayback() async -> Int? { guard modes.contains(.playback), let audioPCMPlayer @@ -140,7 +143,12 @@ public final class AudioController { logger.error("Please pass [.playback] to the AudioController initializer") return nil } - return audioPCMPlayer.interruptPlayback() + return await audioPCMPlayer.interruptPlayback() + } + + /// Whether queued assistant audio is still audibly playing. + public var isPlaybackActive: Bool { + audioPCMPlayer?.isPlaybackActive ?? false } /// Suspends until all currently queued audio buffers have played. diff --git a/Tests/OpenAITests/RealtimeAudioConversionTests.swift b/Tests/OpenAITests/RealtimeAudioConversionTests.swift index 53e2591..7be3883 100644 --- a/Tests/OpenAITests/RealtimeAudioConversionTests.swift +++ b/Tests/OpenAITests/RealtimeAudioConversionTests.swift @@ -5,6 +5,16 @@ import XCTest @testable import SwiftOpenAI final class RealtimeAudioConversionTests: XCTestCase { + func testVoiceProcessingOutputIsTappedAsMono() throws { + let tapFormat = try XCTUnwrap( + MicrophonePCMSampleVendorAE.makeMonoTapFormat(sampleRate: 48_000)) + + XCTAssertEqual(tapFormat.commonFormat, .pcmFormatInt16) + XCTAssertEqual(tapFormat.sampleRate, 48_000) + XCTAssertEqual(tapFormat.channelCount, 1) + XCTAssertFalse(tapFormat.isInterleaved) + } + func testNativeFloatMicrophoneBuffersConvertToRealtimePCM16() throws { let inputFormat = try XCTUnwrap(AVAudioFormat( commonFormat: .pcmFormatFloat32, From c017898fa2968bd465b329be6cba2d2c241ad952 Mon Sep 17 00:00:00 2001 From: jamesrochabrun Date: Mon, 10 Aug 2026 23:56:48 -0700 Subject: [PATCH 2/2] Apply swiftformat declaration ordering to touched audio files --- .../OpenAI/Private/Audio/AudioPCMPlayer.swift | 8 ++++---- .../Audio/MicrophonePCMSampleVendorAE.swift | 16 ++++++++-------- .../OpenAI/Public/Shared/AudioController.swift | 10 +++++----- 3 files changed, 17 insertions(+), 17 deletions(-) diff --git a/Sources/OpenAI/Private/Audio/AudioPCMPlayer.swift b/Sources/OpenAI/Private/Audio/AudioPCMPlayer.swift index df08340..c04f448 100644 --- a/Sources/OpenAI/Private/Audio/AudioPCMPlayer.swift +++ b/Sources/OpenAI/Private/Audio/AudioPCMPlayer.swift @@ -88,6 +88,10 @@ final class AudioPCMPlayer { logger.debug("AudioPCMPlayer is being freed") } + public var isPlaybackActive: Bool { + hasActivePlayback + } + public func playPCM16Audio(from base64String: String, itemID: String?) { guard let audioData = Data(base64Encoded: base64String) else { logger.error("Could not decode base64 string for audio playback") @@ -188,10 +192,6 @@ final class AudioPCMPlayer { playerNodeStopper.stopWithoutWaiting() } - public var isPlaybackActive: Bool { - hasActivePlayback - } - public func waitUntilPlaybackFinishes() async { guard pendingBufferCount > 0 else { return } await withCheckedContinuation { continuation in diff --git a/Sources/OpenAI/Private/Audio/MicrophonePCMSampleVendorAE.swift b/Sources/OpenAI/Private/Audio/MicrophonePCMSampleVendorAE.swift index 6536573..6c097af 100644 --- a/Sources/OpenAI/Private/Audio/MicrophonePCMSampleVendorAE.swift +++ b/Sources/OpenAI/Private/Audio/MicrophonePCMSampleVendorAE.swift @@ -65,6 +65,14 @@ class MicrophonePCMSampleVendorAE: MicrophonePCMSampleVendor { logger.debug("MicrophonePCMSampleVendorAE is being freed") } + nonisolated static func makeMonoTapFormat(sampleRate: Double) -> AVAudioFormat? { + AVAudioFormat( + commonFormat: .pcmFormatInt16, + sampleRate: sampleRate, + channels: 1, + interleaved: false) + } + func start() throws -> AsyncStream { let outputFormat = inputNode.outputFormat(forBus: 0) guard outputFormat.sampleRate > 0 else { @@ -114,14 +122,6 @@ class MicrophonePCMSampleVendorAE: MicrophonePCMSampleVendor { private var continuation: AsyncStream.Continuation? private var hasLoggedFirstBuffer = false - nonisolated static func makeMonoTapFormat(sampleRate: Double) -> AVAudioFormat? { - AVAudioFormat( - commonFormat: .pcmFormatInt16, - sampleRate: sampleRate, - channels: 1, - interleaved: false) - } - private nonisolated func installTapNonIsolated( inputNode: AVAudioInputNode, bufferSize: AVAudioFrameCount, diff --git a/Sources/OpenAI/Public/Shared/AudioController.swift b/Sources/OpenAI/Public/Shared/AudioController.swift index 167489d..d32d0b2 100644 --- a/Sources/OpenAI/Public/Shared/AudioController.swift +++ b/Sources/OpenAI/Public/Shared/AudioController.swift @@ -86,6 +86,11 @@ public final class AudioController { public let modes: [Mode] + /// Whether queued assistant audio is still audibly playing. + public var isPlaybackActive: Bool { + audioPCMPlayer?.isPlaybackActive ?? false + } + /// Installs the microphone tap and starts the shared audio engine. Call this once before expecting /// playback from a controller configured with both record and playback modes. public func micStream() throws -> AsyncStream { @@ -146,11 +151,6 @@ public final class AudioController { return await audioPCMPlayer.interruptPlayback() } - /// Whether queued assistant audio is still audibly playing. - public var isPlaybackActive: Bool { - audioPCMPlayer?.isPlaybackActive ?? false - } - /// Suspends until all currently queued audio buffers have played. public func waitUntilPlaybackFinishes() async { guard