diff --git a/Sources/OpenAI/Private/Audio/AudioPCMPlayer.swift b/Sources/OpenAI/Private/Audio/AudioPCMPlayer.swift index 22eb53c..c04f448 100644 --- a/Sources/OpenAI/Private/Audio/AudioPCMPlayer.swift +++ b/Sources/OpenAI/Private/Audio/AudioPCMPlayer.swift @@ -7,12 +7,38 @@ // #if canImport(AVFoundation) -import AVFoundation +@preconcurrency import AVFoundation import Foundation import OSLog private let logger = Logger(subsystem: "com.swiftopenai", category: "Audio") +private final class AudioPlayerNodeStopper: @unchecked Sendable { + init(playerNode: AVAudioPlayerNode) { + self.playerNode = playerNode + } + + func stop() async { + await withCheckedContinuation { continuation in + queue.async { + self.playerNode.stop() + continuation.resume() + } + } + } + + func stopWithoutWaiting() { + queue.async { + self.playerNode.stop() + } + } + + private let playerNode: AVAudioPlayerNode + private let queue = DispatchQueue( + label: "com.swiftopenai.audio-player-stop", + qos: .default) +} + // MARK: - AudioPCMPlayer /// Playback shares its `AVAudioEngine` with microphone capture. Keeping both directions in this @@ -47,9 +73,13 @@ final class AudioPCMPlayer { let node = AVAudioPlayerNode() audioEngine.attach(node) - audioEngine.connect(node, to: audioEngine.outputNode, format: playableFormat) + // Route through the main mixer: connecting a 24 kHz mono format straight into the output + // node fails AUGraph initialization (-10875) on devices whose hardware format differs; + // the mixer performs the sample-rate conversion to the hardware format. + audioEngine.connect(node, to: audioEngine.mainMixerNode, format: playableFormat) playerNode = node + playerNodeStopper = AudioPlayerNodeStopper(playerNode: node) self.inputFormat = inputFormat self.playableFormat = playableFormat } @@ -58,6 +88,10 @@ final class AudioPCMPlayer { logger.debug("AudioPCMPlayer is being freed") } + public var isPlaybackActive: Bool { + hasActivePlayback + } + public func playPCM16Audio(from base64String: String, itemID: String?) { guard let audioData = Data(base64Encoded: base64String) else { logger.error("Could not decode base64 string for audio playback") @@ -129,14 +163,13 @@ final class AudioPCMPlayer { } } - public func interruptPlayback() -> Int? { + public func interruptPlayback() async -> Int? { guard hasActivePlayback else { - playerNode.stop() + await playerNodeStopper.stop() return nil } logger.debug("Interrupting playback") let playedMilliseconds = Int((Double(playedFrameCount) / playableFormat.sampleRate) * 1000) - playerNode.stop() playbackGeneration += 1 pendingBufferCount = 0 resumePlaybackWaiters() @@ -144,9 +177,21 @@ final class AudioPCMPlayer { hasActivePlayback = false playbackStartSampleTime = nil scheduledFrameCount = 0 + await playerNodeStopper.stop() return playedMilliseconds } + public func stop() { + playbackGeneration += 1 + pendingBufferCount = 0 + resumePlaybackWaiters() + activeItemID = nil + hasActivePlayback = false + playbackStartSampleTime = nil + scheduledFrameCount = 0 + playerNodeStopper.stopWithoutWaiting() + } + public func waitUntilPlaybackFinishes() async { guard pendingBufferCount > 0 else { return } await withCheckedContinuation { continuation in @@ -159,6 +204,7 @@ final class AudioPCMPlayer { private let inputFormat: AVAudioFormat private let playableFormat: AVAudioFormat private let playerNode: AVAudioPlayerNode + private let playerNodeStopper: AudioPlayerNodeStopper private var activeItemID: String? private var hasActivePlayback = false private var playbackStartSampleTime: AVAudioFramePosition? diff --git a/Sources/OpenAI/Private/Audio/MicrophonePCMSampleVendorAE.swift b/Sources/OpenAI/Private/Audio/MicrophonePCMSampleVendorAE.swift index 8899eea..6c097af 100644 --- a/Sources/OpenAI/Private/Audio/MicrophonePCMSampleVendorAE.swift +++ b/Sources/OpenAI/Private/Audio/MicrophonePCMSampleVendorAE.swift @@ -65,11 +65,23 @@ class MicrophonePCMSampleVendorAE: MicrophonePCMSampleVendor { logger.debug("MicrophonePCMSampleVendorAE is being freed") } + nonisolated static func makeMonoTapFormat(sampleRate: Double) -> AVAudioFormat? { + AVAudioFormat( + commonFormat: .pcmFormatInt16, + sampleRate: sampleRate, + channels: 1, + interleaved: false) + } + func start() throws -> AsyncStream { - let tapFormat = inputNode.outputFormat(forBus: 0) - guard tapFormat.sampleRate > 0, tapFormat.channelCount == 1 else { + let outputFormat = inputNode.outputFormat(forBus: 0) + guard outputFormat.sampleRate > 0 else { + throw OpenAIError.audioConfigurationError( + "Realtime microphone input must have a valid sample rate") + } + guard let tapFormat = Self.makeMonoTapFormat(sampleRate: outputFormat.sampleRate) else { throw OpenAIError.audioConfigurationError( - "Realtime microphone input must have a valid mono output format") + "Could not create the mono tap format for realtime microphone input") } // The buffer size argument specifies the target number of audio frames. @@ -97,7 +109,10 @@ class MicrophonePCMSampleVendorAE: MicrophonePCMSampleVendor { continuation?.finish() continuation = nil inputNode.removeTap(onBus: 0) - try? inputNode.setVoiceProcessingEnabled(false) + // Deliberately leave voice processing enabled: each session uses a + // throwaway engine, so the AU is torn down when the engine deallocates. + // Disabling it here blocks on default-QoS CoreAudio reconfiguration and + // trips the priority-inversion diagnostic whenever stop is boosted. microphonePCMSampleVendorCommon.audioConverter = nil } diff --git a/Sources/OpenAI/Public/Shared/AudioController.swift b/Sources/OpenAI/Public/Shared/AudioController.swift index ef4114a..d32d0b2 100644 --- a/Sources/OpenAI/Public/Shared/AudioController.swift +++ b/Sources/OpenAI/Public/Shared/AudioController.swift @@ -49,6 +49,13 @@ public final class AudioController { audioEngine = AVAudioEngine() + // The playback graph must be wired before the capture vendor enables voice processing on + // the input node. Enabling voice processing first and then touching the mixer/player nodes + // leaves the engine unable to initialize (kAudioUnitErr_FailedInitialization, -10875). + if modes.contains(.playback) { + audioPCMPlayer = try await AudioPCMPlayer(audioEngine: audioEngine) + } + if modes.contains(.record) { #if os(macOS) || os(iOS) let needsSharedPlaybackReference = modes.contains(.playback) @@ -64,10 +71,6 @@ public final class AudioController { #endif } - if modes.contains(.playback) { - audioPCMPlayer = try await AudioPCMPlayer(audioEngine: audioEngine) - } - // Capture installs its input tap in `micStream()`. Starting the engine before that tap exists // can leave voice-processing input silent, particularly in Simulator. Playback-only controllers // have no capture tap to wait for and can start immediately. @@ -83,6 +86,11 @@ public final class AudioController { public let modes: [Mode] + /// Whether queued assistant audio is still audibly playing. + public var isPlaybackActive: Bool { + audioPCMPlayer?.isPlaybackActive ?? false + } + /// Installs the microphone tap and starts the shared audio engine. Call this once before expecting /// playback from a controller configured with both record and playback modes. public func micStream() throws -> AsyncStream { @@ -110,7 +118,7 @@ public final class AudioController { } public func stop() { - _ = audioPCMPlayer?.interruptPlayback() + audioPCMPlayer?.stop() audioEngine.stop() microphonePCMSampleVendor?.stop() } @@ -132,7 +140,7 @@ public final class AudioController { /// Stops queued playback and returns how many milliseconds of the current item were heard. @discardableResult - public func interruptPlayback() -> Int? { + public func interruptPlayback() async -> Int? { guard modes.contains(.playback), let audioPCMPlayer @@ -140,7 +148,7 @@ public final class AudioController { logger.error("Please pass [.playback] to the AudioController initializer") return nil } - return audioPCMPlayer.interruptPlayback() + return await audioPCMPlayer.interruptPlayback() } /// Suspends until all currently queued audio buffers have played. diff --git a/Tests/OpenAITests/RealtimeAudioConversionTests.swift b/Tests/OpenAITests/RealtimeAudioConversionTests.swift index 53e2591..7be3883 100644 --- a/Tests/OpenAITests/RealtimeAudioConversionTests.swift +++ b/Tests/OpenAITests/RealtimeAudioConversionTests.swift @@ -5,6 +5,16 @@ import XCTest @testable import SwiftOpenAI final class RealtimeAudioConversionTests: XCTestCase { + func testVoiceProcessingOutputIsTappedAsMono() throws { + let tapFormat = try XCTUnwrap( + MicrophonePCMSampleVendorAE.makeMonoTapFormat(sampleRate: 48_000)) + + XCTAssertEqual(tapFormat.commonFormat, .pcmFormatInt16) + XCTAssertEqual(tapFormat.sampleRate, 48_000) + XCTAssertEqual(tapFormat.channelCount, 1) + XCTAssertFalse(tapFormat.isInterleaved) + } + func testNativeFloatMicrophoneBuffersConvertToRealtimePCM16() throws { let inputFormat = try XCTUnwrap(AVAudioFormat( commonFormat: .pcmFormatFloat32,