From 48ffba60c06329fa68dcde67829a2660ebd75192 Mon Sep 17 00:00:00 2001 From: Chrizz Gelbach Date: Thu, 15 Oct 2026 00:03:09 +0200 Subject: [PATCH] Drop encoder priming and padding for gapless playback Decoders output the encoder delay (priming) and padding (remainder) of formats like AAC and MP3 as audio. The packet table info was only used for the duration, so consecutive entries were separated by a short silence (2112 + remainder frames, about 50 ms, for AAC). - Keep only the decoded frames in [priming, priming + valid) while the decoded position is exact (decoding from the start of the stream) - After a seek, drop the remainder at the end of the stream instead - Frame counts are converted to the output sample rate --- .../Streaming/Audio Entry/AudioEntry.swift | 1 + .../Audio Entry/Models/EntryFrames.swift | 10 +++ .../Streaming/AudioPlayer/AudioPlayer.swift | 2 + .../Processors/AudioFileStreamProcessor.swift | 81 ++++++++++++++++++- 4 files changed, 91 insertions(+), 3 deletions(-) diff --git a/AudioStreaming/Streaming/Audio Entry/AudioEntry.swift b/AudioStreaming/Streaming/Audio Entry/AudioEntry.swift index c948801..0ce2e01 100644 --- a/AudioStreaming/Streaming/Audio Entry/AudioEntry.swift +++ b/AudioStreaming/Streaming/Audio Entry/AudioEntry.swift @@ -100,6 +100,7 @@ class AudioEntry { framesState.played = 0 framesState.queued = 0 framesState.lastFrameQueued = -1 + framesState.decoded = 0 } func has(same source: CoreAudioStreamSource) -> Bool { diff --git a/AudioStreaming/Streaming/Audio Entry/Models/EntryFrames.swift b/AudioStreaming/Streaming/Audio Entry/Models/EntryFrames.swift index 1b4d377..bf6b07c 100644 --- a/AudioStreaming/Streaming/Audio Entry/Models/EntryFrames.swift +++ b/AudioStreaming/Streaming/Audio Entry/Models/EntryFrames.swift @@ -9,4 +9,14 @@ final class EntryFramesState { var queued: Int = 0 var played: Int = 0 var lastFrameQueued: Int = -1 + + /// Gapless info from the packet table, in source frames: encoder delay (priming), audio frames + /// (valid) and padding (remainder). Decoders output priming and remainder like audio; both are + /// dropped. `validFrames` is nil when the stream has no packet table info. + var primingFrames: Int = 0 + var validFrames: Int? + var remainderFrames: Int = 0 + /// Output frames decoded since the start of the stream, including dropped ones. + /// nil after a seek, where the decoded position is not exact. + var decoded: Int? = 0 } diff --git a/AudioStreaming/Streaming/AudioPlayer/AudioPlayer.swift b/AudioStreaming/Streaming/AudioPlayer/AudioPlayer.swift index 13fba34..358344d 100644 --- a/AudioStreaming/Streaming/AudioPlayer/AudioPlayer.swift +++ b/AudioStreaming/Streaming/AudioPlayer/AudioPlayer.swift @@ -1091,6 +1091,8 @@ extension AudioPlayer: AudioStreamSourceDelegate { Logger.debug("Failed to flush remaining packets at EOF: \(flushStatus)", category: .generic) } + fileStreamProcessor.trimTrailingGaplessPadding() + readingEntry.lock.lock() readingEntry.framesState.lastFrameQueued = readingEntry.framesState.queued readingEntry.lock.unlock() diff --git a/AudioStreaming/Streaming/AudioPlayer/Processors/AudioFileStreamProcessor.swift b/AudioStreaming/Streaming/AudioPlayer/Processors/AudioFileStreamProcessor.swift index 77d560e..8944cad 100644 --- a/AudioStreaming/Streaming/AudioPlayer/Processors/AudioFileStreamProcessor.swift +++ b/AudioStreaming/Streaming/AudioPlayer/Processors/AudioFileStreamProcessor.swift @@ -210,6 +210,9 @@ final class AudioFileStreamProcessor { } readingEntry.reset() + readingEntry.lock.lock() + readingEntry.framesState.decoded = nil + readingEntry.lock.unlock() readingEntry.seek(at: Int(seekByteOffset)) rendererContext.waitingForDataAfterSeekFrameCount.write { $0 = 0 } playerContext.setInternalState(to: .waitingForDataAfterSeek) @@ -431,6 +434,9 @@ final class AudioFileStreamProcessor { entry.lock.lock(); defer { entry.lock.unlock() } if pti.mNumberValidFrames > 0 { entry.audioStreamState.dataPacketCount = Double(pti.mNumberValidFrames) / Double(max(1, entry.audioStreamFormat.mFramesPerPacket)) + entry.framesState.primingFrames = Int(pti.mPrimingFrames) + entry.framesState.validFrames = Int(pti.mNumberValidFrames) + entry.framesState.remainderFrames = Int(pti.mRemainderFrames) } } @@ -581,7 +587,7 @@ final class AudioFileStreamProcessor { nil ) - framesAdded = framesToDecode + framesAdded = trimGaplessPadding(bufferIndex: end, framesCount: framesToDecode) if status == AudioConvertStatus.done.rawValue { fillUsedFrames(framesCount: framesAdded) @@ -591,6 +597,12 @@ final class AudioFileStreamProcessor { return } + if framesAdded < framesToDecode { + // dropped frames left a hole before the end of the buffer, continue right after the kept ones + fillUsedFrames(framesCount: framesAdded) + continue packetProcess + } + framesToDecode = start if framesToDecode == 0 { fillUsedFrames(framesCount: framesAdded) @@ -611,7 +623,7 @@ final class AudioFileStreamProcessor { nil ) - framesAdded += framesToDecode + framesAdded += trimGaplessPadding(bufferIndex: 0, framesCount: framesToDecode) if status == AudioConvertStatus.done.rawValue { fillUsedFrames(framesCount: framesAdded) @@ -643,7 +655,7 @@ final class AudioFileStreamProcessor { nil ) - framesAdded = framesToDecode + framesAdded = trimGaplessPadding(bufferIndex: end, framesCount: framesToDecode) if status == AudioConvertStatus.done.rawValue { fillUsedFrames(framesCount: framesAdded) return @@ -676,6 +688,69 @@ final class AudioFileStreamProcessor { bufferList[0].mNumberChannels = rendererContext.audioBuffer.mNumberChannels } + /// Drops the encoder delay (priming) and padding (remainder) that decoders output for formats like + /// AAC and MP3, using the packet table info of the stream. Without this, consecutive entries are + /// separated by a short silence (about 50 ms for AAC). + /// Keeps the decoded frames in [priming, priming + valid) while the decoded position is exact; + /// after a seek, `trimTrailingGaplessPadding()` drops the padding at the end of the stream instead. + /// + /// - parameter bufferIndex: The frame index in the renderer buffer where the decoded frames start + /// - parameter framesCount: The number of frames decoded there + /// - Returns: The number of frames to keep; they are moved to `bufferIndex` + @inline(__always) + private func trimGaplessPadding(bufferIndex: UInt32, framesCount: UInt32) -> UInt32 { + guard framesCount > 0, let entry = playerContext.audioReadingEntry else { return framesCount } + entry.lock.lock() + let framesState = entry.framesState + let decodedStart = framesState.decoded + if let decodedStart { + framesState.decoded = decodedStart + Int(framesCount) + } + let primingFrames = framesState.primingFrames + let validFrames = framesState.validFrames + let sourceSampleRate = entry.audioStreamFormat.mSampleRate + entry.lock.unlock() + + guard let decodedStart, let validFrames, sourceSampleRate > 0 else { return framesCount } + let ratio = outputAudioFormat.mSampleRate / sourceSampleRate + let keepFrom = Int((Double(primingFrames) * ratio).rounded()) + let keepTo = keepFrom + Int((Double(validFrames) * ratio).rounded()) + let first = max(decodedStart, keepFrom) + let last = min(decodedStart + Int(framesCount), keepTo) + guard last > first else { return 0 } + + let leadingFrames = first - decodedStart + let keptFrames = last - first + if leadingFrames > 0, let mData = rendererContext.audioBuffer.mData { + let frameSize = Int(rendererContext.bufferContext.sizeInBytes) + let destination = mData + Int(bufferIndex) * frameSize + memmove(destination, destination + leadingFrames * frameSize, keptFrames * frameSize) + } + return UInt32(keptFrames) + } + + /// Drops the encoder padding (remainder) at the end of the stream when `trimGaplessPadding` could not, + /// because the decoded position is not exact after a seek. + /// Must be called once all packets of the reading entry have been decoded: the padding is the + /// last part written to the renderer buffer then. + func trimTrailingGaplessPadding() { + guard let entry = playerContext.audioReadingEntry else { return } + // same lock order as the renderer: buffer first, then entry + rendererContext.lock.lock(); defer { rendererContext.lock.unlock() } + entry.lock.lock(); defer { entry.lock.unlock() } + + let framesState = entry.framesState + let sourceSampleRate = entry.audioStreamFormat.mSampleRate + guard framesState.decoded == nil, framesState.remainderFrames > 0, sourceSampleRate > 0 else { return } + let outputRemainderFrames = Int((Double(framesState.remainderFrames) * outputAudioFormat.mSampleRate / sourceSampleRate).rounded()) + let notPlayedFrames = framesState.queued - framesState.played + let droppedFrames = min(outputRemainderFrames, notPlayedFrames, Int(rendererContext.bufferContext.frameUsedCount)) + guard droppedFrames > 0 else { return } + + rendererContext.bufferContext.frameUsedCount -= UInt32(droppedFrames) + framesState.queued -= droppedFrames + } + /// Advances the processed frames for buffer and reading entry /// /// - parameter frameCount: An `UInt32` value to be added to the used count of the buffers.