diff --git a/apps/common-app/src/demos/index.ts b/apps/common-app/src/demos/index.ts index 8f1d989b8..c338d6f7d 100644 --- a/apps/common-app/src/demos/index.ts +++ b/apps/common-app/src/demos/index.ts @@ -37,9 +37,8 @@ export const demos: DemoScreen[] = [ { key: 'Crossfade', title: 'Crossfade', - subtitle: - 'Demonstrates crossfading between two audio files.', + subtitle: 'Demonstrates crossfading between two audio files.', icon: icons.ArrowLeftRight, screen: Crossfade, - } + }, ] as const; diff --git a/apps/common-app/src/examples/index.ts b/apps/common-app/src/examples/index.ts index d664f93b1..acceaa8e5 100644 --- a/apps/common-app/src/examples/index.ts +++ b/apps/common-app/src/examples/index.ts @@ -31,6 +31,7 @@ type NavigationParamList = { ConvolverIR: undefined; AudioParamPipeline: undefined; TestScreen: undefined; + LatencyValidation: undefined; }; export type ExampleKey = keyof NavigationParamList; diff --git a/apps/common-app/src/other/LatencyValidation/CorrelationPlot.tsx b/apps/common-app/src/other/LatencyValidation/CorrelationPlot.tsx new file mode 100644 index 000000000..52f8bbb48 --- /dev/null +++ b/apps/common-app/src/other/LatencyValidation/CorrelationPlot.tsx @@ -0,0 +1,149 @@ +import React, { FC, useMemo } from 'react'; +import { StyleSheet, Text, View } from 'react-native'; +import Svg, { Circle, Line, Path, Rect } from 'react-native-svg'; + +import { colors } from '../../styles'; +import type { CorrelationPoint } from './types'; + +interface CorrelationPlotProps { + title: string; + caption?: string; + width: number; + height?: number; + points: CorrelationPoint[]; + bestLagMs: number | null; + expectedLagMs: number; +} + +const CorrelationPlot: FC = ({ + title, + caption, + width, + height = 120, + points, + bestLagMs, + expectedLagMs, +}) => { + const { path, bestX, expectedX, maxScore } = useMemo(() => { + if (points.length === 0) { + return { path: '', bestX: null, expectedX: null, maxScore: 1 }; + } + + const minLag = points[0].lagMs; + const maxLag = points[points.length - 1].lagMs; + const lagSpan = Math.max(maxLag - minLag, 1); + const maxScoreValue = Math.max(...points.map((point) => point.score), 0.001); + + const builtPath = points + .map((point, index) => { + const x = ((point.lagMs - minLag) / lagSpan) * width; + const y = height - (point.score / maxScoreValue) * (height * 0.82) - 8; + return `${index === 0 ? 'M' : 'L'}${x.toFixed(2)},${y.toFixed(2)}`; + }) + .join(' '); + + return { + path: builtPath, + bestX: + bestLagMs !== null + ? ((bestLagMs - minLag) / lagSpan) * width + : null, + expectedX: ((expectedLagMs - minLag) / lagSpan) * width, + maxScore: maxScoreValue, + }; + }, [bestLagMs, expectedLagMs, height, points, width]); + + return ( + + {title} + {caption ? {caption} : null} + + + + + + {expectedX !== null ? ( + + ) : null} + + {bestX !== null ? ( + + ) : null} + + + + {bestX !== null ? ( + + ) : null} + + + + + Peak score max={maxScore.toFixed(3)} + + Best lag {bestLagMs !== null ? `${bestLagMs.toFixed(1)} ms` : '—'} + + + Expected {expectedLagMs.toFixed(1)} ms + + + + ); +}; + +const styles = StyleSheet.create({ + caption: { + color: colors.gray, + fontSize: 12, + lineHeight: 17, + marginBottom: 8, + }, + container: { + backgroundColor: colors.backgroundLight, + borderRadius: 8, + gap: 8, + padding: 12, + }, + legendRow: { + gap: 4, + }, + legendText: { + color: colors.gray, + fontSize: 11, + }, + plotFrame: { + borderColor: colors.separator, + borderRadius: 4, + borderWidth: StyleSheet.hairlineWidth, + overflow: 'hidden', + }, + title: { + color: colors.white, + fontSize: 15, + fontWeight: '700', + }, +}); + +export default CorrelationPlot; diff --git a/apps/common-app/src/other/LatencyValidation/LatencyValidation.tsx b/apps/common-app/src/other/LatencyValidation/LatencyValidation.tsx new file mode 100644 index 000000000..ffde1fba7 --- /dev/null +++ b/apps/common-app/src/other/LatencyValidation/LatencyValidation.tsx @@ -0,0 +1,198 @@ +import React, { FC, useEffect, useRef, useState } from 'react'; +import { Platform, ScrollView, StyleSheet } from 'react-native'; +import { AudioManager } from 'react-native-audio-api'; + +import { Container } from '../../components'; +import { audioContext, audioRecorder } from '../../singletons'; +import { TestUI, type ControlAction } from '../../testComponents'; +import { formatTimestamp } from './helpers'; +import { + cleanupLatencyValidation, + runLatencyValidationSuite, +} from './latencyTests'; +import LoopbackAnalysisPanel from './LoopbackAnalysisPanel'; +import type { LoopbackAnalysis } from './types'; + +type RunnerState = 'idle' | 'running'; + +const LatencyValidation: FC = () => { + const isMountedRef = useRef(true); + const [runnerState, setRunnerState] = useState('idle'); + const [currentStep, setCurrentStep] = useState( + 'Ready to run the speaker-to-mic loopback latency check.' + ); + const [analysis, setAnalysis] = useState(null); + const [liveLog, setLiveLog] = useState([]); + + const appendLog = (message: string) => { + if (!isMountedRef.current) { + return; + } + + setLiveLog((previous) => [ + ...previous, + `${formatTimestamp(Date.now())} ${message}`, + ]); + }; + + useEffect(() => { + return () => { + isMountedRef.current = false; + cleanupLatencyValidation({ audioContext, audioRecorder }).catch((error) => { + console.warn('LatencyValidation cleanup failed', error); + }); + }; + }, []); + + const runTest = async () => { + if (runnerState === 'running') { + return; + } + + setRunnerState('running'); + setAnalysis(null); + setLiveLog([]); + setCurrentStep('Preparing audio session...'); + appendLog('Starting speaker-to-mic loopback check.'); + + try { + const permission = await AudioManager.requestRecordingPermissions(); + if (permission !== 'Granted') { + throw new Error('Recording permission was not granted.'); + } + + if (Platform.OS !== 'web') { + AudioManager.setAudioSessionOptions({ + iosCategory: 'playAndRecord', + // 'measurement' keeps the signal path raw (no AGC/EQ) so the loopback + // delay is accurate, while running the mic+speaker for the shortest + // possible window to keep the idle full-duplex buzz to a minimum. + iosMode: 'measurement', + iosOptions: ['defaultToSpeaker'], + }); + await AudioManager.setAudioSessionActivity(true); + } + + setCurrentStep('Playing beep pattern and analyzing microphone capture...'); + appendLog('Reference pattern scheduled. Recording microphone input.'); + + const result = await runLatencyValidationSuite({ + audioContext, + audioRecorder, + }); + + if (!isMountedRef.current) { + return; + } + + setAnalysis(result); + appendLog( + `Captured ${result.recordedWindowPlot.points.length > 0 ? 'microphone' : 'no'} waveform points for visualization.` + ); + + if (result.scenario.status === 'pass') { + setCurrentStep( + 'Analysis complete: aligned waveform matches the reported latencies.' + ); + } else if (result.scenario.status === 'skipped') { + setCurrentStep( + 'Analysis complete: microphone capture did not contain a matchable beep pattern.' + ); + } else { + setCurrentStep( + 'Analysis complete: waveform alignment exists but latency delta is outside tolerance.' + ); + } + + appendLog(`Test finished with status=${result.scenario.status}.`); + } catch (error) { + const message = + error instanceof Error ? error.message : 'Unknown validation error'; + setCurrentStep(`Loopback check failed: ${message}`); + appendLog(`Test failed: ${message}`); + } finally { + await cleanupLatencyValidation({ audioContext, audioRecorder }); + + if (isMountedRef.current) { + setRunnerState('idle'); + } + } + }; + + const resetTest = async () => { + if (runnerState === 'running') { + return; + } + + setAnalysis(null); + setLiveLog([]); + setCurrentStep('Ready to run the speaker-to-mic loopback latency check.'); + await cleanupLatencyValidation({ audioContext, audioRecorder }); + }; + + if (Platform.OS === 'web') { + return ( + + + + ); + } + + const controlActions: ControlAction[] = [ + { + title: 'Run Test', + onPress: () => { + runTest(); + }, + disabled: runnerState === 'running', + width: 120, + }, + { + title: 'Reset', + onPress: () => { + resetTest(); + }, + disabled: runnerState === 'running', + width: 90, + }, + ]; + + return ( + + + + + + + + + + + ); +}; + +const styles = StyleSheet.create({ + scrollContent: { + gap: 12, + paddingBottom: 32, + paddingHorizontal: 18, + paddingTop: 18, + }, + scrollView: { + flex: 1, + }, +}); + +export default LatencyValidation; diff --git a/apps/common-app/src/other/LatencyValidation/LoopbackAnalysisPanel.tsx b/apps/common-app/src/other/LatencyValidation/LoopbackAnalysisPanel.tsx new file mode 100644 index 000000000..bf8d0f949 --- /dev/null +++ b/apps/common-app/src/other/LatencyValidation/LoopbackAnalysisPanel.tsx @@ -0,0 +1,238 @@ +import React, { FC } from 'react'; +import { StyleSheet, Text, View } from 'react-native'; + +import { TestUI } from '../../testComponents'; +import { colors, layout } from '../../styles'; +import CorrelationPlot from './CorrelationPlot'; +import { formatMs } from './helpers'; +import type { LoopbackAnalysis } from './types'; +import WaveformPlot from './WaveformPlot'; + +interface LoopbackAnalysisPanelProps { + analysis: LoopbackAnalysis | null; +} + +const InfoRow: FC<{ label: string; value: string }> = ({ label, value }) => ( + + {label} + {value} + +); + +const LoopbackAnalysisPanel: FC = ({ analysis }) => { + if (!analysis) { + return ( + + ); + } + + const plotWidth = layout.screenWidth - 36; + const { scenario } = analysis; + + return ( + + + + {analysis.methodology.map((step, index) => ( + + {index + 1}. {step} + + ))} + + + + + + + + + + + + + + + + + + + + + {analysis.overlayPlots.length > 0 ? ( + + ) : null} + + + + {analysis.correlationProfile.length > 0 ? ( + + ) : null} + + + ({ + id: `${scenario.id}-${step.id}`, + message: step.message, + status: step.status, + details: step.details, + })), + }, + ]} + emptyMessage="" + /> + + ); +}; + +const styles = StyleSheet.create({ + container: { + gap: 14, + }, + infoLabel: { + color: colors.gray, + flex: 1, + fontSize: 13, + }, + infoRow: { + flexDirection: 'row', + gap: 12, + justifyContent: 'space-between', + }, + infoValue: { + color: colors.white, + flex: 1, + fontSize: 13, + fontVariant: ['tabular-nums'], + textAlign: 'right', + }, + methodologyCard: { + backgroundColor: colors.backgroundLight, + borderRadius: 8, + gap: 8, + padding: 12, + }, + methodologyStep: { + color: colors.gray, + fontSize: 13, + lineHeight: 18, + }, + metricsCard: { + backgroundColor: colors.backgroundLight, + borderRadius: 8, + gap: 10, + padding: 12, + }, +}); + +export default LoopbackAnalysisPanel; diff --git a/apps/common-app/src/other/LatencyValidation/WaveformPlot.tsx b/apps/common-app/src/other/LatencyValidation/WaveformPlot.tsx new file mode 100644 index 000000000..4612b481d --- /dev/null +++ b/apps/common-app/src/other/LatencyValidation/WaveformPlot.tsx @@ -0,0 +1,218 @@ +import React, { FC, useMemo } from 'react'; +import { StyleSheet, Text, View } from 'react-native'; +import Svg, { Line, Path, Rect } from 'react-native-svg'; + +import { colors } from '../../styles'; +import type { WaveformSeries } from './types'; + +interface PlotMarker { + ratio: number; + label: string; + color: string; +} + +interface WaveformPlotProps { + title: string; + caption?: string; + width: number; + height?: number; + series: WaveformSeries[]; + markers?: PlotMarker[]; + xAxisLabel?: string; + sharedScale?: boolean; +} + +function buildEnvelopePath( + points: number[], + width: number, + height: number, + scaleMax: number +): string { + if (points.length === 0) { + return ''; + } + + const maxVal = Math.max(scaleMax, 0.001); + + return points + .map((point, index) => { + const x = + points.length <= 1 ? 0 : (index / (points.length - 1)) * width; + const y = height - (Math.abs(point) / maxVal) * (height * 0.82) - 6; + return `${index === 0 ? 'M' : 'L'}${x.toFixed(2)},${y.toFixed(2)}`; + }) + .join(' '); +} + +const WaveformPlot: FC = ({ + title, + caption, + width, + height = 120, + series, + markers = [], + xAxisLabel, + sharedScale = false, +}) => { + const scaleMax = useMemo(() => { + if (!sharedScale) { + return Math.max( + ...series.flatMap((entry) => entry.points.map((point) => Math.abs(point))), + 0.001 + ); + } + + return Math.max( + ...series.flatMap((entry) => entry.points.map((point) => Math.abs(point))), + 0.001 + ); + }, [series, sharedScale]); + + const paths = useMemo( + () => + series.map((entry) => { + const entryMax = sharedScale + ? scaleMax + : Math.max(...entry.points.map((point) => Math.abs(point)), 0.001); + + return { + ...entry, + path: buildEnvelopePath(entry.points, width, height, entryMax), + }; + }), + [height, scaleMax, series, sharedScale, width] + ); + + return ( + + {title} + {caption ? {caption} : null} + + + + + + + {markers.map((marker) => ( + + ))} + + {paths.map((entry) => ( + + ))} + + + + + {series.map((entry) => ( + + + {entry.label} + + ))} + + + {markers.length > 0 ? ( + + {markers.map((marker) => ( + + {marker.label} + + ))} + + ) : null} + + {xAxisLabel ? {xAxisLabel} : null} + + ); +}; + +const styles = StyleSheet.create({ + axisLabel: { + color: colors.gray, + fontSize: 11, + marginTop: 4, + }, + caption: { + color: colors.gray, + fontSize: 12, + lineHeight: 17, + marginBottom: 8, + }, + container: { + backgroundColor: colors.backgroundLight, + borderRadius: 8, + gap: 8, + padding: 12, + }, + legendItem: { + alignItems: 'center', + flexDirection: 'row', + gap: 6, + }, + legendRow: { + flexDirection: 'row', + flexWrap: 'wrap', + gap: 12, + }, + legendSwatch: { + borderRadius: 2, + height: 10, + width: 10, + }, + legendText: { + color: colors.gray, + fontSize: 11, + }, + markerRow: { + flexDirection: 'row', + flexWrap: 'wrap', + gap: 10, + }, + markerText: { + fontSize: 11, + fontWeight: '600', + }, + plotFrame: { + borderColor: colors.separator, + borderRadius: 4, + borderWidth: StyleSheet.hairlineWidth, + overflow: 'hidden', + }, + title: { + color: colors.white, + fontSize: 15, + fontWeight: '700', + }, +}); + +export default WaveformPlot; diff --git a/apps/common-app/src/other/LatencyValidation/helpers.ts b/apps/common-app/src/other/LatencyValidation/helpers.ts new file mode 100644 index 000000000..ea7d36c01 --- /dev/null +++ b/apps/common-app/src/other/LatencyValidation/helpers.ts @@ -0,0 +1,403 @@ +export async function sleep(ms: number): Promise { + return new Promise((resolve) => setTimeout(resolve, ms)); +} + +export function formatMs(seconds: number): string { + return `${(seconds * 1000).toFixed(2)} ms`; +} + +export function formatTimestamp(timestamp: number): string { + return new Date(timestamp).toLocaleTimeString([], { + hour: '2-digit', + minute: '2-digit', + second: '2-digit', + }); +} + +export interface BeepPattern { + samples: Float32Array; + durationSeconds: number; + beepCount: number; + frequenciesHz: number[]; +} + +const BEEP_PATTERN = { + beepDurationMs: 45, + gapDurationMs: 70, + peakAmplitude: 0.85, + frequenciesHz: [880, 1046, 1244, 1046, 880], +} as const; + +const PLOT_MAX_POINTS = 320; +const ALIGNMENT_BLOCK_SIZE = 128; +const BASELINE_WINDOW_SECONDS = 0.15; + +function msToSamples(durationMs: number, sampleRate: number): number { + return Math.max(1, Math.round((durationMs / 1000) * sampleRate)); +} + +function createHannWindow(length: number): Float32Array { + const window = new Float32Array(length); + + if (length <= 1) { + window[0] = 1; + return window; + } + + const denominator = length - 1; + for (let index = 0; index < length; index += 1) { + window[index] = 0.5 * (1 - Math.cos((2 * Math.PI * index) / denominator)); + } + + return window; +} + +function writeBeepTone( + destination: Float32Array, + offset: number, + frequencyHz: number, + sampleRate: number, + beepSamples: number, + envelope: Float32Array, + peakAmplitude: number +): void { + const phaseIncrement = (2 * Math.PI * frequencyHz) / sampleRate; + + for (let index = 0; index < beepSamples; index += 1) { + destination[offset + index] = + peakAmplitude * envelope[index] * Math.sin(phaseIncrement * index); + } +} + +export function buildBeepReferencePattern(sampleRate: number): BeepPattern { + const { beepDurationMs, gapDurationMs, peakAmplitude, frequenciesHz } = + BEEP_PATTERN; + const beepSamples = msToSamples(beepDurationMs, sampleRate); + const gapSamples = msToSamples(gapDurationMs, sampleRate); + const beepCount = frequenciesHz.length; + const totalSamples = beepCount * beepSamples + (beepCount - 1) * gapSamples; + const samples = new Float32Array(totalSamples); + const envelope = createHannWindow(beepSamples); + + let offset = 0; + for (let beepIndex = 0; beepIndex < beepCount; beepIndex += 1) { + writeBeepTone( + samples, + offset, + frequenciesHz[beepIndex], + sampleRate, + beepSamples, + envelope, + peakAmplitude + ); + + offset += beepSamples; + if (beepIndex < beepCount - 1) { + offset += gapSamples; + } + } + + return { + samples, + durationSeconds: totalSamples / sampleRate, + beepCount, + frequenciesHz: [...frequenciesHz], + }; +} + +export function appendChannelSamples( + recorded: number[], + buffer: { + numberOfChannels: number; + getChannelData: (channel: number) => Float32Array; + }, + channel = 0 +): void { + const data = buffer.getChannelData(channel); + for (let index = 0; index < data.length; index += 1) { + recorded.push(data[index]); + } +} + +export function estimateNoiseFloor(samples: number[], windowLength: number): number { + const end = Math.min(samples.length, windowLength); + if (end === 0) { + return 0; + } + + let sumSquares = 0; + for (let index = 0; index < end; index += 1) { + sumSquares += samples[index] * samples[index]; + } + + return Math.sqrt(sumSquares / end); +} + +export function prepareRecordedSamples( + samples: number[], + sampleRate: number +): { + processed: Float32Array; + noiseFloor: number; + baselineLength: number; +} { + const baselineLength = Math.min( + samples.length, + Math.round(BASELINE_WINDOW_SECONDS * sampleRate) + ); + const noiseFloor = estimateNoiseFloor(samples, baselineLength); + + let mean = 0; + for (let index = 0; index < baselineLength; index += 1) { + mean += samples[index]; + } + mean /= Math.max(baselineLength, 1); + + const gateThreshold = Math.max(noiseFloor * 2.5, 0.002); + const processed = new Float32Array(samples.length); + + for (let index = 0; index < samples.length; index += 1) { + const centered = samples[index] - mean; + processed[index] = + Math.abs(centered) < gateThreshold ? 0 : centered; + } + + return { + processed, + noiseFloor, + baselineLength, + }; +} + +export function downsampleEnvelopeForPlot( + samples: Float32Array | number[], + maxPoints = PLOT_MAX_POINTS +): number[] { + if (samples.length === 0) { + return []; + } + + const blockSize = Math.max(1, Math.ceil(samples.length / maxPoints)); + const points: number[] = []; + + for (let blockIndex = 0; blockIndex < maxPoints; blockIndex += 1) { + const start = blockIndex * blockSize; + if (start >= samples.length) { + break; + } + + const end = Math.min(samples.length, start + blockSize); + let peak = 0; + + for (let index = start; index < end; index += 1) { + peak = Math.max(peak, Math.abs(samples[index])); + } + + points.push(peak); + } + + return points; +} + +export function sliceSamples( + samples: Float32Array | number[], + startSample: number, + length: number +): Float32Array { + const end = Math.min(samples.length, startSample + length); + const slice = new Float32Array(Math.max(0, end - startSample)); + + for (let index = startSample; index < end; index += 1) { + slice[index - startSample] = samples[index]; + } + + return slice; +} + +function downsampleEnvelope( + samples: Float32Array | number[], + blockSize: number +): Float32Array { + const length = Math.ceil(samples.length / blockSize); + const envelope = new Float32Array(length); + + for (let blockIndex = 0; blockIndex < length; blockIndex += 1) { + const start = blockIndex * blockSize; + const end = Math.min(samples.length, start + blockSize); + let peak = 0; + + for (let index = start; index < end; index += 1) { + peak = Math.max(peak, Math.abs(samples[index])); + } + + envelope[blockIndex] = peak; + } + + return envelope; +} + +function normalizedCorrelation( + reference: Float32Array, + recorded: Float32Array, + lag: number +): number { + const overlap = Math.min(reference.length, recorded.length - lag); + if (overlap <= 0) { + return -1; + } + + let dot = 0; + let refEnergy = 0; + let recEnergy = 0; + + for (let index = 0; index < overlap; index += 1) { + const refSample = reference[index]; + const recSample = recorded[lag + index]; + dot += refSample * recSample; + refEnergy += refSample * refSample; + recEnergy += recSample * recSample; + } + + const norm = Math.sqrt(refEnergy * recEnergy); + return norm > 0 ? dot / norm : -1; +} + +export interface CorrelationPoint { + lagMs: number; + score: number; +} + +export interface AlignmentResult { + lagSamples: number; + correlation: number; + lagSeconds: number; + coarseLagSamples: number; + coarseCorrelation: number; + correlationProfile: CorrelationPoint[]; + searchStartLagSamples: number; + searchEndLagSamples: number; +} + +export function findPatternAlignment( + reference: Float32Array, + recordedSamples: Float32Array | number[], + sampleRate: number, + expectedLagSeconds?: number +): AlignmentResult | null { + if (recordedSamples.length < reference.length) { + return null; + } + + const recorded = + recordedSamples instanceof Float32Array + ? recordedSamples + : Float32Array.from(recordedSamples); + const blockSize = ALIGNMENT_BLOCK_SIZE; + const refEnvelope = downsampleEnvelope(reference, blockSize); + const recEnvelope = downsampleEnvelope(recorded, blockSize); + const maxBlockLag = recEnvelope.length - refEnvelope.length; + + if (maxBlockLag < 0) { + return null; + } + + const expectedBlockLag = + expectedLagSeconds !== undefined + ? Math.round((expectedLagSeconds * sampleRate) / blockSize) + : null; + const searchRadiusBlocks = Math.ceil((0.8 * sampleRate) / blockSize); + const minBlockLag = Math.max( + 0, + expectedBlockLag !== null ? expectedBlockLag - searchRadiusBlocks : 0 + ); + const maxBlockLagSearch = Math.min( + maxBlockLag, + expectedBlockLag !== null + ? expectedBlockLag + searchRadiusBlocks + : maxBlockLag + ); + + let bestBlockLag = 0; + let bestBlockScore = -1; + const correlationProfile: CorrelationPoint[] = []; + + for (let blockLag = minBlockLag; blockLag <= maxBlockLagSearch; blockLag += 1) { + const score = normalizedCorrelation(refEnvelope, recEnvelope, blockLag); + correlationProfile.push({ + lagMs: ((blockLag * blockSize) / sampleRate) * 1000, + score, + }); + + if (score > bestBlockScore) { + bestBlockScore = score; + bestBlockLag = blockLag; + } + } + + if (bestBlockScore < 0.2) { + correlationProfile.length = 0; + let fallbackBestLag = 0; + let fallbackBestScore = -1; + + for (let blockLag = 0; blockLag <= maxBlockLag; blockLag += 1) { + const score = normalizedCorrelation(refEnvelope, recEnvelope, blockLag); + correlationProfile.push({ + lagMs: ((blockLag * blockSize) / sampleRate) * 1000, + score, + }); + + if (score > fallbackBestScore) { + fallbackBestScore = score; + fallbackBestLag = blockLag; + } + } + + if (fallbackBestScore < 0.25) { + return null; + } + + bestBlockLag = fallbackBestLag; + bestBlockScore = fallbackBestScore; + } + + const coarseLagSamples = bestBlockLag * blockSize; + const refineRadius = blockSize * 2; + const minSampleLag = Math.max(0, coarseLagSamples - refineRadius); + const maxSampleLag = Math.min( + recorded.length - reference.length, + coarseLagSamples + refineRadius + ); + + let bestLagSamples = coarseLagSamples; + let bestSampleScore = -1; + + for (let lag = minSampleLag; lag <= maxSampleLag; lag += 1) { + const score = normalizedCorrelation(reference, recorded, lag); + if (score > bestSampleScore) { + bestSampleScore = score; + bestLagSamples = lag; + } + } + + return { + lagSamples: bestLagSamples, + correlation: bestSampleScore, + lagSeconds: bestLagSamples / sampleRate, + coarseLagSamples, + coarseCorrelation: bestBlockScore, + correlationProfile, + searchStartLagSamples: minBlockLag * blockSize, + searchEndLagSamples: maxBlockLagSearch * blockSize, + }; +} + +export function buildRecordedWindow( + recordedSamples: Float32Array | number[], + centerLagSamples: number, + windowSeconds: number, + sampleRate: number +): Float32Array { + const windowSamples = Math.round(windowSeconds * sampleRate); + const startSample = Math.max(0, centerLagSamples - Math.floor(windowSamples / 2)); + return sliceSamples(recordedSamples, startSample, windowSamples); +} diff --git a/apps/common-app/src/other/LatencyValidation/index.ts b/apps/common-app/src/other/LatencyValidation/index.ts new file mode 100644 index 000000000..aea9fe34b --- /dev/null +++ b/apps/common-app/src/other/LatencyValidation/index.ts @@ -0,0 +1 @@ +export { default } from './LatencyValidation'; diff --git a/apps/common-app/src/other/LatencyValidation/latencyTests.ts b/apps/common-app/src/other/LatencyValidation/latencyTests.ts new file mode 100644 index 000000000..c71fdd611 --- /dev/null +++ b/apps/common-app/src/other/LatencyValidation/latencyTests.ts @@ -0,0 +1,528 @@ +import { + AudioManager, + type AudioContext, + type AudioRecorder, +} from 'react-native-audio-api'; +import { Platform } from 'react-native'; + +import { + appendChannelSamples, + buildBeepReferencePattern, + buildRecordedWindow, + downsampleEnvelopeForPlot, + findPatternAlignment, + formatMs, + prepareRecordedSamples, + sleep, +} from './helpers'; +import type { + LatencySnapshot, + LatencyTestScenario, + LatencyTestStep, + LoopbackAnalysis, + WaveformSeries, +} from './types'; + +// Keep the live full-duplex window (mic + speaker both active) as short as +// possible: the raw loopback path produces an idle buzz, so we only settle the +// recorder briefly and use a short scheduling lead before the beep plays. +const RECORDER_SETTLE_MS = 70; +const PLAY_LEAD_TIME_S = 0.07; +const LOOPBACK_TOLERANCE_MS = 120; +const MIN_CORRELATION = 0.45; +const PATTERN_WINDOW_PADDING_SECONDS = 0.12; + +interface LatencyTestContext { + audioContext: AudioContext; + audioRecorder: AudioRecorder; +} + +function readSnapshot( + audioContext: AudioContext, + audioRecorder: AudioRecorder +): LatencySnapshot { + return { + base: audioContext.baseLatency, + output: audioContext.outputLatency, + input: audioRecorder.inputLatency, + sampleRate: audioContext.sampleRate, + contextState: audioContext.state, + recorderActive: audioRecorder.isRecording(), + }; +} + +function createScenario( + id: string, + title: string, + startedAt: number, + steps: LatencyTestStep[] +): LatencyTestScenario { + const status = steps.some((step) => step.status === 'fail') + ? 'fail' + : steps.every((step) => step.status === 'skipped') + ? 'skipped' + : 'pass'; + + return { + id, + title, + status, + durationMs: Date.now() - startedAt, + steps, + }; +} + +function buildMethodology(patternBeepCount: number): string[] { + return [ + `Build a ${patternBeepCount}-beep reference waveform in JavaScript and play it once through AudioContext.destination.`, + 'Record only during a short pre-roll and playback window. The recorder is not routed to the speaker.', + 'Subtract the pre-beep noise floor from the microphone capture before correlation and plotting.', + 'Estimate arrival time using schedule offset + baseLatency + outputLatency + inputLatency.', + 'Cross-correlate amplitude envelopes, then compare both the best-match shift and the system-reported shift against the reference.', + ]; +} + +function buildEmptyPlots(): Pick< + LoopbackAnalysis, + | 'referencePlot' + | 'recordedWindowPlot' + | 'overlayPlots' + | 'systemShiftOverlayPlots' + | 'correlationProfile' + | 'bestAlignmentMarkerRatio' + | 'systemShiftMarkerRatio' +> { + return { + referencePlot: { label: 'Reference envelope', color: '#38ACDD', points: [] }, + recordedWindowPlot: { + label: 'Mic envelope (denoised)', + color: '#FFD61E', + points: [], + }, + overlayPlots: [], + systemShiftOverlayPlots: [], + correlationProfile: [], + bestAlignmentMarkerRatio: null, + systemShiftMarkerRatio: null, + }; +} + +function buildAnalysisPlots(params: { + referencePattern: ReturnType; + processedRecorded: Float32Array; + sampleRate: number; + expectedLagSeconds: number; + alignment: ReturnType; +}): Pick< + LoopbackAnalysis, + | 'referencePlot' + | 'recordedWindowPlot' + | 'overlayPlots' + | 'systemShiftOverlayPlots' + | 'correlationProfile' + | 'bestAlignmentMarkerRatio' + | 'systemShiftMarkerRatio' + | 'recordedDurationSeconds' +> { + const { + referencePattern, + processedRecorded, + sampleRate, + expectedLagSeconds, + alignment, + } = params; + + const patternWindowSeconds = + referencePattern.durationSeconds + PATTERN_WINDOW_PADDING_SECONDS * 2; + const centerLagSamples = alignment?.lagSamples ?? + Math.round(expectedLagSeconds * sampleRate); + const expectedLagSamples = Math.round(expectedLagSeconds * sampleRate); + const recordedWindow = buildRecordedWindow( + processedRecorded, + centerLagSamples, + patternWindowSeconds, + sampleRate + ); + const expectedWindow = buildRecordedWindow( + processedRecorded, + expectedLagSamples, + patternWindowSeconds, + sampleRate + ); + const windowStartSample = Math.max( + 0, + centerLagSamples - Math.floor((patternWindowSeconds * sampleRate) / 2) + ); + const expectedWindowStartSample = Math.max( + 0, + expectedLagSamples - Math.floor((patternWindowSeconds * sampleRate) / 2) + ); + + const referencePlot: WaveformSeries = { + label: 'Reference envelope', + color: '#38ACDD', + points: downsampleEnvelopeForPlot(referencePattern.samples), + }; + + const recordedWindowPlot: WaveformSeries = { + label: 'Mic envelope (denoised)', + color: '#FFD61E', + points: downsampleEnvelopeForPlot(recordedWindow), + }; + + const overlayPlots: WaveformSeries[] = + alignment === null + ? [] + : [ + { + label: 'Reference', + color: '#38ACDD', + points: downsampleEnvelopeForPlot(referencePattern.samples), + }, + { + label: 'Matched mic shift', + color: '#FF7A7A', + points: downsampleEnvelopeForPlot( + processedRecorded.slice( + alignment.lagSamples, + alignment.lagSamples + referencePattern.samples.length + ) + ), + }, + ]; + + const systemShiftOverlayPlots: WaveformSeries[] = [ + { + label: 'Reference', + color: '#38ACDD', + points: downsampleEnvelopeForPlot(referencePattern.samples), + }, + { + label: 'System-latency shift', + color: '#9BE564', + points: downsampleEnvelopeForPlot( + processedRecorded.slice( + expectedLagSamples, + expectedLagSamples + referencePattern.samples.length + ) + ), + }, + ]; + + return { + referencePlot, + recordedWindowPlot, + overlayPlots, + systemShiftOverlayPlots, + correlationProfile: alignment?.correlationProfile ?? [], + bestAlignmentMarkerRatio: + recordedWindow.length > 0 + ? Math.min( + 1, + Math.max(0, (centerLagSamples - windowStartSample) / recordedWindow.length) + ) + : null, + systemShiftMarkerRatio: + expectedWindow.length > 0 + ? Math.min( + 1, + Math.max( + 0, + (expectedLagSamples - expectedWindowStartSample) / expectedWindow.length + ) + ) + : null, + recordedDurationSeconds: recordedWindow.length / sampleRate, + }; +} + +function createEmptyAnalysis( + scenario: LatencyTestScenario, + partial: Partial +): LoopbackAnalysis { + const emptyPlots = buildEmptyPlots(); + + return { + scenario, + snapshot: partial.snapshot ?? { + base: 0, + output: 0, + input: 0, + sampleRate: 44100, + contextState: 'suspended', + recorderActive: false, + }, + recordingStartContextTime: partial.recordingStartContextTime ?? 0, + playTime: partial.playTime ?? 0, + scheduleOffsetSeconds: partial.scheduleOffsetSeconds ?? 0, + reportedRoundTripLatency: partial.reportedRoundTripLatency ?? 0, + expectedLatencySeconds: partial.expectedLatencySeconds ?? 0, + measuredLatencySeconds: partial.measuredLatencySeconds ?? null, + correlation: partial.correlation ?? null, + deltaMs: partial.deltaMs ?? null, + referenceDurationSeconds: partial.referenceDurationSeconds ?? 0, + recordedDurationSeconds: partial.recordedDurationSeconds ?? 0, + noiseFloor: partial.noiseFloor ?? null, + methodology: partial.methodology ?? buildMethodology(5), + ...emptyPlots, + ...partial, + }; +} + +async function runLoopbackLatencyTest( + context: LatencyTestContext +): Promise { + const startedAt = Date.now(); + const steps: LatencyTestStep[] = []; + const { audioContext, audioRecorder } = context; + + await stopLoopbackAudioIO(context); + await sleep(100); + + if (audioContext.state === 'suspended') { + await audioContext.resume(); + } + + const recordingStartContextTime = audioContext.currentTime; + const startResult = await audioRecorder.start(); + if (startResult.status !== 'success') { + steps.push({ + id: 'loopback-recorder-start', + message: 'Recorder is available for loopback measurement', + status: 'fail', + details: startResult.message, + }); + + return createEmptyAnalysis( + createScenario( + 'loopback-latency', + 'Speaker-to-mic loopback check', + startedAt, + steps + ), + { recordingStartContextTime } + ); + } + + const referencePattern = buildBeepReferencePattern(audioContext.sampleRate); + const recordedSamples: number[] = []; + + const callbackResult = audioRecorder.onAudioReady( + { + sampleRate: audioContext.sampleRate, + bufferLength: 1024, + channelCount: 1, + }, + (event) => { + if (!event.buffer) { + return; + } + + appendChannelSamples(recordedSamples, event.buffer); + } + ); + + if (callbackResult.status === 'error') { + steps.push({ + id: 'loopback-callback', + message: 'Recorder callback is available for loopback measurement', + status: 'fail', + details: callbackResult.message, + }); + + return createEmptyAnalysis( + createScenario( + 'loopback-latency', + 'Speaker-to-mic loopback check', + startedAt, + steps + ), + { + recordingStartContextTime, + referenceDurationSeconds: referencePattern.durationSeconds, + referencePlot: { + label: 'Reference envelope', + color: '#38ACDD', + points: downsampleEnvelopeForPlot(referencePattern.samples), + }, + methodology: buildMethodology(referencePattern.beepCount), + } + ); + } + + await sleep(RECORDER_SETTLE_MS); + + const snapshot = readSnapshot(audioContext, audioRecorder); + const playTime = audioContext.currentTime + PLAY_LEAD_TIME_S; + const scheduleOffsetSeconds = playTime - recordingStartContextTime; + const reportedRoundTripLatency = + snapshot.base + snapshot.output + snapshot.input; + const expectedLatencySeconds = + scheduleOffsetSeconds + reportedRoundTripLatency; + + const referenceBuffer = audioContext.createBuffer( + 1, + referencePattern.samples.length, + audioContext.sampleRate + ); + referenceBuffer.getChannelData(0).set(referencePattern.samples); + + const source = audioContext.createBufferSource(); + source.buffer = referenceBuffer; + source.connect(audioContext.destination); + source.start(playTime); + source.stop(playTime + referencePattern.durationSeconds); + + // Tail must still be long enough to capture the last beep after the full + // round-trip latency, but no longer, to avoid extra idle buzz. 0.2 s covers + // realistic loopback delays (schedule + base + output + input). + const listenWindowSeconds = + PLAY_LEAD_TIME_S + referencePattern.durationSeconds + 0.2; + await sleep(listenWindowSeconds * 1000); + + audioRecorder.clearOnAudioReady(); + await stopLoopbackAudioIO(context); + + const { processed, noiseFloor } = prepareRecordedSamples( + recordedSamples, + audioContext.sampleRate + ); + + const alignment = findPatternAlignment( + referencePattern.samples, + processed, + audioContext.sampleRate, + expectedLatencySeconds + ); + + const plotData = buildAnalysisPlots({ + referencePattern, + processedRecorded: processed, + sampleRate: audioContext.sampleRate, + expectedLagSeconds: expectedLatencySeconds, + alignment, + }); + + if (!alignment) { + steps.push({ + id: 'loopback-pattern-detected', + message: 'Recorded waveform matches the generated beep pattern', + status: 'skipped', + details: `Captured ${recordedSamples.length} samples (${formatMs(recordedSamples.length / audioContext.sampleRate)}) with noiseFloor=${noiseFloor.toFixed(4)} but could not align the ${referencePattern.beepCount}-beep reference.`, + }); + + return createEmptyAnalysis( + createScenario( + 'loopback-latency', + 'Speaker-to-mic loopback check', + startedAt, + steps + ), + { + snapshot, + recordingStartContextTime, + playTime, + scheduleOffsetSeconds, + reportedRoundTripLatency, + expectedLatencySeconds, + referenceDurationSeconds: referencePattern.durationSeconds, + recordedDurationSeconds: plotData.recordedDurationSeconds, + noiseFloor, + methodology: buildMethodology(referencePattern.beepCount), + referencePlot: plotData.referencePlot, + recordedWindowPlot: plotData.recordedWindowPlot, + systemShiftOverlayPlots: plotData.systemShiftOverlayPlots, + correlationProfile: plotData.correlationProfile, + systemShiftMarkerRatio: plotData.systemShiftMarkerRatio, + } + ); + } + + const measuredLatencySeconds = alignment.lagSeconds; + const deltaMs = (measuredLatencySeconds - expectedLatencySeconds) * 1000; + const pass = + alignment.correlation >= MIN_CORRELATION && + measuredLatencySeconds >= + expectedLatencySeconds - LOOPBACK_TOLERANCE_MS / 1000 && + measuredLatencySeconds <= + expectedLatencySeconds + LOOPBACK_TOLERANCE_MS / 1000; + + steps.push({ + id: 'loopback-pattern-detected', + message: 'Recorded waveform matches the generated beep pattern', + status: alignment.correlation >= MIN_CORRELATION ? 'pass' : 'fail', + details: `correlation=${alignment.correlation.toFixed(3)}, coarse=${alignment.coarseCorrelation.toFixed(3)}, alignedAt=${formatMs(measuredLatencySeconds)}, noiseFloor=${noiseFloor.toFixed(4)}`, + }); + + steps.push({ + id: 'loopback-latency-match', + message: + 'Aligned loopback delay matches baseLatency + outputLatency + inputLatency', + status: pass ? 'pass' : 'fail', + details: `measured=${formatMs(measuredLatencySeconds)}, expected=${formatMs(expectedLatencySeconds)} (schedule ${formatMs(scheduleOffsetSeconds)} + base ${formatMs(snapshot.base)} + output ${formatMs(snapshot.output)} + input ${formatMs(snapshot.input)}), delta=${deltaMs.toFixed(2)} ms`, + }); + + return createEmptyAnalysis( + createScenario( + 'loopback-latency', + 'Speaker-to-mic loopback check', + startedAt, + steps + ), + { + snapshot, + recordingStartContextTime, + playTime, + scheduleOffsetSeconds, + reportedRoundTripLatency, + expectedLatencySeconds, + measuredLatencySeconds, + correlation: alignment.correlation, + deltaMs, + referenceDurationSeconds: referencePattern.durationSeconds, + recordedDurationSeconds: plotData.recordedDurationSeconds, + noiseFloor, + methodology: buildMethodology(referencePattern.beepCount), + referencePlot: plotData.referencePlot, + recordedWindowPlot: plotData.recordedWindowPlot, + overlayPlots: plotData.overlayPlots, + systemShiftOverlayPlots: plotData.systemShiftOverlayPlots, + correlationProfile: plotData.correlationProfile, + bestAlignmentMarkerRatio: plotData.bestAlignmentMarkerRatio, + systemShiftMarkerRatio: plotData.systemShiftMarkerRatio, + } + ); +} + +export async function runLatencyValidationSuite( + context: LatencyTestContext +): Promise { + return runLoopbackLatencyTest(context); +} + +export async function stopLoopbackAudioIO( + context: LatencyTestContext +): Promise { + const { audioContext, audioRecorder } = context; + + audioRecorder.clearOnAudioReady(); + audioRecorder.disconnect(); + + if (audioRecorder.isRecording()) { + await audioRecorder.stop(); + } + + if (audioContext.state === 'running') { + await audioContext.suspend(); + } + + if (Platform.OS !== 'web') { + await AudioManager.setAudioSessionActivity(false); + } +} + +export async function cleanupLatencyValidation( + context: LatencyTestContext +): Promise { + await stopLoopbackAudioIO(context); +} diff --git a/apps/common-app/src/other/LatencyValidation/types.ts b/apps/common-app/src/other/LatencyValidation/types.ts new file mode 100644 index 000000000..7caba6ed3 --- /dev/null +++ b/apps/common-app/src/other/LatencyValidation/types.ts @@ -0,0 +1,60 @@ +export interface LatencySnapshot { + base: number; + output: number; + input: number; + sampleRate: number; + contextState: string; + recorderActive: boolean; +} + +export type LatencyTestStatus = 'pass' | 'fail' | 'info' | 'skipped'; + +export interface LatencyTestStep { + id: string; + message: string; + status: LatencyTestStatus; + details?: string; +} + +export interface LatencyTestScenario { + id: string; + title: string; + status: 'pass' | 'fail' | 'skipped'; + durationMs: number; + steps: LatencyTestStep[]; +} + +export interface WaveformSeries { + label: string; + color: string; + points: number[]; +} + +export interface CorrelationPoint { + lagMs: number; + score: number; +} + +export interface LoopbackAnalysis { + scenario: LatencyTestScenario; + snapshot: LatencySnapshot; + recordingStartContextTime: number; + playTime: number; + scheduleOffsetSeconds: number; + reportedRoundTripLatency: number; + expectedLatencySeconds: number; + measuredLatencySeconds: number | null; + correlation: number | null; + deltaMs: number | null; + referenceDurationSeconds: number; + recordedDurationSeconds: number; + referencePlot: WaveformSeries; + recordedWindowPlot: WaveformSeries; + overlayPlots: WaveformSeries[]; + systemShiftOverlayPlots: WaveformSeries[]; + correlationProfile: CorrelationPoint[]; + bestAlignmentMarkerRatio: number | null; + systemShiftMarkerRatio: number | null; + noiseFloor: number | null; + methodology: string[]; +} diff --git a/apps/common-app/src/other/index.ts b/apps/common-app/src/other/index.ts index c4a1d8660..c98e9f1c2 100644 --- a/apps/common-app/src/other/index.ts +++ b/apps/common-app/src/other/index.ts @@ -5,6 +5,7 @@ import { TestScreen } from '../../../../packages/test-app-screen'; import AudioParamPipeline from './AudioParamPipeline'; import AudioPipelineStress from './AudioPipelineStress'; +import LatencyValidation from './LatencyValidation'; /** Screens shown under the 'Other' bottom tab (stress tests, internal tooling). */ export const otherScreens: Example[] = [ @@ -26,4 +27,10 @@ export const otherScreens: Example[] = [ Icon: icons.TestTube, screen: TestScreen, }, + { + key: 'LatencyValidation', + title: 'Latency Validation', + Icon: icons.CircleCheck, + screen: LatencyValidation, + }, ]; diff --git a/apps/fabric-example/ios/Podfile.lock b/apps/fabric-example/ios/Podfile.lock index 26b95a6a6..fb6ef5fc4 100644 --- a/apps/fabric-example/ios/Podfile.lock +++ b/apps/fabric-example/ios/Podfile.lock @@ -2523,7 +2523,7 @@ EXTERNAL SOURCES: SPEC CHECKSUMS: FBLazyVector: c00c20551d40126351a6783c47ce75f5b374851b - hermes-engine: c399a2e224a0b13c589d76b4fc05e14bdd76fa88 + hermes-engine: 91023181d4bc5948b457de5314623fbfe4f8604e RCTDeprecation: 3bb167081b134461cfeb875ff7ae1945f8635257 RCTRequired: 74839f55d5058a133a0bc4569b0afec750957f64 RCTSwiftUI: 87a316382f3eab4dd13d2a0d0fd2adcce917361a @@ -2532,7 +2532,7 @@ SPEC CHECKSUMS: React: 1b1536b9099195944034e65b1830f463caaa8390 React-callinvoker: 6dff6d17d1d6cc8fdf85468a649bafed473c65f5 React-Core: 00faa4d038298089a1d5a5b21dde8660c4f0820d - React-Core-prebuilt: ab26be1216323aea7c76f96ca450bffa7bcd4a72 + React-Core-prebuilt: a6d614de037caff7898424dfc22915ec792de921 React-CoreModules: a17807f849bfd86045b0b9a75ec8c19373b482f6 React-cxxreact: c7b53ace5827be54048288bce5c55f337c41e95f React-debug: e1f00fcd2cef58a2897471a6d76a4ef5f5f90c74 @@ -2596,7 +2596,7 @@ SPEC CHECKSUMS: ReactAppDependencyProvider: 5787b37b8e2e51dfeab697ec031cc7c4080dcea2 ReactCodegen: d07ee3c8db75b43d1cbe479ae6affebf9925c733 ReactCommon: fe2a3af8975e63efa60f95fca8c34dc85deee360 - ReactNativeDependencies: 212738cc51e6c4cc34ee487890497d6f41979ec0 + ReactNativeDependencies: 4d5ce2683b6d74f7c686bf90a88c7d381295cf3c RNAudioAPI: 6bba1527c091e2702e3d879de7564ef1ccf30a78 RNAudioWorklets: febe470be646585d9b8b0de320a5fb8684cb012c RNGestureHandler: 187c5c7936abf427bc4d22d6c3b1ac80ad1f63c0 diff --git a/packages/audiodocs/docs/core/audio-context.mdx b/packages/audiodocs/docs/core/audio-context.mdx index aec04234f..4a0b4a654 100644 --- a/packages/audiodocs/docs/core/audio-context.mdx +++ b/packages/audiodocs/docs/core/audio-context.mdx @@ -2,6 +2,7 @@ sidebar_position: 2 --- +import { ReadOnly } from '@site/src/components/Badges'; import { Optional } from '@site/src/components/Badges'; # AudioContext @@ -29,7 +30,11 @@ constructor(options?: AudioContextOptions) ## Properties -`AudioContext` does not define any additional properties. +| Name | Type | Description | | +| :----: | :----: | :-------- | :-: | +| `baseLatency` | `number` | Processing latency in seconds incurred by the `AudioContext` passing audio from the `AudioDestinationNode` to the audio subsystem. Does not include hardware output latency or audio graph processing time. Equals the render callback duration (`callbackFrames / sampleRate`). Returns `0` when the context is not running. | | +| `outputLatency` | `number` | Estimated output latency in seconds — the interval between when the audio system is given a buffer and when the first sample is played by the output device. Can change when the audio route changes; re-query before sync-sensitive work. Returns `0` when the context is not running. On iOS backed by `AVAudioSession.outputLatency`; on Android sampled via Oboe timestamps. | | + Inherits all properties from [`BaseAudioContext`](/docs/core/base-audio-context#properties). diff --git a/packages/audiodocs/docs/inputs/audio-recorder.mdx b/packages/audiodocs/docs/inputs/audio-recorder.mdx index d4c0d1db0..d36628771 100644 --- a/packages/audiodocs/docs/inputs/audio-recorder.mdx +++ b/packages/audiodocs/docs/inputs/audio-recorder.mdx @@ -365,8 +365,9 @@ const audioRecorder = new AudioRecorder(); ## Properties | Name | Type | Description | -| :----: | :----: | :-------- | +| :--- | :--- | :--- | | `options` | [`AudioRecorderFileOptions`](#audiorecorderfileoptions) \| `null` | Active file-output configuration after [`enableFileOutput()`](#enablefileoutput), or `null` when file output is disabled. | +| `inputLatency` | `number` | Returns the estimated **full** input latency in seconds (from analog input to app delivery). Mirrors [`AudioContext.outputLatency`](/docs/core/audio-context#properties) semantics. Calculated using underlying platform properties (`AVAudioSession` on iOS; Oboe timestamps/burst sizes on Android). Value may change when the audio route changes. Returns `0` when idle. | ## Methods diff --git a/packages/audiodocs/docs/other/web-audio-api-coverage.mdx b/packages/audiodocs/docs/other/web-audio-api-coverage.mdx index f40ed597a..741c8703c 100644 --- a/packages/audiodocs/docs/other/web-audio-api-coverage.mdx +++ b/packages/audiodocs/docs/other/web-audio-api-coverage.mdx @@ -41,7 +41,7 @@ We will do our best to ship it as soon as possible! | PeriodicWave | ✅ | | StereoPannerNode | ✅ | | WaveShaperNode | ✅ | -| AudioContext | 🚧 | Available props and methods: `close`, `suspend`, `resume` | +| AudioContext | 🚧 | Available props and methods: `baseLatency`, `outputLatency`, `close`, `suspend`, `resume` | | BaseAudioContext | 🚧 | Available props and methods: `currentTime`, `destination`, `listener`, `sampleRate`, `state`, `decodeAudioData`, all create methods for available or partially implemented nodes | | AudioListener | 🚧 | No effect until PannerNode. | | AudioSinkInfo | ❌ | diff --git a/packages/react-native-audio-api/android/src/main/cpp/audioapi/android/core/AndroidAudioRecorder.cpp b/packages/react-native-audio-api/android/src/main/cpp/audioapi/android/core/AndroidAudioRecorder.cpp index eaa32085d..faa9bb2be 100644 --- a/packages/react-native-audio-api/android/src/main/cpp/audioapi/android/core/AndroidAudioRecorder.cpp +++ b/packages/react-native-audio-api/android/src/main/cpp/audioapi/android/core/AndroidAudioRecorder.cpp @@ -2,6 +2,7 @@ #include #include #include +#include #if !RN_AUDIO_API_FFMPEG_DISABLED #include @@ -17,9 +18,10 @@ #include #include +#include #include +#include #include -#include #include #include @@ -30,7 +32,8 @@ AndroidAudioRecorder::AndroidAudioRecorder( : AudioRecorder(audioEventHandlerRegistry), streamSampleRate_(0.0), streamChannelCount_(0), - streamMaxBufferSizeInFrames_(0) {} + streamMaxBufferSizeInFrames_(0), + latencyMonitor_([this] { return getLatencyMillis(); }) {} /// @brief Destructor ensures that the audio stream and each output type are closed and flushed up remaining data. /// callable from the JS thread or handled by audio thread (if js dropped recorder first). @@ -50,12 +53,8 @@ AndroidAudioRecorder::~AndroidAudioRecorder() { if (adapterNodeHandle_ != nullptr) { static_cast(adapterNodeHandle_->audioNode.get())->adapterCleanup(); } - // oboe could be handling stopping and closing the stream, sanity check just in case - if (mStream_ != nullptr) { - mStream_->requestStop(); - mStream_->close(); - mStream_.reset(); - } + + cleanup(); } /// @brief Creates and opens the Oboe audio input stream for recording. @@ -64,6 +63,7 @@ AndroidAudioRecorder::~AndroidAudioRecorder() { /// Callable from the JS thread only. /// @returns Success status or Error status with message. Result AndroidAudioRecorder::openAudioStream() { + std::scoped_lock streamLock(streamMutex_); if (mStream_ != nullptr) { return Result::Ok(None); } @@ -101,7 +101,7 @@ Result AndroidAudioRecorder::openAudioStream() { /// Most likely this was due to alpha version mistakes, but in case of problems leaving this here. (ㆆ _ ㆆ) /// @returns On success, returns the file URI where the recording is being saved (if file output is enabled). Result AndroidAudioRecorder::start(const std::string &fileNameOverride) { - std::scoped_lock startLock(callbackMutex_, fileWriterMutex_, adapterNodeMutex_); + std::scoped_lock startLock(callbackMutex_, fileWriterMutex_, adapterNodeMutex_, streamMutex_); if (!isIdle()) { return Result::Err("Recorder is already recording"); @@ -157,6 +157,8 @@ Result AndroidAudioRecorder::start(const std::string &fil } state_.store(RecorderState::Recording, std::memory_order_release); + latencyMonitor_.start(); + return Result::Ok(None); } @@ -166,6 +168,7 @@ Result AndroidAudioRecorder::start(const std::string &fil /// NOTE: due to the file access nature on Android, the size might sometimes be zeroed (really long files). Result, double, double>, std::string> AndroidAudioRecorder::stop() { + latencyMonitor_.stop(); std::shared_ptr fileWriter; std::shared_ptr dataCallback; std::shared_ptr adapterNodeHandle; @@ -176,7 +179,7 @@ AndroidAudioRecorder::stop() { bool hadFileOutput = false; { - std::scoped_lock stopLock(callbackMutex_, fileWriterMutex_, adapterNodeMutex_); + std::scoped_lock stopLock(callbackMutex_, fileWriterMutex_, adapterNodeMutex_, streamMutex_); if (isIdle()) { return Result, double, double>, std::string>::Err( @@ -359,6 +362,8 @@ void AndroidAudioRecorder::disableFileOutput() { /// For session without active file output, this method acts same as stop(). /// This method should be called from the JS thread only. void AndroidAudioRecorder::pause() { + latencyMonitor_.stop(); + std::scoped_lock streamLock(streamMutex_); if (!isRecording()) { return; } @@ -370,12 +375,14 @@ void AndroidAudioRecorder::pause() { /// @brief Resumes the audio recording stream if it was previously paused. /// This method should be called from the JS thread only. void AndroidAudioRecorder::resume() { + std::scoped_lock streamLock(streamMutex_); if (!isPaused()) { return; } mStream_->start(0); state_.store(RecorderState::Recording, std::memory_order_release); + latencyMonitor_.start(); } /// @brief Sets the callback to be invoked when audio data is ready. @@ -514,7 +521,9 @@ oboe::DataCallbackResult AndroidAudioRecorder::onAudioReady( } bool AndroidAudioRecorder::isRecording() const { - return state_.load(std::memory_order_acquire) == RecorderState::Recording && + std::scoped_lock streamLock(streamMutex_); + return mStream_ != nullptr && + state_.load(std::memory_order_acquire) == RecorderState::Recording && mStream_->getState() == oboe::StreamState::Started; } @@ -527,21 +536,31 @@ bool AndroidAudioRecorder::isIdle() const { } void AndroidAudioRecorder::cleanup() { + latencyMonitor_.stop(); + std::scoped_lock streamLock(streamMutex_); state_.store(RecorderState::Idle, std::memory_order_release); if (mStream_ != nullptr) { + mStream_->requestStop(); mStream_->close(); mStream_.reset(); } } /// @brief onError callback that is invoked by the Oboe stream when an error occurs. -/// This method runs on the audio thread. +/// This method runs on a background thread spawned by Oboe as per AudioStreamAAudio::internalErrorCallback. /// If the error is a disconnection, it attempts to reopen the stream and resume recording. /// @param oboeStream Pointer to the Oboe audio stream. /// @param error The oboe::Result error code. void AndroidAudioRecorder::onErrorAfterClose(oboe::AudioStream *stream, oboe::Result error) { + std::scoped_lock streamLock(streamMutex_); if (error == oboe::Result::ErrorDisconnected) { + + // Since this runs on a background thread, it can be delayed, so do not teardown an already healthy stream. + if (mStream_.get() != stream) { + return; + } + cleanup(); auto streamResult = openAudioStream(); @@ -563,7 +582,39 @@ void AndroidAudioRecorder::onErrorAfterClose(oboe::AudioStream *stream, oboe::Re mStream_->requestStart(); state_.store(RecorderState::Recording, std::memory_order_release); + latencyMonitor_.start(); } } +std::optional AndroidAudioRecorder::getLatencyMillis() { + std::unique_lock lock(streamMutex_, std::try_to_lock); + if (lock.owns_lock() && isRecording()) { + auto latencyResult = mStream_->calculateLatencyMillis(); + if (latencyResult) { + return latencyResult.value() / 1000.0; + } + } + return std::nullopt; +} + +double AndroidAudioRecorder::getInputLatency() const { + double smoothed = latencyMonitor_.getSmoothedLatency(); + if (smoothed > 0.0) { + return smoothed; + } + + std::scoped_lock streamLock(streamMutex_); + if (mStream_ == nullptr || isIdle()) { + return 0.0; + } + + double minBase = 0.0; + if (mStream_->getFramesPerBurst() > 0 && streamSampleRate_ > 0.0f) { + minBase = static_cast(mStream_->getFramesPerBurst()) / streamSampleRate_; + } + + auto latencyResult = mStream_->calculateLatencyMillis(); + return latencyResult ? std::max(latencyResult.value() / 1000.0, minBase) : minBase; +} + } // namespace audioapi diff --git a/packages/react-native-audio-api/android/src/main/cpp/audioapi/android/core/AndroidAudioRecorder.h b/packages/react-native-audio-api/android/src/main/cpp/audioapi/android/core/AndroidAudioRecorder.h index 03cced2f5..34a6b030f 100644 --- a/packages/react-native-audio-api/android/src/main/cpp/audioapi/android/core/AndroidAudioRecorder.h +++ b/packages/react-native-audio-api/android/src/main/cpp/audioapi/android/core/AndroidAudioRecorder.h @@ -6,10 +6,14 @@ #include #include #include +#include +#include #include #include #include +#include + namespace audioapi { class AudioFileProperties; @@ -49,6 +53,8 @@ class AndroidAudioRecorder : public oboe::AudioStreamCallback, void connect(const std::shared_ptr &node) override; void disconnect() override; + [[nodiscard]] double getInputLatency() const override; + oboe::DataCallbackResult onAudioReady(oboe::AudioStream *oboeStream, void *audioData, int32_t numFrames) override; void onErrorAfterClose(oboe::AudioStream *oboeStream, oboe::Result error) override; @@ -56,7 +62,7 @@ class AndroidAudioRecorder : public oboe::AudioStreamCallback, private: std::shared_ptr deinterleavingBuffer_; - float streamSampleRate_; + std::atomic streamSampleRate_; int32_t streamChannelCount_; int32_t streamMaxBufferSizeInFrames_; @@ -70,6 +76,9 @@ class AndroidAudioRecorder : public oboe::AudioStreamCallback, Result setupFileWriter( const std::shared_ptr &properties, const std::string &fileNameOverride = ""); + + LatencyMonitor latencyMonitor_; + [[nodiscard]] std::optional getLatencyMillis(); }; } // namespace audioapi diff --git a/packages/react-native-audio-api/android/src/main/cpp/audioapi/android/core/AudioPlayer.cpp b/packages/react-native-audio-api/android/src/main/cpp/audioapi/android/core/AudioPlayer.cpp index 3d9f5968a..bb9055bfe 100644 --- a/packages/react-native-audio-api/android/src/main/cpp/audioapi/android/core/AudioPlayer.cpp +++ b/packages/react-native-audio-api/android/src/main/cpp/audioapi/android/core/AudioPlayer.cpp @@ -26,9 +26,11 @@ AudioPlayer::AudioPlayer( channelCount_(channelCount), isRunning_(false), driverMutex_(driverMutex), - context_(context) {} + context_(context), + latencyMonitor_([this] { return getLatencyMillis(); }) {} bool AudioPlayer::openAudioStream() { + std::scoped_lock lock(streamMutex_); AudioStreamBuilder builder; builder.setSharingMode(SharingMode::Exclusive) @@ -55,9 +57,15 @@ bool AudioPlayer::openAudioStream() { } bool AudioPlayer::start() { + std::scoped_lock lock(streamMutex_); if (mStream_ != nullptr) { auto result = mStream_->requestStart() == oboe::Result::OK; isRunning_.store(result, std::memory_order_release); + + if (result) { + latencyMonitor_.start(); + } + return result; } @@ -65,13 +73,17 @@ bool AudioPlayer::start() { } void AudioPlayer::stop() { + latencyMonitor_.stop(); + std::scoped_lock lock(streamMutex_); if (mStream_ != nullptr) { isRunning_.store(false, std::memory_order_release); + lastCallbackFrameCount_.store(0, std::memory_order_release); mStream_->requestStop(); } } bool AudioPlayer::resume() { + std::scoped_lock lock(streamMutex_); if (isRunning()) { return true; } @@ -79,6 +91,11 @@ bool AudioPlayer::resume() { if (mStream_ != nullptr) { auto result = mStream_->requestStart() == oboe::Result::OK; isRunning_.store(result, std::memory_order_release); + + if (result) { + latencyMonitor_.start(); + } + return result; } @@ -86,6 +103,8 @@ bool AudioPlayer::resume() { } void AudioPlayer::suspend() { + latencyMonitor_.stop(); + std::scoped_lock lock(streamMutex_); if (mStream_ != nullptr) { isRunning_.store(false, std::memory_order_release); mStream_->requestPause(); @@ -93,6 +112,8 @@ void AudioPlayer::suspend() { } void AudioPlayer::cleanup() { + latencyMonitor_.stop(); + std::scoped_lock lock(streamMutex_); isInitialized_.store(false, std::memory_order_release); if (mStream_ != nullptr) { @@ -102,6 +123,7 @@ void AudioPlayer::cleanup() { } bool AudioPlayer::isRunning() const { + std::scoped_lock lock(streamMutex_); return mStream_ != nullptr && mStream_->getState() == oboe::StreamState::Started && isRunning_.load(std::memory_order_acquire); } @@ -112,6 +134,10 @@ AudioPlayer::onAudioReady(AudioStream *oboeStream, void *audioData, int32_t numF return DataCallbackResult::Continue; } + if (numFrames > 0) { + lastCallbackFrameCount_.store(numFrames, std::memory_order_release); + } + const CurrentRenderScope renderScope(currentRenders_); auto *buffer = static_cast(audioData); @@ -145,7 +171,7 @@ void AudioPlayer::onErrorAfterClose(oboe::AudioStream *stream, oboe::Result erro } // Serialize with start()/resume()/suspend()/close() on the JS / promise-pool threads. - std::scoped_lock lock(*driverMutex_); + std::scoped_lock lock(*driverMutex_, streamMutex_); auto context = context_.lock(); if (context == nullptr || context->isClosed()) { @@ -163,4 +189,41 @@ void AudioPlayer::onErrorAfterClose(oboe::AudioStream *stream, oboe::Result erro resume(); } } + +std::optional AudioPlayer::getLatencyMillis() { + std::unique_lock lock(streamMutex_, std::try_to_lock); + if (lock.owns_lock() && isRunning()) { + auto latencyResult = mStream_->calculateLatencyMillis(); + if (latencyResult) { + return latencyResult.value() / 1000.0; + } + } + return std::nullopt; +} + +double AudioPlayer::getBaseLatency() const { + std::scoped_lock lock(streamMutex_); + if (mStream_ == nullptr || !isInitialized_.load(std::memory_order_acquire) || !isRunning()) { + return 0.0; + } + return static_cast(RENDER_QUANTUM_SIZE) / static_cast(sampleRate_); +} + +double AudioPlayer::getOutputLatency() const { + double smoothed = latencyMonitor_.getSmoothedLatency(); + if (smoothed > 0.0) { + return smoothed; + } + + std::scoped_lock lock(streamMutex_); + if (mStream_ == nullptr || !isInitialized_.load(std::memory_order_acquire) || !isRunning()) { + return 0.0; + } + + int32_t burst = mStream_->getFramesPerBurst(); + double minBase = static_cast(burst > 0 ? burst : RENDER_QUANTUM_SIZE) / sampleRate_; + + auto latencyResult = mStream_->calculateLatencyMillis(); + return latencyResult ? std::max(latencyResult.value() / 1000.0, minBase) : minBase; +} } // namespace audioapi diff --git a/packages/react-native-audio-api/android/src/main/cpp/audioapi/android/core/AudioPlayer.h b/packages/react-native-audio-api/android/src/main/cpp/audioapi/android/core/AudioPlayer.h index e27af5546..bb280db24 100644 --- a/packages/react-native-audio-api/android/src/main/cpp/audioapi/android/core/AudioPlayer.h +++ b/packages/react-native-audio-api/android/src/main/cpp/audioapi/android/core/AudioPlayer.h @@ -1,7 +1,10 @@ #pragma once +#include #include #include +#include +#include #include #include #include @@ -41,26 +44,35 @@ class AudioPlayer : public AudioStreamDataCallback, [[nodiscard]] bool isRunning() const; + [[nodiscard]] double getBaseLatency() const; + [[nodiscard]] double getOutputLatency() const; + DataCallbackResult onAudioReady(AudioStream *oboeStream, void *audioData, int32_t numFrames) override; - void onErrorAfterClose(AudioStream *audioStream, Result error) override; + void onErrorAfterClose(AudioStream *audioStream, oboe::Result error) override; private: std::function renderAudio_; std::atomic ¤tRenders_; std::shared_ptr mStream_; + mutable std::recursive_mutex streamMutex_; std::shared_ptr buffer_; std::atomic isInitialized_{false}; float sampleRate_; int channelCount_; std::atomic isRunning_; + /// Updated on the audio thread from each Oboe callback `numFrames`. + std::atomic lastCallbackFrameCount_{0}; std::mutex *driverMutex_; std::weak_ptr context_; bool openAudioStream(); facebook::jni::global_ref nativeAudioPlayer_; + + LatencyMonitor latencyMonitor_; + [[nodiscard]] std::optional getLatencyMillis(); }; } // namespace audioapi diff --git a/packages/react-native-audio-api/android/src/main/cpp/audioapi/android/core/utils/latency/LatencyMonitor.cpp b/packages/react-native-audio-api/android/src/main/cpp/audioapi/android/core/utils/latency/LatencyMonitor.cpp new file mode 100644 index 000000000..c0e23c245 --- /dev/null +++ b/packages/react-native-audio-api/android/src/main/cpp/audioapi/android/core/utils/latency/LatencyMonitor.cpp @@ -0,0 +1,53 @@ + +#include +#include +#include +#include +#include +#include + +namespace audioapi { + +LatencyMonitor::LatencyMonitor(LatencyCallback fetchLatencyCb) + : stopThread_(true), smoothedLatency_(0.0), fetchCallback_(std::move(fetchLatencyCb)) {} + +double LatencyMonitor::getSmoothedLatency() const { + return smoothedLatency_.load(std::memory_order_acquire); +} + +void LatencyMonitor::start() { + stop(); + smoothedLatency_.store(0.0, std::memory_order_release); + stopThread_.store(false, std::memory_order_release); + monitorThread_ = std::thread(&LatencyMonitor::pollingLoop, this); +} + +void LatencyMonitor::pollingLoop() { + while (!stopThread_.load(std::memory_order_acquire)) { + std::this_thread::sleep_for(std::chrono::milliseconds(LATENCY_MONITOR_SLEEP_DURATION_MS)); + + auto rawLatencyOpt = fetchCallback_(); + + if (rawLatencyOpt.has_value()) { + double rawLatency = rawLatencyOpt.value(); + double currentSmoothed = smoothedLatency_.load(std::memory_order_acquire); + + if (currentSmoothed == 0.0) { + smoothedLatency_.store(rawLatency, std::memory_order_release); + } else { + double newSmoothed = (LATENCY_MONITOR_SMOOTHING_FACTOR * rawLatency) + + ((1.0 - LATENCY_MONITOR_SMOOTHING_FACTOR) * currentSmoothed); + smoothedLatency_.store(newSmoothed, std::memory_order_release); + } + } + } +} + +void LatencyMonitor::stop() { + stopThread_.store(true, std::memory_order_release); + if (monitorThread_.joinable()) { + monitorThread_.join(); + } +} + +}; // namespace audioapi diff --git a/packages/react-native-audio-api/android/src/main/cpp/audioapi/android/core/utils/latency/LatencyMonitor.h b/packages/react-native-audio-api/android/src/main/cpp/audioapi/android/core/utils/latency/LatencyMonitor.h new file mode 100644 index 000000000..2e9ff4969 --- /dev/null +++ b/packages/react-native-audio-api/android/src/main/cpp/audioapi/android/core/utils/latency/LatencyMonitor.h @@ -0,0 +1,39 @@ +#pragma once + +#include +#include +#include +#include +#include +#include +#include + +namespace audioapi { + +inline constexpr auto LATENCY_MONITOR_SLEEP_DURATION_MS = std::chrono::milliseconds(10); +inline constexpr auto LATENCY_MONITOR_SMOOTHING_FACTOR = 0.1; +inline constexpr auto LATENCY_MONITOR_CALLBACK_SIZE = sizeof(std::size_t); + +using LatencyCallback = FatFunction()>; + +class LatencyMonitor { + public: + explicit LatencyMonitor(LatencyCallback fetchLatencyCb); + ~LatencyMonitor() { + stop(); + } + DELETE_COPY_AND_MOVE(LatencyMonitor); + void start(); + void stop(); + [[nodiscard]] double getSmoothedLatency() const; + + private: + std::atomic stopThread_; + std::atomic smoothedLatency_; + std::thread monitorThread_; + LatencyCallback fetchCallback_; + + void pollingLoop(); +}; + +} // namespace audioapi \ No newline at end of file diff --git a/packages/react-native-audio-api/common/cpp/audioapi/HostObjects/AudioContextHostObject.cpp b/packages/react-native-audio-api/common/cpp/audioapi/HostObjects/AudioContextHostObject.cpp index 1fdc3c820..c898fb8c5 100644 --- a/packages/react-native-audio-api/common/cpp/audioapi/HostObjects/AudioContextHostObject.cpp +++ b/packages/react-native-audio-api/common/cpp/audioapi/HostObjects/AudioContextHostObject.cpp @@ -18,6 +18,8 @@ AudioContextHostObject::AudioContextHostObject( std::make_shared(sampleRate, audioEventHandlerRegistry), runtime, callInvoker) { + addGetters(JSI_EXPORT_PROPERTY_GETTER(AudioContextHostObject, outputLatency)); + addGetters(JSI_EXPORT_PROPERTY_GETTER(AudioContextHostObject, baseLatency)); addFunctions( JSI_EXPORT_FUNCTION(AudioContextHostObject, close), JSI_EXPORT_FUNCTION(AudioContextHostObject, resume), @@ -69,6 +71,16 @@ JSI_HOST_FUNCTION_IMPL(AudioContextHostObject, suspend) { return promise; } +JSI_PROPERTY_GETTER_IMPL(AudioContextHostObject, outputLatency) { + auto audioContext = std::static_pointer_cast(context_); + return {audioContext->getOutputLatency()}; +} + +JSI_PROPERTY_GETTER_IMPL(AudioContextHostObject, baseLatency) { + auto audioContext = std::static_pointer_cast(context_); + return {audioContext->getBaseLatency()}; +} + JSI_HOST_FUNCTION_IMPL(AudioContextHostObject, createMediaElementSource) { auto sourceObject = args[0].asObject(runtime); auto fileSourceHostObject = sourceObject.getHostObject(runtime); diff --git a/packages/react-native-audio-api/common/cpp/audioapi/HostObjects/AudioContextHostObject.h b/packages/react-native-audio-api/common/cpp/audioapi/HostObjects/AudioContextHostObject.h index 99104aa08..4952c6c85 100644 --- a/packages/react-native-audio-api/common/cpp/audioapi/HostObjects/AudioContextHostObject.h +++ b/packages/react-native-audio-api/common/cpp/audioapi/HostObjects/AudioContextHostObject.h @@ -23,5 +23,8 @@ class AudioContextHostObject : public BaseAudioContextHostObject { JSI_HOST_FUNCTION_DECL(resume); JSI_HOST_FUNCTION_DECL(suspend); JSI_HOST_FUNCTION_DECL(createMediaElementSource); + + JSI_PROPERTY_GETTER_DECL(outputLatency); + JSI_PROPERTY_GETTER_DECL(baseLatency); }; } // namespace audioapi diff --git a/packages/react-native-audio-api/common/cpp/audioapi/HostObjects/inputs/AudioRecorderHostObject.cpp b/packages/react-native-audio-api/common/cpp/audioapi/HostObjects/inputs/AudioRecorderHostObject.cpp index f787a1d6b..5fe0476c4 100644 --- a/packages/react-native-audio-api/common/cpp/audioapi/HostObjects/inputs/AudioRecorderHostObject.cpp +++ b/packages/react-native-audio-api/common/cpp/audioapi/HostObjects/inputs/AudioRecorderHostObject.cpp @@ -47,6 +47,8 @@ AudioRecorderHostObject::AudioRecorderHostObject( JSI_EXPORT_FUNCTION(AudioRecorderHostObject, setOnError), JSI_EXPORT_FUNCTION(AudioRecorderHostObject, clearOnError), JSI_EXPORT_FUNCTION(AudioRecorderHostObject, getCurrentDuration)); + + addGetters(JSI_EXPORT_PROPERTY_GETTER(AudioRecorderHostObject, inputLatency)); } JSI_HOST_FUNCTION_IMPL(AudioRecorderHostObject, start) { @@ -219,4 +221,8 @@ JSI_HOST_FUNCTION_IMPL(AudioRecorderHostObject, getCurrentDuration) { return jsi::Value(duration); } +JSI_PROPERTY_GETTER_IMPL(AudioRecorderHostObject, inputLatency) { + return jsi::Value(audioRecorder_->getInputLatency()); +} + } // namespace audioapi diff --git a/packages/react-native-audio-api/common/cpp/audioapi/HostObjects/inputs/AudioRecorderHostObject.h b/packages/react-native-audio-api/common/cpp/audioapi/HostObjects/inputs/AudioRecorderHostObject.h index db31596be..6fda9690e 100644 --- a/packages/react-native-audio-api/common/cpp/audioapi/HostObjects/inputs/AudioRecorderHostObject.h +++ b/packages/react-native-audio-api/common/cpp/audioapi/HostObjects/inputs/AudioRecorderHostObject.h @@ -41,6 +41,8 @@ class AudioRecorderHostObject : public HostObject { JSI_HOST_FUNCTION_DECL(getCurrentDuration); + JSI_PROPERTY_GETTER_DECL(inputLatency); + private: std::shared_ptr audioRecorder_; std::shared_ptr promiseVendor_; diff --git a/packages/react-native-audio-api/common/cpp/audioapi/core/AudioContext.cpp b/packages/react-native-audio-api/common/cpp/audioapi/core/AudioContext.cpp index ad35c1bfe..95b43596a 100644 --- a/packages/react-native-audio-api/common/cpp/audioapi/core/AudioContext.cpp +++ b/packages/react-native-audio-api/common/cpp/audioapi/core/AudioContext.cpp @@ -135,4 +135,20 @@ bool AudioContext::isDriverRunning() const { return audioPlayer_->isRunning(); } +double AudioContext::getBaseLatency() const { + if (audioPlayer_ == nullptr) { + return 0.0; + } + + return audioPlayer_->getBaseLatency(); +} + +double AudioContext::getOutputLatency() const { + if (audioPlayer_ == nullptr) { + return 0.0; + } + + return audioPlayer_->getOutputLatency(); +} + } // namespace audioapi diff --git a/packages/react-native-audio-api/common/cpp/audioapi/core/AudioContext.h b/packages/react-native-audio-api/common/cpp/audioapi/core/AudioContext.h index 2dbe5c312..2eaf63f8f 100644 --- a/packages/react-native-audio-api/common/cpp/audioapi/core/AudioContext.h +++ b/packages/react-native-audio-api/common/cpp/audioapi/core/AudioContext.h @@ -35,6 +35,14 @@ class AudioContext : public BaseAudioContext { /// @note This method must be called before the audio context can be used for processing audio. void initialize(const AudioDestinationNode *destination) final; + /// @brief Returns the base latency of the audio context in seconds. + /// @returns The base latency in seconds. + [[nodiscard]] double getBaseLatency() const; + + /// @brief Returns the output latency of the audio context in seconds. + /// @returns The output latency in seconds. + [[nodiscard]] double getOutputLatency() const; + private: #ifdef RN_AUDIO_API_NODE std::shared_ptr audioPlayer_; diff --git a/packages/react-native-audio-api/common/cpp/audioapi/core/inputs/AudioRecorder.h b/packages/react-native-audio-api/common/cpp/audioapi/core/inputs/AudioRecorder.h index 534d8e7ee..295df0b79 100644 --- a/packages/react-native-audio-api/common/cpp/audioapi/core/inputs/AudioRecorder.h +++ b/packages/react-native-audio-api/common/cpp/audioapi/core/inputs/AudioRecorder.h @@ -61,6 +61,8 @@ class AudioRecorder { virtual bool isPaused() const = 0; virtual bool isIdle() const = 0; + [[nodiscard]] virtual double getInputLatency() const = 0; + protected: bool wantsCallback() const; bool wantsFileOutput() const; @@ -79,6 +81,7 @@ class AudioRecorder { mutable std::mutex fileWriterMutex_; std::mutex errorCallbackMutex_; mutable std::mutex adapterNodeMutex_; + mutable std::recursive_mutex streamMutex_; std::atomic errorCallbackId_{0}; diff --git a/packages/react-native-audio-api/common/cpp/test/src/utils/WsolaTimeStretcherTest.cpp b/packages/react-native-audio-api/common/cpp/test/src/utils/WsolaTimeStretcherTest.cpp index 59bc14524..48e84e7ce 100644 --- a/packages/react-native-audio-api/common/cpp/test/src/utils/WsolaTimeStretcherTest.cpp +++ b/packages/react-native-audio-api/common/cpp/test/src/utils/WsolaTimeStretcherTest.cpp @@ -1,5 +1,5 @@ #include -#include +#include #include #include diff --git a/packages/react-native-audio-api/ios/audioapi/ios/core/IOSAudioPlayer.h b/packages/react-native-audio-api/ios/audioapi/ios/core/IOSAudioPlayer.h index f0dda71b6..4e3cb8e97 100644 --- a/packages/react-native-audio-api/ios/audioapi/ios/core/IOSAudioPlayer.h +++ b/packages/react-native-audio-api/ios/audioapi/ios/core/IOSAudioPlayer.h @@ -33,6 +33,9 @@ class IOSAudioPlayer { bool isRunning() const; + [[nodiscard]] double getBaseLatency() const; + [[nodiscard]] double getOutputLatency() const; + private: void clearPendingSaved(); /// Audio-thread only. Always pulls the graph in steps of RENDER_QUANTUM_SIZE; if the system @@ -42,6 +45,7 @@ class IOSAudioPlayer { std::shared_ptr audioBuffer_; NativeAudioPlayer *audioPlayer_; + float sampleRate_; std::function renderAudio_; std::atomic ¤tRenders_; int channelCount_; diff --git a/packages/react-native-audio-api/ios/audioapi/ios/core/IOSAudioPlayer.mm b/packages/react-native-audio-api/ios/audioapi/ios/core/IOSAudioPlayer.mm index 85f07aeb2..cfd860e3e 100644 --- a/packages/react-native-audio-api/ios/audioapi/ios/core/IOSAudioPlayer.mm +++ b/packages/react-native-audio-api/ios/audioapi/ios/core/IOSAudioPlayer.mm @@ -8,6 +8,7 @@ #include #include #include +#include #include namespace audioapi { @@ -20,6 +21,7 @@ : audioBuffer_(nullptr), audioPlayer_(nullptr), renderAudio_(renderAudio), + sampleRate_(sampleRate), currentRenders_(currentRenders), channelCount_(channelCount), isRunning_(false), @@ -176,4 +178,23 @@ audioBuffer_ = nullptr; } +double IOSAudioPlayer::getBaseLatency() const +{ + if (!isRunning()) { + return 0.0; + } + + return static_cast(RENDER_QUANTUM_SIZE) / static_cast(sampleRate_); +} + +double IOSAudioPlayer::getOutputLatency() const +{ + if (!isRunning()) { + return 0.0; + } + + AudioSessionManager *sessionManager = [AudioSessionManager sharedInstance]; + return [sessionManager outputLatencySeconds] + [sessionManager ioBufferDurationSeconds]; +} + } // namespace audioapi diff --git a/packages/react-native-audio-api/ios/audioapi/ios/core/IOSAudioRecorder.h b/packages/react-native-audio-api/ios/audioapi/ios/core/IOSAudioRecorder.h index 0d6631518..3ecfa37f2 100644 --- a/packages/react-native-audio-api/ios/audioapi/ios/core/IOSAudioRecorder.h +++ b/packages/react-native-audio-api/ios/audioapi/ios/core/IOSAudioRecorder.h @@ -13,6 +13,8 @@ typedef struct objc_object NativeAudioRecorder; #include #include +#include +#include #include #include #include @@ -54,6 +56,8 @@ class IOSAudioRecorder : public AudioRecorder { uint64_t callbackId) override; void clearOnAudioReadyCallback() override; + [[nodiscard]] double getInputLatency() const override; + protected: NativeAudioRecorder *nativeRecorder_; @@ -65,6 +69,9 @@ class IOSAudioRecorder : public AudioRecorder { const std::string &fileNameOverride = ""); std::vector recordingSegmentPaths_; + std::atomic streamSampleRate_{0.0f}; + /// Updated on the audio thread from each input callback `numFrames`. + std::atomic lastCallbackFrameCount_{0}; }; } // namespace audioapi diff --git a/packages/react-native-audio-api/ios/audioapi/ios/core/IOSAudioRecorder.mm b/packages/react-native-audio-api/ios/audioapi/ios/core/IOSAudioRecorder.mm index d434a171b..2f1bf413a 100644 --- a/packages/react-native-audio-api/ios/audioapi/ios/core/IOSAudioRecorder.mm +++ b/packages/react-native-audio-api/ios/audioapi/ios/core/IOSAudioRecorder.mm @@ -108,6 +108,10 @@ static void cleanupStartedRecorder( : AudioRecorder(audioEventHandlerRegistry) { AudioReceiverBlock receiverBlock = ^(const AudioBufferList *inputBuffer, int numFrames) { + if (numFrames > 0) { + lastCallbackFrameCount_.store(numFrames, std::memory_order_release); + } + if (usesFileOutput()) { if (auto lock = Locker::tryLock(fileWriterMutex_)) { fileWriter_->writeAudioData(inputBuffer, numFrames); @@ -228,6 +232,8 @@ static void cleanupStartedRecorder( // Estimate the maximum input buffer lengths that can be expected from the sink node size_t maxInputBufferLength = [nativeRecorder_ getResolvedBufferSize]; + streamSampleRate_ = static_cast(recorderFormatSampleRate(inputFormat)); + lastCallbackFrameCount_.store(0, std::memory_order_release); bool fileWasOpened = false; if (wantsFileOutput()) { @@ -310,6 +316,8 @@ static void cleanupStartedRecorder( [nativeRecorder_ setInputArmed:false]; state_.store(RecorderState::Idle, std::memory_order_release); + lastCallbackFrameCount_.store(0, std::memory_order_release); + streamSampleRate_ = 0.0f; [nativeRecorder_ stop]; hadFileOutput = usesFileOutput(); @@ -610,6 +618,25 @@ static void cleanupStartedRecorder( return Result::Ok(None); } +double IOSAudioRecorder::getInputLatency() const +{ + if (isIdle()) { + return 0.0; + } + + AudioSessionManager *sessionManager = [AudioSessionManager sharedInstance]; + + double baseLatency = 0.0; + const int32_t callbackFrames = lastCallbackFrameCount_.load(std::memory_order_acquire); + if (callbackFrames > 0 && streamSampleRate_ > 0.0f) { + baseLatency = static_cast(callbackFrames) / static_cast(streamSampleRate_); + } else { + baseLatency = [sessionManager ioBufferDurationSeconds]; + } + + return baseLatency + [sessionManager inputLatencySeconds]; +} + /// @brief Clears the audio data callback. /// If the recorder is currently active, it will stop invoking the callback immediately. /// This method should be called from the JS thread only. diff --git a/packages/react-native-audio-api/ios/audioapi/ios/system/AudioSessionManager.h b/packages/react-native-audio-api/ios/audioapi/ios/system/AudioSessionManager.h index e3ff5bcec..472fcce70 100644 --- a/packages/react-native-audio-api/ios/audioapi/ios/system/AudioSessionManager.h +++ b/packages/react-native-audio-api/ios/audioapi/ios/system/AudioSessionManager.h @@ -56,4 +56,8 @@ resolve:(RCTPromiseResolveBlock)resolve reject:(RCTPromiseRejectBlock)reject; +- (double)outputLatencySeconds; +- (double)inputLatencySeconds; +- (double)ioBufferDurationSeconds; + @end diff --git a/packages/react-native-audio-api/ios/audioapi/ios/system/AudioSessionManager.mm b/packages/react-native-audio-api/ios/audioapi/ios/system/AudioSessionManager.mm index d11583738..31262c90e 100644 --- a/packages/react-native-audio-api/ios/audioapi/ios/system/AudioSessionManager.mm +++ b/packages/react-native-audio-api/ios/audioapi/ios/system/AudioSessionManager.mm @@ -560,4 +560,20 @@ - (AVAudioSessionCategoryOptions)optionsFromArray:(NSArray *)optionsArray return options; } + +- (double)outputLatencySeconds +{ + return self.audioSession.outputLatency; +} + +- (double)inputLatencySeconds +{ + return self.audioSession.inputLatency; +} + +- (double)ioBufferDurationSeconds +{ + return self.audioSession.IOBufferDuration; +} + @end diff --git a/packages/react-native-audio-api/src/core/AudioContext.ts b/packages/react-native-audio-api/src/core/AudioContext.ts index 5e3b7feee..9f7b4fb62 100644 --- a/packages/react-native-audio-api/src/core/AudioContext.ts +++ b/packages/react-native-audio-api/src/core/AudioContext.ts @@ -19,6 +19,14 @@ export default class AudioContext extends BaseAudioContext { ); } + public get baseLatency(): number { + return (this.context as IAudioContext).baseLatency; + } + + public get outputLatency(): number { + return (this.context as IAudioContext).outputLatency; + } + async close(): Promise { return (this.context as IAudioContext).close(); } diff --git a/packages/react-native-audio-api/src/core/AudioRecorder.ts b/packages/react-native-audio-api/src/core/AudioRecorder.ts index de44b981f..e8f5db8a9 100644 --- a/packages/react-native-audio-api/src/core/AudioRecorder.ts +++ b/packages/react-native-audio-api/src/core/AudioRecorder.ts @@ -186,6 +186,10 @@ export default class AudioRecorder { return this.recorder.getCurrentDuration(); } + get inputLatency(): number { + return this.recorder.inputLatency; + } + onError(callback: (error: OnRecorderErrorEventType) => void): void { if (this.onErrorSubscription) { this.recorder.clearOnError(); diff --git a/packages/react-native-audio-api/src/jsi-interfaces.ts b/packages/react-native-audio-api/src/jsi-interfaces.ts index 32b139aa3..4ccd741a5 100644 --- a/packages/react-native-audio-api/src/jsi-interfaces.ts +++ b/packages/react-native-audio-api/src/jsi-interfaces.ts @@ -70,6 +70,8 @@ export interface IBaseAudioContext { } export interface IAudioContext extends IBaseAudioContext { + readonly baseLatency: number; + readonly outputLatency: number; createMediaElementSource: ( mediaElement: IAudioFileSourceNode ) => IMediaElementAudioSourceNode; @@ -324,6 +326,8 @@ export interface IAudioRecorder { getCurrentDuration: () => number; getFilePath: () => string | null; + + readonly inputLatency: number; } export interface IAudioDecoder { diff --git a/packages/react-native-audio-api/src/mock/index.ts b/packages/react-native-audio-api/src/mock/index.ts index 8b556e705..72a6f64f4 100644 --- a/packages/react-native-audio-api/src/mock/index.ts +++ b/packages/react-native-audio-api/src/mock/index.ts @@ -559,6 +559,10 @@ class BaseAudioContextMock { return this._state; } + get baseLatency(): number { + return 0.005; + } + createBuffer( numberOfChannels: number, length: number, @@ -653,6 +657,10 @@ class AudioContextMock extends BaseAudioContextMock { super(options); } + get outputLatency(): number { + return 0.01; + } + close(): Promise { this._state = 'closed'; return Promise.resolve(); @@ -796,6 +804,10 @@ class AudioRecorderMock { return this._currentDuration; } + getInputLatency(): number { + return 0.01; + } + onError( callback: (error: Record | undefined) => void ): void { diff --git a/packages/react-native-audio-api/src/web-core/AudioContext.web.ts b/packages/react-native-audio-api/src/web-core/AudioContext.web.ts index 7638fbf1f..2b3de8cf1 100644 --- a/packages/react-native-audio-api/src/web-core/AudioContext.web.ts +++ b/packages/react-native-audio-api/src/web-core/AudioContext.web.ts @@ -46,6 +46,14 @@ export default class AudioContext implements BaseAudioContext { return this.context.state as ContextState; } + public get baseLatency(): number { + return this.context.baseLatency; + } + + public get outputLatency(): number { + return this.context.outputLatency; + } + createOscillator(): OscillatorNode { return new OscillatorNode(this); } diff --git a/packages/react-native-audio-api/tests/mock.test.ts b/packages/react-native-audio-api/tests/mock.test.ts index d21936bba..166a91b3d 100644 --- a/packages/react-native-audio-api/tests/mock.test.ts +++ b/packages/react-native-audio-api/tests/mock.test.ts @@ -9,6 +9,8 @@ describe('React Native Audio API Mocks', () => { expect(context.sampleRate).toBe(44100); expect(context.currentTime).toBe(0); expect(context.state).toBe('running'); + expect(context.baseLatency).toBe(0.005); + expect(context.outputLatency).toBe(0.01); expect(context.destination).toBeInstanceOf(MockAPI.AudioDestinationNode); }); @@ -42,6 +44,7 @@ describe('React Native Audio API Mocks', () => { expect(context.sampleRate).toBe(44100); expect(context.length).toBe(44100); + expect(context.baseLatency).toBe(0.005); }); it('should start rendering and return an AudioBuffer', async () => { @@ -215,6 +218,7 @@ describe('React Native Audio API Mocks', () => { expect(recorder.isRecording()).toBe(false); expect(recorder.isPaused()).toBe(false); expect(recorder.getCurrentDuration()).toBe(0); + expect(recorder.getInputLatency()).toBe(0.01); expect(recorder.options).toBeNull(); });