From 32fc5be7f3c511a0da9ccc480366cbf774ca85a9 Mon Sep 17 00:00:00 2001 From: Jordan Ritter Date: Wed, 29 Jul 2026 18:52:10 -0700 Subject: [PATCH 1/2] feat: support OpenAI transcription replay --- src/recorder.ts | 50 +++-- src/server.ts | 6 +- src/stream-collapse.ts | 50 +++++ src/transcription.ts | 63 ++++++- src/types.ts | 2 + src/ws-realtime.ts | 410 +++++++++++++++++++++++++++++++++++------ 6 files changed, 507 insertions(+), 74 deletions(-) diff --git a/src/recorder.ts b/src/recorder.ts index eb299688..d228d944 100644 --- a/src/recorder.ts +++ b/src/recorder.ts @@ -615,7 +615,9 @@ export async function proxyAndRecord( // A single Gemini turn can interleave audio with a functionCall and/or // text/thought parts; preserve those companion modalities so the tool call // / content / reasoning are not silently dropped when audio is present. - if (collapsed.audioB64) { + if (collapsed.transcription) { + fixtureResponse = { transcription: collapsed.transcription }; + } else if (collapsed.audioB64) { const audioToolCallsSpread = collapsed.toolCalls && collapsed.toolCalls.length > 0 ? { @@ -777,7 +779,12 @@ export async function proxyAndRecord( // NOTE: base64 embeddings are decoded unconditionally inside // buildFixtureResponse regardless of the request's `encoding_format`, so // there is no need to re-parse it here — it was a dead param. - fixtureResponse = buildFixtureResponse(parsedResponse, upstreamStatus, defaults.logger); + fixtureResponse = buildFixtureResponse( + parsedResponse, + upstreamStatus, + defaults.logger, + request, + ); } } @@ -1435,7 +1442,12 @@ function toToolCallArguments(raw: unknown): string { * Detect the response format from the parsed upstream JSON and convert * it into an aimock FixtureResponse. */ -function buildFixtureResponse(parsed: unknown, status: number, logger?: Logger): FixtureResponse { +function buildFixtureResponse( + parsed: unknown, + status: number, + logger?: Logger, + request?: ChatCompletionRequest, +): FixtureResponse { if (parsed === null || parsed === undefined) { // Raw / unparseable response — save as error return { @@ -1536,16 +1548,17 @@ function buildFixtureResponse(parsed: unknown, status: number, logger?: Logger): return { images }; } - // OpenAI transcription: { text: "...", ... } - // Tightened: a bare `text` string alongside a single incidental `language` or - // `duration` field is too weak — many non-transcription payloads carry a - // `text` plus a `duration`-like number. Require an explicit - // `task: "transcribe"`, OR BOTH `language` and `duration` (the verbose - // transcription shape). Also reject anything carrying clear non-transcription - // markers (chat completions, events, etc.) so they route to their own branch. + // OpenAI transcription: { text: "...", ... }. Modern gpt-transcribe + // responses may be the minimal { text, languages?, usage? } shape, so trust + // that shape only on an audio transcription/translation request. Other + // endpoints still need the legacy markers to avoid misclassifying text APIs. + const isTranscriptionRequest = + request?._endpointType === "transcription" || request?._endpointType === "translation"; const looksLikeTranscription = typeof obj.text === "string" && - (obj.task === "transcribe" || (obj.language !== undefined && obj.duration !== undefined)) && + (isTranscriptionRequest || + obj.task === "transcribe" || + (obj.language !== undefined && obj.duration !== undefined)) && !("choices" in obj) && !("candidates" in obj) && !("object" in obj) && @@ -1556,7 +1569,22 @@ function buildFixtureResponse(parsed: unknown, status: number, logger?: Logger): transcription: { text: obj.text as string, ...(obj.language ? { language: String(obj.language) } : {}), + ...(Array.isArray(obj.languages) + ? { + languages: obj.languages + .filter( + (language): language is Record => + typeof language === "object" && + language !== null && + typeof language.code === "string", + ) + .map((language) => ({ code: language.code as string })), + } + : {}), ...(obj.duration !== undefined ? { duration: Number(obj.duration) } : {}), + ...(obj.usage && typeof obj.usage === "object" + ? { usage: obj.usage as Record } + : {}), ...(Array.isArray(obj.words) ? { words: obj.words } : {}), ...(Array.isArray(obj.segments) ? { segments: obj.segments } : {}), }, diff --git a/src/server.ts b/src/server.ts index 2e9bb4b8..5859ec72 100644 --- a/src/server.ts +++ b/src/server.ts @@ -2914,10 +2914,14 @@ export async function createServer( upgradeHeaders: req.headers, }); } else if (pathname === REALTIME_PATH) { - const model = parsedUrl.searchParams.get("model") ?? "gpt-realtime-2"; + const transcriptionIntent = parsedUrl.searchParams.get("intent") === "transcription"; + const model = transcriptionIntent + ? "gpt-transcribe" + : (parsedUrl.searchParams.get("model") ?? "gpt-realtime-2"); handleWebSocketRealtime(ws, fixtures, journal, { ...defaults, model, + transcriptionIntent, testId: wsTestId, upgradeHeaders: req.headers, }); diff --git a/src/stream-collapse.ts b/src/stream-collapse.ts index b9a4d292..7d48e183 100644 --- a/src/stream-collapse.ts +++ b/src/stream-collapse.ts @@ -18,6 +18,11 @@ import { isHarmonyContent, parseHarmonyContent } from "./harmony.js"; export interface CollapseResult { content?: string; + transcription?: { + text: string; + languages?: Array<{ code: string }>; + usage?: Record; + }; reasoning?: string; /** * The real cryptographic `signature` value captured from an Anthropic @@ -280,6 +285,10 @@ function extractSSEData(lines: string[]): string | undefined { export function collapseOpenAISSE(body: string): CollapseResult { const lines = splitSSEEvents(body); let content = ""; + let transcript = ""; + let transcriptSeen = false; + let transcriptLanguages: Array<{ code: string }> | undefined; + let transcriptUsage: Record | undefined; let reasoning = ""; const webSearchQueries: string[] = []; let droppedChunks = 0; @@ -321,6 +330,29 @@ export function collapseOpenAISSE(body: string): CollapseResult { } // Responses API reasoning events + if (parsed.type === "transcript.text.delta" && typeof parsed.delta === "string") { + transcript += parsed.delta; + transcriptSeen = true; + continue; + } + if (parsed.type === "transcript.text.done" && typeof parsed.text === "string") { + transcript = parsed.text; + transcriptSeen = true; + if (Array.isArray(parsed.languages)) { + transcriptLanguages = parsed.languages + .filter( + (language): language is Record => + typeof language === "object" && + language !== null && + typeof language.code === "string", + ) + .map((language) => ({ code: language.code as string })); + } + if (parsed.usage && typeof parsed.usage === "object") { + transcriptUsage = parsed.usage as Record; + } + continue; + } if ( parsed.type === "response.reasoning_summary_text.delta" && typeof parsed.delta === "string" @@ -472,6 +504,15 @@ export function collapseOpenAISSE(body: string): CollapseResult { ...(tc.id ? { id: tc.id } : {}), })); return { + ...(transcriptSeen + ? { + transcription: { + text: transcript, + ...(transcriptLanguages ? { languages: transcriptLanguages } : {}), + ...(transcriptUsage ? { usage: transcriptUsage } : {}), + }, + } + : {}), ...(blocks ? { blocks } : {}), ...(content ? { content } : {}), // Fallback-only: harmonyToolCalls are populated ONLY in the @@ -491,6 +532,15 @@ export function collapseOpenAISSE(body: string): CollapseResult { } return { + ...(transcriptSeen + ? { + transcription: { + text: transcript, + ...(transcriptLanguages ? { languages: transcriptLanguages } : {}), + ...(transcriptUsage ? { usage: transcriptUsage } : {}), + }, + } + : {}), content, ...(reasoning ? { reasoning } : {}), ...(webSearchQueries.length > 0 ? { webSearches: webSearchQueries } : {}), diff --git a/src/transcription.ts b/src/transcription.ts index 5426511a..34b37e96 100644 --- a/src/transcription.ts +++ b/src/transcription.ts @@ -14,10 +14,11 @@ import { strictNoMatchLogLine, } from "./helpers.js"; import { matchFixtureDiagnostic } from "./router.js"; -import { writeErrorResponse } from "./sse-writer.js"; +import { calculateDelay, delay, writeErrorResponse } from "./sse-writer.js"; import type { Journal } from "./journal.js"; import { applyChaos } from "./chaos.js"; import { proxyAndRecord } from "./recorder.js"; +import { createInterruptionSignal } from "./interruption.js"; /** * Extract the multipart boundary string from a Content-Type header. @@ -100,6 +101,7 @@ export async function handleTranscription( const model = extractFormField(raw, "model", boundary) ?? "whisper-1"; const responseFormat = extractFormField(raw, "response_format", boundary) ?? "json"; + const stream = extractFormField(raw, "stream", boundary) === "true"; const syntheticReq: ChatCompletionRequest = { model, @@ -254,7 +256,7 @@ export async function handleTranscription( return; } - journal.add({ + const journalEntry = journal.add({ method, path, headers: flattenHeaders(req.headers), @@ -263,6 +265,55 @@ export async function handleTranscription( }); const t = response.transcription; + + // Only the transcription endpoint streams modern models. Whisper-1 ignores + // `stream=true`, and translations continue to return their JSON response. + if (endpointType === "transcription" && stream && model !== "whisper-1") { + const done = { + type: "transcript.text.done", + text: t.text, + ...(t.languages !== undefined ? { languages: t.languages } : {}), + ...(t.usage !== undefined ? { usage: t.usage } : {}), + }; + const latency = fixture.latency ?? defaults.latency; + const chunkSize = Math.max(1, fixture.chunkSize ?? defaults.chunkSize); + const replaySpeed = fixture.replaySpeed ?? defaults.replaySpeed; + res.writeHead(200, { + "Content-Type": "text/event-stream", + "Cache-Control": "no-cache", + Connection: "keep-alive", + }); + let chunkIndex = 0; + const interruption = createInterruptionSignal(fixture); + for (let index = 0; index < t.text.length; index += chunkSize) { + const chunkDelay = calculateDelay( + chunkIndex, + fixture.streamingProfile, + latency, + fixture.recordedTimings, + replaySpeed, + ); + if (chunkDelay > 0) await delay(chunkDelay, interruption?.signal); + if (interruption?.signal.aborted) break; + res.write( + `data: ${JSON.stringify({ type: "transcript.text.delta", delta: t.text.slice(index, index + chunkSize) })}\n\n`, + ); + interruption?.tick(); + chunkIndex++; + } + if (interruption?.signal.aborted) { + journalEntry.response.interrupted = true; + journalEntry.response.interruptReason = interruption.reason(); + interruption.cleanup(); + res.destroy(); + return; + } + res.write(`data: ${JSON.stringify(done)}\n\n`); + res.end(); + interruption?.cleanup(); + return; + } + const useVerbose = responseFormat === "verbose_json" || t.words != null || t.segments != null; if (useVerbose) { @@ -282,6 +333,12 @@ export async function handleTranscription( res.end(JSON.stringify(verboseBody)); } else { res.writeHead(200, { "Content-Type": "application/json" }); - res.end(JSON.stringify({ text: t.text })); + res.end( + JSON.stringify({ + text: t.text, + ...(t.languages !== undefined ? { languages: t.languages } : {}), + ...(t.usage !== undefined ? { usage: t.usage } : {}), + }), + ); } } diff --git a/src/types.ts b/src/types.ts index ab8f88ca..6de282c9 100644 --- a/src/types.ts +++ b/src/types.ts @@ -285,7 +285,9 @@ export interface TranscriptionResponse { transcription: { text: string; language?: string; + languages?: Array<{ code: string }>; duration?: number; + usage?: Record; words?: Array<{ word: string; start: number; end: number }>; segments?: Array<{ id: number; text: string; start: number; end: number }>; }; diff --git a/src/ws-realtime.ts b/src/ws-realtime.ts index 4b1ad1e3..dac8ffdd 100644 --- a/src/ws-realtime.ts +++ b/src/ws-realtime.ts @@ -21,6 +21,7 @@ import { isTextResponse, isToolCallResponse, isContentWithToolCallsResponse, + isTranscriptionResponse, isErrorResponse, resolveFixtureBlocks, resolveResponse, @@ -62,10 +63,10 @@ interface SessionConfig { input_audio_format: string | null; output_audio_format: string | null; input_audio_noise_reduction: { type: string } | null; - input_audio_transcription: { model: string } | null; + input_audio_transcription: { model: string; language?: string; prompt?: string } | null; turn_detection: unknown | null; temperature: number; - type: "conversation" | "transcription" | "translation"; + type: "conversation" | "realtime" | "transcription" | "translation"; reasoning: { effort: string } | null; } @@ -81,6 +82,58 @@ interface RealtimeMessage { }; } +function isLiveTranscriptionModel(model: string): boolean { + return ( + model === "gpt-transcribe" || + model === "gpt-live-transcribe" || + model.startsWith("gpt-live-transcribe-") || + model === "gpt-4o-transcribe" || + model.startsWith("gpt-4o-mini-transcribe") || + model === "whisper-1" + ); +} + +function transcriptionModel(session: SessionConfig): string { + return session.input_audio_transcription?.model ?? session.model; +} + +function isLiveTranscriptionSession(session: SessionConfig): boolean { + return ( + isLiveTranscriptionModel(transcriptionModel(session)) && + (session.type === "transcription" || session.input_audio_transcription !== null) + ); +} + +function serializeSession(session: SessionConfig, sessionId?: string): Record { + return { + ...(sessionId ? { id: sessionId } : {}), + object: "realtime.session", + model: session.model, + expires_at: Math.floor(Date.now() / 1000) + 3600, + modalities: session.modalities, + instructions: session.instructions, + tools: session.tools, + tool_choice: "auto", + temperature: session.temperature, + max_response_output_tokens: "inf", + audio: { + input: { + format: session.input_audio_format ? { type: session.input_audio_format } : null, + noise_reduction: session.input_audio_noise_reduction, + transcription: session.input_audio_transcription, + turn_detection: session.turn_detection, + }, + output: { + format: session.output_audio_format ? { type: session.output_audio_format } : null, + voice: session.voice, + }, + }, + type: + session.type === "conversation" || session.type === "realtime" ? "realtime" : session.type, + reasoning: session.reasoning, + }; +} + // ─── Conversion helpers ───────────────────────────────────────────────────── export function realtimeItemsToMessages( @@ -259,10 +312,13 @@ function translateGAToBeta(event: Record): Record) }; if (session.audio && typeof session.audio === "object") { const audio = session.audio as Record; - session.voice = audio.voice; - session.input_audio_format = audio.input_audio_format; - session.output_audio_format = audio.output_audio_format; - session.input_audio_transcription = audio.input_audio_transcription; + const input = audio.input as Record | undefined; + const output = audio.output as Record | undefined; + session.voice = output?.voice; + session.input_audio_format = (input?.format as Record | undefined)?.type; + session.output_audio_format = (output?.format as Record | undefined)?.type; + session.input_audio_transcription = input?.transcription; + session.input_audio_noise_reduction = input?.noise_reduction; delete session.audio; } delete session.type; @@ -313,6 +369,7 @@ export function handleWebSocketRealtime( requestTransform?: (req: ChatCompletionRequest) => ChatCompletionRequest; testId?: string; upgradeHeaders?: import("node:http").IncomingHttpHeaders; + transcriptionIntent?: boolean; }, ): void { const { logger } = defaults; @@ -334,7 +391,7 @@ export function handleWebSocketRealtime( input_audio_transcription: null, turn_detection: null, temperature: 0.8, - type: "conversation", + type: defaults.transcriptionIntent ? "transcription" : "conversation", reasoning: null, }; @@ -345,28 +402,7 @@ export function handleWebSocketRealtime( ws, { type: "session.created", - session: { - id: sessionId, - object: "realtime.session", - model: session.model, - expires_at: Math.floor(Date.now() / 1000) + 3600, - modalities: session.modalities, - instructions: session.instructions, - tools: session.tools, - tool_choice: "auto", - temperature: session.temperature, - max_response_output_tokens: "inf", - audio: { - voice: session.voice, - input_audio_format: session.input_audio_format, - output_audio_format: session.output_audio_format, - input_audio_noise_reduction: session.input_audio_noise_reduction, - input_audio_transcription: session.input_audio_transcription, - }, - turn_detection: session.turn_detection, - type: session.type, - reasoning: session.reasoning, - }, + session: serializeSession(session, sessionId), }, isBeta, ); @@ -414,6 +450,7 @@ async function processMessage( requestTransform?: (req: ChatCompletionRequest) => ChatCompletionRequest; testId?: string; upgradeHeaders?: import("node:http").IncomingHttpHeaders; + transcriptionIntent?: boolean; }, session: SessionConfig, conversationItems: RealtimeItem[], @@ -436,13 +473,24 @@ async function processMessage( const msgType = parsed.type; - // ── session.update ──────────────────────────────────────────────────── - if (msgType === "session.update") { + // ── session.update / transcription_session.update ───────────────────── + const isTranscriptionSessionUpdate = msgType === "transcription_session.update"; + if (msgType === "session.update" || isTranscriptionSessionUpdate) { + if (isTranscriptionSessionUpdate && !defaults.transcriptionIntent) { + buildErrorRealtimeEvent( + ws, + "transcription_session.update requires a realtime connection with intent=transcription", + isBeta, + "invalid_request_error", + "invalid_session_config", + ); + return; + } if (parsed.session) { const s = parsed.session; // Validate session.type value before applying any mutations - const validTypes = new Set(["conversation", "transcription", "translation"]); + const validTypes = new Set(["conversation", "realtime", "transcription", "translation"]); if ((s as Record).type !== undefined) { if (!validTypes.has((s as Record).type as string)) { sendEvent( @@ -464,10 +512,22 @@ async function processMessage( // Capture full pre-mutation snapshot for rollback on validation failure const prevSession = { ...session }; + if (s.model !== undefined && s.model !== session.model) { + buildErrorRealtimeEvent( + ws, + "The session model is selected when the connection is established and cannot be changed", + isBeta, + "invalid_request_error", + "invalid_session_config", + ); + return; + } + + if (isTranscriptionSessionUpdate) session.type = "transcription"; + if (s.instructions !== undefined) session.instructions = s.instructions; if (s.tools !== undefined) session.tools = s.tools; if (s.modalities !== undefined) session.modalities = s.modalities; - if (s.model !== undefined) session.model = s.model; if (s.temperature !== undefined) session.temperature = s.temperature; if ((s as Record).type !== undefined) session.type = (s as Record).type as SessionConfig["type"]; @@ -486,7 +546,27 @@ async function processMessage( if (audio.input_audio_transcription !== undefined) session.input_audio_transcription = audio.input_audio_transcription as { model: string; + language?: string; + prompt?: string; } | null; + // Current Realtime session shape nests input transcription under + // session.audio.input.transcription. + if (audio.input && typeof audio.input === "object") { + const input = audio.input as Record; + if (input.transcription !== undefined) + session.input_audio_transcription = input.transcription as { + model: string; + language?: string; + prompt?: string; + } | null; + if (input.noise_reduction !== undefined) + session.input_audio_noise_reduction = input.noise_reduction as { type: string } | null; + if (input.turn_detection !== undefined) session.turn_detection = input.turn_detection; + if (input.format && typeof input.format === "object") { + const format = input.format as Record; + if (typeof format.type === "string") session.input_audio_format = format.type; + } + } } // Beta flat fields (backward compat) if (s.voice !== undefined) session.voice = s.voice; @@ -517,14 +597,19 @@ async function processMessage( "gpt-realtime-translate", ]); - if (session.type === "transcription" && !transcriptionModels.has(session.model)) { + const candidateTranscriptionModel = transcriptionModel(session); + if ( + session.type === "transcription" && + !transcriptionModels.has(candidateTranscriptionModel) && + !isLiveTranscriptionSession(session) + ) { Object.assign(session, prevSession); sendEvent( ws, { type: "error", error: { - message: `Model ${s.model ?? prevSession.model} does not support session type transcription`, + message: `Model ${candidateTranscriptionModel} does not support session type transcription`, type: "invalid_request_error", code: "invalid_session_config", }, @@ -554,28 +639,8 @@ async function processMessage( sendEvent( ws, { - type: "session.updated", - session: { - object: "realtime.session", - model: session.model, - expires_at: Math.floor(Date.now() / 1000) + 3600, - modalities: session.modalities, - instructions: session.instructions, - tools: session.tools, - tool_choice: "auto", - temperature: session.temperature, - max_response_output_tokens: "inf", - audio: { - voice: session.voice, - input_audio_format: session.input_audio_format, - output_audio_format: session.output_audio_format, - input_audio_noise_reduction: session.input_audio_noise_reduction, - input_audio_transcription: session.input_audio_transcription, - }, - turn_detection: session.turn_detection, - type: session.type, - reasoning: session.reasoning, - }, + type: isTranscriptionSessionUpdate ? "transcription_session.updated" : "session.updated", + session: serializeSession(session), }, isBeta, ); @@ -630,8 +695,13 @@ async function processMessage( // ── input_audio_buffer.commit ────────────────────────────────────── if (msgType === "input_audio_buffer.commit") { sendEvent(ws, { type: "input_audio_buffer.committed" }, isBeta); - // In transcription/translation mode, add a placeholder user item - if (session.type === "transcription" || session.type === "translation") { + // Transcription sessions can be configured through the documented input + // transcription field even when the connection model itself is realtime. + if ( + session.type === "transcription" || + session.type === "translation" || + isLiveTranscriptionSession(session) + ) { const audioItem: RealtimeItem = { type: "message", id: realtimeId("item"), @@ -647,6 +717,17 @@ async function processMessage( }, isBeta, ); + if (isLiveTranscriptionSession(session)) { + await emitLiveTranscriptionEvents( + ws, + fixtures, + journal, + defaults, + session, + audioItem, + isBeta, + ); + } } return; } @@ -666,6 +747,217 @@ async function processMessage( // Unknown message type — ignore silently (matches OpenAI behavior) } +async function emitLiveTranscriptionEvents( + ws: WebSocketConnection, + fixtures: Fixture[], + journal: Journal, + defaults: { + latency: number; + chunkSize: number; + replaySpeed?: number; + model: string; + logger: Logger; + strict?: boolean; + requestTransform?: (req: ChatCompletionRequest) => ChatCompletionRequest; + testId?: string; + upgradeHeaders?: import("node:http").IncomingHttpHeaders; + }, + session: SessionConfig, + audioItem: RealtimeItem, + isBeta: boolean, +): Promise { + const realtimeContextHeader = defaults.upgradeHeaders?.["x-aimock-context"]; + const realtimeContext = + typeof realtimeContextHeader === "string" + ? realtimeContextHeader + : Array.isArray(realtimeContextHeader) && realtimeContextHeader.length > 0 + ? realtimeContextHeader[0] + : undefined; + const request: ChatCompletionRequest = { + model: transcriptionModel(session), + messages: realtimeItemsToMessages([audioItem], undefined, defaults.logger), + _endpointType: "realtime-transcription", + _context: realtimeContext, + }; + const testId = defaults.testId ?? DEFAULT_TEST_ID; + const { fixture, skippedBySequenceOrTurn } = matchFixtureDiagnostic( + fixtures, + request, + journal.getFixtureMatchCountsForTest(testId), + defaults.requestTransform, + ); + const itemId = audioItem.id ?? realtimeId("item"); + + if (!fixture) { + if (resolveStrictMode(defaults.strict, defaults.upgradeHeaders)) { + const strictMessage = strictNoMatchMessage(skippedBySequenceOrTurn); + defaults.logger.error(strictNoMatchLogLine("WS", "/v1/realtime", skippedBySequenceOrTurn)); + journal.add({ + method: "WS", + path: "/v1/realtime", + headers: flattenHeaders(defaults.upgradeHeaders ?? {}), + body: request, + response: { + status: 503, + fixture: null, + ...strictOverrideField(defaults.strict, defaults.upgradeHeaders), + }, + }); + ws.close(1008, strictMessage); + return; + } + + journal.add({ + method: "WS", + path: "/v1/realtime", + headers: flattenHeaders(defaults.upgradeHeaders ?? {}), + body: request, + response: { + status: 404, + fixture: null, + ...strictOverrideField(defaults.strict, defaults.upgradeHeaders), + }, + }); + sendLiveTranscriptionFailure( + ws, + itemId, + { message: "No fixture matched", type: "invalid_request_error", code: "no_fixture_match" }, + isBeta, + ); + return; + } + journal.incrementFixtureMatchCount(fixture, fixtures, testId); + + const response = await resolveResponse(fixture, request); + if (isErrorResponse(response)) { + journal.add({ + method: "WS", + path: "/v1/realtime", + headers: flattenHeaders(defaults.upgradeHeaders ?? {}), + body: request, + response: { status: response.status ?? 500, fixture }, + }); + sendLiveTranscriptionFailure( + ws, + itemId, + { + message: response.error.message, + type: response.error.type ?? "server_error", + ...(response.error.code !== undefined && { code: response.error.code }), + }, + isBeta, + ); + return; + } + if (!isTranscriptionResponse(response)) { + journal.add({ + method: "WS", + path: "/v1/realtime", + headers: flattenHeaders(defaults.upgradeHeaders ?? {}), + body: request, + response: { status: 500, fixture }, + }); + sendLiveTranscriptionFailure( + ws, + itemId, + { message: "Fixture response is not a transcription type", type: "server_error" }, + isBeta, + ); + return; + } + + const transcript = response.transcription.text; + const usage = response.transcription.usage ?? { + type: "tokens", + input_tokens: 0, + output_tokens: 0, + total_tokens: 0, + }; + const journalEntry = journal.add({ + method: "WS", + path: "/v1/realtime", + headers: flattenHeaders(defaults.upgradeHeaders ?? {}), + body: request, + response: { status: 200, fixture }, + }); + const latency = fixture.latency ?? defaults.latency; + const chunkSize = Math.max(1, fixture.chunkSize ?? defaults.chunkSize); + const replaySpeed = fixture.replaySpeed ?? defaults.replaySpeed; + const interruption = createInterruptionSignal(fixture); + let chunkIndex = 0; + for (let index = 0; index < transcript.length; index += chunkSize) { + const chunkDelay = calculateDelay( + chunkIndex, + fixture.streamingProfile, + latency, + fixture.recordedTimings, + replaySpeed, + ); + if (chunkDelay > 0) await delay(chunkDelay, interruption?.signal); + if (interruption?.signal.aborted || ws.isClosed) { + if (interruption?.signal.aborted) { + journalEntry.response.interrupted = true; + journalEntry.response.interruptReason = interruption.reason(); + ws.destroy(); + } + interruption?.cleanup(); + return; + } + sendEvent( + ws, + { + type: "conversation.item.input_audio_transcription.delta", + item_id: itemId, + content_index: 0, + delta: transcript.slice(index, index + chunkSize), + }, + isBeta, + ); + interruption?.tick(); + chunkIndex++; + } + if (interruption?.signal.aborted) { + journalEntry.response.interrupted = true; + journalEntry.response.interruptReason = interruption.reason(); + ws.destroy(); + interruption.cleanup(); + return; + } + sendEvent( + ws, + { + type: "conversation.item.input_audio_transcription.completed", + item_id: itemId, + content_index: 0, + transcript, + usage, + ...(response.transcription.languages !== undefined + ? { languages: response.transcription.languages } + : {}), + }, + isBeta, + ); + interruption?.cleanup(); +} + +function sendLiveTranscriptionFailure( + ws: WebSocketConnection, + itemId: string, + error: { message: string; type: string; code?: string }, + isBeta: boolean, +): void { + sendEvent( + ws, + { + type: "conversation.item.input_audio_transcription.failed", + item_id: itemId, + content_index: 0, + error, + }, + isBeta, + ); +} + async function handleResponseCreate( ws: WebSocketConnection, fixtures: Fixture[], From 23aca199c441d64435558a27f6d94f4023e81300 Mon Sep 17 00:00:00 2001 From: Jordan Ritter Date: Wed, 29 Jul 2026 18:52:11 -0700 Subject: [PATCH 2/2] test: cover transcription replay contracts --- src/__tests__/multimedia-record.test.ts | 170 ++++++++++ src/__tests__/multimedia.test.ts | 235 +++++++++++++- src/__tests__/stream-collapse.test.ts | 10 + src/__tests__/ws-api-conformance.test.ts | 9 +- src/__tests__/ws-realtime.test.ts | 376 ++++++++++++++++++++--- 5 files changed, 755 insertions(+), 45 deletions(-) diff --git a/src/__tests__/multimedia-record.test.ts b/src/__tests__/multimedia-record.test.ts index 4d915ba1..314794b2 100644 --- a/src/__tests__/multimedia-record.test.ts +++ b/src/__tests__/multimedia-record.test.ts @@ -17,6 +17,7 @@ import * as fs from "node:fs"; import * as os from "node:os"; import * as path from "node:path"; import { proxyAndRecord } from "../recorder.js"; +import { createServer } from "../server.js"; import type { Fixture, RecordConfig, ChatCompletionRequest } from "../types.js"; import { Logger } from "../logger.js"; @@ -77,6 +78,21 @@ function makeTmpDir(): string { return fs.mkdtempSync(path.join(os.tmpdir(), "aimock-mm-record-")); } +async function requestStreamingTranscription(url: string): Promise { + const formData = new FormData(); + formData.append("file", new Blob(["audio"], { type: "audio/wav" }), "audio.wav"); + formData.append("model", "gpt-transcribe"); + formData.append("stream", "true"); + return fetch(`${url}/v1/audio/transcriptions`, { method: "POST", body: formData }); +} + +async function requestJsonTranscription(url: string): Promise { + const formData = new FormData(); + formData.append("file", new Blob(["audio"], { type: "audio/wav" }), "audio.wav"); + formData.append("model", "gpt-transcribe"); + return fetch(`${url}/v1/audio/transcriptions`, { method: "POST", body: formData }); +} + // --------------------------------------------------------------------------- // Tests: buildFixtureResponse detection via proxyAndRecord // --------------------------------------------------------------------------- @@ -207,6 +223,160 @@ describe("multimedia record: image response detection", () => { }); describe("multimedia record: transcription response detection", () => { + it("records upstream gpt-transcribe SSE and replays it through the local handler", async () => { + const fixturePath = makeTmpDir(); + const upstreamBody = + 'data: {"type":"transcript.text.delta","delta":"Recorded "}\n\n' + + 'data: {"type":"transcript.text.delta","delta":"stream"}\n\n' + + 'data: {"type":"transcript.text.done","text":"Recorded stream","languages":[{"code":"en"}],"usage":{"type":"tokens","input_tokens":4,"output_tokens":5,"total_tokens":9}}\n\n'; + const { server: upstream, url } = await createUpstream((_req, res) => { + res.writeHead(200, { "Content-Type": "text/event-stream" }); + res.end(upstreamBody); + }); + const fixtures: Fixture[] = []; + const recorder = await createServer(fixtures, { + port: 0, + record: { providers: { openai: url }, fixturePath }, + }); + + try { + const recorded = await requestStreamingTranscription(recorder.url); + expect(recorded.status).toBe(200); + expect(await recorded.text()).toBe(upstreamBody); + expect(fixtures).toHaveLength(1); + expect(fixtures[0].response).toEqual({ + transcription: { + text: "Recorded stream", + languages: [{ code: "en" }], + usage: { type: "tokens", input_tokens: 4, output_tokens: 5, total_tokens: 9 }, + }, + }); + + const savedFixture = JSON.parse( + fs.readFileSync( + path.join( + fixturePath, + fs.readdirSync(fixturePath).find((file) => file.endsWith(".json"))!, + ), + "utf8", + ), + ); + expect(savedFixture.fixtures[0].response).toEqual({ + transcription: { + text: "Recorded stream", + languages: [{ code: "en" }], + usage: { type: "tokens", input_tokens: 4, output_tokens: 5, total_tokens: 9 }, + }, + }); + + const replay = await requestStreamingTranscription(recorder.url); + expect(replay.status).toBe(200); + expect(await replay.text()).toBe( + 'data: {"type":"transcript.text.delta","delta":"Recorded stream"}\n\n' + + 'data: {"type":"transcript.text.done","text":"Recorded stream","languages":[{"code":"en"}],"usage":{"type":"tokens","input_tokens":4,"output_tokens":5,"total_tokens":9}}\n\n', + ); + } finally { + await closeServer(recorder.server); + await closeServer(upstream); + fs.rmSync(fixturePath, { recursive: true, force: true }); + } + }); + + it("records and replays the modern gpt-transcribe text languages and usage response", async () => { + const fixturePath = makeTmpDir(); + const { server, url } = await createUpstream((_req, res) => { + res.writeHead(200, { "Content-Type": "application/json" }); + res.end( + JSON.stringify({ + text: "Modern transcript", + languages: [{ code: "en" }, { code: "es" }], + usage: { type: "tokens", input_tokens: 4, output_tokens: 5, total_tokens: 9 }, + }), + ); + }); + + try { + const fixtures: Fixture[] = []; + const record: RecordConfig = { providers: { openai: url }, fixturePath }; + const logger = new Logger("silent"); + const request: ChatCompletionRequest = { + model: "gpt-transcribe", + messages: [], + _endpointType: "transcription", + }; + + const { req, res } = createMockReqRes("/v1/audio/transcriptions"); + await proxyAndRecord(req, res, request, "openai", "/v1/audio/transcriptions", fixtures, { + record, + logger, + }); + + const response = fixtures[0].response as { + transcription?: { + text: string; + languages?: Array<{ code: string }>; + usage?: Record; + }; + }; + expect(response.transcription).toEqual({ + text: "Modern transcript", + languages: [{ code: "en" }, { code: "es" }], + usage: { type: "tokens", input_tokens: 4, output_tokens: 5, total_tokens: 9 }, + }); + } finally { + await closeServer(server); + fs.rmSync(fixturePath, { recursive: true, force: true }); + } + }); + + it("records and locally replays JSON transcription languages and usage", async () => { + const fixturePath = makeTmpDir(); + const { server: upstream, url } = await createUpstream((_req, res) => { + res.writeHead(200, { "Content-Type": "application/json" }); + res.end( + JSON.stringify({ + text: "Recorded JSON transcript", + languages: [{ code: "en" }, { code: "fr" }], + usage: { type: "tokens", input_tokens: 3, output_tokens: 4, total_tokens: 7 }, + }), + ); + }); + const fixtures: Fixture[] = []; + const recorder = await createServer(fixtures, { + port: 0, + record: { providers: { openai: url }, fixturePath }, + }); + + try { + const recorded = await requestJsonTranscription(recorder.url); + expect(recorded.status).toBe(200); + expect(await recorded.json()).toEqual({ + text: "Recorded JSON transcript", + languages: [{ code: "en" }, { code: "fr" }], + usage: { type: "tokens", input_tokens: 3, output_tokens: 4, total_tokens: 7 }, + }); + expect(fixtures[0].response).toEqual({ + transcription: { + text: "Recorded JSON transcript", + languages: [{ code: "en" }, { code: "fr" }], + usage: { type: "tokens", input_tokens: 3, output_tokens: 4, total_tokens: 7 }, + }, + }); + + const replay = await requestJsonTranscription(recorder.url); + expect(replay.status).toBe(200); + expect(await replay.json()).toEqual({ + text: "Recorded JSON transcript", + languages: [{ code: "en" }, { code: "fr" }], + usage: { type: "tokens", input_tokens: 3, output_tokens: 4, total_tokens: 7 }, + }); + } finally { + await closeServer(recorder.server); + await closeServer(upstream); + fs.rmSync(fixturePath, { recursive: true, force: true }); + } + }); + it("detects OpenAI transcription response", async () => { const fixturePath = makeTmpDir(); const { server, url } = await createUpstream((_req, res) => { diff --git a/src/__tests__/multimedia.test.ts b/src/__tests__/multimedia.test.ts index 1ee8686f..af7b78df 100644 --- a/src/__tests__/multimedia.test.ts +++ b/src/__tests__/multimedia.test.ts @@ -1,6 +1,32 @@ import { describe, test, expect } from "vitest"; import { LLMock } from "../llmock.js"; +async function readSSEFrameTimes( + response: Response, +): Promise<{ body: string; frameTimes: number[] }> { + const reader = response.body?.getReader(); + if (!reader) throw new Error("Expected an SSE response body"); + + const decoder = new TextDecoder(); + let body = ""; + let pending = ""; + const frameTimes: number[] = []; + while (true) { + const { done, value } = await reader.read(); + if (done) break; + const chunk = decoder.decode(value, { stream: true }); + body += chunk; + pending += chunk; + while (pending.includes("\n\n")) { + const boundary = pending.indexOf("\n\n"); + pending = pending.slice(boundary + 2); + frameTimes.push(Date.now()); + } + } + body += decoder.decode(); + return { body, frameTimes }; +} + describe("image generation", () => { test("image generation returns fixture (OpenAI format)", async () => { const mock = new LLMock({ port: 0 }); @@ -87,7 +113,7 @@ describe("image generation", () => { }); describe("audio transcription", () => { - test("transcription returns text", async () => { + test("gpt-transcribe returns a nonstreaming transcription", async () => { const mock = new LLMock({ port: 0 }); mock.addFixture({ match: { endpoint: "transcription" }, @@ -97,7 +123,7 @@ describe("audio transcription", () => { const formData = new FormData(); formData.append("file", new Blob(["fake audio"], { type: "audio/wav" }), "test.wav"); - formData.append("model", "whisper-1"); + formData.append("model", "gpt-transcribe"); const res = await fetch(`${mock.url}/v1/audio/transcriptions`, { method: "POST", @@ -110,6 +136,187 @@ describe("audio transcription", () => { await mock.stop(); }); + test("gpt-transcribe streams transcript delta and completion events", async () => { + const mock = new LLMock({ port: 0 }); + mock.addFixture({ + match: { endpoint: "transcription" }, + response: { transcription: { text: "Welcome" } }, + }); + await mock.start(); + + const formData = new FormData(); + formData.append("file", new Blob(["fake audio"], { type: "audio/wav" }), "test.wav"); + formData.append("model", "gpt-transcribe"); + formData.append("stream", "true"); + + const res = await fetch(`${mock.url}/v1/audio/transcriptions`, { + method: "POST", + headers: { Authorization: "Bearer test" }, + body: formData, + }); + + expect(res.status).toBe(200); + expect(res.headers.get("content-type")).toContain("text/event-stream"); + expect(await res.text()).toBe( + 'data: {"type":"transcript.text.delta","delta":"Welcome"}\n\ndata: {"type":"transcript.text.done","text":"Welcome"}\n\n', + ); + await mock.stop(); + }); + + test("gpt-transcribe stream schedules fixture chunks across separate SSE frames", async () => { + const mock = new LLMock({ port: 0, chunkSize: 2, latency: 40 }); + mock.addFixture({ + match: { endpoint: "transcription" }, + response: { transcription: { text: "abcdef" } }, + chunkSize: 2, + latency: 40, + }); + await mock.start(); + + const formData = new FormData(); + formData.append("file", new Blob(["fake audio"], { type: "audio/wav" }), "test.wav"); + formData.append("model", "gpt-transcribe"); + formData.append("stream", "true"); + + const res = await fetch(`${mock.url}/v1/audio/transcriptions`, { + method: "POST", + headers: { Authorization: "Bearer test" }, + body: formData, + }); + + const { body, frameTimes } = await readSSEFrameTimes(res); + const deltas = [...body.matchAll(/"type":"transcript\.text\.delta","delta":"([^"]+)"/g)].map( + (match) => match[1], + ); + expect(deltas).toEqual(["ab", "cd", "ef"]); + expect(frameTimes).toHaveLength(4); + expect(frameTimes[1] - frameTimes[0]).toBeGreaterThanOrEqual(20); + expect(frameTimes[2] - frameTimes[1]).toBeGreaterThanOrEqual(20); + await mock.stop(); + }); + + test("gpt-transcribe stream records fixture truncation before its done event", async () => { + const mock = new LLMock({ port: 0, chunkSize: 2, latency: 1 }); + mock.addFixture({ + match: { endpoint: "transcription", model: "gpt-transcribe" }, + response: { transcription: { text: "abcdefgh" } }, + chunkSize: 2, + truncateAfterChunks: 2, + }); + await mock.start(); + + const formData = new FormData(); + formData.append("file", new Blob(["fake audio"], { type: "audio/wav" }), "test.wav"); + formData.append("model", "gpt-transcribe"); + formData.append("stream", "true"); + const res = await fetch(`${mock.url}/v1/audio/transcriptions`, { + method: "POST", + body: formData, + }); + + await expect(res.text()).rejects.toThrow(); + expect(mock.journal.getLast()).toMatchObject({ + response: { interrupted: true, interruptReason: "truncateAfterChunks" }, + }); + await mock.stop(); + }); + + test("whisper-1 keeps its JSON response when stream is requested", async () => { + const mock = new LLMock({ port: 0 }); + mock.addFixture({ + match: { endpoint: "transcription" }, + response: { transcription: { text: "Legacy transcript" } }, + }); + await mock.start(); + + const formData = new FormData(); + formData.append("file", new Blob(["fake audio"], { type: "audio/wav" }), "test.wav"); + formData.append("model", "whisper-1"); + formData.append("stream", "true"); + + const res = await fetch(`${mock.url}/v1/audio/transcriptions`, { + method: "POST", + headers: { Authorization: "Bearer test" }, + body: formData, + }); + + expect(res.headers.get("content-type")).toContain("application/json"); + expect(await res.json()).toEqual({ text: "Legacy transcript" }); + await mock.stop(); + }); + + test("gpt-transcribe stream errors remain JSON in strict mode", async () => { + const mock = new LLMock({ port: 0, strict: true }); + await mock.start(); + + const formData = new FormData(); + formData.append("file", new Blob(["fake audio"], { type: "audio/wav" }), "test.wav"); + formData.append("model", "gpt-transcribe"); + formData.append("stream", "true"); + + const res = await fetch(`${mock.url}/v1/audio/transcriptions`, { + method: "POST", + headers: { Authorization: "Bearer test" }, + body: formData, + }); + + expect(res.status).toBe(503); + expect(res.headers.get("content-type")).toContain("application/json"); + expect(await res.json()).toMatchObject({ + error: { code: "no_fixture_match", type: "invalid_request_error" }, + }); + await mock.stop(); + }); + + test("gpt-transcribe stream no-match remains a non-strict 404 JSON error", async () => { + const mock = new LLMock({ port: 0 }); + await mock.start(); + + const formData = new FormData(); + formData.append("file", new Blob(["fake audio"], { type: "audio/wav" }), "test.wav"); + formData.append("model", "gpt-transcribe"); + formData.append("stream", "true"); + const res = await fetch(`${mock.url}/v1/audio/transcriptions`, { + method: "POST", + body: formData, + }); + + expect(res.status).toBe(404); + expect(res.headers.get("content-type")).toContain("application/json"); + expect(await res.json()).toMatchObject({ + error: { code: "no_fixture_match", type: "invalid_request_error" }, + }); + await mock.stop(); + }); + + test("gpt-transcribe stream ErrorResponse remains JSON", async () => { + const mock = new LLMock({ port: 0 }); + mock.addFixture({ + match: { endpoint: "transcription", model: "gpt-transcribe" }, + response: { + error: { message: "Audio quota exhausted", type: "rate_limit_error", code: "quota" }, + status: 429, + }, + }); + await mock.start(); + + const formData = new FormData(); + formData.append("file", new Blob(["fake audio"], { type: "audio/wav" }), "test.wav"); + formData.append("model", "gpt-transcribe"); + formData.append("stream", "true"); + const res = await fetch(`${mock.url}/v1/audio/transcriptions`, { + method: "POST", + body: formData, + }); + + expect(res.status).toBe(429); + expect(res.headers.get("content-type")).toContain("application/json"); + expect(await res.json()).toMatchObject({ + error: { message: "Audio quota exhausted", type: "rate_limit_error", code: "quota" }, + }); + await mock.stop(); + }); + test("verbose transcription includes words and segments", async () => { const mock = new LLMock({ port: 0 }); mock.addFixture({ @@ -168,6 +375,30 @@ describe("audio translation", () => { await mock.stop(); }); + test("translation keeps its JSON response when gpt-transcribe requests stream", async () => { + const mock = new LLMock({ port: 0 }); + mock.addFixture({ + match: { endpoint: "translation" }, + response: { transcription: { text: "Translated" } }, + }); + await mock.start(); + + const formData = new FormData(); + formData.append("file", new Blob(["fake audio"], { type: "audio/wav" }), "test.wav"); + formData.append("model", "gpt-transcribe"); + formData.append("stream", "true"); + + const res = await fetch(`${mock.url}/v1/audio/translations`, { + method: "POST", + headers: { Authorization: "Bearer test" }, + body: formData, + }); + + expect(res.headers.get("content-type")).toContain("application/json"); + expect(await res.json()).toEqual({ text: "Translated" }); + await mock.stop(); + }); + test("verbose translation includes task=translate", async () => { const mock = new LLMock({ port: 0 }); mock.addFixture({ diff --git a/src/__tests__/stream-collapse.test.ts b/src/__tests__/stream-collapse.test.ts index d8789f19..0cc50b8e 100644 --- a/src/__tests__/stream-collapse.test.ts +++ b/src/__tests__/stream-collapse.test.ts @@ -19,6 +19,16 @@ import type { Fixture } from "../types.js"; // --------------------------------------------------------------------------- describe("collapseOpenAISSE", () => { + it("preserves typed transcription metadata, including an empty transcript", () => { + const languageResult = collapseOpenAISSE( + 'data: {"type":"transcript.text.done","text":"hello","languages":[{"code":"en"}]}\n\n', + ); + const emptyResult = collapseOpenAISSE('data: {"type":"transcript.text.done","text":""}\n\n'); + + expect(languageResult.transcription).toEqual({ text: "hello", languages: [{ code: "en" }] }); + expect(emptyResult).toEqual({ transcription: { text: "" }, content: "" }); + }); + it("collapses text content from SSE chunks", () => { const body = [ `data: ${JSON.stringify({ id: "chatcmpl-123", choices: [{ delta: { role: "assistant" } }] })}`, diff --git a/src/__tests__/ws-api-conformance.test.ts b/src/__tests__/ws-api-conformance.test.ts index edd8a787..f6ee2866 100644 --- a/src/__tests__/ws-api-conformance.test.ts +++ b/src/__tests__/ws-api-conformance.test.ts @@ -615,14 +615,11 @@ describe("GA Realtime conformance", () => { expect(frame.type).toBe("session.created"); const session = frame.session; expect(session).toHaveProperty("audio"); - expect(session).toHaveProperty("type", "conversation"); + expect(session).toHaveProperty("type", "realtime"); expect(session).toHaveProperty("reasoning"); expect(session.audio).toMatchObject({ - voice: null, - input_audio_format: null, - output_audio_format: null, - input_audio_noise_reduction: null, - input_audio_transcription: null, + input: { format: null, noise_reduction: null, transcription: null }, + output: { format: null, voice: null }, }); }); diff --git a/src/__tests__/ws-realtime.test.ts b/src/__tests__/ws-realtime.test.ts index 518e1409..3830a3f9 100644 --- a/src/__tests__/ws-realtime.test.ts +++ b/src/__tests__/ws-realtime.test.ts @@ -72,6 +72,13 @@ function sessionUpdate(config: Record): string { return JSON.stringify({ type: "session.update", session: config }); } +function transcriptionSessionUpdate(model: string): string { + return JSON.stringify({ + type: "transcription_session.update", + session: { input_audio_transcription: { model } }, + }); +} + function functionCallOutputItem(callId: string, output: string): string { return JSON.stringify({ type: "conversation.item.create", @@ -146,16 +153,14 @@ describe("WebSocket /v1/realtime", () => { expect(typeof session.expires_at).toBe("number"); expect(session.max_response_output_tokens).toBe("inf"); expect(session.tool_choice).toBe("auto"); - expect(session.type).toBe("conversation"); + expect(session.type).toBe("realtime"); expect(session.reasoning).toBeNull(); // GA nested audio config const audio = session.audio as Record; - expect(audio).toBeDefined(); - expect(audio.voice).toBeNull(); - expect(audio.input_audio_format).toBeNull(); - expect(audio.output_audio_format).toBeNull(); - expect(audio.input_audio_noise_reduction).toBeNull(); - expect(audio.input_audio_transcription).toBeNull(); + expect(audio).toEqual({ + input: { format: null, noise_reduction: null, transcription: null, turn_detection: null }, + output: { format: null, voice: null }, + }); ws.close(); }); @@ -185,11 +190,11 @@ describe("WebSocket /v1/realtime", () => { expect(typeof session.expires_at).toBe("number"); expect(session.max_response_output_tokens).toBe("inf"); expect(session.tool_choice).toBe("auto"); - expect(session.type).toBe("conversation"); + expect(session.type).toBe("realtime"); // GA nested audio config const audio = session.audio as Record; expect(audio).toBeDefined(); - expect(audio.voice).toBeNull(); + expect((audio.output as Record).voice).toBeNull(); ws.close(); }); @@ -743,7 +748,7 @@ describe("WebSocket /v1/realtime", () => { ws.close(); }); - it("session.update updates modalities, model, and temperature", async () => { + it("session.update rejects mutation of an established connection model", async () => { instance = await createServer(allFixtures); const ws = await connectWebSocket(instance.url, "/v1/realtime"); @@ -759,19 +764,7 @@ describe("WebSocket /v1/realtime", () => { const raw = await ws.waitForMessages(2); const event = JSON.parse(raw[1]) as WSEvent; - expect(event.type).toBe("session.updated"); - const session = event.session as Record; - expect(session.modalities).toEqual(["text", "audio"]); - expect(session.model).toBe("gpt-4o-mini-realtime"); - expect(session.temperature).toBe(0.5); - expect(session.object).toBe("realtime.session"); - expect(typeof session.expires_at).toBe("number"); - expect(session.max_response_output_tokens).toBe("inf"); - expect(session.tool_choice).toBe("auto"); - expect(session.type).toBe("conversation"); - // GA nested audio config - const audio = session.audio as Record; - expect(audio).toBeDefined(); + expect(event).toMatchObject({ type: "error", error: { code: "invalid_session_config" } }); ws.close(); }); @@ -1035,7 +1028,9 @@ describe("WebSocket /v1/realtime", () => { expect(event.type).toBe("session.updated"); const session = event.session as Record; const audio = session.audio as Record; - expect(audio.input_audio_noise_reduction).toEqual({ type: "near_field" }); + expect((audio.input as Record).noise_reduction).toEqual({ + type: "near_field", + }); ws.close(); }); @@ -1061,7 +1056,7 @@ describe("WebSocket /v1/realtime", () => { expect(event.type).toBe("session.updated"); const session = event.session as Record; const audio = session.audio as Record; - expect(audio.input_audio_transcription).toEqual({ model: "whisper-1" }); + expect((audio.input as Record).transcription).toEqual({ model: "whisper-1" }); ws.close(); }); @@ -1092,9 +1087,15 @@ describe("WebSocket /v1/realtime", () => { expect(event.type).toBe("session.updated"); const session = event.session as Record; const audio = session.audio as Record; - expect(audio.voice).toBe("alloy"); - expect(audio.input_audio_format).toBe("pcm16"); - expect(audio.output_audio_format).toBe("pcm16"); + expect(audio).toEqual({ + input: { + format: { type: "pcm16" }, + noise_reduction: null, + transcription: null, + turn_detection: null, + }, + output: { format: { type: "pcm16" }, voice: "alloy" }, + }); expect(session.modalities).toEqual(["text", "audio"]); ws.close(); @@ -1290,13 +1291,13 @@ describe("WebSocket /v1/realtime", () => { // ── Translate/Whisper session types + audio buffer ───────────────────── it("accepts transcription session type and acknowledges audio buffer commit", async () => { instance = await createServer(allFixtures); - const ws = await connectWebSocket(instance.url, "/v1/realtime"); + const ws = await connectWebSocket(instance.url, "/v1/realtime?model=gpt-4o-transcribe"); // Skip session.created await ws.waitForMessages(1); // Update session to transcription mode with transcribe model - ws.send(sessionUpdate({ type: "transcription", model: "gpt-4o-transcribe" })); + ws.send(sessionUpdate({ type: "transcription" })); const updateRaw = await ws.waitForMessages(2); const updateEvent = parseEvents(updateRaw.slice(1))[0]; @@ -1326,9 +1327,310 @@ describe("WebSocket /v1/realtime", () => { ws.close(); }); + it("streams live transcription after documented transcription_session.update", async () => { + const transcriptionFixture: Fixture = { + match: { endpoint: "realtime-transcription" }, + response: { transcription: { text: "Live caption" } }, + }; + instance = await createServer([transcriptionFixture]); + const ws = await connectWebSocket( + instance.url, + "/v1/realtime?model=gpt-realtime&intent=transcription", + ); + + await ws.waitForMessages(1); // session.created + ws.send(transcriptionSessionUpdate("gpt-live-transcribe-2026-07-01")); + const update = parseEvents(await ws.waitForMessages(2))[1]; + expect(update.type).toBe("transcription_session.updated"); + expect(update.session).toMatchObject({ + audio: { + input: { transcription: { model: "gpt-live-transcribe-2026-07-01" } }, + }, + }); + + ws.send(JSON.stringify({ type: "input_audio_buffer.commit" })); + const raw = await ws.waitForMessages(6); + const events = parseEvents(raw.slice(2)); + const delta = events.find( + (event) => event.type === "conversation.item.input_audio_transcription.delta", + ); + const completed = events.find( + (event) => event.type === "conversation.item.input_audio_transcription.completed", + ); + + expect(delta).toMatchObject({ content_index: 0, delta: "Live caption" }); + expect(completed).toMatchObject({ content_index: 0, transcript: "Live caption" }); + expect(completed!.item_id).toBe(delta!.item_id); + ws.close(); + }); + + it("schedules live transcription deltas across separate socket frames", async () => { + const transcriptionFixture: Fixture = { + match: { endpoint: "realtime-transcription" }, + response: { transcription: { text: "abcdef" } }, + chunkSize: 2, + latency: 40, + }; + instance = await createServer([transcriptionFixture], { chunkSize: 2, latency: 40 }); + const ws = await connectWebSocket( + instance.url, + "/v1/realtime?model=gpt-realtime&intent=transcription", + ); + + await ws.waitForMessages(1); + ws.send(transcriptionSessionUpdate("gpt-live-transcribe")); + await ws.waitForMessages(2); + const startedAt = Date.now(); + ws.send(JSON.stringify({ type: "input_audio_buffer.commit" })); + + const firstDelta = await ws.waitForMessages(5); + const firstDeltaAt = Date.now(); + await ws.waitForMessages(6); + const secondDeltaAt = Date.now(); + await ws.waitForMessages(7); + const thirdDeltaAt = Date.now(); + const events = parseEvents(await ws.waitForMessages(8)); + + expect(firstDeltaAt - startedAt).toBeGreaterThanOrEqual(20); + expect(secondDeltaAt - firstDeltaAt).toBeGreaterThanOrEqual(20); + expect(thirdDeltaAt - secondDeltaAt).toBeGreaterThanOrEqual(20); + expect(parseEvents(firstDelta.slice(4))[0]).toMatchObject({ delta: "ab" }); + expect(events[7]).toMatchObject({ + type: "conversation.item.input_audio_transcription.completed", + transcript: "abcdef", + }); + ws.close(); + }); + + it("uses the versioned model configured in session.audio.input.transcription", async () => { + const transcriptionFixture: Fixture = { + match: { endpoint: "realtime-transcription", model: "gpt-live-transcribe-2026-07-01" }, + response: { transcription: { text: "Nested config caption" } }, + }; + instance = await createServer([transcriptionFixture]); + const ws = await connectWebSocket(instance.url, "/v1/realtime?model=gpt-realtime"); + + await ws.waitForMessages(1); // session.created + ws.send( + sessionUpdate({ + audio: { + input: { transcription: { model: "gpt-live-transcribe-2026-07-01" } }, + }, + }), + ); + await ws.waitForMessages(2); // session.updated + + ws.send(JSON.stringify({ type: "input_audio_buffer.commit" })); + const events = parseEvents(await ws.waitForMessages(7)); + const deltas = events + .filter((event) => event.type === "conversation.item.input_audio_transcription.delta") + .map((event) => event.delta) + .join(""); + expect(deltas).toBe("Nested config caption"); + expect(events[6]).toMatchObject({ + type: "conversation.item.input_audio_transcription.completed", + transcript: "Nested config caption", + }); + ws.close(); + }); + + it("abruptly destroys a documented transcription socket after truncateAfterChunks", async () => { + const fixture: Fixture = { + match: { endpoint: "realtime-transcription", model: "gpt-live-transcribe-2026-07-01" }, + response: { transcription: { text: "abcdefgh" } }, + chunkSize: 2, + truncateAfterChunks: 2, + }; + instance = await createServer([fixture]); + const ws = await connectWebSocket( + instance.url, + "/v1/realtime?intent=transcription&model=ignored-by-intent", + ); + + await ws.waitForMessages(1); + ws.send(transcriptionSessionUpdate("gpt-live-transcribe-2026-07-01")); + await ws.waitForMessages(2); + ws.send(JSON.stringify({ type: "input_audio_buffer.commit" })); + + await ws.waitForClose(); + await expect(ws.waitForCloseFrame()).rejects.toThrow("without a close frame"); + const events = parseEvents(ws.getMessages()); + expect( + events.filter((event) => event.type === "conversation.item.input_audio_transcription.delta"), + ).toHaveLength(2); + expect(events).not.toContainEqual( + expect.objectContaining({ type: "conversation.item.input_audio_transcription.completed" }), + ); + expect(instance.journal.getLast()).toMatchObject({ + body: { _endpointType: "realtime-transcription", model: "gpt-live-transcribe-2026-07-01" }, + response: { interrupted: true, interruptReason: "truncateAfterChunks" }, + }); + }); + + it("abruptly destroys a documented transcription socket after disconnectAfterMs", async () => { + const fixture: Fixture = { + match: { endpoint: "realtime-transcription", model: "gpt-live-transcribe-2026-07-01" }, + response: { transcription: { text: "abcdefgh" } }, + chunkSize: 2, + latency: 60, + disconnectAfterMs: 15, + }; + instance = await createServer([fixture]); + const ws = await connectWebSocket(instance.url, "/v1/realtime?intent=transcription"); + + await ws.waitForMessages(1); + ws.send(transcriptionSessionUpdate("gpt-live-transcribe-2026-07-01")); + await ws.waitForMessages(2); + ws.send(JSON.stringify({ type: "input_audio_buffer.commit" })); + + await ws.waitForClose(); + await expect(ws.waitForCloseFrame()).rejects.toThrow("without a close frame"); + const events = parseEvents(ws.getMessages()); + expect(events).not.toContainEqual( + expect.objectContaining({ type: "conversation.item.input_audio_transcription.delta" }), + ); + expect(events).not.toContainEqual( + expect.objectContaining({ type: "conversation.item.input_audio_transcription.completed" }), + ); + expect(instance.journal.getLast()).toMatchObject({ + body: { _endpointType: "realtime-transcription", model: "gpt-live-transcribe-2026-07-01" }, + response: { interrupted: true, interruptReason: "disconnectAfterMs" }, + }); + }); + + it("closes documented transcription sockets in strict mode when a versioned fixture misses", async () => { + instance = await createServer([], { strict: true }); + const ws = await connectWebSocket(instance.url, "/v1/realtime?intent=transcription"); + + await ws.waitForMessages(1); // session.created + ws.send(transcriptionSessionUpdate("gpt-live-transcribe-2026-07-01")); + await ws.waitForMessages(2); // session.updated + ws.send(JSON.stringify({ type: "input_audio_buffer.commit" })); + + const close = await ws.waitForCloseFrame(); + expect(close.code).toBe(1008); + expect(instance.journal.getLast()).toMatchObject({ + response: { status: 503, fixture: null }, + body: { + _endpointType: "realtime-transcription", + model: "gpt-live-transcribe-2026-07-01", + }, + }); + }); + + it("emits a transcription failure event for a documented versioned ErrorResponse fixture", async () => { + const errorFixture: Fixture = { + match: { endpoint: "realtime-transcription", model: "gpt-live-transcribe-2026-07-01" }, + response: { + error: { message: "Rate limited", type: "rate_limit_error", code: "rate_limit" }, + status: 429, + }, + }; + instance = await createServer([errorFixture]); + const ws = await connectWebSocket(instance.url, "/v1/realtime?intent=transcription"); + + await ws.waitForMessages(1); // session.created + ws.send(transcriptionSessionUpdate("gpt-live-transcribe-2026-07-01")); + await ws.waitForMessages(2); // session.updated + ws.send(JSON.stringify({ type: "input_audio_buffer.commit" })); + + const raw = await ws.waitForMessages(5); + const events = parseEvents(raw); + const item = events[3].item as Record; + expect(events[4]).toMatchObject({ + type: "conversation.item.input_audio_transcription.failed", + item_id: item.id, + content_index: 0, + error: { message: "Rate limited", type: "rate_limit_error", code: "rate_limit" }, + }); + expect(instance.journal.getLast()).toMatchObject({ + response: { status: 429, fixture: errorFixture }, + }); + ws.close(); + }); + + it("emits a transcription failure event for a non-strict live transcription no-match", async () => { + instance = await createServer([]); + const ws = await connectWebSocket(instance.url, "/v1/realtime?model=gpt-live-transcribe"); + + await ws.waitForMessages(1); // session.created + ws.send(sessionUpdate({ type: "transcription" })); + await ws.waitForMessages(2); // session.updated + ws.send(JSON.stringify({ type: "input_audio_buffer.commit" })); + + const raw = await ws.waitForMessages(5); + const events = parseEvents(raw); + const item = events[3].item as Record; + expect(events[4]).toMatchObject({ + type: "conversation.item.input_audio_transcription.failed", + item_id: item.id, + content_index: 0, + error: { + message: "No fixture matched", + type: "invalid_request_error", + code: "no_fixture_match", + }, + }); + expect(instance.journal.getLast()).toMatchObject({ response: { status: 404, fixture: null } }); + ws.close(); + }); + + it("matches context-scoped live transcription fixtures", async () => { + const contextFixture: Fixture = { + match: { endpoint: "realtime-transcription", context: "call-42" }, + response: { transcription: { text: "Context transcript" } }, + }; + instance = await createServer([contextFixture]); + const ws = await connectWebSocket(instance.url, "/v1/realtime?model=gpt-live-transcribe", { + "x-aimock-context": "call-42", + }); + + await ws.waitForMessages(1); // session.created + ws.send(sessionUpdate({ type: "transcription" })); + await ws.waitForMessages(2); // session.updated + ws.send(JSON.stringify({ type: "input_audio_buffer.commit" })); + + const raw = await ws.waitForMessages(6); + const completed = parseEvents(raw)[5]; + expect(completed).toMatchObject({ + type: "conversation.item.input_audio_transcription.completed", + transcript: "Context transcript", + }); + expect(instance.journal.getLast()).toMatchObject({ body: { _context: "call-42" } }); + ws.close(); + }); + + it("emits a server failure for a non-transcription live transcription fixture", async () => { + const invalidFixture: Fixture = { + match: { endpoint: "realtime-transcription" }, + response: { content: "not a transcript" }, + }; + instance = await createServer([invalidFixture]); + const ws = await connectWebSocket(instance.url, "/v1/realtime?model=gpt-live-transcribe"); + + await ws.waitForMessages(1); // session.created + ws.send(sessionUpdate({ type: "transcription" })); + await ws.waitForMessages(2); // session.updated + ws.send(JSON.stringify({ type: "input_audio_buffer.commit" })); + + const raw = await ws.waitForMessages(5); + const events = parseEvents(raw); + const item = events[3].item as Record; + expect(events[4]).toMatchObject({ + type: "conversation.item.input_audio_transcription.failed", + item_id: item.id, + content_index: 0, + error: { message: "Fixture response is not a transcription type", type: "server_error" }, + }); + expect(instance.journal.getLast()).toMatchObject({ + response: { status: 500, fixture: invalidFixture }, + }); + ws.close(); + }); + it("input_audio_buffer.append is silently accepted", async () => { instance = await createServer(allFixtures); - const ws = await connectWebSocket(instance.url, "/v1/realtime"); + const ws = await connectWebSocket(instance.url, "/v1/realtime?model=gpt-4o-transcribe"); await ws.waitForMessages(1); // session.created @@ -1385,11 +1687,11 @@ describe("WebSocket /v1/realtime", () => { it("accepts translation session type", async () => { instance = await createServer(allFixtures); - const ws = await connectWebSocket(instance.url, "/v1/realtime"); + const ws = await connectWebSocket(instance.url, "/v1/realtime?model=gpt-4o-transcribe"); await ws.waitForMessages(1); // session.created - ws.send(sessionUpdate({ type: "translation", model: "gpt-4o-transcribe" })); + ws.send(sessionUpdate({ type: "translation" })); const raw = await ws.waitForMessages(2); const event = parseEvents(raw.slice(1))[0]; @@ -1425,7 +1727,7 @@ describe("WebSocket /v1/realtime", () => { await ws.waitForMessages(1); // session.created - ws.send(sessionUpdate({ type: "translation", model: "gpt-realtime-mini" })); + ws.send(sessionUpdate({ type: "translation" })); const raw = await ws.waitForMessages(2); const event = parseEvents(raw.slice(1))[0]; @@ -1440,11 +1742,11 @@ describe("WebSocket /v1/realtime", () => { it("audio buffer commit in translation mode adds placeholder conversation item", async () => { instance = await createServer(allFixtures); - const ws = await connectWebSocket(instance.url, "/v1/realtime"); + const ws = await connectWebSocket(instance.url, "/v1/realtime?model=gpt-4o-transcribe"); await ws.waitForMessages(1); // session.created - ws.send(sessionUpdate({ type: "translation", model: "gpt-4o-transcribe" })); + ws.send(sessionUpdate({ type: "translation" })); await ws.waitForMessages(2); // session.updated ws.send(JSON.stringify({ type: "input_audio_buffer.commit" })); @@ -1920,7 +2222,7 @@ describe("WebSocket /v1/realtime", () => { const session = event3.session as Record; // Model and type should still be the pre-rejection values expect(session.model).toBe("gpt-realtime-2"); - expect(session.type).toBe("conversation"); + expect(session.type).toBe("realtime"); expect(session.instructions).toBe("Updated instructions"); ws.close();