From 1dbfe76c625b4557375a7a1ff25dd7204f29b885 Mon Sep 17 00:00:00 2001 From: "t3-code[bot]" <269035359+t3-code[bot]@users.noreply.github.com> Date: Sun, 2 Aug 2026 04:33:13 +0000 Subject: [PATCH 1/6] feat: add voice dictation beta --- .../settings/DesktopClientSettings.test.ts | 5 + apps/server/src/http.ts | 46 ++++++ apps/server/src/httpCors.ts | 3 + apps/server/src/server.ts | 2 + apps/server/src/transcription.test.ts | 66 ++++++++ apps/server/src/transcription.ts | 113 +++++++++++++ apps/web/src/components/chat/ChatComposer.tsx | 63 ++++++++ .../chat/VoiceTranscriptionPanel.tsx | 68 ++++++++ .../components/settings/BetaSettingsPanel.tsx | 142 +++++++++++++++++ .../src/components/settings/settingsSearch.ts | 5 + apps/web/src/hooks/useVoiceTranscription.ts | 149 ++++++++++++++++++ apps/web/src/lib/voiceTranscription.ts | 47 ++++++ packages/contracts/src/settings.ts | 21 +++ scripts/build-desktop-artifact.test.ts | 3 + scripts/build-desktop-artifact.ts | 3 + 15 files changed, 736 insertions(+) create mode 100644 apps/server/src/transcription.test.ts create mode 100644 apps/server/src/transcription.ts create mode 100644 apps/web/src/components/chat/VoiceTranscriptionPanel.tsx create mode 100644 apps/web/src/hooks/useVoiceTranscription.ts create mode 100644 apps/web/src/lib/voiceTranscription.ts diff --git a/apps/desktop/src/settings/DesktopClientSettings.test.ts b/apps/desktop/src/settings/DesktopClientSettings.test.ts index 8d76ea83a33..baa5499d995 100644 --- a/apps/desktop/src/settings/DesktopClientSettings.test.ts +++ b/apps/desktop/src/settings/DesktopClientSettings.test.ts @@ -33,6 +33,11 @@ const clientSettings: ClientSettings = { sidebarV2Enabled: false, sidebarV2ConfiguredByUser: false, timestampFormat: "24-hour", + voiceTranscriptionEnabled: true, + voiceTranscriptionProvider: "local", + voiceTranscriptionBaseUrl: "http://127.0.0.1:8080/v1", + voiceTranscriptionModel: "whisper-1", + voiceTranscriptionApiKey: "", wordWrap: true, }; diff --git a/apps/server/src/http.ts b/apps/server/src/http.ts index 5a380be8fe2..0272f6d7012 100644 --- a/apps/server/src/http.ts +++ b/apps/server/src/http.ts @@ -40,8 +40,10 @@ import { } from "./auth/http.ts"; import * as ServerEnvironment from "./environment/ServerEnvironment.ts"; import { browserApiCorsAllowedHeaders, browserApiCorsAllowedMethods } from "./httpCors.ts"; +import { forwardWhisperTranscription, MAX_TRANSCRIPTION_AUDIO_BYTES } from "./transcription.ts"; const OTLP_TRACES_PROXY_PATH = "/api/observability/v1/traces"; +const TRANSCRIPTION_PATH = "/api/transcription"; const LOOPBACK_HOSTNAMES = new Set(["127.0.0.1", "::1", "localhost"]); const DESKTOP_RENDERER_ORIGINS = ["t3code://app", "t3code-dev://app"]; const GZIP_MIN_BYTES = 1024; @@ -247,6 +249,50 @@ export const otlpTracesProxyRouteLayer = HttpRouter.add( ), ); +export const transcriptionRouteLayer = HttpRouter.add( + "POST", + TRANSCRIPTION_PATH, + Effect.gen(function* () { + yield* authenticateRawRouteWithScope(AuthOrchestrationOperateScope); + const request = yield* HttpServerRequest.HttpServerRequest; + const declaredLength = Number(request.headers["content-length"] ?? "0"); + if (Number.isFinite(declaredLength) && declaredLength > MAX_TRANSCRIPTION_AUDIO_BYTES) { + return HttpServerResponse.jsonUnsafe( + { error: "The recording exceeds the 25 MB limit." }, + { status: 413 }, + ); + } + + const body = yield* request.arrayBuffer; + const audio = new Uint8Array(body); + const result = yield* Effect.tryPromise(() => + forwardWhisperTranscription({ + audio, + audioMimeType: request.headers["content-type"] ?? "audio/webm", + baseUrl: request.headers["x-t3-transcription-base-url"] ?? "", + model: request.headers["x-t3-transcription-model"] ?? "", + apiKey: request.headers["x-t3-transcription-api-key"] ?? "", + }), + ).pipe( + Effect.orElseSucceed(() => ({ + ok: false as const, + status: 502, + message: "Transcription failed unexpectedly.", + })), + ); + + return result.ok + ? HttpServerResponse.jsonUnsafe({ text: result.text }) + : HttpServerResponse.jsonUnsafe({ error: result.message }, { status: result.status }); + }).pipe( + Effect.catchTags({ + EnvironmentAuthInvalidError: HttpServerRespondable.toResponse, + EnvironmentInternalError: HttpServerRespondable.toResponse, + EnvironmentScopeRequiredError: HttpServerRespondable.toResponse, + }), + ), +); + export const assetRouteLayer = HttpRouter.add( "GET", `${ASSET_ROUTE_PREFIX}/*`, diff --git a/apps/server/src/httpCors.ts b/apps/server/src/httpCors.ts index aeb8dbce5a5..b41f65e1fb1 100644 --- a/apps/server/src/httpCors.ts +++ b/apps/server/src/httpCors.ts @@ -5,6 +5,9 @@ export const browserApiCorsAllowedHeaders = [ "traceparent", "content-type", "dpop", + "x-t3-transcription-api-key", + "x-t3-transcription-base-url", + "x-t3-transcription-model", ] as const; export const browserApiCorsHeaders = { diff --git a/apps/server/src/server.ts b/apps/server/src/server.ts index 05657af6d48..62ad7c9bd3e 100644 --- a/apps/server/src/server.ts +++ b/apps/server/src/server.ts @@ -13,6 +13,7 @@ import * as ServerConfig from "./config.ts"; import * as HttpResponseCompression from "./httpCompression/HttpResponseCompression.ts"; import { otlpTracesProxyRouteLayer, + transcriptionRouteLayer, assetRouteLayer, serverEnvironmentHttpApiLayer, staticAndDevRouteLayer, @@ -417,6 +418,7 @@ export const makeRoutesLayer = Layer.mergeAll( Layer.provide(environmentAuthenticatedAuthLayer), ), otlpTracesProxyRouteLayer, + transcriptionRouteLayer, assetRouteLayer, staticAndDevRouteLayer, websocketRpcRouteLayer, diff --git a/apps/server/src/transcription.test.ts b/apps/server/src/transcription.test.ts new file mode 100644 index 00000000000..44dd0a135e4 --- /dev/null +++ b/apps/server/src/transcription.test.ts @@ -0,0 +1,66 @@ +import { expect, it } from "@effect/vitest"; +import { describe, vi } from "vite-plus/test"; + +import { forwardWhisperTranscription, resolveWhisperTranscriptionUrl } from "./transcription.ts"; + +describe("resolveWhisperTranscriptionUrl", () => { + it("adds the OpenAI-compatible transcription path", () => { + expect(resolveWhisperTranscriptionUrl("http://127.0.0.1:8080/v1/")?.toString()).toBe( + "http://127.0.0.1:8080/v1/audio/transcriptions", + ); + expect( + resolveWhisperTranscriptionUrl( + "https://api.groq.com/openai/v1/audio/transcriptions", + )?.toString(), + ).toBe("https://api.groq.com/openai/v1/audio/transcriptions"); + }); + + it("rejects non-http and credential-bearing URLs", () => { + expect(resolveWhisperTranscriptionUrl("file:///tmp/whisper")).toBeNull(); + expect(resolveWhisperTranscriptionUrl("https://user:pass@example.com/v1")).toBeNull(); + }); +}); + +describe("forwardWhisperTranscription", () => { + it("sends OpenAI-compatible multipart audio with BYOK authorization", async () => { + const fetchImpl = vi.fn(async (_input: string | URL | Request, init?: RequestInit) => { + expect(init?.headers).toEqual({ authorization: "Bearer groq-key" }); + expect(init?.body).toBeInstanceOf(FormData); + const form = init?.body as FormData; + expect(form.get("model")).toBe("whisper-large-v3-turbo"); + expect(form.get("file")).toBeInstanceOf(Blob); + return Response.json({ text: "hello from the microphone" }); + }); + + await expect( + forwardWhisperTranscription( + { + audio: new Uint8Array([1, 2, 3]), + audioMimeType: "audio/webm;codecs=opus", + baseUrl: "https://api.groq.com/openai/v1", + model: "whisper-large-v3-turbo", + apiKey: "groq-key", + }, + fetchImpl, + ), + ).resolves.toEqual({ ok: true, text: "hello from the microphone" }); + }); + + it("returns a safe provider error", async () => { + const fetchImpl = vi.fn(async () => + Response.json({ error: { message: "bad key" } }, { status: 401 }), + ); + await expect( + forwardWhisperTranscription( + { + audio: new Uint8Array([1]), + audioMimeType: "audio/webm", + baseUrl: "https://example.com/v1", + model: "whisper-1", + apiKey: "bad", + }, + fetchImpl, + ), + ).resolves.toEqual({ ok: false, status: 502, message: "bad key" }); + }); +}); diff --git a/apps/server/src/transcription.ts b/apps/server/src/transcription.ts new file mode 100644 index 00000000000..5d380bafa88 --- /dev/null +++ b/apps/server/src/transcription.ts @@ -0,0 +1,113 @@ +const TRANSCRIPTION_PATH = "/audio/transcriptions"; + +export const MAX_TRANSCRIPTION_AUDIO_BYTES = 25 * 1024 * 1024; + +export interface WhisperTranscriptionInput { + readonly audio: Uint8Array; + readonly audioMimeType: string; + readonly baseUrl: string; + readonly model: string; + readonly apiKey: string; +} + +type TranscriptionFetch = (input: string | URL | Request, init?: RequestInit) => Promise; + +export type WhisperTranscriptionResult = + | { readonly ok: true; readonly text: string } + | { readonly ok: false; readonly status: number; readonly message: string }; + +export function resolveWhisperTranscriptionUrl(baseUrl: string): URL | null { + try { + const url = new URL(baseUrl.trim()); + if (url.protocol !== "http:" && url.protocol !== "https:") return null; + if (url.username || url.password) return null; + url.search = ""; + url.hash = ""; + url.pathname = url.pathname.replace(/\/$/, ""); + if (!url.pathname.endsWith(TRANSCRIPTION_PATH)) { + url.pathname += TRANSCRIPTION_PATH; + } + return url; + } catch { + return null; + } +} + +function audioFileExtension(mimeType: string): string { + if (mimeType.includes("ogg")) return "ogg"; + if (mimeType.includes("mp4") || mimeType.includes("m4a")) return "m4a"; + if (mimeType.includes("wav")) return "wav"; + return "webm"; +} + +export async function forwardWhisperTranscription( + input: WhisperTranscriptionInput, + fetchImpl: TranscriptionFetch = globalThis.fetch, +): Promise { + const endpoint = resolveWhisperTranscriptionUrl(input.baseUrl); + if (!endpoint) { + return { ok: false, status: 400, message: "Enter a valid HTTP transcription endpoint." }; + } + if (!input.model.trim()) { + return { ok: false, status: 400, message: "Enter a transcription model." }; + } + if (input.audio.byteLength === 0) { + return { ok: false, status: 400, message: "The recording was empty." }; + } + if (input.audio.byteLength > MAX_TRANSCRIPTION_AUDIO_BYTES) { + return { ok: false, status: 413, message: "The recording exceeds the 25 MB limit." }; + } + + const mimeType = input.audioMimeType.split(";", 1)[0]?.trim() || "audio/webm"; + const form = new FormData(); + form.set("model", input.model.trim()); + form.set( + "file", + new Blob([input.audio], { type: mimeType }), + `recording.${audioFileExtension(mimeType)}`, + ); + + let response: Response; + try { + response = await fetchImpl(endpoint, { + method: "POST", + headers: input.apiKey ? { authorization: `Bearer ${input.apiKey}` } : undefined, + body: form, + signal: AbortSignal.timeout(120_000), + }); + } catch { + return { ok: false, status: 502, message: "Could not reach the transcription provider." }; + } + + let payload: unknown; + try { + payload = await response.json(); + } catch { + return { ok: false, status: 502, message: "The transcription provider returned invalid JSON." }; + } + + if (!response.ok) { + const providerMessage = + typeof payload === "object" && + payload !== null && + "error" in payload && + typeof payload.error === "object" && + payload.error !== null && + "message" in payload.error && + typeof payload.error.message === "string" + ? payload.error.message + : null; + return { + ok: false, + status: 502, + message: providerMessage?.slice(0, 300) || "The transcription provider rejected the request.", + }; + } + + const text = + typeof payload === "object" && payload !== null && "text" in payload ? payload.text : undefined; + if (typeof text !== "string") { + return { ok: false, status: 502, message: "The transcription response did not contain text." }; + } + return { ok: true, text: text.trim() }; +} diff --git a/apps/web/src/components/chat/ChatComposer.tsx b/apps/web/src/components/chat/ChatComposer.tsx index e92ecd497e3..dfcdbeeb402 100644 --- a/apps/web/src/components/chat/ChatComposer.tsx +++ b/apps/web/src/components/chat/ChatComposer.tsx @@ -107,6 +107,8 @@ import { buildExpandedImagePreview, type ExpandedImagePreview } from "./Expanded import { basenameOfPath } from "../../pierre-icons"; import { cn, randomUUID } from "~/lib/utils"; import { Separator } from "../ui/separator"; +import { useVoiceTranscription } from "../../hooks/useVoiceTranscription"; +import { VoiceTranscriptionPanel } from "./VoiceTranscriptionPanel"; function ComposerCommandMenuLayer(props: { anchor: HTMLElement | null; children: ReactNode }) { const [position, setPosition] = useState<{ @@ -179,6 +181,7 @@ import { LockOpenIcon, PenLineIcon, SparklesIcon, + MicIcon, XIcon, } from "lucide-react"; import { proposedPlanTitle } from "../../proposedPlan"; @@ -1269,6 +1272,30 @@ export const ChatComposer = memo(function ChatComposer(props: ChatComposerProps) [composerDraftTarget, setComposerDraftPrompt], ); + const appendVoiceTranscript = useCallback( + (transcript: string) => { + const currentPrompt = promptRef.current; + const boundary = currentPrompt.length > 0 && !/\s$/.test(currentPrompt) ? " " : ""; + const nextPrompt = `${currentPrompt}${boundary}${transcript}`; + promptRef.current = nextPrompt; + setPrompt(nextPrompt); + const nextCursor = collapseExpandedComposerCursor(nextPrompt, nextPrompt.length); + setComposerCursor(nextCursor); + setComposerTrigger(null); + scheduleComposerFocus(); + }, + [promptRef, scheduleComposerFocus, setPrompt], + ); + const voiceTranscription = useVoiceTranscription({ + config: { + provider: settings.voiceTranscriptionProvider, + baseUrl: settings.voiceTranscriptionBaseUrl, + model: settings.voiceTranscriptionModel, + apiKey: settings.voiceTranscriptionApiKey, + }, + onTranscript: appendVoiceTranscript, + }); + const addComposerImage = useCallback( (image: ComposerImageAttachment) => { addComposerDraftImage(composerDraftTarget, image); @@ -3102,6 +3129,19 @@ export const ChatComposer = memo(function ChatComposer(props: ChatComposerProps) + {settings.voiceTranscriptionEnabled && voiceTranscription.status !== "idle" ? ( + + ) : settings.voiceTranscriptionEnabled && voiceTranscription.error ? ( +

+ {voiceTranscription.error} +

+ ) : null} + {/* Bottom toolbar */} {isComposerCollapsedMobile ? null : activePendingApproval ? (
@@ -3205,6 +3245,29 @@ export const ChatComposer = memo(function ChatComposer(props: ChatComposerProps) } className="flex shrink-0 flex-nowrap items-center justify-end gap-2" > + {settings.voiceTranscriptionEnabled && + voiceTranscription.status === "idle" && + !isComposerApprovalState && + pendingUserInputs.length === 0 ? ( + + void voiceTranscription.start()} + aria-label="Start dictation" + > + + + } + /> + Start dictation + + ) : null} `voice-waveform-${index}`); + +function formatElapsed(elapsedMs: number): string { + const seconds = Math.floor(elapsedMs / 1_000); + return `${Math.floor(seconds / 60)}:${String(seconds % 60).padStart(2, "0")}`; +} + +export function VoiceTranscriptionPanel({ + status, + elapsedMs, + levels, + onStop, +}: { + readonly status: Exclude; + readonly elapsedMs: number; + readonly levels: readonly number[]; + readonly onStop: () => void; +}) { + return ( +
+ {status === "recording" ? ( + + ) : ( +
+ + transcribing with your configured provider... +
+ )} + + {formatElapsed(elapsedMs)} + + {status === "recording" ? ( + + ) : null} +
+ ); +} diff --git a/apps/web/src/components/settings/BetaSettingsPanel.tsx b/apps/web/src/components/settings/BetaSettingsPanel.tsx index 740d3048f0e..c1f2b60d622 100644 --- a/apps/web/src/components/settings/BetaSettingsPanel.tsx +++ b/apps/web/src/components/settings/BetaSettingsPanel.tsx @@ -1,3 +1,8 @@ +import { + DEFAULT_VOICE_TRANSCRIPTION_BASE_URL, + DEFAULT_VOICE_TRANSCRIPTION_MODEL, + type VoiceTranscriptionProvider, +} from "@t3tools/contracts"; import { useEffect, useState } from "react"; import { @@ -5,7 +10,12 @@ import { useSidebarV2Enabled, useUpdateClientSettings, } from "../../hooks/useSettings"; +import { + GROQ_TRANSCRIPTION_BASE_URL, + GROQ_TRANSCRIPTION_MODEL, +} from "../../lib/voiceTranscription"; import { Input } from "../ui/input"; +import { Select, SelectItem, SelectPopup, SelectTrigger, SelectValue } from "../ui/select"; import { Switch } from "../ui/switch"; import { SettingsPageContainer, SettingsRow, SettingsSection } from "./settingsLayout"; import { searchableSetting } from "./settingsSearch"; @@ -60,8 +70,41 @@ export function BetaSettingsPanel() { const sidebarAutoSettleAfterDays = useClientSettings( (settings) => settings.sidebarAutoSettleAfterDays, ); + const voiceTranscriptionEnabled = useClientSettings( + (settings) => settings.voiceTranscriptionEnabled, + ); + const voiceTranscriptionProvider = useClientSettings( + (settings) => settings.voiceTranscriptionProvider, + ); + const voiceTranscriptionBaseUrl = useClientSettings( + (settings) => settings.voiceTranscriptionBaseUrl, + ); + const voiceTranscriptionModel = useClientSettings((settings) => settings.voiceTranscriptionModel); + const voiceTranscriptionApiKey = useClientSettings( + (settings) => settings.voiceTranscriptionApiKey, + ); const updateSettings = useUpdateClientSettings(); + const selectVoiceProvider = (provider: VoiceTranscriptionProvider) => { + if (provider === "local") { + updateSettings({ + voiceTranscriptionProvider: provider, + voiceTranscriptionBaseUrl: DEFAULT_VOICE_TRANSCRIPTION_BASE_URL, + voiceTranscriptionModel: DEFAULT_VOICE_TRANSCRIPTION_MODEL, + }); + return; + } + if (provider === "groq") { + updateSettings({ + voiceTranscriptionProvider: provider, + voiceTranscriptionBaseUrl: GROQ_TRANSCRIPTION_BASE_URL, + voiceTranscriptionModel: GROQ_TRANSCRIPTION_MODEL, + }); + return; + } + updateSettings({ voiceTranscriptionProvider: provider }); + }; + return ( @@ -114,6 +157,105 @@ export function BetaSettingsPanel() { ) : null} ) : null} + + updateSettings({ voiceTranscriptionEnabled: Boolean(checked) }) + } + aria-label="Enable voice dictation beta" + /> + } + /> + {voiceTranscriptionEnabled ? ( + <> + + selectVoiceProvider(value as VoiceTranscriptionProvider) + } + > + + + {voiceTranscriptionProvider === "local" + ? "Local" + : voiceTranscriptionProvider === "groq" + ? "Groq" + : "Custom"} + + + + + Local + + + Groq + + + Custom + + + + } + /> + + updateSettings({ voiceTranscriptionBaseUrl: event.target.value }) + } + placeholder="http://127.0.0.1:8080/v1" + aria-label="Transcription endpoint" + spellCheck={false} + /> + } + /> + + updateSettings({ voiceTranscriptionModel: event.target.value }) + } + placeholder="whisper-1" + aria-label="Transcription model" + spellCheck={false} + /> + } + /> + + updateSettings({ voiceTranscriptionApiKey: event.target.value }) + } + placeholder={voiceTranscriptionProvider === "local" ? "Optional" : "Required"} + aria-label="Transcription API key" + /> + } + /> + + ) : null} ); diff --git a/apps/web/src/components/settings/settingsSearch.ts b/apps/web/src/components/settings/settingsSearch.ts index 4ead6eff4d7..e80b41a4e0e 100644 --- a/apps/web/src/components/settings/settingsSearch.ts +++ b/apps/web/src/components/settings/settingsSearch.ts @@ -157,6 +157,11 @@ export const SETTINGS_SEARCH_ITEMS = [ to: "/settings/beta", targetId: "sidebar-v2", }, + { + id: "voice-dictation", + title: "Voice dictation", + to: "/settings/beta", + }, { id: "archive", title: "Archived threads", diff --git a/apps/web/src/hooks/useVoiceTranscription.ts b/apps/web/src/hooks/useVoiceTranscription.ts new file mode 100644 index 00000000000..061739cef37 --- /dev/null +++ b/apps/web/src/hooks/useVoiceTranscription.ts @@ -0,0 +1,149 @@ +import { useCallback, useEffect, useRef, useState } from "react"; + +import { transcribeVoiceRecording, type VoiceTranscriptionConfig } from "../lib/voiceTranscription"; + +const LEVEL_COUNT = 36; +const MAX_RECORDING_MS = 5 * 60 * 1_000; +const MIME_TYPES = ["audio/webm;codecs=opus", "audio/ogg;codecs=opus", "audio/mp4"]; + +export type VoiceTranscriptionStatus = "idle" | "recording" | "transcribing"; + +function supportedMimeType(): string | undefined { + if (typeof MediaRecorder === "undefined") return undefined; + return MIME_TYPES.find((mimeType) => MediaRecorder.isTypeSupported(mimeType)); +} + +export function useVoiceTranscription({ + config, + onTranscript, +}: { + readonly config: VoiceTranscriptionConfig; + readonly onTranscript: (text: string) => void; +}) { + const [status, setStatus] = useState("idle"); + const [elapsedMs, setElapsedMs] = useState(0); + const [levels, setLevels] = useState(() => Array(LEVEL_COUNT).fill(0.12)); + const [error, setError] = useState(null); + const recorderRef = useRef(null); + const streamRef = useRef(null); + const audioContextRef = useRef(null); + const intervalsRef = useRef([]); + const timeoutRef = useRef(null); + const startedAtRef = useRef(0); + const mountedRef = useRef(true); + const configRef = useRef(config); + const onTranscriptRef = useRef(onTranscript); + configRef.current = config; + onTranscriptRef.current = onTranscript; + + const cleanupCapture = useCallback(() => { + for (const interval of intervalsRef.current) window.clearInterval(interval); + intervalsRef.current = []; + if (timeoutRef.current !== null) window.clearTimeout(timeoutRef.current); + timeoutRef.current = null; + streamRef.current?.getTracks().forEach((track) => track.stop()); + streamRef.current = null; + void audioContextRef.current?.close(); + audioContextRef.current = null; + recorderRef.current = null; + }, []); + + useEffect(() => { + mountedRef.current = true; + return () => { + mountedRef.current = false; + const recorder = recorderRef.current; + if (recorder?.state === "recording") recorder.stop(); + cleanupCapture(); + }; + }, [cleanupCapture]); + + const stop = useCallback(() => { + const recorder = recorderRef.current; + if (recorder?.state === "recording") recorder.stop(); + }, []); + + const start = useCallback(async () => { + if (status !== "idle") return; + setError(null); + if (!navigator.mediaDevices?.getUserMedia || typeof MediaRecorder === "undefined") { + setError("Microphone recording is not supported on this device."); + return; + } + + try { + const stream = await navigator.mediaDevices.getUserMedia({ + audio: { echoCancellation: true, noiseSuppression: true, autoGainControl: true }, + }); + if (!mountedRef.current) { + stream.getTracks().forEach((track) => track.stop()); + return; + } + + const mimeType = supportedMimeType(); + const recorder = new MediaRecorder(stream, mimeType ? { mimeType } : undefined); + const chunks: Blob[] = []; + streamRef.current = stream; + recorderRef.current = recorder; + recorder.addEventListener("dataavailable", (event) => { + if (event.data.size > 0) chunks.push(event.data); + }); + recorder.addEventListener("error", () => { + if (mountedRef.current) setError("The microphone stopped unexpectedly."); + }); + recorder.addEventListener("stop", () => { + const blob = new Blob(chunks, { type: recorder.mimeType || mimeType || "audio/webm" }); + cleanupCapture(); + if (!mountedRef.current) return; + setStatus("transcribing"); + void transcribeVoiceRecording(blob, configRef.current) + .then((text) => { + if (!mountedRef.current) return; + if (text) onTranscriptRef.current(text); + setStatus("idle"); + setElapsedMs(0); + }) + .catch((cause: unknown) => { + if (!mountedRef.current) return; + setError(cause instanceof Error ? cause.message : "Voice transcription failed."); + setStatus("idle"); + }); + }); + + const audioContext = new AudioContext(); + audioContextRef.current = audioContext; + const analyser = audioContext.createAnalyser(); + analyser.fftSize = 128; + audioContext.createMediaStreamSource(stream).connect(analyser); + const samples = new Uint8Array(analyser.frequencyBinCount); + intervalsRef.current.push( + window.setInterval(() => { + analyser.getByteFrequencyData(samples); + setLevels( + Array.from({ length: LEVEL_COUNT }, (_, index) => { + const sampleIndex = Math.floor((index / LEVEL_COUNT) * samples.length); + return Math.max(0.1, (samples[sampleIndex] ?? 0) / 255); + }), + ); + }, 80), + ); + startedAtRef.current = Date.now(); + intervalsRef.current.push( + window.setInterval(() => setElapsedMs(Date.now() - startedAtRef.current), 250), + ); + timeoutRef.current = window.setTimeout(() => recorder.stop(), MAX_RECORDING_MS); + recorder.start(250); + setStatus("recording"); + } catch (cause) { + cleanupCapture(); + setStatus("idle"); + setError( + cause instanceof DOMException && cause.name === "NotAllowedError" + ? "Microphone permission was denied. Allow access and try again." + : "Could not start the microphone.", + ); + } + }, [cleanupCapture, status]); + + return { status, elapsedMs, levels, error, start, stop } as const; +} diff --git a/apps/web/src/lib/voiceTranscription.ts b/apps/web/src/lib/voiceTranscription.ts new file mode 100644 index 00000000000..05499b4c4fc --- /dev/null +++ b/apps/web/src/lib/voiceTranscription.ts @@ -0,0 +1,47 @@ +import type { VoiceTranscriptionProvider } from "@t3tools/contracts"; + +import { readDesktopPrimaryBearerToken } from "../environments/primary/desktopAuth"; +import { resolvePrimaryEnvironmentHttpUrl } from "../environments/primary/target"; + +export const GROQ_TRANSCRIPTION_BASE_URL = "https://api.groq.com/openai/v1"; +export const GROQ_TRANSCRIPTION_MODEL = "whisper-large-v3-turbo"; + +export interface VoiceTranscriptionConfig { + readonly provider: VoiceTranscriptionProvider; + readonly baseUrl: string; + readonly model: string; + readonly apiKey: string; +} + +export async function transcribeVoiceRecording( + audio: Blob, + config: VoiceTranscriptionConfig, +): Promise { + const bearerToken = await readDesktopPrimaryBearerToken(); + const response = await globalThis.fetch(resolvePrimaryEnvironmentHttpUrl("/api/transcription"), { + method: "POST", + credentials: bearerToken ? "omit" : "include", + headers: { + ...(bearerToken ? { authorization: `Bearer ${bearerToken}` } : {}), + "content-type": audio.type || "audio/webm", + "x-t3-transcription-base-url": config.baseUrl, + "x-t3-transcription-model": config.model, + ...(config.apiKey ? { "x-t3-transcription-api-key": config.apiKey } : {}), + }, + body: audio, + }); + + const payload = (await response.json().catch(() => null)) as { + readonly text?: unknown; + readonly error?: unknown; + } | null; + if (!response.ok) { + throw new Error( + typeof payload?.error === "string" ? payload.error : "Voice transcription failed.", + ); + } + if (typeof payload?.text !== "string") { + throw new Error("The transcription response did not contain text."); + } + return payload.text.trim(); +} diff --git a/packages/contracts/src/settings.ts b/packages/contracts/src/settings.ts index 7edda2e52e5..13b21c21a8d 100644 --- a/packages/contracts/src/settings.ts +++ b/packages/contracts/src/settings.ts @@ -62,6 +62,11 @@ export const EnvironmentIdentificationMode = Schema.Literals(["artwork", "pill", export type EnvironmentIdentificationMode = typeof EnvironmentIdentificationMode.Type; export const DEFAULT_ENVIRONMENT_IDENTIFICATION_MODE: EnvironmentIdentificationMode = "artwork"; +export const VoiceTranscriptionProvider = Schema.Literals(["local", "groq", "custom"]); +export type VoiceTranscriptionProvider = typeof VoiceTranscriptionProvider.Type; +export const DEFAULT_VOICE_TRANSCRIPTION_BASE_URL = "http://127.0.0.1:8080/v1"; +export const DEFAULT_VOICE_TRANSCRIPTION_MODEL = "whisper-1"; + export const ClientSettingsSchema = Schema.Struct({ autoOpenPlanSidebar: Schema.Boolean.pipe(Schema.withDecodingDefault(Effect.succeed(false))), confirmThreadArchive: Schema.Boolean.pipe(Schema.withDecodingDefault(Effect.succeed(false))), @@ -130,6 +135,17 @@ export const ClientSettingsSchema = Schema.Struct({ timestampFormat: TimestampFormat.pipe( Schema.withDecodingDefault(Effect.succeed(DEFAULT_TIMESTAMP_FORMAT)), ), + voiceTranscriptionEnabled: Schema.Boolean.pipe(Schema.withDecodingDefault(Effect.succeed(false))), + voiceTranscriptionProvider: VoiceTranscriptionProvider.pipe( + Schema.withDecodingDefault(Effect.succeed("local" as const)), + ), + voiceTranscriptionBaseUrl: TrimmedString.pipe( + Schema.withDecodingDefault(Effect.succeed(DEFAULT_VOICE_TRANSCRIPTION_BASE_URL)), + ), + voiceTranscriptionModel: TrimmedString.pipe( + Schema.withDecodingDefault(Effect.succeed(DEFAULT_VOICE_TRANSCRIPTION_MODEL)), + ), + voiceTranscriptionApiKey: Schema.String.pipe(Schema.withDecodingDefault(Effect.succeed(""))), wordWrap: Schema.Boolean.pipe(Schema.withDecodingDefault(Effect.succeed(true))), }); export type ClientSettings = typeof ClientSettingsSchema.Type; @@ -712,6 +728,11 @@ export const ClientSettingsPatch = Schema.Struct({ sidebarV2Enabled: Schema.optionalKey(Schema.Boolean), sidebarV2ConfiguredByUser: Schema.optionalKey(Schema.Boolean), timestampFormat: Schema.optionalKey(TimestampFormat), + voiceTranscriptionEnabled: Schema.optionalKey(Schema.Boolean), + voiceTranscriptionProvider: Schema.optionalKey(VoiceTranscriptionProvider), + voiceTranscriptionBaseUrl: Schema.optionalKey(TrimmedString), + voiceTranscriptionModel: Schema.optionalKey(TrimmedString), + voiceTranscriptionApiKey: Schema.optionalKey(Schema.String), wordWrap: Schema.optionalKey(Schema.Boolean), }); export type ClientSettingsPatch = typeof ClientSettingsPatch.Type; diff --git a/scripts/build-desktop-artifact.test.ts b/scripts/build-desktop-artifact.test.ts index 7d2b7410a9e..d6c53c5be9f 100644 --- a/scripts/build-desktop-artifact.test.ts +++ b/scripts/build-desktop-artifact.test.ts @@ -349,6 +349,9 @@ it.layer(NodeServices.layer)("build-desktop-artifact", (it) => { assert.notProperty(mac, "asarUnpack"); assert.notProperty(linux, "asarUnpack"); assert.deepStrictEqual(win.asarUnpack, WINDOWS_ASAR_UNPACK); + assert.deepStrictEqual((mac.mac as Record).extendInfo, { + NSMicrophoneUsageDescription: "T3 Code uses the microphone for voice dictation.", + }); // Linux must register the renderer schemes so the generated .desktop // entry advertises MimeType=x-scheme-handler/t3code; for OAuth deep links. assert.deepStrictEqual((linux.linux as Record).protocols, [ diff --git a/scripts/build-desktop-artifact.ts b/scripts/build-desktop-artifact.ts index a3c99969256..970b523944f 100644 --- a/scripts/build-desktop-artifact.ts +++ b/scripts/build-desktop-artifact.ts @@ -1568,6 +1568,9 @@ export const createBuildConfig = Effect.fn("createBuildConfig")(function* ( target: target === "dmg" ? [target, "zip"] : [target], icon: "icon.icns", category: "public.app-category.developer-tools", + extendInfo: { + NSMicrophoneUsageDescription: "T3 Code uses the microphone for voice dictation.", + }, protocols: [ { name: "T3 Code", From b5aa8b65cca6480e1fe8cb0e8d63a2a4c8198e29 Mon Sep 17 00:00:00 2001 From: "t3-code[bot]" <269035359+t3-code[bot]@users.noreply.github.com> Date: Sun, 2 Aug 2026 04:53:37 +0000 Subject: [PATCH 2/6] fix: simplify voice dictation providers --- .../settings/DesktopClientSettings.test.ts | 4 +- apps/server/src/http.ts | 52 ++-- apps/server/src/httpCors.ts | 3 +- apps/server/src/server.test.ts | 4 + apps/server/src/transcription.test.ts | 96 ++++---- apps/server/src/transcription.ts | 223 +++++++++++------- apps/web/src/components/chat/ChatComposer.tsx | 4 +- .../components/settings/BetaSettingsPanel.tsx | 92 +------- apps/web/src/hooks/useVoiceTranscription.ts | 21 +- apps/web/src/lib/voiceTranscription.ts | 13 +- packages/contracts/src/settings.ts | 18 +- 11 files changed, 261 insertions(+), 269 deletions(-) diff --git a/apps/desktop/src/settings/DesktopClientSettings.test.ts b/apps/desktop/src/settings/DesktopClientSettings.test.ts index baa5499d995..27185a7acad 100644 --- a/apps/desktop/src/settings/DesktopClientSettings.test.ts +++ b/apps/desktop/src/settings/DesktopClientSettings.test.ts @@ -34,9 +34,7 @@ const clientSettings: ClientSettings = { sidebarV2ConfiguredByUser: false, timestampFormat: "24-hour", voiceTranscriptionEnabled: true, - voiceTranscriptionProvider: "local", - voiceTranscriptionBaseUrl: "http://127.0.0.1:8080/v1", - voiceTranscriptionModel: "whisper-1", + voiceTranscriptionProvider: "openai", voiceTranscriptionApiKey: "", wordWrap: true, }; diff --git a/apps/server/src/http.ts b/apps/server/src/http.ts index 0272f6d7012..22372757ec3 100644 --- a/apps/server/src/http.ts +++ b/apps/server/src/http.ts @@ -40,7 +40,13 @@ import { } from "./auth/http.ts"; import * as ServerEnvironment from "./environment/ServerEnvironment.ts"; import { browserApiCorsAllowedHeaders, browserApiCorsAllowedMethods } from "./httpCors.ts"; -import { forwardWhisperTranscription, MAX_TRANSCRIPTION_AUDIO_BYTES } from "./transcription.ts"; +import { + forwardVoiceTranscription, + MAX_TRANSCRIPTION_AUDIO_BYTES, + readTranscriptionAudio, + resolveTranscriptionProvider, + TranscriptionInputError, +} from "./transcription.ts"; const OTLP_TRACES_PROXY_PATH = "/api/observability/v1/traces"; const TRANSCRIPTION_PATH = "/api/transcription"; @@ -263,32 +269,38 @@ export const transcriptionRouteLayer = HttpRouter.add( ); } - const body = yield* request.arrayBuffer; - const audio = new Uint8Array(body); - const result = yield* Effect.tryPromise(() => - forwardWhisperTranscription({ - audio, - audioMimeType: request.headers["content-type"] ?? "audio/webm", - baseUrl: request.headers["x-t3-transcription-base-url"] ?? "", - model: request.headers["x-t3-transcription-model"] ?? "", - apiKey: request.headers["x-t3-transcription-api-key"] ?? "", - }), - ).pipe( - Effect.orElseSucceed(() => ({ - ok: false as const, - status: 502, - message: "Transcription failed unexpectedly.", - })), + const provider = resolveTranscriptionProvider( + request.headers["x-t3-transcription-provider"] ?? "", ); + if (!provider) { + return yield* new TranscriptionInputError({ reason: "invalid_provider" }); + } - return result.ok - ? HttpServerResponse.jsonUnsafe({ text: result.text }) - : HttpServerResponse.jsonUnsafe({ error: result.message }, { status: result.status }); + const audio = yield* readTranscriptionAudio(request.stream); + const text = yield* forwardVoiceTranscription({ + audio, + audioMimeType: request.headers["content-type"] ?? "audio/webm", + provider, + apiKey: request.headers["x-t3-transcription-api-key"] ?? "", + }); + return HttpServerResponse.jsonUnsafe({ text }); }).pipe( Effect.catchTags({ EnvironmentAuthInvalidError: HttpServerRespondable.toResponse, EnvironmentInternalError: HttpServerRespondable.toResponse, EnvironmentScopeRequiredError: HttpServerRespondable.toResponse, + TranscriptionAudioTooLargeError: (error) => + Effect.succeed(HttpServerResponse.jsonUnsafe({ error: error.message }, { status: 413 })), + TranscriptionBodyReadError: (error) => + Effect.succeed(HttpServerResponse.jsonUnsafe({ error: error.message }, { status: 400 })), + TranscriptionInputError: (error) => + Effect.succeed(HttpServerResponse.jsonUnsafe({ error: error.message }, { status: 400 })), + TranscriptionProviderError: (error) => + Effect.succeed(HttpServerResponse.jsonUnsafe({ error: error.message }, { status: 502 })), + TranscriptionRequestError: (error) => + Effect.succeed(HttpServerResponse.jsonUnsafe({ error: error.message }, { status: 502 })), + TranscriptionResponseError: (error) => + Effect.succeed(HttpServerResponse.jsonUnsafe({ error: error.message }, { status: 502 })), }), ), ); diff --git a/apps/server/src/httpCors.ts b/apps/server/src/httpCors.ts index b41f65e1fb1..a29d8f271d3 100644 --- a/apps/server/src/httpCors.ts +++ b/apps/server/src/httpCors.ts @@ -6,8 +6,7 @@ export const browserApiCorsAllowedHeaders = [ "content-type", "dpop", "x-t3-transcription-api-key", - "x-t3-transcription-base-url", - "x-t3-transcription-model", + "x-t3-transcription-provider", ] as const; export const browserApiCorsHeaders = { diff --git a/apps/server/src/server.test.ts b/apps/server/src/server.test.ts index 74d3fd2d594..52cc50835d2 100644 --- a/apps/server/src/server.test.ts +++ b/apps/server/src/server.test.ts @@ -1300,6 +1300,8 @@ const assertBrowserApiCorsPreflightHeaders = ( "content-type", "dpop", "traceparent", + "x-t3-transcription-api-key", + "x-t3-transcription-provider", ]); }; const crossOriginClientOrigin = "http://remote-client.test:3773"; @@ -4209,6 +4211,8 @@ it.layer(NodeServices.layer)("server router seam", (it) => { "content-type", "dpop", "traceparent", + "x-t3-transcription-api-key", + "x-t3-transcription-provider", ]); }).pipe(Effect.provide(NodeHttpServer.layerTest)), ); diff --git a/apps/server/src/transcription.test.ts b/apps/server/src/transcription.test.ts index 44dd0a135e4..3bbd526e0f5 100644 --- a/apps/server/src/transcription.test.ts +++ b/apps/server/src/transcription.test.ts @@ -1,66 +1,50 @@ import { expect, it } from "@effect/vitest"; -import { describe, vi } from "vite-plus/test"; +import * as Effect from "effect/Effect"; +import * as Stream from "effect/Stream"; +import { describe } from "vite-plus/test"; -import { forwardWhisperTranscription, resolveWhisperTranscriptionUrl } from "./transcription.ts"; +import { + MAX_TRANSCRIPTION_AUDIO_BYTES, + readTranscriptionAudio, + resolveTranscriptionProvider, + transcriptionProviderConfig, +} from "./transcription.ts"; -describe("resolveWhisperTranscriptionUrl", () => { - it("adds the OpenAI-compatible transcription path", () => { - expect(resolveWhisperTranscriptionUrl("http://127.0.0.1:8080/v1/")?.toString()).toBe( - "http://127.0.0.1:8080/v1/audio/transcriptions", - ); - expect( - resolveWhisperTranscriptionUrl( - "https://api.groq.com/openai/v1/audio/transcriptions", - )?.toString(), - ).toBe("https://api.groq.com/openai/v1/audio/transcriptions"); +describe("transcription providers", () => { + it("uses a fixed OpenAI endpoint and model", () => { + expect(resolveTranscriptionProvider("openai")).toBe("openai"); + expect(transcriptionProviderConfig("openai")).toEqual({ + endpoint: "https://api.openai.com/v1/audio/transcriptions", + model: "gpt-4o-mini-transcribe", + }); }); - it("rejects non-http and credential-bearing URLs", () => { - expect(resolveWhisperTranscriptionUrl("file:///tmp/whisper")).toBeNull(); - expect(resolveWhisperTranscriptionUrl("https://user:pass@example.com/v1")).toBeNull(); + it("uses a fixed Groq endpoint and rejects custom providers", () => { + expect(resolveTranscriptionProvider("groq")).toBe("groq"); + expect(transcriptionProviderConfig("groq")).toEqual({ + endpoint: "https://api.groq.com/openai/v1/audio/transcriptions", + model: "whisper-large-v3-turbo", + }); + expect(resolveTranscriptionProvider("http://127.0.0.1:8080/v1")).toBeNull(); }); }); -describe("forwardWhisperTranscription", () => { - it("sends OpenAI-compatible multipart audio with BYOK authorization", async () => { - const fetchImpl = vi.fn(async (_input: string | URL | Request, init?: RequestInit) => { - expect(init?.headers).toEqual({ authorization: "Bearer groq-key" }); - expect(init?.body).toBeInstanceOf(FormData); - const form = init?.body as FormData; - expect(form.get("model")).toBe("whisper-large-v3-turbo"); - expect(form.get("file")).toBeInstanceOf(Blob); - return Response.json({ text: "hello from the microphone" }); - }); - - await expect( - forwardWhisperTranscription( - { - audio: new Uint8Array([1, 2, 3]), - audioMimeType: "audio/webm;codecs=opus", - baseUrl: "https://api.groq.com/openai/v1", - model: "whisper-large-v3-turbo", - apiKey: "groq-key", - }, - fetchImpl, - ), - ).resolves.toEqual({ ok: true, text: "hello from the microphone" }); - }); +describe("readTranscriptionAudio", () => { + it.effect("combines streamed audio chunks", () => + Effect.gen(function* () { + const audio = yield* readTranscriptionAudio( + Stream.make(new Uint8Array([1, 2]), new Uint8Array([3, 4])), + ); + expect(Array.from(audio)).toEqual([1, 2, 3, 4]); + }), + ); - it("returns a safe provider error", async () => { - const fetchImpl = vi.fn(async () => - Response.json({ error: { message: "bad key" } }, { status: 401 }), - ); - await expect( - forwardWhisperTranscription( - { - audio: new Uint8Array([1]), - audioMimeType: "audio/webm", - baseUrl: "https://example.com/v1", - model: "whisper-1", - apiKey: "bad", - }, - fetchImpl, - ), - ).resolves.toEqual({ ok: false, status: 502, message: "bad key" }); - }); + it.effect("stops when streamed audio exceeds the limit", () => + Effect.gen(function* () { + const error = yield* readTranscriptionAudio( + Stream.make(new Uint8Array(MAX_TRANSCRIPTION_AUDIO_BYTES), new Uint8Array([1])), + ).pipe(Effect.flip); + expect(error._tag).toBe("TranscriptionAudioTooLargeError"); + }), + ); }); diff --git a/apps/server/src/transcription.ts b/apps/server/src/transcription.ts index 5d380bafa88..3fc111289f8 100644 --- a/apps/server/src/transcription.ts +++ b/apps/server/src/transcription.ts @@ -1,38 +1,127 @@ -const TRANSCRIPTION_PATH = "/audio/transcriptions"; +import type { VoiceTranscriptionProvider } from "@t3tools/contracts"; +import * as Effect from "effect/Effect"; +import * as Schema from "effect/Schema"; +import * as Stream from "effect/Stream"; +import { HttpClient, HttpClientRequest, HttpClientResponse } from "effect/unstable/http"; export const MAX_TRANSCRIPTION_AUDIO_BYTES = 25 * 1024 * 1024; -export interface WhisperTranscriptionInput { +const PROVIDERS = { + openai: { + endpoint: "https://api.openai.com/v1/audio/transcriptions", + model: "gpt-4o-mini-transcribe", + }, + groq: { + endpoint: "https://api.groq.com/openai/v1/audio/transcriptions", + model: "whisper-large-v3-turbo", + }, +} as const satisfies Record; + +export interface VoiceTranscriptionInput { readonly audio: Uint8Array; readonly audioMimeType: string; - readonly baseUrl: string; - readonly model: string; + readonly provider: VoiceTranscriptionProvider; readonly apiKey: string; } -type TranscriptionFetch = (input: string | URL | Request, init?: RequestInit) => Promise; - -export type WhisperTranscriptionResult = - | { readonly ok: true; readonly text: string } - | { readonly ok: false; readonly status: number; readonly message: string }; - -export function resolveWhisperTranscriptionUrl(baseUrl: string): URL | null { - try { - const url = new URL(baseUrl.trim()); - if (url.protocol !== "http:" && url.protocol !== "https:") return null; - if (url.username || url.password) return null; - url.search = ""; - url.hash = ""; - url.pathname = url.pathname.replace(/\/$/, ""); - if (!url.pathname.endsWith(TRANSCRIPTION_PATH)) { - url.pathname += TRANSCRIPTION_PATH; - } - return url; - } catch { - return null; +export class TranscriptionAudioTooLargeError extends Schema.TaggedErrorClass()( + "TranscriptionAudioTooLargeError", + { receivedBytes: Schema.Number }, +) { + override get message(): string { + return "The recording exceeds the 25 MB limit."; + } +} + +export class TranscriptionBodyReadError extends Schema.TaggedErrorClass()( + "TranscriptionBodyReadError", + { cause: Schema.Defect() }, +) { + override get message(): string { + return "Could not read the recording."; + } +} + +export class TranscriptionInputError extends Schema.TaggedErrorClass()( + "TranscriptionInputError", + { reason: Schema.Literals(["empty_audio", "invalid_provider", "missing_api_key"]) }, +) { + override get message(): string { + if (this.reason === "empty_audio") return "The recording was empty."; + if (this.reason === "invalid_provider") return "Select OpenAI or Groq for transcription."; + return "Add an API key for the selected transcription provider."; + } +} + +export class TranscriptionRequestError extends Schema.TaggedErrorClass()( + "TranscriptionRequestError", + { + provider: Schema.String, + cause: Schema.Defect(), + }, +) { + override get message(): string { + return "Could not reach the transcription provider."; } } +export class TranscriptionProviderError extends Schema.TaggedErrorClass()( + "TranscriptionProviderError", + { + provider: Schema.String, + providerStatus: Schema.Int, + }, +) { + override get message(): string { + return "The transcription provider rejected the request. Check the provider and API key."; + } +} + +export class TranscriptionResponseError extends Schema.TaggedErrorClass()( + "TranscriptionResponseError", + { + provider: Schema.String, + cause: Schema.Defect(), + }, +) { + override get message(): string { + return "The transcription provider returned an invalid response."; + } +} + +const TranscriptionResponse = Schema.Struct({ text: Schema.String }); + +export function resolveTranscriptionProvider(provider: string): VoiceTranscriptionProvider | null { + return provider === "openai" || provider === "groq" ? provider : null; +} + +export function transcriptionProviderConfig(provider: VoiceTranscriptionProvider) { + return PROVIDERS[provider]; +} + +export const readTranscriptionAudio = (stream: Stream.Stream) => + stream.pipe( + Stream.mapError((cause) => new TranscriptionBodyReadError({ cause })), + Stream.runFoldEffect( + () => ({ chunks: [] as Uint8Array[], size: 0 }), + (accumulator, chunk) => { + const size = accumulator.size + chunk.byteLength; + return size > MAX_TRANSCRIPTION_AUDIO_BYTES + ? Effect.fail(new TranscriptionAudioTooLargeError({ receivedBytes: size })) + : Effect.succeed({ chunks: [...accumulator.chunks, chunk], size }); + }, + ), + Effect.map(({ chunks, size }) => { + const audio = new Uint8Array(size); + let offset = 0; + for (const chunk of chunks) { + audio.set(chunk, offset); + offset += chunk.byteLength; + } + return audio; + }), + ); + function audioFileExtension(mimeType: string): string { if (mimeType.includes("ogg")) return "ogg"; if (mimeType.includes("mp4") || mimeType.includes("m4a")) return "m4a"; @@ -40,74 +129,50 @@ function audioFileExtension(mimeType: string): string { return "webm"; } -export async function forwardWhisperTranscription( - input: WhisperTranscriptionInput, - fetchImpl: TranscriptionFetch = globalThis.fetch, -): Promise { - const endpoint = resolveWhisperTranscriptionUrl(input.baseUrl); - if (!endpoint) { - return { ok: false, status: 400, message: "Enter a valid HTTP transcription endpoint." }; - } - if (!input.model.trim()) { - return { ok: false, status: 400, message: "Enter a transcription model." }; - } +export const forwardVoiceTranscription = Effect.fn("voiceTranscription.forward")(function* ( + input: VoiceTranscriptionInput, +) { if (input.audio.byteLength === 0) { - return { ok: false, status: 400, message: "The recording was empty." }; + return yield* new TranscriptionInputError({ reason: "empty_audio" }); } if (input.audio.byteLength > MAX_TRANSCRIPTION_AUDIO_BYTES) { - return { ok: false, status: 413, message: "The recording exceeds the 25 MB limit." }; + return yield* new TranscriptionAudioTooLargeError({ + receivedBytes: input.audio.byteLength, + }); + } + const apiKey = input.apiKey.trim(); + if (!apiKey) { + return yield* new TranscriptionInputError({ reason: "missing_api_key" }); } + const providerConfig = transcriptionProviderConfig(input.provider); const mimeType = input.audioMimeType.split(";", 1)[0]?.trim() || "audio/webm"; const form = new FormData(); - form.set("model", input.model.trim()); + form.set("model", providerConfig.model); form.set( "file", new Blob([input.audio], { type: mimeType }), `recording.${audioFileExtension(mimeType)}`, ); - let response: Response; - try { - response = await fetchImpl(endpoint, { - method: "POST", - headers: input.apiKey ? { authorization: `Bearer ${input.apiKey}` } : undefined, - body: form, - signal: AbortSignal.timeout(120_000), - }); - } catch { - return { ok: false, status: 502, message: "Could not reach the transcription provider." }; - } - - let payload: unknown; - try { - payload = await response.json(); - } catch { - return { ok: false, status: 502, message: "The transcription provider returned invalid JSON." }; - } + const httpClient = yield* HttpClient.HttpClient; + const response = yield* HttpClientRequest.post(providerConfig.endpoint).pipe( + HttpClientRequest.bearerToken(apiKey), + HttpClientRequest.bodyFormData(form), + httpClient.execute, + Effect.timeout("2 minutes"), + Effect.mapError((cause) => new TranscriptionRequestError({ provider: input.provider, cause })), + ); - if (!response.ok) { - const providerMessage = - typeof payload === "object" && - payload !== null && - "error" in payload && - typeof payload.error === "object" && - payload.error !== null && - "message" in payload.error && - typeof payload.error.message === "string" - ? payload.error.message - : null; - return { - ok: false, - status: 502, - message: providerMessage?.slice(0, 300) || "The transcription provider rejected the request.", - }; + if (response.status < 200 || response.status >= 300) { + return yield* new TranscriptionProviderError({ + provider: input.provider, + providerStatus: response.status, + }); } - const text = - typeof payload === "object" && payload !== null && "text" in payload ? payload.text : undefined; - if (typeof text !== "string") { - return { ok: false, status: 502, message: "The transcription response did not contain text." }; - } - return { ok: true, text: text.trim() }; -} + const payload = yield* HttpClientResponse.schemaBodyJson(TranscriptionResponse)(response).pipe( + Effect.mapError((cause) => new TranscriptionResponseError({ provider: input.provider, cause })), + ); + return payload.text.trim(); +}); diff --git a/apps/web/src/components/chat/ChatComposer.tsx b/apps/web/src/components/chat/ChatComposer.tsx index dfcdbeeb402..033987bd04c 100644 --- a/apps/web/src/components/chat/ChatComposer.tsx +++ b/apps/web/src/components/chat/ChatComposer.tsx @@ -1289,8 +1289,6 @@ export const ChatComposer = memo(function ChatComposer(props: ChatComposerProps) const voiceTranscription = useVoiceTranscription({ config: { provider: settings.voiceTranscriptionProvider, - baseUrl: settings.voiceTranscriptionBaseUrl, - model: settings.voiceTranscriptionModel, apiKey: settings.voiceTranscriptionApiKey, }, onTranscript: appendVoiceTranscript, @@ -3129,7 +3127,7 @@ export const ChatComposer = memo(function ChatComposer(props: ChatComposerProps)
- {settings.voiceTranscriptionEnabled && voiceTranscription.status !== "idle" ? ( + {voiceTranscription.status !== "idle" ? ( settings.voiceTranscriptionProvider, ); - const voiceTranscriptionBaseUrl = useClientSettings( - (settings) => settings.voiceTranscriptionBaseUrl, - ); - const voiceTranscriptionModel = useClientSettings((settings) => settings.voiceTranscriptionModel); const voiceTranscriptionApiKey = useClientSettings( (settings) => settings.voiceTranscriptionApiKey, ); const updateSettings = useUpdateClientSettings(); - const selectVoiceProvider = (provider: VoiceTranscriptionProvider) => { - if (provider === "local") { - updateSettings({ - voiceTranscriptionProvider: provider, - voiceTranscriptionBaseUrl: DEFAULT_VOICE_TRANSCRIPTION_BASE_URL, - voiceTranscriptionModel: DEFAULT_VOICE_TRANSCRIPTION_MODEL, - }); - return; - } - if (provider === "groq") { - updateSettings({ - voiceTranscriptionProvider: provider, - voiceTranscriptionBaseUrl: GROQ_TRANSCRIPTION_BASE_URL, - voiceTranscriptionModel: GROQ_TRANSCRIPTION_MODEL, - }); - return; - } - updateSettings({ voiceTranscriptionProvider: provider }); - }; - return ( @@ -174,72 +142,36 @@ export function BetaSettingsPanel() { <> - selectVoiceProvider(value as VoiceTranscriptionProvider) + updateSettings({ + voiceTranscriptionProvider: value as VoiceTranscriptionProvider, + voiceTranscriptionApiKey: "", + }) } > - {voiceTranscriptionProvider === "local" - ? "Local" - : voiceTranscriptionProvider === "groq" - ? "Groq" - : "Custom"} + {voiceTranscriptionProvider === "openai" ? "OpenAI" : "Groq"} - - Local + + OpenAI Groq - - Custom - } /> - updateSettings({ voiceTranscriptionBaseUrl: event.target.value }) - } - placeholder="http://127.0.0.1:8080/v1" - aria-label="Transcription endpoint" - spellCheck={false} - /> - } - /> - - updateSettings({ voiceTranscriptionModel: event.target.value }) - } - placeholder="whisper-1" - aria-label="Transcription model" - spellCheck={false} - /> - } - /> - updateSettings({ voiceTranscriptionApiKey: event.target.value }) } - placeholder={voiceTranscriptionProvider === "local" ? "Optional" : "Required"} + placeholder="Required" aria-label="Transcription API key" /> } diff --git a/apps/web/src/hooks/useVoiceTranscription.ts b/apps/web/src/hooks/useVoiceTranscription.ts index 061739cef37..4902b46399f 100644 --- a/apps/web/src/hooks/useVoiceTranscription.ts +++ b/apps/web/src/hooks/useVoiceTranscription.ts @@ -25,6 +25,7 @@ export function useVoiceTranscription({ const [levels, setLevels] = useState(() => Array(LEVEL_COUNT).fill(0.12)); const [error, setError] = useState(null); const recorderRef = useRef(null); + const startingRef = useRef(false); const streamRef = useRef(null); const audioContextRef = useRef(null); const intervalsRef = useRef([]); @@ -52,6 +53,7 @@ export function useVoiceTranscription({ mountedRef.current = true; return () => { mountedRef.current = false; + startingRef.current = false; const recorder = recorderRef.current; if (recorder?.state === "recording") recorder.stop(); cleanupCapture(); @@ -64,9 +66,11 @@ export function useVoiceTranscription({ }, []); const start = useCallback(async () => { - if (status !== "idle") return; + if (startingRef.current || status !== "idle") return; + startingRef.current = true; setError(null); if (!navigator.mediaDevices?.getUserMedia || typeof MediaRecorder === "undefined") { + startingRef.current = false; setError("Microphone recording is not supported on this device."); return; } @@ -77,24 +81,31 @@ export function useVoiceTranscription({ }); if (!mountedRef.current) { stream.getTracks().forEach((track) => track.stop()); + startingRef.current = false; return; } + streamRef.current = stream; const mimeType = supportedMimeType(); const recorder = new MediaRecorder(stream, mimeType ? { mimeType } : undefined); const chunks: Blob[] = []; - streamRef.current = stream; + let recordingFailed = false; recorderRef.current = recorder; recorder.addEventListener("dataavailable", (event) => { if (event.data.size > 0) chunks.push(event.data); }); recorder.addEventListener("error", () => { - if (mountedRef.current) setError("The microphone stopped unexpectedly."); + recordingFailed = true; + cleanupCapture(); + if (mountedRef.current) { + setStatus("idle"); + setError("The microphone stopped unexpectedly."); + } }); recorder.addEventListener("stop", () => { const blob = new Blob(chunks, { type: recorder.mimeType || mimeType || "audio/webm" }); cleanupCapture(); - if (!mountedRef.current) return; + if (!mountedRef.current || recordingFailed) return; setStatus("transcribing"); void transcribeVoiceRecording(blob, configRef.current) .then((text) => { @@ -133,8 +144,10 @@ export function useVoiceTranscription({ ); timeoutRef.current = window.setTimeout(() => recorder.stop(), MAX_RECORDING_MS); recorder.start(250); + startingRef.current = false; setStatus("recording"); } catch (cause) { + startingRef.current = false; cleanupCapture(); setStatus("idle"); setError( diff --git a/apps/web/src/lib/voiceTranscription.ts b/apps/web/src/lib/voiceTranscription.ts index 05499b4c4fc..8fcee72cb17 100644 --- a/apps/web/src/lib/voiceTranscription.ts +++ b/apps/web/src/lib/voiceTranscription.ts @@ -3,13 +3,8 @@ import type { VoiceTranscriptionProvider } from "@t3tools/contracts"; import { readDesktopPrimaryBearerToken } from "../environments/primary/desktopAuth"; import { resolvePrimaryEnvironmentHttpUrl } from "../environments/primary/target"; -export const GROQ_TRANSCRIPTION_BASE_URL = "https://api.groq.com/openai/v1"; -export const GROQ_TRANSCRIPTION_MODEL = "whisper-large-v3-turbo"; - export interface VoiceTranscriptionConfig { readonly provider: VoiceTranscriptionProvider; - readonly baseUrl: string; - readonly model: string; readonly apiKey: string; } @@ -17,6 +12,9 @@ export async function transcribeVoiceRecording( audio: Blob, config: VoiceTranscriptionConfig, ): Promise { + const apiKey = config.apiKey.trim(); + if (!apiKey) throw new Error("Add an API key for the selected transcription provider."); + const bearerToken = await readDesktopPrimaryBearerToken(); const response = await globalThis.fetch(resolvePrimaryEnvironmentHttpUrl("/api/transcription"), { method: "POST", @@ -24,9 +22,8 @@ export async function transcribeVoiceRecording( headers: { ...(bearerToken ? { authorization: `Bearer ${bearerToken}` } : {}), "content-type": audio.type || "audio/webm", - "x-t3-transcription-base-url": config.baseUrl, - "x-t3-transcription-model": config.model, - ...(config.apiKey ? { "x-t3-transcription-api-key": config.apiKey } : {}), + "x-t3-transcription-provider": config.provider, + "x-t3-transcription-api-key": apiKey, }, body: audio, }); diff --git a/packages/contracts/src/settings.ts b/packages/contracts/src/settings.ts index 13b21c21a8d..a5fde0637fe 100644 --- a/packages/contracts/src/settings.ts +++ b/packages/contracts/src/settings.ts @@ -62,10 +62,8 @@ export const EnvironmentIdentificationMode = Schema.Literals(["artwork", "pill", export type EnvironmentIdentificationMode = typeof EnvironmentIdentificationMode.Type; export const DEFAULT_ENVIRONMENT_IDENTIFICATION_MODE: EnvironmentIdentificationMode = "artwork"; -export const VoiceTranscriptionProvider = Schema.Literals(["local", "groq", "custom"]); +export const VoiceTranscriptionProvider = Schema.Literals(["openai", "groq"]); export type VoiceTranscriptionProvider = typeof VoiceTranscriptionProvider.Type; -export const DEFAULT_VOICE_TRANSCRIPTION_BASE_URL = "http://127.0.0.1:8080/v1"; -export const DEFAULT_VOICE_TRANSCRIPTION_MODEL = "whisper-1"; export const ClientSettingsSchema = Schema.Struct({ autoOpenPlanSidebar: Schema.Boolean.pipe(Schema.withDecodingDefault(Effect.succeed(false))), @@ -137,15 +135,9 @@ export const ClientSettingsSchema = Schema.Struct({ ), voiceTranscriptionEnabled: Schema.Boolean.pipe(Schema.withDecodingDefault(Effect.succeed(false))), voiceTranscriptionProvider: VoiceTranscriptionProvider.pipe( - Schema.withDecodingDefault(Effect.succeed("local" as const)), + Schema.withDecodingDefault(Effect.succeed("openai" as const)), ), - voiceTranscriptionBaseUrl: TrimmedString.pipe( - Schema.withDecodingDefault(Effect.succeed(DEFAULT_VOICE_TRANSCRIPTION_BASE_URL)), - ), - voiceTranscriptionModel: TrimmedString.pipe( - Schema.withDecodingDefault(Effect.succeed(DEFAULT_VOICE_TRANSCRIPTION_MODEL)), - ), - voiceTranscriptionApiKey: Schema.String.pipe(Schema.withDecodingDefault(Effect.succeed(""))), + voiceTranscriptionApiKey: TrimmedString.pipe(Schema.withDecodingDefault(Effect.succeed(""))), wordWrap: Schema.Boolean.pipe(Schema.withDecodingDefault(Effect.succeed(true))), }); export type ClientSettings = typeof ClientSettingsSchema.Type; @@ -730,9 +722,7 @@ export const ClientSettingsPatch = Schema.Struct({ timestampFormat: Schema.optionalKey(TimestampFormat), voiceTranscriptionEnabled: Schema.optionalKey(Schema.Boolean), voiceTranscriptionProvider: Schema.optionalKey(VoiceTranscriptionProvider), - voiceTranscriptionBaseUrl: Schema.optionalKey(TrimmedString), - voiceTranscriptionModel: Schema.optionalKey(TrimmedString), - voiceTranscriptionApiKey: Schema.optionalKey(Schema.String), + voiceTranscriptionApiKey: Schema.optionalKey(TrimmedString), wordWrap: Schema.optionalKey(Schema.Boolean), }); export type ClientSettingsPatch = typeof ClientSettingsPatch.Type; From bf3ecf0e4051a69aebb9a1b704432220c8e01946 Mon Sep 17 00:00:00 2001 From: "t3-code[bot]" <269035359+t3-code[bot]@users.noreply.github.com> Date: Sun, 2 Aug 2026 04:56:48 +0000 Subject: [PATCH 3/6] fix: split transcription input errors --- apps/server/src/http.ts | 12 +++++++++--- apps/server/src/transcription.ts | 30 +++++++++++++++++++++++------- 2 files changed, 32 insertions(+), 10 deletions(-) diff --git a/apps/server/src/http.ts b/apps/server/src/http.ts index 22372757ec3..fbc9db32860 100644 --- a/apps/server/src/http.ts +++ b/apps/server/src/http.ts @@ -45,7 +45,9 @@ import { MAX_TRANSCRIPTION_AUDIO_BYTES, readTranscriptionAudio, resolveTranscriptionProvider, - TranscriptionInputError, + TranscriptionApiKeyMissingError, + TranscriptionEmptyAudioError, + TranscriptionProviderUnsupportedError, } from "./transcription.ts"; const OTLP_TRACES_PROXY_PATH = "/api/observability/v1/traces"; @@ -273,7 +275,7 @@ export const transcriptionRouteLayer = HttpRouter.add( request.headers["x-t3-transcription-provider"] ?? "", ); if (!provider) { - return yield* new TranscriptionInputError({ reason: "invalid_provider" }); + return yield* new TranscriptionProviderUnsupportedError(); } const audio = yield* readTranscriptionAudio(request.stream); @@ -291,12 +293,16 @@ export const transcriptionRouteLayer = HttpRouter.add( EnvironmentScopeRequiredError: HttpServerRespondable.toResponse, TranscriptionAudioTooLargeError: (error) => Effect.succeed(HttpServerResponse.jsonUnsafe({ error: error.message }, { status: 413 })), + TranscriptionApiKeyMissingError: (error) => + Effect.succeed(HttpServerResponse.jsonUnsafe({ error: error.message }, { status: 400 })), TranscriptionBodyReadError: (error) => Effect.succeed(HttpServerResponse.jsonUnsafe({ error: error.message }, { status: 400 })), - TranscriptionInputError: (error) => + TranscriptionEmptyAudioError: (error) => Effect.succeed(HttpServerResponse.jsonUnsafe({ error: error.message }, { status: 400 })), TranscriptionProviderError: (error) => Effect.succeed(HttpServerResponse.jsonUnsafe({ error: error.message }, { status: 502 })), + TranscriptionProviderUnsupportedError: (error) => + Effect.succeed(HttpServerResponse.jsonUnsafe({ error: error.message }, { status: 400 })), TranscriptionRequestError: (error) => Effect.succeed(HttpServerResponse.jsonUnsafe({ error: error.message }, { status: 502 })), TranscriptionResponseError: (error) => diff --git a/apps/server/src/transcription.ts b/apps/server/src/transcription.ts index 3fc111289f8..ee488408be3 100644 --- a/apps/server/src/transcription.ts +++ b/apps/server/src/transcription.ts @@ -42,13 +42,29 @@ export class TranscriptionBodyReadError extends Schema.TaggedErrorClass()( - "TranscriptionInputError", - { reason: Schema.Literals(["empty_audio", "invalid_provider", "missing_api_key"]) }, +export class TranscriptionEmptyAudioError extends Schema.TaggedErrorClass()( + "TranscriptionEmptyAudioError", + {}, +) { + override get message(): string { + return "The recording was empty."; + } +} + +export class TranscriptionProviderUnsupportedError extends Schema.TaggedErrorClass()( + "TranscriptionProviderUnsupportedError", + {}, +) { + override get message(): string { + return "Select OpenAI or Groq for transcription."; + } +} + +export class TranscriptionApiKeyMissingError extends Schema.TaggedErrorClass()( + "TranscriptionApiKeyMissingError", + {}, ) { override get message(): string { - if (this.reason === "empty_audio") return "The recording was empty."; - if (this.reason === "invalid_provider") return "Select OpenAI or Groq for transcription."; return "Add an API key for the selected transcription provider."; } } @@ -133,7 +149,7 @@ export const forwardVoiceTranscription = Effect.fn("voiceTranscription.forward") input: VoiceTranscriptionInput, ) { if (input.audio.byteLength === 0) { - return yield* new TranscriptionInputError({ reason: "empty_audio" }); + return yield* new TranscriptionEmptyAudioError(); } if (input.audio.byteLength > MAX_TRANSCRIPTION_AUDIO_BYTES) { return yield* new TranscriptionAudioTooLargeError({ @@ -142,7 +158,7 @@ export const forwardVoiceTranscription = Effect.fn("voiceTranscription.forward") } const apiKey = input.apiKey.trim(); if (!apiKey) { - return yield* new TranscriptionInputError({ reason: "missing_api_key" }); + return yield* new TranscriptionApiKeyMissingError(); } const providerConfig = transcriptionProviderConfig(input.provider); From dbbe34921979c7235d80e6fc62f9936b1f251e7b Mon Sep 17 00:00:00 2001 From: "t3-code[bot]" <269035359+t3-code[bot]@users.noreply.github.com> Date: Sun, 2 Aug 2026 05:05:42 +0000 Subject: [PATCH 4/6] fix: bound transcription response processing --- apps/server/src/transcription.ts | 41 +++++++++++++++++++------------- 1 file changed, 25 insertions(+), 16 deletions(-) diff --git a/apps/server/src/transcription.ts b/apps/server/src/transcription.ts index ee488408be3..dfdea7415a0 100644 --- a/apps/server/src/transcription.ts +++ b/apps/server/src/transcription.ts @@ -122,9 +122,11 @@ export const readTranscriptionAudio = (stream: Stream.Stream ({ chunks: [] as Uint8Array[], size: 0 }), (accumulator, chunk) => { const size = accumulator.size + chunk.byteLength; - return size > MAX_TRANSCRIPTION_AUDIO_BYTES - ? Effect.fail(new TranscriptionAudioTooLargeError({ receivedBytes: size })) - : Effect.succeed({ chunks: [...accumulator.chunks, chunk], size }); + if (size > MAX_TRANSCRIPTION_AUDIO_BYTES) { + return Effect.fail(new TranscriptionAudioTooLargeError({ receivedBytes: size })); + } + accumulator.chunks.push(chunk); + return Effect.succeed({ chunks: accumulator.chunks, size }); }, ), Effect.map(({ chunks, size }) => { @@ -172,23 +174,30 @@ export const forwardVoiceTranscription = Effect.fn("voiceTranscription.forward") ); const httpClient = yield* HttpClient.HttpClient; - const response = yield* HttpClientRequest.post(providerConfig.endpoint).pipe( + const payload = yield* HttpClientRequest.post(providerConfig.endpoint).pipe( HttpClientRequest.bearerToken(apiKey), HttpClientRequest.bodyFormData(form), httpClient.execute, - Effect.timeout("2 minutes"), Effect.mapError((cause) => new TranscriptionRequestError({ provider: input.provider, cause })), - ); - - if (response.status < 200 || response.status >= 300) { - return yield* new TranscriptionProviderError({ - provider: input.provider, - providerStatus: response.status, - }); - } - - const payload = yield* HttpClientResponse.schemaBodyJson(TranscriptionResponse)(response).pipe( - Effect.mapError((cause) => new TranscriptionResponseError({ provider: input.provider, cause })), + Effect.flatMap((response) => + Effect.gen(function* () { + if (response.status < 200 || response.status >= 300) { + return yield* new TranscriptionProviderError({ + provider: input.provider, + providerStatus: response.status, + }); + } + return yield* HttpClientResponse.schemaBodyJson(TranscriptionResponse)(response).pipe( + Effect.mapError( + (cause) => new TranscriptionResponseError({ provider: input.provider, cause }), + ), + ); + }), + ), + Effect.timeout("2 minutes"), + Effect.catchTag("TimeoutError", (cause) => + Effect.fail(new TranscriptionRequestError({ provider: input.provider, cause })), + ), ); return payload.text.trim(); }); From ecbafb6d5e2bc0d96ac7763bbcff37f40eacdff1 Mon Sep 17 00:00:00 2001 From: "t3-code[bot]" <269035359+t3-code[bot]@users.noreply.github.com> Date: Sun, 2 Aug 2026 05:09:45 +0000 Subject: [PATCH 5/6] fix: follow effect recovery conventions --- apps/server/src/transcription.ts | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/apps/server/src/transcription.ts b/apps/server/src/transcription.ts index dfdea7415a0..f5ecc6033e1 100644 --- a/apps/server/src/transcription.ts +++ b/apps/server/src/transcription.ts @@ -195,9 +195,10 @@ export const forwardVoiceTranscription = Effect.fn("voiceTranscription.forward") }), ), Effect.timeout("2 minutes"), - Effect.catchTag("TimeoutError", (cause) => - Effect.fail(new TranscriptionRequestError({ provider: input.provider, cause })), - ), + Effect.catchTags({ + TimeoutError: (cause) => + Effect.fail(new TranscriptionRequestError({ provider: input.provider, cause })), + }), ); return payload.text.trim(); }); From 14315f5b3af3a4ba05f479ce2e5c79389a1d731b Mon Sep 17 00:00:00 2001 From: maria-rcks Date: Sun, 2 Aug 2026 19:49:34 -0400 Subject: [PATCH 6/6] feat: improve voice dictation configuration and recording --- .../settings/DesktopClientSettings.test.ts | 1 + apps/server/src/http.ts | 71 +++++++- apps/server/src/httpCors.ts | 1 + apps/server/src/server.test.ts | 2 + apps/server/src/transcription.test.ts | 84 +++++++++- apps/server/src/transcription.ts | 106 +++++++++++- apps/web/src/components/chat/ChatComposer.tsx | 56 +++++-- .../chat/VoiceTranscriptionPanel.tsx | 67 ++++---- .../components/settings/BetaSettingsPanel.tsx | 155 +++++++++++++++++- apps/web/src/hooks/useVoiceTranscription.ts | 71 ++++++-- apps/web/src/lib/voiceTranscription.test.ts | 70 ++++++++ apps/web/src/lib/voiceTranscription.ts | 87 +++++++++- docs/README.md | 1 + docs/user/voice-dictation.md | 28 ++++ packages/contracts/src/settings.test.ts | 16 ++ packages/contracts/src/settings.ts | 2 + 16 files changed, 720 insertions(+), 98 deletions(-) create mode 100644 apps/web/src/lib/voiceTranscription.test.ts create mode 100644 docs/user/voice-dictation.md diff --git a/apps/desktop/src/settings/DesktopClientSettings.test.ts b/apps/desktop/src/settings/DesktopClientSettings.test.ts index 27185a7acad..38bba6fa3ae 100644 --- a/apps/desktop/src/settings/DesktopClientSettings.test.ts +++ b/apps/desktop/src/settings/DesktopClientSettings.test.ts @@ -36,6 +36,7 @@ const clientSettings: ClientSettings = { voiceTranscriptionEnabled: true, voiceTranscriptionProvider: "openai", voiceTranscriptionApiKey: "", + voiceTranscriptionModel: "", wordWrap: true, }; diff --git a/apps/server/src/http.ts b/apps/server/src/http.ts index fbc9db32860..6afc5063cbb 100644 --- a/apps/server/src/http.ts +++ b/apps/server/src/http.ts @@ -42,16 +42,17 @@ import * as ServerEnvironment from "./environment/ServerEnvironment.ts"; import { browserApiCorsAllowedHeaders, browserApiCorsAllowedMethods } from "./httpCors.ts"; import { forwardVoiceTranscription, + listVoiceTranscriptionModels, MAX_TRANSCRIPTION_AUDIO_BYTES, readTranscriptionAudio, resolveTranscriptionProvider, - TranscriptionApiKeyMissingError, - TranscriptionEmptyAudioError, + transcriptionEnvironmentApiKeyStatus, TranscriptionProviderUnsupportedError, } from "./transcription.ts"; const OTLP_TRACES_PROXY_PATH = "/api/observability/v1/traces"; const TRANSCRIPTION_PATH = "/api/transcription"; +const TRANSCRIPTION_MODELS_PATH = "/api/transcription/models"; const LOOPBACK_HOSTNAMES = new Set(["127.0.0.1", "::1", "localhost"]); const DESKTOP_RENDERER_ORIGINS = ["t3code://app", "t3code-dev://app"]; const GZIP_MIN_BYTES = 1024; @@ -257,7 +258,26 @@ export const otlpTracesProxyRouteLayer = HttpRouter.add( ), ); -export const transcriptionRouteLayer = HttpRouter.add( +const transcriptionConfigRouteLayer = HttpRouter.add( + "GET", + TRANSCRIPTION_PATH, + Effect.gen(function* () { + yield* authenticateRawRouteWithScope(AuthOrchestrationOperateScope); + const [openai, groq] = yield* Effect.all([ + transcriptionEnvironmentApiKeyStatus("openai"), + transcriptionEnvironmentApiKeyStatus("groq"), + ]); + return HttpServerResponse.jsonUnsafe({ openai, groq }); + }).pipe( + Effect.catchTags({ + EnvironmentAuthInvalidError: HttpServerRespondable.toResponse, + EnvironmentInternalError: HttpServerRespondable.toResponse, + EnvironmentScopeRequiredError: HttpServerRespondable.toResponse, + }), + ), +); + +const transcriptionUploadRouteLayer = HttpRouter.add( "POST", TRANSCRIPTION_PATH, Effect.gen(function* () { @@ -284,6 +304,7 @@ export const transcriptionRouteLayer = HttpRouter.add( audioMimeType: request.headers["content-type"] ?? "audio/webm", provider, apiKey: request.headers["x-t3-transcription-api-key"] ?? "", + model: request.headers["x-t3-transcription-model"] ?? "", }); return HttpServerResponse.jsonUnsafe({ text }); }).pipe( @@ -299,6 +320,44 @@ export const transcriptionRouteLayer = HttpRouter.add( Effect.succeed(HttpServerResponse.jsonUnsafe({ error: error.message }, { status: 400 })), TranscriptionEmptyAudioError: (error) => Effect.succeed(HttpServerResponse.jsonUnsafe({ error: error.message }, { status: 400 })), + TranscriptionModelMissingError: (error) => + Effect.succeed(HttpServerResponse.jsonUnsafe({ error: error.message }, { status: 400 })), + TranscriptionProviderError: (error) => + Effect.succeed(HttpServerResponse.jsonUnsafe({ error: error.message }, { status: 502 })), + TranscriptionProviderUnsupportedError: (error) => + Effect.succeed(HttpServerResponse.jsonUnsafe({ error: error.message }, { status: 400 })), + TranscriptionRequestError: (error) => + Effect.succeed(HttpServerResponse.jsonUnsafe({ error: error.message }, { status: 502 })), + TranscriptionResponseError: (error) => + Effect.succeed(HttpServerResponse.jsonUnsafe({ error: error.message }, { status: 502 })), + }), + ), +); + +const transcriptionModelsRouteLayer = HttpRouter.add( + "GET", + TRANSCRIPTION_MODELS_PATH, + Effect.gen(function* () { + yield* authenticateRawRouteWithScope(AuthOrchestrationOperateScope); + const request = yield* HttpServerRequest.HttpServerRequest; + const provider = resolveTranscriptionProvider( + request.headers["x-t3-transcription-provider"] ?? "", + ); + if (!provider) { + return yield* new TranscriptionProviderUnsupportedError(); + } + const models = yield* listVoiceTranscriptionModels({ + provider, + apiKey: request.headers["x-t3-transcription-api-key"] ?? "", + }); + return HttpServerResponse.jsonUnsafe({ models }); + }).pipe( + Effect.catchTags({ + EnvironmentAuthInvalidError: HttpServerRespondable.toResponse, + EnvironmentInternalError: HttpServerRespondable.toResponse, + EnvironmentScopeRequiredError: HttpServerRespondable.toResponse, + TranscriptionApiKeyMissingError: (error) => + Effect.succeed(HttpServerResponse.jsonUnsafe({ error: error.message }, { status: 400 })), TranscriptionProviderError: (error) => Effect.succeed(HttpServerResponse.jsonUnsafe({ error: error.message }, { status: 502 })), TranscriptionProviderUnsupportedError: (error) => @@ -311,6 +370,12 @@ export const transcriptionRouteLayer = HttpRouter.add( ), ); +export const transcriptionRouteLayer = Layer.mergeAll( + transcriptionConfigRouteLayer, + transcriptionModelsRouteLayer, + transcriptionUploadRouteLayer, +); + export const assetRouteLayer = HttpRouter.add( "GET", `${ASSET_ROUTE_PREFIX}/*`, diff --git a/apps/server/src/httpCors.ts b/apps/server/src/httpCors.ts index a29d8f271d3..6212a958e16 100644 --- a/apps/server/src/httpCors.ts +++ b/apps/server/src/httpCors.ts @@ -6,6 +6,7 @@ export const browserApiCorsAllowedHeaders = [ "content-type", "dpop", "x-t3-transcription-api-key", + "x-t3-transcription-model", "x-t3-transcription-provider", ] as const; diff --git a/apps/server/src/server.test.ts b/apps/server/src/server.test.ts index 52cc50835d2..295d005fb19 100644 --- a/apps/server/src/server.test.ts +++ b/apps/server/src/server.test.ts @@ -1301,6 +1301,7 @@ const assertBrowserApiCorsPreflightHeaders = ( "dpop", "traceparent", "x-t3-transcription-api-key", + "x-t3-transcription-model", "x-t3-transcription-provider", ]); }; @@ -4212,6 +4213,7 @@ it.layer(NodeServices.layer)("server router seam", (it) => { "dpop", "traceparent", "x-t3-transcription-api-key", + "x-t3-transcription-model", "x-t3-transcription-provider", ]); }).pipe(Effect.provide(NodeHttpServer.layerTest)), diff --git a/apps/server/src/transcription.test.ts b/apps/server/src/transcription.test.ts index 3bbd526e0f5..85e954eec89 100644 --- a/apps/server/src/transcription.test.ts +++ b/apps/server/src/transcription.test.ts @@ -1,32 +1,102 @@ import { expect, it } from "@effect/vitest"; +import * as ConfigProvider from "effect/ConfigProvider"; import * as Effect from "effect/Effect"; import * as Stream from "effect/Stream"; +import { HttpClient, HttpClientRequest, HttpClientResponse } from "effect/unstable/http"; import { describe } from "vite-plus/test"; import { + forwardVoiceTranscription, + listVoiceTranscriptionModels, MAX_TRANSCRIPTION_AUDIO_BYTES, readTranscriptionAudio, resolveTranscriptionProvider, + transcriptionEnvironmentApiKeyStatus, transcriptionProviderConfig, } from "./transcription.ts"; describe("transcription providers", () => { - it("uses a fixed OpenAI endpoint and model", () => { + it("uses fixed OpenAI and Groq configurations", () => { expect(resolveTranscriptionProvider("openai")).toBe("openai"); expect(transcriptionProviderConfig("openai")).toEqual({ endpoint: "https://api.openai.com/v1/audio/transcriptions", - model: "gpt-4o-mini-transcribe", + modelsEndpoint: "https://api.openai.com/v1/models", + apiKeyEnvironmentVariable: "OPENAI_API_KEY", }); - }); - - it("uses a fixed Groq endpoint and rejects custom providers", () => { expect(resolveTranscriptionProvider("groq")).toBe("groq"); expect(transcriptionProviderConfig("groq")).toEqual({ endpoint: "https://api.groq.com/openai/v1/audio/transcriptions", - model: "whisper-large-v3-turbo", + modelsEndpoint: "https://api.groq.com/openai/v1/models", + apiKeyEnvironmentVariable: "GROQ_API_KEY", }); - expect(resolveTranscriptionProvider("http://127.0.0.1:8080/v1")).toBeNull(); + expect(resolveTranscriptionProvider("custom")).toBeNull(); }); + + it.effect("loads accessible transcription models with the provider API key", () => + Effect.gen(function* () { + let capturedRequest: HttpClientRequest.HttpClientRequest | undefined; + const client = HttpClient.make((request) => + Effect.sync(() => { + capturedRequest = request; + return HttpClientResponse.fromWeb( + request, + Response.json({ + data: [ + { id: "gpt-4o" }, + { id: " whisper-1 " }, + { id: "gpt-4o-mini-transcribe" }, + { id: "gpt-4o-mini-transcribe" }, + ], + }), + ); + }), + ); + + const models = yield* listVoiceTranscriptionModels({ + provider: "openai", + apiKey: "client-openai-key", + }).pipe(Effect.provideService(HttpClient.HttpClient, client)); + + expect(models).toEqual(["gpt-4o-mini-transcribe", "whisper-1"]); + expect(capturedRequest?.url).toBe("https://api.openai.com/v1/models"); + expect(capturedRequest?.headers.authorization).toBe("Bearer client-openai-key"); + }), + ); + + it.effect("uses a provider API key from the server environment", () => + Effect.gen(function* () { + let capturedRequest: HttpClientRequest.HttpClientRequest | undefined; + const client = HttpClient.make((request) => + Effect.sync(() => { + capturedRequest = request; + return HttpClientResponse.fromWeb(request, Response.json({ text: " groq transcript " })); + }), + ); + + const transcript = yield* forwardVoiceTranscription({ + audio: new Uint8Array([1, 2, 3]), + audioMimeType: "audio/webm;codecs=opus", + provider: "groq", + apiKey: "", + model: "whisper-large-v3", + }).pipe(Effect.provideService(HttpClient.HttpClient, client)); + + expect(yield* transcriptionEnvironmentApiKeyStatus("groq")).toBe(true); + expect(yield* transcriptionEnvironmentApiKeyStatus("openai")).toBe(false); + expect(transcript).toBe("groq transcript"); + expect(capturedRequest?.url).toBe("https://api.groq.com/openai/v1/audio/transcriptions"); + expect(capturedRequest?.headers.authorization).toBe("Bearer env-groq-key"); + expect(capturedRequest?.body._tag).toBe("FormData"); + if (capturedRequest?.body._tag === "FormData") { + expect(capturedRequest.body.formData.get("model")).toBe("whisper-large-v3"); + expect(capturedRequest.body.formData.get("file")).toBeInstanceOf(Blob); + } + }).pipe( + Effect.provide( + ConfigProvider.layer(ConfigProvider.fromEnv({ env: { GROQ_API_KEY: "env-groq-key" } })), + ), + ), + ); }); describe("readTranscriptionAudio", () => { diff --git a/apps/server/src/transcription.ts b/apps/server/src/transcription.ts index f5ecc6033e1..1b167c18123 100644 --- a/apps/server/src/transcription.ts +++ b/apps/server/src/transcription.ts @@ -1,4 +1,5 @@ import type { VoiceTranscriptionProvider } from "@t3tools/contracts"; +import * as Config from "effect/Config"; import * as Effect from "effect/Effect"; import * as Schema from "effect/Schema"; import * as Stream from "effect/Stream"; @@ -9,19 +10,30 @@ export const MAX_TRANSCRIPTION_AUDIO_BYTES = 25 * 1024 * 1024; const PROVIDERS = { openai: { endpoint: "https://api.openai.com/v1/audio/transcriptions", - model: "gpt-4o-mini-transcribe", + modelsEndpoint: "https://api.openai.com/v1/models", + apiKeyEnvironmentVariable: "OPENAI_API_KEY", }, groq: { endpoint: "https://api.groq.com/openai/v1/audio/transcriptions", - model: "whisper-large-v3-turbo", + modelsEndpoint: "https://api.groq.com/openai/v1/models", + apiKeyEnvironmentVariable: "GROQ_API_KEY", }, -} as const satisfies Record; +} as const satisfies Record< + VoiceTranscriptionProvider, + { endpoint: string; modelsEndpoint: string; apiKeyEnvironmentVariable: string } +>; export interface VoiceTranscriptionInput { readonly audio: Uint8Array; readonly audioMimeType: string; readonly provider: VoiceTranscriptionProvider; readonly apiKey: string; + readonly model: string; +} + +export interface VoiceTranscriptionModelsInput { + readonly provider: VoiceTranscriptionProvider; + readonly apiKey: string; } export class TranscriptionAudioTooLargeError extends Schema.TaggedErrorClass()( @@ -65,7 +77,16 @@ export class TranscriptionApiKeyMissingError extends Schema.TaggedErrorClass()( + "TranscriptionModelMissingError", + {}, +) { + override get message(): string { + return "Select a transcription model."; } } @@ -106,6 +127,10 @@ export class TranscriptionResponseError extends Schema.TaggedErrorClass 0; +}); + +const resolveTranscriptionApiKey = Effect.fn("voiceTranscription.resolveApiKey")(function* ( + input: VoiceTranscriptionModelsInput, +) { + const providerConfig = transcriptionProviderConfig(input.provider); + const apiKey = + input.apiKey.trim() || + (yield* Config.string(providerConfig.apiKeyEnvironmentVariable).pipe( + Config.withDefault(""), + )).trim(); + if (!apiKey) { + return yield* new TranscriptionApiKeyMissingError(); + } + return apiKey; +}); + +export const listVoiceTranscriptionModels = Effect.fn("voiceTranscription.listModels")(function* ( + input: VoiceTranscriptionModelsInput, +) { + const providerConfig = transcriptionProviderConfig(input.provider); + const apiKey = yield* resolveTranscriptionApiKey(input); + const httpClient = yield* HttpClient.HttpClient; + const payload = yield* HttpClientRequest.get(providerConfig.modelsEndpoint).pipe( + HttpClientRequest.bearerToken(apiKey), + httpClient.execute, + Effect.mapError((cause) => new TranscriptionRequestError({ provider: input.provider, cause })), + Effect.flatMap((response) => + Effect.gen(function* () { + if (response.status < 200 || response.status >= 300) { + return yield* new TranscriptionProviderError({ + provider: input.provider, + providerStatus: response.status, + }); + } + return yield* HttpClientResponse.schemaBodyJson(ModelsResponse)(response).pipe( + Effect.mapError( + (cause) => new TranscriptionResponseError({ provider: input.provider, cause }), + ), + ); + }), + ), + Effect.timeout("30 seconds"), + Effect.catchTags({ + TimeoutError: (cause) => + Effect.fail(new TranscriptionRequestError({ provider: input.provider, cause })), + }), + ); + + const models = [...new Set(payload.data.map(({ id }) => id.trim()).filter(Boolean))].sort(); + const transcriptionModels = models.filter((model) => TRANSCRIPTION_MODEL_ID_PATTERN.test(model)); + return transcriptionModels.length > 0 ? transcriptionModels : models; +}); + export const readTranscriptionAudio = (stream: Stream.Stream) => stream.pipe( Stream.mapError((cause) => new TranscriptionBodyReadError({ cause })), @@ -158,15 +245,16 @@ export const forwardVoiceTranscription = Effect.fn("voiceTranscription.forward") receivedBytes: input.audio.byteLength, }); } - const apiKey = input.apiKey.trim(); - if (!apiKey) { - return yield* new TranscriptionApiKeyMissingError(); + const providerConfig = transcriptionProviderConfig(input.provider); + const model = input.model.trim(); + if (!model) { + return yield* new TranscriptionModelMissingError(); } + const apiKey = yield* resolveTranscriptionApiKey(input); - const providerConfig = transcriptionProviderConfig(input.provider); const mimeType = input.audioMimeType.split(";", 1)[0]?.trim() || "audio/webm"; const form = new FormData(); - form.set("model", providerConfig.model); + form.set("model", model); form.set( "file", new Blob([input.audio], { type: mimeType }), diff --git a/apps/web/src/components/chat/ChatComposer.tsx b/apps/web/src/components/chat/ChatComposer.tsx index 033987bd04c..b27b368eb90 100644 --- a/apps/web/src/components/chat/ChatComposer.tsx +++ b/apps/web/src/components/chat/ChatComposer.tsx @@ -182,6 +182,7 @@ import { PenLineIcon, SparklesIcon, MicIcon, + SquareIcon, XIcon, } from "lucide-react"; import { proposedPlanTitle } from "../../proposedPlan"; @@ -1290,6 +1291,7 @@ export const ChatComposer = memo(function ChatComposer(props: ChatComposerProps) config: { provider: settings.voiceTranscriptionProvider, apiKey: settings.voiceTranscriptionApiKey, + model: settings.voiceTranscriptionModel, }, onTranscript: appendVoiceTranscript, }); @@ -3127,14 +3129,9 @@ export const ChatComposer = memo(function ChatComposer(props: ChatComposerProps) - {voiceTranscription.status !== "idle" ? ( - - ) : settings.voiceTranscriptionEnabled && voiceTranscription.error ? ( + {voiceTranscription.status === "idle" && + settings.voiceTranscriptionEnabled && + voiceTranscription.error ? (

{voiceTranscription.error}

@@ -3160,7 +3157,20 @@ export const ChatComposer = memo(function ChatComposer(props: ChatComposerProps) showMobilePendingAnswerActions && "hidden sm:flex", )} > -
+ {voiceTranscription.status !== "idle" ? ( + + ) : null} +
{noProviderAvailable ? ( } /> - Start dictation + + {voiceTranscription.status === "recording" + ? "Stop dictation" + : "Start dictation"} + ) : null} `voice-waveform-${index}`); function formatElapsed(elapsedMs: number): string { const seconds = Math.floor(elapsedMs / 1_000); @@ -14,55 +11,61 @@ export function VoiceTranscriptionPanel({ status, elapsedMs, levels, - onStop, }: { readonly status: Exclude; readonly elapsedMs: number; readonly levels: readonly number[]; - readonly onStop: () => void; }) { + const waveformPath = levels + .map((level, index) => { + if (level <= 0.01) return ""; + const amplitude = Math.max(1.75, level * 14); + return `M ${index + 0.5} ${16 - amplitude} V ${16 + amplitude}`; + }) + .join(" "); + return (
{status === "recording" ? ( - + + + ) : (
- transcribing with your configured provider... + Transcribing…
)} {formatElapsed(elapsedMs)} - {status === "recording" ? ( - - ) : null}
); } diff --git a/apps/web/src/components/settings/BetaSettingsPanel.tsx b/apps/web/src/components/settings/BetaSettingsPanel.tsx index a6c29a9129a..2cd183feebe 100644 --- a/apps/web/src/components/settings/BetaSettingsPanel.tsx +++ b/apps/web/src/components/settings/BetaSettingsPanel.tsx @@ -6,6 +6,10 @@ import { useSidebarV2Enabled, useUpdateClientSettings, } from "../../hooks/useSettings"; +import { + listVoiceTranscriptionModels, + readVoiceTranscriptionEnvironmentStatus, +} from "../../lib/voiceTranscription"; import { Input } from "../ui/input"; import { Select, SelectItem, SelectPopup, SelectTrigger, SelectValue } from "../ui/select"; import { Switch } from "../ui/switch"; @@ -15,6 +19,10 @@ import { searchableSetting } from "./settingsSearch"; const AUTO_SETTLE_MIN_DAYS = 1; const AUTO_SETTLE_MAX_DAYS = 90; const AUTO_SETTLE_DEFAULT_DAYS = 3; +const TRANSCRIPTION_API_KEY_ENV = { + openai: "OPENAI_API_KEY", + groq: "GROQ_API_KEY", +} as const; function AutoSettleDaysInput({ value, @@ -71,8 +79,100 @@ export function BetaSettingsPanel() { const voiceTranscriptionApiKey = useClientSettings( (settings) => settings.voiceTranscriptionApiKey, ); + const voiceTranscriptionModel = useClientSettings((settings) => settings.voiceTranscriptionModel); + const [environmentApiKeys, setEnvironmentApiKeys] = useState({ + openai: false, + groq: false, + }); + const [transcriptionModels, setTranscriptionModels] = useState([]); + const [transcriptionModelsLoading, setTranscriptionModelsLoading] = useState(false); + const [transcriptionModelsError, setTranscriptionModelsError] = useState(null); const updateSettings = useUpdateClientSettings(); + useEffect(() => { + if (!voiceTranscriptionEnabled) return; + let active = true; + void readVoiceTranscriptionEnvironmentStatus() + .then((status) => { + if (active) setEnvironmentApiKeys(status); + }) + .catch(() => undefined); + return () => { + active = false; + }; + }, [voiceTranscriptionEnabled]); + + const transcriptionProviderLabel = voiceTranscriptionProvider === "openai" ? "OpenAI" : "Groq"; + const transcriptionApiKeyEnvironmentVariable = + TRANSCRIPTION_API_KEY_ENV[voiceTranscriptionProvider]; + const hasEnvironmentApiKey = environmentApiKeys[voiceTranscriptionProvider]; + const hasTranscriptionApiKey = voiceTranscriptionApiKey.trim().length > 0 || hasEnvironmentApiKey; + + useEffect(() => { + if (!voiceTranscriptionEnabled || !hasTranscriptionApiKey) { + setTranscriptionModels([]); + setTranscriptionModelsLoading(false); + setTranscriptionModelsError(null); + return; + } + + let active = true; + setTranscriptionModelsLoading(true); + setTranscriptionModelsError(null); + const timeout = window.setTimeout( + () => { + void listVoiceTranscriptionModels({ + provider: voiceTranscriptionProvider, + apiKey: voiceTranscriptionApiKey, + }) + .then((models) => { + if (!active) return; + setTranscriptionModels(models); + setTranscriptionModelsLoading(false); + }) + .catch((cause: unknown) => { + if (!active) return; + setTranscriptionModels([]); + setTranscriptionModelsLoading(false); + setTranscriptionModelsError( + cause instanceof Error ? cause.message : "Could not load transcription models.", + ); + }); + }, + voiceTranscriptionApiKey.trim() ? 400 : 0, + ); + + return () => { + active = false; + window.clearTimeout(timeout); + }; + }, [ + hasTranscriptionApiKey, + voiceTranscriptionApiKey, + voiceTranscriptionEnabled, + voiceTranscriptionProvider, + ]); + + useEffect(() => { + if ( + transcriptionModels.length > 0 && + voiceTranscriptionModel && + !transcriptionModels.includes(voiceTranscriptionModel) + ) { + updateSettings({ voiceTranscriptionModel: "" }); + } + }, [transcriptionModels, updateSettings, voiceTranscriptionModel]); + + const transcriptionModelDescription = !hasTranscriptionApiKey + ? `Add an API key to load models available from ${transcriptionProviderLabel}.` + : transcriptionModelsLoading + ? `Loading models available from ${transcriptionProviderLabel}…` + : transcriptionModelsError + ? transcriptionModelsError + : transcriptionModels.length === 0 + ? `${transcriptionProviderLabel} did not return any models.` + : `Loaded from ${transcriptionProviderLabel} using the configured API key.`; + return ( @@ -142,7 +242,7 @@ export function BetaSettingsPanel() { <> - - {voiceTranscriptionProvider === "openai" ? "OpenAI" : "Groq"} - + {transcriptionProviderLabel} @@ -170,8 +269,12 @@ export function BetaSettingsPanel() { } /> - updateSettings({ voiceTranscriptionApiKey: event.target.value }) + updateSettings({ + voiceTranscriptionApiKey: event.target.value, + voiceTranscriptionModel: "", + }) + } + placeholder={ + hasEnvironmentApiKey + ? `Using ${transcriptionApiKeyEnvironmentVariable}` + : "Required" } - placeholder="Required" - aria-label="Transcription API key" + aria-label={`${transcriptionProviderLabel} transcription API key`} /> } /> + { + if (value !== null) updateSettings({ voiceTranscriptionModel: value }); + }} + > + + + {voiceTranscriptionModel || + (transcriptionModelsLoading ? "Loading models…" : "Select model")} + + + + {transcriptionModels.map((model) => ( + + {model} + + ))} + + + } + /> ) : null} diff --git a/apps/web/src/hooks/useVoiceTranscription.ts b/apps/web/src/hooks/useVoiceTranscription.ts index 4902b46399f..22e776dafc7 100644 --- a/apps/web/src/hooks/useVoiceTranscription.ts +++ b/apps/web/src/hooks/useVoiceTranscription.ts @@ -2,9 +2,10 @@ import { useCallback, useEffect, useRef, useState } from "react"; import { transcribeVoiceRecording, type VoiceTranscriptionConfig } from "../lib/voiceTranscription"; -const LEVEL_COUNT = 36; +const LEVEL_COUNT = 160; const MAX_RECORDING_MS = 5 * 60 * 1_000; const MIME_TYPES = ["audio/webm;codecs=opus", "audio/ogg;codecs=opus", "audio/mp4"]; +const FLAT_LEVELS = Array(LEVEL_COUNT).fill(0); export type VoiceTranscriptionStatus = "idle" | "recording" | "transcribing"; @@ -22,10 +23,11 @@ export function useVoiceTranscription({ }) { const [status, setStatus] = useState("idle"); const [elapsedMs, setElapsedMs] = useState(0); - const [levels, setLevels] = useState(() => Array(LEVEL_COUNT).fill(0.12)); + const [levels, setLevels] = useState(FLAT_LEVELS); const [error, setError] = useState(null); const recorderRef = useRef(null); const startingRef = useRef(false); + const cancelStartingRef = useRef(false); const streamRef = useRef(null); const audioContextRef = useRef(null); const intervalsRef = useRef([]); @@ -54,6 +56,7 @@ export function useVoiceTranscription({ return () => { mountedRef.current = false; startingRef.current = false; + cancelStartingRef.current = true; const recorder = recorderRef.current; if (recorder?.state === "recording") recorder.stop(); cleanupCapture(); @@ -61,27 +64,51 @@ export function useVoiceTranscription({ }, [cleanupCapture]); const stop = useCallback(() => { + if (startingRef.current) { + cancelStartingRef.current = true; + cleanupCapture(); + if (mountedRef.current) { + setStatus("idle"); + setElapsedMs(0); + } + return; + } const recorder = recorderRef.current; if (recorder?.state === "recording") recorder.stop(); - }, []); + }, [cleanupCapture]); const start = useCallback(async () => { if (startingRef.current || status !== "idle") return; startingRef.current = true; setError(null); + setElapsedMs(0); + setLevels(FLAT_LEVELS); + if (!configRef.current.model.trim()) { + startingRef.current = false; + setError("Select a transcription model in Settings."); + return; + } if (!navigator.mediaDevices?.getUserMedia || typeof MediaRecorder === "undefined") { startingRef.current = false; setError("Microphone recording is not supported on this device."); return; } + cancelStartingRef.current = false; + setStatus("recording"); try { + const audioContext = new AudioContext(); + audioContextRef.current = audioContext; + if (audioContext.state === "suspended") { + void audioContext.resume().catch(() => undefined); + } const stream = await navigator.mediaDevices.getUserMedia({ audio: { echoCancellation: true, noiseSuppression: true, autoGainControl: true }, }); - if (!mountedRef.current) { + if (!mountedRef.current || cancelStartingRef.current) { stream.getTracks().forEach((track) => track.stop()); startingRef.current = false; + cancelStartingRef.current = false; return; } @@ -121,22 +148,29 @@ export function useVoiceTranscription({ }); }); - const audioContext = new AudioContext(); - audioContextRef.current = audioContext; const analyser = audioContext.createAnalyser(); - analyser.fftSize = 128; + analyser.fftSize = 256; + const silentOutput = audioContext.createGain(); + silentOutput.gain.value = 0; audioContext.createMediaStreamSource(stream).connect(analyser); - const samples = new Uint8Array(analyser.frequencyBinCount); + analyser.connect(silentOutput); + silentOutput.connect(audioContext.destination); + if (audioContext.state === "suspended") { + void audioContext.resume().catch(() => undefined); + } + const samples = new Uint8Array(analyser.fftSize); intervalsRef.current.push( window.setInterval(() => { - analyser.getByteFrequencyData(samples); - setLevels( - Array.from({ length: LEVEL_COUNT }, (_, index) => { - const sampleIndex = Math.floor((index / LEVEL_COUNT) * samples.length); - return Math.max(0.1, (samples[sampleIndex] ?? 0) / 255); - }), - ); - }, 80), + analyser.getByteTimeDomainData(samples); + let squaredAmplitude = 0; + for (const sample of samples) { + const amplitude = (sample - 128) / 128; + squaredAmplitude += amplitude * amplitude; + } + const rootMeanSquare = Math.sqrt(squaredAmplitude / samples.length); + const nextLevel = Math.min(1, Math.max(0, (rootMeanSquare - 0.008) * 9)); + setLevels((current) => [...current.slice(1), nextLevel]); + }, 50), ); startedAtRef.current = Date.now(); intervalsRef.current.push( @@ -145,10 +179,13 @@ export function useVoiceTranscription({ timeoutRef.current = window.setTimeout(() => recorder.stop(), MAX_RECORDING_MS); recorder.start(250); startingRef.current = false; - setStatus("recording"); } catch (cause) { startingRef.current = false; cleanupCapture(); + if (cancelStartingRef.current) { + cancelStartingRef.current = false; + return; + } setStatus("idle"); setError( cause instanceof DOMException && cause.name === "NotAllowedError" diff --git a/apps/web/src/lib/voiceTranscription.test.ts b/apps/web/src/lib/voiceTranscription.test.ts new file mode 100644 index 00000000000..b82a1962f21 --- /dev/null +++ b/apps/web/src/lib/voiceTranscription.test.ts @@ -0,0 +1,70 @@ +import { afterEach, describe, expect, it, vi } from "vite-plus/test"; + +import { + listVoiceTranscriptionModels, + voiceTranscriptionRequestHeaders, +} from "./voiceTranscription"; + +afterEach(() => { + vi.unstubAllGlobals(); +}); + +describe("voiceTranscriptionRequestHeaders", () => { + it("sends a client-provided OpenAI key", () => { + expect( + voiceTranscriptionRequestHeaders("audio/webm", { + provider: "openai", + apiKey: " openai-key ", + model: " gpt-4o-mini-transcribe ", + }), + ).toEqual({ + "content-type": "audio/webm", + "x-t3-transcription-provider": "openai", + "x-t3-transcription-model": "gpt-4o-mini-transcribe", + "x-t3-transcription-api-key": "openai-key", + }); + }); + + it("omits an empty key so the server can use the provider environment variable", () => { + expect( + voiceTranscriptionRequestHeaders("audio/mp4", { + provider: "groq", + apiKey: "", + model: "whisper-large-v3", + }), + ).toEqual({ + "content-type": "audio/mp4", + "x-t3-transcription-provider": "groq", + "x-t3-transcription-model": "whisper-large-v3", + }); + }); +}); + +describe("listVoiceTranscriptionModels", () => { + it("loads the provider models through the connected T3 server", async () => { + const fetchMock = vi.fn().mockResolvedValue( + Response.json({ + models: ["gpt-4o-mini-transcribe", "whisper-1"], + }), + ); + vi.stubGlobal("window", { + location: { + href: "http://localhost:3773/settings", + origin: "http://localhost:3773", + }, + }); + vi.stubGlobal("fetch", fetchMock); + + await expect( + listVoiceTranscriptionModels({ provider: "openai", apiKey: "client-key" }), + ).resolves.toEqual(["gpt-4o-mini-transcribe", "whisper-1"]); + expect(fetchMock).toHaveBeenCalledWith("http://localhost:3773/api/transcription/models", { + method: "GET", + credentials: "include", + headers: { + "x-t3-transcription-api-key": "client-key", + "x-t3-transcription-provider": "openai", + }, + }); + }); +}); diff --git a/apps/web/src/lib/voiceTranscription.ts b/apps/web/src/lib/voiceTranscription.ts index 8fcee72cb17..99a19cb783e 100644 --- a/apps/web/src/lib/voiceTranscription.ts +++ b/apps/web/src/lib/voiceTranscription.ts @@ -6,24 +6,99 @@ import { resolvePrimaryEnvironmentHttpUrl } from "../environments/primary/target export interface VoiceTranscriptionConfig { readonly provider: VoiceTranscriptionProvider; readonly apiKey: string; + readonly model: string; +} + +export type VoiceTranscriptionProviderConfig = Omit; + +export interface VoiceTranscriptionEnvironmentStatus { + readonly openai: boolean; + readonly groq: boolean; +} + +export function voiceTranscriptionRequestHeaders( + contentType: string, + config: VoiceTranscriptionConfig, +): Record { + const apiKey = config.apiKey.trim(); + return { + "content-type": contentType, + "x-t3-transcription-provider": config.provider, + "x-t3-transcription-model": config.model.trim(), + ...(apiKey ? { "x-t3-transcription-api-key": apiKey } : {}), + }; +} + +function voiceTranscriptionProviderHeaders( + config: VoiceTranscriptionProviderConfig, +): Record { + const apiKey = config.apiKey.trim(); + return { + "x-t3-transcription-provider": config.provider, + ...(apiKey ? { "x-t3-transcription-api-key": apiKey } : {}), + }; +} + +export async function readVoiceTranscriptionEnvironmentStatus(): Promise { + const bearerToken = await readDesktopPrimaryBearerToken(); + const response = await globalThis.fetch(resolvePrimaryEnvironmentHttpUrl("/api/transcription"), { + method: "GET", + credentials: bearerToken ? "omit" : "include", + ...(bearerToken ? { headers: { authorization: `Bearer ${bearerToken}` } } : {}), + }); + if (!response.ok) throw new Error("Could not read transcription provider settings."); + + const payload = (await response.json()) as { readonly openai?: unknown; readonly groq?: unknown }; + return { + openai: payload.openai === true, + groq: payload.groq === true, + }; +} + +export async function listVoiceTranscriptionModels( + config: VoiceTranscriptionProviderConfig, +): Promise { + const bearerToken = await readDesktopPrimaryBearerToken(); + const response = await globalThis.fetch( + resolvePrimaryEnvironmentHttpUrl("/api/transcription/models"), + { + method: "GET", + credentials: bearerToken ? "omit" : "include", + headers: { + ...(bearerToken ? { authorization: `Bearer ${bearerToken}` } : {}), + ...voiceTranscriptionProviderHeaders(config), + }, + }, + ); + const payload = (await response.json().catch(() => null)) as { + readonly models?: unknown; + readonly error?: unknown; + } | null; + if (!response.ok) { + throw new Error( + typeof payload?.error === "string" ? payload.error : "Could not load transcription models.", + ); + } + if ( + !Array.isArray(payload?.models) || + !payload.models.every((model) => typeof model === "string") + ) { + throw new Error("The transcription model response was invalid."); + } + return payload.models; } export async function transcribeVoiceRecording( audio: Blob, config: VoiceTranscriptionConfig, ): Promise { - const apiKey = config.apiKey.trim(); - if (!apiKey) throw new Error("Add an API key for the selected transcription provider."); - const bearerToken = await readDesktopPrimaryBearerToken(); const response = await globalThis.fetch(resolvePrimaryEnvironmentHttpUrl("/api/transcription"), { method: "POST", credentials: bearerToken ? "omit" : "include", headers: { ...(bearerToken ? { authorization: `Bearer ${bearerToken}` } : {}), - "content-type": audio.type || "audio/webm", - "x-t3-transcription-provider": config.provider, - "x-t3-transcription-api-key": apiKey, + ...voiceTranscriptionRequestHeaders(audio.type || "audio/webm", config), }, body: audio, }); diff --git a/docs/README.md b/docs/README.md index bc359826a04..70ef5c32e2e 100644 --- a/docs/README.md +++ b/docs/README.md @@ -5,6 +5,7 @@ - [Install and first run](./user/install.md) - [Permission modes](./user/permission-modes.md) - [Keyboard shortcuts](./user/keybindings.md) +- [Voice dictation](./user/voice-dictation.md) - [Remote access](./user/remote-access.md) - [Keeping app and server in sync](./user/updating.md) - [Source control integrations](./user/source-control.md) diff --git a/docs/user/voice-dictation.md b/docs/user/voice-dictation.md new file mode 100644 index 00000000000..e34e51a6bdc --- /dev/null +++ b/docs/user/voice-dictation.md @@ -0,0 +1,28 @@ +# Voice Dictation + +Voice dictation records from the message composer and inserts the transcription into your draft. +It is available in the web and desktop clients on browsers that support microphone recording. + +Enable it in **Settings** → **Beta features** → **Voice dictation**. The composer then shows a +microphone action. Select it to start recording and stop it when you are finished. Recordings stop +automatically after five minutes. + +## Providers and API Keys + +Choose **OpenAI** or **Groq**. T3 Code supplies the provider's transcription endpoint. After an +API key is available, T3 Code loads the models that key can access from the provider and lets you +select the transcription model. Model IDs are not bundled into T3 Code, so newly available models +can appear without an app update. + +You can enter a key in the client, or configure it in the environment that runs the connected T3 +Code server: + +- OpenAI uses `OPENAI_API_KEY` +- Groq uses `GROQ_API_KEY` + +When an environment key is available, the API key input indicates that it is already configured. +A key entered in the input overrides the environment key and stays in that client's local +settings. Environment key values are never sent to the client. + +Recordings are sent through the connected T3 Code server to the selected provider. Recordings +larger than 25 MB are rejected. diff --git a/packages/contracts/src/settings.test.ts b/packages/contracts/src/settings.test.ts index 2bc61d72f21..992a65cb2e4 100644 --- a/packages/contracts/src/settings.test.ts +++ b/packages/contracts/src/settings.test.ts @@ -111,6 +111,22 @@ describe("ClientSettings sidebar v2", () => { }); }); +describe("ClientSettings voice transcription", () => { + it("defaults to OpenAI and accepts Groq", () => { + const settings = decodeClientSettings({}); + + expect(settings.voiceTranscriptionProvider).toBe("openai"); + expect(settings.voiceTranscriptionModel).toBe(""); + expect( + decodeClientSettingsPatch({ voiceTranscriptionProvider: "groq" }).voiceTranscriptionProvider, + ).toBe("groq"); + expect( + decodeClientSettingsPatch({ voiceTranscriptionModel: " whisper-large-v3 " }) + .voiceTranscriptionModel, + ).toBe("whisper-large-v3"); + }); +}); + describe("ServerSettings.providerInstances (slice-2 invariant)", () => { it("defaults to an empty record so legacy configs without the key still decode", () => { expect(DEFAULT_SERVER_SETTINGS.providerInstances).toEqual({}); diff --git a/packages/contracts/src/settings.ts b/packages/contracts/src/settings.ts index a5fde0637fe..fbb6668b951 100644 --- a/packages/contracts/src/settings.ts +++ b/packages/contracts/src/settings.ts @@ -138,6 +138,7 @@ export const ClientSettingsSchema = Schema.Struct({ Schema.withDecodingDefault(Effect.succeed("openai" as const)), ), voiceTranscriptionApiKey: TrimmedString.pipe(Schema.withDecodingDefault(Effect.succeed(""))), + voiceTranscriptionModel: TrimmedString.pipe(Schema.withDecodingDefault(Effect.succeed(""))), wordWrap: Schema.Boolean.pipe(Schema.withDecodingDefault(Effect.succeed(true))), }); export type ClientSettings = typeof ClientSettingsSchema.Type; @@ -723,6 +724,7 @@ export const ClientSettingsPatch = Schema.Struct({ voiceTranscriptionEnabled: Schema.optionalKey(Schema.Boolean), voiceTranscriptionProvider: Schema.optionalKey(VoiceTranscriptionProvider), voiceTranscriptionApiKey: Schema.optionalKey(TrimmedString), + voiceTranscriptionModel: Schema.optionalKey(TrimmedString), wordWrap: Schema.optionalKey(Schema.Boolean), }); export type ClientSettingsPatch = typeof ClientSettingsPatch.Type;