diff --git a/apps/desktop/src/settings/DesktopClientSettings.test.ts b/apps/desktop/src/settings/DesktopClientSettings.test.ts index 8d76ea83a33..38bba6fa3ae 100644 --- a/apps/desktop/src/settings/DesktopClientSettings.test.ts +++ b/apps/desktop/src/settings/DesktopClientSettings.test.ts @@ -33,6 +33,10 @@ const clientSettings: ClientSettings = { sidebarV2Enabled: false, sidebarV2ConfiguredByUser: false, timestampFormat: "24-hour", + voiceTranscriptionEnabled: true, + voiceTranscriptionProvider: "openai", + voiceTranscriptionApiKey: "", + voiceTranscriptionModel: "", wordWrap: true, }; diff --git a/apps/server/src/http.ts b/apps/server/src/http.ts index 5a380be8fe2..6afc5063cbb 100644 --- a/apps/server/src/http.ts +++ b/apps/server/src/http.ts @@ -40,8 +40,19 @@ import { } from "./auth/http.ts"; import * as ServerEnvironment from "./environment/ServerEnvironment.ts"; import { browserApiCorsAllowedHeaders, browserApiCorsAllowedMethods } from "./httpCors.ts"; +import { + forwardVoiceTranscription, + listVoiceTranscriptionModels, + MAX_TRANSCRIPTION_AUDIO_BYTES, + readTranscriptionAudio, + resolveTranscriptionProvider, + transcriptionEnvironmentApiKeyStatus, + TranscriptionProviderUnsupportedError, +} from "./transcription.ts"; const OTLP_TRACES_PROXY_PATH = "/api/observability/v1/traces"; +const TRANSCRIPTION_PATH = "/api/transcription"; +const TRANSCRIPTION_MODELS_PATH = "/api/transcription/models"; const LOOPBACK_HOSTNAMES = new Set(["127.0.0.1", "::1", "localhost"]); const DESKTOP_RENDERER_ORIGINS = ["t3code://app", "t3code-dev://app"]; const GZIP_MIN_BYTES = 1024; @@ -247,6 +258,124 @@ export const otlpTracesProxyRouteLayer = HttpRouter.add( ), ); +const transcriptionConfigRouteLayer = HttpRouter.add( + "GET", + TRANSCRIPTION_PATH, + Effect.gen(function* () { + yield* authenticateRawRouteWithScope(AuthOrchestrationOperateScope); + const [openai, groq] = yield* Effect.all([ + transcriptionEnvironmentApiKeyStatus("openai"), + transcriptionEnvironmentApiKeyStatus("groq"), + ]); + return HttpServerResponse.jsonUnsafe({ openai, groq }); + }).pipe( + Effect.catchTags({ + EnvironmentAuthInvalidError: HttpServerRespondable.toResponse, + EnvironmentInternalError: HttpServerRespondable.toResponse, + EnvironmentScopeRequiredError: HttpServerRespondable.toResponse, + }), + ), +); + +const transcriptionUploadRouteLayer = HttpRouter.add( + "POST", + TRANSCRIPTION_PATH, + Effect.gen(function* () { + yield* authenticateRawRouteWithScope(AuthOrchestrationOperateScope); + const request = yield* HttpServerRequest.HttpServerRequest; + const declaredLength = Number(request.headers["content-length"] ?? "0"); + if (Number.isFinite(declaredLength) && declaredLength > MAX_TRANSCRIPTION_AUDIO_BYTES) { + return HttpServerResponse.jsonUnsafe( + { error: "The recording exceeds the 25 MB limit." }, + { status: 413 }, + ); + } + + const provider = resolveTranscriptionProvider( + request.headers["x-t3-transcription-provider"] ?? "", + ); + if (!provider) { + return yield* new TranscriptionProviderUnsupportedError(); + } + + const audio = yield* readTranscriptionAudio(request.stream); + const text = yield* forwardVoiceTranscription({ + audio, + audioMimeType: request.headers["content-type"] ?? "audio/webm", + provider, + apiKey: request.headers["x-t3-transcription-api-key"] ?? "", + model: request.headers["x-t3-transcription-model"] ?? "", + }); + return HttpServerResponse.jsonUnsafe({ text }); + }).pipe( + Effect.catchTags({ + EnvironmentAuthInvalidError: HttpServerRespondable.toResponse, + EnvironmentInternalError: HttpServerRespondable.toResponse, + EnvironmentScopeRequiredError: HttpServerRespondable.toResponse, + TranscriptionAudioTooLargeError: (error) => + Effect.succeed(HttpServerResponse.jsonUnsafe({ error: error.message }, { status: 413 })), + TranscriptionApiKeyMissingError: (error) => + Effect.succeed(HttpServerResponse.jsonUnsafe({ error: error.message }, { status: 400 })), + TranscriptionBodyReadError: (error) => + Effect.succeed(HttpServerResponse.jsonUnsafe({ error: error.message }, { status: 400 })), + TranscriptionEmptyAudioError: (error) => + Effect.succeed(HttpServerResponse.jsonUnsafe({ error: error.message }, { status: 400 })), + TranscriptionModelMissingError: (error) => + Effect.succeed(HttpServerResponse.jsonUnsafe({ error: error.message }, { status: 400 })), + TranscriptionProviderError: (error) => + Effect.succeed(HttpServerResponse.jsonUnsafe({ error: error.message }, { status: 502 })), + TranscriptionProviderUnsupportedError: (error) => + Effect.succeed(HttpServerResponse.jsonUnsafe({ error: error.message }, { status: 400 })), + TranscriptionRequestError: (error) => + Effect.succeed(HttpServerResponse.jsonUnsafe({ error: error.message }, { status: 502 })), + TranscriptionResponseError: (error) => + Effect.succeed(HttpServerResponse.jsonUnsafe({ error: error.message }, { status: 502 })), + }), + ), +); + +const transcriptionModelsRouteLayer = HttpRouter.add( + "GET", + TRANSCRIPTION_MODELS_PATH, + Effect.gen(function* () { + yield* authenticateRawRouteWithScope(AuthOrchestrationOperateScope); + const request = yield* HttpServerRequest.HttpServerRequest; + const provider = resolveTranscriptionProvider( + request.headers["x-t3-transcription-provider"] ?? "", + ); + if (!provider) { + return yield* new TranscriptionProviderUnsupportedError(); + } + const models = yield* listVoiceTranscriptionModels({ + provider, + apiKey: request.headers["x-t3-transcription-api-key"] ?? "", + }); + return HttpServerResponse.jsonUnsafe({ models }); + }).pipe( + Effect.catchTags({ + EnvironmentAuthInvalidError: HttpServerRespondable.toResponse, + EnvironmentInternalError: HttpServerRespondable.toResponse, + EnvironmentScopeRequiredError: HttpServerRespondable.toResponse, + TranscriptionApiKeyMissingError: (error) => + Effect.succeed(HttpServerResponse.jsonUnsafe({ error: error.message }, { status: 400 })), + TranscriptionProviderError: (error) => + Effect.succeed(HttpServerResponse.jsonUnsafe({ error: error.message }, { status: 502 })), + TranscriptionProviderUnsupportedError: (error) => + Effect.succeed(HttpServerResponse.jsonUnsafe({ error: error.message }, { status: 400 })), + TranscriptionRequestError: (error) => + Effect.succeed(HttpServerResponse.jsonUnsafe({ error: error.message }, { status: 502 })), + TranscriptionResponseError: (error) => + Effect.succeed(HttpServerResponse.jsonUnsafe({ error: error.message }, { status: 502 })), + }), + ), +); + +export const transcriptionRouteLayer = Layer.mergeAll( + transcriptionConfigRouteLayer, + transcriptionModelsRouteLayer, + transcriptionUploadRouteLayer, +); + export const assetRouteLayer = HttpRouter.add( "GET", `${ASSET_ROUTE_PREFIX}/*`, diff --git a/apps/server/src/httpCors.ts b/apps/server/src/httpCors.ts index aeb8dbce5a5..6212a958e16 100644 --- a/apps/server/src/httpCors.ts +++ b/apps/server/src/httpCors.ts @@ -5,6 +5,9 @@ export const browserApiCorsAllowedHeaders = [ "traceparent", "content-type", "dpop", + "x-t3-transcription-api-key", + "x-t3-transcription-model", + "x-t3-transcription-provider", ] as const; export const browserApiCorsHeaders = { diff --git a/apps/server/src/server.test.ts b/apps/server/src/server.test.ts index 74d3fd2d594..295d005fb19 100644 --- a/apps/server/src/server.test.ts +++ b/apps/server/src/server.test.ts @@ -1300,6 +1300,9 @@ const assertBrowserApiCorsPreflightHeaders = ( "content-type", "dpop", "traceparent", + "x-t3-transcription-api-key", + "x-t3-transcription-model", + "x-t3-transcription-provider", ]); }; const crossOriginClientOrigin = "http://remote-client.test:3773"; @@ -4209,6 +4212,9 @@ it.layer(NodeServices.layer)("server router seam", (it) => { "content-type", "dpop", "traceparent", + "x-t3-transcription-api-key", + "x-t3-transcription-model", + "x-t3-transcription-provider", ]); }).pipe(Effect.provide(NodeHttpServer.layerTest)), ); diff --git a/apps/server/src/server.ts b/apps/server/src/server.ts index 05657af6d48..62ad7c9bd3e 100644 --- a/apps/server/src/server.ts +++ b/apps/server/src/server.ts @@ -13,6 +13,7 @@ import * as ServerConfig from "./config.ts"; import * as HttpResponseCompression from "./httpCompression/HttpResponseCompression.ts"; import { otlpTracesProxyRouteLayer, + transcriptionRouteLayer, assetRouteLayer, serverEnvironmentHttpApiLayer, staticAndDevRouteLayer, @@ -417,6 +418,7 @@ export const makeRoutesLayer = Layer.mergeAll( Layer.provide(environmentAuthenticatedAuthLayer), ), otlpTracesProxyRouteLayer, + transcriptionRouteLayer, assetRouteLayer, staticAndDevRouteLayer, websocketRpcRouteLayer, diff --git a/apps/server/src/transcription.test.ts b/apps/server/src/transcription.test.ts new file mode 100644 index 00000000000..85e954eec89 --- /dev/null +++ b/apps/server/src/transcription.test.ts @@ -0,0 +1,120 @@ +import { expect, it } from "@effect/vitest"; +import * as ConfigProvider from "effect/ConfigProvider"; +import * as Effect from "effect/Effect"; +import * as Stream from "effect/Stream"; +import { HttpClient, HttpClientRequest, HttpClientResponse } from "effect/unstable/http"; +import { describe } from "vite-plus/test"; + +import { + forwardVoiceTranscription, + listVoiceTranscriptionModels, + MAX_TRANSCRIPTION_AUDIO_BYTES, + readTranscriptionAudio, + resolveTranscriptionProvider, + transcriptionEnvironmentApiKeyStatus, + transcriptionProviderConfig, +} from "./transcription.ts"; + +describe("transcription providers", () => { + it("uses fixed OpenAI and Groq configurations", () => { + expect(resolveTranscriptionProvider("openai")).toBe("openai"); + expect(transcriptionProviderConfig("openai")).toEqual({ + endpoint: "https://api.openai.com/v1/audio/transcriptions", + modelsEndpoint: "https://api.openai.com/v1/models", + apiKeyEnvironmentVariable: "OPENAI_API_KEY", + }); + expect(resolveTranscriptionProvider("groq")).toBe("groq"); + expect(transcriptionProviderConfig("groq")).toEqual({ + endpoint: "https://api.groq.com/openai/v1/audio/transcriptions", + modelsEndpoint: "https://api.groq.com/openai/v1/models", + apiKeyEnvironmentVariable: "GROQ_API_KEY", + }); + expect(resolveTranscriptionProvider("custom")).toBeNull(); + }); + + it.effect("loads accessible transcription models with the provider API key", () => + Effect.gen(function* () { + let capturedRequest: HttpClientRequest.HttpClientRequest | undefined; + const client = HttpClient.make((request) => + Effect.sync(() => { + capturedRequest = request; + return HttpClientResponse.fromWeb( + request, + Response.json({ + data: [ + { id: "gpt-4o" }, + { id: " whisper-1 " }, + { id: "gpt-4o-mini-transcribe" }, + { id: "gpt-4o-mini-transcribe" }, + ], + }), + ); + }), + ); + + const models = yield* listVoiceTranscriptionModels({ + provider: "openai", + apiKey: "client-openai-key", + }).pipe(Effect.provideService(HttpClient.HttpClient, client)); + + expect(models).toEqual(["gpt-4o-mini-transcribe", "whisper-1"]); + expect(capturedRequest?.url).toBe("https://api.openai.com/v1/models"); + expect(capturedRequest?.headers.authorization).toBe("Bearer client-openai-key"); + }), + ); + + it.effect("uses a provider API key from the server environment", () => + Effect.gen(function* () { + let capturedRequest: HttpClientRequest.HttpClientRequest | undefined; + const client = HttpClient.make((request) => + Effect.sync(() => { + capturedRequest = request; + return HttpClientResponse.fromWeb(request, Response.json({ text: " groq transcript " })); + }), + ); + + const transcript = yield* forwardVoiceTranscription({ + audio: new Uint8Array([1, 2, 3]), + audioMimeType: "audio/webm;codecs=opus", + provider: "groq", + apiKey: "", + model: "whisper-large-v3", + }).pipe(Effect.provideService(HttpClient.HttpClient, client)); + + expect(yield* transcriptionEnvironmentApiKeyStatus("groq")).toBe(true); + expect(yield* transcriptionEnvironmentApiKeyStatus("openai")).toBe(false); + expect(transcript).toBe("groq transcript"); + expect(capturedRequest?.url).toBe("https://api.groq.com/openai/v1/audio/transcriptions"); + expect(capturedRequest?.headers.authorization).toBe("Bearer env-groq-key"); + expect(capturedRequest?.body._tag).toBe("FormData"); + if (capturedRequest?.body._tag === "FormData") { + expect(capturedRequest.body.formData.get("model")).toBe("whisper-large-v3"); + expect(capturedRequest.body.formData.get("file")).toBeInstanceOf(Blob); + } + }).pipe( + Effect.provide( + ConfigProvider.layer(ConfigProvider.fromEnv({ env: { GROQ_API_KEY: "env-groq-key" } })), + ), + ), + ); +}); + +describe("readTranscriptionAudio", () => { + it.effect("combines streamed audio chunks", () => + Effect.gen(function* () { + const audio = yield* readTranscriptionAudio( + Stream.make(new Uint8Array([1, 2]), new Uint8Array([3, 4])), + ); + expect(Array.from(audio)).toEqual([1, 2, 3, 4]); + }), + ); + + it.effect("stops when streamed audio exceeds the limit", () => + Effect.gen(function* () { + const error = yield* readTranscriptionAudio( + Stream.make(new Uint8Array(MAX_TRANSCRIPTION_AUDIO_BYTES), new Uint8Array([1])), + ).pipe(Effect.flip); + expect(error._tag).toBe("TranscriptionAudioTooLargeError"); + }), + ); +}); diff --git a/apps/server/src/transcription.ts b/apps/server/src/transcription.ts new file mode 100644 index 00000000000..1b167c18123 --- /dev/null +++ b/apps/server/src/transcription.ts @@ -0,0 +1,292 @@ +import type { VoiceTranscriptionProvider } from "@t3tools/contracts"; +import * as Config from "effect/Config"; +import * as Effect from "effect/Effect"; +import * as Schema from "effect/Schema"; +import * as Stream from "effect/Stream"; +import { HttpClient, HttpClientRequest, HttpClientResponse } from "effect/unstable/http"; + +export const MAX_TRANSCRIPTION_AUDIO_BYTES = 25 * 1024 * 1024; + +const PROVIDERS = { + openai: { + endpoint: "https://api.openai.com/v1/audio/transcriptions", + modelsEndpoint: "https://api.openai.com/v1/models", + apiKeyEnvironmentVariable: "OPENAI_API_KEY", + }, + groq: { + endpoint: "https://api.groq.com/openai/v1/audio/transcriptions", + modelsEndpoint: "https://api.groq.com/openai/v1/models", + apiKeyEnvironmentVariable: "GROQ_API_KEY", + }, +} as const satisfies Record< + VoiceTranscriptionProvider, + { endpoint: string; modelsEndpoint: string; apiKeyEnvironmentVariable: string } +>; + +export interface VoiceTranscriptionInput { + readonly audio: Uint8Array; + readonly audioMimeType: string; + readonly provider: VoiceTranscriptionProvider; + readonly apiKey: string; + readonly model: string; +} + +export interface VoiceTranscriptionModelsInput { + readonly provider: VoiceTranscriptionProvider; + readonly apiKey: string; +} + +export class TranscriptionAudioTooLargeError extends Schema.TaggedErrorClass()( + "TranscriptionAudioTooLargeError", + { receivedBytes: Schema.Number }, +) { + override get message(): string { + return "The recording exceeds the 25 MB limit."; + } +} + +export class TranscriptionBodyReadError extends Schema.TaggedErrorClass()( + "TranscriptionBodyReadError", + { cause: Schema.Defect() }, +) { + override get message(): string { + return "Could not read the recording."; + } +} + +export class TranscriptionEmptyAudioError extends Schema.TaggedErrorClass()( + "TranscriptionEmptyAudioError", + {}, +) { + override get message(): string { + return "The recording was empty."; + } +} + +export class TranscriptionProviderUnsupportedError extends Schema.TaggedErrorClass()( + "TranscriptionProviderUnsupportedError", + {}, +) { + override get message(): string { + return "Select OpenAI or Groq for transcription."; + } +} + +export class TranscriptionApiKeyMissingError extends Schema.TaggedErrorClass()( + "TranscriptionApiKeyMissingError", + {}, +) { + override get message(): string { + return "Add an API key or configure the provider's API key environment variable."; + } +} + +export class TranscriptionModelMissingError extends Schema.TaggedErrorClass()( + "TranscriptionModelMissingError", + {}, +) { + override get message(): string { + return "Select a transcription model."; + } +} + +export class TranscriptionRequestError extends Schema.TaggedErrorClass()( + "TranscriptionRequestError", + { + provider: Schema.String, + cause: Schema.Defect(), + }, +) { + override get message(): string { + return "Could not reach the transcription provider."; + } +} + +export class TranscriptionProviderError extends Schema.TaggedErrorClass()( + "TranscriptionProviderError", + { + provider: Schema.String, + providerStatus: Schema.Int, + }, +) { + override get message(): string { + return "The transcription provider rejected the request. Check the provider and API key."; + } +} + +export class TranscriptionResponseError extends Schema.TaggedErrorClass()( + "TranscriptionResponseError", + { + provider: Schema.String, + cause: Schema.Defect(), + }, +) { + override get message(): string { + return "The transcription provider returned an invalid response."; + } +} + +const TranscriptionResponse = Schema.Struct({ text: Schema.String }); +const ModelsResponse = Schema.Struct({ + data: Schema.Array(Schema.Struct({ id: Schema.String })), +}); +const TRANSCRIPTION_MODEL_ID_PATTERN = /(?:transcri|whisper|speech[-_ ]?to[-_ ]?text)/i; + +export function resolveTranscriptionProvider(provider: string): VoiceTranscriptionProvider | null { + return provider === "openai" || provider === "groq" ? provider : null; +} + +export function transcriptionProviderConfig(provider: VoiceTranscriptionProvider) { + return PROVIDERS[provider]; +} + +export const transcriptionEnvironmentApiKeyStatus = Effect.fn( + "transcriptionEnvironmentApiKeyStatus", +)(function* (provider: VoiceTranscriptionProvider) { + const providerConfig = transcriptionProviderConfig(provider); + const value = yield* Config.string(providerConfig.apiKeyEnvironmentVariable).pipe( + Config.withDefault(""), + ); + return value.trim().length > 0; +}); + +const resolveTranscriptionApiKey = Effect.fn("voiceTranscription.resolveApiKey")(function* ( + input: VoiceTranscriptionModelsInput, +) { + const providerConfig = transcriptionProviderConfig(input.provider); + const apiKey = + input.apiKey.trim() || + (yield* Config.string(providerConfig.apiKeyEnvironmentVariable).pipe( + Config.withDefault(""), + )).trim(); + if (!apiKey) { + return yield* new TranscriptionApiKeyMissingError(); + } + return apiKey; +}); + +export const listVoiceTranscriptionModels = Effect.fn("voiceTranscription.listModels")(function* ( + input: VoiceTranscriptionModelsInput, +) { + const providerConfig = transcriptionProviderConfig(input.provider); + const apiKey = yield* resolveTranscriptionApiKey(input); + const httpClient = yield* HttpClient.HttpClient; + const payload = yield* HttpClientRequest.get(providerConfig.modelsEndpoint).pipe( + HttpClientRequest.bearerToken(apiKey), + httpClient.execute, + Effect.mapError((cause) => new TranscriptionRequestError({ provider: input.provider, cause })), + Effect.flatMap((response) => + Effect.gen(function* () { + if (response.status < 200 || response.status >= 300) { + return yield* new TranscriptionProviderError({ + provider: input.provider, + providerStatus: response.status, + }); + } + return yield* HttpClientResponse.schemaBodyJson(ModelsResponse)(response).pipe( + Effect.mapError( + (cause) => new TranscriptionResponseError({ provider: input.provider, cause }), + ), + ); + }), + ), + Effect.timeout("30 seconds"), + Effect.catchTags({ + TimeoutError: (cause) => + Effect.fail(new TranscriptionRequestError({ provider: input.provider, cause })), + }), + ); + + const models = [...new Set(payload.data.map(({ id }) => id.trim()).filter(Boolean))].sort(); + const transcriptionModels = models.filter((model) => TRANSCRIPTION_MODEL_ID_PATTERN.test(model)); + return transcriptionModels.length > 0 ? transcriptionModels : models; +}); + +export const readTranscriptionAudio = (stream: Stream.Stream) => + stream.pipe( + Stream.mapError((cause) => new TranscriptionBodyReadError({ cause })), + Stream.runFoldEffect( + () => ({ chunks: [] as Uint8Array[], size: 0 }), + (accumulator, chunk) => { + const size = accumulator.size + chunk.byteLength; + if (size > MAX_TRANSCRIPTION_AUDIO_BYTES) { + return Effect.fail(new TranscriptionAudioTooLargeError({ receivedBytes: size })); + } + accumulator.chunks.push(chunk); + return Effect.succeed({ chunks: accumulator.chunks, size }); + }, + ), + Effect.map(({ chunks, size }) => { + const audio = new Uint8Array(size); + let offset = 0; + for (const chunk of chunks) { + audio.set(chunk, offset); + offset += chunk.byteLength; + } + return audio; + }), + ); + +function audioFileExtension(mimeType: string): string { + if (mimeType.includes("ogg")) return "ogg"; + if (mimeType.includes("mp4") || mimeType.includes("m4a")) return "m4a"; + if (mimeType.includes("wav")) return "wav"; + return "webm"; +} + +export const forwardVoiceTranscription = Effect.fn("voiceTranscription.forward")(function* ( + input: VoiceTranscriptionInput, +) { + if (input.audio.byteLength === 0) { + return yield* new TranscriptionEmptyAudioError(); + } + if (input.audio.byteLength > MAX_TRANSCRIPTION_AUDIO_BYTES) { + return yield* new TranscriptionAudioTooLargeError({ + receivedBytes: input.audio.byteLength, + }); + } + const providerConfig = transcriptionProviderConfig(input.provider); + const model = input.model.trim(); + if (!model) { + return yield* new TranscriptionModelMissingError(); + } + const apiKey = yield* resolveTranscriptionApiKey(input); + + const mimeType = input.audioMimeType.split(";", 1)[0]?.trim() || "audio/webm"; + const form = new FormData(); + form.set("model", model); + form.set( + "file", + new Blob([input.audio], { type: mimeType }), + `recording.${audioFileExtension(mimeType)}`, + ); + + const httpClient = yield* HttpClient.HttpClient; + const payload = yield* HttpClientRequest.post(providerConfig.endpoint).pipe( + HttpClientRequest.bearerToken(apiKey), + HttpClientRequest.bodyFormData(form), + httpClient.execute, + Effect.mapError((cause) => new TranscriptionRequestError({ provider: input.provider, cause })), + Effect.flatMap((response) => + Effect.gen(function* () { + if (response.status < 200 || response.status >= 300) { + return yield* new TranscriptionProviderError({ + provider: input.provider, + providerStatus: response.status, + }); + } + return yield* HttpClientResponse.schemaBodyJson(TranscriptionResponse)(response).pipe( + Effect.mapError( + (cause) => new TranscriptionResponseError({ provider: input.provider, cause }), + ), + ); + }), + ), + Effect.timeout("2 minutes"), + Effect.catchTags({ + TimeoutError: (cause) => + Effect.fail(new TranscriptionRequestError({ provider: input.provider, cause })), + }), + ); + return payload.text.trim(); +}); diff --git a/apps/web/src/components/chat/ChatComposer.tsx b/apps/web/src/components/chat/ChatComposer.tsx index e92ecd497e3..b27b368eb90 100644 --- a/apps/web/src/components/chat/ChatComposer.tsx +++ b/apps/web/src/components/chat/ChatComposer.tsx @@ -107,6 +107,8 @@ import { buildExpandedImagePreview, type ExpandedImagePreview } from "./Expanded import { basenameOfPath } from "../../pierre-icons"; import { cn, randomUUID } from "~/lib/utils"; import { Separator } from "../ui/separator"; +import { useVoiceTranscription } from "../../hooks/useVoiceTranscription"; +import { VoiceTranscriptionPanel } from "./VoiceTranscriptionPanel"; function ComposerCommandMenuLayer(props: { anchor: HTMLElement | null; children: ReactNode }) { const [position, setPosition] = useState<{ @@ -179,6 +181,8 @@ import { LockOpenIcon, PenLineIcon, SparklesIcon, + MicIcon, + SquareIcon, XIcon, } from "lucide-react"; import { proposedPlanTitle } from "../../proposedPlan"; @@ -1269,6 +1273,29 @@ export const ChatComposer = memo(function ChatComposer(props: ChatComposerProps) [composerDraftTarget, setComposerDraftPrompt], ); + const appendVoiceTranscript = useCallback( + (transcript: string) => { + const currentPrompt = promptRef.current; + const boundary = currentPrompt.length > 0 && !/\s$/.test(currentPrompt) ? " " : ""; + const nextPrompt = `${currentPrompt}${boundary}${transcript}`; + promptRef.current = nextPrompt; + setPrompt(nextPrompt); + const nextCursor = collapseExpandedComposerCursor(nextPrompt, nextPrompt.length); + setComposerCursor(nextCursor); + setComposerTrigger(null); + scheduleComposerFocus(); + }, + [promptRef, scheduleComposerFocus, setPrompt], + ); + const voiceTranscription = useVoiceTranscription({ + config: { + provider: settings.voiceTranscriptionProvider, + apiKey: settings.voiceTranscriptionApiKey, + model: settings.voiceTranscriptionModel, + }, + onTranscript: appendVoiceTranscript, + }); + const addComposerImage = useCallback( (image: ComposerImageAttachment) => { addComposerDraftImage(composerDraftTarget, image); @@ -3102,6 +3129,14 @@ export const ChatComposer = memo(function ChatComposer(props: ChatComposerProps) + {voiceTranscription.status === "idle" && + settings.voiceTranscriptionEnabled && + voiceTranscription.error ? ( +

+ {voiceTranscription.error} +

+ ) : null} + {/* Bottom toolbar */} {isComposerCollapsedMobile ? null : activePendingApproval ? (
@@ -3122,7 +3157,20 @@ export const ChatComposer = memo(function ChatComposer(props: ChatComposerProps) showMobilePendingAnswerActions && "hidden sm:flex", )} > -
+ {voiceTranscription.status !== "idle" ? ( + + ) : null} +
{noProviderAvailable ? ( + } + /> + + {voiceTranscription.status === "recording" + ? "Stop dictation" + : "Start dictation"} + + + ) : null} ; + readonly elapsedMs: number; + readonly levels: readonly number[]; +}) { + const waveformPath = levels + .map((level, index) => { + if (level <= 0.01) return ""; + const amplitude = Math.max(1.75, level * 14); + return `M ${index + 0.5} ${16 - amplitude} V ${16 + amplitude}`; + }) + .join(" "); + + return ( +
+ {status === "recording" ? ( + + ) : ( +
+ + Transcribing… +
+ )} + + {formatElapsed(elapsedMs)} + +
+ ); +} diff --git a/apps/web/src/components/settings/BetaSettingsPanel.tsx b/apps/web/src/components/settings/BetaSettingsPanel.tsx index 740d3048f0e..2cd183feebe 100644 --- a/apps/web/src/components/settings/BetaSettingsPanel.tsx +++ b/apps/web/src/components/settings/BetaSettingsPanel.tsx @@ -1,3 +1,4 @@ +import type { VoiceTranscriptionProvider } from "@t3tools/contracts"; import { useEffect, useState } from "react"; import { @@ -5,7 +6,12 @@ import { useSidebarV2Enabled, useUpdateClientSettings, } from "../../hooks/useSettings"; +import { + listVoiceTranscriptionModels, + readVoiceTranscriptionEnvironmentStatus, +} from "../../lib/voiceTranscription"; import { Input } from "../ui/input"; +import { Select, SelectItem, SelectPopup, SelectTrigger, SelectValue } from "../ui/select"; import { Switch } from "../ui/switch"; import { SettingsPageContainer, SettingsRow, SettingsSection } from "./settingsLayout"; import { searchableSetting } from "./settingsSearch"; @@ -13,6 +19,10 @@ import { searchableSetting } from "./settingsSearch"; const AUTO_SETTLE_MIN_DAYS = 1; const AUTO_SETTLE_MAX_DAYS = 90; const AUTO_SETTLE_DEFAULT_DAYS = 3; +const TRANSCRIPTION_API_KEY_ENV = { + openai: "OPENAI_API_KEY", + groq: "GROQ_API_KEY", +} as const; function AutoSettleDaysInput({ value, @@ -60,8 +70,109 @@ export function BetaSettingsPanel() { const sidebarAutoSettleAfterDays = useClientSettings( (settings) => settings.sidebarAutoSettleAfterDays, ); + const voiceTranscriptionEnabled = useClientSettings( + (settings) => settings.voiceTranscriptionEnabled, + ); + const voiceTranscriptionProvider = useClientSettings( + (settings) => settings.voiceTranscriptionProvider, + ); + const voiceTranscriptionApiKey = useClientSettings( + (settings) => settings.voiceTranscriptionApiKey, + ); + const voiceTranscriptionModel = useClientSettings((settings) => settings.voiceTranscriptionModel); + const [environmentApiKeys, setEnvironmentApiKeys] = useState({ + openai: false, + groq: false, + }); + const [transcriptionModels, setTranscriptionModels] = useState([]); + const [transcriptionModelsLoading, setTranscriptionModelsLoading] = useState(false); + const [transcriptionModelsError, setTranscriptionModelsError] = useState(null); const updateSettings = useUpdateClientSettings(); + useEffect(() => { + if (!voiceTranscriptionEnabled) return; + let active = true; + void readVoiceTranscriptionEnvironmentStatus() + .then((status) => { + if (active) setEnvironmentApiKeys(status); + }) + .catch(() => undefined); + return () => { + active = false; + }; + }, [voiceTranscriptionEnabled]); + + const transcriptionProviderLabel = voiceTranscriptionProvider === "openai" ? "OpenAI" : "Groq"; + const transcriptionApiKeyEnvironmentVariable = + TRANSCRIPTION_API_KEY_ENV[voiceTranscriptionProvider]; + const hasEnvironmentApiKey = environmentApiKeys[voiceTranscriptionProvider]; + const hasTranscriptionApiKey = voiceTranscriptionApiKey.trim().length > 0 || hasEnvironmentApiKey; + + useEffect(() => { + if (!voiceTranscriptionEnabled || !hasTranscriptionApiKey) { + setTranscriptionModels([]); + setTranscriptionModelsLoading(false); + setTranscriptionModelsError(null); + return; + } + + let active = true; + setTranscriptionModelsLoading(true); + setTranscriptionModelsError(null); + const timeout = window.setTimeout( + () => { + void listVoiceTranscriptionModels({ + provider: voiceTranscriptionProvider, + apiKey: voiceTranscriptionApiKey, + }) + .then((models) => { + if (!active) return; + setTranscriptionModels(models); + setTranscriptionModelsLoading(false); + }) + .catch((cause: unknown) => { + if (!active) return; + setTranscriptionModels([]); + setTranscriptionModelsLoading(false); + setTranscriptionModelsError( + cause instanceof Error ? cause.message : "Could not load transcription models.", + ); + }); + }, + voiceTranscriptionApiKey.trim() ? 400 : 0, + ); + + return () => { + active = false; + window.clearTimeout(timeout); + }; + }, [ + hasTranscriptionApiKey, + voiceTranscriptionApiKey, + voiceTranscriptionEnabled, + voiceTranscriptionProvider, + ]); + + useEffect(() => { + if ( + transcriptionModels.length > 0 && + voiceTranscriptionModel && + !transcriptionModels.includes(voiceTranscriptionModel) + ) { + updateSettings({ voiceTranscriptionModel: "" }); + } + }, [transcriptionModels, updateSettings, voiceTranscriptionModel]); + + const transcriptionModelDescription = !hasTranscriptionApiKey + ? `Add an API key to load models available from ${transcriptionProviderLabel}.` + : transcriptionModelsLoading + ? `Loading models available from ${transcriptionProviderLabel}…` + : transcriptionModelsError + ? transcriptionModelsError + : transcriptionModels.length === 0 + ? `${transcriptionProviderLabel} did not return any models.` + : `Loaded from ${transcriptionProviderLabel} using the configured API key.`; + return ( @@ -114,6 +225,106 @@ export function BetaSettingsPanel() { ) : null} ) : null} + + updateSettings({ voiceTranscriptionEnabled: Boolean(checked) }) + } + aria-label="Enable voice dictation beta" + /> + } + /> + {voiceTranscriptionEnabled ? ( + <> + + updateSettings({ + voiceTranscriptionProvider: value as VoiceTranscriptionProvider, + voiceTranscriptionApiKey: "", + voiceTranscriptionModel: "", + }) + } + > + + {transcriptionProviderLabel} + + + + OpenAI + + + Groq + + + + } + /> + + updateSettings({ + voiceTranscriptionApiKey: event.target.value, + voiceTranscriptionModel: "", + }) + } + placeholder={ + hasEnvironmentApiKey + ? `Using ${transcriptionApiKeyEnvironmentVariable}` + : "Required" + } + aria-label={`${transcriptionProviderLabel} transcription API key`} + /> + } + /> + { + if (value !== null) updateSettings({ voiceTranscriptionModel: value }); + }} + > + + + {voiceTranscriptionModel || + (transcriptionModelsLoading ? "Loading models…" : "Select model")} + + + + {transcriptionModels.map((model) => ( + + {model} + + ))} + + + } + /> + + ) : null} ); diff --git a/apps/web/src/components/settings/settingsSearch.ts b/apps/web/src/components/settings/settingsSearch.ts index 4ead6eff4d7..e80b41a4e0e 100644 --- a/apps/web/src/components/settings/settingsSearch.ts +++ b/apps/web/src/components/settings/settingsSearch.ts @@ -157,6 +157,11 @@ export const SETTINGS_SEARCH_ITEMS = [ to: "/settings/beta", targetId: "sidebar-v2", }, + { + id: "voice-dictation", + title: "Voice dictation", + to: "/settings/beta", + }, { id: "archive", title: "Archived threads", diff --git a/apps/web/src/hooks/useVoiceTranscription.ts b/apps/web/src/hooks/useVoiceTranscription.ts new file mode 100644 index 00000000000..22e776dafc7 --- /dev/null +++ b/apps/web/src/hooks/useVoiceTranscription.ts @@ -0,0 +1,199 @@ +import { useCallback, useEffect, useRef, useState } from "react"; + +import { transcribeVoiceRecording, type VoiceTranscriptionConfig } from "../lib/voiceTranscription"; + +const LEVEL_COUNT = 160; +const MAX_RECORDING_MS = 5 * 60 * 1_000; +const MIME_TYPES = ["audio/webm;codecs=opus", "audio/ogg;codecs=opus", "audio/mp4"]; +const FLAT_LEVELS = Array(LEVEL_COUNT).fill(0); + +export type VoiceTranscriptionStatus = "idle" | "recording" | "transcribing"; + +function supportedMimeType(): string | undefined { + if (typeof MediaRecorder === "undefined") return undefined; + return MIME_TYPES.find((mimeType) => MediaRecorder.isTypeSupported(mimeType)); +} + +export function useVoiceTranscription({ + config, + onTranscript, +}: { + readonly config: VoiceTranscriptionConfig; + readonly onTranscript: (text: string) => void; +}) { + const [status, setStatus] = useState("idle"); + const [elapsedMs, setElapsedMs] = useState(0); + const [levels, setLevels] = useState(FLAT_LEVELS); + const [error, setError] = useState(null); + const recorderRef = useRef(null); + const startingRef = useRef(false); + const cancelStartingRef = useRef(false); + const streamRef = useRef(null); + const audioContextRef = useRef(null); + const intervalsRef = useRef([]); + const timeoutRef = useRef(null); + const startedAtRef = useRef(0); + const mountedRef = useRef(true); + const configRef = useRef(config); + const onTranscriptRef = useRef(onTranscript); + configRef.current = config; + onTranscriptRef.current = onTranscript; + + const cleanupCapture = useCallback(() => { + for (const interval of intervalsRef.current) window.clearInterval(interval); + intervalsRef.current = []; + if (timeoutRef.current !== null) window.clearTimeout(timeoutRef.current); + timeoutRef.current = null; + streamRef.current?.getTracks().forEach((track) => track.stop()); + streamRef.current = null; + void audioContextRef.current?.close(); + audioContextRef.current = null; + recorderRef.current = null; + }, []); + + useEffect(() => { + mountedRef.current = true; + return () => { + mountedRef.current = false; + startingRef.current = false; + cancelStartingRef.current = true; + const recorder = recorderRef.current; + if (recorder?.state === "recording") recorder.stop(); + cleanupCapture(); + }; + }, [cleanupCapture]); + + const stop = useCallback(() => { + if (startingRef.current) { + cancelStartingRef.current = true; + cleanupCapture(); + if (mountedRef.current) { + setStatus("idle"); + setElapsedMs(0); + } + return; + } + const recorder = recorderRef.current; + if (recorder?.state === "recording") recorder.stop(); + }, [cleanupCapture]); + + const start = useCallback(async () => { + if (startingRef.current || status !== "idle") return; + startingRef.current = true; + setError(null); + setElapsedMs(0); + setLevels(FLAT_LEVELS); + if (!configRef.current.model.trim()) { + startingRef.current = false; + setError("Select a transcription model in Settings."); + return; + } + if (!navigator.mediaDevices?.getUserMedia || typeof MediaRecorder === "undefined") { + startingRef.current = false; + setError("Microphone recording is not supported on this device."); + return; + } + + cancelStartingRef.current = false; + setStatus("recording"); + try { + const audioContext = new AudioContext(); + audioContextRef.current = audioContext; + if (audioContext.state === "suspended") { + void audioContext.resume().catch(() => undefined); + } + const stream = await navigator.mediaDevices.getUserMedia({ + audio: { echoCancellation: true, noiseSuppression: true, autoGainControl: true }, + }); + if (!mountedRef.current || cancelStartingRef.current) { + stream.getTracks().forEach((track) => track.stop()); + startingRef.current = false; + cancelStartingRef.current = false; + return; + } + + streamRef.current = stream; + const mimeType = supportedMimeType(); + const recorder = new MediaRecorder(stream, mimeType ? { mimeType } : undefined); + const chunks: Blob[] = []; + let recordingFailed = false; + recorderRef.current = recorder; + recorder.addEventListener("dataavailable", (event) => { + if (event.data.size > 0) chunks.push(event.data); + }); + recorder.addEventListener("error", () => { + recordingFailed = true; + cleanupCapture(); + if (mountedRef.current) { + setStatus("idle"); + setError("The microphone stopped unexpectedly."); + } + }); + recorder.addEventListener("stop", () => { + const blob = new Blob(chunks, { type: recorder.mimeType || mimeType || "audio/webm" }); + cleanupCapture(); + if (!mountedRef.current || recordingFailed) return; + setStatus("transcribing"); + void transcribeVoiceRecording(blob, configRef.current) + .then((text) => { + if (!mountedRef.current) return; + if (text) onTranscriptRef.current(text); + setStatus("idle"); + setElapsedMs(0); + }) + .catch((cause: unknown) => { + if (!mountedRef.current) return; + setError(cause instanceof Error ? cause.message : "Voice transcription failed."); + setStatus("idle"); + }); + }); + + const analyser = audioContext.createAnalyser(); + analyser.fftSize = 256; + const silentOutput = audioContext.createGain(); + silentOutput.gain.value = 0; + audioContext.createMediaStreamSource(stream).connect(analyser); + analyser.connect(silentOutput); + silentOutput.connect(audioContext.destination); + if (audioContext.state === "suspended") { + void audioContext.resume().catch(() => undefined); + } + const samples = new Uint8Array(analyser.fftSize); + intervalsRef.current.push( + window.setInterval(() => { + analyser.getByteTimeDomainData(samples); + let squaredAmplitude = 0; + for (const sample of samples) { + const amplitude = (sample - 128) / 128; + squaredAmplitude += amplitude * amplitude; + } + const rootMeanSquare = Math.sqrt(squaredAmplitude / samples.length); + const nextLevel = Math.min(1, Math.max(0, (rootMeanSquare - 0.008) * 9)); + setLevels((current) => [...current.slice(1), nextLevel]); + }, 50), + ); + startedAtRef.current = Date.now(); + intervalsRef.current.push( + window.setInterval(() => setElapsedMs(Date.now() - startedAtRef.current), 250), + ); + timeoutRef.current = window.setTimeout(() => recorder.stop(), MAX_RECORDING_MS); + recorder.start(250); + startingRef.current = false; + } catch (cause) { + startingRef.current = false; + cleanupCapture(); + if (cancelStartingRef.current) { + cancelStartingRef.current = false; + return; + } + setStatus("idle"); + setError( + cause instanceof DOMException && cause.name === "NotAllowedError" + ? "Microphone permission was denied. Allow access and try again." + : "Could not start the microphone.", + ); + } + }, [cleanupCapture, status]); + + return { status, elapsedMs, levels, error, start, stop } as const; +} diff --git a/apps/web/src/lib/voiceTranscription.test.ts b/apps/web/src/lib/voiceTranscription.test.ts new file mode 100644 index 00000000000..b82a1962f21 --- /dev/null +++ b/apps/web/src/lib/voiceTranscription.test.ts @@ -0,0 +1,70 @@ +import { afterEach, describe, expect, it, vi } from "vite-plus/test"; + +import { + listVoiceTranscriptionModels, + voiceTranscriptionRequestHeaders, +} from "./voiceTranscription"; + +afterEach(() => { + vi.unstubAllGlobals(); +}); + +describe("voiceTranscriptionRequestHeaders", () => { + it("sends a client-provided OpenAI key", () => { + expect( + voiceTranscriptionRequestHeaders("audio/webm", { + provider: "openai", + apiKey: " openai-key ", + model: " gpt-4o-mini-transcribe ", + }), + ).toEqual({ + "content-type": "audio/webm", + "x-t3-transcription-provider": "openai", + "x-t3-transcription-model": "gpt-4o-mini-transcribe", + "x-t3-transcription-api-key": "openai-key", + }); + }); + + it("omits an empty key so the server can use the provider environment variable", () => { + expect( + voiceTranscriptionRequestHeaders("audio/mp4", { + provider: "groq", + apiKey: "", + model: "whisper-large-v3", + }), + ).toEqual({ + "content-type": "audio/mp4", + "x-t3-transcription-provider": "groq", + "x-t3-transcription-model": "whisper-large-v3", + }); + }); +}); + +describe("listVoiceTranscriptionModels", () => { + it("loads the provider models through the connected T3 server", async () => { + const fetchMock = vi.fn().mockResolvedValue( + Response.json({ + models: ["gpt-4o-mini-transcribe", "whisper-1"], + }), + ); + vi.stubGlobal("window", { + location: { + href: "http://localhost:3773/settings", + origin: "http://localhost:3773", + }, + }); + vi.stubGlobal("fetch", fetchMock); + + await expect( + listVoiceTranscriptionModels({ provider: "openai", apiKey: "client-key" }), + ).resolves.toEqual(["gpt-4o-mini-transcribe", "whisper-1"]); + expect(fetchMock).toHaveBeenCalledWith("http://localhost:3773/api/transcription/models", { + method: "GET", + credentials: "include", + headers: { + "x-t3-transcription-api-key": "client-key", + "x-t3-transcription-provider": "openai", + }, + }); + }); +}); diff --git a/apps/web/src/lib/voiceTranscription.ts b/apps/web/src/lib/voiceTranscription.ts new file mode 100644 index 00000000000..99a19cb783e --- /dev/null +++ b/apps/web/src/lib/voiceTranscription.ts @@ -0,0 +1,119 @@ +import type { VoiceTranscriptionProvider } from "@t3tools/contracts"; + +import { readDesktopPrimaryBearerToken } from "../environments/primary/desktopAuth"; +import { resolvePrimaryEnvironmentHttpUrl } from "../environments/primary/target"; + +export interface VoiceTranscriptionConfig { + readonly provider: VoiceTranscriptionProvider; + readonly apiKey: string; + readonly model: string; +} + +export type VoiceTranscriptionProviderConfig = Omit; + +export interface VoiceTranscriptionEnvironmentStatus { + readonly openai: boolean; + readonly groq: boolean; +} + +export function voiceTranscriptionRequestHeaders( + contentType: string, + config: VoiceTranscriptionConfig, +): Record { + const apiKey = config.apiKey.trim(); + return { + "content-type": contentType, + "x-t3-transcription-provider": config.provider, + "x-t3-transcription-model": config.model.trim(), + ...(apiKey ? { "x-t3-transcription-api-key": apiKey } : {}), + }; +} + +function voiceTranscriptionProviderHeaders( + config: VoiceTranscriptionProviderConfig, +): Record { + const apiKey = config.apiKey.trim(); + return { + "x-t3-transcription-provider": config.provider, + ...(apiKey ? { "x-t3-transcription-api-key": apiKey } : {}), + }; +} + +export async function readVoiceTranscriptionEnvironmentStatus(): Promise { + const bearerToken = await readDesktopPrimaryBearerToken(); + const response = await globalThis.fetch(resolvePrimaryEnvironmentHttpUrl("/api/transcription"), { + method: "GET", + credentials: bearerToken ? "omit" : "include", + ...(bearerToken ? { headers: { authorization: `Bearer ${bearerToken}` } } : {}), + }); + if (!response.ok) throw new Error("Could not read transcription provider settings."); + + const payload = (await response.json()) as { readonly openai?: unknown; readonly groq?: unknown }; + return { + openai: payload.openai === true, + groq: payload.groq === true, + }; +} + +export async function listVoiceTranscriptionModels( + config: VoiceTranscriptionProviderConfig, +): Promise { + const bearerToken = await readDesktopPrimaryBearerToken(); + const response = await globalThis.fetch( + resolvePrimaryEnvironmentHttpUrl("/api/transcription/models"), + { + method: "GET", + credentials: bearerToken ? "omit" : "include", + headers: { + ...(bearerToken ? { authorization: `Bearer ${bearerToken}` } : {}), + ...voiceTranscriptionProviderHeaders(config), + }, + }, + ); + const payload = (await response.json().catch(() => null)) as { + readonly models?: unknown; + readonly error?: unknown; + } | null; + if (!response.ok) { + throw new Error( + typeof payload?.error === "string" ? payload.error : "Could not load transcription models.", + ); + } + if ( + !Array.isArray(payload?.models) || + !payload.models.every((model) => typeof model === "string") + ) { + throw new Error("The transcription model response was invalid."); + } + return payload.models; +} + +export async function transcribeVoiceRecording( + audio: Blob, + config: VoiceTranscriptionConfig, +): Promise { + const bearerToken = await readDesktopPrimaryBearerToken(); + const response = await globalThis.fetch(resolvePrimaryEnvironmentHttpUrl("/api/transcription"), { + method: "POST", + credentials: bearerToken ? "omit" : "include", + headers: { + ...(bearerToken ? { authorization: `Bearer ${bearerToken}` } : {}), + ...voiceTranscriptionRequestHeaders(audio.type || "audio/webm", config), + }, + body: audio, + }); + + const payload = (await response.json().catch(() => null)) as { + readonly text?: unknown; + readonly error?: unknown; + } | null; + if (!response.ok) { + throw new Error( + typeof payload?.error === "string" ? payload.error : "Voice transcription failed.", + ); + } + if (typeof payload?.text !== "string") { + throw new Error("The transcription response did not contain text."); + } + return payload.text.trim(); +} diff --git a/docs/README.md b/docs/README.md index bc359826a04..70ef5c32e2e 100644 --- a/docs/README.md +++ b/docs/README.md @@ -5,6 +5,7 @@ - [Install and first run](./user/install.md) - [Permission modes](./user/permission-modes.md) - [Keyboard shortcuts](./user/keybindings.md) +- [Voice dictation](./user/voice-dictation.md) - [Remote access](./user/remote-access.md) - [Keeping app and server in sync](./user/updating.md) - [Source control integrations](./user/source-control.md) diff --git a/docs/user/voice-dictation.md b/docs/user/voice-dictation.md new file mode 100644 index 00000000000..e34e51a6bdc --- /dev/null +++ b/docs/user/voice-dictation.md @@ -0,0 +1,28 @@ +# Voice Dictation + +Voice dictation records from the message composer and inserts the transcription into your draft. +It is available in the web and desktop clients on browsers that support microphone recording. + +Enable it in **Settings** → **Beta features** → **Voice dictation**. The composer then shows a +microphone action. Select it to start recording and stop it when you are finished. Recordings stop +automatically after five minutes. + +## Providers and API Keys + +Choose **OpenAI** or **Groq**. T3 Code supplies the provider's transcription endpoint. After an +API key is available, T3 Code loads the models that key can access from the provider and lets you +select the transcription model. Model IDs are not bundled into T3 Code, so newly available models +can appear without an app update. + +You can enter a key in the client, or configure it in the environment that runs the connected T3 +Code server: + +- OpenAI uses `OPENAI_API_KEY` +- Groq uses `GROQ_API_KEY` + +When an environment key is available, the API key input indicates that it is already configured. +A key entered in the input overrides the environment key and stays in that client's local +settings. Environment key values are never sent to the client. + +Recordings are sent through the connected T3 Code server to the selected provider. Recordings +larger than 25 MB are rejected. diff --git a/packages/contracts/src/settings.test.ts b/packages/contracts/src/settings.test.ts index 2bc61d72f21..992a65cb2e4 100644 --- a/packages/contracts/src/settings.test.ts +++ b/packages/contracts/src/settings.test.ts @@ -111,6 +111,22 @@ describe("ClientSettings sidebar v2", () => { }); }); +describe("ClientSettings voice transcription", () => { + it("defaults to OpenAI and accepts Groq", () => { + const settings = decodeClientSettings({}); + + expect(settings.voiceTranscriptionProvider).toBe("openai"); + expect(settings.voiceTranscriptionModel).toBe(""); + expect( + decodeClientSettingsPatch({ voiceTranscriptionProvider: "groq" }).voiceTranscriptionProvider, + ).toBe("groq"); + expect( + decodeClientSettingsPatch({ voiceTranscriptionModel: " whisper-large-v3 " }) + .voiceTranscriptionModel, + ).toBe("whisper-large-v3"); + }); +}); + describe("ServerSettings.providerInstances (slice-2 invariant)", () => { it("defaults to an empty record so legacy configs without the key still decode", () => { expect(DEFAULT_SERVER_SETTINGS.providerInstances).toEqual({}); diff --git a/packages/contracts/src/settings.ts b/packages/contracts/src/settings.ts index 7edda2e52e5..fbb6668b951 100644 --- a/packages/contracts/src/settings.ts +++ b/packages/contracts/src/settings.ts @@ -62,6 +62,9 @@ export const EnvironmentIdentificationMode = Schema.Literals(["artwork", "pill", export type EnvironmentIdentificationMode = typeof EnvironmentIdentificationMode.Type; export const DEFAULT_ENVIRONMENT_IDENTIFICATION_MODE: EnvironmentIdentificationMode = "artwork"; +export const VoiceTranscriptionProvider = Schema.Literals(["openai", "groq"]); +export type VoiceTranscriptionProvider = typeof VoiceTranscriptionProvider.Type; + export const ClientSettingsSchema = Schema.Struct({ autoOpenPlanSidebar: Schema.Boolean.pipe(Schema.withDecodingDefault(Effect.succeed(false))), confirmThreadArchive: Schema.Boolean.pipe(Schema.withDecodingDefault(Effect.succeed(false))), @@ -130,6 +133,12 @@ export const ClientSettingsSchema = Schema.Struct({ timestampFormat: TimestampFormat.pipe( Schema.withDecodingDefault(Effect.succeed(DEFAULT_TIMESTAMP_FORMAT)), ), + voiceTranscriptionEnabled: Schema.Boolean.pipe(Schema.withDecodingDefault(Effect.succeed(false))), + voiceTranscriptionProvider: VoiceTranscriptionProvider.pipe( + Schema.withDecodingDefault(Effect.succeed("openai" as const)), + ), + voiceTranscriptionApiKey: TrimmedString.pipe(Schema.withDecodingDefault(Effect.succeed(""))), + voiceTranscriptionModel: TrimmedString.pipe(Schema.withDecodingDefault(Effect.succeed(""))), wordWrap: Schema.Boolean.pipe(Schema.withDecodingDefault(Effect.succeed(true))), }); export type ClientSettings = typeof ClientSettingsSchema.Type; @@ -712,6 +721,10 @@ export const ClientSettingsPatch = Schema.Struct({ sidebarV2Enabled: Schema.optionalKey(Schema.Boolean), sidebarV2ConfiguredByUser: Schema.optionalKey(Schema.Boolean), timestampFormat: Schema.optionalKey(TimestampFormat), + voiceTranscriptionEnabled: Schema.optionalKey(Schema.Boolean), + voiceTranscriptionProvider: Schema.optionalKey(VoiceTranscriptionProvider), + voiceTranscriptionApiKey: Schema.optionalKey(TrimmedString), + voiceTranscriptionModel: Schema.optionalKey(TrimmedString), wordWrap: Schema.optionalKey(Schema.Boolean), }); export type ClientSettingsPatch = typeof ClientSettingsPatch.Type; diff --git a/scripts/build-desktop-artifact.test.ts b/scripts/build-desktop-artifact.test.ts index 7d2b7410a9e..d6c53c5be9f 100644 --- a/scripts/build-desktop-artifact.test.ts +++ b/scripts/build-desktop-artifact.test.ts @@ -349,6 +349,9 @@ it.layer(NodeServices.layer)("build-desktop-artifact", (it) => { assert.notProperty(mac, "asarUnpack"); assert.notProperty(linux, "asarUnpack"); assert.deepStrictEqual(win.asarUnpack, WINDOWS_ASAR_UNPACK); + assert.deepStrictEqual((mac.mac as Record).extendInfo, { + NSMicrophoneUsageDescription: "T3 Code uses the microphone for voice dictation.", + }); // Linux must register the renderer schemes so the generated .desktop // entry advertises MimeType=x-scheme-handler/t3code; for OAuth deep links. assert.deepStrictEqual((linux.linux as Record).protocols, [ diff --git a/scripts/build-desktop-artifact.ts b/scripts/build-desktop-artifact.ts index a3c99969256..970b523944f 100644 --- a/scripts/build-desktop-artifact.ts +++ b/scripts/build-desktop-artifact.ts @@ -1568,6 +1568,9 @@ export const createBuildConfig = Effect.fn("createBuildConfig")(function* ( target: target === "dmg" ? [target, "zip"] : [target], icon: "icon.icns", category: "public.app-category.developer-tools", + extendInfo: { + NSMicrophoneUsageDescription: "T3 Code uses the microphone for voice dictation.", + }, protocols: [ { name: "T3 Code",