From 3bb795023f798b32fd9def91961e03a9a3b6233a Mon Sep 17 00:00:00 2001 From: akadotsh Date: Thu, 5 Nov 2026 20:02:09 +0530 Subject: [PATCH 1/9] feat: add Cartesia speech and transcription adapters --- .changeset/soft-voices-speak.md | 5 + README.md | 1 + docs/adapters/cartesia.md | 82 +++++++++++++ docs/config.json | 9 +- docs/getting-started/overview.md | 1 + docs/media/text-to-speech.md | 3 +- docs/media/transcription.md | 3 +- packages/ai-cartesia/LICENSE | 21 ++++ packages/ai-cartesia/README.md | 33 +++++ packages/ai-cartesia/package.json | 45 +++++++ packages/ai-cartesia/src/client.ts | 67 ++++++++++ packages/ai-cartesia/src/index.ts | 18 +++ packages/ai-cartesia/src/model-meta.ts | 11 ++ packages/ai-cartesia/src/speech.ts | 116 ++++++++++++++++++ packages/ai-cartesia/src/transcription.ts | 92 ++++++++++++++ .../tests/cartesia-adapters.test.ts | 88 +++++++++++++ packages/ai-cartesia/tsconfig.json | 8 ++ packages/ai-cartesia/vite.config.ts | 37 ++++++ pnpm-lock.yaml | 19 +++ testing/e2e/global-setup.ts | 54 ++++++++ testing/e2e/package.json | 1 + testing/e2e/src/components/TTSUI.tsx | 10 +- testing/e2e/src/lib/feature-support.ts | 2 + testing/e2e/src/lib/media-providers.ts | 14 +++ testing/e2e/src/lib/providers.ts | 6 + testing/e2e/src/lib/types.ts | 2 + testing/e2e/tests/test-matrix.ts | 1 + 27 files changed, 742 insertions(+), 7 deletions(-) create mode 100644 .changeset/soft-voices-speak.md create mode 100644 docs/adapters/cartesia.md create mode 100644 packages/ai-cartesia/LICENSE create mode 100644 packages/ai-cartesia/README.md create mode 100644 packages/ai-cartesia/package.json create mode 100644 packages/ai-cartesia/src/client.ts create mode 100644 packages/ai-cartesia/src/index.ts create mode 100644 packages/ai-cartesia/src/model-meta.ts create mode 100644 packages/ai-cartesia/src/speech.ts create mode 100644 packages/ai-cartesia/src/transcription.ts create mode 100644 packages/ai-cartesia/tests/cartesia-adapters.test.ts create mode 100644 packages/ai-cartesia/tsconfig.json create mode 100644 packages/ai-cartesia/vite.config.ts diff --git a/.changeset/soft-voices-speak.md b/.changeset/soft-voices-speak.md new file mode 100644 index 0000000000..45018c6f09 --- /dev/null +++ b/.changeset/soft-voices-speak.md @@ -0,0 +1,5 @@ +--- +'@tanstack/ai-cartesia': minor +--- + +Add Cartesia text-to-speech and batch transcription adapters. diff --git a/README.md b/README.md index f902cb8230..3c2bc86b02 100644 --- a/README.md +++ b/README.md @@ -240,6 +240,7 @@ Official adapters include: | [`@tanstack/ai-grok`](https://tanstack.com/ai/latest/docs/adapters/grok) | xAI Grok chat, images, and realtime | | [`@tanstack/ai-groq`](https://tanstack.com/ai/latest/docs/adapters/groq) | Groq low-latency inference | | [`@tanstack/ai-elevenlabs`](https://tanstack.com/ai/latest/docs/adapters/elevenlabs) | ElevenLabs realtime voice, speech, transcription, music, and sound effects | +| [`@tanstack/ai-cartesia`](https://tanstack.com/ai/latest/docs/adapters/cartesia) | Cartesia Sonic speech and Ink Whisper transcription | | [`@tanstack/ai-byteplus`](https://tanstack.com/ai/latest/docs/adapters/byteplus) | BytePlus Seed chat, Seedance video, Seedream image, and Seed Speech TTS/ASR | | [`@tanstack/ai-fal`](https://tanstack.com/ai/latest/docs/adapters/fal) | fal.ai image, video, audio, speech, and transcription models | | [`@tanstack/ai-reactor`](https://tanstack.com/ai/latest/docs/adapters/reactor) | Reactor live world and video generation (Orbis, Happy Oyster, LingBot, Helios, FastH3) | diff --git a/docs/adapters/cartesia.md b/docs/adapters/cartesia.md new file mode 100644 index 0000000000..cf626b59a9 --- /dev/null +++ b/docs/adapters/cartesia.md @@ -0,0 +1,82 @@ +--- +title: Cartesia +id: cartesia-adapter +order: 10 +description: "Generate speech with Sonic 3.6 and transcribe recordings with Ink Whisper in TanStack AI." +keywords: + - tanstack ai + - cartesia + - sonic + - ink whisper + - text to speech + - transcription +--- + +Call `cartesiaSpeech()` to generate audio with Sonic, or call `cartesiaTranscription()` to transcribe a recording with Ink Whisper. + +## 1. Install + + + +react: @tanstack/ai @tanstack/ai-cartesia +vue: @tanstack/ai @tanstack/ai-cartesia +solid: @tanstack/ai @tanstack/ai-cartesia +svelte: @tanstack/ai @tanstack/ai-cartesia +preact: @tanstack/ai @tanstack/ai-cartesia +angular: @tanstack/ai @tanstack/ai-cartesia +vanilla: @tanstack/ai @tanstack/ai-cartesia +octane: @tanstack/ai @tanstack/ai-cartesia + + + +## 2. Set the API key + +Put your Cartesia key in the environment of your server: + +```bash +CARTESIA_API_KEY=sk_car_... +``` + +## 3. Generate speech + +Pass a [Cartesia voice ID](https://docs.cartesia.ai/build-with-cartesia/tts-models/latest#voice-selection) to `generateSpeech()`. The result contains base64 audio in MP3 format by default. + +```typescript +import { generateSpeech } from '@tanstack/ai' +import { cartesiaSpeech } from '@tanstack/ai-cartesia' + +const result = await generateSpeech({ + adapter: cartesiaSpeech('sonic-3.6'), + text: 'Hello from Cartesia.', + voice: 'db6b0ed5-d5d3-463d-ae85-518a07d3c2b4', + threadId: 'speech-demo', +}) + +console.log(result.format, result.contentType) +``` + +Set `format` to `mp3`, `wav`, or `pcm`. Set `speed` on `generateSpeech()` to control playback speed. Use `modelOptions` for `language`, `normalization`, `pronunciationDictId`, or `volume`. + +## 4. Transcribe a recording + +`cartesiaTranscription()` accepts an audio file, a Blob, an ArrayBuffer, or a base64 string. Use `responseFormat: 'verbose_json'` to request word timestamps. + +```typescript +import { readFile } from 'node:fs/promises' +import { generateTranscription } from '@tanstack/ai' +import { cartesiaTranscription } from '@tanstack/ai-cartesia' + +const bytes = await readFile('./recording.mp3') +const audio = new File([bytes], 'recording.mp3', { type: 'audio/mpeg' }) + +const result = await generateTranscription({ + adapter: cartesiaTranscription('ink-whisper'), + audio, + responseFormat: 'verbose_json', + threadId: 'transcription-demo', +}) + +console.log(result.text, result.words) +``` + +For browser input and server routes, see [Text-to-Speech](../media/text-to-speech) and [Transcription](../media/transcription). Both calls now use Cartesia from your server. diff --git a/docs/config.json b/docs/config.json index 3059eb1ed9..f00ee36e59 100644 --- a/docs/config.json +++ b/docs/config.json @@ -942,13 +942,13 @@ "label": "Text-to-Speech", "to": "media/text-to-speech", "addedAt": "2026-04-15", - "updatedAt": "2026-10-22" + "updatedAt": "2026-11-05" }, { "label": "Transcription", "to": "media/transcription", "addedAt": "2026-04-15", - "updatedAt": "2026-10-22" + "updatedAt": "2026-11-05" }, { "label": "Transcription: Advanced", @@ -1880,6 +1880,11 @@ "addedAt": "2026-04-15", "updatedAt": "2026-10-22" }, + { + "label": "Cartesia", + "to": "adapters/cartesia", + "addedAt": "2026-11-05" + }, { "label": "fal.ai", "to": "adapters/fal", diff --git a/docs/getting-started/overview.md b/docs/getting-started/overview.md index a867d14c2e..c4c0cc14d0 100644 --- a/docs/getting-started/overview.md +++ b/docs/getting-started/overview.md @@ -137,6 +137,7 @@ Each adapter is its own package. Install only the ones you use. | `@tanstack/ai-grok` | xAI Grok. | | `@tanstack/ai-bedrock` | Amazon Bedrock (Claude, Nova, Llama, and more through AWS). | | `@tanstack/ai-byteplus` | BytePlus (Seed chat, Seedance video, Seedream image, Seed Speech). | +| `@tanstack/ai-cartesia` | Cartesia (Sonic speech and Ink Whisper transcription). | | `@tanstack/ai-fal` | fal (image and video generation). | | `@tanstack/ai-reactor` | Reactor (live world and video generation: Orbis, Happy Oyster, LingBot, Helios, FastH3). | | `@tanstack/ai-worldlabs` | World Labs Marble (persistent 3D world generation). | diff --git a/docs/media/text-to-speech.md b/docs/media/text-to-speech.md index eec45cb579..bcb423165e 100644 --- a/docs/media/text-to-speech.md +++ b/docs/media/text-to-speech.md @@ -2,7 +2,7 @@ title: Text-to-Speech id: text-to-speech order: 3 -description: "Turn text into spoken audio with generateSpeech() and play it in the browser. Works with OpenAI, Gemini, ElevenLabs, BytePlus, fal.ai, Grok, and Groq." +description: "Turn text into spoken audio with generateSpeech() and play it in the browser. Works with OpenAI, Gemini, ElevenLabs, Cartesia, BytePlus, fal.ai, Grok, and Groq." keywords: - tanstack ai - text-to-speech @@ -110,6 +110,7 @@ Swap the adapter to change the provider. The rest of the route stays the same. | OpenAI | `openaiSpeech` from `@tanstack/ai-openai` | `tts-1-hd`, `tts-1`, `gpt-4o-mini-tts` | `OPENAI_API_KEY` | | Gemini (experimental) | `geminiSpeech` from `@tanstack/ai-gemini` | `gemini-3.1-flash-tts-preview`, `gemini-2.5-flash-preview-tts`, `gemini-2.5-pro-preview-tts` | `GOOGLE_API_KEY` or `GEMINI_API_KEY` | | ElevenLabs | `elevenlabsSpeech` from `@tanstack/ai-elevenlabs` | `eleven_v4`, `eleven_v4_turbo`, `eleven_v3`, and more | `ELEVENLABS_API_KEY` | +| Cartesia | `cartesiaSpeech` from `@tanstack/ai-cartesia` | `sonic-3.6` | `CARTESIA_API_KEY` | | BytePlus | `byteplusSpeech` from `@tanstack/ai-byteplus` | `seed-audio-1.0` | `BYTEPLUS_VOICE_API_KEY` | | fal.ai | `falSpeech` from `@tanstack/ai-fal` | Kokoro, ElevenLabs, MiniMax, Gemini TTS, Chatterbox, and more | `FAL_KEY` | | Grok | `grokSpeech` from `@tanstack/ai-grok` | `grok-tts` | `XAI_API_KEY` | diff --git a/docs/media/transcription.md b/docs/media/transcription.md index cc80d692e4..ac667b4538 100644 --- a/docs/media/transcription.md +++ b/docs/media/transcription.md @@ -2,7 +2,7 @@ title: Transcription id: transcription order: 4 -description: "Turn recorded speech into text with generateTranscription() and the useTranscription hook. Works with OpenAI, ElevenLabs, Groq, xAI, BytePlus, fal.ai, Cloudflare, and Lovable." +description: "Turn recorded speech into text with generateTranscription() and the useTranscription hook. Works with OpenAI, ElevenLabs, Cartesia, Groq, xAI, BytePlus, fal.ai, Cloudflare, and Lovable." keywords: - tanstack ai - transcription @@ -156,6 +156,7 @@ Swap the adapter to change the provider. Each adapter reads its key from the env | --- | --- | --- | --- | | OpenAI | `openaiTranscription` | `gpt-4o-transcribe`, `gpt-4o-mini-transcribe`, `gpt-4o-transcribe-diarize`, `whisper-1` | `OPENAI_API_KEY` | | ElevenLabs | `elevenlabsTranscription` | `scribe_v2`, `scribe_v2_medical` | `ELEVENLABS_API_KEY` | +| Cartesia | `cartesiaTranscription` | `ink-whisper` | `CARTESIA_API_KEY` | | Groq | `groqTranscription` | `whisper-large-v3-turbo`, `whisper-large-v3` | `GROQ_API_KEY` | | xAI | `grokTranscription` | `grok-stt` | `XAI_API_KEY` | | BytePlus | `byteplusTranscription` | `seed-asr` | `BYTEPLUS_VOICE_API_KEY` | diff --git a/packages/ai-cartesia/LICENSE b/packages/ai-cartesia/LICENSE new file mode 100644 index 0000000000..308cb68dc0 --- /dev/null +++ b/packages/ai-cartesia/LICENSE @@ -0,0 +1,21 @@ +MIT License + +Copyright (c) 2025 Tanner Linsley + +Permission is hereby granted, free of charge, to any person obtaining a copy +of this software and associated documentation files (the "Software"), to deal +in the Software without restriction, including without limitation the rights +to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +copies of the Software, and to permit persons to whom the Software is +furnished to do so, subject to the following conditions: + +The above copyright notice and this permission notice shall be included in all +copies or substantial portions of the Software. + +THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE +SOFTWARE. diff --git a/packages/ai-cartesia/README.md b/packages/ai-cartesia/README.md new file mode 100644 index 0000000000..a2144f75b1 --- /dev/null +++ b/packages/ai-cartesia/README.md @@ -0,0 +1,33 @@ +
+ + + + TanStack AI + +
+ +
+ +# @tanstack/ai-cartesia + +Cartesia text-to-speech and batch transcription adapters for TanStack AI. + +```bash +pnpm add @tanstack/ai @tanstack/ai-cartesia +``` + +Set `CARTESIA_API_KEY` on your server. Then generate speech: + +```ts +import { generateSpeech } from '@tanstack/ai' +import { cartesiaSpeech } from '@tanstack/ai-cartesia' + +const result = await generateSpeech({ + adapter: cartesiaSpeech('sonic-3.6'), + text: 'Hello from Cartesia.', + voice: 'db6b0ed5-d5d3-463d-ae85-518a07d3c2b4', + threadId: 'speech-demo', +}) +``` + +For transcription, use `cartesiaTranscription('ink-whisper')` with `generateTranscription()`. To pass a key directly, use `createCartesiaSpeech()` or `createCartesiaTranscription()`. diff --git a/packages/ai-cartesia/package.json b/packages/ai-cartesia/package.json new file mode 100644 index 0000000000..a697bd18c3 --- /dev/null +++ b/packages/ai-cartesia/package.json @@ -0,0 +1,45 @@ +{ + "name": "@tanstack/ai-cartesia", + "version": "0.0.0", + "description": "Cartesia speech and transcription adapters for TanStack AI", + "author": "Tanner Linsley", + "license": "MIT", + "homepage": "https://tanstack.com/ai", + "repository": { + "type": "git", + "url": "git+https://github.com/TanStack/ai.git", + "directory": "packages/ai-cartesia" + }, + "type": "module", + "module": "./dist/esm/index.js", + "types": "./dist/esm/index.d.ts", + "exports": { + ".": { + "types": "./dist/esm/index.d.ts", + "import": "./dist/esm/index.js" + } + }, + "sideEffects": false, + "files": ["dist", "src"], + "scripts": { + "build": "vite build", + "clean": "premove ./build ./dist", + "test:build": "publint --strict", + "test:coverage": "vitest run --coverage --coverage.include='src/**' --coverage.reporter=text-summary --coverage.reporter=json-summary", + "test:lib": "vitest run", + "test:types": "tsc", + "test:oxlint": "oxlint src --type-aware" + }, + "keywords": ["ai", "tanstack", "cartesia", "text-to-speech", "transcription"], + "devDependencies": { + "@tanstack/ai": "workspace:*", + "@vitest/coverage-v8": "4.1.10", + "vite": "^8.2.1" + }, + "peerDependencies": { + "@tanstack/ai": "workspace:^" + }, + "dependencies": { + "@tanstack/ai-utils": "workspace:^" + } +} diff --git a/packages/ai-cartesia/src/client.ts b/packages/ai-cartesia/src/client.ts new file mode 100644 index 0000000000..2a3e9bf327 --- /dev/null +++ b/packages/ai-cartesia/src/client.ts @@ -0,0 +1,67 @@ +import { base64ToArrayBuffer } from '@tanstack/ai-utils' + +export interface CartesiaConfig { + apiKey?: string + baseURL?: string + defaultHeaders?: Record +} + +export function getCartesiaApiKeyFromEnv(): string { + const key = process.env.CARTESIA_API_KEY + if (!key) throw new Error('CARTESIA_API_KEY is required') + return key +} + +export class CartesiaClient { + private readonly apiKey: string + private readonly baseURL: string + private readonly headers: Record + + constructor(config: CartesiaConfig = {}) { + this.apiKey = config.apiKey ?? getCartesiaApiKeyFromEnv() + this.baseURL = (config.baseURL ?? 'https://api.cartesia.ai').replace( + /\/$/, + '', + ) + this.headers = config.defaultHeaders ?? {} + } + + async post( + path: string, + body: BodyInit, + signal?: AbortSignal, + ): Promise { + const response = await fetch(`${this.baseURL}${path}`, { + method: 'POST', + headers: { + ...this.headers, + Authorization: `Bearer ${this.apiKey}`, + 'Cartesia-Version': '2026-08-14', + ...(typeof body === 'string' + ? { 'Content-Type': 'application/json' } + : {}), + }, + body, + signal, + }) + if (!response.ok) { + const message = await response.text() + throw new Error(`Cartesia API error ${response.status}: ${message}`) + } + return response + } +} + +export function toAudioFile(audio: string | File | Blob | ArrayBuffer): File { + if (audio instanceof File) return audio + if (audio instanceof Blob) { + return new File([audio], 'audio', { type: audio.type || 'audio/mpeg' }) + } + if (audio instanceof ArrayBuffer) { + return new File([audio], 'audio.mp3', { type: 'audio/mpeg' }) + } + const match = /^data:([^;,]+);base64,(.*)$/s.exec(audio) + const mimeType = match?.[1] ?? 'audio/mpeg' + const data = match?.[2] ?? audio + return new File([base64ToArrayBuffer(data)], 'audio', { type: mimeType }) +} diff --git a/packages/ai-cartesia/src/index.ts b/packages/ai-cartesia/src/index.ts new file mode 100644 index 0000000000..f036a0e2db --- /dev/null +++ b/packages/ai-cartesia/src/index.ts @@ -0,0 +1,18 @@ +export { + CartesiaSpeechAdapter, + cartesiaSpeech, + createCartesiaSpeech, +} from './speech' +export { + CartesiaTranscriptionAdapter, + cartesiaTranscription, + createCartesiaTranscription, +} from './transcription' +export { + CARTESIA_TTS_MODELS, + CARTESIA_TRANSCRIPTION_MODELS, +} from './model-meta' +export { getCartesiaApiKeyFromEnv } from './client' +export type { CartesiaSpeechOptions } from './speech' +export type { CartesiaConfig } from './client' +export type { CartesiaTTSModel, CartesiaTranscriptionModel } from './model-meta' diff --git a/packages/ai-cartesia/src/model-meta.ts b/packages/ai-cartesia/src/model-meta.ts new file mode 100644 index 0000000000..a75bb7733f --- /dev/null +++ b/packages/ai-cartesia/src/model-meta.ts @@ -0,0 +1,11 @@ +export const CARTESIA_TTS_MODELS = [ + 'sonic-3.6', + 'sonic-3.5', + 'sonic-3', + 'sonic-latest', +] as const +export type CartesiaTTSModel = (typeof CARTESIA_TTS_MODELS)[number] + +export const CARTESIA_TRANSCRIPTION_MODELS = ['ink-whisper'] as const +export type CartesiaTranscriptionModel = + (typeof CARTESIA_TRANSCRIPTION_MODELS)[number] diff --git a/packages/ai-cartesia/src/speech.ts b/packages/ai-cartesia/src/speech.ts new file mode 100644 index 0000000000..7645ad7d11 --- /dev/null +++ b/packages/ai-cartesia/src/speech.ts @@ -0,0 +1,116 @@ +import { BaseTTSAdapter } from '@tanstack/ai/adapters' +import { arrayBufferToBase64 } from '@tanstack/ai-utils' +import { CartesiaClient } from './client' +import type { TTSOptions, TTSResult } from '@tanstack/ai' +import type { CartesiaConfig } from './client' +import type { CartesiaTTSModel } from './model-meta' + +export interface CartesiaSpeechOptions { + language?: string + normalization?: string + pronunciationDictId?: string + volume?: number +} + +export class CartesiaSpeechAdapter< + TModel extends CartesiaTTSModel, +> extends BaseTTSAdapter { + readonly name = 'cartesia' as const + private readonly client: CartesiaClient + + constructor(model: TModel, config?: CartesiaConfig) { + super(model) + this.client = new CartesiaClient(config) + } + + async generateSpeech( + options: TTSOptions, + ): Promise { + const { logger } = options + logger.request( + `activity=generateSpeech provider=cartesia model=${this.model}`, + { + provider: this.name, + model: this.model, + }, + ) + try { + if (!options.voice) throw new Error('Cartesia speech requires a voice ID') + const format = options.format ?? 'mp3' + if (format !== 'mp3' && format !== 'wav' && format !== 'pcm') { + throw new Error(`Cartesia speech does not support format '${format}'`) + } + const outputFormat = + format === 'mp3' + ? { container: 'mp3', sample_rate: 44100, bit_rate: 128000 } + : { + container: format === 'wav' ? 'wav' : 'raw', + encoding: 'pcm_s16le', + sample_rate: 44100, + } + const modelOptions = options.modelOptions + const response = await this.client.post( + '/tts/bytes', + JSON.stringify({ + model_id: this.model, + transcript: options.text, + voice: options.voice, + output_format: outputFormat, + ...(modelOptions?.language + ? { language: modelOptions.language } + : {}), + ...(modelOptions?.normalization + ? { normalization: modelOptions.normalization } + : {}), + ...(modelOptions?.pronunciationDictId + ? { pronunciation_dict_id: modelOptions.pronunciationDictId } + : {}), + ...(options.speed != null || modelOptions?.volume != null + ? { + generation_config: { + ...(options.speed != null ? { speed: options.speed } : {}), + ...(modelOptions?.volume != null + ? { volume: modelOptions.volume } + : {}), + }, + } + : {}), + }), + options.abortSignal, + ) + return { + id: response.headers.get('x-request-id') ?? this.generateId(), + model: this.model, + audio: arrayBufferToBase64(await response.arrayBuffer()), + format, + contentType: + format === 'mp3' + ? 'audio/mpeg' + : format === 'wav' + ? 'audio/wav' + : 'audio/pcm', + } + } catch (error) { + logger.errors('cartesia.generateSpeech fatal', { + error, + source: 'cartesia.generateSpeech', + }) + throw error + } + } +} + +export function cartesiaSpeech( + model: TModel, + config?: CartesiaConfig, +): CartesiaSpeechAdapter { + return new CartesiaSpeechAdapter(model, config) +} + +export function createCartesiaSpeech( + model: TModel, + apiKey: string, + config?: Omit, +): CartesiaSpeechAdapter { + return new CartesiaSpeechAdapter(model, { ...config, apiKey }) +} diff --git a/packages/ai-cartesia/src/transcription.ts b/packages/ai-cartesia/src/transcription.ts new file mode 100644 index 0000000000..eadcf904b6 --- /dev/null +++ b/packages/ai-cartesia/src/transcription.ts @@ -0,0 +1,92 @@ +import { BaseTranscriptionAdapter } from '@tanstack/ai/adapters' +import { CartesiaClient, toAudioFile } from './client' +import type { TranscriptionOptions, TranscriptionResult } from '@tanstack/ai' +import type { CartesiaConfig } from './client' +import type { CartesiaTranscriptionModel } from './model-meta' + +interface CartesiaTranscript { + text: string + request_id?: string + language?: string + duration?: number + words?: Array<{ word: string; start: number; end: number }> +} + +export class CartesiaTranscriptionAdapter< + TModel extends CartesiaTranscriptionModel, +> extends BaseTranscriptionAdapter { + readonly name = 'cartesia' as const + private readonly client: CartesiaClient + + constructor(model: TModel, config?: CartesiaConfig) { + super(model) + this.client = new CartesiaClient(config) + } + + async transcribe( + options: TranscriptionOptions, + ): Promise { + const { logger } = options + logger.request( + `activity=transcription provider=cartesia model=${this.model}`, + { + provider: this.name, + model: this.model, + }, + ) + try { + if (options.prompt) + throw new Error('Cartesia batch transcription does not support prompt') + if ( + options.responseFormat && + options.responseFormat !== 'json' && + options.responseFormat !== 'verbose_json' + ) { + throw new Error( + `Cartesia batch transcription does not support responseFormat '${options.responseFormat}'`, + ) + } + const form = new FormData() + form.append('file', toAudioFile(options.audio)) + form.append('model', this.model) + if (options.language) form.append('language', options.language) + if (options.responseFormat === 'verbose_json') + form.append('timestamp_granularities[]', 'word') + const response = await this.client.post('/stt', form, options.abortSignal) + const result = (await response.json()) as CartesiaTranscript + return { + id: result.request_id ?? this.generateId(), + model: this.model, + text: result.text, + ...(result.language ? { language: result.language } : {}), + ...(result.duration != null ? { duration: result.duration } : {}), + ...(result.words ? { words: result.words } : {}), + } + } catch (error) { + logger.errors('cartesia.transcribe fatal', { + error, + source: 'cartesia.transcribe', + }) + throw error + } + } +} + +export function cartesiaTranscription< + TModel extends CartesiaTranscriptionModel, +>( + model: TModel, + config?: CartesiaConfig, +): CartesiaTranscriptionAdapter { + return new CartesiaTranscriptionAdapter(model, config) +} + +export function createCartesiaTranscription< + TModel extends CartesiaTranscriptionModel, +>( + model: TModel, + apiKey: string, + config?: Omit, +): CartesiaTranscriptionAdapter { + return new CartesiaTranscriptionAdapter(model, { ...config, apiKey }) +} diff --git a/packages/ai-cartesia/tests/cartesia-adapters.test.ts b/packages/ai-cartesia/tests/cartesia-adapters.test.ts new file mode 100644 index 0000000000..3af3db302d --- /dev/null +++ b/packages/ai-cartesia/tests/cartesia-adapters.test.ts @@ -0,0 +1,88 @@ +import { afterEach, describe, expect, it, vi } from 'vitest' +import { resolveDebugOption } from '@tanstack/ai/adapter-internals' +import { createCartesiaSpeech, createCartesiaTranscription } from '../src/index' + +const logger = resolveDebugOption(false) + +afterEach(() => vi.unstubAllGlobals()) + +describe('Cartesia adapters', () => { + it('sends Sonic speech options and returns audio', async () => { + const fetchMock = vi.fn( + async (_input: RequestInfo | URL, _init?: RequestInit) => + new Response(new Uint8Array([1, 2, 3]), { + headers: { 'x-request-id': 'speech-1' }, + }), + ) + vi.stubGlobal('fetch', fetchMock) + + const result = await createCartesiaSpeech( + 'sonic-3.6', + 'test-key', + ).generateSpeech({ + model: 'sonic-3.6', + text: 'hello', + voice: 'voice-1', + speed: 1.2, + modelOptions: { language: 'en', volume: 0.8 }, + logger, + }) + + const [url, init] = fetchMock.mock.calls[0]! + expect(url).toBe('https://api.cartesia.ai/tts/bytes') + expect(init?.headers).toMatchObject({ + Authorization: 'Bearer test-key', + 'Cartesia-Version': '2026-08-14', + }) + expect(JSON.parse(String(init?.body))).toMatchObject({ + model_id: 'sonic-3.6', + transcript: 'hello', + voice: 'voice-1', + output_format: { container: 'mp3', sample_rate: 44100, bit_rate: 128000 }, + language: 'en', + generation_config: { speed: 1.2, volume: 0.8 }, + }) + expect(result).toMatchObject({ + id: 'speech-1', + audio: 'AQID', + format: 'mp3', + }) + }) + + it('sends Ink Whisper audio and maps word timings', async () => { + const fetchMock = vi.fn( + async (_input: RequestInfo | URL, _init?: RequestInit) => + Response.json({ + text: 'hello', + request_id: 'stt-1', + language: 'en', + duration: 1.5, + words: [{ word: 'hello', start: 0, end: 1.5 }], + }), + ) + vi.stubGlobal('fetch', fetchMock) + + const result = await createCartesiaTranscription( + 'ink-whisper', + 'test-key', + ).transcribe({ + model: 'ink-whisper', + audio: new File([new Uint8Array([1, 2])], 'voice.wav', { + type: 'audio/wav', + }), + responseFormat: 'verbose_json', + language: 'en', + logger, + }) + + const [url, init] = fetchMock.mock.calls[0]! + expect(url).toBe('https://api.cartesia.ai/stt') + expect(init?.body).toBeInstanceOf(FormData) + const form = init?.body as FormData + expect(form.get('model')).toBe('ink-whisper') + expect(form.get('language')).toBe('en') + expect(form.get('timestamp_granularities[]')).toBe('word') + expect(result).toMatchObject({ id: 'stt-1', text: 'hello', duration: 1.5 }) + expect(result.words).toEqual([{ word: 'hello', start: 0, end: 1.5 }]) + }) +}) diff --git a/packages/ai-cartesia/tsconfig.json b/packages/ai-cartesia/tsconfig.json new file mode 100644 index 0000000000..c38689f4ea --- /dev/null +++ b/packages/ai-cartesia/tsconfig.json @@ -0,0 +1,8 @@ +{ + "extends": "../../tsconfig.base.json", + "compilerOptions": { + "outDir": "dist" + }, + "include": ["src", "tests"], + "exclude": ["node_modules", "dist"] +} diff --git a/packages/ai-cartesia/vite.config.ts b/packages/ai-cartesia/vite.config.ts new file mode 100644 index 0000000000..11f5b20b70 --- /dev/null +++ b/packages/ai-cartesia/vite.config.ts @@ -0,0 +1,37 @@ +import { defineConfig, mergeConfig } from 'vitest/config' +import { tanstackViteConfig } from '@tanstack/vite-config' +import packageJson from './package.json' + +const config = defineConfig({ + test: { + name: packageJson.name, + dir: './', + watch: false, + + globals: true, + environment: 'node', + include: ['tests/**/*.test.ts'], + coverage: { + provider: 'v8', + reporter: ['text', 'json', 'html', 'lcov'], + exclude: [ + 'node_modules/', + 'dist/', + 'tests/', + '**/*.test.ts', + '**/*.config.ts', + '**/types.ts', + ], + include: ['src/**/*.ts'], + }, + }, +}) + +export default mergeConfig( + config, + tanstackViteConfig({ + entry: ['./src/index.ts'], + srcDir: './src', + cjs: false, + }), +) diff --git a/pnpm-lock.yaml b/pnpm-lock.yaml index 1a01816aff..90c5ce658e 100644 --- a/pnpm-lock.yaml +++ b/pnpm-lock.yaml @@ -2627,6 +2627,22 @@ importers: specifier: ^8.3.8 version: 8.3.8(@types/node@24.13.6)(esbuild@0.28.2)(jiti@2.7.0)(less@4.6.6)(sass-embedded@1.108.2)(sass@1.108.2)(terser@5.44.1)(tsx@4.23.15)(yaml@2.9.0) + packages/ai-cartesia: + dependencies: + '@tanstack/ai-utils': + specifier: workspace:^ + version: link:../ai-utils + devDependencies: + '@tanstack/ai': + specifier: workspace:* + version: link:../ai + '@vitest/coverage-v8': + specifier: 4.1.10 + version: 4.1.10(vitest@4.1.11) + vite: + specifier: ^8.2.1 + version: 8.3.0(@types/node@24.13.6)(esbuild@0.28.2)(jiti@2.7.0)(less@4.6.6)(sass-embedded@1.107.1)(sass@1.107.1)(terser@5.44.1)(tsx@4.23.12)(yaml@2.9.0) + packages/ai-claude-code: devDependencies: '@tanstack/ai': @@ -4396,6 +4412,9 @@ importers: '@tanstack/ai-byteplus': specifier: workspace:* version: link:../../packages/ai-byteplus + '@tanstack/ai-cartesia': + specifier: workspace:* + version: link:../../packages/ai-cartesia '@tanstack/ai-claude-code': specifier: workspace:* version: link:../../packages/ai-claude-code diff --git a/testing/e2e/global-setup.ts b/testing/e2e/global-setup.ts index 900ba7bf3c..7b1248e5e6 100644 --- a/testing/e2e/global-setup.ts +++ b/testing/e2e/global-setup.ts @@ -49,6 +49,8 @@ export default async function globalSetup() { // pattern above. mock.mount('/v1/text-to-speech', elevenLabsTTSMount()) mock.mount('/v1/speech-to-text', elevenLabsSTTMount()) + mock.mount('/tts/bytes', cartesiaTTSMount()) + mock.mount('/stt', cartesiaSTTMount()) // Gemini TTS hits the standard Gemini generateContent endpoint // (POST /v1beta/models/{model}:generateContent) with @@ -1674,6 +1676,58 @@ function byteplusASRMount(): Mountable { } } +function cartesiaTTSMount(): Mountable { + return { + async handleRequest(req, res, pathname): Promise { + if (pathname !== '/' || req.method !== 'POST') return false + const body = await readJsonRequestBody(req) + const output = asRecord(body?.output_format) + if ( + body?.model_id !== 'sonic-3.6' || + typeof body.transcript !== 'string' || + typeof body.voice !== 'string' || + output?.container !== 'mp3' || + req.headers['cartesia-version'] !== '2026-08-14' + ) { + return rejectVoiceRequest(res, 'Invalid Cartesia speech request') + } + res.statusCode = 200 + res.setHeader('Content-Type', 'audio/mpeg') + res.end(FAKE_MP3_BYTES) + return true + }, + } +} + +function cartesiaSTTMount(): Mountable { + return { + async handleRequest(req, res, pathname): Promise { + if (pathname !== '/' || req.method !== 'POST') return false + const body = await readBody(req) + if ( + !body.includes('name="file"') || + !body.includes('name="model"') || + !body.includes('ink-whisper') || + req.headers['cartesia-version'] !== '2026-08-14' + ) { + return rejectVoiceRequest(res, 'Invalid Cartesia transcription request') + } + res.statusCode = 200 + res.setHeader('Content-Type', 'application/json') + res.end( + JSON.stringify({ + type: 'transcript', + text: 'I would like to buy a Fender Stratocaster please', + request_id: 'cartesia-e2e', + language: 'en', + duration: 2.4, + }), + ) + return true + }, + } +} + /** * Mounts Claude-shaped responses for server-tool regression coverage. The * streaming response reproduces issue #604, while the request validator diff --git a/testing/e2e/package.json b/testing/e2e/package.json index 00de379746..4154362272 100644 --- a/testing/e2e/package.json +++ b/testing/e2e/package.json @@ -25,6 +25,7 @@ "@tanstack/ai-anthropic": "workspace:*", "@tanstack/ai-bedrock": "workspace:*", "@tanstack/ai-byteplus": "workspace:*", + "@tanstack/ai-cartesia": "workspace:*", "@tanstack/ai-claude-code": "workspace:*", "@tanstack/ai-client": "workspace:*", "@tanstack/ai-cloudflare": "workspace:*", diff --git a/testing/e2e/src/components/TTSUI.tsx b/testing/e2e/src/components/TTSUI.tsx index c585d91f9a..0f7aad02b6 100644 --- a/testing/e2e/src/components/TTSUI.tsx +++ b/testing/e2e/src/components/TTSUI.tsx @@ -15,9 +15,9 @@ interface TTSUIProps { aimockPort?: number } -// ElevenLabs requires a voice on every TTS call. The other providers infer -// one from their adapter config, so we only attach this when needed. +// ElevenLabs and Cartesia require a voice on every TTS call. const ELEVENLABS_DEFAULT_VOICE = '21m00Tcm4TlvDq8ikWAM' +const CARTESIA_DEFAULT_VOICE = 'db6b0ed5-d5d3-463d-ae85-518a07d3c2b4' // Deliberately synthetic voice ids: the dialogue mounts only check that each // turn carries *a* voice, and fake-but-plausible provider ids get copied into @@ -30,7 +30,11 @@ const DIALOGUE_TURNS: Array = [ export function TTSUI({ provider, mode, testId, aimockPort }: TTSUIProps) { const [text, setText] = useState('') const defaultVoice = - provider === 'elevenlabs' ? ELEVENLABS_DEFAULT_VOICE : undefined + provider === 'elevenlabs' + ? ELEVENLABS_DEFAULT_VOICE + : provider === 'cartesia' + ? CARTESIA_DEFAULT_VOICE + : undefined const connectionOptions = () => { const body = { provider, testId, aimockPort } diff --git a/testing/e2e/src/lib/feature-support.ts b/testing/e2e/src/lib/feature-support.ts index 8cecc8cc27..faea10829d 100644 --- a/testing/e2e/src/lib/feature-support.ts +++ b/testing/e2e/src/lib/feature-support.ts @@ -475,6 +475,7 @@ export const matrix: Record> = { 'gemini', 'grok', 'elevenlabs', + 'cartesia', 'byteplus', 'lovable', ]), @@ -489,6 +490,7 @@ export const matrix: Record> = { 'grok', 'groq', 'elevenlabs', + 'cartesia', 'byteplus', 'lovable', ]), diff --git a/testing/e2e/src/lib/media-providers.ts b/testing/e2e/src/lib/media-providers.ts index be75700452..c0edad7fe6 100644 --- a/testing/e2e/src/lib/media-providers.ts +++ b/testing/e2e/src/lib/media-providers.ts @@ -21,6 +21,10 @@ import { createElevenLabsSpeech, createElevenLabsTranscription, } from '@tanstack/ai-elevenlabs' +import { + createCartesiaSpeech, + createCartesiaTranscription, +} from '@tanstack/ai-cartesia' import { createBytePlusImage, createBytePlusSpeech, @@ -159,6 +163,11 @@ export function createTTSAdapter( baseUrl: llmockBase(aimockPort), headers, }), + cartesia: () => + createCartesiaSpeech('sonic-3.6', DUMMY_KEY, { + baseURL: llmockBase(aimockPort), + defaultHeaders: headers, + }), // Seed Speech is a separate BytePlus product from Ark with its own key and // its own host, so this takes the bare mock base — the adapter appends // `/api/v3/tts/create`, which `byteplusTTSMount()` serves. @@ -207,6 +216,11 @@ export function createTranscriptionAdapter( baseUrl: llmockBase(aimockPort), headers, }), + cartesia: () => + createCartesiaTranscription('ink-whisper', DUMMY_KEY, { + baseURL: llmockBase(aimockPort), + defaultHeaders: headers, + }), // Same Seed Speech host as the TTS adapter above; the adapter appends // `/api/v3/auc/bigmodel/recognize/flash`, served by `byteplusASRMount()`. byteplus: () => diff --git a/testing/e2e/src/lib/providers.ts b/testing/e2e/src/lib/providers.ts index 456878d3b3..92f7837b44 100644 --- a/testing/e2e/src/lib/providers.ts +++ b/testing/e2e/src/lib/providers.ts @@ -115,6 +115,7 @@ const defaultModels: Record = { // it out of text features, but we still need an entry to satisfy the // Record constraint. elevenlabs: '', + cartesia: '', llmgateway: 'gpt-5.6-terra', cloudflare: '@cf/zai-org/glm-5.3-flash', neon: 'gemini-3-8-flash', @@ -399,6 +400,11 @@ export function createTextAdapter( 'ElevenLabs has no text/chat adapter — use createTTSAdapter or createTranscriptionAdapter.', ) }, + cartesia: () => { + throw new Error( + 'Cartesia has no text/chat adapter — use createTTSAdapter or createTranscriptionAdapter.', + ) + }, llmgateway: () => createChatOptions({ adapter: createLLMGatewayText(model as 'gpt-5.6-terra', DUMMY_KEY, { diff --git a/testing/e2e/src/lib/types.ts b/testing/e2e/src/lib/types.ts index a0a78fdff8..8999f55285 100644 --- a/testing/e2e/src/lib/types.ts +++ b/testing/e2e/src/lib/types.ts @@ -23,6 +23,7 @@ export type Provider = | 'mistral' | 'byteplus' | 'elevenlabs' + | 'cartesia' | 'llmgateway' | 'cloudflare' | 'neon' @@ -89,6 +90,7 @@ export const ALL_PROVIDERS: Provider[] = [ 'mistral', 'byteplus', 'elevenlabs', + 'cartesia', 'llmgateway', 'cloudflare', 'neon', diff --git a/testing/e2e/tests/test-matrix.ts b/testing/e2e/tests/test-matrix.ts index dbb17aa71e..23e417af66 100644 --- a/testing/e2e/tests/test-matrix.ts +++ b/testing/e2e/tests/test-matrix.ts @@ -36,6 +36,7 @@ export const providers: Provider[] = [ 'mistral', 'byteplus', 'elevenlabs', + 'cartesia', 'llmgateway', 'cloudflare', 'neon', From fbd44a84cee3acfc6929d5ecaff3e836a1100e81 Mon Sep 17 00:00:00 2001 From: akadotsh Date: Sat, 7 Nov 2026 15:43:28 +0530 Subject: [PATCH 2/9] fix: address Cartesia adapter review feedback --- docs/config.json | 2 +- docs/media/text-to-speech.md | 2 + packages/ai-cartesia/src/client.ts | 25 +++--- packages/ai-cartesia/src/speech.ts | 5 ++ packages/ai-cartesia/src/transcription.ts | 5 ++ .../tests/cartesia-adapters.test.ts | 87 ++++++++++++++++++- testing/e2e/global-setup.ts | 45 +++++++--- testing/e2e/package.json | 6 +- testing/e2e/src/components/TTSUI.tsx | 1 + testing/e2e/src/lib/media-providers.ts | 2 + testing/e2e/src/lib/providers.ts | 1 + testing/e2e/tests/transcription.spec.ts | 20 +++++ testing/e2e/tests/tts.spec.ts | 47 ++++++++++ 13 files changed, 218 insertions(+), 30 deletions(-) diff --git a/docs/config.json b/docs/config.json index f00ee36e59..514ca25023 100644 --- a/docs/config.json +++ b/docs/config.json @@ -942,7 +942,7 @@ "label": "Text-to-Speech", "to": "media/text-to-speech", "addedAt": "2026-04-15", - "updatedAt": "2026-11-05" + "updatedAt": "2026-11-07" }, { "label": "Transcription", diff --git a/docs/media/text-to-speech.md b/docs/media/text-to-speech.md index bcb423165e..730a277878 100644 --- a/docs/media/text-to-speech.md +++ b/docs/media/text-to-speech.md @@ -105,6 +105,8 @@ Each provider has a fixed set of voices. If none of them fit, [create your own v Swap the adapter to change the provider. The rest of the route stays the same. +For Cartesia, pass a voice ID as `voice` to `generateSpeech()`. In the route above, replace the default `'alloy'` with a Cartesia voice ID, such as `'db6b0ed5-d5d3-463d-ae85-518a07d3c2b4'`. + | Provider | Adapter | Models | API key | | --- | --- | --- | --- | | OpenAI | `openaiSpeech` from `@tanstack/ai-openai` | `tts-1-hd`, `tts-1`, `gpt-4o-mini-tts` | `OPENAI_API_KEY` | diff --git a/packages/ai-cartesia/src/client.ts b/packages/ai-cartesia/src/client.ts index 2a3e9bf327..9808d17192 100644 --- a/packages/ai-cartesia/src/client.ts +++ b/packages/ai-cartesia/src/client.ts @@ -6,17 +6,21 @@ export interface CartesiaConfig { defaultHeaders?: Record } +/** Read the Cartesia API key from the server environment. */ export function getCartesiaApiKeyFromEnv(): string { - const key = process.env.CARTESIA_API_KEY + const key = + typeof process === 'undefined' ? undefined : process.env.CARTESIA_API_KEY if (!key) throw new Error('CARTESIA_API_KEY is required') return key } +/** Send authenticated requests to the Cartesia API. */ export class CartesiaClient { private readonly apiKey: string private readonly baseURL: string private readonly headers: Record + /** Configure the API key, endpoint, and additional request headers. */ constructor(config: CartesiaConfig = {}) { this.apiKey = config.apiKey ?? getCartesiaApiKeyFromEnv() this.baseURL = (config.baseURL ?? 'https://api.cartesia.ai').replace( @@ -26,21 +30,21 @@ export class CartesiaClient { this.headers = config.defaultHeaders ?? {} } + /** Send a JSON or multipart POST request to Cartesia. */ async post( path: string, body: BodyInit, signal?: AbortSignal, ): Promise { + const headers = new Headers(this.headers) + headers.set('Authorization', `Bearer ${this.apiKey}`) + headers.set('Cartesia-Version', '2026-08-14') + if (typeof body === 'string') + headers.set('Content-Type', 'application/json') + else headers.delete('Content-Type') const response = await fetch(`${this.baseURL}${path}`, { method: 'POST', - headers: { - ...this.headers, - Authorization: `Bearer ${this.apiKey}`, - 'Cartesia-Version': '2026-08-14', - ...(typeof body === 'string' - ? { 'Content-Type': 'application/json' } - : {}), - }, + headers, body, signal, }) @@ -52,6 +56,7 @@ export class CartesiaClient { } } +/** Convert an audio input to a file for multipart transcription. */ export function toAudioFile(audio: string | File | Blob | ArrayBuffer): File { if (audio instanceof File) return audio if (audio instanceof Blob) { @@ -60,7 +65,7 @@ export function toAudioFile(audio: string | File | Blob | ArrayBuffer): File { if (audio instanceof ArrayBuffer) { return new File([audio], 'audio.mp3', { type: 'audio/mpeg' }) } - const match = /^data:([^;,]+);base64,(.*)$/s.exec(audio) + const match = /^data:([^;,]+)(?:;[^;,]+)*;base64,(.*)$/is.exec(audio) const mimeType = match?.[1] ?? 'audio/mpeg' const data = match?.[2] ?? audio return new File([base64ToArrayBuffer(data)], 'audio', { type: mimeType }) diff --git a/packages/ai-cartesia/src/speech.ts b/packages/ai-cartesia/src/speech.ts index 7645ad7d11..61319b9023 100644 --- a/packages/ai-cartesia/src/speech.ts +++ b/packages/ai-cartesia/src/speech.ts @@ -12,17 +12,20 @@ export interface CartesiaSpeechOptions { volume?: number } +/** Generate speech with a Cartesia Sonic model. */ export class CartesiaSpeechAdapter< TModel extends CartesiaTTSModel, > extends BaseTTSAdapter { readonly name = 'cartesia' as const private readonly client: CartesiaClient + /** Configure the model and API connection for speech generation. */ constructor(model: TModel, config?: CartesiaConfig) { super(model) this.client = new CartesiaClient(config) } + /** Generate audio for text using the requested voice and format. */ async generateSpeech( options: TTSOptions, ): Promise { @@ -100,6 +103,7 @@ export class CartesiaSpeechAdapter< } } +/** Create a speech adapter using the environment API key or supplied config. */ export function cartesiaSpeech( model: TModel, config?: CartesiaConfig, @@ -107,6 +111,7 @@ export function cartesiaSpeech( return new CartesiaSpeechAdapter(model, config) } +/** Create a speech adapter with an explicit API key. */ export function createCartesiaSpeech( model: TModel, apiKey: string, diff --git a/packages/ai-cartesia/src/transcription.ts b/packages/ai-cartesia/src/transcription.ts index eadcf904b6..34b8ad5c75 100644 --- a/packages/ai-cartesia/src/transcription.ts +++ b/packages/ai-cartesia/src/transcription.ts @@ -12,17 +12,20 @@ interface CartesiaTranscript { words?: Array<{ word: string; start: number; end: number }> } +/** Transcribe recorded audio with a Cartesia model. */ export class CartesiaTranscriptionAdapter< TModel extends CartesiaTranscriptionModel, > extends BaseTranscriptionAdapter { readonly name = 'cartesia' as const private readonly client: CartesiaClient + /** Configure the model and API connection for transcription. */ constructor(model: TModel, config?: CartesiaConfig) { super(model) this.client = new CartesiaClient(config) } + /** Transcribe audio and return text and available word timings. */ async transcribe( options: TranscriptionOptions, ): Promise { @@ -72,6 +75,7 @@ export class CartesiaTranscriptionAdapter< } } +/** Create a transcription adapter using the environment API key or supplied config. */ export function cartesiaTranscription< TModel extends CartesiaTranscriptionModel, >( @@ -81,6 +85,7 @@ export function cartesiaTranscription< return new CartesiaTranscriptionAdapter(model, config) } +/** Create a transcription adapter with an explicit API key. */ export function createCartesiaTranscription< TModel extends CartesiaTranscriptionModel, >( diff --git a/packages/ai-cartesia/tests/cartesia-adapters.test.ts b/packages/ai-cartesia/tests/cartesia-adapters.test.ts index 3af3db302d..74bd591496 100644 --- a/packages/ai-cartesia/tests/cartesia-adapters.test.ts +++ b/packages/ai-cartesia/tests/cartesia-adapters.test.ts @@ -1,6 +1,11 @@ import { afterEach, describe, expect, it, vi } from 'vitest' import { resolveDebugOption } from '@tanstack/ai/adapter-internals' import { createCartesiaSpeech, createCartesiaTranscription } from '../src/index' +import { + CartesiaClient, + getCartesiaApiKeyFromEnv, + toAudioFile, +} from '../src/client' const logger = resolveDebugOption(false) @@ -30,10 +35,9 @@ describe('Cartesia adapters', () => { const [url, init] = fetchMock.mock.calls[0]! expect(url).toBe('https://api.cartesia.ai/tts/bytes') - expect(init?.headers).toMatchObject({ - Authorization: 'Bearer test-key', - 'Cartesia-Version': '2026-08-14', - }) + const headers = new Headers(init?.headers) + expect(headers.get('authorization')).toBe('Bearer test-key') + expect(headers.get('cartesia-version')).toBe('2026-08-14') expect(JSON.parse(String(init?.body))).toMatchObject({ model_id: 'sonic-3.6', transcript: 'hello', @@ -49,6 +53,81 @@ describe('Cartesia adapters', () => { }) }) + it.each([ + ['wav', 'wav'], + ['pcm', 'raw'], + ] as const)('sends %s output options', async (format, container) => { + const fetchMock = vi.fn( + async (_input: RequestInfo | URL, _init?: RequestInit) => + new Response(new Uint8Array([1, 2, 3])), + ) + vi.stubGlobal('fetch', fetchMock) + const result = await createCartesiaSpeech( + 'sonic-3.6', + 'test-key', + ).generateSpeech({ + model: 'sonic-3.6', + text: 'hello', + voice: 'voice-1', + format, + logger, + }) + expect( + JSON.parse(String(fetchMock.mock.calls[0]?.[1]?.body)).output_format, + ).toEqual({ + container, + encoding: 'pcm_s16le', + sample_rate: 44100, + }) + expect(result).toMatchObject({ + format, + audio: 'AQID', + contentType: `audio/${format}`, + }) + }) + + it('uses the required headers for JSON and multipart requests', async () => { + const fetchMock = vi.fn( + async (_input: RequestInfo | URL, _init?: RequestInit) => new Response(), + ) + vi.stubGlobal('fetch', fetchMock) + const client = new CartesiaClient({ + apiKey: 'key', + defaultHeaders: { + authorization: 'wrong', + 'content-type': 'wrong', + 'cartesia-version': 'wrong', + 'x-test-id': 'test', + }, + }) + await client.post('/tts/bytes', '{}') + await client.post('/stt', new FormData()) + const jsonHeaders = new Headers(fetchMock.mock.calls[0]?.[1]?.headers) + const formHeaders = new Headers(fetchMock.mock.calls[1]?.[1]?.headers) + expect(jsonHeaders.get('authorization')).toBe('Bearer key') + expect(jsonHeaders.get('content-type')).toBe('application/json') + expect(jsonHeaders.get('cartesia-version')).toBe('2026-08-14') + expect(formHeaders.get('authorization')).toBe('Bearer key') + expect(formHeaders.has('content-type')).toBe(false) + expect(formHeaders.get('x-test-id')).toBe('test') + }) + + it('reports a missing key when process is unavailable', () => { + vi.stubGlobal('process', undefined) + expect(() => getCartesiaApiKeyFromEnv()).toThrow( + 'CARTESIA_API_KEY is required', + ) + }) + + it('decodes data URLs with MIME parameters and plain base64', async () => { + const file = toAudioFile('data:audio/wav;codec=pcm;rate=24000;base64,AQID') + expect(file.type).toBe('audio/wav') + expect([...new Uint8Array(await file.arrayBuffer())]).toEqual([1, 2, 3]) + const fallback = toAudioFile('AQID') + expect(fallback.type).toBe('audio/mpeg') + expect([...new Uint8Array(await fallback.arrayBuffer())]).toEqual([1, 2, 3]) + }) + it('sends Ink Whisper audio and maps word timings', async () => { const fetchMock = vi.fn( async (_input: RequestInfo | URL, _init?: RequestInit) => diff --git a/testing/e2e/global-setup.ts b/testing/e2e/global-setup.ts index 7b1248e5e6..726ffadc21 100644 --- a/testing/e2e/global-setup.ts +++ b/testing/e2e/global-setup.ts @@ -1187,9 +1187,9 @@ const FAKE_AUDIO_BY_FORMAT: Record = { /** * Minimal RIFF/WAV container: a 44-byte header describing 16-bit mono PCM at - * 24 kHz, followed by silence. + * the requested sample rate, followed by silence. */ -function buildSilentWav(): Buffer { +function buildSilentWav(sampleRate = 24000): Buffer { const samples = Buffer.alloc(64) const header = Buffer.alloc(44) header.write('RIFF', 0, 'ascii') @@ -1199,8 +1199,8 @@ function buildSilentWav(): Buffer { header.writeUInt32LE(16, 16) // PCM chunk size header.writeUInt16LE(1, 20) // format: PCM header.writeUInt16LE(1, 22) // channels - header.writeUInt32LE(24000, 24) // sample rate - header.writeUInt32LE(24000 * 2, 28) // byte rate + header.writeUInt32LE(sampleRate, 24) // sample rate + header.writeUInt32LE(sampleRate * 2, 28) // byte rate header.writeUInt16LE(2, 32) // block align header.writeUInt16LE(16, 34) // bits per sample header.write('data', 36, 'ascii') @@ -1676,6 +1676,7 @@ function byteplusASRMount(): Mountable { } } +/** Validate Cartesia speech requests and return audio in the requested format. */ function cartesiaTTSMount(): Mountable { return { async handleRequest(req, res, pathname): Promise { @@ -1684,30 +1685,50 @@ function cartesiaTTSMount(): Mountable { const output = asRecord(body?.output_format) if ( body?.model_id !== 'sonic-3.6' || - typeof body.transcript !== 'string' || + body.transcript !== 'welcome to the guitar store' || typeof body.voice !== 'string' || - output?.container !== 'mp3' || + (output?.container !== 'mp3' && + output?.container !== 'wav' && + output?.container !== 'raw') || req.headers['cartesia-version'] !== '2026-08-14' ) { return rejectVoiceRequest(res, 'Invalid Cartesia speech request') } res.statusCode = 200 - res.setHeader('Content-Type', 'audio/mpeg') - res.end(FAKE_MP3_BYTES) + const format = output.container === 'raw' ? 'pcm' : output.container + res.setHeader( + 'Content-Type', + format === 'mp3' ? 'audio/mpeg' : `audio/${format}`, + ) + res.end( + format === 'wav' ? buildSilentWav(44100) : FAKE_AUDIO_BY_FORMAT[format], + ) return true }, } } +/** Validate Cartesia transcription uploads and return a fixed transcript. */ function cartesiaSTTMount(): Mountable { return { async handleRequest(req, res, pathname): Promise { if (pathname !== '/' || req.method !== 'POST') return false - const body = await readBody(req) + const chunks: Array = [] + for await (const chunk of req) chunks.push(Buffer.from(chunk)) + const contentType = req.headers['content-type'] ?? '' + let form: FormData + try { + form = await new Response(Buffer.concat(chunks), { + headers: { 'content-type': contentType }, + }).formData() + } catch { + return rejectVoiceRequest(res, 'Invalid Cartesia transcription request') + } + const file = form.get('file') if ( - !body.includes('name="file"') || - !body.includes('name="model"') || - !body.includes('ink-whisper') || + !(file instanceof File) || + !Buffer.from(await file.arrayBuffer()).equals(Buffer.from('Hello')) || + form.get('model') !== 'ink-whisper' || req.headers['cartesia-version'] !== '2026-08-14' ) { return rejectVoiceRequest(res, 'Invalid Cartesia transcription request') diff --git a/testing/e2e/package.json b/testing/e2e/package.json index 4154362272..a4273b2302 100644 --- a/testing/e2e/package.json +++ b/testing/e2e/package.json @@ -3,11 +3,11 @@ "private": true, "type": "module", "scripts": { - "dev": "vite dev --port 3010", + "dev": "pnpm --dir ../.. run build:all && vite dev --port 3010", "build": "vite build", "test:types": "tsc --noEmit", - "test:e2e": "playwright test", - "test:e2e:ui": "playwright test --ui", + "test:e2e": "pnpm --dir ../.. run build:all && playwright test", + "test:e2e:ui": "pnpm --dir ../.. run build:all && playwright test --ui", "record": "LLMOCK_RECORD=true pnpm dev", "postinstall": "playwright install chromium" }, diff --git a/testing/e2e/src/components/TTSUI.tsx b/testing/e2e/src/components/TTSUI.tsx index 0f7aad02b6..eeb7014f0d 100644 --- a/testing/e2e/src/components/TTSUI.tsx +++ b/testing/e2e/src/components/TTSUI.tsx @@ -27,6 +27,7 @@ const DIALOGUE_TURNS: Array = [ { text: 'what can I play for you', voice: 'e2e-voice-b' }, ] +/** Render the speech generation test UI for the selected provider and transport. */ export function TTSUI({ provider, mode, testId, aimockPort }: TTSUIProps) { const [text, setText] = useState('') const defaultVoice = diff --git a/testing/e2e/src/lib/media-providers.ts b/testing/e2e/src/lib/media-providers.ts index c0edad7fe6..dac84b3118 100644 --- a/testing/e2e/src/lib/media-providers.ts +++ b/testing/e2e/src/lib/media-providers.ts @@ -137,6 +137,7 @@ export function createImageAdapter( return factory() } +/** Create a speech adapter that sends requests to the test mock. */ export function createTTSAdapter( provider: Provider, aimockPort?: number, @@ -187,6 +188,7 @@ export function createTTSAdapter( return factory() } +/** Create a transcription adapter that sends requests to the test mock. */ export function createTranscriptionAdapter( provider: Provider, aimockPort?: number, diff --git a/testing/e2e/src/lib/providers.ts b/testing/e2e/src/lib/providers.ts index 92f7837b44..b606912c31 100644 --- a/testing/e2e/src/lib/providers.ts +++ b/testing/e2e/src/lib/providers.ts @@ -124,6 +124,7 @@ const defaultModels: Record = { muse: 'muse-ember-1.0', } +/** Create the selected chat adapter with the test mock configuration. */ export function createTextAdapter( provider: Provider, modelOverride?: string, diff --git a/testing/e2e/tests/transcription.spec.ts b/testing/e2e/tests/transcription.spec.ts index faf1cd0edd..bb7fbe72a7 100644 --- a/testing/e2e/tests/transcription.spec.ts +++ b/testing/e2e/tests/transcription.spec.ts @@ -2,6 +2,26 @@ import { test, expect } from './fixtures' import { clickGenerate, waitForGenerationComplete, featureUrl } from './helpers' import { providersFor } from './test-matrix' +test('cartesia mock rejects an empty file or incorrect model', async ({ + testId, + aimockPort, +}) => { + for (const [audio, model] of [ + ['', 'ink-whisper'], + ['Hello', 'wrong'], + ] as const) { + const form = new FormData() + form.append('file', new File([audio], 'audio.mp3', { type: 'audio/mpeg' })) + form.append('model', model) + const response = await fetch(`http://127.0.0.1:${aimockPort}/stt`, { + method: 'POST', + headers: { 'Cartesia-Version': '2026-08-14', 'X-Test-Id': testId }, + body: form, + }) + expect(response.status).toBe(400) + } +}) + for (const provider of providersFor('transcription')) { test.describe(`${provider} -- transcription`, () => { test('sse -- transcribes audio via SSE connection', async ({ diff --git a/testing/e2e/tests/tts.spec.ts b/testing/e2e/tests/tts.spec.ts index 2129a94287..2e946a81d5 100644 --- a/testing/e2e/tests/tts.spec.ts +++ b/testing/e2e/tests/tts.spec.ts @@ -1,5 +1,6 @@ import { generateSpeech } from '@tanstack/ai' import { createElevenLabsSpeech } from '@tanstack/ai-elevenlabs' +import { createCartesiaSpeech } from '@tanstack/ai-cartesia' import { test, expect } from './fixtures' import { fillTextInput, @@ -9,6 +10,52 @@ import { } from './helpers' import { providersFor } from './test-matrix' +for (const format of ['wav', 'pcm'] as const) { + test(`cartesia -- generates ${format} audio`, async ({ + testId, + aimockPort, + }) => { + const result = await generateSpeech({ + threadId: testId, + adapter: createCartesiaSpeech('sonic-3.6', 'e2e-dummy', { + baseURL: `http://127.0.0.1:${aimockPort}`, + defaultHeaders: { 'X-Test-Id': testId }, + }), + text: 'welcome to the guitar store', + voice: 'test-voice', + format, + }) + const audio = Buffer.from(result.audio, 'base64') + expect(result.format).toBe(format) + expect(result.contentType).toBe(`audio/${format}`) + if (format === 'wav') { + expect(audio.toString('ascii', 0, 4)).toBe('RIFF') + expect(audio.readUInt32LE(24)).toBe(44100) + } else expect(audio).toEqual(Buffer.alloc(32)) + }) +} + +test('cartesia mock rejects a different transcript', async ({ + testId, + aimockPort, +}) => { + const response = await fetch(`http://127.0.0.1:${aimockPort}/tts/bytes`, { + method: 'POST', + headers: { + 'Content-Type': 'application/json', + 'Cartesia-Version': '2026-08-14', + 'X-Test-Id': testId, + }, + body: JSON.stringify({ + model_id: 'sonic-3.6', + transcript: 'wrong', + voice: 'test-voice', + output_format: { container: 'mp3' }, + }), + }) + expect(response.status).toBe(400) +}) + test.describe('elevenlabs -- tts formats', () => { test('returns playable WAV from the PCM endpoint', async ({ page, From acdd02aae89039290cc3d1a8d98f14c60129832d Mon Sep 17 00:00:00 2001 From: Tom Beckenham <34339192+tombeckenham@users.noreply.github.com> Date: Mon, 9 Nov 2026 14:06:22 +1100 Subject: [PATCH 3/9] fix(ai-cartesia): reject invalid speech controls before fetch Cartesia accepts speed from 0.6 to 1.5 and volume from 0.5 to 2. Throw before the request, reject empty audio and a transcript with no text, and stop the e2e scripts from building the repo again. --- docs/adapters/cartesia.md | 12 +++- docs/config.json | 3 +- docs/media/text-to-speech.md | 4 +- docs/media/transcription.md | 6 +- packages/ai-cartesia/src/speech.ts | 24 ++++++- packages/ai-cartesia/src/transcription.ts | 3 + .../tests/cartesia-adapters.test.ts | 69 +++++++++++++++++++ testing/e2e/package.json | 6 +- 8 files changed, 114 insertions(+), 13 deletions(-) diff --git a/docs/adapters/cartesia.md b/docs/adapters/cartesia.md index cf626b59a9..b340d07643 100644 --- a/docs/adapters/cartesia.md +++ b/docs/adapters/cartesia.md @@ -55,7 +55,13 @@ const result = await generateSpeech({ console.log(result.format, result.contentType) ``` -Set `format` to `mp3`, `wav`, or `pcm`. Set `speed` on `generateSpeech()` to control playback speed. Use `modelOptions` for `language`, `normalization`, `pronunciationDictId`, or `volume`. +Set `format` to `mp3`, `wav`, or `pcm`. `pcm` audio is mono `pcm_s16le` at 44100 Hz and has no header. + +`speed` on `generateSpeech()` sets the speaking rate. Valid values are from 0.6 to 1.5. Values outside that range throw before the request. + +`modelOptions.volume` sets the loudness. Valid values are from 0.5 to 2. Values outside that range throw before the request. + +Use `modelOptions` for `language`, `normalization`, and `pronunciationDictId`. ## 4. Transcribe a recording @@ -79,4 +85,6 @@ const result = await generateTranscription({ console.log(result.text, result.words) ``` -For browser input and server routes, see [Text-to-Speech](../media/text-to-speech) and [Transcription](../media/transcription). Both calls now use Cartesia from your server. +For browser input and server routes, see [Text-to-Speech](../media/text-to-speech) and [Transcription](../media/transcription). The samples on those pages use OpenAI. + +For Cartesia, call `cartesiaSpeech` or `cartesiaTranscription`. Set `CARTESIA_API_KEY` on the server. Pass a Cartesia voice id to `generateSpeech()`. diff --git a/docs/config.json b/docs/config.json index 514ca25023..fff815b21f 100644 --- a/docs/config.json +++ b/docs/config.json @@ -1883,7 +1883,8 @@ { "label": "Cartesia", "to": "adapters/cartesia", - "addedAt": "2026-11-05" + "addedAt": "2026-11-05", + "updatedAt": "2026-11-09" }, { "label": "fal.ai", diff --git a/docs/media/text-to-speech.md b/docs/media/text-to-speech.md index 730a277878..5223ba9a54 100644 --- a/docs/media/text-to-speech.md +++ b/docs/media/text-to-speech.md @@ -105,14 +105,14 @@ Each provider has a fixed set of voices. If none of them fit, [create your own v Swap the adapter to change the provider. The rest of the route stays the same. -For Cartesia, pass a voice ID as `voice` to `generateSpeech()`. In the route above, replace the default `'alloy'` with a Cartesia voice ID, such as `'db6b0ed5-d5d3-463d-ae85-518a07d3c2b4'`. +For Cartesia, use `cartesiaSpeech` from `@tanstack/ai-cartesia` in that route. Pass a Cartesia voice id as `voice`, for example `'db6b0ed5-d5d3-463d-ae85-518a07d3c2b4'`. Set `CARTESIA_API_KEY` on the server. | Provider | Adapter | Models | API key | | --- | --- | --- | --- | | OpenAI | `openaiSpeech` from `@tanstack/ai-openai` | `tts-1-hd`, `tts-1`, `gpt-4o-mini-tts` | `OPENAI_API_KEY` | | Gemini (experimental) | `geminiSpeech` from `@tanstack/ai-gemini` | `gemini-3.1-flash-tts-preview`, `gemini-2.5-flash-preview-tts`, `gemini-2.5-pro-preview-tts` | `GOOGLE_API_KEY` or `GEMINI_API_KEY` | | ElevenLabs | `elevenlabsSpeech` from `@tanstack/ai-elevenlabs` | `eleven_v4`, `eleven_v4_turbo`, `eleven_v3`, and more | `ELEVENLABS_API_KEY` | -| Cartesia | `cartesiaSpeech` from `@tanstack/ai-cartesia` | `sonic-3.6` | `CARTESIA_API_KEY` | +| Cartesia | `cartesiaSpeech` from `@tanstack/ai-cartesia` | `sonic-3.6`, `sonic-3.5`, `sonic-3`, `sonic-latest` | `CARTESIA_API_KEY` | | BytePlus | `byteplusSpeech` from `@tanstack/ai-byteplus` | `seed-audio-1.0` | `BYTEPLUS_VOICE_API_KEY` | | fal.ai | `falSpeech` from `@tanstack/ai-fal` | Kokoro, ElevenLabs, MiniMax, Gemini TTS, Chatterbox, and more | `FAL_KEY` | | Grok | `grokSpeech` from `@tanstack/ai-grok` | `grok-tts` | `XAI_API_KEY` | diff --git a/docs/media/transcription.md b/docs/media/transcription.md index ac667b4538..616c9225e0 100644 --- a/docs/media/transcription.md +++ b/docs/media/transcription.md @@ -145,7 +145,7 @@ console.log(result.text) - A `File` or a `Blob`. - An `ArrayBuffer`. -- A base64 string. The OpenAI and Groq adapters treat plain base64 as `audio/mpeg`. +- A base64 string. The OpenAI, Groq, and Cartesia adapters treat plain base64 as `audio/mpeg`. - A data URL, such as `data:audio/webm;base64,...`. The type in the URL goes to the provider. ## Providers @@ -172,8 +172,8 @@ Need speaker labels? Use `gpt-4o-transcribe-diarize`, or BytePlus with `enable_s | --- | --- | --- | | `audio` | `string \| File \| Blob \| ArrayBuffer` | The audio to transcribe (required) | | `language` | `string` | The language in ISO-639-1 format, for example `"en"`, `"es"`, or `"ja"`. A correct language makes the result more accurate and faster. | -| `prompt` | `string` | Words or names the model must expect, for example `"Technical terms: API, SDK, CLI"`. `gpt-4o-transcribe-diarize` does not support it. | -| `responseFormat` | `'json' \| 'text' \| 'srt' \| 'verbose_json' \| 'vtt'` | The output format. `verbose_json` adds timestamps and segments. | +| `prompt` | `string` | Words or names the model must expect, for example `"Technical terms: API, SDK, CLI"`. `gpt-4o-transcribe-diarize` and Cartesia do not support `prompt`. | +| `responseFormat` | `'json' \| 'text' \| 'srt' \| 'verbose_json' \| 'vtt'` | The output format. `verbose_json` adds timestamps and segments. Cartesia accepts only `json` and `verbose_json`. For Cartesia, `verbose_json` returns word timestamps and does not return segments. | | `modelOptions` | `object` | Provider options. They are typed for the model you pass. | | `debug` | `DebugOption` | Log the request and every raw provider chunk. See [Debug Logging](../advanced/debug-logging). | diff --git a/packages/ai-cartesia/src/speech.ts b/packages/ai-cartesia/src/speech.ts index 61319b9023..58675fbed0 100644 --- a/packages/ai-cartesia/src/speech.ts +++ b/packages/ai-cartesia/src/speech.ts @@ -43,6 +43,23 @@ export class CartesiaSpeechAdapter< if (format !== 'mp3' && format !== 'wav' && format !== 'pcm') { throw new Error(`Cartesia speech does not support format '${format}'`) } + if ( + options.speed != null && + (options.speed < 0.6 || options.speed > 1.5) + ) { + throw new Error( + `Cartesia speech speed must be from 0.6 to 1.5 (received ${options.speed})`, + ) + } + const modelOptions = options.modelOptions + if ( + modelOptions?.volume != null && + (modelOptions.volume < 0.5 || modelOptions.volume > 2) + ) { + throw new Error( + `Cartesia speech volume must be from 0.5 to 2 (received ${modelOptions.volume})`, + ) + } const outputFormat = format === 'mp3' ? { container: 'mp3', sample_rate: 44100, bit_rate: 128000 } @@ -51,7 +68,6 @@ export class CartesiaSpeechAdapter< encoding: 'pcm_s16le', sample_rate: 44100, } - const modelOptions = options.modelOptions const response = await this.client.post( '/tts/bytes', JSON.stringify({ @@ -81,10 +97,14 @@ export class CartesiaSpeechAdapter< }), options.abortSignal, ) + const bytes = await response.arrayBuffer() + if (bytes.byteLength === 0) { + throw new Error('Cartesia speech returned empty audio') + } return { id: response.headers.get('x-request-id') ?? this.generateId(), model: this.model, - audio: arrayBufferToBase64(await response.arrayBuffer()), + audio: arrayBufferToBase64(bytes), format, contentType: format === 'mp3' diff --git a/packages/ai-cartesia/src/transcription.ts b/packages/ai-cartesia/src/transcription.ts index 34b8ad5c75..e0850bca27 100644 --- a/packages/ai-cartesia/src/transcription.ts +++ b/packages/ai-cartesia/src/transcription.ts @@ -57,6 +57,9 @@ export class CartesiaTranscriptionAdapter< form.append('timestamp_granularities[]', 'word') const response = await this.client.post('/stt', form, options.abortSignal) const result = (await response.json()) as CartesiaTranscript + if (typeof result.text !== 'string') { + throw new Error('Cartesia transcription response has no text') + } return { id: result.request_id ?? this.generateId(), model: this.model, diff --git a/packages/ai-cartesia/tests/cartesia-adapters.test.ts b/packages/ai-cartesia/tests/cartesia-adapters.test.ts index 74bd591496..6e776cbc07 100644 --- a/packages/ai-cartesia/tests/cartesia-adapters.test.ts +++ b/packages/ai-cartesia/tests/cartesia-adapters.test.ts @@ -164,4 +164,73 @@ describe('Cartesia adapters', () => { expect(result).toMatchObject({ id: 'stt-1', text: 'hello', duration: 1.5 }) expect(result.words).toEqual([{ word: 'hello', start: 0, end: 1.5 }]) }) + + it('throws the Cartesia status and body on a non-OK response', async () => { + vi.stubGlobal( + 'fetch', + vi.fn(async () => new Response('nope', { status: 400 })), + ) + await expect( + createCartesiaSpeech('sonic-3.6', 'test-key').generateSpeech({ + model: 'sonic-3.6', + text: 'hello', + voice: 'voice-1', + logger, + }), + ).rejects.toThrow('Cartesia API error 400: nope') + }) + + it('rejects speed and volume outside the Cartesia range before fetch', async () => { + const fetchMock = vi.fn(async () => new Response(new Uint8Array([1]))) + vi.stubGlobal('fetch', fetchMock) + const speech = createCartesiaSpeech('sonic-3.6', 'test-key') + await expect( + speech.generateSpeech({ + model: 'sonic-3.6', + text: 'hello', + voice: 'voice-1', + speed: 0.5, + logger, + }), + ).rejects.toThrow('Cartesia speech speed must be from 0.6 to 1.5') + await expect( + speech.generateSpeech({ + model: 'sonic-3.6', + text: 'hello', + voice: 'voice-1', + modelOptions: { volume: 3 }, + logger, + }), + ).rejects.toThrow('Cartesia speech volume must be from 0.5 to 2') + expect(fetchMock).not.toHaveBeenCalled() + }) + + it('rejects empty speech audio', async () => { + vi.stubGlobal( + 'fetch', + vi.fn(async () => new Response(new Uint8Array())), + ) + await expect( + createCartesiaSpeech('sonic-3.6', 'test-key').generateSpeech({ + model: 'sonic-3.6', + text: 'hello', + voice: 'voice-1', + logger, + }), + ).rejects.toThrow('Cartesia speech returned empty audio') + }) + + it('rejects a transcript response with no text', async () => { + vi.stubGlobal( + 'fetch', + vi.fn(async () => Response.json({ type: 'transcript' })), + ) + await expect( + createCartesiaTranscription('ink-whisper', 'test-key').transcribe({ + model: 'ink-whisper', + audio: new File([new Uint8Array([1])], 'a.mp3', { type: 'audio/mpeg' }), + logger, + }), + ).rejects.toThrow('Cartesia transcription response has no text') + }) }) diff --git a/testing/e2e/package.json b/testing/e2e/package.json index a4273b2302..4154362272 100644 --- a/testing/e2e/package.json +++ b/testing/e2e/package.json @@ -3,11 +3,11 @@ "private": true, "type": "module", "scripts": { - "dev": "pnpm --dir ../.. run build:all && vite dev --port 3010", + "dev": "vite dev --port 3010", "build": "vite build", "test:types": "tsc --noEmit", - "test:e2e": "pnpm --dir ../.. run build:all && playwright test", - "test:e2e:ui": "pnpm --dir ../.. run build:all && playwright test --ui", + "test:e2e": "playwright test", + "test:e2e:ui": "playwright test --ui", "record": "LLMOCK_RECORD=true pnpm dev", "postinstall": "playwright install chromium" }, From a347f97726cc3fa1d38d9a61144c950e8470bed7 Mon Sep 17 00:00:00 2001 From: "autofix-ci[bot]" <114827586+autofix-ci[bot]@users.noreply.github.com> Date: Mon, 9 Nov 2026 03:09:11 +0000 Subject: [PATCH 4/9] ci: apply automated fixes --- README.md | 2 +- packages/ai-cartesia/package.json | 13 +++++++++++-- 2 files changed, 12 insertions(+), 3 deletions(-) diff --git a/README.md b/README.md index 3c2bc86b02..f7bab56995 100644 --- a/README.md +++ b/README.md @@ -240,7 +240,7 @@ Official adapters include: | [`@tanstack/ai-grok`](https://tanstack.com/ai/latest/docs/adapters/grok) | xAI Grok chat, images, and realtime | | [`@tanstack/ai-groq`](https://tanstack.com/ai/latest/docs/adapters/groq) | Groq low-latency inference | | [`@tanstack/ai-elevenlabs`](https://tanstack.com/ai/latest/docs/adapters/elevenlabs) | ElevenLabs realtime voice, speech, transcription, music, and sound effects | -| [`@tanstack/ai-cartesia`](https://tanstack.com/ai/latest/docs/adapters/cartesia) | Cartesia Sonic speech and Ink Whisper transcription | +| [`@tanstack/ai-cartesia`](https://tanstack.com/ai/latest/docs/adapters/cartesia) | Cartesia Sonic speech and Ink Whisper transcription | | [`@tanstack/ai-byteplus`](https://tanstack.com/ai/latest/docs/adapters/byteplus) | BytePlus Seed chat, Seedance video, Seedream image, and Seed Speech TTS/ASR | | [`@tanstack/ai-fal`](https://tanstack.com/ai/latest/docs/adapters/fal) | fal.ai image, video, audio, speech, and transcription models | | [`@tanstack/ai-reactor`](https://tanstack.com/ai/latest/docs/adapters/reactor) | Reactor live world and video generation (Orbis, Happy Oyster, LingBot, Helios, FastH3) | diff --git a/packages/ai-cartesia/package.json b/packages/ai-cartesia/package.json index a697bd18c3..67f1dc5f21 100644 --- a/packages/ai-cartesia/package.json +++ b/packages/ai-cartesia/package.json @@ -20,7 +20,10 @@ } }, "sideEffects": false, - "files": ["dist", "src"], + "files": [ + "dist", + "src" + ], "scripts": { "build": "vite build", "clean": "premove ./build ./dist", @@ -30,7 +33,13 @@ "test:types": "tsc", "test:oxlint": "oxlint src --type-aware" }, - "keywords": ["ai", "tanstack", "cartesia", "text-to-speech", "transcription"], + "keywords": [ + "ai", + "tanstack", + "cartesia", + "text-to-speech", + "transcription" + ], "devDependencies": { "@tanstack/ai": "workspace:*", "@vitest/coverage-v8": "4.1.10", From 8faae72b234aecac4599538ab3cf63d6860b80f0 Mon Sep 17 00:00:00 2001 From: Tom Beckenham <34339192+tombeckenham@users.noreply.github.com> Date: Tue, 17 Nov 2026 19:56:01 +1100 Subject: [PATCH 5/9] refactor(ai-cartesia): use the official Cartesia SDK Replace the hand-written fetch client and response type with @cartesia/cartesia-js. The SDK supplies the request types, the Cartesia-Version header, and an injectable fetch. - Reject a data URL that is not base64 instead of decoding it as base64. - Unit tests pass fetch in the config instead of stubbing the global. - Remove three e2e tests that called the mock and ran no adapter code. - List the Cartesia adapters in the media-generation skill. --- packages/ai-cartesia/package.json | 1 + packages/ai-cartesia/src/client.ts | 61 ++++--------- packages/ai-cartesia/src/speech.ts | 43 ++++----- packages/ai-cartesia/src/transcription.ts | 36 ++++---- .../tests/cartesia-adapters.test.ts | 90 ++++++------------- .../skills/ai-core/media-generation/SKILL.md | 10 ++- pnpm-lock.yaml | 15 ++++ testing/e2e/tests/transcription.spec.ts | 20 ----- testing/e2e/tests/tts.spec.ts | 21 ----- 9 files changed, 103 insertions(+), 194 deletions(-) diff --git a/packages/ai-cartesia/package.json b/packages/ai-cartesia/package.json index 67f1dc5f21..d46d1001dc 100644 --- a/packages/ai-cartesia/package.json +++ b/packages/ai-cartesia/package.json @@ -49,6 +49,7 @@ "@tanstack/ai": "workspace:^" }, "dependencies": { + "@cartesia/cartesia-js": "^4.2.0", "@tanstack/ai-utils": "workspace:^" } } diff --git a/packages/ai-cartesia/src/client.ts b/packages/ai-cartesia/src/client.ts index 9808d17192..31f5a5ecb0 100644 --- a/packages/ai-cartesia/src/client.ts +++ b/packages/ai-cartesia/src/client.ts @@ -1,10 +1,12 @@ +import { Cartesia } from '@cartesia/cartesia-js' import { base64ToArrayBuffer } from '@tanstack/ai-utils' +import type { ClientOptions } from '@cartesia/cartesia-js' -export interface CartesiaConfig { - apiKey?: string - baseURL?: string - defaultHeaders?: Record -} +/** The Cartesia SDK client options that the adapters accept. */ +export type CartesiaConfig = Pick< + ClientOptions, + 'apiKey' | 'baseURL' | 'defaultHeaders' | 'fetch' +> /** Read the Cartesia API key from the server environment. */ export function getCartesiaApiKeyFromEnv(): string { @@ -14,46 +16,12 @@ export function getCartesiaApiKeyFromEnv(): string { return key } -/** Send authenticated requests to the Cartesia API. */ -export class CartesiaClient { - private readonly apiKey: string - private readonly baseURL: string - private readonly headers: Record - - /** Configure the API key, endpoint, and additional request headers. */ - constructor(config: CartesiaConfig = {}) { - this.apiKey = config.apiKey ?? getCartesiaApiKeyFromEnv() - this.baseURL = (config.baseURL ?? 'https://api.cartesia.ai').replace( - /\/$/, - '', - ) - this.headers = config.defaultHeaders ?? {} - } - - /** Send a JSON or multipart POST request to Cartesia. */ - async post( - path: string, - body: BodyInit, - signal?: AbortSignal, - ): Promise { - const headers = new Headers(this.headers) - headers.set('Authorization', `Bearer ${this.apiKey}`) - headers.set('Cartesia-Version', '2026-08-14') - if (typeof body === 'string') - headers.set('Content-Type', 'application/json') - else headers.delete('Content-Type') - const response = await fetch(`${this.baseURL}${path}`, { - method: 'POST', - headers, - body, - signal, - }) - if (!response.ok) { - const message = await response.text() - throw new Error(`Cartesia API error ${response.status}: ${message}`) - } - return response - } +/** Create the Cartesia SDK client for an adapter. */ +export function createCartesiaClient(config: CartesiaConfig = {}): Cartesia { + return new Cartesia({ + ...config, + apiKey: config.apiKey ?? getCartesiaApiKeyFromEnv(), + }) } /** Convert an audio input to a file for multipart transcription. */ @@ -66,6 +34,9 @@ export function toAudioFile(audio: string | File | Blob | ArrayBuffer): File { return new File([audio], 'audio.mp3', { type: 'audio/mpeg' }) } const match = /^data:([^;,]+)(?:;[^;,]+)*;base64,(.*)$/is.exec(audio) + if (!match && audio.startsWith('data:')) { + throw new Error('Cartesia transcription needs a base64 data URL') + } const mimeType = match?.[1] ?? 'audio/mpeg' const data = match?.[2] ?? audio return new File([base64ToArrayBuffer(data)], 'audio', { type: mimeType }) diff --git a/packages/ai-cartesia/src/speech.ts b/packages/ai-cartesia/src/speech.ts index 58675fbed0..81fe40d6df 100644 --- a/packages/ai-cartesia/src/speech.ts +++ b/packages/ai-cartesia/src/speech.ts @@ -1,13 +1,15 @@ import { BaseTTSAdapter } from '@tanstack/ai/adapters' import { arrayBufferToBase64 } from '@tanstack/ai-utils' -import { CartesiaClient } from './client' +import { createCartesiaClient } from './client' +import type { Cartesia } from '@cartesia/cartesia-js' import type { TTSOptions, TTSResult } from '@tanstack/ai' import type { CartesiaConfig } from './client' import type { CartesiaTTSModel } from './model-meta' -export interface CartesiaSpeechOptions { - language?: string - normalization?: string +export interface CartesiaSpeechOptions extends Pick< + Cartesia.TTSGenerateParams, + 'language' | 'normalization' +> { pronunciationDictId?: string volume?: number } @@ -17,12 +19,12 @@ export class CartesiaSpeechAdapter< TModel extends CartesiaTTSModel, > extends BaseTTSAdapter { readonly name = 'cartesia' as const - private readonly client: CartesiaClient + private readonly client: Cartesia /** Configure the model and API connection for speech generation. */ constructor(model: TModel, config?: CartesiaConfig) { super(model) - this.client = new CartesiaClient(config) + this.client = createCartesiaClient(config) } /** Generate audio for text using the requested voice and format. */ @@ -60,7 +62,7 @@ export class CartesiaSpeechAdapter< `Cartesia speech volume must be from 0.5 to 2 (received ${modelOptions.volume})`, ) } - const outputFormat = + const outputFormat: Cartesia.TTSGenerateParams['output_format'] = format === 'mp3' ? { container: 'mp3', sample_rate: 44100, bit_rate: 128000 } : { @@ -68,34 +70,25 @@ export class CartesiaSpeechAdapter< encoding: 'pcm_s16le', sample_rate: 44100, } - const response = await this.client.post( - '/tts/bytes', - JSON.stringify({ + const response = await this.client.tts.generate( + { model_id: this.model, transcript: options.text, voice: options.voice, output_format: outputFormat, - ...(modelOptions?.language - ? { language: modelOptions.language } - : {}), - ...(modelOptions?.normalization - ? { normalization: modelOptions.normalization } - : {}), - ...(modelOptions?.pronunciationDictId - ? { pronunciation_dict_id: modelOptions.pronunciationDictId } - : {}), + language: modelOptions?.language, + normalization: modelOptions?.normalization, + pronunciation_dict_id: modelOptions?.pronunciationDictId, ...(options.speed != null || modelOptions?.volume != null ? { generation_config: { - ...(options.speed != null ? { speed: options.speed } : {}), - ...(modelOptions?.volume != null - ? { volume: modelOptions.volume } - : {}), + speed: options.speed, + volume: modelOptions?.volume, }, } : {}), - }), - options.abortSignal, + }, + { signal: options.abortSignal }, ) const bytes = await response.arrayBuffer() if (bytes.byteLength === 0) { diff --git a/packages/ai-cartesia/src/transcription.ts b/packages/ai-cartesia/src/transcription.ts index e0850bca27..4aa544f961 100644 --- a/packages/ai-cartesia/src/transcription.ts +++ b/packages/ai-cartesia/src/transcription.ts @@ -1,28 +1,21 @@ import { BaseTranscriptionAdapter } from '@tanstack/ai/adapters' -import { CartesiaClient, toAudioFile } from './client' +import { createCartesiaClient, toAudioFile } from './client' +import type { Cartesia } from '@cartesia/cartesia-js' import type { TranscriptionOptions, TranscriptionResult } from '@tanstack/ai' import type { CartesiaConfig } from './client' import type { CartesiaTranscriptionModel } from './model-meta' -interface CartesiaTranscript { - text: string - request_id?: string - language?: string - duration?: number - words?: Array<{ word: string; start: number; end: number }> -} - /** Transcribe recorded audio with a Cartesia model. */ export class CartesiaTranscriptionAdapter< TModel extends CartesiaTranscriptionModel, > extends BaseTranscriptionAdapter { readonly name = 'cartesia' as const - private readonly client: CartesiaClient + private readonly client: Cartesia /** Configure the model and API connection for transcription. */ constructor(model: TModel, config?: CartesiaConfig) { super(model) - this.client = new CartesiaClient(config) + this.client = createCartesiaClient(config) } /** Transcribe audio and return text and available word timings. */ @@ -49,14 +42,19 @@ export class CartesiaTranscriptionAdapter< `Cartesia batch transcription does not support responseFormat '${options.responseFormat}'`, ) } - const form = new FormData() - form.append('file', toAudioFile(options.audio)) - form.append('model', this.model) - if (options.language) form.append('language', options.language) - if (options.responseFormat === 'verbose_json') - form.append('timestamp_granularities[]', 'word') - const response = await this.client.post('/stt', form, options.abortSignal) - const result = (await response.json()) as CartesiaTranscript + const result = await this.client.stt.transcribe( + { + file: toAudioFile(options.audio), + model: this.model, + // The core option is a plain string. Cartesia rejects a code that + // it does not know. + language: + options.language as Cartesia.STTTranscribeParams['language'], + timestamp_granularities: + options.responseFormat === 'verbose_json' ? ['word'] : undefined, + }, + { signal: options.abortSignal }, + ) if (typeof result.text !== 'string') { throw new Error('Cartesia transcription response has no text') } diff --git a/packages/ai-cartesia/tests/cartesia-adapters.test.ts b/packages/ai-cartesia/tests/cartesia-adapters.test.ts index 6e776cbc07..6fbec04011 100644 --- a/packages/ai-cartesia/tests/cartesia-adapters.test.ts +++ b/packages/ai-cartesia/tests/cartesia-adapters.test.ts @@ -1,11 +1,7 @@ import { afterEach, describe, expect, it, vi } from 'vitest' import { resolveDebugOption } from '@tanstack/ai/adapter-internals' import { createCartesiaSpeech, createCartesiaTranscription } from '../src/index' -import { - CartesiaClient, - getCartesiaApiKeyFromEnv, - toAudioFile, -} from '../src/client' +import { getCartesiaApiKeyFromEnv, toAudioFile } from '../src/client' const logger = resolveDebugOption(false) @@ -19,12 +15,10 @@ describe('Cartesia adapters', () => { headers: { 'x-request-id': 'speech-1' }, }), ) - vi.stubGlobal('fetch', fetchMock) - const result = await createCartesiaSpeech( - 'sonic-3.6', - 'test-key', - ).generateSpeech({ + const result = await createCartesiaSpeech('sonic-3.6', 'test-key', { + fetch: fetchMock, + }).generateSpeech({ model: 'sonic-3.6', text: 'hello', voice: 'voice-1', @@ -61,11 +55,9 @@ describe('Cartesia adapters', () => { async (_input: RequestInfo | URL, _init?: RequestInit) => new Response(new Uint8Array([1, 2, 3])), ) - vi.stubGlobal('fetch', fetchMock) - const result = await createCartesiaSpeech( - 'sonic-3.6', - 'test-key', - ).generateSpeech({ + const result = await createCartesiaSpeech('sonic-3.6', 'test-key', { + fetch: fetchMock, + }).generateSpeech({ model: 'sonic-3.6', text: 'hello', voice: 'voice-1', @@ -86,32 +78,6 @@ describe('Cartesia adapters', () => { }) }) - it('uses the required headers for JSON and multipart requests', async () => { - const fetchMock = vi.fn( - async (_input: RequestInfo | URL, _init?: RequestInit) => new Response(), - ) - vi.stubGlobal('fetch', fetchMock) - const client = new CartesiaClient({ - apiKey: 'key', - defaultHeaders: { - authorization: 'wrong', - 'content-type': 'wrong', - 'cartesia-version': 'wrong', - 'x-test-id': 'test', - }, - }) - await client.post('/tts/bytes', '{}') - await client.post('/stt', new FormData()) - const jsonHeaders = new Headers(fetchMock.mock.calls[0]?.[1]?.headers) - const formHeaders = new Headers(fetchMock.mock.calls[1]?.[1]?.headers) - expect(jsonHeaders.get('authorization')).toBe('Bearer key') - expect(jsonHeaders.get('content-type')).toBe('application/json') - expect(jsonHeaders.get('cartesia-version')).toBe('2026-08-14') - expect(formHeaders.get('authorization')).toBe('Bearer key') - expect(formHeaders.has('content-type')).toBe(false) - expect(formHeaders.get('x-test-id')).toBe('test') - }) - it('reports a missing key when process is unavailable', () => { vi.stubGlobal('process', undefined) expect(() => getCartesiaApiKeyFromEnv()).toThrow( @@ -126,6 +92,9 @@ describe('Cartesia adapters', () => { const fallback = toAudioFile('AQID') expect(fallback.type).toBe('audio/mpeg') expect([...new Uint8Array(await fallback.arrayBuffer())]).toEqual([1, 2, 3]) + expect(() => toAudioFile('data:audio/wav,abc')).toThrow( + 'Cartesia transcription needs a base64 data URL', + ) }) it('sends Ink Whisper audio and maps word timings', async () => { @@ -139,11 +108,13 @@ describe('Cartesia adapters', () => { words: [{ word: 'hello', start: 0, end: 1.5 }], }), ) - vi.stubGlobal('fetch', fetchMock) const result = await createCartesiaTranscription( 'ink-whisper', 'test-key', + { + fetch: fetchMock, + }, ).transcribe({ model: 'ink-whisper', audio: new File([new Uint8Array([1, 2])], 'voice.wav', { @@ -154,7 +125,9 @@ describe('Cartesia adapters', () => { logger, }) - const [url, init] = fetchMock.mock.calls[0]! + // The SDK first calls a custom fetch with 'data:,' to find its Response + // class. The upload is the last call. + const [url, init] = fetchMock.mock.calls.at(-1)! expect(url).toBe('https://api.cartesia.ai/stt') expect(init?.body).toBeInstanceOf(FormData) const form = init?.body as FormData @@ -166,24 +139,23 @@ describe('Cartesia adapters', () => { }) it('throws the Cartesia status and body on a non-OK response', async () => { - vi.stubGlobal( - 'fetch', - vi.fn(async () => new Response('nope', { status: 400 })), - ) await expect( - createCartesiaSpeech('sonic-3.6', 'test-key').generateSpeech({ + createCartesiaSpeech('sonic-3.6', 'test-key', { + fetch: async () => new Response('nope', { status: 400 }), + }).generateSpeech({ model: 'sonic-3.6', text: 'hello', voice: 'voice-1', logger, }), - ).rejects.toThrow('Cartesia API error 400: nope') + ).rejects.toThrow('400 nope') }) it('rejects speed and volume outside the Cartesia range before fetch', async () => { const fetchMock = vi.fn(async () => new Response(new Uint8Array([1]))) - vi.stubGlobal('fetch', fetchMock) - const speech = createCartesiaSpeech('sonic-3.6', 'test-key') + const speech = createCartesiaSpeech('sonic-3.6', 'test-key', { + fetch: fetchMock, + }) await expect( speech.generateSpeech({ model: 'sonic-3.6', @@ -206,12 +178,10 @@ describe('Cartesia adapters', () => { }) it('rejects empty speech audio', async () => { - vi.stubGlobal( - 'fetch', - vi.fn(async () => new Response(new Uint8Array())), - ) await expect( - createCartesiaSpeech('sonic-3.6', 'test-key').generateSpeech({ + createCartesiaSpeech('sonic-3.6', 'test-key', { + fetch: async () => new Response(new Uint8Array()), + }).generateSpeech({ model: 'sonic-3.6', text: 'hello', voice: 'voice-1', @@ -221,12 +191,10 @@ describe('Cartesia adapters', () => { }) it('rejects a transcript response with no text', async () => { - vi.stubGlobal( - 'fetch', - vi.fn(async () => Response.json({ type: 'transcript' })), - ) await expect( - createCartesiaTranscription('ink-whisper', 'test-key').transcribe({ + createCartesiaTranscription('ink-whisper', 'test-key', { + fetch: async () => Response.json({ type: 'transcript' }), + }).transcribe({ model: 'ink-whisper', audio: new File([new Uint8Array([1])], 'a.mp3', { type: 'audio/mpeg' }), logger, diff --git a/packages/ai/skills/ai-core/media-generation/SKILL.md b/packages/ai/skills/ai-core/media-generation/SKILL.md index 598d1a1501..1573bf67b3 100644 --- a/packages/ai/skills/ai-core/media-generation/SKILL.md +++ b/packages/ai/skills/ai-core/media-generation/SKILL.md @@ -5,8 +5,8 @@ description: > activity-specific adapters: generateImage() with openaiImage/geminiImage/byteplusImage, generateAudio() with geminiAudio/falAudio, generateVideo() with async polling (openaiVideo/geminiVideo/grokVideo/falVideo/byteplusVideo/openRouterVideo, - per-model typed durations), generateSpeech() with openaiSpeech/byteplusSpeech/elevenlabsSpeech, - generateTranscription() with openaiTranscription/byteplusTranscription, + per-model typed durations), generateSpeech() with openaiSpeech/byteplusSpeech/elevenlabsSpeech/cartesiaSpeech, + generateTranscription() with openaiTranscription/byteplusTranscription/cartesiaTranscription, generateVoice() with elevenlabsVoiceDesign (create a voice, then speak with it). React hooks: useGenerateImage, useGenerateAudio, @@ -404,7 +404,11 @@ const { generate, result, isLoading } = useGenerateAudio({ ### 3. Text-to-Speech Adapters include `openaiSpeech` (tts-1, tts-1-hd, gpt-4o-mini-tts), -`byteplusSpeech` (`seed-audio-1.0`), and `elevenlabsSpeech` (`eleven_v4`). +`byteplusSpeech` (`seed-audio-1.0`), `elevenlabsSpeech` (`eleven_v4`), and +`cartesiaSpeech` (`sonic-3.6`). + +`cartesiaSpeech` requires a Cartesia voice id as `voice`. It accepts +`format: 'mp3' | 'wav' | 'pcm'`. `speed` must be from 0.6 to 1.5. `elevenlabsSpeech` accepts `format: 'mp3' | 'pcm' | 'opus' | 'wav'`. WAV output contains 44.1 kHz, 16-bit mono PCM with a RIFF header. diff --git a/pnpm-lock.yaml b/pnpm-lock.yaml index 90c5ce658e..de96868ce4 100644 --- a/pnpm-lock.yaml +++ b/pnpm-lock.yaml @@ -2629,6 +2629,9 @@ importers: packages/ai-cartesia: dependencies: + '@cartesia/cartesia-js': + specifier: ^4.2.0 + version: 4.2.0(ws@8.21.3) '@tanstack/ai-utils': specifier: workspace:^ version: link:../ai-utils @@ -5988,6 +5991,14 @@ packages: '@bufbuild/protobuf@2.15.0': resolution: {integrity: sha512-DAheWUkVr/SJTWCc+lg9dhY0eN4SaWlf4+bG1KzHeXbnqt0AfB/NX0Z+VunGlM1ki1B4zVvye27MpKh/svySUA==} + '@cartesia/cartesia-js@4.2.0': + resolution: {integrity: sha512-v5djlNl226LSYg9Q/TecHd7ClK8He6CeXtp3uCxdiK500WMgPXkOSdd5AxokWypQDkFdiGj54455oKFNFeq5jw==} + peerDependencies: + ws: ^8.18.0 + peerDependenciesMeta: + ws: + optional: true + '@cfworker/json-schema@4.1.1': resolution: {integrity: sha512-gAmrUZSGtKc3AiBL71iNWxDsyUC5uMaKKGdvzYsBoTW/xi42JQHl7eKV2OYzCUqvc+D2RCcf7EXY2iCyFIk6og==} @@ -22756,6 +22767,10 @@ snapshots: '@bufbuild/protobuf@2.15.0': {} + '@cartesia/cartesia-js@4.2.0(ws@8.21.3)': + optionalDependencies: + ws: 8.21.3 + '@cfworker/json-schema@4.1.1': {} '@changesets/apply-release-plan@7.1.0': diff --git a/testing/e2e/tests/transcription.spec.ts b/testing/e2e/tests/transcription.spec.ts index bb7fbe72a7..faf1cd0edd 100644 --- a/testing/e2e/tests/transcription.spec.ts +++ b/testing/e2e/tests/transcription.spec.ts @@ -2,26 +2,6 @@ import { test, expect } from './fixtures' import { clickGenerate, waitForGenerationComplete, featureUrl } from './helpers' import { providersFor } from './test-matrix' -test('cartesia mock rejects an empty file or incorrect model', async ({ - testId, - aimockPort, -}) => { - for (const [audio, model] of [ - ['', 'ink-whisper'], - ['Hello', 'wrong'], - ] as const) { - const form = new FormData() - form.append('file', new File([audio], 'audio.mp3', { type: 'audio/mpeg' })) - form.append('model', model) - const response = await fetch(`http://127.0.0.1:${aimockPort}/stt`, { - method: 'POST', - headers: { 'Cartesia-Version': '2026-08-14', 'X-Test-Id': testId }, - body: form, - }) - expect(response.status).toBe(400) - } -}) - for (const provider of providersFor('transcription')) { test.describe(`${provider} -- transcription`, () => { test('sse -- transcribes audio via SSE connection', async ({ diff --git a/testing/e2e/tests/tts.spec.ts b/testing/e2e/tests/tts.spec.ts index 2e946a81d5..2fa8c314dd 100644 --- a/testing/e2e/tests/tts.spec.ts +++ b/testing/e2e/tests/tts.spec.ts @@ -35,27 +35,6 @@ for (const format of ['wav', 'pcm'] as const) { }) } -test('cartesia mock rejects a different transcript', async ({ - testId, - aimockPort, -}) => { - const response = await fetch(`http://127.0.0.1:${aimockPort}/tts/bytes`, { - method: 'POST', - headers: { - 'Content-Type': 'application/json', - 'Cartesia-Version': '2026-08-14', - 'X-Test-Id': testId, - }, - body: JSON.stringify({ - model_id: 'sonic-3.6', - transcript: 'wrong', - voice: 'test-voice', - output_format: { container: 'mp3' }, - }), - }) - expect(response.status).toBe(400) -}) - test.describe('elevenlabs -- tts formats', () => { test('returns playable WAV from the PCM endpoint', async ({ page, From 4ccd933b77455611be2f10c8353f4998fa43b0a6 Mon Sep 17 00:00:00 2001 From: Tom Beckenham <34339192+tombeckenham@users.noreply.github.com> Date: Tue, 17 Nov 2026 20:03:29 +1100 Subject: [PATCH 6/9] fix(ai-cartesia): name audio uploads with a file extension A Blob or base64 input was uploaded as `audio` with no extension. Name it `audio.` from the MIME type, as the OpenAI and Groq adapters do. --- packages/ai-cartesia/src/client.ts | 24 ++++++++++++------- .../tests/cartesia-adapters.test.ts | 8 +++++++ 2 files changed, 23 insertions(+), 9 deletions(-) diff --git a/packages/ai-cartesia/src/client.ts b/packages/ai-cartesia/src/client.ts index 31f5a5ecb0..1b3501f56c 100644 --- a/packages/ai-cartesia/src/client.ts +++ b/packages/ai-cartesia/src/client.ts @@ -24,20 +24,26 @@ export function createCartesiaClient(config: CartesiaConfig = {}): Cartesia { }) } +/** Name the upload `audio.` from its MIME type, as speech APIs expect. */ +function audioFile(data: BlobPart, type: string): File { + const ext = + type === 'audio/mpeg' + ? 'mp3' + : (type.split(/[/;]/)[1]?.replace(/^x-/, '') ?? 'mp3') + return new File([data], `audio.${ext}`, { type }) +} + /** Convert an audio input to a file for multipart transcription. */ export function toAudioFile(audio: string | File | Blob | ArrayBuffer): File { if (audio instanceof File) return audio - if (audio instanceof Blob) { - return new File([audio], 'audio', { type: audio.type || 'audio/mpeg' }) - } - if (audio instanceof ArrayBuffer) { - return new File([audio], 'audio.mp3', { type: 'audio/mpeg' }) - } + if (audio instanceof Blob) return audioFile(audio, audio.type || 'audio/mpeg') + if (audio instanceof ArrayBuffer) return audioFile(audio, 'audio/mpeg') const match = /^data:([^;,]+)(?:;[^;,]+)*;base64,(.*)$/is.exec(audio) if (!match && audio.startsWith('data:')) { throw new Error('Cartesia transcription needs a base64 data URL') } - const mimeType = match?.[1] ?? 'audio/mpeg' - const data = match?.[2] ?? audio - return new File([base64ToArrayBuffer(data)], 'audio', { type: mimeType }) + return audioFile( + base64ToArrayBuffer(match?.[2] ?? audio), + match?.[1] ?? 'audio/mpeg', + ) } diff --git a/packages/ai-cartesia/tests/cartesia-adapters.test.ts b/packages/ai-cartesia/tests/cartesia-adapters.test.ts index 6fbec04011..ccc76d4e09 100644 --- a/packages/ai-cartesia/tests/cartesia-adapters.test.ts +++ b/packages/ai-cartesia/tests/cartesia-adapters.test.ts @@ -88,9 +88,17 @@ describe('Cartesia adapters', () => { it('decodes data URLs with MIME parameters and plain base64', async () => { const file = toAudioFile('data:audio/wav;codec=pcm;rate=24000;base64,AQID') expect(file.type).toBe('audio/wav') + expect(file.name).toBe('audio.wav') expect([...new Uint8Array(await file.arrayBuffer())]).toEqual([1, 2, 3]) const fallback = toAudioFile('AQID') expect(fallback.type).toBe('audio/mpeg') + expect(fallback.name).toBe('audio.mp3') + expect( + toAudioFile(new Blob(['a'], { type: 'audio/webm;codecs=opus' })).name, + ).toBe('audio.webm') + expect(toAudioFile(new Blob(['a'], { type: 'audio/x-wav' })).name).toBe( + 'audio.wav', + ) expect([...new Uint8Array(await fallback.arrayBuffer())]).toEqual([1, 2, 3]) expect(() => toAudioFile('data:audio/wav,abc')).toThrow( 'Cartesia transcription needs a base64 data URL', From 34c11af93dfadf503caaa0a0cb0a9998f2ea21a0 Mon Sep 17 00:00:00 2001 From: Tom Beckenham <34339192+tombeckenham@users.noreply.github.com> Date: Tue, 17 Nov 2026 20:04:37 +1100 Subject: [PATCH 7/9] chore(ai-cartesia): match the Vitest and Vite versions on main The merge with main left pnpm-lock.yaml with no entry for @vitest/coverage-v8@4.1.10, so a frozen install failed in CI. --- packages/ai-cartesia/package.json | 4 ++-- pnpm-lock.yaml | 8 ++++---- 2 files changed, 6 insertions(+), 6 deletions(-) diff --git a/packages/ai-cartesia/package.json b/packages/ai-cartesia/package.json index d46d1001dc..ed609ccfc1 100644 --- a/packages/ai-cartesia/package.json +++ b/packages/ai-cartesia/package.json @@ -42,8 +42,8 @@ ], "devDependencies": { "@tanstack/ai": "workspace:*", - "@vitest/coverage-v8": "4.1.10", - "vite": "^8.2.1" + "@vitest/coverage-v8": "4.1.12", + "vite": "^8.3.8" }, "peerDependencies": { "@tanstack/ai": "workspace:^" diff --git a/pnpm-lock.yaml b/pnpm-lock.yaml index de96868ce4..08bab44ed1 100644 --- a/pnpm-lock.yaml +++ b/pnpm-lock.yaml @@ -2640,11 +2640,11 @@ importers: specifier: workspace:* version: link:../ai '@vitest/coverage-v8': - specifier: 4.1.10 - version: 4.1.10(vitest@4.1.11) + specifier: 4.1.12 + version: 4.1.12(vitest@4.1.12) vite: - specifier: ^8.2.1 - version: 8.3.0(@types/node@24.13.6)(esbuild@0.28.2)(jiti@2.7.0)(less@4.6.6)(sass-embedded@1.107.1)(sass@1.107.1)(terser@5.44.1)(tsx@4.23.12)(yaml@2.9.0) + specifier: ^8.3.8 + version: 8.3.8(@types/node@24.13.6)(esbuild@0.28.2)(jiti@2.7.0)(less@4.6.6)(sass-embedded@1.108.2)(sass@1.108.2)(terser@5.44.1)(tsx@4.23.15)(yaml@2.9.0) packages/ai-claude-code: devDependencies: From 81a017bb51d1d6fc28f26c673bc7f6b79ddcfbcf Mon Sep 17 00:00:00 2001 From: Tom Beckenham <34339192+tombeckenham@users.noreply.github.com> Date: Sat, 21 Nov 2026 15:06:57 +1100 Subject: [PATCH 8/9] Fix: document Cartesia voice and transcription language Show a Cartesia voice id in the text-to-speech route. Say that a missing transcription language means English. List cartesiaTranscription limits in the media-generation skill. --- docs/adapters/cartesia.md | 4 ++- docs/config.json | 6 ++-- docs/media/text-to-speech.md | 28 +++++++++++++++++-- docs/media/transcription.md | 2 ++ packages/ai-cartesia/src/transcription.ts | 4 +-- .../skills/ai-core/media-generation/SKILL.md | 10 +++++-- 6 files changed, 44 insertions(+), 10 deletions(-) diff --git a/docs/adapters/cartesia.md b/docs/adapters/cartesia.md index b340d07643..5376baf6b1 100644 --- a/docs/adapters/cartesia.md +++ b/docs/adapters/cartesia.md @@ -67,6 +67,8 @@ Use `modelOptions` for `language`, `normalization`, and `pronunciationDictId`. `cartesiaTranscription()` accepts an audio file, a Blob, an ArrayBuffer, or a base64 string. Use `responseFormat: 'verbose_json'` to request word timestamps. +If you omit `language`, Cartesia uses English and translates other speech into English. When the recording is not English, pass `language`. + ```typescript import { readFile } from 'node:fs/promises' import { generateTranscription } from '@tanstack/ai' @@ -85,6 +87,6 @@ const result = await generateTranscription({ console.log(result.text, result.words) ``` -For browser input and server routes, see [Text-to-Speech](../media/text-to-speech) and [Transcription](../media/transcription). The samples on those pages use OpenAI. +For browser input and server routes, see [Text-to-Speech](../media/text-to-speech) and [Transcription](../media/transcription). The Text-to-Speech page includes a Cartesia route. The main Transcription sample uses OpenAI. For Cartesia, call `cartesiaSpeech` or `cartesiaTranscription`. Set `CARTESIA_API_KEY` on the server. Pass a Cartesia voice id to `generateSpeech()`. diff --git a/docs/config.json b/docs/config.json index fff815b21f..cadecfc3d4 100644 --- a/docs/config.json +++ b/docs/config.json @@ -942,13 +942,13 @@ "label": "Text-to-Speech", "to": "media/text-to-speech", "addedAt": "2026-04-15", - "updatedAt": "2026-11-07" + "updatedAt": "2026-11-21" }, { "label": "Transcription", "to": "media/transcription", "addedAt": "2026-04-15", - "updatedAt": "2026-11-05" + "updatedAt": "2026-11-21" }, { "label": "Transcription: Advanced", @@ -1884,7 +1884,7 @@ "label": "Cartesia", "to": "adapters/cartesia", "addedAt": "2026-11-05", - "updatedAt": "2026-11-09" + "updatedAt": "2026-11-21" }, { "label": "fal.ai", diff --git a/docs/media/text-to-speech.md b/docs/media/text-to-speech.md index 5223ba9a54..f76987d14c 100644 --- a/docs/media/text-to-speech.md +++ b/docs/media/text-to-speech.md @@ -103,9 +103,33 @@ Each provider has a fixed set of voices. If none of them fit, [create your own v ## Pick a provider -Swap the adapter to change the provider. The rest of the route stays the same. +Swap the adapter to change the provider. -For Cartesia, use `cartesiaSpeech` from `@tanstack/ai-cartesia` in that route. Pass a Cartesia voice id as `voice`, for example `'db6b0ed5-d5d3-463d-ae85-518a07d3c2b4'`. Set `CARTESIA_API_KEY` on the server. +For Cartesia, use this route. `voice` is a Cartesia voice id. Set `CARTESIA_API_KEY` on the server. + +```typescript +import { + generateSpeech, + generationParamsFromRequest, + toServerSentEventsResponse, +} from '@tanstack/ai' +import { cartesiaSpeech } from '@tanstack/ai-cartesia' + +export async function POST(request: Request) { + const { input, threadId } = await generationParamsFromRequest('tts', request) + + const stream = generateSpeech({ + adapter: cartesiaSpeech('sonic-3.6'), + text: input.text, + threadId, + voice: input.voice ?? 'db6b0ed5-d5d3-463d-ae85-518a07d3c2b4', + stream: true, + }) + return toServerSentEventsResponse(stream) +} +``` + +`alloy` is an OpenAI voice. Cartesia rejects it. | Provider | Adapter | Models | API key | | --- | --- | --- | --- | diff --git a/docs/media/transcription.md b/docs/media/transcription.md index 616c9225e0..59c1469bc9 100644 --- a/docs/media/transcription.md +++ b/docs/media/transcription.md @@ -166,6 +166,8 @@ Swap the adapter to change the provider. Each adapter reads its key from the env Need speaker labels? Use `gpt-4o-transcribe-diarize`, or BytePlus with `enable_speaker_info`. Examples for each provider are in [Transcription: Advanced](./transcription-advanced). +If you omit `language` for Cartesia, the request uses English and translates other speech into English. When the recording is not English, pass `language`. + ## Common options | Option | Type | Description | diff --git a/packages/ai-cartesia/src/transcription.ts b/packages/ai-cartesia/src/transcription.ts index 4aa544f961..48f5f4b056 100644 --- a/packages/ai-cartesia/src/transcription.ts +++ b/packages/ai-cartesia/src/transcription.ts @@ -46,8 +46,8 @@ export class CartesiaTranscriptionAdapter< { file: toAudioFile(options.audio), model: this.model, - // The core option is a plain string. Cartesia rejects a code that - // it does not know. + // Omit language and Cartesia uses English and translates other + // speech. The cast only narrows the string. language: options.language as Cartesia.STTTranscribeParams['language'], timestamp_granularities: diff --git a/packages/ai/skills/ai-core/media-generation/SKILL.md b/packages/ai/skills/ai-core/media-generation/SKILL.md index 1573bf67b3..980845ebd8 100644 --- a/packages/ai/skills/ai-core/media-generation/SKILL.md +++ b/packages/ai/skills/ai-core/media-generation/SKILL.md @@ -587,9 +587,15 @@ OpenAI, Gemini, and Cloudflare have fixed voice catalogs and will not get one. ### 5. Audio Transcription Adapters: `openaiTranscription` (whisper-1, gpt-4o-transcribe, -gpt-4o-mini-transcribe, gpt-4o-transcribe-diarize) and `byteplusTranscription` +gpt-4o-mini-transcribe, gpt-4o-transcribe-diarize), `byteplusTranscription` (`seed-asr` — synchronous, no polling; audio up to 2 hours / 100 MB; also reads -**`BYTEPLUS_VOICE_API_KEY`**). +**`BYTEPLUS_VOICE_API_KEY`**), and `cartesiaTranscription` (`ink-whisper`). + +`cartesiaTranscription` accepts `responseFormat` `json` or `verbose_json`. +`verbose_json` fills `result.words` (`{ word, start, end }`) and leaves +`segments` empty. A `prompt` throws. Omit `language` only for English audio: +Cartesia translates other speech into English. When the recording is not +English, pass `language` (ISO-639-1). > **Capturing audio in the browser:** Use `useAudioRecorder` from `@tanstack/ai-react` to record directly in the browser, then pass the recording as the `audio` input to `generate()`, or use `recording.part` as a prompt part in chat/generation calls. No transcoding or extra dependencies required — the recorder returns the native browser format (`audio/webm` or `audio/mp4`). For transcription, wrap it as a `data:` URL so the provider gets the real content type; passing raw `recording.base64` makes the adapter assume `audio/mpeg` and mislabel the webm/mp4 bytes. > From 269c09e0080c1f58686b0a53b3c7861398451d75 Mon Sep 17 00:00:00 2001 From: Tom Beckenham <34339192+tombeckenham@users.noreply.github.com> Date: Sat, 21 Nov 2026 15:23:34 +1100 Subject: [PATCH 9/9] chore(ai-cartesia): depend on Cartesia SDK 5.1 Speech and transcription calls stay the same. SDK 5 changes managed agents, which this package does not call. --- packages/ai-cartesia/package.json | 2 +- pnpm-lock.yaml | 10 +++++----- 2 files changed, 6 insertions(+), 6 deletions(-) diff --git a/packages/ai-cartesia/package.json b/packages/ai-cartesia/package.json index ed609ccfc1..998ec5dc8c 100644 --- a/packages/ai-cartesia/package.json +++ b/packages/ai-cartesia/package.json @@ -49,7 +49,7 @@ "@tanstack/ai": "workspace:^" }, "dependencies": { - "@cartesia/cartesia-js": "^4.2.0", + "@cartesia/cartesia-js": "^5.1.0", "@tanstack/ai-utils": "workspace:^" } } diff --git a/pnpm-lock.yaml b/pnpm-lock.yaml index 08bab44ed1..2493991973 100644 --- a/pnpm-lock.yaml +++ b/pnpm-lock.yaml @@ -2630,8 +2630,8 @@ importers: packages/ai-cartesia: dependencies: '@cartesia/cartesia-js': - specifier: ^4.2.0 - version: 4.2.0(ws@8.21.3) + specifier: ^5.1.0 + version: 5.1.0(ws@8.21.3) '@tanstack/ai-utils': specifier: workspace:^ version: link:../ai-utils @@ -5991,8 +5991,8 @@ packages: '@bufbuild/protobuf@2.15.0': resolution: {integrity: sha512-DAheWUkVr/SJTWCc+lg9dhY0eN4SaWlf4+bG1KzHeXbnqt0AfB/NX0Z+VunGlM1ki1B4zVvye27MpKh/svySUA==} - '@cartesia/cartesia-js@4.2.0': - resolution: {integrity: sha512-v5djlNl226LSYg9Q/TecHd7ClK8He6CeXtp3uCxdiK500WMgPXkOSdd5AxokWypQDkFdiGj54455oKFNFeq5jw==} + '@cartesia/cartesia-js@5.1.0': + resolution: {integrity: sha512-KTnyJrV0Hl9/ElprFexh+XEpnybE+65KoBwTUJf9q3+3ziTd+wGn8ttm3mP9HVJPKKm8TuklmdVOhL6Lx4UOJA==} peerDependencies: ws: ^8.18.0 peerDependenciesMeta: @@ -22767,7 +22767,7 @@ snapshots: '@bufbuild/protobuf@2.15.0': {} - '@cartesia/cartesia-js@4.2.0(ws@8.21.3)': + '@cartesia/cartesia-js@5.1.0(ws@8.21.3)': optionalDependencies: ws: 8.21.3