diff --git a/.changeset/soft-voices-speak.md b/.changeset/soft-voices-speak.md new file mode 100644 index 0000000000..45018c6f09 --- /dev/null +++ b/.changeset/soft-voices-speak.md @@ -0,0 +1,5 @@ +--- +'@tanstack/ai-cartesia': minor +--- + +Add Cartesia text-to-speech and batch transcription adapters. diff --git a/README.md b/README.md index f902cb8230..f7bab56995 100644 --- a/README.md +++ b/README.md @@ -240,6 +240,7 @@ Official adapters include: | [`@tanstack/ai-grok`](https://tanstack.com/ai/latest/docs/adapters/grok) | xAI Grok chat, images, and realtime | | [`@tanstack/ai-groq`](https://tanstack.com/ai/latest/docs/adapters/groq) | Groq low-latency inference | | [`@tanstack/ai-elevenlabs`](https://tanstack.com/ai/latest/docs/adapters/elevenlabs) | ElevenLabs realtime voice, speech, transcription, music, and sound effects | +| [`@tanstack/ai-cartesia`](https://tanstack.com/ai/latest/docs/adapters/cartesia) | Cartesia Sonic speech and Ink Whisper transcription | | [`@tanstack/ai-byteplus`](https://tanstack.com/ai/latest/docs/adapters/byteplus) | BytePlus Seed chat, Seedance video, Seedream image, and Seed Speech TTS/ASR | | [`@tanstack/ai-fal`](https://tanstack.com/ai/latest/docs/adapters/fal) | fal.ai image, video, audio, speech, and transcription models | | [`@tanstack/ai-reactor`](https://tanstack.com/ai/latest/docs/adapters/reactor) | Reactor live world and video generation (Orbis, Happy Oyster, LingBot, Helios, FastH3) | diff --git a/docs/adapters/cartesia.md b/docs/adapters/cartesia.md new file mode 100644 index 0000000000..5376baf6b1 --- /dev/null +++ b/docs/adapters/cartesia.md @@ -0,0 +1,92 @@ +--- +title: Cartesia +id: cartesia-adapter +order: 10 +description: "Generate speech with Sonic 3.6 and transcribe recordings with Ink Whisper in TanStack AI." +keywords: + - tanstack ai + - cartesia + - sonic + - ink whisper + - text to speech + - transcription +--- + +Call `cartesiaSpeech()` to generate audio with Sonic, or call `cartesiaTranscription()` to transcribe a recording with Ink Whisper. + +## 1. Install + + + +react: @tanstack/ai @tanstack/ai-cartesia +vue: @tanstack/ai @tanstack/ai-cartesia +solid: @tanstack/ai @tanstack/ai-cartesia +svelte: @tanstack/ai @tanstack/ai-cartesia +preact: @tanstack/ai @tanstack/ai-cartesia +angular: @tanstack/ai @tanstack/ai-cartesia +vanilla: @tanstack/ai @tanstack/ai-cartesia +octane: @tanstack/ai @tanstack/ai-cartesia + + + +## 2. Set the API key + +Put your Cartesia key in the environment of your server: + +```bash +CARTESIA_API_KEY=sk_car_... +``` + +## 3. Generate speech + +Pass a [Cartesia voice ID](https://docs.cartesia.ai/build-with-cartesia/tts-models/latest#voice-selection) to `generateSpeech()`. The result contains base64 audio in MP3 format by default. + +```typescript +import { generateSpeech } from '@tanstack/ai' +import { cartesiaSpeech } from '@tanstack/ai-cartesia' + +const result = await generateSpeech({ + adapter: cartesiaSpeech('sonic-3.6'), + text: 'Hello from Cartesia.', + voice: 'db6b0ed5-d5d3-463d-ae85-518a07d3c2b4', + threadId: 'speech-demo', +}) + +console.log(result.format, result.contentType) +``` + +Set `format` to `mp3`, `wav`, or `pcm`. `pcm` audio is mono `pcm_s16le` at 44100 Hz and has no header. + +`speed` on `generateSpeech()` sets the speaking rate. Valid values are from 0.6 to 1.5. Values outside that range throw before the request. + +`modelOptions.volume` sets the loudness. Valid values are from 0.5 to 2. Values outside that range throw before the request. + +Use `modelOptions` for `language`, `normalization`, and `pronunciationDictId`. + +## 4. Transcribe a recording + +`cartesiaTranscription()` accepts an audio file, a Blob, an ArrayBuffer, or a base64 string. Use `responseFormat: 'verbose_json'` to request word timestamps. + +If you omit `language`, Cartesia uses English and translates other speech into English. When the recording is not English, pass `language`. + +```typescript +import { readFile } from 'node:fs/promises' +import { generateTranscription } from '@tanstack/ai' +import { cartesiaTranscription } from '@tanstack/ai-cartesia' + +const bytes = await readFile('./recording.mp3') +const audio = new File([bytes], 'recording.mp3', { type: 'audio/mpeg' }) + +const result = await generateTranscription({ + adapter: cartesiaTranscription('ink-whisper'), + audio, + responseFormat: 'verbose_json', + threadId: 'transcription-demo', +}) + +console.log(result.text, result.words) +``` + +For browser input and server routes, see [Text-to-Speech](../media/text-to-speech) and [Transcription](../media/transcription). The Text-to-Speech page includes a Cartesia route. The main Transcription sample uses OpenAI. + +For Cartesia, call `cartesiaSpeech` or `cartesiaTranscription`. Set `CARTESIA_API_KEY` on the server. Pass a Cartesia voice id to `generateSpeech()`. diff --git a/docs/config.json b/docs/config.json index 3059eb1ed9..cadecfc3d4 100644 --- a/docs/config.json +++ b/docs/config.json @@ -942,13 +942,13 @@ "label": "Text-to-Speech", "to": "media/text-to-speech", "addedAt": "2026-04-15", - "updatedAt": "2026-10-22" + "updatedAt": "2026-11-21" }, { "label": "Transcription", "to": "media/transcription", "addedAt": "2026-04-15", - "updatedAt": "2026-10-22" + "updatedAt": "2026-11-21" }, { "label": "Transcription: Advanced", @@ -1880,6 +1880,12 @@ "addedAt": "2026-04-15", "updatedAt": "2026-10-22" }, + { + "label": "Cartesia", + "to": "adapters/cartesia", + "addedAt": "2026-11-05", + "updatedAt": "2026-11-21" + }, { "label": "fal.ai", "to": "adapters/fal", diff --git a/docs/getting-started/overview.md b/docs/getting-started/overview.md index a867d14c2e..c4c0cc14d0 100644 --- a/docs/getting-started/overview.md +++ b/docs/getting-started/overview.md @@ -137,6 +137,7 @@ Each adapter is its own package. Install only the ones you use. | `@tanstack/ai-grok` | xAI Grok. | | `@tanstack/ai-bedrock` | Amazon Bedrock (Claude, Nova, Llama, and more through AWS). | | `@tanstack/ai-byteplus` | BytePlus (Seed chat, Seedance video, Seedream image, Seed Speech). | +| `@tanstack/ai-cartesia` | Cartesia (Sonic speech and Ink Whisper transcription). | | `@tanstack/ai-fal` | fal (image and video generation). | | `@tanstack/ai-reactor` | Reactor (live world and video generation: Orbis, Happy Oyster, LingBot, Helios, FastH3). | | `@tanstack/ai-worldlabs` | World Labs Marble (persistent 3D world generation). | diff --git a/docs/media/text-to-speech.md b/docs/media/text-to-speech.md index eec45cb579..f76987d14c 100644 --- a/docs/media/text-to-speech.md +++ b/docs/media/text-to-speech.md @@ -2,7 +2,7 @@ title: Text-to-Speech id: text-to-speech order: 3 -description: "Turn text into spoken audio with generateSpeech() and play it in the browser. Works with OpenAI, Gemini, ElevenLabs, BytePlus, fal.ai, Grok, and Groq." +description: "Turn text into spoken audio with generateSpeech() and play it in the browser. Works with OpenAI, Gemini, ElevenLabs, Cartesia, BytePlus, fal.ai, Grok, and Groq." keywords: - tanstack ai - text-to-speech @@ -103,13 +103,40 @@ Each provider has a fixed set of voices. If none of them fit, [create your own v ## Pick a provider -Swap the adapter to change the provider. The rest of the route stays the same. +Swap the adapter to change the provider. + +For Cartesia, use this route. `voice` is a Cartesia voice id. Set `CARTESIA_API_KEY` on the server. + +```typescript +import { + generateSpeech, + generationParamsFromRequest, + toServerSentEventsResponse, +} from '@tanstack/ai' +import { cartesiaSpeech } from '@tanstack/ai-cartesia' + +export async function POST(request: Request) { + const { input, threadId } = await generationParamsFromRequest('tts', request) + + const stream = generateSpeech({ + adapter: cartesiaSpeech('sonic-3.6'), + text: input.text, + threadId, + voice: input.voice ?? 'db6b0ed5-d5d3-463d-ae85-518a07d3c2b4', + stream: true, + }) + return toServerSentEventsResponse(stream) +} +``` + +`alloy` is an OpenAI voice. Cartesia rejects it. | Provider | Adapter | Models | API key | | --- | --- | --- | --- | | OpenAI | `openaiSpeech` from `@tanstack/ai-openai` | `tts-1-hd`, `tts-1`, `gpt-4o-mini-tts` | `OPENAI_API_KEY` | | Gemini (experimental) | `geminiSpeech` from `@tanstack/ai-gemini` | `gemini-3.1-flash-tts-preview`, `gemini-2.5-flash-preview-tts`, `gemini-2.5-pro-preview-tts` | `GOOGLE_API_KEY` or `GEMINI_API_KEY` | | ElevenLabs | `elevenlabsSpeech` from `@tanstack/ai-elevenlabs` | `eleven_v4`, `eleven_v4_turbo`, `eleven_v3`, and more | `ELEVENLABS_API_KEY` | +| Cartesia | `cartesiaSpeech` from `@tanstack/ai-cartesia` | `sonic-3.6`, `sonic-3.5`, `sonic-3`, `sonic-latest` | `CARTESIA_API_KEY` | | BytePlus | `byteplusSpeech` from `@tanstack/ai-byteplus` | `seed-audio-1.0` | `BYTEPLUS_VOICE_API_KEY` | | fal.ai | `falSpeech` from `@tanstack/ai-fal` | Kokoro, ElevenLabs, MiniMax, Gemini TTS, Chatterbox, and more | `FAL_KEY` | | Grok | `grokSpeech` from `@tanstack/ai-grok` | `grok-tts` | `XAI_API_KEY` | diff --git a/docs/media/transcription.md b/docs/media/transcription.md index cc80d692e4..59c1469bc9 100644 --- a/docs/media/transcription.md +++ b/docs/media/transcription.md @@ -2,7 +2,7 @@ title: Transcription id: transcription order: 4 -description: "Turn recorded speech into text with generateTranscription() and the useTranscription hook. Works with OpenAI, ElevenLabs, Groq, xAI, BytePlus, fal.ai, Cloudflare, and Lovable." +description: "Turn recorded speech into text with generateTranscription() and the useTranscription hook. Works with OpenAI, ElevenLabs, Cartesia, Groq, xAI, BytePlus, fal.ai, Cloudflare, and Lovable." keywords: - tanstack ai - transcription @@ -145,7 +145,7 @@ console.log(result.text) - A `File` or a `Blob`. - An `ArrayBuffer`. -- A base64 string. The OpenAI and Groq adapters treat plain base64 as `audio/mpeg`. +- A base64 string. The OpenAI, Groq, and Cartesia adapters treat plain base64 as `audio/mpeg`. - A data URL, such as `data:audio/webm;base64,...`. The type in the URL goes to the provider. ## Providers @@ -156,6 +156,7 @@ Swap the adapter to change the provider. Each adapter reads its key from the env | --- | --- | --- | --- | | OpenAI | `openaiTranscription` | `gpt-4o-transcribe`, `gpt-4o-mini-transcribe`, `gpt-4o-transcribe-diarize`, `whisper-1` | `OPENAI_API_KEY` | | ElevenLabs | `elevenlabsTranscription` | `scribe_v2`, `scribe_v2_medical` | `ELEVENLABS_API_KEY` | +| Cartesia | `cartesiaTranscription` | `ink-whisper` | `CARTESIA_API_KEY` | | Groq | `groqTranscription` | `whisper-large-v3-turbo`, `whisper-large-v3` | `GROQ_API_KEY` | | xAI | `grokTranscription` | `grok-stt` | `XAI_API_KEY` | | BytePlus | `byteplusTranscription` | `seed-asr` | `BYTEPLUS_VOICE_API_KEY` | @@ -165,14 +166,16 @@ Swap the adapter to change the provider. Each adapter reads its key from the env Need speaker labels? Use `gpt-4o-transcribe-diarize`, or BytePlus with `enable_speaker_info`. Examples for each provider are in [Transcription: Advanced](./transcription-advanced). +If you omit `language` for Cartesia, the request uses English and translates other speech into English. When the recording is not English, pass `language`. + ## Common options | Option | Type | Description | | --- | --- | --- | | `audio` | `string \| File \| Blob \| ArrayBuffer` | The audio to transcribe (required) | | `language` | `string` | The language in ISO-639-1 format, for example `"en"`, `"es"`, or `"ja"`. A correct language makes the result more accurate and faster. | -| `prompt` | `string` | Words or names the model must expect, for example `"Technical terms: API, SDK, CLI"`. `gpt-4o-transcribe-diarize` does not support it. | -| `responseFormat` | `'json' \| 'text' \| 'srt' \| 'verbose_json' \| 'vtt'` | The output format. `verbose_json` adds timestamps and segments. | +| `prompt` | `string` | Words or names the model must expect, for example `"Technical terms: API, SDK, CLI"`. `gpt-4o-transcribe-diarize` and Cartesia do not support `prompt`. | +| `responseFormat` | `'json' \| 'text' \| 'srt' \| 'verbose_json' \| 'vtt'` | The output format. `verbose_json` adds timestamps and segments. Cartesia accepts only `json` and `verbose_json`. For Cartesia, `verbose_json` returns word timestamps and does not return segments. | | `modelOptions` | `object` | Provider options. They are typed for the model you pass. | | `debug` | `DebugOption` | Log the request and every raw provider chunk. See [Debug Logging](../advanced/debug-logging). | diff --git a/packages/ai-cartesia/LICENSE b/packages/ai-cartesia/LICENSE new file mode 100644 index 0000000000..308cb68dc0 --- /dev/null +++ b/packages/ai-cartesia/LICENSE @@ -0,0 +1,21 @@ +MIT License + +Copyright (c) 2025 Tanner Linsley + +Permission is hereby granted, free of charge, to any person obtaining a copy +of this software and associated documentation files (the "Software"), to deal +in the Software without restriction, including without limitation the rights +to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +copies of the Software, and to permit persons to whom the Software is +furnished to do so, subject to the following conditions: + +The above copyright notice and this permission notice shall be included in all +copies or substantial portions of the Software. + +THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE +SOFTWARE. diff --git a/packages/ai-cartesia/README.md b/packages/ai-cartesia/README.md new file mode 100644 index 0000000000..a2144f75b1 --- /dev/null +++ b/packages/ai-cartesia/README.md @@ -0,0 +1,33 @@ +
+ + + + TanStack AI + +
+ +
+ +# @tanstack/ai-cartesia + +Cartesia text-to-speech and batch transcription adapters for TanStack AI. + +```bash +pnpm add @tanstack/ai @tanstack/ai-cartesia +``` + +Set `CARTESIA_API_KEY` on your server. Then generate speech: + +```ts +import { generateSpeech } from '@tanstack/ai' +import { cartesiaSpeech } from '@tanstack/ai-cartesia' + +const result = await generateSpeech({ + adapter: cartesiaSpeech('sonic-3.6'), + text: 'Hello from Cartesia.', + voice: 'db6b0ed5-d5d3-463d-ae85-518a07d3c2b4', + threadId: 'speech-demo', +}) +``` + +For transcription, use `cartesiaTranscription('ink-whisper')` with `generateTranscription()`. To pass a key directly, use `createCartesiaSpeech()` or `createCartesiaTranscription()`. diff --git a/packages/ai-cartesia/package.json b/packages/ai-cartesia/package.json new file mode 100644 index 0000000000..998ec5dc8c --- /dev/null +++ b/packages/ai-cartesia/package.json @@ -0,0 +1,55 @@ +{ + "name": "@tanstack/ai-cartesia", + "version": "0.0.0", + "description": "Cartesia speech and transcription adapters for TanStack AI", + "author": "Tanner Linsley", + "license": "MIT", + "homepage": "https://tanstack.com/ai", + "repository": { + "type": "git", + "url": "git+https://github.com/TanStack/ai.git", + "directory": "packages/ai-cartesia" + }, + "type": "module", + "module": "./dist/esm/index.js", + "types": "./dist/esm/index.d.ts", + "exports": { + ".": { + "types": "./dist/esm/index.d.ts", + "import": "./dist/esm/index.js" + } + }, + "sideEffects": false, + "files": [ + "dist", + "src" + ], + "scripts": { + "build": "vite build", + "clean": "premove ./build ./dist", + "test:build": "publint --strict", + "test:coverage": "vitest run --coverage --coverage.include='src/**' --coverage.reporter=text-summary --coverage.reporter=json-summary", + "test:lib": "vitest run", + "test:types": "tsc", + "test:oxlint": "oxlint src --type-aware" + }, + "keywords": [ + "ai", + "tanstack", + "cartesia", + "text-to-speech", + "transcription" + ], + "devDependencies": { + "@tanstack/ai": "workspace:*", + "@vitest/coverage-v8": "4.1.12", + "vite": "^8.3.8" + }, + "peerDependencies": { + "@tanstack/ai": "workspace:^" + }, + "dependencies": { + "@cartesia/cartesia-js": "^5.1.0", + "@tanstack/ai-utils": "workspace:^" + } +} diff --git a/packages/ai-cartesia/src/client.ts b/packages/ai-cartesia/src/client.ts new file mode 100644 index 0000000000..1b3501f56c --- /dev/null +++ b/packages/ai-cartesia/src/client.ts @@ -0,0 +1,49 @@ +import { Cartesia } from '@cartesia/cartesia-js' +import { base64ToArrayBuffer } from '@tanstack/ai-utils' +import type { ClientOptions } from '@cartesia/cartesia-js' + +/** The Cartesia SDK client options that the adapters accept. */ +export type CartesiaConfig = Pick< + ClientOptions, + 'apiKey' | 'baseURL' | 'defaultHeaders' | 'fetch' +> + +/** Read the Cartesia API key from the server environment. */ +export function getCartesiaApiKeyFromEnv(): string { + const key = + typeof process === 'undefined' ? undefined : process.env.CARTESIA_API_KEY + if (!key) throw new Error('CARTESIA_API_KEY is required') + return key +} + +/** Create the Cartesia SDK client for an adapter. */ +export function createCartesiaClient(config: CartesiaConfig = {}): Cartesia { + return new Cartesia({ + ...config, + apiKey: config.apiKey ?? getCartesiaApiKeyFromEnv(), + }) +} + +/** Name the upload `audio.` from its MIME type, as speech APIs expect. */ +function audioFile(data: BlobPart, type: string): File { + const ext = + type === 'audio/mpeg' + ? 'mp3' + : (type.split(/[/;]/)[1]?.replace(/^x-/, '') ?? 'mp3') + return new File([data], `audio.${ext}`, { type }) +} + +/** Convert an audio input to a file for multipart transcription. */ +export function toAudioFile(audio: string | File | Blob | ArrayBuffer): File { + if (audio instanceof File) return audio + if (audio instanceof Blob) return audioFile(audio, audio.type || 'audio/mpeg') + if (audio instanceof ArrayBuffer) return audioFile(audio, 'audio/mpeg') + const match = /^data:([^;,]+)(?:;[^;,]+)*;base64,(.*)$/is.exec(audio) + if (!match && audio.startsWith('data:')) { + throw new Error('Cartesia transcription needs a base64 data URL') + } + return audioFile( + base64ToArrayBuffer(match?.[2] ?? audio), + match?.[1] ?? 'audio/mpeg', + ) +} diff --git a/packages/ai-cartesia/src/index.ts b/packages/ai-cartesia/src/index.ts new file mode 100644 index 0000000000..f036a0e2db --- /dev/null +++ b/packages/ai-cartesia/src/index.ts @@ -0,0 +1,18 @@ +export { + CartesiaSpeechAdapter, + cartesiaSpeech, + createCartesiaSpeech, +} from './speech' +export { + CartesiaTranscriptionAdapter, + cartesiaTranscription, + createCartesiaTranscription, +} from './transcription' +export { + CARTESIA_TTS_MODELS, + CARTESIA_TRANSCRIPTION_MODELS, +} from './model-meta' +export { getCartesiaApiKeyFromEnv } from './client' +export type { CartesiaSpeechOptions } from './speech' +export type { CartesiaConfig } from './client' +export type { CartesiaTTSModel, CartesiaTranscriptionModel } from './model-meta' diff --git a/packages/ai-cartesia/src/model-meta.ts b/packages/ai-cartesia/src/model-meta.ts new file mode 100644 index 0000000000..a75bb7733f --- /dev/null +++ b/packages/ai-cartesia/src/model-meta.ts @@ -0,0 +1,11 @@ +export const CARTESIA_TTS_MODELS = [ + 'sonic-3.6', + 'sonic-3.5', + 'sonic-3', + 'sonic-latest', +] as const +export type CartesiaTTSModel = (typeof CARTESIA_TTS_MODELS)[number] + +export const CARTESIA_TRANSCRIPTION_MODELS = ['ink-whisper'] as const +export type CartesiaTranscriptionModel = + (typeof CARTESIA_TRANSCRIPTION_MODELS)[number] diff --git a/packages/ai-cartesia/src/speech.ts b/packages/ai-cartesia/src/speech.ts new file mode 100644 index 0000000000..81fe40d6df --- /dev/null +++ b/packages/ai-cartesia/src/speech.ts @@ -0,0 +1,134 @@ +import { BaseTTSAdapter } from '@tanstack/ai/adapters' +import { arrayBufferToBase64 } from '@tanstack/ai-utils' +import { createCartesiaClient } from './client' +import type { Cartesia } from '@cartesia/cartesia-js' +import type { TTSOptions, TTSResult } from '@tanstack/ai' +import type { CartesiaConfig } from './client' +import type { CartesiaTTSModel } from './model-meta' + +export interface CartesiaSpeechOptions extends Pick< + Cartesia.TTSGenerateParams, + 'language' | 'normalization' +> { + pronunciationDictId?: string + volume?: number +} + +/** Generate speech with a Cartesia Sonic model. */ +export class CartesiaSpeechAdapter< + TModel extends CartesiaTTSModel, +> extends BaseTTSAdapter { + readonly name = 'cartesia' as const + private readonly client: Cartesia + + /** Configure the model and API connection for speech generation. */ + constructor(model: TModel, config?: CartesiaConfig) { + super(model) + this.client = createCartesiaClient(config) + } + + /** Generate audio for text using the requested voice and format. */ + async generateSpeech( + options: TTSOptions, + ): Promise { + const { logger } = options + logger.request( + `activity=generateSpeech provider=cartesia model=${this.model}`, + { + provider: this.name, + model: this.model, + }, + ) + try { + if (!options.voice) throw new Error('Cartesia speech requires a voice ID') + const format = options.format ?? 'mp3' + if (format !== 'mp3' && format !== 'wav' && format !== 'pcm') { + throw new Error(`Cartesia speech does not support format '${format}'`) + } + if ( + options.speed != null && + (options.speed < 0.6 || options.speed > 1.5) + ) { + throw new Error( + `Cartesia speech speed must be from 0.6 to 1.5 (received ${options.speed})`, + ) + } + const modelOptions = options.modelOptions + if ( + modelOptions?.volume != null && + (modelOptions.volume < 0.5 || modelOptions.volume > 2) + ) { + throw new Error( + `Cartesia speech volume must be from 0.5 to 2 (received ${modelOptions.volume})`, + ) + } + const outputFormat: Cartesia.TTSGenerateParams['output_format'] = + format === 'mp3' + ? { container: 'mp3', sample_rate: 44100, bit_rate: 128000 } + : { + container: format === 'wav' ? 'wav' : 'raw', + encoding: 'pcm_s16le', + sample_rate: 44100, + } + const response = await this.client.tts.generate( + { + model_id: this.model, + transcript: options.text, + voice: options.voice, + output_format: outputFormat, + language: modelOptions?.language, + normalization: modelOptions?.normalization, + pronunciation_dict_id: modelOptions?.pronunciationDictId, + ...(options.speed != null || modelOptions?.volume != null + ? { + generation_config: { + speed: options.speed, + volume: modelOptions?.volume, + }, + } + : {}), + }, + { signal: options.abortSignal }, + ) + const bytes = await response.arrayBuffer() + if (bytes.byteLength === 0) { + throw new Error('Cartesia speech returned empty audio') + } + return { + id: response.headers.get('x-request-id') ?? this.generateId(), + model: this.model, + audio: arrayBufferToBase64(bytes), + format, + contentType: + format === 'mp3' + ? 'audio/mpeg' + : format === 'wav' + ? 'audio/wav' + : 'audio/pcm', + } + } catch (error) { + logger.errors('cartesia.generateSpeech fatal', { + error, + source: 'cartesia.generateSpeech', + }) + throw error + } + } +} + +/** Create a speech adapter using the environment API key or supplied config. */ +export function cartesiaSpeech( + model: TModel, + config?: CartesiaConfig, +): CartesiaSpeechAdapter { + return new CartesiaSpeechAdapter(model, config) +} + +/** Create a speech adapter with an explicit API key. */ +export function createCartesiaSpeech( + model: TModel, + apiKey: string, + config?: Omit, +): CartesiaSpeechAdapter { + return new CartesiaSpeechAdapter(model, { ...config, apiKey }) +} diff --git a/packages/ai-cartesia/src/transcription.ts b/packages/ai-cartesia/src/transcription.ts new file mode 100644 index 0000000000..48f5f4b056 --- /dev/null +++ b/packages/ai-cartesia/src/transcription.ts @@ -0,0 +1,98 @@ +import { BaseTranscriptionAdapter } from '@tanstack/ai/adapters' +import { createCartesiaClient, toAudioFile } from './client' +import type { Cartesia } from '@cartesia/cartesia-js' +import type { TranscriptionOptions, TranscriptionResult } from '@tanstack/ai' +import type { CartesiaConfig } from './client' +import type { CartesiaTranscriptionModel } from './model-meta' + +/** Transcribe recorded audio with a Cartesia model. */ +export class CartesiaTranscriptionAdapter< + TModel extends CartesiaTranscriptionModel, +> extends BaseTranscriptionAdapter { + readonly name = 'cartesia' as const + private readonly client: Cartesia + + /** Configure the model and API connection for transcription. */ + constructor(model: TModel, config?: CartesiaConfig) { + super(model) + this.client = createCartesiaClient(config) + } + + /** Transcribe audio and return text and available word timings. */ + async transcribe( + options: TranscriptionOptions, + ): Promise { + const { logger } = options + logger.request( + `activity=transcription provider=cartesia model=${this.model}`, + { + provider: this.name, + model: this.model, + }, + ) + try { + if (options.prompt) + throw new Error('Cartesia batch transcription does not support prompt') + if ( + options.responseFormat && + options.responseFormat !== 'json' && + options.responseFormat !== 'verbose_json' + ) { + throw new Error( + `Cartesia batch transcription does not support responseFormat '${options.responseFormat}'`, + ) + } + const result = await this.client.stt.transcribe( + { + file: toAudioFile(options.audio), + model: this.model, + // Omit language and Cartesia uses English and translates other + // speech. The cast only narrows the string. + language: + options.language as Cartesia.STTTranscribeParams['language'], + timestamp_granularities: + options.responseFormat === 'verbose_json' ? ['word'] : undefined, + }, + { signal: options.abortSignal }, + ) + if (typeof result.text !== 'string') { + throw new Error('Cartesia transcription response has no text') + } + return { + id: result.request_id ?? this.generateId(), + model: this.model, + text: result.text, + ...(result.language ? { language: result.language } : {}), + ...(result.duration != null ? { duration: result.duration } : {}), + ...(result.words ? { words: result.words } : {}), + } + } catch (error) { + logger.errors('cartesia.transcribe fatal', { + error, + source: 'cartesia.transcribe', + }) + throw error + } + } +} + +/** Create a transcription adapter using the environment API key or supplied config. */ +export function cartesiaTranscription< + TModel extends CartesiaTranscriptionModel, +>( + model: TModel, + config?: CartesiaConfig, +): CartesiaTranscriptionAdapter { + return new CartesiaTranscriptionAdapter(model, config) +} + +/** Create a transcription adapter with an explicit API key. */ +export function createCartesiaTranscription< + TModel extends CartesiaTranscriptionModel, +>( + model: TModel, + apiKey: string, + config?: Omit, +): CartesiaTranscriptionAdapter { + return new CartesiaTranscriptionAdapter(model, { ...config, apiKey }) +} diff --git a/packages/ai-cartesia/tests/cartesia-adapters.test.ts b/packages/ai-cartesia/tests/cartesia-adapters.test.ts new file mode 100644 index 0000000000..ccc76d4e09 --- /dev/null +++ b/packages/ai-cartesia/tests/cartesia-adapters.test.ts @@ -0,0 +1,212 @@ +import { afterEach, describe, expect, it, vi } from 'vitest' +import { resolveDebugOption } from '@tanstack/ai/adapter-internals' +import { createCartesiaSpeech, createCartesiaTranscription } from '../src/index' +import { getCartesiaApiKeyFromEnv, toAudioFile } from '../src/client' + +const logger = resolveDebugOption(false) + +afterEach(() => vi.unstubAllGlobals()) + +describe('Cartesia adapters', () => { + it('sends Sonic speech options and returns audio', async () => { + const fetchMock = vi.fn( + async (_input: RequestInfo | URL, _init?: RequestInit) => + new Response(new Uint8Array([1, 2, 3]), { + headers: { 'x-request-id': 'speech-1' }, + }), + ) + + const result = await createCartesiaSpeech('sonic-3.6', 'test-key', { + fetch: fetchMock, + }).generateSpeech({ + model: 'sonic-3.6', + text: 'hello', + voice: 'voice-1', + speed: 1.2, + modelOptions: { language: 'en', volume: 0.8 }, + logger, + }) + + const [url, init] = fetchMock.mock.calls[0]! + expect(url).toBe('https://api.cartesia.ai/tts/bytes') + const headers = new Headers(init?.headers) + expect(headers.get('authorization')).toBe('Bearer test-key') + expect(headers.get('cartesia-version')).toBe('2026-08-14') + expect(JSON.parse(String(init?.body))).toMatchObject({ + model_id: 'sonic-3.6', + transcript: 'hello', + voice: 'voice-1', + output_format: { container: 'mp3', sample_rate: 44100, bit_rate: 128000 }, + language: 'en', + generation_config: { speed: 1.2, volume: 0.8 }, + }) + expect(result).toMatchObject({ + id: 'speech-1', + audio: 'AQID', + format: 'mp3', + }) + }) + + it.each([ + ['wav', 'wav'], + ['pcm', 'raw'], + ] as const)('sends %s output options', async (format, container) => { + const fetchMock = vi.fn( + async (_input: RequestInfo | URL, _init?: RequestInit) => + new Response(new Uint8Array([1, 2, 3])), + ) + const result = await createCartesiaSpeech('sonic-3.6', 'test-key', { + fetch: fetchMock, + }).generateSpeech({ + model: 'sonic-3.6', + text: 'hello', + voice: 'voice-1', + format, + logger, + }) + expect( + JSON.parse(String(fetchMock.mock.calls[0]?.[1]?.body)).output_format, + ).toEqual({ + container, + encoding: 'pcm_s16le', + sample_rate: 44100, + }) + expect(result).toMatchObject({ + format, + audio: 'AQID', + contentType: `audio/${format}`, + }) + }) + + it('reports a missing key when process is unavailable', () => { + vi.stubGlobal('process', undefined) + expect(() => getCartesiaApiKeyFromEnv()).toThrow( + 'CARTESIA_API_KEY is required', + ) + }) + + it('decodes data URLs with MIME parameters and plain base64', async () => { + const file = toAudioFile('data:audio/wav;codec=pcm;rate=24000;base64,AQID') + expect(file.type).toBe('audio/wav') + expect(file.name).toBe('audio.wav') + expect([...new Uint8Array(await file.arrayBuffer())]).toEqual([1, 2, 3]) + const fallback = toAudioFile('AQID') + expect(fallback.type).toBe('audio/mpeg') + expect(fallback.name).toBe('audio.mp3') + expect( + toAudioFile(new Blob(['a'], { type: 'audio/webm;codecs=opus' })).name, + ).toBe('audio.webm') + expect(toAudioFile(new Blob(['a'], { type: 'audio/x-wav' })).name).toBe( + 'audio.wav', + ) + expect([...new Uint8Array(await fallback.arrayBuffer())]).toEqual([1, 2, 3]) + expect(() => toAudioFile('data:audio/wav,abc')).toThrow( + 'Cartesia transcription needs a base64 data URL', + ) + }) + + it('sends Ink Whisper audio and maps word timings', async () => { + const fetchMock = vi.fn( + async (_input: RequestInfo | URL, _init?: RequestInit) => + Response.json({ + text: 'hello', + request_id: 'stt-1', + language: 'en', + duration: 1.5, + words: [{ word: 'hello', start: 0, end: 1.5 }], + }), + ) + + const result = await createCartesiaTranscription( + 'ink-whisper', + 'test-key', + { + fetch: fetchMock, + }, + ).transcribe({ + model: 'ink-whisper', + audio: new File([new Uint8Array([1, 2])], 'voice.wav', { + type: 'audio/wav', + }), + responseFormat: 'verbose_json', + language: 'en', + logger, + }) + + // The SDK first calls a custom fetch with 'data:,' to find its Response + // class. The upload is the last call. + const [url, init] = fetchMock.mock.calls.at(-1)! + expect(url).toBe('https://api.cartesia.ai/stt') + expect(init?.body).toBeInstanceOf(FormData) + const form = init?.body as FormData + expect(form.get('model')).toBe('ink-whisper') + expect(form.get('language')).toBe('en') + expect(form.get('timestamp_granularities[]')).toBe('word') + expect(result).toMatchObject({ id: 'stt-1', text: 'hello', duration: 1.5 }) + expect(result.words).toEqual([{ word: 'hello', start: 0, end: 1.5 }]) + }) + + it('throws the Cartesia status and body on a non-OK response', async () => { + await expect( + createCartesiaSpeech('sonic-3.6', 'test-key', { + fetch: async () => new Response('nope', { status: 400 }), + }).generateSpeech({ + model: 'sonic-3.6', + text: 'hello', + voice: 'voice-1', + logger, + }), + ).rejects.toThrow('400 nope') + }) + + it('rejects speed and volume outside the Cartesia range before fetch', async () => { + const fetchMock = vi.fn(async () => new Response(new Uint8Array([1]))) + const speech = createCartesiaSpeech('sonic-3.6', 'test-key', { + fetch: fetchMock, + }) + await expect( + speech.generateSpeech({ + model: 'sonic-3.6', + text: 'hello', + voice: 'voice-1', + speed: 0.5, + logger, + }), + ).rejects.toThrow('Cartesia speech speed must be from 0.6 to 1.5') + await expect( + speech.generateSpeech({ + model: 'sonic-3.6', + text: 'hello', + voice: 'voice-1', + modelOptions: { volume: 3 }, + logger, + }), + ).rejects.toThrow('Cartesia speech volume must be from 0.5 to 2') + expect(fetchMock).not.toHaveBeenCalled() + }) + + it('rejects empty speech audio', async () => { + await expect( + createCartesiaSpeech('sonic-3.6', 'test-key', { + fetch: async () => new Response(new Uint8Array()), + }).generateSpeech({ + model: 'sonic-3.6', + text: 'hello', + voice: 'voice-1', + logger, + }), + ).rejects.toThrow('Cartesia speech returned empty audio') + }) + + it('rejects a transcript response with no text', async () => { + await expect( + createCartesiaTranscription('ink-whisper', 'test-key', { + fetch: async () => Response.json({ type: 'transcript' }), + }).transcribe({ + model: 'ink-whisper', + audio: new File([new Uint8Array([1])], 'a.mp3', { type: 'audio/mpeg' }), + logger, + }), + ).rejects.toThrow('Cartesia transcription response has no text') + }) +}) diff --git a/packages/ai-cartesia/tsconfig.json b/packages/ai-cartesia/tsconfig.json new file mode 100644 index 0000000000..c38689f4ea --- /dev/null +++ b/packages/ai-cartesia/tsconfig.json @@ -0,0 +1,8 @@ +{ + "extends": "../../tsconfig.base.json", + "compilerOptions": { + "outDir": "dist" + }, + "include": ["src", "tests"], + "exclude": ["node_modules", "dist"] +} diff --git a/packages/ai-cartesia/vite.config.ts b/packages/ai-cartesia/vite.config.ts new file mode 100644 index 0000000000..11f5b20b70 --- /dev/null +++ b/packages/ai-cartesia/vite.config.ts @@ -0,0 +1,37 @@ +import { defineConfig, mergeConfig } from 'vitest/config' +import { tanstackViteConfig } from '@tanstack/vite-config' +import packageJson from './package.json' + +const config = defineConfig({ + test: { + name: packageJson.name, + dir: './', + watch: false, + + globals: true, + environment: 'node', + include: ['tests/**/*.test.ts'], + coverage: { + provider: 'v8', + reporter: ['text', 'json', 'html', 'lcov'], + exclude: [ + 'node_modules/', + 'dist/', + 'tests/', + '**/*.test.ts', + '**/*.config.ts', + '**/types.ts', + ], + include: ['src/**/*.ts'], + }, + }, +}) + +export default mergeConfig( + config, + tanstackViteConfig({ + entry: ['./src/index.ts'], + srcDir: './src', + cjs: false, + }), +) diff --git a/packages/ai/skills/ai-core/media-generation/SKILL.md b/packages/ai/skills/ai-core/media-generation/SKILL.md index 598d1a1501..980845ebd8 100644 --- a/packages/ai/skills/ai-core/media-generation/SKILL.md +++ b/packages/ai/skills/ai-core/media-generation/SKILL.md @@ -5,8 +5,8 @@ description: > activity-specific adapters: generateImage() with openaiImage/geminiImage/byteplusImage, generateAudio() with geminiAudio/falAudio, generateVideo() with async polling (openaiVideo/geminiVideo/grokVideo/falVideo/byteplusVideo/openRouterVideo, - per-model typed durations), generateSpeech() with openaiSpeech/byteplusSpeech/elevenlabsSpeech, - generateTranscription() with openaiTranscription/byteplusTranscription, + per-model typed durations), generateSpeech() with openaiSpeech/byteplusSpeech/elevenlabsSpeech/cartesiaSpeech, + generateTranscription() with openaiTranscription/byteplusTranscription/cartesiaTranscription, generateVoice() with elevenlabsVoiceDesign (create a voice, then speak with it). React hooks: useGenerateImage, useGenerateAudio, @@ -404,7 +404,11 @@ const { generate, result, isLoading } = useGenerateAudio({ ### 3. Text-to-Speech Adapters include `openaiSpeech` (tts-1, tts-1-hd, gpt-4o-mini-tts), -`byteplusSpeech` (`seed-audio-1.0`), and `elevenlabsSpeech` (`eleven_v4`). +`byteplusSpeech` (`seed-audio-1.0`), `elevenlabsSpeech` (`eleven_v4`), and +`cartesiaSpeech` (`sonic-3.6`). + +`cartesiaSpeech` requires a Cartesia voice id as `voice`. It accepts +`format: 'mp3' | 'wav' | 'pcm'`. `speed` must be from 0.6 to 1.5. `elevenlabsSpeech` accepts `format: 'mp3' | 'pcm' | 'opus' | 'wav'`. WAV output contains 44.1 kHz, 16-bit mono PCM with a RIFF header. @@ -583,9 +587,15 @@ OpenAI, Gemini, and Cloudflare have fixed voice catalogs and will not get one. ### 5. Audio Transcription Adapters: `openaiTranscription` (whisper-1, gpt-4o-transcribe, -gpt-4o-mini-transcribe, gpt-4o-transcribe-diarize) and `byteplusTranscription` +gpt-4o-mini-transcribe, gpt-4o-transcribe-diarize), `byteplusTranscription` (`seed-asr` — synchronous, no polling; audio up to 2 hours / 100 MB; also reads -**`BYTEPLUS_VOICE_API_KEY`**). +**`BYTEPLUS_VOICE_API_KEY`**), and `cartesiaTranscription` (`ink-whisper`). + +`cartesiaTranscription` accepts `responseFormat` `json` or `verbose_json`. +`verbose_json` fills `result.words` (`{ word, start, end }`) and leaves +`segments` empty. A `prompt` throws. Omit `language` only for English audio: +Cartesia translates other speech into English. When the recording is not +English, pass `language` (ISO-639-1). > **Capturing audio in the browser:** Use `useAudioRecorder` from `@tanstack/ai-react` to record directly in the browser, then pass the recording as the `audio` input to `generate()`, or use `recording.part` as a prompt part in chat/generation calls. No transcoding or extra dependencies required — the recorder returns the native browser format (`audio/webm` or `audio/mp4`). For transcription, wrap it as a `data:` URL so the provider gets the real content type; passing raw `recording.base64` makes the adapter assume `audio/mpeg` and mislabel the webm/mp4 bytes. > diff --git a/pnpm-lock.yaml b/pnpm-lock.yaml index 1a01816aff..2493991973 100644 --- a/pnpm-lock.yaml +++ b/pnpm-lock.yaml @@ -2627,6 +2627,25 @@ importers: specifier: ^8.3.8 version: 8.3.8(@types/node@24.13.6)(esbuild@0.28.2)(jiti@2.7.0)(less@4.6.6)(sass-embedded@1.108.2)(sass@1.108.2)(terser@5.44.1)(tsx@4.23.15)(yaml@2.9.0) + packages/ai-cartesia: + dependencies: + '@cartesia/cartesia-js': + specifier: ^5.1.0 + version: 5.1.0(ws@8.21.3) + '@tanstack/ai-utils': + specifier: workspace:^ + version: link:../ai-utils + devDependencies: + '@tanstack/ai': + specifier: workspace:* + version: link:../ai + '@vitest/coverage-v8': + specifier: 4.1.12 + version: 4.1.12(vitest@4.1.12) + vite: + specifier: ^8.3.8 + version: 8.3.8(@types/node@24.13.6)(esbuild@0.28.2)(jiti@2.7.0)(less@4.6.6)(sass-embedded@1.108.2)(sass@1.108.2)(terser@5.44.1)(tsx@4.23.15)(yaml@2.9.0) + packages/ai-claude-code: devDependencies: '@tanstack/ai': @@ -4396,6 +4415,9 @@ importers: '@tanstack/ai-byteplus': specifier: workspace:* version: link:../../packages/ai-byteplus + '@tanstack/ai-cartesia': + specifier: workspace:* + version: link:../../packages/ai-cartesia '@tanstack/ai-claude-code': specifier: workspace:* version: link:../../packages/ai-claude-code @@ -5969,6 +5991,14 @@ packages: '@bufbuild/protobuf@2.15.0': resolution: {integrity: sha512-DAheWUkVr/SJTWCc+lg9dhY0eN4SaWlf4+bG1KzHeXbnqt0AfB/NX0Z+VunGlM1ki1B4zVvye27MpKh/svySUA==} + '@cartesia/cartesia-js@5.1.0': + resolution: {integrity: sha512-KTnyJrV0Hl9/ElprFexh+XEpnybE+65KoBwTUJf9q3+3ziTd+wGn8ttm3mP9HVJPKKm8TuklmdVOhL6Lx4UOJA==} + peerDependencies: + ws: ^8.18.0 + peerDependenciesMeta: + ws: + optional: true + '@cfworker/json-schema@4.1.1': resolution: {integrity: sha512-gAmrUZSGtKc3AiBL71iNWxDsyUC5uMaKKGdvzYsBoTW/xi42JQHl7eKV2OYzCUqvc+D2RCcf7EXY2iCyFIk6og==} @@ -22737,6 +22767,10 @@ snapshots: '@bufbuild/protobuf@2.15.0': {} + '@cartesia/cartesia-js@5.1.0(ws@8.21.3)': + optionalDependencies: + ws: 8.21.3 + '@cfworker/json-schema@4.1.1': {} '@changesets/apply-release-plan@7.1.0': diff --git a/testing/e2e/global-setup.ts b/testing/e2e/global-setup.ts index 900ba7bf3c..726ffadc21 100644 --- a/testing/e2e/global-setup.ts +++ b/testing/e2e/global-setup.ts @@ -49,6 +49,8 @@ export default async function globalSetup() { // pattern above. mock.mount('/v1/text-to-speech', elevenLabsTTSMount()) mock.mount('/v1/speech-to-text', elevenLabsSTTMount()) + mock.mount('/tts/bytes', cartesiaTTSMount()) + mock.mount('/stt', cartesiaSTTMount()) // Gemini TTS hits the standard Gemini generateContent endpoint // (POST /v1beta/models/{model}:generateContent) with @@ -1185,9 +1187,9 @@ const FAKE_AUDIO_BY_FORMAT: Record = { /** * Minimal RIFF/WAV container: a 44-byte header describing 16-bit mono PCM at - * 24 kHz, followed by silence. + * the requested sample rate, followed by silence. */ -function buildSilentWav(): Buffer { +function buildSilentWav(sampleRate = 24000): Buffer { const samples = Buffer.alloc(64) const header = Buffer.alloc(44) header.write('RIFF', 0, 'ascii') @@ -1197,8 +1199,8 @@ function buildSilentWav(): Buffer { header.writeUInt32LE(16, 16) // PCM chunk size header.writeUInt16LE(1, 20) // format: PCM header.writeUInt16LE(1, 22) // channels - header.writeUInt32LE(24000, 24) // sample rate - header.writeUInt32LE(24000 * 2, 28) // byte rate + header.writeUInt32LE(sampleRate, 24) // sample rate + header.writeUInt32LE(sampleRate * 2, 28) // byte rate header.writeUInt16LE(2, 32) // block align header.writeUInt16LE(16, 34) // bits per sample header.write('data', 36, 'ascii') @@ -1674,6 +1676,79 @@ function byteplusASRMount(): Mountable { } } +/** Validate Cartesia speech requests and return audio in the requested format. */ +function cartesiaTTSMount(): Mountable { + return { + async handleRequest(req, res, pathname): Promise { + if (pathname !== '/' || req.method !== 'POST') return false + const body = await readJsonRequestBody(req) + const output = asRecord(body?.output_format) + if ( + body?.model_id !== 'sonic-3.6' || + body.transcript !== 'welcome to the guitar store' || + typeof body.voice !== 'string' || + (output?.container !== 'mp3' && + output?.container !== 'wav' && + output?.container !== 'raw') || + req.headers['cartesia-version'] !== '2026-08-14' + ) { + return rejectVoiceRequest(res, 'Invalid Cartesia speech request') + } + res.statusCode = 200 + const format = output.container === 'raw' ? 'pcm' : output.container + res.setHeader( + 'Content-Type', + format === 'mp3' ? 'audio/mpeg' : `audio/${format}`, + ) + res.end( + format === 'wav' ? buildSilentWav(44100) : FAKE_AUDIO_BY_FORMAT[format], + ) + return true + }, + } +} + +/** Validate Cartesia transcription uploads and return a fixed transcript. */ +function cartesiaSTTMount(): Mountable { + return { + async handleRequest(req, res, pathname): Promise { + if (pathname !== '/' || req.method !== 'POST') return false + const chunks: Array = [] + for await (const chunk of req) chunks.push(Buffer.from(chunk)) + const contentType = req.headers['content-type'] ?? '' + let form: FormData + try { + form = await new Response(Buffer.concat(chunks), { + headers: { 'content-type': contentType }, + }).formData() + } catch { + return rejectVoiceRequest(res, 'Invalid Cartesia transcription request') + } + const file = form.get('file') + if ( + !(file instanceof File) || + !Buffer.from(await file.arrayBuffer()).equals(Buffer.from('Hello')) || + form.get('model') !== 'ink-whisper' || + req.headers['cartesia-version'] !== '2026-08-14' + ) { + return rejectVoiceRequest(res, 'Invalid Cartesia transcription request') + } + res.statusCode = 200 + res.setHeader('Content-Type', 'application/json') + res.end( + JSON.stringify({ + type: 'transcript', + text: 'I would like to buy a Fender Stratocaster please', + request_id: 'cartesia-e2e', + language: 'en', + duration: 2.4, + }), + ) + return true + }, + } +} + /** * Mounts Claude-shaped responses for server-tool regression coverage. The * streaming response reproduces issue #604, while the request validator diff --git a/testing/e2e/package.json b/testing/e2e/package.json index 00de379746..4154362272 100644 --- a/testing/e2e/package.json +++ b/testing/e2e/package.json @@ -25,6 +25,7 @@ "@tanstack/ai-anthropic": "workspace:*", "@tanstack/ai-bedrock": "workspace:*", "@tanstack/ai-byteplus": "workspace:*", + "@tanstack/ai-cartesia": "workspace:*", "@tanstack/ai-claude-code": "workspace:*", "@tanstack/ai-client": "workspace:*", "@tanstack/ai-cloudflare": "workspace:*", diff --git a/testing/e2e/src/components/TTSUI.tsx b/testing/e2e/src/components/TTSUI.tsx index c585d91f9a..eeb7014f0d 100644 --- a/testing/e2e/src/components/TTSUI.tsx +++ b/testing/e2e/src/components/TTSUI.tsx @@ -15,9 +15,9 @@ interface TTSUIProps { aimockPort?: number } -// ElevenLabs requires a voice on every TTS call. The other providers infer -// one from their adapter config, so we only attach this when needed. +// ElevenLabs and Cartesia require a voice on every TTS call. const ELEVENLABS_DEFAULT_VOICE = '21m00Tcm4TlvDq8ikWAM' +const CARTESIA_DEFAULT_VOICE = 'db6b0ed5-d5d3-463d-ae85-518a07d3c2b4' // Deliberately synthetic voice ids: the dialogue mounts only check that each // turn carries *a* voice, and fake-but-plausible provider ids get copied into @@ -27,10 +27,15 @@ const DIALOGUE_TURNS: Array = [ { text: 'what can I play for you', voice: 'e2e-voice-b' }, ] +/** Render the speech generation test UI for the selected provider and transport. */ export function TTSUI({ provider, mode, testId, aimockPort }: TTSUIProps) { const [text, setText] = useState('') const defaultVoice = - provider === 'elevenlabs' ? ELEVENLABS_DEFAULT_VOICE : undefined + provider === 'elevenlabs' + ? ELEVENLABS_DEFAULT_VOICE + : provider === 'cartesia' + ? CARTESIA_DEFAULT_VOICE + : undefined const connectionOptions = () => { const body = { provider, testId, aimockPort } diff --git a/testing/e2e/src/lib/feature-support.ts b/testing/e2e/src/lib/feature-support.ts index 8cecc8cc27..faea10829d 100644 --- a/testing/e2e/src/lib/feature-support.ts +++ b/testing/e2e/src/lib/feature-support.ts @@ -475,6 +475,7 @@ export const matrix: Record> = { 'gemini', 'grok', 'elevenlabs', + 'cartesia', 'byteplus', 'lovable', ]), @@ -489,6 +490,7 @@ export const matrix: Record> = { 'grok', 'groq', 'elevenlabs', + 'cartesia', 'byteplus', 'lovable', ]), diff --git a/testing/e2e/src/lib/media-providers.ts b/testing/e2e/src/lib/media-providers.ts index be75700452..dac84b3118 100644 --- a/testing/e2e/src/lib/media-providers.ts +++ b/testing/e2e/src/lib/media-providers.ts @@ -21,6 +21,10 @@ import { createElevenLabsSpeech, createElevenLabsTranscription, } from '@tanstack/ai-elevenlabs' +import { + createCartesiaSpeech, + createCartesiaTranscription, +} from '@tanstack/ai-cartesia' import { createBytePlusImage, createBytePlusSpeech, @@ -133,6 +137,7 @@ export function createImageAdapter( return factory() } +/** Create a speech adapter that sends requests to the test mock. */ export function createTTSAdapter( provider: Provider, aimockPort?: number, @@ -159,6 +164,11 @@ export function createTTSAdapter( baseUrl: llmockBase(aimockPort), headers, }), + cartesia: () => + createCartesiaSpeech('sonic-3.6', DUMMY_KEY, { + baseURL: llmockBase(aimockPort), + defaultHeaders: headers, + }), // Seed Speech is a separate BytePlus product from Ark with its own key and // its own host, so this takes the bare mock base — the adapter appends // `/api/v3/tts/create`, which `byteplusTTSMount()` serves. @@ -178,6 +188,7 @@ export function createTTSAdapter( return factory() } +/** Create a transcription adapter that sends requests to the test mock. */ export function createTranscriptionAdapter( provider: Provider, aimockPort?: number, @@ -207,6 +218,11 @@ export function createTranscriptionAdapter( baseUrl: llmockBase(aimockPort), headers, }), + cartesia: () => + createCartesiaTranscription('ink-whisper', DUMMY_KEY, { + baseURL: llmockBase(aimockPort), + defaultHeaders: headers, + }), // Same Seed Speech host as the TTS adapter above; the adapter appends // `/api/v3/auc/bigmodel/recognize/flash`, served by `byteplusASRMount()`. byteplus: () => diff --git a/testing/e2e/src/lib/providers.ts b/testing/e2e/src/lib/providers.ts index 456878d3b3..b606912c31 100644 --- a/testing/e2e/src/lib/providers.ts +++ b/testing/e2e/src/lib/providers.ts @@ -115,6 +115,7 @@ const defaultModels: Record = { // it out of text features, but we still need an entry to satisfy the // Record constraint. elevenlabs: '', + cartesia: '', llmgateway: 'gpt-5.6-terra', cloudflare: '@cf/zai-org/glm-5.3-flash', neon: 'gemini-3-8-flash', @@ -123,6 +124,7 @@ const defaultModels: Record = { muse: 'muse-ember-1.0', } +/** Create the selected chat adapter with the test mock configuration. */ export function createTextAdapter( provider: Provider, modelOverride?: string, @@ -399,6 +401,11 @@ export function createTextAdapter( 'ElevenLabs has no text/chat adapter — use createTTSAdapter or createTranscriptionAdapter.', ) }, + cartesia: () => { + throw new Error( + 'Cartesia has no text/chat adapter — use createTTSAdapter or createTranscriptionAdapter.', + ) + }, llmgateway: () => createChatOptions({ adapter: createLLMGatewayText(model as 'gpt-5.6-terra', DUMMY_KEY, { diff --git a/testing/e2e/src/lib/types.ts b/testing/e2e/src/lib/types.ts index a0a78fdff8..8999f55285 100644 --- a/testing/e2e/src/lib/types.ts +++ b/testing/e2e/src/lib/types.ts @@ -23,6 +23,7 @@ export type Provider = | 'mistral' | 'byteplus' | 'elevenlabs' + | 'cartesia' | 'llmgateway' | 'cloudflare' | 'neon' @@ -89,6 +90,7 @@ export const ALL_PROVIDERS: Provider[] = [ 'mistral', 'byteplus', 'elevenlabs', + 'cartesia', 'llmgateway', 'cloudflare', 'neon', diff --git a/testing/e2e/tests/test-matrix.ts b/testing/e2e/tests/test-matrix.ts index dbb17aa71e..23e417af66 100644 --- a/testing/e2e/tests/test-matrix.ts +++ b/testing/e2e/tests/test-matrix.ts @@ -36,6 +36,7 @@ export const providers: Provider[] = [ 'mistral', 'byteplus', 'elevenlabs', + 'cartesia', 'llmgateway', 'cloudflare', 'neon', diff --git a/testing/e2e/tests/tts.spec.ts b/testing/e2e/tests/tts.spec.ts index 2129a94287..2fa8c314dd 100644 --- a/testing/e2e/tests/tts.spec.ts +++ b/testing/e2e/tests/tts.spec.ts @@ -1,5 +1,6 @@ import { generateSpeech } from '@tanstack/ai' import { createElevenLabsSpeech } from '@tanstack/ai-elevenlabs' +import { createCartesiaSpeech } from '@tanstack/ai-cartesia' import { test, expect } from './fixtures' import { fillTextInput, @@ -9,6 +10,31 @@ import { } from './helpers' import { providersFor } from './test-matrix' +for (const format of ['wav', 'pcm'] as const) { + test(`cartesia -- generates ${format} audio`, async ({ + testId, + aimockPort, + }) => { + const result = await generateSpeech({ + threadId: testId, + adapter: createCartesiaSpeech('sonic-3.6', 'e2e-dummy', { + baseURL: `http://127.0.0.1:${aimockPort}`, + defaultHeaders: { 'X-Test-Id': testId }, + }), + text: 'welcome to the guitar store', + voice: 'test-voice', + format, + }) + const audio = Buffer.from(result.audio, 'base64') + expect(result.format).toBe(format) + expect(result.contentType).toBe(`audio/${format}`) + if (format === 'wav') { + expect(audio.toString('ascii', 0, 4)).toBe('RIFF') + expect(audio.readUInt32LE(24)).toBe(44100) + } else expect(audio).toEqual(Buffer.alloc(32)) + }) +} + test.describe('elevenlabs -- tts formats', () => { test('returns playable WAV from the PCM endpoint', async ({ page,