diff --git a/README.md b/README.md index 47ed6ad..7d0df3b 100644 --- a/README.md +++ b/README.md @@ -42,12 +42,13 @@ To get started [Setup](#setup) and [Configure](#configure) the server. No API Ke [![View Live Demo](https://img.shields.io/badge/view_live_demo-green?style=for-the-badge&logo=chatbot&logoColor=white)](https://cssnr.github.io/vitepress-chat/) -- Client: https://github.com/cssnr/vitepress-chat -- Server: https://github.com/cssnr/chat-server +- Client: +- Server: ### Features - Works with Claude, OpenAI, Gemini and OpenAI Compatible Providers +- Chat, Completion, and Object Endpoints - Live Stream Results to Client - Automatic Input Token Caching - Automatic Retry on API Failures @@ -107,7 +108,9 @@ Environment Variables. | `BASE_URL` | `https://opencode.ai/zen/v1` | OpenAI Compatible Provider Base URL | | [PROVIDER_OPTIONS](#PROVIDER_OPTIONS) | - | Provider Options JSON String | | `MAX_TOKENS` | - | Max Output Tokens | -| `INSTRUCTIONS` | - | Fallback System Instructions | +| `CHAT_INSTRUCTIONS` | - | System Instructions for Chat | +| `COMPLETION_INSTRUCTIONS` | - | System Instructions for Completion | +| `OBJECT_INSTRUCTIONS` | - | System Instructions for Object | | `AI_SDK_LOG_WARNINGS` | - | Disable SDK Warnings | | `CORS_ORIGINS` | - | Allowed CORS Origins (supports \*) | | `PORT` | `3000` | Server Port | @@ -145,23 +148,72 @@ The value is only checked for valid JSON at startup and will fail at runtime if ## Client +### Endpoints + +| Endpoint | Method | Description | +| :------------ | :----: | :------------------------------------------------------------------------------------------------------------------------------------- | +| `/chat` | `POST` | Use with [useChat](https://ai-sdk.dev/docs/reference/ai-sdk-ui/use-chat) and [VitePress Chat](https://cssnr.github.io/vitepress-chat/) | +| `/completion` | `POST` | Use with [useCompletion](https://ai-sdk.dev/docs/reference/ai-sdk-ui/use-completion) | +| `/object` | `POST` | Use with [useObject](https://ai-sdk.dev/docs/reference/ai-sdk-ui/use-object) | + +### Chat + To send System Instructions from the client, add them to the body. ```typescript const chat = new Chat({ transport: new DefaultChatTransport({ - api: 'https://chat-server.cssnr.com/', + api: 'https://chat-server.cssnr.com/chat', headers: { Authorization: 'Basic Abc123=' }, body: { system: 'You are a helpful assistant.' }, }), }) ``` +Reference: + +### Completion + +```typescript +import { useCompletion } from '@ai-sdk/react' + +const { completion, complete, isLoading, stop } = useCompletion({ + api: 'https://chat-server.cssnr.com/completion', + headers: { Authorization: 'Basic Abc123=' }, + body: { system: 'You are a helpful assistant.' }, +}) +``` + +Reference: + +### Object + +```typescript +import { useObject } from '@ai-sdk/react' + +const { object, submit } = useObject({ + api: 'https://chat-server.cssnr.com/object', + schema: z.object({ name: z.string(), age: z.number() }), + headers: { Authorization: 'Basic Abc123=' }, +}) + +submit({ + system: 'You are a helpful assistant.', + prompt: 'Extract the name and age from: John is 30 years old.', + output: { + type: 'object', + properties: { name: { type: 'string' }, age: { type: 'number' } }, + }, +}) +``` + +Reference: + ### VitePress Chat Plugin The client is currently available as a VitePress Plugin. -- https://github.com/cssnr/vitepress-chat +- [![View Documentation](https://img.shields.io/badge/view_documentation-blue?style=for-the-badge&logo=googledocs&logoColor=white)](https://cssnr.github.io/vitepress-chat/) diff --git a/src/index.ts b/src/index.ts index 2379805..6e16701 100644 --- a/src/index.ts +++ b/src/index.ts @@ -7,10 +7,16 @@ import { google } from '@ai-sdk/google' import { openai } from '@ai-sdk/openai' import { createOpenAICompatible } from '@ai-sdk/openai-compatible' import { - streamText, + type GenerateTextEndEvent, + Output, + consumeStream, + convertToModelMessages, createUIMessageStream, + jsonSchema, + pipeTextStreamToResponse, pipeUIMessageStreamToResponse, - convertToModelMessages, + streamText, + toTextStream, toUIMessageStream, } from 'ai' @@ -42,7 +48,11 @@ const maxOutputTokens = process.env.MAX_TOKENS ? Number.parseInt(process.env.MAX_TOKENS) : undefined console.log(`maxOutputTokens: ${maxOutputTokens}`) -console.log(`INSTRUCTIONS: ${process.env.INSTRUCTIONS}`) +console.log( + `CHAT_INSTRUCTIONS: ${process.env.INSTRUCTIONS || process.env.CHAT_INSTRUCTIONS}`, // NOSONAR +) +console.log(`COMPLETION_INSTRUCTIONS: ${process.env.COMPLETION_INSTRUCTIONS}`) +console.log(`OBJECT_INSTRUCTIONS: ${process.env.OBJECT_INSTRUCTIONS}`) const model = getModel() console.log(`Loaded modelId: ${model.modelId}`) @@ -61,21 +71,23 @@ app.listen(port, () => console.log(`Listening on PORT: ${port}`)) // app.get('/app-health-check', (_req, res) => res.sendStatus(200)) -app.post('/', async (req: Request, res: Response) => { +app.post(['/', '/chat'], async (req: Request, res: Response) => { // console.log('req.headers:', req.headers) // console.log('authorization:', req.headers.authorization) const { messages, system } = req.body - // if (system) console.log('system:', system.substring(0, 512)) + // console.log('system:', system?.substring(0, 512)) const modelMessages = await convertToModelMessages(messages) - console.log('modelMessages:', modelMessages.length) + // console.log('modelMessages:', modelMessages.length) const stream = createUIMessageStream({ execute: ({ writer }) => { const result = streamText({ model: model, messages: modelMessages, - system: system || process.env.INSTRUCTIONS, + system: system || process.env.INSTRUCTIONS || process.env.CHAT_INSTRUCTIONS, maxOutputTokens, providerOptions, + onError: onStreamError, + onEnd: onStreamEnd, }) writer.merge(toUIMessageStream({ stream: result.stream })) }, @@ -84,6 +96,52 @@ app.post('/', async (req: Request, res: Response) => { pipeUIMessageStreamToResponse({ response: res, stream }) }) +app.post('/completion', async (req: Request, res: Response) => { + const { prompt, system } = req.body + console.log('prompt:', prompt?.length) + console.log('system:', system?.length) + const result = streamText({ + model: model, + prompt, + system: system || process.env.COMPLETION_INSTRUCTIONS, + maxOutputTokens, + providerOptions, + onError: onStreamError, + onEnd: onStreamEnd, + }) + const stream = createUIMessageStream({ + execute: ({ writer }) => { + writer.merge(toUIMessageStream({ stream: result.stream })) + }, + }) + pipeUIMessageStreamToResponse({ + response: res, + stream, + consumeSseStream: consumeStream, + }) +}) + +app.post('/object', async (req: Request, res: Response) => { + const { output, prompt, system } = req.body + console.log('output:', output ? 'SET' : undefined) + console.log('prompt:', prompt?.length) + console.log('system:', system?.length) + const result = streamText({ + model: model, + prompt, + system: system || process.env.OBJECT_INSTRUCTIONS, + maxOutputTokens, + providerOptions, + output: output ? Output.object({ schema: jsonSchema(output) }) : Output.json(), + onError: onStreamError, + onEnd: onStreamEnd, + }) + pipeTextStreamToResponse({ + response: res, + stream: toTextStream({ stream: result.stream }), + }) +}) + function corsCallback( origin: string | undefined, callback: (err: Error | null, origin?: boolean) => void, @@ -126,3 +184,14 @@ function getProviderOptions() { console.error('parsing PROVIDER_OPTIONS as JSON') } } + +function onStreamError(error: unknown) { + console.log('error:', error) +} + +function onStreamEnd({ finalStep, finishReason, text, usage }: GenerateTextEndEvent) { + console.log('reasoning:', finalStep.reasoningText) + console.log('response:', text) + console.log('usage:', usage) + console.log('finishReason:', finishReason) +}