diff --git a/.env.sample b/.env.sample index be3fdec8..92384631 100644 --- a/.env.sample +++ b/.env.sample @@ -133,9 +133,20 @@ GOOGLE_CLIENT_SECRET= # Optional — Embedding # ----------------------------------------------------------------------------- -# Base URL for the embedding API (omit for OpenAI default) +# Base URL for the embedding API (omit for OpenAI default). Any +# OpenAI-compatible /embeddings gateway works, e.g. to route through OpenRouter: +# EMBEDDING_BASE_URL=https://openrouter.ai/api/v1 +# EMBEDDING_MODEL=openai/text-embedding-3-small +# EMBEDDING_API_KEY= # EMBEDDING_BASE_URL= +# Embedding model name (default: text-embedding-3-small). Override when a +# gateway namespaces the model differently (OpenRouter needs the "openai/" +# prefix). The replacement MUST emit 1536-dim vectors (the stored column is +# halfvec(1536)) and be the same underlying model as whatever produced your +# existing vectors — otherwise semantic search silently degrades. +# EMBEDDING_MODEL= + # Timeout per embedding API call (ms) # EMBEDDING_TIMEOUT_MS= diff --git a/SELF_HOST.md b/SELF_HOST.md index 0c1902cd..76244dfb 100644 --- a/SELF_HOST.md +++ b/SELF_HOST.md @@ -51,6 +51,20 @@ cp .env.sample .env Set the required values: - `EMBEDDING_API_KEY` — your OpenAI (or compatible) API key. +- Optional: route embeddings through an OpenAI-compatible gateway instead of + OpenAI directly by setting `EMBEDDING_BASE_URL` plus `EMBEDDING_MODEL` (some + gateways namespace model names). For OpenRouter: + + ```bash + EMBEDDING_BASE_URL=https://openrouter.ai/api/v1 + EMBEDDING_MODEL=openai/text-embedding-3-small + EMBEDDING_API_KEY= + ``` + + The model must emit 1536-dim vectors (the stored column is `halfvec(1536)`) + and must be the same underlying model that produced your existing vectors — + switching to a genuinely different model puts stored and query vectors in + different vector spaces and silently degrades semantic search. - Optional: `EMBEDDING_TOKENIZE_THREADS` controls worker threads used for OpenAI tokenization/truncation. It auto-sizes by default (`max(1, min(available CPU cores - 1, 4))`); set `0` to run inline in very diff --git a/packages/server/config.ts b/packages/server/config.ts index c18a935e..3a3cf4df 100644 --- a/packages/server/config.ts +++ b/packages/server/config.ts @@ -4,6 +4,18 @@ /** * Embedding model configuration. * All engines use the same embedding model for consistency. + * + * `model` is the default only — it is overridable per-deploy via EMBEDDING_MODEL + * (see buildEmbeddingConfig), because an OpenAI-compatible gateway may namespace + * the same underlying model differently (OpenRouter, for instance, serves + * text-embedding-3-small as "openai/text-embedding-3-small"). + * + * `dimensions` is deliberately NOT configurable: the memory table's embedding + * column is halfvec(1536), so a different width would not round-trip. Any + * replacement model must emit 1536-dim vectors, and it must be embedding- + * compatible with whatever produced the vectors already stored — mixing models + * silently degrades semantic search, since stored and query vectors would live + * in different vector spaces. */ export const embeddingConstants = { model: "text-embedding-3-small", diff --git a/packages/server/start.test.ts b/packages/server/start.test.ts index e7d57529..de429fd7 100644 --- a/packages/server/start.test.ts +++ b/packages/server/start.test.ts @@ -17,6 +17,7 @@ const TOUCHED_ENV = [ "EMBEDDING_TIMEOUT_MS", "EMBEDDING_MAX_PARALLEL_CALLS", "EMBEDDING_BASE_URL", + "EMBEDDING_MODEL", ] as const; describe("buildEmbeddingConfig", () => { @@ -68,4 +69,28 @@ describe("buildEmbeddingConfig", () => { "EMBEDDING_API_KEY is required", ); }); + + test("defaults model to text-embedding-3-small", () => { + expect(buildEmbeddingConfig().model).toBe("text-embedding-3-small"); + }); + + test("forwards an explicit EMBEDDING_MODEL (gateway-namespaced name)", () => { + process.env.EMBEDDING_MODEL = "openai/text-embedding-3-small"; + expect(buildEmbeddingConfig().model).toBe("openai/text-embedding-3-small"); + }); + + test("falls back to the default model when EMBEDDING_MODEL is empty", () => { + process.env.EMBEDDING_MODEL = ""; + expect(buildEmbeddingConfig().model).toBe("text-embedding-3-small"); + }); + + test("keeps dimensions at 1536 regardless of the model override", () => { + process.env.EMBEDDING_MODEL = "openai/text-embedding-3-small"; + expect(buildEmbeddingConfig().dimensions).toBe(1536); + }); + + test("forwards EMBEDDING_BASE_URL for gateway routing", () => { + process.env.EMBEDDING_BASE_URL = "https://openrouter.ai/api/v1"; + expect(buildEmbeddingConfig().baseUrl).toBe("https://openrouter.ai/api/v1"); + }); }); diff --git a/packages/server/start.ts b/packages/server/start.ts index 02fe7bfc..40b8f48d 100644 --- a/packages/server/start.ts +++ b/packages/server/start.ts @@ -73,6 +73,14 @@ function parseIntEnv( * Build the embedding config from environment variables. Requires * EMBEDDING_API_KEY (the server won't boot without it). Used as the default * when `StartServerOptions.embeddingConfig` is not supplied. + * + * The provider is always the OpenAI SDK client, which speaks the + * OpenAI-compatible /embeddings protocol — so pointing EMBEDDING_BASE_URL at a + * gateway (e.g. OpenRouter's https://openrouter.ai/api/v1) plus that gateway's + * key in EMBEDDING_API_KEY is all that's needed to route through it. + * EMBEDDING_MODEL exists because gateways rename models (OpenRouter wants + * "openai/text-embedding-3-small"). Dimensions stay fixed at 1536 to match the + * halfvec(1536) column — see embeddingConstants. */ export function buildEmbeddingConfig(): EmbeddingConfig { const apiKey = process.env.EMBEDDING_API_KEY; @@ -109,7 +117,7 @@ export function buildEmbeddingConfig(): EmbeddingConfig { return { provider: "openai", - model: embeddingConstants.model, + model: process.env.EMBEDDING_MODEL || embeddingConstants.model, dimensions: embeddingConstants.dimensions, apiKey, baseUrl: process.env.EMBEDDING_BASE_URL, diff --git a/scripts/embed-query.ts b/scripts/embed-query.ts index 3f67b040..0405d5fb 100644 --- a/scripts/embed-query.ts +++ b/scripts/embed-query.ts @@ -64,7 +64,9 @@ if (!apiKey) { const embedding = await generateEmbedding(query, { provider: "openai", - model: "text-embedding-3-small", + // Mirror the server's EMBEDDING_MODEL override so a query vector generated + // here matches the model that produced the stored vectors. + model: process.env.EMBEDDING_MODEL || "text-embedding-3-small", dimensions: 1536, apiKey, baseUrl: process.env.EMBEDDING_BASE_URL,