diff --git a/apps/memos-local-openclaw/index.ts b/apps/memos-local-openclaw/index.ts index 89152f2e6..ca08ef167 100644 --- a/apps/memos-local-openclaw/index.ts +++ b/apps/memos-local-openclaw/index.ts @@ -258,6 +258,7 @@ const memosLocalPlugin = { const store = new SqliteStore(ctx.config.storage!.dbPath!, ctx.log); const embedder = new Embedder(ctx.config.embedding, ctx.log, ctx.openclawAPI); + store.configureVectorIndex?.(embedder.dimensions); const worker = new IngestWorker(store, embedder, ctx); const engine = new RecallEngine(store, embedder, ctx); const evidenceTag = ctx.config.capture?.evidenceWrapperTag ?? DEFAULTS.evidenceWrapperTag; diff --git a/apps/memos-local-openclaw/package.json b/apps/memos-local-openclaw/package.json index 2aa9b4a81..d3c95af85 100644 --- a/apps/memos-local-openclaw/package.json +++ b/apps/memos-local-openclaw/package.json @@ -55,6 +55,7 @@ "posthog-node": "^5.28.0", "puppeteer": "^24.38.0", "semver": "^7.7.4", + "sqlite-vec": "^0.1.9", "uuid": "^10.0.0" }, "devDependencies": { diff --git a/apps/memos-local-openclaw/scripts/native-binding.cjs b/apps/memos-local-openclaw/scripts/native-binding.cjs index b3683fab6..1b76d65df 100644 --- a/apps/memos-local-openclaw/scripts/native-binding.cjs +++ b/apps/memos-local-openclaw/scripts/native-binding.cjs @@ -32,6 +32,37 @@ function validateNativeBinding(bindingPath, loadBinding = defaultLoadBinding) { } } +function validateSqliteVecExtension( + sqliteVec, + createDatabase, + fsImpl = fs, +) { + if (!sqliteVec || typeof sqliteVec.getLoadablePath !== "function" || typeof sqliteVec.load !== "function") { + return { ok: false, reason: "missing", message: "sqlite-vec package is unavailable" }; + } + + let db; + try { + const extensionPath = sqliteVec.getLoadablePath(); + if (!extensionPath || !fsImpl.existsSync(extensionPath)) { + return { + ok: false, + reason: "missing-platform-binary", + message: `sqlite-vec platform binary not found: ${extensionPath || "unknown"}`, + }; + } + + db = createDatabase(); + sqliteVec.load(db); + const row = db.prepare("SELECT vec_version() AS version").get(); + return { ok: true, reason: "ok", message: "", version: row?.version || "unknown" }; + } catch (error) { + return { ok: false, reason: "load-error", message: errorMessage(error) }; + } finally { + try { db?.close(); } catch { /* best effort */ } + } +} + function quarantineNativeBinding( bindingPath, fsImpl = fs, @@ -74,4 +105,5 @@ module.exports = { defaultLoadBinding, quarantineNativeBinding, validateNativeBinding, + validateSqliteVecExtension, }; diff --git a/apps/memos-local-openclaw/scripts/postinstall.cjs b/apps/memos-local-openclaw/scripts/postinstall.cjs index c229a3b0e..54cc15542 100644 --- a/apps/memos-local-openclaw/scripts/postinstall.cjs +++ b/apps/memos-local-openclaw/scripts/postinstall.cjs @@ -4,7 +4,11 @@ const { spawnSync } = require("child_process"); const path = require("path"); const fs = require("fs"); -const { quarantineNativeBinding, validateNativeBinding } = require("./native-binding.cjs"); +const { + quarantineNativeBinding, + validateNativeBinding, + validateSqliteVecExtension, +} = require("./native-binding.cjs"); const RESET = "\x1b[0m"; const GREEN = "\x1b[32m"; @@ -137,7 +141,7 @@ try { function ensureDependencies() { phase(0, "Check core dependencies / 检测核心依赖"); - const coreDeps = ["@sinclair/typebox", "uuid", "@huggingface/transformers"]; + const coreDeps = ["@sinclair/typebox", "uuid", "@huggingface/transformers", "sqlite-vec"]; const missing = []; for (const dep of coreDeps) { try { @@ -482,6 +486,21 @@ ${YELLOW}${BOLD} ╚═══════════════════ } } +try { + const SqliteDatabase = require("better-sqlite3"); + const sqliteVec = require("sqlite-vec"); + const status = validateSqliteVecExtension(sqliteVec, () => new SqliteDatabase(":memory:")); + if (status.ok) { + ok(`sqlite-vec is ready (v${status.version}).`); + } else { + warn(`sqlite-vec is unavailable (${status.reason}): ${status.message}`); + log("Vector search will use the built-in brute-force fallback."); + } +} catch (e) { + warn(`sqlite-vec validation failed: ${e.message}`); + log("Vector search will use the built-in brute-force fallback."); +} + /* ═══════════════════════════════════════════════════════════ * Phase 3: Interactive LAN Sharing Setup * ═══════════════════════════════════════════════════════════ */ diff --git a/apps/memos-local-openclaw/scripts/re-embed.ts b/apps/memos-local-openclaw/scripts/re-embed.ts index 8b3d492a3..eba7fc20d 100644 --- a/apps/memos-local-openclaw/scripts/re-embed.ts +++ b/apps/memos-local-openclaw/scripts/re-embed.ts @@ -146,6 +146,8 @@ export async function runReembed(opts: CliOpts): Promise<{ processed: number; fa return { processed: 0, failed: 0, planned: ids.length }; } + store.configureVectorIndex(embedder.dimensions); + let processed = 0; let failed = 0; for (let i = 0; i < ids.length; i += opts.batchSize) { diff --git a/apps/memos-local-openclaw/scripts/start-viewer.ts b/apps/memos-local-openclaw/scripts/start-viewer.ts index 91bf3c7b5..9c66fbb3b 100644 --- a/apps/memos-local-openclaw/scripts/start-viewer.ts +++ b/apps/memos-local-openclaw/scripts/start-viewer.ts @@ -55,6 +55,7 @@ async function main() { }, log, ); + store.configureVectorIndex(embedder.dimensions); const port = parseInt(process.env.VIEWER_PORT ?? "18799", 10); const ctx = buildContext(dataDir, process.cwd(), undefined, log); diff --git a/apps/memos-local-openclaw/src/embedding/cache.ts b/apps/memos-local-openclaw/src/embedding/cache.ts new file mode 100644 index 000000000..23a6f5d91 --- /dev/null +++ b/apps/memos-local-openclaw/src/embedding/cache.ts @@ -0,0 +1,95 @@ +import { createHash } from "node:crypto"; +import type { Logger } from "../types"; + +interface CacheEntry { + vector: number[]; + timestamp: number; +} + +export interface CacheOptions { + maxSize: number; + ttlMs: number; + now?: () => number; +} + +export class EmbeddingCache { + private readonly cache = new Map(); + private readonly now: () => number; + + constructor( + private readonly options: CacheOptions, + private readonly log?: Logger, + ) { + this.now = options.now ?? Date.now; + } + + async get(signature: string, text: string): Promise { + const key = this.cacheKey(signature, text); + const entry = this.cache.get(key); + if (!entry) return null; + + if (this.now() - entry.timestamp >= this.options.ttlMs) { + this.cache.delete(key); + this.log?.debug(`[EmbeddingCache] Entry expired for key: ${key.slice(0, 16)}...`); + return null; + } + + this.cache.delete(key); + this.cache.set(key, entry); + this.log?.debug(`[EmbeddingCache] Cache hit for key: ${key.slice(0, 16)}...`); + return [...entry.vector]; + } + + async set(signature: string, text: string, vector: number[]): Promise { + if (this.options.maxSize <= 0 || this.options.ttlMs <= 0) return; + + const key = this.cacheKey(signature, text); + this.cache.delete(key); + while (this.cache.size >= this.options.maxSize) { + const oldestKey = this.cache.keys().next().value as string | undefined; + if (!oldestKey) break; + this.cache.delete(oldestKey); + this.log?.debug(`[EmbeddingCache] Evicted LRU entry: ${oldestKey.slice(0, 16)}...`); + } + + this.cache.set(key, { vector: [...vector], timestamp: this.now() }); + this.log?.debug(`[EmbeddingCache] Cached embedding for key: ${key.slice(0, 16)}...`); + } + + async has(signature: string, text: string): Promise { + return (await this.get(signature, text)) !== null; + } + + getStats(): { size: number; maxSize: number; ttlMs: number } { + return { + size: this.cache.size, + maxSize: this.options.maxSize, + ttlMs: this.options.ttlMs, + }; + } + + clear(): void { + this.cache.clear(); + this.log?.debug("[EmbeddingCache] Cache cleared"); + } + + private cacheKey(signature: string, text: string): string { + return createHash("sha256").update(signature).update("\0").update(text).digest("hex"); + } +} + +export const DEFAULT_CACHE_OPTIONS: CacheOptions = { + maxSize: 1000, + ttlMs: 60 * 60 * 1000, +}; + +let globalCache: EmbeddingCache | null = null; + +export function getGlobalCache(log?: Logger): EmbeddingCache { + if (!globalCache) globalCache = new EmbeddingCache(DEFAULT_CACHE_OPTIONS, log); + return globalCache; +} + +export function resetGlobalCache(): void { + globalCache = null; +} diff --git a/apps/memos-local-openclaw/src/embedding/index.ts b/apps/memos-local-openclaw/src/embedding/index.ts index 712fef9d2..58a6847e4 100644 --- a/apps/memos-local-openclaw/src/embedding/index.ts +++ b/apps/memos-local-openclaw/src/embedding/index.ts @@ -4,17 +4,65 @@ import { embedGemini } from "./providers/gemini"; import { embedCohere, embedCohereQuery } from "./providers/cohere"; import { embedVoyage } from "./providers/voyage"; import { embedMistral } from "./providers/mistral"; +import { embedOllama } from "./providers/ollama"; import { embedLocal } from "./local"; import { modelHealth } from "../ingest/providers"; +import { EmbeddingCache, getGlobalCache } from "./cache"; type EmbeddingInputKind = "document" | "query"; export class Embedder { + private cache: EmbeddingCache; + constructor( private cfg: EmbeddingConfig | undefined, private log: Logger, private openclawAPI?: OpenClawAPI, - ) {} + ) { + // Use global cache singleton to share cache across instances + this.cache = getGlobalCache(log); + } + + /** + * Get embedding for query with caching support + */ + async embedQueryWithCache(text: string): Promise { + const cached = await this.cache.get(this.signature, text); + if (cached) { + this.log.debug(`[Embedder] Cache hit for query: "${text.slice(0, 50)}..."`); + return cached; + } + + // Generate embedding + const startTime = Date.now(); + const vector = await this.embedQuery(text); + const duration = Date.now() - startTime; + + if (vector.length === this.dimensions) { + await this.cache.set(this.signature, text, vector); + this.log.debug(`[Embedder] Cached embedding (${duration}ms) for query: "${text.slice(0, 50)}..."`); + } else { + this.log.debug( + `[Embedder] Skipped query cache because expected ${this.dimensions} dimensions but received ${vector.length}`, + ); + } + + return vector; + } + + /** + * Clear embedding cache + */ + clearCache(): void { + this.cache.clear(); + } + + /** + * Get cache statistics + */ + getCacheStats(): { size: number; maxSize: number; ttlMs: number } { + return this.cache.getStats(); + } get provider(): string { if (this.cfg?.provider === "openclaw" && this.cfg.capabilities?.hostEmbedding !== true) { @@ -25,6 +73,7 @@ export class Embedder { get model(): string { if (this.provider === "local") return ""; + if (this.provider === "ollama") return this.cfg?.model ?? "nomic-embed-text"; return this.cfg?.model ?? ""; } @@ -92,6 +141,8 @@ export class Embedder { result = await embedMistral(texts, cfg!, this.log); break; case "voyage": result = await embedVoyage(texts, cfg!, this.log); break; + case "ollama": + result = await embedOllama(texts, cfg!, this.log); break; case "local": default: result = await embedLocal(texts, this.log); break; diff --git a/apps/memos-local-openclaw/src/embedding/providers/ollama.ts b/apps/memos-local-openclaw/src/embedding/providers/ollama.ts new file mode 100644 index 000000000..cbc70350d --- /dev/null +++ b/apps/memos-local-openclaw/src/embedding/providers/ollama.ts @@ -0,0 +1,44 @@ +import type { EmbeddingConfig, Logger } from "../../types"; + +interface OllamaEmbedResponse { + embeddings?: unknown; +} + +export async function embedOllama( + texts: string[], + cfg: EmbeddingConfig, + log: Logger, +): Promise { + if (texts.length === 0) return []; + + const endpoint = cfg.endpoint ?? "http://localhost:11434"; + const model = cfg.model ?? "nomic-embed-text"; + const baseUrl = endpoint.replace(/\/+$/, ""); + const url = baseUrl.endsWith("/api/embed") ? baseUrl : `${baseUrl}/api/embed`; + + log.debug(`Calling Ollama embedding API for ${texts.length} texts`); + const response = await fetch(url, { + method: "POST", + headers: { + "Content-Type": "application/json", + ...cfg.headers, + }, + body: JSON.stringify({ model, input: texts }), + signal: AbortSignal.timeout(cfg.timeoutMs ?? 60_000), + }); + + if (!response.ok) { + const body = await response.text(); + throw new Error(`Ollama embedding failed (${response.status}): ${body}`); + } + + const payload = await response.json() as OllamaEmbedResponse; + if (!Array.isArray(payload.embeddings) + || payload.embeddings.length !== texts.length + || payload.embeddings.some((vector) => !Array.isArray(vector) + || vector.some((value) => typeof value !== "number"))) { + throw new Error("Ollama embedding response has an invalid embeddings shape"); + } + + return payload.embeddings as number[][]; +} diff --git a/apps/memos-local-openclaw/src/index.ts b/apps/memos-local-openclaw/src/index.ts index bc2a09f8e..bf4d9f7aa 100644 --- a/apps/memos-local-openclaw/src/index.ts +++ b/apps/memos-local-openclaw/src/index.ts @@ -61,6 +61,7 @@ export function initPlugin(opts: PluginInitOptions = {}): MemosLocalPlugin { const store = new SqliteStore(ctx.config.storage!.dbPath!, ctx.log); const embedder = new Embedder(ctx.config.embedding, ctx.log, ctx.openclawAPI); + store.configureVectorIndex?.(embedder.dimensions); warnOnEmbeddingMismatch(store, embedder, ctx.log); const worker = new IngestWorker(store, embedder, ctx); const engine = new RecallEngine(store, embedder, ctx); diff --git a/apps/memos-local-openclaw/src/recall/engine.ts b/apps/memos-local-openclaw/src/recall/engine.ts index 100e6467c..4d6e96595 100644 --- a/apps/memos-local-openclaw/src/recall/engine.ts +++ b/apps/memos-local-openclaw/src/recall/engine.ts @@ -60,11 +60,19 @@ export class RecallEngine { let vecCandidates: Array<{ chunkId: string; score: number }> = []; if (query) { try { - const queryVec = await this.embedder.embedQuery(query); + const queryVec = await this.embedQuery(query); const maxChunks = recallCfg.vectorSearchMaxChunks && recallCfg.vectorSearchMaxChunks > 0 ? recallCfg.vectorSearchMaxChunks : undefined; - vecCandidates = vectorSearch(this.store, queryVec, candidatePool, maxChunks, ownerFilter, excludeSessionKey); + vecCandidates = vectorSearch( + this.store, + queryVec, + candidatePool, + maxChunks, + ownerFilter, + excludeSessionKey, + this.ctx.log, + ); } catch (err) { this.ctx.log.warn(`Vector search failed, using FTS only: ${err}`); } @@ -111,17 +119,12 @@ export class RecallEngine { } try { - const qv = await this.embedder.embedQuery(query).catch(() => null); + const qv = await this.embedQuery(query).catch(() => null); if (qv) { const memEmbs = this.store.getVisibleHubMemoryEmbeddings("__hub__"); const scored: Array<{ id: string; score: number }> = []; for (const e of memEmbs) { - let dot = 0, nA = 0, nB = 0; - const len = Math.min(qv.length, e.vector.length); - for (let i = 0; i < len; i++) { - dot += qv[i] * e.vector[i]; nA += qv[i] * qv[i]; nB += e.vector[i] * e.vector[i]; - } - const sim = nA > 0 && nB > 0 ? dot / (Math.sqrt(nA) * Math.sqrt(nB)) : 0; + const sim = cosineSimilarity(qv, e.vector); if (sim > 0.3) scored.push({ id: `hubmem:${e.memoryId}`, score: sim }); } scored.sort((a, b) => b.score - a.score); @@ -310,10 +313,10 @@ export class RecallEngine { // FTS on name + description const ftsCandidates = this.store.skillFtsSearch(query, TOP_CANDIDATES, scope, currentOwner); - // Vector search on description embedding + // Vector search on description embedding (with caching) let vecCandidates: Array<{ skillId: string; score: number }> = []; try { - const queryVec = await this.embedder.embedQuery(query); + const queryVec = await this.embedQuery(query); const allEmb = this.store.getSkillEmbeddings(scope, currentOwner); vecCandidates = allEmb.map((row) => ({ skillId: row.skillId, @@ -387,6 +390,13 @@ export class RecallEngine { // Fallback: return all candidates return candidates.map((_, i) => i); } + + private embedQuery(text: string): Promise { + const cachedEmbed = (this.embedder as Partial).embedQueryWithCache; + return typeof cachedEmbed === "function" + ? cachedEmbed.call(this.embedder, text) + : this.embedder.embedQuery(text); + } } function makeExcerpt(content: string): string { diff --git a/apps/memos-local-openclaw/src/storage/sqlite.ts b/apps/memos-local-openclaw/src/storage/sqlite.ts index cd6402ffc..1a448fb6f 100644 --- a/apps/memos-local-openclaw/src/storage/sqlite.ts +++ b/apps/memos-local-openclaw/src/storage/sqlite.ts @@ -1,12 +1,32 @@ import Database from "better-sqlite3"; import { createHash } from "crypto"; +import { createRequire } from "node:module"; import * as fs from "fs"; import * as path from "path"; import type { Chunk, ChunkRef, DedupStatus, Task, TaskStatus, Skill, SkillStatus, SkillVisibility, SkillVersion, TaskSkillLink, TaskSkillRelation, Logger } from "../types"; import type { SharedVisibility, UserInfo, UserRole, UserStatus } from "../sharing/types"; +interface SqliteVecModule { + load(db: Database.Database): void; +} + +const requireFromModule = createRequire(import.meta.url); +let sqliteVecModule: SqliteVecModule | null | undefined; + +function loadSqliteVecModule(): SqliteVecModule | null { + if (sqliteVecModule !== undefined) return sqliteVecModule; + try { + sqliteVecModule = requireFromModule("sqlite-vec") as SqliteVecModule; + } catch { + sqliteVecModule = null; + } + return sqliteVecModule; +} + export class SqliteStore { private db: Database.Database; + private vecExtensionLoaded = false; + private vecIndexDimensions: number | null = null; constructor(dbPath: string, private log: Logger) { fs.mkdirSync(path.dirname(dbPath), { recursive: true }); @@ -258,6 +278,163 @@ export class SqliteStore { `); } + // ─── sqlite-vec Index ─── + + configureVectorIndex(dimensions: number): boolean { + if (!Number.isInteger(dimensions) || dimensions <= 0) { + this.log.warn(`sqlite-vec disabled: invalid embedding dimensions ${dimensions}`); + return false; + } + + try { + const sqliteVec = loadSqliteVecModule(); + if (!sqliteVec) throw new Error("sqlite-vec package is unavailable"); + sqliteVec.load(this.db); + + const row = this.db.prepare( + "SELECT sql FROM sqlite_schema WHERE type = 'table' AND name = 'vec_chunks'", + ).get() as { sql: string } | undefined; + const schema = row?.sql ?? ""; + const hasExpectedSchema = new RegExp(`FLOAT\\s*\\[${dimensions}\\]`, "i").test(schema) + && /distance_metric\s*=\s*cosine/i.test(schema) + && /\bowner\s+TEXT\b/i.test(schema) + && /\bsession_key\s+TEXT\b/i.test(schema); + + if (!hasExpectedSchema) { + this.db.exec("DROP TABLE IF EXISTS vec_chunks"); + this.db.exec(` + CREATE VIRTUAL TABLE vec_chunks USING vec0( + chunk_id TEXT PRIMARY KEY, + embedding FLOAT[${dimensions}] distance_metric=cosine, + owner TEXT, + session_key TEXT + ) + `); + this.backfillVectorIndex(dimensions); + } else if (!this.vectorIndexRowCountMatches(dimensions)) { + this.db.prepare("DELETE FROM vec_chunks").run(); + this.backfillVectorIndex(dimensions); + } + + this.vecIndexDimensions = dimensions; + this.vecExtensionLoaded = true; + this.log.info(`sqlite-vec index ready (${dimensions} dimensions, cosine distance)`); + return true; + } catch (err) { + this.vecExtensionLoaded = false; + this.vecIndexDimensions = null; + this.log.warn(`sqlite-vec unavailable; using brute-force vector search: ${err}`); + return false; + } + } + + hasVecIndex(): boolean { + return this.vecExtensionLoaded && this.vecIndexDimensions !== null; + } + + searchVecChunks( + queryVec: number[], + topK: number, + ownerFilter?: string[], + excludeSessionKey?: string, + ): Array<{ chunkId: string; distance: number }> { + if (!this.hasVecIndex()) throw new Error("sqlite-vec index is not configured"); + if (queryVec.length !== this.vecIndexDimensions) { + throw new Error( + `sqlite-vec dimension mismatch: index=${this.vecIndexDimensions}, query=${queryVec.length}`, + ); + } + if (topK <= 0) return []; + + let sql = ` + SELECT chunk_id, distance + FROM vec_chunks + WHERE embedding MATCH ? AND k = ? + `; + const params: unknown[] = [JSON.stringify(queryVec), topK]; + if (ownerFilter && ownerFilter.length > 0) { + sql += ` AND owner IN (${ownerFilter.map(() => "?").join(",")})`; + params.push(...ownerFilter); + } + if (excludeSessionKey) { + sql += " AND session_key != ?"; + params.push(excludeSessionKey); + } + sql += " ORDER BY distance"; + + const rows = this.db.prepare(sql).all(...params) as Array<{ chunk_id: string; distance: number }>; + + return rows.map((row) => ({ chunkId: row.chunk_id, distance: row.distance })); + } + + private backfillVectorIndex(dimensions: number): void { + const rows = this.db.prepare(` + SELECT e.chunk_id, e.vector, c.owner, c.session_key + FROM embeddings e + JOIN chunks c ON c.id = e.chunk_id + WHERE e.dimensions = ? AND c.dedup_status = 'active' + `).all(dimensions) as Array<{ chunk_id: string; vector: Buffer; owner: string; session_key: string }>; + const insert = this.db.prepare( + "INSERT OR REPLACE INTO vec_chunks (chunk_id, embedding, owner, session_key) VALUES (?, ?, ?, ?)", + ); + const backfill = this.db.transaction((items: Array<{ + chunk_id: string; + vector: Buffer; + owner: string; + session_key: string; + }>) => { + for (const item of items) { + insert.run(item.chunk_id, item.vector, item.owner, item.session_key); + } + }); + backfill(rows); + } + + private vectorIndexRowCountMatches(dimensions: number): boolean { + const expected = this.db.prepare(` + SELECT COUNT(*) AS count + FROM embeddings e + JOIN chunks c ON c.id = e.chunk_id + WHERE e.dimensions = ? AND c.dedup_status = 'active' + `).get(dimensions) as { count: number }; + const indexed = this.db.prepare("SELECT COUNT(*) AS count FROM vec_chunks").get() as { count: number }; + return expected.count === indexed.count; + } + + private syncVectorIndex(chunkId: string, vector: number[]): void { + if (!this.hasVecIndex()) return; + if (vector.length !== this.vecIndexDimensions) { + this.deleteVectorIndexEntry(chunkId); + this.log.debug( + `sqlite-vec skipped ${chunkId}: index=${this.vecIndexDimensions}, vector=${vector.length}`, + ); + return; + } + + const chunk = this.db.prepare( + "SELECT dedup_status, owner, session_key FROM chunks WHERE id = ?", + ).get(chunkId) as { dedup_status: string; owner: string; session_key: string } | undefined; + if (!chunk || chunk.dedup_status !== "active") { + this.deleteVectorIndexEntry(chunkId); + return; + } + + this.db.prepare( + "INSERT OR REPLACE INTO vec_chunks (chunk_id, embedding, owner, session_key) VALUES (?, ?, ?, ?)", + ).run(chunkId, JSON.stringify(vector), chunk.owner, chunk.session_key); + } + + private deleteVectorIndexEntry(chunkId: string): void { + if (!this.hasVecIndex()) return; + try { + this.db.prepare("DELETE FROM vec_chunks WHERE chunk_id = ?").run(chunkId); + } catch (err) { + this.vecExtensionLoaded = false; + this.vecIndexDimensions = null; + this.log.warn(`sqlite-vec delete failed for ${chunkId}; index disabled: ${err}`); + } + } + private migrateOwnerFields(): void { const chunkCols = this.db.prepare("PRAGMA table_info(chunks)").all() as Array<{ name: string }>; if (!chunkCols.some((c) => c.name === "owner")) { @@ -1201,6 +1378,7 @@ export class SqliteStore { this.db.prepare( "UPDATE chunks SET dedup_status = ?, dedup_target = ?, dedup_reason = ?, updated_at = ? WHERE id = ?", ).run(status, targetChunkId, reason, Date.now(), chunkId); + this.deleteVectorIndexEntry(chunkId); } updateSummary(chunkId: string, summary: string): void { @@ -1217,9 +1395,17 @@ export class SqliteStore { INSERT OR REPLACE INTO embeddings (chunk_id, vector, dimensions, provider, model, updated_at) VALUES (?, ?, ?, ?, ?, ?) `).run(chunkId, buf, vector.length, producer?.provider ?? "", producer?.model ?? "", Date.now()); + try { + this.syncVectorIndex(chunkId, vector); + } catch (err) { + this.vecExtensionLoaded = false; + this.vecIndexDimensions = null; + this.log.warn(`sqlite-vec update failed for ${chunkId}; index disabled: ${err}`); + } } deleteEmbedding(chunkId: string): void { + this.deleteVectorIndexEntry(chunkId); this.db.prepare("DELETE FROM embeddings WHERE chunk_id = ?").run(chunkId); } @@ -1629,17 +1815,34 @@ export class SqliteStore { // ─── Delete ─── deleteChunk(chunkId: string): boolean { + this.deleteVectorIndexEntry(chunkId); const result = this.db.prepare("DELETE FROM chunks WHERE id = ?").run(chunkId); return result.changes > 0; } deleteSession(sessionKey: string): number { + if (this.hasVecIndex()) { + const rows = this.db.prepare("SELECT id FROM chunks WHERE session_key = ?").all(sessionKey) as Array<{ id: string }>; + const remove = this.db.transaction((items: Array<{ id: string }>) => { + for (const row of items) this.deleteVectorIndexEntry(row.id); + }); + remove(rows); + } const result = this.db.prepare("DELETE FROM chunks WHERE session_key = ?").run(sessionKey); return result.changes; } deleteAll(): number { this.db.exec("PRAGMA foreign_keys = OFF"); + if (this.hasVecIndex()) { + try { + this.db.prepare("DELETE FROM vec_chunks").run(); + } catch (err) { + this.vecExtensionLoaded = false; + this.vecIndexDimensions = null; + this.log.warn(`deleteAll: sqlite-vec index disabled after clear failed: ${err}`); + } + } try { this.db.exec("DROP TRIGGER IF EXISTS tasks_fts_ai"); this.db.exec("DROP TRIGGER IF EXISTS tasks_fts_ad"); diff --git a/apps/memos-local-openclaw/src/storage/vector.ts b/apps/memos-local-openclaw/src/storage/vector.ts index 488f287bf..332990139 100644 --- a/apps/memos-local-openclaw/src/storage/vector.ts +++ b/apps/memos-local-openclaw/src/storage/vector.ts @@ -1,6 +1,7 @@ +import type { Logger } from "../types"; import type { SqliteStore } from "./sqlite"; -export function cosineSimilarity(a: number[], b: number[]): number { +export function cosineSimilarity(a: ArrayLike, b: ArrayLike): number { if (a.length !== b.length) return 0; let dot = 0; let normA = 0; @@ -20,10 +21,8 @@ export interface VectorHit { } /** - * Brute-force vector search over stored embeddings. - * When maxChunks > 0, only searches the most recent maxChunks chunks (uses index; avoids full scan as data grows). - * When excludeSessionKey is set, chunks whose session_key equals it are filtered out before scoring, - * so the caller can suppress recall of the current conversation session. + * Uses sqlite-vec metadata for owner/session filters. A recent-chunk window + * keeps the existing SQL implementation so the cap remains exact. */ export function vectorSearch( store: SqliteStore, @@ -32,14 +31,40 @@ export function vectorSearch( maxChunks?: number, ownerFilter?: string[], excludeSessionKey?: string, + log?: Logger, ): VectorHit[] { + const requiresRecentWindow = maxChunks != null && maxChunks > 0; + + const hasConfiguredIndex = typeof store.hasVecIndex === "function" && store.hasVecIndex(); + if (isVecIndexAvailable() && hasConfiguredIndex && !requiresRecentWindow) { + try { + return store.searchVecChunks(queryVec, topK, ownerFilter, excludeSessionKey).map((result) => ({ + chunkId: result.chunkId, + score: 1 - result.distance, + })); + } catch (err) { + log?.warn(`Indexed vector search failed; using brute force: ${err}`); + } + } + const all = maxChunks != null && maxChunks > 0 ? store.getRecentEmbeddings(maxChunks, ownerFilter, excludeSessionKey) : store.getAllEmbeddings(ownerFilter, excludeSessionKey); - const scored: VectorHit[] = all.map((row) => ({ + const scored = all.map((row) => ({ chunkId: row.chunkId, score: cosineSimilarity(queryVec, row.vector), })); scored.sort((a, b) => b.score - a.score); return scored.slice(0, topK); } + +export function isVecIndexAvailable(): boolean { + return process.env.MEMOS_USE_VEC_INDEX !== "false"; +} + +export function getSearchMode(): { useIndex: boolean; reason: string } { + if (!isVecIndexAvailable()) { + return { useIndex: false, reason: "MEMOS_USE_VEC_INDEX=false" }; + } + return { useIndex: true, reason: "sqlite-vec indexed search" }; +} diff --git a/apps/memos-local-openclaw/src/types.ts b/apps/memos-local-openclaw/src/types.ts index 0eec78cbe..dcc5b900e 100644 --- a/apps/memos-local-openclaw/src/types.ts +++ b/apps/memos-local-openclaw/src/types.ts @@ -166,6 +166,7 @@ export type EmbeddingProvider = | "cohere" | "mistral" | "voyage" + | "ollama" | "local" | "openclaw"; diff --git a/apps/memos-local-openclaw/tests/embedding-cache.test.ts b/apps/memos-local-openclaw/tests/embedding-cache.test.ts new file mode 100644 index 000000000..6ed176dfc --- /dev/null +++ b/apps/memos-local-openclaw/tests/embedding-cache.test.ts @@ -0,0 +1,47 @@ +import { describe, expect, it } from "vitest"; + +import { EmbeddingCache } from "../src/embedding/cache"; + +describe("EmbeddingCache", () => { + it("returns a cached embedding", async () => { + const cache = new EmbeddingCache({ maxSize: 2, ttlMs: 1_000 }); + await cache.set("openai:model-a:3", "query", [1, 2, 3]); + + expect(await cache.get("openai:model-a:3", "query")).toEqual([1, 2, 3]); + expect(await cache.has("openai:model-a:3", "query")).toBe(true); + }); + + it("expires entries after the configured TTL", async () => { + let now = 100; + const cache = new EmbeddingCache({ maxSize: 2, ttlMs: 50, now: () => now }); + await cache.set("openai:model-a:3", "query", [1, 2, 3]); + + now = 149; + expect(await cache.get("openai:model-a:3", "query")).toEqual([1, 2, 3]); + now = 150; + expect(await cache.get("openai:model-a:3", "query")).toBeNull(); + }); + + it("evicts the least recently used entry", async () => { + const cache = new EmbeddingCache({ maxSize: 2, ttlMs: 1_000 }); + const signature = "openai:model-a:1"; + await cache.set(signature, "first", [1]); + await cache.set(signature, "second", [2]); + await cache.get(signature, "first"); + await cache.set(signature, "third", [3]); + + expect(await cache.get(signature, "first")).toEqual([1]); + expect(await cache.get(signature, "second")).toBeNull(); + expect(await cache.get(signature, "third")).toEqual([3]); + }); + + it("keys identical text by provider, model, and dimensions", async () => { + const cache = new EmbeddingCache({ maxSize: 4, ttlMs: 1_000 }); + await cache.set("openai:model-a:3", "same query", [1, 0, 0]); + await cache.set("openai:model-b:3", "same query", [0, 1, 0]); + + expect(await cache.get("openai:model-a:3", "same query")).toEqual([1, 0, 0]); + expect(await cache.get("openai:model-b:3", "same query")).toEqual([0, 1, 0]); + expect(await cache.get("openai:model-a:4", "same query")).toBeNull(); + }); +}); diff --git a/apps/memos-local-openclaw/tests/embedding-ollama.test.ts b/apps/memos-local-openclaw/tests/embedding-ollama.test.ts new file mode 100644 index 000000000..7f92ab8ca --- /dev/null +++ b/apps/memos-local-openclaw/tests/embedding-ollama.test.ts @@ -0,0 +1,83 @@ +import { afterEach, describe, expect, it, vi } from "vitest"; + +import { embedOllama } from "../src/embedding/providers/ollama"; +import type { EmbeddingConfig, Logger } from "../src/types"; + +const noopLog: Logger = { + debug: () => {}, + info: () => {}, + warn: () => {}, + error: () => {}, +}; + +afterEach(() => { + vi.unstubAllGlobals(); +}); + +describe("Ollama embedding provider", () => { + it("sends a batch to /api/embed with an embedding model default", async () => { + const fetchMock = vi.fn(async (_url: string, _init?: RequestInit) => ({ + ok: true, + json: async () => ({ embeddings: [[1, 0], [0, 1]] }), + }) as Response); + vi.stubGlobal("fetch", fetchMock); + + const vectors = await embedOllama( + ["first", "second"], + { provider: "ollama", endpoint: "http://ollama.test/" }, + noopLog, + ); + + expect(vectors).toEqual([[1, 0], [0, 1]]); + expect(fetchMock).toHaveBeenCalledTimes(1); + const [url, init] = fetchMock.mock.calls[0]; + expect(url).toBe("http://ollama.test/api/embed"); + expect(JSON.parse(String(init?.body))).toEqual({ + model: "nomic-embed-text", + input: ["first", "second"], + }); + }); + + it("does not append /api/embed twice", async () => { + const fetchMock = vi.fn(async (_url: string, _init?: RequestInit) => ({ + ok: true, + json: async () => ({ embeddings: [[1, 0]] }), + }) as Response); + vi.stubGlobal("fetch", fetchMock); + + await embedOllama( + ["query"], + { provider: "ollama", endpoint: "http://ollama.test/api/embed/", model: "mxbai-embed-large" }, + noopLog, + ); + + expect(fetchMock.mock.calls[0][0]).toBe("http://ollama.test/api/embed"); + }); + + it("reports HTTP errors", async () => { + vi.stubGlobal("fetch", vi.fn(async () => ({ + ok: false, + status: 503, + text: async () => "model unavailable", + }) as Response)); + + await expect(embedOllama( + ["query"], + { provider: "ollama" } as EmbeddingConfig, + noopLog, + )).rejects.toThrow("Ollama embedding failed (503): model unavailable"); + }); + + it("rejects malformed or incomplete batches", async () => { + vi.stubGlobal("fetch", vi.fn(async () => ({ + ok: true, + json: async () => ({ embeddings: [[1, 0]] }), + }) as Response)); + + await expect(embedOllama( + ["first", "second"], + { provider: "ollama" } as EmbeddingConfig, + noopLog, + )).rejects.toThrow("invalid embeddings shape"); + }); +}); diff --git a/apps/memos-local-openclaw/tests/embedding-reembed.test.ts b/apps/memos-local-openclaw/tests/embedding-reembed.test.ts index 6d98e0b06..0773215c9 100644 --- a/apps/memos-local-openclaw/tests/embedding-reembed.test.ts +++ b/apps/memos-local-openclaw/tests/embedding-reembed.test.ts @@ -410,7 +410,11 @@ describe("re-embed CLI", () => { // Verify the DB embeddings table is unchanged const checkDb = new Database(dbPath, { readonly: true }); const rows = checkDb.prepare("SELECT chunk_id, provider, model FROM embeddings ORDER BY chunk_id").all(); + const vecTable = checkDb.prepare( + "SELECT COUNT(*) AS count FROM sqlite_schema WHERE name = 'vec_chunks'", + ).get() as { count: number }; checkDb.close(); + expect(vecTable.count).toBe(0); expect(rows).toEqual([ { chunk_id: "c1", provider: "local", model: "" }, { chunk_id: "c2", provider: "openai", model: "old-model" }, diff --git a/apps/memos-local-openclaw/tests/package-manifest.test.ts b/apps/memos-local-openclaw/tests/package-manifest.test.ts index a199e9bfc..c5ef5c7fc 100644 --- a/apps/memos-local-openclaw/tests/package-manifest.test.ts +++ b/apps/memos-local-openclaw/tests/package-manifest.test.ts @@ -14,6 +14,7 @@ describe("package manifest", () => { files: string[]; openclaw: { extensions: string[] }; scripts: Record; + dependencies: Record; }; const pluginJson = JSON.parse( fs.readFileSync(path.join(root, "openclaw.plugin.json"), "utf-8"), @@ -31,5 +32,6 @@ describe("package manifest", () => { expect(packageJson.files).toEqual(expect.arrayContaining(["dist", "tsconfig.json"])); expect(packageJson.files).not.toContain("index.ts"); expect(packageJson.scripts.prepublishOnly).toBe("npm run build"); + expect(packageJson.dependencies["sqlite-vec"]).toBe("^0.1.9"); }); }); diff --git a/apps/memos-local-openclaw/tests/postinstall-native-binding.test.ts b/apps/memos-local-openclaw/tests/postinstall-native-binding.test.ts index 995820cf9..b3c18e278 100644 --- a/apps/memos-local-openclaw/tests/postinstall-native-binding.test.ts +++ b/apps/memos-local-openclaw/tests/postinstall-native-binding.test.ts @@ -5,6 +5,7 @@ const require = createRequire(import.meta.url); const { quarantineNativeBinding, validateNativeBinding, + validateSqliteVecExtension, } = require("../scripts/native-binding.cjs"); describe("postinstall native binding validation", () => { @@ -134,4 +135,36 @@ describe("postinstall native binding validation", () => { expect(renames).toHaveLength(2); expect(renames[0][1]).not.toBe(renames[1][1]); }); + + it("accepts a loadable sqlite-vec platform extension", () => { + let closed = false; + const db = { + prepare: () => ({ get: () => ({ version: "0.1.9" }) }), + close: () => { closed = true; }, + }; + const sqliteVec = { + getLoadablePath: () => "/tmp/vec0.dylib", + load: (candidate: unknown) => expect(candidate).toBe(db), + }; + + const result = validateSqliteVecExtension( + sqliteVec, + () => db, + { existsSync: () => true }, + ); + + expect(result).toEqual({ ok: true, reason: "ok", message: "", version: "0.1.9" }); + expect(closed).toBe(true); + }); + + it("reports a missing sqlite-vec platform binary", () => { + const result = validateSqliteVecExtension( + { getLoadablePath: () => "/tmp/vec0.dylib", load: () => {} }, + () => { throw new Error("database should not be opened"); }, + { existsSync: () => false }, + ); + + expect(result.ok).toBe(false); + expect(result.reason).toBe("missing-platform-binary"); + }); }); diff --git a/apps/memos-local-openclaw/tests/recall-skill-vector-sort.test.ts b/apps/memos-local-openclaw/tests/recall-skill-vector-sort.test.ts new file mode 100644 index 000000000..ed7c732b4 --- /dev/null +++ b/apps/memos-local-openclaw/tests/recall-skill-vector-sort.test.ts @@ -0,0 +1,41 @@ +import { describe, expect, it, vi } from "vitest"; + +import { RecallEngine } from "../src/recall/engine"; +import type { Embedder } from "../src/embedding"; +import type { SqliteStore } from "../src/storage/sqlite"; +import type { PluginContext } from "../src/types"; + +describe("RecallEngine skill vector ranking", () => { + it("sorts candidates by descending cosine score", async () => { + const store = { + skillFtsSearch: vi.fn(() => []), + getSkillEmbeddings: vi.fn(() => [ + { skillId: "low", vector: [0, 1] }, + { skillId: "high", vector: [1, 0] }, + ]), + getSkill: vi.fn((id: string) => ({ + id, + name: id, + description: id, + owner: "agent:main", + visibility: "private", + })), + } as unknown as SqliteStore; + const embedder = { + embedQueryWithCache: vi.fn(async () => [1, 0]), + embedQuery: vi.fn(async () => [1, 0]), + } as unknown as Embedder; + const context = { + stateDir: "/tmp", + workspaceDir: "/tmp", + config: {}, + log: { debug() {}, info() {}, warn() {}, error() {} }, + } as unknown as PluginContext; + const engine = new RecallEngine(store, embedder, context); + vi.spyOn(engine as any, "judgeSkillRelevance").mockResolvedValue([0, 1]); + + const results = await engine.searchSkills("query", "self", "agent:main"); + + expect(results.map((result) => result.skillId)).toEqual(["high", "low"]); + }); +}); diff --git a/apps/memos-local-openclaw/tests/sqlite-vec.test.ts b/apps/memos-local-openclaw/tests/sqlite-vec.test.ts new file mode 100644 index 000000000..73f11a40a --- /dev/null +++ b/apps/memos-local-openclaw/tests/sqlite-vec.test.ts @@ -0,0 +1,180 @@ +import { afterEach, beforeEach, describe, expect, it, vi } from "vitest"; +import fs from "node:fs"; +import os from "node:os"; +import path from "node:path"; + +import { SqliteStore } from "../src/storage/sqlite"; +import { vectorSearch } from "../src/storage/vector"; +import type { Chunk, Logger } from "../src/types"; + +let store: SqliteStore; +let tmpDir: string; +let warn: ReturnType; +let originalIndexSetting: string | undefined; + +function chunk(id: string, createdAt: number, overrides: Partial = {}): Chunk { + return { + id, + sessionKey: "session-1", + turnId: "turn-1", + seq: 0, + role: "user", + content: id, + kind: "paragraph", + summary: id, + embedding: null, + taskId: null, + skillId: null, + owner: "agent:main", + dedupStatus: "active", + dedupTarget: null, + dedupReason: null, + mergeCount: 0, + lastHitAt: null, + mergeHistory: "[]", + createdAt, + updatedAt: createdAt, + ...overrides, + }; +} + +beforeEach(() => { + originalIndexSetting = process.env.MEMOS_USE_VEC_INDEX; + delete process.env.MEMOS_USE_VEC_INDEX; + tmpDir = fs.mkdtempSync(path.join(os.tmpdir(), "memos-sqlite-vec-")); + warn = vi.fn(); + const log: Logger = { debug: () => {}, info: () => {}, warn, error: () => {} }; + store = new SqliteStore(path.join(tmpDir, "test.db"), log); +}); + +afterEach(() => { + store.close(); + fs.rmSync(tmpDir, { recursive: true, force: true }); + if (originalIndexSetting === undefined) delete process.env.MEMOS_USE_VEC_INDEX; + else process.env.MEMOS_USE_VEC_INDEX = originalIndexSetting; +}); + +describe("sqlite-vec search", () => { + it("derives the vec0 dimensions and uses cosine distance", () => { + expect(store.configureVectorIndex(3)).toBe(true); + + const db = (store as unknown as { db: { + prepare(sql: string): { get(): { sql: string } }; + } }).db; + const schema = db.prepare( + "SELECT sql FROM sqlite_schema WHERE name = 'vec_chunks'", + ).get().sql; + expect(schema).toMatch(/FLOAT\s*\[3\]/i); + expect(schema).toMatch(/distance_metric\s*=\s*cosine/i); + expect(schema).toMatch(/owner\s+TEXT/i); + expect(schema).toMatch(/session_key\s+TEXT/i); + }); + + it("returns the same ranking and scores as brute-force cosine search", () => { + expect(store.configureVectorIndex(3)).toBe(true); + const now = Date.now(); + const vectors: Array<[string, number[]]> = [ + ["exact", [1, 0, 0]], + ["near", [0.8, 0.2, 0]], + ["far", [0, 1, 0]], + ]; + for (const [id, vector] of vectors) { + store.insertChunk(chunk(id, now)); + store.upsertEmbedding(id, vector); + } + + const indexed = vectorSearch(store, [1, 0, 0], 3); + process.env.MEMOS_USE_VEC_INDEX = "false"; + const bruteForce = vectorSearch(store, [1, 0, 0], 3); + + expect(indexed.map((hit) => hit.chunkId)).toEqual(bruteForce.map((hit) => hit.chunkId)); + indexed.forEach((hit, index) => { + expect(hit.score).toBeCloseTo(bruteForce[index].score, 5); + }); + }); + + it("backfills matching stored embeddings when the index is created", () => { + store.insertChunk(chunk("existing", Date.now())); + store.upsertEmbedding("existing", [1, 0, 0]); + + expect(store.configureVectorIndex(3)).toBe(true); + + expect(store.searchVecChunks([1, 0, 0], 1)).toEqual([ + { chunkId: "existing", distance: 0 }, + ]); + }); + + it("repairs an incomplete existing index on configuration", () => { + expect(store.configureVectorIndex(3)).toBe(true); + store.insertChunk(chunk("missing-from-index", Date.now())); + store.upsertEmbedding("missing-from-index", [1, 0, 0]); + const db = (store as unknown as { db: { + prepare(sql: string): { run(...args: unknown[]): unknown }; + } }).db; + db.prepare("DELETE FROM vec_chunks WHERE chunk_id = ?").run("missing-from-index"); + + expect(store.configureVectorIndex(3)).toBe(true); + + expect(store.searchVecChunks([1, 0, 0], 1)[0].chunkId).toBe("missing-from-index"); + }); + + it("falls back on dimension mismatch", () => { + expect(store.configureVectorIndex(3)).toBe(true); + store.insertChunk(chunk("item", Date.now())); + store.upsertEmbedding("item", [1, 0, 0]); + + const result = vectorSearch(store, [1, 0], 1, undefined, undefined, undefined, { + debug: () => {}, info: () => {}, warn, error: () => {}, + }); + + expect(result).toEqual([{ chunkId: "item", score: 0 }]); + expect(warn).toHaveBeenCalledWith(expect.stringContaining("dimension mismatch")); + }); + + it("applies owner and excluded-session filters inside the index", () => { + expect(store.configureVectorIndex(3)).toBe(true); + const indexedSearch = vi.spyOn(store, "searchVecChunks"); + const now = Date.now(); + store.insertChunk(chunk("current-private", now, { + owner: "agent:alpha", + sessionKey: "current-session", + })); + store.upsertEmbedding("current-private", [1, 0, 0]); + store.insertChunk(chunk("public-history", now, { + owner: "public", + sessionKey: "old-session", + })); + store.upsertEmbedding("public-history", [0.9, 0.1, 0]); + store.insertChunk(chunk("other-private", now, { + owner: "agent:beta", + sessionKey: "old-session", + })); + store.upsertEmbedding("other-private", [1, 0, 0]); + + const result = vectorSearch( + store, + [1, 0, 0], + 5, + undefined, + ["agent:alpha", "public"], + "current-session", + ); + + expect(indexedSearch).toHaveBeenCalled(); + expect(result.map((hit) => hit.chunkId)).toEqual(["public-history"]); + }); + + it("honors vectorSearchMaxChunks by using the filtered fallback", () => { + expect(store.configureVectorIndex(3)).toBe(true); + const indexedSearch = vi.spyOn(store, "searchVecChunks"); + store.insertChunk(chunk("old", 100)); + store.upsertEmbedding("old", [1, 0, 0]); + store.insertChunk(chunk("recent", 200)); + store.upsertEmbedding("recent", [0, 1, 0]); + + const result = vectorSearch(store, [1, 0, 0], 5, 1); + + expect(indexedSearch).not.toHaveBeenCalled(); + expect(result.map((hit) => hit.chunkId)).toEqual(["recent"]); + }); +});