diff --git a/apps/memos-local-openclaw/scripts/re-embed.ts b/apps/memos-local-openclaw/scripts/re-embed.ts new file mode 100644 index 000000000..8b3d492a3 --- /dev/null +++ b/apps/memos-local-openclaw/scripts/re-embed.ts @@ -0,0 +1,222 @@ +#!/usr/bin/env tsx +/** + * scripts/re-embed.ts — re-embed chunks under the currently configured + * embedding provider/model. See issue #1333. + * + * Usage: + * pnpm exec tsx scripts/re-embed.ts [--missing-only] [--dry-run] + * [--limit N] [--batch-size N] + * [--db PATH] [--config PATH] + */ +import * as fs from "fs"; +import * as os from "os"; +import * as path from "path"; +import { SqliteStore } from "../src/storage/sqlite"; +import { Embedder } from "../src/embedding"; +import { resolveConfig } from "../src/config"; +import type { Logger, MemosLocalConfig } from "../src/types"; + +interface CliOpts { + missingOnly: boolean; + dryRun: boolean; + limit?: number; + batchSize: number; + dbPath?: string; + configPath: string; + help: boolean; +} + +export function parseArgs(argv: string[]): CliOpts { + const opts: CliOpts = { + missingOnly: false, + dryRun: false, + batchSize: 32, + configPath: path.join(os.homedir(), ".openclaw", "openclaw.json"), + help: false, + }; + for (let i = 0; i < argv.length; i++) { + const a = argv[i]; + if (a === "--missing-only") opts.missingOnly = true; + else if (a === "--dry-run") opts.dryRun = true; + else if (a === "--limit") opts.limit = Number(argv[++i]); + else if (a === "--batch-size") opts.batchSize = Number(argv[++i]); + else if (a === "--db") opts.dbPath = argv[++i]; + else if (a === "--config") opts.configPath = argv[++i]; + else if (a === "-h" || a === "--help") opts.help = true; + else throw new Error(`Unknown argument: ${a}`); + } + return opts; +} + +const HELP = `Re-embed chunks under the currently configured embedding provider/model. + +Usage: tsx scripts/re-embed.ts [options] + + --missing-only Only re-embed chunks that have no embedding row. + --dry-run Print the plan only; do not write embeddings. + --limit Process at most n chunks. + --batch-size Embed n chunks per provider call (default 32). + --db Override DB path (default reads ~/.openclaw config). + --config Override openclaw.json path (default ~/.openclaw/openclaw.json). + -h, --help Show this help. + +Behaviour: scans for chunks whose embedding row's (provider, model, dimensions) +does not match the current Embedder — plus rows tagged as 'legacy' (empty +producer columns, from before this feature) and chunks with no embedding row. +Re-embeds them in batches; never deletes existing data. Re-running picks up +where it left off because the candidate list is re-computed each run. +`; + +function makeLog(): Logger { + return { + debug: (...a) => console.debug("[re-embed]", ...a), + info: (...a) => console.info("[re-embed]", ...a), + warn: (...a) => console.warn("[re-embed]", ...a), + error: (...a) => console.error("[re-embed]", ...a), + }; +} + +function loadPluginConfig(configPath: string): Partial { + if (!fs.existsSync(configPath)) { + throw new Error(`Config not found: ${configPath}`); + } + const raw = JSON.parse(fs.readFileSync(configPath, "utf-8")) as Record; + const plugins = (raw.plugins as Record | undefined) ?? {}; + const entries = ((plugins as { entries?: Record }).entries ?? {}) as Record }>; + const configs = ((plugins as { configs?: Record }).configs ?? {}) as Record }>; + const candidate = + entries["memos-local"]?.config ?? + entries["memos-local-openclaw-plugin"]?.config ?? + configs["memos-local"]?.config ?? + {}; + return candidate; +} + +function formatStats(label: string, s: { + total: number; + matched: number; + mismatched: number; + legacy: number; + missing: number; + current: { provider: string; model: string; dimensions: number }; + byProducer: Array<{ provider: string; model: string; dimensions: number; count: number }>; +}): string { + const lines: string[] = []; + lines.push(`── ${label} ──`); + lines.push(` current: ${s.current.provider}:${s.current.model}:${s.current.dimensions}`); + lines.push(` total embeddings: ${s.total}`); + lines.push(` matched: ${s.matched}`); + lines.push(` mismatched: ${s.mismatched}`); + lines.push(` legacy: ${s.legacy} (pre-tagging rows; treated as needing re-embed)`); + lines.push(` missing: ${s.missing} (active chunks with no embedding row)`); + lines.push(` by producer:`); + for (const b of s.byProducer) { + const tag = b.provider === "" ? "(legacy)" : `${b.provider}/${b.model || "(no model)"}`; + lines.push(` ${tag} dim=${b.dimensions} count=${b.count}`); + } + return lines.join("\n"); +} + +export async function runReembed(opts: CliOpts): Promise<{ processed: number; failed: number; planned: number }> { + const log = makeLog(); + const pluginCfg = loadPluginConfig(opts.configPath); + const stateDir = path.join(os.homedir(), ".openclaw"); + const resolved = resolveConfig(pluginCfg, stateDir); + const dbPath = opts.dbPath ?? resolved.storage!.dbPath!; + log.info(`config: ${opts.configPath}`); + log.info(`db: ${dbPath}`); + + const store = new SqliteStore(dbPath, log); + const embedder = new Embedder(resolved.embedding, log); + const producer = { provider: embedder.provider, model: embedder.model }; + const current = { provider: embedder.provider, model: embedder.model, dimensions: embedder.dimensions }; + + const before = store.getEmbeddingStats(current); + console.log(formatStats("before", before)); + + const ids = store.listChunkIdsForReembed(current, { + missingOnly: opts.missingOnly, + limit: opts.limit, + }); + console.log(`\nplanned re-embed: ${ids.length} chunk(s)`); + + if (opts.dryRun) { + console.log("(dry-run: no writes performed)"); + store.close(); + return { processed: 0, failed: 0, planned: ids.length }; + } + + let processed = 0; + let failed = 0; + for (let i = 0; i < ids.length; i += opts.batchSize) { + const batchIds = ids.slice(i, i + opts.batchSize); + const texts: string[] = []; + const keep: string[] = []; + for (const id of batchIds) { + const chunk = store.getChunk(id); + if (!chunk) continue; + const text = chunk.summary || (chunk.content ?? "").slice(0, 500); + if (!text) continue; + keep.push(id); + texts.push(text); + } + if (keep.length === 0) continue; + + try { + const vecs = await embedder.embed(texts); + for (let j = 0; j < keep.length; j++) { + if (vecs[j]) { + store.upsertEmbedding(keep[j], vecs[j], producer); + processed++; + } + } + console.log(` batch ${i / opts.batchSize + 1}: re-embedded ${keep.length} chunk(s) (cumulative ${processed}/${ids.length})`); + } catch (err) { + failed += keep.length; + log.warn(`batch ${i / opts.batchSize + 1} failed; continuing: ${err}`); + } + } + + const after = store.getEmbeddingStats(current); + console.log("\n" + formatStats("after", after)); + console.log(`\ndone: processed=${processed} failed=${failed} planned=${ids.length}`); + + store.close(); + return { processed, failed, planned: ids.length }; +} + +async function main(): Promise { + let opts: CliOpts; + try { + opts = parseArgs(process.argv.slice(2)); + } catch (err) { + console.error(String(err)); + console.error(HELP); + process.exit(2); + } + if (opts.help) { + console.log(HELP); + return; + } + const result = await runReembed(opts); + if (result.failed > 0) process.exitCode = 1; +} + +// Node will run main() when invoked directly. We avoid the `import.meta.url` +// trick for vitest compatibility and just check require.main or treat any +// direct script execution as the entry point. +const isDirect = (() => { + try { + // tsx + ESM: process.argv[1] is the script path + return process.argv[1] && process.argv[1].endsWith("re-embed.ts"); + } catch { + return false; + } +})(); + +if (isDirect) { + main().catch((err) => { + console.error("re-embed failed:", err); + process.exit(1); + }); +} diff --git a/apps/memos-local-openclaw/src/embedding/index.ts b/apps/memos-local-openclaw/src/embedding/index.ts index 2adc7cac7..0a5e88067 100644 --- a/apps/memos-local-openclaw/src/embedding/index.ts +++ b/apps/memos-local-openclaw/src/embedding/index.ts @@ -21,11 +21,25 @@ export class Embedder { return this.cfg?.provider ?? "local"; } + get model(): string { + if (this.provider === "local") return ""; + return this.cfg?.model ?? ""; + } + get dimensions(): number { if (this.provider === "local") return 384; return this.cfg?.dimensions ?? 1536; } + /** + * Canonical identity of the embedding space this embedder produces vectors in. + * Format: "provider:model:dimensions". Used to detect when stored vectors + * were produced by a different model than the live config. + */ + get signature(): string { + return `${this.provider}:${this.model}:${this.dimensions}`; + } + async embed(texts: string[]): Promise { const batchSize = this.cfg?.batchSize ?? 32; const results: number[][] = []; diff --git a/apps/memos-local-openclaw/src/hub/server.ts b/apps/memos-local-openclaw/src/hub/server.ts index d8b697d79..bc61cecf5 100644 --- a/apps/memos-local-openclaw/src/hub/server.ts +++ b/apps/memos-local-openclaw/src/hub/server.ts @@ -201,11 +201,12 @@ export class HubServer { private embedChunksAsync(chunkIds: string[], chunks: Array<{ id: string; summary?: string; content?: string }>): void { const embedder = this.opts.embedder; if (!embedder) return; + const producer = { provider: embedder.provider, model: embedder.model }; const texts = chunks.map(c => c.summary || (c.content ? c.content.slice(0, 500) : "")); embedder.embed(texts).then((vectors) => { for (let i = 0; i < vectors.length; i++) { if (vectors[i]) { - this.opts.store.upsertHubEmbedding(chunkIds[i], new Float32Array(vectors[i])); + this.opts.store.upsertHubEmbedding(chunkIds[i], new Float32Array(vectors[i]), producer); } } this.opts.log.info(`hub: embedded ${vectors.filter(Boolean).length}/${chunkIds.length} shared chunks`); @@ -217,10 +218,11 @@ export class HubServer { private embedSkillAsync(skillId: string, name: string, description: string, sourceUserId: string, sourceSkillId: string): void { const embedder = this.opts.embedder; if (!embedder) return; + const producer = { provider: embedder.provider, model: embedder.model }; const text = `${name}: ${description}`; embedder.embed([text]).then((vectors) => { if (vectors[0]) { - this.opts.store.upsertHubSkillEmbedding(skillId, Array.from(vectors[0]), sourceUserId, sourceSkillId); + this.opts.store.upsertHubSkillEmbedding(skillId, Array.from(vectors[0]), sourceUserId, sourceSkillId, producer); this.opts.log.info(`hub: embedded shared skill ${skillId}`); } }).catch((err) => { @@ -230,6 +232,8 @@ export class HubServer { private backfillMemoryEmbeddings(): void { if (!this.opts.embedder) return; + const embedder = this.opts.embedder; + const producer = { provider: embedder.provider, model: embedder.model }; try { const all = this.opts.store.listHubMemories({ limit: 500 }); const missing = all.filter(m => { @@ -238,11 +242,11 @@ export class HubServer { if (missing.length === 0) return; this.opts.log.info(`hub: backfilling embeddings for ${missing.length} hub memories`); const texts = missing.map(m => (m.summary || m.content || "").slice(0, 500)); - this.opts.embedder.embed(texts).then((vectors) => { + embedder.embed(texts).then((vectors) => { let count = 0; for (let i = 0; i < vectors.length; i++) { if (vectors[i]) { - this.opts.store.upsertHubMemoryEmbedding(missing[i].id, new Float32Array(vectors[i])); + this.opts.store.upsertHubMemoryEmbedding(missing[i].id, new Float32Array(vectors[i]), producer); count++; } } @@ -258,11 +262,12 @@ export class HubServer { private embedMemoryAsync(memoryId: string, summary: string, content: string): void { const embedder = this.opts.embedder; if (!embedder) return; + const producer = { provider: embedder.provider, model: embedder.model }; const text = (summary || content || "").slice(0, 500); if (!text) return; embedder.embed([text]).then((vectors) => { if (vectors[0]) { - this.opts.store.upsertHubMemoryEmbedding(memoryId, new Float32Array(vectors[0])); + this.opts.store.upsertHubMemoryEmbedding(memoryId, new Float32Array(vectors[0]), producer); this.opts.log.info(`hub: embedded shared memory ${memoryId}`); } }).catch((err) => { diff --git a/apps/memos-local-openclaw/src/index.ts b/apps/memos-local-openclaw/src/index.ts index abf3e3590..bc2a09f8e 100644 --- a/apps/memos-local-openclaw/src/index.ts +++ b/apps/memos-local-openclaw/src/index.ts @@ -61,6 +61,7 @@ export function initPlugin(opts: PluginInitOptions = {}): MemosLocalPlugin { const store = new SqliteStore(ctx.config.storage!.dbPath!, ctx.log); const embedder = new Embedder(ctx.config.embedding, ctx.log, ctx.openclawAPI); + warnOnEmbeddingMismatch(store, embedder, ctx.log); const worker = new IngestWorker(store, embedder, ctx); const engine = new RecallEngine(store, embedder, ctx); @@ -115,5 +116,31 @@ function defaultStateDir(): string { return `${home}/.openclaw`; } +/** + * One-shot init check (issue #1333): if the `embeddings` table has rows + * tagged with a different producer than the live Embedder, or untagged + * legacy rows from a previous version, emit a single warn line so the + * user knows to run `scripts/re-embed.ts`. Best-effort: never throws. + */ +function warnOnEmbeddingMismatch(store: SqliteStore, embedder: Embedder, log: Logger): void { + try { + const stats = store.getEmbeddingStats({ + provider: embedder.provider, + model: embedder.model, + dimensions: embedder.dimensions, + }); + if (stats.total === 0) return; + const stale = stats.legacy + stats.mismatched; + if (stale === 0) return; + log.warn( + `embedding model mismatch detected: ${stats.legacy} legacy + ${stats.mismatched} from prior models, ` + + `${stats.matched} match current (${embedder.signature}). Vector recall is degraded for non-matching rows — ` + + `run \`pnpm exec tsx scripts/re-embed.ts\` to re-embed.`, + ); + } catch (err) { + log.debug(`warnOnEmbeddingMismatch skipped: ${err}`); + } +} + // Re-export types for consumers export type { MemosLocalConfig, ToolDefinition, SearchResult, SearchHit, TimelineResult, GetResult } from "./types"; diff --git a/apps/memos-local-openclaw/src/ingest/worker.ts b/apps/memos-local-openclaw/src/ingest/worker.ts index d62ab4a2a..c2921719b 100644 --- a/apps/memos-local-openclaw/src/ingest/worker.ts +++ b/apps/memos-local-openclaw/src/ingest/worker.ts @@ -286,7 +286,10 @@ export class IngestWorker { this.store.insertChunk(chunk); if (embedding && dedupStatus === "active") { - this.store.upsertEmbedding(chunkId, embedding); + this.store.upsertEmbedding(chunkId, embedding, { + provider: this.embedder.provider, + model: this.embedder.model, + }); } this.ctx.log.debug(`Stored chunk=${chunkId} kind=${kind} role=${msg.role} dedup=${dedupStatus} len=${content.length} hasVec=${!!embedding && dedupStatus === "active"}`); diff --git a/apps/memos-local-openclaw/src/storage/sqlite.ts b/apps/memos-local-openclaw/src/storage/sqlite.ts index c4f658ec3..642fafeff 100644 --- a/apps/memos-local-openclaw/src/storage/sqlite.ts +++ b/apps/memos-local-openclaw/src/storage/sqlite.ts @@ -120,9 +120,43 @@ export class SqliteStore { this.migrateHubUserIdentityFields(); this.migrateClientHubConnectionIdentityFields(); this.migrateTeamSharingInstanceId(); + this.migrateEmbeddingProducerColumns(); this.log.debug("Database schema initialized"); } + /** + * Tag every cached embedding row with the producer that created it. + * Adds `provider TEXT NOT NULL DEFAULT ''` + `model TEXT NOT NULL DEFAULT ''` + * to every embedding-shaped table. Idempotent: skips tables that already + * have the columns. `dimensions` already exists on every target table. + */ + private migrateEmbeddingProducerColumns(): void { + const tables = [ + "embeddings", + "skill_embeddings", + "task_embeddings", + "hub_embeddings", + "hub_skill_embeddings", + "hub_memory_embeddings", + ]; + for (const table of tables) { + try { + const cols = this.db.prepare(`PRAGMA table_info(${table})`).all() as Array<{ name: string }>; + if (cols.length === 0) continue; // table didn't exist yet + if (!cols.some((c) => c.name === "provider")) { + this.db.exec(`ALTER TABLE ${table} ADD COLUMN provider TEXT NOT NULL DEFAULT ''`); + this.log.info(`Migrated: added provider column to ${table}`); + } + if (!cols.some((c) => c.name === "model")) { + this.db.exec(`ALTER TABLE ${table} ADD COLUMN model TEXT NOT NULL DEFAULT ''`); + this.log.info(`Migrated: added model column to ${table}`); + } + } catch (err) { + this.log.warn(`migrateEmbeddingProducerColumns(${table}) failed: ${err}`); + } + } + } + private migrateChunksIndexesForRecall(): void { this.db.exec("CREATE INDEX IF NOT EXISTS idx_chunks_dedup_created ON chunks(dedup_status, created_at DESC)"); } @@ -1177,12 +1211,12 @@ export class SqliteStore { ); } - upsertEmbedding(chunkId: string, vector: number[]): void { + upsertEmbedding(chunkId: string, vector: number[], producer?: { provider?: string; model?: string }): void { const buf = Buffer.from(new Float32Array(vector).buffer); this.db.prepare(` - INSERT OR REPLACE INTO embeddings (chunk_id, vector, dimensions, updated_at) - VALUES (?, ?, ?, ?) - `).run(chunkId, buf, vector.length, Date.now()); + INSERT OR REPLACE INTO embeddings (chunk_id, vector, dimensions, provider, model, updated_at) + VALUES (?, ?, ?, ?, ?, ?) + `).run(chunkId, buf, vector.length, producer?.provider ?? "", producer?.model ?? "", Date.now()); } deleteEmbedding(chunkId: string): void { @@ -1420,6 +1454,87 @@ export class SqliteStore { return Array.from(new Float32Array(row.vector.buffer, row.vector.byteOffset, row.dimensions)); } + // ─── Embedding model signature reporting (issue #1333) ─── + + /** + * Snapshot of cached vector rows compared against the live `current` + * embedder signature. `legacy` are rows that pre-date the producer + * tagging columns (provider = ''). `mismatched` are rows whose + * producer was tagged but differs from `current`. `missing` counts + * active chunks that have no embedding row at all. + */ + getEmbeddingStats(current: { provider: string; model: string; dimensions: number }): { + total: number; + matched: number; + mismatched: number; + legacy: number; + missing: number; + current: { provider: string; model: string; dimensions: number }; + byProducer: Array<{ provider: string; model: string; dimensions: number; count: number }>; + } { + const total = (this.db.prepare("SELECT COUNT(*) as c FROM embeddings").get() as { c: number }).c; + const matched = (this.db.prepare( + "SELECT COUNT(*) as c FROM embeddings WHERE provider = ? AND model = ? AND dimensions = ?", + ).get(current.provider, current.model, current.dimensions) as { c: number }).c; + const legacy = (this.db.prepare( + "SELECT COUNT(*) as c FROM embeddings WHERE provider = ''", + ).get() as { c: number }).c; + const mismatched = (this.db.prepare( + "SELECT COUNT(*) as c FROM embeddings WHERE provider != '' AND NOT (provider = ? AND model = ? AND dimensions = ?)", + ).get(current.provider, current.model, current.dimensions) as { c: number }).c; + const missing = (this.db.prepare(` + SELECT COUNT(*) as c FROM chunks c + WHERE c.dedup_status = 'active' + AND NOT EXISTS (SELECT 1 FROM embeddings e WHERE e.chunk_id = c.id) + `).get() as { c: number }).c; + + const byProducer = (this.db.prepare( + "SELECT provider, model, dimensions, COUNT(*) as count FROM embeddings GROUP BY provider, model, dimensions ORDER BY count DESC", + ).all() as Array<{ provider: string; model: string; dimensions: number; count: number }>); + + return { total, matched, mismatched, legacy, missing, current, byProducer }; + } + + /** + * Chunk ids that should be re-embedded under `current`. By default returns + * every chunk whose embedding row's producer doesn't match (including legacy + * empty rows) plus chunks with no embedding row at all. With + * `missingOnly: true`, returns only the latter. Ordered by `created_at ASC` + * so re-embed runs are deterministic and resumable. + */ + listChunkIdsForReembed( + current: { provider: string; model: string; dimensions: number }, + opts: { missingOnly?: boolean; limit?: number } = {}, + ): string[] { + const limit = opts.limit ?? Number.MAX_SAFE_INTEGER; + if (opts.missingOnly) { + const rows = this.db.prepare(` + SELECT c.id as id FROM chunks c + WHERE c.dedup_status = 'active' + AND NOT EXISTS (SELECT 1 FROM embeddings e WHERE e.chunk_id = c.id) + ORDER BY c.created_at ASC + LIMIT ? + `).all(limit) as Array<{ id: string }>; + return rows.map((r) => r.id); + } + const rows = this.db.prepare(` + SELECT c.id as id FROM chunks c + WHERE c.dedup_status = 'active' + AND ( + NOT EXISTS (SELECT 1 FROM embeddings e WHERE e.chunk_id = c.id) + OR EXISTS ( + SELECT 1 FROM embeddings e + WHERE e.chunk_id = c.id + AND NOT (e.provider = ? AND e.model = ? AND e.dimensions = ?) + ) + ) + ORDER BY c.created_at ASC + LIMIT ? + `).all(current.provider, current.model, current.dimensions, limit) as Array<{ id: string }>; + return rows.map((r) => r.id); + } + + // ─── Update ─── updateChunk(chunkId: string, fields: { summary?: string; content?: string; role?: string; kind?: string; owner?: string }): boolean { @@ -1828,12 +1943,12 @@ export class SqliteStore { .run(visibility, Date.now(), skillId); } - upsertSkillEmbedding(skillId: string, vector: number[]): void { + upsertSkillEmbedding(skillId: string, vector: number[], producer?: { provider?: string; model?: string }): void { const buf = Buffer.from(new Float32Array(vector).buffer); this.db.prepare(` - INSERT OR REPLACE INTO skill_embeddings (skill_id, vector, dimensions, updated_at) - VALUES (?, ?, ?, ?) - `).run(skillId, buf, vector.length, Date.now()); + INSERT OR REPLACE INTO skill_embeddings (skill_id, vector, dimensions, provider, model, updated_at) + VALUES (?, ?, ?, ?, ?, ?) + `).run(skillId, buf, vector.length, producer?.provider ?? "", producer?.model ?? "", Date.now()); } getSkillEmbedding(skillId: string): number[] | null { @@ -1908,12 +2023,12 @@ export class SqliteStore { // ─── Task Embeddings & Search ─── - upsertTaskEmbedding(taskId: string, vector: number[]): void { + upsertTaskEmbedding(taskId: string, vector: number[], producer?: { provider?: string; model?: string }): void { const buf = Buffer.from(new Float32Array(vector).buffer); this.db.prepare(` - INSERT OR REPLACE INTO task_embeddings (task_id, vector, dimensions, updated_at) - VALUES (?, ?, ?, ?) - `).run(taskId, buf, vector.length, Date.now()); + INSERT OR REPLACE INTO task_embeddings (task_id, vector, dimensions, provider, model, updated_at) + VALUES (?, ?, ?, ?, ?, ?) + `).run(taskId, buf, vector.length, producer?.provider ?? "", producer?.model ?? "", Date.now()); } getTaskEmbeddings(owner?: string): Array<{ taskId: string; vector: number[] }> { @@ -2365,19 +2480,21 @@ export class SqliteStore { return row ? rowToHubSkill(row) : null; } - upsertHubSkillEmbedding(skillId: string, vector: number[], sourceUserId: string, sourceSkillId: string): void { + upsertHubSkillEmbedding(skillId: string, vector: number[], sourceUserId: string, sourceSkillId: string, producer?: { provider?: string; model?: string }): void { if (!sourceUserId || !sourceSkillId) throw new Error("sourceUserId and sourceSkillId are required for hub skill embedding upserts"); const canonicalSkillId = this.resolveCanonicalHubSkillId(skillId, sourceUserId, sourceSkillId); const buf = Buffer.allocUnsafe(vector.length * 4); for (let i = 0; i < vector.length; i++) buf.writeFloatLE(vector[i], i * 4); this.db.prepare(` - INSERT INTO hub_skill_embeddings (skill_id, vector, dimensions, updated_at) - VALUES (?, ?, ?, ?) + INSERT INTO hub_skill_embeddings (skill_id, vector, dimensions, provider, model, updated_at) + VALUES (?, ?, ?, ?, ?, ?) ON CONFLICT(skill_id) DO UPDATE SET vector = excluded.vector, dimensions = excluded.dimensions, + provider = excluded.provider, + model = excluded.model, updated_at = excluded.updated_at - `).run(canonicalSkillId, buf, vector.length, Date.now()); + `).run(canonicalSkillId, buf, vector.length, producer?.provider ?? "", producer?.model ?? "", Date.now()); } getHubSkillEmbedding(skillId: string): number[] | null { @@ -2400,13 +2517,13 @@ export class SqliteStore { })); } - upsertHubMemoryEmbedding(memoryId: string, vector: Float32Array): void { + upsertHubMemoryEmbedding(memoryId: string, vector: Float32Array, producer?: { provider?: string; model?: string }): void { const buf = Buffer.from(vector.buffer, vector.byteOffset, vector.byteLength); this.db.prepare(` - INSERT INTO hub_memory_embeddings (memory_id, vector, dimensions, updated_at) - VALUES (?, ?, ?, ?) - ON CONFLICT(memory_id) DO UPDATE SET vector = excluded.vector, dimensions = excluded.dimensions, updated_at = excluded.updated_at - `).run(memoryId, buf, vector.length, Date.now()); + INSERT INTO hub_memory_embeddings (memory_id, vector, dimensions, provider, model, updated_at) + VALUES (?, ?, ?, ?, ?, ?) + ON CONFLICT(memory_id) DO UPDATE SET vector = excluded.vector, dimensions = excluded.dimensions, provider = excluded.provider, model = excluded.model, updated_at = excluded.updated_at + `).run(memoryId, buf, vector.length, producer?.provider ?? "", producer?.model ?? "", Date.now()); } getHubMemoryEmbedding(memoryId: string): Float32Array | null { @@ -2452,13 +2569,13 @@ export class SqliteStore { return rows.map((row, idx) => ({ hit: row, rank: idx + 1 })); } - upsertHubEmbedding(chunkId: string, vector: Float32Array): void { + upsertHubEmbedding(chunkId: string, vector: Float32Array, producer?: { provider?: string; model?: string }): void { const buf = Buffer.from(vector.buffer, vector.byteOffset, vector.byteLength); this.db.prepare(` - INSERT INTO hub_embeddings (chunk_id, vector, dimensions, updated_at) - VALUES (?, ?, ?, ?) - ON CONFLICT(chunk_id) DO UPDATE SET vector = excluded.vector, dimensions = excluded.dimensions, updated_at = excluded.updated_at - `).run(chunkId, buf, vector.length, Date.now()); + INSERT INTO hub_embeddings (chunk_id, vector, dimensions, provider, model, updated_at) + VALUES (?, ?, ?, ?, ?, ?) + ON CONFLICT(chunk_id) DO UPDATE SET vector = excluded.vector, dimensions = excluded.dimensions, provider = excluded.provider, model = excluded.model, updated_at = excluded.updated_at + `).run(chunkId, buf, vector.length, producer?.provider ?? "", producer?.model ?? "", Date.now()); } getHubEmbedding(chunkId: string): Float32Array | null { diff --git a/apps/memos-local-openclaw/src/viewer/server.ts b/apps/memos-local-openclaw/src/viewer/server.ts index 85541d567..e3474cc70 100644 --- a/apps/memos-local-openclaw/src/viewer/server.ts +++ b/apps/memos-local-openclaw/src/viewer/server.ts @@ -1278,8 +1278,9 @@ export class ViewerServer { private embedTaskInBackground(taskId: string, text: string): void { if (!this.embedder || !text.trim()) return; - this.embedder.embed([text]).then((vecs: number[][]) => { - if (vecs.length > 0) this.store.upsertTaskEmbedding(taskId, vecs[0]); + const embedder = this.embedder; + embedder.embed([text]).then((vecs: number[][]) => { + if (vecs.length > 0) this.store.upsertTaskEmbedding(taskId, vecs[0], { provider: embedder.provider, model: embedder.model }); }).catch(() => {}); } @@ -1419,8 +1420,9 @@ export class ViewerServer { const sv = this.store.getLatestSkillVersion(skillId); if (sv) { const text = `${skill.name}: ${skill.description}`; - this.embedder.embed([text]).then((vecs: number[][]) => { - if (vecs.length > 0) this.store.upsertSkillEmbedding(skillId, vecs[0]); + const embedder = this.embedder; + embedder.embed([text]).then((vecs: number[][]) => { + if (vecs.length > 0) this.store.upsertSkillEmbedding(skillId, vecs[0], { provider: embedder.provider, model: embedder.model }); }).catch(() => {}); } } @@ -4379,7 +4381,7 @@ export class ViewerServer { this.store.updateChunkSummaryAndContent(targetId, dedupResult.mergedSummary, row.text); try { const [newEmb] = await this.embedder.embed([dedupResult.mergedSummary]); - if (newEmb) this.store.upsertEmbedding(targetId, newEmb); + if (newEmb) this.store.upsertEmbedding(targetId, newEmb, { provider: this.embedder.provider, model: this.embedder.model }); } catch { /* best-effort */ } dedupStatus = "merged"; dedupTarget = targetId; @@ -4420,7 +4422,7 @@ export class ViewerServer { this.store.insertChunk(chunk); if (embedding && dedupStatus === "active") { - this.store.upsertEmbedding(chunkId, embedding); + this.store.upsertEmbedding(chunkId, embedding, { provider: this.embedder.provider, model: this.embedder.model }); } totalStored++; @@ -4612,7 +4614,7 @@ export class ViewerServer { const targetId = candidates[dedupResult.targetIndex - 1]?.chunkId; if (targetId) { this.store.updateChunkSummaryAndContent(targetId, dedupResult.mergedSummary, content); - try { const [newEmb] = await this.embedder.embed([dedupResult.mergedSummary]); if (newEmb) this.store.upsertEmbedding(targetId, newEmb); } catch { /* best-effort */ } + try { const [newEmb] = await this.embedder.embed([dedupResult.mergedSummary]); if (newEmb) this.store.upsertEmbedding(targetId, newEmb, { provider: this.embedder.provider, model: this.embedder.model }); } catch { /* best-effort */ } dedupStatus = "merged"; dedupTarget = targetId; dedupReason = dedupResult.reason; } } @@ -4635,7 +4637,7 @@ export class ViewerServer { }; this.store.insertChunk(chunk); - if (embedding && dedupStatus === "active") this.store.upsertEmbedding(chunkId, embedding); + if (embedding && dedupStatus === "active") this.store.upsertEmbedding(chunkId, embedding, { provider: this.embedder.provider, model: this.embedder.model }); totalStored++; send("item", { index: idx, total: totalMsgs, status: dedupStatus === "active" ? "stored" : dedupStatus, preview: content.slice(0, 120), summary: summary.slice(0, 80), source: file, agent: agentId, role: msgRole, stepFailures }); diff --git a/apps/memos-local-openclaw/tests/embedding-reembed.test.ts b/apps/memos-local-openclaw/tests/embedding-reembed.test.ts new file mode 100644 index 000000000..6d98e0b06 --- /dev/null +++ b/apps/memos-local-openclaw/tests/embedding-reembed.test.ts @@ -0,0 +1,419 @@ +import { describe, it, expect, beforeEach, afterEach } from "vitest"; +import * as fs from "fs"; +import * as path from "path"; +import * as os from "os"; +import Database from "better-sqlite3"; +import { SqliteStore } from "../src/storage/sqlite"; +import { Embedder } from "../src/embedding"; +import { initPlugin } from "../src/index"; +import { parseArgs, runReembed } from "../scripts/re-embed"; +import type { Chunk, Logger, EmbeddingConfig } from "../src/types"; + +const noopLog: Logger = { + debug: () => {}, + info: () => {}, + warn: () => {}, + error: () => {}, +}; + +let store: SqliteStore; +let tmpDir: string; +let dbPath: string; + +beforeEach(() => { + tmpDir = fs.mkdtempSync(path.join(os.tmpdir(), "memos-reembed-test-")); + dbPath = path.join(tmpDir, "test.db"); + store = new SqliteStore(dbPath, noopLog); +}); + +afterEach(() => { + store.close(); + fs.rmSync(tmpDir, { recursive: true, force: true }); +}); + +function makeChunk(overrides: Partial = {}): Chunk { + return { + id: overrides.id ?? "chunk-1", + sessionKey: overrides.sessionKey ?? "session-1", + turnId: "turn-1", + seq: 0, + role: "user", + content: "Hello world", + kind: "paragraph", + summary: "Greeting", + embedding: null, + taskId: null, + skillId: null, + owner: "agent:main", + dedupStatus: "active", + dedupTarget: null, + dedupReason: null, + mergeCount: 0, + lastHitAt: null, + mergeHistory: "[]", + createdAt: Date.now(), + updatedAt: Date.now(), + ...overrides, + }; +} + +function rawEmbeddingRow(p: string, chunkId: string): { + provider: string; + model: string; + dimensions: number; +} | undefined { + const db = new Database(p, { readonly: true }); + try { + return db + .prepare("SELECT provider, model, dimensions FROM embeddings WHERE chunk_id = ?") + .get(chunkId) as + | { provider: string; model: string; dimensions: number } + | undefined; + } finally { + db.close(); + } +} + +describe("Embedding producer columns (TC-1, TC-2, TC-3)", () => { + it("migration adds provider + model columns with NOT NULL DEFAULT ''", () => { + const dbRO = new Database(dbPath, { readonly: true }); + try { + const cols = dbRO.prepare("PRAGMA table_info(embeddings)").all() as Array<{ + name: string; + type: string; + notnull: number; + dflt_value: string | null; + }>; + const provider = cols.find((c) => c.name === "provider"); + const model = cols.find((c) => c.name === "model"); + expect(provider, "embeddings.provider column should exist").toBeDefined(); + expect(model, "embeddings.model column should exist").toBeDefined(); + expect(provider!.notnull).toBe(1); + expect(model!.notnull).toBe(1); + expect(provider!.dflt_value).toMatch(/''|""/); + expect(model!.dflt_value).toMatch(/''|""/); + } finally { + dbRO.close(); + } + }); + + it("upsertEmbedding without producer (back-compat) stores empty strings", () => { + store.insertChunk(makeChunk({ id: "c1" })); + store.upsertEmbedding("c1", [0.1, 0.2, 0.3]); + + const row = rawEmbeddingRow(dbPath, "c1"); + expect(row).toBeDefined(); + expect(row!.provider).toBe(""); + expect(row!.model).toBe(""); + expect(row!.dimensions).toBe(3); + + const v = store.getEmbedding("c1"); + expect(v).not.toBeNull(); + expect(v!).toHaveLength(3); + expect(v![0]).toBeCloseTo(0.1, 5); + }); + + it("upsertEmbedding with producer persists provider + model", () => { + store.insertChunk(makeChunk({ id: "c1" })); + store.upsertEmbedding("c1", [0.1, 0.2, 0.3], { + provider: "openai", + model: "text-embedding-3-small", + }); + + const row = rawEmbeddingRow(dbPath, "c1"); + expect(row).toBeDefined(); + expect(row!.provider).toBe("openai"); + expect(row!.model).toBe("text-embedding-3-small"); + expect(row!.dimensions).toBe(3); + + // overwrite with a different producer + store.upsertEmbedding("c1", [0.4, 0.5, 0.6], { + provider: "openai", + model: "text-embedding-3-large", + }); + const row2 = rawEmbeddingRow(dbPath, "c1"); + expect(row2!.model).toBe("text-embedding-3-large"); + }); +}); + +describe("getEmbeddingStats + listChunkIdsForReembed (TC-4, TC-5, TC-6)", () => { + beforeEach(() => { + // 6 chunks total. 5 with embeddings, 1 without. + const now = Date.now(); + store.insertChunk(makeChunk({ id: "c1", createdAt: now + 1 })); + store.insertChunk(makeChunk({ id: "c2", createdAt: now + 2 })); + store.insertChunk(makeChunk({ id: "c3", createdAt: now + 3 })); + store.insertChunk(makeChunk({ id: "c4", createdAt: now + 4 })); + store.insertChunk(makeChunk({ id: "c5", createdAt: now + 5 })); + store.insertChunk(makeChunk({ id: "c6", createdAt: now + 6 })); // no embedding + + // matched (×2): openai/text-embedding-3-small/1536 + store.upsertEmbedding("c1", Array(1536).fill(0.1), { + provider: "openai", + model: "text-embedding-3-small", + }); + store.upsertEmbedding("c2", Array(1536).fill(0.2), { + provider: "openai", + model: "text-embedding-3-small", + }); + // mismatched: different model + store.upsertEmbedding("c3", Array(3072).fill(0.3), { + provider: "openai", + model: "text-embedding-3-large", + }); + // mismatched: different provider + store.upsertEmbedding("c4", Array(384).fill(0.4), { + provider: "local", + model: "", + }); + // legacy: no producer info + store.upsertEmbedding("c5", Array(1536).fill(0.5)); + }); + + it("getEmbeddingStats reports matched/mismatched/legacy/missing", () => { + const stats = store.getEmbeddingStats({ + provider: "openai", + model: "text-embedding-3-small", + dimensions: 1536, + }); + + expect(stats.total).toBe(5); + expect(stats.matched).toBe(2); + // c3 + c4 are explicit mismatch (both have non-empty provider != current) + expect(stats.mismatched).toBe(2); + expect(stats.legacy).toBe(1); + expect(stats.missing).toBe(1); // c6 has chunk but no embedding row + + expect(stats.current.provider).toBe("openai"); + expect(stats.current.model).toBe("text-embedding-3-small"); + expect(stats.current.dimensions).toBe(1536); + + // byProducer has all three buckets + const producers = stats.byProducer.map((b) => `${b.provider}|${b.model}|${b.dimensions}`).sort(); + expect(producers).toEqual([ + "||1536", + "local||384", + "openai|text-embedding-3-large|3072", + "openai|text-embedding-3-small|1536", + ].sort()); + }); + + it("listChunkIdsForReembed default returns mismatched + legacy + missing", () => { + const ids = store.listChunkIdsForReembed({ + provider: "openai", + model: "text-embedding-3-small", + dimensions: 1536, + }); + expect(new Set(ids)).toEqual(new Set(["c3", "c4", "c5", "c6"])); + }); + + it("listChunkIdsForReembed missingOnly returns only chunks without embedding row", () => { + const ids = store.listChunkIdsForReembed( + { + provider: "openai", + model: "text-embedding-3-small", + dimensions: 1536, + }, + { missingOnly: true }, + ); + expect(ids).toEqual(["c6"]); + }); + + it("listChunkIdsForReembed respects limit", () => { + const ids = store.listChunkIdsForReembed( + { + provider: "openai", + model: "text-embedding-3-small", + dimensions: 1536, + }, + { limit: 2 }, + ); + expect(ids.length).toBe(2); + }); +}); + +describe("Embedder.signature (TC-7)", () => { + it("returns provider:model:dimensions for an explicit config", () => { + const cfg: EmbeddingConfig = { + provider: "openai", + model: "text-embedding-3-small", + dimensions: 1536, + }; + const e = new Embedder(cfg, noopLog); + expect(e.signature).toBe("openai:text-embedding-3-small:1536"); + expect(e.model).toBe("text-embedding-3-small"); + }); + + it("falls back to local::384 when config is undefined", () => { + const e = new Embedder(undefined, noopLog); + expect(e.signature).toBe("local::384"); + expect(e.provider).toBe("local"); + expect(e.model).toBe(""); + expect(e.dimensions).toBe(384); + }); + + it("treats openclaw provider without hostEmbedding as local", () => { + const cfg: EmbeddingConfig = { + provider: "openclaw", + model: "any", + dimensions: 1536, + capabilities: { hostEmbedding: false }, + }; + const e = new Embedder(cfg, noopLog); + // provider downshifts to "local", model still surfaces; signature is canonical for current Embedder identity + expect(e.provider).toBe("local"); + expect(e.signature.startsWith("local:")).toBe(true); + }); +}); + +describe("Init-time mismatch warning (TC-9)", () => { + it("emits a warn line when legacy or mismatched rows are present", () => { + // 1. Pre-create the DB with one legacy embedding row + const initStore = new SqliteStore(dbPath, noopLog); + initStore.insertChunk(makeChunk({ id: "c-legacy" })); + initStore.upsertEmbedding("c-legacy", Array(1536).fill(0.1)); + initStore.close(); + + // 2. Build a log that captures warn calls + const warns: string[] = []; + const captureLog: Logger = { + debug: () => {}, + info: () => {}, + warn: (msg) => warns.push(String(msg)), + error: () => {}, + }; + + // 3. Spin up initPlugin pointing at the same DB with a fresh config + const stateDir = path.join(tmpDir, "state"); + fs.mkdirSync(stateDir, { recursive: true }); + const plugin = initPlugin({ + stateDir, + workspaceDir: tmpDir, + config: { + storage: { dbPath }, + embedding: { + provider: "openai", + model: "text-embedding-3-small", + dimensions: 1536, + }, + }, + log: captureLog, + }); + + // 4. Expect at least one warn mentioning legacy / mismatch and the script path + expect(warns.some((w) => /embedding model mismatch/i.test(w))).toBe(true); + expect(warns.some((w) => /scripts\/re-embed\.ts/.test(w))).toBe(true); + + // shutdown is async; tests don't need to await on it + void plugin.shutdown(); + }); + + it("stays quiet when the DB has no embeddings at all", () => { + const warns: string[] = []; + const captureLog: Logger = { + debug: () => {}, + info: () => {}, + warn: (msg) => warns.push(String(msg)), + error: () => {}, + }; + const stateDir = path.join(tmpDir, "state2"); + fs.mkdirSync(stateDir, { recursive: true }); + const plugin = initPlugin({ + stateDir, + workspaceDir: tmpDir, + config: { + storage: { dbPath: path.join(stateDir, "empty.db") }, + embedding: { provider: "openai", model: "x", dimensions: 1536 }, + }, + log: captureLog, + }); + + expect(warns.some((w) => /embedding model mismatch/i.test(w))).toBe(false); + void plugin.shutdown(); + }); +}); + +describe("re-embed CLI", () => { + it("parseArgs supports the documented flags", () => { + const opts = parseArgs([ + "--missing-only", + "--dry-run", + "--limit", + "5", + "--batch-size", + "2", + "--db", + "/tmp/foo.db", + "--config", + "/tmp/openclaw.json", + ]); + expect(opts.missingOnly).toBe(true); + expect(opts.dryRun).toBe(true); + expect(opts.limit).toBe(5); + expect(opts.batchSize).toBe(2); + expect(opts.dbPath).toBe("/tmp/foo.db"); + expect(opts.configPath).toBe("/tmp/openclaw.json"); + }); + + it("parseArgs rejects unknown args", () => { + expect(() => parseArgs(["--what"])).toThrow(/Unknown argument/); + }); + + it("dry-run reports planned count and writes nothing", async () => { + // Build a DB with: 1 matched, 1 mismatched, 1 missing. + const setup = new SqliteStore(dbPath, noopLog); + setup.insertChunk(makeChunk({ id: "c1" })); + setup.insertChunk(makeChunk({ id: "c2" })); + setup.insertChunk(makeChunk({ id: "c3" })); // no embedding + setup.upsertEmbedding("c1", Array(8).fill(0.1), { + provider: "local", + model: "", + }); + setup.upsertEmbedding("c2", Array(8).fill(0.2), { + provider: "openai", + model: "old-model", + }); + setup.close(); + + // Write a minimal openclaw.json the script can read + const stateDir = path.join(tmpDir, "state"); + fs.mkdirSync(stateDir, { recursive: true }); + const configPath = path.join(tmpDir, "openclaw.json"); + fs.writeFileSync( + configPath, + JSON.stringify({ + plugins: { + entries: { + "memos-local": { + config: { + storage: { dbPath }, + // No `embedding` block → local fallback (dim 384) + }, + }, + }, + }, + }), + ); + + const result = await runReembed({ + configPath, + dbPath, + missingOnly: false, + dryRun: true, + batchSize: 32, + help: false, + }); + // planned = c1 mismatched (local|||8 vs local||384) + c2 mismatched + c3 missing = 3 + expect(result.planned).toBe(3); + expect(result.processed).toBe(0); + + // Verify the DB embeddings table is unchanged + const checkDb = new Database(dbPath, { readonly: true }); + const rows = checkDb.prepare("SELECT chunk_id, provider, model FROM embeddings ORDER BY chunk_id").all(); + checkDb.close(); + expect(rows).toEqual([ + { chunk_id: "c1", provider: "local", model: "" }, + { chunk_id: "c2", provider: "openai", model: "old-model" }, + ]); + }); +});