From 3a944f6e0966f215746ac1f78dffdd9fe89f75de Mon Sep 17 00:00:00 2001 From: Levy Tate <78818969+iLevyTate@users.noreply.github.com> Date: Fri, 6 Nov 2026 22:39:28 -0500 Subject: [PATCH] fix(vector-db): save only what restore reads, and make the size limit a reported error A collection is saved as one JSON string, and V8 cannot build a string longer than buffer.constants.MAX_STRING_LENGTH (~537 M chars). On a large library the passage index reached ~461 M chars. Past the limit every save threw RangeError, was retried as a transient failure, the retry counter reset in `finally`, and each later write re-ran a multi-second stringify that had to fail, while new passages were lost on restart. - Save only `docs.docs`. Restore has read nothing else since v2.0.0 and rebuilds every index by re-inserting documents. Orama's full export also carried the vector index (a second copy of every embedding) and the text and sort indexes: more than half of each file. Old full-export files still restore, and older builds read the new ones. - A collection whose JSON is too long to build raises `orama.too-large:`, the other collections still save, and it is not stringified again until its document count drops below where it failed. A warning is logged once per launch past 85% of the limit. - A file that decompresses past the limit is no longer quarantined as corrupt. The collection starts empty for the session and that stand-in is never saved, blob or sidecar, so the unread file is not overwritten. A reset or rebuild replaces the stand-in and saving resumes. - `finally` cleared no timer before re-arming, so a write during a save left two timers and a second save started right after the first. - Startup sweeps `*.tmp*` files over an hour old from the vector-db folder (the existing userData sweep is top-level only); interrupted saves left hundreds of MB there. --- CHANGELOG.md | 17 ++ src/main/services/OramaVectorService.js | 202 ++++++++++++- test/OramaVectorService.saveSafety.test.js | 314 +++++++++++++++++++++ 3 files changed, 525 insertions(+), 8 deletions(-) create mode 100644 test/OramaVectorService.saveSafety.test.js diff --git a/CHANGELOG.md b/CHANGELOG.md index b4e1b19..b40ddd0 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -126,6 +126,11 @@ Changes you will notice: - Search index saves write only the collections that changed, refresh the `.bak` copy at most every 30 minutes, and are spaced by how long a save takes (5 s to 2 min). On a large library one save had been about 785 MB. + - Each search index file now holds only the documents, which is all a restart reads back. The + index structures Orama rebuilds on load, including a second copy of every embedding, were more + than half of each file. Files written by earlier versions still load, and earlier versions can + read the new files. + - A write that landed during a save no longer starts a second save right after the first one. - Analysis history saves wait for 2 s of quiet and are at least 10 s apart, holding a change for at most about 10 s; quitting writes at once. A crash can lose at most those last seconds of entries. @@ -167,6 +172,18 @@ Changes you will notice: ### Fixed +- **A search index too large to save is reported instead of silently not saving.** A collection is + saved as one block of text, and V8 cannot build one longer than about 537 million characters. Past + that, every save failed, was retried as if the disk had hiccuped, and new passages were lost at + each restart. Now the app shows an error naming the index, saves everything else, and does not + retry that index until files are removed from it. It warns in the log once a save passes 85% of + the limit. +- **A search index file too large to read is no longer renamed as corrupt.** It is left on disk as + it is and never saved over, so no data is lost, and an error says search results are incomplete. + Resetting that index clears the error. +- **Half-written search index files are cleaned up.** A save interrupted by a crash or a full disk + left its temporary file behind (hundreds of MB each on a large library). Startup now removes those + older than an hour from the search index folder. - **Startup repair re-embeds missing vectors under default settings.** The file-vector backfill asked whether embedding was allowed after organizing, which is refused under the default "during analysis" timing, so missing file vectors were never rebuilt and the file was marked skipped. diff --git a/src/main/services/OramaVectorService.js b/src/main/services/OramaVectorService.js index 50b7de8..9ab9528 100644 --- a/src/main/services/OramaVectorService.js +++ b/src/main/services/OramaVectorService.js @@ -59,6 +59,7 @@ const { createSingletonHelpers } = require('../../shared/singletonFactory'); const { AI_DEFAULTS, IPC_CHANNELS } = require('../../shared/constants'); const { ERROR_CODES } = require('../../shared/errorCodes'); const { isDiskFullError } = require('../../shared/errorClassifier'); +const { sweepOrphanUserDataTmpFiles } = require('./userDataTmpSweeper'); const { attachErrorCode } = require('../../shared/errorHandlingUtils'); const { replaceFileWithRetry } = require('../../shared/atomicFile'); const { tryParse: safeJsonTryParse } = require('../../shared/safeJsonOps'); @@ -132,6 +133,46 @@ const STORAGE_FULL_RETRY_MS = 60_000; const PERSIST_COMPRESSION_ENABLED = String(process.env.STRATOSORT_ORAMA_COMPRESS || 'true').toLowerCase() !== 'false'; +// The longest string V8 can build. A collection's saved JSON is one string, +// so past this the save throws instead of writing. Warn well before it. +const MAX_SAVED_JSON_CHARS = require('buffer').constants.MAX_STRING_LENGTH; +const SAVED_JSON_WARN_RATIO = 0.85; + +/** + * Whether an error is V8 refusing to build a string that long: a save whose + * JSON outgrew MAX_SAVED_JSON_CHARS, or a restore whose file decompresses + * past it. Neither means the data on disk is damaged. + * @param {any} error + * @returns {boolean} + */ +function isStringTooLongError(error) { + if (!error) return false; + if (error.code === 'ERR_STRING_TOO_LONG') return true; + return error instanceof RangeError && /invalid string length/i.test(String(error.message)); +} + +/** + * The saved form of a collection: its documents and nothing else. + * + * Restore reads only `docs.docs` and rebuilds every index by inserting the + * documents again (_createOrRestoreDatabase; every release since v2.0.0 does + * the same, so older builds still read this). Orama's full export also holds + * the vector index, a second copy of every embedding, plus the text and sort + * indexes: more than half of each save, written and parsed every time and + * never read back. + * + * @param {any} db + * @returns {Promise} + */ +async function serializeCollectionForDisk(db) { + const docsStore = db?.data?.docs; + if (!docsStore || typeof docsStore.docs !== 'object' || docsStore.docs === null) { + // Not a full Orama instance: keep the full export rather than guess. + return /** @type {any} */ (await persist(db, 'json')); + } + return JSON.stringify({ docs: { docs: docsStore.docs, count: docsStore.count } }); +} + const writeFileAtomic = async (/** @type {any} */ filePath, /** @type {any} */ data) => { const tempPath = `${filePath}.tmp.${require('crypto').randomUUID()}`; try { @@ -298,6 +339,24 @@ class OramaVectorService extends EventEmitter { this._savedSidecarSizes = new Map(); /** @type {Map} when each collection's .bak was last refreshed */ this._bakRotatedAt = new Map(); + /** + * Collections whose saved JSON outgrew MAX_SAVED_JSON_CHARS, with their + * document count at the time. Their save is skipped until the count drops + * below it, instead of re-running a multi-second stringify that must fail. + * @type {Map} + */ + this._tooLargeToSave = new Map(); + /** + * Collections whose file was too large to read at startup, with the + * empty collection that stood in for them. While that stand-in is still + * the live collection, neither its blob nor its sidecar is saved: either + * would overwrite the unread data on disk. A reset or rebuild replaces + * it, and saving resumes. + * @type {Map} + */ + this._tooLargeToRead = new Map(); + /** @type {Set} collections already warned about nearing the limit */ + this._nearSizeLimitWarned = new Set(); this._lastFullPersistAt = 0; this._forceFullPersist = false; this._lastPersistDurationMs = 0; @@ -1394,6 +1453,16 @@ class OramaVectorService extends EventEmitter { this._dataPath = await this._resolveVectorDataPath(); await fs.mkdir(this._dataPath, { recursive: true }); + // Saves interrupted by a crash or a full disk leave hundreds of MB of + // half-written temp files here. Nothing in this process has written + // one yet, and the sweep only takes files over an hour old. + await sweepOrphanUserDataTmpFiles({ userDataDir: this._dataPath, logger }).catch( + (/** @type {any} */ error) => { + logger.debug('[OramaVectorService] Temp file sweep failed', { error: error?.message }); + } + ); + this._tooLargeToSave.clear(); + this._tooLargeToRead.clear(); await this._refreshEmbeddingDimension({ reason: 'initialize' }); @@ -1673,13 +1742,18 @@ class OramaVectorService extends EventEmitter { let data; let usedCompressed = false; let dataSource = null; // tracks where `data` came from for accurate quarantine + // A file that decompresses past the longest string V8 can build is not + // damaged; it is only too large for this reader. Never quarantine it. + let tooLargeToRead = false; try { const compressed = await fs.readFile(compressedPath); data = (await uncompress(compressed)).toString('utf8'); usedCompressed = true; dataSource = compressedPath; } catch (error) { - if (error?.code !== 'ENOENT') { + if (isStringTooLongError(error)) { + tooLargeToRead = true; + } else if (error?.code !== 'ENOENT') { // LZ4 file exists but is corrupt — back it up so it can be inspected, // then fall through to the uncompressed JSON fallback. logger.error('[OramaVectorService] Compressed persistence corrupt, backing up', { @@ -1702,7 +1776,9 @@ class OramaVectorService extends EventEmitter { data = await fs.readFile(persistPath, 'utf-8'); dataSource = persistPath; } catch (jsonError) { - if (jsonError?.code !== 'ENOENT') { + if (isStringTooLongError(jsonError)) { + tooLargeToRead = true; + } else if (jsonError?.code !== 'ENOENT') { logger.warn('[OramaVectorService] Uncompressed primary read failed', { name, error: jsonError.message @@ -1729,7 +1805,9 @@ class OramaVectorService extends EventEmitter { `[OramaVectorService] Restored ${name} from .bak rolling backup; primary was unreadable` ); } catch (bakError) { - if (bakError?.code !== 'ENOENT') { + if (isStringTooLongError(bakError)) { + tooLargeToRead = true; + } else if (bakError?.code !== 'ENOENT') { logger.debug(`[OramaVectorService] .bak fallback failed for ${name}`, { error: bakError.message }); @@ -1741,6 +1819,17 @@ class OramaVectorService extends EventEmitter { // This is the expected state on a fresh install — return a clean DB // without logging error noise. if (!data) { + if (tooLargeToRead) { + // Start empty for this session, but never save it: an empty + // collection written over the unread file would lose it for good. + const standIn = await create({ + schema, + components: { tokenizer: ORAMA_TOKENIZER_CONFIG } + }); + this._tooLargeToRead.set(name, standIn); + this._raiseCollectionTooLarge(name, 'read'); + return standIn; + } return await create({ schema, components: { tokenizer: ORAMA_TOKENIZER_CONFIG } }); } @@ -2149,8 +2238,8 @@ class OramaVectorService extends EventEmitter { // Handle compression format migration if (!PERSIST_COMPRESSION_ENABLED && usedCompressed) { try { - const json = await persist(/** @type {any} */ (db), 'json'); - await writeFileAtomic(persistPath, /** @type {any} */ (json)); + const json = await serializeCollectionForDisk(db); + await writeFileAtomic(persistPath, json); await fs.rename(compressedPath, `${compressedPath}.legacy.${Date.now()}`); logger.info(`[OramaVectorService] Migrated ${name} persistence to JSON`); } catch (migrateError) { @@ -2265,6 +2354,80 @@ class OramaVectorService extends EventEmitter { } } + /** + * Tell the user a collection has outgrown what one save file can hold. + * Logged once per occurrence and raised as a degraded flag, so the size + * limit is a visible error rather than a save that silently stops. + * @private + * @param {string} name + * @param {'save' | 'read'} phase + * @param {Record} [detail] + */ + _raiseCollectionTooLarge(name, phase, detail = {}) { + logger.error( + phase === 'save' + ? `[OramaVectorService] ${name} is too large to save as one file; new changes to it are kept only until the app closes` + : `[OramaVectorService] ${name} is too large to read; it is left untouched on disk and not saved over`, + { collection: name, limitChars: MAX_SAVED_JSON_CHARS, ...detail } + ); + const tracker = getDegradedStateTrackerSafe(); + if (!tracker) return; + tracker.raise({ + code: `orama.too-large:${name}`, + subsystem: 'vector-db', + severity: 'error', + message: + phase === 'save' + ? `The search index "${name}" has grown past what can be saved. Changes to it are lost when the app closes. Removing files from the library frees space.` + : `The search index "${name}" was too large to load. It was left on disk unchanged, and search results will be incomplete.`, + detail: { collection: name, phase, limitChars: MAX_SAVED_JSON_CHARS, ...detail } + }); + } + + /** + * Whether a collection is still the empty stand-in for a file too large to + * read. Once a reset or rebuild replaces it, the block lifts. + * @private + * @param {string} name + * @returns {boolean} + */ + _isUnreadStandIn(name) { + const standIn = this._tooLargeToRead.get(name); + if (!standIn) return false; + if (standIn === this._databases[name]) return true; + this._tooLargeToRead.delete(name); + this._clearCollectionTooLarge(name); + return false; + } + + /** + * @private + * @param {string} name + */ + _clearCollectionTooLarge(name) { + logger.info(`[OramaVectorService] ${name} fits in one save file again; saving resumed`); + getDegradedStateTrackerSafe()?.clear?.(`orama.too-large:${name}`); + } + + /** + * Warn once per launch when a collection's saved JSON passes + * SAVED_JSON_WARN_RATIO of the limit, while there is still room to act. + * @private + * @param {string} name + * @param {number} chars + */ + _warnIfNearSizeLimit(name, chars) { + if (chars < MAX_SAVED_JSON_CHARS * SAVED_JSON_WARN_RATIO) return; + if (this._nearSizeLimitWarned.has(name)) return; + this._nearSizeLimitWarned.add(name); + logger.warn(`[OramaVectorService] ${name} is nearing the largest size one save file can hold`, { + collection: name, + chars, + limitChars: MAX_SAVED_JSON_CHARS, + percent: Math.round((chars / MAX_SAVED_JSON_CHARS) * 100) + }); + } + /** * One save attempt per STORAGE_FULL_RETRY_MS while the disk is full. The * retry counter is left one short of the limit, so each attempt that fails @@ -2349,7 +2512,7 @@ class OramaVectorService extends EventEmitter { !this._lastFullPersistAt || saveStartedAt - this._lastFullPersistAt >= FULL_PERSIST_INTERVAL_MS; const toSave = Object.keys(this._databases).filter( - (name) => fullSave || this._collectionNeedsSave(name) + (name) => !this._isUnreadStandIn(name) && (fullSave || this._collectionNeedsSave(name)) ); // Unmark before serializing: a write that lands while this save runs marks // its collection again, so the next save picks it up. @@ -2364,6 +2527,8 @@ class OramaVectorService extends EventEmitter { /** @type {any} */ let sidecarError = null; let blobsWritten = false; + /** @type {Set} collections whose blob was over the size limit this round */ + const oversized = new Set(); try { // Persist embedding sidecar maps before Orama JSON blobs so we always @@ -2414,7 +2579,24 @@ class OramaVectorService extends EventEmitter { const db = this._databases[name]; const persistPath = path.join(this._dataPath, `${name}.json`); const compressedPath = `${persistPath}.lz4`; - const data = /** @type {any} */ (await persist(db, 'json')); + const docCount = Number(db?.data?.docs?.count); + const failedAtCount = this._tooLargeToSave.get(name); + if (failedAtCount !== undefined && !(docCount < failedAtCount)) { + oversized.add(name); + continue; + } + let data; + try { + data = await serializeCollectionForDisk(db); + } catch (error) { + if (!isStringTooLongError(error)) throw error; + this._tooLargeToSave.set(name, Number.isFinite(docCount) ? docCount : 0); + this._raiseCollectionTooLarge(name, 'save', { documents: docCount }); + oversized.add(name); + continue; + } + if (this._tooLargeToSave.delete(name)) this._clearCollectionTooLarge(name); + this._warnIfNearSizeLimit(name, data.length); // Keep a single-generation `.bak` of the active persistence. Previously // the rotation `rename(active → active.bak)` happened BEFORE // writeFileAtomic, opening a window where neither the active file nor @@ -2516,7 +2698,8 @@ class OramaVectorService extends EventEmitter { { collections: toSave, full: fullSave, - durationMs: this._lastPersistDurationMs + durationMs: this._lastPersistDurationMs, + ...(oversized.size > 0 ? { tooLargeToSave: [...oversized] } : {}) } ); this._clearStorageFull(); @@ -2563,6 +2746,9 @@ class OramaVectorService extends EventEmitter { // fired during our run was dropped (guard on line 368). Reschedule so // those writes are not silently orphaned. if (this._persistPending && !this._isShuttingDown) { + // A write during the save armed its own debounce timer. Replace it, + // or both fire and the second save starts right after this one. + if (this._persistTimer) clearTimeout(this._persistTimer); this._persistTimer = setTimeout(() => { if (!this._isPersisting) { this._currentPersistPromise = this._doPersist(); diff --git a/test/OramaVectorService.saveSafety.test.js b/test/OramaVectorService.saveSafety.test.js new file mode 100644 index 0000000..d443b27 --- /dev/null +++ b/test/OramaVectorService.saveSafety.test.js @@ -0,0 +1,314 @@ +/** + * The vector store saves what restore reads, and its size limit is an error + * the user sees, never a save that silently stops. + * + * A collection is saved as one JSON string. On a real library the passage + * index reached 461 million characters, about 75 MB short of the longest + * string V8 can build (536,870,888). Past that, every save threw RangeError, + * the retry counter reset, and each later write re-ran a doomed multi-second + * stringify while new passages were lost on restart. Most of that string was + * Orama's index export, which restore never reads. + */ + +const path = require('path'); +const fsp = require('fs').promises; + +const mockTracker = { raise: jest.fn(), clear: jest.fn() }; +const TOO_LARGE_MARKER = 'stratosort-too-large-to-read'; + +jest.mock('electron', () => ({ app: { getPath: jest.fn() } })); +jest.mock('../src/shared/logger', () => { + const logger = { info: jest.fn(), warn: jest.fn(), error: jest.fn(), debug: jest.fn() }; + return { createLogger: () => logger, logger }; +}); +jest.mock('../src/shared/singletonFactory', () => ({ + createSingletonHelpers: () => ({ + getInstance: jest.fn(), + createInstance: jest.fn(), + registerWithContainer: jest.fn(), + resetInstance: jest.fn() + }) +})); +jest.mock('../src/main/services/DegradedStateTracker', () => ({ + getInstance: () => mockTracker, + raiseStorageFull: jest.fn(), + clearStorageFull: jest.fn(() => true), + STORAGE_FULL_SOURCES: { VECTOR_STORE: 'vector-store' } +})); +jest.mock('../src/shared/lz4Codec', () => { + const real = jest.requireActual('../src/shared/lz4Codec'); + return { + ...real, + // A file holding the marker stands in for one that decompresses past + // the longest string V8 can build: its toString throws like Node does. + uncompress: jest.fn(async (input) => { + const out = await real.uncompress(input); + if (!out.includes(TOO_LARGE_MARKER)) return out; + return { + toString() { + const error = new Error('Cannot create a string longer than 0x1fffffe8 characters'); + // @ts-ignore + error.code = 'ERR_STRING_TOO_LONG'; + throw error; + } + }; + }) + }; +}); +jest.mock('../src/main/llamaUtils', () => ({ + getEmbeddingModel: jest.fn(() => 'nomic-embed-text-v1.5-Q8_0.gguf'), + loadLlamaConfig: jest.fn().mockResolvedValue({ + selectedEmbeddingModel: 'nomic-embed-text-v1.5-Q8_0.gguf' + }) +})); + +const { app } = require('electron'); +const { logger } = require('../src/shared/logger'); +const { compress, uncompress } = jest.requireActual('../src/shared/lz4Codec'); +const { OramaVectorService } = require('../src/main/services/OramaVectorService'); + +const TEMP_DIR = path.join(__dirname, 'temp-orama-save-safety'); +const vector = (value = 0.1) => new Array(768).fill(value); + +describe('OramaVectorService save safety', () => { + /** @type {any} */ + let service; + + beforeEach(async () => { + await fsp.rm(TEMP_DIR, { recursive: true, force: true }).catch(() => {}); + await fsp.mkdir(TEMP_DIR, { recursive: true }); + app.getPath.mockReturnValue(TEMP_DIR); + jest.clearAllMocks(); + service = new OramaVectorService(); + await service.initialize(); + }); + + afterEach(async () => { + jest.restoreAllMocks(); + if (service?.shutdown) await service.shutdown(); + }); + + afterAll(async () => { + await fsp.rm(TEMP_DIR, { recursive: true, force: true }).catch(() => {}); + }); + + /** One save, after any save already running (the first write starts one). */ + async function save() { + while (service._currentPersistPromise || service._isPersisting) { + await service._currentPersistPromise; + service._currentPersistPromise = null; + if (service._isPersisting) await new Promise((resolve) => setTimeout(resolve, 10)); + } + service._persistPending = true; + await service._doPersist(); + } + + async function readSaved(name) { + const raw = await fsp.readFile(path.join(service._dataPath, `${name}.json.lz4`)); + return JSON.parse((await uncompress(raw)).toString('utf8')); + } + + /** Shut down (which saves), optionally change the files, then start again. */ + async function restart(beforeStart = async () => {}) { + await service.shutdown(); + await beforeStart(); + service = new OramaVectorService(); + await service.initialize(); + } + + const persistFailures = () => + logger.error.mock.calls.filter(([message]) => String(message).includes('Persistence failed')) + .length; + + describe('what a save writes', () => { + test('the saved file holds the documents and none of the index export', async () => { + await service.upsertFile({ id: 'file:a', vector: vector(), meta: { path: '/x/a.txt' } }); + await save(); + + const saved = await readSaved('files'); + expect(Object.keys(saved)).toEqual(['docs']); + const docs = Object.values(saved.docs.docs); + expect(docs).toHaveLength(1); + expect(docs[0]).toEqual(expect.objectContaining({ id: 'file:a' })); + }); + + test('a file saved that way comes back with its vector after a restart', async () => { + await service.upsertFile({ id: 'file:a', vector: vector(0.3), meta: { path: '/x/a.txt' } }); + await save(); + await restart(); + + const doc = await service.getFile('file:a'); + expect(doc).toEqual(expect.objectContaining({ id: 'file:a' })); + expect(Array.from(doc.embedding)).toHaveLength(768); + expect(doc.embedding[0]).toBeCloseTo(0.3); + }); + + test('a file in the old full-export format still restores', async () => { + // Written by an earlier release: Orama's whole export, index included. + const { save: oramaSave } = jest.requireActual('@orama/orama'); + await service.upsertFile({ id: 'file:old', vector: vector(), meta: { path: '/x/o.txt' } }); + // What the persistence plugin's persist(db, 'json') writes. + const fullExport = JSON.stringify(oramaSave(service._databases.files)); + expect(Object.keys(JSON.parse(fullExport))).toEqual( + expect.arrayContaining(['docs', 'index', 'sorting']) + ); + const dir = service._dataPath; + await restart(async () => { + await fsp.writeFile( + path.join(dir, 'files.json.lz4'), + await compress(Buffer.from(fullExport, 'utf8')) + ); + }); + + expect(await service.getFile('file:old')).toEqual( + expect.objectContaining({ id: 'file:old' }) + ); + }); + }); + + describe('a collection too large to save', () => { + /** Make stringify fail only for the files collection's saved JSON. */ + function failFilesStringify() { + const realStringify = JSON.stringify; + const calls = { files: 0 }; + jest.spyOn(JSON, 'stringify').mockImplementation(function (value, ...rest) { + const docs = value?.docs?.docs; + if (docs && Object.values(docs).some((doc) => doc?.id === 'file:a')) { + calls.files += 1; + throw new RangeError('Invalid string length'); + } + return realStringify.call(JSON, value, ...rest); + }); + return calls; + } + + test('is reported, the other collections still save, and it is not retried as a failure', async () => { + await save(); + await service.upsertFile({ id: 'file:a', vector: vector(), meta: { path: '/x/a.txt' } }); + await service.upsertFolder({ id: 'folder:a', vector: vector(), name: 'A', path: '/x' }); + const before = await fsp.readFile(path.join(service._dataPath, 'files.json.lz4')); + failFilesStringify(); + + await save(); + + expect(mockTracker.raise).toHaveBeenCalledWith( + expect.objectContaining({ code: 'orama.too-large:files', severity: 'error' }) + ); + expect(persistFailures()).toBe(0); + expect(service._persistPending).toBe(false); + expect(await fsp.readFile(path.join(service._dataPath, 'files.json.lz4'))).toEqual(before); + const folders = await readSaved('folders'); + expect(Object.values(folders.docs.docs).map((doc) => doc.id)).toContain('folder:a'); + }); + + test('is not stringified again until it shrinks, then saving resumes', async () => { + await save(); + await service.upsertFile({ id: 'file:a', vector: vector(), meta: { path: '/x/a.txt' } }); + await service.upsertFile({ id: 'file:b', vector: vector(), meta: { path: '/x/b.txt' } }); + const calls = failFilesStringify(); + await save(); + expect(calls.files).toBe(1); + + // Growing does not try again: the same multi-second stringify would fail. + await service.upsertFile({ id: 'file:c', vector: vector(), meta: { path: '/x/c.txt' } }); + await save(); + expect(calls.files).toBe(1); + + // Fewer documents than when it failed: worth one more try. + jest.restoreAllMocks(); + await service.deleteFileEmbedding('file:b'); + await service.deleteFileEmbedding('file:c'); + await save(); + + expect(mockTracker.clear).toHaveBeenCalledWith('orama.too-large:files'); + const ids = Object.values((await readSaved('files')).docs.docs).map((doc) => doc.id); + expect(ids).toEqual(['file:a']); + }); + }); + + describe('a collection too large to read', () => { + async function plantTooLargeFileChunks() { + await service.batchUpsertFileChunks([ + { id: 'chunk:a:0', vector: vector(), meta: { fileId: 'file:a', chunkIndex: 0 } } + ]); + const dir = service._dataPath; + await restart(async () => { + await fsp.writeFile( + path.join(dir, 'fileChunks.json.lz4'), + await compress(Buffer.from(`{"docs":{"docs":{},"note":"${TOO_LARGE_MARKER}"}}`, 'utf8')) + ); + }); + } + + test('is left on disk as it is, not renamed as corrupt', async () => { + await plantTooLargeFileChunks(); + + const names = await fsp.readdir(service._dataPath); + expect(names).toContain('fileChunks.json.lz4'); + expect(names.some((name) => name.includes('.corrupt.'))).toBe(false); + expect(mockTracker.raise).toHaveBeenCalledWith( + expect.objectContaining({ code: 'orama.too-large:fileChunks' }) + ); + }); + + test('is never saved over, blob or sidecar, while it stays unread', async () => { + await plantTooLargeFileChunks(); + const blob = await fsp.readFile(path.join(service._dataPath, 'fileChunks.json.lz4')); + const sidecar = await fsp.readFile(path.join(service._dataPath, 'fileChunks.sidecar.json')); + + await service.batchUpsertFileChunks([ + { id: 'chunk:b:0', vector: vector(), meta: { fileId: 'file:b', chunkIndex: 0 } } + ]); + await service.persistAll(); + + expect(await fsp.readFile(path.join(service._dataPath, 'fileChunks.json.lz4'))).toEqual(blob); + expect(await fsp.readFile(path.join(service._dataPath, 'fileChunks.sidecar.json'))).toEqual( + sidecar + ); + }); + + test('a deliberate reset replaces it and saving resumes', async () => { + await plantTooLargeFileChunks(); + + await service.resetFileChunks(); + + const saved = await readSaved('fileChunks'); + expect(saved.docs.docs).toEqual({}); + expect(mockTracker.clear).toHaveBeenCalledWith('orama.too-large:fileChunks'); + }); + }); + + test('a write during a save leaves one save scheduled after it, not two', async () => { + await save(); + await service.upsertFile({ id: 'file:a', vector: vector(), meta: { path: '/x/a.txt' } }); + const clearSpy = jest.spyOn(global, 'clearTimeout'); + + service._persistPending = true; + const running = service._doPersist(); + // A write lands mid-save and arms the debounce timer. + service._schedulePersist(); + const armedDuringSave = service._persistTimer; + await running; + + expect(service._persistTimer).not.toBe(armedDuringSave); + expect(clearSpy).toHaveBeenCalledWith(armedDuringSave); + }); + + test('startup removes half-written temp files from old saves and keeps recent ones', async () => { + const dir = service._dataPath; + await service.shutdown(); + const stale = path.join(dir, 'fileChunks.json.lz4.tmp.a05413d8'); + const fresh = path.join(dir, 'fileChunks.json.lz4.tmp.cd6e4624'); + await fsp.writeFile(stale, 'partial'); + await fsp.writeFile(fresh, 'partial'); + const twoHoursAgo = new Date(Date.now() - 2 * 60 * 60 * 1000); + await fsp.utimes(stale, twoHoursAgo, twoHoursAgo); + + service = new OramaVectorService(); + await service.initialize(); + + const names = await fsp.readdir(dir); + expect(names).not.toContain(path.basename(stale)); + expect(names).toContain(path.basename(fresh)); + }); +});