diff --git a/.changeset/lucky-pugs-repeat.md b/.changeset/lucky-pugs-repeat.md new file mode 100644 index 0000000000..a77eda0960 --- /dev/null +++ b/.changeset/lucky-pugs-repeat.md @@ -0,0 +1,34 @@ +--- +'@objectstack/driver-sql': patch +'@objectstack/service-datasource': patch +'@objectstack/runtime': patch +'@objectstack/cli': patch +--- + +`os migrate plan` no longer creates a database on a project that has never been started (#6743) + +`migrate plan` is a dry run, and since #3917 it has reported the boot-time +create-table DDL and the artifact seed instead of performing them. It still +brought the database file itself into existence, though: SQLite creates the +file at open, so a `plan` in a fresh project left behind a 0-table +`.objectstack/data/objectstack.db` — a write side effect from a read-only +command, and one that erased the only signal ("no database file yet") by which +the next command can tell a never-started project from a started one. + +A missing SQLite target is now opened as an empty in-memory database instead of +being created. **The plan output is unchanged**, deliberately: a database with +zero tables is exactly what a freshly created empty file is, so "every table +needs creating" — the true and useful answer for a new project — still prints, +and the `Database:` line still names the real target path rather than the +in-memory stand-in. + +New driver capability, additive and off by default: +`SqlDriverConfig.sqliteAbsentFile` (`'create'` | `'empty-in-memory'`, default +`'create'`). Every existing caller keeps SQLite's own create-if-absent +behaviour. It is threaded to the driver as a host-composition option +(`createDefaultDatasourceDriverFactory`, `DefaultDatasourcePlugin`, +`createStandaloneStack`), not as an authorable `datasource.config` key — a +datasource must not be able to declare itself into never persisting. + +`os migrate apply` deliberately does **not** use it: it boots deferred too, but +flushes the deferred DDL after confirmation and needs a real file to flush into. diff --git a/packages/cli/src/commands/migrate/plan.ts b/packages/cli/src/commands/migrate/plan.ts index 5b87b9dd23..7c916cc10a 100644 --- a/packages/cli/src/commands/migrate/plan.ts +++ b/packages/cli/src/commands/migrate/plan.ts @@ -40,6 +40,16 @@ import { * printed a single line is now REPORTED as pending work instead of performed. * A database another process is using is reported too — as a warning, not a * refusal, since a plan writes nothing either way. + * + * Since #6743 the dry run also stops short of CREATING the database. The + * deferral #3917 introduced covered the DDL and the seed but not the open + * itself, so a `plan` in a never-started project still left a 0-table + * `.objectstack/data/objectstack.db` behind — a write side effect from a + * read-only command, and one that made "this project has no database yet" + * unobservable to whatever ran next. A missing sqlite target is now opened as + * an empty in-memory database: the plan is unchanged (an empty database is an + * empty database, and "every table needs creating" is the true answer for a + * fresh project), and nothing is written to disk. */ export default class MigratePlan extends Command { static override description = @@ -77,7 +87,19 @@ export default class MigratePlan extends Command { let stack; try { - stack = await bootSchemaStack({ jsonOutput: flags.json, databaseUrl: flags['database-url'], deferSchemaDdl: true }); + stack = await bootSchemaStack({ + jsonOutput: flags.json, + databaseUrl: flags['database-url'], + deferSchemaDdl: true, + // A plan writes nothing — so it must not bring a database file into + // existence either (#6743). On a never-started project the target is + // opened as an empty in-memory database: the plan below is unchanged + // (an empty database is an empty database), and no file, `-wal` or + // `-shm` is left behind. `os migrate apply` deliberately does NOT set + // this — it flushes the deferred DDL after confirmation and needs a + // real file to flush into. + readOnlyProbe: true, + }); } catch (error: any) { if (flags.json) { await emitJson({ error: error.message }, 0, { compact: true }); this.exit(1); } printError(error.message || String(error)); diff --git a/packages/cli/src/utils/schema-migrate.readonly-probe.integration.test.ts b/packages/cli/src/utils/schema-migrate.readonly-probe.integration.test.ts new file mode 100644 index 0000000000..3458706c40 --- /dev/null +++ b/packages/cli/src/utils/schema-migrate.readonly-probe.integration.test.ts @@ -0,0 +1,215 @@ +// Copyright (c) 2026 ObjectStack. Licensed under the Apache-2.0 license. + +/** + * End-to-end acceptance for #6743: `os migrate plan` on a project that has + * never been started must leave NO database behind — and must print exactly + * the plan it printed before. + * + * #3917 made the boot stop WRITING (no create-table DDL, no seed rows), and + * `schema-migrate.deferred-ddl.integration.test.ts` pins that half against a + * database that already exists. The half left over — and the one this file + * pins — is the OPEN itself: SQLite creates a database file the moment a + * connection issues its first statement, so a `plan` in a fresh project still + * produced a 0-table `.objectstack/data/objectstack.db`. A dry run that leaves + * a file behind also destroys the one signal "this project has no database + * yet", which every following command reads by asking whether the file exists. + * + * Two assertions, and BOTH are load-bearing (the `domain:cli` ruling on the + * issue is explicit that the fix must not be paid for with the report): + * + * 1. **No file.** Not just `objectstack.db` — the `-wal` and `-shm` + * companions too, and the `data/` directory itself. A pin that names only + * the `.db` path misses two of the three files an interrupted WAL + * connection can strand. + * 2. **Same report.** On a fresh project "every table needs creating" is not + * a false report; it is the true answer, and it is what a user most wants + * from the first `plan` of a new project. So the pending-work set is + * compared against a control boot that DOES create the file, and the + * database label is asserted to still name the real target path rather + * than the `:memory:` stand-in the probe actually read. + */ + +import { describe, it, expect, beforeEach, afterEach } from 'vitest'; +import { mkdtempSync, writeFileSync, mkdirSync, rmSync, existsSync, readdirSync } from 'node:fs'; +import { tmpdir } from 'node:os'; +import { join } from 'node:path'; +import { bootSchemaStack, type PendingSchemaWork } from './schema-migrate.js'; + +const ARTIFACT = { + id: 'probe_smoke', + name: 'Probe Smoke', + objects: [ + { name: 'probe_widget', fields: { name: { type: 'text', required: true }, colour: { type: 'text' } } }, + { name: 'probe_gadget', fields: { label: { type: 'text' } } }, + ], +}; + +/** + * The env vars that outrank the unified project default (#6469). Every one of + * them must be absent or the boot resolves somewhere else entirely and this + * test silently stops testing anything. + */ +const OVERRIDING_ENV = [ + 'OS_DATABASE_URL', + 'DATABASE_URL', + 'TURSO_DATABASE_URL', + 'OS_DATABASE_DRIVER', + 'OS_HOME', +] as const; + +describe('os migrate plan on a fresh project — no database is created (#6743)', () => { + let dir: string; + let dataDir: string; + let dbFile: string; + const savedEnv: Record = {}; + + beforeEach(() => { + dir = mkdtempSync(join(tmpdir(), 'os-probe-')); + mkdirSync(join(dir, 'dist'), { recursive: true }); + writeFileSync(join(dir, 'dist', 'objectstack.json'), JSON.stringify(ARTIFACT)); + // Deliberately NOT created — the whole point is that the boot must not + // bring it into existence. This is the unified default of #6469. + dataDir = join(dir, '.objectstack', 'data'); + dbFile = join(dataDir, 'objectstack.db'); + + for (const key of OVERRIDING_ENV) { + savedEnv[key] = process.env[key]; + delete process.env[key]; + } + savedEnv.OS_ARTIFACT_PATH = process.env.OS_ARTIFACT_PATH; + savedEnv.NODE_ENV = process.env.NODE_ENV; + process.env.OS_ARTIFACT_PATH = join(dir, 'dist', 'objectstack.json'); + process.env.NODE_ENV = 'production'; // no dev auto-reconcile, no wasm step-down + }); + + afterEach(() => { + for (const key of OVERRIDING_ENV) { + if (savedEnv[key] === undefined) delete process.env[key]; + else process.env[key] = savedEnv[key]; + } + process.env.OS_ARTIFACT_PATH = savedEnv.OS_ARTIFACT_PATH; + process.env.NODE_ENV = savedEnv.NODE_ENV; + try { rmSync(dir, { recursive: true, force: true }); } catch { /* ignore */ } + }); + + /** Every path the sqlite target could have brought into existence. */ + function databaseArtifactsOnDisk(): string[] { + const found: string[] = []; + for (const path of [dbFile, `${dbFile}-wal`, `${dbFile}-shm`]) { + if (existsSync(path)) found.push(path); + } + // Anything else the boot dropped in `data/` counts too — a companion this + // list does not name is still debris. + if (existsSync(dataDir)) { + for (const entry of readdirSync(dataDir)) { + const full = join(dataDir, entry); + if (!found.includes(full)) found.push(full); + } + } + return found.sort(); + } + + /** Comparable projection of the plan's "New (additive)" section. */ + function pendingShape(pending: PendingSchemaWork[]): Array<{ table: string; kind: string; columns: number }> { + return pending + .map((p) => ({ table: p.table, kind: p.kind, columns: p.columns.length })) + .sort((a, b) => a.table.localeCompare(b.table)); + } + + it('leaves no .objectstack/data/ at all — file, -wal and -shm alike', async () => { + expect(existsSync(dataDir)).toBe(false); + + const stack = await bootSchemaStack({ + jsonOutput: false, + deferSchemaDdl: true, + readOnlyProbe: true, + projectRoot: dir, + }); + try { + // The plan is computed and non-empty — this is a real run, not a boot + // that failed early and left nothing behind for uninteresting reasons. + expect(stack.pendingSchemaWork.length).toBeGreaterThan(0); + expect(await stack.driver!.detectManagedDrift()).toEqual([]); + } finally { + await stack.shutdown(); + } + + expect(databaseArtifactsOnDisk()).toEqual([]); + // The directory itself is the other half of the write side effect: a + // `mkdir -p` here would recreate exactly the state whose absence tells the + // next command that this project has never been started. + expect(existsSync(dataDir)).toBe(false); + }, 60_000); + + it('prints the same plan as a boot that does create the file, and still names the real target', async () => { + const probe = await bootSchemaStack({ + jsonOutput: false, + deferSchemaDdl: true, + readOnlyProbe: true, + projectRoot: dir, + }); + const probePending = pendingShape(probe.pendingSchemaWork); + const probeLabel = probe.dbLabel; + const probeExamined = probe.managedTableCount; + await probe.shutdown(); + + expect(existsSync(dataDir)).toBe(false); + + // Control: the pre-#6743 behaviour, on the same fresh project. + const control = await bootSchemaStack({ + jsonOutput: false, + deferSchemaDdl: true, + projectRoot: dir, + }); + const controlPending = pendingShape(control.pendingSchemaWork); + const controlLabel = control.dbLabel; + const controlExamined = control.managedTableCount; + await control.shutdown(); + + // The control is what proves the fixture is a genuine fresh project: it + // creates the file this issue is about. + expect(existsSync(dbFile)).toBe(true); + + expect(probePending).toEqual(controlPending); + expect(probeExamined).toBe(controlExamined); + // The one line that would betray the stand-in. `describeDb` reads the + // driver's DECLARED config, which the driver keeps pointing at the real + // file even while its Knex instance holds `:memory:`. + expect(probeLabel).toBe(controlLabel); + expect(probeLabel).toContain('objectstack.db'); + expect(probeLabel).not.toContain(':memory:'); + }, 60_000); + + it('does NOT arm itself for a boot that will write — os migrate apply keeps its file', async () => { + // `apply` boots with `deferSchemaDdl` too and then flushes the deferred DDL + // after the operator confirms. If the absent-file redirect ever became + // implied by `deferSchemaDdl`, that flush would land in an ephemeral + // database and the operator's migration would silently evaporate. + const stack = await bootSchemaStack({ + jsonOutput: false, + deferSchemaDdl: true, + projectRoot: dir, + }); + try { + const created = await stack.flushSchemaDdl(); + expect(created.length).toBeGreaterThan(0); + } finally { + await stack.shutdown(); + } + + expect(existsSync(dbFile)).toBe(true); + + // And the tables really are in the FILE, not in a stand-in that vanished. + const after = await bootSchemaStack({ + jsonOutput: false, + deferSchemaDdl: true, + readOnlyProbe: true, + projectRoot: dir, + }); + try { + expect(after.pendingSchemaWork).toEqual([]); + } finally { + await after.shutdown(); + } + }, 60_000); +}); diff --git a/packages/cli/src/utils/schema-migrate.ts b/packages/cli/src/utils/schema-migrate.ts index eaa894682e..83f2dd3a46 100644 --- a/packages/cli/src/utils/schema-migrate.ts +++ b/packages/cli/src/utils/schema-migrate.ts @@ -189,6 +189,28 @@ export async function bootSchemaStack( * tables to exist. */ deferSchemaDdl?: boolean; + /** + * Boot WITHOUT BRINGING A DATABASE INTO EXISTENCE (#6743). + * + * `deferSchemaDdl` stopped the boot from writing DDL and seed rows, but the + * sqlite driver still opened its target in SQLite's default create-if-absent + * mode — so `os migrate plan` on a never-started project left a 0-table + * `.objectstack/data/objectstack.db` (plus its `-wal`/`-shm` on an unclean + * exit) behind: a write side effect from a command that calls itself a dry + * run, and one that makes "this project has no database yet" unobservable + * to the next command. + * + * With this set, a missing sqlite file is opened as an empty `:memory:` + * database instead. A database with zero tables is exactly what a freshly + * created empty file is, so the plan is byte-for-byte the one printed + * before — the report was never the defect and must not pay for the fix. + * + * ⚠️ NOT implied by `deferSchemaDdl`, and it must not become so: + * `os migrate apply` also boots deferred, then FLUSHES the deferred DDL + * once the operator confirms. Writes into the `:memory:` stand-in would be + * discarded at disconnect, so `apply` keeps the default. + */ + readOnlyProbe?: boolean; /** * Project root the booted stack scopes its on-disk state to — the default * sqlite database and the metadata FileSystemRepository @@ -216,6 +238,7 @@ export async function bootSchemaStack( projectRoot: opts.projectRoot ?? process.cwd(), ...(opts.databaseUrl ? { databaseUrl: opts.databaseUrl } : {}), ...(defer ? { skipSeedData: true } : {}), + ...(opts.readOnlyProbe ? { sqliteAbsentFile: 'empty-in-memory' as const } : {}), }); // No HTTP, no cluster — this is a one-shot schema operation. diff --git a/packages/drivers/driver-sql/src/index.ts b/packages/drivers/driver-sql/src/index.ts index 5c83909899..a9b4322504 100644 --- a/packages/drivers/driver-sql/src/index.ts +++ b/packages/drivers/driver-sql/src/index.ts @@ -3,9 +3,14 @@ import { SqlDriver } from './sql-driver.js'; export { SqlDriver }; +// The absent-file open decision (#6743). Exported as a VALUE because the +// service layer's native→wasm step-down resolves the same answer for its +// fallback rung — one judgement, two call sites, no second `existsSync`. +export { resolveSqliteAbsentFileTarget } from './sql-driver.js'; export type { SqlDriverConfig, SqliteJournalMode, + SqliteAbsentFileMode, IntrospectedSchema, IntrospectedTable, IntrospectedColumn, diff --git a/packages/drivers/driver-sql/src/sql-driver-sqlite-absent-file.test.ts b/packages/drivers/driver-sql/src/sql-driver-sqlite-absent-file.test.ts new file mode 100644 index 0000000000..e1f3e3bcd4 --- /dev/null +++ b/packages/drivers/driver-sql/src/sql-driver-sqlite-absent-file.test.ts @@ -0,0 +1,177 @@ +// Copyright (c) 2026 ObjectStack. Licensed under the Apache-2.0 license. + +/** + * `sqliteAbsentFile` — the driver's open-mode contract (#6743). + * + * The defect this closes lived one layer below every dry-run guard the CLI had + * already built: `os migrate plan` deferred its DDL (#3917) and suppressed its + * seed, and still created a 0-table database, because SQLite brings the file + * into existence at OPEN — for this driver, at the `PRAGMA auto_vacuum` that + * `connect()` issues first. + * + * The judgement lives here rather than in the CLI on purpose. A caller-side + * `existsSync` would be a second opinion about the driver's own open + * semantics, free to drift from it; the driver is the only place that knows + * what it is about to open. + */ + +import { describe, it, expect, beforeEach, afterEach } from 'vitest'; +import { mkdtempSync, rmSync, existsSync, readdirSync, writeFileSync, mkdirSync } from 'node:fs'; +import { tmpdir } from 'node:os'; +import { join } from 'node:path'; +import { SqlDriver, resolveSqliteAbsentFileTarget } from './index.js'; + +describe('resolveSqliteAbsentFileTarget — the one absent-file judgement (#6743)', () => { + let dir: string; + + beforeEach(() => { dir = mkdtempSync(join(tmpdir(), 'os-absent-')); }); + afterEach(() => { try { rmSync(dir, { recursive: true, force: true }); } catch { /* ignore */ } }); + + it('redirects a missing file to :memory: only under empty-in-memory', () => { + const missing = join(dir, 'nope.db'); + + expect(resolveSqliteAbsentFileTarget(missing, 'empty-in-memory')) + .toEqual({ filename: ':memory:', openedEmptyInMemory: true }); + // Default and explicit 'create' are the same answer: SQLite's own. + expect(resolveSqliteAbsentFileTarget(missing, 'create')) + .toEqual({ filename: missing, openedEmptyInMemory: false }); + expect(resolveSqliteAbsentFileTarget(missing, undefined)) + .toEqual({ filename: missing, openedEmptyInMemory: false }); + }); + + it('leaves an existing file alone — the mode is about creation, not access', () => { + const present = join(dir, 'there.db'); + writeFileSync(present, ''); + expect(resolveSqliteAbsentFileTarget(present, 'empty-in-memory')) + .toEqual({ filename: present, openedEmptyInMemory: false }); + }); + + it('passes pseudo-filenames through untouched — they are already fileless', () => { + for (const pseudo of [':memory:', ':other:']) { + expect(resolveSqliteAbsentFileTarget(pseudo, 'empty-in-memory')) + .toEqual({ filename: pseudo, openedEmptyInMemory: false }); + } + }); +}); + +describe('SqlDriver({ sqliteAbsentFile }) — opening must not create (#6743)', () => { + let dir: string; + + beforeEach(() => { dir = mkdtempSync(join(tmpdir(), 'os-absent-drv-')); }); + afterEach(() => { try { rmSync(dir, { recursive: true, force: true }); } catch { /* ignore */ } }); + + const build = (filename: string, mode?: 'create' | 'empty-in-memory') => + new SqlDriver({ + client: 'better-sqlite3', + connection: { filename }, + useNullAsDefault: true, + ...(mode ? { sqliteAbsentFile: mode } : {}), + } as any); + + it('creates nothing on disk — not the file, not -wal, not -shm, not the directory', async () => { + const dataDir = join(dir, 'data'); + const dbFile = join(dataDir, 'objectstack.db'); + const driver = build(dbFile, 'empty-in-memory'); + try { + // `connect()` is where it used to happen: `ensureDatabaseExists()` + // mkdir'd the directory and the first PRAGMA created the file. + await driver.connect(); + // A real, usable connection — the report a plan renders is computed + // against exactly this. + const rows: any = await (driver as any).knex.raw( + "SELECT name FROM sqlite_master WHERE type = 'table'", + ); + expect(rows.filter((r: { name: string }) => !r.name.startsWith('sqlite_'))).toEqual([]); + expect(driver.sqliteOpenedEmptyInMemory).toBe(true); + } finally { + await driver.disconnect(); + } + + expect(existsSync(dbFile)).toBe(false); + expect(existsSync(`${dbFile}-wal`)).toBe(false); + expect(existsSync(`${dbFile}-shm`)).toBe(false); + expect(existsSync(dataDir)).toBe(false); + }); + + it('keeps config naming the DECLARED target so the CLI can still print it', async () => { + const dbFile = join(dir, 'data', 'objectstack.db'); + const driver = build(dbFile, 'empty-in-memory'); + try { + await driver.connect(); + // `describeDriverConnection` renders this. Rewriting it to `:memory:` + // would change `os migrate plan`'s "Database:" line, which the ruling on + // #6743 explicitly refuses to pay for the hygiene fix. + expect(((driver as any).config as any).connection.filename).toBe(dbFile); + } finally { + await driver.disconnect(); + } + }); + + it('opens an EXISTING file normally — the mode changes creation, not access', async () => { + const dbFile = join(dir, 'existing.db'); + const seed = build(dbFile); + await seed.connect(); + await (seed as any).knex.schema.createTable('kept', (t: any) => { t.string('id'); }); + await seed.disconnect(); + expect(existsSync(dbFile)).toBe(true); + + const driver = build(dbFile, 'empty-in-memory'); + try { + await driver.connect(); + expect(driver.sqliteOpenedEmptyInMemory).toBe(false); + const rows: any = await (driver as any).knex.raw( + "SELECT name FROM sqlite_master WHERE type = 'table'", + ); + expect(rows.map((r: { name: string }) => r.name)).toContain('kept'); + } finally { + await driver.disconnect(); + } + }); + + it('without the option, the file is created — the behaviour every existing caller keeps', async () => { + const dataDir = join(dir, 'default-mode'); + const dbFile = join(dataDir, 'objectstack.db'); + const driver = build(dbFile); + try { + await driver.connect(); + expect(driver.sqliteOpenedEmptyInMemory).toBe(false); + } finally { + await driver.disconnect(); + } + expect(existsSync(dbFile)).toBe(true); + }); + + it('is inert for :memory: and for non-sqlite dialects', async () => { + const mem = build(':memory:', 'empty-in-memory'); + try { + await mem.connect(); + expect(mem.sqliteOpenedEmptyInMemory).toBe(false); + } finally { + await mem.disconnect(); + } + + // A pg config is never opened here; the point is only that the option is + // stripped from what Knex receives rather than handed on as an option pg + // has never heard of. + const pg = new SqlDriver({ + client: 'pg', + connection: { host: 'localhost', database: 'nope' }, + sqliteAbsentFile: 'empty-in-memory', + } as any); + expect(((pg as any).config as any).sqliteAbsentFile).toBeUndefined(); + expect(pg.sqliteOpenedEmptyInMemory).toBe(false); + }); + + it('a directory that already exists is not disturbed', async () => { + const dataDir = join(dir, 'preexisting'); + mkdirSync(dataDir, { recursive: true }); + writeFileSync(join(dataDir, 'unrelated.txt'), 'keep me'); + const driver = build(join(dataDir, 'objectstack.db'), 'empty-in-memory'); + try { + await driver.connect(); + } finally { + await driver.disconnect(); + } + expect(readdirSync(dataDir)).toEqual(['unrelated.txt']); + }); +}); diff --git a/packages/drivers/driver-sql/src/sql-driver.ts b/packages/drivers/driver-sql/src/sql-driver.ts index 284645a10b..bf61beda86 100644 --- a/packages/drivers/driver-sql/src/sql-driver.ts +++ b/packages/drivers/driver-sql/src/sql-driver.ts @@ -70,6 +70,7 @@ import { import knex, { Knex } from 'knex'; import { nanoid } from 'nanoid'; import { createHash } from 'node:crypto'; +import { existsSync } from 'node:fs'; import { currentPerfTiming, perfNow, type PerfTiming } from '@objectstack/observability'; /** @@ -2243,6 +2244,52 @@ export type ReadPresentationKind = 'datetime' | 'date' | 'time' | 'boolean' | 'n */ export type SqliteJournalMode = 'wal' | 'delete'; +/** + * What to do when a **file-backed** SQLite datasource points at a path that + * does not exist yet (#6743). + * + * - `'create'` — SQLite's own behaviour, and the default: the file is brought + * into existence by the first statement on the connection. + * - `'empty-in-memory'` — do NOT create it. Open an ephemeral `:memory:` + * database instead, which introspects as a database with zero tables — the + * same thing a freshly-created empty file introspects as. For a read-only + * caller (`os migrate plan`) the report is therefore identical while the + * filesystem is left untouched. + * + * `'empty-in-memory'` is for callers that only ever READ. A caller that may + * write MUST leave it at `'create'`: writes land in the ephemeral database and + * are lost at disconnect. `os migrate apply` is exactly that caller — it boots + * with `deferSchemaDdl` too, but flushes the deferred DDL after confirmation, + * so it must keep `'create'`. Deferring the DDL and never opening the file are + * different promises; #3917 made the first, this makes the second. + */ +export type SqliteAbsentFileMode = 'create' | 'empty-in-memory'; + +/** + * Resolve the filename a SQLite connection should actually open, given the + * declared target and the {@link SqliteAbsentFileMode} the host asked for. + * + * The ONE place the "does the target exist?" judgement is made (#6743). It is + * exported so the service layer's native→wasm step-down can resolve the same + * answer for its fallback rung rather than open-coding a second `existsSync` + * that is free to drift — the two-places objection that ruled out doing this + * check up in the CLI. + * + * Pseudo-filenames (`:memory:` and anything else `:`-prefixed) and non-file + * dialects are already fileless, so they are returned untouched. + */ +export function resolveSqliteAbsentFileTarget( + filename: string, + mode: SqliteAbsentFileMode | undefined, +): { filename: string; openedEmptyInMemory: boolean } { + if (mode !== 'empty-in-memory') return { filename, openedEmptyInMemory: false }; + if (typeof filename !== 'string' || filename === '' || filename.startsWith(':')) { + return { filename, openedEmptyInMemory: false }; + } + if (existsSync(filename)) return { filename, openedEmptyInMemory: false }; + return { filename: ':memory:', openedEmptyInMemory: true }; +} + export type SqlDriverConfig = Knex.Config & { schemaMode?: SchemaMode; /** @@ -2265,6 +2312,18 @@ export type SqlDriverConfig = Knex.Config & { * @see {@link SqlDriver.applySqliteJournalMode} */ sqliteJournalMode?: SqliteJournalMode; + /** + * What to do when the **file-backed** SQLite target does not exist (#6743). + * Defaults to `'create'` — SQLite's own behaviour, and what every existing + * caller gets, since this option changes nothing unless it is passed. + * + * Ignored for `:memory:` and for non-SQLite dialects, neither of which can + * bring a database file into existence. + * + * @see {@link SqliteAbsentFileMode} + * @see {@link SqlDriver.sqliteOpenedEmptyInMemory} + */ + sqliteAbsentFile?: SqliteAbsentFileMode; }; // ── SQL Driver ─────────────────────────────────────────────────────────────── @@ -2595,18 +2654,67 @@ export class SqlDriver implements IDataDriver { /** Object defs `initObjects` registered but did not physically sync while {@link deferredDdl}. */ protected deferredSchemaObjects = new Map }>(); + /** Backing field for {@link sqliteOpenedEmptyInMemory} (#6743). */ + private openedEmptyInMemory = false; + constructor(config: SqlDriverConfig) { - // `schemaMode` / `autoMigrate` / `sqliteJournalMode` are ObjectStack - // concerns, not Knex options — strip them before handing the config to Knex. - const { schemaMode, autoMigrate, sqliteJournalMode, ...knexConfig } = config; + // `schemaMode` / `autoMigrate` / `sqliteJournalMode` / `sqliteAbsentFile` + // are ObjectStack concerns, not Knex options — strip them before handing + // the config to Knex. + const { schemaMode, autoMigrate, sqliteJournalMode, sqliteAbsentFile, ...knexConfig } = config; this.schemaMode = schemaMode ?? 'managed'; this.autoMigrate = autoMigrate ?? 'off'; this.declaredJournalMode = sqliteJournalMode; + // `this.config` keeps the DECLARED target, deliberately: it is what + // `describeDriverConnection` renders, so `os migrate plan` still names the + // database the plan is about rather than the `:memory:` stand-in it read + // (#6743 — the ruling's "same report as today, byte for byte"). Only the + // Knex instance below is redirected. this.config = knexConfig; - this.knex = knex(SqlDriver.withConnectBound(knexConfig)); + this.knex = knex(SqlDriver.withConnectBound(this.knexConfigFor(knexConfig, sqliteAbsentFile))); this.installQueryTiming(); } + /** + * Whether this driver opened an ephemeral `:memory:` database because its + * declared SQLite file did not exist and the host asked for + * `sqliteAbsentFile: 'empty-in-memory'` (#6743). + * + * `false` for every driver that did not ask for that mode — which is every + * driver that existed before it. + */ + public get sqliteOpenedEmptyInMemory(): boolean { + return this.openedEmptyInMemory; + } + + /** + * Apply {@link SqliteAbsentFileMode} to the Knex config, redirecting a + * missing SQLite file to `:memory:` before Knex ever opens it. + * + * Has to happen HERE rather than in `connect()`: the filename is baked into + * the Knex instance at construction, and better-sqlite3 opens (and therefore + * creates) the file on the first statement — which for this driver is the + * `PRAGMA auto_vacuum` in {@link connect}. + */ + private knexConfigFor( + knexConfig: Knex.Config, + mode: SqliteAbsentFileMode | undefined, + ): Knex.Config { + if (mode !== 'empty-in-memory') return knexConfig; + const conn = (knexConfig as { connection?: unknown }).connection; + const declared = typeof conn === 'string' ? conn : (conn as { filename?: unknown })?.filename; + if (typeof declared !== 'string') return knexConfig; + const resolved = resolveSqliteAbsentFileTarget(declared, mode); + if (!resolved.openedEmptyInMemory) return knexConfig; + this.openedEmptyInMemory = true; + return { + ...knexConfig, + connection: typeof conn === 'string' + ? resolved.filename + : { ...(conn as object), filename: resolved.filename }, + } as Knex.Config; + } + /** * Default bound on establishing ONE connection (framework#3769). * @@ -8845,6 +8953,14 @@ export class SqlDriver implements IDataDriver { */ protected sqliteFilename(): string | null { if (!this.isSqlite) return null; + // Redirected to `:memory:` because the declared file did not exist and the + // host asked not to create one (#6743). `this.config` still names the + // declared target — on purpose, so the CLI can print it — but NO on-disk + // file backs this connection, which is precisely what this method reports. + // Both callers depend on that reading: `ensureDatabaseExists` would + // otherwise `mkdir` the state directory this mode exists to avoid, and + // `applySqliteJournalMode` would try to put an in-memory database in WAL. + if (this.openedEmptyInMemory) return null; const conn = (this.config as any).connection; const filename = typeof conn === 'string' ? conn : conn?.filename; if (typeof filename !== 'string' || filename === '') return null; diff --git a/packages/runtime/src/default-datasource-plugin.ts b/packages/runtime/src/default-datasource-plugin.ts index d93bff94df..39d6d2f517 100644 --- a/packages/runtime/src/default-datasource-plugin.ts +++ b/packages/runtime/src/default-datasource-plugin.ts @@ -8,6 +8,7 @@ import { type ConnectableDatasource, type IDatasourceDriverFactory, } from '@objectstack/service-datasource'; +import type { SqliteAbsentFileMode } from '@objectstack/driver-sql'; /** * DefaultDatasourcePlugin — the `default` datasource as a DECLARATION @@ -63,6 +64,15 @@ export interface DefaultDatasourceDefinition { export interface DefaultDatasourcePluginOptions { /** Arms the shared factory's dev sqlite step-down (#2229) + loosen-only self-heal passthroughs. */ dev?: boolean; + /** + * Forwarded to the shared factory: what a `sqlite` default does when its + * file does not exist (#6743). `'empty-in-memory'` is for read-only boots + * (`os migrate plan`); the default `'create'` is every other boot. + * + * Ignored when {@link factory} is injected — a host that brings its own + * factory owns its own open semantics. + */ + sqliteAbsentFile?: SqliteAbsentFileMode; /** * Host-injected driver factory. Defaults to the shared open-core factory * (`createDefaultDatasourceDriverFactory`). The seam exists for hosts whose @@ -98,6 +108,7 @@ export class DefaultDatasourcePlugin implements Plugin { private readonly def: DefaultDatasourceDefinition; private readonly dev?: boolean; + private readonly sqliteAbsentFile?: SqliteAbsentFileMode; private readonly factory?: IDatasourceDriverFactory; /** The init()-time local connection service — held for destroy()'s teardown. */ private connection?: DatasourceConnectionService; @@ -105,6 +116,7 @@ export class DefaultDatasourcePlugin implements Plugin { constructor(def: DefaultDatasourceDefinition, opts: DefaultDatasourcePluginOptions = {}) { this.def = def; this.dev = opts.dev; + this.sqliteAbsentFile = opts.sqliteAbsentFile; this.factory = opts.factory; } @@ -121,7 +133,10 @@ export class DefaultDatasourcePlugin implements Plugin { init = async (ctx: PluginContext) => { const connection = new DatasourceConnectionService({ - factory: () => this.factory ?? createDefaultDatasourceDriverFactory({ dev: this.dev }), + factory: () => this.factory ?? createDefaultDatasourceDriverFactory({ + dev: this.dev, + ...(this.sqliteAbsentFile ? { sqliteAbsentFile: this.sqliteAbsentFile } : {}), + }), engine: () => { try { return ctx.getService('data'); diff --git a/packages/runtime/src/standalone-stack.ts b/packages/runtime/src/standalone-stack.ts index 47909f29ff..fd834a1c44 100644 --- a/packages/runtime/src/standalone-stack.ts +++ b/packages/runtime/src/standalone-stack.ts @@ -53,7 +53,7 @@ */ import { resolve as resolvePath } from 'node:path'; -import { mkdirSync } from 'node:fs'; +import { mkdirSync, existsSync } from 'node:fs'; import { homedir } from 'node:os'; import { z } from 'zod'; import { stampSearchPinyinEnabled } from '@objectstack/types'; @@ -199,6 +199,21 @@ export const StandaloneStackConfigSchema = z.object({ * operator's live database before they have confirmed anything. */ skipSeedData: z.boolean().optional(), + /** + * What this boot does when the default sqlite database file does not + * exist yet (#6743). `'empty-in-memory'` opens an ephemeral database + * instead of creating the file, and suppresses the host's `mkdir` of the + * state directory with it — so a read-only boot on a never-started + * project leaves the filesystem exactly as it found it. Defaults to + * `'create'`. + * + * ⚠️ READ-ONLY BOOTS ONLY. This is NOT implied by `skipSeedData` / + * `deferSchemaDdl`, and deliberately so: `os migrate apply` boots deferred + * too and then FLUSHES the deferred DDL once the operator confirms, so it + * needs a real file. `os migrate plan` never writes and is the caller this + * exists for. + */ + sqliteAbsentFile: z.enum(['create', 'empty-in-memory']).optional(), }); export type StandaloneStackConfig = z.input; @@ -583,7 +598,15 @@ export async function createStandaloneStack(config?: StandaloneStackConfig): Pro // sqlite (better-sqlite3) driverId = 'sqlite'; const filename = sqliteFilenameFromUrl(dbUrl, 'sqlite'); - mkdirSync(resolvePath(filename, '..'), { recursive: true }); + // The host's filesystem prep. Skipped for a read-only boot whose target + // does not exist (#6743): creating `/data/` is the other half + // of the write side effect `os migrate plan` was leaving behind, and a + // `mkdir -p` here would recreate the very directory the driver is about + // to decline to put a file in. When the file DOES exist the directory + // does too, so this costs the mode nothing. + if (!(cfg.sqliteAbsentFile === 'empty-in-memory' && !existsSync(filename))) { + mkdirSync(resolvePath(filename, '..'), { recursive: true }); + } driverConfig = { filename }; } else { // Unreachable by construction — and making it unreachable is half the @@ -602,7 +625,11 @@ export async function createStandaloneStack(config?: StandaloneStackConfig): Pro } const defaultDatasourcePlugin = new DefaultDatasourcePlugin( { driver: driverId, config: driverConfig }, - { dev: factoryDev, ...(hostFactory ? { factory: hostFactory } : {}) }, + { + dev: factoryDev, + ...(cfg.sqliteAbsentFile ? { sqliteAbsentFile: cfg.sqliteAbsentFile } : {}), + ...(hostFactory ? { factory: hostFactory } : {}), + }, ); const artifactBundle = await loadArtifactBundle(artifactPath, { diff --git a/packages/services/service-datasource/src/default-datasource-driver-factory.ts b/packages/services/service-datasource/src/default-datasource-driver-factory.ts index 3aa3995be8..c2f9d77601 100644 --- a/packages/services/service-datasource/src/default-datasource-driver-factory.ts +++ b/packages/services/service-datasource/src/default-datasource-driver-factory.ts @@ -53,6 +53,7 @@ import type { DatasourceDriverHandle, } from './contracts/index.js'; import { assertDatasourcePoolSupported } from './datasource-pool-support.js'; +import type { SqliteAbsentFileMode } from '@objectstack/driver-sql'; /** * Driver-id resolution comes from the spec since #4410 — this file used to keep @@ -331,6 +332,23 @@ export interface DefaultDatasourceDriverFactoryOptions { * failure is NOT silently swapped for a different engine (fail-closed). */ dev?: boolean; + /** + * What a `sqlite` construction does when its file does not exist (#6743). + * `'empty-in-memory'` opens an ephemeral database instead of creating the + * file — for hosts that only READ (`os migrate plan`). Defaults to + * `'create'`, so no existing host changes behaviour. + * + * A host-composition option rather than a datasource `config` key on + * purpose: it describes what THIS BOOT is allowed to do, not a property of + * the datasource, and `SqliteConfigSchema` is strict — an authorable key + * here would invite `objectstack.config.ts` to declare a database that + * silently never persists. + * + * Applies to the `sqlite` kind only. `sqlite-wasm` is constructed directly + * from a filename and is deliberately left alone; every other kind connects + * to a server that this process cannot bring into existence anyway. + */ + sqliteAbsentFile?: SqliteAbsentFileMode; } export function createDefaultDatasourceDriverFactory( @@ -403,6 +421,7 @@ export function createDefaultDatasourceDriverFactory( dev: options.dev, ...(schemaMode ? { schemaMode } : {}), ...(autoMigrate ? { autoMigrate } : {}), + ...(options.sqliteAbsentFile ? { sqliteAbsentFile: options.sqliteAbsentFile } : {}), }); return toHandle(resolved.driver, () => sqlServerVersion(resolved.driver, 'sqlite')); } diff --git a/packages/services/service-datasource/src/sqlite-driver-fallback.test.ts b/packages/services/service-datasource/src/sqlite-driver-fallback.test.ts index 8e83fb59d8..16f63b91ca 100644 --- a/packages/services/service-datasource/src/sqlite-driver-fallback.test.ts +++ b/packages/services/service-datasource/src/sqlite-driver-fallback.test.ts @@ -1,6 +1,7 @@ // Copyright (c) 2025 ObjectStack. Licensed under the Apache-2.0 license. import { describe, it, expect, beforeEach, afterAll, vi } from 'vitest'; +import { existsSync } from 'node:fs'; import { resolveSqliteDriver, NATIVE_SQLITE_WASM_FALLBACK_WARNING, @@ -42,7 +43,24 @@ vi.mock('@objectstack/driver-sql', () => { } async disconnect(): Promise {} } - return { SqlDriver }; + /** + * The real `resolveSqliteAbsentFileTarget` (#6743), restated here because + * this suite mocks the whole driver package away and the step-down under + * test now consults it for the wasm rung. Kept deliberately literal — it is + * six lines of pure logic with no driver state — and the assertions below + * pin the OUTCOME (which filename each rung receives), so a drift between + * this restatement and the real one shows up as a failure there rather than + * as silently absent coverage. + */ + const resolveSqliteAbsentFileTarget = (filename: string, mode: string | undefined) => { + if (mode !== 'empty-in-memory') return { filename, openedEmptyInMemory: false }; + if (typeof filename !== 'string' || filename === '' || filename.startsWith(':')) { + return { filename, openedEmptyInMemory: false }; + } + if (existsSync(filename)) return { filename, openedEmptyInMemory: false }; + return { filename: ':memory:', openedEmptyInMemory: true }; + }; + return { SqlDriver, resolveSqliteAbsentFileTarget }; }); vi.mock('@objectstack/driver-sqlite-wasm', () => { @@ -146,6 +164,37 @@ describe('resolveSqliteDriver — native better-sqlite3 → wasm → in-memory s }); }); + it('forwards sqliteAbsentFile to the native driver — the step-down does not decide it (#6743)', async () => { + await resolveSqliteDriver({ filename: '/tmp/os-absent-never.db', dev: true, sqliteAbsentFile: 'empty-in-memory' }); + expect(state.nativeConfigs[0]).toMatchObject({ sqliteAbsentFile: 'empty-in-memory' }); + }); + + it('omits sqliteAbsentFile entirely when not asked for — no caller changes behaviour (#6743)', async () => { + await resolveSqliteDriver({ filename: ':memory:', dev: true }); + expect(state.nativeConfigs[0]).not.toHaveProperty('sqliteAbsentFile'); + }); + + it('the wasm rung opens :memory: too, so a step-down cannot create the file either (#6743)', async () => { + // The native rung is what normally applies the redirect, and here it is + // exactly the rung that fails. `SqliteWasmDriver` takes a bare filename and + // has no absent-file mode of its own, so without this the dev step-down + // would quietly create the very file `os migrate plan` declined to create. + state.nativeFails = true; + const missing = '/tmp/os-absent-wasm-never.db'; + expect(existsSync(missing)).toBe(false); + + const resolved = await resolveSqliteDriver({ + filename: missing, + dev: true, + sqliteAbsentFile: 'empty-in-memory', + warn: vi.fn(), + }); + + expect(resolved.engine).toBe('sqlite-wasm'); + expect(state.wasmConfigs[0]).toMatchObject({ filename: ':memory:', persist: 'on-disconnect' }); + expect(existsSync(missing)).toBe(false); + }); + it('production (dev=false) is fail-closed — returns native unprobed, never degrades', async () => { state.nativeFails = true; const warn = vi.fn(); diff --git a/packages/services/service-datasource/src/sqlite-driver-fallback.ts b/packages/services/service-datasource/src/sqlite-driver-fallback.ts index 2561a4a710..6258dab896 100644 --- a/packages/services/service-datasource/src/sqlite-driver-fallback.ts +++ b/packages/services/service-datasource/src/sqlite-driver-fallback.ts @@ -33,6 +33,8 @@ * hoists it into one place shared by every sqlite construction site. */ +import type { SqliteAbsentFileMode } from '@objectstack/driver-sql'; + /** Which engine the resolver ultimately produced. */ export type SqliteFallbackEngine = 'better-sqlite3' | 'sqlite-wasm' | 'memory'; @@ -56,6 +58,13 @@ export interface ResolveSqliteDriverOptions { autoMigrate?: 'off' | 'safe'; /** Forwarded to the SQL drivers (external schema mode, ADR-0015). */ schemaMode?: string; + /** + * What to do when {@link filename} names a file that does not exist (#6743). + * Forwarded to the native driver, and applied to the wasm rung too — a + * step-down must not create the file the caller asked us not to create. + * Defaults to `'create'`, i.e. SQLite's own behaviour. + */ + sqliteAbsentFile?: SqliteAbsentFileMode; /** * Warning sink for the step-down messages. Defaults to `console.warn`. * `serve.ts` passes a `chalk.yellow` wrapper so the banner stays consistent. @@ -113,7 +122,7 @@ export async function resolveSqliteDriver( } }); - const { SqlDriver } = await import('@objectstack/driver-sql'); + const { SqlDriver, resolveSqliteAbsentFileTarget } = await import('@objectstack/driver-sql'); const buildNative = () => new SqlDriver({ @@ -122,8 +131,18 @@ export async function resolveSqliteDriver( useNullAsDefault: true, ...(opts.autoMigrate ? { autoMigrate: opts.autoMigrate } : {}), ...(opts.schemaMode ? { schemaMode: opts.schemaMode } : {}), + ...(opts.sqliteAbsentFile ? { sqliteAbsentFile: opts.sqliteAbsentFile } : {}), } as any); + /** + * The filename the wasm rung must open (#6743). `SqliteWasmDriver` has no + * `sqliteAbsentFile` of its own — it is constructed straight from a filename + * — so the step-down resolves the driver's OWN judgement here rather than + * re-deciding it, and a fresh project that falls through to wasm still + * leaves no file behind. + */ + const wasmFilename = resolveSqliteAbsentFileTarget(filename, opts.sqliteAbsentFile).filename; + // Production: never silently swap engines. Construct the native driver and // hand it back UNPROBED — exactly the historical behavior. A native load // failure surfaces loudly at first use (fail-closed). @@ -165,11 +184,11 @@ export async function resolveSqliteDriver( try { const { SqliteWasmDriver } = await import('@objectstack/driver-sqlite-wasm'); wasmDriver = new SqliteWasmDriver({ - filename, + filename: wasmFilename, // Match the existing construction sites: ephemeral DBs flush on // disconnect; a persistent file flushes on every write so AI-authored // data survives an unclean dev-server kill. - persist: isEphemeralFilename(filename) ? 'on-disconnect' : 'on-write', + persist: isEphemeralFilename(wasmFilename) ? 'on-disconnect' : 'on-write', } as any); await wasmDriver.connect(); wasmOk = true;