From be7f9ee9eb102697c486de8d36764aa96c0dd001 Mon Sep 17 00:00:00 2001 From: Matias Fernandez <7898112+MatiasFernandez@users.noreply.github.com> Date: Mon, 19 Oct 2026 10:03:01 -0300 Subject: [PATCH 1/3] Check an external database in one place, and wait for it to start The window and the GemDB Shell each had their own copy of "is the external database up", and neither waited for a stone still at Startup. Both now call assertExternalDatabaseUp, which waits as waitForStoneReady does for a database of our own. For an external database the messages no longer name GemDB's own stone log (which has nothing of it) and no longer tell the user to start it again: the administrator runs it. Co-Authored-By: Claude Sonnet 5.5 --- src/__tests__/externalLifecycle.test.ts | 8 ++- src/__tests__/startProcesses.test.ts | 83 ++++++++++++++++++++++++- src/cliMain.ts | 22 +++---- src/lifecycle.ts | 10 +-- src/processes.ts | 40 ++++++++++-- 5 files changed, 138 insertions(+), 25 deletions(-) diff --git a/src/__tests__/externalLifecycle.test.ts b/src/__tests__/externalLifecycle.test.ts index e2a1c3c..5eb086b 100644 --- a/src/__tests__/externalLifecycle.test.ts +++ b/src/__tests__/externalLifecycle.test.ts @@ -45,7 +45,13 @@ vi.mock('../processes', async (importOriginal) => { const actual = await importOriginal(); return { ExternalDatabaseError: actual.ExternalDatabaseError, - externalDatabaseDownError: actual.externalDatabaseDownError, + // The real check's verdict on the faked listing; its waiting is covered in startProcesses.test.ts. + assertExternalDatabaseUp: async (opts: { listener: boolean }) => { + const external = actual.externalDatabaseDownError; + if (!findStone() || (opts.listener && !findNetldi())) { + throw external({ stone: 'stone', netldi: 'netldi' }); + } + }, listProcesses: () => [], findStone: () => findStone(), findNetldi: () => findNetldi(), diff --git a/src/__tests__/startProcesses.test.ts b/src/__tests__/startProcesses.test.ts index b554958..15e9703 100644 --- a/src/__tests__/startProcesses.test.ts +++ b/src/__tests__/startProcesses.test.ts @@ -68,8 +68,15 @@ vi.mock('../lock', async (importOriginal) => { const { expectedEnginePath } = await import('../paths'); const { STONE_LOCK_NAME } = await import('../lock'); -const { ensureListener, ensureProcesses, stoneReady, stoneState, waitForStoneReady } = - await import('../processes'); +const { + EXTERNAL_SHUTTING_DOWN_MESSAGE, + assertExternalDatabaseUp, + ensureListener, + ensureProcesses, + stoneReady, + stoneState, + waitForStoneReady, +} = await import('../processes'); const { parseGslist } = await import('../gslist'); function row(status: string, type: 'Stone' | 'Netldi', name: string): string { @@ -495,3 +502,75 @@ describe('waiting for the stone lock', () => { expect(outcome.error?.message).toContain(STONE_LOCK_NAME); }); }); + +describe('checking an external database', () => { + const EXTERNAL_STONE = row('OK', 'Stone', 'gs64stone'); + const EXTERNAL_LISTENER = row('OK', 'Netldi', 'gs64ldi'); + + beforeEach(() => { + const gemstone = path.join(root, 'external'); + fs.mkdirSync(path.join(gemstone, 'bin'), { recursive: true }); + fs.writeFileSync(path.join(gemstone, 'bin', 'gslist'), ''); + __setSetting('gemdb.externalDatabase.gemstone', gemstone); + }); + + it('passes when the stone and the listener are up', async () => { + listings = [[EXTERNAL_STONE, EXTERNAL_LISTENER].join('\n')]; + + await expect(assertExternalDatabaseUp({ listener: true })).resolves.toBeUndefined(); + }); + + it('says the administrator runs it when there is no stone, without waiting', async () => { + listings = ['']; + + await expect(assertExternalDatabaseUp({ listener: false })).rejects.toThrow( + /administrator runs\. Ask them to start it/, + ); + expect(gslistCalls).toBe(1); + }); + + it('waits for a stone that is still starting', async () => { + const starting = row('Startup', 'Stone', 'gs64stone'); + listings = [starting, starting, [EXTERNAL_STONE, EXTERNAL_LISTENER].join('\n')]; + const report = vi.fn(); + + await assertExternalDatabaseUp({ listener: true, report }); + + expect(report).toHaveBeenCalledWith('Waiting for the database to finish starting…'); + }); + + it('names no log of ours when a stone is still starting after a minute', async () => { + listings = [row('Startup', 'Stone', 'gs64stone')]; + + const failure = await waitForStoneReady(undefined, { pollMs: 1, timeoutMs: 20 }).catch( + (e: Error) => e, + ); + + expect((failure as Error).message).toBe('The database has been starting for over a minute.'); + }); + + it('says the administrator runs it for a stone at Shutdown, even after it was starting', async () => { + listings = [row('Startup', 'Stone', 'gs64stone'), row('Shutdown', 'Stone', 'gs64stone')]; + + await expect(assertExternalDatabaseUp({ listener: true })).rejects.toThrow( + EXTERNAL_SHUTTING_DOWN_MESSAGE, + ); + }); + + it('names no log of ours when a stone that was starting disappears', async () => { + listings = [row('Startup', 'Stone', 'gs64stone'), '']; + + const failure = await assertExternalDatabaseUp({ listener: false }).catch((e: Error) => e); + + expect((failure as Error).message).toBe('The database stopped while it was starting.'); + }); + + it('requires a listener only when asked to', async () => { + listings = [EXTERNAL_STONE]; + + await expect(assertExternalDatabaseUp({ listener: false })).resolves.toBeUndefined(); + await expect(assertExternalDatabaseUp({ listener: true })).rejects.toThrow( + /NetLDI gs64ldi.*administrator runs/, + ); + }); +}); diff --git a/src/cliMain.ts b/src/cliMain.ts index 2ac6783..a92d1b3 100644 --- a/src/cliMain.ts +++ b/src/cliMain.ts @@ -1,13 +1,7 @@ import { abortIdleSessionsAfterMinutes, externalDatabase } from './config'; import { assertSafeToStart } from './database'; import { errorMessage } from './log'; -import { - ensureProcesses, - externalDatabaseDownError, - findNetldi, - findStone, - listProcesses, -} from './processes'; +import { assertExternalDatabaseUp, ensureProcesses } from './processes'; import { PyRepl, ReplSession } from './pyRepl'; import { runPythonInSession } from './pythonQueries'; import { GciSession, setInputHandler } from './session'; @@ -39,21 +33,23 @@ import { GciSession, setInputHandler } from './session'; * comes from editor settings the Shell does not have: `system.conf` keeps * whatever the window's last start wrote. * - * An external database is only checked, as the window checks it: GemDB starts - * none of it. Errors are reported rather than thrown: the shell prompt (or the + * An external database is only checked, as the window checks it (waiting for + * a stone still starting): GemDB starts none of it. Errors are reported rather than thrown: the shell prompt (or the * exit code) is the caller's answer. */ +function reportStart(message: string): void { + process.stderr.write(`gemdb: ${message.charAt(0).toLowerCase()}${message.slice(1)}\r\n`); +} + async function ensureDatabase(): Promise { try { const external = externalDatabase(); if (external) { - const running = listProcesses(); - if (!findStone(running) || !findNetldi(running)) throw externalDatabaseDownError(external); + await assertExternalDatabaseUp({ listener: true, report: reportStart }); } else { await ensureProcesses({ beforeStoneStart: assertSafeToStart, - report: (message) => - process.stderr.write(`gemdb: ${message.charAt(0).toLowerCase()}${message.slice(1)}\r\n`), + report: reportStart, }); } return true; diff --git a/src/lifecycle.ts b/src/lifecycle.ts index c02bf64..79b814e 100644 --- a/src/lifecycle.ts +++ b/src/lifecycle.ts @@ -60,14 +60,12 @@ import { mcpPath, } from './paths'; import { + assertExternalDatabaseUp, ensureListener, ensureProcesses, - externalDatabaseDownError, - findNetldi, findStone, isListening, isRunning, - listProcesses, stopNetldi, stopStone, } from './processes'; @@ -893,8 +891,10 @@ async function startProcesses( // in terms of what to do about it. const external = externalDatabase(); if (external) { - const running = listProcesses(); - if (!findStone(running) || !findNetldi(running)) throw externalDatabaseDownError(external); + await assertExternalDatabaseUp({ + listener: true, + report: (message) => progress?.report({ message }), + }); log('The database is running.'); return { startedStone: false, startedNetldi: false }; } diff --git a/src/processes.ts b/src/processes.ts index c6f084c..a2bc8d7 100644 --- a/src/processes.ts +++ b/src/processes.ts @@ -222,6 +222,13 @@ export function stoneState(processes: EngineProcess[]): StoneState { export const SHUTTING_DOWN_MESSAGE = 'The database is shutting down. Start it again once it has stopped.'; +/** + * {@link SHUTTING_DOWN_MESSAGE} for an external database, which nobody here + * can start again: its administrator runs it. + */ +export const EXTERNAL_SHUTTING_DOWN_MESSAGE = + "The database is shutting down. This machine's administrator runs it; ask them to start it again once it has stopped."; + /** * Wait for a listed stone to finish starting. * @@ -234,12 +241,16 @@ export const SHUTTING_DOWN_MESSAGE = * from `wait.listing` when the caller has one that recent, rather than asking * gslist twice in a row. Returns the last listing, so the caller need not * take another. + * + * For an external database the messages do not point at GemDB's own log, which + * has nothing of its stone, and say who to ask rather than to start it again. */ export async function waitForStoneReady( report?: (message: string) => void, wait: { pollMs?: number; timeoutMs?: number; listing?: EngineProcess[] } = {}, ): Promise { - const stoneLog = path.join(databaseLogPath(), `${STONE_NAME}.log`); + const external = externalDatabase() !== undefined; + const seeLog = external ? '' : ` See ${path.join(databaseLogPath(), `${STONE_NAME}.log`)}.`; const deadline = Date.now() + (wait.timeoutMs ?? STONE_LOCK_TIMEOUT_MS); let running = wait.listing ?? (await listProcessesAsync()); let waiting = false; @@ -249,9 +260,9 @@ export async function waitForStoneReady( case 'other': return running; case 'stopping': - throw new Error(SHUTTING_DOWN_MESSAGE); + throw new Error(external ? EXTERNAL_SHUTTING_DOWN_MESSAGE : SHUTTING_DOWN_MESSAGE); case 'absent': - throw new Error(`The database stopped while it was starting. See ${stoneLog}.`); + throw new Error(`The database stopped while it was starting.${seeLog}`); case 'starting': break; } @@ -261,7 +272,7 @@ export async function waitForStoneReady( log('The database is still starting; waiting for it.'); } if (Date.now() > deadline) { - throw new Error(`The database has been starting for over a minute. See ${stoneLog}.`); + throw new Error(`The database has been starting for over a minute.${seeLog}`); } await new Promise((resolve) => setTimeout(resolve, wait.pollMs ?? STONE_LOCK_POLL_MS)); running = await listProcessesAsync(); @@ -303,6 +314,27 @@ export function externalDatabaseDownError(external: { ); } +/** + * Check that an external database is up, waiting for a stone that is still + * starting. GemDB starts none of it, so a stone that is absent or shutting + * down is an error that says who to ask. NetLDI is required only when the + * caller will open sessions through it (`listener`), and means listed, as it + * does for GemDB's own database. The one check the window and the GemDB Shell + * share. + */ +export async function assertExternalDatabaseUp(opts: { + listener: boolean; + report?: (message: string) => void; +}): Promise { + const external = externalDatabase(); + if (!external) return; + const listed = await listProcessesAsync(); + // Absent is not "stopped while it was starting": nothing was starting. + if (!findStone(listed)) throw externalDatabaseDownError(external); + const running = await waitForStoneReady(opts.report, { listing: listed }); + if (opts.listener && !findNetldi(running)) throw externalDatabaseDownError(external); +} + /** * Bring up whichever of the stone and the session listener is not already * up, and say which of them this call started. From 3b6329f1c9ddb9801a2c3d585bbf293cbf691c3c Mon Sep 17 00:00:00 2001 From: Matias Fernandez <7898112+MatiasFernandez@users.noreply.github.com> Date: Mon, 19 Oct 2026 10:07:47 -0300 Subject: [PATCH 2/3] Hand every cold start of `gemdb` to the shell program A file, -c or -m run that found the stone down started it from ~130 lines of bash that duplicated ensureProcesses' lock, start and wait, had drifted from it, skipped assertSafeToStart (so a pin move ended in a raw 4045), and treated an external database differently from the window and the Shell. The wrapper now keeps only the fast path for a stone that reads `ok`. Every other case runs `gemdb-shell.js --ensure-stone`, which goes through bringUp: the same ensureProcesses and guards as the Shell, starting the stone only, since a file run is a linked topaz and never uses NetLDI (ensureProcesses takes a `listener` option for this). - The stone lock now reports that it is waiting, so a CLI run behind a window's start says so instead of printing nothing for up to 150 s. - A mistyped file or a -c with no code fails before anything starts. - The comments that said bash takes the lock now say that only wrappers from 1.7.0 and earlier do, which is why the lock's format must stay as it is. Co-Authored-By: Claude Sonnet 5.5 --- CHANGELOG.md | 10 + CLAUDE.md | 13 +- src/__integration__/cli.test.ts | 4 +- src/__tests__/cli.test.ts | 387 +++++++++---------------- src/__tests__/cliStart.test.ts | 88 ++++++ src/__tests__/externalDatabase.test.ts | 4 +- src/__tests__/lock.test.ts | 35 +++ src/__tests__/startProcesses.test.ts | 63 ++++ src/cli.ts | 224 +++----------- src/cliMain.ts | 61 ++-- src/cliStart.ts | 39 +++ src/lock.ts | 24 +- src/processes.ts | 39 ++- 13 files changed, 519 insertions(+), 472 deletions(-) create mode 100644 src/__tests__/cliStart.test.ts create mode 100644 src/cliStart.ts diff --git a/CHANGELOG.md b/CHANGELOG.md index 28a63e3..635a777 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -44,6 +44,16 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 `gemdb app.py`, or of **Run Python File in GemDB**, stop the program as Ctrl+C would. On Linux it usually did not: the program carried on out of sight, its `finally` blocks unrun, holding its database session. macOS was not affected. +- **The window and the GemDB Shell wait for an external database that is still starting.** They + used to report one at `Startup` as up, and the first login then failed. A stone that is shutting + down is now said to be run by the machine's administrator, who is the one to start it again. +- **`gemdb file.py`, `-c` and `-m` start the database the way the window and the GemDB Shell do.** + A database left by an older engine is now refused with the sentence that says what to delete, + instead of a raw 4045 at the first login, and a lack of room or an NFS root path is refused + too. A `gemdb` run that finds the database being started by another process now + says it is waiting, and waits as long as the window does, instead of giving up after ten seconds. + A run starts the database only, not the session listener it never uses. A mistyped file name or + a `-c` with no code no longer starts the database before failing. ## [1.8.0] - 2026-10-18 diff --git a/CLAUDE.md b/CLAUDE.md index d50901b..f285545 100644 --- a/CLAUDE.md +++ b/CLAUDE.md @@ -62,8 +62,12 @@ force-stop uses), and "ready" means gslist status `OK`, not "listed": a stone at `Startup` is listed and refuses logins, so wait on it, never start it. `stoneState` reads the rest, case-insensitively since the engine prints mixed case: `Shutdown` fails at once, and any other listed status (`exe deleted`, -`frozen`, ...) goes on to the login as it always did. The wrapper in `cli.ts` -makes the same calls in shell; change both together. +`frozen`, ...) goes on to the login as it always did. The `gemdb` wrapper in +`cli.ts` keeps no copy of this: it only fast-paths a stone that reads `ok`, and +every other case (including an external database) runs +`gemdb-shell.js --ensure-stone`, which calls `bringUp` (`cliStart.ts`) and so +`ensureProcesses` with `assertSafeToStart`. File mode starts the stone only +(`listener: false`): it is a linked topaz and never goes through NetLDI. ## CI @@ -219,7 +223,10 @@ bundle and `node_modules/koffi` (this platform's binary only) to `process.execPath` recorded at generation time, `ELECTRON_RUN_AS_NODE=1`, a PATH `node` as fallback. "Open GemDB Shell" opens a terminal on that wrapper, so the shell is out of the extension host entirely: a wedged FFI call is a -dead tab, not a dead window. The reason this exists is measured topaz history: +dead tab, not a dead window. The same bundle is the wrapper's cold start: +`--ensure-stone` runs before anything touches stdin, so a file, `-c` or `-m` +run that finds the stone down needs the bundle too (and refuses without it, +with the Shell's messages, rather than falling back to bash). The reason this exists is measured topaz history: Grail's own topaz REPL (`gemdb`'s old no-argument handoff) wraps evaluation in `on: Error do:`, and Grail's Python exceptions descend from `AbstractException`, not `Error` — so a `ZeroDivisionError` printed a diff --git a/src/__integration__/cli.test.ts b/src/__integration__/cli.test.ts index 36f84a0..d61b217 100644 --- a/src/__integration__/cli.test.ts +++ b/src/__integration__/cli.test.ts @@ -5,7 +5,7 @@ import * as path from 'path'; import { afterAll, beforeAll, describe, expect, it } from 'vitest'; import { cliPath } from '../cli'; import { bundledGrailStamp, stageGrail } from '../grail'; -import { isRunning, stopStone } from '../processes'; +import { isListening, isRunning, stopStone } from '../processes'; import { createDatabaseWithPython, Fixture, @@ -411,6 +411,8 @@ close expect(ran.stdout).toContain('back'); expect(ran.code).toBe(0); expect(isRunning()).toBe(true); + // A file run is a linked topaz: the listener is none of its business. + expect(isListening()).toBe(false); }); }); diff --git a/src/__tests__/cli.test.ts b/src/__tests__/cli.test.ts index 72dabb9..2150686 100644 --- a/src/__tests__/cli.test.ts +++ b/src/__tests__/cli.test.ts @@ -121,7 +121,7 @@ describe('writeCliScripts', () => { 'topaz -L -q -I "$ROOT/bin/gemdb-run-init.tpz" -P "$ROOT/bin/gemdb-run.tpz"', ); expect(script).toContain('GEMDB_TOPAZ_LOG="$(mktemp'); - expect(script).toMatch(/if \[ -z "\$STATUS" \]; then\n\s+sed .* "\$GEMDB_TOPAZ_LOG" >&2\n/); + expect(script).toMatch(/if \[ -z "\$RUN_STATUS" \]; then\n\s+sed .* "\$GEMDB_TOPAZ_LOG" >&2\n/); }); it('tidies up after the run however it ends, topaz included', () => { @@ -204,8 +204,9 @@ describe('writeCliScripts', () => { writeCliScripts(ext); - // File and module mode do not need the bundle, so the wrapper is written - // regardless; its shell branch reports the gap at run time instead. + // A run against a database that is already up does not need the bundle, so + // the wrapper is written regardless; a cold start and the Shell report the + // gap at run time instead. expect(fs.existsSync(cliPath())).toBe(true); expect(fs.existsSync(path.join(root, 'bin', 'gemdb-shell.js'))).toBe(false); expect(fs.readFileSync(cliPath(), 'utf8')).toContain('has no shell program'); @@ -347,41 +348,35 @@ describe('putCliOnPath', () => { }); /** - * Starting the stone is a check-then-act with nothing between the check and - * the start, and every `gemdb` command runs it -- so two ordinary commands - * close together are enough to start two stones on one extent. That happened: - * two stoned processes held the same extent0.dbf read-write for a week, and - * `gslist` showed one row for them because it keys Stone rows by name. + * What the wrapper decides about starting the database, with stand-ins for + * everything it calls. It only recognises a stone that answers (`ok`) and runs + * against it; every other case is handed to the shell program's + * `--ensure-stone`, which is where the lock, the guards and the waiting live + * (and are tested, in processes.test.ts and cliStart.test.ts). * - * The stubs stand in for the engine binaries the wrapper calls by absolute - * path. `startstone` records that it ran and leaves a marker holding the - * stone's status; `gslist` reports a stone, with that status, only once the - * marker exists, so a wrapper that re-checks after taking the lock sees what - * a real one would. Each `gslist` call is counted in `gslist.calls`. + * `gslist` reports a stone with the status in `stone.up`; the shell program is + * a script that records how it was called and, when it exits 0, leaves the + * stone `OK` as a real start would; `topaz` records that the run began and + * reports exit status 0. */ describe('starting the database', () => { - function installEngineStubs(): { starts: string } { + const callsLog = (name: string): string => path.join(root, `${name}.log`); + + function readCalls(name: string): string[] { + return fs.existsSync(callsLog(name)) + ? fs.readFileSync(callsLog(name), 'utf8').split('\n').filter(Boolean) + : []; + } + + function installStubs(): void { const bin = path.join(expectedEnginePath(), 'bin'); fs.mkdirSync(bin, { recursive: true }); const marker = path.join(root, 'stone.up'); - const starts = path.join(root, 'starts.log'); - const countdown = path.join(root, 'stone.countdown'); - const next = path.join(root, 'stone.next'); - const calls = path.join(root, 'gslist.calls'); - fs.writeFileSync( path.join(bin, 'gslist'), [ '#!/bin/sh', - `echo "$*" >> '${calls}'`, - // A stone that changes state on its own: see stoneBecomes. - `if [ -f '${countdown}' ]; then`, - ` n=$(cat '${countdown}')`, - ` if [ "$n" -le 0 ]; then`, - ` rm -f '${countdown}'`, - ` if [ -f '${next}' ]; then mv '${next}' '${marker}'; else rm -f '${marker}'; fi`, - ` else echo $((n - 1)) > '${countdown}'; fi`, - 'fi', + `echo "$*" >> '${callsLog('gslist')}'`, // The shape of a real -cvl row: status, version, owner, pid, port, date, type, name. `[ -f '${marker}' ] && echo "$(cat '${marker}') 4.0.0.a4 me 1 1 Oct 13 15:57 Stone gemdb"`, 'exit 0', @@ -389,278 +384,176 @@ describe('starting the database', () => { ].join('\n'), ); fs.writeFileSync( - path.join(bin, 'startstone'), - `#!/bin/sh\nsleep 0.2\necho started >> '${starts}'\necho OK > '${marker}'\nexit 0\n`, + path.join(bin, 'topaz'), + `#!/bin/sh\necho "$*" >> '${callsLog('topaz')}'\necho "\${ELECTRON_RUN_AS_NODE:-unset}" >> '${callsLog('topazenv')}'\necho 0 > "$GEMDB_STATUS_FILE"\nexit 0\n`, ); fs.chmodSync(path.join(bin, 'gslist'), 0o755); - fs.chmodSync(path.join(bin, 'startstone'), 0o755); - return { starts }; + fs.chmodSync(path.join(bin, 'topaz'), 0o755); + // Run by the real node the wrapper recorded; records its arguments and + // what its stdin is, and starts the stone unless told to fail. + fs.mkdirSync(path.join(root, 'bin'), { recursive: true }); + fs.writeFileSync( + path.join(root, 'bin', 'gemdb-shell.js'), + [ + "const fs = require('fs');", + 'const stdin = fs.fstatSync(0);', + `const terminal = require('tty').isatty(0);`, + `fs.appendFileSync('${callsLog('shell')}', process.argv.slice(2).join(' ') + (terminal ? ' tty' : stdin.isCharacterDevice() ? ' null' : ' other') + '\\n');`, + `const fail = fs.existsSync('${path.join(root, 'ensure.fails')}');`, + "if (fail) { process.stderr.write('gemdb: the database could not be started.\\n'); process.exit(1); }", + `fs.writeFileSync('${marker}', 'OK');`, + '', + ].join('\n'), + ); } - /** - * Leave a stone at `status` that becomes `then` (or disappears, when - * undefined) after `calls` gslist calls — the wrapper's own pace rather than - * the clock's, since how long it takes to reach its stone checks varies. - */ - function stoneBecomes(status: string, then: string | undefined, calls: number): void { + function stoneIs(status: string): void { fs.writeFileSync(path.join(root, 'stone.up'), status); - if (then !== undefined) fs.writeFileSync(path.join(root, 'stone.next'), then); - fs.writeFileSync(path.join(root, 'stone.countdown'), String(calls)); - } - - function gslistCalls(): number { - const calls = path.join(root, 'gslist.calls'); - return fs.existsSync(calls) - ? fs.readFileSync(calls, 'utf8').split('\n').filter(Boolean).length - : 0; } - /** Run the wrapper on a path that does not exist: it exits after the - * stone-start block and before topaz, which is the part under test. */ - function runWrapper(): Promise { + function run(args: string[]): Promise<{ code: number; stderr: string }> { return new Promise((resolve) => { - execFile(cliPath(), [path.join(root, 'nothing-here.py')], (_error, _stdout, stderr) => - resolve(stderr), + // The editor's own environment may carry the variable under test. + const { ELECTRON_RUN_AS_NODE: _unused, ...env } = process.env; + execFile(cliPath(), args, { env }, (error, _stdout, stderr) => + resolve({ code: error ? Number(error.code) : 0, stderr }), ); }); } - it('locks on the same name the extension does', () => { - // Two doors start this stone: the extension on activation and this wrapper - // on any command. A lock each would not be a lock -- so if one side is - // renamed, this fails rather than quietly reopening the race. + /** A script file to run: arguments that exist, so only the start is under test. */ + function aProgram(): string { + const file = path.join(root, 'program.py'); + fs.writeFileSync(file, 'print(1)\n'); + return file; + } + + beforeEach(() => { writeCliScripts(ext); - expect(fs.readFileSync(cliPath(), 'utf8')).toContain(STONE_LOCK_NAME); + installStubs(); }); - it('takes over a lock whose owner is gone, rather than blocking for ever', async () => { - writeCliScripts(ext); - const { starts } = installEngineStubs(); - // Debris from a crash: a held lock naming a pid that is not running. - const lock = path.join(root, '.gemdb-stone.lock'); - fs.mkdirSync(lock); - fs.writeFileSync(path.join(lock, 'pid'), '999999\n'); + it('runs against a stone that answers without calling the shell program', async () => { + stoneIs('OK'); - await runWrapper(); + const { code } = await run([aProgram()]); - expect(fs.existsSync(starts)).toBe(true); - expect(fs.existsSync(lock)).toBe(false); + expect(code).toBe(0); + expect(readCalls('gslist')).toHaveLength(1); + expect(readCalls('shell')).toEqual([]); + expect(readCalls('topaz')).toHaveLength(1); }); - /** Make a path look ten seconds old, past the grace given to a half-written lock. */ - function backdate(target: string): void { - const then = new Date(Date.now() - 10_000); - fs.utimesSync(target, then, then); - } + it('hands a stone that is not running to the shell program, once, with no terminal on its stdin', async () => { + const { code } = await run([aProgram()]); - it('leaves alone a lock whose pid is not written yet', async () => { - // mkdir and the pid write are two steps, so a fresh empty lock is another - // process mid-claim. Removing it would start a second stone beside theirs. - writeCliScripts(ext); - const { starts } = installEngineStubs(); - const lock = path.join(root, '.gemdb-stone.lock'); - fs.mkdirSync(lock); - // The "other process" finishes: the stone comes up while the wrapper waits. - setTimeout(() => fs.writeFileSync(path.join(root, 'stone.up'), 'OK'), 300); - - await runWrapper(); - - expect(fs.existsSync(lock)).toBe(true); - expect(fs.existsSync(starts)).toBe(false); + expect(code).toBe(0); + expect(readCalls('shell')).toEqual(['--ensure-stone null']); + expect(readCalls('topaz')).toHaveLength(1); }); - it('takes over a lock that has had no pid for longer than any claim takes', async () => { - writeCliScripts(ext); - const { starts } = installEngineStubs(); - const lock = path.join(root, '.gemdb-stone.lock'); - fs.mkdirSync(lock); - backdate(lock); - - await runWrapper(); + it('leaves the shell program’s environment out of the program it then runs', async () => { + await run([aProgram()]); - expect(fs.existsSync(starts)).toBe(true); - expect(fs.existsSync(lock)).toBe(false); + expect(readCalls('shell')).toHaveLength(1); + expect(readCalls('topazenv')).toEqual(['unset']); }); - it('does not remove a lock that someone else took over while it was starting', async () => { - // If ours was judged stale and stolen mid-start, the lock is the thief's now. - writeCliScripts(ext); - const bin = path.join(expectedEnginePath(), 'bin'); - installEngineStubs(); - const lock = path.join(root, '.gemdb-stone.lock'); - fs.writeFileSync( - path.join(bin, 'startstone'), - `#!/bin/sh\necho ${process.pid} > '${lock}/pid'\necho OK > '${path.join(root, 'stone.up')}'\nexit 0\n`, - ); - - await runWrapper(); + it('hands -c and -m runs over the same way', async () => { + await run(['-c', 'print(1)']); + fs.rmSync(path.join(root, 'stone.up')); + await run(['-m', 'json.tool']); - expect(fs.readFileSync(path.join(lock, 'pid'), 'utf8').trim()).toBe(String(process.pid)); + expect(readCalls('shell')).toEqual(['--ensure-stone null', '--ensure-stone null']); }); - /** The stone lock and steal guard a stealer left when it died inside the guard. */ - function leaveDeadStealersDebris(): void { - const lock = path.join(root, '.gemdb-stone.lock'); - fs.mkdirSync(lock); - fs.writeFileSync(path.join(lock, 'pid'), '999999\n'); - fs.mkdirSync(`${lock}.steal`); - backdate(`${lock}.steal`); - } + // Whatever is not plain `ok` is the shell program's to judge: waiting for a + // stone that is starting, refusing one that is stopping, letting a stone + // that reads "exe deleted" go on to its login. + for (const status of ['Startup', 'Recovery', 'Shutdown', 'exe deleted', 'frozen']) { + it(`hands a stone that reads "${status}" to the shell program`, async () => { + stoneIs(status); + + await run([aProgram()]); - /** What is left beside the stone lock: the guard, or a tomb it was moved to. */ - function guardLeftovers(): string[] { - return fs.readdirSync(root).filter((name) => name.startsWith('.gemdb-stone.lock.steal')); + expect(readCalls('shell')).toEqual(['--ensure-stone null']); + }); } - it('takes over the steal guard of a stealer that died inside it', async () => { - writeCliScripts(ext); - const { starts } = installEngineStubs(); - leaveDeadStealersDebris(); + it('stops where the shell program fails, showing what it said, and runs nothing', async () => { + fs.writeFileSync(path.join(root, 'ensure.fails'), ''); - await runWrapper(); + const { code, stderr } = await run([aProgram()]); - expect(fs.existsSync(starts)).toBe(true); - expect(guardLeftovers()).toEqual([]); + expect(code).toBe(1); + expect(stderr).toContain('gemdb: the database could not be started.'); + expect(readCalls('topaz')).toEqual([]); }); - it('starts the stone once when several commands race over a dead stealer’s guard', async () => { - // A smoke test of the takeover under contention, not a reproduction of the - // race it closes: that window is too narrow to hit with natural timing, - // and this passes against delete-in-place too. No assertion on leftovers: - // a guard put back in the rare three-stealer case can outlive the run. + it('hands an external database to the shell program too, when it is not answering', async () => { + __setSetting('gemdb.externalDatabase.gemstone', expectedEnginePath()); + __setSetting('gemdb.externalDatabase.globalDirectory', root); + __setSetting('gemdb.externalDatabase.stone', 'gemdb'); writeCliScripts(ext); - const { starts } = installEngineStubs(); - leaveDeadStealersDebris(); + // Staging put the extension's own bundle over the stand-in. + installStubs(); + stoneIs('Startup'); - await Promise.all([runWrapper(), runWrapper(), runWrapper(), runWrapper()]); + await run([aProgram()]); - expect(fs.readFileSync(starts, 'utf8').trim().split('\n')).toHaveLength(1); + expect(readCalls('shell')).toEqual(['--ensure-stone null']); }); - it('starts the stone once when several commands race', async () => { - writeCliScripts(ext); - const { starts } = installEngineStubs(); + it('refuses without a shell program rather than starting the database itself', async () => { + fs.rmSync(path.join(root, 'bin', 'gemdb-shell.js')); - await Promise.all([runWrapper(), runWrapper(), runWrapper(), runWrapper()]); + const { code, stderr } = await run([aProgram()]); - const started = fs.existsSync(starts) - ? fs.readFileSync(starts, 'utf8').trim().split('\n').filter(Boolean).length - : 0; - expect(started).toBe(1); - }); - it('waits for a stone that is still starting instead of starting a second one', async () => { - // Listed at Startup with nobody holding the lock: a crash being recovered, - // or a start by a GemDB that does not wait for OK. - writeCliScripts(ext); - const { starts } = installEngineStubs(); - // Still starting for the first check, the re-check in the lock, and the - // first wait for it. - stoneBecomes('Startup', 'OK', 3); - - const stderr = await runWrapper(); - - expect(fs.existsSync(starts)).toBe(false); - expect(stderr).toContain('waiting for the database to finish starting'); - expect(stderr).toContain("can't open file"); + expect(code).toBe(1); + expect(stderr).toContain('this installation has no shell program'); + expect(readCalls('topaz')).toEqual([]); }); - it('names the stone log when a stone that was starting disappears', async () => { - writeCliScripts(ext); - const { starts } = installEngineStubs(); - stoneBecomes('Startup', undefined, 3); + it('refuses without a Node runtime to run the shell program', async () => { + const wrapper = fs.readFileSync(cliPath(), 'utf8'); + fs.writeFileSync( + cliPath(), + wrapper + .replace(`NODE="${process.execPath}"`, 'NODE="/nowhere/node"') + .replace('command -v node', 'command -v no-such-node-anywhere'), + ); - const stderr = await runWrapper(); + const { code, stderr } = await run([aProgram()]); - expect(fs.existsSync(starts)).toBe(false); - expect(stderr).toMatch(/stopped while it was starting\. See .*gemdb\.log/); + expect(code).toBe(1); + expect(stderr).toContain('no Node runtime found'); + expect(readCalls('topaz')).toEqual([]); }); - // Statuses the engine prints that are neither OK nor starting. They went on - // to the login before GemDB waited for OK, and still do: "exe deleted" is a - // stone left running when the engine tree was replaced, and it takes logins. - for (const status of ['exe deleted', 'frozen', 'full']) { - it(`goes on to the login for a stone that reads "${status}", asking once`, async () => { - writeCliScripts(ext); - const { starts } = installEngineStubs(); - fs.writeFileSync(path.join(root, 'stone.up'), status); - - const stderr = await runWrapper(); - - expect(fs.existsSync(starts)).toBe(false); - expect(gslistCalls()).toBe(1); - expect(stderr).toContain("can't open file"); - }); - } - - it('says a stone at Shutdown is shutting down, at once, rather than waiting on it', async () => { - writeCliScripts(ext); - const { starts } = installEngineStubs(); - fs.writeFileSync(path.join(root, 'stone.up'), 'Shutdown'); - - const stderr = await runWrapper(); + it('fails a file that is not there without looking for the stone or starting it', async () => { + const { code, stderr } = await run([path.join(root, 'nothing-here.py')]); - expect(fs.existsSync(starts)).toBe(false); - expect(gslistCalls()).toBe(1); - expect(stderr).toContain('the database is shutting down'); - expect(stderr).not.toContain("can't open file"); + expect(code).toBe(2); + expect(stderr).toContain("can't open file"); + expect(readCalls('gslist')).toEqual([]); + expect(readCalls('shell')).toEqual([]); }); - describe('for a database this machine’s administrator runs', () => { - /** The stubbed engine, configured as someone else's, with a stone named as the stubs name it. */ - function external(): void { - __setSetting('gemdb.externalDatabase.gemstone', expectedEnginePath()); - __setSetting('gemdb.externalDatabase.globalDirectory', root); - __setSetting('gemdb.externalDatabase.stone', 'gemdb'); - } - - const lock = (): string => path.join(root, STONE_LOCK_NAME); - - it('starts nothing and takes no lock when the stone is not running', async () => { - external(); - writeCliScripts(ext); - const { starts } = installEngineStubs(); - - const stderr = await runWrapper(); - - expect(fs.existsSync(starts)).toBe(false); - expect(fs.existsSync(lock())).toBe(false); - expect(stderr).toContain("It is run by this machine's administrator"); - }); - - it('waits for a stone that is still starting, and never starts one', async () => { - external(); - writeCliScripts(ext); - const { starts } = installEngineStubs(); - stoneBecomes('Startup', 'OK', 2); - - const stderr = await runWrapper(); + it('fails -c with no code without looking for the stone or starting it', async () => { + const { code, stderr } = await run(['-c']); - expect(fs.existsSync(starts)).toBe(false); - expect(fs.existsSync(lock())).toBe(false); - expect(stderr).toContain('waiting for the database to finish starting'); - expect(stderr).toContain("can't open file"); - }); - - it('does not start a stone that disappears while starting, or point at a log of ours', async () => { - external(); - writeCliScripts(ext); - const { starts } = installEngineStubs(); - stoneBecomes('Startup', undefined, 2); - - const stderr = await runWrapper(); - - expect(fs.existsSync(starts)).toBe(false); - expect(stderr).toContain('stopped while it was starting.'); - expect(stderr).not.toContain('gemdb.log'); - }); + expect(code).toBe(2); + expect(stderr).toContain('-c requires an argument'); + expect(readCalls('gslist')).toEqual([]); + expect(readCalls('shell')).toEqual([]); + }); - it('generates no way to reach startstone at all', () => { - external(); - writeCliScripts(ext); + it('carries nothing of its own that starts a stone or takes the lock', () => { + const wrapper = fs.readFileSync(cliPath(), 'utf8'); - const wrapper = fs.readFileSync(cliPath(), 'utf8'); - expect(wrapper).not.toContain('bin/startstone'); - expect(wrapper).not.toContain(STONE_LOCK_NAME); - }); + // Every start goes through the shell program, so there is one lock + // mechanism and one set of guards, not a bash copy that drifts. + expect(wrapper).not.toContain('startstone'); + expect(wrapper).not.toContain(STONE_LOCK_NAME); }); }); diff --git a/src/__tests__/cliStart.test.ts b/src/__tests__/cliStart.test.ts new file mode 100644 index 0000000..f4f8107 --- /dev/null +++ b/src/__tests__/cliStart.test.ts @@ -0,0 +1,88 @@ +import { beforeEach, describe, expect, it, vi } from 'vitest'; + +/** + * What `gemdb` does to bring the database up, with the engine's side faked: + * which checks run before a start, and whether the listener is asked for. + */ + +let external: object | undefined; +const ensureProcesses = vi.fn(async (_opts: unknown) => ({ + startedStone: true, + startedNetldi: false, +})); +const assertExternalDatabaseUp = vi.fn(async (_opts: unknown) => {}); +const assertSafeToStart = vi.fn(); + +vi.mock('../config', () => ({ externalDatabase: () => external })); +vi.mock('../database', () => ({ assertSafeToStart: () => assertSafeToStart() })); +vi.mock('../processes', () => ({ + ensureProcesses: (opts: unknown) => ensureProcesses(opts), + assertExternalDatabaseUp: (opts: unknown) => assertExternalDatabaseUp(opts), +})); + +const { bringUp } = await import('../cliStart'); + +beforeEach(() => { + external = undefined; + ensureProcesses.mockClear(); + assertExternalDatabaseUp.mockClear(); + assertSafeToStart.mockClear(); +}); + +describe('bringing the database up for the gemdb command', () => { + it('runs the same guards before a start that the editor does', async () => { + const report = vi.fn(); + + await bringUp({ listener: false, report }); + + const opts = ensureProcesses.mock.calls[0]?.[0] as { + beforeStoneStart: () => void; + report: unknown; + }; + opts.beforeStoneStart(); + expect(assertSafeToStart).toHaveBeenCalledTimes(1); + expect(opts.report).toBe(report); + }); + + it('passes on that the caller already looked', async () => { + await bringUp({ listener: false, report: () => {}, skipQuickCheck: true }); + + expect(ensureProcesses.mock.calls[0]?.[0]).toMatchObject({ skipQuickCheck: true }); + }); + + it('starts the stone alone for a file run, and the listener too for the Shell', async () => { + await bringUp({ listener: false, report: () => {} }); + await bringUp({ listener: true, report: () => {} }); + + expect( + ensureProcesses.mock.calls.map(([opts]) => (opts as { listener: boolean }).listener), + ).toEqual([false, true]); + }); + + it('only checks an external database, whichever way it is asked', async () => { + external = { stone: 'gs64stone' }; + const report = vi.fn(); + + await bringUp({ listener: false, report }); + + expect(ensureProcesses).not.toHaveBeenCalled(); + expect(assertExternalDatabaseUp).toHaveBeenCalledWith({ listener: false, report }); + }); + + it('says why an external database that is down cannot be used', async () => { + external = { stone: 'gs64stone' }; + assertExternalDatabaseUp.mockRejectedValueOnce(new Error('The database is not running')); + + await expect(bringUp({ listener: true, report: () => {} })).rejects.toThrow( + 'The database is not running', + ); + }); + + it('lets a refused start through to the caller, which says it', async () => { + ensureProcesses.mockRejectedValueOnce(new Error('delete this directory')); + + await expect(bringUp({ listener: false, report: () => {} })).rejects.toThrow( + 'delete this directory', + ); + }); +}); diff --git a/src/__tests__/externalDatabase.test.ts b/src/__tests__/externalDatabase.test.ts index 849795e..cce141a 100644 --- a/src/__tests__/externalDatabase.test.ts +++ b/src/__tests__/externalDatabase.test.ts @@ -263,7 +263,9 @@ describe('the gemdb command', () => { expect(script).toContain('export GEMSTONE_GLOBAL_DIR="/srv/gemstone"'); expect(script).toContain(`export GEMDB_EXTERNAL_PASSWORD_FILE="${file}"`); expect(script).not.toContain('GEMSTONE_SYS_CONF'); - expect(script).toContain("It is run by this machine's administrator."); + // Starting is the shell program's judgement, which for an external + // database only checks. + expect(script).not.toContain('startstone'); const driver = path.join(path.dirname(cliPath()), 'gemdb-run.tpz'); const text = fs.readFileSync(driver, 'utf8'); diff --git a/src/__tests__/lock.test.ts b/src/__tests__/lock.test.ts index 10a2aaa..3eecc5e 100644 --- a/src/__tests__/lock.test.ts +++ b/src/__tests__/lock.test.ts @@ -290,6 +290,41 @@ describe('withStoneLock', () => { expect(fs.existsSync(stoneLock())).toBe(true); }); + it('says once that it is waiting when the lock is held', async () => { + fs.mkdirSync(stoneLock()); + writeStonePid(`${process.ppid}\n`); + setTimeout(() => fs.rmSync(stoneLock(), { recursive: true }), 60); + const report = vi.fn(); + + await withStoneLock(async () => 'started', { pollMs: 10, report }); + + expect(report).toHaveBeenCalledTimes(1); + expect(report).toHaveBeenCalledWith( + 'Waiting for another process to finish starting the database…', + ); + }); + + it('says it is this window that is starting the database when it is', async () => { + fs.mkdirSync(stoneLock()); + writeStonePid(`${process.pid}\n`); + setTimeout(() => fs.rmSync(stoneLock(), { recursive: true }), 40); + const report = vi.fn(); + + await withStoneLock(async () => 'started', { pollMs: 10, report }); + + expect(report).toHaveBeenCalledWith( + 'This window is already starting the database; waiting for it…', + ); + }); + + it('says nothing when it gets the lock at once', async () => { + const report = vi.fn(); + + await withStoneLock(async () => 'started', { report }); + + expect(report).not.toHaveBeenCalled(); + }); + it('fails naming the lock when a live holder never lets go', async () => { fs.mkdirSync(stoneLock()); writeStonePid(`${process.ppid}\n`); diff --git a/src/__tests__/startProcesses.test.ts b/src/__tests__/startProcesses.test.ts index 15e9703..ba356b5 100644 --- a/src/__tests__/startProcesses.test.ts +++ b/src/__tests__/startProcesses.test.ts @@ -170,6 +170,24 @@ describe('ensureProcesses', () => { expect(gslistCalls).toBe(3); }); + it('asks gslist one time fewer on a cold start when the caller just looked', async () => { + listings = ['', STONE_UP]; + + const started = await ensureProcesses({ listener: false, skipQuickCheck: true }); + + expect(started).toEqual({ startedStone: true, startedNetldi: false }); + expect(gslistCalls).toBe(2); + }); + + it('still takes the lock and starts nothing when the stone turns out to be up', async () => { + listings = [STONE_UP]; + + const started = await ensureProcesses({ listener: false, skipQuickCheck: true }); + + expect(started).toEqual({ startedStone: false, startedNetldi: false }); + expect(spawned).toEqual([]); + }); + it('starts both, guards first, when neither is up', async () => { // Before the lock, inside it, then the stone after starting it. listings = ['', '', STONE_UP]; @@ -294,6 +312,51 @@ describe('ensureProcesses', () => { }); }); +describe('starting the stone alone', () => { + it('starts the stone, guards first, and never the listener', async () => { + listings = ['', '', STONE_UP]; + const beforeStoneStart = vi.fn(() => { + expect(spawned).toEqual([]); + }); + + const started = await ensureProcesses({ beforeStoneStart, listener: false }); + + expect(started).toEqual({ startedStone: true, startedNetldi: false }); + expect(beforeStoneStart).toHaveBeenCalledTimes(1); + expect(spawned).toEqual(['startstone']); + }); + + it('takes no lock for a stone that is up with no listener', async () => { + listings = [STONE_UP]; + + const started = await ensureProcesses({ listener: false }); + + expect(started).toEqual({ startedStone: false, startedNetldi: false }); + expect(gslistCalls).toBe(1); + expect(satisfied).toBeUndefined(); + expect(spawned).toEqual([]); + }); + + it('still waits for a stone that is starting, and starts no listener beside it', async () => { + listings = [STONE_STARTING, STONE_STARTING, STONE_UP]; + + const started = await ensureProcesses({ listener: false }); + + expect(started).toEqual({ startedStone: false, startedNetldi: false }); + expect(spawned).toEqual([]); + }); + + it('stops waiting for the lock once the stone is up, listener or not', async () => { + listings = [STONE_STARTING]; + const done = ensureProcesses({ listener: false }); + await new Promise((resolve) => setImmediate(resolve)); + listings = [STONE_UP]; + + expect(await satisfied?.()).toBe(true); + await done; + }); +}); + describe('a stone that is still starting', () => { it('is listed, but not ready for a login', () => { const processes = parseGslist(STONE_STARTING); diff --git a/src/cli.ts b/src/cli.ts index c93d1bf..222ea6f 100644 --- a/src/cli.ts +++ b/src/cli.ts @@ -117,8 +117,10 @@ const TOPAZ_INIT = `output push $GEMDB_TOPAZ_LOG only * machine's is kept — the same pruning .vscodeignore applies to the copy the * extension itself ships. A build without the bundle (a fresh checkout that * has not run `npm run bundle`) stages nothing and says so; the wrapper then - * reports the missing shell at run time rather than failing here, because the - * file and module modes still work without it. + * reports the missing shell at run time rather than failing here, because a + * file or module run against a database that is already up works without it. + * A cold start does not: it is the shell program's `--ensure-stone` that starts + * the database. */ function stageShell(extensionPath: string): void { const bundle = path.join(extensionPath, 'out', 'gemdb-shell.js'); @@ -483,12 +485,11 @@ case "\${1:-}" in exit 0 ;; esac -# No arguments: the GemDB Shell — the same Python prompt the editor opens, -# on this terminal. It brings the database up itself (a shell session needs -# the listener too), so it runs before the stone check below. The editor's -# own Node runtime is recorded here; PATH is the fallback for when the editor -# has moved since this file was written. -if [ "$#" -eq 0 ]; then +# The shell program runs under the editor's own Node runtime, recorded here +# when this file was written; PATH is the fallback for when the editor has +# moved since. Both the GemDB Shell and a cold start (below) run it, so both +# find it the same way and fail with the same words. +find_node() { SHELL_JS="$ROOT/bin/gemdb-shell.js" if [ ! -f "$SHELL_JS" ]; then echo "gemdb: this installation has no shell program. Open VS Code so GemDB can finish setting up, then retry." >&2 @@ -503,180 +504,57 @@ if [ "$#" -eq 0 ]; then export GEMDB_ENGINE_VERSION="${engineVersion()}" export GEMDB_ABORT_IDLE_MINUTES="${abortIdleSessionsAfterMinutes()}" export ELECTRON_RUN_AS_NODE=1 +} + +# No arguments: the GemDB Shell — the same Python prompt the editor opens, +# on this terminal. It brings the database up itself (a shell session needs +# the listener too), so it runs before the stone check below. +if [ "$#" -eq 0 ]; then + find_node exec "$NODE" "$SHELL_JS" fi +# Arguments that are wrong fail here, with a message and exit code shaped like +# CPython's, before anything starts a database for them: a typo must not cost +# a start. Only the checks are here; -c's file is made after the start below. +if [ "$1" = "-c" ]; then + [ "$#" -ge 2 ] || { echo "gemdb: -c requires an argument" >&2; exit 2; } +elif [ "$1" != "-m" ] && [ ! -e "$1" ]; then + echo "gemdb: can't open file '$1': No such file or directory" >&2 + exit 2 +fi + # The database must be up — a linked gem still needs the stone. Starting it # here is the same judgement the editor makes: running Python is the request. # -# Under a lock, because this is a check-then-act and EVERY gemdb command runs -# it. Two commands close together — a terminal one while the editor starts the -# database, say — would otherwise both see no stone and both start one, and -# nothing downstream refuses the second: two stoned processes held one -# extent0.dbf read-write for a week on a developer machine. It went unnoticed -# because gslist keys Stone rows by name, so it shows ONE row for two stones -# with the same name; the second only became visible when the first stopped -# and it took over the row. -# -# mkdir is the primitive: it creates or fails, with no window between the test -# and the claim, and it is there on a stock macOS where flock is not. -# -# What the stone's status says decides what to do. Not listed: start one. A -# stone still at Startup -- recovering from a crash, or another process's start -# not finished -- is listed, and a second startstone beside it is the -# two-stones case above, so it is waited for until it is past Startup, since a -# login now would fail. One at Shutdown is being stopped, and saying so beats -# waiting a minute for it. Anything else goes on to the login, which judges it: -# a stone left running when the engine tree was replaced reads "exe deleted" -# and takes logins. +# The wrapper only recognises the common case. A stone that answers is just +# run against, which is every run after the first, and costs one gslist. Every +# other case -- no stone, one still starting or shutting down, a status that +# needs a closer look, an external database -- goes to the shell program, which +# makes the judgement the editor and the GemDB Shell make: the same lock, the +# same pre-start guards (engine version, NFS, space), the same wait. It starts +# the stone and nothing else: a file run is a linked topaz and never goes +# through NetLDI. Not run on a terminal's stdin: --ensure-stone reads none, and +# a shell program old enough not to know the flag stops at "needs a terminal" +# rather than opening a Shell in the middle of a file run. # # The status as gslist -cvl lists it, lowercased (the engine prints "OK", # "Startup", "frozen"), or nothing when the stone is not listed. One call per # question: -v asks the stone itself, which takes about five seconds when it is # frozen (measured). A row is status, version, owner, pid, port, three date # fields, type and name, and "exe deleted" is the one two-word status. A row of -# any other shape is still this stone, so it answers something: nothing would -# mean "start one". +# any other shape is still this stone, so it answers something other than "ok". stone_status() { "$GEMSTONE/bin/gslist" -cvl 2>/dev/null | awk -v s="$STONE" '$(NF-1) == "Stone" && $NF == s { print tolower(NF == 10 ? $1 : NF == 11 ? $1 " " $2 : "listed"); exit }' } -starting() { case "$1" in startup|recovery) return 0 ;; *) return 1 ;; esac; } -shutting_down() { - echo "gemdb: the database is shutting down. Run this again once it has stopped." >&2 - exit 1 -} -# Wait while the stone is starting, from the status last seen ($1). $2 is where -# to look when it fails. Waits are timed by the clock, not by counting polls: -# each poll is a gslist call, and those vary from milliseconds to seconds. -wait_for_stone() { - local st="$1" deadline=$(( $(date +%s) + 60 )) said="" - while starting "$st"; do - if [ "$(date +%s)" -ge "$deadline" ]; then - echo "gemdb: the database has been starting for over a minute.$2" >&2 - exit 1 - fi - [ -z "$said" ] && said=1 && echo "gemdb: waiting for the database to finish starting…" >&2 - sleep 0.1 - st=$(stone_status) - done - [ "$st" = shutdown ] && shutting_down - if [ -z "$st" ]; then - echo "gemdb: the database stopped while it was starting.$2" >&2 - exit 1 - fi -} - -STATUS=$(stone_status) -[ "$STATUS" = shutdown ] && shutting_down -${ - external - ? `# This machine's administrator runs the database, so a stopped stone is theirs -# to start: starting it here would run it as this user. So no lock and no -# start, only a wait for one that is still starting. -if [ -z "$STATUS" ]; then - echo "gemdb: the database (stone $STONE) is not running. It is run by this machine's administrator." >&2 - exit 1 -fi -wait_for_stone "$STATUS" "" -` - : `if [ -z "$STATUS" ] || starting "$STATUS"; then - STONE_LOCK="$ROOT/.gemdb-stone.lock" - # mkdir and the pid write are two steps, here and in the extension, so a - # lock with no pid yet is one being taken right now. It is debris only once - # it is older than a few seconds. stat spells mtime differently on GNU and - # macOS; GNU is tried first because its -f means something else. - older_than_5s() { - local m; m=$(stat -c %Y "$1" 2>/dev/null || stat -f %m "$1" 2>/dev/null) || return 1 - [ -n "$m" ] && [ $(( $(date +%s) - m )) -gt 5 ] - } - stone_lock_is_stale() { - [ -d "$STONE_LOCK" ] || return 1 - local owner; owner=$(cat "$STONE_LOCK/pid" 2>/dev/null) - case "$owner" in - ''|*[!0-9]*) older_than_5s "$STONE_LOCK" ;; - *) ! kill -0 "$owner" 2>/dev/null ;; - esac - } - release_stone_lock() { - [ "$(cat "$STONE_LOCK/pid" 2>/dev/null)" = "$$" ] && rm -rf "$STONE_LOCK" - } - # A steal guard older than a few seconds is debris from a stealer that died - # inside it. Taking it over is the extension's scheme: move it to a name of - # our own and check it is the guard we judged old (same inode and mtime), - # since deleting it in place could delete a fresh one someone took - # meanwhile. A live one moved by mistake is replaced with mkdir, never by - # moving ours back: mv onto a directory moves into it, and rmdir could never - # clear the guard again. This closes the two-stealer race; three at once - # over a dead guard can still pair up, until Step 5's kernel locks. - inode_and_mtime() { stat -c '%i %Y' "$1" 2>/dev/null || stat -f '%i %m' "$1" 2>/dev/null; } - take_stale_guard() { - local guard="$STONE_LOCK.steal" seen tomb took=1 - seen=$(inode_and_mtime "$guard") && [ -n "$seen" ] || return 1 - [ $(( $(date +%s) - \${seen#* } )) -gt 5 ] || return 1 - tomb="$guard.$$.$RANDOM" - mv "$guard" "$tomb" 2>/dev/null || return 1 - if [ "$(inode_and_mtime "$tomb")" = "$seen" ]; then - mkdir "$guard" 2>/dev/null && took=0 - else - mkdir "$guard" 2>/dev/null - fi - rm -rf "$tomb" - return "$took" - } - held="" - lock_deadline=$(( $(date +%s) + 10 )) - while [ "$(date +%s)" -lt "$lock_deadline" ]; do - if mkdir "$STONE_LOCK" 2>/dev/null; then - held=yes - echo $$ > "$STONE_LOCK/pid" 2>/dev/null - break - fi - # Held by someone. A lock whose owner is gone is debris from a crash and - # must not block every later command; one whose owner is alive is a start - # already in flight, and waiting for it is the whole point. Stealing goes - # through the same .steal guard the extension takes, re-checked inside it, - # so two stealers cannot both remove a lock and the second remove the - # first one's fresh claim. - if stone_lock_is_stale; then - if mkdir "$STONE_LOCK.steal" 2>/dev/null || take_stale_guard; then - stone_lock_is_stale && rm -rf "$STONE_LOCK" - rmdir "$STONE_LOCK.steal" 2>/dev/null - continue - fi - fi - STATUS=$(stone_status) - [ -n "$STATUS" ] && ! starting "$STATUS" && break - sleep 0.1 - done - if [ -n "$held" ]; then - # Re-check inside the lock. Between our first check and this one, whoever - # we queued behind may have started it -- which is the case the bare - # check-then-act got wrong. Listed, not ready: one still starting is - # waited for below, never started again. - STATUS=$(stone_status) - if [ -z "$STATUS" ]; then - echo "gemdb: starting the database…" >&2 - if ! "$GEMSTONE/bin/startstone" -l "$ROOT/db/log/$STONE.log" "$STONE" >/dev/null 2>&1; then - release_stone_lock - echo "gemdb: the database at $ROOT could not be started. See $ROOT/db/log/$STONE.log" >&2 - exit 1 - fi - STATUS=$(stone_status) - fi - release_stone_lock - elif [ -z "$STATUS" ]; then - echo "gemdb: the database is not running and another process has been starting it for" >&2 - echo "gemdb: ten seconds. See $ROOT/db/log/$STONE.log, or remove $STONE_LOCK if nothing is." >&2 - exit 1 - fi - - # Ours or someone else's, a stone still starting is waited for: a login now - # would fail. A minute, as the extension waits. - wait_for_stone "$STATUS" " See $ROOT/db/log/$STONE.log" +STONE_STATUS=$(stone_status) +if [ "$STONE_STATUS" != ok ]; then + # A subshell: find_node's exports (ELECTRON_RUN_AS_NODE above all) are for + # the shell program, not for the user's program the run starts next, which + # would otherwise see them on a cold start and not on a warm one. + ( find_node; "$NODE" "$SHELL_JS" --ensure-stone &2; exit 2; } CODE_FILE="$(mktemp "\${TMPDIR:-/tmp}/gemdb-c-XXXXXX").py" printf '%s\\n' "$2" > "$CODE_FILE" shift 2 set -- "$CODE_FILE" "$@" fi -# A file that is not there should fail here, with a message and exit code -# shaped like CPython's, not as a Smalltalk stack. -if [ "$1" != "-m" ] && [ ! -e "$1" ]; then - echo "gemdb: can't open file '$1': No such file or directory" >&2 - exit 2 -fi - # topaz cannot carry an exit status out of a run block, so the driver writes # it to a file and this wrapper becomes the exit code. See gemdb-run.tpz. # Everything topaz itself prints goes to GEMDB_TOPAZ_LOG instead of the @@ -763,13 +633,13 @@ TOPAZ_RUNNING=yes topaz -L -q -I "$ROOT/bin/gemdb-run-init.tpz" -P "$ROOT/bin/gemdb-run.tpz" ${TOPAZ_TUNING} -- "$@" RC=$? TOPAZ_RUNNING="" -STATUS="$(cat "$GEMDB_STATUS_FILE" 2>/dev/null || true)" -if [ -z "$STATUS" ]; then +RUN_STATUS="$(cat "$GEMDB_STATUS_FILE" 2>/dev/null || true)" +if [ -z "$RUN_STATUS" ]; then sed -e '/^topaz[^>]*> run$/,/^%$/d' -e '/^topaz[^>]*> /d' "$GEMDB_TOPAZ_LOG" >&2 fi -case "$STATUS" in +case "$RUN_STATUS" in ''|*[!0-9]*) [ "$RC" -ne 0 ] && exit "$RC"; exit 1 ;; - *) exit "$STATUS" ;; + *) exit "$RUN_STATUS" ;; esac `; diff --git a/src/cliMain.ts b/src/cliMain.ts index a92d1b3..4ededd8 100644 --- a/src/cliMain.ts +++ b/src/cliMain.ts @@ -1,7 +1,6 @@ -import { abortIdleSessionsAfterMinutes, externalDatabase } from './config'; -import { assertSafeToStart } from './database'; +import { abortIdleSessionsAfterMinutes } from './config'; import { errorMessage } from './log'; -import { assertExternalDatabaseUp, ensureProcesses } from './processes'; +import { bringUp } from './cliStart'; import { PyRepl, ReplSession } from './pyRepl'; import { runPythonInSession } from './pythonQueries'; import { GciSession, setInputHandler } from './session'; @@ -23,35 +22,25 @@ import { GciSession, setInputHandler } from './session'; */ /** - * Bring the database up, the same judgement the wrapper makes for a file run: - * asking for the shell is asking for a running database. Unlike the wrapper's - * file mode — a linked gem, needing only the stone — a shell session arrives - * through the listener, so both are ensured. Both go through - * `ensureProcesses`, the same locked path the window takes, so a Shell opened while a window is - * starting the database waits for it rather than racing it. The stone gets the - * window's pre-start guards too, all but statmonitor, whose configuration - * comes from editor settings the Shell does not have: `system.conf` keeps - * whatever the window's last start wrote. - * - * An external database is only checked, as the window checks it (waiting for - * a stone still starting): GemDB starts none of it. Errors are reported rather than thrown: the shell prompt (or the - * exit code) is the caller's answer. + * What a step of bringing the database up says on stderr: `gemdb: starting the + * database…`. `eol` is `\r\n` once raw mode may be on, where a bare `\n` would + * stairstep the message. */ -function reportStart(message: string): void { - process.stderr.write(`gemdb: ${message.charAt(0).toLowerCase()}${message.slice(1)}\r\n`); +function reporter(eol: string): (message: string) => void { + return (message) => + process.stderr.write(`gemdb: ${message.charAt(0).toLowerCase()}${message.slice(1)}${eol}`); } +/** + * Bring the database up for the Shell, which asking for is asking for a + * running one: unlike a file run (a linked gem, needing only the stone) a + * Shell session arrives through the listener, so both are ensured. See + * {@link bringUp}. Errors are reported rather than thrown: the shell prompt + * (or the exit code) is the caller's answer. + */ async function ensureDatabase(): Promise { try { - const external = externalDatabase(); - if (external) { - await assertExternalDatabaseUp({ listener: true, report: reportStart }); - } else { - await ensureProcesses({ - beforeStoneStart: assertSafeToStart, - report: reportStart, - }); - } + await bringUp({ listener: true, report: reporter('\r\n') }); return true; } catch (e) { // Raw mode may be on by now, so bare `\n` would stairstep the message. @@ -60,6 +49,23 @@ async function ensureDatabase(): Promise { } } +/** + * `--ensure-stone`: what the wrapper runs when a file, `-c` or `-m` run finds + * the stone down. Starts the stone alone, since file mode never goes through + * NetLDI, and exits 0 or 1. Runs before anything looks at stdin: the wrapper + * hands it `/dev/null`, and it is no terminal program. + */ +async function ensureStone(): Promise { + try { + // The wrapper only runs this after finding the stone not answering. + await bringUp({ listener: false, report: reporter('\n'), skipQuickCheck: true }); + } catch (e) { + process.stderr.write(`gemdb: ${errorMessage(e)}\n`); + process.exit(1); + } + process.exit(0); +} + /** * How often the shell checks whether its session has been idle long enough to * refresh. The setting is in minutes, so a minute is fine-grained enough. @@ -102,6 +108,7 @@ function login(): ReplSession { } async function main(): Promise { + if (process.argv[2] === '--ensure-stone') return ensureStone(); if (!process.stdin.isTTY || !process.stdout.isTTY) { process.stderr.write( 'gemdb: the GemDB Shell needs a terminal. To run a program, pass a file, -m, or -c.\n', diff --git a/src/cliStart.ts b/src/cliStart.ts new file mode 100644 index 0000000..fe8b5bf --- /dev/null +++ b/src/cliStart.ts @@ -0,0 +1,39 @@ +import { externalDatabase } from './config'; +import { assertSafeToStart } from './database'; +import { assertExternalDatabaseUp, ensureProcesses } from './processes'; + +/** + * Bring the database up for the `gemdb` command: what the GemDB Shell does at + * startup, and what the wrapper's `--ensure-stone` does for a file, `-c` or + * `-m` run that finds the stone down. + * + * Its own module, rather than a function in `cliMain.ts`, because that file + * runs `main()` as it is imported and cannot be loaded by a test. + * + * Own database: the same `ensureProcesses` the window uses, with the stone's + * pre-start guards (engine version, NFS, space, room for the extent) run + * before any start. `listener` says whether the caller opens sessions through + * NetLDI. The Shell does; a file run is a linked topaz and never does, so it + * starts the stone alone. + * + * External database: only checked, waiting for a stone that is still + * starting, since GemDB starts none of it. + * + * Throws with a sentence for the user; `report` hears each step as it begins. + */ +export async function bringUp(opts: { + listener: boolean; + report: (message: string) => void; + skipQuickCheck?: boolean; +}): Promise { + if (externalDatabase()) { + await assertExternalDatabaseUp(opts); + return; + } + await ensureProcesses({ + beforeStoneStart: assertSafeToStart, + report: opts.report, + listener: opts.listener, + skipQuickCheck: opts.skipQuickCheck, + }); +} diff --git a/src/lock.ts b/src/lock.ts index 0186ac5..bc23a0d 100644 --- a/src/lock.ts +++ b/src/lock.ts @@ -481,12 +481,15 @@ function safeRead(file: string): string | undefined { } /** - * The name of the stone lock, shared with the generated `gemdb` wrapper. + * The name of the stone lock, shared with older `gemdb` wrappers. * - * Exported because the wrapper takes the same lock in shell, and a lock each - * would not be a lock at all: both doors start the same stone -- the editor on - * activation, a terminal command on any invocation -- and two commands close - * together is all it takes. Two stoned processes held one extent0.dbf + * The current wrapper takes it through `ensureProcesses` (`gemdb-shell.js + * --ensure-stone`); wrappers written by 1.8.0 and earlier still take it in + * bash, and one of those can be on the same root path, written by another + * editor or version. So the name, the directory format and the pid file must + * not change until those wrappers are gone. A lock each would not be a lock at + * all: both doors start the same stone -- the editor on activation, a terminal + * command on any invocation -- and two commands close together is all it takes. Two stoned processes held one extent0.dbf * read-write for a week on a developer machine before anyone noticed, because * gslist keys Stone rows by name and shows one row for two stones. */ @@ -520,9 +523,9 @@ export const STONE_LOCK_IDLE_MS = 30_000; * * A DIRECTORY rather than a file, unlike the setup lock above: `mkdir` is the * atomic create-or-fail primitive available to both this and the shell - * wrapper, and `flock` is not on a stock macOS. The owning pid goes in a file + * wrappers older versions wrote, and `flock` is not on a stock macOS. The owning pid goes in a file * inside it, so a lock left by a crash can be told from one held by a live - * process -- the same test the wrapper makes. + * process -- the same test an older wrapper makes in bash. * * Waiting is the point: a caller that stepped aside used to carry on as if * the stone were up — to the listener and the Grail file-in — while the @@ -540,6 +543,8 @@ export async function withStoneLock( wait: { satisfied?: () => boolean | Promise; progressing?: () => boolean; + /** Told once, when the lock turns out to be held: for a caller whose `log` goes nowhere. */ + report?: (message: string) => void; pollMs?: number; timeoutMs?: number; } = {}, @@ -562,6 +567,11 @@ export async function withStoneLock( ? 'This window is already starting the database; waiting for it.' : 'Another process is starting the database; waiting for it.', ); + wait.report?.( + ours + ? 'This window is already starting the database; waiting for it…' + : 'Waiting for another process to finish starting the database…', + ); } await sleep(wait.pollMs ?? STONE_LOCK_POLL_MS); if (await wait.satisfied?.()) return undefined; diff --git a/src/processes.ts b/src/processes.ts index a2bc8d7..aa7e449 100644 --- a/src/processes.ts +++ b/src/processes.ts @@ -339,9 +339,10 @@ export async function assertExternalDatabaseUp(opts: { * Bring up whichever of the stone and the session listener is not already * up, and say which of them this call started. * - * Every start of GemDB's own database goes through here — the window's and - * the GemDB Shell's (the listener a declined force-stop puts back goes through - * {@link ensureListener}) — and all of it happens under the stone lock the `gemdb` wrapper also takes. Checking + * Every start of GemDB's own database goes through here — the window's, the + * GemDB Shell's and the `gemdb` wrapper's (the listener a declined force-stop + * puts back goes through {@link ensureListener}) — and all of it happens under + * the stone lock, which wrappers older than this one also take in bash. Checking * and starting have to be one step: checked outside the lock, two callers * that both found the listener missing both ran `startnetldi`, and the loser * reported a failure over a database that was up. So the look happens inside @@ -353,19 +354,36 @@ export async function assertExternalDatabaseUp(opts: { * the guards that refuse a stone that should not start belong there, not on * every call. `report` hears each step as it begins, for a progress * notification or a terminal. + * + * `listener: false` is for a caller that never opens a session through + * NetLDI, as the wrapper's file mode (a linked topaz) never does: the stone + * is all it needs, so a missing listener neither takes the lock nor is + * started. Starting it there would only add a failure mode, and hand the + * caller's environment to every session NetLDI forks later. + * + * `skipQuickCheck` is for a caller that has just looked and found the stone + * not answering, as the wrapper has before it runs `--ensure-stone`: the + * lock-free listing that would find everything up is skipped, and the first + * look is the one inside the lock. */ export async function ensureProcesses( - opts: { beforeStoneStart?: () => void; report?: (message: string) => void } = {}, + opts: { + beforeStoneStart?: () => void; + report?: (message: string) => void; + listener?: boolean; + skipQuickCheck?: boolean; + } = {}, ): Promise<{ startedStone: boolean; startedNetldi: boolean }> { + const wantListener = opts.listener ?? true; requireOwnDatabase('start'); // Every notebook cell lands here, and nearly always both are up: one // listing and no lock then. Never for a stone starting or stopping, which // the lock and the wait below are for. - const before = await listProcessesAsync(); + const before = opts.skipQuickCheck ? [] : await listProcessesAsync(); const stoneUp = ['ready', 'other'].includes(stoneState(before)); - if (stoneUp && findNetldi(before)) { + if (stoneUp && (!wantListener || findNetldi(before))) { log('The database is already running.'); - log('The session listener is already running.'); + if (wantListener) log('The session listener is already running.'); return { startedStone: false, startedNetldi: false }; } let seen = before; @@ -396,7 +414,9 @@ export async function ensureProcesses( }); let startedNetldi = false; - if (!findNetldi(running)) { + if (!wantListener) { + // Nothing here opens a session through it. + } else if (!findNetldi(running)) { opts.report?.('Starting the session listener…'); startedNetldi = await startNetldi(); } else { @@ -407,8 +427,9 @@ export async function ensureProcesses( { satisfied: async () => { seen = await listProcessesAsync(); - return stoneReady(seen) && listenerReady(seen); + return stoneReady(seen) && (!wantListener || listenerReady(seen)); }, + report: opts.report, // A stone listed and not going down means whoever holds the lock is past // startstone: waiting on it, or starting the listener beside it. progressing: () => ['starting', 'ready', 'other'].includes(stoneState(seen)), From 9f1b613d067f8c21581ec5f2c74feda5ec490c48 Mon Sep 17 00:00:00 2001 From: Matias Fernandez <7898112+MatiasFernandez@users.noreply.github.com> Date: Mon, 19 Oct 2026 10:09:28 -0300 Subject: [PATCH 3/3] Refuse `gemdb` runs against a database without Python A database Grail had not been filed into yet answered a file, -c, -m or Shell start with driver compile errors, or a 4051 when there was no account yet. The wrapper now tests the Grail stamp for an own database, before the fast path, and the Shell and --ensure-stone do the same in bringUp. All three say GRAIL_NOT_INSTALLED, which points at "GemDB: Start GemDB". GRAIL_MISSING keeps its own wording: it now only meets a database whose stamp is present and whose Python is gone, which "Start GemDB" does not repair. The message holds double quotes, so the wrapper writes it through shellQuote. Co-Authored-By: Claude Sonnet 5.5 --- CHANGELOG.md | 5 +++ src/__integration__/cli.test.ts | 42 +++++++++++++++++++- src/__integration__/repl.test.ts | 20 +++++++++- src/__tests__/cli.test.ts | 66 +++++++++++++++++++++++++++++++- src/__tests__/cliStart.test.ts | 22 +++++++++++ src/cli.ts | 23 ++++++++++- src/cliStart.ts | 8 ++++ src/grailMessages.ts | 15 ++++++++ 8 files changed, 196 insertions(+), 5 deletions(-) create mode 100644 src/grailMessages.ts diff --git a/CHANGELOG.md b/CHANGELOG.md index 635a777..66c6da5 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -54,6 +54,11 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 says it is waiting, and waits as long as the window does, instead of giving up after ten seconds. A run starts the database only, not the session listener it never uses. A mistyped file name or a `-c` with no code no longer starts the database before failing. +- **`gemdb` says so when Python has not been installed into the database yet.** Run on a database + GemDB Code had not finished setting up, a file, `-c`, `-m` and the GemDB Shell printed compile + errors from the driver, or a refused login, and the GemDB Shell in a terminal could start a + database with no Python in it. They now stop with "Python support is not installed in this + database yet. Run "GemDB: Start GemDB" in VS Code and let it finish", before starting anything. ## [1.8.0] - 2026-10-18 diff --git a/src/__integration__/cli.test.ts b/src/__integration__/cli.test.ts index d61b217..99b89a8 100644 --- a/src/__integration__/cli.test.ts +++ b/src/__integration__/cli.test.ts @@ -4,7 +4,9 @@ import * as os from 'os'; import * as path from 'path'; import { afterAll, beforeAll, describe, expect, it } from 'vitest'; import { cliPath } from '../cli'; -import { bundledGrailStamp, stageGrail } from '../grail'; +import { bundledGrailStamp, recordGrailInstalled, stageGrail } from '../grail'; +import { GRAIL_NOT_INSTALLED } from '../grailMessages'; +import { grailStampPath } from '../paths'; import { isListening, isRunning, stopStone } from '../processes'; import { createDatabaseWithPython, @@ -75,6 +77,9 @@ beforeAll(async () => { createDatabaseWithPython(fixture); // Python already filed in — see testExtentPath stageGrail(ext); // stages Grail and generates the CLI + // The extent already has Python; the stamp is what the wrapper checks, and + // staging deleted it with the rest of grail/. + recordGrailInstalled(ext); await startStoneAlone(); fs.writeFileSync(path.join(workDir, 'hello.py'), 'print("hello from", 6 * 7)\n'); @@ -416,6 +421,41 @@ close }); }); +describe.skipIf(!ready || !canMakeFixture())('the gemdb command without Python', () => { + let aside: string; + + beforeAll(() => { + aside = `${grailStampPath()}.aside`; + fs.renameSync(grailStampPath(), aside); + }); + + afterAll(() => { + fs.renameSync(aside, grailStampPath()); + }); + + it.each([ + ['a file', ['hello.py']], + ['-c', ['-c', 'print(1)']], + ['-m', ['-m', 'json.tool']], + ])('refuses %s, saying where Python comes from, on a running stone', async (_mode, args) => { + const ran = await gemdb(...args); + + expect(ran.code).toBe(1); + expect(ran.stderr).toBe(`gemdb: ${GRAIL_NOT_INSTALLED}\n`); + expect(ran.stdout).toBe(''); + }); + + it('refuses before starting a stone that is down', async () => { + await stopStone(false); + + const ran = await gemdb('hello.py'); + + expect(ran.code).toBe(1); + expect(ran.stderr).toBe(`gemdb: ${GRAIL_NOT_INSTALLED}\n`); + expect(isRunning()).toBe(false); + }); +}); + /** See database.test.ts — skipIf is evaluated during collection. */ function canMakeFixture(): boolean { const probe = makeFixture(); diff --git a/src/__integration__/repl.test.ts b/src/__integration__/repl.test.ts index 17fee73..690ee60 100644 --- a/src/__integration__/repl.test.ts +++ b/src/__integration__/repl.test.ts @@ -3,7 +3,9 @@ import * as fs from 'fs'; import * as path from 'path'; import { afterAll, beforeAll, describe, expect, it } from 'vitest'; import { cliPath } from '../cli'; -import { bundledGrailStamp, stageGrail } from '../grail'; +import { bundledGrailStamp, recordGrailInstalled, stageGrail } from '../grail'; +import { GRAIL_NOT_INSTALLED } from '../grailMessages'; +import { grailStampPath } from '../paths'; import { isRunning, stopNetldi, stopStone } from '../processes'; import { createDatabaseWithPython, Fixture, haveTestExtent, makeFixture } from './fixture'; @@ -83,6 +85,8 @@ beforeAll(() => { if (!fixture) return; createDatabaseWithPython(fixture); // Python already filed in — see testExtentPath stageGrail(ext); // stages Grail, the CLI, and the shell bundle + // The extent already has Python; staging deleted the stamp with grail/. + recordGrailInstalled(ext); }); afterAll(async () => { @@ -175,6 +179,20 @@ send "exit()\\r" expect(ran.code).toBe(0); expect(isRunning()).toBe(true); // the shell brought the stone up itself }); + + it('refuses to open without Python in the database, even on a running stone', async () => { + const aside = `${grailStampPath()}.aside`; + fs.renameSync(grailStampPath(), aside); + try { + const ran = await driveShell(''); + + expect(ran.code).toBe(1); + expect(ran.transcript).toContain(`gemdb: ${GRAIL_NOT_INSTALLED}`); + expect(ran.transcript).not.toContain('>>> '); + } finally { + fs.renameSync(aside, grailStampPath()); + } + }); }); /** See database.test.ts — skipIf is evaluated during collection. */ diff --git a/src/__tests__/cli.test.ts b/src/__tests__/cli.test.ts index 2150686..9319f9e 100644 --- a/src/__tests__/cli.test.ts +++ b/src/__tests__/cli.test.ts @@ -6,7 +6,8 @@ import { afterEach, beforeEach, describe, expect, it } from 'vitest'; import { __resetSettings, __setSetting } from '../__mocks__/vscode'; import { cliPath, ensureCliCurrent, putCliOnPath, writeCliScripts } from '../cli'; import { STONE_LOCK_NAME } from '../lock'; -import { cliStampPath, expectedEnginePath } from '../paths'; +import { GRAIL_NOT_INSTALLED } from '../grailMessages'; +import { cliStampPath, expectedEnginePath, grailStampPath } from '../paths'; /** * The generator, not the command: what the files say and where they land. @@ -428,9 +429,15 @@ describe('starting the database', () => { return file; } + function stampGrail(): void { + fs.mkdirSync(path.dirname(grailStampPath()), { recursive: true }); + fs.writeFileSync(grailStampPath(), 'grail=test\n'); + } + beforeEach(() => { writeCliScripts(ext); installStubs(); + stampGrail(); }); it('runs against a stone that answers without calling the shell program', async () => { @@ -548,6 +555,63 @@ describe('starting the database', () => { expect(readCalls('shell')).toEqual([]); }); + describe('in a database Python has not been filed into', () => { + beforeEach(() => { + fs.rmSync(grailStampPath()); + }); + + // Exact, quotes and all: the message holds double quotes, and one pasted + // unescaped into the script would end its echo early. + const refusal = `gemdb: ${GRAIL_NOT_INSTALLED}\n`; + + for (const mode of ['a file', '-c', '-m']) { + it(`refuses ${mode}, whatever the stone is doing, without calling anything`, async () => { + stoneIs('OK'); + const args = + mode === 'a file' + ? [aProgram()] + : mode === '-c' + ? ['-c', 'print(1)'] + : ['-m', 'json.tool']; + + const { code, stderr } = await run(args); + + expect(code).toBe(1); + expect(stderr).toBe(refusal); + expect(readCalls('gslist')).toEqual([]); + expect(readCalls('shell')).toEqual([]); + expect(readCalls('topaz')).toEqual([]); + }); + } + + it('still answers --version and --help', async () => { + const { code } = await run(['--version']); + + expect(code).toBe(0); + }); + + it('still reports a bad argument as one, before the missing Python', async () => { + const { code, stderr } = await run([path.join(root, 'nothing-here.py')]); + + expect(code).toBe(2); + expect(stderr).toContain("can't open file"); + }); + + it('is not asked of an external database, whose Python someone else filed in', async () => { + __setSetting('gemdb.externalDatabase.gemstone', expectedEnginePath()); + __setSetting('gemdb.externalDatabase.globalDirectory', root); + __setSetting('gemdb.externalDatabase.stone', 'gemdb'); + writeCliScripts(ext); + installStubs(); + stoneIs('OK'); + + const { code } = await run([aProgram()]); + + expect(code).toBe(0); + expect(readCalls('topaz')).toHaveLength(1); + }); + }); + it('carries nothing of its own that starts a stone or takes the lock', () => { const wrapper = fs.readFileSync(cliPath(), 'utf8'); diff --git a/src/__tests__/cliStart.test.ts b/src/__tests__/cliStart.test.ts index f4f8107..ad95e52 100644 --- a/src/__tests__/cliStart.test.ts +++ b/src/__tests__/cliStart.test.ts @@ -6,6 +6,7 @@ import { beforeEach, describe, expect, it, vi } from 'vitest'; */ let external: object | undefined; +let grailInstalled = true; const ensureProcesses = vi.fn(async (_opts: unknown) => ({ startedStone: true, startedNetldi: false, @@ -14,6 +15,7 @@ const assertExternalDatabaseUp = vi.fn(async (_opts: unknown) => {}); const assertSafeToStart = vi.fn(); vi.mock('../config', () => ({ externalDatabase: () => external })); +vi.mock('../paths', () => ({ grailInstalled: () => grailInstalled })); vi.mock('../database', () => ({ assertSafeToStart: () => assertSafeToStart() })); vi.mock('../processes', () => ({ ensureProcesses: (opts: unknown) => ensureProcesses(opts), @@ -21,9 +23,11 @@ vi.mock('../processes', () => ({ })); const { bringUp } = await import('../cliStart'); +const { GRAIL_NOT_INSTALLED } = await import('../grailMessages'); beforeEach(() => { external = undefined; + grailInstalled = true; ensureProcesses.mockClear(); assertExternalDatabaseUp.mockClear(); assertSafeToStart.mockClear(); @@ -85,4 +89,22 @@ describe('bringing the database up for the gemdb command', () => { 'delete this directory', ); }); + + it('refuses an own database that has no Python in it, before starting anything', async () => { + grailInstalled = false; + + await expect(bringUp({ listener: false, report: () => {} })).rejects.toThrow( + GRAIL_NOT_INSTALLED, + ); + expect(ensureProcesses).not.toHaveBeenCalled(); + }); + + it('does not ask an external database for Python', async () => { + external = { stone: 'gs64stone' }; + grailInstalled = false; + + await bringUp({ listener: true, report: () => {} }); + + expect(assertExternalDatabaseUp).toHaveBeenCalled(); + }); }); diff --git a/src/cli.ts b/src/cli.ts index 222ea6f..ae00071 100644 --- a/src/cli.ts +++ b/src/cli.ts @@ -11,8 +11,10 @@ import { rootPath, stoneName, } from './config'; +import { GRAIL_NOT_INSTALLED } from './grailMessages'; import { log } from './log'; -import { cliStampPath, enginePath, grailPath } from './paths'; +import { shellQuote } from './osConfig'; +import { cliStampPath, enginePath, grailPath, grailStampPath } from './paths'; import { sharedLibraryExtension } from './platform'; /** @@ -524,7 +526,24 @@ elif [ "$1" != "-m" ] && [ ! -e "$1" ]; then exit 2 fi -# The database must be up — a linked gem still needs the stone. Starting it +${ + external + ? '' + : // Its own database only: an external one has Python filed in by whoever + // runs it. The same test as grailInstalled() in paths.ts. + `# Python has to be in the database before anything is run against it. A +# database Grail has not been filed into yet would only answer a run with +# compile errors, or a login refused for want of an account. The stamp is +# written when the file-in succeeds. Before the stone check, so it holds on +# a running stone too. +GRAIL_STAMP="${grailStampPath()}" +if [ ! -f "$GRAIL_STAMP" ]; then + echo ${shellQuote(`gemdb: ${GRAIL_NOT_INSTALLED}`)} >&2 + exit 1 +fi + +` +}# The database must be up — a linked gem still needs the stone. Starting it # here is the same judgement the editor makes: running Python is the request. # # The wrapper only recognises the common case. A stone that answers is just diff --git a/src/cliStart.ts b/src/cliStart.ts index fe8b5bf..e0e9f57 100644 --- a/src/cliStart.ts +++ b/src/cliStart.ts @@ -1,5 +1,7 @@ import { externalDatabase } from './config'; import { assertSafeToStart } from './database'; +import { GRAIL_NOT_INSTALLED } from './grailMessages'; +import { grailInstalled } from './paths'; import { assertExternalDatabaseUp, ensureProcesses } from './processes'; /** @@ -16,6 +18,11 @@ import { assertExternalDatabaseUp, ensureProcesses } from './processes'; * NetLDI. The Shell does; a file run is a linked topaz and never does, so it * starts the stone alone. * + * Python must already be in an own database: a stone that is up with no Grail + * in it fails every run with errors that name neither cause nor cure, so this + * is refused first, running stone or not. That is `GRAIL_NOT_INSTALLED`, for a + * database nobody has filed Grail into yet. + * * External database: only checked, waiting for a stone that is still * starting, since GemDB starts none of it. * @@ -30,6 +37,7 @@ export async function bringUp(opts: { await assertExternalDatabaseUp(opts); return; } + if (!grailInstalled()) throw new Error(GRAIL_NOT_INSTALLED); await ensureProcesses({ beforeStoneStart: assertSafeToStart, report: opts.report, diff --git a/src/grailMessages.ts b/src/grailMessages.ts new file mode 100644 index 0000000..9fe0c62 --- /dev/null +++ b/src/grailMessages.ts @@ -0,0 +1,15 @@ +/** + * What to say when Python is not in the database yet. Its own module so the + * extension and the shell bundle can both import it: it must not live in + * `lifecycle.ts` or anything that reaches telemetry, which the shell bundle + * refuses. + * + * This is the sentence for a database Grail has not been filed into **yet**: + * the first start is still running in the window, or has not happened. "Start + * GemDB" is the cure. It is not `GRAIL_MISSING` (`pythonQueries.ts`), which + * names a database whose stamp says Python is installed and which has lost it, + * and which only "Reinstall the Python Execution Engine" repairs. + */ +export const GRAIL_NOT_INSTALLED = + 'Python support is not installed in this database yet. ' + + 'Run "GemDB: Start GemDB" in VS Code and let it finish, then run this again.';