From eb80eee0593696a7ad911609b4740c41f3691dd9 Mon Sep 17 00:00:00 2001 From: hhuumm Date: Fri, 28 Aug 2026 19:32:08 -0400 Subject: [PATCH] feat: persist app runtime history --- desktop/src/app-control.tsx | 2 +- desktop/src/main.ts | 171 +++++++++++++++++++++++++++++--- docs/architecture.md | 2 +- docs/threat-model.md | 1 + src/app-log-store.ts | 192 ++++++++++++++++++++++++++++++++++++ src/database.ts | 24 ++++- test/app-log-store.test.ts | 96 ++++++++++++++++++ test/database.test.ts | 9 ++ 8 files changed, 483 insertions(+), 14 deletions(-) create mode 100644 src/app-log-store.ts create mode 100644 test/app-log-store.test.ts diff --git a/desktop/src/app-control.tsx b/desktop/src/app-control.tsx index 5279574..de5b570 100644 --- a/desktop/src/app-control.tsx +++ b/desktop/src/app-control.tsx @@ -42,7 +42,7 @@ export function AppControl({ projectId, status, onStatus }: { projectId: string; -
App output
{status?.logs.length ? status.logs.join("") : "Start the app to see its output here."}
+
App outputPersisted locally
{status?.logs.length ? status.logs.join("") : "Start the app to see its output here."}
; } diff --git a/desktop/src/main.ts b/desktop/src/main.ts index 29bc081..8f4d611 100644 --- a/desktop/src/main.ts +++ b/desktop/src/main.ts @@ -55,6 +55,11 @@ import { } from "../../src/project-onboarding"; import { removeProjectRegistration } from "../../src/project-registry"; import { assertGitHubPublicationAllowed, parseGitHubAuthStatus, type GitHubStatus } from "../../src/github-auth"; +import { + AppLogStore, + formatAppLogEntry, + type AppLogStream, +} from "../../src/app-log-store"; declare const MAIN_WINDOW_VITE_DEV_SERVER_URL: string | undefined; declare const MAIN_WINDOW_VITE_NAME: string; if (started) app.quit(); @@ -64,7 +69,16 @@ const pmThreads = new Map(); const pmTurns = new KeyedSerialQueue(); const pmActivity = new Map(); const appProcesses = new Map>(); -const appProcessState = new Map(); +type AppProcessRuntime = { + startedAt?: string; + sessionId?: string; + stopping?: boolean; + persistenceError?: boolean; + logs: string[]; +}; +const appProcessState = new Map(); +const appLogStores = new Map(); +let appIsQuitting = false; const defaultGlobalSettings: GlobalSettings = { defaultProvider: "codex", scaffold: { mode: "guided", autoDeploy: false, onboardingPrompt: "You are onboarding me into a new project. Begin by asking focused questions about the users, problem, core workflow, constraints, technology preferences, and definition of success. Summarize decisions as we go. Create small implementation tasks only after requirements are agreed, then continue helping me shape and build the project incrementally." }, github: { allowWrites: false } }; const file = () => join(app.getPath("userData"), "state.json"); function load(): DesktopState { @@ -324,6 +338,8 @@ function registerIPC() { state.runs = []; pmThreads.delete(project.id); appProcessState.delete(project.id); + appLogStores.get(project.id)?.close(); + appLogStores.delete(project.id); save(state); return hydrate(); }); @@ -498,8 +514,70 @@ function saveAppConfig(input: AppConfig, project: Project) { mkdirSync(dirname(appConfigPath(project)), { recursive: true }); writeFileSync(appConfigPath(project), JSON.stringify({ command, workingDirectory, url }, null, 2)); } +function projectAppLogStore(project: Project): AppLogStore { + const existing = appLogStores.get(project.id); + if (existing) return existing; + const store = new AppLogStore( + join(project.repo, ".codepilot", "codepilot.db"), + ); + appLogStores.set(project.id, store); + return store; +} +function persistedAppLogs(project: Project): string[] { + try { + return projectAppLogStore(project).recent(500).map(formatAppLogEntry); + } catch { + return []; + } +} +function appendAppLog( + project: Project, + state: AppProcessRuntime, + stream: AppLogStream, + payload: string, +): void { + const timestamp = new Date().toISOString(); + state.logs.push(formatAppLogEntry({ timestamp, stream, payload })); + state.logs = state.logs.slice(-500); + if (!state.sessionId || appIsQuitting) return; + try { + projectAppLogStore(project).append( + state.sessionId, + stream, + payload, + timestamp, + ); + } catch (error) { + if (state.persistenceError) return; + state.persistenceError = true; + state.logs.push( + formatAppLogEntry({ + timestamp: new Date().toISOString(), + stream: "system", + payload: `App log persistence failed: ${error instanceof Error ? error.message : String(error)}`, + }), + ); + } +} +function endAppLogSession( + project: Project, + state: AppProcessRuntime, + input: { exitCode?: number; reason: string }, +): void { + if (!state.sessionId) return; + try { + projectAppLogStore(project).endSession(state.sessionId, input); + } catch { + state.persistenceError = true; + } +} function projectAppStatus(project: Project) { - const process = appProcesses.get(project.id), state = appProcessState.get(project.id) ?? { logs: [] }; + const process = appProcesses.get(project.id); + let state = appProcessState.get(project.id); + if (!state) { + state = { logs: persistedAppLogs(project) }; + appProcessState.set(project.id, state); + } return { configured: Boolean(readAppConfig(project).command), running: Boolean(process && process.exitCode === null), ...(process?.pid ? { pid: process.pid } : {}), ...(state.startedAt ? { startedAt: state.startedAt } : {}), config: readAppConfig(project), logs: state.logs }; } function projectEnvironment(...directories: string[]): NodeJS.ProcessEnv { @@ -527,23 +605,77 @@ function startProjectApp(project: Project) { if (!config.command) throw new Error("Configure a start command first"); const cwd = resolve(project.repo, config.workingDirectory); if (!existsSync(cwd)) throw new Error("Configured working directory does not exist"); - const state = { startedAt: new Date().toISOString(), logs: [] as string[] }; - const child = spawn(config.command, { cwd, shell: true, windowsHide: true, env: { ...isolatedAgentEnvironment(), ...projectEnvironment(project.repo, cwd) } }); - const append = (prefix: string, chunk: Buffer) => { state.logs.push(`${prefix}${chunk.toString("utf8")}`); state.logs = state.logs.slice(-200); }; - child.stdout?.on("data", (chunk: Buffer) => append("", chunk)); - child.stderr?.on("data", (chunk: Buffer) => append("[stderr] ", chunk)); - child.once("error", (error) => state.logs.push(`[error] ${error.message}`)); - child.once("exit", (code) => { state.logs.push(`[codePilot] App exited with code ${code ?? "unknown"}.`); appProcesses.delete(project.id); }); + const startedAt = new Date().toISOString(); + const journal = projectAppLogStore(project); + const sessionId = journal.startSession({ + command: config.command, + workingDirectory: config.workingDirectory, + url: config.url, + startedAt, + }); + const state: AppProcessRuntime = { + startedAt, + sessionId, + logs: journal.recent(500).map(formatAppLogEntry), + }; + let child: ReturnType; + try { + child = spawn(config.command, { cwd, shell: true, windowsHide: true, env: { ...isolatedAgentEnvironment(), ...projectEnvironment(project.repo, cwd) } }); + } catch (error) { + const message = error instanceof Error ? error.message : String(error); + journal.append(sessionId, "system", `App could not start: ${message}`); + journal.endSession(sessionId, { reason: "spawn-error" }); + throw error; + } + appendAppLog( + project, + state, + "system", + `App started${child.pid ? ` with PID ${child.pid}` : ""}.`, + ); + child.stdout?.on("data", (chunk: Buffer) => + appendAppLog(project, state, "stdout", chunk.toString("utf8")), + ); + child.stderr?.on("data", (chunk: Buffer) => + appendAppLog(project, state, "stderr", chunk.toString("utf8")), + ); + child.once("error", (error) => { + if (appIsQuitting) return; + appendAppLog(project, state, "system", `App error: ${error.message}`); + endAppLogSession(project, state, { reason: "spawn-error" }); + }); + child.once("exit", (code) => { + if (!appIsQuitting && !state.stopping) + appendAppLog( + project, + state, + "system", + `App exited with code ${code ?? "unknown"}.`, + ); + if (!appIsQuitting) + endAppLogSession(project, state, { + ...(typeof code === "number" ? { exitCode: code } : {}), + reason: state.stopping ? "user-stop" : "process-exit", + }); + if (appProcesses.get(project.id) === child) + appProcesses.delete(project.id); + }); appProcesses.set(project.id, child); appProcessState.set(project.id, state); return projectAppStatus(project); } function stopProjectApp(project: Project) { const child = appProcesses.get(project.id); if (child?.pid) { + const state = appProcessState.get(project.id); + if (state) { + state.stopping = true; + appendAppLog(project, state, "system", "App stopped by user."); + endAppLogSession(project, state, { reason: "user-stop" }); + } if (process.platform === "win32") { try { execFileSync("taskkill", ["/pid", String(child.pid), "/T", "/F"], { windowsHide: true }); } catch { child.kill(); } } else child.kill("SIGTERM"); - appProcesses.delete(project.id); - appProcessState.get(project.id)?.logs.push("[codePilot] App stopped by user."); + if (appProcesses.get(project.id) === child) + appProcesses.delete(project.id); } return projectAppStatus(project); } @@ -1302,9 +1434,26 @@ app.on("window-all-closed", () => { }); app.on("before-quit", () => { pmServer.stop(); + const projects = new Map(load().projects.map((project) => [project.id, project])); + for (const [projectId] of appProcesses) { + const project = projects.get(projectId); + const state = appProcessState.get(projectId); + if (!project || !state) continue; + state.stopping = true; + appendAppLog( + project, + state, + "system", + "App stopped because CodePilot is quitting.", + ); + endAppLogSession(project, state, { reason: "codepilot-quit" }); + } + appIsQuitting = true; for (const child of appProcesses.values()) { if (!child.pid) continue; if (process.platform === "win32") { try { execFileSync("taskkill", ["/pid", String(child.pid), "/T", "/F"], { windowsHide: true }); } catch { child.kill(); } } else child.kill("SIGTERM"); } + for (const store of appLogStores.values()) store.close(); + appLogStores.clear(); }); diff --git a/docs/architecture.md b/docs/architecture.md index b449e8e..f77aa35 100644 --- a/docs/architecture.md +++ b/docs/architecture.md @@ -53,7 +53,7 @@ GitHub authentication is owned by the official `gh` CLI. The desktop launches it ### Persistence -One SQLite database owns runs, tasks, attempts, bounded events, worker results, leases, and desktop project-manager records. Initialization is centralized and schema version is recorded with SQLite `user_version`. Interrupted running tasks are marked failed and their runs require review on desktop restart; automatic resume is not yet implemented. +One SQLite database owns runs, tasks, attempts, bounded worker events, worker results, leases, desktop project-manager records, and App Core runtime history. Each app launch creates a session with its command, working directory, URL, start/end times, exit code, and stop reason. Timestamped stdout, stderr, and system events are hydrated after desktop restarts. Runtime retention is bounded to the latest 100 sessions and 20,000 events per project, while the App Core pane renders the latest 500 events. Initialization is centralized and schema version is recorded with SQLite `user_version`. Interrupted running tasks are marked failed and their runs require review on desktop restart; automatic resume is not yet implemented. ## Current limitations diff --git a/docs/threat-model.md b/docs/threat-model.md index 5a08d66..e19fc04 100644 --- a/docs/threat-model.md +++ b/docs/threat-model.md @@ -42,6 +42,7 @@ codePilot v0.1 is local developer tooling for repositories, objectives, provider - Validation commands use a shell and can execute arbitrary operator-supplied text. - Cooperative file leases do not prevent a provider from ignoring the protocol. - SQLite state is local and is not tamper-evident. +- App Core stdout and stderr are retained locally in the project database. Applications should not print credentials or other secrets; retained logs must be inspected before sharing `.codepilot/` data. - The experimental Docker files do not yet represent an enforced manager-owned sandbox. - Explicit GitHub publication uses the current user's authenticated `gh` and Git credentials to push the generated branch and open a pull request. Repository permissions are enforced by GitHub, not codePilot. - Explicit repository cloning runs Git with the current user's network access and credentials; the cloned repository remains untrusted until the operator reviews it. diff --git a/src/app-log-store.ts b/src/app-log-store.ts new file mode 100644 index 0000000..7c904da --- /dev/null +++ b/src/app-log-store.ts @@ -0,0 +1,192 @@ +import { randomUUID } from "node:crypto"; +import { mkdirSync } from "node:fs"; +import { dirname } from "node:path"; +import { DatabaseSync } from "node:sqlite"; +import { initializeDatabase } from "./database.js"; + +export type AppLogStream = "stdout" | "stderr" | "system"; + +export type AppLogEntry = { + id: number; + sessionId: string; + timestamp: string; + stream: AppLogStream; + payload: string; +}; + +export type AppSessionInput = { + command: string; + workingDirectory: string; + url: string; + startedAt?: string; +}; + +export type AppLogStoreOptions = { + maxEvents?: number; + maxSessions?: number; + maxPayloadCharacters?: number; +}; + +const DEFAULT_MAX_EVENTS = 20_000; +const DEFAULT_MAX_SESSIONS = 100; +const DEFAULT_MAX_PAYLOAD_CHARACTERS = 64 * 1024; + +export class AppLogStore implements Disposable { + readonly #database: DatabaseSync; + readonly #maxEvents: number; + readonly #maxSessions: number; + readonly #maxPayloadCharacters: number; + + constructor(path: string, options: AppLogStoreOptions = {}) { + mkdirSync(dirname(path), { recursive: true }); + this.#database = new DatabaseSync(path); + initializeDatabase(this.#database); + this.#maxEvents = positiveInteger(options.maxEvents, DEFAULT_MAX_EVENTS); + this.#maxSessions = positiveInteger( + options.maxSessions, + DEFAULT_MAX_SESSIONS, + ); + this.#maxPayloadCharacters = positiveInteger( + options.maxPayloadCharacters, + DEFAULT_MAX_PAYLOAD_CHARACTERS, + ); + } + + startSession(input: AppSessionInput): string { + const id = randomUUID(); + this.#database + .prepare(` + INSERT INTO app_sessions ( + id, command, working_directory, url, started_at + ) VALUES (?, ?, ?, ?, ?) + `) + .run( + id, + input.command, + input.workingDirectory, + input.url, + input.startedAt ?? new Date().toISOString(), + ); + this.#pruneSessions(); + return id; + } + + append( + sessionId: string, + stream: AppLogStream, + payload: string, + timestamp = new Date().toISOString(), + ): void { + if (!payload) return; + const insert = this.#database.prepare(` + INSERT INTO app_events (session_id, timestamp, stream, payload) + VALUES (?, ?, ?, ?) + `); + this.#database.exec("BEGIN IMMEDIATE"); + try { + for (let offset = 0; offset < payload.length; ) { + const next = payload.slice( + offset, + offset + this.#maxPayloadCharacters, + ); + insert.run(sessionId, timestamp, stream, next); + offset += next.length; + } + this.#database.exec("COMMIT"); + } catch (error) { + this.#database.exec("ROLLBACK"); + throw error; + } + this.#pruneEvents(); + } + + endSession( + sessionId: string, + input: { + exitCode?: number; + reason: string; + endedAt?: string; + }, + ): void { + this.#database + .prepare(` + UPDATE app_sessions + SET ended_at = COALESCE(ended_at, ?), + exit_code = COALESCE(exit_code, ?), + stop_reason = COALESCE(stop_reason, ?) + WHERE id = ? + `) + .run( + input.endedAt ?? new Date().toISOString(), + input.exitCode ?? null, + input.reason, + sessionId, + ); + } + + recent(limit = 500): AppLogEntry[] { + const boundedLimit = Math.min(5_000, positiveInteger(limit, 500)); + return ( + this.#database + .prepare(` + SELECT id, + session_id AS sessionId, + timestamp, + stream, + payload + FROM app_events + ORDER BY id DESC + LIMIT ? + `) + .all(boundedLimit) as AppLogEntry[] + ).reverse(); + } + + close(): void { + this.#database.close(); + } + + [Symbol.dispose](): void { + this.close(); + } + + #pruneEvents(): void { + this.#database + .prepare(` + DELETE FROM app_events + WHERE id <= COALESCE(( + SELECT id + FROM app_events + ORDER BY id DESC + LIMIT 1 OFFSET ? + ), 0) + `) + .run(this.#maxEvents); + } + + #pruneSessions(): void { + this.#database + .prepare(` + DELETE FROM app_sessions + WHERE id IN ( + SELECT id + FROM app_sessions + ORDER BY started_at DESC, id DESC + LIMIT -1 OFFSET ? + ) + `) + .run(this.#maxSessions); + } +} + +export function formatAppLogEntry( + entry: Pick, +): string { + const stream = entry.stream === "stdout" ? "" : `[${entry.stream}] `; + const line = `[${entry.timestamp}] ${stream}${entry.payload}`; + return line.endsWith("\n") ? line : `${line}\n`; +} + +function positiveInteger(value: number | undefined, fallback: number): number { + return Number.isInteger(value) && Number(value) > 0 ? Number(value) : fallback; +} diff --git a/src/database.ts b/src/database.ts index de22eae..9ce707f 100644 --- a/src/database.ts +++ b/src/database.ts @@ -1,6 +1,6 @@ import { DatabaseSync } from "node:sqlite"; -export const DATABASE_VERSION = 2; +export const DATABASE_VERSION = 3; /** * Own the complete project-control schema in one place. Every process may open @@ -61,6 +61,28 @@ export function initializeDatabase(database: DatabaseSync): void { expires_at TEXT NOT NULL ); CREATE INDEX IF NOT EXISTS file_checkouts_owner ON file_checkouts(owner); + CREATE TABLE IF NOT EXISTS app_sessions ( + id TEXT PRIMARY KEY, + command TEXT NOT NULL, + working_directory TEXT NOT NULL, + url TEXT NOT NULL, + started_at TEXT NOT NULL, + ended_at TEXT, + exit_code INTEGER, + stop_reason TEXT + ); + CREATE INDEX IF NOT EXISTS app_sessions_started_at_idx + ON app_sessions(started_at DESC); + CREATE TABLE IF NOT EXISTS app_events ( + id INTEGER PRIMARY KEY AUTOINCREMENT, + session_id TEXT NOT NULL, + timestamp TEXT NOT NULL, + stream TEXT NOT NULL CHECK(stream IN ('stdout', 'stderr', 'system')), + payload TEXT NOT NULL, + FOREIGN KEY(session_id) REFERENCES app_sessions(id) ON DELETE CASCADE + ); + CREATE INDEX IF NOT EXISTS app_events_session_id_idx + ON app_events(session_id, id); CREATE TABLE IF NOT EXISTS pm_tasks ( id TEXT PRIMARY KEY, title TEXT NOT NULL, diff --git a/test/app-log-store.test.ts b/test/app-log-store.test.ts new file mode 100644 index 0000000..a63f8bb --- /dev/null +++ b/test/app-log-store.test.ts @@ -0,0 +1,96 @@ +import assert from "node:assert/strict"; +import { mkdtemp } from "node:fs/promises"; +import { tmpdir } from "node:os"; +import { join } from "node:path"; +import { DatabaseSync } from "node:sqlite"; +import test from "node:test"; +import { + AppLogStore, + formatAppLogEntry, +} from "../src/app-log-store.js"; + +test("persists structured app sessions and output across store instances", async () => { + const root = await mkdtemp(join(tmpdir(), "codepilot-app-logs-")); + const path = join(root, "codepilot.db"); + const startedAt = "2026-08-28T12:00:00.000Z"; + const endedAt = "2026-08-28T12:00:01.000Z"; + + const first = new AppLogStore(path); + const sessionId = first.startSession({ + command: "npm start", + workingDirectory: ".", + url: "http://localhost:3000", + startedAt, + }); + first.append(sessionId, "stdout", "ready\n", startedAt); + first.append(sessionId, "stderr", "warning", endedAt); + first.endSession(sessionId, { exitCode: 0, reason: "process-exit", endedAt }); + first.close(); + + using second = new AppLogStore(path); + assert.deepEqual( + second.recent().map(({ stream, payload }) => ({ stream, payload })), + [ + { stream: "stdout", payload: "ready\n" }, + { stream: "stderr", payload: "warning" }, + ], + ); + using sessionDatabase = new DatabaseSync(path, { readOnly: true }); + const session = sessionDatabase + .prepare( + "SELECT ended_at, exit_code, stop_reason FROM app_sessions WHERE id = ?", + ) + .get(sessionId) as Record; + assert.equal(session.ended_at, endedAt); + assert.equal(session.exit_code, 0); + assert.equal(session.stop_reason, "process-exit"); +}); + +test("bounds retained history while preserving chunked payload data", async () => { + const root = await mkdtemp(join(tmpdir(), "codepilot-app-retention-")); + const path = join(root, "codepilot.db"); + using store = new AppLogStore(path, { + maxEvents: 3, + maxSessions: 2, + maxPayloadCharacters: 4, + }); + const first = store.startSession({ + command: "one", + workingDirectory: ".", + url: "http://localhost:3000", + }); + store.append(first, "stdout", "abcdefgh"); + const second = store.startSession({ + command: "two", + workingDirectory: ".", + url: "http://localhost:3001", + }); + store.append(second, "stdout", "middle"); + const third = store.startSession({ + command: "three", + workingDirectory: ".", + url: "http://localhost:3002", + }); + store.append(third, "system", "stopped"); + + assert.deepEqual( + store.recent().map((entry) => entry.payload), + ["le", "stop", "ped"], + ); + using database = new DatabaseSync(path, { readOnly: true }); + const row = database + .prepare("SELECT COUNT(*) count FROM app_sessions") + .get() as Record; + assert.equal(row.count, 2); +}); + +test("formats persisted streams for the existing App Core output", () => { + assert.equal( + formatAppLogEntry({ + timestamp: "2026-08-28T12:00:00.000Z", + stream: "stderr", + payload: "warning", + }), + "[2026-08-28T12:00:00.000Z] [stderr] warning\n", + ); +}); diff --git a/test/database.test.ts b/test/database.test.ts index b515047..565e063 100644 --- a/test/database.test.ts +++ b/test/database.test.ts @@ -33,6 +33,15 @@ test("upgrades the legacy task schema and records its database version", async ( assert.ok(columns.has("external_id")); assert.ok(columns.has("dependencies")); assert.ok(columns.has("attempts")); + const tables = new Set( + ( + database + .prepare("SELECT name FROM sqlite_master WHERE type = 'table'") + .all() as Array> + ).map((row) => String(row.name)), + ); + assert.ok(tables.has("app_sessions")); + assert.ok(tables.has("app_events")); const version = database.prepare("PRAGMA user_version").get() as Record; assert.equal(Number(version.user_version), DATABASE_VERSION); });