diff --git a/apps/dev-playground/evals.config.ts b/apps/dev-playground/evals.config.ts new file mode 100644 index 00000000..6ad057c1 --- /dev/null +++ b/apps/dev-playground/evals.config.ts @@ -0,0 +1,15 @@ +import { defineEvalConfig } from "@databricks/appkit/beta"; + +/** + * Root eval config for the dev-playground. `webServer` lets `appkit agent eval` + * boot the app on demand (reusing an already-running dev server) instead of + * requiring it to be started by hand. + */ +export default defineEvalConfig({ + baseUrl: "http://localhost:8000", + webServer: { + // Monorepo fixture command; a template project would use `npm run dev`. + command: "pnpm --filter=dev-playground dev", + timeoutMs: 90_000, + }, +}); diff --git a/packages/appkit/src/evals/discover.ts b/packages/appkit/src/evals/discover.ts index 96bd6a74..2ca02786 100644 --- a/packages/appkit/src/evals/discover.ts +++ b/packages/appkit/src/evals/discover.ts @@ -95,6 +95,17 @@ export function discoverEvalFiles(rootDir: string): DiscoveredEval[] { ); } +/** + * Path to the root `evals.config.ts` (from {@link defineEvalConfig}) at + * `/evals.config.ts`, or `undefined` when absent. The root config + * holds run-wide settings (`baseUrl`, `webServer`); it's distinct from the + * per-agent configs found by {@link discoverEvalConfigs}. + */ +export function findRootEvalConfig(rootDir: string): string | undefined { + const file = path.join(rootDir, "evals.config.ts"); + return isFile(file) ? file : undefined; +} + /** * Discover the per-agent `evals.config.ts` (from {@link defineEvalConfig}) at * `/config/agents//evals/evals.config.ts`. Config is per-agent: diff --git a/packages/appkit/src/evals/index.ts b/packages/appkit/src/evals/index.ts index 0fa5aadb..b7fb84c7 100644 --- a/packages/appkit/src/evals/index.ts +++ b/packages/appkit/src/evals/index.ts @@ -19,6 +19,7 @@ export { type DiscoveredEvalConfig, discoverEvalConfigs, discoverEvalFiles, + findRootEvalConfig, } from "./discover"; export { createHttpDriver, type HttpDriverOptions } from "./http-driver"; export { @@ -49,6 +50,7 @@ export { type RunEvalOptions, runEval } from "./run-eval"; export { type EvalProgress, type EvalRunSummary, + loadRootEvalConfig, type RunEvalsOptions, runBounded, runEvalsInDir, @@ -63,6 +65,7 @@ export type { EvalDefinition, EvalDriver, EvalResult, + EvalWebServer, Matcher, MatchResult, Severity, diff --git a/packages/appkit/src/evals/run-evals.ts b/packages/appkit/src/evals/run-evals.ts index d5347f51..ad3b9c91 100644 --- a/packages/appkit/src/evals/run-evals.ts +++ b/packages/appkit/src/evals/run-evals.ts @@ -6,6 +6,7 @@ import { type DiscoveredEval, discoverEvalConfigs, discoverEvalFiles, + findRootEvalConfig, } from "./discover"; import { createHttpDriver } from "./http-driver"; import { configureJudge } from "./judge"; @@ -126,6 +127,20 @@ async function loadEvalConfig(file: string): Promise { return resolveConfigDefault(mod); } +/** + * Load the root `evals.config.ts` under `rootDir` (the project root), or return + * `undefined` when there is none. This is the run-wide config carrying + * `baseUrl`/`webServer`; the CLI reads it to resolve options and manage the + * app-under-test lifecycle before calling {@link runEvalsInDir}. + */ +export async function loadRootEvalConfig( + rootDir: string, +): Promise { + const file = findRootEvalConfig(rootDir); + if (!file) return undefined; + return loadEvalConfig(file); +} + /** * Unwrap the config default export across module-interop shapes (see * {@link resolveEvalDefault}). A config has no `.test`, so the first plain diff --git a/packages/appkit/src/evals/tests/discover.test.ts b/packages/appkit/src/evals/tests/discover.test.ts index aefc64ca..d3c55922 100644 --- a/packages/appkit/src/evals/tests/discover.test.ts +++ b/packages/appkit/src/evals/tests/discover.test.ts @@ -2,7 +2,11 @@ import { mkdirSync, mkdtempSync, rmSync, writeFileSync } from "node:fs"; import { tmpdir } from "node:os"; import path from "node:path"; import { afterEach, beforeEach, describe, expect, test } from "vitest"; -import { discoverEvalConfigs, discoverEvalFiles } from "../discover"; +import { + discoverEvalConfigs, + discoverEvalFiles, + findRootEvalConfig, +} from "../discover"; let root: string; @@ -59,3 +63,15 @@ describe("discoverEvalConfigs", () => { expect(discoverEvalConfigs(root)).toEqual([]); }); }); + +describe("findRootEvalConfig", () => { + test("finds a root evals.config.ts", () => { + write("evals.config.ts"); + expect(findRootEvalConfig(root)).toBe(path.join(root, "evals.config.ts")); + }); + + test("returns undefined when absent (and ignores per-agent configs)", () => { + write("config/agents/support/evals/evals.config.ts"); + expect(findRootEvalConfig(root)).toBeUndefined(); + }); +}); diff --git a/packages/appkit/src/evals/types.ts b/packages/appkit/src/evals/types.ts index e65b7fc3..a2ef151b 100644 --- a/packages/appkit/src/evals/types.ts +++ b/packages/appkit/src/evals/types.ts @@ -167,7 +167,39 @@ export interface EvalDefinition { test(t: TestContext): Promise | void; } -/** Per-directory config from `evals.config.ts`. */ +/** + * Auto-start config for the app under test, à la Playwright's `webServer`. When + * set in a root `evals.config.ts`, the CLI boots the app before running evals + * and tears it down after — so you don't have to start the server by hand. + */ +export interface EvalWebServer { + /** Shell command that starts the app, e.g. `"npm run dev"`. */ + command: string; + /** + * URL polled until it answers before evals start. Defaults to the run's + * `baseUrl` (`--url`). Readiness = any HTTP response (a 404 still proves the + * server is up). + */ + url?: string; + /** How long to wait for `url` to answer before giving up. Defaults to 60s. */ + timeoutMs?: number; + /** + * When `true` (default), reuse a server already answering at `url` instead of + * spawning one — so a running `dev` server is used as-is. Set `false` to + * always spawn a fresh server. + */ + reuseExisting?: boolean; +} + +/** + * Eval config from `evals.config.ts` (via {@link defineEvalConfig}). + * + * Two scopes share this shape: a **root** `evals.config.ts` (project root) may + * set run-wide settings — `baseUrl` and `webServer` — plus defaults for + * `maxConcurrency`/`timeoutMs`; a **per-agent** `config/agents//evals/evals.config.ts` + * sets only that agent's `maxConcurrency`/`timeoutMs` overrides (`baseUrl`/ + * `webServer` there are ignored — server lifecycle is run-wide). + */ export interface EvalConfig { /** LLM judge config. Defaults to the agent's own serving endpoint. */ judge?: { model?: string }; @@ -175,6 +207,10 @@ export interface EvalConfig { maxConcurrency?: number; /** Default per-eval timeout. */ timeoutMs?: number; + /** Base URL of the app to drive (root config only). Overridden by `--url`. */ + baseUrl?: string; + /** Auto-start the app under test (root config only). */ + webServer?: EvalWebServer; } /** The outcome of running one eval. */ diff --git a/packages/shared/src/cli/commands/agent/eval.ts b/packages/shared/src/cli/commands/agent/eval.ts index 9bc69d17..d8ece64c 100644 --- a/packages/shared/src/cli/commands/agent/eval.ts +++ b/packages/shared/src/cli/commands/agent/eval.ts @@ -1,3 +1,4 @@ +import { type ChildProcess, spawn } from "node:child_process"; import fs from "node:fs"; import { Command, Option } from "commander"; @@ -48,6 +49,7 @@ interface EvalRunner { host?: string; token?: string; }): unknown; + loadRootEvalConfig(rootDir: string): Promise; evalGlyph(result: unknown): string; formatEvalDetail(result: unknown): string[]; formatSummaryLine(results: unknown[]): string; @@ -56,6 +58,19 @@ interface EvalRunner { summarize(results: unknown[]): { allPassed: boolean; passRate: number }; } +/** Subset of `@databricks/appkit/beta`'s `EvalConfig` the CLI reads. */ +interface EvalConfig { + maxConcurrency?: number; + timeoutMs?: number; + baseUrl?: string; + webServer?: { + command: string; + url?: string; + timeoutMs?: number; + reuseExisting?: boolean; + }; +} + /** * Loaded at runtime from the consuming project so this command (which ships in * `@databricks/shared`) doesn't take a build-time dependency on appkit. The @@ -90,8 +105,83 @@ function positiveInt(raw: string | undefined): number | undefined { return n > 0 ? n : undefined; } +/** True when `url` answers with any HTTP response (a 404 still proves it's up). */ +async function isServerUp(url: string): Promise { + try { + await fetch(url, { signal: AbortSignal.timeout(2000) }); + return true; + } catch { + return false; + } +} + +/** + * Start the app under test per a root config's `webServer`, à la Playwright: + * reuse a server already answering at `url` (unless `reuseExisting: false`), + * else spawn `command`, poll `url` until it answers or `timeoutMs` elapses. + * Returns a `stop()` that kills the spawned process group (a no-op when the + * server was reused). Logs go to stderr so a machine reporter's stdout stays + * clean. Throws if the server never comes up. + */ +async function startWebServer( + webServer: NonNullable, + baseUrl: string, +): Promise<{ stop: () => void }> { + const url = webServer.url ?? baseUrl; + const reuse = webServer.reuseExisting !== false; + const noop = { stop: () => {} }; + + if (reuse && (await isServerUp(url))) { + console.error(`Reusing server already running at ${url}`); + return noop; + } + + console.error(`Starting web server: ${webServer.command}`); + // `detached` + a negative-PID kill lets us tear down the whole process group + // (dev servers spawn child processes). stdout/stderr inherit so the user sees + // build output; the server's stdout is not our report stream. + const child: ChildProcess = spawn(webServer.command, { + shell: true, + detached: true, + stdio: "inherit", + }); + + let exited = false; + child.on("exit", () => { + exited = true; + }); + + const stop = (): void => { + if (exited || child.pid === undefined) return; + try { + process.kill(-child.pid, "SIGTERM"); + } catch { + // Group already gone, or never became a leader — best-effort. + } + }; + + const deadline = Date.now() + (webServer.timeoutMs ?? 60_000); + try { + while (Date.now() < deadline) { + if (exited) throw new Error("web server exited before becoming ready"); + if (await isServerUp(url)) { + console.error(`Web server ready at ${url}`); + return { stop }; + } + await new Promise((r) => setTimeout(r, 500)); + } + } catch (err) { + stop(); + throw err; + } + stop(); + throw new Error( + `web server did not respond at ${url} within ${webServer.timeoutMs ?? 60_000}ms`, + ); +} + interface EvalOptions { - url: string; + url?: string; strict?: boolean; root?: string; header?: string[]; @@ -116,6 +206,15 @@ async function runAgentEval( ): Promise { const runner = await loadRunner(); + // Root `evals.config.ts` (project root) carries run-wide settings — baseUrl, + // webServer, and defaults for concurrency/timeout. A CLI flag always wins. + const rootDir = opts.root ?? process.cwd(); + const config = (await runner.loadRootEvalConfig(rootDir)) ?? {}; + + // Base URL: --url flag > config.baseUrl > built-in default. `--url` has no + // commander default so an unset flag is undefined and lets config win. + const baseUrl = opts.url ?? config.baseUrl ?? "http://localhost:8000"; + // Databricks credentials shared by auth resolution and the workspace client: // an explicit flag/DATABRICKS_* env wins, else the SDK resolves from the CLI // profile. @@ -150,11 +249,13 @@ async function runAgentEval( const warehouseId = opts.warehouse ?? process.env.DATABRICKS_WAREHOUSE_ID; const workspaceClient = runner.resolveWorkspaceClient(credentials); - // Drive up to N evals/rows concurrently (default serial). Ignore junk input. - const maxConcurrency = positiveInt(opts.concurrency); + // Drive up to N evals/rows concurrently (default serial). --concurrency flag + // wins over the root config; junk input is ignored. + const maxConcurrency = positiveInt(opts.concurrency) ?? config.maxConcurrency; - // Runner-level default per-eval timeout (ms). A per-eval `timeoutMs` wins. - const timeoutMs = positiveInt(opts.timeout); + // Runner-level default per-eval timeout (ms). --timeout flag wins over the + // root config; a per-eval `timeoutMs` overrides both (applied in the runner). + const timeoutMs = positiveInt(opts.timeout) ?? config.timeoutMs; // Extra attempts for evals that fail on an infra error (turn/timeout). Junk // or negative input falls back to no retries. @@ -175,7 +276,7 @@ async function runAgentEval( switch (event.type) { case "discovered": info( - `Running ${event.total} eval${event.total === 1 ? "" : "s"} against ${opts.url}\n`, + `Running ${event.total} eval${event.total === 1 ? "" : "s"} against ${baseUrl}\n`, ); break; case "run-created": @@ -198,22 +299,33 @@ async function runAgentEval( } }; - const summary = await runner.runEvalsInDir({ - rootDir: opts.root, - baseUrl: opts.url, - filter, - tags: opts.tag, - strict: opts.strict, - headers: opts.header ? parseHeaders(opts.header) : undefined, - mlflow, - judge, - workspaceClient, - warehouseId, - maxConcurrency, - timeoutMs, - retries, - onEvent, - }); + // Boot the app under test if the root config declares a webServer (reuses an + // already-running server unless told otherwise); always torn down after. + const server = config.webServer + ? await startWebServer(config.webServer, baseUrl) + : undefined; + + let summary: EvalRunSummary; + try { + summary = await runner.runEvalsInDir({ + rootDir, + baseUrl, + filter, + tags: opts.tag, + strict: opts.strict, + headers: opts.header ? parseHeaders(opts.header) : undefined, + mlflow, + judge, + workspaceClient, + warehouseId, + maxConcurrency, + timeoutMs, + retries, + onEvent, + }); + } finally { + server?.stop(); + } // The final human summary always shows (stderr for machine reporters so it // never pollutes the report on stdout/file). @@ -288,7 +400,10 @@ export const agentEvalCommand = new Command("eval") "[filter]", "Only run evals whose / contains this substring (or an exact agent id)", ) - .option("--url ", "Base URL of the running app", "http://localhost:3000") + .option( + "--url ", + "Base URL of the app to drive (default: evals.config.ts baseUrl, else http://localhost:8000)", + ) .option("--strict", "Fail on soft-assertion misses too", false) .option( "--root ",