feat(appkit): auto-start app under test for agent evals (stack 6/6)

apps/dev-playground/evals.config.ts

@@ -0,0 +1,15 @@
+import { defineEvalConfig } from "@databricks/appkit/beta";
+
+/**
+ * Root eval config for the dev-playground. `webServer` lets `appkit agent eval`
+ * boot the app on demand (reusing an already-running dev server) instead of
+ * requiring it to be started by hand.
+ */
+export default defineEvalConfig({
+  baseUrl: "http://localhost:8000",
+  webServer: {
+    // Monorepo fixture command; a template project would use `npm run dev`.
+    command: "pnpm --filter=dev-playground dev",
+    timeoutMs: 90_000,
+  },
+});

packages/appkit/src/evals/discover.ts

@@ -95,6 +95,17 @@ export function discoverEvalFiles(rootDir: string): DiscoveredEval[] {
   );
 }
 
+/**
+ * Path to the root `evals.config.ts` (from {@link defineEvalConfig}) at
+ * `<rootDir>/evals.config.ts`, or `undefined` when absent. The root config
+ * holds run-wide settings (`baseUrl`, `webServer`); it's distinct from the
+ * per-agent configs found by {@link discoverEvalConfigs}.
+ */
+export function findRootEvalConfig(rootDir: string): string | undefined {
+  const file = path.join(rootDir, "evals.config.ts");
+  return isFile(file) ? file : undefined;
+}
+
 /**
  * Discover the per-agent `evals.config.ts` (from {@link defineEvalConfig}) at
  * `<rootDir>/config/agents/<agent>/evals/evals.config.ts`. Config is per-agent:

packages/appkit/src/evals/index.ts

@@ -19,6 +19,7 @@ export {
   type DiscoveredEvalConfig,
   discoverEvalConfigs,
   discoverEvalFiles,
+  findRootEvalConfig,
 } from "./discover";
 export { createHttpDriver, type HttpDriverOptions } from "./http-driver";
 export {
@@ -49,6 +50,7 @@ export { type RunEvalOptions, runEval } from "./run-eval";
 export {
   type EvalProgress,
   type EvalRunSummary,
+  loadRootEvalConfig,
   type RunEvalsOptions,
   runBounded,
   runEvalsInDir,
@@ -63,6 +65,7 @@ export type {
   EvalDefinition,
   EvalDriver,
   EvalResult,
+  EvalWebServer,
   Matcher,
   MatchResult,
   Severity,

packages/appkit/src/evals/run-evals.ts

@@ -6,6 +6,7 @@ import {
   type DiscoveredEval,
   discoverEvalConfigs,
   discoverEvalFiles,
+  findRootEvalConfig,
 } from "./discover";
 import { createHttpDriver } from "./http-driver";
 import { configureJudge } from "./judge";
@@ -126,6 +127,20 @@ async function loadEvalConfig(file: string): Promise<EvalConfig | undefined> {
   return resolveConfigDefault(mod);
 }
 
+/**
+ * Load the root `evals.config.ts` under `rootDir` (the project root), or return
+ * `undefined` when there is none. This is the run-wide config carrying
+ * `baseUrl`/`webServer`; the CLI reads it to resolve options and manage the
+ * app-under-test lifecycle before calling {@link runEvalsInDir}.
+ */
+export async function loadRootEvalConfig(
+  rootDir: string,
+): Promise<EvalConfig | undefined> {
+  const file = findRootEvalConfig(rootDir);
+  if (!file) return undefined;
+  return loadEvalConfig(file);
+}
+
 /**
  * Unwrap the config default export across module-interop shapes (see
  * {@link resolveEvalDefault}). A config has no `.test`, so the first plain

packages/appkit/src/evals/tests/discover.test.ts

@@ -2,7 +2,11 @@ import { mkdirSync, mkdtempSync, rmSync, writeFileSync } from "node:fs";
 import { tmpdir } from "node:os";
 import path from "node:path";
 import { afterEach, beforeEach, describe, expect, test } from "vitest";
-import { discoverEvalConfigs, discoverEvalFiles } from "../discover";
+import {
+  discoverEvalConfigs,
+  discoverEvalFiles,
+  findRootEvalConfig,
+} from "../discover";
 
 let root: string;
 
@@ -59,3 +63,15 @@ describe("discoverEvalConfigs", () => {
     expect(discoverEvalConfigs(root)).toEqual([]);
   });
 });
+
+describe("findRootEvalConfig", () => {
+  test("finds a root evals.config.ts", () => {
+    write("evals.config.ts");
+    expect(findRootEvalConfig(root)).toBe(path.join(root, "evals.config.ts"));
+  });
+
+  test("returns undefined when absent (and ignores per-agent configs)", () => {
+    write("config/agents/support/evals/evals.config.ts");
+    expect(findRootEvalConfig(root)).toBeUndefined();
+  });
+});

packages/appkit/src/evals/types.ts

@@ -167,14 +167,50 @@ export interface EvalDefinition {
   test(t: TestContext): Promise<void> | void;
 }
 
-/** Per-directory config from `evals.config.ts`. */
+/**
+ * Auto-start config for the app under test, à la Playwright's `webServer`. When
+ * set in a root `evals.config.ts`, the CLI boots the app before running evals
+ * and tears it down after — so you don't have to start the server by hand.
+ */
+export interface EvalWebServer {
+  /** Shell command that starts the app, e.g. `"npm run dev"`. */
+  command: string;
+  /**
+   * URL polled until it answers before evals start. Defaults to the run's
+   * `baseUrl` (`--url`). Readiness = any HTTP response (a 404 still proves the
+   * server is up).
+   */
+  url?: string;
+  /** How long to wait for `url` to answer before giving up. Defaults to 60s. */
+  timeoutMs?: number;
+  /**
+   * When `true` (default), reuse a server already answering at `url` instead of
+   * spawning one — so a running `dev` server is used as-is. Set `false` to
+   * always spawn a fresh server.
+   */
+  reuseExisting?: boolean;
+}
+
+/**
+ * Eval config from `evals.config.ts` (via {@link defineEvalConfig}).
+ *
+ * Two scopes share this shape: a **root** `evals.config.ts` (project root) may
+ * set run-wide settings — `baseUrl` and `webServer` — plus defaults for
+ * `maxConcurrency`/`timeoutMs`; a **per-agent** `config/agents/<id>/evals/evals.config.ts`
+ * sets only that agent's `maxConcurrency`/`timeoutMs` overrides (`baseUrl`/
+ * `webServer` there are ignored — server lifecycle is run-wide).
+ */
 export interface EvalConfig {
   /** LLM judge config. Defaults to the agent's own serving endpoint. */
   judge?: { model?: string };
   /** Max evals to run concurrently. */
   maxConcurrency?: number;
   /** Default per-eval timeout. */
   timeoutMs?: number;
+  /** Base URL of the app to drive (root config only). Overridden by `--url`. */
+  baseUrl?: string;
+  /** Auto-start the app under test (root config only). */
+  webServer?: EvalWebServer;
 }
 
 /** The outcome of running one eval. */

packages/shared/src/cli/commands/agent/eval.ts

@@ -1,3 +1,4 @@
+import { type ChildProcess, spawn } from "node:child_process";
 import fs from "node:fs";
 import { Command, Option } from "commander";
 
@@ -48,6 +49,7 @@ interface EvalRunner {
     host?: string;
     token?: string;
   }): unknown;
+  loadRootEvalConfig(rootDir: string): Promise<EvalConfig | undefined>;
   evalGlyph(result: unknown): string;
   formatEvalDetail(result: unknown): string[];
   formatSummaryLine(results: unknown[]): string;
@@ -56,6 +58,19 @@ interface EvalRunner {
   summarize(results: unknown[]): { allPassed: boolean; passRate: number };
 }
 
+/** Subset of `@databricks/appkit/beta`'s `EvalConfig` the CLI reads. */
+interface EvalConfig {
+  maxConcurrency?: number;
+  timeoutMs?: number;
+  baseUrl?: string;
+  webServer?: {
+    command: string;
+    url?: string;
+    timeoutMs?: number;
+    reuseExisting?: boolean;
+  };
+}
+
 /**
  * Loaded at runtime from the consuming project so this command (which ships in
  * `@databricks/shared`) doesn't take a build-time dependency on appkit. The
@@ -90,8 +105,83 @@ function positiveInt(raw: string | undefined): number | undefined {
   return n > 0 ? n : undefined;
 }
 
+/** True when `url` answers with any HTTP response (a 404 still proves it's up). */
+async function isServerUp(url: string): Promise<boolean> {
+  try {
+    await fetch(url, { signal: AbortSignal.timeout(2000) });
+    return true;
+  } catch {
+    return false;
+  }
+}
+
+/**
+ * Start the app under test per a root config's `webServer`, à la Playwright:
+ * reuse a server already answering at `url` (unless `reuseExisting: false`),
+ * else spawn `command`, poll `url` until it answers or `timeoutMs` elapses.
+ * Returns a `stop()` that kills the spawned process group (a no-op when the
+ * server was reused). Logs go to stderr so a machine reporter's stdout stays
+ * clean. Throws if the server never comes up.
+ */
+async function startWebServer(
+  webServer: NonNullable<EvalConfig["webServer"]>,
+  baseUrl: string,
+): Promise<{ stop: () => void }> {
+  const url = webServer.url ?? baseUrl;
+  const reuse = webServer.reuseExisting !== false;
+  const noop = { stop: () => {} };
+
+  if (reuse && (await isServerUp(url))) {
+    console.error(`Reusing server already running at ${url}`);
+    return noop;
+  }
+
+  console.error(`Starting web server: ${webServer.command}`);
+  // `detached` + a negative-PID kill lets us tear down the whole process group
+  // (dev servers spawn child processes). stdout/stderr inherit so the user sees
+  // build output; the server's stdout is not our report stream.
+  const child: ChildProcess = spawn(webServer.command, {
+    shell: true,
+    detached: true,
+    stdio: "inherit",
+  });
+
+  let exited = false;
+  child.on("exit", () => {
+    exited = true;
+  });
+
+  const stop = (): void => {
+    if (exited || child.pid === undefined) return;
+    try {
+      process.kill(-child.pid, "SIGTERM");
+    } catch {
+      // Group already gone, or never became a leader — best-effort.
+    }
+  };
+
+  const deadline = Date.now() + (webServer.timeoutMs ?? 60_000);
+  try {
+    while (Date.now() < deadline) {
+      if (exited) throw new Error("web server exited before becoming ready");
+      if (await isServerUp(url)) {
+        console.error(`Web server ready at ${url}`);
+        return { stop };
+      }
+      await new Promise((r) => setTimeout(r, 500));
+    }
+  } catch (err) {
+    stop();
+    throw err;
+  }
+  stop();
+  throw new Error(
+    `web server did not respond at ${url} within ${webServer.timeoutMs ?? 60_000}ms`,
+  );
+}
+
 interface EvalOptions {
-  url: string;
+  url?: string;
   strict?: boolean;
   root?: string;
   header?: string[];
@@ -116,6 +206,15 @@ async function runAgentEval(
 ): Promise<void> {
   const runner = await loadRunner();
 
+  // Root `evals.config.ts` (project root) carries run-wide settings — baseUrl,
+  // webServer, and defaults for concurrency/timeout. A CLI flag always wins.
+  const rootDir = opts.root ?? process.cwd();
+  const config = (await runner.loadRootEvalConfig(rootDir)) ?? {};
+
+  // Base URL: --url flag > config.baseUrl > built-in default. `--url` has no
+  // commander default so an unset flag is undefined and lets config win.
+  const baseUrl = opts.url ?? config.baseUrl ?? "http://localhost:8000";
+
   // Databricks credentials shared by auth resolution and the workspace client:
   // an explicit flag/DATABRICKS_* env wins, else the SDK resolves from the CLI
   // profile.
@@ -150,11 +249,13 @@ async function runAgentEval(
   const warehouseId = opts.warehouse ?? process.env.DATABRICKS_WAREHOUSE_ID;
   const workspaceClient = runner.resolveWorkspaceClient(credentials);
 
-  // Drive up to N evals/rows concurrently (default serial). Ignore junk input.
-  const maxConcurrency = positiveInt(opts.concurrency);
+  // Drive up to N evals/rows concurrently (default serial). --concurrency flag
+  // wins over the root config; junk input is ignored.
+  const maxConcurrency = positiveInt(opts.concurrency) ?? config.maxConcurrency;
 
-  // Runner-level default per-eval timeout (ms). A per-eval `timeoutMs` wins.
-  const timeoutMs = positiveInt(opts.timeout);
+  // Runner-level default per-eval timeout (ms). --timeout flag wins over the
+  // root config; a per-eval `timeoutMs` overrides both (applied in the runner).
+  const timeoutMs = positiveInt(opts.timeout) ?? config.timeoutMs;
 
   // Extra attempts for evals that fail on an infra error (turn/timeout). Junk
   // or negative input falls back to no retries.
@@ -175,7 +276,7 @@ async function runAgentEval(
     switch (event.type) {
       case "discovered":
         info(
-          `Running ${event.total} eval${event.total === 1 ? "" : "s"} against ${opts.url}\n`,
+          `Running ${event.total} eval${event.total === 1 ? "" : "s"} against ${baseUrl}\n`,
         );
         break;
       case "run-created":
@@ -198,22 +299,33 @@ async function runAgentEval(
     }
   };
 
-  const summary = await runner.runEvalsInDir({
-    rootDir: opts.root,
-    baseUrl: opts.url,
-    filter,
-    tags: opts.tag,
-    strict: opts.strict,
-    headers: opts.header ? parseHeaders(opts.header) : undefined,
-    mlflow,
-    judge,
-    workspaceClient,
-    warehouseId,
-    maxConcurrency,
-    timeoutMs,
-    retries,
-    onEvent,
-  });
+  // Boot the app under test if the root config declares a webServer (reuses an
+  // already-running server unless told otherwise); always torn down after.
+  const server = config.webServer
+    ? await startWebServer(config.webServer, baseUrl)
+    : undefined;
+
+  let summary: EvalRunSummary;
+  try {
+    summary = await runner.runEvalsInDir({
+      rootDir,
+      baseUrl,
+      filter,
+      tags: opts.tag,
+      strict: opts.strict,
+      headers: opts.header ? parseHeaders(opts.header) : undefined,
+      mlflow,
+      judge,
+      workspaceClient,
+      warehouseId,
+      maxConcurrency,
+      timeoutMs,
+      retries,
+      onEvent,
+    });
+  } finally {
+    server?.stop();
+  }
 
   // The final human summary always shows (stderr for machine reporters so it
   // never pollutes the report on stdout/file).
@@ -288,7 +400,10 @@ export const agentEvalCommand = new Command("eval")
     "[filter]",
     "Only run evals whose <agent>/<id> contains this substring (or an exact agent id)",
   )
-  .option("--url <url>", "Base URL of the running app", "http://localhost:3000")
+  .option(
+    "--url <url>",
+    "Base URL of the app to drive (default: evals.config.ts baseUrl, else http://localhost:8000)",
+  )
   .option("--strict", "Fail on soft-assertion misses too", false)
   .option(
     "--root <dir>",