From 57f9d8420a8c2b3d7116c77da2410eb4137dd1b0 Mon Sep 17 00:00:00 2001 From: MarioCadenas Date: Mon, 13 Jul 2026 14:00:23 +0200 Subject: [PATCH 1/2] docs: document the agent evaluation framework Signed-off-by: MarioCadenas --- docs/docs/plugins/agents.md | 238 ++++++++++++++++++++++++++++++++++++ 1 file changed, 238 insertions(+) diff --git a/docs/docs/plugins/agents.md b/docs/docs/plugins/agents.md index 2ae56c86..fa5e8510 100644 --- a/docs/docs/plugins/agents.md +++ b/docs/docs/plugins/agents.md @@ -411,6 +411,244 @@ appkit.agents.reload(); // re-scan the directory appkit.agents.getThreads(userId); // list user's threads ``` +## Evaluating agents + +AppKit ships an eve-style eval framework for the agents you build here. You author evals in TypeScript with `defineEval`, drive the agent by sending it messages, and assert on its reply and tool usage with deterministic matchers or LLM judges. Evals run against a **running app** over HTTP (`--url`), and — with Databricks creds and an experiment — report to MLflow as native "Evaluation runs" with per-assertion and per-judge feedback attached to each turn's trace. The eval API is part of the beta surface: import it from `@databricks/appkit/beta`. + +Evals live beside each agent: `config/agents//evals/*.eval.ts`. Each file default-exports one `defineEval({ test })`. The agent under test defaults to the parent `` directory; set `agent:` to target a different one. + +### A first eval + +```ts +// config/agents/query/evals/smoke.eval.ts +import { defineEval } from "@databricks/appkit/beta"; + +export default defineEval({ + description: "Query agent responds to a greeting", + async test(t) { + await t.send("Hi there!"); + t.succeeded(); // gate: the turn completed without an agent/stream error + }, +}); +``` + +Start the app, then run the evals against it: + +```bash +# the app must be running and reachable at --url +appkit agent eval --url http://localhost:3000 + +# scope to one agent/eval by substring, and point at a project root +appkit agent eval query --root apps/dev-playground --url http://localhost:3000 +``` + +The positional `[filter]` matches evals whose `/` contains the substring (or an exact agent id). The command discovers every `*.eval.ts` under `config/agents/*/evals/`, drives each against the running app, and exits non-zero if any gate fails. + +### Assertions + +Every assertion returns a chainable handle. Assertions are **gates by default** — a failure fails the eval (non-zero exit). Chain `.soft()` to demote to a tracked-only metric, `.gate()` to promote a soft assertion back, or `.atLeast(n)` to set the pass threshold on a scored assertion. + +| Assertion | Passes when | +|---|---| +| `t.succeeded()` | The last turn completed without an agent/stream error. | +| `t.calledTool(name)` | The agent called `name` during the run. | +| `t.calledToolWith(name, expected)` | `name` was called with arguments that deep-contain `expected` (every key in `expected` matches recursively; extra args are ignored). | +| `t.check(value, matcher)` | `value` satisfies the matcher — `includes(substring)`, `equals(expected)`, or `matches(pattern)`. | + +```ts +import { defineEval, includes } from "@databricks/appkit/beta"; + +export default defineEval({ + description: "Helper agent answers a math question", + agent: "helper", + async test(t) { + await t.send("What is 2 + 2?"); + t.succeeded(); // gate + t.check(t.reply, includes("4")).soft(); // tracked metric, won't fail the gate + }, +}); +``` + +```ts +// deep-partial tool-arg check +await t.send("What's the weather in Brooklyn?"); +t.calledTool("get_weather"); +t.calledToolWith("get_weather", { city: "Brooklyn" }); +``` + +Call `t.skip("reason")` to skip an eval, and read `t.reply`, `t.toolCalls`, and `t.sessionId` to inspect the last turn. + +### LLM-as-judge + +`t.judge.*` scores the last reply with an LLM judge (via `autoevals` pointed at a Databricks serving endpoint). Each judge returns a scored assertion (0..1) that **gates by default** — a miss fails the eval. Chain `.atLeast(n)` to set the pass threshold, or `.soft()` to track it only. Judges require a judge model: pass `--judge-model ` (or set `APPKIT_JUDGE_MODEL`) plus Databricks auth; without one, `t.judge.*` throws with a clear message. + +```ts +async test(t) { + await t.send("What's the weather in Brooklyn?"); + t.succeeded(); + + // closedQA needs no ground truth — it judges the reply against a question. + (await t.judge.closedQA( + "Does the response describe weather conditions for Brooklyn?", + )).atLeast(0.5); +} +``` + +- `t.judge.factuality(expected)` — score the reply against an expected reference answer. +- `t.judge.closedQA(criteria)` — score whether the reply answers the question, per `criteria`. +- `t.judge.custom(spec)` — a prompt-template judge (`{ name, promptTemplate, choiceScores }`), the TS analog of MLflow's `@scorer`. + +Guard judge calls with `isJudgeConfigured()` when an eval should still exercise the drive path without a judge model configured: + +```ts +import { defineEval, isJudgeConfigured } from "@databricks/appkit/beta"; +// ... +if (isJudgeConfigured()) { + (await t.judge.closedQA(guideline)).atLeast(0.5); +} +``` + +### Conversations + +Each `t.send` is one user turn. How you sequence them controls the thread: + +```ts +// One-shot: a single turn. +await t.send("Summarize Q3 revenue."); +t.succeeded(); + +// Multi-turn: consecutive sends share one thread, so the agent sees history. +await t.send("Show me the orders table."); +await t.send("Now filter it to last week."); +t.succeeded(); + +// t.reset() drops the conversation: the next send opens a fresh thread with +// no history. Use it to run several independent one-shot checks in one test. +await t.send("What's 2 + 2?"); +t.check(t.reply, includes("4")); +t.reset(); +await t.send("What's the capital of France?"); +t.check(t.reply, includes("Paris")); +``` + +### Datasets + +Add `dataset: { table }` to sweep a Databricks **managed evaluation dataset** — a Unity Catalog `catalog.schema.table` with `inputs`/`expectations` columns. The eval runs once per row; the runner binds each row's `inputs` to `t.input` and `expectations` to `t.expected`. Reading the dataset requires a workspace client and warehouse (`--warehouse` + auth). + +```ts +import { defineEval, isJudgeConfigured, userTurns } from "@databricks/appkit/beta"; + +export default defineEval({ + description: "Query agent satisfies each dataset row's guidelines", + dataset: { table: "main.mario.appkit_eval_dataset" }, // optional `limit?` + async test(t) { + // Replay every user turn in the row against one thread, so the agent sees + // the accumulating conversation. A single-user-turn row sends once. + for (const turn of userTurns(t.input)) { + await t.send(turn); + } + t.succeeded(); + + if (isJudgeConfigured()) { + for (const guideline of guidelines(t.expected)) { + (await t.judge.closedQA(guideline)).atLeast(0.5); + } + } + }, +}); +``` + +The row shapes match the MLflow managed-dataset UI: + +``` +inputs {"messages":[{"role":"user","content":"..."}]} +expectations {"guidelines":{"value":["...","..."]}} (optional) +``` + +`userTurns(t.input)` extracts every `role: "user"` message content in order from the `{messages:[...]}` input — a row can carry a full multi-turn conversation, and replaying each user turn against one thread lets the agent build up history (interleaved assistant/system turns are ignored; the agent generates its own). Read `expectations.guidelines.value` yourself; the UI wraps the array as `{value: [...]}`: + +```ts +function guidelines(expected: Record | undefined): string[] { + const g = (expected?.guidelines as { value?: unknown } | undefined)?.value; + return Array.isArray(g) ? g.map(String) : []; +} +``` + +Run a dataset eval: + +```bash +appkit agent eval dataset --root apps/dev-playground --url http://localhost:3000 \ + --profile --warehouse --judge-model +``` + +### Running evals & CI + +`appkit agent eval [filter]` — run agent evals (`config/agents//evals/*.eval.ts`) against a running app. + +| Flag | Description | +|---|---| +| `[filter]` | Only run evals whose `/` contains this substring (or an exact agent id) | +| `--url ` | Base URL of the running app (default `http://localhost:3000`) | +| `--strict` | Fail on soft-assertion misses too | +| `--root ` | Project root containing `config/agents/` (default: cwd) | +| `--header ` | Extra request header as `'Key: value'` (repeatable) | +| `--tag ` | Only run evals tagged with one of these tags (repeatable) | +| `--profile ` | Databricks CLI profile to authenticate with via OAuth (default: `DATABRICKS_CONFIG_PROFILE`) | +| `--databricks-host ` | Databricks host for writing MLflow assessments (default: `DATABRICKS_HOST`) | +| `--databricks-token ` | Databricks token for writing MLflow assessments (default: `DATABRICKS_TOKEN`) | +| `--experiment ` | MLflow experiment id for the evaluation run (default: `MLFLOW_EXPERIMENT_ID`) | +| `--warehouse ` | SQL warehouse id for reading managed evaluation datasets (default: `DATABRICKS_WAREHOUSE_ID`) | +| `--judge-model ` | Databricks serving endpoint to use as the LLM judge for `t.judge.*` (default: `APPKIT_JUDGE_MODEL`) | +| `--concurrency ` | Max evals/dataset rows to drive concurrently (default: 1, serial) | +| `--timeout ` | Default per-eval timeout in ms (a per-eval `timeoutMs` overrides it) | +| `--retries ` | Re-run an eval up to N times when it fails on an infra error (turn/timeout); assertion failures are not retried | +| `--min-pass-rate ` | Gate on aggregate pass rate (0..1) instead of requiring every eval to pass; exit 1 when below | +| `--reporter ` | Report format: `text` (live console), `json` (dashboards), or `junit` (CI test reporters) | +| `--output ` | Write the json/junit report to this file instead of stdout (ignored for text) | + +Notes: + +- **Auth is OAuth-first.** `--profile ` mints an OAuth token from your Databricks CLI profile — no PAT needed. An explicit `--databricks-host`/`--databricks-token` (or the `DATABRICKS_*` env vars) wins over the profile. +- **`--retries` only absorbs infra flakiness.** A retry fires only when an eval throws or times out (`result.error` set); a wrong reply is real signal and is never retried. Each attempt gets a fresh driver. +- **Gating.** By default the run exits non-zero if any eval fails. `--min-pass-rate 0.9` switches to threshold mode: it exits non-zero only when the aggregate pass rate drops below the threshold. `--strict` additionally treats soft-assertion misses as failures. +- **CI reports.** `--reporter junit --output results.xml` writes a JUnit file for CI test reporters; `--reporter json` emits machine-readable results. In machine reporters, human-facing lines go to stderr so stdout stays clean for the report. + +```bash +# CI: gate on 90% pass rate, emit JUnit, authenticate + report to MLflow +appkit agent eval --url "$APP_URL" \ + --profile ci \ + --experiment "$MLFLOW_EXPERIMENT_ID" \ + --concurrency 4 --retries 1 \ + --min-pass-rate 0.9 \ + --reporter junit --output eval-results.xml +``` + +### Per-directory config + +Drop an `evals.config.ts` beside an agent's evals to set defaults for that agent's runs: + +```ts +// config/agents/query/evals/evals.config.ts +import { defineEvalConfig } from "@databricks/appkit/beta"; + +export default defineEvalConfig({ + maxConcurrency: 4, // run up to 4 evals/rows concurrently + timeoutMs: 30_000, // default per-eval timeout +}); +``` + +Precedence: a **CLI flag** wins over the **`evals.config.ts`** value, which wins over the **built-in default** (concurrency `1`, no timeout). A per-eval `def.timeoutMs` overrides both for that eval. + +### MLflow reporting + +When `--experiment ` (or `MLFLOW_EXPERIMENT_ID`) is set together with Databricks auth, the runner creates a native MLflow **Evaluation run** up front. As each eval runs against the app, its turn trace is linked to the run, and every assertion and judge score is written back as feedback on that trace: + +- Each deterministic assertion becomes a `CODE`-sourced Feedback with a boolean value. +- Each judge assertion becomes an `LLM_JUDGE`-sourced Feedback with its numeric 0..1 score and rationale. +- An overall `appkit_eval` pass/fail Feedback is attached per eval, and aggregate metrics are logged when the run finishes. + +Skip `--experiment` (and `MLFLOW_EXPERIMENT_ID`) to run evals purely locally with no MLflow side effects — the CLI prints a reminder that the evaluation run was skipped. + ## Frontmatter schema | Key | Type | Notes | From a1ed8080a60673bef25c8564f4090e3edf335eff Mon Sep 17 00:00:00 2001 From: MarioCadenas Date: Mon, 13 Jul 2026 14:24:19 +0200 Subject: [PATCH 2/2] refactor(appkit): dedupe agent-dir listing, concurrency max, and cli int parsing Signed-off-by: MarioCadenas --- packages/appkit/src/evals/discover.ts | 35 ++++++++----------- packages/appkit/src/evals/run-evals.ts | 11 +++--- .../shared/src/cli/commands/agent/eval.ts | 24 +++++-------- 3 files changed, 29 insertions(+), 41 deletions(-) diff --git a/packages/appkit/src/evals/discover.ts b/packages/appkit/src/evals/discover.ts index 33cacf0f..96bd6a74 100644 --- a/packages/appkit/src/evals/discover.ts +++ b/packages/appkit/src/evals/discover.ts @@ -35,6 +35,19 @@ function isFile(p: string): boolean { } } +/** List the agent directory names under `/config/agents/` (empty if absent). */ +function listAgents(rootDir: string): { agentsDir: string; agents: string[] } { + const agentsDir = path.join(rootDir, "config", "agents"); + try { + const agents = readdirSync(agentsDir).filter((n) => + isDir(path.join(agentsDir, n)), + ); + return { agentsDir, agents }; + } catch { + return { agentsDir, agents: [] }; + } +} + /** Recursively collect `*.eval.ts` files (skips `evals.config.ts`). */ function walkEvalFiles(dir: string): string[] { const out: string[] = []; @@ -61,18 +74,9 @@ function walkEvalFiles(dir: string): string[] { * dir with `.eval.ts` stripped. Returns a stable, sorted list. */ export function discoverEvalFiles(rootDir: string): DiscoveredEval[] { - const agentsDir = path.join(rootDir, "config", "agents"); + const { agentsDir, agents } = listAgents(rootDir); const out: DiscoveredEval[] = []; - let agents: string[]; - try { - agents = readdirSync(agentsDir).filter((n) => - isDir(path.join(agentsDir, n)), - ); - } catch { - return out; - } - for (const agent of agents) { const evalsDir = path.join(agentsDir, agent, "evals"); if (!isDir(evalsDir)) continue; @@ -98,18 +102,9 @@ export function discoverEvalFiles(rootDir: string): DiscoveredEval[] { * config file are omitted. Returns a stable, sorted list. */ export function discoverEvalConfigs(rootDir: string): DiscoveredEvalConfig[] { - const agentsDir = path.join(rootDir, "config", "agents"); + const { agentsDir, agents } = listAgents(rootDir); const out: DiscoveredEvalConfig[] = []; - let agents: string[]; - try { - agents = readdirSync(agentsDir).filter((n) => - isDir(path.join(agentsDir, n)), - ); - } catch { - return out; - } - for (const agent of agents) { const file = path.join(agentsDir, agent, "evals", "evals.config.ts"); if (isFile(file)) out.push({ file, agent }); diff --git a/packages/appkit/src/evals/run-evals.ts b/packages/appkit/src/evals/run-evals.ts index 1bd39338..d5347f51 100644 --- a/packages/appkit/src/evals/run-evals.ts +++ b/packages/appkit/src/evals/run-evals.ts @@ -308,13 +308,12 @@ export async function runEvalsInDir( // `evals.config.ts` `maxConcurrency` governs the single shared work pool, so // it can't be applied per-agent without splitting the pool. CLI wins; else // the highest value any agent's config requests (the pool ceiling); else 1. - const configMaxConcurrency = [...configs.values()] + const configConcurrencies = [...configs.values()] .map((c) => c.maxConcurrency) - .filter((n): n is number => typeof n === "number") - .reduce( - (max, n) => (max === undefined ? n : Math.max(max, n)), - undefined, - ); + .filter((n): n is number => typeof n === "number"); + const configMaxConcurrency = configConcurrencies.length + ? Math.max(...configConcurrencies) + : undefined; const maxConcurrency = options.maxConcurrency ?? configMaxConcurrency ?? 1; const total = loaded.length; diff --git a/packages/shared/src/cli/commands/agent/eval.ts b/packages/shared/src/cli/commands/agent/eval.ts index dbba2cc1..9bc69d17 100644 --- a/packages/shared/src/cli/commands/agent/eval.ts +++ b/packages/shared/src/cli/commands/agent/eval.ts @@ -84,6 +84,12 @@ function parseHeaders(values: string[]): Record { return headers; } +/** Parse a positive-integer CLI option; junk, zero, or negative → undefined. */ +function positiveInt(raw: string | undefined): number | undefined { + const n = raw ? Number.parseInt(raw, 10) : Number.NaN; + return n > 0 ? n : undefined; +} + interface EvalOptions { url: string; strict?: boolean; @@ -145,26 +151,14 @@ async function runAgentEval( const workspaceClient = runner.resolveWorkspaceClient(credentials); // Drive up to N evals/rows concurrently (default serial). Ignore junk input. - const parsedConcurrency = opts.concurrency - ? Number.parseInt(opts.concurrency, 10) - : undefined; - const maxConcurrency = - parsedConcurrency && parsedConcurrency > 0 ? parsedConcurrency : undefined; + const maxConcurrency = positiveInt(opts.concurrency); // Runner-level default per-eval timeout (ms). A per-eval `timeoutMs` wins. - const parsedTimeout = opts.timeout - ? Number.parseInt(opts.timeout, 10) - : undefined; - const timeoutMs = - parsedTimeout && parsedTimeout > 0 ? parsedTimeout : undefined; + const timeoutMs = positiveInt(opts.timeout); // Extra attempts for evals that fail on an infra error (turn/timeout). Junk // or negative input falls back to no retries. - const parsedRetries = opts.retries - ? Number.parseInt(opts.retries, 10) - : undefined; - const retries = - parsedRetries && parsedRetries > 0 ? parsedRetries : undefined; + const retries = positiveInt(opts.retries); // In a machine reporter (json/junit), stdout is reserved for the report (it // may be piped), so human-facing lines go to stderr and the per-eval live