From 2bdb1aaa0447346d25ccb76b5af1fb571603337e Mon Sep 17 00:00:00 2001 From: MarioCadenas Date: Fri, 26 Jun 2026 19:01:56 +0200 Subject: [PATCH 1/3] feat(appkit): add agent eval framework with native mlflow evaluation runs eve-style eval authoring (defineEval + t-context + matchers) discovered from config/agents//evals/*.eval.ts and run via 'appkit agent eval' against a running app. Streams per-eval progress and gates CI via exit code. When Databricks creds + an experiment are set, it creates a real MLflow evaluation run (mlflow.runType=genai_evaluate): each eval's trace links to the run, pass/fail is written as feedback assessments, and aggregate metrics are logged. All via the MLflow REST API. Signed-off-by: MarioCadenas --- .../config/agents/query/evals/smoke.eval.ts | 14 ++ .../config/agents/query/evals/sum.eval.ts | 13 ++ .../agents/query/evals/tool-call.eval.ts | 16 ++ packages/appkit/src/beta.ts | 3 +- packages/appkit/src/evals/define-eval.ts | 32 +++ packages/appkit/src/evals/discover.ts | 76 +++++++ packages/appkit/src/evals/http-driver.ts | 148 ++++++++++++++ packages/appkit/src/evals/index.ts | 40 ++++ packages/appkit/src/evals/matchers.ts | 25 +++ packages/appkit/src/evals/mlflow-report.ts | 113 +++++++++++ packages/appkit/src/evals/mlflow-rest.ts | 39 ++++ packages/appkit/src/evals/mlflow-run.ts | 117 +++++++++++ packages/appkit/src/evals/report.ts | 76 +++++++ packages/appkit/src/evals/run-eval.ts | 155 ++++++++++++++ packages/appkit/src/evals/run-evals.ts | 166 +++++++++++++++ .../appkit/src/evals/tests/discover.test.ts | 42 ++++ .../appkit/src/evals/tests/matchers.test.ts | 19 ++ .../src/evals/tests/mlflow-report.test.ts | 57 ++++++ .../appkit/src/evals/tests/mlflow-run.test.ts | 30 +++ .../appkit/src/evals/tests/report.test.ts | 54 +++++ .../src/evals/tests/resolve-default.test.ts | 25 +++ .../appkit/src/evals/tests/run-eval.test.ts | 117 +++++++++++ packages/appkit/src/evals/types.ts | 128 ++++++++++++ .../shared/src/cli/commands/agent/eval.ts | 191 ++++++++++++++++++ .../shared/src/cli/commands/agent/index.ts | 19 ++ packages/shared/src/cli/index.ts | 2 + 26 files changed, 1716 insertions(+), 1 deletion(-) create mode 100644 apps/dev-playground/config/agents/query/evals/smoke.eval.ts create mode 100644 apps/dev-playground/config/agents/query/evals/sum.eval.ts create mode 100644 apps/dev-playground/config/agents/query/evals/tool-call.eval.ts create mode 100644 packages/appkit/src/evals/define-eval.ts create mode 100644 packages/appkit/src/evals/discover.ts create mode 100644 packages/appkit/src/evals/http-driver.ts create mode 100644 packages/appkit/src/evals/index.ts create mode 100644 packages/appkit/src/evals/matchers.ts create mode 100644 packages/appkit/src/evals/mlflow-report.ts create mode 100644 packages/appkit/src/evals/mlflow-rest.ts create mode 100644 packages/appkit/src/evals/mlflow-run.ts create mode 100644 packages/appkit/src/evals/report.ts create mode 100644 packages/appkit/src/evals/run-eval.ts create mode 100644 packages/appkit/src/evals/run-evals.ts create mode 100644 packages/appkit/src/evals/tests/discover.test.ts create mode 100644 packages/appkit/src/evals/tests/matchers.test.ts create mode 100644 packages/appkit/src/evals/tests/mlflow-report.test.ts create mode 100644 packages/appkit/src/evals/tests/mlflow-run.test.ts create mode 100644 packages/appkit/src/evals/tests/report.test.ts create mode 100644 packages/appkit/src/evals/tests/resolve-default.test.ts create mode 100644 packages/appkit/src/evals/tests/run-eval.test.ts create mode 100644 packages/appkit/src/evals/types.ts create mode 100644 packages/shared/src/cli/commands/agent/eval.ts create mode 100644 packages/shared/src/cli/commands/agent/index.ts diff --git a/apps/dev-playground/config/agents/query/evals/smoke.eval.ts b/apps/dev-playground/config/agents/query/evals/smoke.eval.ts new file mode 100644 index 00000000..de3f2d4d --- /dev/null +++ b/apps/dev-playground/config/agents/query/evals/smoke.eval.ts @@ -0,0 +1,14 @@ +import { defineEval } from "@databricks/appkit/beta"; + +/** + * Example eval. The agent defaults to this directory's name (`query`). + * Run with: + * pnpm exec appkit agent eval query --root apps/dev-playground --url http://localhost:8000 + */ +export default defineEval({ + description: "Query dispatcher responds to a greeting", + async test(t) { + await t.send("Hi there!"); + t.succeeded(); // gate: the turn completed + }, +}); diff --git a/apps/dev-playground/config/agents/query/evals/sum.eval.ts b/apps/dev-playground/config/agents/query/evals/sum.eval.ts new file mode 100644 index 00000000..9bc905e6 --- /dev/null +++ b/apps/dev-playground/config/agents/query/evals/sum.eval.ts @@ -0,0 +1,13 @@ +import { defineEval, includes } from "@databricks/appkit/beta"; + +export default defineEval({ + description: "Helper agent smoke test", + // Target the default code-defined agent. Drop this to use the `query` agent + // (the parent directory name). + agent: "helper", + async test(t) { + await t.send("What is 2 + 2?"); + t.succeeded(); // gate: the turn completed + t.check(t.reply, includes("4")).soft(); // tracked metric, won't fail the gate + }, +}); diff --git a/apps/dev-playground/config/agents/query/evals/tool-call.eval.ts b/apps/dev-playground/config/agents/query/evals/tool-call.eval.ts new file mode 100644 index 00000000..47221330 --- /dev/null +++ b/apps/dev-playground/config/agents/query/evals/tool-call.eval.ts @@ -0,0 +1,16 @@ +import { defineEval } from "@databricks/appkit/beta"; + +/** + * Tool-call eval: the default `helper` agent should call its `get_weather` + * tool when asked about the weather. (Targets `helper` explicitly — the parent + * directory only determines discovery, not which agent runs.) + */ +export default defineEval({ + description: "Helper agent calls the get_weather tool", + agent: "helper", + async test(t) { + await t.send("What's the weather in Brooklyn?"); + t.succeeded(); + t.calledTool("get_weather"); + }, +}); diff --git a/packages/appkit/src/beta.ts b/packages/appkit/src/beta.ts index 3f5bba80..16881c61 100644 --- a/packages/appkit/src/beta.ts +++ b/packages/appkit/src/beta.ts @@ -47,7 +47,8 @@ export { tool, toolsFromRegistry, } from "./core/agent/tools"; - +// Agent evaluation (eve-style authoring, reports to MLflow) +export * from "./evals"; // Agent types export type { AgentDefinition, diff --git a/packages/appkit/src/evals/define-eval.ts b/packages/appkit/src/evals/define-eval.ts new file mode 100644 index 00000000..9b8ff907 --- /dev/null +++ b/packages/appkit/src/evals/define-eval.ts @@ -0,0 +1,32 @@ +import type { EvalConfig, EvalDefinition } from "./types"; + +/** + * Define an agent eval. Default-export the result from a + * `config/agents//evals/*.eval.ts` file. + * + * @example + * ```ts + * import { defineEval, includes } from "@databricks/appkit/beta"; + * + * export default defineEval({ + * description: "Weather agent basic coverage", + * async test(t) { + * await t.send("What's the weather in Brooklyn?"); + * t.succeeded(); + * t.calledTool("get_weather"); + * t.check(t.reply, includes("Sunny")); + * }, + * }); + * ``` + */ +export function defineEval(def: EvalDefinition): EvalDefinition { + if (typeof def.test !== "function") { + throw new Error("defineEval: `test` must be a function"); + } + return def; +} + +/** Define per-directory eval config. Default-export from `evals.config.ts`. */ +export function defineEvalConfig(config: EvalConfig): EvalConfig { + return config; +} diff --git a/packages/appkit/src/evals/discover.ts b/packages/appkit/src/evals/discover.ts new file mode 100644 index 00000000..e60ae403 --- /dev/null +++ b/packages/appkit/src/evals/discover.ts @@ -0,0 +1,76 @@ +import { readdirSync, statSync } from "node:fs"; +import path from "node:path"; + +/** An eval file found under `config/agents//evals/`. */ +export interface DiscoveredEval { + /** Absolute path to the `*.eval.ts` file. */ + file: string; + /** Id relative to the agent's evals dir, without `.eval.ts` (e.g. `weather/basic`). */ + id: string; + /** The agent id (the `config/agents/` directory name). */ + agent: string; +} + +function isDir(p: string): boolean { + try { + return statSync(p).isDirectory(); + } catch { + return false; + } +} + +/** Recursively collect `*.eval.ts` files (skips `evals.config.ts`). */ +function walkEvalFiles(dir: string): string[] { + const out: string[] = []; + let entries: string[]; + try { + entries = readdirSync(dir); + } catch { + return out; + } + for (const entry of entries) { + const full = path.join(dir, entry); + if (isDir(full)) { + out.push(...walkEvalFiles(full)); + } else if (entry.endsWith(".eval.ts")) { + out.push(full); + } + } + return out; +} + +/** + * Discover evals under `/config/agents//evals/`. The agent id + * is the directory name; the eval id is the file path relative to that evals + * dir with `.eval.ts` stripped. Returns a stable, sorted list. + */ +export function discoverEvalFiles(rootDir: string): DiscoveredEval[] { + const agentsDir = path.join(rootDir, "config", "agents"); + const out: DiscoveredEval[] = []; + + let agents: string[]; + try { + agents = readdirSync(agentsDir).filter((n) => + isDir(path.join(agentsDir, n)), + ); + } catch { + return out; + } + + for (const agent of agents) { + const evalsDir = path.join(agentsDir, agent, "evals"); + if (!isDir(evalsDir)) continue; + for (const file of walkEvalFiles(evalsDir)) { + const id = path + .relative(evalsDir, file) + .replace(/\.eval\.ts$/, "") + .split(path.sep) + .join("/"); + out.push({ file, id, agent }); + } + } + + return out.sort( + (a, b) => a.agent.localeCompare(b.agent) || a.id.localeCompare(b.id), + ); +} diff --git a/packages/appkit/src/evals/http-driver.ts b/packages/appkit/src/evals/http-driver.ts new file mode 100644 index 00000000..0803a6e3 --- /dev/null +++ b/packages/appkit/src/evals/http-driver.ts @@ -0,0 +1,148 @@ +import type { DriveResult, EvalDriver } from "./types"; + +export interface HttpDriverOptions { + /** Base URL of the running app, e.g. `http://localhost:3000`. */ + baseUrl: string; + /** Agent alias to target. Omit to use the app's default agent. */ + agent?: string; + /** Extra request headers (e.g. auth for a deployed app). */ + headers?: Record; + /** Chat endpoint path. Defaults to `/api/agents/chat`. */ + path?: string; + /** MLflow run id to link each turn's trace to (for evaluation runs). */ + mlflowRunId?: string; +} + +/** Parse a single Responses-API SSE `data:` payload into the running totals. */ +function applyEvent( + event: Record, + state: { + reply: string; + toolCalls: string[]; + seen: Set; + ok: boolean; + traceId?: string; + }, + setThread: (id: string) => void, +): void { + const type = event.type; + if ( + type === "response.output_text.delta" && + typeof event.delta === "string" + ) { + state.reply += event.delta; + return; + } + if ( + type === "response.output_item.added" || + type === "response.output_item.done" + ) { + const item = event.item as + | { type?: string; name?: string; call_id?: string } + | undefined; + if (item?.type === "function_call" && item.name) { + const key = item.call_id ?? item.name; + if (!state.seen.has(key)) { + state.seen.add(key); + state.toolCalls.push(item.name); + } + } + return; + } + if (type === "error" || type === "response.failed") { + state.ok = false; + return; + } + if (type === "appkit.metadata") { + const data = event.data as + | { threadId?: string; mlflowTraceId?: string } + | undefined; + if (data?.threadId) setThread(data.threadId); + if (data?.mlflowTraceId) state.traceId = data.mlflowTraceId; + } +} + +/** + * Drives an agent by POSTing to a running app's chat endpoint and parsing the + * SSE response. Keeps the thread id across `send`s so multi-turn evals share a + * conversation. Agent/stream errors surface as `succeeded: false` rather than + * throwing, so `t.succeeded()` can assert on them. + */ +export function createHttpDriver(options: HttpDriverOptions): EvalDriver { + const chatPath = options.path ?? "/api/agents/chat"; + let threadId: string | undefined; + + return { + async send(message: string): Promise { + let res: Response; + try { + res = await fetch(`${options.baseUrl}${chatPath}`, { + method: "POST", + headers: { "content-type": "application/json", ...options.headers }, + body: JSON.stringify({ + message, + ...(options.agent ? { agent: options.agent } : {}), + ...(threadId ? { threadId } : {}), + ...(options.mlflowRunId + ? { mlflowRunId: options.mlflowRunId } + : {}), + }), + }); + } catch { + return { reply: "", toolCalls: [], succeeded: false }; + } + + if (!res.ok || !res.body) { + return { + reply: "", + toolCalls: [], + succeeded: false, + sessionId: threadId, + }; + } + + const state = { + reply: "", + toolCalls: [] as string[], + seen: new Set(), + ok: true, + traceId: undefined as string | undefined, + }; + const reader = res.body.getReader(); + const decoder = new TextDecoder(); + let buffer = ""; + + try { + while (true) { + const { done, value } = await reader.read(); + if (done) break; + buffer += decoder.decode(value, { stream: true }); + const lines = buffer.split("\n"); + buffer = lines.pop() ?? ""; + for (const line of lines) { + if (!line.startsWith("data: ")) continue; + const data = line.slice(6).trim(); + if (!data || data === "[DONE]") continue; + try { + applyEvent(JSON.parse(data), state, (id) => { + threadId = id; + }); + } catch { + // skip malformed event lines + } + } + } + } finally { + reader.releaseLock(); + } + + return { + reply: state.reply, + toolCalls: state.toolCalls, + succeeded: state.ok, + sessionId: threadId, + traceId: state.traceId, + }; + }, + }; +} diff --git a/packages/appkit/src/evals/index.ts b/packages/appkit/src/evals/index.ts new file mode 100644 index 00000000..7143d993 --- /dev/null +++ b/packages/appkit/src/evals/index.ts @@ -0,0 +1,40 @@ +export { defineEval, defineEvalConfig } from "./define-eval"; +export { type DiscoveredEval, discoverEvalFiles } from "./discover"; +export { createHttpDriver, type HttpDriverOptions } from "./http-driver"; +export { equals, includes, matches } from "./matchers"; +export { + type Assessment, + buildAssessment, + type MlflowReportOptions, + type ReportOutcome, + reportToMlflow, +} from "./mlflow-report"; +export { + type EvalSummary, + evalGlyph, + formatEvalDetail, + formatEvalHeadline, + formatEvalResults, + formatSummaryLine, + summarize, +} from "./report"; +export { type RunEvalOptions, runEval } from "./run-eval"; +export { + type EvalProgress, + type EvalRunSummary, + type RunEvalsOptions, + runEvalsInDir, +} from "./run-evals"; +export type { + AssertionHandle, + AssertionResult, + DriveResult, + EvalConfig, + EvalDefinition, + EvalDriver, + EvalResult, + Matcher, + MatchResult, + Severity, + TestContext, +} from "./types"; diff --git a/packages/appkit/src/evals/matchers.ts b/packages/appkit/src/evals/matchers.ts new file mode 100644 index 00000000..1e57796f --- /dev/null +++ b/packages/appkit/src/evals/matchers.ts @@ -0,0 +1,25 @@ +import type { Matcher } from "./types"; + +/** Passes when the value contains `substring`. */ +export function includes(substring: string): Matcher { + return (value) => ({ + pass: value.includes(substring), + detail: `expected to include ${JSON.stringify(substring)}`, + }); +} + +/** Passes when the value equals `expected` exactly. */ +export function equals(expected: string): Matcher { + return (value) => ({ + pass: value === expected, + detail: `expected to equal ${JSON.stringify(expected)}`, + }); +} + +/** Passes when the value matches `pattern`. */ +export function matches(pattern: RegExp): Matcher { + return (value) => ({ + pass: pattern.test(value), + detail: `expected to match ${pattern}`, + }); +} diff --git a/packages/appkit/src/evals/mlflow-report.ts b/packages/appkit/src/evals/mlflow-report.ts new file mode 100644 index 00000000..f801bbd6 --- /dev/null +++ b/packages/appkit/src/evals/mlflow-report.ts @@ -0,0 +1,113 @@ +import { normalizeHost } from "./mlflow-rest"; +import type { EvalResult } from "./types"; + +/** A Feedback assessment in the MLflow REST proto-JSON shape. */ +export interface Assessment { + trace_id: string; + assessment_name: string; + source: { source_type: "CODE" | "HUMAN" | "LLM_JUDGE"; source_id: string }; + feedback: { value: unknown }; + rationale?: string; + metadata?: Record; +} + +export interface MlflowReportOptions { + /** Databricks workspace host (scheme optional — normalized). */ + host: string; + /** Bearer token for the MLflow REST API. */ + token: string; +} + +export interface ReportOutcome { + written: number; + skipped: number; + failures: Array<{ traceId: string; status?: number; error?: string }>; +} + +/** + * Build the single pass/fail Feedback assessment for an eval result. Returns + * undefined when there's no trace to attach to or the eval was skipped. + */ +export function buildAssessment(result: EvalResult): Assessment | undefined { + if (!result.traceId || result.skipped) return undefined; + + const failed = result.assertions.filter((a) => !a.pass); + const rationale = result.error + ? `error: ${result.error}` + : failed.length + ? failed + .map( + (a) => + `${a.severity}:${a.label}${a.detail ? ` (${a.detail})` : ""}`, + ) + .join("; ") + : "all assertions passed"; + + return { + trace_id: result.traceId, + assessment_name: "appkit_eval", + source: { source_type: "CODE", source_id: "appkit-eval" }, + feedback: { value: result.passed }, + rationale, + metadata: { eval_id: result.id }, + }; +} + +async function postAssessment( + host: string, + token: string, + assessment: Assessment, +): Promise<{ ok: boolean; status?: number; error?: string }> { + const url = `${normalizeHost(host)}/api/3.0/mlflow/traces/${encodeURIComponent( + assessment.trace_id, + )}/assessments`; + try { + const res = await fetch(url, { + method: "POST", + headers: { + "content-type": "application/json", + authorization: `Bearer ${token}`, + }, + body: JSON.stringify({ assessment }), + }); + if (!res.ok) { + const text = await res.text().catch(() => ""); + return { ok: false, status: res.status, error: text.slice(0, 500) }; + } + return { ok: true }; + } catch (err) { + return { + ok: false, + error: err instanceof Error ? err.message : String(err), + }; + } +} + +/** + * Write one pass/fail assessment per eval result to the Databricks MLflow REST + * API. Never throws — failures are collected so the run still reports. + */ +export async function reportToMlflow( + results: EvalResult[], + options: MlflowReportOptions, +): Promise { + const outcome: ReportOutcome = { written: 0, skipped: 0, failures: [] }; + for (const result of results) { + const assessment = buildAssessment(result); + if (!assessment) { + outcome.skipped++; + continue; + } + const res = await postAssessment(options.host, options.token, assessment); + if (res.ok) { + outcome.written++; + } else { + outcome.failures.push({ + traceId: assessment.trace_id, + status: res.status, + error: res.error, + }); + } + } + return outcome; +} diff --git a/packages/appkit/src/evals/mlflow-rest.ts b/packages/appkit/src/evals/mlflow-rest.ts new file mode 100644 index 00000000..a89912d2 --- /dev/null +++ b/packages/appkit/src/evals/mlflow-rest.ts @@ -0,0 +1,39 @@ +/** Shared helpers for talking to the Databricks/MLflow REST API. */ + +export interface MlflowRestOptions { + /** Databricks workspace host (scheme optional — normalized). */ + host: string; + /** Bearer token for the MLflow REST API. */ + token: string; +} + +/** Ensure the host has a scheme (Databricks env often lacks `https://`). */ +export function normalizeHost(raw: string): string { + const h = raw.trim().replace(/\/+$/, ""); + return /^https?:\/\//i.test(h) ? h : `https://${h}`; +} + +/** + * POST JSON to an MLflow REST endpoint. Returns the parsed JSON body, or throws + * with the status + response text so callers can surface a precise error. + */ +export async function mlflowPost( + options: MlflowRestOptions, + path: string, + body: unknown, +): Promise { + const res = await fetch(`${normalizeHost(options.host)}${path}`, { + method: "POST", + headers: { + "content-type": "application/json", + authorization: `Bearer ${options.token}`, + }, + body: JSON.stringify(body), + }); + if (!res.ok) { + const text = await res.text().catch(() => ""); + throw new Error(`${path} -> ${res.status} ${text.slice(0, 500)}`); + } + const text = await res.text(); + return (text ? JSON.parse(text) : {}) as T; +} diff --git a/packages/appkit/src/evals/mlflow-run.ts b/packages/appkit/src/evals/mlflow-run.ts new file mode 100644 index 00000000..5806a322 --- /dev/null +++ b/packages/appkit/src/evals/mlflow-run.ts @@ -0,0 +1,117 @@ +import { type MlflowRestOptions, mlflowPost } from "./mlflow-rest"; +import type { EvalResult } from "./types"; + +/** Run tag value that makes a run appear under the experiment's "Evaluation runs". */ +const GENAI_EVALUATE_RUN_TYPE = "genai_evaluate"; + +interface CreateRunResponse { + run: { info: { run_id?: string; run_uuid?: string } }; +} + +interface MlflowMetric { + key: string; + value: number; + timestamp: number; + step: number; +} + +/** + * Create an MLflow run tagged as a GenAI evaluation, so it shows under the + * experiment's "Evaluation runs". Returns the run id; link traces to it via the + * `mlflow.sourceRun` trace metadata and log results before finishing. + */ +export async function createEvalRun( + options: MlflowRestOptions & { + experimentId: string; + runName?: string; + startTime: number; + }, +): Promise { + const created = await mlflowPost( + options, + "/api/2.0/mlflow/runs/create", + { + experiment_id: options.experimentId, + start_time: options.startTime, + ...(options.runName ? { run_name: options.runName } : {}), + }, + ); + const runId = created.run?.info?.run_id ?? created.run?.info?.run_uuid; + if (!runId) { + throw new Error("runs/create returned no run id"); + } + await mlflowPost(options, "/api/2.0/mlflow/runs/set-tag", { + run_id: runId, + key: "mlflow.runType", + value: GENAI_EVALUATE_RUN_TYPE, + }); + return runId; +} + +/** Aggregate per-eval results into MLflow run metrics. */ +export function aggregateMetrics( + results: EvalResult[], + timestamp: number, +): MlflowMetric[] { + const scored = results.filter((r) => !r.skipped); + const passed = scored.filter((r) => r.passed).length; + return [ + { key: "eval/total", value: results.length, timestamp, step: 0 }, + { key: "eval/scored", value: scored.length, timestamp, step: 0 }, + { key: "eval/passed", value: passed, timestamp, step: 0 }, + { + key: "eval/pass_rate", + value: scored.length ? passed / scored.length : 0, + timestamp, + step: 0, + }, + ]; +} + +export interface FinishOutcome { + finished: boolean; + /** Metric logging is best-effort; set when it failed (the run is still finished). */ + metricsError?: string; + /** Set when the FINISHED update itself failed (run may be left RUNNING). */ + finishError?: string; +} + +/** + * Log aggregate metrics (best-effort) and mark the eval run FINISHED. Never + * throws — a metric-logging failure must not prevent the run from being closed, + * or it would be left stuck in RUNNING forever. + */ +export async function finishEvalRun( + options: MlflowRestOptions & { + runId: string; + results: EvalResult[]; + endTime: number; + }, +): Promise { + const outcome: FinishOutcome = { finished: false }; + + const metrics = aggregateMetrics(options.results, options.endTime); + if (metrics.length) { + try { + await mlflowPost(options, "/api/2.0/mlflow/runs/log-batch", { + run_id: options.runId, + metrics, + }); + } catch (err) { + outcome.metricsError = err instanceof Error ? err.message : String(err); + } + } + + try { + await mlflowPost(options, "/api/2.0/mlflow/runs/update", { + run_id: options.runId, + status: "FINISHED", + end_time: options.endTime, + }); + outcome.finished = true; + } catch (err) { + outcome.finishError = err instanceof Error ? err.message : String(err); + } + + return outcome; +} diff --git a/packages/appkit/src/evals/report.ts b/packages/appkit/src/evals/report.ts new file mode 100644 index 00000000..2e42c5b1 --- /dev/null +++ b/packages/appkit/src/evals/report.ts @@ -0,0 +1,76 @@ +import type { EvalResult } from "./types"; + +export interface EvalSummary { + total: number; + passed: number; + failed: number; + skipped: number; + /** True when no eval failed (skips don't count as failures). */ + allPassed: boolean; +} + +export function summarize(results: EvalResult[]): EvalSummary { + let passed = 0; + let failed = 0; + let skipped = 0; + for (const r of results) { + if (r.skipped) skipped++; + else if (r.passed) passed++; + else failed++; + } + return { + total: results.length, + passed, + failed, + skipped, + allPassed: failed === 0, + }; +} + +/** Status glyph for a single eval result. */ +export function evalGlyph(result: EvalResult): string { + if (result.skipped) return "−"; + return result.passed ? "✓" : "✗"; +} + +/** The one-line header for a single eval result (no failure detail). */ +export function formatEvalHeadline(result: EvalResult): string { + if (result.skipped) { + return `− ${result.id} (skipped${ + result.skipped.reason ? `: ${result.skipped.reason}` : "" + })`; + } + return `${evalGlyph(result)} ${result.id}${ + result.description ? ` — ${result.description}` : "" + }`; +} + +/** Indented detail lines for a failing eval (error + failing assertions). */ +export function formatEvalDetail(result: EvalResult): string[] { + const lines: string[] = []; + if (result.error) lines.push(` error: ${result.error}`); + for (const a of result.assertions) { + if (a.pass) continue; + const tag = a.severity === "soft" ? "soft" : "gate"; + lines.push(` ✗ [${tag}] ${a.label}${a.detail ? ` — ${a.detail}` : ""}`); + } + return lines; +} + +/** The final PASS/FAIL summary line. */ +export function formatSummaryLine(results: EvalResult[]): string { + const s = summarize(results); + return `${s.allPassed ? "PASS" : "FAIL"} — ${s.passed} passed, ${s.failed} failed, ${s.skipped} skipped (${s.total} total)`; +} + +/** Render all results as a human-readable console report (non-streaming). */ +export function formatEvalResults(results: EvalResult[]): string { + const lines: string[] = []; + for (const r of results) { + lines.push(formatEvalHeadline(r)); + lines.push(...formatEvalDetail(r)); + } + lines.push(""); + lines.push(formatSummaryLine(results)); + return lines.join("\n"); +} diff --git a/packages/appkit/src/evals/run-eval.ts b/packages/appkit/src/evals/run-eval.ts new file mode 100644 index 00000000..637508f2 --- /dev/null +++ b/packages/appkit/src/evals/run-eval.ts @@ -0,0 +1,155 @@ +import type { + AssertionHandle, + AssertionResult, + EvalDefinition, + EvalDriver, + EvalResult, + Matcher, + TestContext, +} from "./types"; + +/** Thrown by `t.skip()` to unwind the test and mark the eval skipped. */ +class SkipSignal extends Error { + constructor(public reason?: string) { + super("eval skipped"); + this.name = "SkipSignal"; + } +} + +export interface RunEvalOptions { + /** Stable id for the eval (e.g. its file path relative to the evals dir). */ + id: string; + /** Drives the agent and returns reply/tool-calls/success per `send`. */ + driver: EvalDriver; + /** When true, soft assertion failures also fail the eval. */ + strict?: boolean; +} + +/** + * Run a single eval against a driver. Never throws for assertion or agent + * failures — those become a non-passing {@link EvalResult}. Only a malformed + * eval definition surfaces as `result.error`. + */ +export async function runEval( + def: EvalDefinition, + options: RunEvalOptions, +): Promise { + const assertions: AssertionResult[] = []; + let reply = ""; + let toolCalls: string[] = []; + let sessionId: string | undefined; + let lastTraceId: string | undefined; + let lastSucceeded = false; + + const record = ( + label: string, + pass: boolean, + score?: number, + detail?: string, + ): AssertionHandle => { + const result: AssertionResult = { + label, + severity: "gate", + pass, + score, + detail, + }; + assertions.push(result); + const handle: AssertionHandle = { + gate() { + result.severity = "gate"; + return handle; + }, + soft() { + result.severity = "soft"; + return handle; + }, + atLeast(threshold: number) { + result.severity = "soft"; + result.pass = (result.score ?? (result.pass ? 1 : 0)) >= threshold; + return handle; + }, + }; + return handle; + }; + + const t: TestContext = { + async send(message) { + const r = await options.driver.send(message); + reply = r.reply; + toolCalls = r.toolCalls; + sessionId = r.sessionId; + lastSucceeded = r.succeeded; + if (r.traceId) lastTraceId = r.traceId; + }, + get reply() { + return reply; + }, + get toolCalls() { + return toolCalls; + }, + get sessionId() { + return sessionId; + }, + succeeded() { + return record( + "succeeded", + lastSucceeded, + undefined, + lastSucceeded ? undefined : "agent turn did not complete successfully", + ); + }, + calledTool(name) { + return record( + `calledTool(${name})`, + toolCalls.includes(name), + undefined, + `expected tool "${name}" to be called (called: ${ + toolCalls.length ? toolCalls.join(", ") : "none" + })`, + ); + }, + check(value: string, matcher: Matcher) { + const m = matcher(value); + return record("check", m.pass, m.score, m.detail); + }, + skip(reason) { + throw new SkipSignal(reason); + }, + }; + + try { + await def.test(t); + } catch (err) { + if (err instanceof SkipSignal) { + return { + id: options.id, + description: def.description, + skipped: { reason: err.reason }, + assertions, + passed: true, + traceId: lastTraceId, + }; + } + return { + id: options.id, + description: def.description, + assertions, + passed: false, + error: err instanceof Error ? err.message : String(err), + traceId: lastTraceId, + }; + } + + const passed = assertions.every( + (a) => a.pass || (a.severity === "soft" && !options.strict), + ); + + return { + id: options.id, + description: def.description, + assertions, + passed, + traceId: lastTraceId, + }; +} diff --git a/packages/appkit/src/evals/run-evals.ts b/packages/appkit/src/evals/run-evals.ts new file mode 100644 index 00000000..4cbd51c0 --- /dev/null +++ b/packages/appkit/src/evals/run-evals.ts @@ -0,0 +1,166 @@ +import { pathToFileURL } from "node:url"; +import { discoverEvalFiles } from "./discover"; +import { createHttpDriver } from "./http-driver"; +import { type ReportOutcome, reportToMlflow } from "./mlflow-report"; +import { createEvalRun, type FinishOutcome, finishEvalRun } from "./mlflow-run"; +import { runEval } from "./run-eval"; +import type { EvalDefinition, EvalResult } from "./types"; + +export interface RunEvalsOptions { + /** Project root containing `config/agents/`. Defaults to `process.cwd()`. */ + rootDir?: string; + /** Base URL of the running app to drive, e.g. `http://localhost:3000`. */ + baseUrl: string; + /** Substring filter on `/` (or an exact agent id). */ + filter?: string; + /** Soft assertion failures also fail the eval. */ + strict?: boolean; + /** Extra request headers for the driver (e.g. auth for a deployed app). */ + headers?: Record; + /** + * When set, create a native MLflow "Evaluation run": each eval's trace is + * linked to the run, pass/fail is written as feedback, and aggregate metrics + * are logged. Requires Databricks creds + the target experiment. + */ + mlflow?: { host: string; token: string; experimentId: string }; + /** Wall-clock timestamp (ms) for run create/finish — pass `Date.now()`. */ + now?: number; + /** Progress callback, invoked as evals are discovered, started, and finished. */ + onEvent?: (event: EvalProgress) => void; +} + +export type EvalProgress = + | { type: "discovered"; total: number } + | { type: "run-created"; runId: string } + | { type: "start"; id: string; index: number; total: number } + | { type: "result"; result: EvalResult; index: number; total: number }; + +export interface EvalRunSummary { + results: EvalResult[]; + /** Present when an MLflow evaluation run was created. */ + mlflow?: { runId: string; report: ReportOutcome; finish: FinishOutcome }; +} + +/** + * Load a `*.eval.ts` file and return its default-exported {@link EvalDefinition}. + * Uses tsx's programmatic loader so TypeScript eval files run without a build + * step. The specifier is indirected so the type checker doesn't try to resolve + * tsx's internal entry. + */ +async function loadEval(file: string): Promise { + const tsxApi = "tsx/esm/api"; + let tsImport: (specifier: string, parentURL: string) => Promise; + try { + ({ tsImport } = (await import(tsxApi)) as { + tsImport: (specifier: string, parentURL: string) => Promise; + }); + } catch { + throw new Error( + "Running .eval.ts files requires `tsx`. Install it as a dev dependency (`pnpm add -D tsx`).", + ); + } + + const mod = await tsImport(pathToFileURL(file).href, import.meta.url); + const def = resolveEvalDefault(mod); + if (!def) { + throw new Error(`${file}: must default-export defineEval({ test })`); + } + return def; +} + +/** + * Unwrap the eval default export across module-interop shapes. Depending on + * whether the eval file is treated as ESM or CJS, the value lands at + * `mod.default` (ESM), `mod.default.default` (CJS `__esModule` double-wrap), or + * `mod` itself. Returns the first candidate that looks like an eval. + */ +export function resolveEvalDefault(mod: unknown): EvalDefinition | undefined { + const seen = new Set(); + let candidate: unknown = mod; + for (let i = 0; i < 4 && candidate && !seen.has(candidate); i++) { + if (typeof (candidate as EvalDefinition).test === "function") { + return candidate as EvalDefinition; + } + seen.add(candidate); + candidate = (candidate as { default?: unknown }).default; + } + return undefined; +} + +/** + * Discover, load, and run every eval under each agent's `evals/` dir, driving + * the agents on a running app. Never throws for an individual eval — load/run + * failures become non-passing {@link EvalResult}s. + */ +export async function runEvalsInDir( + options: RunEvalsOptions, +): Promise { + const root = options.rootDir ?? process.cwd(); + const now = options.now ?? Date.now(); + let discovered = discoverEvalFiles(root); + + if (options.filter) { + const f = options.filter; + discovered = discovered.filter( + (d) => d.agent === f || `${d.agent}/${d.id}`.includes(f), + ); + } + + const emit = options.onEvent ?? (() => {}); + const total = discovered.length; + emit({ type: "discovered", total }); + + // Create the MLflow evaluation run up front so each eval's trace can be + // linked to it as it runs. + let runId: string | undefined; + if (options.mlflow) { + runId = await createEvalRun({ + ...options.mlflow, + runName: `appkit-eval ${new Date(now).toISOString()}`, + startTime: now, + }); + emit({ type: "run-created", runId }); + } + + const results: EvalResult[] = []; + for (let index = 0; index < discovered.length; index++) { + const d = discovered[index]; + const id = `${d.agent}/${d.id}`; + emit({ type: "start", id, index, total }); + + let result: EvalResult; + try { + const def = await loadEval(d.file); + const driver = createHttpDriver({ + baseUrl: options.baseUrl, + agent: def.agent ?? d.agent, + headers: options.headers, + mlflowRunId: runId, + }); + result = await runEval(def, { id, driver, strict: options.strict }); + } catch (err) { + result = { + id, + assertions: [], + passed: false, + error: err instanceof Error ? err.message : String(err), + }; + } + + results.push(result); + emit({ type: "result", result, index, total }); + } + + if (options.mlflow && runId) { + const report = await reportToMlflow(results, options.mlflow); + const finish = await finishEvalRun({ + ...options.mlflow, + runId, + results, + endTime: options.now ?? Date.now(), + }); + return { results, mlflow: { runId, report, finish } }; + } + + return { results }; +} diff --git a/packages/appkit/src/evals/tests/discover.test.ts b/packages/appkit/src/evals/tests/discover.test.ts new file mode 100644 index 00000000..6eb7c220 --- /dev/null +++ b/packages/appkit/src/evals/tests/discover.test.ts @@ -0,0 +1,42 @@ +import { mkdirSync, mkdtempSync, rmSync, writeFileSync } from "node:fs"; +import { tmpdir } from "node:os"; +import path from "node:path"; +import { afterEach, beforeEach, describe, expect, test } from "vitest"; +import { discoverEvalFiles } from "../discover"; + +let root: string; + +function write(rel: string, content = "export default {}") { + const full = path.join(root, rel); + mkdirSync(path.dirname(full), { recursive: true }); + writeFileSync(full, content); +} + +beforeEach(() => { + root = mkdtempSync(path.join(tmpdir(), "appkit-evals-")); +}); +afterEach(() => { + rmSync(root, { recursive: true, force: true }); +}); + +describe("discoverEvalFiles", () => { + test("finds *.eval.ts per agent, derives id + agent, ignores config", () => { + write("config/agents/support/evals/basic.eval.ts"); + write("config/agents/support/evals/nested/deep.eval.ts"); + write("config/agents/support/evals/evals.config.ts"); + write("config/agents/analyst/evals/sql.eval.ts"); + write("config/agents/no-evals/agent.md", "# agent"); + + const found = discoverEvalFiles(root); + + expect(found.map((f) => `${f.agent}/${f.id}`)).toEqual([ + "analyst/sql", + "support/basic", + "support/nested/deep", + ]); + }); + + test("returns empty when there is no config/agents dir", () => { + expect(discoverEvalFiles(root)).toEqual([]); + }); +}); diff --git a/packages/appkit/src/evals/tests/matchers.test.ts b/packages/appkit/src/evals/tests/matchers.test.ts new file mode 100644 index 00000000..03653b52 --- /dev/null +++ b/packages/appkit/src/evals/tests/matchers.test.ts @@ -0,0 +1,19 @@ +import { describe, expect, test } from "vitest"; +import { equals, includes, matches } from "../matchers"; + +describe("eval matchers", () => { + test("includes", () => { + expect(includes("Sunny")("It is Sunny today").pass).toBe(true); + expect(includes("Rainy")("It is Sunny today").pass).toBe(false); + }); + + test("equals", () => { + expect(equals("yes")("yes").pass).toBe(true); + expect(equals("yes")("Yes").pass).toBe(false); + }); + + test("matches", () => { + expect(matches(/^\d+ rows$/)("42 rows").pass).toBe(true); + expect(matches(/^\d+ rows$/)("forty rows").pass).toBe(false); + }); +}); diff --git a/packages/appkit/src/evals/tests/mlflow-report.test.ts b/packages/appkit/src/evals/tests/mlflow-report.test.ts new file mode 100644 index 00000000..14155640 --- /dev/null +++ b/packages/appkit/src/evals/tests/mlflow-report.test.ts @@ -0,0 +1,57 @@ +import { describe, expect, test } from "vitest"; +import { buildAssessment } from "../mlflow-report"; +import type { EvalResult } from "../types"; + +describe("buildAssessment", () => { + test("builds a pass assessment with trace id and source", () => { + const result: EvalResult = { + id: "support/weather", + traceId: "tr-123", + assertions: [{ label: "succeeded", severity: "gate", pass: true }], + passed: true, + }; + const a = buildAssessment(result); + expect(a).toEqual({ + trace_id: "tr-123", + assessment_name: "appkit_eval", + source: { source_type: "CODE", source_id: "appkit-eval" }, + feedback: { value: true }, + rationale: "all assertions passed", + metadata: { eval_id: "support/weather" }, + }); + }); + + test("fail assessment summarizes failing assertions in the rationale", () => { + const a = buildAssessment({ + id: "support/x", + traceId: "tr-9", + assertions: [ + { label: "succeeded", severity: "gate", pass: true }, + { + label: "calledTool(get_weather)", + severity: "gate", + pass: false, + detail: "not called", + }, + ], + passed: false, + }); + expect(a?.feedback.value).toBe(false); + expect(a?.rationale).toContain("gate:calledTool(get_weather) (not called)"); + }); + + test("returns undefined without a trace id or when skipped", () => { + expect( + buildAssessment({ id: "x", assertions: [], passed: true }), + ).toBeUndefined(); + expect( + buildAssessment({ + id: "x", + traceId: "tr-1", + assertions: [], + passed: true, + skipped: { reason: "no data" }, + }), + ).toBeUndefined(); + }); +}); diff --git a/packages/appkit/src/evals/tests/mlflow-run.test.ts b/packages/appkit/src/evals/tests/mlflow-run.test.ts new file mode 100644 index 00000000..8c4318cd --- /dev/null +++ b/packages/appkit/src/evals/tests/mlflow-run.test.ts @@ -0,0 +1,30 @@ +import { describe, expect, test } from "vitest"; +import { aggregateMetrics } from "../mlflow-run"; +import type { EvalResult } from "../types"; + +describe("aggregateMetrics", () => { + test("counts total/scored/passed and computes pass_rate (skips excluded)", () => { + const results: EvalResult[] = [ + { id: "a", assertions: [], passed: true }, + { id: "b", assertions: [], passed: false }, + { id: "c", assertions: [], passed: true, skipped: { reason: "x" } }, + ]; + const metrics = aggregateMetrics(results, 1000); + const byKey = Object.fromEntries(metrics.map((m) => [m.key, m.value])); + expect(byKey["eval/total"]).toBe(3); + expect(byKey["eval/scored"]).toBe(2); + expect(byKey["eval/passed"]).toBe(1); + expect(byKey["eval/pass_rate"]).toBe(0.5); + expect(metrics.every((m) => m.timestamp === 1000 && m.step === 0)).toBe( + true, + ); + }); + + test("pass_rate is 0 when nothing is scored", () => { + const metrics = aggregateMetrics( + [{ id: "a", assertions: [], passed: true, skipped: {} }], + 1, + ); + expect(metrics.find((m) => m.key === "eval/pass_rate")?.value).toBe(0); + }); +}); diff --git a/packages/appkit/src/evals/tests/report.test.ts b/packages/appkit/src/evals/tests/report.test.ts new file mode 100644 index 00000000..c02c6884 --- /dev/null +++ b/packages/appkit/src/evals/tests/report.test.ts @@ -0,0 +1,54 @@ +import { describe, expect, test } from "vitest"; +import { formatEvalResults, summarize } from "../report"; +import type { EvalResult } from "../types"; + +const results: EvalResult[] = [ + { + id: "a/pass", + assertions: [{ label: "succeeded", severity: "gate", pass: true }], + passed: true, + }, + { + id: "a/fail", + assertions: [ + { + label: "calledTool(x)", + severity: "gate", + pass: false, + detail: "not called", + }, + ], + passed: false, + }, + { + id: "a/skip", + assertions: [], + passed: true, + skipped: { reason: "no data" }, + }, +]; + +describe("eval reporting", () => { + test("summarize counts pass/fail/skip and allPassed", () => { + expect(summarize(results)).toEqual({ + total: 3, + passed: 1, + failed: 1, + skipped: 1, + allPassed: false, + }); + }); + + test("summarize allPassed is true when nothing failed", () => { + expect(summarize([results[0], results[2]]).allPassed).toBe(true); + }); + + test("formatEvalResults shows status, failing assertions, and a summary line", () => { + const out = formatEvalResults(results); + expect(out).toContain("✓ a/pass"); + expect(out).toContain("✗ a/fail"); + expect(out).toContain("[gate] calledTool(x) — not called"); + expect(out).toContain("a/skip (skipped: no data)"); + expect(out).toContain("FAIL — 1 passed, 1 failed, 1 skipped (3 total)"); + }); +}); diff --git a/packages/appkit/src/evals/tests/resolve-default.test.ts b/packages/appkit/src/evals/tests/resolve-default.test.ts new file mode 100644 index 00000000..40079226 --- /dev/null +++ b/packages/appkit/src/evals/tests/resolve-default.test.ts @@ -0,0 +1,25 @@ +import { describe, expect, test } from "vitest"; +import { resolveEvalDefault } from "../run-evals"; + +const def = { description: "x", test: async () => {} }; + +describe("resolveEvalDefault (module interop)", () => { + test("pure ESM: mod.default", () => { + expect(resolveEvalDefault({ default: def })).toBe(def); + }); + + test("CJS __esModule double-wrap: mod.default.default", () => { + expect( + resolveEvalDefault({ default: { __esModule: true, default: def } }), + ).toBe(def); + }); + + test("direct: mod is the def", () => { + expect(resolveEvalDefault(def)).toBe(def); + }); + + test("no default export → undefined", () => { + expect(resolveEvalDefault({ default: { notTest: 1 } })).toBeUndefined(); + expect(resolveEvalDefault(null)).toBeUndefined(); + }); +}); diff --git a/packages/appkit/src/evals/tests/run-eval.test.ts b/packages/appkit/src/evals/tests/run-eval.test.ts new file mode 100644 index 00000000..abc78030 --- /dev/null +++ b/packages/appkit/src/evals/tests/run-eval.test.ts @@ -0,0 +1,117 @@ +import { describe, expect, test } from "vitest"; +import { defineEval } from "../define-eval"; +import { includes } from "../matchers"; +import { runEval } from "../run-eval"; +import type { DriveResult, EvalDriver } from "../types"; + +/** Driver that returns a fixed result for every send. */ +function fakeDriver(result: Partial): EvalDriver { + return { + send: async () => ({ + reply: "", + toolCalls: [], + succeeded: true, + ...result, + }), + }; +} + +describe("runEval", () => { + test("passes when all gate assertions pass", async () => { + const def = defineEval({ + async test(t) { + await t.send("weather?"); + t.succeeded(); + t.calledTool("get_weather"); + t.check(t.reply, includes("Sunny")); + }, + }); + const result = await runEval(def, { + id: "weather", + driver: fakeDriver({ + reply: "It is Sunny", + toolCalls: ["get_weather"], + succeeded: true, + }), + }); + expect(result.passed).toBe(true); + expect(result.assertions).toHaveLength(3); + expect(result.assertions.every((a) => a.pass)).toBe(true); + }); + + test("fails when a gate assertion fails", async () => { + const def = defineEval({ + async test(t) { + await t.send("weather?"); + t.calledTool("get_weather"); + }, + }); + const result = await runEval(def, { + id: "no-tool", + driver: fakeDriver({ reply: "I don't know", toolCalls: [] }), + }); + expect(result.passed).toBe(false); + expect(result.assertions[0].pass).toBe(false); + }); + + test("soft failures don't fail the eval unless strict", async () => { + const def = defineEval({ + async test(t) { + await t.send("hi"); + t.calledTool("get_weather").soft(); + }, + }); + const lenient = await runEval(def, { + id: "soft", + driver: fakeDriver({ toolCalls: [] }), + }); + expect(lenient.passed).toBe(true); + + const strict = await runEval(def, { + id: "soft", + driver: fakeDriver({ toolCalls: [] }), + strict: true, + }); + expect(strict.passed).toBe(false); + }); + + test("succeeded() reflects the driver's success flag", async () => { + const def = defineEval({ + async test(t) { + await t.send("hi"); + t.succeeded(); + }, + }); + const result = await runEval(def, { + id: "failed-turn", + driver: fakeDriver({ succeeded: false }), + }); + expect(result.passed).toBe(false); + }); + + test("t.skip marks the eval skipped and passing", async () => { + const def = defineEval({ + test(t) { + t.skip("no fixture data"); + }, + }); + const result = await runEval(def, { + id: "skipped", + driver: fakeDriver({}), + }); + expect(result.skipped?.reason).toBe("no fixture data"); + expect(result.passed).toBe(true); + expect(result.assertions).toHaveLength(0); + }); + + test("a thrown error becomes a non-passing result, not an exception", async () => { + const def = defineEval({ + async test() { + throw new Error("boom"); + }, + }); + const result = await runEval(def, { id: "boom", driver: fakeDriver({}) }); + expect(result.passed).toBe(false); + expect(result.error).toBe("boom"); + }); +}); diff --git a/packages/appkit/src/evals/types.ts b/packages/appkit/src/evals/types.ts new file mode 100644 index 00000000..c6ef02e1 --- /dev/null +++ b/packages/appkit/src/evals/types.ts @@ -0,0 +1,128 @@ +/** + * Agent evaluation primitives — an eve-style authoring API that runs against + * AppKit agents and reports to Databricks MLflow. + * + * Evals live in `config/agents//evals/*.eval.ts`, each default-exporting a + * {@link EvalDefinition} via {@link defineEval}. A runner drives the agent + * (today over HTTP against a running app), and the `test` function asserts on + * the reply and tool usage with deterministic matchers (and, later, LLM judges). + */ + +/** Result of a deterministic matcher run against a value. */ +export interface MatchResult { + pass: boolean; + /** Optional 0..1 score for scored matchers (similarity, judges). */ + score?: number; + /** Human-readable explanation, shown on failure. */ + detail?: string; +} + +/** A deterministic matcher: inspects a string value and returns a result. */ +export type Matcher = (value: string) => MatchResult; + +/** Whether an assertion fails the eval (`gate`) or is tracked only (`soft`). */ +export type Severity = "gate" | "soft"; + +/** A single recorded assertion outcome. */ +export interface AssertionResult { + label: string; + severity: Severity; + pass: boolean; + score?: number; + detail?: string; +} + +/** + * Chainable handle returned by every assertion to control its severity. + * Mirrors eve: assertions are gates by default; `.soft()` demotes to a tracked + * metric; `.atLeast(n)` is a soft, score-thresholded assertion. + */ +export interface AssertionHandle { + /** Promote to a hard gate — failure fails the eval (non-zero exit). */ + gate(): AssertionHandle; + /** Demote to a tracked metric — doesn't fail unless running with `strict`. */ + soft(): AssertionHandle; + /** Soft assertion that passes only when the score is at least `threshold`. */ + atLeast(threshold: number): AssertionHandle; +} + +/** What a driver returns for a single `t.send`. */ +export interface DriveResult { + /** The final assistant message text. */ + reply: string; + /** Names of tools the agent called during the turn. */ + toolCalls: string[]; + /** Whether the turn completed without an agent/stream error. */ + succeeded: boolean; + /** Thread/session id, when the driver exposes one. */ + sessionId?: string; + /** MLflow trace id for the turn, when tracing is enabled on the app. */ + traceId?: string; +} + +/** + * Abstraction over how the agent is driven. The HTTP driver posts to a running + * app's agents endpoint; future drivers (in-process) implement the same shape. + */ +export interface EvalDriver { + send(message: string): Promise; +} + +/** The `t` context passed to an eval's `test` function. */ +export interface TestContext { + /** Send a user message to the agent and capture its response. */ + send(message: string): Promise; + /** The last assistant reply. */ + readonly reply: string; + /** Tools called during the last turn. */ + readonly toolCalls: string[]; + /** The current session/thread id, if any. */ + readonly sessionId: string | undefined; + /** Assert the last turn completed successfully (gate by default). */ + succeeded(): AssertionHandle; + /** Assert a tool was called during the run (gate by default). */ + calledTool(name: string): AssertionHandle; + /** Assert a value against a matcher, e.g. `t.check(t.reply, includes("Sunny"))`. */ + check(value: string, matcher: Matcher): AssertionHandle; + /** Skip this eval with an optional reason. */ + skip(reason?: string): never; +} + +/** A single eval, default-exported from a `*.eval.ts` file. */ +export interface EvalDefinition { + /** Short human description, shown in reports. */ + description?: string; + /** Target agent id. Defaults to the eval's parent `config/agents/` dir. */ + agent?: string; + /** Free-form tags for filtering. */ + tags?: string[]; + /** Per-eval timeout. */ + timeoutMs?: number; + /** The eval body: drive the agent and assert on its behavior. */ + test(t: TestContext): Promise | void; +} + +/** Per-directory config from `evals.config.ts`. */ +export interface EvalConfig { + /** LLM judge config. Defaults to the agent's own serving endpoint. */ + judge?: { model?: string }; + /** Max evals to run concurrently. */ + maxConcurrency?: number; + /** Default per-eval timeout. */ + timeoutMs?: number; +} + +/** The outcome of running one eval. */ +export interface EvalResult { + id: string; + description?: string; + /** Set when the eval called `t.skip`. */ + skipped?: { reason?: string }; + assertions: AssertionResult[]; + /** True when all gates passed (and, under strict, all soft assertions too). */ + passed: boolean; + /** Set when the eval threw before completing. */ + error?: string; + /** MLflow trace id of the eval's last turn, for attaching assessments. */ + traceId?: string; +} diff --git a/packages/shared/src/cli/commands/agent/eval.ts b/packages/shared/src/cli/commands/agent/eval.ts new file mode 100644 index 00000000..ee692423 --- /dev/null +++ b/packages/shared/src/cli/commands/agent/eval.ts @@ -0,0 +1,191 @@ +import { Command } from "commander"; + +interface EvalRunSummary { + results: unknown[]; + mlflow?: { + runId: string; + report: { + written: number; + skipped: number; + failures: Array<{ traceId: string; status?: number; error?: string }>; + }; + finish: { finished: boolean; metricsError?: string; finishError?: string }; + }; +} + +type EvalProgress = + | { type: "discovered"; total: number } + | { type: "run-created"; runId: string } + | { type: "start"; id: string; index: number; total: number } + | { type: "result"; result: unknown; index: number; total: number }; + +/** Subset of `@databricks/appkit/beta`'s eval runner used by this command. */ +interface EvalRunner { + runEvalsInDir(opts: { + rootDir?: string; + baseUrl: string; + filter?: string; + strict?: boolean; + headers?: Record; + mlflow?: { host: string; token: string; experimentId: string }; + onEvent?: (event: EvalProgress) => void; + }): Promise; + evalGlyph(result: unknown): string; + formatEvalDetail(result: unknown): string[]; + formatSummaryLine(results: unknown[]): string; + summarize(results: unknown[]): { allPassed: boolean }; +} + +/** + * Loaded at runtime from the consuming project so this command (which ships in + * `@databricks/shared`) doesn't take a build-time dependency on appkit. The + * specifier is a variable so the type checker treats it as `any`. + */ +async function loadRunner(): Promise { + const spec = "@databricks/appkit/beta"; + try { + return (await import(spec)) as unknown as EvalRunner; + } catch (err) { + throw new Error( + "Could not load @databricks/appkit. Run `appkit agent eval` from a " + + "project with @databricks/appkit installed. " + + `Cause: ${err instanceof Error ? err.message : String(err)}`, + ); + } +} + +function parseHeaders(values: string[]): Record { + const headers: Record = {}; + for (const v of values) { + const i = v.indexOf(":"); + if (i === -1) continue; + headers[v.slice(0, i).trim()] = v.slice(i + 1).trim(); + } + return headers; +} + +interface EvalOptions { + url: string; + strict?: boolean; + root?: string; + header?: string[]; + databricksHost?: string; + databricksToken?: string; + experiment?: string; +} + +async function runAgentEval( + filter: string | undefined, + opts: EvalOptions, +): Promise { + const runner = await loadRunner(); + + // Create a native MLflow "Evaluation run" when Databricks creds + an + // experiment are available (traces live in the app; the run + scores are + // driven from here, so this side needs creds). + const host = opts.databricksHost ?? process.env.DATABRICKS_HOST; + const token = opts.databricksToken ?? process.env.DATABRICKS_TOKEN; + const experimentId = opts.experiment ?? process.env.MLFLOW_EXPERIMENT_ID; + const mlflow = + host && token && experimentId ? { host, token, experimentId } : undefined; + + // Stream progress as evals run, instead of going silent until the end. + const onEvent = (event: EvalProgress): void => { + switch (event.type) { + case "discovered": + console.log( + `Running ${event.total} eval${event.total === 1 ? "" : "s"} against ${opts.url}\n`, + ); + break; + case "run-created": + console.log(`MLflow evaluation run: ${event.runId}\n`); + break; + case "start": + process.stdout.write( + `▸ [${event.index + 1}/${event.total}] ${event.id} … `, + ); + break; + case "result": { + process.stdout.write(`${runner.evalGlyph(event.result)}\n`); + for (const line of runner.formatEvalDetail(event.result)) { + console.log(line); + } + break; + } + } + }; + + const summary = await runner.runEvalsInDir({ + rootDir: opts.root, + baseUrl: opts.url, + filter, + strict: opts.strict, + headers: opts.header ? parseHeaders(opts.header) : undefined, + mlflow, + onEvent, + }); + console.log(`\n${runner.formatSummaryLine(summary.results)}`); + + if (summary.mlflow) { + const { report, finish } = summary.mlflow; + console.log( + `MLflow: ${report.written} assessment(s) written` + + (report.skipped ? `, ${report.skipped} skipped` : "") + + (report.failures.length ? `, ${report.failures.length} failed` : ""), + ); + for (const f of report.failures) { + console.error( + ` ✗ trace ${f.traceId}: ${f.status ?? ""} ${f.error ?? ""}`.trim(), + ); + } + if (finish.metricsError) { + console.error(` ⚠ metrics not logged: ${finish.metricsError}`); + } + if (!finish.finished) { + console.error( + ` ✗ run left RUNNING — failed to finish: ${finish.finishError ?? "unknown"}`, + ); + } + } else { + console.log( + "\nMLflow evaluation run skipped — set DATABRICKS_HOST + DATABRICKS_TOKEN" + + " + MLFLOW_EXPERIMENT_ID (or the matching flags) to create one.", + ); + } + + if (!runner.summarize(summary.results).allPassed) { + process.exitCode = 1; + } +} + +export const agentEvalCommand = new Command("eval") + .description( + "Run agent evals (config/agents//evals/*.eval.ts) against a running app", + ) + .argument( + "[filter]", + "Only run evals whose / contains this substring (or an exact agent id)", + ) + .option("--url ", "Base URL of the running app", "http://localhost:3000") + .option("--strict", "Fail on soft-assertion misses too", false) + .option( + "--root ", + "Project root containing config/agents/ (default: cwd)", + ) + .option( + "--header ", + "Extra request header as 'Key: value' (repeatable)", + ) + .option( + "--databricks-host ", + "Databricks host for writing MLflow assessments (default: DATABRICKS_HOST)", + ) + .option( + "--databricks-token ", + "Databricks token for writing MLflow assessments (default: DATABRICKS_TOKEN)", + ) + .option( + "--experiment ", + "MLflow experiment id for the evaluation run (default: MLFLOW_EXPERIMENT_ID)", + ) + .action(runAgentEval); diff --git a/packages/shared/src/cli/commands/agent/index.ts b/packages/shared/src/cli/commands/agent/index.ts new file mode 100644 index 00000000..3f31a92d --- /dev/null +++ b/packages/shared/src/cli/commands/agent/index.ts @@ -0,0 +1,19 @@ +import { Command } from "commander"; +import { agentEvalCommand } from "./eval"; + +/** + * Parent command for agent development operations. + * Subcommands: + * - eval: Run agent evals against a running app + */ +export const agentCommand = new Command("agent") + .description("Agent development commands") + .addCommand(agentEvalCommand) + .addHelpText( + "after", + ` +Examples: + $ appkit agent eval + $ appkit agent eval support --strict + $ appkit agent eval --url https://my-app.databricksapps.com`, + ); diff --git a/packages/shared/src/cli/index.ts b/packages/shared/src/cli/index.ts index aa60157c..fe96dd69 100644 --- a/packages/shared/src/cli/index.ts +++ b/packages/shared/src/cli/index.ts @@ -4,6 +4,7 @@ import { readFileSync } from "node:fs"; import { dirname, join } from "node:path"; import { fileURLToPath } from "node:url"; import { Command } from "commander"; +import { agentCommand } from "./commands/agent/index.js"; import { codemodCommand } from "./commands/codemod/index.js"; import { docsCommand } from "./commands/docs.js"; import { generateTypesCommand } from "./commands/generate-types.js"; @@ -28,5 +29,6 @@ cmd.addCommand(lintCommand); cmd.addCommand(docsCommand); cmd.addCommand(pluginCommand); cmd.addCommand(codemodCommand); +cmd.addCommand(agentCommand); await cmd.parseAsync(); From ab65f504ea00ac88bccc5da6d178446b401276a5 Mon Sep 17 00:00:00 2001 From: MarioCadenas Date: Mon, 29 Jun 2026 17:55:16 +0200 Subject: [PATCH 2/3] feat(appkit): add llm-as-judge and mlflow trace/eval parity Extend the agent eval framework and tighten MLflow output to match the native `mlflow.genai.evaluate` experience: - LLM-as-judge via autoevals (factuality, closedQA, custom), pointed at a Databricks serving endpoint; exposed through `t.judge.*`. - One Feedback assessment per assertion (judges as LLM_JUDGE with score + rationale) plus an overall `appkit_eval`; assessment names sanitized to `[A-Za-z0-9_-]` since the API rejects dots. - Trace-table parity: set Request/Response previews and the `mlflow.traceName` tag (the Trace-name column reads the tag, not the span name). - Eval runs carry `mlflow.source.name`/`type` tags so linked traces show Source and Run name; live chat traces have no run so those stay empty. - Example judge eval under config/agents/query/evals. Signed-off-by: MarioCadenas --- .../config/agents/query/evals/judge.eval.ts | 25 +++ .../api/appkit/Function.buildAssessment.md | 18 ++ .../api/appkit/Function.createHttpDriver.md | 20 ++ docs/docs/api/appkit/Function.defineEval.md | 34 ++++ .../api/appkit/Function.defineEvalConfig.md | 17 ++ .../api/appkit/Function.discoverEvalFiles.md | 19 ++ docs/docs/api/appkit/Function.equals.md | 17 ++ docs/docs/api/appkit/Function.evalGlyph.md | 17 ++ .../api/appkit/Function.formatEvalDetail.md | 17 ++ .../api/appkit/Function.formatEvalHeadline.md | 17 ++ .../api/appkit/Function.formatEvalResults.md | 17 ++ .../api/appkit/Function.formatSummaryLine.md | 17 ++ docs/docs/api/appkit/Function.includes.md | 17 ++ docs/docs/api/appkit/Function.matches.md | 17 ++ .../api/appkit/Function.reportToMlflow.md | 19 ++ docs/docs/api/appkit/Function.runEval.md | 20 ++ .../docs/api/appkit/Function.runEvalsInDir.md | 19 ++ docs/docs/api/appkit/Function.summarize.md | 15 ++ .../api/appkit/Interface.AssertionHandle.md | 53 +++++ .../api/appkit/Interface.AssertionResult.md | 43 ++++ docs/docs/api/appkit/Interface.Assessment.md | 74 +++++++ .../api/appkit/Interface.DiscoveredEval.md | 33 +++ docs/docs/api/appkit/Interface.DriveResult.md | 53 +++++ docs/docs/api/appkit/Interface.EvalConfig.md | 41 ++++ .../api/appkit/Interface.EvalDefinition.md | 63 ++++++ docs/docs/api/appkit/Interface.EvalDriver.md | 22 ++ docs/docs/api/appkit/Interface.EvalResult.md | 75 +++++++ .../api/appkit/Interface.EvalRunSummary.md | 41 ++++ docs/docs/api/appkit/Interface.EvalSummary.md | 43 ++++ .../api/appkit/Interface.HttpDriverOptions.md | 51 +++++ docs/docs/api/appkit/Interface.MatchResult.md | 31 +++ .../appkit/Interface.MlflowReportOptions.md | 21 ++ .../api/appkit/Interface.ReportOutcome.md | 47 +++++ .../api/appkit/Interface.RunEvalOptions.md | 31 +++ .../api/appkit/Interface.RunEvalsOptions.md | 115 +++++++++++ docs/docs/api/appkit/Interface.TestContext.md | 128 ++++++++++++ .../docs/api/appkit/TypeAlias.EvalProgress.md | 25 +++ docs/docs/api/appkit/TypeAlias.Matcher.md | 17 ++ docs/docs/api/appkit/TypeAlias.Severity.md | 7 + docs/docs/api/appkit/index.md | 38 ++++ docs/docs/api/appkit/typedoc-sidebar.ts | 190 ++++++++++++++++++ packages/appkit/package.json | 1 + packages/appkit/src/evals/index.ts | 9 +- packages/appkit/src/evals/judge.ts | 108 ++++++++++ packages/appkit/src/evals/mlflow-report.ts | 88 +++++--- packages/appkit/src/evals/mlflow-run.ts | 17 +- packages/appkit/src/evals/run-eval.ts | 40 ++++ packages/appkit/src/evals/run-evals.ts | 10 + packages/appkit/src/evals/tests/judge.test.ts | 22 ++ .../src/evals/tests/mlflow-report.test.ts | 82 +++++--- packages/appkit/src/evals/types.ts | 21 ++ packages/appkit/src/plugins/agents/agents.ts | 15 +- packages/appkit/src/plugins/agents/mlflow.ts | 27 +++ .../shared/src/cli/commands/agent/eval.ts | 14 ++ pnpm-lock.yaml | 189 ++++++++++++++++- 55 files changed, 2149 insertions(+), 78 deletions(-) create mode 100644 apps/dev-playground/config/agents/query/evals/judge.eval.ts create mode 100644 docs/docs/api/appkit/Function.buildAssessment.md create mode 100644 docs/docs/api/appkit/Function.createHttpDriver.md create mode 100644 docs/docs/api/appkit/Function.defineEval.md create mode 100644 docs/docs/api/appkit/Function.defineEvalConfig.md create mode 100644 docs/docs/api/appkit/Function.discoverEvalFiles.md create mode 100644 docs/docs/api/appkit/Function.equals.md create mode 100644 docs/docs/api/appkit/Function.evalGlyph.md create mode 100644 docs/docs/api/appkit/Function.formatEvalDetail.md create mode 100644 docs/docs/api/appkit/Function.formatEvalHeadline.md create mode 100644 docs/docs/api/appkit/Function.formatEvalResults.md create mode 100644 docs/docs/api/appkit/Function.formatSummaryLine.md create mode 100644 docs/docs/api/appkit/Function.includes.md create mode 100644 docs/docs/api/appkit/Function.matches.md create mode 100644 docs/docs/api/appkit/Function.reportToMlflow.md create mode 100644 docs/docs/api/appkit/Function.runEval.md create mode 100644 docs/docs/api/appkit/Function.runEvalsInDir.md create mode 100644 docs/docs/api/appkit/Function.summarize.md create mode 100644 docs/docs/api/appkit/Interface.AssertionHandle.md create mode 100644 docs/docs/api/appkit/Interface.AssertionResult.md create mode 100644 docs/docs/api/appkit/Interface.Assessment.md create mode 100644 docs/docs/api/appkit/Interface.DiscoveredEval.md create mode 100644 docs/docs/api/appkit/Interface.DriveResult.md create mode 100644 docs/docs/api/appkit/Interface.EvalConfig.md create mode 100644 docs/docs/api/appkit/Interface.EvalDefinition.md create mode 100644 docs/docs/api/appkit/Interface.EvalDriver.md create mode 100644 docs/docs/api/appkit/Interface.EvalResult.md create mode 100644 docs/docs/api/appkit/Interface.EvalRunSummary.md create mode 100644 docs/docs/api/appkit/Interface.EvalSummary.md create mode 100644 docs/docs/api/appkit/Interface.HttpDriverOptions.md create mode 100644 docs/docs/api/appkit/Interface.MatchResult.md create mode 100644 docs/docs/api/appkit/Interface.MlflowReportOptions.md create mode 100644 docs/docs/api/appkit/Interface.ReportOutcome.md create mode 100644 docs/docs/api/appkit/Interface.RunEvalOptions.md create mode 100644 docs/docs/api/appkit/Interface.RunEvalsOptions.md create mode 100644 docs/docs/api/appkit/Interface.TestContext.md create mode 100644 docs/docs/api/appkit/TypeAlias.EvalProgress.md create mode 100644 docs/docs/api/appkit/TypeAlias.Matcher.md create mode 100644 docs/docs/api/appkit/TypeAlias.Severity.md create mode 100644 packages/appkit/src/evals/judge.ts create mode 100644 packages/appkit/src/evals/tests/judge.test.ts diff --git a/apps/dev-playground/config/agents/query/evals/judge.eval.ts b/apps/dev-playground/config/agents/query/evals/judge.eval.ts new file mode 100644 index 00000000..039045bd --- /dev/null +++ b/apps/dev-playground/config/agents/query/evals/judge.eval.ts @@ -0,0 +1,25 @@ +import { defineEval } from "@databricks/appkit/beta"; + +/** + * LLM-as-judge eval: scores the helper agent's weather answer with a judge + * model (via autoevals → a Databricks serving endpoint). + * + * Requires a judge model: + * appkit agent eval query --root apps/dev-playground --url http://localhost:8001 \ + * --judge-model databricks-claude-sonnet-4-5 + * (or set APPKIT_JUDGE_MODEL). Without it, t.judge.* errors with a clear message. + */ +export default defineEval({ + description: "Helper weather answer is relevant (LLM judge)", + agent: "helper", + async test(t) { + await t.send("What's the weather in Brooklyn?"); + t.succeeded(); + // closedQA needs no ground truth — it judges the reply against a question. + ( + await t.judge.closedQA( + "Does the response describe weather conditions for Brooklyn?", + ) + ).atLeast(0.5); + }, +}); diff --git a/docs/docs/api/appkit/Function.buildAssessment.md b/docs/docs/api/appkit/Function.buildAssessment.md new file mode 100644 index 00000000..62c08b2b --- /dev/null +++ b/docs/docs/api/appkit/Function.buildAssessment.md @@ -0,0 +1,18 @@ +# Function: buildAssessment() + +```ts +function buildAssessment(result: EvalResult): Assessment | undefined; +``` + +Build the single pass/fail Feedback assessment for an eval result. Returns +undefined when there's no trace to attach to or the eval was skipped. + +## Parameters + +| Parameter | Type | +| ------ | ------ | +| `result` | [`EvalResult`](Interface.EvalResult.md) | + +## Returns + +[`Assessment`](Interface.Assessment.md) \| `undefined` diff --git a/docs/docs/api/appkit/Function.createHttpDriver.md b/docs/docs/api/appkit/Function.createHttpDriver.md new file mode 100644 index 00000000..b373be10 --- /dev/null +++ b/docs/docs/api/appkit/Function.createHttpDriver.md @@ -0,0 +1,20 @@ +# Function: createHttpDriver() + +```ts +function createHttpDriver(options: HttpDriverOptions): EvalDriver; +``` + +Drives an agent by POSTing to a running app's chat endpoint and parsing the +SSE response. Keeps the thread id across `send`s so multi-turn evals share a +conversation. Agent/stream errors surface as `succeeded: false` rather than +throwing, so `t.succeeded()` can assert on them. + +## Parameters + +| Parameter | Type | +| ------ | ------ | +| `options` | [`HttpDriverOptions`](Interface.HttpDriverOptions.md) | + +## Returns + +[`EvalDriver`](Interface.EvalDriver.md) diff --git a/docs/docs/api/appkit/Function.defineEval.md b/docs/docs/api/appkit/Function.defineEval.md new file mode 100644 index 00000000..a47e8662 --- /dev/null +++ b/docs/docs/api/appkit/Function.defineEval.md @@ -0,0 +1,34 @@ +# Function: defineEval() + +```ts +function defineEval(def: EvalDefinition): EvalDefinition; +``` + +Define an agent eval. Default-export the result from a +`config/agents//evals/*.eval.ts` file. + +## Parameters + +| Parameter | Type | +| ------ | ------ | +| `def` | [`EvalDefinition`](Interface.EvalDefinition.md) | + +## Returns + +[`EvalDefinition`](Interface.EvalDefinition.md) + +## Example + +```ts +import { defineEval, includes } from "@databricks/appkit/beta"; + +export default defineEval({ + description: "Weather agent basic coverage", + async test(t) { + await t.send("What's the weather in Brooklyn?"); + t.succeeded(); + t.calledTool("get_weather"); + t.check(t.reply, includes("Sunny")); + }, +}); +``` diff --git a/docs/docs/api/appkit/Function.defineEvalConfig.md b/docs/docs/api/appkit/Function.defineEvalConfig.md new file mode 100644 index 00000000..2710a598 --- /dev/null +++ b/docs/docs/api/appkit/Function.defineEvalConfig.md @@ -0,0 +1,17 @@ +# Function: defineEvalConfig() + +```ts +function defineEvalConfig(config: EvalConfig): EvalConfig; +``` + +Define per-directory eval config. Default-export from `evals.config.ts`. + +## Parameters + +| Parameter | Type | +| ------ | ------ | +| `config` | [`EvalConfig`](Interface.EvalConfig.md) | + +## Returns + +[`EvalConfig`](Interface.EvalConfig.md) diff --git a/docs/docs/api/appkit/Function.discoverEvalFiles.md b/docs/docs/api/appkit/Function.discoverEvalFiles.md new file mode 100644 index 00000000..746b45e5 --- /dev/null +++ b/docs/docs/api/appkit/Function.discoverEvalFiles.md @@ -0,0 +1,19 @@ +# Function: discoverEvalFiles() + +```ts +function discoverEvalFiles(rootDir: string): DiscoveredEval[]; +``` + +Discover evals under `/config/agents//evals/`. The agent id +is the directory name; the eval id is the file path relative to that evals +dir with `.eval.ts` stripped. Returns a stable, sorted list. + +## Parameters + +| Parameter | Type | +| ------ | ------ | +| `rootDir` | `string` | + +## Returns + +[`DiscoveredEval`](Interface.DiscoveredEval.md)[] diff --git a/docs/docs/api/appkit/Function.equals.md b/docs/docs/api/appkit/Function.equals.md new file mode 100644 index 00000000..6d93b837 --- /dev/null +++ b/docs/docs/api/appkit/Function.equals.md @@ -0,0 +1,17 @@ +# Function: equals() + +```ts +function equals(expected: string): Matcher; +``` + +Passes when the value equals `expected` exactly. + +## Parameters + +| Parameter | Type | +| ------ | ------ | +| `expected` | `string` | + +## Returns + +[`Matcher`](TypeAlias.Matcher.md) diff --git a/docs/docs/api/appkit/Function.evalGlyph.md b/docs/docs/api/appkit/Function.evalGlyph.md new file mode 100644 index 00000000..edef1483 --- /dev/null +++ b/docs/docs/api/appkit/Function.evalGlyph.md @@ -0,0 +1,17 @@ +# Function: evalGlyph() + +```ts +function evalGlyph(result: EvalResult): string; +``` + +Status glyph for a single eval result. + +## Parameters + +| Parameter | Type | +| ------ | ------ | +| `result` | [`EvalResult`](Interface.EvalResult.md) | + +## Returns + +`string` diff --git a/docs/docs/api/appkit/Function.formatEvalDetail.md b/docs/docs/api/appkit/Function.formatEvalDetail.md new file mode 100644 index 00000000..f07f83a1 --- /dev/null +++ b/docs/docs/api/appkit/Function.formatEvalDetail.md @@ -0,0 +1,17 @@ +# Function: formatEvalDetail() + +```ts +function formatEvalDetail(result: EvalResult): string[]; +``` + +Indented detail lines for a failing eval (error + failing assertions). + +## Parameters + +| Parameter | Type | +| ------ | ------ | +| `result` | [`EvalResult`](Interface.EvalResult.md) | + +## Returns + +`string`[] diff --git a/docs/docs/api/appkit/Function.formatEvalHeadline.md b/docs/docs/api/appkit/Function.formatEvalHeadline.md new file mode 100644 index 00000000..f6d22ee8 --- /dev/null +++ b/docs/docs/api/appkit/Function.formatEvalHeadline.md @@ -0,0 +1,17 @@ +# Function: formatEvalHeadline() + +```ts +function formatEvalHeadline(result: EvalResult): string; +``` + +The one-line header for a single eval result (no failure detail). + +## Parameters + +| Parameter | Type | +| ------ | ------ | +| `result` | [`EvalResult`](Interface.EvalResult.md) | + +## Returns + +`string` diff --git a/docs/docs/api/appkit/Function.formatEvalResults.md b/docs/docs/api/appkit/Function.formatEvalResults.md new file mode 100644 index 00000000..2607a52b --- /dev/null +++ b/docs/docs/api/appkit/Function.formatEvalResults.md @@ -0,0 +1,17 @@ +# Function: formatEvalResults() + +```ts +function formatEvalResults(results: EvalResult[]): string; +``` + +Render all results as a human-readable console report (non-streaming). + +## Parameters + +| Parameter | Type | +| ------ | ------ | +| `results` | [`EvalResult`](Interface.EvalResult.md)[] | + +## Returns + +`string` diff --git a/docs/docs/api/appkit/Function.formatSummaryLine.md b/docs/docs/api/appkit/Function.formatSummaryLine.md new file mode 100644 index 00000000..355f9295 --- /dev/null +++ b/docs/docs/api/appkit/Function.formatSummaryLine.md @@ -0,0 +1,17 @@ +# Function: formatSummaryLine() + +```ts +function formatSummaryLine(results: EvalResult[]): string; +``` + +The final PASS/FAIL summary line. + +## Parameters + +| Parameter | Type | +| ------ | ------ | +| `results` | [`EvalResult`](Interface.EvalResult.md)[] | + +## Returns + +`string` diff --git a/docs/docs/api/appkit/Function.includes.md b/docs/docs/api/appkit/Function.includes.md new file mode 100644 index 00000000..3ff1ab7d --- /dev/null +++ b/docs/docs/api/appkit/Function.includes.md @@ -0,0 +1,17 @@ +# Function: includes() + +```ts +function includes(substring: string): Matcher; +``` + +Passes when the value contains `substring`. + +## Parameters + +| Parameter | Type | +| ------ | ------ | +| `substring` | `string` | + +## Returns + +[`Matcher`](TypeAlias.Matcher.md) diff --git a/docs/docs/api/appkit/Function.matches.md b/docs/docs/api/appkit/Function.matches.md new file mode 100644 index 00000000..6839fa5c --- /dev/null +++ b/docs/docs/api/appkit/Function.matches.md @@ -0,0 +1,17 @@ +# Function: matches() + +```ts +function matches(pattern: RegExp): Matcher; +``` + +Passes when the value matches `pattern`. + +## Parameters + +| Parameter | Type | +| ------ | ------ | +| `pattern` | `RegExp` | + +## Returns + +[`Matcher`](TypeAlias.Matcher.md) diff --git a/docs/docs/api/appkit/Function.reportToMlflow.md b/docs/docs/api/appkit/Function.reportToMlflow.md new file mode 100644 index 00000000..5d3c3b7b --- /dev/null +++ b/docs/docs/api/appkit/Function.reportToMlflow.md @@ -0,0 +1,19 @@ +# Function: reportToMlflow() + +```ts +function reportToMlflow(results: EvalResult[], options: MlflowReportOptions): Promise; +``` + +Write one pass/fail assessment per eval result to the Databricks MLflow REST +API. Never throws — failures are collected so the run still reports. + +## Parameters + +| Parameter | Type | +| ------ | ------ | +| `results` | [`EvalResult`](Interface.EvalResult.md)[] | +| `options` | [`MlflowReportOptions`](Interface.MlflowReportOptions.md) | + +## Returns + +`Promise`\<[`ReportOutcome`](Interface.ReportOutcome.md)\> diff --git a/docs/docs/api/appkit/Function.runEval.md b/docs/docs/api/appkit/Function.runEval.md new file mode 100644 index 00000000..9f0fd1a0 --- /dev/null +++ b/docs/docs/api/appkit/Function.runEval.md @@ -0,0 +1,20 @@ +# Function: runEval() + +```ts +function runEval(def: EvalDefinition, options: RunEvalOptions): Promise; +``` + +Run a single eval against a driver. Never throws for assertion or agent +failures — those become a non-passing [EvalResult](Interface.EvalResult.md). Only a malformed +eval definition surfaces as `result.error`. + +## Parameters + +| Parameter | Type | +| ------ | ------ | +| `def` | [`EvalDefinition`](Interface.EvalDefinition.md) | +| `options` | [`RunEvalOptions`](Interface.RunEvalOptions.md) | + +## Returns + +`Promise`\<[`EvalResult`](Interface.EvalResult.md)\> diff --git a/docs/docs/api/appkit/Function.runEvalsInDir.md b/docs/docs/api/appkit/Function.runEvalsInDir.md new file mode 100644 index 00000000..903559b1 --- /dev/null +++ b/docs/docs/api/appkit/Function.runEvalsInDir.md @@ -0,0 +1,19 @@ +# Function: runEvalsInDir() + +```ts +function runEvalsInDir(options: RunEvalsOptions): Promise; +``` + +Discover, load, and run every eval under each agent's `evals/` dir, driving +the agents on a running app. Never throws for an individual eval — load/run +failures become non-passing [EvalResult](Interface.EvalResult.md)s. + +## Parameters + +| Parameter | Type | +| ------ | ------ | +| `options` | [`RunEvalsOptions`](Interface.RunEvalsOptions.md) | + +## Returns + +`Promise`\<[`EvalRunSummary`](Interface.EvalRunSummary.md)\> diff --git a/docs/docs/api/appkit/Function.summarize.md b/docs/docs/api/appkit/Function.summarize.md new file mode 100644 index 00000000..959817e6 --- /dev/null +++ b/docs/docs/api/appkit/Function.summarize.md @@ -0,0 +1,15 @@ +# Function: summarize() + +```ts +function summarize(results: EvalResult[]): EvalSummary; +``` + +## Parameters + +| Parameter | Type | +| ------ | ------ | +| `results` | [`EvalResult`](Interface.EvalResult.md)[] | + +## Returns + +[`EvalSummary`](Interface.EvalSummary.md) diff --git a/docs/docs/api/appkit/Interface.AssertionHandle.md b/docs/docs/api/appkit/Interface.AssertionHandle.md new file mode 100644 index 00000000..02e3c746 --- /dev/null +++ b/docs/docs/api/appkit/Interface.AssertionHandle.md @@ -0,0 +1,53 @@ +# Interface: AssertionHandle + +Chainable handle returned by every assertion to control its severity. +Mirrors eve: assertions are gates by default; `.soft()` demotes to a tracked +metric; `.atLeast(n)` is a soft, score-thresholded assertion. + +## Methods + +### atLeast() + +```ts +atLeast(threshold: number): AssertionHandle; +``` + +Soft assertion that passes only when the score is at least `threshold`. + +#### Parameters + +| Parameter | Type | +| ------ | ------ | +| `threshold` | `number` | + +#### Returns + +`AssertionHandle` + +*** + +### gate() + +```ts +gate(): AssertionHandle; +``` + +Promote to a hard gate — failure fails the eval (non-zero exit). + +#### Returns + +`AssertionHandle` + +*** + +### soft() + +```ts +soft(): AssertionHandle; +``` + +Demote to a tracked metric — doesn't fail unless running with `strict`. + +#### Returns + +`AssertionHandle` diff --git a/docs/docs/api/appkit/Interface.AssertionResult.md b/docs/docs/api/appkit/Interface.AssertionResult.md new file mode 100644 index 00000000..df3e986b --- /dev/null +++ b/docs/docs/api/appkit/Interface.AssertionResult.md @@ -0,0 +1,43 @@ +# Interface: AssertionResult + +A single recorded assertion outcome. + +## Properties + +### detail? + +```ts +optional detail: string; +``` + +*** + +### label + +```ts +label: string; +``` + +*** + +### pass + +```ts +pass: boolean; +``` + +*** + +### score? + +```ts +optional score: number; +``` + +*** + +### severity + +```ts +severity: Severity; +``` diff --git a/docs/docs/api/appkit/Interface.Assessment.md b/docs/docs/api/appkit/Interface.Assessment.md new file mode 100644 index 00000000..d3537cf0 --- /dev/null +++ b/docs/docs/api/appkit/Interface.Assessment.md @@ -0,0 +1,74 @@ +# Interface: Assessment + +A Feedback assessment in the MLflow REST proto-JSON shape. + +## Properties + +### assessment\_name + +```ts +assessment_name: string; +``` + +*** + +### feedback + +```ts +feedback: { + value: unknown; +}; +``` + +#### value + +```ts +value: unknown; +``` + +*** + +### metadata? + +```ts +optional metadata: Record; +``` + +*** + +### rationale? + +```ts +optional rationale: string; +``` + +*** + +### source + +```ts +source: { + source_id: string; + source_type: "CODE" | "HUMAN" | "LLM_JUDGE"; +}; +``` + +#### source\_id + +```ts +source_id: string; +``` + +#### source\_type + +```ts +source_type: "CODE" | "HUMAN" | "LLM_JUDGE"; +``` + +*** + +### trace\_id + +```ts +trace_id: string; +``` diff --git a/docs/docs/api/appkit/Interface.DiscoveredEval.md b/docs/docs/api/appkit/Interface.DiscoveredEval.md new file mode 100644 index 00000000..376aa4c4 --- /dev/null +++ b/docs/docs/api/appkit/Interface.DiscoveredEval.md @@ -0,0 +1,33 @@ +# Interface: DiscoveredEval + +An eval file found under `config/agents//evals/`. + +## Properties + +### agent + +```ts +agent: string; +``` + +The agent id (the `config/agents/` directory name). + +*** + +### file + +```ts +file: string; +``` + +Absolute path to the `*.eval.ts` file. + +*** + +### id + +```ts +id: string; +``` + +Id relative to the agent's evals dir, without `.eval.ts` (e.g. `weather/basic`). diff --git a/docs/docs/api/appkit/Interface.DriveResult.md b/docs/docs/api/appkit/Interface.DriveResult.md new file mode 100644 index 00000000..15625c1a --- /dev/null +++ b/docs/docs/api/appkit/Interface.DriveResult.md @@ -0,0 +1,53 @@ +# Interface: DriveResult + +What a driver returns for a single `t.send`. + +## Properties + +### reply + +```ts +reply: string; +``` + +The final assistant message text. + +*** + +### sessionId? + +```ts +optional sessionId: string; +``` + +Thread/session id, when the driver exposes one. + +*** + +### succeeded + +```ts +succeeded: boolean; +``` + +Whether the turn completed without an agent/stream error. + +*** + +### toolCalls + +```ts +toolCalls: string[]; +``` + +Names of tools the agent called during the turn. + +*** + +### traceId? + +```ts +optional traceId: string; +``` + +MLflow trace id for the turn, when tracing is enabled on the app. diff --git a/docs/docs/api/appkit/Interface.EvalConfig.md b/docs/docs/api/appkit/Interface.EvalConfig.md new file mode 100644 index 00000000..eb8227fc --- /dev/null +++ b/docs/docs/api/appkit/Interface.EvalConfig.md @@ -0,0 +1,41 @@ +# Interface: EvalConfig + +Per-directory config from `evals.config.ts`. + +## Properties + +### judge? + +```ts +optional judge: { + model?: string; +}; +``` + +LLM judge config. Defaults to the agent's own serving endpoint. + +#### model? + +```ts +optional model: string; +``` + +*** + +### maxConcurrency? + +```ts +optional maxConcurrency: number; +``` + +Max evals to run concurrently. + +*** + +### timeoutMs? + +```ts +optional timeoutMs: number; +``` + +Default per-eval timeout. diff --git a/docs/docs/api/appkit/Interface.EvalDefinition.md b/docs/docs/api/appkit/Interface.EvalDefinition.md new file mode 100644 index 00000000..e579ce34 --- /dev/null +++ b/docs/docs/api/appkit/Interface.EvalDefinition.md @@ -0,0 +1,63 @@ +# Interface: EvalDefinition + +A single eval, default-exported from a `*.eval.ts` file. + +## Properties + +### agent? + +```ts +optional agent: string; +``` + +Target agent id. Defaults to the eval's parent `config/agents/` dir. + +*** + +### description? + +```ts +optional description: string; +``` + +Short human description, shown in reports. + +*** + +### tags? + +```ts +optional tags: string[]; +``` + +Free-form tags for filtering. + +*** + +### timeoutMs? + +```ts +optional timeoutMs: number; +``` + +Per-eval timeout. + +## Methods + +### test() + +```ts +test(t: TestContext): void | Promise; +``` + +The eval body: drive the agent and assert on its behavior. + +#### Parameters + +| Parameter | Type | +| ------ | ------ | +| `t` | [`TestContext`](Interface.TestContext.md) | + +#### Returns + +`void` \| `Promise`\<`void`\> diff --git a/docs/docs/api/appkit/Interface.EvalDriver.md b/docs/docs/api/appkit/Interface.EvalDriver.md new file mode 100644 index 00000000..69d81a05 --- /dev/null +++ b/docs/docs/api/appkit/Interface.EvalDriver.md @@ -0,0 +1,22 @@ +# Interface: EvalDriver + +Abstraction over how the agent is driven. The HTTP driver posts to a running +app's agents endpoint; future drivers (in-process) implement the same shape. + +## Methods + +### send() + +```ts +send(message: string): Promise; +``` + +#### Parameters + +| Parameter | Type | +| ------ | ------ | +| `message` | `string` | + +#### Returns + +`Promise`\<[`DriveResult`](Interface.DriveResult.md)\> diff --git a/docs/docs/api/appkit/Interface.EvalResult.md b/docs/docs/api/appkit/Interface.EvalResult.md new file mode 100644 index 00000000..94a6c9af --- /dev/null +++ b/docs/docs/api/appkit/Interface.EvalResult.md @@ -0,0 +1,75 @@ +# Interface: EvalResult + +The outcome of running one eval. + +## Properties + +### assertions + +```ts +assertions: AssertionResult[]; +``` + +*** + +### description? + +```ts +optional description: string; +``` + +*** + +### error? + +```ts +optional error: string; +``` + +Set when the eval threw before completing. + +*** + +### id + +```ts +id: string; +``` + +*** + +### passed + +```ts +passed: boolean; +``` + +True when all gates passed (and, under strict, all soft assertions too). + +*** + +### skipped? + +```ts +optional skipped: { + reason?: string; +}; +``` + +Set when the eval called `t.skip`. + +#### reason? + +```ts +optional reason: string; +``` + +*** + +### traceId? + +```ts +optional traceId: string; +``` + +MLflow trace id of the eval's last turn, for attaching assessments. diff --git a/docs/docs/api/appkit/Interface.EvalRunSummary.md b/docs/docs/api/appkit/Interface.EvalRunSummary.md new file mode 100644 index 00000000..ec9176c0 --- /dev/null +++ b/docs/docs/api/appkit/Interface.EvalRunSummary.md @@ -0,0 +1,41 @@ +# Interface: EvalRunSummary + +## Properties + +### mlflow? + +```ts +optional mlflow: { + finish: FinishOutcome; + report: ReportOutcome; + runId: string; +}; +``` + +Present when an MLflow evaluation run was created. + +#### finish + +```ts +finish: FinishOutcome; +``` + +#### report + +```ts +report: ReportOutcome; +``` + +#### runId + +```ts +runId: string; +``` + +*** + +### results + +```ts +results: EvalResult[]; +``` diff --git a/docs/docs/api/appkit/Interface.EvalSummary.md b/docs/docs/api/appkit/Interface.EvalSummary.md new file mode 100644 index 00000000..3c8fc1e6 --- /dev/null +++ b/docs/docs/api/appkit/Interface.EvalSummary.md @@ -0,0 +1,43 @@ +# Interface: EvalSummary + +## Properties + +### allPassed + +```ts +allPassed: boolean; +``` + +True when no eval failed (skips don't count as failures). + +*** + +### failed + +```ts +failed: number; +``` + +*** + +### passed + +```ts +passed: number; +``` + +*** + +### skipped + +```ts +skipped: number; +``` + +*** + +### total + +```ts +total: number; +``` diff --git a/docs/docs/api/appkit/Interface.HttpDriverOptions.md b/docs/docs/api/appkit/Interface.HttpDriverOptions.md new file mode 100644 index 00000000..8bb1f25e --- /dev/null +++ b/docs/docs/api/appkit/Interface.HttpDriverOptions.md @@ -0,0 +1,51 @@ +# Interface: HttpDriverOptions + +## Properties + +### agent? + +```ts +optional agent: string; +``` + +Agent alias to target. Omit to use the app's default agent. + +*** + +### baseUrl + +```ts +baseUrl: string; +``` + +Base URL of the running app, e.g. `http://localhost:3000`. + +*** + +### headers? + +```ts +optional headers: Record; +``` + +Extra request headers (e.g. auth for a deployed app). + +*** + +### mlflowRunId? + +```ts +optional mlflowRunId: string; +``` + +MLflow run id to link each turn's trace to (for evaluation runs). + +*** + +### path? + +```ts +optional path: string; +``` + +Chat endpoint path. Defaults to `/api/agents/chat`. diff --git a/docs/docs/api/appkit/Interface.MatchResult.md b/docs/docs/api/appkit/Interface.MatchResult.md new file mode 100644 index 00000000..a8744ef0 --- /dev/null +++ b/docs/docs/api/appkit/Interface.MatchResult.md @@ -0,0 +1,31 @@ +# Interface: MatchResult + +Result of a deterministic matcher run against a value. + +## Properties + +### detail? + +```ts +optional detail: string; +``` + +Human-readable explanation, shown on failure. + +*** + +### pass + +```ts +pass: boolean; +``` + +*** + +### score? + +```ts +optional score: number; +``` + +Optional 0..1 score for scored matchers (similarity, judges). diff --git a/docs/docs/api/appkit/Interface.MlflowReportOptions.md b/docs/docs/api/appkit/Interface.MlflowReportOptions.md new file mode 100644 index 00000000..22733c8b --- /dev/null +++ b/docs/docs/api/appkit/Interface.MlflowReportOptions.md @@ -0,0 +1,21 @@ +# Interface: MlflowReportOptions + +## Properties + +### host + +```ts +host: string; +``` + +Databricks workspace host (scheme optional — normalized). + +*** + +### token + +```ts +token: string; +``` + +Bearer token for the MLflow REST API. diff --git a/docs/docs/api/appkit/Interface.ReportOutcome.md b/docs/docs/api/appkit/Interface.ReportOutcome.md new file mode 100644 index 00000000..f42136a9 --- /dev/null +++ b/docs/docs/api/appkit/Interface.ReportOutcome.md @@ -0,0 +1,47 @@ +# Interface: ReportOutcome + +## Properties + +### failures + +```ts +failures: { + error?: string; + status?: number; + traceId: string; +}[]; +``` + +#### error? + +```ts +optional error: string; +``` + +#### status? + +```ts +optional status: number; +``` + +#### traceId + +```ts +traceId: string; +``` + +*** + +### skipped + +```ts +skipped: number; +``` + +*** + +### written + +```ts +written: number; +``` diff --git a/docs/docs/api/appkit/Interface.RunEvalOptions.md b/docs/docs/api/appkit/Interface.RunEvalOptions.md new file mode 100644 index 00000000..16a6b819 --- /dev/null +++ b/docs/docs/api/appkit/Interface.RunEvalOptions.md @@ -0,0 +1,31 @@ +# Interface: RunEvalOptions + +## Properties + +### driver + +```ts +driver: EvalDriver; +``` + +Drives the agent and returns reply/tool-calls/success per `send`. + +*** + +### id + +```ts +id: string; +``` + +Stable id for the eval (e.g. its file path relative to the evals dir). + +*** + +### strict? + +```ts +optional strict: boolean; +``` + +When true, soft assertion failures also fail the eval. diff --git a/docs/docs/api/appkit/Interface.RunEvalsOptions.md b/docs/docs/api/appkit/Interface.RunEvalsOptions.md new file mode 100644 index 00000000..5fe5bbd9 --- /dev/null +++ b/docs/docs/api/appkit/Interface.RunEvalsOptions.md @@ -0,0 +1,115 @@ +# Interface: RunEvalsOptions + +## Properties + +### baseUrl + +```ts +baseUrl: string; +``` + +Base URL of the running app to drive, e.g. `http://localhost:3000`. + +*** + +### filter? + +```ts +optional filter: string; +``` + +Substring filter on `/` (or an exact agent id). + +*** + +### headers? + +```ts +optional headers: Record; +``` + +Extra request headers for the driver (e.g. auth for a deployed app). + +*** + +### mlflow? + +```ts +optional mlflow: { + experimentId: string; + host: string; + token: string; +}; +``` + +When set, create a native MLflow "Evaluation run": each eval's trace is +linked to the run, pass/fail is written as feedback, and aggregate metrics +are logged. Requires Databricks creds + the target experiment. + +#### experimentId + +```ts +experimentId: string; +``` + +#### host + +```ts +host: string; +``` + +#### token + +```ts +token: string; +``` + +*** + +### now? + +```ts +optional now: number; +``` + +Wall-clock timestamp (ms) for run create/finish — pass `Date.now()`. + +*** + +### onEvent()? + +```ts +optional onEvent: (event: EvalProgress) => void; +``` + +Progress callback, invoked as evals are discovered, started, and finished. + +#### Parameters + +| Parameter | Type | +| ------ | ------ | +| `event` | [`EvalProgress`](TypeAlias.EvalProgress.md) | + +#### Returns + +`void` + +*** + +### rootDir? + +```ts +optional rootDir: string; +``` + +Project root containing `config/agents/`. Defaults to `process.cwd()`. + +*** + +### strict? + +```ts +optional strict: boolean; +``` + +Soft assertion failures also fail the eval. diff --git a/docs/docs/api/appkit/Interface.TestContext.md b/docs/docs/api/appkit/Interface.TestContext.md new file mode 100644 index 00000000..5f249952 --- /dev/null +++ b/docs/docs/api/appkit/Interface.TestContext.md @@ -0,0 +1,128 @@ +# Interface: TestContext + +The `t` context passed to an eval's `test` function. + +## Properties + +### reply + +```ts +readonly reply: string; +``` + +The last assistant reply. + +*** + +### sessionId + +```ts +readonly sessionId: string | undefined; +``` + +The current session/thread id, if any. + +*** + +### toolCalls + +```ts +readonly toolCalls: string[]; +``` + +Tools called during the last turn. + +## Methods + +### calledTool() + +```ts +calledTool(name: string): AssertionHandle; +``` + +Assert a tool was called during the run (gate by default). + +#### Parameters + +| Parameter | Type | +| ------ | ------ | +| `name` | `string` | + +#### Returns + +[`AssertionHandle`](Interface.AssertionHandle.md) + +*** + +### check() + +```ts +check(value: string, matcher: Matcher): AssertionHandle; +``` + +Assert a value against a matcher, e.g. `t.check(t.reply, includes("Sunny"))`. + +#### Parameters + +| Parameter | Type | +| ------ | ------ | +| `value` | `string` | +| `matcher` | [`Matcher`](TypeAlias.Matcher.md) | + +#### Returns + +[`AssertionHandle`](Interface.AssertionHandle.md) + +*** + +### send() + +```ts +send(message: string): Promise; +``` + +Send a user message to the agent and capture its response. + +#### Parameters + +| Parameter | Type | +| ------ | ------ | +| `message` | `string` | + +#### Returns + +`Promise`\<`void`\> + +*** + +### skip() + +```ts +skip(reason?: string): never; +``` + +Skip this eval with an optional reason. + +#### Parameters + +| Parameter | Type | +| ------ | ------ | +| `reason?` | `string` | + +#### Returns + +`never` + +*** + +### succeeded() + +```ts +succeeded(): AssertionHandle; +``` + +Assert the last turn completed successfully (gate by default). + +#### Returns + +[`AssertionHandle`](Interface.AssertionHandle.md) diff --git a/docs/docs/api/appkit/TypeAlias.EvalProgress.md b/docs/docs/api/appkit/TypeAlias.EvalProgress.md new file mode 100644 index 00000000..d1814ba6 --- /dev/null +++ b/docs/docs/api/appkit/TypeAlias.EvalProgress.md @@ -0,0 +1,25 @@ +# Type Alias: EvalProgress + +```ts +type EvalProgress = + | { + total: number; + type: "discovered"; +} + | { + runId: string; + type: "run-created"; +} + | { + id: string; + index: number; + total: number; + type: "start"; +} + | { + index: number; + result: EvalResult; + total: number; + type: "result"; +}; +``` diff --git a/docs/docs/api/appkit/TypeAlias.Matcher.md b/docs/docs/api/appkit/TypeAlias.Matcher.md new file mode 100644 index 00000000..f22e4878 --- /dev/null +++ b/docs/docs/api/appkit/TypeAlias.Matcher.md @@ -0,0 +1,17 @@ +# Type Alias: Matcher() + +```ts +type Matcher = (value: string) => MatchResult; +``` + +A deterministic matcher: inspects a string value and returns a result. + +## Parameters + +| Parameter | Type | +| ------ | ------ | +| `value` | `string` | + +## Returns + +[`MatchResult`](Interface.MatchResult.md) diff --git a/docs/docs/api/appkit/TypeAlias.Severity.md b/docs/docs/api/appkit/TypeAlias.Severity.md new file mode 100644 index 00000000..575bad86 --- /dev/null +++ b/docs/docs/api/appkit/TypeAlias.Severity.md @@ -0,0 +1,7 @@ +# Type Alias: Severity + +```ts +type Severity = "gate" | "soft"; +``` + +Whether an assertion fails the eval (`gate`) or is tracked only (`soft`). diff --git a/docs/docs/api/appkit/index.md b/docs/docs/api/appkit/index.md index 4d995051..a2102558 100644 --- a/docs/docs/api/appkit/index.md +++ b/docs/docs/api/appkit/index.md @@ -39,15 +39,27 @@ surface with `@databricks/appkit/beta`. Not meant for application imports. | [AgentRunContext](Interface.AgentRunContext.md) | - | | [AgentsPluginConfig](Interface.AgentsPluginConfig.md) | Base configuration interface for AppKit plugins | | [AgentToolDefinition](Interface.AgentToolDefinition.md) | - | +| [AssertionHandle](Interface.AssertionHandle.md) | Chainable handle returned by every assertion to control its severity. Mirrors eve: assertions are gates by default; `.soft()` demotes to a tracked metric; `.atLeast(n)` is a soft, score-thresholded assertion. | +| [AssertionResult](Interface.AssertionResult.md) | A single recorded assertion outcome. | +| [Assessment](Interface.Assessment.md) | A Feedback assessment in the MLflow REST proto-JSON shape. | | [AutoInheritToolsConfig](Interface.AutoInheritToolsConfig.md) | Auto-inherit configuration. When enabled for a given agent origin, agents with no explicit `tools:` declaration receive every registered ToolProvider plugin tool whose author marked `autoInheritable: true`. Tools without that flag — destructive, state-mutating, or privilege-sensitive — never spread automatically and must be wired via `tools:` (object or function form in code, `plugin:NAME` entries in markdown frontmatter). | | [BasePluginConfig](Interface.BasePluginConfig.md) | Base configuration interface for AppKit plugins | | [CacheConfig](Interface.CacheConfig.md) | Configuration for the CacheInterceptor. Controls TTL, size limits, storage backend, and probabilistic cleanup. | | [DatabaseCredential](Interface.DatabaseCredential.md) | Database credentials with OAuth token for Postgres connection | +| [DiscoveredEval](Interface.DiscoveredEval.md) | An eval file found under `config/agents//evals/`. | +| [DriveResult](Interface.DriveResult.md) | What a driver returns for a single `t.send`. | | [EndpointConfig](Interface.EndpointConfig.md) | - | +| [EvalConfig](Interface.EvalConfig.md) | Per-directory config from `evals.config.ts`. | +| [EvalDefinition](Interface.EvalDefinition.md) | A single eval, default-exported from a `*.eval.ts` file. | +| [EvalDriver](Interface.EvalDriver.md) | Abstraction over how the agent is driven. The HTTP driver posts to a running app's agents endpoint; future drivers (in-process) implement the same shape. | +| [EvalResult](Interface.EvalResult.md) | The outcome of running one eval. | +| [EvalRunSummary](Interface.EvalRunSummary.md) | - | +| [EvalSummary](Interface.EvalSummary.md) | - | | [FilePolicyUser](Interface.FilePolicyUser.md) | Minimal user identity passed to the policy function. | | [FileResource](Interface.FileResource.md) | Describes the file or directory being acted upon. | | [FunctionTool](Interface.FunctionTool.md) | - | | [GenerateDatabaseCredentialRequest](Interface.GenerateDatabaseCredentialRequest.md) | Request parameters for generating database OAuth credentials | +| [HttpDriverOptions](Interface.HttpDriverOptions.md) | - | | [IJobsConfig](Interface.IJobsConfig.md) | Configuration for the Jobs plugin. | | [ITelemetry](Interface.ITelemetry.md) | Plugin-facing interface for OpenTelemetry instrumentation. Provides a thin abstraction over OpenTelemetry APIs for plugins. | | [JobAPI](Interface.JobAPI.md) | User-facing API for a single configured job. | @@ -56,22 +68,28 @@ surface with `@databricks/appkit/beta`. Not meant for application imports. | [LakebasePool](Interface.LakebasePool.md) | Subset of `pg.Pool` exposed by the Lakebase plugin. | | [LakebasePoolConfig](Interface.LakebasePoolConfig.md) | Configuration for creating a Lakebase connection pool | | [LakebasePoolManager](Interface.LakebasePoolManager.md) | Manages multiple Lakebase connection pools keyed by an identifier (e.g. userId). | +| [MatchResult](Interface.MatchResult.md) | Result of a deterministic matcher run against a value. | | [McpConnectAllResult](Interface.McpConnectAllResult.md) | Per-endpoint outcome of [AppKitMcpClient.connectAll](Class.AppKitMcpClient.md#connectall). Callers (the agents plugin in particular) use the split to warn at startup when some MCP servers are unreachable without aborting boot for the rest. | | [Message](Interface.Message.md) | - | +| [MlflowReportOptions](Interface.MlflowReportOptions.md) | - | | [PluginManifest](Interface.PluginManifest.md) | Plugin manifest that declares metadata and resource requirements. Attached to plugin classes as a static property. Extends the shared PluginManifest with strict resource types. | | [PluginToolkitProvider](Interface.PluginToolkitProvider.md) | Minimum shape every entry in the [Plugins](TypeAlias.Plugins.md) map must expose. Core plugins (analytics, files, genie, lakebase) implement this directly via their `.toolkit()` method. The agents plugin and standalone `runAgent` synthesize this shape for any registered plugin that doesn't implement `.toolkit()` directly (falling back to `getAgentTools()` walking). | | [PromptContext](Interface.PromptContext.md) | Context passed to `baseSystemPrompt` callbacks. | | [RegisteredAgent](Interface.RegisteredAgent.md) | - | +| [ReportOutcome](Interface.ReportOutcome.md) | - | | [RequestedClaims](Interface.RequestedClaims.md) | Optional claims for fine-grained Unity Catalog table permissions When specified, the returned token will be scoped to only the requested tables | | [RequestedResource](Interface.RequestedResource.md) | Resource to request permissions for in Unity Catalog | | [ResourceEntry](Interface.ResourceEntry.md) | Internal representation of a resource in the registry. Extends ResourceRequirement with resolution state and plugin ownership. | | [ResourceRequirement](Interface.ResourceRequirement.md) | Declares a resource requirement for a plugin. Can be defined statically in a manifest or dynamically via getResourceRequirements(). | | [RunAgentInput](Interface.RunAgentInput.md) | - | | [RunAgentResult](Interface.RunAgentResult.md) | - | +| [RunEvalOptions](Interface.RunEvalOptions.md) | - | +| [RunEvalsOptions](Interface.RunEvalsOptions.md) | - | | [ServingEndpointEntry](Interface.ServingEndpointEntry.md) | Shape of a single registry entry. | | [ServingEndpointRegistry](Interface.ServingEndpointRegistry.md) | Registry interface for serving endpoint type generation. Empty by default — augmented by the Vite type generator's `.d.ts` output via module augmentation. When populated, provides autocomplete for alias names and typed request/response/chunk per endpoint. | | [StreamExecutionSettings](Interface.StreamExecutionSettings.md) | Execution settings for streaming endpoints. Extends PluginExecutionSettings with SSE stream configuration. | | [TelemetryConfig](Interface.TelemetryConfig.md) | OpenTelemetry configuration for AppKit applications | +| [TestContext](Interface.TestContext.md) | The `t` context passed to an eval's `test` function. | | [Thread](Interface.Thread.md) | - | | [ThreadStore](Interface.ThreadStore.md) | - | | [ToolAnnotations](Interface.ToolAnnotations.md) | - | @@ -92,6 +110,7 @@ surface with `@databricks/appkit/beta`. Not meant for application imports. | [AgentToolsFn](TypeAlias.AgentToolsFn.md) | Function form of `AgentDefinition.tools`. Receives the typed [Plugins](TypeAlias.Plugins.md) map and returns a tool record. Invoked exactly once at setup (or once per `runAgent` call in standalone mode); the result is cached as the agent's resolved tool record. | | [BaseSystemPromptOption](TypeAlias.BaseSystemPromptOption.md) | - | | [ConfigSchema](TypeAlias.ConfigSchema.md) | Configuration schema definition for plugin config. Re-exported from the standard JSON Schema Draft 7 types. | +| [EvalProgress](TypeAlias.EvalProgress.md) | - | | [ExecutionResult](TypeAlias.ExecutionResult.md) | Discriminated union for plugin execution results. | | [FileAction](TypeAlias.FileAction.md) | Every action the files plugin can perform. | | [FilePolicy](TypeAlias.FilePolicy.md) | A policy function that decides whether `user` may perform `action` on `resource`. Return `true` to allow, `false` to deny. | @@ -99,12 +118,14 @@ surface with `@databricks/appkit/beta`. Not meant for application imports. | [IAppRouter](TypeAlias.IAppRouter.md) | Express router type for plugin route registration | | [JobHandle](TypeAlias.JobHandle.md) | Job handle returned by `appkit.jobs("etl")`. Supports OBO access via `.asUser(req)`. | | [JobsExport](TypeAlias.JobsExport.md) | Public API shape of the jobs plugin. Callable to select a job by key. | +| [Matcher](TypeAlias.Matcher.md) | A deterministic matcher: inspects a string value and returns a result. | | [PluginData](TypeAlias.PluginData.md) | Tuple of plugin class, config, and name. Created by `toPlugin()` and passed to `createApp()`. | | [Plugins](TypeAlias.Plugins.md) | Plugin map passed to the function form of [AgentDefinition.tools](Interface.AgentDefinition.md#tools). Each entry exposes a `.toolkit(opts?)` method that returns a record of [ToolkitEntry](Interface.ToolkitEntry.md) markers ready to be spread into a tool record. | | [ResolvedToolEntry](TypeAlias.ResolvedToolEntry.md) | Internal tool-index entry after a tool record has been resolved to a dispatchable form. | | [ResourceFieldEntry](TypeAlias.ResourceFieldEntry.md) | - | | [ResourcePermission](TypeAlias.ResourcePermission.md) | Union of all possible permission levels across all resource types. | | [ServingFactory](TypeAlias.ServingFactory.md) | Factory function returned by `AppKit.serving`. | +| [Severity](TypeAlias.Severity.md) | Whether an assertion fails the eval (`gate`) or is tracked only (`soft`). | | [ToolRegistry](TypeAlias.ToolRegistry.md) | - | | [ToPlugin](TypeAlias.ToPlugin.md) | Factory function type returned by `toPlugin()`. Accepts optional config and returns a PluginData tuple. | @@ -124,14 +145,25 @@ surface with `@databricks/appkit/beta`. Not meant for application imports. | [agentIdFromMarkdownPath](Function.agentIdFromMarkdownPath.md) | Derives the logical agent id from a markdown path. When the file is named `agent.md`, the id is the parent directory name (folder-based layout); otherwise the id is the file stem (e.g. legacy single-file paths). | | [appKitServingTypesPlugin](Function.appKitServingTypesPlugin.md) | Vite plugin to generate TypeScript types for AppKit serving endpoints. Fetches OpenAPI schemas from Databricks and generates a .d.ts with ServingEndpointRegistry module augmentation. | | [appKitTypesPlugin](Function.appKitTypesPlugin.md) | Vite plugin to generate types for AppKit queries. Calls generateFromEntryPoint under the hood. | +| [buildAssessment](Function.buildAssessment.md) | Build the single pass/fail Feedback assessment for an eval result. Returns undefined when there's no trace to attach to or the eval was skipped. | | [createAgent](Function.createAgent.md) | Pure factory for agent definitions. Returns the passed-in definition after cycle-detecting the sub-agent graph. Accepts the full `AgentDefinition` shape and is safe to call at module top-level. | | [createApp](Function.createApp.md) | Bootstraps AppKit with the provided configuration. | +| [createHttpDriver](Function.createHttpDriver.md) | Drives an agent by POSTing to a running app's chat endpoint and parsing the SSE response. Keeps the thread id across `send`s so multi-turn evals share a conversation. Agent/stream errors surface as `succeeded: false` rather than throwing, so `t.succeeded()` can assert on them. | | [createLakebasePool](Function.createLakebasePool.md) | Create a Lakebase pool with appkit's logger integration. Telemetry automatically uses appkit's OpenTelemetry configuration via global registry. | | [createLakebasePoolManager](Function.createLakebasePoolManager.md) | Create a pool manager that maintains per-key Lakebase connection pools. | +| [defineEval](Function.defineEval.md) | Define an agent eval. Default-export the result from a `config/agents//evals/*.eval.ts` file. | +| [defineEvalConfig](Function.defineEvalConfig.md) | Define per-directory eval config. Default-export from `evals.config.ts`. | | [defineTool](Function.defineTool.md) | Defines a single tool entry for a plugin's internal registry. | +| [discoverEvalFiles](Function.discoverEvalFiles.md) | Discover evals under `/config/agents//evals/`. The agent id is the directory name; the eval id is the file path relative to that evals dir with `.eval.ts` stripped. Returns a stable, sorted list. | +| [equals](Function.equals.md) | Passes when the value equals `expected` exactly. | +| [evalGlyph](Function.evalGlyph.md) | Status glyph for a single eval result. | | [executeFromRegistry](Function.executeFromRegistry.md) | Validates tool-call arguments against the entry's schema and invokes its handler. On validation failure, returns an LLM-friendly error string (matching the behavior of `tool()`) rather than throwing, so the model can self-correct on its next turn. | | [extractServingEndpoints](Function.extractServingEndpoints.md) | Extract serving endpoint config from a server file by AST-parsing it. Looks for `serving({ endpoints: { alias: { env: "..." }, ... } })` calls and extracts the endpoint alias names and their environment variable mappings. | | [findServerFile](Function.findServerFile.md) | Find the server entry file by checking candidate paths in order. | +| [formatEvalDetail](Function.formatEvalDetail.md) | Indented detail lines for a failing eval (error + failing assertions). | +| [formatEvalHeadline](Function.formatEvalHeadline.md) | The one-line header for a single eval result (no failure detail). | +| [formatEvalResults](Function.formatEvalResults.md) | Render all results as a human-readable console report (non-streaming). | +| [formatSummaryLine](Function.formatSummaryLine.md) | The final PASS/FAIL summary line. | | [functionToolToDefinition](Function.functionToolToDefinition.md) | - | | [generateDatabaseCredential](Function.generateDatabaseCredential.md) | Generate OAuth credentials for Postgres database connection using the proper Postgres API. | | [getExecutionContext](Function.getExecutionContext.md) | Get the current execution context. | @@ -141,15 +173,21 @@ surface with `@databricks/appkit/beta`. Not meant for application imports. | [getResourceRequirements](Function.getResourceRequirements.md) | Gets the resource requirements from a plugin's manifest. | | [getUsernameWithApiLookup](Function.getUsernameWithApiLookup.md) | Resolves the PostgreSQL username for a Lakebase connection. | | [getWorkspaceClient](Function.getWorkspaceClient.md) | Get workspace client from config or SDK default auth chain | +| [includes](Function.includes.md) | Passes when the value contains `substring`. | | [isFunctionTool](Function.isFunctionTool.md) | - | | [isHostedTool](Function.isHostedTool.md) | - | | [isSQLTypeMarker](Function.isSQLTypeMarker.md) | Type guard to check if a value is a SQL type marker | | [isToolkitEntry](Function.isToolkitEntry.md) | Type guard for `ToolkitEntry` — used by the agents plugin to differentiate toolkit references from inline tools in a mixed `tools` record. | | [loadAgentFromFile](Function.loadAgentFromFile.md) | Loads a single markdown agent file and resolves its frontmatter against registered plugin toolkits + ambient tool library. | | [loadAgentsFromDir](Function.loadAgentsFromDir.md) | Scans a directory for one subdirectory per agent, each containing `agent.md` (frontmatter + body). Produces an `AgentDefinition` record keyed by agent id (folder name). Throws on frontmatter errors or unresolved references. Returns an empty map if the directory does not exist. | +| [matches](Function.matches.md) | Passes when the value matches `pattern`. | | [mcpServer](Function.mcpServer.md) | Factory for declaring a custom MCP server tool. | | [parseTextToolCalls](Function.parseTextToolCalls.md) | Parses text-based tool calls from model output. | +| [reportToMlflow](Function.reportToMlflow.md) | Write one pass/fail assessment per eval result to the Databricks MLflow REST API. Never throws — failures are collected so the run still reports. | | [resolveHostedTools](Function.resolveHostedTools.md) | - | | [runAgent](Function.runAgent.md) | Standalone agent execution without `createApp`. Resolves the adapter, binds inline tools, and drives the adapter's `run()` loop to completion. | +| [runEval](Function.runEval.md) | Run a single eval against a driver. Never throws for assertion or agent failures — those become a non-passing [EvalResult](Interface.EvalResult.md). Only a malformed eval definition surfaces as `result.error`. | +| [runEvalsInDir](Function.runEvalsInDir.md) | Discover, load, and run every eval under each agent's `evals/` dir, driving the agents on a running app. Never throws for an individual eval — load/run failures become non-passing [EvalResult](Interface.EvalResult.md)s. | +| [summarize](Function.summarize.md) | - | | [tool](Function.tool.md) | Factory for defining function tools with Zod schemas. | | [toolsFromRegistry](Function.toolsFromRegistry.md) | Produces the `AgentToolDefinition[]` a ToolProvider exposes to the LLM, deriving `parameters` JSON Schema from each entry's Zod schema. | diff --git a/docs/docs/api/appkit/typedoc-sidebar.ts b/docs/docs/api/appkit/typedoc-sidebar.ts index c74d6232..db2537bd 100644 --- a/docs/docs/api/appkit/typedoc-sidebar.ts +++ b/docs/docs/api/appkit/typedoc-sidebar.ts @@ -127,6 +127,21 @@ const typedocSidebar: SidebarsConfig = { id: "api/appkit/Interface.AgentToolDefinition", label: "AgentToolDefinition" }, + { + type: "doc", + id: "api/appkit/Interface.AssertionHandle", + label: "AssertionHandle" + }, + { + type: "doc", + id: "api/appkit/Interface.AssertionResult", + label: "AssertionResult" + }, + { + type: "doc", + id: "api/appkit/Interface.Assessment", + label: "Assessment" + }, { type: "doc", id: "api/appkit/Interface.AutoInheritToolsConfig", @@ -147,11 +162,51 @@ const typedocSidebar: SidebarsConfig = { id: "api/appkit/Interface.DatabaseCredential", label: "DatabaseCredential" }, + { + type: "doc", + id: "api/appkit/Interface.DiscoveredEval", + label: "DiscoveredEval" + }, + { + type: "doc", + id: "api/appkit/Interface.DriveResult", + label: "DriveResult" + }, { type: "doc", id: "api/appkit/Interface.EndpointConfig", label: "EndpointConfig" }, + { + type: "doc", + id: "api/appkit/Interface.EvalConfig", + label: "EvalConfig" + }, + { + type: "doc", + id: "api/appkit/Interface.EvalDefinition", + label: "EvalDefinition" + }, + { + type: "doc", + id: "api/appkit/Interface.EvalDriver", + label: "EvalDriver" + }, + { + type: "doc", + id: "api/appkit/Interface.EvalResult", + label: "EvalResult" + }, + { + type: "doc", + id: "api/appkit/Interface.EvalRunSummary", + label: "EvalRunSummary" + }, + { + type: "doc", + id: "api/appkit/Interface.EvalSummary", + label: "EvalSummary" + }, { type: "doc", id: "api/appkit/Interface.FilePolicyUser", @@ -172,6 +227,11 @@ const typedocSidebar: SidebarsConfig = { id: "api/appkit/Interface.GenerateDatabaseCredentialRequest", label: "GenerateDatabaseCredentialRequest" }, + { + type: "doc", + id: "api/appkit/Interface.HttpDriverOptions", + label: "HttpDriverOptions" + }, { type: "doc", id: "api/appkit/Interface.IJobsConfig", @@ -212,6 +272,11 @@ const typedocSidebar: SidebarsConfig = { id: "api/appkit/Interface.LakebasePoolManager", label: "LakebasePoolManager" }, + { + type: "doc", + id: "api/appkit/Interface.MatchResult", + label: "MatchResult" + }, { type: "doc", id: "api/appkit/Interface.McpConnectAllResult", @@ -222,6 +287,11 @@ const typedocSidebar: SidebarsConfig = { id: "api/appkit/Interface.Message", label: "Message" }, + { + type: "doc", + id: "api/appkit/Interface.MlflowReportOptions", + label: "MlflowReportOptions" + }, { type: "doc", id: "api/appkit/Interface.PluginManifest", @@ -242,6 +312,11 @@ const typedocSidebar: SidebarsConfig = { id: "api/appkit/Interface.RegisteredAgent", label: "RegisteredAgent" }, + { + type: "doc", + id: "api/appkit/Interface.ReportOutcome", + label: "ReportOutcome" + }, { type: "doc", id: "api/appkit/Interface.RequestedClaims", @@ -272,6 +347,16 @@ const typedocSidebar: SidebarsConfig = { id: "api/appkit/Interface.RunAgentResult", label: "RunAgentResult" }, + { + type: "doc", + id: "api/appkit/Interface.RunEvalOptions", + label: "RunEvalOptions" + }, + { + type: "doc", + id: "api/appkit/Interface.RunEvalsOptions", + label: "RunEvalsOptions" + }, { type: "doc", id: "api/appkit/Interface.ServingEndpointEntry", @@ -292,6 +377,11 @@ const typedocSidebar: SidebarsConfig = { id: "api/appkit/Interface.TelemetryConfig", label: "TelemetryConfig" }, + { + type: "doc", + id: "api/appkit/Interface.TestContext", + label: "TestContext" + }, { type: "doc", id: "api/appkit/Interface.Thread", @@ -373,6 +463,11 @@ const typedocSidebar: SidebarsConfig = { id: "api/appkit/TypeAlias.ConfigSchema", label: "ConfigSchema" }, + { + type: "doc", + id: "api/appkit/TypeAlias.EvalProgress", + label: "EvalProgress" + }, { type: "doc", id: "api/appkit/TypeAlias.ExecutionResult", @@ -408,6 +503,11 @@ const typedocSidebar: SidebarsConfig = { id: "api/appkit/TypeAlias.JobsExport", label: "JobsExport" }, + { + type: "doc", + id: "api/appkit/TypeAlias.Matcher", + label: "Matcher" + }, { type: "doc", id: "api/appkit/TypeAlias.PluginData", @@ -438,6 +538,11 @@ const typedocSidebar: SidebarsConfig = { id: "api/appkit/TypeAlias.ServingFactory", label: "ServingFactory" }, + { + type: "doc", + id: "api/appkit/TypeAlias.Severity", + label: "Severity" + }, { type: "doc", id: "api/appkit/TypeAlias.ToolRegistry", @@ -495,6 +600,11 @@ const typedocSidebar: SidebarsConfig = { id: "api/appkit/Function.appKitTypesPlugin", label: "appKitTypesPlugin" }, + { + type: "doc", + id: "api/appkit/Function.buildAssessment", + label: "buildAssessment" + }, { type: "doc", id: "api/appkit/Function.createAgent", @@ -505,6 +615,11 @@ const typedocSidebar: SidebarsConfig = { id: "api/appkit/Function.createApp", label: "createApp" }, + { + type: "doc", + id: "api/appkit/Function.createHttpDriver", + label: "createHttpDriver" + }, { type: "doc", id: "api/appkit/Function.createLakebasePool", @@ -515,11 +630,36 @@ const typedocSidebar: SidebarsConfig = { id: "api/appkit/Function.createLakebasePoolManager", label: "createLakebasePoolManager" }, + { + type: "doc", + id: "api/appkit/Function.defineEval", + label: "defineEval" + }, + { + type: "doc", + id: "api/appkit/Function.defineEvalConfig", + label: "defineEvalConfig" + }, { type: "doc", id: "api/appkit/Function.defineTool", label: "defineTool" }, + { + type: "doc", + id: "api/appkit/Function.discoverEvalFiles", + label: "discoverEvalFiles" + }, + { + type: "doc", + id: "api/appkit/Function.equals", + label: "equals" + }, + { + type: "doc", + id: "api/appkit/Function.evalGlyph", + label: "evalGlyph" + }, { type: "doc", id: "api/appkit/Function.executeFromRegistry", @@ -535,6 +675,26 @@ const typedocSidebar: SidebarsConfig = { id: "api/appkit/Function.findServerFile", label: "findServerFile" }, + { + type: "doc", + id: "api/appkit/Function.formatEvalDetail", + label: "formatEvalDetail" + }, + { + type: "doc", + id: "api/appkit/Function.formatEvalHeadline", + label: "formatEvalHeadline" + }, + { + type: "doc", + id: "api/appkit/Function.formatEvalResults", + label: "formatEvalResults" + }, + { + type: "doc", + id: "api/appkit/Function.formatSummaryLine", + label: "formatSummaryLine" + }, { type: "doc", id: "api/appkit/Function.functionToolToDefinition", @@ -580,6 +740,11 @@ const typedocSidebar: SidebarsConfig = { id: "api/appkit/Function.getWorkspaceClient", label: "getWorkspaceClient" }, + { + type: "doc", + id: "api/appkit/Function.includes", + label: "includes" + }, { type: "doc", id: "api/appkit/Function.isFunctionTool", @@ -610,6 +775,11 @@ const typedocSidebar: SidebarsConfig = { id: "api/appkit/Function.loadAgentsFromDir", label: "loadAgentsFromDir" }, + { + type: "doc", + id: "api/appkit/Function.matches", + label: "matches" + }, { type: "doc", id: "api/appkit/Function.mcpServer", @@ -620,6 +790,11 @@ const typedocSidebar: SidebarsConfig = { id: "api/appkit/Function.parseTextToolCalls", label: "parseTextToolCalls" }, + { + type: "doc", + id: "api/appkit/Function.reportToMlflow", + label: "reportToMlflow" + }, { type: "doc", id: "api/appkit/Function.resolveHostedTools", @@ -630,6 +805,21 @@ const typedocSidebar: SidebarsConfig = { id: "api/appkit/Function.runAgent", label: "runAgent" }, + { + type: "doc", + id: "api/appkit/Function.runEval", + label: "runEval" + }, + { + type: "doc", + id: "api/appkit/Function.runEvalsInDir", + label: "runEvalsInDir" + }, + { + type: "doc", + id: "api/appkit/Function.summarize", + label: "summarize" + }, { type: "doc", id: "api/appkit/Function.tool", diff --git a/packages/appkit/package.json b/packages/appkit/package.json index 6a1541c3..2b50cc25 100644 --- a/packages/appkit/package.json +++ b/packages/appkit/package.json @@ -78,6 +78,7 @@ "@opentelemetry/semantic-conventions": "1.38.0", "@types/semver": "7.7.1", "apache-arrow": "21.1.0", + "autoevals": "^0.3.0", "dotenv": "16.6.1", "express": "4.22.0", "get-port": "7.2.0", diff --git a/packages/appkit/src/evals/index.ts b/packages/appkit/src/evals/index.ts index 7143d993..bcfecfe0 100644 --- a/packages/appkit/src/evals/index.ts +++ b/packages/appkit/src/evals/index.ts @@ -1,10 +1,16 @@ export { defineEval, defineEvalConfig } from "./define-eval"; export { type DiscoveredEval, discoverEvalFiles } from "./discover"; export { createHttpDriver, type HttpDriverOptions } from "./http-driver"; +export { + configureJudge, + isJudgeConfigured, + type JudgeConfig, + type JudgeScore, +} from "./judge"; export { equals, includes, matches } from "./matchers"; export { type Assessment, - buildAssessment, + buildAssessments, type MlflowReportOptions, type ReportOutcome, reportToMlflow, @@ -28,6 +34,7 @@ export { export type { AssertionHandle, AssertionResult, + CustomJudgeSpec, DriveResult, EvalConfig, EvalDefinition, diff --git a/packages/appkit/src/evals/judge.ts b/packages/appkit/src/evals/judge.ts new file mode 100644 index 00000000..7588d997 --- /dev/null +++ b/packages/appkit/src/evals/judge.ts @@ -0,0 +1,108 @@ +import { normalizeHost } from "./mlflow-rest"; + +/** + * LLM-as-judge scoring via the `autoevals` library (the same scorers eve uses), + * pointed at a Databricks serving endpoint. autoevals talks to an + * OpenAI-compatible API; Databricks Model Serving exposes one at + * `/serving-endpoints`, so we set `OPENAI_BASE_URL`/`OPENAI_API_KEY` and + * use the judge endpoint name as the model. + * + * There is no public REST to call Databricks' built-in judges directly (they're + * Python/SDK-only and the rubric prompts live in the mlflow package), so we run + * autoevals' equivalent scorers against a Databricks judge model. + */ +type AutoEvals = typeof import("autoevals"); + +let mod: AutoEvals | undefined; +let enabled = false; + +export interface JudgeConfig { + /** Databricks host (scheme optional). */ + host: string; + /** Bearer token for the serving endpoint. */ + token: string; + /** Serving endpoint name used as the judge model. */ + model: string; +} + +/** A normalized judge result. `score` is 0..1. */ +export interface JudgeScore { + score: number; + rationale?: string; +} + +/** + * Configure the judge once. Sets the OpenAI-compatible client env autoevals + * reads and the default judge model. No-op-safe: on failure, judging stays + * disabled and {@link isJudgeConfigured} returns false. + */ +export async function configureJudge(config: JudgeConfig): Promise { + try { + mod = await import("autoevals"); + process.env.OPENAI_BASE_URL = `${normalizeHost(config.host)}/serving-endpoints`; + process.env.OPENAI_API_KEY = config.token; + mod.init({ defaultModel: config.model }); + enabled = true; + } catch { + enabled = false; + } +} + +export function isJudgeConfigured(): boolean { + return enabled; +} + +/** Normalize an autoevals `Score` into a `JudgeScore`. */ +export function toJudgeScore(s: { + score?: number | null; + metadata?: Record; +}): JudgeScore { + const rationale = s.metadata?.rationale; + return { + score: typeof s.score === "number" ? s.score : 0, + rationale: typeof rationale === "string" ? rationale : undefined, + }; +} + +function ensure(): AutoEvals { + if (!enabled || !mod) { + throw new Error( + "LLM judge is not configured. Set --judge-model (and DATABRICKS_HOST/DATABRICKS_TOKEN) to use t.judge.*", + ); + } + return mod; +} + +/** Factuality of `output` vs an `expected` reference. */ +export async function judgeFactuality(args: { + input: string; + output: string; + expected: string; +}): Promise { + return toJudgeScore(await ensure().Factuality(args)); +} + +/** Whether `output` answers the question in `input`, optionally constrained by `criteria`. */ +export async function judgeClosedQA(args: { + input: string; + output: string; + criteria: string; +}): Promise { + return toJudgeScore(await ensure().ClosedQA(args)); +} + +/** + * A custom LLM judge defined by a prompt template + choice→score map — the + * TypeScript analog of MLflow's custom `@scorer`. + */ +export async function judgeCustom( + spec: { + name: string; + promptTemplate: string; + choiceScores: Record; + }, + args: { input: string; output: string }, +): Promise { + const scorer = ensure().LLMClassifierFromTemplate(spec); + return toJudgeScore(await scorer(args)); +} diff --git a/packages/appkit/src/evals/mlflow-report.ts b/packages/appkit/src/evals/mlflow-report.ts index f801bbd6..2136782c 100644 --- a/packages/appkit/src/evals/mlflow-report.ts +++ b/packages/appkit/src/evals/mlflow-report.ts @@ -25,32 +25,58 @@ export interface ReportOutcome { } /** - * Build the single pass/fail Feedback assessment for an eval result. Returns - * undefined when there's no trace to attach to or the eval was skipped. + * MLflow assessment names reject `.` (and we avoid spaces/parens too), so map + * anything outside `[A-Za-z0-9_-]` to `_`. The judge check on the raw label + * (`judge.`-prefixed) is unaffected — it runs before sanitization. */ -export function buildAssessment(result: EvalResult): Assessment | undefined { - if (!result.traceId || result.skipped) return undefined; +function sanitizeName(label: string): string { + return label.replace(/[^A-Za-z0-9_-]/g, "_"); +} - const failed = result.assertions.filter((a) => !a.pass); - const rationale = result.error - ? `error: ${result.error}` - : failed.length - ? failed - .map( - (a) => - `${a.severity}:${a.label}${a.detail ? ` (${a.detail})` : ""}`, - ) - .join("; ") - : "all assertions passed"; +/** + * Build the Feedback assessments for an eval result: one per assertion (judge + * assertions tagged `LLM_JUDGE` with their numeric score + rationale, so they + * render as judge feedback in MLflow) plus an overall `appkit_eval` pass/fail. + * Returns [] when there's no trace to attach to or the eval was skipped. + */ +export function buildAssessments(result: EvalResult): Assessment[] { + if (!result.traceId || result.skipped) return []; + const traceId = result.traceId; + const out: Assessment[] = []; + const used = new Map(); - return { - trace_id: result.traceId, + for (const a of result.assertions) { + const isJudge = a.label.startsWith("judge."); + const base = sanitizeName(a.label); + const seen = used.get(base) ?? 0; + used.set(base, seen + 1); + out.push({ + trace_id: traceId, + assessment_name: seen === 0 ? base : `${base}_${seen + 1}`, + source: isJudge + ? { source_type: "LLM_JUDGE", source_id: "appkit-judge" } + : { source_type: "CODE", source_id: "appkit-eval" }, + // Judges report a numeric score; deterministic assertions a boolean. + feedback: { value: a.score ?? a.pass }, + rationale: a.detail, + metadata: { eval_id: result.id, severity: a.severity }, + }); + } + + out.push({ + trace_id: traceId, assessment_name: "appkit_eval", source: { source_type: "CODE", source_id: "appkit-eval" }, feedback: { value: result.passed }, - rationale, + rationale: result.error + ? `error: ${result.error}` + : result.passed + ? "all gates passed" + : "one or more gates failed", metadata: { eval_id: result.id }, - }; + }); + + return out; } async function postAssessment( @@ -93,20 +119,22 @@ export async function reportToMlflow( ): Promise { const outcome: ReportOutcome = { written: 0, skipped: 0, failures: [] }; for (const result of results) { - const assessment = buildAssessment(result); - if (!assessment) { + const assessments = buildAssessments(result); + if (assessments.length === 0) { outcome.skipped++; continue; } - const res = await postAssessment(options.host, options.token, assessment); - if (res.ok) { - outcome.written++; - } else { - outcome.failures.push({ - traceId: assessment.trace_id, - status: res.status, - error: res.error, - }); + for (const assessment of assessments) { + const res = await postAssessment(options.host, options.token, assessment); + if (res.ok) { + outcome.written++; + } else { + outcome.failures.push({ + traceId: assessment.trace_id, + status: res.status, + error: res.error, + }); + } } } return outcome; diff --git a/packages/appkit/src/evals/mlflow-run.ts b/packages/appkit/src/evals/mlflow-run.ts index 5806a322..4253bc83 100644 --- a/packages/appkit/src/evals/mlflow-run.ts +++ b/packages/appkit/src/evals/mlflow-run.ts @@ -4,6 +4,13 @@ import type { EvalResult } from "./types"; /** Run tag value that makes a run appear under the experiment's "Evaluation runs". */ const GENAI_EVALUATE_RUN_TYPE = "genai_evaluate"; +/** + * Source tags on the eval run. Traces linked via `mlflow.sourceRun` surface the + * run's source in the traces-table "Source" column (mirrors how Python's + * `evaluate()` shows the script name), so tagging the run is what populates it. + */ +const EVAL_SOURCE_NAME = "appkit agent eval"; + interface CreateRunResponse { run: { info: { run_id?: string; run_uuid?: string } }; } @@ -34,17 +41,17 @@ export async function createEvalRun( experiment_id: options.experimentId, start_time: options.startTime, ...(options.runName ? { run_name: options.runName } : {}), + tags: [ + { key: "mlflow.runType", value: GENAI_EVALUATE_RUN_TYPE }, + { key: "mlflow.source.name", value: EVAL_SOURCE_NAME }, + { key: "mlflow.source.type", value: "LOCAL" }, + ], }, ); const runId = created.run?.info?.run_id ?? created.run?.info?.run_uuid; if (!runId) { throw new Error("runs/create returned no run id"); } - await mlflowPost(options, "/api/2.0/mlflow/runs/set-tag", { - run_id: runId, - key: "mlflow.runType", - value: GENAI_EVALUATE_RUN_TYPE, - }); return runId; } diff --git a/packages/appkit/src/evals/run-eval.ts b/packages/appkit/src/evals/run-eval.ts index 637508f2..e227af75 100644 --- a/packages/appkit/src/evals/run-eval.ts +++ b/packages/appkit/src/evals/run-eval.ts @@ -1,3 +1,4 @@ +import { judgeClosedQA, judgeCustom, judgeFactuality } from "./judge"; import type { AssertionHandle, AssertionResult, @@ -8,6 +9,9 @@ import type { TestContext, } from "./types"; +/** Default pass threshold for an LLM-judge score (0..1) before `.atLeast()`. */ +const DEFAULT_JUDGE_THRESHOLD = 0.5; + /** Thrown by `t.skip()` to unwind the test and mark the eval skipped. */ class SkipSignal extends Error { constructor(public reason?: string) { @@ -36,6 +40,7 @@ export async function runEval( ): Promise { const assertions: AssertionResult[] = []; let reply = ""; + let lastInput = ""; let toolCalls: string[] = []; let sessionId: string | undefined; let lastTraceId: string | undefined; @@ -73,8 +78,18 @@ export async function runEval( return handle; }; + // LLM-judge assertions are scored and soft by default; the caller chains + // `.atLeast(n)` to set the pass threshold or `.gate()` to promote. + const recordJudge = ( + label: string, + score: number, + rationale?: string, + ): AssertionHandle => + record(label, score >= DEFAULT_JUDGE_THRESHOLD, score, rationale).soft(); + const t: TestContext = { async send(message) { + lastInput = message; const r = await options.driver.send(message); reply = r.reply; toolCalls = r.toolCalls; @@ -113,6 +128,31 @@ export async function runEval( const m = matcher(value); return record("check", m.pass, m.score, m.detail); }, + judge: { + async factuality(expected) { + const { score, rationale } = await judgeFactuality({ + input: lastInput, + output: reply, + expected, + }); + return recordJudge("judge.factuality", score, rationale); + }, + async closedQA(criteria) { + const { score, rationale } = await judgeClosedQA({ + input: lastInput, + output: reply, + criteria, + }); + return recordJudge("judge.closedQA", score, rationale); + }, + async custom(spec) { + const { score, rationale } = await judgeCustom(spec, { + input: lastInput, + output: reply, + }); + return recordJudge(`judge.${spec.name}`, score, rationale); + }, + }, skip(reason) { throw new SkipSignal(reason); }, diff --git a/packages/appkit/src/evals/run-evals.ts b/packages/appkit/src/evals/run-evals.ts index 4cbd51c0..2a531add 100644 --- a/packages/appkit/src/evals/run-evals.ts +++ b/packages/appkit/src/evals/run-evals.ts @@ -1,6 +1,7 @@ import { pathToFileURL } from "node:url"; import { discoverEvalFiles } from "./discover"; import { createHttpDriver } from "./http-driver"; +import { configureJudge } from "./judge"; import { type ReportOutcome, reportToMlflow } from "./mlflow-report"; import { createEvalRun, type FinishOutcome, finishEvalRun } from "./mlflow-run"; import { runEval } from "./run-eval"; @@ -23,6 +24,11 @@ export interface RunEvalsOptions { * are logged. Requires Databricks creds + the target experiment. */ mlflow?: { host: string; token: string; experimentId: string }; + /** + * When set, enable `t.judge.*` LLM-as-judge scoring via autoevals against a + * Databricks serving endpoint (`model`). + */ + judge?: { host: string; token: string; model: string }; /** Wall-clock timestamp (ms) for run create/finish — pass `Date.now()`. */ now?: number; /** Progress callback, invoked as evals are discovered, started, and finished. */ @@ -110,6 +116,10 @@ export async function runEvalsInDir( const total = discovered.length; emit({ type: "discovered", total }); + if (options.judge) { + await configureJudge(options.judge); + } + // Create the MLflow evaluation run up front so each eval's trace can be // linked to it as it runs. let runId: string | undefined; diff --git a/packages/appkit/src/evals/tests/judge.test.ts b/packages/appkit/src/evals/tests/judge.test.ts new file mode 100644 index 00000000..7052d6a9 --- /dev/null +++ b/packages/appkit/src/evals/tests/judge.test.ts @@ -0,0 +1,22 @@ +import { describe, expect, test } from "vitest"; +import { isJudgeConfigured, toJudgeScore } from "../judge"; + +describe("judge score mapping", () => { + test("toJudgeScore reads score and rationale from an autoevals Score", () => { + expect( + toJudgeScore({ score: 0.8, metadata: { rationale: "mostly correct" } }), + ).toEqual({ score: 0.8, rationale: "mostly correct" }); + }); + + test("toJudgeScore defaults a missing/null score to 0 and omits rationale", () => { + expect(toJudgeScore({ score: null })).toEqual({ score: 0 }); + expect(toJudgeScore({})).toEqual({ score: 0 }); + expect(toJudgeScore({ score: 1, metadata: { rationale: 42 } })).toEqual({ + score: 1, + }); + }); + + test("judging is disabled until configured", () => { + expect(isJudgeConfigured()).toBe(false); + }); +}); diff --git a/packages/appkit/src/evals/tests/mlflow-report.test.ts b/packages/appkit/src/evals/tests/mlflow-report.test.ts index 14155640..db658c13 100644 --- a/packages/appkit/src/evals/tests/mlflow-report.test.ts +++ b/packages/appkit/src/evals/tests/mlflow-report.test.ts @@ -1,57 +1,73 @@ import { describe, expect, test } from "vitest"; -import { buildAssessment } from "../mlflow-report"; +import { buildAssessments } from "../mlflow-report"; import type { EvalResult } from "../types"; -describe("buildAssessment", () => { - test("builds a pass assessment with trace id and source", () => { +describe("buildAssessments", () => { + test("emits one feedback per assertion plus an overall appkit_eval", () => { const result: EvalResult = { id: "support/weather", traceId: "tr-123", - assertions: [{ label: "succeeded", severity: "gate", pass: true }], + assertions: [ + { label: "succeeded", severity: "gate", pass: true }, + { + label: "judge.closedQA", + severity: "soft", + pass: true, + score: 0.9, + detail: "clearly relevant", + }, + ], passed: true, }; - const a = buildAssessment(result); - expect(a).toEqual({ - trace_id: "tr-123", - assessment_name: "appkit_eval", - source: { source_type: "CODE", source_id: "appkit-eval" }, - feedback: { value: true }, - rationale: "all assertions passed", - metadata: { eval_id: "support/weather" }, - }); + const out = buildAssessments(result); + expect(out.map((a) => a.assessment_name)).toEqual([ + "succeeded", + "judge_closedQA", + "appkit_eval", + ]); + + const judge = out.find((a) => a.assessment_name === "judge_closedQA"); + expect(judge?.source.source_type).toBe("LLM_JUDGE"); + expect(judge?.feedback.value).toBe(0.9); // numeric score, not boolean + expect(judge?.rationale).toBe("clearly relevant"); + + const succeeded = out.find((a) => a.assessment_name === "succeeded"); + expect(succeeded?.source.source_type).toBe("CODE"); + expect(succeeded?.feedback.value).toBe(true); + + const overall = out.find((a) => a.assessment_name === "appkit_eval"); + expect(overall?.feedback.value).toBe(true); }); - test("fail assessment summarizes failing assertions in the rationale", () => { - const a = buildAssessment({ - id: "support/x", - traceId: "tr-9", + test("sanitizes and de-duplicates assertion names", () => { + const out = buildAssessments({ + id: "x", + traceId: "tr-1", assertions: [ - { label: "succeeded", severity: "gate", pass: true }, - { - label: "calledTool(get_weather)", - severity: "gate", - pass: false, - detail: "not called", - }, + { label: "calledTool(get_weather)", severity: "gate", pass: true }, + { label: "check", severity: "gate", pass: true }, + { label: "check", severity: "gate", pass: true }, ], - passed: false, + passed: true, }); - expect(a?.feedback.value).toBe(false); - expect(a?.rationale).toContain("gate:calledTool(get_weather) (not called)"); + const names = out.map((a) => a.assessment_name); + expect(names).toContain("calledTool_get_weather_"); + expect(names).toContain("check"); + expect(names).toContain("check_2"); }); - test("returns undefined without a trace id or when skipped", () => { - expect( - buildAssessment({ id: "x", assertions: [], passed: true }), - ).toBeUndefined(); + test("returns [] without a trace id or when skipped", () => { + expect(buildAssessments({ id: "x", assertions: [], passed: true })).toEqual( + [], + ); expect( - buildAssessment({ + buildAssessments({ id: "x", traceId: "tr-1", assertions: [], passed: true, skipped: { reason: "no data" }, }), - ).toBeUndefined(); + ).toEqual([]); }); }); diff --git a/packages/appkit/src/evals/types.ts b/packages/appkit/src/evals/types.ts index c6ef02e1..16a7d3be 100644 --- a/packages/appkit/src/evals/types.ts +++ b/packages/appkit/src/evals/types.ts @@ -84,10 +84,31 @@ export interface TestContext { calledTool(name: string): AssertionHandle; /** Assert a value against a matcher, e.g. `t.check(t.reply, includes("Sunny"))`. */ check(value: string, matcher: Matcher): AssertionHandle; + /** + * LLM-as-judge scoring of the last reply (via autoevals → a Databricks judge + * model). Each returns a scored, soft-by-default assertion; chain `.atLeast(n)` + * to set the pass threshold or `.gate()` to make it a hard gate. Requires the + * judge to be configured (`--judge-model`). + */ + judge: { + /** Score factuality of the reply against an expected reference. */ + factuality(expected: string): Promise; + /** Score whether the reply answers the question, per optional `criteria`. */ + closedQA(criteria: string): Promise; + /** A custom prompt-template judge (the TS analog of MLflow's `@scorer`). */ + custom(spec: CustomJudgeSpec): Promise; + }; /** Skip this eval with an optional reason. */ skip(reason?: string): never; } +/** A custom LLM-judge definition: a prompt template and choice→score mapping. */ +export interface CustomJudgeSpec { + name: string; + promptTemplate: string; + choiceScores: Record; +} + /** A single eval, default-exported from a `*.eval.ts` file. */ export interface EvalDefinition { /** Short human description, shown in reports. */ diff --git a/packages/appkit/src/plugins/agents/agents.ts b/packages/appkit/src/plugins/agents/agents.ts index 16289be3..533d3268 100644 --- a/packages/appkit/src/plugins/agents/agents.ts +++ b/packages/appkit/src/plugins/agents/agents.ts @@ -56,6 +56,7 @@ import { currentTraceId, initAgentTracing, linkTraceToRun, + updateTracePreview, withAgentSpan, } from "./mlflow"; import { @@ -1086,7 +1087,7 @@ export class AgentsPlugin extends Plugin implements ToolProvider { // No-op passthrough unless an MLflow experiment is bound. await withAgentSpan( { - name: registered.name ?? "agent", + name: registered.name || "agent", type: "AGENT", inputs: { messages: thread.messages.map((m) => ({ @@ -1159,6 +1160,18 @@ export class AgentsPlugin extends Plugin implements ToolProvider { }); } + // Populate the trace-level Request/Response columns and the trace + // name in the MLflow traces table (span inputs/outputs and the span + // name don't fill these). + const lastUserMessage = [...thread.messages] + .reverse() + .find((m) => m.role === "user")?.content; + updateTracePreview({ + request: lastUserMessage, + response: fullContent || undefined, + name: registered.name || "agent", + }); + // Surface the MLflow trace id so eval runs can attach assessments // to this turn's trace. No-op when tracing is disabled. const mlflowTraceId = currentTraceId(); diff --git a/packages/appkit/src/plugins/agents/mlflow.ts b/packages/appkit/src/plugins/agents/mlflow.ts index 4d3230d0..6c4a310e 100644 --- a/packages/appkit/src/plugins/agents/mlflow.ts +++ b/packages/appkit/src/plugins/agents/mlflow.ts @@ -115,6 +115,33 @@ export function linkTraceToRun(runId: string): void { } } +/** + * Populate the trace-level Request/Response columns and the trace name shown in + * the MLflow traces table (span-level inputs/outputs don't fill these, and the + * Trace-name column reads the `mlflow.traceName` tag, not the root span name). + * The Source column is run-derived (via `mlflow.sourceRun`), so a live chat + * turn with no run leaves it empty — it's only set for eval runs, where the run + * itself carries the source tags. No-op when tracing is off. + */ +export function updateTracePreview(opts: { + request?: string; + response?: string; + name?: string; +}): void { + if (!enabled || !mlflow) return; + try { + mlflow.updateCurrentTrace({ + ...(opts.request !== undefined ? { requestPreview: opts.request } : {}), + ...(opts.response !== undefined + ? { responsePreview: opts.response } + : {}), + ...(opts.name ? { tags: { "mlflow.traceName": opts.name } } : {}), + }); + } catch (err) { + logger.warn("Failed to update trace preview: %O", err); + } +} + /** Flush buffered traces (e.g. on shutdown). No-op when tracing is disabled. */ export async function flushAgentTraces(): Promise { if (!enabled || !mlflow) return; diff --git a/packages/shared/src/cli/commands/agent/eval.ts b/packages/shared/src/cli/commands/agent/eval.ts index ee692423..fa327567 100644 --- a/packages/shared/src/cli/commands/agent/eval.ts +++ b/packages/shared/src/cli/commands/agent/eval.ts @@ -28,6 +28,7 @@ interface EvalRunner { strict?: boolean; headers?: Record; mlflow?: { host: string; token: string; experimentId: string }; + judge?: { host: string; token: string; model: string }; onEvent?: (event: EvalProgress) => void; }): Promise; evalGlyph(result: unknown): string; @@ -72,6 +73,7 @@ interface EvalOptions { databricksHost?: string; databricksToken?: string; experiment?: string; + judgeModel?: string; } async function runAgentEval( @@ -89,6 +91,13 @@ async function runAgentEval( const mlflow = host && token && experimentId ? { host, token, experimentId } : undefined; + // LLM-as-judge: reuse the Databricks creds + a judge serving endpoint. + const judgeModel = opts.judgeModel ?? process.env.APPKIT_JUDGE_MODEL; + const judge = + judgeModel && host && token + ? { host, token, model: judgeModel } + : undefined; + // Stream progress as evals run, instead of going silent until the end. const onEvent = (event: EvalProgress): void => { switch (event.type) { @@ -122,6 +131,7 @@ async function runAgentEval( strict: opts.strict, headers: opts.header ? parseHeaders(opts.header) : undefined, mlflow, + judge, onEvent, }); console.log(`\n${runner.formatSummaryLine(summary.results)}`); @@ -188,4 +198,8 @@ export const agentEvalCommand = new Command("eval") "--experiment ", "MLflow experiment id for the evaluation run (default: MLFLOW_EXPERIMENT_ID)", ) + .option( + "--judge-model ", + "Databricks serving endpoint to use as the LLM judge for t.judge.* (default: APPKIT_JUDGE_MODEL)", + ) .action(runAgentEval); diff --git a/pnpm-lock.yaml b/pnpm-lock.yaml index 4042be1b..d641f38b 100644 --- a/pnpm-lock.yaml +++ b/pnpm-lock.yaml @@ -302,6 +302,9 @@ importers: apache-arrow: specifier: 21.1.0 version: 21.1.0 + autoevals: + specifier: ^0.3.0 + version: 0.3.0(ws@8.18.3(bufferutil@4.0.9))(zod@4.3.6) dotenv: specifier: 16.6.1 version: 16.6.1 @@ -5649,6 +5652,12 @@ packages: autocomplete.js@0.37.1: resolution: {integrity: sha512-PgSe9fHYhZEsm/9jggbjtVsGXJkPLvd+9mC7gZJ662vVL5CRWEtm/mIrrzCx0MrNxHVwxD5d00UOn6NsmL2LUQ==} + autoevals@0.3.0: + resolution: {integrity: sha512-4CEzBVhjVBHvk46s+DBcgmEfOdM+zXEoCkvvDqYO2IWdVpZKUJANfX8BvfE5vfcGy24on9zW21Zf7V9cUJdbfg==} + engines: {npm: please-use-pnpm, pnpm: '>=10.27.0', yarn: please-use-pnpm} + peerDependencies: + zod: ^3.25.0 || ^4.0.0 + autoprefixer@10.4.21: resolution: {integrity: sha512-O+A6LWV5LDHSJD3LjHYoNi4VLsj/Whi7k6zG12xTYaU4cQ8oxQGckXNX8cRHK5yOZ/ppVHe0ZBXGzSV9jXdVbQ==} engines: {node: ^10 || ^12 || >=14} @@ -5750,6 +5759,9 @@ packages: resolution: {integrity: sha512-Ceh+7ox5qe7LJuLHoY0feh3pHuUDHAcRUeyL2VYghZwfpkNIy/+8Ocg0a3UuSoYzavmylwuLWQOf3hl0jjMMIw==} engines: {node: '>=8'} + binary-search@1.3.6: + resolution: {integrity: sha512-nbE1WxOTTrUWIfsfZ4aHGYu5DOuNkbxGokjV6Z2kxfJK3uaAb8zNK1muzOeipoLHZjInT4Br88BHpzevc681xA==} + birpc@4.0.0: resolution: {integrity: sha512-LShSxJP0KTmd101b6DRyGBj57LZxSDYWKitQNW/mi8GRMvZb078Uf9+pveax1DrVL89vm7mWe+TovdI/UDOuPw==} @@ -5951,6 +5963,9 @@ packages: resolution: {integrity: sha512-WDrybc/gKFpTYQutKIK6UvfcuxijIZfMfXaYm8NMsPQxSYvf+13fXUJ4rztGGbJcBQ/GF55gvrZ0Bc0bj/mqvg==} engines: {node: '>=20.18.1'} + cheminfo-types@1.15.0: + resolution: {integrity: sha512-shv45WN2u0yN9EHH1bisNrv+fy4Cw+eLM5lOoriP67mePrwbHZ1kJqg90C8GEU7K1A8gJsicEoVZHcuBbuul/w==} + chevrotain-allstar@0.3.1: resolution: {integrity: sha512-b7g+y9A0v4mxCW1qUhf3BSVPg+/NvGErk/dOkrDaHA0nQIQGAtrOjlX//9OQtRlSCy+x9rfB5N8yC71lH1nvMw==} peerDependencies: @@ -6140,6 +6155,15 @@ packages: resolution: {integrity: sha512-9mAqGPHLakhCLeNyxPkK4xVo746zQ/czLH1Ky+vkitMnWfWZps8r0qXuwhwizagCRttsL4lfG4pIOvaWLpAP0w==} engines: {node: '>= 0.8.0'} + compute-cosine-similarity@1.1.0: + resolution: {integrity: sha512-FXhNx0ILLjGi9Z9+lglLzM12+0uoTnYkHm7GiadXDAr0HGVLm25OivUS1B/LPkbzzvlcXz/1EvWg9ZYyJSdhTw==} + + compute-dot@1.1.0: + resolution: {integrity: sha512-L5Ocet4DdMrXboss13K59OK23GXjiSia7+7Ukc7q4Bl+RVpIXK2W9IHMbWDZkh+JUEvJAwOKRaJDiFUa1LTnJg==} + + compute-l2norm@1.1.0: + resolution: {integrity: sha512-6EHh1Elj90eU28SXi+h2PLnTQvZmkkHWySpoFz+WOlVNLz3DQoC4ISUHSV9n5jMxPHtKGJ01F4uu2PsXBB8sSg==} + concat-map@0.0.1: resolution: {integrity: sha512-/Srv4dswyQNBfohGpz9o6Yb3Gz3SrUDqBH5rTuhGR7ahtlbYKnVxw2bCFMRljaA7EXHaXZ8wsHdodFvbkhKmqg==} @@ -7314,6 +7338,9 @@ packages: resolution: {integrity: sha512-7yAQpD2UMJzLi1Dqv7qFYnPbaPx7ZfFK6PiIxQ4PfkGPyNyl2Ugx+a/umUonmKqjhM4DnfbMvdX6otXq83soQQ==} engines: {node: ^12.20 || >= 14.13} + fft.js@4.0.4: + resolution: {integrity: sha512-f9c00hphOgeQTlDyavwTtu6RiK8AIFjD6+jvXkNkpeQ7rirK3uFWVpalkoS4LAwbdX7mfZ8aoBfFVQX1Re/8aw==} + figures@3.2.0: resolution: {integrity: sha512-yaduQFRKLXYOGgEn6AZau90j3ggSOyiqXU0F9JZfeXYhNa+Jk4X+s45A2zg5jns87GAFa34BBm2kXw4XpNcbdg==} engines: {node: '>=8'} @@ -8072,6 +8099,9 @@ packages: is-alphanumerical@2.0.1: resolution: {integrity: sha512-hmbYhX/9MUMF5uh7tOXyK/n0ZvWpad5caBA17GsC6vyuCqaWliRG5K1qS9inmUhEMaOBIW7/whAnSwveW/LtZw==} + is-any-array@3.0.0: + resolution: {integrity: sha512-o4h+tylWykC4BD1vaejp6gDxoM13bwW8FGuNs4yIKpj8xbBJcRxJx8vZpq0dCr7ZDEfeKjmsi/euolKhX6f/ww==} + is-arrayish@0.2.1: resolution: {integrity: sha512-zz06S8t0ozoDXMG+ube26zeCTNXcKIPJZJi8hBrF4idCLms4CG9QtK7qBl1boi5ODzFpjswb5JPmHCbMpjaYzg==} @@ -8307,6 +8337,10 @@ packages: joi@17.13.3: resolution: {integrity: sha512-otDA4ldcIx+ZXsKHWmp0YizCweVRZG96J10b0FevjfuncLO1oX59THoAmHkNubYJ+9gWsYsp5k8v4ib6oDv1fA==} + js-levenshtein@1.1.6: + resolution: {integrity: sha512-X2BB11YZtrRqY4EnQcLX5Rh373zbK4alC1FW7D7MBhL2gtcC17cTnr6DmfHZeS0s2rTHjUTMMHfG7gO8SSdw+g==} + engines: {node: '>=0.10.0'} + js-tokens@4.0.0: resolution: {integrity: sha512-RdJUflcE3cUzKiMqQgsCu06FPu9UdIJO0beYbPhHN4k6apgJtifcoCtT9bcxOpYBtpD2kCM6Sbzg4CausW/PKQ==} @@ -8527,6 +8561,9 @@ packages: resolution: {integrity: sha512-/vlFKAoH5Cgt3Ie+JLhRbwOsCQePABiU3tJ1egGvyQ+33R/vcwM2Zl2QR/LzjsBeItPt3oSVXapn+m4nQDvpzw==} engines: {node: '>=14'} + linear-sum-assignment@1.0.9: + resolution: {integrity: sha512-1T2Ek3sxpt2mBHeBFMRJEikiIK/yIOwf+mrxv/DkAU/5ddnCMndZL//hFH7QuHa1tbaQADzsf9t7rkGZKqoFfQ==} + lines-and-columns@1.2.4: resolution: {integrity: sha512-7ylylesZQ/PV29jhEDl3Ufjo6ZX7gCqJr5F7PKrqc93v7fzSymt1BpwEU8nAUXs8qzzvqhbjhK5QZg6Mt/HkBg==} @@ -9070,6 +9107,24 @@ packages: engines: {node: '>=10'} hasBin: true + ml-array-max@2.0.0: + resolution: {integrity: sha512-QQZ4kENwpWmyNb98UXRDFXrmtIXuXtt1+bSbda/2KA85+F+rrJP8hZk6QOkCQXM2Th9mUDYdq/PNByPdT9ID4A==} + + ml-array-min@2.0.0: + resolution: {integrity: sha512-GRj6Ky6sW9vGL6yIjgsHmXZ9YgrdmcQ8nCxPqEGeKc6dkfYg1XDYxGFxADUjNuZyoCd5PUscWAS4N+cFaX6hFg==} + + ml-array-rescale@2.0.0: + resolution: {integrity: sha512-2GGtKfSno94/kIloWGvpp/U5Q5vLvLrza+SAaGsLeo6Xj4mEbA6Gqx+oTfZFkxnd1grT2X007HfJNs3T5BsiVg==} + + ml-matrix@6.12.2: + resolution: {integrity: sha512-GC+BnW+pBh8Auap8goAxY0senAmF0IEoc3HNVSfnfbvGw0buuDIYb9kAKMS1l+GiwJ1rfK2bzJ8IHhwjzATSFA==} + + ml-spectra-processing@14.29.0: + resolution: {integrity: sha512-825CS864krbjMv7OB0mbjgAmyOL5ymj1OGa0gAzz1h1Dcd3Eeol2DaOimSiPYmRhW+iYhpeQnb7cSU0mlSK6+g==} + + ml-xsadd@3.0.1: + resolution: {integrity: sha512-Fz2q6dwgzGM8wYKGArTUTZDGa4lQFA2Vi6orjGeTVRy22ZnQFKlJuwS9n8NRviqz1KHAHAzdKJwbnYhdo38uYg==} + mlflow-tracing@0.1.3: resolution: {integrity: sha512-Koqkwaid5ubGHuLprBP6J7Su70WddlD11f2vgzgxbFFHYKsAsJatMGvjIck5CkyhT/gMUyBqpA3Lkl+zC3W3uQ==} engines: {node: '>=18'} @@ -9104,6 +9159,10 @@ packages: resolution: {integrity: sha512-2eznPJP8z2BFLX50tf0LuODrpINqP1RVIm/CObbTcBRITQgmC/TjcREF1NeTBzIcR5XO/ukWo+YHOjBbFwIupg==} hasBin: true + mustache@4.2.0: + resolution: {integrity: sha512-71ippSywq5Yb7/tVYyGbkBggbU8H3u5Rz56fH60jGFgr8uHwxs+aSKeqmluIVzM0m0kB7xQjKS6qPfd0b2ZoqQ==} + hasBin: true + mute-stream@2.0.0: resolution: {integrity: sha512-WWdIxpyjEn+FhQJQQv9aQAYlHoNVdzIzUySNV1gHUPDSdZJ3yZn7pAAbQcV7B56Mvu881q9FZV+0Vx2xC44VWA==} engines: {node: ^18.17.0 || >=20.5.0} @@ -9337,6 +9396,17 @@ packages: resolution: {integrity: sha512-7x81NCL719oNbsq/3mh+hVrAWmFuEYUqrq/Iw3kUzH8ReypT9QQ0BLoJS7/G9k6N81XjW4qHWtjWwe/9eLy1EQ==} engines: {node: '>=12'} + openai@6.44.0: + resolution: {integrity: sha512-09/gH+8jH0RgUwsgWHAaxsKGRT5zVZ95IaJUnqAWj6XejIBmnFRwq2WUIF37VtDEsmGrtPmvCs5+yBSeZGWvkA==} + peerDependencies: + ws: ^8.18.0 + zod: ^3.25 || ^4.0 + peerDependenciesMeta: + ws: + optional: true + zod: + optional: true + opener@1.5.2: resolution: {integrity: sha512-ur5UIdyw5Y7yEj9wLzhqXiy6GZ3Mwx0yGI+5sMn2r0N0v3cKJvUmFH5yPP+WXh9e0xfyzyJX95D8l088DNFj7A==} hasBin: true @@ -11687,6 +11757,12 @@ packages: resolution: {integrity: sha512-hVDIBwsRruT73PbK7uP5ebUt+ezEtCmzZz3F59BSr2F6OVFnJ/6h8liuvdLrQ88Xmnk6/+xGGuq+pG9WwTuy3A==} engines: {node: ^20.17.0 || >=22.9.0} + validate.io-array@1.0.6: + resolution: {integrity: sha512-DeOy7CnPEziggrOO5CZhVKJw6S3Yi7e9e65R1Nl/RTN1vTQKnzjfvks0/8kQ40FP/dsjRAOd4hxmJ7uLa6vxkg==} + + validate.io-function@1.0.2: + resolution: {integrity: sha512-LlFybRJEriSuBnUhQyG5bwglhh50EpTL2ul23MPIuR1odjO7XaMLFV8vHGwp7AZciFxtYOeiSCT5st+XSPONiQ==} + validator@13.15.26: resolution: {integrity: sha512-spH26xU080ydGggxRyR1Yhcbgx+j3y5jbNXk/8L+iRvdIEQ4uTRH2Sgf2dokud6Q4oAtsbNvJ1Ft+9xmm6IZcA==} engines: {node: '>= 0.10'} @@ -12111,6 +12187,11 @@ packages: resolution: {integrity: sha512-CzhO+pFNo8ajLM2d2IW/R93ipy99LWjtwblvC1RsoSUMZgyLbYFr221TnSNT7GjGdYui6P459mw9JH/g/zW2ug==} engines: {node: '>=18'} + zod-to-json-schema@3.25.0: + resolution: {integrity: sha512-HvWtU2UG41LALjajJrML6uQejQhNJx+JBO9IflpSja4R03iNWfKXrj6W2h7ljuLyc1nKS+9yDyL/9tD1U/yBnQ==} + peerDependencies: + zod: ^3.25 || ^4 + zod-validation-error@4.0.2: resolution: {integrity: sha512-Q6/nZLe6jxuU80qb/4uJ4t5v2VEZ44lzQjPDhYJNztRQ4wyWc6VF3D3Kb/fAuPetZQnhS3hnajCf9CsWesghLQ==} engines: {node: '>=18.0.0'} @@ -13328,7 +13409,7 @@ snapshots: '@commitlint/config-validator@19.8.1': dependencies: '@commitlint/types': 19.8.1 - ajv: 8.17.1 + ajv: 8.18.0 '@commitlint/ensure@19.8.1': dependencies: @@ -18193,9 +18274,9 @@ snapshots: '@opentelemetry/api': 1.9.0 zod: 4.3.6 - ajv-formats@2.1.1(ajv@8.17.1): + ajv-formats@2.1.1(ajv@8.18.0): optionalDependencies: - ajv: 8.17.1 + ajv: 8.18.0 ajv-formats@3.0.1(ajv@8.18.0): optionalDependencies: @@ -18205,9 +18286,9 @@ snapshots: dependencies: ajv: 6.12.6 - ajv-keywords@5.1.0(ajv@8.17.1): + ajv-keywords@5.1.0(ajv@8.18.0): dependencies: - ajv: 8.17.1 + ajv: 8.18.0 fast-deep-equal: 3.1.3 ajv@6.12.6: @@ -18352,6 +18433,20 @@ snapshots: dependencies: immediate: 3.3.0 + autoevals@0.3.0(ws@8.18.3(bufferutil@4.0.9))(zod@4.3.6): + dependencies: + ajv: 8.18.0 + compute-cosine-similarity: 1.1.0 + js-levenshtein: 1.1.6 + js-yaml: 4.1.1 + linear-sum-assignment: 1.0.9 + mustache: 4.2.0 + openai: 6.44.0(ws@8.18.3(bufferutil@4.0.9))(zod@4.3.6) + zod: 4.3.6 + zod-to-json-schema: 3.25.0(zod@4.3.6) + transitivePeerDependencies: + - ws + autoprefixer@10.4.21(postcss@8.5.6): dependencies: browserslist: 4.28.1 @@ -18452,6 +18547,8 @@ snapshots: binary-extensions@2.3.0: {} + binary-search@1.3.6: {} + birpc@4.0.0: {} bl@4.1.0: @@ -18747,6 +18844,8 @@ snapshots: undici: 7.24.5 whatwg-mimetype: 4.0.0 + cheminfo-types@1.15.0: {} + chevrotain-allstar@0.3.1(chevrotain@11.0.3): dependencies: chevrotain: 11.0.3 @@ -18932,6 +19031,23 @@ snapshots: transitivePeerDependencies: - supports-color + compute-cosine-similarity@1.1.0: + dependencies: + compute-dot: 1.1.0 + compute-l2norm: 1.1.0 + validate.io-array: 1.0.6 + validate.io-function: 1.0.2 + + compute-dot@1.1.0: + dependencies: + validate.io-array: 1.0.6 + validate.io-function: 1.0.2 + + compute-l2norm@1.1.0: + dependencies: + validate.io-array: 1.0.6 + validate.io-function: 1.0.2 + concat-map@0.0.1: {} concat-stream@2.0.0: @@ -20121,6 +20237,8 @@ snapshots: node-domexception: 1.0.0 web-streams-polyfill: 3.3.3 + fft.js@4.0.4: {} + figures@3.2.0: dependencies: escape-string-regexp: 1.0.5 @@ -21106,6 +21224,8 @@ snapshots: is-alphabetical: 2.0.1 is-decimal: 2.0.1 + is-any-array@3.0.0: {} + is-arrayish@0.2.1: {} is-binary-path@2.1.0: @@ -21308,6 +21428,8 @@ snapshots: '@sideway/formula': 3.0.1 '@sideway/pinpoint': 2.0.0 + js-levenshtein@1.1.6: {} + js-tokens@4.0.0: {} js-tokens@9.0.1: {} @@ -21516,6 +21638,12 @@ snapshots: lilconfig@3.1.3: {} + linear-sum-assignment@1.0.9: + dependencies: + cheminfo-types: 1.15.0 + ml-matrix: 6.12.2 + ml-spectra-processing: 14.29.0 + lines-and-columns@1.2.4: {} linkify-it@5.0.0: @@ -22354,6 +22482,36 @@ snapshots: mkdirp@3.0.1: {} + ml-array-max@2.0.0: + dependencies: + is-any-array: 3.0.0 + + ml-array-min@2.0.0: + dependencies: + is-any-array: 3.0.0 + + ml-array-rescale@2.0.0: + dependencies: + is-any-array: 3.0.0 + ml-array-max: 2.0.0 + ml-array-min: 2.0.0 + + ml-matrix@6.12.2: + dependencies: + is-any-array: 3.0.0 + ml-array-rescale: 2.0.0 + + ml-spectra-processing@14.29.0: + dependencies: + binary-search: 1.3.6 + cheminfo-types: 1.15.0 + fft.js: 4.0.4 + is-any-array: 3.0.0 + ml-matrix: 6.12.2 + ml-xsadd: 3.0.1 + + ml-xsadd@3.0.1: {} + mlflow-tracing@0.1.3: dependencies: '@databricks/sdk-experimental': 0.15.0 @@ -22393,6 +22551,8 @@ snapshots: dns-packet: 5.6.1 thunky: 1.1.0 + mustache@4.2.0: {} + mute-stream@2.0.0: {} nanoid@3.3.11: {} @@ -22616,6 +22776,11 @@ snapshots: is-docker: 2.2.1 is-wsl: 2.2.0 + openai@6.44.0(ws@8.18.3(bufferutil@4.0.9))(zod@4.3.6): + optionalDependencies: + ws: 8.18.3(bufferutil@4.0.9) + zod: 4.3.6 + opener@1.5.2: {} optionator@0.9.4: @@ -24189,9 +24354,9 @@ snapshots: schema-utils@4.3.3: dependencies: '@types/json-schema': 7.0.15 - ajv: 8.17.1 - ajv-formats: 2.1.1(ajv@8.17.1) - ajv-keywords: 5.1.0(ajv@8.17.1) + ajv: 8.18.0 + ajv-formats: 2.1.1(ajv@8.18.0) + ajv-keywords: 5.1.0(ajv@8.18.0) search-insights@2.17.3: {} @@ -25205,6 +25370,10 @@ snapshots: validate-npm-package-name@7.0.2: {} + validate.io-array@1.0.6: {} + + validate.io-function@1.0.2: {} + validator@13.15.26: {} value-equal@1.0.1: {} @@ -25719,6 +25888,10 @@ snapshots: yoctocolors@2.1.2: {} + zod-to-json-schema@3.25.0(zod@4.3.6): + dependencies: + zod: 4.3.6 + zod-validation-error@4.0.2(zod@4.1.13): dependencies: zod: 4.1.13 From 0b5571942f0e2317a9a9353193bb06e91896ad5b Mon Sep 17 00:00:00 2001 From: MarioCadenas Date: Fri, 10 Jul 2026 11:21:49 +0200 Subject: [PATCH 3/3] refactor(appkit): add mlflow connector with profile-based oauth Introduce connectors/mlflow as the shared REST + auth layer for MLflow, so the eval runner (and future callers) stop threading host/token and hand-rolling fetch/URL logic: - MlflowClient owns {host, token}: normalizes the host once, exposes post() (throws) for runs/* and postResult() (structured failure) for best-effort assessment writes, plus servingEndpointsUrl() for the judge. - resolveDatabricksAuth() mints an OAuth bearer from a CLI profile via the SDK WorkspaceClient (the AppKit-native path), so `agent eval` no longer requires a hand-set DATABRICKS_TOKEN. Adds an `--profile` flag. - Eval run create/finish, assessment reporting, and the judge take the client; the agents plugin's host normalization now delegates to the connector's normalizeHost. The mlflow-tracing SDK wrapper stays in the agents plugin: it manages a process-global provider (like TelemetryManager) and has an agent-shaped API, so it isn't a connector. Signed-off-by: MarioCadenas --- packages/appkit/src/connectors/index.ts | 1 + packages/appkit/src/connectors/mlflow/auth.ts | 55 ++++++++++++ .../appkit/src/connectors/mlflow/client.ts | 90 +++++++++++++++++++ .../appkit/src/connectors/mlflow/index.ts | 6 ++ .../connectors/mlflow/tests/client.test.ts | 82 +++++++++++++++++ packages/appkit/src/evals/index.ts | 9 +- packages/appkit/src/evals/judge.ts | 10 +-- packages/appkit/src/evals/mlflow-report.ts | 46 ++-------- packages/appkit/src/evals/mlflow-rest.ts | 39 -------- packages/appkit/src/evals/mlflow-run.ts | 15 ++-- packages/appkit/src/evals/run-evals.ts | 23 +++-- packages/appkit/src/plugins/agents/mlflow.ts | 3 +- .../shared/src/cli/commands/agent/eval.ts | 32 +++++-- 13 files changed, 305 insertions(+), 106 deletions(-) create mode 100644 packages/appkit/src/connectors/mlflow/auth.ts create mode 100644 packages/appkit/src/connectors/mlflow/client.ts create mode 100644 packages/appkit/src/connectors/mlflow/index.ts create mode 100644 packages/appkit/src/connectors/mlflow/tests/client.test.ts delete mode 100644 packages/appkit/src/evals/mlflow-rest.ts diff --git a/packages/appkit/src/connectors/index.ts b/packages/appkit/src/connectors/index.ts index 5fad31d9..742e069d 100644 --- a/packages/appkit/src/connectors/index.ts +++ b/packages/appkit/src/connectors/index.ts @@ -3,5 +3,6 @@ export * from "./genie"; export * from "./jobs"; export * from "./lakebase"; export * from "./mcp"; +export * from "./mlflow"; export * from "./sql-warehouse"; export * from "./vector-search"; diff --git a/packages/appkit/src/connectors/mlflow/auth.ts b/packages/appkit/src/connectors/mlflow/auth.ts new file mode 100644 index 00000000..adbe6d63 --- /dev/null +++ b/packages/appkit/src/connectors/mlflow/auth.ts @@ -0,0 +1,55 @@ +import { WorkspaceClient } from "@databricks/sdk-experimental"; + +/** Resolved Databricks host + bearer token for the eval runner's REST calls. */ +export interface DatabricksAuth { + host: string; + token: string; +} + +export interface ResolveDatabricksAuthOptions { + /** `~/.databrickscfg` profile to authenticate with (e.g. `dogfood`). */ + profile?: string; + /** Explicit host; wins over the profile/SDK-resolved host when set. */ + host?: string; + /** Explicit bearer token; when set, no OAuth is minted (PAT/CI path). */ + token?: string; +} + +/** + * Resolve `{host, token}` for the eval runner the same way the rest of AppKit + * authenticates: construct a Databricks `WorkspaceClient` and let its config + * mint (and later refresh) an OAuth bearer from the CLI profile — no hand-set + * PAT required. An explicit host/token still wins (PAT or CI env), so the SDK + * is only consulted for whatever isn't supplied. + * + * Returns `undefined` when neither an explicit token nor a resolvable profile + * yields a bearer, so the caller can treat auth as simply unavailable. + */ +export async function resolveDatabricksAuth( + options: ResolveDatabricksAuthOptions = {}, +): Promise { + // Fully explicit — no need to touch the SDK. + if (options.host && options.token) { + return { host: options.host, token: options.token }; + } + + try { + const client = new WorkspaceClient( + options.profile ? { profile: options.profile } : {}, + ); + const headers = new Headers(); + // Mints the OAuth access token (or reuses a PAT from the profile) and adds + // an `Authorization: Bearer ` header — the same call the connectors + // use before each request. + await client.config.authenticate(headers); + const token = + options.token ?? headers.get("authorization")?.replace(/^Bearer\s+/i, ""); + const host = + options.host ?? + (await client.config.getHost()).toString().replace(/\/+$/, ""); + if (!token || !host) return undefined; + return { host, token }; + } catch { + return undefined; + } +} diff --git a/packages/appkit/src/connectors/mlflow/client.ts b/packages/appkit/src/connectors/mlflow/client.ts new file mode 100644 index 00000000..b764c9fb --- /dev/null +++ b/packages/appkit/src/connectors/mlflow/client.ts @@ -0,0 +1,90 @@ +/** Shared client for talking to the Databricks/MLflow REST API. */ + +/** Ensure the host has a scheme (Databricks env often lacks `https://`). */ +export function normalizeHost(raw: string): string { + const h = raw.trim().replace(/\/+$/, ""); + return /^https?:\/\//i.test(h) ? h : `https://${h}`; +} + +/** Structured result for a best-effort POST that must not throw. */ +export interface PostResult { + ok: boolean; + status?: number; + error?: string; +} + +/** + * A thin client over the Databricks workspace REST API, owning the host + bearer + * token so callers (eval-run creation, assessment writes, the judge's serving + * endpoint) don't each re-derive URLs or re-attach auth. The host is normalized + * once at construction. + */ +export class MlflowClient { + /** Normalized workspace base URL (scheme guaranteed, no trailing slash). */ + readonly baseUrl: string; + private readonly token: string; + + constructor(host: string, token: string) { + this.baseUrl = normalizeHost(host); + this.token = token; + } + + private headers(): Record { + return { + "content-type": "application/json", + authorization: `Bearer ${this.token}`, + }; + } + + /** + * POST JSON to an MLflow REST endpoint. Returns the parsed JSON body, or + * throws with the status + response text so callers can surface a precise + * error. Use for calls whose failure should abort (e.g. `runs/create`). + */ + async post(path: string, body: unknown): Promise { + const res = await fetch(`${this.baseUrl}${path}`, { + method: "POST", + headers: this.headers(), + body: JSON.stringify(body), + }); + if (!res.ok) { + const text = await res.text().catch(() => ""); + throw new Error(`${path} -> ${res.status} ${text.slice(0, 500)}`); + } + const text = await res.text(); + return (text ? JSON.parse(text) : {}) as T; + } + + /** + * POST JSON without throwing: returns `{ ok, status, error }` so best-effort + * writes (e.g. per-trace assessments) can be collected and reported without + * aborting the run. + */ + async postResult(path: string, body: unknown): Promise { + try { + const res = await fetch(`${this.baseUrl}${path}`, { + method: "POST", + headers: this.headers(), + body: JSON.stringify(body), + }); + if (!res.ok) { + const text = await res.text().catch(() => ""); + return { ok: false, status: res.status, error: text.slice(0, 500) }; + } + return { ok: true }; + } catch (err) { + return { + ok: false, + error: err instanceof Error ? err.message : String(err), + }; + } + } + + /** + * OpenAI-compatible base URL for Databricks Model Serving, used as the judge's + * `OPENAI_BASE_URL`. Same workspace host + token as the MLflow REST calls. + */ + servingEndpointsUrl(): string { + return `${this.baseUrl}/serving-endpoints`; + } +} diff --git a/packages/appkit/src/connectors/mlflow/index.ts b/packages/appkit/src/connectors/mlflow/index.ts new file mode 100644 index 00000000..62506288 --- /dev/null +++ b/packages/appkit/src/connectors/mlflow/index.ts @@ -0,0 +1,6 @@ +export { + type DatabricksAuth, + type ResolveDatabricksAuthOptions, + resolveDatabricksAuth, +} from "./auth"; +export { MlflowClient, normalizeHost, type PostResult } from "./client"; diff --git a/packages/appkit/src/connectors/mlflow/tests/client.test.ts b/packages/appkit/src/connectors/mlflow/tests/client.test.ts new file mode 100644 index 00000000..32c39609 --- /dev/null +++ b/packages/appkit/src/connectors/mlflow/tests/client.test.ts @@ -0,0 +1,82 @@ +import { afterEach, describe, expect, test, vi } from "vitest"; +import { MlflowClient, normalizeHost } from "../client"; + +describe("normalizeHost", () => { + test("adds https:// when missing and strips trailing slashes", () => { + expect(normalizeHost("workspace.cloud.databricks.com")).toBe( + "https://workspace.cloud.databricks.com", + ); + expect(normalizeHost("https://host.com/")).toBe("https://host.com"); + expect(normalizeHost("http://host.com//")).toBe("http://host.com"); + }); +}); + +describe("MlflowClient", () => { + afterEach(() => { + vi.unstubAllGlobals(); + }); + + test("normalizes host once and derives the serving-endpoints URL", () => { + const client = new MlflowClient("host.databricks.com", "tok"); + expect(client.baseUrl).toBe("https://host.databricks.com"); + expect(client.servingEndpointsUrl()).toBe( + "https://host.databricks.com/serving-endpoints", + ); + }); + + test("post() sends bearer auth to the normalized URL and parses JSON", async () => { + const fetchMock = vi.fn( + async (_url: string, _init: RequestInit) => + new Response(JSON.stringify({ ok: 1 })), + ); + vi.stubGlobal("fetch", fetchMock); + + const client = new MlflowClient("host.com", "secret"); + const body = await client.post("/api/2.0/mlflow/runs/create", { a: 1 }); + + expect(body).toEqual({ ok: 1 }); + const [url, init] = fetchMock.mock.calls[0]; + expect(url).toBe("https://host.com/api/2.0/mlflow/runs/create"); + expect(init.method).toBe("POST"); + expect((init.headers as Record).authorization).toBe( + "Bearer secret", + ); + }); + + test("post() throws with status + body on a non-2xx response", async () => { + vi.stubGlobal( + "fetch", + vi.fn(async () => new Response("boom", { status: 400 })), + ); + const client = new MlflowClient("host.com", "t"); + await expect(client.post("/p", {})).rejects.toThrow(/400 boom/); + }); + + test("postResult() returns a structured failure instead of throwing", async () => { + vi.stubGlobal( + "fetch", + vi.fn(async () => new Response("nope", { status: 403 })), + ); + const client = new MlflowClient("host.com", "t"); + expect(await client.postResult("/p", {})).toEqual({ + ok: false, + status: 403, + error: "nope", + }); + }); + + test("postResult() reports ok on success and network errors without throwing", async () => { + const fetchMock = vi + .fn() + .mockResolvedValueOnce(new Response(null, { status: 200 })) + .mockRejectedValueOnce(new Error("ECONNREFUSED")); + vi.stubGlobal("fetch", fetchMock); + + const client = new MlflowClient("host.com", "t"); + expect(await client.postResult("/p", {})).toEqual({ ok: true }); + expect(await client.postResult("/p", {})).toEqual({ + ok: false, + error: "ECONNREFUSED", + }); + }); +}); diff --git a/packages/appkit/src/evals/index.ts b/packages/appkit/src/evals/index.ts index bcfecfe0..506d05cb 100644 --- a/packages/appkit/src/evals/index.ts +++ b/packages/appkit/src/evals/index.ts @@ -1,3 +1,11 @@ +export { + type DatabricksAuth, + MlflowClient, + normalizeHost, + type PostResult, + type ResolveDatabricksAuthOptions, + resolveDatabricksAuth, +} from "../connectors/mlflow"; export { defineEval, defineEvalConfig } from "./define-eval"; export { type DiscoveredEval, discoverEvalFiles } from "./discover"; export { createHttpDriver, type HttpDriverOptions } from "./http-driver"; @@ -11,7 +19,6 @@ export { equals, includes, matches } from "./matchers"; export { type Assessment, buildAssessments, - type MlflowReportOptions, type ReportOutcome, reportToMlflow, } from "./mlflow-report"; diff --git a/packages/appkit/src/evals/judge.ts b/packages/appkit/src/evals/judge.ts index 7588d997..1f640af6 100644 --- a/packages/appkit/src/evals/judge.ts +++ b/packages/appkit/src/evals/judge.ts @@ -1,4 +1,4 @@ -import { normalizeHost } from "./mlflow-rest"; +import type { MlflowClient } from "../connectors/mlflow"; /** * LLM-as-judge scoring via the `autoevals` library (the same scorers eve uses), @@ -17,8 +17,8 @@ let mod: AutoEvals | undefined; let enabled = false; export interface JudgeConfig { - /** Databricks host (scheme optional). */ - host: string; + /** Client for the workspace hosting the judge serving endpoint. */ + client: MlflowClient; /** Bearer token for the serving endpoint. */ token: string; /** Serving endpoint name used as the judge model. */ @@ -39,7 +39,7 @@ export interface JudgeScore { export async function configureJudge(config: JudgeConfig): Promise { try { mod = await import("autoevals"); - process.env.OPENAI_BASE_URL = `${normalizeHost(config.host)}/serving-endpoints`; + process.env.OPENAI_BASE_URL = config.client.servingEndpointsUrl(); process.env.OPENAI_API_KEY = config.token; mod.init({ defaultModel: config.model }); enabled = true; @@ -67,7 +67,7 @@ export function toJudgeScore(s: { function ensure(): AutoEvals { if (!enabled || !mod) { throw new Error( - "LLM judge is not configured. Set --judge-model (and DATABRICKS_HOST/DATABRICKS_TOKEN) to use t.judge.*", + "LLM judge is not configured. Pass --judge-model and authenticate via --profile (or DATABRICKS_HOST/DATABRICKS_TOKEN) to use t.judge.*", ); } return mod; diff --git a/packages/appkit/src/evals/mlflow-report.ts b/packages/appkit/src/evals/mlflow-report.ts index 2136782c..859ffaf9 100644 --- a/packages/appkit/src/evals/mlflow-report.ts +++ b/packages/appkit/src/evals/mlflow-report.ts @@ -1,4 +1,4 @@ -import { normalizeHost } from "./mlflow-rest"; +import type { MlflowClient } from "../connectors/mlflow"; import type { EvalResult } from "./types"; /** A Feedback assessment in the MLflow REST proto-JSON shape. */ @@ -11,13 +11,6 @@ export interface Assessment { metadata?: Record; } -export interface MlflowReportOptions { - /** Databricks workspace host (scheme optional — normalized). */ - host: string; - /** Bearer token for the MLflow REST API. */ - token: string; -} - export interface ReportOutcome { written: number; skipped: number; @@ -79,34 +72,9 @@ export function buildAssessments(result: EvalResult): Assessment[] { return out; } -async function postAssessment( - host: string, - token: string, - assessment: Assessment, -): Promise<{ ok: boolean; status?: number; error?: string }> { - const url = `${normalizeHost(host)}/api/3.0/mlflow/traces/${encodeURIComponent( - assessment.trace_id, - )}/assessments`; - try { - const res = await fetch(url, { - method: "POST", - headers: { - "content-type": "application/json", - authorization: `Bearer ${token}`, - }, - body: JSON.stringify({ assessment }), - }); - if (!res.ok) { - const text = await res.text().catch(() => ""); - return { ok: false, status: res.status, error: text.slice(0, 500) }; - } - return { ok: true }; - } catch (err) { - return { - ok: false, - error: err instanceof Error ? err.message : String(err), - }; - } +/** REST path for writing a Feedback assessment onto a trace. */ +function assessmentPath(traceId: string): string { + return `/api/3.0/mlflow/traces/${encodeURIComponent(traceId)}/assessments`; } /** @@ -114,8 +82,8 @@ async function postAssessment( * API. Never throws — failures are collected so the run still reports. */ export async function reportToMlflow( + client: MlflowClient, results: EvalResult[], - options: MlflowReportOptions, ): Promise { const outcome: ReportOutcome = { written: 0, skipped: 0, failures: [] }; for (const result of results) { @@ -125,7 +93,9 @@ export async function reportToMlflow( continue; } for (const assessment of assessments) { - const res = await postAssessment(options.host, options.token, assessment); + const res = await client.postResult(assessmentPath(assessment.trace_id), { + assessment, + }); if (res.ok) { outcome.written++; } else { diff --git a/packages/appkit/src/evals/mlflow-rest.ts b/packages/appkit/src/evals/mlflow-rest.ts deleted file mode 100644 index a89912d2..00000000 --- a/packages/appkit/src/evals/mlflow-rest.ts +++ /dev/null @@ -1,39 +0,0 @@ -/** Shared helpers for talking to the Databricks/MLflow REST API. */ - -export interface MlflowRestOptions { - /** Databricks workspace host (scheme optional — normalized). */ - host: string; - /** Bearer token for the MLflow REST API. */ - token: string; -} - -/** Ensure the host has a scheme (Databricks env often lacks `https://`). */ -export function normalizeHost(raw: string): string { - const h = raw.trim().replace(/\/+$/, ""); - return /^https?:\/\//i.test(h) ? h : `https://${h}`; -} - -/** - * POST JSON to an MLflow REST endpoint. Returns the parsed JSON body, or throws - * with the status + response text so callers can surface a precise error. - */ -export async function mlflowPost( - options: MlflowRestOptions, - path: string, - body: unknown, -): Promise { - const res = await fetch(`${normalizeHost(options.host)}${path}`, { - method: "POST", - headers: { - "content-type": "application/json", - authorization: `Bearer ${options.token}`, - }, - body: JSON.stringify(body), - }); - if (!res.ok) { - const text = await res.text().catch(() => ""); - throw new Error(`${path} -> ${res.status} ${text.slice(0, 500)}`); - } - const text = await res.text(); - return (text ? JSON.parse(text) : {}) as T; -} diff --git a/packages/appkit/src/evals/mlflow-run.ts b/packages/appkit/src/evals/mlflow-run.ts index 4253bc83..dadcfc38 100644 --- a/packages/appkit/src/evals/mlflow-run.ts +++ b/packages/appkit/src/evals/mlflow-run.ts @@ -1,4 +1,4 @@ -import { type MlflowRestOptions, mlflowPost } from "./mlflow-rest"; +import type { MlflowClient } from "../connectors/mlflow"; import type { EvalResult } from "./types"; /** Run tag value that makes a run appear under the experiment's "Evaluation runs". */ @@ -28,14 +28,14 @@ interface MlflowMetric { * `mlflow.sourceRun` trace metadata and log results before finishing. */ export async function createEvalRun( - options: MlflowRestOptions & { + client: MlflowClient, + options: { experimentId: string; runName?: string; startTime: number; }, ): Promise { - const created = await mlflowPost( - options, + const created = await client.post( "/api/2.0/mlflow/runs/create", { experiment_id: options.experimentId, @@ -89,7 +89,8 @@ export interface FinishOutcome { * or it would be left stuck in RUNNING forever. */ export async function finishEvalRun( - options: MlflowRestOptions & { + client: MlflowClient, + options: { runId: string; results: EvalResult[]; endTime: number; @@ -100,7 +101,7 @@ export async function finishEvalRun( const metrics = aggregateMetrics(options.results, options.endTime); if (metrics.length) { try { - await mlflowPost(options, "/api/2.0/mlflow/runs/log-batch", { + await client.post("/api/2.0/mlflow/runs/log-batch", { run_id: options.runId, metrics, }); @@ -110,7 +111,7 @@ export async function finishEvalRun( } try { - await mlflowPost(options, "/api/2.0/mlflow/runs/update", { + await client.post("/api/2.0/mlflow/runs/update", { run_id: options.runId, status: "FINISHED", end_time: options.endTime, diff --git a/packages/appkit/src/evals/run-evals.ts b/packages/appkit/src/evals/run-evals.ts index 2a531add..f31dc38a 100644 --- a/packages/appkit/src/evals/run-evals.ts +++ b/packages/appkit/src/evals/run-evals.ts @@ -1,4 +1,5 @@ import { pathToFileURL } from "node:url"; +import { MlflowClient } from "../connectors/mlflow"; import { discoverEvalFiles } from "./discover"; import { createHttpDriver } from "./http-driver"; import { configureJudge } from "./judge"; @@ -117,15 +118,22 @@ export async function runEvalsInDir( emit({ type: "discovered", total }); if (options.judge) { - await configureJudge(options.judge); + await configureJudge({ + client: new MlflowClient(options.judge.host, options.judge.token), + token: options.judge.token, + model: options.judge.model, + }); } // Create the MLflow evaluation run up front so each eval's trace can be - // linked to it as it runs. + // linked to it as it runs. One client is shared by run create/finish and the + // per-trace assessment writes. let runId: string | undefined; + let mlflowClient: MlflowClient | undefined; if (options.mlflow) { - runId = await createEvalRun({ - ...options.mlflow, + mlflowClient = new MlflowClient(options.mlflow.host, options.mlflow.token); + runId = await createEvalRun(mlflowClient, { + experimentId: options.mlflow.experimentId, runName: `appkit-eval ${new Date(now).toISOString()}`, startTime: now, }); @@ -161,10 +169,9 @@ export async function runEvalsInDir( emit({ type: "result", result, index, total }); } - if (options.mlflow && runId) { - const report = await reportToMlflow(results, options.mlflow); - const finish = await finishEvalRun({ - ...options.mlflow, + if (mlflowClient && runId) { + const report = await reportToMlflow(mlflowClient, results); + const finish = await finishEvalRun(mlflowClient, { runId, results, endTime: options.now ?? Date.now(), diff --git a/packages/appkit/src/plugins/agents/mlflow.ts b/packages/appkit/src/plugins/agents/mlflow.ts index 6c4a310e..aad6b41d 100644 --- a/packages/appkit/src/plugins/agents/mlflow.ts +++ b/packages/appkit/src/plugins/agents/mlflow.ts @@ -1,4 +1,5 @@ import type { LiveSpan } from "mlflow-tracing"; +import { normalizeHost } from "../../connectors/mlflow"; import { createLogger } from "../../logging/logger"; const logger = createLogger("agents"); @@ -25,7 +26,7 @@ function experimentId(): string | undefined { function normalizedDatabricksHost(): string | undefined { const raw = process.env.DATABRICKS_HOST?.trim(); if (!raw) return undefined; - return /^https?:\/\//i.test(raw) ? raw : `https://${raw}`; + return normalizeHost(raw); } /** diff --git a/packages/shared/src/cli/commands/agent/eval.ts b/packages/shared/src/cli/commands/agent/eval.ts index fa327567..41390890 100644 --- a/packages/shared/src/cli/commands/agent/eval.ts +++ b/packages/shared/src/cli/commands/agent/eval.ts @@ -31,6 +31,11 @@ interface EvalRunner { judge?: { host: string; token: string; model: string }; onEvent?: (event: EvalProgress) => void; }): Promise; + resolveDatabricksAuth(opts: { + profile?: string; + host?: string; + token?: string; + }): Promise<{ host: string; token: string } | undefined>; evalGlyph(result: unknown): string; formatEvalDetail(result: unknown): string[]; formatSummaryLine(results: unknown[]): string; @@ -70,6 +75,7 @@ interface EvalOptions { strict?: boolean; root?: string; header?: string[]; + profile?: string; databricksHost?: string; databricksToken?: string; experiment?: string; @@ -82,11 +88,19 @@ async function runAgentEval( ): Promise { const runner = await loadRunner(); - // Create a native MLflow "Evaluation run" when Databricks creds + an - // experiment are available (traces live in the app; the run + scores are - // driven from here, so this side needs creds). - const host = opts.databricksHost ?? process.env.DATABRICKS_HOST; - const token = opts.databricksToken ?? process.env.DATABRICKS_TOKEN; + // Resolve Databricks host + bearer the AppKit-native way: an explicit + // host/token (or DATABRICKS_* env) wins; otherwise the SDK mints an OAuth + // token from the CLI profile — so no hand-set PAT is required. + const auth = await runner.resolveDatabricksAuth({ + profile: opts.profile ?? process.env.DATABRICKS_CONFIG_PROFILE, + host: opts.databricksHost ?? process.env.DATABRICKS_HOST, + token: opts.databricksToken ?? process.env.DATABRICKS_TOKEN, + }); + const host = auth?.host; + const token = auth?.token; + + // Create a native MLflow "Evaluation run" when creds + an experiment are + // available (traces live in the app; the run + scores are driven from here). const experimentId = opts.experiment ?? process.env.MLFLOW_EXPERIMENT_ID; const mlflow = host && token && experimentId ? { host, token, experimentId } : undefined; @@ -158,8 +172,8 @@ async function runAgentEval( } } else { console.log( - "\nMLflow evaluation run skipped — set DATABRICKS_HOST + DATABRICKS_TOKEN" + - " + MLFLOW_EXPERIMENT_ID (or the matching flags) to create one.", + "\nMLflow evaluation run skipped — pass --experiment (or set" + + " MLFLOW_EXPERIMENT_ID) plus --profile/--databricks-host to create one.", ); } @@ -186,6 +200,10 @@ export const agentEvalCommand = new Command("eval") "--header ", "Extra request header as 'Key: value' (repeatable)", ) + .option( + "--profile ", + "Databricks CLI profile to authenticate with via OAuth (default: DATABRICKS_CONFIG_PROFILE)", + ) .option( "--databricks-host ", "Databricks host for writing MLflow assessments (default: DATABRICKS_HOST)",