diff --git a/apps/dev-playground/config/agents/query/evals/dataset.eval.ts b/apps/dev-playground/config/agents/query/evals/dataset.eval.ts new file mode 100644 index 00000000..98cdd9d3 --- /dev/null +++ b/apps/dev-playground/config/agents/query/evals/dataset.eval.ts @@ -0,0 +1,53 @@ +import { defineEval, isJudgeConfigured } from "@databricks/appkit/beta"; + +/** + * Dataset-driven eval: runs once per row of a Databricks managed evaluation + * dataset (a Unity Catalog table with `inputs`/`expectations` columns). The + * runner binds each row to `t.input`/`t.expected`. + * + * Run it (reading the dataset needs a workspace client + warehouse; judging the + * guidelines needs a judge model): + * appkit agent eval dataset --root apps/dev-playground --url http://localhost:8000 \ + * --profile --warehouse --judge-model + * + * Row shape produced by the MLflow managed-dataset UI: + * inputs {"messages":[{"role":"user","content":"..."}]} + * expectations {"guidelines":{"value":["...","..."]}} (optional) + */ + +/** Pull the last user message out of an MLflow `{messages:[...]}` input. */ +function userMessage(input: Record): string { + const messages = Array.isArray(input.messages) + ? (input.messages as Array<{ role?: string; content?: string }>) + : []; + const last = [...messages].reverse().find((m) => m.role === "user"); + return last?.content ?? ""; +} + +/** Read `expectations.guidelines` — the UI wraps the array as `{value: [...]}`. */ +function guidelines(expected: Record | undefined): string[] { + const g = (expected?.guidelines as { value?: unknown } | undefined)?.value; + return Array.isArray(g) ? g.map(String) : []; +} + +export default defineEval({ + description: "Query agent satisfies each dataset row's guidelines", + // Point at your own managed evaluation dataset (catalog.schema.table). + dataset: { table: "main.mario.appkit_eval_dataset" }, + async test(t) { + // One turn per row. For a multi-turn conversation, call `t.send` again + // (same thread); to start an independent turn in the same test, `t.reset()`. + await t.send(userMessage(t.input)); + t.succeeded(); + + // Each guideline is judged against the reply — gate by default, so a miss + // fails the eval (chain `.soft()` to only track it). Skipped cleanly when no + // judge model is configured, so the eval still exercises the dataset read + + // drive path without one. + if (isJudgeConfigured()) { + for (const guideline of guidelines(t.expected)) { + (await t.judge.closedQA(guideline)).atLeast(0.5); + } + } + }, +}); diff --git a/packages/appkit/src/connectors/mlflow/auth.ts b/packages/appkit/src/connectors/mlflow/auth.ts index adbe6d63..d00a121c 100644 --- a/packages/appkit/src/connectors/mlflow/auth.ts +++ b/packages/appkit/src/connectors/mlflow/auth.ts @@ -34,9 +34,8 @@ export async function resolveDatabricksAuth( } try { - const client = new WorkspaceClient( - options.profile ? { profile: options.profile } : {}, - ); + const client = resolveWorkspaceClient(options); + if (!client) return undefined; const headers = new Headers(); // Mints the OAuth access token (or reuses a PAT from the profile) and adds // an `Authorization: Bearer ` header — the same call the connectors @@ -53,3 +52,29 @@ export async function resolveDatabricksAuth( return undefined; } } + +/** + * Construct a Databricks `WorkspaceClient` for the eval runner — the object the + * SDK-backed connectors (e.g. `SQLWarehouseConnector`) take. An explicit + * host+token builds a PAT client; otherwise the profile (or ambient config) is + * used and the SDK resolves credentials, minting OAuth as needed. Returns + * `undefined` if construction throws (missing/invalid config). + */ +export function resolveWorkspaceClient( + options: ResolveDatabricksAuthOptions = {}, +): WorkspaceClient | undefined { + try { + if (options.host && options.token) { + return new WorkspaceClient({ + host: options.host, + token: options.token, + authType: "pat", + }); + } + return new WorkspaceClient( + options.profile ? { profile: options.profile } : {}, + ); + } catch { + return undefined; + } +} diff --git a/packages/appkit/src/connectors/mlflow/index.ts b/packages/appkit/src/connectors/mlflow/index.ts index 62506288..db8351be 100644 --- a/packages/appkit/src/connectors/mlflow/index.ts +++ b/packages/appkit/src/connectors/mlflow/index.ts @@ -2,5 +2,6 @@ export { type DatabricksAuth, type ResolveDatabricksAuthOptions, resolveDatabricksAuth, + resolveWorkspaceClient, } from "./auth"; export { MlflowClient, normalizeHost, type PostResult } from "./client"; diff --git a/packages/appkit/src/evals/dataset.ts b/packages/appkit/src/evals/dataset.ts new file mode 100644 index 00000000..dc0cac13 --- /dev/null +++ b/packages/appkit/src/evals/dataset.ts @@ -0,0 +1,74 @@ +import type { WorkspaceClient } from "@databricks/sdk-experimental"; +import { SQLWarehouseConnector } from "../connectors"; + +/** + * One row of a managed evaluation dataset. `inputs` are the kwargs passed to the + * agent for the turn; `expectations` (when present) is the row's ground truth / + * guidelines. Mirrors the `{inputs, expectations}` shape of `mlflow.genai` + * datasets and of the Unity Catalog table backing a managed eval dataset. + */ +export interface DatasetRow { + inputs: Record; + expectations?: Record; +} + +export interface ReadEvalDatasetOptions { + /** Fully-qualified UC table: `catalog.schema.table`. */ + table: string; + /** SQL warehouse id to run the read against. */ + warehouseId: string; + /** Optional row cap. */ + limit?: number; +} + +/** A managed eval dataset is a UC table; only 3-level names are valid. */ +const UC_TABLE = /^[A-Za-z0-9_]+\.[A-Za-z0-9_]+\.[A-Za-z0-9_]+$/; + +/** Coerce a cell (already JSON-parsed by the connector for JSON columns) to a record. */ +function toRecord(value: unknown): Record | undefined { + if (value && typeof value === "object" && !Array.isArray(value)) { + return value as Record; + } + return undefined; +} + +/** + * Read a Databricks managed evaluation dataset (a Unity Catalog table with + * `inputs`/`expectations` columns) into rows, over the public SQL Statement + * Execution API. Reuses {@link SQLWarehouseConnector} for submit/poll/transform + * — its result transform already JSON-parses string columns into objects, so + * `inputs`/`expectations` come back as records whether the table stores them as + * JSON strings or structs. + * + * The Python `mlflow.genai.datasets` API needs a Spark session (no TS + * equivalent), so we read the backing table directly. + */ +export async function readEvalDataset( + client: WorkspaceClient, + options: ReadEvalDatasetOptions, +): Promise { + if (!UC_TABLE.test(options.table)) { + throw new Error( + `Invalid dataset table "${options.table}" — expected catalog.schema.table`, + ); + } + + const limit = + typeof options.limit === "number" + ? ` LIMIT ${Math.floor(options.limit)}` + : ""; + const connector = new SQLWarehouseConnector({}); + const response = await connector.executeStatement(client, { + warehouse_id: options.warehouseId, + statement: `SELECT inputs, expectations FROM ${options.table}${limit}`, + }); + + const rows = + (response.result as { data?: Array> } | undefined) + ?.data ?? []; + + return rows.map((row) => ({ + inputs: toRecord(row.inputs) ?? {}, + expectations: toRecord(row.expectations), + })); +} diff --git a/packages/appkit/src/evals/http-driver.ts b/packages/appkit/src/evals/http-driver.ts index 0803a6e3..f4d8e402 100644 --- a/packages/appkit/src/evals/http-driver.ts +++ b/packages/appkit/src/evals/http-driver.ts @@ -73,6 +73,9 @@ export function createHttpDriver(options: HttpDriverOptions): EvalDriver { let threadId: string | undefined; return { + reset(): void { + threadId = undefined; + }, async send(message: string): Promise { let res: Response; try { diff --git a/packages/appkit/src/evals/index.ts b/packages/appkit/src/evals/index.ts index 506d05cb..5f3258e3 100644 --- a/packages/appkit/src/evals/index.ts +++ b/packages/appkit/src/evals/index.ts @@ -5,7 +5,13 @@ export { type PostResult, type ResolveDatabricksAuthOptions, resolveDatabricksAuth, + resolveWorkspaceClient, } from "../connectors/mlflow"; +export { + type DatasetRow, + type ReadEvalDatasetOptions, + readEvalDataset, +} from "./dataset"; export { defineEval, defineEvalConfig } from "./define-eval"; export { type DiscoveredEval, discoverEvalFiles } from "./discover"; export { createHttpDriver, type HttpDriverOptions } from "./http-driver"; diff --git a/packages/appkit/src/evals/run-eval.ts b/packages/appkit/src/evals/run-eval.ts index e227af75..c0ec2a57 100644 --- a/packages/appkit/src/evals/run-eval.ts +++ b/packages/appkit/src/evals/run-eval.ts @@ -1,3 +1,4 @@ +import type { DatasetRow } from "./dataset"; import { judgeClosedQA, judgeCustom, judgeFactuality } from "./judge"; import type { AssertionHandle, @@ -27,6 +28,8 @@ export interface RunEvalOptions { driver: EvalDriver; /** When true, soft assertion failures also fail the eval. */ strict?: boolean; + /** Dataset row bound to `t.input`/`t.expected` for dataset-driven evals. */ + row?: DatasetRow; } /** @@ -70,7 +73,8 @@ export async function runEval( return handle; }, atLeast(threshold: number) { - result.severity = "soft"; + // Re-threshold the score; keep the current severity (gate unless the + // caller also chained `.soft()`). result.pass = (result.score ?? (result.pass ? 1 : 0)) >= threshold; return handle; }, @@ -78,14 +82,15 @@ export async function runEval( return handle; }; - // LLM-judge assertions are scored and soft by default; the caller chains - // `.atLeast(n)` to set the pass threshold or `.gate()` to promote. + // LLM-judge assertions are scored and gate by default (a miss fails the eval, + // like other assertions). The caller chains `.atLeast(n)` to change the pass + // threshold, or `.soft()` to demote to a tracked-only metric. const recordJudge = ( label: string, score: number, rationale?: string, ): AssertionHandle => - record(label, score >= DEFAULT_JUDGE_THRESHOLD, score, rationale).soft(); + record(label, score >= DEFAULT_JUDGE_THRESHOLD, score, rationale); const t: TestContext = { async send(message) { @@ -97,6 +102,9 @@ export async function runEval( lastSucceeded = r.succeeded; if (r.traceId) lastTraceId = r.traceId; }, + reset() { + options.driver.reset?.(); + }, get reply() { return reply; }, @@ -106,6 +114,12 @@ export async function runEval( get sessionId() { return sessionId; }, + get input() { + return options.row?.inputs ?? {}; + }, + get expected() { + return options.row?.expectations; + }, succeeded() { return record( "succeeded", diff --git a/packages/appkit/src/evals/run-evals.ts b/packages/appkit/src/evals/run-evals.ts index f31dc38a..c304e454 100644 --- a/packages/appkit/src/evals/run-evals.ts +++ b/packages/appkit/src/evals/run-evals.ts @@ -1,5 +1,7 @@ import { pathToFileURL } from "node:url"; +import type { WorkspaceClient } from "@databricks/sdk-experimental"; import { MlflowClient } from "../connectors/mlflow"; +import { type DatasetRow, readEvalDataset } from "./dataset"; import { discoverEvalFiles } from "./discover"; import { createHttpDriver } from "./http-driver"; import { configureJudge } from "./judge"; @@ -30,6 +32,13 @@ export interface RunEvalsOptions { * Databricks serving endpoint (`model`). */ judge?: { host: string; token: string; model: string }; + /** + * Workspace client used to read managed evaluation datasets (for evals that + * declare `dataset`). Required alongside {@link warehouseId} for those evals. + */ + workspaceClient?: WorkspaceClient; + /** SQL warehouse id used to read managed evaluation datasets. */ + warehouseId?: string; /** Wall-clock timestamp (ms) for run create/finish — pass `Date.now()`. */ now?: number; /** Progress callback, invoked as evals are discovered, started, and finished. */ @@ -141,32 +150,87 @@ export async function runEvalsInDir( } const results: EvalResult[] = []; + // `total` counts eval files, not dataset rows: row counts aren't known until + // each file loads. Per-row detail is carried in the result id (`[row i/n]`). for (let index = 0; index < discovered.length; index++) { const d = discovered[index]; const id = `${d.agent}/${d.id}`; - emit({ type: "start", id, index, total }); - let result: EvalResult; + let def: EvalDefinition | undefined; try { - const def = await loadEval(d.file); - const driver = createHttpDriver({ - baseUrl: options.baseUrl, - agent: def.agent ?? d.agent, - headers: options.headers, - mlflowRunId: runId, - }); - result = await runEval(def, { id, driver, strict: options.strict }); + def = await loadEval(d.file); } catch (err) { - result = { + emit({ type: "start", id, index, total }); + const result: EvalResult = { id, assertions: [], passed: false, error: err instanceof Error ? err.message : String(err), }; + results.push(result); + emit({ type: "result", result, index, total }); + continue; } - results.push(result); - emit({ type: "result", result, index, total }); + // Resolve dataset rows up front; a dataset eval with no rows still runs + // once with an empty row so a misconfiguration surfaces as a result. + let rows: Array = [undefined]; + let datasetError: string | undefined; + if (def.dataset) { + if (!options.workspaceClient || !options.warehouseId) { + datasetError = + "dataset eval requires a workspace client and warehouse (pass --warehouse)"; + } else { + try { + rows = await readEvalDataset(options.workspaceClient, { + table: def.dataset.table, + warehouseId: options.warehouseId, + limit: def.dataset.limit, + }); + if (rows.length === 0) rows = [undefined]; + } catch (err) { + datasetError = err instanceof Error ? err.message : String(err); + } + } + } + + for (let r = 0; r < rows.length; r++) { + const rowId = + def.dataset && rows.length > 1 + ? `${id} [row ${r + 1}/${rows.length}]` + : id; + emit({ type: "start", id: rowId, index, total }); + + let result: EvalResult; + if (datasetError) { + result = { + id: rowId, + assertions: [], + passed: false, + error: datasetError, + }; + } else { + // A fresh driver per row: each row is an independent conversation, so + // its thread must not carry over the previous row's history. (Multiple + // `t.send`s within one row still share the thread — that's the driver's + // per-instance behavior.) + const driver = createHttpDriver({ + baseUrl: options.baseUrl, + agent: def.agent ?? d.agent, + headers: options.headers, + mlflowRunId: runId, + }); + result = await runEval(def, { + id: rowId, + driver, + strict: options.strict, + row: rows[r], + }); + } + + results.push(result); + emit({ type: "result", result, index, total }); + } } if (mlflowClient && runId) { diff --git a/packages/appkit/src/evals/tests/dataset.test.ts b/packages/appkit/src/evals/tests/dataset.test.ts new file mode 100644 index 00000000..b6787cd8 --- /dev/null +++ b/packages/appkit/src/evals/tests/dataset.test.ts @@ -0,0 +1,92 @@ +import { beforeEach, describe, expect, test, vi } from "vitest"; + +// Mock the connectors barrel so readEvalDataset uses a fake SQLWarehouseConnector. +const executeStatement = vi.fn(); +vi.mock("../../connectors", () => ({ + SQLWarehouseConnector: class { + executeStatement = executeStatement; + }, +})); + +import { readEvalDataset } from "../dataset"; + +const client = {} as never; + +describe("readEvalDataset", () => { + beforeEach(() => { + executeStatement.mockReset(); + }); + + test("maps connector rows to {inputs, expectations}", async () => { + executeStatement.mockResolvedValue({ + result: { + data: [ + { + inputs: { query: "What is MLflow?" }, + expectations: { expected_facts: ["MLflow is an ML platform"] }, + }, + { inputs: { query: "Weather?" }, expectations: null }, + ], + }, + }); + + const rows = await readEvalDataset(client, { + table: "main.default.eval_ds", + warehouseId: "wh1", + }); + + expect(rows).toEqual([ + { + inputs: { query: "What is MLflow?" }, + expectations: { expected_facts: ["MLflow is an ML platform"] }, + }, + { inputs: { query: "Weather?" }, expectations: undefined }, + ]); + + // SELECT targets the table; no LIMIT when unset. + const [, input] = executeStatement.mock.calls[0]; + expect(input.warehouse_id).toBe("wh1"); + expect(input.statement).toBe( + "SELECT inputs, expectations FROM main.default.eval_ds", + ); + }); + + test("applies an integer LIMIT when provided", async () => { + executeStatement.mockResolvedValue({ result: { data: [] } }); + await readEvalDataset(client, { + table: "cat.sch.tbl", + warehouseId: "wh1", + limit: 5, + }); + expect(executeStatement.mock.calls[0][1].statement).toBe( + "SELECT inputs, expectations FROM cat.sch.tbl LIMIT 5", + ); + }); + + test("defaults a missing/non-object inputs cell to {}", async () => { + executeStatement.mockResolvedValue({ + result: { data: [{ inputs: null }, { inputs: "oops" }] }, + }); + const rows = await readEvalDataset(client, { + table: "cat.sch.tbl", + warehouseId: "wh1", + }); + expect(rows).toEqual([ + { inputs: {}, expectations: undefined }, + { inputs: {}, expectations: undefined }, + ]); + }); + + test("rejects a non-3-level table name without querying", async () => { + await expect( + readEvalDataset(client, { table: "schema.table", warehouseId: "wh1" }), + ).rejects.toThrow(/catalog\.schema\.table/); + await expect( + readEvalDataset(client, { + table: "cat.sch.tbl; DROP TABLE x", + warehouseId: "wh1", + }), + ).rejects.toThrow(/Invalid dataset table/); + expect(executeStatement).not.toHaveBeenCalled(); + }); +}); diff --git a/packages/appkit/src/evals/tests/run-eval.test.ts b/packages/appkit/src/evals/tests/run-eval.test.ts index abc78030..18970120 100644 --- a/packages/appkit/src/evals/tests/run-eval.test.ts +++ b/packages/appkit/src/evals/tests/run-eval.test.ts @@ -1,4 +1,4 @@ -import { describe, expect, test } from "vitest"; +import { describe, expect, test, vi } from "vitest"; import { defineEval } from "../define-eval"; import { includes } from "../matchers"; import { runEval } from "../run-eval"; @@ -114,4 +114,76 @@ describe("runEval", () => { expect(result.passed).toBe(false); expect(result.error).toBe("boom"); }); + + test("atLeast() gates by default — a below-threshold score fails the eval", async () => { + // A scored matcher standing in for a judge (which shares the same handle). + const scored = (score: number) => (): { pass: boolean; score: number } => ({ + pass: score >= 0.5, + score, + }); + const def = defineEval({ + async test(t) { + await t.send("q"); + t.check(t.reply, scored(0.2)).atLeast(0.5); // gate, below threshold + }, + }); + const result = await runEval(def, { + id: "gate", + driver: fakeDriver({ reply: "x" }), + }); + expect(result.passed).toBe(false); + expect(result.assertions[0].severity).toBe("gate"); + }); + + test("atLeast().soft() demotes so a below-threshold score only tracks", async () => { + const scored = (score: number) => (): { pass: boolean; score: number } => ({ + pass: score >= 0.5, + score, + }); + const def = defineEval({ + async test(t) { + await t.send("q"); + t.check(t.reply, scored(0.2)).atLeast(0.5).soft(); + }, + }); + const result = await runEval(def, { + id: "soft-judge", + driver: fakeDriver({ reply: "x" }), + }); + expect(result.passed).toBe(true); + expect(result.assertions[0].severity).toBe("soft"); + expect(result.assertions[0].pass).toBe(false); + }); + + test("t.reset() forwards to the driver to start a fresh conversation", async () => { + const reset = vi.fn(); + const driver: EvalDriver = { + send: async () => ({ reply: "", toolCalls: [], succeeded: true }), + reset, + }; + const def = defineEval({ + async test(t) { + await t.send("first"); + t.reset(); + await t.send("second"); + }, + }); + await runEval(def, { id: "reset", driver }); + expect(reset).toHaveBeenCalledTimes(1); + }); + + test("t.reset() is a no-op when the driver has no reset", async () => { + const def = defineEval({ + async test(t) { + t.reset(); + await t.send("hi"); + }, + }); + // fakeDriver has no reset(); this must not throw. + const result = await runEval(def, { + id: "no-reset", + driver: fakeDriver({}), + }); + expect(result.passed).toBe(true); + }); }); diff --git a/packages/appkit/src/evals/types.ts b/packages/appkit/src/evals/types.ts index 16a7d3be..03860fd8 100644 --- a/packages/appkit/src/evals/types.ts +++ b/packages/appkit/src/evals/types.ts @@ -42,7 +42,11 @@ export interface AssertionHandle { gate(): AssertionHandle; /** Demote to a tracked metric — doesn't fail unless running with `strict`. */ soft(): AssertionHandle; - /** Soft assertion that passes only when the score is at least `threshold`. */ + /** + * Set the pass threshold for a scored assertion: it passes only when the + * score is at least `threshold`. Keeps the current severity (gate unless also + * chained with `.soft()`). + */ atLeast(threshold: number): AssertionHandle; } @@ -66,18 +70,39 @@ export interface DriveResult { */ export interface EvalDriver { send(message: string): Promise; + /** + * Drop the current conversation so the next `send` starts a fresh thread. + * Optional: drivers without a session concept omit it. + */ + reset?(): void; } /** The `t` context passed to an eval's `test` function. */ export interface TestContext { /** Send a user message to the agent and capture its response. */ send(message: string): Promise; + /** + * Start a fresh conversation: the next `send` opens a new thread with no + * history. Use to run several independent one-shot checks in one test. + * Consecutive `send`s (without a `reset`) stay in one multi-turn conversation. + */ + reset(): void; /** The last assistant reply. */ readonly reply: string; /** Tools called during the last turn. */ readonly toolCalls: string[]; /** The current session/thread id, if any. */ readonly sessionId: string | undefined; + /** + * The current dataset row's `inputs` when the eval is dataset-driven (see + * {@link EvalDefinition.dataset}); `{}` for a plain single-run eval. + */ + readonly input: Record; + /** + * The current dataset row's `expectations` (ground truth / guidelines), or + * `undefined` when the row has none or the eval isn't dataset-driven. + */ + readonly expected: Record | undefined; /** Assert the last turn completed successfully (gate by default). */ succeeded(): AssertionHandle; /** Assert a tool was called during the run (gate by default). */ @@ -86,9 +111,10 @@ export interface TestContext { check(value: string, matcher: Matcher): AssertionHandle; /** * LLM-as-judge scoring of the last reply (via autoevals → a Databricks judge - * model). Each returns a scored, soft-by-default assertion; chain `.atLeast(n)` - * to set the pass threshold or `.gate()` to make it a hard gate. Requires the - * judge to be configured (`--judge-model`). + * model). Each returns a scored assertion that gates by default (a miss fails + * the eval); chain `.atLeast(n)` to change the pass threshold or `.soft()` to + * demote to a tracked-only metric. Requires the judge to be configured + * (`--judge-model`). */ judge: { /** Score factuality of the reply against an expected reference. */ @@ -119,6 +145,13 @@ export interface EvalDefinition { tags?: string[]; /** Per-eval timeout. */ timeoutMs?: number; + /** + * Run this eval once per row of a Databricks managed evaluation dataset (a + * Unity Catalog `catalog.schema.table` with `inputs`/`expectations` columns). + * Each row is bound to `t.input`/`t.expected`. Requires the runner to have a + * workspace client + warehouse (`--warehouse`). Omit for a single-run eval. + */ + dataset?: { table: string; limit?: number }; /** The eval body: drive the agent and assert on its behavior. */ test(t: TestContext): Promise | void; } diff --git a/packages/shared/src/cli/commands/agent/eval.ts b/packages/shared/src/cli/commands/agent/eval.ts index 41390890..3261d51c 100644 --- a/packages/shared/src/cli/commands/agent/eval.ts +++ b/packages/shared/src/cli/commands/agent/eval.ts @@ -29,6 +29,8 @@ interface EvalRunner { headers?: Record; mlflow?: { host: string; token: string; experimentId: string }; judge?: { host: string; token: string; model: string }; + workspaceClient?: unknown; + warehouseId?: string; onEvent?: (event: EvalProgress) => void; }): Promise; resolveDatabricksAuth(opts: { @@ -36,6 +38,11 @@ interface EvalRunner { host?: string; token?: string; }): Promise<{ host: string; token: string } | undefined>; + resolveWorkspaceClient(opts: { + profile?: string; + host?: string; + token?: string; + }): unknown; evalGlyph(result: unknown): string; formatEvalDetail(result: unknown): string[]; formatSummaryLine(results: unknown[]): string; @@ -80,6 +87,7 @@ interface EvalOptions { databricksToken?: string; experiment?: string; judgeModel?: string; + warehouse?: string; } async function runAgentEval( @@ -112,6 +120,15 @@ async function runAgentEval( ? { host, token, model: judgeModel } : undefined; + // Managed-dataset reads: a workspace client (same profile/host/token) + a SQL + // warehouse. Only needed by evals that declare `dataset`. + const warehouseId = opts.warehouse ?? process.env.DATABRICKS_WAREHOUSE_ID; + const workspaceClient = runner.resolveWorkspaceClient({ + profile: opts.profile ?? process.env.DATABRICKS_CONFIG_PROFILE, + host: opts.databricksHost ?? process.env.DATABRICKS_HOST, + token: opts.databricksToken ?? process.env.DATABRICKS_TOKEN, + }); + // Stream progress as evals run, instead of going silent until the end. const onEvent = (event: EvalProgress): void => { switch (event.type) { @@ -146,6 +163,8 @@ async function runAgentEval( headers: opts.header ? parseHeaders(opts.header) : undefined, mlflow, judge, + workspaceClient, + warehouseId, onEvent, }); console.log(`\n${runner.formatSummaryLine(summary.results)}`); @@ -216,6 +235,10 @@ export const agentEvalCommand = new Command("eval") "--experiment ", "MLflow experiment id for the evaluation run (default: MLFLOW_EXPERIMENT_ID)", ) + .option( + "--warehouse ", + "SQL warehouse id for reading managed evaluation datasets (default: DATABRICKS_WAREHOUSE_ID)", + ) .option( "--judge-model ", "Databricks serving endpoint to use as the LLM judge for t.judge.* (default: APPKIT_JUDGE_MODEL)",