feat(appkit): eval suite features — concurrency, retries, reporters (stack 4/5)

apps/dev-playground/config/agents/query/evals/dataset.eval.ts

@@ -1,4 +1,8 @@
-import { defineEval, isJudgeConfigured } from "@databricks/appkit/beta";
+import {
+  defineEval,
+  isJudgeConfigured,
+  userTurns,
+} from "@databricks/appkit/beta";
 
 /**
  * Dataset-driven eval: runs once per row of a Databricks managed evaluation
@@ -13,17 +17,12 @@ import { defineEval, isJudgeConfigured } from "@databricks/appkit/beta";
  * Row shape produced by the MLflow managed-dataset UI:
  *   inputs        {"messages":[{"role":"user","content":"..."}]}
  *   expectations  {"guidelines":{"value":["...","..."]}}   (optional)
+ *
+ * A row's `messages` can be a full multi-turn conversation. We replay each USER
+ * turn in order against one shared thread (below); interleaved assistant turns
+ * in the row are ignored — the agent generates its own responses.
  */
 
-/** Pull the last user message out of an MLflow `{messages:[...]}` input. */
-function userMessage(input: Record<string, unknown>): string {
-  const messages = Array.isArray(input.messages)
-    ? (input.messages as Array<{ role?: string; content?: string }>)
-    : [];
-  const last = [...messages].reverse().find((m) => m.role === "user");
-  return last?.content ?? "";
-}
-
 /** Read `expectations.guidelines` — the UI wraps the array as `{value: [...]}`. */
 function guidelines(expected: Record<string, unknown> | undefined): string[] {
   const g = (expected?.guidelines as { value?: unknown } | undefined)?.value;
@@ -35,9 +34,12 @@ export default defineEval({
   // Point at your own managed evaluation dataset (catalog.schema.table).
   dataset: { table: "main.mario.appkit_eval_dataset" },
   async test(t) {
-    // One turn per row. For a multi-turn conversation, call `t.send` again
-    // (same thread); to start an independent turn in the same test, `t.reset()`.
-    await t.send(userMessage(t.input));
+    // Replay every user turn in the row against one thread, so the agent sees
+    // the accumulating conversation. A single-user-turn row sends once. The
+    // runner gives each row a fresh driver, so rows don't bleed into each other.
+    for (const turn of userTurns(t.input)) {
+      await t.send(turn);
+    }
     t.succeeded();
 
     // Each guideline is judged against the reply — gate by default, so a miss

packages/appkit/src/evals/dataset.ts

@@ -21,6 +21,25 @@ export interface ReadEvalDatasetOptions {
   limit?: number;
 }
 
+/**
+ * Extract every user-message content, in order, from an MLflow
+ * `{"messages":[{"role":"user","content":"..."}]}` input. A dataset row can
+ * carry a full multi-turn conversation; replaying these against one thread (one
+ * `t.send` per returned string) lets the agent see the accumulating history.
+ *
+ * Only `role === "user"` turns are returned — any interleaved `assistant`/
+ * `system` messages in the row are ignored, since the agent generates its own
+ * responses; you never inject the dataset's assistant turns. A single-user-turn
+ * row yields a one-element array (backward compatible); a row with no `messages`
+ * yields `[]`.
+ */
+export function userTurns(input: Record<string, unknown>): string[] {
+  const messages = Array.isArray(input.messages)
+    ? (input.messages as Array<{ role?: string; content?: string }>)
+    : [];
+  return messages.filter((m) => m.role === "user").map((m) => m.content ?? "");
+}
+
 /** A managed eval dataset is a UC table; only 3-level names are valid. */
 const UC_TABLE = /^[A-Za-z0-9_]+\.[A-Za-z0-9_]+\.[A-Za-z0-9_]+$/;
 

packages/appkit/src/evals/discover.ts

@@ -11,6 +11,14 @@ export interface DiscoveredEval {
   agent: string;
 }
 
+/** A per-agent `evals.config.ts` found under `config/agents/<agent>/evals/`. */
+export interface DiscoveredEvalConfig {
+  /** Absolute path to the `evals.config.ts` file. */
+  file: string;
+  /** The agent id whose evals this config applies to. */
+  agent: string;
+}
+
 function isDir(p: string): boolean {
   try {
     return statSync(p).isDirectory();
@@ -19,6 +27,14 @@ function isDir(p: string): boolean {
   }
 }
 
+function isFile(p: string): boolean {
+  try {
+    return statSync(p).isFile();
+  } catch {
+    return false;
+  }
+}
+
 /** Recursively collect `*.eval.ts` files (skips `evals.config.ts`). */
 function walkEvalFiles(dir: string): string[] {
   const out: string[] = [];
@@ -74,3 +90,30 @@ export function discoverEvalFiles(rootDir: string): DiscoveredEval[] {
     (a, b) => a.agent.localeCompare(b.agent) || a.id.localeCompare(b.id),
   );
 }
+
+/**
+ * Discover the per-agent `evals.config.ts` (from {@link defineEvalConfig}) at
+ * `<rootDir>/config/agents/<agent>/evals/evals.config.ts`. Config is per-agent:
+ * each agent's config applies only to that agent's evals. Agents without a
+ * config file are omitted. Returns a stable, sorted list.
+ */
+export function discoverEvalConfigs(rootDir: string): DiscoveredEvalConfig[] {
+  const agentsDir = path.join(rootDir, "config", "agents");
+  const out: DiscoveredEvalConfig[] = [];
+
+  let agents: string[];
+  try {
+    agents = readdirSync(agentsDir).filter((n) =>
+      isDir(path.join(agentsDir, n)),
+    );
+  } catch {
+    return out;
+  }
+
+  for (const agent of agents) {
+    const file = path.join(agentsDir, agent, "evals", "evals.config.ts");
+    if (isFile(file)) out.push({ file, agent });
+  }
+
+  return out.sort((a, b) => a.agent.localeCompare(b.agent));
+}

packages/appkit/src/evals/http-driver.ts

@@ -13,13 +13,30 @@ export interface HttpDriverOptions {
   mlflowRunId?: string;
 }
 
+/** A single captured tool call, deduped by `call_id ?? name`. */
+type ToolCall = { name: string; args: Record<string, unknown> };
+
+/** Parse a function-call `arguments` JSON string; `{}` on missing/invalid. */
+function parseArgs(raw: unknown): Record<string, unknown> {
+  if (typeof raw !== "string" || raw.trim() === "") return {};
+  try {
+    const parsed = JSON.parse(raw);
+    return parsed !== null &&
+      typeof parsed === "object" &&
+      !Array.isArray(parsed)
+      ? (parsed as Record<string, unknown>)
+      : {};
+  } catch {
+    return {};
+  }
+}
+
 /** Parse a single Responses-API SSE `data:` payload into the running totals. */
 function applyEvent(
   event: Record<string, unknown>,
   state: {
     reply: string;
-    toolCalls: string[];
-    seen: Set<string>;
+    toolCalls: Map<string, ToolCall>;
     ok: boolean;
     traceId?: string;
   },
@@ -38,13 +55,18 @@ function applyEvent(
     type === "response.output_item.done"
   ) {
     const item = event.item as
-      | { type?: string; name?: string; call_id?: string }
+      | { type?: string; name?: string; call_id?: string; arguments?: string }
       | undefined;
     if (item?.type === "function_call" && item.name) {
       const key = item.call_id ?? item.name;
-      if (!state.seen.has(key)) {
-        state.seen.add(key);
-        state.toolCalls.push(item.name);
+      const args = parseArgs(item.arguments);
+      const existing = state.toolCalls.get(key);
+      if (!existing) {
+        state.toolCalls.set(key, { name: item.name, args });
+      } else if (Object.keys(args).length > 0) {
+        // The initial `added` event may carry empty args while the later
+        // `done` carries the full arguments — keep the fuller set.
+        existing.args = args;
       }
     }
     return;
@@ -92,22 +114,27 @@ export function createHttpDriver(options: HttpDriverOptions): EvalDriver {
           }),
         });
       } catch {
-        return { reply: "", toolCalls: [], succeeded: false };
+        return {
+          reply: "",
+          toolCalls: [],
+          toolCallDetails: [],
+          succeeded: false,
+        };
       }
 
       if (!res.ok || !res.body) {
         return {
           reply: "",
           toolCalls: [],
+          toolCallDetails: [],
           succeeded: false,
           sessionId: threadId,
         };
       }
 
       const state = {
         reply: "",
-        toolCalls: [] as string[],
-        seen: new Set<string>(),
+        toolCalls: new Map<string, ToolCall>(),
         ok: true,
         traceId: undefined as string | undefined,
       };
@@ -139,9 +166,11 @@ export function createHttpDriver(options: HttpDriverOptions): EvalDriver {
         reader.releaseLock();
       }
 
+      const toolCallDetails = [...state.toolCalls.values()];
       return {
         reply: state.reply,
-        toolCalls: state.toolCalls,
+        toolCalls: toolCallDetails.map((c) => c.name),
+        toolCallDetails,
         succeeded: state.ok,
         sessionId: threadId,
         traceId: state.traceId,

packages/appkit/src/evals/index.ts

@@ -11,9 +11,15 @@ export {
   type DatasetRow,
   type ReadEvalDatasetOptions,
   readEvalDataset,
+  userTurns,
 } from "./dataset";
 export { defineEval, defineEvalConfig } from "./define-eval";
-export { type DiscoveredEval, discoverEvalFiles } from "./discover";
+export {
+  type DiscoveredEval,
+  type DiscoveredEvalConfig,
+  discoverEvalConfigs,
+  discoverEvalFiles,
+} from "./discover";
 export { createHttpDriver, type HttpDriverOptions } from "./http-driver";
 export {
   configureJudge,
@@ -34,6 +40,8 @@ export {
   formatEvalDetail,
   formatEvalHeadline,
   formatEvalResults,
+  formatResultsJson,
+  formatResultsJUnit,
   formatSummaryLine,
   summarize,
 } from "./report";
@@ -42,7 +50,9 @@ export {
   type EvalProgress,
   type EvalRunSummary,
   type RunEvalsOptions,
+  runBounded,
   runEvalsInDir,
+  runWithRetries,
 } from "./run-evals";
 export type {
   AssertionHandle,

packages/appkit/src/evals/report.ts

@@ -7,6 +7,8 @@ export interface EvalSummary {
   skipped: number;
   /** True when no eval failed (skips don't count as failures). */
   allPassed: boolean;
+  /** Fraction of scored (non-skipped) evals that passed, 0..1 (1 when none scored). */
+  passRate: number;
 }
 
 export function summarize(results: EvalResult[]): EvalSummary {
@@ -18,12 +20,14 @@ export function summarize(results: EvalResult[]): EvalSummary {
     else if (r.passed) passed++;
     else failed++;
   }
+  const scored = passed + failed;
   return {
     total: results.length,
     passed,
     failed,
     skipped,
     allPassed: failed === 0,
+    passRate: scored === 0 ? 1 : passed / scored,
   };
 }
 
@@ -74,3 +78,67 @@ export function formatEvalResults(results: EvalResult[]): string {
   lines.push(formatSummaryLine(results));
   return lines.join("\n");
 }
+
+/**
+ * Render results as a machine-readable JSON report (2-space indented):
+ * `{ summary: EvalSummary, results: EvalResult[] }`. Faithful to the types —
+ * every field present on a result round-trips.
+ */
+export function formatResultsJson(results: EvalResult[]): string {
+  return JSON.stringify({ summary: summarize(results), results }, null, 2);
+}
+
+/** Escape a value for use in XML text/attribute content. */
+function escapeXml(value: string): string {
+  return value
+    .replace(/&/g, "&")
+    .replace(/</g, "&lt;")
+    .replace(/>/g, "&gt;")
+    .replace(/"/g, "&quot;")
+    .replace(/'/g, "&apos;");
+}
+
+/** One-line reason a result failed: its error, else its failing gate labels. */
+function failureMessage(result: EvalResult): string {
+  if (result.error) return result.error;
+  const gates = result.assertions
+    .filter((a) => !a.pass && a.severity === "gate")
+    .map((a) => (a.detail ? `${a.label} — ${a.detail}` : a.label));
+  return gates.length ? gates.join("; ") : "eval failed";
+}
+
+/**
+ * Render results as JUnit XML for standard CI test reporters: a single
+ * `<testsuite name="appkit-agent-evals">` with one `<testcase>` per result.
+ * Failures carry a `<failure>` (error or failing-gate summary); skips a
+ * `<skipped>`. All attribute/text values are XML-escaped.
+ */
+export function formatResultsJUnit(results: EvalResult[]): string {
+  const s = summarize(results);
+  const lines: string[] = [];
+  lines.push('<?xml version="1.0" encoding="UTF-8"?>');
+  lines.push(
+    `<testsuite name="appkit-agent-evals" tests="${s.total}" failures="${s.failed}" skipped="${s.skipped}">`,
+  );
+  for (const r of results) {
+    const open = `  <testcase name="${escapeXml(r.id)}" classname="agent-eval"`;
+    if (r.skipped) {
+      lines.push(`${open}>`);
+      lines.push(
+        r.skipped.reason
+          ? `    <skipped message="${escapeXml(r.skipped.reason)}"/>`
+          : "    <skipped/>",
+      );
+      lines.push("  </testcase>");
+    } else if (!r.passed) {
+      const message = failureMessage(r);
+      lines.push(`${open}>`);
+      lines.push(`    <failure message="${escapeXml(message)}"/>`);
+      lines.push("  </testcase>");
+    } else {
+      lines.push(`${open}/>`);
+    }
+  }
+  lines.push("</testsuite>");
+  return lines.join("\n");
+}