feat(appkit): agent eval framework, judge, mlflow connector (stack 2/5)

apps/dev-playground/config/agents/query/evals/judge.eval.ts

@@ -0,0 +1,25 @@
+import { defineEval } from "@databricks/appkit/beta";
+
+/**
+ * LLM-as-judge eval: scores the helper agent's weather answer with a judge
+ * model (via autoevals → a Databricks serving endpoint).
+ *
+ * Requires a judge model:
+ *   appkit agent eval query --root apps/dev-playground --url http://localhost:8001 \
+ *     --judge-model databricks-claude-sonnet-4-5
+ * (or set APPKIT_JUDGE_MODEL). Without it, t.judge.* errors with a clear message.
+ */
+export default defineEval({
+  description: "Helper weather answer is relevant (LLM judge)",
+  agent: "helper",
+  async test(t) {
+    await t.send("What's the weather in Brooklyn?");
+    t.succeeded();
+    // closedQA needs no ground truth — it judges the reply against a question.
+    (
+      await t.judge.closedQA(
+        "Does the response describe weather conditions for Brooklyn?",
+      )
+    ).atLeast(0.5);
+  },
+});

apps/dev-playground/config/agents/query/evals/smoke.eval.ts

@@ -0,0 +1,14 @@
+import { defineEval } from "@databricks/appkit/beta";
+
+/**
+ * Example eval. The agent defaults to this directory's name (`query`).
+ * Run with:
+ *   pnpm exec appkit agent eval query --root apps/dev-playground --url http://localhost:8000
+ */
+export default defineEval({
+  description: "Query dispatcher responds to a greeting",
+  async test(t) {
+    await t.send("Hi there!");
+    t.succeeded(); // gate: the turn completed
+  },
+});

apps/dev-playground/config/agents/query/evals/sum.eval.ts

@@ -0,0 +1,13 @@
+import { defineEval, includes } from "@databricks/appkit/beta";
+
+export default defineEval({
+  description: "Helper agent smoke test",
+  // Target the default code-defined agent. Drop this to use the `query` agent
+  // (the parent directory name).
+  agent: "helper",
+  async test(t) {
+    await t.send("What is 2 + 2?");
+    t.succeeded(); // gate: the turn completed
+    t.check(t.reply, includes("4")).soft(); // tracked metric, won't fail the gate
+  },
+});

apps/dev-playground/config/agents/query/evals/tool-call.eval.ts

@@ -0,0 +1,16 @@
+import { defineEval } from "@databricks/appkit/beta";
+
+/**
+ * Tool-call eval: the default `helper` agent should call its `get_weather`
+ * tool when asked about the weather. (Targets `helper` explicitly — the parent
+ * directory only determines discovery, not which agent runs.)
+ */
+export default defineEval({
+  description: "Helper agent calls the get_weather tool",
+  agent: "helper",
+  async test(t) {
+    await t.send("What's the weather in Brooklyn?");
+    t.succeeded();
+    t.calledTool("get_weather");
+  },
+});