diff --git a/apps/dev-playground/server/agents/classifier/agent.ts b/apps/dev-playground/server/agents/classifier/agent.ts new file mode 100644 index 000000000..929067a27 --- /dev/null +++ b/apps/dev-playground/server/agents/classifier/agent.ts @@ -0,0 +1,22 @@ +import { createAgent } from "@databricks/appkit/beta"; +import { z } from "zod"; + +// Structured-output demo: a tool-free agent whose answer is validated against +// its `output` Zod schema instead of returned as text. Discovered from the +// folder name (`classifier`). See docs/plugins/agents.md "Structured output" +// for how the object surfaces on /chat, /invocations, and in-process runAgent. +export default createAgent({ + instructions: + "You are a support-ticket triage classifier. Read the user's message and " + + "classify it into one category, decide whether it is urgent (the user is " + + "blocked or reports an outage), and write a one-sentence summary.", + output: z.object({ + category: z + .enum(["billing", "bug", "feature_request", "how_to", "other"]) + .describe("The single best-fit category for the ticket."), + urgent: z + .boolean() + .describe("True only if the user is blocked or reports an outage."), + summary: z.string().describe("A one-sentence summary of the request."), + }), +}); diff --git a/docs/docs/plugins/agents.md b/docs/docs/plugins/agents.md index 19b0f6da2..b691e7362 100644 --- a/docs/docs/plugins/agents.md +++ b/docs/docs/plugins/agents.md @@ -313,6 +313,47 @@ const result = await runAgent(classifier, { MCP hosted tools (`mcpServer(...)`) still require `agents()` (they need a live MCP client). Supervisor-API hosted tools (`supervisorTools.*`), by contrast, **work in standalone `runAgent`** — the adapter has everything it needs to execute them server-side. This makes batch-eval / CI use of supervisor agents possible without `createApp`. Plugin tool dispatch in standalone mode runs as the service principal (no OBO) and **bypasses the agents-plugin approval gate** — treat standalone runAgent as a trusted-prompt environment (CI, batch eval, internal scripts), not as an exposed user-facing surface. +## Structured output + +Give a code agent an `output` Zod schema and its final answer is validated against that schema instead of returned as freeform text: + +```ts +import { createAgent } from "@databricks/appkit/beta"; +import { z } from "zod"; + +export default createAgent({ + instructions: "Classify the support ticket.", + output: z.object({ + category: z.enum(["billing", "bug", "feature_request", "how_to", "other"]), + urgent: z.boolean(), + summary: z.string(), + }), +}); +``` + +The parsed, schema-valid object shows up on every non-streaming surface and on the stream: + +| Surface | Where the object appears | +| --- | --- | +| `POST /invocations`, `POST /responses` | top-level `output_parsed` field on the JSON envelope (next to `output`) | +| `POST /chat` (SSE) | one final `appkit.structured_output` event (`{ data }`) **after** the streamed text | +| in-process `runAgent` | `RunAgentResult.output`, statically typed as `z.infer` of the schema | + +```ts +import { runAgent } from "@databricks/appkit/beta"; + +const { output } = await runAgent(classifier, { messages: ticket }); +output?.category; // "billing" | "bug" | … | undefined — fully typed +``` + +A per-call override is also available for one-off shapes: `runAgent(agent, input, { output: SomeSchema })` (it takes precedence over the agent's own schema and retypes the result). + +**How it works.** The agent answers normally first (streaming its visible text on `/chat`). Structured output is then produced by a dedicated **non-streaming** completion constrained by the schema via `response_format` — Databricks rejects `response_format` under streaming, so the structured call can't be the streamed one. The answer is validated as-is first (a cheap pre-check for a model that already emitted JSON); otherwise the non-streaming pass reformats it. Either way the JSON is validated with Zod; on a mismatch AppKit re-prompts with the validation errors (up to two retries, invisible to the `/chat` text stream) and then throws a `StructuredOutputError` carrying the last raw output — it never returns partial or unvalidated data. If an endpoint rejects `response_format` outright, AppKit strips it and retries, relying on the prompt plus Zod validation. + +:::note Scope +Structured output is for **code-config agents** (`createAgent`). Markdown `agent.md` agents can't carry a schema, and sub-agents return text into their parent's context (typing buys nothing there). See the live API reference (`npx @databricks/appkit docs "appkit API reference"`) for the exact `createAgent` / `runAgent` / `StructuredOutputError` signatures. +::: + ## Adding agents to an existing app Already have an app and want to add agents? What you touch depends on the kind: diff --git a/packages/appkit/src/agents/databricks.ts b/packages/appkit/src/agents/databricks.ts index 097f4fdba..51b7b063c 100644 --- a/packages/appkit/src/agents/databricks.ts +++ b/packages/appkit/src/agents/databricks.ts @@ -7,6 +7,7 @@ import type { } from "shared"; import { + query as servingQuery, type StreamBody, stream as servingStream, } from "../connectors/serving/client"; @@ -70,6 +71,51 @@ function applyGenerationParams( } } +/** + * True when `err`'s message names structured output / `response_format`. Used + * to strip the param and retry when an endpoint rejects it — some endpoints + * don't support it at all, and Databricks returns the coarse + * `INVALID_PARAMETER_VALUE` ("Structured output is not currently supported…") + * with no granular code to key on. The Zod boundary in the resolver is the + * real guarantee, so a stray strip is harmless (validation still runs); that + * lets this stay a simple message test rather than probing status codes. + * + * ponytail: substring match — Databricks has no granular code for a + * response_format rejection; tighten to a code check if one ever ships. + */ +function isResponseFormatRejection(err: unknown): boolean { + const msg = err instanceof Error ? err.message : String(err); + return /response_format|json[_ ]schema|structured output/i.test(msg); +} + +/** Pull the assistant message text out of a NON-streaming chat completion. */ +function extractMessageContent(parsed: unknown): string { + if (!isRecord(parsed)) return ""; + const choices = parsed.choices; + if (!Array.isArray(choices) || choices.length === 0) return ""; + const first = choices[0]; + if (!isRecord(first)) return ""; + const message = first.message; + if (!isRecord(message)) return ""; + const content = message.content; + if (typeof content === "string") return content; + // Harmony/array content: concatenate the text parts. + if (Array.isArray(content)) { + let text = ""; + for (const part of content) { + if ( + isRecord(part) && + part.type === "text" && + typeof part.text === "string" + ) { + text += part.text; + } + } + return text; + } + return ""; +} + function extractLlamaToolJsonSlice(text: string): string | undefined { const start = text.indexOf("[{"); if (start < 0) return undefined; @@ -118,6 +164,12 @@ function reasoningPartText(part: Record): string { return text; } +/** Non-streaming sibling of {@link StreamBody}; returns the parsed JSON body. */ +type QueryBody = ( + body: Record, + signal?: AbortSignal, +) => Promise; + /** * Escape-hatch options: provide an `endpointUrl` + `authenticate()` and the * adapter uses a bare `fetch()` to call it. Useful for tests and for pointing @@ -146,6 +198,8 @@ interface RawFetchAdapterOptions { */ interface StreamBodyAdapterOptions { streamBody: StreamBody; + /** Non-streaming transport for structured-output completions. */ + queryBody?: QueryBody; maxSteps?: number; maxTokens?: number; generationParams?: GenerationParams; @@ -284,6 +338,8 @@ interface DeltaToolCall { */ export class DatabricksAdapter implements AgentAdapter { private streamBody: StreamBody; + /** Non-streaming transport; present only when structured output is usable. */ + private queryBody?: QueryBody; private maxSteps: number; private maxTokens: number; private generationParams: GenerationParams; @@ -304,11 +360,13 @@ export class DatabricksAdapter implements AgentAdapter { if (isStreamBodyOptions(options)) { this.streamBody = options.streamBody; + this.queryBody = options.queryBody; } else { const { endpointUrl, authenticate } = options; - this.streamBody = async (body, signal) => { - const fetchSignal = - signal ?? AbortSignal.timeout(RAW_FETCH_DEFAULT_TIMEOUT_MS); + const doFetch = async ( + body: Record, + signal?: AbortSignal, + ): Promise => { const authHeaders = await authenticate(); const response = await fetch(endpointUrl, { method: "POST", @@ -318,7 +376,7 @@ export class DatabricksAdapter implements AgentAdapter { ...authHeaders, }, body: JSON.stringify(body), - signal: fetchSignal, + signal: signal ?? AbortSignal.timeout(RAW_FETCH_DEFAULT_TIMEOUT_MS), }); if (!response.ok) { const errorText = await response.text().catch(() => "Unknown error"); @@ -326,9 +384,16 @@ export class DatabricksAdapter implements AgentAdapter { `Databricks API error (${response.status}): ${errorText}`, ); } + return response; + }; + this.streamBody = async (body, signal) => { + const response = await doFetch(body, signal); if (!response.body) throw new Error("No response body"); return response.body; }; + // Non-streaming sibling for structured-output completions. + this.queryBody = async (body, signal) => + (await doFetch({ ...body, stream: false }, signal)).json(); } } @@ -364,6 +429,13 @@ export class DatabricksAdapter implements AgentAdapter { body, signal, ), + queryBody: (body, signal) => + servingQuery( + workspaceClient as unknown as Parameters[0], + endpointName, + body, + signal, + ), maxSteps, maxTokens, generationParams, @@ -480,6 +552,27 @@ export class DatabricksAdapter implements AgentAdapter { yield { type: "status", status: "running" }; + // Tool-free structuring pass (the resolver drives it via + // `run({ tools: [], outputSchema })`): a non-streaming, schema-constrained + // completion. The visible answer streams on the path below. + if (input.outputSchema && input.tools.length === 0) { + let text: string; + try { + text = await this.structuredCompletion( + messages, + input.outputSchema, + context, + ); + } catch (err) { + const msg = + err instanceof Error ? err.message : "Structured request failed"; + yield { type: "status", status: "error", error: msg }; + throw err; + } + yield { type: "message", content: text }; + return; + } + for (let step = 0; step < this.maxSteps; step++) { if (context.signal?.aborted) break; @@ -559,6 +652,48 @@ export class DatabricksAdapter implements AgentAdapter { } } + /** + * One tool-free, non-streaming completion constrained by `schema` via + * `response_format` (Databricks rejects `response_format` under + * `stream: true`). Returns the raw message text for the caller to validate; + * on a rejection, strips `response_format` and retries once (see + * {@link isResponseFormatRejection}). + */ + private async structuredCompletion( + messages: OpenAIMessage[], + schema: Record, + context: AgentRunContext, + ): Promise { + if (!this.queryBody) { + throw new Error( + "DatabricksAdapter: structured output requires a non-streaming transport. " + + "Build the adapter via DatabricksAdapter.fromServingEndpoint / fromModelServing.", + ); + } + const body: Record = { + messages, + max_tokens: this.maxTokens, + response_format: { + type: "json_schema", + json_schema: { name: "structured_output", schema, strict: true }, + }, + }; + applyGenerationParams(body, this.generationParams); + + let parsed: unknown; + try { + parsed = await this.queryBody(body, context.signal); + } catch (err) { + if (isResponseFormatRejection(err)) { + delete body.response_format; + parsed = await this.queryBody(body, context.signal); + } else { + throw err; + } + } + return extractMessageContent(parsed); + } + private async *streamCompletion( messages: OpenAIMessage[], tools: OpenAITool[], diff --git a/packages/appkit/src/agents/tests/databricks.test.ts b/packages/appkit/src/agents/tests/databricks.test.ts index 4fff9c9a0..4bd53febc 100644 --- a/packages/appkit/src/agents/tests/databricks.test.ts +++ b/packages/appkit/src/agents/tests/databricks.test.ts @@ -1283,3 +1283,183 @@ describe("parseTextToolCalls", () => { expect(parseTextToolCalls(`${filler}${suffix}`)).toEqual([]); }); }); + +describe("DatabricksAdapter structured output", () => { + const originalFetch = globalThis.fetch; + + afterEach(() => { + globalThis.fetch = originalFetch; + mockAuthenticate.mockClear(); + }); + + interface FakeResponse { + ok: boolean; + status?: number; + /** SSE chunks for the streaming path. */ + chunks?: string[]; + /** Parsed body for the non-streaming (structured) path. */ + json?: unknown; + text?: string; + } + + /** Records each request body and replays queued responses in order. */ + function capturingFetch( + bodies: Array>, + responses: FakeResponse[], + ): typeof globalThis.fetch { + let call = 0; + return vi.fn().mockImplementation((_url, init) => { + if (init?.body) bodies.push(JSON.parse(init.body)); + const r = responses[Math.min(call, responses.length - 1)]; + call++; + return Promise.resolve({ + ok: r.ok, + status: r.status ?? (r.ok ? 200 : 400), + body: r.ok ? createReadableStream(r.chunks ?? []) : null, + json: () => Promise.resolve(r.json), + text: () => Promise.resolve(r.text ?? ""), + }); + }); + } + + /** A non-streaming chat-completion body with a single JSON message. */ + function completion(content: string): unknown { + return { choices: [{ message: { role: "assistant", content } }] }; + } + + async function collect( + gen: AsyncGenerator, + ): Promise { + const events: AgentEvent[] = []; + for await (const ev of gen) events.push(ev); + return events; + } + + const outputSchema = { + type: "object", + properties: { answer: { type: "string" } }, + required: ["answer"], + }; + + test("tool-free structured run is NON-streaming with response_format", async () => { + const bodies: Array> = []; + globalThis.fetch = capturingFetch(bodies, [ + { ok: true, json: completion('{"answer":"hi"}') }, + ]); + + const adapter = createAdapter(); + const events = await collect( + adapter.run( + { + messages: createTestMessages(), + tools: [], + threadId: "t1", + outputSchema, + }, + { executeTool: vi.fn() }, + ), + ); + + expect(bodies).toHaveLength(1); + expect(bodies[0].stream).toBe(false); + expect(bodies[0].response_format).toEqual({ + type: "json_schema", + json_schema: { + name: "structured_output", + schema: outputSchema, + strict: true, + }, + }); + expect(bodies[0].tools).toBeUndefined(); + // The JSON arrives as a single `message` event (not streamed deltas). + expect(events).toContainEqual({ + type: "message", + content: '{"answer":"hi"}', + }); + }); + + test("does NOT send response_format when tools are present (stays streaming)", async () => { + const bodies: Array> = []; + globalThis.fetch = capturingFetch(bodies, [ + { ok: true, chunks: [textDelta("done"), sseChunk("[DONE]")] }, + ]); + + const adapter = createAdapter(); + await collect( + adapter.run( + { + messages: createTestMessages(), + tools: createTestTools(), + threadId: "t1", + outputSchema, + }, + { executeTool: vi.fn() }, + ), + ); + + expect(bodies[0].response_format).toBeUndefined(); + expect(bodies[0].tools).toBeDefined(); + expect(bodies[0].stream).toBe(true); + }); + + test("400 naming structured output strips response_format and retries once", async () => { + const bodies: Array> = []; + globalThis.fetch = capturingFetch(bodies, [ + { + ok: false, + status: 400, + text: "INVALID_PARAMETER_VALUE: Structured output is not currently supported with streaming.", + }, + { ok: true, json: completion('{"answer":"ok"}') }, + ]); + + const adapter = createAdapter(); + const events = await collect( + adapter.run( + { + messages: createTestMessages(), + tools: [], + threadId: "t1", + outputSchema, + }, + { executeTool: vi.fn() }, + ), + ); + + // First body carried response_format; the retry stripped it. Both non-streaming. + expect(bodies).toHaveLength(2); + expect(bodies[0].response_format).toBeDefined(); + expect(bodies[1].response_format).toBeUndefined(); + expect(events).toContainEqual({ + type: "message", + content: '{"answer":"ok"}', + }); + expect(events).not.toContainEqual( + expect.objectContaining({ type: "status", status: "error" }), + ); + }); + + test("a 400 NOT naming the param propagates (no strip-retry)", async () => { + const bodies: Array> = []; + globalThis.fetch = capturingFetch(bodies, [ + { ok: false, status: 400, text: "Bad request: token limit exceeded" }, + ]); + + const adapter = createAdapter(); + await expect( + collect( + adapter.run( + { + messages: createTestMessages(), + tools: [], + threadId: "t1", + outputSchema, + }, + { executeTool: vi.fn() }, + ), + ), + ).rejects.toThrow(/token limit exceeded/); + // Only the original request — no retry for an unrelated 400. + expect(bodies).toHaveLength(1); + }); +}); diff --git a/packages/appkit/src/beta.ts b/packages/appkit/src/beta.ts index 4de7ba79c..acfdefacf 100644 --- a/packages/appkit/src/beta.ts +++ b/packages/appkit/src/beta.ts @@ -42,6 +42,7 @@ export { export { createAgent } from "./core/agent/create-agent"; export { type RunAgentInput, + type RunAgentOptions, type RunAgentResult, runAgent, } from "./core/agent/run-agent"; diff --git a/packages/appkit/src/connectors/serving/client.ts b/packages/appkit/src/connectors/serving/client.ts index de9d0465c..662a6941a 100644 --- a/packages/appkit/src/connectors/serving/client.ts +++ b/packages/appkit/src/connectors/serving/client.ts @@ -101,6 +101,37 @@ export async function streamPath( return response.contents; } +/** + * POSTs `body` to a serving endpoint as a non-streaming request (forces + * `stream: false`) and returns the parsed JSON (`{ choices: [{ message }] }`). + * Used by the structured-output path. + * + * @internal Not part of the public AppKit surface. Like {@link streamPath}, + * the endpoint name is caller-controlled and hard-coded by internal callers; + * do not expose to user input (workspace-credentialled SSRF). + */ +export async function query( + client: ApiClientLike, + endpointName: string, + body: Record, + signal?: AbortSignal, +): Promise { + const { stream: _stream, ...cleanBody } = body; + const context = contextFromAbortSignal(signal); + return client.apiClient.request( + { + path: `/serving-endpoints/${encodeURIComponent(endpointName)}/invocations`, + method: "POST", + headers: new Headers({ + "Content-Type": "application/json", + Accept: "application/json", + }), + payload: { ...cleanBody, stream: false }, + }, + context, + ); +} + /** * Returns the raw SSE byte stream from a serving endpoint. Thin wrapper over * {@link streamPath} that handles serving-specific URL encoding and forces diff --git a/packages/appkit/src/core/agent/create-agent.ts b/packages/appkit/src/core/agent/create-agent.ts index 67c589317..b946097c0 100644 --- a/packages/appkit/src/core/agent/create-agent.ts +++ b/packages/appkit/src/core/agent/create-agent.ts @@ -1,3 +1,5 @@ +import type { z } from "zod"; + import { ConfigurationError } from "../../errors"; import type { AgentDefinition } from "./types"; @@ -30,8 +32,25 @@ const AGENT_BRAND: unique symbol = Symbol.for("appkit.agent"); * }, * }); * ``` + * + * @example Structured output + * ```ts + * const classify = createAgent({ + * instructions: "Classify the ticket.", + * output: z.object({ category: z.string(), urgent: z.boolean() }), + * }); + * // In-process, the result is typed via z.infer: + * const { output } = await runAgent(classify, { messages: "..." }); + * output?.category; // string | undefined + * ``` */ -export function createAgent(def: AgentDefinition): AgentDefinition { +export function createAgent( + def: AgentDefinition> & { output: S }, +): AgentDefinition>; +export function createAgent(def: AgentDefinition): AgentDefinition; +export function createAgent( + def: AgentDefinition, +): AgentDefinition { detectCycles(def); // Non-enumerable + in-place: identity, JSON, and spread are unaffected. Object.defineProperty(def, AGENT_BRAND, { @@ -58,11 +77,11 @@ export function isCreatedAgent(value: unknown): value is AgentDefinition { * Walks the `agents: { ... }` sub-agent tree via DFS and throws if a cycle is * found. Cycles would cause infinite recursion at tool-invocation time. */ -function detectCycles(def: AgentDefinition): void { - const visiting = new Set(); - const visited = new Set(); +function detectCycles(def: AgentDefinition): void { + const visiting = new Set>(); + const visited = new Set>(); - const walk = (current: AgentDefinition, path: string[]): void => { + const walk = (current: AgentDefinition, path: string[]): void => { if (visited.has(current)) return; if (visiting.has(current)) { throw new ConfigurationError( diff --git a/packages/appkit/src/core/agent/run-agent.ts b/packages/appkit/src/core/agent/run-agent.ts index 57a659686..05ed4be76 100644 --- a/packages/appkit/src/core/agent/run-agent.ts +++ b/packages/appkit/src/core/agent/run-agent.ts @@ -9,6 +9,7 @@ import type { PluginData, ToolProvider, } from "shared"; +import type { z } from "zod"; import { isSupervisorTool, @@ -18,6 +19,10 @@ import { import { createLogger } from "../../logging/logger"; import { consumeAdapterStream } from "./consume-adapter-stream"; import { createPluginsProxy } from "./plugins-map"; +import { + buildStructuringPass, + resolveStructuredOutput, +} from "./structured-output"; import { resolveToolkitFromProvider } from "./toolkit-resolver"; import { type FunctionTool, @@ -25,6 +30,7 @@ import { isFunctionTool, } from "./tools/function-tool"; import { isHostedTool } from "./tools/hosted-tools"; +import { toToolJSONSchema } from "./tools/json-schema"; import type { AgentDefinition, AgentTool, @@ -51,11 +57,28 @@ export interface RunAgentInput { plugins?: PluginData[]; } -export interface RunAgentResult { +/** Per-call options for {@link runAgent}. */ +export interface RunAgentOptions { + /** + * Structured-output schema override for this call. Takes precedence over the + * agent's own `output` schema. When either is set, `runAgent` validates the + * final answer and populates {@link RunAgentResult.output}, throwing a + * `StructuredOutputError` if it can't produce a valid object. + */ + output?: z.ZodType; +} + +export interface RunAgentResult { /** Aggregated text output from all `message_delta` events. */ text: string; /** Every event the adapter yielded, in order. Useful for inspection/tests. */ events: AgentEvent[]; + /** + * Parsed, schema-validated object — present only when the agent (or the + * per-call override) declared an `output` schema. Statically typed via + * `z.infer` when the agent was built with `createAgent({ output })`. + */ + output?: TOutput; } /** @@ -86,10 +109,22 @@ export interface RunAgentResult { * `PluginContext`) throw at standalone-init time with a clear "use * createApp instead" message — not mid-stream. */ +// Per-call schema override drives the result type via z.infer. +export function runAgent( + def: AgentDefinition, + input: RunAgentInput, + options: RunAgentOptions & { output: S }, +): Promise>>; +// No override — the result type comes from the agent's own `output` schema. +export function runAgent( + def: AgentDefinition, + input: RunAgentInput, +): Promise>; export async function runAgent( - def: AgentDefinition, + def: AgentDefinition, input: RunAgentInput, -): Promise { + options?: RunAgentOptions, +): Promise> { // Single shared cache for the whole call graph: parent + every nested // sub-agent dispatch share constructed plugin instances. Without this, // each nested `runAgent` would build its own cache, re-instantiate every @@ -97,14 +132,43 @@ export async function runAgent( // (e.g. query result caches, connection pools). const providerCache = new Map(); await initStandalonePlugins(input.plugins ?? [], providerCache); - return runAgentInternal(def, input, providerCache); + + const { text, events, adapter, baseMessages } = await runAgentInternal( + def, + input, + providerCache, + ); + + const schema = options?.output ?? def.output; + if (!schema) return { text, events }; + + // Structured output is produced by a separate tool-free, non-streaming + // structuring pass (Databricks rejects response_format under streaming), + // so the main run above always streams the visible answer normally. + const output = await resolveStructuredOutput({ + schema, + baseMessages, + finalText: text, + runStructuringPass: buildStructuringPass(adapter, toToolJSONSchema(schema)), + signal: input.signal, + }); + return { text, events, output }; +} + +interface RawRunResult { + text: string; + events: AgentEvent[]; + /** Adapter used for the run — reused for the structuring pass. */ + adapter: AgentAdapter; + /** Normalized system + input messages the run saw (structuring-pass seed). */ + baseMessages: Message[]; } async function runAgentInternal( - def: AgentDefinition, + def: AgentDefinition, input: RunAgentInput, providerCache: Map, -): Promise { +): Promise { const adapter = await resolveAdapter(def); const messages = normalizeMessages(input.messages, def.instructions); const toolIndex = buildStandaloneToolIndex( @@ -193,7 +257,12 @@ async function runAgentInternal( }, }); - return { text, events }; + return { + text, + events, + adapter, + baseMessages: messages, + }; } /** @@ -263,7 +332,9 @@ async function initStandalonePlugins( } } -async function resolveAdapter(def: AgentDefinition): Promise { +async function resolveAdapter( + def: AgentDefinition, +): Promise { const { model } = def; if (!model) { const { DatabricksAdapter } = await import("../../agents/databricks"); @@ -344,7 +415,7 @@ type StandaloneEntry = * references throw a named "not registered" error via the proxy. */ function buildStandaloneToolIndex( - def: AgentDefinition, + def: AgentDefinition, plugins: PluginData[], providerCache: Map, ): Map { @@ -389,7 +460,7 @@ function buildStandaloneToolIndex( * directly (no construction at toolkit-call time). */ function resolveDefTools( - def: AgentDefinition, + def: AgentDefinition, plugins: PluginData[], providerCache: Map, ): AgentTools { diff --git a/packages/appkit/src/core/agent/structured-output.ts b/packages/appkit/src/core/agent/structured-output.ts new file mode 100644 index 000000000..d2ba34e55 --- /dev/null +++ b/packages/appkit/src/core/agent/structured-output.ts @@ -0,0 +1,165 @@ +import { randomUUID } from "node:crypto"; + +import type { AgentAdapter, Message } from "shared"; +import type { z } from "zod"; + +import { StructuredOutputError } from "../../errors"; +import { consumeAdapterStream } from "./consume-adapter-stream"; +import { formatZodIssues } from "./tools/tool"; + +/** Re-prompted structuring passes after the first convert pass (so ≤3 total). Separate from the tool-call / maxSteps budget. */ +const MAX_VALIDATION_RETRIES = 2; + +const CONVERT_INSTRUCTION = + "Convert the assistant's answer above into JSON matching the provided schema. " + + "Output only the JSON, with no prose, explanation, or code fences."; + +function retryInstruction(errors: string): string { + return ( + "Your previous response did not match the required schema. " + + `Validation errors: ${errors}. ` + + "Return only JSON matching the schema, with no prose or code fences." + ); +} + +/** + * Runs one tool-free, schema-constrained completion and returns the raw text. + * Injected so the resolver stays free of any adapter / MLflow dependency; see + * {@link buildStructuringPass}. + */ +export type StructuringPass = ( + messages: Message[], + signal?: AbortSignal, +) => Promise; + +interface ResolveStructuredOutputParams { + /** Schema the final object is validated against. */ + schema: z.ZodType; + /** The conversation the main run saw (system + thread), without the final answer. */ + baseMessages: Message[]; + /** The main run's final assistant text — the answer to reformat into JSON. */ + finalText: string; + /** Runs one tool-free, schema-constrained completion (see {@link StructuringPass}). */ + runStructuringPass: StructuringPass; + signal?: AbortSignal; +} + +/** + * Turns an agent's answer into a schema-validated object. Validates `finalText` + * as-is first (skipping a structuring round-trip when the model already emitted + * valid JSON); otherwise re-prompts a tool-free structuring pass with the + * flattened Zod issues, up to {@link MAX_VALIDATION_RETRIES} times. Throws + * {@link StructuredOutputError} on exhaustion — never returns partial data. + */ +export async function resolveStructuredOutput( + params: ResolveStructuredOutputParams, +): Promise { + const { schema, baseMessages, finalText, runStructuringPass, signal } = + params; + + const direct = parseAndValidate(schema, finalText); + if (direct.ok) return direct.value; + + let lastRaw = await runStructuringPass( + structuringMessages(baseMessages, finalText, CONVERT_INSTRUCTION), + signal, + ); + + for (let retries = 0; ; retries++) { + const parsed = parseAndValidate(schema, lastRaw); + if (parsed.ok) return parsed.value; + + if (retries >= MAX_VALIDATION_RETRIES) { + throw new StructuredOutputError( + `Agent output did not match the required schema after ` + + `${MAX_VALIDATION_RETRIES + 1} attempts: ${parsed.error}`, + { lastRaw }, + ); + } + + lastRaw = await runStructuringPass( + structuringMessages( + baseMessages, + lastRaw, + retryInstruction(parsed.error), + ), + signal, + ); + } +} + +/** Appends the latest answer (as an assistant turn) + a user instruction. */ +function structuringMessages( + base: Message[], + assistantText: string, + instruction: string, +): Message[] { + return [ + ...base, + { + id: randomUUID(), + role: "assistant", + content: assistantText, + createdAt: new Date(), + }, + { + id: randomUUID(), + role: "user", + content: instruction, + createdAt: new Date(), + }, + ]; +} + +type ParseResult = { ok: true; value: T } | { ok: false; error: string }; + +function parseAndValidate( + schema: z.ZodType, + raw: string, +): ParseResult { + let json: unknown; + try { + json = JSON.parse(stripCodeFences(raw)); + } catch { + return { ok: false, error: "response was not valid JSON" }; + } + const result = schema.safeParse(json); + if (result.success) return { ok: true, value: result.data }; + return { ok: false, error: formatZodIssues(result.error) }; +} + +/** Strip a leading/trailing Markdown code fence — models often wrap JSON in ` ```json … ``` `. */ +function stripCodeFences(text: string): string { + const trimmed = text.trim(); + if (!trimmed.startsWith("```")) return trimmed; + return trimmed + .replace(/^```(?:json)?\s*\n?/i, "") + .replace(/\n?```$/, "") + .trim(); +} + +/** + * Builds a {@link StructuringPass}: one tool-free, schema-constrained + * `adapter.run()` consumed to its text. `executeTool` throws — a tool-free + * pass must never dispatch one. + */ +export function buildStructuringPass( + adapter: AgentAdapter, + outputSchema: Record, +): StructuringPass { + return (messages, signal) => + consumeAdapterStream( + adapter.run( + { messages, tools: [], threadId: randomUUID(), signal, outputSchema }, + { + executeTool: () => { + throw new Error( + "structured-output structuring pass is tool-free and must not call a tool", + ); + }, + signal, + }, + ), + { signal }, + ); +} diff --git a/packages/appkit/src/core/agent/tests/run-agent-structured.test.ts b/packages/appkit/src/core/agent/tests/run-agent-structured.test.ts new file mode 100644 index 000000000..1cad26ef7 --- /dev/null +++ b/packages/appkit/src/core/agent/tests/run-agent-structured.test.ts @@ -0,0 +1,138 @@ +import type { AgentAdapter, AgentInput } from "shared"; +import { describe, expect, test } from "vitest"; +import { z } from "zod"; + +import { StructuredOutputError } from "../../../errors"; +import { createAgent } from "../create-agent"; +import { runAgent } from "../run-agent"; +import { tool } from "../tools/tool"; + +const schema = z.object({ answer: z.string(), score: z.number() }); + +/** Fake adapter that records each run() input and emits a scripted text per call. */ +function recordingAdapter( + texts: string[], +): AgentAdapter & { calls: AgentInput[] } { + const calls: AgentInput[] = []; + let i = 0; + return { + calls, + async *run(input) { + calls.push(input); + const text = texts[Math.min(i, texts.length - 1)]; + i++; + yield { type: "status", status: "running" }; + yield { type: "message_delta", content: text }; + }, + }; +} + +describe("runAgent structured output", () => { + test("answer already valid JSON: direct pre-check, no structuring pass", async () => { + const adapter = recordingAdapter(['{"answer":"hi","score":1}']); + const agent = createAgent({ + instructions: "classify", + model: adapter, + output: schema, + }); + + const result = await runAgent(agent, { messages: "hello" }); + + expect(result.output).toEqual({ answer: "hi", score: 1 }); + // Statically typed via z.infer — compiles only if the type is right. + const typed: { answer: string; score: number } | undefined = result.output; + expect(typed?.answer).toBe("hi"); + + // One run only; the main run never carries outputSchema. + expect(adapter.calls).toHaveLength(1); + expect(adapter.calls[0].outputSchema).toBeUndefined(); + }); + + test("prose answer: a tool-free structuring pass produces the object", async () => { + const adapter = recordingAdapter([ + "The answer is hi with a score of 1.", // main run: prose + '{"answer":"hi","score":1}', // structuring pass: JSON + ]); + const agent = createAgent({ + instructions: "classify", + model: adapter, + output: schema, + }); + + const result = await runAgent(agent, { messages: "hello" }); + + expect(result.output).toEqual({ answer: "hi", score: 1 }); + expect(adapter.calls).toHaveLength(2); + // Main run: no outputSchema. Structuring pass: tool-free + schema-constrained. + expect(adapter.calls[0].outputSchema).toBeUndefined(); + expect(adapter.calls[1].tools).toEqual([]); + expect(adapter.calls[1].outputSchema).toBeDefined(); + }); + + test("tool-having agent structures identically (tool only on the main run)", async () => { + const adapter = recordingAdapter([ + "Looks like a billing issue, not urgent.", + '{"answer":"billing","score":0}', + ]); + const agent = createAgent({ + instructions: "classify", + model: adapter, + output: schema, + tools: { + noop: tool({ + name: "noop", + description: "does nothing", + schema: z.object({}), + execute: () => "ok", + }), + }, + }); + + const result = await runAgent(agent, { messages: "hello" }); + + expect(result.output).toEqual({ answer: "billing", score: 0 }); + expect(adapter.calls).toHaveLength(2); + expect(adapter.calls[0].tools).toHaveLength(1); // main run has the tool + expect(adapter.calls[1].tools).toEqual([]); // structuring pass is tool-free + expect(adapter.calls[1].outputSchema).toBeDefined(); + }); + + test("per-call { output } override drives structured output when the agent has none", async () => { + const adapter = recordingAdapter(['{"answer":"o","score":2}']); + const agent = createAgent({ instructions: "x", model: adapter }); + + const result = await runAgent( + agent, + { messages: "hi" }, + { output: schema }, + ); + + expect(result.output).toEqual({ answer: "o", score: 2 }); + }); + + test("throws StructuredOutputError when output never validates", async () => { + const adapter = recordingAdapter(["not json at all"]); + const agent = createAgent({ + instructions: "x", + model: adapter, + output: schema, + }); + + await expect(runAgent(agent, { messages: "hi" })).rejects.toBeInstanceOf( + StructuredOutputError, + ); + // main run + convert pass + 2 retry passes. + expect(adapter.calls).toHaveLength(4); + }); + + test("no output schema: result.output is undefined, no extra runs", async () => { + const adapter = recordingAdapter(["just some prose"]); + const agent = createAgent({ instructions: "x", model: adapter }); + + const result = await runAgent(agent, { messages: "hi" }); + + expect(result.output).toBeUndefined(); + expect(result.text).toBe("just some prose"); + expect(adapter.calls).toHaveLength(1); + }); +}); diff --git a/packages/appkit/src/core/agent/tests/structured-output.test.ts b/packages/appkit/src/core/agent/tests/structured-output.test.ts new file mode 100644 index 000000000..5699d0a9f --- /dev/null +++ b/packages/appkit/src/core/agent/tests/structured-output.test.ts @@ -0,0 +1,115 @@ +import type { Message } from "shared"; +import { describe, expect, test, vi } from "vitest"; +import { z } from "zod"; + +import { StructuredOutputError } from "../../../errors"; +import { + resolveStructuredOutput, + type StructuringPass, +} from "../structured-output"; + +const schema = z.object({ category: z.string(), urgent: z.boolean() }); + +function baseMessages(): Message[] { + return [ + { id: "s", role: "system", content: "classify", createdAt: new Date() }, + { id: "u", role: "user", content: "help me", createdAt: new Date() }, + ]; +} + +describe("resolveStructuredOutput", () => { + test("validates the answer as-is when it is already valid JSON (no pass)", async () => { + const pass = vi.fn(); + const output = await resolveStructuredOutput({ + schema, + baseMessages: baseMessages(), + finalText: JSON.stringify({ category: "billing", urgent: true }), + runStructuringPass: pass, + }); + + expect(output).toEqual({ category: "billing", urgent: true }); + expect(pass).not.toHaveBeenCalled(); + }); + + test("reformats a prose answer via one structuring pass", async () => { + const pass = vi + .fn() + .mockResolvedValue(JSON.stringify({ category: "sales", urgent: false })); + + const output = await resolveStructuredOutput({ + schema, + baseMessages: baseMessages(), + finalText: "This looks like a sales question, not urgent.", + runStructuringPass: pass, + }); + + expect(output).toEqual({ category: "sales", urgent: false }); + expect(pass).toHaveBeenCalledTimes(1); + // The prose answer is appended as an assistant turn + a convert instruction. + const [msgs] = pass.mock.calls[0]; + expect(msgs.at(-2)).toMatchObject({ role: "assistant" }); + expect(msgs.at(-1)?.role).toBe("user"); + expect(msgs.at(-1)?.content).toMatch(/JSON matching the provided schema/i); + }); + + test("strips code fences before parsing", async () => { + const output = await resolveStructuredOutput({ + schema, + baseMessages: baseMessages(), + finalText: '```json\n{"category":"support","urgent":false}\n```', + runStructuringPass: vi.fn(), + }); + expect(output).toEqual({ category: "support", urgent: false }); + }); + + test("re-prompts with zod errors on validation failure, then succeeds", async () => { + const pass = vi + .fn() + .mockResolvedValueOnce(JSON.stringify({ category: "billing" })) // missing urgent + .mockResolvedValueOnce( + JSON.stringify({ category: "billing", urgent: true }), + ); + + const output = await resolveStructuredOutput({ + schema, + baseMessages: baseMessages(), + finalText: JSON.stringify({ category: "billing" }), // invalid — triggers a pass + runStructuringPass: pass, + }); + + expect(output).toEqual({ category: "billing", urgent: true }); + // pass #1 (convert) invalid -> pass #2 (retry w/ errors) valid. + expect(pass).toHaveBeenCalledTimes(2); + // The FIRST pass is the convert instruction; the retry carries the zod error. + expect(pass.mock.calls[0][0].at(-1)?.content).toMatch( + /JSON matching the provided schema/i, + ); + const retryMsg = pass.mock.calls[1][0].at(-1)?.content ?? ""; + expect(retryMsg).toMatch(/did not match the required schema/i); + expect(retryMsg).toMatch(/urgent/); + }); + + test("throws StructuredOutputError with lastRaw after retries exhausted", async () => { + const pass = vi.fn().mockResolvedValue("still not json"); + + await expect( + resolveStructuredOutput({ + schema, + baseMessages: baseMessages(), + finalText: "not json either", + runStructuringPass: pass, + }), + ).rejects.toMatchObject({ + name: "StructuredOutputError", + lastRaw: "still not json", + }); + + // convert pass + 2 retry passes = 3 structuring attempts. + expect(pass).toHaveBeenCalledTimes(3); + }); + + test("StructuredOutputError does not leak lastRaw via clientMessage", () => { + const err = new StructuredOutputError("boom", { lastRaw: "secret raw" }); + expect(err.clientMessage).not.toContain("secret raw"); + }); +}); diff --git a/packages/appkit/src/core/agent/tools/tool.ts b/packages/appkit/src/core/agent/tools/tool.ts index 62092f217..22ff6bc92 100644 --- a/packages/appkit/src/core/agent/tools/tool.ts +++ b/packages/appkit/src/core/agent/tools/tool.ts @@ -84,15 +84,25 @@ export function tool(config: ToolConfig): FunctionTool { }; } +/** + * Compact one-line rendering of a ZodError's issues: `path: message` per issue, + * joined by `; ` (root-level issues render as `(root)`). Stable across zod v4. + * Shared by {@link formatZodError} and the structured-output re-prompt. + */ +export function formatZodIssues(error: z.ZodError): string { + return error.issues + .map((issue) => { + const field = issue.path.length > 0 ? issue.path.join(".") : "(root)"; + return `${field}: ${issue.message}`; + }) + .join("; "); +} + /** * Formats a Zod validation error into an LLM-friendly string. * * Example: `Invalid arguments for get_weather: city: Invalid input: expected string, received undefined` */ export function formatZodError(error: z.ZodError, toolName: string): string { - const parts = error.issues.map((issue) => { - const field = issue.path.length > 0 ? issue.path.join(".") : "(root)"; - return `${field}: ${issue.message}`; - }); - return `Invalid arguments for ${toolName}: ${parts.join("; ")}`; + return `Invalid arguments for ${toolName}: ${formatZodIssues(error)}`; } diff --git a/packages/appkit/src/core/agent/types.ts b/packages/appkit/src/core/agent/types.ts index 53dae3c87..a816130b7 100644 --- a/packages/appkit/src/core/agent/types.ts +++ b/packages/appkit/src/core/agent/types.ts @@ -5,6 +5,7 @@ import type { ThreadStore, ToolAnnotations, } from "shared"; +import type { z } from "zod"; import type { GenerationParams } from "../../agents/databricks"; import type { McpHostPolicyConfig } from "../../connectors/mcp"; @@ -124,7 +125,7 @@ export type AgentTools = Record; */ export type AgentToolsFn = (plugins: Plugins) => AgentTools; -export interface AgentDefinition { +export interface AgentDefinition { /** * Stable identifier for the agent. **Optional and informational** — * when the definition is registered via `agents: { foo: def }` (code) or @@ -154,6 +155,15 @@ export interface AgentDefinition { default?: boolean; /** System prompt body. For markdown-loaded agents this is the file body. */ instructions: string; + /** + * Optional Zod schema the agent's final answer is validated against; when + * set, the agent returns a typed object instead of text — surfaced as + * `output_parsed` on `/invocations`, a final `structured_output` SSE event + * on `/chat`, and `RunAgentResult.output` in-process. Prefer {@link + * createAgent} so the in-process result is typed via `z.infer`. Code-config + * agents only — markdown agents can't carry a schema. + */ + output?: z.ZodType; /** * Model adapter (or endpoint-name string sugar for * `DatabricksAdapter.fromServingEndpoint({ endpointName })`). Optional — @@ -399,6 +409,15 @@ export interface RegisteredAgent { generationParams?: GenerationParams; /** Mirrors `AgentDefinition.ephemeral` — skip thread persistence. */ ephemeral?: boolean; + /** + * Mirrors `AgentDefinition.output` — the Zod schema the final answer is + * validated against. Present only for agents that declared structured + * output. Untyped here (the registry `Map` is string-keyed); `z.infer` + * typing lives on `createAgent`/`runAgent`. + */ + output?: z.ZodType; + /** The {@link output} schema converted to JSON Schema, memoized at registration. Present iff `output` is. */ + outputSchema?: Record; /** * Resolved per-agent skill catalog (visibility + collision rules applied). * Present when any skill is visible to this agent; drives the always-on diff --git a/packages/appkit/src/errors/index.ts b/packages/appkit/src/errors/index.ts index a367b843c..25dbb9c69 100644 --- a/packages/appkit/src/errors/index.ts +++ b/packages/appkit/src/errors/index.ts @@ -26,5 +26,6 @@ export { ConnectionError } from "./connection"; export { ExecutionError } from "./execution"; export { InitializationError } from "./initialization"; export { ServerError } from "./server"; +export { StructuredOutputError } from "./structured-output"; export { TunnelError } from "./tunnel"; export { ValidationError } from "./validation"; diff --git a/packages/appkit/src/errors/structured-output.ts b/packages/appkit/src/errors/structured-output.ts new file mode 100644 index 000000000..4c4c38ec0 --- /dev/null +++ b/packages/appkit/src/errors/structured-output.ts @@ -0,0 +1,47 @@ +import { AppKitError } from "./base"; + +/** + * Thrown when an agent with an `output` schema could not produce a + * schema-valid value within the structuring retry budget — the caller gets a + * valid object or this error, never partial data. Carries the last raw model + * output (`lastRaw`) for debugging, server-side only: it is never returned to + * the client, which sees the generic {@link clientMessage}. + * + * @example + * ```typescript + * try { + * const { output } = await runAgent(classifier, { messages: "..." }); + * } catch (e) { + * if (e instanceof StructuredOutputError) { + * // model couldn't produce schema-valid JSON; e.lastRaw has the attempt + * } + * } + * ``` + */ +export class StructuredOutputError extends AppKitError { + readonly code = "STRUCTURED_OUTPUT_ERROR"; + readonly statusCode = 500; + readonly isRetryable = false; + + /** + * Last raw model output that failed to parse/validate. Server-side only — + * kept off {@link clientMessage} since it can echo arbitrary model text. + */ + readonly lastRaw?: string; + + constructor( + message: string, + options?: { + cause?: Error; + context?: Record; + lastRaw?: string; + }, + ) { + super(message, options); + this.lastRaw = options?.lastRaw; + } + + override get clientMessage(): string { + return this._clientMessage ?? "The agent could not produce a valid result"; + } +} diff --git a/packages/appkit/src/index.ts b/packages/appkit/src/index.ts index 127cad384..f3bcded02 100644 --- a/packages/appkit/src/index.ts +++ b/packages/appkit/src/index.ts @@ -48,6 +48,7 @@ export { ExecutionError, InitializationError, ServerError, + StructuredOutputError, TunnelError, ValidationError, } from "./errors"; diff --git a/packages/appkit/src/plugins/agents/agents.ts b/packages/appkit/src/plugins/agents/agents.ts index 8ebbf32c3..6f4ff38d3 100644 --- a/packages/appkit/src/plugins/agents/agents.ts +++ b/packages/appkit/src/plugins/agents/agents.ts @@ -27,6 +27,10 @@ import type { ResolvedSkillCatalog, SkillDefinition, } from "../../core/agent/skills"; +import { + buildStructuringPass, + resolveStructuredOutput, +} from "../../core/agent/structured-output"; import { resolveToolkitFromProvider } from "../../core/agent/toolkit-resolver"; import { functionToolToDefinition, @@ -34,6 +38,7 @@ import { isHostedTool, resolveHostedTools, } from "../../core/agent/tools"; +import { toToolJSONSchema } from "../../core/agent/tools/json-schema"; import type { AgentDefinition, AgentsPluginConfig, @@ -65,6 +70,7 @@ import { initAgentTracing, linkTraceToRun, traceAgent, + traceTool, } from "./mlflow"; import { composePromptForAgent } from "./prompt"; import { @@ -493,6 +499,9 @@ export class AgentsPlugin extends Plugin implements ToolProvider { maxTokens: def.maxTokens, generationParams: def.generationParams, ephemeral: def.ephemeral, + output: def.output, + // Convert once here — the schema is fixed per agent; re-deriving per request would be wasted. + outputSchema: def.output ? toToolJSONSchema(def.output) : undefined, skills, }; } @@ -1332,6 +1341,24 @@ export class AgentsPlugin extends Plugin implements ToolProvider { outboundEvents.push(evt); } } + + // After the visible text, resolve structured output and emit one + // final `structured_output` event — any retries happen here, + // invisibly to the stream; a throw surfaces via the driver's catch. + if (registered.output) { + const data = await this.runStructuredOutput( + registered, + messagesWithSystem, + fullContent, + signal, + ); + for (const evt of translator.translate({ + type: "structured_output", + data, + })) { + outboundEvents.push(evt); + } + } }, ); @@ -1428,6 +1455,8 @@ export class AgentsPlugin extends Plugin implements ToolProvider { // Assigned inside the span below (the only place the active trace id // resolves), read into the response envelope after. let mlflowTraceId: string | undefined; + // Parsed structured output, when the agent declared an `output` schema. + let outputParsed: unknown; const runState: RunState = { req, @@ -1515,6 +1544,17 @@ export class AgentsPlugin extends Plugin implements ToolProvider { }); } + // Resolve structured output for the `output_parsed` envelope field; + // a throw is caught below as a 500. + if (registered.output) { + outputParsed = await this.runStructuredOutput( + registered, + messagesWithSystem, + fullContent, + signal, + ); + } + mlflowTraceId = currentTraceId(); }, ); @@ -1571,10 +1611,42 @@ export class AgentsPlugin extends Plugin implements ToolProvider { ...(runState.toolErrors.length > 0 ? { tool_errors: runState.toolErrors } : {}), + // Parsed structured output — the non-streaming counterpart of the + // `structured_output` SSE event. + ...(outputParsed !== undefined ? { output_parsed: outputParsed } : {}), output: [message], }); } + /** + * Resolves the agent's structured output from the run's final text, wrapped + * in a TOOL span so it nests under the turn's AGENT span. Assumes the agent + * declared an `output` schema. + */ + private runStructuredOutput( + registered: RegisteredAgent, + baseMessages: Message[], + finalText: string, + signal: AbortSignal, + ): Promise { + const { output: schema, outputSchema } = registered; + if (!schema || !outputSchema) { + throw new Error("runStructuredOutput called without an output schema"); + } + return traceTool("structured_output", { schema: outputSchema }, () => + resolveStructuredOutput({ + schema, + baseMessages, + finalText, + signal, + runStructuringPass: buildStructuringPass( + registered.adapter, + outputSchema, + ), + }), + ); + } + private dispatchSkillTool( entry: Extract, args: unknown, diff --git a/packages/appkit/src/plugins/agents/event-translator.ts b/packages/appkit/src/plugins/agents/event-translator.ts index ecaa2aab5..fb9980e70 100644 --- a/packages/appkit/src/plugins/agents/event-translator.ts +++ b/packages/appkit/src/plugins/agents/event-translator.ts @@ -60,6 +60,14 @@ export class AgentEventTranslator { sequence_number: this.seqNum++, }, ]; + case "structured_output": + return [ + { + type: "appkit.structured_output", + data: event.data, + sequence_number: this.seqNum++, + }, + ]; case "approval_pending": return [ { diff --git a/packages/appkit/src/plugins/agents/tests/event-translator.test.ts b/packages/appkit/src/plugins/agents/tests/event-translator.test.ts index 28fd474b3..2a974f64d 100644 --- a/packages/appkit/src/plugins/agents/tests/event-translator.test.ts +++ b/packages/appkit/src/plugins/agents/tests/event-translator.test.ts @@ -130,6 +130,20 @@ describe("AgentEventTranslator", () => { } }); + test("translates structured_output to appkit.structured_output extension event", () => { + const translator = new AgentEventTranslator(); + const events = translator.translate({ + type: "structured_output", + data: { category: "billing", urgent: true }, + }); + + expect(events).toHaveLength(1); + expect(events[0].type).toBe("appkit.structured_output"); + if (events[0].type === "appkit.structured_output") { + expect(events[0].data).toEqual({ category: "billing", urgent: true }); + } + }); + test("status:complete triggers finalize with response.completed", () => { const translator = new AgentEventTranslator(); translator.translate({ type: "message_delta", content: "Hi" }); diff --git a/packages/shared/src/agent.ts b/packages/shared/src/agent.ts index 811092b84..36978593a 100644 --- a/packages/shared/src/agent.ts +++ b/packages/shared/src/agent.ts @@ -129,6 +129,15 @@ export type AgentEvent = error?: string; } | { type: "metadata"; data: Record } + | { + /** + * Emitted by the agents plugin (not adapters) after the streamed text: + * the parsed, schema-validated object. Wire event: + * {@link AppKitStructuredOutputEvent}. + */ + type: "structured_output"; + data: unknown; + } | { /** * Emitted by the agents plugin (not adapters) when a mutating tool call @@ -245,6 +254,19 @@ export interface AppKitMetadataEvent { * arrives before the server-side timeout, the call is auto-denied and the * agent receives a denial string as the tool output. */ +/** + * Emitted once on `/chat`, after the streamed assistant text, when the agent + * declared an `output` schema. `data` is the parsed, schema-validated object. + * The `appkit.` prefix matches the other AppKit-injected wire events + * (`appkit.thinking`, `appkit.metadata`); the equivalent non-streaming field + * is `output_parsed`. + */ +export interface AppKitStructuredOutputEvent { + type: "appkit.structured_output"; + data: unknown; + sequence_number: number; +} + export interface AppKitApprovalPendingEvent { type: "appkit.approval_pending"; approval_id: string; @@ -264,6 +286,7 @@ export type ResponseStreamEvent = | ResponseFailedEvent | AppKitThinkingEvent | AppKitMetadataEvent + | AppKitStructuredOutputEvent | AppKitApprovalPendingEvent; // --------------------------------------------------------------------------- @@ -275,6 +298,14 @@ export interface AgentInput { tools: AgentToolDefinition[]; threadId: string; signal?: AbortSignal; + /** + * JSON Schema to constrain a tool-free completion to, for adapters that + * support server-side structured output (OpenAI-compatible `response_format`). + * Adapters that can't ignore it — the structured-output resolver then falls + * back to prompt + Zod validation. Already stripped of the top-level + * `$schema` key by {@link toToolJSONSchema}. + */ + outputSchema?: Record; /** * Adapter-specific opaque payloads, keyed by adapter namespace. The * shared contract intentionally does not enumerate keys — see each