From b769a8c5b110e10e4df1b4c7255729222df4cc42 Mon Sep 17 00:00:00 2001 From: ran Date: Tue, 28 Jul 2026 19:40:56 +0300 Subject: [PATCH 01/12] feat(responses): synthesize reasoning.encrypted_content when requested Real OpenAI returns an opaque `encrypted_content` blob on reasoning items only when a request opts in via `include: ["reasoning.encrypted_content"]` (the stateless-replay path used by agent-framework-openai >= 1.11.0). aimock never modeled it, so a replayed reasoning item carried no blob. On that path agent-framework-openai >= 1.11.0 hard-fails a reasoning + multi-tool chain on the follow-up request (microsoft/agent-framework#7233): it parses a reasoning item with an id but no protected_data and rejects the request before sending. Emit a deterministic synthetic encrypted_content on reasoning items, gated on the request's `include` so only opted-in requests see it (mirrors real OpenAI; every other consumer's replay stays byte-identical). Covers the streaming and non-streaming reasoning builders. --- src/__tests__/responses.test.ts | 81 +++++++++++++++++++++++++++++ src/responses.ts | 91 ++++++++++++++++++++++++++++++--- 2 files changed, 165 insertions(+), 7 deletions(-) diff --git a/src/__tests__/responses.test.ts b/src/__tests__/responses.test.ts index 1139fdbf..f80e25b8 100644 --- a/src/__tests__/responses.test.ts +++ b/src/__tests__/responses.test.ts @@ -1878,6 +1878,87 @@ describe("reasoning_summary_text.done includes item_id", () => { }); }); +// ─── reasoning.encrypted_content (microsoft/agent-framework#7233) ──────────── +// +// Real OpenAI returns an opaque `encrypted_content` blob on reasoning items only +// when the request opts in via `include: ["reasoning.encrypted_content"]` (the +// stateless-replay path used by agent-framework-openai >= 1.11.0). aimock now +// synthesizes a placeholder so that client can replay the reasoning item instead +// of hard-failing the reasoning + multi-tool follow-up request. + +describe("reasoning.encrypted_content emission", () => { + function reasoningItems(events: SSEEvent[]): { type: string; encrypted_content?: string }[] { + return events + .filter( + (e) => + (e.type === "response.output_item.added" || e.type === "response.output_item.done") && + (e.item as { type?: string })?.type === "reasoning", + ) + .map((e) => e.item as { type: string; encrypted_content?: string }); + } + + it("builder omits encrypted_content by default (flag off)", () => { + const events = buildTextStreamEvents("result", "gpt-5-test", 100, "thinking..."); + const items = reasoningItems(events); + expect(items.length).toBeGreaterThan(0); + for (const item of items) expect(item.encrypted_content).toBeUndefined(); + }); + + it("builder emits encrypted_content on the done item when flag on", () => { + const events = buildTextStreamEvents( + "result", + "gpt-5-test", + 100, + "thinking...", + undefined, + undefined, + true, + ); + const done = events.find( + (e) => + e.type === "response.output_item.done" && + (e.item as { type?: string })?.type === "reasoning", + ); + expect(done).toBeDefined(); + const item = done!.item as { id: string; encrypted_content?: string }; + expect(typeof item.encrypted_content).toBe("string"); + expect(item.encrypted_content!.length).toBeGreaterThan(0); + }); + + const reasoningFixture: Fixture = { + match: { userMessage: "compare" }, + response: { content: "AAPL vs MSFT", reasoning: "Look up AAPL, then MSFT." }, + }; + + it("streams encrypted_content when the request includes reasoning.encrypted_content", async () => { + instance = await createServer([reasoningFixture]); + const res = await post(`${instance.url}/v1/responses`, { + model: "gpt-5-test", + input: [{ role: "user", content: "compare" }], + stream: true, + include: ["reasoning.encrypted_content"], + }); + expect(res.status).toBe(200); + const items = reasoningItems(parseResponsesSSEEvents(res.body)); + const done = items.find((i) => i.encrypted_content !== undefined); + expect(done).toBeDefined(); + expect(typeof done!.encrypted_content).toBe("string"); + }); + + it("omits encrypted_content when the request does not opt in", async () => { + instance = await createServer([reasoningFixture]); + const res = await post(`${instance.url}/v1/responses`, { + model: "gpt-5-test", + input: [{ role: "user", content: "compare" }], + stream: true, + }); + expect(res.status).toBe(200); + const items = reasoningItems(parseResponsesSSEEvents(res.body)); + expect(items.length).toBeGreaterThan(0); + for (const item of items) expect(item.encrypted_content).toBeUndefined(); + }); +}); + // ─── Bug fix: multi-fco after single item_reference ───────────────────────── describe("multi-fco after single item_reference", () => { diff --git a/src/responses.ts b/src/responses.ts index 5d0149ed..17b8b581 100644 --- a/src/responses.ts +++ b/src/responses.ts @@ -292,6 +292,7 @@ export function buildTextStreamEvents( reasoning?: string, webSearches?: string[], overrides?: ResponseOverrides, + emitEncryptedReasoning = false, ): ResponsesSSEEvent[] { const { respId, created, events, prefixOutputItems, nextOutputIndex } = buildResponsePreamble( model, @@ -299,6 +300,7 @@ export function buildTextStreamEvents( reasoning, webSearches, overrides, + emitEncryptedReasoning, ); const { events: msgEvents, msgItem } = buildMessageOutputEvents( @@ -331,6 +333,7 @@ export function buildToolCallStreamEvents( reasoning?: string, webSearches?: string[], overrides?: ResponseOverrides, + emitEncryptedReasoning = false, ): ResponsesSSEEvent[] { const { respId, created, events, prefixOutputItems, nextOutputIndex } = buildResponsePreamble( model, @@ -338,6 +341,7 @@ export function buildToolCallStreamEvents( reasoning, webSearches, overrides, + emitEncryptedReasoning, ); const fcOutputItems: object[] = []; @@ -418,10 +422,47 @@ export function buildToolCallStreamEvents( return events; } +/** + * Whether the incoming Responses request opted into encrypted reasoning via + * `include: ["reasoning.encrypted_content"]`. Real OpenAI only returns the + * encrypted reasoning blob when the caller asks for it this way — the + * stateless-replay path used by `agent-framework-openai` >= 1.11.0. Gating on it + * keeps every other consumer's replay byte-identical. + */ +function requestOptsIntoEncryptedReasoning(req: ResponsesRequest): boolean { + const include = (req as { include?: unknown }).include; + return Array.isArray(include) && include.includes("reasoning.encrypted_content"); +} + +/** + * Synthetic stand-in for OpenAI's `reasoning.encrypted_content`. aimock has no + * real ciphertext, so it emits a deterministic placeholder keyed on the + * reasoning id. Consumers store it (e.g. as agent-framework `protected_data`) + * and replay it back verbatim on the next request; aimock matches on content and + * ignores the echoed blob, so any stable string is safe. Emitted only when the + * request opts in (see `requestOptsIntoEncryptedReasoning`). Without it, + * agent-framework-openai >= 1.11.0 hard-fails a reasoning + multi-tool chain on + * the follow-up request. See microsoft/agent-framework#7233. + */ +function syntheticEncryptedReasoning(reasoningId: string): string { + return `aimock-encrypted-reasoning:${reasoningId}`; +} + +/** Spreadable `encrypted_content` field for a reasoning item, or empty when off. */ +function encryptedReasoningField( + reasoningId: string, + emitEncryptedReasoning: boolean, +): Record { + return emitEncryptedReasoning + ? { encrypted_content: syntheticEncryptedReasoning(reasoningId) } + : {}; +} + function buildReasoningStreamEvents( reasoning: string, model: string, chunkSize: number, + emitEncryptedReasoning = false, ): ResponsesSSEEvent[] { const reasoningId = generateId("rs"); const events: ResponsesSSEEvent[] = []; @@ -433,6 +474,7 @@ function buildReasoningStreamEvents( type: "reasoning", id: reasoningId, summary: [], + ...encryptedReasoningField(reasoningId, emitEncryptedReasoning), }, }); @@ -478,6 +520,7 @@ function buildReasoningStreamEvents( type: "reasoning", id: reasoningId, summary: [{ type: "summary_text", text: reasoning }], + ...encryptedReasoningField(reasoningId, emitEncryptedReasoning), }, }); @@ -536,6 +579,7 @@ function buildResponsePreamble( reasoning?: string, webSearches?: string[], overrides?: ResponseOverrides, + emitEncryptedReasoning = false, ): PreambleResult { const respId = overrides?.id ?? responseId(); const created = overrides?.created ?? Math.floor(Date.now() / 1000); @@ -568,7 +612,12 @@ function buildResponsePreamble( }); if (reasoning) { - const reasoningEvents = buildReasoningStreamEvents(reasoning, model, chunkSize); + const reasoningEvents = buildReasoningStreamEvents( + reasoning, + model, + chunkSize, + emitEncryptedReasoning, + ); events.push(...reasoningEvents); const doneEvent = reasoningEvents.find( (e) => @@ -725,14 +774,21 @@ function buildFunctionCallOutputEvents( // ─── Non-streaming response builders ──────────────────────────────────────── -function buildOutputPrefix(content: string, reasoning?: string, webSearches?: string[]): object[] { +function buildOutputPrefix( + content: string, + reasoning?: string, + webSearches?: string[], + emitEncryptedReasoning = false, +): object[] { const output: object[] = []; if (reasoning) { + const reasoningId = generateId("rs"); output.push({ type: "reasoning", - id: generateId("rs"), + id: reasoningId, summary: [{ type: "summary_text", text: reasoning }], + ...encryptedReasoningField(reasoningId, emitEncryptedReasoning), }); } @@ -780,10 +836,11 @@ function buildTextResponse( reasoning?: string, webSearches?: string[], overrides?: ResponseOverrides, + emitEncryptedReasoning = false, ): object { return buildResponseEnvelope( model, - buildOutputPrefix(content, reasoning, webSearches), + buildOutputPrefix(content, reasoning, webSearches, emitEncryptedReasoning), overrides, ); } @@ -794,13 +851,16 @@ function buildToolCallResponse( reasoning?: string, webSearches?: string[], overrides?: ResponseOverrides, + emitEncryptedReasoning = false, ): object { const output: object[] = []; if (reasoning) { + const reasoningId = generateId("rs"); output.push({ type: "reasoning", - id: generateId("rs"), + id: reasoningId, summary: [{ type: "summary_text", text: reasoning }], + ...encryptedReasoningField(reasoningId, emitEncryptedReasoning), }); } if (webSearches && webSearches.length > 0) { @@ -835,6 +895,7 @@ export function buildContentWithToolCallsStreamEvents( webSearches?: string[], overrides?: ResponseOverrides, blocks?: FixtureBlock[], + emitEncryptedReasoning = false, ): ResponsesSSEEvent[] { const { respId, created, events, prefixOutputItems, nextOutputIndex } = buildResponsePreamble( model, @@ -842,6 +903,7 @@ export function buildContentWithToolCallsStreamEvents( reasoning, webSearches, overrides, + emitEncryptedReasoning, ); // The output items assembled in emission order (after any reasoning / @@ -944,6 +1006,7 @@ function buildContentWithToolCallsResponse( webSearches?: string[], overrides?: ResponseOverrides, blocks?: FixtureBlock[], + emitEncryptedReasoning = false, ): object { if (blocks && blocks.length > 0) { // NEW PATH: the non-streaming `output[]` array is positionally observable, @@ -954,10 +1017,12 @@ function buildContentWithToolCallsResponse( const ordered = resolveFixtureBlocks(blocks); const output: object[] = []; if (reasoning) { + const reasoningId = generateId("rs"); output.push({ type: "reasoning", - id: generateId("rs"), + id: reasoningId, summary: [{ type: "summary_text", text: reasoning }], + ...encryptedReasoningField(reasoningId, emitEncryptedReasoning), }); } if (webSearches && webSearches.length > 0) { @@ -987,7 +1052,7 @@ function buildContentWithToolCallsResponse( } // LEGACY PATH: message item first, then function_call items — unchanged. - const output = buildOutputPrefix(content, reasoning, webSearches); + const output = buildOutputPrefix(content, reasoning, webSearches, emitEncryptedReasoning); for (const tc of toolCalls) { output.push(buildFunctionCallOutputItem(tc)); } @@ -1085,6 +1150,12 @@ export async function handleResponses( completionReq._endpointType = "chat"; completionReq._context = getContext(req); + // Emit a synthetic `reasoning.encrypted_content` only when the request opted in + // via `include` (mirrors real OpenAI). This is what agent-framework-openai + // >= 1.11.0 sends on stateless-replay requests; without the blob it hard-fails + // a reasoning + multi-tool chain. See microsoft/agent-framework#7233. + const emitEncryptedReasoning = requestOptsIntoEncryptedReasoning(responsesReq); + const testId = getTestId(req); const { fixture, skippedBySequenceOrTurn } = matchFixtureDiagnostic( fixtures, @@ -1261,6 +1332,7 @@ export async function handleResponses( response.webSearches, overrides, response.blocks, + emitEncryptedReasoning, ); res.writeHead(200, { "Content-Type": "application/json" }); res.end(JSON.stringify(body)); @@ -1274,6 +1346,7 @@ export async function handleResponses( response.webSearches, overrides, response.blocks, + emitEncryptedReasoning, ); const interruption = createInterruptionSignal(fixture); const completed = await writeResponsesSSEStream(res, events, { @@ -1319,6 +1392,7 @@ export async function handleResponses( effReasoning, response.webSearches, overrides, + emitEncryptedReasoning, ); res.writeHead(200, { "Content-Type": "application/json" }); res.end(JSON.stringify(body)); @@ -1330,6 +1404,7 @@ export async function handleResponses( effReasoning, response.webSearches, overrides, + emitEncryptedReasoning, ); const interruption = createInterruptionSignal(fixture); const completed = await writeResponsesSSEStream(res, events, { @@ -1375,6 +1450,7 @@ export async function handleResponses( effReasoning, response.webSearches, overrides, + emitEncryptedReasoning, ); res.writeHead(200, { "Content-Type": "application/json" }); res.end(JSON.stringify(body)); @@ -1386,6 +1462,7 @@ export async function handleResponses( effReasoning, response.webSearches, overrides, + emitEncryptedReasoning, ); const interruption = createInterruptionSignal(fixture); const completed = await writeResponsesSSEStream(res, events, { From 1157c54173c5ca2181ed7fa41726f76f56c3b891 Mon Sep 17 00:00:00 2001 From: ran Date: Wed, 29 Jul 2026 10:42:52 +0300 Subject: [PATCH 02/12] =?UTF-8?q?fix(responses):=20address=20review=20?= =?UTF-8?q?=E2=80=94=20blob=20only=20on=20done,=20store=20gate,=20WS,=20co?= =?UTF-8?q?verage?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Follow-up to review on #342: - encrypted_content now only on the terminal reasoning item (done / non-streaming output[]), never on `added` — matches real OpenAI and avoids masking clients that read reasoning at `added` (langchain-ai/langchainjs#10844). - Gate on `include: ["reasoning.encrypted_content"]` OR stateless `store: false` (openai-python 2.46.0 made the blob default-on for non-stored responses); was include-only. - Wire the WebSocket Responses transport too (export the gate, thread `include` / `store` through the rebuilt request) so #7233 doesn't still reproduce over WS. - base64 the placeholder (opaque like the real field); declare `include` / `store` on ResponsesRequest instead of casting; drop the unused `model` param; collapse the 5 hand-built reasoning items into one `buildReasoningOutputItem`. - Tests: pin added-vs-done placement, id→blob determinism, both gate triggers, response.completed.output, and all three non-streaming builders; add an opted-in drift leg. CHANGELOG entry added. --- CHANGELOG.md | 1 + src/__tests__/drift/openai-responses.drift.ts | 34 ++++ src/__tests__/responses.test.ts | 158 +++++++++++++----- src/responses.ts | 141 +++++++++------- src/ws-responses.ts | 10 ++ 5 files changed, 236 insertions(+), 108 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index dfd4df7b..52cd1aa7 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -4,6 +4,7 @@ ### Added +- **Reasoning `encrypted_content` on the Responses API.** aimock now synthesizes an opaque (base64) `reasoning.encrypted_content` blob on the TERMINAL reasoning item (`response.output_item.done` and non-streaming `output[]`), never on the in-progress `added` item — matching real OpenAI (a populated blob at `added` would mask clients that read reasoning before `done`, e.g. langchain-ai/langchainjs#10844). Emitted only when the request wants it: `include: ["reasoning.encrypted_content"]` OR a stateless request (`store: false` / ZDR). Stored / opted-out replays stay byte-identical. This lets `agent-framework-openai` >= 1.11.0 replay reasoning-paired tool calls instead of hard-failing the follow-up on a stateless reasoning + multi-tool chain (root cause: microsoft/agent-framework#7233). Applies to both the HTTP and WebSocket Responses transports. Known limitation: aimock skips inbound reasoning items, so it cannot reproduce OpenAI's `invalid_encrypted_content` rejection of a mismatched blob/id. - **OpenRouter chat / LLM router simulation.** Requests whose original path starts with `/api/v1/` (point any OpenAI SDK at a `baseURL` ending `/api/v1`) are detected as OpenRouter and shaped to match real OpenRouter bytes: a `gen-` id prefix (a fixture `id` override still wins), a top-level `provider` (default = the winning model slug's author, fixture-overridable via `provider`), both `finish_reason` and `native_finish_reason` on every choice/delta, an always-present `system_fingerprint` and `service_tier` (null by default), `message.reasoning`, and a rich `usage` with a **fixture-scriptable** `cost` + `cost_details` (emitted only when a fixture supplies a cost — never fabricated), plus `is_byok` / `prompt_tokens_details` / `completion_tokens_details` when overridden. Callers on the plain OpenAI `/v1/...` base are byte-for-byte unchanged. - **`models[]` fallback (router failover) simulation.** When the request body carries `models: [...]`, aimock walks `[model, ...models]` in order and serves the first fixture returning a NON-error response; a `429`/`503` error fixture on a candidate simulates a runtime provider failure and falls through to the next. The winning slug is echoed back as the top-level `model`, so a test asserts failover by reading `response.model`. (Deliberate non-goal: an unknown/invalid model is a fixture miss, not OpenRouter's up-front invalid-model 400.) - **OpenRouter discovery endpoints** `GET /api/v1/models` (catalog synthesized from loaded chat fixtures), `GET /api/v1/key`, and `GET /api/v1/credits`, matching the real response shapes. diff --git a/src/__tests__/drift/openai-responses.drift.ts b/src/__tests__/drift/openai-responses.drift.ts index 6d5231cf..43a4396b 100644 --- a/src/__tests__/drift/openai-responses.drift.ts +++ b/src/__tests__/drift/openai-responses.drift.ts @@ -558,6 +558,40 @@ describe("OpenAI Responses API reasoning drift", () => { expect(reasoningAdded, "no output_item.added with type=reasoning").toBeDefined(); }); + it("opted-in reasoning carries encrypted_content ONLY on the terminal item", async () => { + // Real OpenAI populates reasoning.encrypted_content on the `done` item (and + // response.completed.output), never on `added`, when the caller opts in via + // include or a stateless (store:false) request. See microsoft/agent-framework#7233. + const res = await httpPost(`${reasoningInstance.url}/v1/responses`, { + model: "gpt-4o-mini", + input: [{ role: "user", content: "Think carefully" }], + stream: true, + include: ["reasoning.encrypted_content"], + }); + expect(res.status).toBe(200); + + const events = parseTypedSSE(res.body); + const reasoningItemOf = (kind: "added" | "done") => + events.find( + (e) => + e.type === `response.output_item.${kind}` && + (e.data as { item?: { type?: string } }).item?.type === "reasoning", + )?.data as { item?: { encrypted_content?: string } } | undefined; + + const added = reasoningItemOf("added"); + const done = reasoningItemOf("done"); + expect(added?.item, "no reasoning output_item.added").toBeDefined(); + expect(done?.item, "no reasoning output_item.done").toBeDefined(); + expect( + added!.item!.encrypted_content, + "encrypted_content must be absent on added", + ).toBeUndefined(); + expect(typeof done!.item!.encrypted_content, "encrypted_content must be present on done").toBe( + "string", + ); + expect(done!.item!.encrypted_content!.length).toBeGreaterThan(0); + }); + it("reasoning event shapes include item_id, output_index, summary_index", async () => { const res = await httpPost(`${reasoningInstance.url}/v1/responses`, { model: "gpt-4o-mini", diff --git a/src/__tests__/responses.test.ts b/src/__tests__/responses.test.ts index f80e25b8..cf5f25d8 100644 --- a/src/__tests__/responses.test.ts +++ b/src/__tests__/responses.test.ts @@ -1880,31 +1880,58 @@ describe("reasoning_summary_text.done includes item_id", () => { // ─── reasoning.encrypted_content (microsoft/agent-framework#7233) ──────────── // -// Real OpenAI returns an opaque `encrypted_content` blob on reasoning items only -// when the request opts in via `include: ["reasoning.encrypted_content"]` (the -// stateless-replay path used by agent-framework-openai >= 1.11.0). aimock now -// synthesizes a placeholder so that client can replay the reasoning item instead +// Real OpenAI populates an opaque `encrypted_content` blob on the TERMINAL +// reasoning item (never the in-progress `added` item) when the request opts in +// via `include: ["reasoning.encrypted_content"]` OR is stateless (`store:false`) +// — the path used by agent-framework-openai >= 1.11.0. aimock synthesizes a +// deterministic placeholder so that client can replay the reasoning item instead // of hard-failing the reasoning + multi-tool follow-up request. describe("reasoning.encrypted_content emission", () => { - function reasoningItems(events: SSEEvent[]): { type: string; encrypted_content?: string }[] { - return events - .filter( - (e) => - (e.type === "response.output_item.added" || e.type === "response.output_item.done") && - (e.item as { type?: string })?.type === "reasoning", - ) - .map((e) => e.item as { type: string; encrypted_content?: string }); + // Mirror of `syntheticEncryptedReasoning` in responses.ts — pins the id→blob + // relationship so a non-deterministic (e.g. Math.random) blob is caught. + const expectedBlob = (id: string): string => + Buffer.from(`aimock-encrypted-reasoning:${id}`).toString("base64"); + + function reasoningItem( + events: SSEEvent[], + kind: "added" | "done", + ): { id: string; encrypted_content?: string } | undefined { + const ev = events.find( + (e) => + e.type === `response.output_item.${kind}` && + (e.item as { type?: string })?.type === "reasoning", + ); + return ev?.item as { id: string; encrypted_content?: string } | undefined; } - it("builder omits encrypted_content by default (flag off)", () => { + const reasoningTextFixture: Fixture = { + match: { userMessage: "compare" }, + response: { content: "AAPL vs MSFT", reasoning: "Look up AAPL, then MSFT." }, + }; + const reasoningToolFixture: Fixture = { + match: { userMessage: "toolreason" }, + response: { toolCalls: [{ name: "get_stock", arguments: "{}" }], reasoning: "Call the tool." }, + }; + const reasoningContentToolFixture: Fixture = { + match: { userMessage: "bothreason" }, + response: { + content: "done", + toolCalls: [{ name: "get_stock", arguments: "{}" }], + reasoning: "Think, then call.", + }, + }; + + // --- Builder-level: placement (added vs done) + determinism --- + + it("builder omits encrypted_content everywhere when the flag is off", () => { const events = buildTextStreamEvents("result", "gpt-5-test", 100, "thinking..."); - const items = reasoningItems(events); - expect(items.length).toBeGreaterThan(0); - for (const item of items) expect(item.encrypted_content).toBeUndefined(); + expect(reasoningItem(events, "added")!.encrypted_content).toBeUndefined(); + expect(reasoningItem(events, "done")!.encrypted_content).toBeUndefined(); }); - it("builder emits encrypted_content on the done item when flag on", () => { + it("builder emits the blob ONLY on the done item, and it is deterministic by id", () => { + // trailing `true` = emitEncryptedReasoning const events = buildTextStreamEvents( "result", "gpt-5-test", @@ -1914,48 +1941,87 @@ describe("reasoning.encrypted_content emission", () => { undefined, true, ); - const done = events.find( - (e) => - e.type === "response.output_item.done" && - (e.item as { type?: string })?.type === "reasoning", - ); - expect(done).toBeDefined(); - const item = done!.item as { id: string; encrypted_content?: string }; - expect(typeof item.encrypted_content).toBe("string"); - expect(item.encrypted_content!.length).toBeGreaterThan(0); + const added = reasoningItem(events, "added")!; + const done = reasoningItem(events, "done")!; + expect(added.encrypted_content).toBeUndefined(); // never on `added` (langchainjs#10844) + expect(done.encrypted_content).toBe(expectedBlob(done.id)); // pins presence + determinism }); - const reasoningFixture: Fixture = { - match: { userMessage: "compare" }, - response: { content: "AAPL vs MSFT", reasoning: "Look up AAPL, then MSFT." }, - }; + // --- HTTP streaming: gate triggers + completed.output --- - it("streams encrypted_content when the request includes reasoning.encrypted_content", async () => { - instance = await createServer([reasoningFixture]); - const res = await post(`${instance.url}/v1/responses`, { + async function streamReasoning(body: Record): Promise { + instance = await createServer([reasoningTextFixture]); + const res = await post(`${instance!.url}/v1/responses`, { model: "gpt-5-test", input: [{ role: "user", content: "compare" }], stream: true, - include: ["reasoning.encrypted_content"], + ...body, }); expect(res.status).toBe(200); - const items = reasoningItems(parseResponsesSSEEvents(res.body)); - const done = items.find((i) => i.encrypted_content !== undefined); - expect(done).toBeDefined(); - expect(typeof done!.encrypted_content).toBe("string"); + return parseResponsesSSEEvents(res.body); + } + + it("emits the blob when the request opts in via include", async () => { + const events = await streamReasoning({ include: ["reasoning.encrypted_content"] }); + expect(reasoningItem(events, "added")!.encrypted_content).toBeUndefined(); + const done = reasoningItem(events, "done")!; + expect(done.encrypted_content).toBe(expectedBlob(done.id)); + // Also present on the reasoning item inside response.completed.output. + const completed = events.find((e) => e.type === "response.completed") as SSEEvent & { + response: { output: { type: string; encrypted_content?: string }[] }; + }; + const outputReasoning = completed.response.output.find((o) => o.type === "reasoning"); + expect(outputReasoning!.encrypted_content).toBe(done.encrypted_content); }); - it("omits encrypted_content when the request does not opt in", async () => { - instance = await createServer([reasoningFixture]); - const res = await post(`${instance.url}/v1/responses`, { + it("emits the blob for a stateless request (store:false) even without include", async () => { + const events = await streamReasoning({ store: false }); + const done = reasoningItem(events, "done")!; + expect(done.encrypted_content).toBe(expectedBlob(done.id)); + }); + + it("omits the blob for a stored request that did not opt in", async () => { + const events = await streamReasoning({}); + expect(reasoningItem(events, "added")!.encrypted_content).toBeUndefined(); + expect(reasoningItem(events, "done")!.encrypted_content).toBeUndefined(); + }); + + // --- Non-streaming builders: all three carry the blob when opted in --- + + async function nonStreamReasoning( + fixture: Fixture, + userMessage: string, + ): Promise<{ type: string; encrypted_content?: string; id: string }[]> { + instance = await createServer([fixture]); + const res = await post(`${instance!.url}/v1/responses`, { model: "gpt-5-test", - input: [{ role: "user", content: "compare" }], - stream: true, + input: [{ role: "user", content: userMessage }], + stream: false, + include: ["reasoning.encrypted_content"], }); expect(res.status).toBe(200); - const items = reasoningItems(parseResponsesSSEEvents(res.body)); - expect(items.length).toBeGreaterThan(0); - for (const item of items) expect(item.encrypted_content).toBeUndefined(); + const body = JSON.parse(res.body) as { + output: { type: string; encrypted_content?: string; id: string }[]; + }; + return body.output; + } + + it("non-streaming text response carries the blob (buildOutputPrefix)", async () => { + const output = await nonStreamReasoning(reasoningTextFixture, "compare"); + const r = output.find((o) => o.type === "reasoning")!; + expect(r.encrypted_content).toBe(expectedBlob(r.id)); + }); + + it("non-streaming tool-call response carries the blob (buildToolCallResponse)", async () => { + const output = await nonStreamReasoning(reasoningToolFixture, "toolreason"); + const r = output.find((o) => o.type === "reasoning")!; + expect(r.encrypted_content).toBe(expectedBlob(r.id)); + }); + + it("non-streaming content+tool response carries the blob (buildContentWithToolCallsResponse)", async () => { + const output = await nonStreamReasoning(reasoningContentToolFixture, "bothreason"); + const r = output.find((o) => o.type === "reasoning")!; + expect(r.encrypted_content).toBe(expectedBlob(r.id)); }); }); diff --git a/src/responses.ts b/src/responses.ts index 17b8b581..acfee10b 100644 --- a/src/responses.ts +++ b/src/responses.ts @@ -74,6 +74,12 @@ interface ResponsesRequest { temperature?: number; max_output_tokens?: number; response_format?: { type: string; [key: string]: unknown }; + // Additional output data requested by the caller, e.g. + // "reasoning.encrypted_content" (gates encrypted-reasoning emission). + include?: string[]; + // Server-side storage flag. `false` (ZDR / stateless replay) is one of the + // triggers for encrypted reasoning; see `requestWantsEncryptedReasoning`. + store?: boolean; [key: string]: unknown; } @@ -423,44 +429,65 @@ export function buildToolCallStreamEvents( } /** - * Whether the incoming Responses request opted into encrypted reasoning via - * `include: ["reasoning.encrypted_content"]`. Real OpenAI only returns the - * encrypted reasoning blob when the caller asks for it this way — the - * stateless-replay path used by `agent-framework-openai` >= 1.11.0. Gating on it - * keeps every other consumer's replay byte-identical. + * Whether the incoming Responses request should receive encrypted reasoning. + * + * Real OpenAI populates `reasoning.encrypted_content` when the caller opts in + * with `include: ["reasoning.encrypted_content"]` OR whenever the response is + * not server-side stored (`store: false` / ZDR) — the latter became the default + * trigger in openai-python 2.46.0. `agent-framework-openai` >= 1.11.0 drives its + * stateless-replay path with both signals set. Gating on either keeps aimock in + * step with real OpenAI while leaving stored / opted-out replays byte-identical. + * Exported so the WebSocket Responses transport can share one gate. */ -function requestOptsIntoEncryptedReasoning(req: ResponsesRequest): boolean { - const include = (req as { include?: unknown }).include; - return Array.isArray(include) && include.includes("reasoning.encrypted_content"); +export function requestWantsEncryptedReasoning(req: ResponsesRequest): boolean { + const include = req.include; + if (Array.isArray(include) && include.includes("reasoning.encrypted_content")) return true; + return req.store === false; } /** - * Synthetic stand-in for OpenAI's `reasoning.encrypted_content`. aimock has no - * real ciphertext, so it emits a deterministic placeholder keyed on the - * reasoning id. Consumers store it (e.g. as agent-framework `protected_data`) - * and replay it back verbatim on the next request; aimock matches on content and - * ignores the echoed blob, so any stable string is safe. Emitted only when the - * request opts in (see `requestOptsIntoEncryptedReasoning`). Without it, + * Synthetic stand-in for OpenAI's opaque `reasoning.encrypted_content`. aimock + * has no real ciphertext; it emits a base64 placeholder derived from the + * reasoning id (stable for a given item, opaque like the real field). Consumers + * store it (e.g. as agent-framework `protected_data`) and replay it verbatim; + * aimock matches on content and ignores the echoed blob. Without it, * agent-framework-openai >= 1.11.0 hard-fails a reasoning + multi-tool chain on * the follow-up request. See microsoft/agent-framework#7233. + * + * aimock skips inbound reasoning items entirely, so it cannot reproduce + * OpenAI's `invalid_encrypted_content` rejection of a mismatched blob/id — a + * known replay-only limitation. */ function syntheticEncryptedReasoning(reasoningId: string): string { - return `aimock-encrypted-reasoning:${reasoningId}`; + return Buffer.from(`aimock-encrypted-reasoning:${reasoningId}`).toString("base64"); } -/** Spreadable `encrypted_content` field for a reasoning item, or empty when off. */ -function encryptedReasoningField( +/** + * Build a Responses `reasoning` output item. Pass `summaryText` for the terminal + * (`done` / non-streaming) item — it carries the summary and, when + * `emitEncrypted`, the encrypted blob. Omit `summaryText` for the in-progress + * (`added`) item: empty summary and — matching real OpenAI — never a blob (a + * populated blob at `added` would mask clients that read reasoning before + * `done`; see langchain-ai/langchainjs#10844). + */ +function buildReasoningOutputItem( reasoningId: string, - emitEncryptedReasoning: boolean, -): Record { - return emitEncryptedReasoning - ? { encrypted_content: syntheticEncryptedReasoning(reasoningId) } - : {}; + opts: { summaryText?: string; emitEncrypted?: boolean } = {}, +): Record { + const item: Record = { + type: "reasoning", + id: reasoningId, + summary: + opts.summaryText === undefined ? [] : [{ type: "summary_text", text: opts.summaryText }], + }; + if (opts.summaryText !== undefined && opts.emitEncrypted) { + item.encrypted_content = syntheticEncryptedReasoning(reasoningId); + } + return item; } function buildReasoningStreamEvents( reasoning: string, - model: string, chunkSize: number, emitEncryptedReasoning = false, ): ResponsesSSEEvent[] { @@ -470,12 +497,7 @@ function buildReasoningStreamEvents( events.push({ type: "response.output_item.added", output_index: 0, - item: { - type: "reasoning", - id: reasoningId, - summary: [], - ...encryptedReasoningField(reasoningId, emitEncryptedReasoning), - }, + item: buildReasoningOutputItem(reasoningId), }); events.push({ @@ -516,12 +538,10 @@ function buildReasoningStreamEvents( events.push({ type: "response.output_item.done", output_index: 0, - item: { - type: "reasoning", - id: reasoningId, - summary: [{ type: "summary_text", text: reasoning }], - ...encryptedReasoningField(reasoningId, emitEncryptedReasoning), - }, + item: buildReasoningOutputItem(reasoningId, { + summaryText: reasoning, + emitEncrypted: emitEncryptedReasoning, + }), }); return events; @@ -614,7 +634,6 @@ function buildResponsePreamble( if (reasoning) { const reasoningEvents = buildReasoningStreamEvents( reasoning, - model, chunkSize, emitEncryptedReasoning, ); @@ -783,13 +802,12 @@ function buildOutputPrefix( const output: object[] = []; if (reasoning) { - const reasoningId = generateId("rs"); - output.push({ - type: "reasoning", - id: reasoningId, - summary: [{ type: "summary_text", text: reasoning }], - ...encryptedReasoningField(reasoningId, emitEncryptedReasoning), - }); + output.push( + buildReasoningOutputItem(generateId("rs"), { + summaryText: reasoning, + emitEncrypted: emitEncryptedReasoning, + }), + ); } if (webSearches && webSearches.length > 0) { @@ -855,13 +873,12 @@ function buildToolCallResponse( ): object { const output: object[] = []; if (reasoning) { - const reasoningId = generateId("rs"); - output.push({ - type: "reasoning", - id: reasoningId, - summary: [{ type: "summary_text", text: reasoning }], - ...encryptedReasoningField(reasoningId, emitEncryptedReasoning), - }); + output.push( + buildReasoningOutputItem(generateId("rs"), { + summaryText: reasoning, + emitEncrypted: emitEncryptedReasoning, + }), + ); } if (webSearches && webSearches.length > 0) { for (const query of webSearches) { @@ -1017,13 +1034,12 @@ function buildContentWithToolCallsResponse( const ordered = resolveFixtureBlocks(blocks); const output: object[] = []; if (reasoning) { - const reasoningId = generateId("rs"); - output.push({ - type: "reasoning", - id: reasoningId, - summary: [{ type: "summary_text", text: reasoning }], - ...encryptedReasoningField(reasoningId, emitEncryptedReasoning), - }); + output.push( + buildReasoningOutputItem(generateId("rs"), { + summaryText: reasoning, + emitEncrypted: emitEncryptedReasoning, + }), + ); } if (webSearches && webSearches.length > 0) { for (const query of webSearches) { @@ -1150,11 +1166,12 @@ export async function handleResponses( completionReq._endpointType = "chat"; completionReq._context = getContext(req); - // Emit a synthetic `reasoning.encrypted_content` only when the request opted in - // via `include` (mirrors real OpenAI). This is what agent-framework-openai - // >= 1.11.0 sends on stateless-replay requests; without the blob it hard-fails - // a reasoning + multi-tool chain. See microsoft/agent-framework#7233. - const emitEncryptedReasoning = requestOptsIntoEncryptedReasoning(responsesReq); + // Emit a synthetic `reasoning.encrypted_content` only when the request wants it + // (opted in via `include`, or stateless `store: false` — mirrors real OpenAI). + // This is what agent-framework-openai >= 1.11.0 sends on stateless-replay + // requests; without the blob it hard-fails a reasoning + multi-tool chain. + // See microsoft/agent-framework#7233. + const emitEncryptedReasoning = requestWantsEncryptedReasoning(responsesReq); const testId = getTestId(req); const { fixture, skippedBySequenceOrTurn } = matchFixtureDiagnostic( diff --git a/src/ws-responses.ts b/src/ws-responses.ts index acef2f13..f1f2ea70 100644 --- a/src/ws-responses.ts +++ b/src/ws-responses.ts @@ -10,6 +10,7 @@ import type { ChatCompletionRequest, Fixture } from "./types.js"; import { matchFixtureDiagnostic } from "./router.js"; import { responsesToCompletionRequest, + requestWantsEncryptedReasoning, buildTextStreamEvents, buildToolCallStreamEvents, buildContentWithToolCallsStreamEvents, @@ -172,8 +173,14 @@ async function processMessage( stream: parsed.stream, temperature: parsed.temperature, max_output_tokens: parsed.max_output_tokens, + include: (parsed as { include?: string[] }).include, + store: (parsed as { store?: boolean }).store, }; + // Gate encrypted-reasoning emission identically to the HTTP transport so the + // agent-framework#7233 stateless-replay path works over WebSocket too. + const emitEncryptedReasoning = requestWantsEncryptedReasoning(responsesReq); + const completionReq = responsesToCompletionRequest(responsesReq); completionReq._endpointType = "chat"; const contextHeader = defaults.upgradeHeaders?.["x-aimock-context"]; @@ -283,6 +290,7 @@ async function processMessage( response.webSearches, extractOverrides(response), response.blocks, + emitEncryptedReasoning, ); const interruption = createInterruptionSignal(fixture); @@ -326,6 +334,7 @@ async function processMessage( ), response.webSearches, extractOverrides(response), + emitEncryptedReasoning, ); const interruption = createInterruptionSignal(fixture); const completed = await sendEvents( @@ -370,6 +379,7 @@ async function processMessage( ), response.webSearches, extractOverrides(response), + emitEncryptedReasoning, ); const interruption = createInterruptionSignal(fixture); const completed = await sendEvents( From a3a544615e5c301faa6466e1882ba3bece49ec5a Mon Sep 17 00:00:00 2001 From: ran Date: Wed, 29 Jul 2026 23:59:05 +0300 Subject: [PATCH 03/12] =?UTF-8?q?test(responses):=20close=20review=20round?= =?UTF-8?q?=202=20=E2=80=94=20WS=20+=20streaming=20coverage,=20decouple=20?= =?UTF-8?q?blob?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Follow-up to jpr5's second pass on #342: - Decouple the blob from summary presence: `buildReasoningOutputItem` now emits encrypted_content on `emitEncrypted` alone (empty-summary + blob is expressible). - WS transport coverage: new ws-responses.test.ts cases assert the blob over the WebSocket surface for content+reasoning and tool+reasoning, both gate triggers (include / store:false), and absence when not opted in. (Mutation-checked: hardcoding the WS gate to false fails 3 tests.) - Streaming coverage for every builder: buildTextStreamEvents, buildToolCallStreamEvents, buildContentWithToolCallsStreamEvents (legacy + blocks branches) + response.completed.output; and the non-streaming blocks branch of buildContentWithToolCallsResponse (the old test never entered it). (Mutation-checked: dropping the blob fails 10 tests.) - `added` placement is no longer pinned as absence (real OpenAI populates it; aimock omits it — both legal). Flag-off still pins genuine absence. - Drift: removed the ineffective local leg (couldn't fail); added the real spec anchor (encrypted_content on the terminal item in sdk-shapes.ts). - Docstring: correct the store-branch wording (include is the real-client trigger; store:false is the secondary/captured one). --- src/__tests__/drift/openai-responses.drift.ts | 40 +----- src/__tests__/drift/sdk-shapes.ts | 6 + src/__tests__/responses.test.ts | 124 +++++++++++++----- src/__tests__/ws-responses.test.ts | 99 ++++++++++++++ src/responses.ts | 33 +++-- 5 files changed, 220 insertions(+), 82 deletions(-) diff --git a/src/__tests__/drift/openai-responses.drift.ts b/src/__tests__/drift/openai-responses.drift.ts index 43a4396b..654cacb7 100644 --- a/src/__tests__/drift/openai-responses.drift.ts +++ b/src/__tests__/drift/openai-responses.drift.ts @@ -558,39 +558,13 @@ describe("OpenAI Responses API reasoning drift", () => { expect(reasoningAdded, "no output_item.added with type=reasoning").toBeDefined(); }); - it("opted-in reasoning carries encrypted_content ONLY on the terminal item", async () => { - // Real OpenAI populates reasoning.encrypted_content on the `done` item (and - // response.completed.output), never on `added`, when the caller opts in via - // include or a stateless (store:false) request. See microsoft/agent-framework#7233. - const res = await httpPost(`${reasoningInstance.url}/v1/responses`, { - model: "gpt-4o-mini", - input: [{ role: "user", content: "Think carefully" }], - stream: true, - include: ["reasoning.encrypted_content"], - }); - expect(res.status).toBe(200); - - const events = parseTypedSSE(res.body); - const reasoningItemOf = (kind: "added" | "done") => - events.find( - (e) => - e.type === `response.output_item.${kind}` && - (e.data as { item?: { type?: string } }).item?.type === "reasoning", - )?.data as { item?: { encrypted_content?: string } } | undefined; - - const added = reasoningItemOf("added"); - const done = reasoningItemOf("done"); - expect(added?.item, "no reasoning output_item.added").toBeDefined(); - expect(done?.item, "no reasoning output_item.done").toBeDefined(); - expect( - added!.item!.encrypted_content, - "encrypted_content must be absent on added", - ).toBeUndefined(); - expect(typeof done!.item!.encrypted_content, "encrypted_content must be present on done").toBe( - "string", - ); - expect(done!.item!.encrypted_content!.length).toBeGreaterThan(0); - }); + // NOTE: encrypted_content emission is enforced by the unit + WS suites + // (responses.test.ts / ws-responses.test.ts), which run in CI and fail on a + // dropped blob. A local-server assertion here would not add coverage (this + // block posts to a local aimock, not api.openai.com, and drift `expect` + // failures are report-collected rather than hard-failing). The live spec + // anchor lives in sdk-shapes.ts (`openaiResponsesReasoningEventShapes` pins + // encrypted_content on the terminal item) for the triangulated legs above. it("reasoning event shapes include item_id, output_index, summary_index", async () => { const res = await httpPost(`${reasoningInstance.url}/v1/responses`, { diff --git a/src/__tests__/drift/sdk-shapes.ts b/src/__tests__/drift/sdk-shapes.ts index c634bec4..e798ac21 100644 --- a/src/__tests__/drift/sdk-shapes.ts +++ b/src/__tests__/drift/sdk-shapes.ts @@ -464,6 +464,12 @@ export function openaiResponsesReasoningEventShapes(): SSEEventShape[] { type: "reasoning", id: "rs_abc123", summary: [{ type: "summary_text", text: "Step by step..." }], + // Real OpenAI carries the encrypted reasoning blob on the terminal + // item when the request opts in (include / store:false). aimock emits + // it there too. Anchored on `done` only — `added` is opportunistic in + // real OpenAI and aimock deliberately omits it, so pinning `added` + // would flag an intentional, legal shape choice as drift. + encrypted_content: "gAAAAAB...", }, }), }, diff --git a/src/__tests__/responses.test.ts b/src/__tests__/responses.test.ts index cf5f25d8..a0019f19 100644 --- a/src/__tests__/responses.test.ts +++ b/src/__tests__/responses.test.ts @@ -1905,8 +1905,16 @@ describe("reasoning.encrypted_content emission", () => { return ev?.item as { id: string; encrypted_content?: string } | undefined; } + // `added` placement is NOT contractual: real OpenAI populates it, aimock omits + // it, and both are legal (field is anyOf:[string,null], non-required). So a + // present blob must be a string, but absence must NOT fail. (Contrast the + // flag-off case below, which pins genuine absence.) + const assertAddedNotPinned = (item: { encrypted_content?: string }): void => { + if (item.encrypted_content !== undefined) expect(typeof item.encrypted_content).toBe("string"); + }; + const reasoningTextFixture: Fixture = { - match: { userMessage: "compare" }, + match: { userMessage: "textreason" }, response: { content: "AAPL vs MSFT", reasoning: "Look up AAPL, then MSFT." }, }; const reasoningToolFixture: Fixture = { @@ -1921,8 +1929,22 @@ describe("reasoning.encrypted_content emission", () => { reasoning: "Think, then call.", }, }; + // `blocks` forces the positional NEW branch of buildContentWithToolCalls* (both + // streaming and non-streaming), which builds its own reasoning item. + const reasoningBlocksFixture: Fixture = { + match: { userMessage: "blocksreason" }, + response: { + content: "done", + toolCalls: [{ name: "get_stock", arguments: "{}" }], + reasoning: "Think, then call.", + blocks: [ + { type: "toolCall", name: "get_stock", arguments: "{}" }, + { type: "text", text: "done" }, + ], + }, + }; - // --- Builder-level: placement (added vs done) + determinism --- + // --- Builder-level: placement + determinism --- it("builder omits encrypted_content everywhere when the flag is off", () => { const events = buildTextStreamEvents("result", "gpt-5-test", 100, "thinking..."); @@ -1930,7 +1952,7 @@ describe("reasoning.encrypted_content emission", () => { expect(reasoningItem(events, "done")!.encrypted_content).toBeUndefined(); }); - it("builder emits the blob ONLY on the done item, and it is deterministic by id", () => { + it("builder emits the blob on the done item, deterministic by id", () => { // trailing `true` = emitEncryptedReasoning const events = buildTextStreamEvents( "result", @@ -1941,19 +1963,22 @@ describe("reasoning.encrypted_content emission", () => { undefined, true, ); - const added = reasoningItem(events, "added")!; const done = reasoningItem(events, "done")!; - expect(added.encrypted_content).toBeUndefined(); // never on `added` (langchainjs#10844) expect(done.encrypted_content).toBe(expectedBlob(done.id)); // pins presence + determinism + assertAddedNotPinned(reasoningItem(events, "added")!); }); - // --- HTTP streaming: gate triggers + completed.output --- + // --- HTTP streaming: every builder + gate triggers + completed.output --- - async function streamReasoning(body: Record): Promise { - instance = await createServer([reasoningTextFixture]); + async function streamReasoning( + fixture: Fixture, + userMessage: string, + body: Record, + ): Promise { + instance = await createServer([fixture]); const res = await post(`${instance!.url}/v1/responses`, { model: "gpt-5-test", - input: [{ role: "user", content: "compare" }], + input: [{ role: "user", content: userMessage }], stream: true, ...body, }); @@ -1961,12 +1986,39 @@ describe("reasoning.encrypted_content emission", () => { return parseResponsesSSEEvents(res.body); } - it("emits the blob when the request opts in via include", async () => { - const events = await streamReasoning({ include: ["reasoning.encrypted_content"] }); - expect(reasoningItem(events, "added")!.encrypted_content).toBeUndefined(); + const OPT_IN = { include: ["reasoning.encrypted_content"] }; + + // One case per streaming builder: buildTextStreamEvents, + // buildToolCallStreamEvents, buildContentWithToolCallsStreamEvents (legacy + + // blocks branches). Each must carry the blob on the done reasoning item. + const streamingCases: [string, Fixture, string][] = [ + ["text (buildTextStreamEvents)", reasoningTextFixture, "textreason"], + ["tool-only (buildToolCallStreamEvents)", reasoningToolFixture, "toolreason"], + [ + "content+tool legacy (buildContentWithToolCallsStreamEvents)", + reasoningContentToolFixture, + "bothreason", + ], + [ + "content+tool blocks (buildContentWithToolCallsStreamEvents)", + reasoningBlocksFixture, + "blocksreason", + ], + ]; + + for (const [label, fixture, msg] of streamingCases) { + it(`streaming ${label} carries the blob on done when opted in`, async () => { + const events = await streamReasoning(fixture, msg, OPT_IN); + const done = reasoningItem(events, "done")!; + expect(done, "no done reasoning item").toBeDefined(); + expect(done.encrypted_content).toBe(expectedBlob(done.id)); + assertAddedNotPinned(reasoningItem(events, "added")!); + }); + } + + it("streaming reflects the blob in response.completed.output", async () => { + const events = await streamReasoning(reasoningTextFixture, "textreason", OPT_IN); const done = reasoningItem(events, "done")!; - expect(done.encrypted_content).toBe(expectedBlob(done.id)); - // Also present on the reasoning item inside response.completed.output. const completed = events.find((e) => e.type === "response.completed") as SSEEvent & { response: { output: { type: string; encrypted_content?: string }[] }; }; @@ -1974,19 +2026,19 @@ describe("reasoning.encrypted_content emission", () => { expect(outputReasoning!.encrypted_content).toBe(done.encrypted_content); }); - it("emits the blob for a stateless request (store:false) even without include", async () => { - const events = await streamReasoning({ store: false }); + it("streaming emits the blob for a stateless request (store:false) without include", async () => { + const events = await streamReasoning(reasoningTextFixture, "textreason", { store: false }); const done = reasoningItem(events, "done")!; expect(done.encrypted_content).toBe(expectedBlob(done.id)); }); - it("omits the blob for a stored request that did not opt in", async () => { - const events = await streamReasoning({}); + it("streaming omits the blob for a stored request that did not opt in", async () => { + const events = await streamReasoning(reasoningTextFixture, "textreason", {}); expect(reasoningItem(events, "added")!.encrypted_content).toBeUndefined(); expect(reasoningItem(events, "done")!.encrypted_content).toBeUndefined(); }); - // --- Non-streaming builders: all three carry the blob when opted in --- + // --- Non-streaming: every builder path (incl. the positional blocks branch) --- async function nonStreamReasoning( fixture: Fixture, @@ -2006,23 +2058,25 @@ describe("reasoning.encrypted_content emission", () => { return body.output; } - it("non-streaming text response carries the blob (buildOutputPrefix)", async () => { - const output = await nonStreamReasoning(reasoningTextFixture, "compare"); - const r = output.find((o) => o.type === "reasoning")!; - expect(r.encrypted_content).toBe(expectedBlob(r.id)); - }); - - it("non-streaming tool-call response carries the blob (buildToolCallResponse)", async () => { - const output = await nonStreamReasoning(reasoningToolFixture, "toolreason"); - const r = output.find((o) => o.type === "reasoning")!; - expect(r.encrypted_content).toBe(expectedBlob(r.id)); - }); + const nonStreamingCases: [string, Fixture, string][] = [ + ["text (buildOutputPrefix)", reasoningTextFixture, "textreason"], + ["tool-only (buildToolCallResponse)", reasoningToolFixture, "toolreason"], + ["content+tool legacy (buildOutputPrefix)", reasoningContentToolFixture, "bothreason"], + [ + "content+tool blocks (buildContentWithToolCallsResponse)", + reasoningBlocksFixture, + "blocksreason", + ], + ]; - it("non-streaming content+tool response carries the blob (buildContentWithToolCallsResponse)", async () => { - const output = await nonStreamReasoning(reasoningContentToolFixture, "bothreason"); - const r = output.find((o) => o.type === "reasoning")!; - expect(r.encrypted_content).toBe(expectedBlob(r.id)); - }); + for (const [label, fixture, msg] of nonStreamingCases) { + it(`non-streaming ${label} carries the blob when opted in`, async () => { + const output = await nonStreamReasoning(fixture, msg); + const r = output.find((o) => o.type === "reasoning")!; + expect(r, "no reasoning output item").toBeDefined(); + expect(r.encrypted_content).toBe(expectedBlob(r.id)); + }); + } }); // ─── Bug fix: multi-fco after single item_reference ───────────────────────── diff --git a/src/__tests__/ws-responses.test.ts b/src/__tests__/ws-responses.test.ts index eda04698..26d35b03 100644 --- a/src/__tests__/ws-responses.test.ts +++ b/src/__tests__/ws-responses.test.ts @@ -784,3 +784,102 @@ describe("WebSocket /v1/responses reasoning capability gating", () => { ws.close(); }); }); + +// ─── WS reasoning.encrypted_content (microsoft/agent-framework#7233) ───────── +// +// The gate must be honored over the WebSocket Responses transport too — the WS +// dispatch rebuilds the request field-by-field, so `include` / `store` and the +// emitted blob need transport-level coverage (deleting the threading otherwise +// leaves every test green). + +describe("WebSocket /v1/responses encrypted reasoning", () => { + const expectedBlob = (id: string): string => + Buffer.from(`aimock-encrypted-reasoning:${id}`).toString("base64"); + + function createMsg(userContent: string, extra: Record): string { + return JSON.stringify({ + type: "response.create", + model: "gpt-4.1", // non-strict keeps reasoning for non-reasoning models (see above) + input: [{ role: "user", content: userContent }], + ...extra, + }); + } + + async function collectUntilCompleted(ws: { + waitForMessages: (n: number) => Promise; + }): Promise { + const maxEvents = 50; + let events: WSEvent[] = []; + for (let count = 1; count <= maxEvents; count++) { + events = parseEvents(await ws.waitForMessages(count)); + if (events[events.length - 1].type === "response.completed") return events; + } + throw new Error( + `response.completed never arrived within ${maxEvents} events ` + + `(last: ${events[events.length - 1]?.type})`, + ); + } + + const reasoningDone = (events: WSEvent[]) => + events.find( + (e) => + e.type === "response.output_item.done" && + (e.item as { type?: string })?.type === "reasoning", + )?.item as { id: string; encrypted_content?: string } | undefined; + + it("emits the blob on the done item when opted in via include (content+reasoning)", async () => { + instance = await createServer(allFixtures); + const ws = await connectWebSocket(instance.url, "/v1/responses"); + ws.send(createMsg("think", { include: ["reasoning.encrypted_content"] })); + + const events = await collectUntilCompleted(ws); + const done = reasoningDone(events); + expect(done, "no done reasoning item").toBeDefined(); + expect(done!.encrypted_content).toBe(expectedBlob(done!.id)); + + // Also reflected in the terminal completed.output. + const completed = events.find((e) => e.type === "response.completed"); + const output = ( + completed!.response as { output: { type: string; encrypted_content?: string }[] } + ).output; + const outReasoning = output.find((o) => o.type === "reasoning"); + expect(outReasoning!.encrypted_content).toBe(done!.encrypted_content); + + ws.close(); + }); + + it("emits the blob for a tool-only reasoning response when opted in", async () => { + instance = await createServer(allFixtures); + const ws = await connectWebSocket(instance.url, "/v1/responses"); + ws.send(createMsg("tool-reason", { include: ["reasoning.encrypted_content"] })); + + const events = await collectUntilCompleted(ws); + const done = reasoningDone(events); + expect(done!.encrypted_content).toBe(expectedBlob(done!.id)); + + ws.close(); + }); + + it("emits the blob for a stateless request (store:false) without include", async () => { + instance = await createServer(allFixtures); + const ws = await connectWebSocket(instance.url, "/v1/responses"); + ws.send(createMsg("think", { store: false })); + + const events = await collectUntilCompleted(ws); + const done = reasoningDone(events); + expect(done!.encrypted_content).toBe(expectedBlob(done!.id)); + + ws.close(); + }); + + it("omits the blob when the request does not opt in", async () => { + instance = await createServer(allFixtures); + const ws = await connectWebSocket(instance.url, "/v1/responses"); + ws.send(createMsg("think", {})); + + const events = await collectUntilCompleted(ws); + expect(reasoningDone(events)!.encrypted_content).toBeUndefined(); + + ws.close(); + }); +}); diff --git a/src/responses.ts b/src/responses.ts index acfee10b..c67e250f 100644 --- a/src/responses.ts +++ b/src/responses.ts @@ -431,13 +431,16 @@ export function buildToolCallStreamEvents( /** * Whether the incoming Responses request should receive encrypted reasoning. * - * Real OpenAI populates `reasoning.encrypted_content` when the caller opts in - * with `include: ["reasoning.encrypted_content"]` OR whenever the response is - * not server-side stored (`store: false` / ZDR) — the latter became the default - * trigger in openai-python 2.46.0. `agent-framework-openai` >= 1.11.0 drives its - * stateless-replay path with both signals set. Gating on either keeps aimock in - * step with real OpenAI while leaving stored / opted-out replays byte-identical. - * Exported so the WebSocket Responses transport can share one gate. + * Two observed triggers, gated on EITHER: + * - `include: ["reasoning.encrypted_content"]` — the explicit opt-in, and what + * `agent-framework-openai` >= 1.11.0 auto-appends on its stateless-replay path + * (it never sends `store`), so this is the branch that fires for real clients. + * - `store: false` — captured responses carry the blob when not server-side + * stored (ZDR) and omit it when stored; kept as a secondary trigger for + * clients that go stateless without opting in via `include`. + * + * Stored / opted-out replays stay byte-identical. Exported so the WebSocket + * Responses transport can share one gate. */ export function requestWantsEncryptedReasoning(req: ResponsesRequest): boolean { const include = req.include; @@ -463,12 +466,14 @@ function syntheticEncryptedReasoning(reasoningId: string): string { } /** - * Build a Responses `reasoning` output item. Pass `summaryText` for the terminal - * (`done` / non-streaming) item — it carries the summary and, when - * `emitEncrypted`, the encrypted blob. Omit `summaryText` for the in-progress - * (`added`) item: empty summary and — matching real OpenAI — never a blob (a - * populated blob at `added` would mask clients that read reasoning before - * `done`; see langchain-ai/langchainjs#10844). + * Build a Responses `reasoning` output item. `summaryText` controls the summary + * (present for the terminal `done` / non-streaming item, omitted for the + * in-progress `added` item); `emitEncrypted` controls the blob INDEPENDENTLY of + * the summary, so a blob with an empty summary is expressible. aimock emits the + * blob only on the terminal item and omits it on `added`: real OpenAI populates + * `added` opportunistically, but the field is `anyOf: [string, null]` and + * non-required, so consumers should read it at `done` and treat `added` as + * best-effort — omitting there is legal and harmless. */ function buildReasoningOutputItem( reasoningId: string, @@ -480,7 +485,7 @@ function buildReasoningOutputItem( summary: opts.summaryText === undefined ? [] : [{ type: "summary_text", text: opts.summaryText }], }; - if (opts.summaryText !== undefined && opts.emitEncrypted) { + if (opts.emitEncrypted) { item.encrypted_content = syntheticEncryptedReasoning(reasoningId); } return item; From 57ad4c030f012d71b3e0126f2db75325824a21bc Mon Sep 17 00:00:00 2001 From: Jordan Ritter Date: Wed, 29 Jul 2026 15:18:21 -0700 Subject: [PATCH 04/12] fix: split encrypted-reasoning drift pin by opt-in so it can actually pass MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The `drift-live-pr` required check failed on every run of this branch with `[critical] item.encrypted_content ... Mock: `, and no code change could have cleared it. `openaiResponsesReasoningEventShapes()` pinned `encrypted_content` on the terminal reasoning item unconditionally, but its only consumer — the "triangulate against SDK expectations" leg — posts an OPTED-OUT request (no `include`, no `store: false`). aimock correctly emits no blob there: `requestWantsEncryptedReasoning()` is false, so `buildReasoningOutputItem` omits the field by design. Triangulating an unconditional SDK pin against a legitimately-absent mock field lands in triangulate()'s "in SDK + real but not mock" branch, which is always critical. So the pin asserted the mock had a field the request never asked for. It reported critical on every run regardless of whether the emission worked, which meant it gated nothing — and because the key was new relative to main, computeDelta() classified it as block[] (new-in-head), failing the required check outright rather than degrading to an advisory. Fix follows the harness's existing convention for variant shapes (separate parameterless functions, as in fal-queue-contract.ts and the realtime text/beta pair) rather than an options bag, and mirrors production's own `buildReasoningOutputItem({ emitEncrypted })` gate: - `openaiResponsesReasoningEventShapes()` — no blob; grades opted-out legs. - `openaiResponsesEncryptedReasoningEventShapes()` — blob; grades opted-in. The existing leg now pins the no-blob variant matching the request it sends, and a new leg posts `include: ["reasoning.encrypted_content"]` (what agent-framework-openai >= 1.11.0 sends on stateless replay) and pins the blob variant. That new leg is what actually gates the emission: deleting the `encrypted_content` assignment in src/responses.ts flips it critical while the opted-out leg stays clean. Verified locally — collector exit 2 -> 0, delta gate BLOCKED -> PASSED, and the sabotage re-reds only the opted-in leg. No production behavior change. --- src/__tests__/drift/openai-responses.drift.ts | 97 +++++++++++++------ src/__tests__/drift/sdk-shapes.ts | 35 ++++++- 2 files changed, 101 insertions(+), 31 deletions(-) diff --git a/src/__tests__/drift/openai-responses.drift.ts b/src/__tests__/drift/openai-responses.drift.ts index 654cacb7..a4bc1edd 100644 --- a/src/__tests__/drift/openai-responses.drift.ts +++ b/src/__tests__/drift/openai-responses.drift.ts @@ -7,12 +7,19 @@ import { describe, it, expect, beforeAll, afterAll } from "vitest"; import { createServer, type ServerInstance } from "../../server.js"; import type { Fixture } from "../../types.js"; -import { extractShape, triangulate, compareSSESequences, formatDriftReport } from "./schema.js"; +import { + extractShape, + triangulate, + compareSSESequences, + formatDriftReport, + type SSEEventShape, +} from "./schema.js"; import { openaiResponsesNonStreamingShape, openaiResponsesTextEventShapes, openaiResponsesToolCallEventShapes, openaiResponsesReasoningEventShapes, + openaiResponsesEncryptedReasoningEventShapes, } from "./sdk-shapes.js"; import { resolveLiveModel, @@ -558,13 +565,14 @@ describe("OpenAI Responses API reasoning drift", () => { expect(reasoningAdded, "no output_item.added with type=reasoning").toBeDefined(); }); - // NOTE: encrypted_content emission is enforced by the unit + WS suites - // (responses.test.ts / ws-responses.test.ts), which run in CI and fail on a - // dropped blob. A local-server assertion here would not add coverage (this - // block posts to a local aimock, not api.openai.com, and drift `expect` - // failures are report-collected rather than hard-failing). The live spec - // anchor lives in sdk-shapes.ts (`openaiResponsesReasoningEventShapes` pins - // encrypted_content on the terminal item) for the triangulated legs above. + // NOTE: encrypted_content emission is ALSO enforced by the unit + WS suites + // (responses.test.ts / ws-responses.test.ts), which hard-fail on a dropped + // blob. The drift-side anchor is the shape pin in sdk-shapes.ts, and it is + // split by opt-in: `openaiResponsesReasoningEventShapes` (no blob) grades the + // opted-OUT legs, `openaiResponsesEncryptedReasoningEventShapes` (blob) grades + // the opted-IN leg below. Pinning the blob unconditionally made every + // opted-out leg report `item.encrypted_content` critical forever — a finding + // that no code change could clear, so it gated nothing. it("reasoning event shapes include item_id, output_index, summary_index", async () => { const res = await httpPost(`${reasoningInstance.url}/v1/responses`, { @@ -621,26 +629,22 @@ describe("OpenAI Responses API reasoning drift", () => { expect((partDoneData.part as { type: string; text: string }).text).toBe(REASONING_TEXT); }); - it("reasoning event shapes triangulate against SDK expectations", async () => { - const sdkEvents = openaiResponsesReasoningEventShapes(); - - const res = await httpPost(`${reasoningInstance.url}/v1/responses`, { - model: "gpt-4o-mini", - input: [{ role: "user", content: "Think carefully" }], - stream: true, - }); - - expect(res.status).toBe(200); - - const mockEvents = parseTypedSSE(res.body); - const mockSSEShapes = mockEvents.map((e) => ({ + // Grade a reasoning SSE body against the SDK shape variant for the request + // that produced it. `scenario` keeps the opted-out and opted-in legs' drift + // keys distinct so the delta gate attributes a finding to the right variant. + // + // Since reasoning is not available on gpt-4o-mini via real API, the SDK shape + // is used as both "expected" and "real" for shape validation. + const triangulateReasoningEvents = ( + sdkEvents: SSEEventShape[], + body: string, + scenario: string, + ) => { + const mockSSEShapes = parseTypedSSE(body).map((e) => ({ type: e.type, dataShape: extractShape(e.data), })); - // Triangulate reasoning-specific events against SDK shapes. - // Since reasoning is not available on gpt-4o-mini via real API, we - // use SDK shapes as both "expected" and "real" for shape validation. for (const sdkEvent of sdkEvents) { const mockEvent = mockSSEShapes.find((m) => m.type === sdkEvent.type); if (!mockEvent) { @@ -649,16 +653,51 @@ describe("OpenAI Responses API reasoning drift", () => { } const diffs = triangulate(sdkEvent.dataShape, sdkEvent.dataShape, mockEvent.dataShape); - const report = formatDriftReport( - `OpenAI Responses Reasoning:${sdkEvent.type}`, - diffs, - "openai-responses", - ); + const report = formatDriftReport(`${scenario}:${sdkEvent.type}`, diffs, "openai-responses"); expect( diffs.filter((d) => d.severity === "critical"), report, ).toEqual([]); } + }; + + it("reasoning event shapes triangulate against SDK expectations", async () => { + // Opted OUT: no `include`, no `store: false` — the terminal item must carry + // NO encrypted_content, so this leg is graded against the no-blob variant. + const res = await httpPost(`${reasoningInstance.url}/v1/responses`, { + model: "gpt-4o-mini", + input: [{ role: "user", content: "Think carefully" }], + stream: true, + }); + + expect(res.status).toBe(200); + + triangulateReasoningEvents( + openaiResponsesReasoningEventShapes(), + res.body, + "OpenAI Responses Reasoning", + ); + }); + + it("opted-in reasoning event shapes carry encrypted_content", async () => { + // Opted IN via `include` — the exact request agent-framework-openai + // >= 1.11.0 sends on its stateless-replay path. This is the leg that GATES + // the emission: drop `encrypted_content` in responses.ts and the terminal + // item reports `item.encrypted_content` critical here. + const res = await httpPost(`${reasoningInstance.url}/v1/responses`, { + model: "gpt-4o-mini", + input: [{ role: "user", content: "Think carefully" }], + stream: true, + include: ["reasoning.encrypted_content"], + }); + + expect(res.status).toBe(200); + + triangulateReasoningEvents( + openaiResponsesEncryptedReasoningEventShapes(), + res.body, + "OpenAI Responses Encrypted Reasoning", + ); }); }); diff --git a/src/__tests__/drift/sdk-shapes.ts b/src/__tests__/drift/sdk-shapes.ts index e798ac21..7fe9eaed 100644 --- a/src/__tests__/drift/sdk-shapes.ts +++ b/src/__tests__/drift/sdk-shapes.ts @@ -401,7 +401,19 @@ export function openaiResponsesToolCallEventShapes(): SSEEventShape[] { ]; } -export function openaiResponsesReasoningEventShapes(): SSEEventShape[] { +/** + * Reasoning SSE event shapes, parameterized on the encrypted-reasoning opt-in. + * + * `emitEncrypted` mirrors `responses.ts`'s own + * `buildReasoningOutputItem({ emitEncrypted })` gate: real OpenAI (and aimock) + * carry `encrypted_content` on the terminal item ONLY when the request opted in + * (`include: ["reasoning.encrypted_content"]` or `store: false`). Pinning it + * unconditionally would report critical drift against every opted-OUT leg, + * whose absent blob is correct behavior — so the two variants are exposed as + * separate shape functions and each drift leg pins the one matching the request + * it actually sends. + */ +function reasoningEventShapes(emitEncrypted: boolean): SSEEventShape[] { return [ { type: "response.output_item.added", @@ -469,13 +481,32 @@ export function openaiResponsesReasoningEventShapes(): SSEEventShape[] { // it there too. Anchored on `done` only — `added` is opportunistic in // real OpenAI and aimock deliberately omits it, so pinning `added` // would flag an intentional, legal shape choice as drift. - encrypted_content: "gAAAAAB...", + ...(emitEncrypted ? { encrypted_content: "gAAAAAB..." } : {}), }, }), }, ]; } +/** + * Opted-OUT reasoning shapes: no `include`, no `store: false`. The terminal item + * carries NO `encrypted_content` — its absence is correct, not drift. + */ +export function openaiResponsesReasoningEventShapes(): SSEEventShape[] { + return reasoningEventShapes(false); +} + +/** + * Opted-IN reasoning shapes: the request asked for + * `include: ["reasoning.encrypted_content"]` (or went stateless with + * `store: false`), so the terminal item MUST carry the blob. This is the variant + * that gates the emission — drop the blob in `responses.ts` and the opted-in + * drift leg reports `item.encrypted_content` critical. + */ +export function openaiResponsesEncryptedReasoningEventShapes(): SSEEventShape[] { + return reasoningEventShapes(true); +} + export function openaiResponsesNonStreamingShape(): ShapeNode { return extractShape({ id: "resp_abc123", From fa264d520d129571e2768c8877e1a0c51e75902f Mon Sep 17 00:00:00 2001 From: Jordan Ritter Date: Wed, 29 Jul 2026 15:19:05 -0700 Subject: [PATCH 05/12] test: cover the encrypted-reasoning LEAK direction on non-streaming builders MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The existing suite proved the `reasoning.encrypted_content` blob IS emitted where it should be, but nothing proved it is NOT emitted when the request never opted in. That is the more dangerous direction for a mock: an unconditional emission silently changes response bytes for every consumer that sent neither `include: ["reasoning.encrypted_content"]` nor `store: false`, breaking the byte-identical stored/opted-out replay. Forcing `emitEncrypted: true` at each of the three non-streaming sites left the entire suite green (4741 passed), so all three were genuinely uncovered: - `buildOutputPrefix` (text + content+tool legacy) - `buildToolCallResponse` (tool-only) - the `blocks` branch of `buildContentWithToolCallsResponse` Add one negative per site, reusing the existing non-streaming case table, each pinning genuine key ABSENCE via `not.toHaveProperty` rather than a falsy value. Every negative fails under exactly its own site's mutation and no other. Also add the missing WS case for the content+toolCalls builder — the WS dispatch threads the gate into each streaming builder separately, and dropping the flag from that one call site left the existing text and tool-only WS cases green. No production code changed. --- src/__tests__/responses.test.ts | 22 +++++++++++++++++++++- src/__tests__/ws-responses.test.ts | 30 ++++++++++++++++++++++++++++++ 2 files changed, 51 insertions(+), 1 deletion(-) diff --git a/src/__tests__/responses.test.ts b/src/__tests__/responses.test.ts index a0019f19..7fa2e97f 100644 --- a/src/__tests__/responses.test.ts +++ b/src/__tests__/responses.test.ts @@ -2043,13 +2043,14 @@ describe("reasoning.encrypted_content emission", () => { async function nonStreamReasoning( fixture: Fixture, userMessage: string, + extra: Record = OPT_IN, ): Promise<{ type: string; encrypted_content?: string; id: string }[]> { instance = await createServer([fixture]); const res = await post(`${instance!.url}/v1/responses`, { model: "gpt-5-test", input: [{ role: "user", content: userMessage }], stream: false, - include: ["reasoning.encrypted_content"], + ...extra, }); expect(res.status).toBe(200); const body = JSON.parse(res.body) as { @@ -2077,6 +2078,25 @@ describe("reasoning.encrypted_content emission", () => { expect(r.encrypted_content).toBe(expectedBlob(r.id)); }); } + + // --- Non-streaming LEAK direction: the same builders must stay silent --- + // + // The positive cases above only prove the blob CAN be emitted; they pass just + // as well if a builder emits it unconditionally. A leak is the worse failure + // for a mock: it silently changes response bytes for every consumer that never + // opted in (no `include`, no `store:false`), so the stored/opted-out replay is + // no longer byte-identical. One negative per builder site — buildOutputPrefix + // (text + content+tool legacy), buildToolCallResponse, and the `blocks` branch + // of buildContentWithToolCallsResponse — each pinning genuine key ABSENCE, not + // merely a falsy value. + for (const [label, fixture, msg] of nonStreamingCases) { + it(`non-streaming ${label} omits the blob when the request did not opt in`, async () => { + const output = await nonStreamReasoning(fixture, msg, {}); + const r = output.find((o) => o.type === "reasoning")!; + expect(r, "no reasoning output item").toBeDefined(); + expect(r).not.toHaveProperty("encrypted_content"); + }); + } }); // ─── Bug fix: multi-fco after single item_reference ───────────────────────── diff --git a/src/__tests__/ws-responses.test.ts b/src/__tests__/ws-responses.test.ts index 26d35b03..f7f8eb4d 100644 --- a/src/__tests__/ws-responses.test.ts +++ b/src/__tests__/ws-responses.test.ts @@ -50,6 +50,19 @@ const toolReasoningFixture: Fixture = { }, }; +// Combined content+toolCalls fixture that also carries reasoning. Distinct match +// key so it drives the WS dispatch's content+toolCalls branch +// (buildContentWithToolCallsStreamEvents) — the third streaming builder, which +// the text-only "think" and tool-only "tool-reason" fixtures never reach. +const contentToolReasoningFixture: Fixture = { + match: { userMessage: "both-reason" }, + response: { + content: "Here you go.", + toolCalls: [{ name: "get_weather", arguments: '{"city":"NYC"}' }], + reasoning: "Let me reason, then call the tool.", + }, +}; + // Combined content+toolCalls fixture carrying an ORDERED `blocks` array placing // the tool call BEFORE the text. On the WebSocket /v1/responses surface this // must yield the function_call output item at a LOWER output_index than the @@ -74,6 +87,7 @@ const allFixtures: Fixture[] = [ reasoningFixture, capabilityReasoningFixture, toolReasoningFixture, + contentToolReasoningFixture, wsBlocksToolFirstFixture, ]; @@ -860,6 +874,22 @@ describe("WebSocket /v1/responses encrypted reasoning", () => { ws.close(); }); + // The WS dispatch threads the gate into each streaming builder separately, so + // the content+toolCalls branch needs its own case — the text and tool-only + // cases above both stay green if that one call site drops the flag. + it("emits the blob for a content+toolCalls reasoning response when opted in", async () => { + instance = await createServer(allFixtures); + const ws = await connectWebSocket(instance.url, "/v1/responses"); + ws.send(createMsg("both-reason", { include: ["reasoning.encrypted_content"] })); + + const events = await collectUntilCompleted(ws); + const done = reasoningDone(events); + expect(done, "no done reasoning item").toBeDefined(); + expect(done!.encrypted_content).toBe(expectedBlob(done!.id)); + + ws.close(); + }); + it("emits the blob for a stateless request (store:false) without include", async () => { instance = await createServer(allFixtures); const ws = await connectWebSocket(instance.url, "/v1/responses"); From 39e82449e721be99efe136428e5c47dc966a8385 Mon Sep 17 00:00:00 2001 From: Jordan Ritter Date: Wed, 29 Jul 2026 15:12:30 -0700 Subject: [PATCH 06/12] docs(changelog): correct the encrypted_content added-item claim The entry said the blob is withheld from the in-progress `added` item "matching real OpenAI". That is backwards, and the claim originated in my review feedback on this PR, not from the author. Recorded OpenAI captures carry a populated blob on `response.output_item.added` beside an empty summary, re-encrypted by `done`; the code comments added in a3a5446 already say so, so the changelog was the last place still asserting parity. The langchainjs#10844 rationale does not hold either: that payload is a hand-written msw mock, and its converter whitelists {id, type, summary} at `added`, so the reported bug reproduces whether or not `added` is populated. Withholding at `added` is a deliberate, contract-legal simplification (the field is `anyOf: [string, null]` and non-required, and no known consumer needs it), so the entry now describes it that way. Also softens "hard-failing" to "related": without the blob agent-framework-openai skips the reasoning item rather than hard-failing the request. --- CHANGELOG.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 52cd1aa7..4ddbc677 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -4,7 +4,7 @@ ### Added -- **Reasoning `encrypted_content` on the Responses API.** aimock now synthesizes an opaque (base64) `reasoning.encrypted_content` blob on the TERMINAL reasoning item (`response.output_item.done` and non-streaming `output[]`), never on the in-progress `added` item — matching real OpenAI (a populated blob at `added` would mask clients that read reasoning before `done`, e.g. langchain-ai/langchainjs#10844). Emitted only when the request wants it: `include: ["reasoning.encrypted_content"]` OR a stateless request (`store: false` / ZDR). Stored / opted-out replays stay byte-identical. This lets `agent-framework-openai` >= 1.11.0 replay reasoning-paired tool calls instead of hard-failing the follow-up on a stateless reasoning + multi-tool chain (root cause: microsoft/agent-framework#7233). Applies to both the HTTP and WebSocket Responses transports. Known limitation: aimock skips inbound reasoning items, so it cannot reproduce OpenAI's `invalid_encrypted_content` rejection of a mismatched blob/id. +- **Reasoning `encrypted_content` on the Responses API.** aimock now synthesizes an opaque (base64) `reasoning.encrypted_content` blob on the TERMINAL reasoning item (`response.output_item.done` and non-streaming `output[]`). Emitted only when the request wants it: `include: ["reasoning.encrypted_content"]` OR a stateless request (`store: false` / ZDR). Stored / opted-out replays stay byte-identical. This lets `agent-framework-openai` >= 1.11.0 replay reasoning-paired tool calls on a stateless reasoning + multi-tool chain (related: microsoft/agent-framework#7233). Applies to both the HTTP and WebSocket Responses transports. Known limitations: the blob is withheld from the in-progress `added` item, which is aimock's own simplification rather than upstream parity — real OpenAI populates `added` too (recorded captures carry a shorter blob there beside an empty summary, re-encrypted by `done`), but the field is `anyOf: [string, null]` and non-required, so omitting it is contract-legal and no known consumer requires it; and aimock skips inbound reasoning items, so it cannot reproduce OpenAI's `invalid_encrypted_content` rejection of a mismatched blob/id. - **OpenRouter chat / LLM router simulation.** Requests whose original path starts with `/api/v1/` (point any OpenAI SDK at a `baseURL` ending `/api/v1`) are detected as OpenRouter and shaped to match real OpenRouter bytes: a `gen-` id prefix (a fixture `id` override still wins), a top-level `provider` (default = the winning model slug's author, fixture-overridable via `provider`), both `finish_reason` and `native_finish_reason` on every choice/delta, an always-present `system_fingerprint` and `service_tier` (null by default), `message.reasoning`, and a rich `usage` with a **fixture-scriptable** `cost` + `cost_details` (emitted only when a fixture supplies a cost — never fabricated), plus `is_byok` / `prompt_tokens_details` / `completion_tokens_details` when overridden. Callers on the plain OpenAI `/v1/...` base are byte-for-byte unchanged. - **`models[]` fallback (router failover) simulation.** When the request body carries `models: [...]`, aimock walks `[model, ...models]` in order and serves the first fixture returning a NON-error response; a `429`/`503` error fixture on a candidate simulates a runtime provider failure and falls through to the next. The winning slug is echoed back as the top-level `model`, so a test asserts failover by reading `response.model`. (Deliberate non-goal: an unknown/invalid model is a fixture miss, not OpenRouter's up-front invalid-model 400.) - **OpenRouter discovery endpoints** `GET /api/v1/models` (catalog synthesized from loaded chat fixtures), `GET /api/v1/key`, and `GET /api/v1/credits`, matching the real response shapes. From f8e5e3d568a0c6ba2f61bb93b77f9c3e2e414816 Mon Sep 17 00:00:00 2001 From: Jordan Ritter Date: Wed, 29 Jul 2026 17:50:50 -0700 Subject: [PATCH 07/12] fix(responses): reach summary-less fixtures with encrypted_content MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Every reasoning item aimock emitted originated in a fixture's declared `reasoning` summary text: all five `buildReasoningOutputItem` call sites sat inside an `if (reasoning)` branch. So a request that opted in via `include: ["reasoning.encrypted_content"]` against a fixture with no declared summary got no reasoning item, and therefore no blob — starving the exact case the feature exists to serve, since a fixture RECORDED from the real stateless agent-framework flow has no summary (OpenAI returns `summary: []` unless summaries are requested). Verified on live bytes before the change: an opted-in gpt-5 request against a summary-less fixture returned `output: [message, function_call]` with zero occurrences of `encrypted_content`, and the SSE stream carried no reasoning events at all. Wire rationale: real OpenAI, for a reasoning-capable model, returns a reasoning item with `summary: []` and a populated `encrypted_content` whether or not summaries were requested — the item is the carrier for the blob the client must replay. So synthesize one, gated on: - the EXPLICIT `include` opt-in ONLY, which is NARROWER than the gate controlling the blob itself. Creating an output item that did not previously exist is a bigger behavior change than adding a field to one already on the wire, so it takes the gate directly observable in the request rather than the inferred `store: false` one, whose supporting capture evidence is not verifiable from this repo. `agent-framework-openai` >= 1.11.0 sends `include` and never sends `store`, so the narrowing costs the feature nothing. - the REQUESTED model's reasoning capability. gpt-4o has no reasoning channel, so an item there would be less faithful, not more. Unlike `resolveReasoningForModel`, which fails open for a declared summary (a recorded summary is evidence the model did reason), this gate is hard: nothing exists to preserve, so a synthesized item on a non-reasoning model would be pure fabrication. The resulting matrix on a reasoning-capable model, all five rows pinned by tests and verified on live bytes: include + summary → item with summary + blob (unchanged) include + NO summary → synthesized `summary: []` + blob (new) store:F + summary → item with summary + blob (unchanged) store:F + NO summary → NO reasoning item (unchanged) neither + either → no blob anywhere (unchanged) Streaming emits `output_item.added` → `output_item.done` with nothing between: an empty `summary` has no parts, so a `reasoning_summary_part`/`_text` event would describe a part that is not on the item. The synthesized item takes `output_index` 0 through the existing preamble accounting, so `nextOutputIndex` shifts the message / function_call / web_search items exactly as a declared summary already does, and every event's `output_index` still equals that item's slot in `response.completed.output`. The recorder is unaffected: stream-collapse derives `reasoning` only from `reasoning_summary_text.delta`, which a summary-less item never emits, so a re-record round-trips to a summary-less fixture rather than fabricating summary text. Both transports are covered — the WS Responses path shares these builders and computes both gates itself, so its rows are pinned separately and the narrowing cannot regress on one transport only. --- CHANGELOG.md | 2 +- src/__tests__/responses.test.ts | 193 ++++++++++++++++++++++++++++ src/__tests__/ws-responses.test.ts | 62 +++++++++ src/responses.ts | 198 +++++++++++++++++++++++------ src/ws-responses.ts | 8 ++ 5 files changed, 426 insertions(+), 37 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 4ddbc677..ddcf4e9c 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -4,7 +4,7 @@ ### Added -- **Reasoning `encrypted_content` on the Responses API.** aimock now synthesizes an opaque (base64) `reasoning.encrypted_content` blob on the TERMINAL reasoning item (`response.output_item.done` and non-streaming `output[]`). Emitted only when the request wants it: `include: ["reasoning.encrypted_content"]` OR a stateless request (`store: false` / ZDR). Stored / opted-out replays stay byte-identical. This lets `agent-framework-openai` >= 1.11.0 replay reasoning-paired tool calls on a stateless reasoning + multi-tool chain (related: microsoft/agent-framework#7233). Applies to both the HTTP and WebSocket Responses transports. Known limitations: the blob is withheld from the in-progress `added` item, which is aimock's own simplification rather than upstream parity — real OpenAI populates `added` too (recorded captures carry a shorter blob there beside an empty summary, re-encrypted by `done`), but the field is `anyOf: [string, null]` and non-required, so omitting it is contract-legal and no known consumer requires it; and aimock skips inbound reasoning items, so it cannot reproduce OpenAI's `invalid_encrypted_content` rejection of a mismatched blob/id. +- **Reasoning `encrypted_content` on the Responses API.** aimock now synthesizes an opaque (base64) `reasoning.encrypted_content` blob on the TERMINAL reasoning item (`response.output_item.done` and non-streaming `output[]`). Emitted only when the request wants it: `include: ["reasoning.encrypted_content"]` OR a stateless request (`store: false` / ZDR). Stored / opted-out replays stay byte-identical. A request carrying the EXPLICIT `include: ["reasoning.encrypted_content"]` opt-in, against a reasoning-CAPABLE model, now gets a reasoning item even when the fixture declares NO `reasoning` summary — a `summary: []` item that exists purely to carry the blob, matching what real OpenAI returns when summaries were never requested. This is the shape a fixture recorded from the real stateless agent-framework flow has, so gating the item on a declared summary starved the exact case the feature exists to serve. The synthesized item leads `output[]` / takes `output_index` 0 and shifts the rest, emits `output_item.added` → `output_item.done` with no summary-text events (there is no summary part to describe), and is suppressed for non-reasoning models (gpt-4o etc.), which have no reasoning channel at all. Note the two gates differ in width on purpose: `store: false` still attaches the blob to a reasoning item that a declared summary already produces, but it does NOT synthesize an item where the fixture declares none — creating an output item that did not previously exist is a bigger behavior change than adding a field to one already on the wire, so it takes the gate that is directly observable in the request. `agent-framework-openai` >= 1.11.0 sends `include` and never `store`, so the narrower gate costs the feature nothing. This lets `agent-framework-openai` >= 1.11.0 replay reasoning-paired tool calls on a stateless reasoning + multi-tool chain (related: microsoft/agent-framework#7233). Applies to both the HTTP and WebSocket Responses transports. Known limitations: the blob is withheld from the in-progress `added` item, which is aimock's own simplification rather than upstream parity — real OpenAI populates `added` too (recorded captures carry a shorter blob there beside an empty summary, re-encrypted by `done`), but the field is `anyOf: [string, null]` and non-required, so omitting it is contract-legal and no known consumer requires it; and aimock skips inbound reasoning items, so it cannot reproduce OpenAI's `invalid_encrypted_content` rejection of a mismatched blob/id. - **OpenRouter chat / LLM router simulation.** Requests whose original path starts with `/api/v1/` (point any OpenAI SDK at a `baseURL` ending `/api/v1`) are detected as OpenRouter and shaped to match real OpenRouter bytes: a `gen-` id prefix (a fixture `id` override still wins), a top-level `provider` (default = the winning model slug's author, fixture-overridable via `provider`), both `finish_reason` and `native_finish_reason` on every choice/delta, an always-present `system_fingerprint` and `service_tier` (null by default), `message.reasoning`, and a rich `usage` with a **fixture-scriptable** `cost` + `cost_details` (emitted only when a fixture supplies a cost — never fabricated), plus `is_byok` / `prompt_tokens_details` / `completion_tokens_details` when overridden. Callers on the plain OpenAI `/v1/...` base are byte-for-byte unchanged. - **`models[]` fallback (router failover) simulation.** When the request body carries `models: [...]`, aimock walks `[model, ...models]` in order and serves the first fixture returning a NON-error response; a `429`/`503` error fixture on a candidate simulates a runtime provider failure and falls through to the next. The winning slug is echoed back as the top-level `model`, so a test asserts failover by reading `response.model`. (Deliberate non-goal: an unknown/invalid model is a fixture miss, not OpenRouter's up-front invalid-model 400.) - **OpenRouter discovery endpoints** `GET /api/v1/models` (catalog synthesized from loaded chat fixtures), `GET /api/v1/key`, and `GET /api/v1/credits`, matching the real response shapes. diff --git a/src/__tests__/responses.test.ts b/src/__tests__/responses.test.ts index 7fa2e97f..8a344bf3 100644 --- a/src/__tests__/responses.test.ts +++ b/src/__tests__/responses.test.ts @@ -2097,6 +2097,199 @@ describe("reasoning.encrypted_content emission", () => { expect(r).not.toHaveProperty("encrypted_content"); }); } + + // --- Summary-LESS fixtures: the blob must still reach an opted-in client ---- + // + // The cases above all declare a `reasoning` summary in the fixture, which is + // what makes the reasoning item exist in the first place. A fixture RECORDED + // from the real stateless agent-framework flow typically has NO summary + // (OpenAI returns `summary: []` unless summaries are explicitly requested), so + // gating the item on a declared summary means the very flow this feature + // exists to serve gets no item and therefore no blob. Real OpenAI, for a + // reasoning-capable model, still returns a reasoning item carrying the blob. + // + // Emission stays gated on the SAME opt-in predicate, so a stored / opted-out + // replay of a summary-less fixture is byte-identical to before (no reasoning + // item at all), and on the requested model's reasoning capability, so a + // gpt-4o replay does not grow an item the real provider would never send. + + const noSummaryTextFixture: Fixture = { + match: { userMessage: "nosumtext" }, + response: { content: "AAPL is up" }, + }; + const noSummaryToolFixture: Fixture = { + match: { userMessage: "nosumtool" }, + response: { toolCalls: [{ name: "get_stock", arguments: "{}" }] }, + }; + const noSummaryContentToolFixture: Fixture = { + match: { userMessage: "nosumboth" }, + response: { content: "checking", toolCalls: [{ name: "get_stock", arguments: "{}" }] }, + }; + const noSummaryBlocksFixture: Fixture = { + match: { userMessage: "nosumblocks" }, + response: { + content: "checking", + toolCalls: [{ name: "get_stock", arguments: "{}" }], + blocks: [ + { type: "toolCall", name: "get_stock", arguments: "{}" }, + { type: "text", text: "checking" }, + ], + }, + }; + + const noSummaryCases: [string, Fixture, string][] = [ + ["text", noSummaryTextFixture, "nosumtext"], + ["tool-only", noSummaryToolFixture, "nosumtool"], + ["content+tool legacy", noSummaryContentToolFixture, "nosumboth"], + ["content+tool blocks", noSummaryBlocksFixture, "nosumblocks"], + ]; + + for (const [label, fixture, msg] of noSummaryCases) { + it(`non-streaming ${label} with NO declared summary carries an empty-summary item + blob when opted in`, async () => { + const output = await nonStreamReasoning(fixture, msg); + const r = output.find((o) => o.type === "reasoning") as + | { id: string; summary: unknown[]; encrypted_content?: string } + | undefined; + expect(r, "no reasoning output item").toBeDefined(); + expect(r!.summary).toEqual([]); + expect(r!.encrypted_content).toBe(expectedBlob(r!.id)); + // Real OpenAI leads the output array with the reasoning item. + expect(output[0].type).toBe("reasoning"); + }); + + it(`streaming ${label} with NO declared summary carries an empty-summary item + blob when opted in`, async () => { + const events = await streamReasoning(fixture, msg, OPT_IN); + const added = reasoningItem(events, "added") as + | { id: string; summary: unknown[]; encrypted_content?: string } + | undefined; + const done = reasoningItem(events, "done") as + | { id: string; summary: unknown[]; encrypted_content?: string } + | undefined; + expect(added, "no added reasoning item").toBeDefined(); + expect(done, "no done reasoning item").toBeDefined(); + expect(added!.summary).toEqual([]); + expect(done!.summary).toEqual([]); + expect(done!.encrypted_content).toBe(expectedBlob(done!.id)); + assertAddedNotPinned(added!); + }); + + // The whole blast-radius argument: nothing changes for a client that did not + // opt in. Pins the ABSENCE of any reasoning item, streaming and not. + it(`${label} with NO declared summary emits NO reasoning item when the request did not opt in`, async () => { + const output = await nonStreamReasoning(fixture, msg, {}); + expect(output.some((o) => o.type === "reasoning")).toBe(false); + + const events = await streamReasoning(fixture, msg, {}); + expect(reasoningItem(events, "added")).toBeUndefined(); + expect(reasoningItem(events, "done")).toBeUndefined(); + expect(events.some((e) => e.type.startsWith("response.reasoning_summary"))).toBe(false); + }); + } + + it("streaming a summary-less item emits added → done with NO summary-text events between", async () => { + const events = await streamReasoning(noSummaryTextFixture, "nosumtext", OPT_IN); + // An empty summary has no parts and no deltas: emitting a + // reasoning_summary_part/text event for it would describe a part that does + // not exist in the item's `summary: []`. + expect(events.filter((e) => e.type.startsWith("response.reasoning_summary"))).toEqual([]); + const seq = events + .filter((e) => (e.item as { type?: string })?.type === "reasoning") + .map((e) => e.type); + expect(seq).toEqual(["response.output_item.added", "response.output_item.done"]); + }); + + it("streaming a synthesized reasoning item keeps output_index aligned with completed.output", async () => { + const events = await streamReasoning(noSummaryContentToolFixture, "nosumboth", OPT_IN); + const completed = events.find((e) => e.type === "response.completed") as SSEEvent & { + response: { output: { type: string; id: string }[] }; + }; + // Reasoning must LEAD the output array and every other item shifts by one. + expect(completed.response.output.map((o) => o.type)).toEqual([ + "reasoning", + "message", + "function_call", + ]); + // Every output_item.done event's output_index must equal that item's slot in + // the final output array — an inserted leading item that failed to shift the + // sequencing would desync here. + const dones = events.filter((e) => e.type === "response.output_item.done"); + for (const d of dones) { + const id = (d.item as { id: string }).id; + expect(d.output_index).toBe(completed.response.output.findIndex((o) => o.id === id)); + } + }); + + it("does NOT synthesize a reasoning item for a non-reasoning model", async () => { + // gpt-4o never emits a reasoning channel, so an opted-in replay against it + // must not grow an item the real provider would never send (aimock#254). + instance = await createServer([noSummaryTextFixture]); + const res = await post(`${instance!.url}/v1/responses`, { + model: "gpt-4o", + input: [{ role: "user", content: "nosumtext" }], + stream: false, + ...OPT_IN, + }); + expect(res.status).toBe(200); + const body = JSON.parse(res.body) as { output: { type: string }[] }; + expect(body.output.some((o) => o.type === "reasoning")).toBe(false); + }); + + // --- The 2x2 matrix: which trigger may CREATE an item vs only decorate one --- + // + // Two gates, deliberately different widths: + // blob on an item being emitted anyway → `include` OR `store: false` + // SYNTHESIZING an item the fixture never declared → `include` ONLY + // + // Creating an output item that did not previously exist is a bigger behavior + // change than adding a field to one already on the wire, so it takes the gate + // that is directly observable in the request rather than the inferred + // `store: false` one. `agent-framework-openai` >= 1.11.0 sends `include` and + // never `store`, so the motivating client is unaffected by the narrowing. + + it("row 3: store:false + a DECLARED summary still carries the blob (non-streaming, unchanged)", async () => { + const output = await nonStreamReasoning(reasoningTextFixture, "textreason", { store: false }); + const r = output.find((o) => o.type === "reasoning")!; + expect(r, "no reasoning output item").toBeDefined(); + expect(r.encrypted_content).toBe(expectedBlob(r.id)); + }); + + // ROW 4 — the whole point of the narrowing, and the row that must never + // silently regress: `store: false` must NOT conjure a reasoning item for a + // fixture that declares no summary. Asserted on both transports' HTTP paths and + // for a reasoning-CAPABLE model, so the model gate cannot be what makes it pass. + it("row 4: store:false + NO declared summary synthesizes NOTHING (non-streaming)", async () => { + const output = await nonStreamReasoning(noSummaryTextFixture, "nosumtext", { store: false }); + expect(output.some((o) => o.type === "reasoning")).toBe(false); + // Guard against the model gate being the reason this passes: the SAME + // fixture and model DO produce the item under the explicit `include` opt-in. + const optedIn = await nonStreamReasoning(noSummaryTextFixture, "nosumtext"); + expect(optedIn.some((o) => o.type === "reasoning")).toBe(true); + }); + + it("row 4: store:false + NO declared summary synthesizes NOTHING (streaming)", async () => { + const events = await streamReasoning(noSummaryTextFixture, "nosumtext", { store: false }); + expect(reasoningItem(events, "added")).toBeUndefined(); + expect(reasoningItem(events, "done")).toBeUndefined(); + const completed = events.find((e) => e.type === "response.completed") as SSEEvent & { + response: { output: { type: string }[] }; + }; + expect(completed.response.output.some((o) => o.type === "reasoning")).toBe(false); + }); + + it("row 4 holds for every builder path (tool-only, content+tool legacy, blocks)", async () => { + for (const [, fixture, msg] of noSummaryCases) { + const output = await nonStreamReasoning(fixture, msg, { store: false }); + expect( + output.some((o) => o.type === "reasoning"), + `${msg} synthesized on store:false`, + ).toBe(false); + const events = await streamReasoning(fixture, msg, { store: false }); + expect( + reasoningItem(events, "done"), + `${msg} synthesized on store:false (stream)`, + ).toBeUndefined(); + } + }); }); // ─── Bug fix: multi-fco after single item_reference ───────────────────────── diff --git a/src/__tests__/ws-responses.test.ts b/src/__tests__/ws-responses.test.ts index f7f8eb4d..226dff88 100644 --- a/src/__tests__/ws-responses.test.ts +++ b/src/__tests__/ws-responses.test.ts @@ -912,4 +912,66 @@ describe("WebSocket /v1/responses encrypted reasoning", () => { ws.close(); }); + + // A fixture with NO declared `reasoning` summary — the shape a capture from the + // real stateless agent-framework flow has — must still reach an opted-in WS + // client with a summary-less reasoning item carrying the blob. Needs a + // reasoning-CAPABLE model (the shared `createMsg` above deliberately uses + // gpt-4.1, for which no reasoning item is synthesized at all). + function createReasoningModelMsg(userContent: string, extra: Record): string { + return JSON.stringify({ + type: "response.create", + model: "gpt-5", + input: [{ role: "user", content: userContent }], + ...extra, + }); + } + + for (const [label, msg] of [ + ["text", "hello"], + ["tool-only", "weather"], + ] as [string, string][]) { + it(`${label} fixture with NO declared summary still carries an empty-summary item + blob`, async () => { + instance = await createServer(allFixtures); + const ws = await connectWebSocket(instance.url, "/v1/responses"); + ws.send(createReasoningModelMsg(msg, { include: ["reasoning.encrypted_content"] })); + + const events = await collectUntilCompleted(ws); + const done = reasoningDone(events) as + | { id: string; summary: unknown[]; encrypted_content?: string } + | undefined; + expect(done, "no done reasoning item").toBeDefined(); + expect(done!.summary).toEqual([]); + expect(done!.encrypted_content).toBe(expectedBlob(done!.id)); + // No summary parts exist on the item, so no summary events describe them. + expect(events.some((e) => e.type.startsWith("response.reasoning_summary"))).toBe(false); + + ws.close(); + }); + + it(`${label} fixture with NO declared summary emits NO reasoning item when opted out`, async () => { + instance = await createServer(allFixtures); + const ws = await connectWebSocket(instance.url, "/v1/responses"); + ws.send(createReasoningModelMsg(msg, {})); + + const events = await collectUntilCompleted(ws); + expect(reasoningDone(events)).toBeUndefined(); + + ws.close(); + }); + + // The narrowing, over WS: `store: false` widens the BLOB gate but not the + // SYNTHESIS gate, so a summary-less fixture still yields no reasoning item. + // The WS dispatch computes both gates itself, so this needs its own case. + it(`${label} fixture with NO declared summary synthesizes NOTHING on store:false`, async () => { + instance = await createServer(allFixtures); + const ws = await connectWebSocket(instance.url, "/v1/responses"); + ws.send(createReasoningModelMsg(msg, { store: false })); + + const events = await collectUntilCompleted(ws); + expect(reasoningDone(events)).toBeUndefined(); + + ws.close(); + }); + } }); diff --git a/src/responses.ts b/src/responses.ts index c67e250f..d34a4f53 100644 --- a/src/responses.ts +++ b/src/responses.ts @@ -38,6 +38,7 @@ import { strictNoMatchMessage, strictNoMatchLogLine, } from "./helpers.js"; +import { isReasoningModel } from "./model-utils.js"; import { matchFixtureDiagnostic, recordMatchOptions } from "./router.js"; import { writeErrorResponse, delay, calculateDelay } from "./sse-writer.js"; import { createInterruptionSignal } from "./interruption.js"; @@ -299,6 +300,7 @@ export function buildTextStreamEvents( webSearches?: string[], overrides?: ResponseOverrides, emitEncryptedReasoning = false, + synthesizeSummarylessReasoning = false, ): ResponsesSSEEvent[] { const { respId, created, events, prefixOutputItems, nextOutputIndex } = buildResponsePreamble( model, @@ -307,6 +309,7 @@ export function buildTextStreamEvents( webSearches, overrides, emitEncryptedReasoning, + synthesizeSummarylessReasoning, ); const { events: msgEvents, msgItem } = buildMessageOutputEvents( @@ -340,6 +343,7 @@ export function buildToolCallStreamEvents( webSearches?: string[], overrides?: ResponseOverrides, emitEncryptedReasoning = false, + synthesizeSummarylessReasoning = false, ): ResponsesSSEEvent[] { const { respId, created, events, prefixOutputItems, nextOutputIndex } = buildResponsePreamble( model, @@ -348,6 +352,7 @@ export function buildToolCallStreamEvents( webSearches, overrides, emitEncryptedReasoning, + synthesizeSummarylessReasoning, ); const fcOutputItems: object[] = []; @@ -443,11 +448,32 @@ export function buildToolCallStreamEvents( * Responses transport can share one gate. */ export function requestWantsEncryptedReasoning(req: ResponsesRequest): boolean { - const include = req.include; - if (Array.isArray(include) && include.includes("reasoning.encrypted_content")) return true; + if (requestIncludesEncryptedReasoning(req)) return true; return req.store === false; } +/** + * Whether the request carries the EXPLICIT `include` opt-in — a strictly + * narrower gate than `requestWantsEncryptedReasoning`, which also fires on + * `store: false`. + * + * This is the gate for SYNTHESIZING a reasoning item that the fixture does not + * declare (see `shouldSynthesizeBlobOnlyReasoning`), and it is deliberately the + * narrow one. Attaching a field to an item aimock was already emitting is a much + * smaller step than conjuring a whole output item that did not previously exist, + * so the larger behavior change gets the gate that is directly observable in the + * request rather than the inferred `store: false` one. `agent-framework-openai` + * >= 1.11.0 auto-appends `include` on its stateless-replay path and never sends + * `store` at all, so the motivating client is fully served by this branch — the + * narrowing costs the feature nothing. + * + * Exported so the WebSocket Responses transport shares one gate. + */ +export function requestIncludesEncryptedReasoning(req: ResponsesRequest): boolean { + const include = req.include; + return Array.isArray(include) && include.includes("reasoning.encrypted_content"); +} + /** * Synthetic stand-in for OpenAI's opaque `reasoning.encrypted_content`. aimock * has no real ciphertext; it emits a base64 placeholder derived from the @@ -491,8 +517,60 @@ function buildReasoningOutputItem( return item; } +/** + * Whether to synthesize a summary-LESS reasoning item that exists purely to + * carry the encrypted blob. + * + * Every reasoning item aimock emits otherwise originates in a fixture's declared + * `reasoning` summary text. But a fixture RECORDED from the real stateless + * agent-framework flow typically has NO summary — OpenAI returns `summary: []` + * unless summaries are explicitly requested — so keying the item on a declared + * summary starves the exact flow this feature exists to serve: fully opted in, + * yet no reasoning item and therefore no blob. Real OpenAI, for a + * reasoning-capable model, still returns a reasoning item (`summary: []`) with + * the blob populated. So synthesize one, gated on: + * + * - the EXPLICIT `include` opt-in ONLY (`requestIncludesEncryptedReasoning`), + * which is NARROWER than the `requestWantsEncryptedReasoning` gate that + * controls the blob itself. A `store: false` request whose fixture declares no + * summary therefore gets NO reasoning item, exactly as before this change. + * Two reasons: (1) creating an output item that did not previously exist is a + * bigger behavior change than adding a field to an item already being + * emitted, so it earns the gate that is directly observable in the request + * rather than the inferred one; (2) the `store: false` trigger's supporting + * capture evidence is not verifiable from this repo, and a larger change must + * not be stacked on an unverifiable premise. `agent-framework-openai` >= + * 1.11.0 sends `include` and never `store`, so nothing is lost. + * - the REQUESTED model's reasoning capability (aimock#254) — emitting a + * reasoning item for gpt-4o would be LESS faithful, not more, since that + * model has no reasoning channel at all. Unlike `resolveReasoningForModel`, + * which fails open for a declared summary (a recorded summary is evidence the + * model did reason), there is nothing to preserve here: a synthesized item on + * a non-reasoning model would be pure fabrication, so this gate is hard. + * + * Note this is INDEPENDENT of `emitEncryptedReasoning`: when a fixture DOES + * declare a summary, the blob still rides on it under either trigger (including + * `store: false`), unchanged. + */ +function shouldSynthesizeBlobOnlyReasoning( + reasoning: string | undefined, + model: string | undefined, + synthesizeSummarylessReasoning: boolean, +): boolean { + if (reasoning) return false; // a declared summary already produces the item + if (!synthesizeSummarylessReasoning) return false; // no explicit `include` opt-in + return isReasoningModel(model); +} + +/** + * `reasoning` is `undefined` for a synthesized blob-only item (see + * `shouldSynthesizeBlobOnlyReasoning`). In that case the item's `summary` is + * `[]`, so the summary-part / summary-text events are SKIPPED entirely — they + * would describe a part that does not exist on the item — leaving a coherent + * `output_item.added` → `output_item.done` pair with nothing between. + */ function buildReasoningStreamEvents( - reasoning: string, + reasoning: string | undefined, chunkSize: number, emitEncryptedReasoning = false, ): ResponsesSSEEvent[] { @@ -505,40 +583,42 @@ function buildReasoningStreamEvents( item: buildReasoningOutputItem(reasoningId), }); - events.push({ - type: "response.reasoning_summary_part.added", - item_id: reasoningId, - output_index: 0, - summary_index: 0, - part: { type: "summary_text", text: "" }, - }); - - for (let i = 0; i < reasoning.length; i += chunkSize) { - const slice = reasoning.slice(i, i + chunkSize); + if (reasoning !== undefined) { events.push({ - type: "response.reasoning_summary_text.delta", + type: "response.reasoning_summary_part.added", item_id: reasoningId, output_index: 0, summary_index: 0, - delta: slice, + part: { type: "summary_text", text: "" }, }); - } - events.push({ - type: "response.reasoning_summary_text.done", - item_id: reasoningId, - output_index: 0, - summary_index: 0, - text: reasoning, - }); + for (let i = 0; i < reasoning.length; i += chunkSize) { + const slice = reasoning.slice(i, i + chunkSize); + events.push({ + type: "response.reasoning_summary_text.delta", + item_id: reasoningId, + output_index: 0, + summary_index: 0, + delta: slice, + }); + } - events.push({ - type: "response.reasoning_summary_part.done", - item_id: reasoningId, - output_index: 0, - summary_index: 0, - part: { type: "summary_text", text: reasoning }, - }); + events.push({ + type: "response.reasoning_summary_text.done", + item_id: reasoningId, + output_index: 0, + summary_index: 0, + text: reasoning, + }); + + events.push({ + type: "response.reasoning_summary_part.done", + item_id: reasoningId, + output_index: 0, + summary_index: 0, + part: { type: "summary_text", text: reasoning }, + }); + } events.push({ type: "response.output_item.done", @@ -605,6 +685,7 @@ function buildResponsePreamble( webSearches?: string[], overrides?: ResponseOverrides, emitEncryptedReasoning = false, + synthesizeSummarylessReasoning = false, ): PreambleResult { const respId = overrides?.id ?? responseId(); const created = overrides?.created ?? Math.floor(Date.now() / 1000); @@ -636,7 +717,10 @@ function buildResponsePreamble( }, }); - if (reasoning) { + if ( + reasoning || + shouldSynthesizeBlobOnlyReasoning(reasoning, model, synthesizeSummarylessReasoning) + ) { const reasoningEvents = buildReasoningStreamEvents( reasoning, chunkSize, @@ -800,13 +884,18 @@ function buildFunctionCallOutputEvents( function buildOutputPrefix( content: string, + model: string, reasoning?: string, webSearches?: string[], emitEncryptedReasoning = false, + synthesizeSummarylessReasoning = false, ): object[] { const output: object[] = []; - if (reasoning) { + if ( + reasoning || + shouldSynthesizeBlobOnlyReasoning(reasoning, model, synthesizeSummarylessReasoning) + ) { output.push( buildReasoningOutputItem(generateId("rs"), { summaryText: reasoning, @@ -860,10 +949,18 @@ function buildTextResponse( webSearches?: string[], overrides?: ResponseOverrides, emitEncryptedReasoning = false, + synthesizeSummarylessReasoning = false, ): object { return buildResponseEnvelope( model, - buildOutputPrefix(content, reasoning, webSearches, emitEncryptedReasoning), + buildOutputPrefix( + content, + model, + reasoning, + webSearches, + emitEncryptedReasoning, + synthesizeSummarylessReasoning, + ), overrides, ); } @@ -875,9 +972,13 @@ function buildToolCallResponse( webSearches?: string[], overrides?: ResponseOverrides, emitEncryptedReasoning = false, + synthesizeSummarylessReasoning = false, ): object { const output: object[] = []; - if (reasoning) { + if ( + reasoning || + shouldSynthesizeBlobOnlyReasoning(reasoning, model, synthesizeSummarylessReasoning) + ) { output.push( buildReasoningOutputItem(generateId("rs"), { summaryText: reasoning, @@ -918,6 +1019,7 @@ export function buildContentWithToolCallsStreamEvents( overrides?: ResponseOverrides, blocks?: FixtureBlock[], emitEncryptedReasoning = false, + synthesizeSummarylessReasoning = false, ): ResponsesSSEEvent[] { const { respId, created, events, prefixOutputItems, nextOutputIndex } = buildResponsePreamble( model, @@ -926,6 +1028,7 @@ export function buildContentWithToolCallsStreamEvents( webSearches, overrides, emitEncryptedReasoning, + synthesizeSummarylessReasoning, ); // The output items assembled in emission order (after any reasoning / @@ -1029,6 +1132,7 @@ function buildContentWithToolCallsResponse( overrides?: ResponseOverrides, blocks?: FixtureBlock[], emitEncryptedReasoning = false, + synthesizeSummarylessReasoning = false, ): object { if (blocks && blocks.length > 0) { // NEW PATH: the non-streaming `output[]` array is positionally observable, @@ -1038,7 +1142,10 @@ function buildContentWithToolCallsResponse( // path's ordering for the same `blocks` fixture. const ordered = resolveFixtureBlocks(blocks); const output: object[] = []; - if (reasoning) { + if ( + reasoning || + shouldSynthesizeBlobOnlyReasoning(reasoning, model, synthesizeSummarylessReasoning) + ) { output.push( buildReasoningOutputItem(generateId("rs"), { summaryText: reasoning, @@ -1073,7 +1180,14 @@ function buildContentWithToolCallsResponse( } // LEGACY PATH: message item first, then function_call items — unchanged. - const output = buildOutputPrefix(content, reasoning, webSearches, emitEncryptedReasoning); + const output = buildOutputPrefix( + content, + model, + reasoning, + webSearches, + emitEncryptedReasoning, + synthesizeSummarylessReasoning, + ); for (const tc of toolCalls) { output.push(buildFunctionCallOutputItem(tc)); } @@ -1177,6 +1291,12 @@ export async function handleResponses( // requests; without the blob it hard-fails a reasoning + multi-tool chain. // See microsoft/agent-framework#7233. const emitEncryptedReasoning = requestWantsEncryptedReasoning(responsesReq); + // Synthesizing a reasoning item the fixture never declared is a bigger step + // than adding a field to one already being emitted, so it takes the NARROWER + // explicit-`include` gate: a `store: false` request with a summary-less fixture + // keeps emitting no reasoning item at all. See + // `shouldSynthesizeBlobOnlyReasoning`. + const synthesizeSummarylessReasoning = requestIncludesEncryptedReasoning(responsesReq); const testId = getTestId(req); const { fixture, skippedBySequenceOrTurn } = matchFixtureDiagnostic( @@ -1355,6 +1475,7 @@ export async function handleResponses( overrides, response.blocks, emitEncryptedReasoning, + synthesizeSummarylessReasoning, ); res.writeHead(200, { "Content-Type": "application/json" }); res.end(JSON.stringify(body)); @@ -1369,6 +1490,7 @@ export async function handleResponses( overrides, response.blocks, emitEncryptedReasoning, + synthesizeSummarylessReasoning, ); const interruption = createInterruptionSignal(fixture); const completed = await writeResponsesSSEStream(res, events, { @@ -1415,6 +1537,7 @@ export async function handleResponses( response.webSearches, overrides, emitEncryptedReasoning, + synthesizeSummarylessReasoning, ); res.writeHead(200, { "Content-Type": "application/json" }); res.end(JSON.stringify(body)); @@ -1427,6 +1550,7 @@ export async function handleResponses( response.webSearches, overrides, emitEncryptedReasoning, + synthesizeSummarylessReasoning, ); const interruption = createInterruptionSignal(fixture); const completed = await writeResponsesSSEStream(res, events, { @@ -1473,6 +1597,7 @@ export async function handleResponses( response.webSearches, overrides, emitEncryptedReasoning, + synthesizeSummarylessReasoning, ); res.writeHead(200, { "Content-Type": "application/json" }); res.end(JSON.stringify(body)); @@ -1485,6 +1610,7 @@ export async function handleResponses( response.webSearches, overrides, emitEncryptedReasoning, + synthesizeSummarylessReasoning, ); const interruption = createInterruptionSignal(fixture); const completed = await writeResponsesSSEStream(res, events, { diff --git a/src/ws-responses.ts b/src/ws-responses.ts index f1f2ea70..8dfb2248 100644 --- a/src/ws-responses.ts +++ b/src/ws-responses.ts @@ -10,6 +10,7 @@ import type { ChatCompletionRequest, Fixture } from "./types.js"; import { matchFixtureDiagnostic } from "./router.js"; import { responsesToCompletionRequest, + requestIncludesEncryptedReasoning, requestWantsEncryptedReasoning, buildTextStreamEvents, buildToolCallStreamEvents, @@ -180,6 +181,10 @@ async function processMessage( // Gate encrypted-reasoning emission identically to the HTTP transport so the // agent-framework#7233 stateless-replay path works over WebSocket too. const emitEncryptedReasoning = requestWantsEncryptedReasoning(responsesReq); + // Synthesizing a reasoning item the fixture never declared takes the NARROWER + // explicit-`include` gate, transport-identically to HTTP — a `store: false` + // request with a summary-less fixture still gets no reasoning item. + const synthesizeSummarylessReasoning = requestIncludesEncryptedReasoning(responsesReq); const completionReq = responsesToCompletionRequest(responsesReq); completionReq._endpointType = "chat"; @@ -291,6 +296,7 @@ async function processMessage( extractOverrides(response), response.blocks, emitEncryptedReasoning, + synthesizeSummarylessReasoning, ); const interruption = createInterruptionSignal(fixture); @@ -335,6 +341,7 @@ async function processMessage( response.webSearches, extractOverrides(response), emitEncryptedReasoning, + synthesizeSummarylessReasoning, ); const interruption = createInterruptionSignal(fixture); const completed = await sendEvents( @@ -380,6 +387,7 @@ async function processMessage( response.webSearches, extractOverrides(response), emitEncryptedReasoning, + synthesizeSummarylessReasoning, ); const interruption = createInterruptionSignal(fixture); const completed = await sendEvents( From ecb05dfb65eb6fd6a95f9ba879658c286919aa13 Mon Sep 17 00:00:00 2001 From: Jordan Ritter Date: Wed, 29 Jul 2026 17:56:23 -0700 Subject: [PATCH 08/12] test: close encrypted-reasoning coverage gaps (leak direction, tautology, stale claim) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The encrypted-reasoning suite had three defects that let real regressions through. All three were confirmed by mutation before being fixed. 1. Leak direction covered only 5 of the 9 gate call sites. Hardcoding the emission ON at responses.ts:1371 (content+tool streaming), responses.ts:1487 (tool-only streaming), ws-responses.ts:293 (WS content+tool) or ws-responses.ts:382 (WS tool-only) left the ENTIRE suite green — a silent byte-level leak for every consumer that never opted in. Both single-case negatives are replaced by a loop generating one negative per streaming builder, so the sweep now catches 9/9 sites. Negatives pin genuine key absence via not.toHaveProperty, not a falsy value. 2. "streaming reflects the blob in response.completed.output" was tautological. buildResponsePreamble pushes the output_item.done event's item object into prefixOutputItems, so completed.response.output's reasoning item is the SAME REFERENCE — comparing the two was undefined === undefined, and the test passed with the feature entirely disabled. It now pins the id-derived base64 independently against completed.output. 3. The section header asserted real OpenAI "never" populates the in-progress `added` item. That is false (it does populate it opportunistically) and contradicted responses.ts's own note. Reworded to describe aimock's deliberate choice to carry the blob only on the terminal item — legal, since the field is anyOf:[string,null] and non-required, but not upstream parity. Also replaces assertAddedNotPinned, a conditional expect whose branch could never execute (aimock builds `added` with no emitEncrypted opt), so all five call sites asserted nothing — proven by putting a throw in the branch and watching 118/118 still pass. Now a real assertion of the omission. Test-only change; src/responses.ts and src/ws-responses.ts are untouched. --- src/__tests__/responses.test.ts | 74 +++++++++++++++++++++--------- src/__tests__/ws-responses.test.ts | 36 +++++++++++---- 2 files changed, 80 insertions(+), 30 deletions(-) diff --git a/src/__tests__/responses.test.ts b/src/__tests__/responses.test.ts index 8a344bf3..6d33345a 100644 --- a/src/__tests__/responses.test.ts +++ b/src/__tests__/responses.test.ts @@ -1880,12 +1880,18 @@ describe("reasoning_summary_text.done includes item_id", () => { // ─── reasoning.encrypted_content (microsoft/agent-framework#7233) ──────────── // -// Real OpenAI populates an opaque `encrypted_content` blob on the TERMINAL -// reasoning item (never the in-progress `added` item) when the request opts in -// via `include: ["reasoning.encrypted_content"]` OR is stateless (`store:false`) -// — the path used by agent-framework-openai >= 1.11.0. aimock synthesizes a -// deterministic placeholder so that client can replay the reasoning item instead -// of hard-failing the reasoning + multi-tool follow-up request. +// Real OpenAI populates an opaque `encrypted_content` blob on the reasoning item +// when the request opts in via `include: ["reasoning.encrypted_content"]` OR is +// stateless (`store:false`) — the path used by agent-framework-openai >= 1.11.0. +// aimock synthesizes a deterministic placeholder so that client can replay the +// reasoning item instead of hard-failing the reasoning + multi-tool follow-up +// request. +// +// aimock carries the blob on the TERMINAL (`output_item.done`) reasoning item and +// deliberately omits it from the in-progress `added` item. That is aimock's own +// CHOICE, not upstream parity — real OpenAI DOES populate `added`, but the field +// is `anyOf: [string, null]` and non-required, so `done` is where consumers must +// read it and omitting it on `added` is legal. describe("reasoning.encrypted_content emission", () => { // Mirror of `syntheticEncryptedReasoning` in responses.ts — pins the id→blob @@ -1905,12 +1911,13 @@ describe("reasoning.encrypted_content emission", () => { return ev?.item as { id: string; encrypted_content?: string } | undefined; } - // `added` placement is NOT contractual: real OpenAI populates it, aimock omits - // it, and both are legal (field is anyOf:[string,null], non-required). So a - // present blob must be a string, but absence must NOT fail. (Contrast the - // flag-off case below, which pins genuine absence.) - const assertAddedNotPinned = (item: { encrypted_content?: string }): void => { - if (item.encrypted_content !== undefined) expect(typeof item.encrypted_content).toBe("string"); + // Pins aimock's deliberate omission of the blob on the in-progress `added` item + // even when the gate is ON (see the header note — legal, but NOT a claim about + // what upstream does). This replaces a conditional-expect helper that could + // never fire: aimock builds the `added` item with no `emitEncrypted` opt at all, + // so the guarded branch was unreachable and every call site asserted nothing. + const expectAddedOmitsBlob = (item: object): void => { + expect(item).not.toHaveProperty("encrypted_content"); }; const reasoningTextFixture: Fixture = { @@ -1965,7 +1972,7 @@ describe("reasoning.encrypted_content emission", () => { ); const done = reasoningItem(events, "done")!; expect(done.encrypted_content).toBe(expectedBlob(done.id)); // pins presence + determinism - assertAddedNotPinned(reasoningItem(events, "added")!); + expectAddedOmitsBlob(reasoningItem(events, "added")!); }); // --- HTTP streaming: every builder + gate triggers + completed.output --- @@ -2012,18 +2019,23 @@ describe("reasoning.encrypted_content emission", () => { const done = reasoningItem(events, "done")!; expect(done, "no done reasoning item").toBeDefined(); expect(done.encrypted_content).toBe(expectedBlob(done.id)); - assertAddedNotPinned(reasoningItem(events, "added")!); + expectAddedOmitsBlob(reasoningItem(events, "added")!); }); } it("streaming reflects the blob in response.completed.output", async () => { const events = await streamReasoning(reasoningTextFixture, "textreason", OPT_IN); - const done = reasoningItem(events, "done")!; const completed = events.find((e) => e.type === "response.completed") as SSEEvent & { - response: { output: { type: string; encrypted_content?: string }[] }; + response: { output: { type: string; id: string; encrypted_content?: string }[] }; }; const outputReasoning = completed.response.output.find((o) => o.type === "reasoning"); - expect(outputReasoning!.encrypted_content).toBe(done.encrypted_content); + expect(outputReasoning, "no reasoning item in completed.output").toBeDefined(); + // Pin the blob INDEPENDENTLY, against the id-derived expectation — NOT against + // the `output_item.done` event's item. `buildResponsePreamble` pushes that very + // object into `prefixOutputItems`, so the two are the SAME REFERENCE and + // comparing them is `undefined === undefined` the moment emission stops: the + // assertion passed with the feature entirely off. + expect(outputReasoning!.encrypted_content).toBe(expectedBlob(outputReasoning!.id)); }); it("streaming emits the blob for a stateless request (store:false) without include", async () => { @@ -2032,11 +2044,29 @@ describe("reasoning.encrypted_content emission", () => { expect(done.encrypted_content).toBe(expectedBlob(done.id)); }); - it("streaming omits the blob for a stored request that did not opt in", async () => { - const events = await streamReasoning(reasoningTextFixture, "textreason", {}); - expect(reasoningItem(events, "added")!.encrypted_content).toBeUndefined(); - expect(reasoningItem(events, "done")!.encrypted_content).toBeUndefined(); - }); + // --- HTTP streaming LEAK direction: the same builders must stay silent --- + // + // The positive cases above only prove the blob CAN be emitted; they pass just as + // well if a call site hardcodes the flag ON. A leak is the worse failure for a + // mock: it silently changes response bytes for every consumer that never opted + // in (no `include`, no `store:false`), so the stored / opted-out replay is no + // longer byte-identical. + // + // `handleResponses` threads the gate into each streaming builder from a SEPARATE + // call site (buildTextStreamEvents, buildToolCallStreamEvents and + // buildContentWithToolCallsStreamEvents), so one negative per case is required — + // with only the text negative present, hardcoding the flag on at either of the + // other two sites left the whole suite green. + for (const [label, fixture, msg] of streamingCases) { + it(`streaming ${label} omits the blob when the request did not opt in`, async () => { + const events = await streamReasoning(fixture, msg, {}); + const done = reasoningItem(events, "done"); + expect(done, "no done reasoning item").toBeDefined(); + // Pin genuine key ABSENCE, not merely a falsy value. + expect(done!).not.toHaveProperty("encrypted_content"); + expect(reasoningItem(events, "added")!).not.toHaveProperty("encrypted_content"); + }); + } // --- Non-streaming: every builder path (incl. the positional blocks branch) --- diff --git a/src/__tests__/ws-responses.test.ts b/src/__tests__/ws-responses.test.ts index 226dff88..9ca4047f 100644 --- a/src/__tests__/ws-responses.test.ts +++ b/src/__tests__/ws-responses.test.ts @@ -902,16 +902,36 @@ describe("WebSocket /v1/responses encrypted reasoning", () => { ws.close(); }); - it("omits the blob when the request does not opt in", async () => { - instance = await createServer(allFixtures); - const ws = await connectWebSocket(instance.url, "/v1/responses"); - ws.send(createMsg("think", {})); + // --- LEAK direction: every WS gate call site must stay silent --- + // + // The positive cases above only prove the blob CAN be emitted; they pass just as + // well if a call site hardcodes the flag ON, which silently changes the bytes + // every consumer that never opted in receives. The WS dispatch threads the gate + // into each streaming builder from a SEPARATE call site, so one negative per + // branch is required — with only the text negative present, hardcoding the flag + // on in either the content+toolCalls or the tool-only branch left the entire + // suite green. + const wsLeakCases: [string, string][] = [ + ["text (buildTextStreamEvents)", "think"], + ["tool-only (buildToolCallStreamEvents)", "tool-reason"], + ["content+toolCalls (buildContentWithToolCallsStreamEvents)", "both-reason"], + ]; + + for (const [label, msg] of wsLeakCases) { + it(`omits the blob for ${label} when the request does not opt in`, async () => { + instance = await createServer(allFixtures); + const ws = await connectWebSocket(instance.url, "/v1/responses"); + ws.send(createMsg(msg, {})); - const events = await collectUntilCompleted(ws); - expect(reasoningDone(events)!.encrypted_content).toBeUndefined(); + const events = await collectUntilCompleted(ws); + const done = reasoningDone(events); + expect(done, "no done reasoning item").toBeDefined(); + // Pin genuine key ABSENCE, not merely a falsy value. + expect(done!).not.toHaveProperty("encrypted_content"); - ws.close(); - }); + ws.close(); + }); + } // A fixture with NO declared `reasoning` summary — the shape a capture from the // real stateless agent-framework flow has — must still reach an opted-in WS From 1ceca60fae2718419919c7b54fdf5b42c4839a36 Mon Sep 17 00:00:00 2001 From: Jordan Ritter Date: Wed, 29 Jul 2026 17:51:10 -0700 Subject: [PATCH 09/12] test: grade the encrypted-reasoning blob by VALUE and isolate the two legs' delta keys MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Three reviewer claims about 57ad4c0's drift gate; verified each locally before changing anything. Two were real, one was overstated. CLAIM 1 (paths filter omits src/responses.ts) — fact TRUE, conclusion FALSE. The filter does omit it, but test-unit.yml has NO paths filter, so `pnpm test` runs on every PR and its value pins hard-fail on a dropped/null/empty blob (11 tests red in responses.test.ts + ws-responses.test.ts against a null blob). So this is a cost/latency scoping choice, not a coverage hole — adding a graded production file would pull in nearly all of src/ and run a ~30-min live-credit job on almost every PR. Documented the deliberate scope on the filter instead of widening it. CLAIM 2 (the two legs collide in delta keying) — TRUE. `computeDelta` keys by `provider::id`, `parseDriftBlock` sets `id: path`, and `DriftEntry.scenario` never enters the key. Verified against the real computeDelta: two entries differing only by scenario collapse to ONE key, and an opted-OUT finding on base demotes a new-in-head opted-IN finding at the same path from block[] to advisory[] — it stops failing the required check. Fixed inside this leg (scenario+event-type path prefix) rather than re-keying the shared delta machinery every surface depends on. CLAIM 3 (the assertion passes on null; opted-out gates nothing) — TRUE, both halves, reproduced against the live suite: - opted IN, blob `null` → schema.ts's null-vs-other exemption → leg GREEN. - opted IN, blob `""` → still kind "string" → leg GREEN. - opted OUT, blob present → "MOCK EXTRA FIELD" grades `info`, and the legs fail only on `critical` → leg GREEN. Only outright ABSENCE was caught. `gradeEncryptedBlob` now grades the terminal item's value in both directions and emits through formatDriftReport with the openai-responses surface marker, so a violation is a routed exit-2 finding for the delta gate rather than a bare expect failure the collector could only quarantine. Red-green via the command test-drift.yml actually gates with (`npx tsx scripts/drift-report-collector.ts --out drift-report-head.json`, then the delta gate) — all four sabotages of src/responses.ts now flip collector exit 0 -> 2 and delta gate PASSED -> BLOCKED, on distinct per-leg delta keys: absent, null and empty block on `OpenAI Responses Encrypted Reasoning:...`, while over-emission blocks on `OpenAI Responses Reasoning:...`. Production restored and byte-identical afterwards. No production behavior change. --- .github/workflows/test-drift.yml | 14 ++ src/__tests__/drift/openai-responses.drift.ts | 150 ++++++++++++++++-- 2 files changed, 153 insertions(+), 11 deletions(-) diff --git a/.github/workflows/test-drift.yml b/.github/workflows/test-drift.yml index 0451767e..ed9d304e 100644 --- a/.github/workflows/test-drift.yml +++ b/.github/workflows/test-drift.yml @@ -3,6 +3,20 @@ on: schedule: - cron: "0 6 * * *" # Daily 6am UTC pull_request: + # DELIBERATE SCOPE — this filter lists DRIFT-GRADING code, not the production + # files the drift legs grade. `drift-live-pr` is a ~30-minute job that burns + # real provider credits on every trigger, and its question is "does THIS diff + # introduce provider-format divergence in the drift harness", not "does the + # mock still emit field X". Adding a graded production file (src/responses.ts, + # src/ws-responses.ts, …) would, by the same logic, pull in nearly every file + # under src/ and run the live leg on almost every PR. + # + # Per-field emission properties are gated by the ALWAYS-ON unit lane instead: + # test-unit.yml has no `paths:` filter, so `pnpm test` runs on every PR and + # hard-fails on a dropped/null/empty value (e.g. the `encrypted_content` + # value pins in src/__tests__/responses.test.ts and ws-responses.test.ts). + # If you came here because a drift assertion "never triggers on the code it + # guards": check the unit lane first — that is where the value pin lives. paths: - "src/agui-types.ts" - "src/__tests__/drift/agui-schema.drift.ts" diff --git a/src/__tests__/drift/openai-responses.drift.ts b/src/__tests__/drift/openai-responses.drift.ts index a4bc1edd..47d3fc42 100644 --- a/src/__tests__/drift/openai-responses.drift.ts +++ b/src/__tests__/drift/openai-responses.drift.ts @@ -13,6 +13,7 @@ import { compareSSESequences, formatDriftReport, type SSEEventShape, + type ShapeDiff, } from "./schema.js"; import { openaiResponsesNonStreamingShape, @@ -566,13 +567,23 @@ describe("OpenAI Responses API reasoning drift", () => { }); // NOTE: encrypted_content emission is ALSO enforced by the unit + WS suites - // (responses.test.ts / ws-responses.test.ts), which hard-fail on a dropped - // blob. The drift-side anchor is the shape pin in sdk-shapes.ts, and it is - // split by opt-in: `openaiResponsesReasoningEventShapes` (no blob) grades the - // opted-OUT legs, `openaiResponsesEncryptedReasoningEventShapes` (blob) grades - // the opted-IN leg below. Pinning the blob unconditionally made every - // opted-out leg report `item.encrypted_content` critical forever — a finding - // that no code change could clear, so it gated nothing. + // (responses.test.ts / ws-responses.test.ts), which hard-fail on a dropped, + // null or empty blob and run in the ALWAYS-ON `pnpm test` lane (test-unit.yml + // has no `paths:` filter). That lane — not this file — is what gates a PR that + // touches src/responses.ts; `test-drift.yml`'s PR `paths:` filter deliberately + // scopes the live PR leg to drift-grading code (see the comment on that + // filter). + // + // The drift-side anchor is the shape pin in sdk-shapes.ts, split by opt-in: + // `openaiResponsesReasoningEventShapes` (no blob) grades the opted-OUT legs, + // `openaiResponsesEncryptedReasoningEventShapes` (blob) grades the opted-IN + // leg below. Pinning the blob unconditionally made every opted-out leg report + // `item.encrypted_content` critical forever — a finding that no code change + // could clear, so it gated nothing. + // + // A SHAPE pin alone is not enough in either direction, so `gradeEncryptedBlob` + // below adds the value-level grading. See its docstring for what shape + // comparison structurally cannot see. it("reasoning event shapes include item_id, output_index, summary_index", async () => { const res = await httpPost(`${reasoningInstance.url}/v1/responses`, { @@ -629,6 +640,20 @@ describe("OpenAI Responses API reasoning drift", () => { expect((partDoneData.part as { type: string; text: string }).text).toBe(REASONING_TEXT); }); + // DELTA-KEY ISOLATION. The collector derives each diff's delta `id` from its + // `path` (`parseDriftBlock`: `id: path`) and `computeDelta` keys findings by + // `provider::id` ONLY — `DriftEntry.scenario` rides along in the report but + // NEVER enters the key. So two findings that differ only by scenario collapse + // last-write-wins: verified against the real `computeDelta`, an opted-OUT + // finding at `item.encrypted_content` sitting on the base report demotes a + // NEW-in-head opted-IN finding at the same path from `block[]` to `advisory[]` + // — i.e. it stops failing the required check. Prefixing every path this file + // reports with its scenario + event type keeps the two legs' delta keys + // disjoint without touching the shared delta machinery that every other drift + // surface depends on. + const scopePaths = (diffs: ShapeDiff[], prefix: string): ShapeDiff[] => + diffs.map((d) => ({ ...d, path: `${prefix}:${d.path}` })); + // Grade a reasoning SSE body against the SDK shape variant for the request // that produced it. `scenario` keeps the opted-out and opted-in legs' drift // keys distinct so the delta gate attributes a finding to the right variant. @@ -652,8 +677,12 @@ describe("OpenAI Responses API reasoning drift", () => { continue; } - const diffs = triangulate(sdkEvent.dataShape, sdkEvent.dataShape, mockEvent.dataShape); - const report = formatDriftReport(`${scenario}:${sdkEvent.type}`, diffs, "openai-responses"); + const context = `${scenario}:${sdkEvent.type}`; + const diffs = scopePaths( + triangulate(sdkEvent.dataShape, sdkEvent.dataShape, mockEvent.dataShape), + context, + ); + const report = formatDriftReport(context, diffs, "openai-responses"); expect( diffs.filter((d) => d.severity === "critical"), @@ -662,6 +691,100 @@ describe("OpenAI Responses API reasoning drift", () => { } }; + /** + * VALUE-level grading of the terminal reasoning item's `encrypted_content`. + * + * A shape pin structurally CANNOT see a degenerate value, in either direction + * (all four cases below were reproduced locally against this suite): + * + * - opted IN, `encrypted_content: null` → kind "null" vs the pin's "string", + * and schema.ts's real-vs-mock check deliberately exempts null-vs-other + * ("Allow null vs other type (optional fields)") → no diff, leg GREEN. + * - opted IN, `encrypted_content: ""` → still kind "string" → no diff, + * leg GREEN. + * - opted OUT, blob present → a field in mock but not in the pin + * grades `info` ("MOCK EXTRA FIELD"), and these legs fail only on + * `critical` → leg GREEN. + * - opted IN, blob absent → the ONE case the shape pin does + * catch (critical). + * + * The blob is the payload agent-framework-openai >= 1.11.0 replays verbatim on + * its stateless path (microsoft/agent-framework#7233), so a null or empty blob + * is exactly as broken as an absent one, and a blob handed to a request that + * never opted in is drift too. + * + * Emitted through `formatDriftReport` with the `openai-responses` surface + * marker so a violation lands as a ROUTED drift finding (collector → exit 2 → + * delta gate) instead of a bare `expect` failure the collector could only + * quarantine (exit 5). + */ + const gradeEncryptedBlob = (body: string, expectBlob: boolean, scenario: string) => { + const label = expectBlob ? "opted-in" : "opted-out"; + const path = `${scenario}:response.output_item.done:item.encrypted_content(value)`; + const diffs: ShapeDiff[] = []; + + const terminal = parseTypedSSE(body) + .filter( + (e) => + e.type === "response.output_item.done" && + (e.data as { item?: { type?: string } }).item?.type === "reasoning", + ) + .at(-1); + const item = terminal + ? (terminal.data as { item: { encrypted_content?: unknown } }).item + : undefined; + + if (item === undefined) { + // Folded into the diff list (not a bare `expect`) so a missing terminal + // item is a routed critical finding rather than a quarantined failure. + diffs.push({ + path, + severity: "critical", + issue: "LLMOCK DRIFT — no terminal response.output_item.done with type=reasoning", + expected: "reasoning item", + real: "reasoning item", + mock: "", + }); + } else { + const blob = item.encrypted_content; + const observed = blob === undefined ? "" : JSON.stringify(blob); + if (expectBlob && (typeof blob !== "string" || blob.length === 0)) { + diffs.push({ + path, + severity: "critical", + issue: + "LLMOCK DRIFT — opted-in request must carry a NON-EMPTY encrypted_content string " + + "(agent-framework replays it verbatim; null/empty is as broken as absent)", + expected: "non-empty string", + real: "non-empty string", + mock: observed, + }); + } else if (!expectBlob && blob !== undefined) { + diffs.push({ + path, + severity: "critical", + issue: + "LLMOCK DRIFT — opted-out request must carry NO encrypted_content " + + "(over-emission hands the blob to a consumer that never asked for it)", + expected: "", + real: "", + mock: observed, + }); + } + } + + const report = formatDriftReport( + `${scenario} (${label} encrypted_content value)`, + diffs, + "openai-responses", + ); + + expect( + diffs.filter((d) => d.severity === "critical"), + report, + ).toEqual([]); + }; + it("reasoning event shapes triangulate against SDK expectations", async () => { // Opted OUT: no `include`, no `store: false` — the terminal item must carry // NO encrypted_content, so this leg is graded against the no-blob variant. @@ -678,13 +801,17 @@ describe("OpenAI Responses API reasoning drift", () => { res.body, "OpenAI Responses Reasoning", ); + // Over-emission check: a blob here means the gate leaked it to a request + // that never opted in. Invisible to the shape pin (grades `info`). + gradeEncryptedBlob(res.body, false, "OpenAI Responses Reasoning"); }); - it("opted-in reasoning event shapes carry encrypted_content", async () => { + it("opted-in reasoning event shapes carry a non-empty encrypted_content", async () => { // Opted IN via `include` — the exact request agent-framework-openai // >= 1.11.0 sends on its stateless-replay path. This is the leg that GATES // the emission: drop `encrypted_content` in responses.ts and the terminal - // item reports `item.encrypted_content` critical here. + // item reports `item.encrypted_content` critical here. The shape pin catches + // ABSENCE; `gradeEncryptedBlob` catches null/empty, which the pin cannot. const res = await httpPost(`${reasoningInstance.url}/v1/responses`, { model: "gpt-4o-mini", input: [{ role: "user", content: "Think carefully" }], @@ -699,5 +826,6 @@ describe("OpenAI Responses API reasoning drift", () => { res.body, "OpenAI Responses Encrypted Reasoning", ); + gradeEncryptedBlob(res.body, true, "OpenAI Responses Encrypted Reasoning"); }); }); From 38f917835f6e130866c7fff3aa5b7e895306e2c9 Mon Sep 17 00:00:00 2001 From: Jordan Ritter Date: Wed, 29 Jul 2026 18:33:25 -0700 Subject: [PATCH 10/12] fix(test): point the orphaned assertAddedNotPinned call site at the surviving helper MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The merge of three branches left `src/__tests__/responses.test.ts` calling a helper that no longer exists. `a3a5446` added `assertAddedNotPinned` plus two call sites; `f8e5e3d` added a third (the summary-less streaming loop) from a different branch; `ecb05df` renamed the helper to `expectAddedOmitsBlob` and updated only the two call sites it could see. The third was orphaned. `pnpm test` died with `ReferenceError: assertAddedNotPinned is not defined` on 4 tests. Nothing else catches it: `tsconfig.json` excludes `src/__tests__`, so `tsc --noEmit` exits 0 on the broken tree, and CI has no typecheck job at all — `pnpm test` was the only guard. Before: Tests 4 failed | 4816 passed | 3 skipped (4823) After: Tests 4820 passed | 3 skipped (4823) --- src/__tests__/responses.test.ts | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/__tests__/responses.test.ts b/src/__tests__/responses.test.ts index 6d33345a..77b2622c 100644 --- a/src/__tests__/responses.test.ts +++ b/src/__tests__/responses.test.ts @@ -2200,7 +2200,7 @@ describe("reasoning.encrypted_content emission", () => { expect(added!.summary).toEqual([]); expect(done!.summary).toEqual([]); expect(done!.encrypted_content).toBe(expectedBlob(done!.id)); - assertAddedNotPinned(added!); + expectAddedOmitsBlob(added!); }); // The whole blast-radius argument: nothing changes for a client that did not From 52e8865957ae64247a8db28e4cd3fac6083d66fe Mon Sep 17 00:00:00 2001 From: Jordan Ritter Date: Wed, 29 Jul 2026 18:36:10 -0700 Subject: [PATCH 11/12] test(responses): pin `include` SELECTIVITY, not just its presence MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Every existing leak negative sends NO `include` at all, so the suite pinned whether the blob gate reads `include` but never that it reads the RIGHT VALUE. Broadening `requestIncludesEncryptedReasoning` from `include.includes("reasoning.encrypted_content")` to any non-empty `include` left all 4823 tests green, and a live aimock instance then handed the blob to a request carrying `include: ["file_search_call.results"]` — byte-level over-emission to a client that never asked for reasoning at all. Adds four negatives using `file_search_call.results`, a real member of the SDK's `ResponseIncludable` union (`'file_search_call.results' | 'message.input_image.image_url' | 'computer_call_output.output.image_url' | 'reasoning.encrypted_content'` in openai@4.104.0), with no `store` so the gate's other branch cannot be what makes them pass: - HTTP streaming: no `encrypted_content` anywhere in the raw SSE bytes - HTTP non-streaming: same, on the JSON body - synthesis gate: an unrelated include value conjures no reasoning item - WS: same, over the WebSocket transport, which reads `include` off the parsed frame itself RED-GREEN (`pnpm test`, full suite): mutate `requestIncludesEncryptedReasoning` body -> `include.length > 0` FAIL responses.test.ts > streaming omits the blob for a non-empty include... FAIL responses.test.ts > non-streaming omits the blob for a non-empty include... FAIL responses.test.ts > a non-empty include ... synthesizes NOTHING FAIL ws-responses.test.ts > omits the blob for a non-empty include... Tests 4 failed | 4820 passed | 3 skipped (4827) mutate `requestWantsEncryptedReasoning`'s include branch -> `include.length > 0` Tests 3 failed | 4821 passed | 3 skipped (4827) restored Test Files 166 passed (166) / Tests 4824 passed | 3 skipped (4827) --- src/__tests__/responses.test.ts | 71 ++++++++++++++++++++++++++++++ src/__tests__/ws-responses.test.ts | 23 ++++++++++ 2 files changed, 94 insertions(+) diff --git a/src/__tests__/responses.test.ts b/src/__tests__/responses.test.ts index 77b2622c..1cc66bd5 100644 --- a/src/__tests__/responses.test.ts +++ b/src/__tests__/responses.test.ts @@ -2320,6 +2320,77 @@ describe("reasoning.encrypted_content emission", () => { ).toBeUndefined(); } }); + + // --- `include` SELECTIVITY: an UNRELATED include value is not an opt-in ----- + // + // Every leak negative above sends NO `include` at all, so together they pin the + // predicate's presence/absence but NOT its selectivity. Broadening + // `requestIncludesEncryptedReasoning` from + // `include.includes("reasoning.encrypted_content")` to any non-empty `include` + // left the ENTIRE suite green (verified by mutation), and a real client sending + // `include: ["file_search_call.results"]` then received a blob it never asked + // for — the same byte-level over-emission the per-call-site negatives close, + // but per include VALUE rather than per call site. + // + // `file_search_call.results` is a genuine member of the SDK's + // `ResponseIncludable` union (openai/resources/responses/responses.d.ts: + // `'file_search_call.results' | 'message.input_image.image_url' | + // 'computer_call_output.output.image_url' | 'reasoning.encrypted_content'`) + // with nothing to do with reasoning, so this exercises a request a client + // actually sends rather than a fabricated string. + // + // `store` is deliberately absent: `requestWantsEncryptedReasoning`'s other + // branch fires on `store: false`, so sending it would make the assertion pass + // for the wrong reason. + const UNRELATED_INCLUDE = { include: ["file_search_call.results"] }; + + it("streaming omits the blob for a non-empty include that lacks the reasoning value", async () => { + instance = await createServer([reasoningTextFixture]); + const res = await post(`${instance!.url}/v1/responses`, { + model: "gpt-5-test", + input: [{ role: "user", content: "textreason" }], + stream: true, + ...UNRELATED_INCLUDE, + }); + expect(res.status).toBe(200); + // NOWHERE in the raw bytes — stronger than an item-level check, and it also + // covers `response.completed.output` and any future carrier of the field. + expect(res.body).not.toContain("encrypted_content"); + const events = parseResponsesSSEEvents(res.body); + expect(reasoningItem(events, "done"), "no done reasoning item").toBeDefined(); + expect(reasoningItem(events, "done")!).not.toHaveProperty("encrypted_content"); + expect(reasoningItem(events, "added")!).not.toHaveProperty("encrypted_content"); + }); + + it("non-streaming omits the blob for a non-empty include that lacks the reasoning value", async () => { + instance = await createServer([reasoningTextFixture]); + const res = await post(`${instance!.url}/v1/responses`, { + model: "gpt-5-test", + input: [{ role: "user", content: "textreason" }], + stream: false, + ...UNRELATED_INCLUDE, + }); + expect(res.status).toBe(200); + expect(res.body).not.toContain("encrypted_content"); + const body = JSON.parse(res.body) as { output: { type: string }[] }; + expect( + body.output.some((o) => o.type === "reasoning"), + "no reasoning output item", + ).toBe(true); + }); + + // The SYNTHESIS gate is the same predicate, so its selectivity needs the same + // negative: an unrelated include value must not conjure an item either. + it("a non-empty include that lacks the reasoning value synthesizes NOTHING", async () => { + const output = await nonStreamReasoning(noSummaryTextFixture, "nosumtext", UNRELATED_INCLUDE); + expect(output.some((o) => o.type === "reasoning")).toBe(false); + const events = await streamReasoning(noSummaryTextFixture, "nosumtext", UNRELATED_INCLUDE); + expect(reasoningItem(events, "done")).toBeUndefined(); + // Guard against the model gate being the reason this passes: the SAME fixture + // and model DO produce the item under the explicit opt-in. + const optedIn = await nonStreamReasoning(noSummaryTextFixture, "nosumtext"); + expect(optedIn.some((o) => o.type === "reasoning")).toBe(true); + }); }); // ─── Bug fix: multi-fco after single item_reference ───────────────────────── diff --git a/src/__tests__/ws-responses.test.ts b/src/__tests__/ws-responses.test.ts index 9ca4047f..f0ac8e92 100644 --- a/src/__tests__/ws-responses.test.ts +++ b/src/__tests__/ws-responses.test.ts @@ -933,6 +933,29 @@ describe("WebSocket /v1/responses encrypted reasoning", () => { }); } + // `include` SELECTIVITY. The leak cases above send NO `include` at all, so they + // pin the predicate's presence/absence but not its selectivity: broadening the + // gate from `include.includes("reasoning.encrypted_content")` to any non-empty + // `include` left the whole suite green. `file_search_call.results` is a genuine + // member of the SDK's `ResponseIncludable` union with nothing to do with + // reasoning, so a real client sending it must not receive a blob. The WS + // dispatch reads `include` off the parsed frame itself, so this needs its own + // transport-level case. No `store` — that is the gate's OTHER branch. + it("omits the blob for a non-empty include that lacks the reasoning value", async () => { + instance = await createServer(allFixtures); + const ws = await connectWebSocket(instance.url, "/v1/responses"); + ws.send(createMsg("think", { include: ["file_search_call.results"] })); + + const events = await collectUntilCompleted(ws); + const done = reasoningDone(events); + expect(done, "no done reasoning item").toBeDefined(); + expect(done!).not.toHaveProperty("encrypted_content"); + // NOWHERE in the frames, which also covers completed.output. + expect(JSON.stringify(events)).not.toContain("encrypted_content"); + + ws.close(); + }); + // A fixture with NO declared `reasoning` summary — the shape a capture from the // real stateless agent-framework flow has — must still reach an opted-in WS // client with a summary-less reasoning item carrying the blob. Needs a From ca417858524ef47437afe8c5d7afa3e3d415f063 Mon Sep 17 00:00:00 2001 From: Jordan Ritter Date: Wed, 29 Jul 2026 18:39:12 -0700 Subject: [PATCH 12/12] test(responses): stop pinning a shape real OpenAI actually emits on `added` MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit `expectAddedOmitsBlob` asserted that the in-progress `response.output_item.added` reasoning item has NO `encrypted_content`, for OPTED-IN requests. That is wrong to pin: real OpenAI populates `added` opportunistically, and the field is `anyOf: [string, null]` and non-required — so aimock omitting it is legal, but it is aimock's own CHOICE, not the provider contract. The commit that introduced the helper said so in its own message and in the production comment on `buildReasoningOutputItem`, then pinned the omission anyway. 9 tests would fail any future change that made aimock MORE faithful. The drift contract in this same tree already draws the line correctly, and this brings the unit suite into line with it: `sdk-shapes.ts` anchors the blob on `done` ONLY — "pinning `added` would flag an intentional, legal shape choice as drift" — and `gradeEncryptedBlob` grades the terminal item alone. Renamed to `expectAddedBlobIsContractLegal`, which now grades only what is contractual for an opted-in request: absent is fine, and if present it must be a string equal to the id-derived `expectedBlob(id)`. It does NOT assert presence, so aimock's current omission keeps passing. Opted-out call sites are untouched — there, absence genuinely IS the contract, and they assert it directly. Nothing real is lost, verified by mutation rather than assumed (`pnpm test`): blob on `added` IN ADDITION to terminal (contract-legal, must survive) before: 9 failed -> after: Tests 4824 passed | 3 skipped (4827) blob on `added` INSTEAD of terminal (the property that matters, P1) still CAUGHT: 17 failed | 4807 passed — every failure via a terminal-item assertion, in both the HTTP and WS suites wrong blob VALUE on `added` when opted in (is the relaxed helper vacuous?) CAUGHT: 9 failed | 4815 passed — exactly the 9 call sites --- src/__tests__/responses.test.ts | 48 ++++++++++++++++++++++++++------- 1 file changed, 38 insertions(+), 10 deletions(-) diff --git a/src/__tests__/responses.test.ts b/src/__tests__/responses.test.ts index 1cc66bd5..46c8d54e 100644 --- a/src/__tests__/responses.test.ts +++ b/src/__tests__/responses.test.ts @@ -1911,13 +1911,41 @@ describe("reasoning.encrypted_content emission", () => { return ev?.item as { id: string; encrypted_content?: string } | undefined; } - // Pins aimock's deliberate omission of the blob on the in-progress `added` item - // even when the gate is ON (see the header note — legal, but NOT a claim about - // what upstream does). This replaces a conditional-expect helper that could - // never fire: aimock builds the `added` item with no `emitEncrypted` opt at all, - // so the guarded branch was unreachable and every call site asserted nothing. - const expectAddedOmitsBlob = (item: object): void => { - expect(item).not.toHaveProperty("encrypted_content"); + // For an OPTED-IN request, grades ONLY what is contractual about the blob on + // the in-progress `added` item: if the field is there it must be a well-formed, + // id-derived blob — absence and presence are BOTH legal. + // + // Real OpenAI populates `added` opportunistically (see the section header and + // the note on `buildReasoningOutputItem` in responses.ts) and + // `encrypted_content` is `anyOf: [string, null]` and non-required, so aimock + // omitting it there is aimock's own CHOICE, not the provider contract. The + // drift contract in this same tree already draws the line exactly here: + // `sdk-shapes.ts` anchors the blob on `done` ONLY, explicitly because "pinning + // `added` would flag an intentional, legal shape choice as drift", and + // `gradeEncryptedBlob` grades the terminal item alone. Pinning the ABSENCE here + // instead made 9 tests fail any future change that moved aimock TOWARD upstream + // parity. + // + // It deliberately does not assert presence either, so aimock's current omission + // keeps passing. Nothing real is lost by relaxing it: the property that matters + // is the blob reaching the TERMINAL item, which every call site pins directly + // via `expect(done.encrypted_content).toBe(expectedBlob(done.id))`, so putting + // the blob on `added` INSTEAD of `done` still fails. + // + // For an OPTED-OUT request, absence on `added` genuinely IS the contract (no + // blob may appear anywhere) and stays asserted at those call sites directly. + const expectAddedBlobIsContractLegal = (item: { + id: string; + encrypted_content?: string; + }): void => { + if (!("encrypted_content" in item)) return; + expect(typeof item.encrypted_content, "encrypted_content on `added` must be a string").toBe( + "string", + ); + // Present means it must be the SAME id-derived blob — a degenerate or + // mismatched value there is as broken as a wrong one on `done` + // (cf. `gradeEncryptedBlob` in the drift suite). + expect(item.encrypted_content).toBe(expectedBlob(item.id)); }; const reasoningTextFixture: Fixture = { @@ -1972,7 +2000,7 @@ describe("reasoning.encrypted_content emission", () => { ); const done = reasoningItem(events, "done")!; expect(done.encrypted_content).toBe(expectedBlob(done.id)); // pins presence + determinism - expectAddedOmitsBlob(reasoningItem(events, "added")!); + expectAddedBlobIsContractLegal(reasoningItem(events, "added")!); }); // --- HTTP streaming: every builder + gate triggers + completed.output --- @@ -2019,7 +2047,7 @@ describe("reasoning.encrypted_content emission", () => { const done = reasoningItem(events, "done")!; expect(done, "no done reasoning item").toBeDefined(); expect(done.encrypted_content).toBe(expectedBlob(done.id)); - expectAddedOmitsBlob(reasoningItem(events, "added")!); + expectAddedBlobIsContractLegal(reasoningItem(events, "added")!); }); } @@ -2200,7 +2228,7 @@ describe("reasoning.encrypted_content emission", () => { expect(added!.summary).toEqual([]); expect(done!.summary).toEqual([]); expect(done!.encrypted_content).toBe(expectedBlob(done!.id)); - expectAddedOmitsBlob(added!); + expectAddedBlobIsContractLegal(added!); }); // The whole blast-radius argument: nothing changes for a client that did not