From 2ae76dc55bd61c940a38540bf8278a783581ae00 Mon Sep 17 00:00:00 2001 From: Eason Liang <136036952+easonLiangWorldedtech@users.noreply.github.com> Date: Fri, 20 Nov 2026 02:21:36 +0800 Subject: [PATCH] fix(api): address requesty createMessage review round 2 - Replace the hand-written joinOnce double. The two-waiter abort-isolation check now runs against the real model cache in requesty.modelCache.spec.ts, where only the handler's fetchModels boundary is stubbed, and asserts that both waiters share one provider fetch and that one waiter's abort leaves the flight's signal un-aborted. - Build the Requesty models-discovery deadline signal from its constant so the merged signal is non-optional. The catch then classifies the deadline without an optional chain, which removes the surviving OptionalChaining mutant instead of claiming it is equivalent. Co-Authored-By: Claude --- .../__tests__/requesty.modelCache.spec.ts | 170 +++++ src/api/providers/__tests__/requesty.spec.ts | 663 +++++++++++++++++- .../fetchers/__tests__/modelCache.spec.ts | 23 + .../fetchers/__tests__/requesty.spec.ts | 154 +++- src/api/providers/fetchers/requesty.ts | 41 +- src/api/providers/requesty.ts | 197 ++++-- src/eslint-suppressions.json | 2 +- 7 files changed, 1178 insertions(+), 72 deletions(-) create mode 100644 src/api/providers/__tests__/requesty.modelCache.spec.ts diff --git a/src/api/providers/__tests__/requesty.modelCache.spec.ts b/src/api/providers/__tests__/requesty.modelCache.spec.ts new file mode 100644 index 0000000000..22cc583abd --- /dev/null +++ b/src/api/providers/__tests__/requesty.modelCache.spec.ts @@ -0,0 +1,170 @@ +// npx vitest run api/providers/__tests__/requesty.modelCache.spec.ts + +// The real model cache runs in this file. requesty.spec.ts replaces getModels() with a double, +// which re-implements the single-flight that this abort behaviour depends on, so a regression in +// dedupedFetch()/joinFlight() could not fail there. Here only the cache's storage edges, the SDK +// client and the handler's catalog fetch are stubbed: the join, the per-waiter abort views and the +// last-waiter release are the code under test. + +vitest.mock("fs", () => ({ + existsSync: vitest.fn().mockReturnValue(false), + readFileSync: vitest.fn().mockReturnValue("{}"), +})) + +vitest.mock("fs/promises", () => ({ + writeFile: vitest.fn().mockResolvedValue(undefined), + readFile: vitest.fn().mockResolvedValue("{}"), + mkdir: vitest.fn().mockResolvedValue(undefined), +})) + +vitest.mock("../../../core/config/ContextProxy", () => ({ + ContextProxy: { + instance: { + globalStorageUri: { + fsPath: "/mock/storage/path", + }, + }, + }, +})) + +vitest.mock("../utils/timeout-config", () => ({ + getApiRequestTimeout: vitest.fn().mockReturnValue(300_000), +})) + +const mockCreate = vitest.fn() + +vitest.mock("openai", () => ({ + default: vitest.fn().mockImplementation(function () { + return { + chat: { + completions: { + create: mockCreate, + }, + }, + } + }), +})) + +vitest.mock("delay", () => ({ + default: vitest.fn(function () { + return Promise.resolve() + }), +})) + +import type { ModelRecord } from "@roo-code/types" + +import { RequestyHandler } from "../requesty" +import { makeApiHandlerOptions, makeCreateMessageMetadata } from "../../../test-utils/api" +import { asyncStreamFrom, collectStream } from "../../../test-utils/stream" +import { withSettleGuard } from "../../../test-utils/settle-guard" +import { flushMacrotasks } from "../../../test-utils/timers" + +const CATALOG: ModelRecord = { + "coding/claude-4-sonnet": { + maxTokens: 8192, + contextWindow: 200000, + supportsImages: true, + supportsPromptCache: true, + inputPrice: 3, + outputPrice: 15, + cacheWritesPrice: 3.75, + cacheReadsPrice: 0.3, + description: "Claude 4 Sonnet", + }, +} + +describe("RequestyHandler model lookup through the real model cache", () => { + it("lets one waiter abort without cancelling the catalog fetch the other is waiting on", async () => { + const handler = new RequestyHandler( + makeApiHandlerOptions({ + requestyApiKey: "isolation-key", + requestyModelId: "coding/claude-4-sonnet", + }), + ) + + // The provider fetch boundary is the only stub. It mirrors an HTTP client: it rejects when the + // signal it was handed fires, so a flight that cancelled its own fetch would fail the waiter + // that was still waiting for it. + let resolveCatalog!: (models: ModelRecord) => void + let notifyFetch!: () => void + const fetchStarted = new Promise((resolve) => { + notifyFetch = resolve + }) + const fetchModelsSpy = vitest.spyOn(handler, "fetchModels").mockImplementation((_options, signal) => { + notifyFetch() + return new Promise((resolve, reject) => { + signal?.addEventListener("abort", () => reject(new Error("canceled")), { once: true }) + resolveCatalog = resolve + }) + }) + + try { + const controllerA = new AbortController() + const controllerB = new AbortController() + const pA = collectStream( + handler.createMessage( + "sys", + [{ role: "user", content: "a" }], + makeCreateMessageMetadata({ abortSignal: controllerA.signal }), + ), + ) + void pA.catch(() => {}) + + // A owns the flight: its provider fetch has started. + await withSettleGuard(fetchStarted) + + const pB = collectStream( + handler.createMessage( + "sys", + [{ role: "user", content: "b" }], + makeCreateMessageMetadata({ abortSignal: controllerB.signal }), + ), + ) + void pB.catch(() => {}) + await flushMacrotasks() + + // B joined the flight A started rather than opening its own: still one provider fetch. + expect(fetchModelsSpy).toHaveBeenCalledTimes(1) + + // The fetch is bound to the flight's own signal, never to either caller's. + const flightSignal = fetchModelsSpy.mock.calls[0]?.[1] + expect(flightSignal).toBeInstanceOf(AbortSignal) + expect(flightSignal).not.toBe(controllerA.signal) + expect(flightSignal).not.toBe(controllerB.signal) + + controllerA.abort() + await expect(pA).rejects.toMatchObject({ + name: "AbortError", + message: "The Requesty request was aborted", + }) + + // A detached only its own wait: the shared fetch is still running for B. + expect(flightSignal?.aborted).toBe(false) + + mockCreate.mockResolvedValue( + asyncStreamFrom([ + { + id: "coding/claude-4-sonnet", + choices: [{ delta: { content: "b-response" } }], + }, + { + id: "test-id", + choices: [{ delta: {} }], + usage: { + prompt_tokens: 1, + completion_tokens: 1, + total_tokens: 2, + }, + }, + ]), + ) + resolveCatalog(CATALOG) + + const chunksB = await withSettleGuard(pB) + expect(chunksB).toContainEqual({ type: "text", text: "b-response" }) + expect(fetchModelsSpy).toHaveBeenCalledTimes(1) + } finally { + fetchModelsSpy.mockRestore() + } + }) +}) diff --git a/src/api/providers/__tests__/requesty.spec.ts b/src/api/providers/__tests__/requesty.spec.ts index 31a245570e..209e2858c2 100644 --- a/src/api/providers/__tests__/requesty.spec.ts +++ b/src/api/providers/__tests__/requesty.spec.ts @@ -16,7 +16,7 @@ import { getModels } from "../fetchers/modelCache" import { RequestyHandler } from "../requesty" import { Package } from "../../../shared/package" import { ApiHandlerCreateMessageMetadata } from "../../index" -import { makeApiHandlerOptions } from "../../../test-utils/api" +import { makeApiHandlerOptions, makeCreateMessageMetadata } from "../../../test-utils/api" import { asyncStreamFrom, collectStream } from "../../../test-utils/stream" import { clearAllMocks } from "../../../test-utils/reset" import { withSettleGuard } from "../../../test-utils/settle-guard" @@ -265,9 +265,173 @@ describe("RequestyHandler", () => { stream_options: { include_usage: true }, temperature: 0, }), + expect.objectContaining({ signal: expect.any(AbortSignal) }), ) }) + it("forwards the settings reasoningEffort to the Requesty request", async () => { + const { getModels } = await import("../fetchers/modelCache") + vitest.mocked(getModels).mockImplementationOnce(async () => ({ + "coding/claude-4-sonnet": { + maxTokens: 8192, + contextWindow: 200000, + supportsImages: true, + supportsPromptCache: true, + supportsReasoningEffort: true, + inputPrice: 3, + outputPrice: 15, + description: "Claude 4 Sonnet", + }, + })) + mockCreate.mockResolvedValue(asyncStreamFrom([{ id: "c1", choices: [{ delta: { content: "ok" } }] }])) + const handler = new RequestyHandler( + makeApiHandlerOptions({ + requestyApiKey: "test-key", + requestyModelId: "coding/claude-4-sonnet", + reasoningEffort: "high", + }), + ) + + const chunks = await collectStream( + handler.createMessage("test system", [{ role: "user", content: "test" }]), + ) + expect(chunks).toHaveLength(1) + expect(mockCreate).toHaveBeenCalledTimes(1) + expect(mockCreate).toHaveBeenCalledWith( + expect.objectContaining({ model: "coding/claude-4-sonnet", reasoning_effort: "high" }), + expect.anything(), + ) + }) + + it("omits reasoning_effort when the settings effort is outside the model's supported set", async () => { + const { getModels } = await import("../fetchers/modelCache") + vitest.mocked(getModels).mockImplementationOnce(async () => ({ + "coding/claude-4-sonnet": { + maxTokens: 8192, + contextWindow: 200000, + supportsImages: true, + supportsPromptCache: true, + supportsReasoningEffort: ["minimal", "low", "medium", "high"], + inputPrice: 3, + outputPrice: 15, + description: "Claude 4 Sonnet", + }, + })) + mockCreate.mockResolvedValue(asyncStreamFrom([{ id: "c1", choices: [{ delta: { content: "ok" } }] }])) + const handler = new RequestyHandler( + makeApiHandlerOptions({ + requestyApiKey: "test-key", + requestyModelId: "coding/claude-4-sonnet", + reasoningEffort: "minimal", + }), + ) + + const chunks = await collectStream( + handler.createMessage("test system", [{ role: "user", content: "test" }]), + ) + expect(chunks).toHaveLength(1) + expect(mockCreate).toHaveBeenCalledTimes(1) + // "minimal" is declared by the model fixture so it survives shouldUseReasoningEffort and + // reaches the handler, where the Requesty allowed-effort filter must drop it. With the + // older ["low", "medium", "high"] fixture the shared gate rejected "minimal" first, so deleting + // that filter left this test green. + expect(mockCreate.mock.calls[0][0]).not.toHaveProperty("reasoning_effort") + }) + + it("forwards the task metadata into the requesty-specific request block", async () => { + mockCreate.mockResolvedValue(asyncStreamFrom([{ id: "c1", choices: [{ delta: { content: "ok" } }] }])) + const handler = new RequestyHandler(mockOptions) + + const chunks = await collectStream( + handler.createMessage("test system", [{ role: "user", content: "test" }], { + taskId: "task-123", + mode: "plan", + }), + ) + expect(chunks).toHaveLength(1) + expect(mockCreate).toHaveBeenCalledTimes(1) + expect(mockCreate).toHaveBeenCalledWith( + expect.objectContaining({ + requesty: { trace_id: "task-123", extra: { mode: "plan" } }, + }), + expect.anything(), + ) + }) + + it("tolerates chunks with empty choices before the first delta", async () => { + mockCreate.mockResolvedValue( + asyncStreamFrom([ + { id: "c1", choices: [] }, + { id: "c2", choices: [{ delta: { content: "after empty" } }] }, + ]), + ) + const handler = new RequestyHandler(mockOptions) + + const chunks = await collectStream( + handler.createMessage("test system", [{ role: "user", content: "test" }]), + ) + expect(chunks).toEqual([{ type: "text", text: "after empty" }]) + }) + + it("streams tool_call_partial chunks when the tool call has no function payload", async () => { + mockCreate.mockResolvedValue( + asyncStreamFrom([ + { + id: "c1", + choices: [ + { + delta: { + tool_calls: [ + { index: 2, id: "call_123" }, + { index: 0, id: "call_456" }, + ], + }, + }, + ], + }, + ]), + ) + const handler = new RequestyHandler(mockOptions) + + const chunks = await collectStream( + handler.createMessage("test system", [{ role: "user", content: "test" }]), + ) + expect(chunks).toEqual([ + { type: "tool_call_partial", index: 2, id: "call_123" }, + { type: "tool_call_partial", index: 0, id: "call_456" }, + ]) + }) + + it("emits the usage chunk once when a final chunk carries no usage", async () => { + mockCreate.mockResolvedValue( + asyncStreamFrom([ + { id: "c1", choices: [{ delta: { content: "text" } }] }, + { id: "c2", choices: [{ delta: {} }], usage: { prompt_tokens: 3, completion_tokens: 4 } }, + { id: "c3", choices: [{ delta: {} }] }, + ]), + ) + const handler = new RequestyHandler(mockOptions) + + const chunks = await collectStream( + handler.createMessage("test system", [{ role: "user", content: "test" }]), + ) + expect(chunks).toHaveLength(2) + expect(chunks[0]).toEqual({ type: "text", text: "text" }) + expect(chunks[1]).toMatchObject({ type: "usage", inputTokens: 3, outputTokens: 4 }) + }) + + it("does not emit a usage chunk when the stream reports no usage", async () => { + mockCreate.mockResolvedValue( + asyncStreamFrom([{ id: "c1", choices: [{ delta: { content: "only text" } }] }]), + ) + const handler = new RequestyHandler(mockOptions) + + const chunks = await collectStream( + handler.createMessage("test system", [{ role: "user", content: "test" }]), + ) + expect(chunks).toEqual([{ type: "text", text: "only text" }]) + }) + it("uses adaptive thinking for Claude Fable 5 when reasoning is enabled", async () => { const handler = new RequestyHandler( makeApiHandlerOptions({ @@ -298,6 +462,7 @@ describe("RequestyHandler", () => { thinking: { type: "adaptive", display: "summarized" }, temperature: undefined, }), + expect.objectContaining({ signal: expect.any(AbortSignal) }), ) }) @@ -331,6 +496,7 @@ describe("RequestyHandler", () => { thinking: { type: "adaptive", display: "summarized" }, temperature: undefined, }), + expect.objectContaining({ signal: expect.any(AbortSignal) }), ) }) @@ -364,6 +530,7 @@ describe("RequestyHandler", () => { thinking: { type: "adaptive", display: "summarized" }, temperature: undefined, }), + expect.objectContaining({ signal: expect.any(AbortSignal) }), ) }) @@ -397,6 +564,7 @@ describe("RequestyHandler", () => { thinking: { type: "adaptive", display: "summarized" }, temperature: undefined, }), + expect.objectContaining({ signal: expect.any(AbortSignal) }), ) }) @@ -535,6 +703,7 @@ describe("RequestyHandler", () => { ]), tool_choice: "auto", }), + expect.objectContaining({ signal: expect.any(AbortSignal) }), ) }) @@ -616,6 +785,498 @@ describe("RequestyHandler", () => { }) }) }) + it("rejects with AbortError when the external signal is pre-aborted", async () => { + const handler = new RequestyHandler(mockOptions) + mockCreate.mockResolvedValue(asyncStreamFrom([{ id: "1", choices: [{ delta: { content: "response" } }] }])) + + const controller = new AbortController() + controller.abort() + const metadata = makeCreateMessageMetadata({ abortSignal: controller.signal }) + + await expect( + handler.createMessage("sys", [{ role: "user", content: "hi" }], metadata).next(), + ).rejects.toMatchObject({ + name: "AbortError", + message: "The Requesty request was aborted", + }) + // The fast-fail guard must reject before model discovery starts. + const { getModels } = await import("../fetchers/modelCache") + expect(vitest.mocked(getModels)).not.toHaveBeenCalled() + }) + + it("rejects with AbortError when the external signal aborts during deferred model discovery", async () => { + const handler = new RequestyHandler(mockOptions) + const controller = new AbortController() + + // Model discovery is deferred: capture the resolver and settle it only at the end of + // the test, so the abort deterministically lands while the lookup is still pending. + // The barrier below (instead of a fixed sleep) synchronizes on the lookup starting. + let resolveModelLookup!: (models: ModelRecord) => void + const deferredModelLookup = new Promise((resolve) => { + resolveModelLookup = resolve + }) + let notifyLookupStarted!: () => void + const lookupStarted = new Promise((resolve) => { + notifyLookupStarted = resolve + }) + const { getModels } = await import("../fetchers/modelCache") + vitest.mocked(getModels).mockImplementationOnce(() => { + notifyLookupStarted() + return deferredModelLookup + }) + + const metadata = makeCreateMessageMetadata({ abortSignal: controller.signal }) + const generator = handler.createMessage("sys", [{ role: "user", content: "hi" }], metadata) + + const nextPromise = generator.next() + // Defensive: the fast-fail path may reject before the barrier below settles, + // which would otherwise surface as an unhandled rejection. + void nextPromise.catch(() => {}) + await withSettleGuard(lookupStarted) + controller.abort() + + await expect(withSettleGuard(nextPromise)).rejects.toMatchObject({ + name: "AbortError", + message: "The Requesty request was aborted", + }) + expect(mockCreate).not.toHaveBeenCalled() + + // Settle the abandoned lookup so it cannot outlive the test. + resolveModelLookup({}) + }) + + it("forwards the per-request cancellation scope to the model lookup and the completion request", async () => { + const handler = new RequestyHandler(mockOptions) + mockCreate.mockResolvedValue(asyncStreamFrom([{ id: "c1", choices: [{ delta: { content: "ok" } }] }])) + + let discoverySignal: AbortSignal | undefined + const { getModels } = await import("../fetchers/modelCache") + vitest.mocked(getModels).mockImplementationOnce((options) => { + discoverySignal = options.signal + return Promise.resolve({ + "coding/claude-4-sonnet": { + maxTokens: 8192, + contextWindow: 200000, + supportsImages: true, + supportsPromptCache: true, + inputPrice: 3, + outputPrice: 15, + description: "Claude 4 Sonnet", + }, + }) + }) + + const controller = new AbortController() + const metadata = makeCreateMessageMetadata({ abortSignal: controller.signal }) + + await collectStream(handler.createMessage("sys", [{ role: "user", content: "hi" }], metadata)) + + // The lookup gets this request's per-request scope rather than nothing: modelCache joins + // each caller through its own abort view, which detaches only this waiter and releases the + // shared fetch when the last waiter leaves (see the signal-isolation suite below). Without + // the scope an abort here would leave the catalog request running to modelCache's own bound + // even though this caller stopped waiting for it. + expect(discoverySignal).toBeInstanceOf(AbortSignal) + expect(discoverySignal).not.toBe(controller.signal) + // One scope covers both phases: the SDK completion is driven by the same signal. + const sdkOptions = mockCreate.mock.calls[0]?.[1] as { signal?: AbortSignal } | undefined + expect(sdkOptions?.signal).toBe(discoverySignal) + }) + + it("aborts the in-flight stream and rejects with AbortError when the external signal aborts", async () => { + const handler = new RequestyHandler(mockOptions) + const controller = new AbortController() + + let requestSignal: AbortSignal | undefined + mockCreate.mockImplementationOnce(async (_params: unknown, options?: { signal?: AbortSignal }) => { + requestSignal = options?.signal + // Emulate the OpenAI SDK: the first chunk arrives, then the in-flight + // response body rejects once the request signal aborts. + return (async function* () { + yield { id: "1", choices: [{ delta: { content: "first" } }] } + await new Promise((resolve) => { + expect(requestSignal).toBeDefined() + if (requestSignal!.aborted) { + resolve() + } else { + requestSignal!.addEventListener("abort", () => resolve(), { once: true }) + } + }) + const abortError = new Error("The user aborted a request") + abortError.name = "AbortError" + throw abortError + })() + }) + + const metadata = makeCreateMessageMetadata({ abortSignal: controller.signal }) + const generator = handler.createMessage("sys", [{ role: "user", content: "hi" }], metadata) + + const chunks: unknown[] = [] + const iteration = (async () => { + for await (const chunk of generator) { + chunks.push(chunk) + if (chunk.type === "text") { + // Abort while the stream is still in flight. + controller.abort() + } + } + })() + + await expect(withSettleGuard(iteration)).rejects.toMatchObject({ + name: "AbortError", + message: "The Requesty request was aborted", + }) + expect(chunks).toContainEqual({ type: "text", text: "first" }) + // The provider must forward its own per-request signal, not the caller's signal: + // the per-request controller isolates this request from the external signal. + expect(requestSignal).toBeInstanceOf(AbortSignal) + expect(requestSignal).not.toBe(controller.signal) + }) + it("does not yield text or tool calls after the signal aborts mid-chunk (pre-yield guards)", async () => { + const handler = new RequestyHandler(mockOptions) + const controller = new AbortController() + // A single chunk carrying reasoning + content + a tool call: each part's yield + // is a suspension point, so an abort can land between any two of them within + // the same chunk. + mockCreate.mockResolvedValue( + asyncStreamFrom([ + { + id: "c1", + choices: [ + { + delta: { + reasoning_content: "reasoning", + content: "content", + tool_calls: [ + { + index: 0, + id: "1", + type: "function", + function: { name: "f", arguments: "{}" }, + }, + ], + }, + }, + ], + }, + ]), + ) + + const metadata = makeCreateMessageMetadata({ abortSignal: controller.signal }) + const generator = handler.createMessage("sys", [{ role: "user", content: "hi" }], metadata) + + const first = await generator.next() + expect(first.value?.type).toBe("reasoning") + // Abort after the reasoning yield: the text and tool-call yields of the same + // chunk must not be emitted. + controller.abort() + + await expect(generator.next()).rejects.toMatchObject({ + name: "AbortError", + message: "The Requesty request was aborted", + }) + }) + it("does not yield tool calls after the signal aborts between the content and tool-call yields of one chunk", async () => { + const handler = new RequestyHandler(mockOptions) + const controller = new AbortController() + // Content + tool calls without reasoning: the content yield comes first, so + // the abort lands between it and the tool-call yield. + mockCreate.mockResolvedValue( + asyncStreamFrom([ + { + id: "c1", + choices: [ + { + delta: { + content: "content", + tool_calls: [ + { + index: 0, + id: "1", + type: "function", + function: { name: "f", arguments: "{}" }, + }, + ], + }, + }, + ], + }, + ]), + ) + + const metadata = makeCreateMessageMetadata({ abortSignal: controller.signal }) + const generator = handler.createMessage("sys", [{ role: "user", content: "hi" }], metadata) + + const first = await generator.next() + expect(first.value?.type).toBe("text") + controller.abort() + + await expect(generator.next()).rejects.toMatchObject({ + name: "AbortError", + message: "The Requesty request was aborted", + }) + }) + it("does not process buffered chunks after an abort and surfaces AbortError instead of completing the stream", async () => { + const handler = new RequestyHandler(mockOptions) + const controller = new AbortController() + let pulls = 0 + mockCreate.mockImplementationOnce(async (_params: unknown, _options?: { signal?: AbortSignal }) => { + return (async function* () { + pulls++ + yield { id: "c1", choices: [{ delta: { reasoning_content: "first" } }] } + pulls++ + // Reasoning shape: its yield is the first yield of the iteration and + // carries no pre-yield guard, so only the top-of-loop break prevents + // it from leaking. + yield { id: "c2", choices: [{ delta: { reasoning_content: "buffered" } }] } + pulls++ + yield { id: "c3", choices: [{ delta: { content: "third" } }] } + })() + }) + + const metadata = makeCreateMessageMetadata({ abortSignal: controller.signal }) + const generator = handler.createMessage("sys", [{ role: "user", content: "hi" }], metadata) + + const first = await generator.next() + expect(first.value?.type).toBe("reasoning") + controller.abort() + + await expect(generator.next()).rejects.toMatchObject({ + name: "AbortError", + message: "The Requesty request was aborted", + }) + // The for-await mechanism pulls the already-buffered chunk before the + // top-of-loop check runs; the break must ensure it is not processed and + // that no chunk beyond it is pulled. + expect(pulls).toBe(2) + }) + it("rejects with AbortError when the stream ends normally after a mid-stream abort (swallowed AbortError)", async () => { + const handler = new RequestyHandler(mockOptions) + const controller = new AbortController() + + // Simulate openai@5.23.2: the SDK stream iterator swallows the mid-stream + // AbortError and returns normally instead of throwing, so the catch in + // createMessage never runs. The per-request signal (second argument) is the + // one the SDK observes. + let requestSignal: AbortSignal | undefined + mockCreate.mockImplementationOnce(async (_params: unknown, options?: { signal?: AbortSignal }) => { + requestSignal = options?.signal + return (async function* () { + yield { id: "1", choices: [{ delta: { content: "partial" } }] } + // Wait for the abort instead of polling: the iterator ends gracefully + // (no throw) once the request signal aborts. + await new Promise((resolve) => { + expect(requestSignal).toBeDefined() + if (requestSignal!.aborted) { + resolve() + } else { + requestSignal!.addEventListener("abort", () => resolve(), { once: true }) + } + }) + })() + }) + + const metadata = makeCreateMessageMetadata({ abortSignal: controller.signal }) + const generator = handler.createMessage("sys", [{ role: "user", content: "hi" }], metadata) + + const first = await generator.next() + expect(first.value).toEqual({ type: "text", text: "partial" }) + // Abort mid-stream, after the first chunk has been yielded. + controller.abort() + + // The stream ended normally, but createMessage must still reject with AbortError. + await expect(withSettleGuard(generator.next())).rejects.toMatchObject({ + name: "AbortError", + message: "The Requesty request was aborted", + }) + }) + it("does not emit buffered chunks after a mid-stream abort (iterator keeps delivering)", async () => { + const handler = new RequestyHandler(mockOptions) + const controller = new AbortController() + + // Simulate openai@5.23.2 delivering a buffered chunk after the abort has already + // fired, then ending the iterator normally (no throw). + let requestSignal: AbortSignal | undefined + mockCreate.mockImplementationOnce(async (_params: unknown, options?: { signal?: AbortSignal }) => { + requestSignal = options?.signal + return (async function* () { + yield { id: "1", choices: [{ delta: { content: "partial" } }] } + // Wait for the abort instead of polling: the buffered chunk is delivered + // once the request signal aborts. + await new Promise((resolve) => { + expect(requestSignal).toBeDefined() + if (requestSignal!.aborted) { + resolve() + } else { + requestSignal!.addEventListener("abort", () => resolve(), { once: true }) + } + }) + yield { id: "2", choices: [{ delta: { content: "after-abort" } }] } + })() + }) + + const metadata = makeCreateMessageMetadata({ abortSignal: controller.signal }) + const generator = handler.createMessage("sys", [{ role: "user", content: "hi" }], metadata) + + const first = await generator.next() + expect(first.value).toEqual({ type: "text", text: "partial" }) + // Abort mid-stream, after the first chunk has been yielded. + controller.abort() + + // The buffered second chunk must not be emitted, and the generator must reject + // with the provider AbortError. + await expect(withSettleGuard(generator.next())).rejects.toMatchObject({ + name: "AbortError", + message: "The Requesty request was aborted", + }) + }) + it("rejects with AbortError when the external signal aborts during request creation", async () => { + const handler = new RequestyHandler(mockOptions) + const controller = new AbortController() + + // Synchronize on request startup (instead of a fixed sleep) so the abort + // deterministically lands while the request is in flight. + let notifyCreateStarted!: () => void + const createStarted = new Promise((resolve) => { + notifyCreateStarted = resolve + }) + mockCreate.mockImplementationOnce(async (_params: unknown, options?: { signal?: AbortSignal }) => { + notifyCreateStarted() + // Emulate the OpenAI SDK: the pending request rejects when the signal aborts. + await new Promise((resolve) => { + if (options?.signal?.aborted) { + resolve() + } else { + options?.signal?.addEventListener("abort", () => resolve(), { once: true }) + } + }) + const abortError = new Error("The user aborted a request") + abortError.name = "AbortError" + throw abortError + }) + + const metadata = makeCreateMessageMetadata({ abortSignal: controller.signal }) + const generator = handler.createMessage("sys", [{ role: "user", content: "hi" }], metadata) + + const nextPromise = generator.next() + // Defensive: the fast-fail path may reject before the barrier below settles, + // which would otherwise surface as an unhandled rejection. + void nextPromise.catch(() => {}) + await withSettleGuard(createStarted) + controller.abort() + + await expect(withSettleGuard(nextPromise)).rejects.toMatchObject({ + name: "AbortError", + message: "The Requesty request was aborted", + }) + }) + + it("rejects with AbortError when the SDK call never settles behind its retry backoff", async () => { + const handler = new RequestyHandler(mockOptions) + const controller = new AbortController() + + // openai@5.23.2 sleeps through the retry backoff (or a server retry-after-ms) without + // observing the signal, so the promise create() returns can still be pending a minute + // after the caller cancelled. Emulate exactly that: a create() that was handed the + // request signal and never settles. + let notifyCreateStarted!: () => void + const createStarted = new Promise((resolve) => { + notifyCreateStarted = resolve + }) + mockCreate.mockImplementationOnce(() => { + notifyCreateStarted() + return new Promise(() => {}) + }) + + const metadata = makeCreateMessageMetadata({ abortSignal: controller.signal }) + const generator = handler.createMessage("sys", [{ role: "user", content: "hi" }], metadata) + const nextPromise = generator.next() + void nextPromise.catch(() => {}) + await withSettleGuard(createStarted) + controller.abort() + + // Without the race at requesty.ts the awaited SDK promise stays pending and this guard + // rejects with its own timeout instead of the provider AbortError. + await expect(withSettleGuard(nextPromise)).rejects.toMatchObject({ + name: "AbortError", + message: "The Requesty request was aborted", + }) + }) + + it("rethrows non-abort creation errors from createMessage", async () => { + const handler = new RequestyHandler(mockOptions) + mockCreate.mockImplementationOnce(async () => { + throw new Error("boom") + }) + + const generator = handler.createMessage("sys", [{ role: "user", content: "hi" }]) + + await expect(collectStream(generator)).rejects.toThrow("boom") + }) + + it("removes the external abort listener when the stream completes", async () => { + const handler = new RequestyHandler(mockOptions) + const controller = new AbortController() + const addSpy = vi.spyOn(controller.signal, "addEventListener") + const removeSpy = vi.spyOn(controller.signal, "removeEventListener") + mockCreate.mockImplementationOnce(async () => { + return asyncStreamFrom([{ id: "1", choices: [{ delta: { content: "done" } }] }]) + }) + + const metadata = makeCreateMessageMetadata({ abortSignal: controller.signal }) + const generator = handler.createMessage("sys", [{ role: "user", content: "hi" }], metadata) + + await collectStream(generator) + + // The cleanup path must remove the exact listener that was registered, not just any + // function: removing a different reference would leave the original abort listener + // attached to the signal. First require the registration to have happened at all, + // so a missing registration cannot silently degrade to an undefined comparison. + expect(addSpy).toHaveBeenCalledTimes(1) + const registeredListener = addSpy.mock.calls[0]?.[1] as EventListener | undefined + expect(typeof registeredListener).toBe("function") + expect(removeSpy).toHaveBeenCalledWith("abort", registeredListener) + addSpy.mockRestore() + removeSpy.mockRestore() + }) + + it("removes the external abort listener when the stream fails without an abort", async () => { + const handler = new RequestyHandler(mockOptions) + const controller = new AbortController() + const addSpy = vi.spyOn(controller.signal, "addEventListener") + const removeSpy = vi.spyOn(controller.signal, "removeEventListener") + mockCreate.mockImplementationOnce(async () => { + throw new Error("boom") + }) + + const metadata = makeCreateMessageMetadata({ abortSignal: controller.signal }) + const generator = handler.createMessage("sys", [{ role: "user", content: "hi" }], metadata) + + await expect(collectStream(generator)).rejects.toThrow("boom") + + // The failure path must clean up just like the success path: the exact registered + // listener is removed even when the request rejects without any abort firing. + expect(addSpy).toHaveBeenCalledTimes(1) + const registeredListener = addSpy.mock.calls[0]?.[1] as EventListener | undefined + expect(typeof registeredListener).toBe("function") + expect(removeSpy).toHaveBeenCalledWith("abort", registeredListener) + addSpy.mockRestore() + removeSpy.mockRestore() + }) + + it("rethrows non-abort stream errors from createMessage", async () => { + const handler = new RequestyHandler(mockOptions) + mockCreate.mockImplementationOnce(async () => { + return (async function* () { + yield { id: "1", choices: [{ delta: { content: "first" } }] } + throw new Error("stream broke") + })() + }) + + const generator = handler.createMessage("sys", [{ role: "user", content: "hi" }]) + + await expect(collectStream(generator)).rejects.toThrow("stream broke") + }) }) describe("completePrompt", () => { diff --git a/src/api/providers/fetchers/__tests__/modelCache.spec.ts b/src/api/providers/fetchers/__tests__/modelCache.spec.ts index 3081b5f3eb..f11710a8d6 100644 --- a/src/api/providers/fetchers/__tests__/modelCache.spec.ts +++ b/src/api/providers/fetchers/__tests__/modelCache.spec.ts @@ -171,6 +171,29 @@ describe("getModels with new GetModelsOptions", () => { expect(result).toEqual(mockModels) }) + it("rejects the caller when its cancellation signal aborts during Requesty model discovery", async () => { + setupCancellationMocks() + // Keep the fetch pending so the caller's abort lands mid-flight: under the + // single-flight contract the caller's signal detaches this waiter (and rejects + // it with the canonical AbortError) instead of being handed to the fetcher. + const fetchModels = vi.fn().mockImplementation(() => new Promise(() => {})) + const handler = makeHandler(REQUESTY_SCOPE, fetchModels) + + const controller = new AbortController() + const promise = getModels( + { + provider: providerIdentifiers.requesty, + apiKey: DUMMY_REQUESTY_KEY, + signal: controller.signal, + }, + handler, + ) + + controller.abort() + + await expect(promise).rejects.toMatchObject({ name: "AbortError" }) + }) + it("dispatches credentialed fetchers through canonical provider identifiers", async () => { const mockModels = { "requesty/canonical-model": { diff --git a/src/api/providers/fetchers/__tests__/requesty.spec.ts b/src/api/providers/fetchers/__tests__/requesty.spec.ts index dfc291b9f3..9b051544f6 100644 --- a/src/api/providers/fetchers/__tests__/requesty.spec.ts +++ b/src/api/providers/fetchers/__tests__/requesty.spec.ts @@ -191,16 +191,75 @@ describe("getRequestyModels", () => { expect(sonnet.supportsTemperature).toBeUndefined() }) - it("passes the caller's abort signal to the catalog request", async () => { + it("threads a wall-clock merged signal and the bounded timeout into the models request", async () => { const controller = new AbortController() mockAxiosGet.mockResolvedValueOnce({ data: { data: [] } }) await getRequestyModels(undefined, undefined, { signal: controller.signal }) + // The shared axios mock accumulates calls across this file's tests, so assert on + // the call this test just made (the last one) rather than a global call count. + const calls = mockAxiosGet.mock.calls + const config = calls[calls.length - 1]?.[1] + expect(config?.signal).toBeInstanceOf(AbortSignal) + // The request signal is the caller's signal merged with the wall-clock + // deadline, not the raw caller signal: axios's timeout option resets when + // headers arrive, so only the merged signal enforces the 10_000 ms limit. + expect(config?.signal).not.toBe(controller.signal) + expect(config?.timeout).toBe(10_000) + + // The caller's abort still propagates through the merged signal. + controller.abort() + expect(config?.signal?.aborted).toBe(true) + }) + + it("applies the bounded timeout as a standalone timeout signal when no signal is provided", async () => { + mockAxiosGet.mockResolvedValueOnce({ data: { data: [] } }) + + await getRequestyModels() + + const calls = mockAxiosGet.mock.calls + const config = calls[calls.length - 1]?.[1] + // Even without a caller signal the request carries the wall-clock deadline + // as its own timeout signal. + expect(config?.signal).toBeInstanceOf(AbortSignal) + expect(config?.signal?.aborted).toBe(false) + expect(config?.timeout).toBe(10_000) + }) + + it("passes a merged request signal and the bounded timeout to the catalog request", async () => { + const controller = new AbortController() + mockAxiosGet.mockResolvedValueOnce({ data: { data: [] } }) + + await getRequestyModels(undefined, undefined, { signal: controller.signal }) + + const calls = mockAxiosGet.mock.calls + const config = calls[calls.length - 1]?.[1] expect(mockAxiosGet).toHaveBeenCalledWith("https://router.requesty.ai/v1/models", { headers: {}, - signal: controller.signal, + signal: config?.signal, + timeout: 10_000, }) + expect(config?.signal).toBeInstanceOf(AbortSignal) + }) + + it("aborts the catalog request when the caller signal aborts after the request signal was merged", async () => { + const controller = new AbortController() + let requestSignal: AbortSignal | undefined + mockAxiosGet.mockImplementation(((_url: string, config?: { signal?: AbortSignal }) => { + requestSignal = config?.signal + // Mirror the HTTP client: reject when the request signal fires. + return new Promise((_resolve, reject) => { + requestSignal?.addEventListener("abort", () => reject(new Error("canceled")), { once: true }) + }) + }) as typeof axios.get) + + const fetchPromise = getRequestyModels(undefined, undefined, { signal: controller.signal }) + expect(requestSignal).toBeInstanceOf(AbortSignal) + expect(requestSignal).not.toBe(controller.signal) + controller.abort() + + await expect(fetchPromise).rejects.toMatchObject({ name: "AbortError" }) }) it("rejects with an AbortError when the signal aborts the pending request", async () => { @@ -222,4 +281,95 @@ describe("getRequestyModels", () => { await expect(fetchPromise).rejects.toMatchObject({ name: "AbortError" }) }) + + it("returns the empty fallback and logs a dedicated timeout when discovery exceeds the deadline", async () => { + vi.useFakeTimers() + const errorSpy = vi.spyOn(console, "error").mockImplementation(() => {}) + // The merged deadline uses the native AbortSignal.timeout, which fake timers cannot + // advance. Rather than scheduling a fake sleep inside the mock (a fixed sleep can either + // flake or hide a missing await), the mock hands back a controller whose abort the test + // itself releases once it has advanced the clock past the deadline. + let reachDeadline: () => void = () => {} + const deadline = new Promise((resolve) => { + reachDeadline = resolve + }) + // Typed double: the mock replaces the AbortSignal.timeout static (optional ms argument, + // AbortSignal return) with a controller the test aborts by hand, because fake timers cannot + // advance the real one. The intermediate unknown names what is deliberately partial - the + // double ignores the ms argument the static declares and only supplies the returned signal. + const timeoutSpy = vi.spyOn(AbortSignal, "timeout").mockImplementation(((_ms: number) => { + const controller = new AbortController() + void deadline.then(() => { + controller.abort(new DOMException("The operation timed out.", "TimeoutError")) + }) + return controller.signal + }) as unknown as typeof AbortSignal.timeout) + try { + // Typed double: axios.get is an overload set (config object, url + config, url + config + + // body) that an inline mock cannot restate; this double supplies only the url/config member + // the fetcher calls, plus the signal member on that config the test subscribes to. + mockAxiosGet.mockImplementationOnce( + ((_url: string, config?: { signal?: AbortSignal }) => + new Promise((_resolve, reject) => { + // Axios rejects with the abort reason when the merged deadline fires. + config?.signal?.addEventListener("abort", () => reject(config.signal?.reason), { + once: true, + }) + })) as unknown as typeof axios.get, + ) + + const pending = getRequestyModels() + let settled = false + void pending.then(() => { + settled = true + }) + // The merged deadline has to be the documented 10 s: any shorter value also produces the + // fallback and the log, so the assertions below are what pin the value. + expect(timeoutSpy).toHaveBeenCalledWith(10_000) + await vi.advanceTimersByTimeAsync(9_999) + // Nothing may settle before the deadline fires. + expect(settled).toBe(false) + // The deadline is reached explicitly instead of by a timer the test would have to + // sleep through: the clock is advanced to just under the documented 10 s, nothing may + // settle, and then the deadline is released. + reachDeadline() + await vi.waitFor(() => expect(settled).toBe(true), { interval: 1 }) + const models = await pending + expect(settled).toBe(true) + // The fallback is intentional: callers use the cached/default model info. + expect(models).toEqual({}) + // The dedicated timeout message, not just any log: a generic fetch failure logs + // "Error fetching Requesty models: ..." and would also satisfy a loose "timed out" + // match, so both halves are asserted. + expect(errorSpy).toHaveBeenCalledWith(expect.stringContaining("timed out after 10000ms")) + expect(errorSpy).not.toHaveBeenCalledWith(expect.stringContaining("Error fetching Requesty models")) + } finally { + timeoutSpy.mockRestore() + errorSpy.mockRestore() + vi.useRealTimers() + } + }) + + it("logs the generic fetch error, not the deadline, when a plain network error rejects", async () => { + const errorSpy = vi.spyOn(console, "error").mockImplementation(() => {}) + try { + // The other half of the timeout test's discrimination: a rejection that carries no + // abort at all (axios surfacing a reset connection) must fall to the generic branch. + // Treating every rejection as deadline-driven - e.g. an always-true aborted check in + // the catch - would keep the timeout suite green and fail here. + mockAxiosGet.mockImplementationOnce(() => { + throw Object.assign(new Error("read ECONNRESET"), { code: "ECONNRESET" }) + }) + + const models = await getRequestyModels() + + // Same fallback as a deadline: callers keep the cached/default model info. + expect(models).toEqual({}) + expect(errorSpy).toHaveBeenCalledWith(expect.stringContaining("Error fetching Requesty models")) + expect(errorSpy).toHaveBeenCalledWith(expect.stringContaining("ECONNRESET")) + expect(errorSpy).not.toHaveBeenCalledWith(expect.stringContaining("timed out after 10000ms")) + } finally { + errorSpy.mockRestore() + } + }) }) diff --git a/src/api/providers/fetchers/requesty.ts b/src/api/providers/fetchers/requesty.ts index c167f6212e..fdd68f8595 100644 --- a/src/api/providers/fetchers/requesty.ts +++ b/src/api/providers/fetchers/requesty.ts @@ -5,7 +5,14 @@ import type { ModelInfo } from "@roo-code/types" import { parseApiPrice } from "../../../shared/cost" import { toRequestyServiceUrl } from "../../../shared/utils/requesty" -import { throwIfAborted } from "../utils/abort-signal" +import { mergeAbortSignals, throwIfAborted } from "../utils/abort-signal" + +// Bounded wall-clock limit for the models discovery request. Without it a hung connection +// would keep the request alive indefinitely. The 10_000 ms figure matches the two +// in-tree fetchers that bound the same call: io-intelligence.ts (axios timeout: 10_000) and +// kimi-code.ts (KIMI_CODE_MODELS_TIMEOUT_MS = 10_000). kenari, opencode-go and nanogpt do not +// bound this call at all, so they are not the precedent. +const REQUESTY_MODELS_TIMEOUT_MS = 10_000 export async function getRequestyModels( baseUrl?: string, @@ -13,6 +20,14 @@ export async function getRequestyModels( opts?: { signal?: AbortSignal }, ): Promise> { const models: Record = {} + // Wall-clock deadline merged into the request signal so the fetch is bounded end to end. + // The deadline is a module constant, so it is always usable and the merged signal is always + // present: it is built directly rather than through mergeAbortSignalAndTimeout, whose return + // type is optional only because its timeout argument is caller-supplied. A non-optional signal + // is what lets the catch below read requestSignal.aborted to tell a deadline abort from a + // plain fetch error. + const timeoutSignal = AbortSignal.timeout(REQUESTY_MODELS_TIMEOUT_MS) + const requestSignal = opts?.signal ? mergeAbortSignals(opts.signal, timeoutSignal) : timeoutSignal try { const headers: Record = {} @@ -24,7 +39,14 @@ export async function getRequestyModels( const resolvedBaseUrl = toRequestyServiceUrl(baseUrl) const modelsUrl = new URL("v1/models", resolvedBaseUrl) - const response = await axios.get(modelsUrl.toString(), { headers, signal: opts?.signal }) + // Axios's own timeout resets once response headers arrive (follow-redirects), + // so a slow body could outlive REQUESTY_MODELS_TIMEOUT_MS; requestSignal above + // carries the wall-clock deadline. + const response = await axios.get(modelsUrl.toString(), { + headers, + signal: requestSignal, + timeout: REQUESTY_MODELS_TIMEOUT_MS, + }) const rawModels = response.data.data for (const rawModel of rawModels) { @@ -84,11 +106,22 @@ export async function getRequestyModels( models[rawModel.id] = modelInfo } } catch (error) { - // Surface cancellation as a rejection: logging and returning here would + // Surface a caller cancellation as a rejection: logging and returning here would // present an aborted fetch to callers as a successful (partial) catalog. throwIfAborted(opts?.signal) - console.error(`Error fetching Requesty models: ${JSON.stringify(error, Object.getOwnPropertyNames(error), 2)}`) + if (requestSignal.aborted) { + // The wall-clock deadline (not a caller abort) fired. Callers intentionally fall back + // to the cached/default model info, so keep returning the partial map but log the + // dedicated timeout so a hung discovery is observable. + console.error( + `Requesty models discovery timed out after ${REQUESTY_MODELS_TIMEOUT_MS}ms; falling back to cached/default model info`, + ) + } else { + console.error( + `Error fetching Requesty models: ${JSON.stringify(error, Object.getOwnPropertyNames(error), 2)}`, + ) + } } return models diff --git a/src/api/providers/requesty.ts b/src/api/providers/requesty.ts index b69afb18f0..a9e8d1c447 100644 --- a/src/api/providers/requesty.ts +++ b/src/api/providers/requesty.ts @@ -37,6 +37,7 @@ import { isRequestAborted, mergeAbortSignalAndTimeout, rejectOnAbort, + resolveModelWithAbort, throwIfAborted, } from "./utils/abort-signal" import { applyRouterToolPreferences } from "./utils/router-tool-preferences" @@ -185,80 +186,148 @@ export class RequestyHandler extends BaseProvider implements SingleCompletionHan messages: Anthropic.Messages.MessageParam[], metadata?: ApiHandlerCreateMessageMetadata, ): ApiStream { - const { - id: model, - info, - maxTokens: max_tokens, - temperature, - reasoningEffort: reasoning_effort, - reasoning: thinking, - } = await this.fetchModel() - - const openAiMessages: OpenAI.Chat.ChatCompletionMessageParam[] = [ - { role: "system", content: systemPrompt }, - ...convertToOpenAiMessages(messages), - ] - - // Map extended efforts to OpenAI Chat Completions-accepted values (omit unsupported) - const allowedEffort = (["low", "medium", "high"] as const).includes(reasoning_effort as any) - ? (reasoning_effort as OpenAI.Chat.Completions.ChatCompletionCreateParamsStreaming["reasoning_effort"]) - : undefined - - const completionParams: RequestyChatCompletionParamsStreaming = { - messages: openAiMessages, - model, - max_tokens, - temperature, - ...(allowedEffort && { reasoning_effort: allowedEffort }), - ...(thinking && { thinking }), - stream: true, - stream_options: { include_usage: true }, - requesty: { trace_id: metadata?.taskId, extra: { mode: metadata?.mode } }, - tools: this.convertToolsForOpenAI(metadata?.tools), - tool_choice: metadata?.tool_choice, - } + // Bridge the external abort signal into a request-local controller using the shared + // helper (pre-aborted guard, once-only listener, retained cleanup). The request keeps a + // signal for the SDK even without an external one: a never-aborting fallback preserves the + // previous behavior of always passing a signal to the client. + const { signal: bridgedSignal, dispose: removeExternalAbortListener } = bridgeExternalAbortSignal( + metadata?.abortSignal, + ) + const requestSignal = bridgedSignal ?? new AbortController().signal - let stream try { - // With streaming params type, SDK returns an async iterable stream - stream = await this.client.chat.completions.create(completionParams) - } catch (error) { - throw handleOpenAIError(error, this.providerName) - } - let lastUsage: any = undefined - - for await (const chunk of stream) { - const delta = chunk.choices[0]?.delta - - const reasoningText = extractReasoningFromDelta(delta) - if (reasoningText) { - yield { type: "reasoning", text: reasoningText } + // Resolve the model inside this request's cancellation scope. resolveModelWithAbort owns + // the three pieces the scope needs: the pre-abort fast path (no catalog request started, + // no shared flight joined), the race that settles this waiter at once when the scope + // aborts while the lookup is pending, and the normalization of an abort-shaped lookup + // failure to the provider's AbortError. + // + // The scope is forwarded to the lookup instead of withheld. modelCache joins every caller + // through its own per-waiter abort view (joinFlight): the view detaches only this waiter + // and aborts the underlying fetch when the last waiter leaves, so one caller's abort can + // never reject the fetch out from under a concurrent caller. Forwarding it is what lets a + // cancellation here release that fetch — stopping a discovery request this abort already + // made pointless rather than letting it run to modelCache's own 15 s bound. + const { + id: model, + info, + maxTokens: max_tokens, + temperature, + reasoningEffort: reasoning_effort, + reasoning: thinking, + } = await resolveModelWithAbort(() => this.fetchModel(requestSignal), requestSignal, this.providerName) + + const openAiMessages: OpenAI.Chat.ChatCompletionMessageParam[] = [ + { role: "system", content: systemPrompt }, + ...convertToOpenAiMessages(messages), + ] + + // Map extended efforts to OpenAI Chat Completions-accepted values (omit unsupported) + const allowedEffort = (["low", "medium", "high"] as const).find((effort) => effort === reasoning_effort) + + const completionParams: RequestyChatCompletionParamsStreaming = { + messages: openAiMessages, + model, + max_tokens, + temperature, + ...(allowedEffort && { reasoning_effort: allowedEffort }), + ...(thinking && { thinking }), + stream: true, + stream_options: { include_usage: true }, + requesty: { trace_id: metadata?.taskId, extra: { mode: metadata?.mode } }, + tools: this.convertToolsForOpenAI(metadata?.tools), + tool_choice: metadata?.tool_choice, } - if (delta?.content) { - yield { type: "text", text: delta.content } + let stream + try { + // With streaming params type, SDK returns an async iterable stream + const pendingStream = this.client.chat.completions.create(completionParams, { signal: requestSignal }) + // Race the creation against the cancellation scope, as completePrompt does below: + // openai@5.23.2 does not observe the signal inside `retryRequest`, which sleeps for the + // backoff (or a server-provided `retry-after-ms`, up to about a minute) before issuing + // the next attempt, so awaiting the SDK call alone can hold createMessage open long + // after the caller cancelled. The race settles at once, and a rejection the SDK wins + // first is still normalized by the catch below. + stream = await rejectOnAbort(pendingStream, requestSignal, this.providerName) + } catch (error) { + // Aborted requests are user-initiated: surface them as AbortError instead of + // a completion error. + if (requestSignal.aborted) { + throw createAbortError(this.providerName) + } + throw handleOpenAIError(error, this.providerName) } + try { + let lastUsage: RequestyUsage | undefined = undefined + + for await (const chunk of stream) { + // The iterator can keep delivering buffered chunks after the abort has already + // fired (openai@5.23.2 swallows the mid-stream AbortError), so re-check the + // signal before processing each chunk. A yield is a suspension point, so an + // abort can land between two yields of the same chunk: the guarded yields + // below re-check before emitting (the first yield of a chunk is covered by + // this check alone — no suspension point separates them). + if (requestSignal.aborted) { + break + } + + const delta = chunk.choices[0]?.delta + + // Yield reasoning chunks before content chunks so consumers see them in model order. + // No pre-yield guard: this is the first yield of the iteration — the + // top-of-loop check runs without a suspension point before it. + const reasoningText = extractReasoningFromDelta(delta) + if (reasoningText) { + yield { type: "reasoning", text: reasoningText } + } - // Handle native tool calls - if (delta && "tool_calls" in delta && Array.isArray(delta.tool_calls)) { - for (const toolCall of delta.tool_calls) { - yield { - type: "tool_call_partial", - index: toolCall.index, - id: toolCall.id, - name: toolCall.function?.name, - arguments: toolCall.function?.arguments, + if (delta?.content) { + // Re-check before emitting: the consumer may have aborted while + // processing a previously yielded part of this chunk. + throwIfAborted(requestSignal) + yield { type: "text", text: delta.content } + } + + // Handle native tool calls + if (delta && "tool_calls" in delta && Array.isArray(delta.tool_calls)) { + for (const toolCall of delta.tool_calls) { + throwIfAborted(requestSignal) + yield { + type: "tool_call_partial", + index: toolCall.index, + id: toolCall.id, + name: toolCall.function?.name, + arguments: toolCall.function?.arguments, + } + } + } + + if (chunk.usage) { + lastUsage = chunk.usage } } - } - if (chunk.usage) { - lastUsage = chunk.usage - } - } + // openai@5.23.2's stream iterator swallows a mid-stream AbortError and returns + // normally instead of throwing, so the catch below would never run: without this + // check, createMessage completes silently after yielding partial output. + if (requestSignal.aborted) { + throw createAbortError(this.providerName) + } - if (lastUsage) { - yield this.processUsageMetrics(lastUsage, info) + if (lastUsage) { + yield this.processUsageMetrics(lastUsage, info) + } + } catch (error) { + // Normalize abort-driven stream failures (SDK abort or timeout errors) to a + // DOM-standard AbortError so callers can detect the aborted request. + if (requestSignal.aborted) { + throw createAbortError(this.providerName) + } + throw error + } + } finally { + removeExternalAbortListener() } } diff --git a/src/eslint-suppressions.json b/src/eslint-suppressions.json index d5b845997a..4a1ecdcc80 100644 --- a/src/eslint-suppressions.json +++ b/src/eslint-suppressions.json @@ -321,7 +321,7 @@ }, "api/providers/requesty.ts": { "@typescript-eslint/no-explicit-any": { - "count": 3 + "count": 1 } }, "api/providers/unbound.ts": {