From a7e9233917abc458fedf9057f76c98606bcbd140 Mon Sep 17 00:00:00 2001 From: Erick Date: Fri, 12 Jun 2026 12:39:13 -0700 Subject: [PATCH 1/2] fix(adapter): fire session.close in daemon thread to unblock event loop MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit on_session_end() called bridge.request("session.close") inline with a 30 s blocking urlopen(). gateway/run.py calls this synchronously from _handle_reset_command (an async fn), so the blocking I/O ran on the asyncio event loop thread, preventing Discord heartbeats from firing and causing forced reconnection after 10–30 s. The session.close response is unused and errors are already suppressed, so the call is semantically fire-and-forget. Moving it to a daemon thread is the correct fix: the event loop is never blocked, the request still goes out, and a 5 s timeout keeps it bounded if the bridge is dead. Reproducer: Violet 2026-06-12 09:54 (agent.log lines 3629–3791). Spec: ~/specs/memos-bridge-blocking-shutdown-spec.md --- .../adapters/hermes/memos_provider/__init__.py | 16 ++++++++++++++-- 1 file changed, 14 insertions(+), 2 deletions(-) diff --git a/apps/memos-local-plugin/adapters/hermes/memos_provider/__init__.py b/apps/memos-local-plugin/adapters/hermes/memos_provider/__init__.py index fd9d0d827..45c7c1f6c 100644 --- a/apps/memos-local-plugin/adapters/hermes/memos_provider/__init__.py +++ b/apps/memos-local-plugin/adapters/hermes/memos_provider/__init__.py @@ -1506,8 +1506,20 @@ def on_session_end(self, messages: list[dict[str, Any]]) -> None: # type: ignor # the core will pause or finalize the open episode according to # topic-boundary rules so interrupted Hermes sessions can resume # into the same task later. - with contextlib.suppress(Exception): - self._bridge.request("session.close", {"sessionId": self._session_id}) + # + # Fire session.close in a daemon thread — the response is unused, so + # this is semantically fire-and-forget. Calling urlopen() inline blocks + # the asyncio event loop (gateway/run.py calls us synchronously from + # _handle_reset_command) and causes Discord heartbeat timeouts when the + # bridge is unresponsive. 5 s timeout keeps it bounded. + _bridge = self._bridge + _sid = self._session_id + + def _close() -> None: + with contextlib.suppress(Exception): + _bridge.request("session.close", {"sessionId": _sid}, timeout=5.0) + + threading.Thread(target=_close, daemon=True).start() def shutdown(self) -> None: # type: ignore[override] self._bridge_keepalive_stop.set() From f22444fca38c082de298f75ab88c0a5843695816 Mon Sep 17 00:00:00 2001 From: Memtensor-AI Date: Tue, 30 Jun 2026 19:56:30 +0800 Subject: [PATCH 2/2] Fix #1897: bug(memos-local-plugin): Hermes background model status checks can trigger paid (#1899) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit * Fix #1897: fix(memos-local-plugin): add LLM circuit breaker for terminal provider errors Issue #1897 reported ~12,900 paid LLM requests in 24 h on Hermes against a DeepSeek key with insufficient balance. The local `system_model_status` row count (12,900) closely tracked the provider-side `request_count` (11,344) for the same billing window. The naming is misleading: `system_model_status` is not a health probe; it is the audit row written once per LLM call (ok / fallback / error) inside `core/llm/client.ts`. With no circuit breaker, every pipeline subscriber (capture / session-relation / reward / L2 / L3 / skill / retrieval LLM filter / world-model) kept firing on every turn / closed episode / induction, generating one paid request each. Add a per-`LlmClient` circuit breaker: - Trips on terminal errors: HTTP 401/402/403 or messages containing `insufficient balance` / `invalid api key` / `unauthorized` / `account suspended` / `billing`. - Open: short-circuits subsequent calls inside the facade without contacting the provider. Throws `MemosError(LLM_UNAVAILABLE)` with `details.circuitOpen=true` so existing catch blocks still work. - Half-open after cool-down (default 5 min, configurable, min 30 s): next call probes the provider; success closes the breaker, terminal failure re-opens it for another cool-down. - Host fallback rescues a call without tripping the breaker — fallback exists precisely to keep going when the primary is down. - Coalesces `system_model_status="circuit_open"` audit rows to at most one per ~25 s while the breaker stays open, so we don't replace paid spam with audit-row spam. - Exposes `circuitOpen` / `circuitOpenUntil` / `circuitOpenedReason` via `LlmClientStats` for the Overview viewer card. - Enabled by default; legacy behaviour available via `circuitBreaker.enabled = false`. Tests: 9 new vitest cases under `tests/unit/llm/client.test.ts` covering trip on 402, trip on "insufficient balance" message, no trip on generic transient, coalescing, half-open close on success, host-fallback rescues without trip, disabled mode, stats fields, and re-open on terminal probe failure. All 59 LLM and 28 pipeline tests pass; `tsc --noEmit` clean. Out of scope (tracked separately): 429 `Retry-After` handling (issue #1620), per-tool rate limits, daily budget caps. * Fix LLM breaker with host fallback --------- Co-authored-by: autodev-bot Co-authored-by: Jiang <33757498+hijzy@users.noreply.github.com> Co-authored-by: GU TIANCHUN <96930846+TianchunGu@users.noreply.github.com> Co-authored-by: Dubberman <48425266+whipser030@users.noreply.github.com> --- apps/memos-local-plugin/core/llm/client.ts | 286 +++++++++++++++--- apps/memos-local-plugin/core/llm/index.ts | 2 + apps/memos-local-plugin/core/llm/types.ts | 40 ++- .../tests/unit/llm/client.test.ts | 238 +++++++++++++++ 4 files changed, 528 insertions(+), 38 deletions(-) diff --git a/apps/memos-local-plugin/core/llm/client.ts b/apps/memos-local-plugin/core/llm/client.ts index 7749a572f..d969d714d 100644 --- a/apps/memos-local-plugin/core/llm/client.ts +++ b/apps/memos-local-plugin/core/llm/client.ts @@ -71,6 +71,124 @@ export function createLlmClientWithProvider( let lastFallbackAt: number | null = null; let lastError: { at: number; message: string } | null = null; + // ─── Circuit breaker state (issue #1897) ───────────────────────────────── + // Per-client breaker that trips on terminal provider errors (401/402/403, + // "insufficient balance", "invalid api key", "unauthorized", "account + // suspended", "billing"). Short-circuits subsequent calls inside the + // facade so the broken provider is not contacted again until cool-down + // elapses. Half-open: the next call after `circuitOpenUntil` probes the + // provider; success closes the breaker, terminal failure re-opens it. + const breakerCfg = config.circuitBreaker ?? {}; + const breakerEnabled = breakerCfg.enabled !== false; + const breakerCooldownMs = Math.max(30_000, breakerCfg.cooldownMs ?? 300_000); + const breakerIsTerminal = breakerCfg.isTerminal ?? defaultIsTerminal; + const breakerNow = breakerCfg.now ?? Date.now; + let circuitOpenUntil: number | null = null; + let circuitOpenedReason: string | null = null; + let lastCircuitOpenStatusAt: number | null = null; + + function breakerIsOpen(): boolean { + if (!breakerEnabled) return false; + if (circuitOpenUntil === null) return false; + if (breakerNow() >= circuitOpenUntil) { + // Cool-down elapsed → transition to half-open. We do NOT clear + // `circuitOpenUntil` yet so the very first probe attempt that + // races with the cool-down boundary doesn't fall through to "no + // breaker" twice. The next call's success/failure handler resets + // or re-opens the breaker explicitly. + return false; + } + return true; + } + + function breakerTrip(err: unknown): void { + if (!breakerEnabled) return; + circuitOpenUntil = breakerNow() + breakerCooldownMs; + circuitOpenedReason = summarizeErrMessage(err); + // Reset the coalescer so the first suppressed call after a fresh + // trip always emits a `circuit_open` row. + lastCircuitOpenStatusAt = null; + facadeLog.warn("circuit_breaker.trip", { + provider: provider.name, + model: config.model, + until: circuitOpenUntil, + reason: circuitOpenedReason, + }); + } + + function breakerRecordSuccess(): void { + if (!breakerEnabled) return; + if (circuitOpenUntil !== null) { + facadeLog.info("circuit_breaker.close", { + provider: provider.name, + model: config.model, + }); + } + circuitOpenUntil = null; + circuitOpenedReason = null; + lastCircuitOpenStatusAt = null; + } + + /** + * Emit a coalesced `circuit_open` audit row. At most one row per + * `cooldownMs/12` window per client — bounds audit-row spam while + * still surfacing the suppressed-call event in the Logs viewer. + * The first suppressed call after a fresh trip always emits. + */ + function maybeEmitCircuitOpenStatus(opts: LlmCallOptions | undefined, op: string): void { + if (!config.onStatus) return; + const at = breakerNow(); + const coalesceWindow = Math.max(5_000, Math.floor(breakerCooldownMs / 12)); + if ( + lastCircuitOpenStatusAt !== null && + at - lastCircuitOpenStatusAt < coalesceWindow + ) { + return; + } + lastCircuitOpenStatusAt = at; + try { + config.onStatus({ + status: "circuit_open", + provider: provider.name, + model: config.model, + message: circuitOpenedReason ?? "(unknown reason)", + at, + durationMs: 0, + op, + episodeId: opts?.episodeId, + phase: opts?.phase, + }); + } catch { + /* status sink errors are non-fatal */ + } + } + + function throwBreakerOpen(): never { + throw makeBreakerOpenError(); + } + + function makeBreakerOpenError(): MemosError { + const until = circuitOpenUntil ?? breakerNow(); + return new MemosError( + ERROR_CODES.LLM_UNAVAILABLE, + `circuit_open: ${circuitOpenedReason ?? "terminal provider error"}`, + { + circuitOpen: true, + until, + provider: provider.name, + model: config.model, + }, + ); + } + + function canUseHostFallback(): boolean { + return ( + config.fallbackToHost === true && + provider.name !== "host" && + getHostLlmBridge() !== null + ); + } + /** * Mark a successful primary-provider call. We **do not** clear * `lastError` / `lastFallbackAt` here — the viewer picks the most @@ -151,6 +269,21 @@ export function createLlmClientWithProvider( opts: LlmCallOptions | undefined, op: string, ): Promise<{ completion: LlmCompletion }> { + // ── Circuit breaker short-circuit ── + // When the breaker is open we never reach the primary provider, so + // no request is generated against the broken paid API. We still + // emit (coalesced) `circuit_open` status rows so the Logs viewer / + // Overview can surface that suppression is happening. + if (breakerIsOpen()) { + maybeEmitCircuitOpenStatus(opts, op); + if (canUseHostFallback()) { + return callHostFallback(makeBreakerOpenError(), messages, input, opts, op, { + keepBreakerOpen: true, + notifyError: false, + }); + } + throwBreakerOpen(); + } requests++; const startedAt = Date.now(); try { @@ -166,6 +299,7 @@ export function createLlmClientWithProvider( }; record(completion, op, messages); const okAt = markOk(); + breakerRecordSuccess(); notifyStatus({ status: "ok", provider: provider.name, @@ -179,45 +313,13 @@ export function createLlmClientWithProvider( return { completion }; } catch (err) { if (shouldFallback(err, config, provider.name)) { - const hostProv = new HostLlmProvider(); + const primaryTerminal = breakerIsTerminal(err); + if (primaryTerminal) breakerTrip(err); try { - const res = await hostProv.complete(messages, input, makeCtx(opts, asProviderLog(rootLogger.child({ channel: "llm.host" })))); - hostFallbacks++; - facadeLog.warn("host.fallback", { - from: provider.name, - op, - reason: summarizeErr(err), + return await callHostFallback(err, messages, input, opts, op, { + keepBreakerOpen: primaryTerminal, + notifyError: true, }); - const completion: LlmCompletion = { - text: res.text, - provider: provider.name, - model: config.model, - finishReason: res.finishReason, - usage: res.usage, - servedBy: "host_fallback", - durationMs: res.durationMs, - }; - record(completion, op, messages); - // The primary provider is still broken even though the host - // bridge saved this call. Tag the slot yellow (`lastFallbackAt`) - // and surface the upstream error to the user via the - // system_error log so they can see *why* fallback engaged. - const fallbackAt = markFallback(err); - notifyOnError(err); - notifyStatus({ - status: "fallback", - provider: provider.name, - model: config.model, - message: summarizeErrMessage(err), - code: err instanceof MemosError ? err.code : undefined, - at: fallbackAt, - durationMs: completion.durationMs, - fallbackProvider: "host", - op, - episodeId: opts?.episodeId, - phase: opts?.phase, - }); - return { completion }; } catch (hostErr) { failures++; const failAt = markFail(hostErr); @@ -225,6 +327,10 @@ export function createLlmClientWithProvider( primary: summarizeErr(err), host: summarizeErr(hostErr), }); + // Primary AND host bridge both failed. Trip on a terminal + // primary error (the one the operator typically needs to fix + // — host bridge failures are usually transient stdio issues). + if (breakerIsTerminal(err)) breakerTrip(err); notifyOnError(hostErr); notifyStatus({ status: "error", @@ -249,6 +355,7 @@ export function createLlmClientWithProvider( } failures++; const failAt = markFail(err); + if (breakerIsTerminal(err)) breakerTrip(err); notifyOnError(err); notifyStatus({ status: "error", @@ -415,6 +522,12 @@ export function createLlmClientWithProvider( const call = buildCallInput(opts, opts?.jsonMode === true); const ctx = makeCtx(opts, asProviderLog(providerLog)); + // Short-circuit stream calls when the breaker is open. We do not + // count a suppressed call against `requests` (no network hit). + if (breakerIsOpen()) { + maybeEmitCircuitOpenStatus(opts, opts?.op ?? "stream"); + throwBreakerOpen(); + } requests++; const start = Date.now(); let acc = ""; @@ -448,6 +561,7 @@ export function createLlmClientWithProvider( if (usage?.promptTokens) totalPromptTokens += usage.promptTokens; if (usage?.completionTokens) totalCompletionTokens += usage.completionTokens; const okAt = markOk(); + breakerRecordSuccess(); notifyStatus({ status: "ok", provider: provider.name, @@ -461,6 +575,7 @@ export function createLlmClientWithProvider( } catch (err) { failures++; const failAt = markFail(err); + if (breakerIsTerminal(err)) breakerTrip(err); facadeLog.error("stream.failed", { err: summarizeErr(err) }); notifyOnError(err); notifyStatus({ @@ -479,6 +594,59 @@ export function createLlmClientWithProvider( } } + async function callHostFallback( + primaryErr: unknown, + messages: LlmMessage[], + input: ProviderCallInput, + opts: LlmCallOptions | undefined, + op: string, + behavior: { keepBreakerOpen: boolean; notifyError: boolean }, + ): Promise<{ completion: LlmCompletion }> { + const hostProv = new HostLlmProvider(); + const res = await hostProv.complete( + messages, + input, + makeCtx(opts, asProviderLog(rootLogger.child({ channel: "llm.host" }))), + ); + hostFallbacks++; + facadeLog.warn("host.fallback", { + from: provider.name, + op, + reason: summarizeErr(primaryErr), + }); + const completion: LlmCompletion = { + text: res.text, + provider: provider.name, + model: config.model, + finishReason: res.finishReason, + usage: res.usage, + servedBy: "host_fallback", + durationMs: res.durationMs, + }; + record(completion, op, messages); + // The primary provider is still broken even though the host bridge + // saved this call. Keep the breaker open for terminal primary + // errors so later calls can go straight to host fallback without + // touching the paid provider again. + const fallbackAt = markFallback(primaryErr); + if (!behavior.keepBreakerOpen) breakerRecordSuccess(); + if (behavior.notifyError) notifyOnError(primaryErr); + notifyStatus({ + status: "fallback", + provider: provider.name, + model: config.model, + message: summarizeErrMessage(primaryErr), + code: primaryErr instanceof MemosError ? primaryErr.code : undefined, + at: fallbackAt, + durationMs: completion.durationMs, + fallbackProvider: "host", + op, + episodeId: opts?.episodeId, + phase: opts?.phase, + }); + return { completion }; + } + const client: LlmClient = { provider: provider.name, model: config.model, @@ -497,6 +665,9 @@ export function createLlmClientWithProvider( lastOkAt, lastFallbackAt, lastError, + circuitOpen: breakerIsOpen(), + circuitOpenUntil, + circuitOpenedReason, }; }, resetStats(): void { @@ -509,6 +680,9 @@ export function createLlmClientWithProvider( lastOkAt = null; lastFallbackAt = null; lastError = null; + circuitOpenUntil = null; + circuitOpenedReason = null; + lastCircuitOpenStatusAt = null; }, async close(): Promise { await provider.close?.(); @@ -522,6 +696,10 @@ export function createLlmClientWithProvider( timeoutMs: config.timeoutMs, maxRetries: config.maxRetries, fallbackToHost: config.fallbackToHost, + circuitBreaker: { + enabled: breakerEnabled, + cooldownMs: breakerCooldownMs, + }, }); return client; @@ -562,6 +740,40 @@ function shouldFallback(err: unknown, config: LlmConfig, providerName: LlmProvid ); } +/** + * Default circuit-breaker classifier for terminal provider errors. + * + * A "terminal" error is one that will keep failing until the operator + * intervenes (top up balance, fix API key, fix model name). Retrying + * such an error just burns paid quota and pollutes the audit log, so + * the breaker opens and short-circuits further calls for the cool- + * down window. Issue #1897 reports the symptom — ~12,900 paid LLM + * requests in 24 h against a key with insufficient balance. + * + * Detection sources, in order: + * 1. `MemosError(LLM_UNAVAILABLE)` with `details.status` ∈ 401/402/403 + * — set by `core/llm/fetcher.ts::httpPostJson` for non-ok HTTP + * responses. + * 2. Well-known lowercase phrases in the error message (so providers + * that return 400 for "Insufficient Balance" — looking at you, + * DeepSeek — are still recognized). + */ +function defaultIsTerminal(err: unknown): boolean { + if (!(err instanceof MemosError)) return false; + if (err.code !== ERROR_CODES.LLM_UNAVAILABLE) return false; + const status = Number((err.details as { status?: unknown } | undefined)?.status); + if (status === 401 || status === 402 || status === 403) return true; + const msg = (err.message ?? "").toLowerCase(); + return ( + msg.includes("insufficient balance") || + msg.includes("invalid api key") || + msg.includes("invalid_api_key") || + msg.includes("unauthorized") || + msg.includes("account suspended") || + msg.includes("billing") + ); +} + // ─── Logger adapter ────────────────────────────────────────────────────────── function asProviderLog(log: Logger): LlmProviderLogger { diff --git a/apps/memos-local-plugin/core/llm/index.ts b/apps/memos-local-plugin/core/llm/index.ts index 847c965f0..a295cd4a4 100644 --- a/apps/memos-local-plugin/core/llm/index.ts +++ b/apps/memos-local-plugin/core/llm/index.ts @@ -28,6 +28,7 @@ export { LocalOnlyLlmProvider } from "./providers/local-only.js"; export * from "./prompts/index.js"; export type { LlmCallOptions, + LlmCircuitBreakerConfig, LlmCompleteJsonOptions, LlmCompletion, LlmClient, @@ -40,6 +41,7 @@ export type { LlmProviderLogger, LlmProviderName, LlmRole, + LlmStatusDetail, LlmStreamChunk, LlmUsage, ProviderCallInput, diff --git a/apps/memos-local-plugin/core/llm/types.ts b/apps/memos-local-plugin/core/llm/types.ts index ddfe80c1e..4a835caee 100644 --- a/apps/memos-local-plugin/core/llm/types.ts +++ b/apps/memos-local-plugin/core/llm/types.ts @@ -47,6 +47,33 @@ export interface LlmConfig { * daemon can display status produced by a separate stdio bridge. */ onStatus?: (detail: LlmStatusDetail) => void; + /** + * Optional circuit breaker config. The breaker trips on terminal + * provider errors (HTTP 401/402/403, or well-known phrases like + * "insufficient balance" / "invalid api key" / "unauthorized" / + * "account suspended" / "billing") and short-circuits subsequent + * calls for a cool-down window. Defaults to enabled. See + * `apps/memos-local-plugin/openspec/changes/.../design.md` + * (issue #1897) for the full state machine. + */ + circuitBreaker?: LlmCircuitBreakerConfig; +} + +export interface LlmCircuitBreakerConfig { + /** Default true. Set false to restore legacy (no-breaker) behavior. */ + enabled?: boolean; + /** + * Cool-down window before the breaker enters half-open. Default + * 300_000 ms (5 minutes); minimum clamped to 30_000 ms. + */ + cooldownMs?: number; + /** + * Override the default classifier. Returns true if the error should + * trip the breaker (terminal / non-recoverable). + */ + isTerminal?: (err: unknown) => boolean; + /** Injected clock for tests. Default `Date.now`. */ + now?: () => number; } export interface LlmErrorDetail { @@ -67,7 +94,7 @@ export interface LlmErrorDetail { } export interface LlmStatusDetail { - status: "ok" | "fallback" | "error"; + status: "ok" | "fallback" | "error" | "circuit_open"; provider: LlmProviderName | string; model: string; message?: string; @@ -260,6 +287,17 @@ export interface LlmClientStats extends LastCallStatus { retries: number; totalPromptTokens: number; totalCompletionTokens: number; + /** + * True while the per-client circuit breaker is open (and any + * cooldown timer has not yet elapsed). When true, further calls are + * short-circuited inside the facade and throw immediately without + * touching the provider. See issue #1897. + */ + circuitOpen: boolean; + /** Epoch ms at which the open breaker becomes eligible for half-open probe. */ + circuitOpenUntil: number | null; + /** Free-text reason from the error that opened the breaker. */ + circuitOpenedReason: string | null; } export interface LlmClient { diff --git a/apps/memos-local-plugin/tests/unit/llm/client.test.ts b/apps/memos-local-plugin/tests/unit/llm/client.test.ts index cd5cf6106..7e904a2c6 100644 --- a/apps/memos-local-plugin/tests/unit/llm/client.test.ts +++ b/apps/memos-local-plugin/tests/unit/llm/client.test.ts @@ -14,6 +14,7 @@ import type { LlmProvider, LlmProviderCtx, LlmProviderName, + LlmStatusDetail, LlmStreamChunk, ProviderCallInput, ProviderCompletion, @@ -277,4 +278,241 @@ describe("llm/client", () => { const client = createLlmClientWithProvider(cfg(), fake); await expect(client.complete([] as LlmMessage[])).rejects.toBeInstanceOf(MemosError); }); + + // ─── Circuit breaker (issue #1897) ────────────────────────────────────── + describe("circuit breaker", () => { + function statusSink(): { rows: LlmStatusDetail[]; push: (d: LlmStatusDetail) => void } { + const rows: LlmStatusDetail[] = []; + return { rows, push: (d) => rows.push(d) }; + } + + it("trips on terminal 402 and short-circuits subsequent calls", async () => { + const sink = statusSink(); + let now = 1_000_000; + const tick = () => now; + const provider = new ThrowingProvider( + new MemosError(ERROR_CODES.LLM_UNAVAILABLE, "HTTP 402 from openai_compatible", { + provider: "openai_compatible", + status: 402, + }), + ); + const client = createLlmClientWithProvider( + cfg({ + onStatus: sink.push, + circuitBreaker: { enabled: true, cooldownMs: 300_000, now: tick }, + }), + provider, + ); + // First call: real provider hit, fails terminally → breaker trips. + await expect(client.complete("first")).rejects.toBeInstanceOf(MemosError); + expect(provider.calls).toBe(1); + // Second call: should be short-circuited; provider must NOT be invoked. + now += 100; + await expect(client.complete("second")).rejects.toMatchObject({ + code: ERROR_CODES.LLM_UNAVAILABLE, + details: { circuitOpen: true }, + }); + expect(provider.calls).toBe(1); + // Stats expose circuit state. + const stats = client.stats(); + expect(stats.circuitOpen).toBe(true); + expect(stats.circuitOpenUntil).toBe(1_000_000 + 300_000); + expect(stats.circuitOpenedReason).toMatch(/402/); + // Audit rows: at least one `error` and one `circuit_open`. + const statuses = sink.rows.map((r) => r.status); + expect(statuses).toContain("error"); + expect(statuses).toContain("circuit_open"); + }); + + it("trips on 'insufficient balance' message regardless of HTTP status", async () => { + const sink = statusSink(); + const provider = new ThrowingProvider( + new MemosError( + ERROR_CODES.LLM_UNAVAILABLE, + "HTTP 400 from openai_compatible: Insufficient Balance", + { provider: "openai_compatible", status: 400 }, + ), + ); + const client = createLlmClientWithProvider( + cfg({ onStatus: sink.push, circuitBreaker: { enabled: true } }), + provider, + ); + await expect(client.complete("x")).rejects.toBeInstanceOf(MemosError); + await expect(client.complete("y")).rejects.toMatchObject({ + details: { circuitOpen: true }, + }); + expect(provider.calls).toBe(1); + }); + + it("does NOT trip on generic LLM_UNAVAILABLE without terminal markers", async () => { + const sink = statusSink(); + const provider = new ThrowingProvider( + new MemosError(ERROR_CODES.LLM_UNAVAILABLE, "transient network blip"), + ); + const client = createLlmClientWithProvider( + cfg({ onStatus: sink.push, circuitBreaker: { enabled: true } }), + provider, + ); + // Two consecutive failures with non-terminal classification → both + // calls reach the provider, breaker stays closed. + await expect(client.complete("x")).rejects.toBeInstanceOf(MemosError); + await expect(client.complete("y")).rejects.toBeInstanceOf(MemosError); + expect(provider.calls).toBe(2); + expect(client.stats().circuitOpen).toBe(false); + }); + + it("coalesces circuit_open status rows within cooldown", async () => { + const sink = statusSink(); + let now = 1_000_000; + const tick = () => now; + const provider = new ThrowingProvider( + new MemosError(ERROR_CODES.LLM_UNAVAILABLE, "401", { status: 401 }), + ); + const client = createLlmClientWithProvider( + cfg({ + onStatus: sink.push, + circuitBreaker: { enabled: true, cooldownMs: 300_000, now: tick }, + }), + provider, + ); + await expect(client.complete("trip")).rejects.toBeTruthy(); + // 20 suppressed calls within 1 second → at most a small number of + // `circuit_open` rows (we expect 1, but tolerate up to 2 in case the + // coalescer counts the very first short-circuit as a separate row). + for (let i = 0; i < 20; i++) { + now += 50; + await expect(client.complete(`spam-${i}`)).rejects.toBeTruthy(); + } + const openRows = sink.rows.filter((r) => r.status === "circuit_open"); + expect(openRows.length).toBeGreaterThanOrEqual(1); + expect(openRows.length).toBeLessThanOrEqual(2); + // Provider was only touched once (the very first call that tripped). + expect(provider.calls).toBe(1); + }); + + it("half-open probes the provider after cooldown and closes on success", async () => { + const sink = statusSink(); + let now = 1_000_000; + const tick = () => now; + let attempt = 0; + const provider: LlmProvider = { + name: "openai_compatible", + async complete() { + attempt++; + if (attempt === 1) { + throw new MemosError(ERROR_CODES.LLM_UNAVAILABLE, "401", { status: 401 }); + } + return { text: "ok", durationMs: 1 }; + }, + }; + const client = createLlmClientWithProvider( + cfg({ + onStatus: sink.push, + circuitBreaker: { enabled: true, cooldownMs: 60_000, now: tick }, + }), + provider, + ); + await expect(client.complete("trip")).rejects.toBeTruthy(); + expect(client.stats().circuitOpen).toBe(true); + // Suppressed call before cooldown elapses. + now += 30_000; + await expect(client.complete("suppressed")).rejects.toMatchObject({ + details: { circuitOpen: true }, + }); + expect(attempt).toBe(1); + // After cooldown, the next call probes the provider. + now += 31_000; // total 61_000 since trip + const r = await client.complete("probe"); + expect(r.text).toBe("ok"); + expect(attempt).toBe(2); + // Breaker closes on success. + expect(client.stats().circuitOpen).toBe(false); + }); + + it("trips on terminal primary error even when host fallback rescues the call", async () => { + const sink = statusSink(); + const provider = new ThrowingProvider( + new MemosError(ERROR_CODES.LLM_UNAVAILABLE, "402", { status: 402 }), + ); + let hostCalls = 0; + registerHostLlmBridge({ + id: "test.host", + async complete() { + hostCalls++; + return { text: "rescued", model: "host-m", durationMs: 1 }; + }, + }); + const client = createLlmClientWithProvider( + cfg({ + fallbackToHost: true, + onStatus: sink.push, + circuitBreaker: { enabled: true }, + }), + provider, + ); + const r = await client.complete("call-1"); + expect(r.servedBy).toBe("host_fallback"); + // The terminal primary error still opens the breaker even though + // host fallback rescued the user-visible call. + expect(client.stats().circuitOpen).toBe(true); + const r2 = await client.complete("call-2"); + expect(r2.servedBy).toBe("host_fallback"); + // The second call goes directly to host fallback and never touches + // the broken paid provider again. + expect(provider.calls).toBe(1); + expect(hostCalls).toBe(2); + expect(sink.rows.map((row) => row.status)).toContain("circuit_open"); + }); + + it("disabled when circuitBreaker.enabled=false (legacy behavior)", async () => { + const provider = new ThrowingProvider( + new MemosError(ERROR_CODES.LLM_UNAVAILABLE, "402", { status: 402 }), + ); + const client = createLlmClientWithProvider( + cfg({ circuitBreaker: { enabled: false } }), + provider, + ); + await expect(client.complete("a")).rejects.toBeTruthy(); + await expect(client.complete("b")).rejects.toBeTruthy(); + await expect(client.complete("c")).rejects.toBeTruthy(); + // All three calls reached the provider. + expect(provider.calls).toBe(3); + expect(client.stats().circuitOpen).toBe(false); + }); + + it("LlmClientStats exposes circuit fields when closed", async () => { + const fake = new FakeProvider("openai_compatible", () => ({ text: "ok", durationMs: 1 })); + const client = createLlmClientWithProvider(cfg(), fake); + await client.complete("x"); + const s = client.stats(); + expect(s.circuitOpen).toBe(false); + expect(s.circuitOpenUntil).toBeNull(); + expect(s.circuitOpenedReason).toBeNull(); + }); + + it("re-opens the breaker if the half-open probe fails terminally again", async () => { + const sink = statusSink(); + let now = 1_000_000; + const tick = () => now; + const provider = new ThrowingProvider( + new MemosError(ERROR_CODES.LLM_UNAVAILABLE, "402", { status: 402 }), + ); + const client = createLlmClientWithProvider( + cfg({ + onStatus: sink.push, + circuitBreaker: { enabled: true, cooldownMs: 60_000, now: tick }, + }), + provider, + ); + await expect(client.complete("trip")).rejects.toBeTruthy(); + expect(client.stats().circuitOpen).toBe(true); + now += 61_000; + // Half-open probe still fails terminally → breaker re-opens. + await expect(client.complete("probe")).rejects.toBeTruthy(); + expect(client.stats().circuitOpen).toBe(true); + expect(client.stats().circuitOpenUntil).toBe(now + 60_000); + // Provider was touched twice total (initial trip + probe). + expect(provider.calls).toBe(2); + }); + }); });