diff --git a/.env.example b/.env.example index e7462b3..9670156 100644 --- a/.env.example +++ b/.env.example @@ -2,6 +2,8 @@ # 项目当前不自动读取 .env；不要提交包含真实密钥的 .env。 MINICODE_MODEL=deepseek-v4-pro +# Optional ordered fallbacks on the same OpenAI-compatible endpoint. +MINICODE_FALLBACK_MODELS= OPENAI_BASE_URL=https://api.deepseek.com MINICODE_MODEL_TIMEOUT=120 MINICODE_MAX_STEPS=12 diff --git a/README.md b/README.md index 797a4b2..b9e1f2e 100644 --- a/README.md +++ b/README.md @@ -4,7 +4,7 @@ MiniCode Rebuild 是一个从零、分阶段实现的本地终端 AI Coding Agen ## 当前状态 -阶段 0“仓库初始化与工程基线”至阶段 10“可观测性、质量与发布准备”已经完成；阶段 11 已实现“长期记忆与检索”，阶段 12 已实现独立高级能力“成本控制”。 +阶段 0“仓库初始化与工程基线”至阶段 10“可观测性、质量与发布准备”已经完成；阶段 11“长期记忆与检索”和阶段 12“成本控制”已经合并，阶段 13“多模型路由与降级”已完成开发和跨平台验证，等待独立 PR 合并。 目前已经具备： @@ -40,11 +40,13 @@ MiniCode Rebuild 是一个从零、分阶段实现的本地终端 AI Coding Agen - 将记忆结果标记为不可信历史数据，模型写入和删除仍经过权限边界； - 为会话设置可选 token 预算，并在请求前估算输入和工具协议成本； - 按剩余额度限制单次模型输出，预算不足时不调用 Provider； +- 按配置顺序尝试主模型与后备模型，只对明确的瞬时故障执行降级； +- 将路由失败和后备模型选择作为脱敏终端事件展示； - 离线检查 Python、运行配置、Provider 配置、会话存储、Skills 与记忆存储 readiness； - 使用 Ruff、Mypy、分支覆盖率、构建、安装和跨平台 CI 作为发布质量门禁； - 执行自动化测试。 -阶段 0 至阶段 10 的基础路线、阶段 11“长期记忆与检索”和阶段 12“成本控制”已经完成。其余高级能力仍必须单独选择、设计、测试和提交。 +阶段 0 至阶段 10 的基础路线、阶段 11“长期记忆与检索”和阶段 12“成本控制”已经完成并合并。阶段 13“多模型路由与降级”的独立 Draft PR 已通过跨平台 CI，合并后才视为正式完成；其余高级能力仍必须单独选择、设计、测试和提交。 ## 长期记忆与检索 @@ -76,6 +78,22 @@ minicode-rebuild --interactive --resume latest --token-budget 50000 该能力用于阻止失控的多步调用，不等同于精确账单上限。输入预算使用跨 Provider 启发式估算；服务端实际计费、缓存 token、推理 token 和价格规则由 Provider 决定。当前请求的真实用量只能在响应返回后得知，因此可能小幅越过估算值，但后续请求会使用更新后的服务端统计重新检查。若 Provider 不返回 usage，累计会话用量也无法精确增长；需要硬货币限额时仍应在 Provider 账户侧设置配额。 +## 多模型路由与降级 + +真实模型模式可以在同一个 OpenAI-compatible 端点上配置一个主模型和最多四个有序后备模型： + +```powershell +$env:MINICODE_MODEL="primary-model" +$env:MINICODE_FALLBACK_MODELS="fallback-fast,fallback-stable" +minicode-rebuild "分析并修复当前错误" +``` + +每次 Agent 模型步骤从主模型开始。只有连接失败、超时、HTTP 408/409/425/429 或 5xx 时，运行时才按顺序尝试下一个模型；HTTP 400/401/403 等永久错误、无法解析的响应、适配器编程错误和 `KeyboardInterrupt` 不会触发降级。每个候选最多调用一次，候选名称必须唯一，主模型不能在后备列表中重复。模型名称最多 256 个字符，并拒绝换行、ANSI 转义等不可打印控制字符。 + +降级过程只输出模型名称、错误类型和是否继续，不输出 Provider 响应正文、请求内容、API Key 或工具参数。成功的后备响应继续经过原有 Agent Loop、工具权限和 token 成本控制；同一个 `ModelRequest.max_output_tokens` 会传递给所有候选。 + +当前阶段只支持同一 API 基址与凭据下的多个模型 ID，不实现跨 Provider 凭据池、负载均衡、健康探测、自动能力评分或并行竞速。`OPENAI_BASE_URL` 不允许嵌入用户名或密码，避免 Readiness 输出泄露 URL 凭据。失败请求可能已经被 Provider 计费，但没有可靠 usage 时本地无法补记；需要硬成本保障时仍应结合 Provider 账户配额。 + ## 可观测性与 Readiness 每次 CLI 会话默认把生命周期元数据追加到工作区 `.minicode-rebuild/events.jsonl`。日志只包含时间、事件名、session ID、工具名、成功状态、错误代码和停止原因；不保存用户提示、工具参数、工具输出或 API Key。该目录已从 Git 和模型通用文件工具中隔离。 @@ -156,6 +174,7 @@ python -m minicode_rebuild --help | 环境变量 | 默认值 | 说明 | |---|---|---| | `MINICODE_MODEL` | `deepseek-v4-pro` | 模型名称 | +| `MINICODE_FALLBACK_MODELS` | 无 | 同一端点上的逗号分隔有序后备模型，最多四个 | | `OPENAI_BASE_URL` | `https://api.deepseek.com` | API 基址或完整 `/chat/completions` 地址 | | `MINICODE_MODEL_TIMEOUT` | `120` | 请求超时秒数，必须是正整数 | | `MINICODE_MAX_STEPS` | `12` | 每轮最大模型调用步数，必须是正整数 | diff --git a/docs/REBUILD_LOG.md b/docs/REBUILD_LOG.md index bc6ea8e..2c4234d 100644 --- a/docs/REBUILD_LOG.md +++ b/docs/REBUILD_LOG.md @@ -8,12 +8,12 @@ | 项目 | 内容 | |---|---| -| 当前阶段 | 阶段 12：成本控制（已完成） | -| 最近完成 | 阶段 12：Token 成本控制 | -| 当前分支 | `codex/phase-12-cost-control` | -| 最新阶段实现提交 | `fa243dc feat(phase-12): add token cost controls` | -| 测试状态 | 阶段相关回归 `107 passed`；全量回归 `298 passed, 3 skipped`；分支覆盖率 `85.58%` | -| 下一步 | 审核并由用户合并 Draft PR #6；其他高级能力继续保持独立阶段 | +| 当前阶段 | 阶段 13：多模型路由与降级（跨平台验证完成，等待合并） | +| 最近完成 | 阶段 13：开发与跨平台验证 | +| 当前分支 | `codex/phase-13-model-routing` | +| 最新阶段实现提交 | `38cd0eb feat(phase-13): add model routing and fallback` | +| 测试状态 | 本地 `339 passed, 3 skipped`、覆盖率 `86.01%`；GitHub 四组 CI 全部通过 | +| 下一步 | 用户审核并合并 Draft PR #7；合并前不开始下一高级能力 | ## 总体架构 @@ -42,6 +42,8 @@ flowchart LR | 9 | Skills、Hooks 与扩展机制 | 已完成 | 按需技能和生命周期扩展点 | `6201245` | | 10 | 可观测性、质量与发布准备 | 已完成 | 日志、质量门禁、安装与发布验证 | `edf1569` | | 11 | 长期记忆与检索 | 已完成 | 工作区本地记忆、按需检索和权限控制 | `d06ad40` | +| 12 | 成本控制 | 已完成 | 会话 token 预算、输出上限和预算门禁 | `fa243dc` | +| 13 | 多模型路由与降级 | 待合并 | 有序候选、瞬时故障降级和脱敏路由事件 | `38cd0eb` | ## 阶段 0：仓库初始化与工程基线 @@ -1871,3 +1873,65 @@ request.max_output_tokens = min(configured_output_limit, available_output) ### 8. 跨平台验证记录 阶段 12 分支推送后创建 Draft PR #6。GitHub Actions 运行 `32097364115` 的 Ubuntu 3.11、Ubuntu 3.13、Windows 3.11、Windows 3.13 四组任务全部通过；PR 以 `master` 为基线并保持 Draft，不自动合并主分支。 + +## 阶段 13：多模型路由与降级 + +### 1. 开发前计划 + +- 只实现高级能力清单中的“多模型路由与降级”，不同时引入 MCP、多 Agent、Worktree 编排、TUI 或跨 Provider 账户管理。 +- 允许在现有 OpenAI-compatible 端点和凭据下配置一个主模型与最多四个有序后备模型；未配置后备模型时保持单适配器行为。 +- 每个候选对同一规范化 `ModelRequest` 最多调用一次，保持工具声明、上下文压缩结果和成本控制输出上限不变。 +- 只对网络传输错误、明确超时/限流状态和 5xx 服务端错误降级；认证、请求格式、畸形响应与程序错误立即暴露。 +- 路由日志只包含模型名称、错误类型、尝试序号与是否继续，不包含异常正文、提示、工具参数或密钥。 +- 为配置顺序与上限、HTTP 错误分类、主模型成功、瞬时失败恢复、永久失败停止、全部候选失败、观察器隔离和 CLI 装配补齐测试。 + +### 2. 威胁模型与非目标 + +- 防止永久错误造成重复请求：400/401/403、畸形 JSON、非法工具调用和普通适配器异常不会切换模型。 +- 防止无限重试：候选总数最多五个，每个模型每个步骤只尝试一次，没有循环、退避线程或后台重放。 +- 防止错误信息泄露：全部候选失败时聚合结果只保留模型名与异常类型，不拼接可能包含 Provider 细节的异常正文。 +- 防止观察器破坏模型恢复：终端路由观察器异常被隔离，不改变路由结果。 +- `KeyboardInterrupt`、`SystemExit` 等控制流异常继续传播，不被误判为可降级故障。 +- 本阶段不实现跨端点 Key 池、并行竞速、随机负载均衡、自动基准测试、模型能力推断或后台健康检查。 + +### 3. 配置与错误分类 + +`MINICODE_MODEL` 仍是主模型；`MINICODE_FALLBACK_MODELS` 使用逗号分隔的优先级顺序。配置加载时去除两侧空白，拒绝空项、重复项、主模型重复和超过四个后备模型。模型名称最多 256 个字符，不允许换行、ANSI 转义等不可打印控制字符，避免候选名称污染 Readiness 和终端路由事件。Readiness 只做离线结构检查并展示最终候选顺序，不向任何模型发送请求；`OPENAI_BASE_URL` 拒绝嵌入用户名或密码，避免展示端点时泄露 URL 凭据。 + +OpenAI-compatible 非 2xx 响应现在归一化为带状态码的 `ModelHTTPError`。HTTP 408、409、425、429 和全部 5xx 视为可能瞬时失败；其他 HTTP 状态、`ModelResponseError` 和未知异常不具备降级资格。网络层的 `ModelTransportError` 可以降级，但异常正文不会进入路由汇总。 + +### 4. 路由执行与现有边界 + +`RoutingModelAdapter` 实现相同的 `ModelAdapter` 协议，因此 Agent Loop 不需要知道当前由哪个模型回答。主模型成功时不会访问后备模型；瞬时失败时按声明顺序串行尝试。后备成功返回普通 `ModelResponse`，其 usage、工具调用和停止原因继续由现有 Agent、会话与成本控制处理。 + +路由器不会修改请求。阶段 12 计算出的 `max_output_tokens`、阶段 7 压缩后的消息和工具 JSON Schema 会原样交给每个候选。预算门禁发生在路由器外部，因此预算不足时不会调用主模型或任何后备模型。 + +CLI 只在实际发生故障或选择后备模型时输出 `[model-route]` 行。该输出不包含 Provider message；全部瞬时候选失败时，Agent 将脱敏的 `ModelRoutingError` 作为既有 `model_error` 停止原因展示。 + +### 5. 验收与安全回归 + +- 阶段相关回归：`101 passed`。 +- 全量回归：`339 passed, 3 skipped`。 +- 分支覆盖率：`86.01%`，达到 `85%` 门槛。 +- Ruff：`All checks passed!`。 +- Mypy：`Success: no issues found in 30 source files`。 +- 测试覆盖主模型短路、传输/限流/5xx 降级、永久错误不降级、请求不变、全部候选失败脱敏、观察器隔离、控制流传播、CLI 真实装配、候选名称终端安全与 Readiness URL 凭据拒绝。 + +当前 Windows 环境的 `3 skipped` 仍是缺少目录符号链接权限的安全测试；Ubuntu CI 将执行这些真实路径。该环境限制与模型路由无关。 + +### 6. 限制与后续边界 + +所有候选当前共享同一个 `OPENAI_BASE_URL`、API Key、超时和请求协议，只改变模型 ID。某些端点可能不支持所有候选模型，模型不存在通常返回 4xx，并按永久配置错误停止，而不会继续尝试掩盖配置问题。 + +瞬时失败请求可能在服务端已消耗资源，但若 Provider 没有返回 usage，本地无法准确计入阶段 12 的会话预算。路由是可用性保护而非免费重试保证；需要严格账单上限时必须同时配置本地输出/会话预算和 Provider 账户配额。 + +### 7. Git 计划 + +- 基线：阶段 12 的 PR #6 已合并至 `master`，合并提交为 `74460e6`。 +- 分支：`codex/phase-13-model-routing`。 +- 实现提交：`38cd0eb feat(phase-13): add model routing and fallback`，作者邮箱已核对为 GitHub 关联地址 `3226077178@qq.com`。 +- 文档使用独立提交收口；分支推送后创建以 `master` 为基线的 Draft PR，不自动合并主分支。 + +### 8. 跨平台验证记录 + +阶段 13 分支推送后创建 Draft PR #7。GitHub Actions 运行 `32332645806` 的 Ubuntu 3.11、Ubuntu 3.13、Windows 3.11、Windows 3.13 四组任务全部通过；PR 以 `master` 为基线、状态为 `MERGEABLE`，保持 Draft 并等待用户审核合并。 diff --git a/src/minicode_rebuild/cli.py b/src/minicode_rebuild/cli.py index 6796f8d..3d6bd0b 100644 --- a/src/minicode_rebuild/cli.py +++ b/src/minicode_rebuild/cli.py @@ -26,8 +26,9 @@ from minicode_rebuild.core import ModelAdapter, ModelResponse, ToolCall from minicode_rebuild.hooks import HookManager from minicode_rebuild.memory import MemoryStoreError -from minicode_rebuild.models import MockModel +from minicode_rebuild.models import MockModel, ModelRoute, RoutingModelAdapter from minicode_rebuild.models.openai_compatible import OpenAICompatibleAdapter +from minicode_rebuild.models.routing import RoutingEvent from minicode_rebuild.observability import ( EventLog, ObservabilityError, @@ -201,11 +202,37 @@ def _select_model( demo: bool, environment: Mapping[str, str], injected: ModelAdapter | None, + output: TextIO, ) -> ModelAdapter: if demo: return injected or _demo_model() + if injected is not None: + return injected model_settings = ModelSettings.from_env(environment) - return injected or OpenAICompatibleAdapter(model_settings) + routes = tuple( + ModelRoute( + name=model_name, + adapter=OpenAICompatibleAdapter( + replace(model_settings, model=model_name, fallback_models=()) + ), + ) + for model_name in model_settings.model_candidates + ) + if len(routes) == 1: + return routes[0].adapter + + def observe_route(event: RoutingEvent) -> None: + if event.status == "failed": + action = "trying next route" if event.retrying else "stopping" + output.write( + f"[model-route] {event.route} failed " + f"({event.error_type}); {action}\n" + ) + elif event.status == "selected": + output.write(f"[model-route] selected fallback {event.route}\n") + output.flush() + + return RoutingModelAdapter(routes, observer=observe_route) def _print_result( @@ -531,6 +558,7 @@ def main( demo=args.demo, environment=env, injected=model, + output=output, ) if args.interactive: permission_prompt = make_permission_prompt(input_stream, output) diff --git a/src/minicode_rebuild/config.py b/src/minicode_rebuild/config.py index 5e9017a..dda314c 100644 --- a/src/minicode_rebuild/config.py +++ b/src/minicode_rebuild/config.py @@ -14,6 +14,8 @@ DEFAULT_OPENAI_BASE_URL = "https://api.deepseek.com" DEFAULT_MODEL_TIMEOUT_SECONDS = 120 DEFAULT_MAX_STEPS = 12 +MAX_FALLBACK_MODELS = 4 +MAX_MODEL_NAME_LENGTH = 256 DEFAULT_SYSTEM_PROMPT = ( "You are a careful local coding assistant. Inspect the workspace with tools " "before making claims, and ask for permission before mutations." @@ -24,6 +26,21 @@ class ModelConfigurationError(ValueError): """Raised when model settings are missing or invalid.""" +def _model_name(value: object, variable: str) -> str: + if not isinstance(value, str): + raise ModelConfigurationError(f"{variable} must be text") + normalized = value.strip() + if not normalized: + raise ModelConfigurationError(f"{variable} must not be empty") + if len(normalized) > MAX_MODEL_NAME_LENGTH: + raise ModelConfigurationError( + f"{variable} must not exceed {MAX_MODEL_NAME_LENGTH} characters" + ) + if not normalized.isprintable(): + raise ModelConfigurationError(f"{variable} must not contain control characters") + return normalized + + @dataclass(frozen=True, slots=True) class ModelSettings: """Validated settings for an OpenAI-compatible model endpoint.""" @@ -32,14 +49,18 @@ class ModelSettings: base_url: str api_key: str = field(repr=False) timeout_seconds: int = DEFAULT_MODEL_TIMEOUT_SECONDS + fallback_models: tuple[str, ...] = () def __post_init__(self) -> None: - model = self.model.strip() + model = _model_name(self.model, "MINICODE_MODEL") base_url = self.base_url.rstrip("/") api_key = self.api_key.strip() + if isinstance(self.fallback_models, str): + raise ModelConfigurationError( + "fallback_models must be a tuple of model names" + ) + fallback_models = tuple(self.fallback_models) - if not model: - raise ModelConfigurationError("MINICODE_MODEL must not be empty") if not api_key: raise ModelConfigurationError( "Set OPENAI_API_KEY or DEEPSEEK_API_KEY before using the real model adapter" @@ -47,6 +68,10 @@ def __post_init__(self) -> None: parsed_url = urlsplit(base_url) if parsed_url.scheme not in {"http", "https"} or not parsed_url.netloc: raise ModelConfigurationError("OPENAI_BASE_URL must be an http(s) URL") + if parsed_url.username is not None or parsed_url.password is not None: + raise ModelConfigurationError( + "OPENAI_BASE_URL must not contain embedded credentials" + ) if parsed_url.query or parsed_url.fragment: raise ModelConfigurationError( "OPENAI_BASE_URL must not contain a query string or fragment" @@ -55,10 +80,26 @@ def __post_init__(self) -> None: raise ModelConfigurationError( "MINICODE_MODEL_TIMEOUT must be greater than zero" ) + if len(fallback_models) > MAX_FALLBACK_MODELS: + raise ModelConfigurationError( + f"MINICODE_FALLBACK_MODELS may contain at most {MAX_FALLBACK_MODELS} models" + ) + normalized_fallbacks: list[str] = [] + for fallback in fallback_models: + normalized_fallbacks.append( + _model_name(fallback, "MINICODE_FALLBACK_MODELS") + ) + if model in normalized_fallbacks or len(set(normalized_fallbacks)) != len( + normalized_fallbacks + ): + raise ModelConfigurationError( + "MINICODE_FALLBACK_MODELS must be unique and exclude MINICODE_MODEL" + ) object.__setattr__(self, "model", model) object.__setattr__(self, "base_url", base_url) object.__setattr__(self, "api_key", api_key) + object.__setattr__(self, "fallback_models", tuple(normalized_fallbacks)) @classmethod def from_env( @@ -84,8 +125,27 @@ def from_env( api_key=env.get("OPENAI_API_KEY") or env.get("DEEPSEEK_API_KEY", ""), timeout_seconds=timeout_seconds, + fallback_models=cls._fallback_models(env), ) + @staticmethod + def _fallback_models(environment: Mapping[str, str]) -> tuple[str, ...]: + raw = environment.get("MINICODE_FALLBACK_MODELS", "").strip() + if not raw: + return () + values = tuple(item.strip() for item in raw.split(",")) + if any(not item for item in values): + raise ModelConfigurationError( + "MINICODE_FALLBACK_MODELS must be a comma-separated list without empty entries" + ) + return values + + @property + def model_candidates(self) -> tuple[str, ...]: + """Return primary then fallback model ids in deterministic order.""" + + return (self.model, *self.fallback_models) + @property def chat_completions_url(self) -> str: """Return the full Chat Completions endpoint.""" diff --git a/src/minicode_rebuild/models/__init__.py b/src/minicode_rebuild/models/__init__.py index 89b0faf..06915b6 100644 --- a/src/minicode_rebuild/models/__init__.py +++ b/src/minicode_rebuild/models/__init__.py @@ -1,5 +1,6 @@ """Model adapter implementations.""" from minicode_rebuild.models.mock import MockModel +from minicode_rebuild.models.routing import ModelRoute, RoutingModelAdapter -__all__ = ["MockModel"] +__all__ = ["MockModel", "ModelRoute", "RoutingModelAdapter"] diff --git a/src/minicode_rebuild/models/errors.py b/src/minicode_rebuild/models/errors.py index 5c078bf..5da08b2 100644 --- a/src/minicode_rebuild/models/errors.py +++ b/src/minicode_rebuild/models/errors.py @@ -11,3 +11,27 @@ class ModelTransportError(ModelError): class ModelResponseError(ModelError): """Raised when a provider response cannot be normalized safely.""" + + +class ModelHTTPError(ModelResponseError): + """HTTP response failure with retryability derived only from its status.""" + + def __init__(self, status: int, message: str) -> None: + self.status = status + self.provider_message = message + super().__init__( + f"OpenAI-compatible API returned {status}: {message}" + ) + + @property + def retryable(self) -> bool: + return self.status in {408, 409, 425, 429} or 500 <= self.status <= 599 + + +class ModelRoutingError(ModelError): + """Raised after every eligible model route failed transiently.""" + + def __init__(self, attempts: tuple[str, ...]) -> None: + self.attempts = attempts + summary = ", ".join(attempts) + super().__init__(f"All model routes failed ({summary})") diff --git a/src/minicode_rebuild/models/openai_compatible.py b/src/minicode_rebuild/models/openai_compatible.py index 9b7be6a..4f988ed 100644 --- a/src/minicode_rebuild/models/openai_compatible.py +++ b/src/minicode_rebuild/models/openai_compatible.py @@ -20,6 +20,7 @@ ToolCall, ) from minicode_rebuild.models.errors import ( + ModelHTTPError, ModelResponseError, ModelTransportError, ) @@ -310,7 +311,5 @@ def complete(self, request: ModelRequest) -> ModelResponse: ) if response.status < 200 or response.status >= 300: message = _error_message(response.body, response.status) - raise ModelResponseError( - f"OpenAI-compatible API returned {response.status}: {message}" - ) + raise ModelHTTPError(response.status, message) return _parse_response(_decode_json(response.body)) diff --git a/src/minicode_rebuild/models/routing.py b/src/minicode_rebuild/models/routing.py new file mode 100644 index 0000000..1033366 --- /dev/null +++ b/src/minicode_rebuild/models/routing.py @@ -0,0 +1,140 @@ +"""Ordered model routing with conservative transient-error failover.""" + +from __future__ import annotations + +from collections.abc import Callable, Iterable +from dataclasses import dataclass + +from minicode_rebuild.core import ModelAdapter, ModelRequest, ModelResponse +from minicode_rebuild.models.errors import ( + ModelHTTPError, + ModelRoutingError, + ModelTransportError, +) + +MAX_MODEL_ROUTES = 5 +MAX_ROUTE_NAME_LENGTH = 256 + + +@dataclass(frozen=True, slots=True) +class ModelRoute: + """One named adapter candidate in priority order.""" + + name: str + adapter: ModelAdapter + + def __post_init__(self) -> None: + if not isinstance(self.name, str): + raise TypeError("model route name must be text") + name = self.name.strip() + if not name: + raise ValueError("model route name must not be empty") + if len(name) > MAX_ROUTE_NAME_LENGTH: + raise ValueError( + f"model route name must not exceed {MAX_ROUTE_NAME_LENGTH} characters" + ) + if not name.isprintable(): + raise ValueError("model route name must not contain control characters") + if not isinstance(self.adapter, ModelAdapter): + raise TypeError("model route adapter must implement ModelAdapter") + object.__setattr__(self, "name", name) + + +@dataclass(frozen=True, slots=True) +class RoutingEvent: + """Redacted routing evidence suitable for terminal display.""" + + route: str + attempt: int + status: str + error_type: str | None = None + retrying: bool = False + + +RoutingObserver = Callable[[RoutingEvent], None] + + +def _retryable(error: Exception) -> bool: + if isinstance(error, ModelTransportError): + return True + return isinstance(error, ModelHTTPError) and error.retryable + + +class RoutingModelAdapter: + """Try ordered models once each, only for known transient failures.""" + + def __init__( + self, + routes: Iterable[ModelRoute], + *, + observer: RoutingObserver | None = None, + ) -> None: + normalized = tuple(routes) + if not normalized: + raise ValueError("at least one model route is required") + if len(normalized) > MAX_MODEL_ROUTES: + raise ValueError(f"at most {MAX_MODEL_ROUTES} model routes are allowed") + if any(not isinstance(route, ModelRoute) for route in normalized): + raise TypeError("routes must contain only ModelRoute instances") + names = tuple(route.name for route in normalized) + if len(set(names)) != len(names): + raise ValueError("model route names must be unique") + self._routes = normalized + self._observer = observer + + @property + def routes(self) -> tuple[ModelRoute, ...]: + return self._routes + + def _notify(self, event: RoutingEvent) -> None: + if self._observer is None: + return + try: + self._observer(event) + except Exception: + return + + def complete(self, request: ModelRequest) -> ModelResponse: + """Return the first success without retrying permanent or malformed errors.""" + + failures: list[str] = [] + for index, route in enumerate(self._routes, start=1): + try: + response = route.adapter.complete(request) + except Exception as exc: + retrying = _retryable(exc) and index < len(self._routes) + self._notify( + RoutingEvent( + route=route.name, + attempt=index, + status="failed", + error_type=type(exc).__name__, + retrying=retrying, + ) + ) + if not _retryable(exc): + raise + failures.append(f"{route.name}:{type(exc).__name__}") + if not retrying: + raise ModelRoutingError(tuple(failures)) from exc + continue + if index > 1: + self._notify( + RoutingEvent( + route=route.name, + attempt=index, + status="selected", + ) + ) + return response + raise AssertionError("validated routes unexpectedly produced no result") + + +__all__ = [ + "MAX_MODEL_ROUTES", + "MAX_ROUTE_NAME_LENGTH", + "ModelRoute", + "RoutingEvent", + "RoutingModelAdapter", + "RoutingObserver", +] diff --git a/src/minicode_rebuild/readiness.py b/src/minicode_rebuild/readiness.py index 614af08..2e42819 100644 --- a/src/minicode_rebuild/readiness.py +++ b/src/minicode_rebuild/readiness.py @@ -61,7 +61,8 @@ def check_readiness( ReadinessCheck( "provider-config", True, - f"model={settings.model} endpoint={settings.chat_completions_url}", + f"models={','.join(settings.model_candidates)} " + f"endpoint={settings.chat_completions_url}", ) ) except ModelConfigurationError as exc: diff --git a/tests/test_cli.py b/tests/test_cli.py index e5f525c..3cd2e71 100644 --- a/tests/test_cli.py +++ b/tests/test_cli.py @@ -9,10 +9,12 @@ import pytest +import minicode_rebuild.cli as cli_module from minicode_rebuild import __version__ from minicode_rebuild.cli import main from minicode_rebuild.core import ModelResponse, TokenUsage, ToolCall from minicode_rebuild.models import MockModel +from minicode_rebuild.models.errors import ModelTransportError def run_module(*arguments: str) -> subprocess.CompletedProcess[str]: @@ -150,6 +152,46 @@ def test_headless_real_model_uses_environment_and_reports_stats( assert model.requests[0].messages[-1].content == "answer once" +def test_real_model_routes_to_configured_fallback( + tmp_path: Path, + monkeypatch: pytest.MonkeyPatch, +) -> None: + created: list[str] = [] + + class FakeAdapter: + def __init__(self, settings) -> None: + self.model = settings.model + created.append(self.model) + + def complete(self, _request): + if self.model == "primary-model": + raise ModelTransportError("offline") + return ModelResponse(content="Recovered", usage=TokenUsage(3, 2)) + + monkeypatch.setattr(cli_module, "OpenAICompatibleAdapter", FakeAdapter) + stdout = StringIO() + + code = main( + ["--cwd", str(tmp_path), "answer once"], + environment={ + "OPENAI_API_KEY": "secret", + "MINICODE_MODEL": "primary-model", + "MINICODE_FALLBACK_MODELS": "fallback-model", + }, + stdin=StringIO(), + stdout=stdout, + stderr=StringIO(), + ) + + assert code == 0 + assert created == ["primary-model", "fallback-model"] + assert "primary-model failed (ModelTransportError); trying next route" in ( + stdout.getvalue() + ) + assert "selected fallback fallback-model" in stdout.getvalue() + assert "Recovered" in stdout.getvalue() + + def test_headless_token_budget_blocks_before_provider_call(tmp_path: Path) -> None: stdout = StringIO() model = MockModel([ModelResponse(content="must not run")]) diff --git a/tests/test_config.py b/tests/test_config.py index a9df6e8..2f6b574 100644 --- a/tests/test_config.py +++ b/tests/test_config.py @@ -30,6 +30,7 @@ def test_settings_allow_openai_compatible_overrides() -> None: "OPENAI_API_KEY": "openai-test-secret", "DEEPSEEK_API_KEY": "deepseek-test-secret", "MINICODE_MODEL_TIMEOUT": "15", + "MINICODE_FALLBACK_MODELS": "fallback-small, fallback-stable", } ) @@ -38,6 +39,12 @@ def test_settings_allow_openai_compatible_overrides() -> None: assert settings.api_key == "openai-test-secret" assert settings.chat_completions_url == "https://example.test/v1/chat/completions" assert settings.timeout_seconds == 15 + assert settings.fallback_models == ("fallback-small", "fallback-stable") + assert settings.model_candidates == ( + "local-model", + "fallback-small", + "fallback-stable", + ) def test_settings_accept_full_chat_completions_url() -> None: @@ -84,6 +91,59 @@ def test_settings_repr_does_not_expose_api_key() -> None: assert "top-secret-value" not in repr(settings) +@pytest.mark.parametrize( + "fallbacks", + [ + "primary", + "secondary,secondary", + "secondary,,tertiary", + "one,two,three,four,five", + "secondary\nforged", + "secondary\x1b[31m", + "x" * 257, + ], +) +def test_settings_reject_invalid_fallback_models(fallbacks: str) -> None: + with pytest.raises(ModelConfigurationError, match="MINICODE_FALLBACK_MODELS"): + ModelSettings.from_env( + { + "OPENAI_API_KEY": "test-secret", + "MINICODE_MODEL": "primary", + "MINICODE_FALLBACK_MODELS": fallbacks, + } + ) + + +def test_settings_constructor_rejects_string_fallback_collection() -> None: + with pytest.raises(ModelConfigurationError, match="fallback_models"): + ModelSettings( + model="primary", + base_url="https://example.test", + api_key="secret", + fallback_models="secondary", # type: ignore[arg-type] + ) + + +def test_settings_reject_control_characters_in_primary_model() -> None: + with pytest.raises(ModelConfigurationError, match="control characters"): + ModelSettings.from_env( + { + "OPENAI_API_KEY": "test-secret", + "MINICODE_MODEL": "primary\nforged", + } + ) + + +def test_settings_reject_base_url_credentials() -> None: + with pytest.raises(ModelConfigurationError, match="embedded credentials"): + ModelSettings.from_env( + { + "OPENAI_API_KEY": "test-secret", + "OPENAI_BASE_URL": "https://user:password@example.test/v1", + } + ) + + def test_runtime_settings_load_cli_environment() -> None: settings = RuntimeSettings.from_env( { diff --git a/tests/test_model_routing.py b/tests/test_model_routing.py new file mode 100644 index 0000000..4f87d01 --- /dev/null +++ b/tests/test_model_routing.py @@ -0,0 +1,201 @@ +from __future__ import annotations + +import pytest + +from minicode_rebuild.core import Message, MessageRole, ModelRequest, ModelResponse +from minicode_rebuild.models import MockModel, ModelRoute, RoutingModelAdapter +from minicode_rebuild.models.errors import ( + ModelHTTPError, + ModelResponseError, + ModelRoutingError, + ModelTransportError, +) +from minicode_rebuild.models.routing import RoutingEvent + +REQUEST = ModelRequest( + messages=(Message(role=MessageRole.USER, content="help"),) +) + + +def route(name: str, *responses: ModelResponse | Exception) -> ModelRoute: + return ModelRoute(name, MockModel(responses)) + + +def test_primary_success_does_not_touch_fallback() -> None: + primary_model = MockModel([ModelResponse(content="primary")]) + fallback_model = MockModel([ModelResponse(content="fallback")]) + events: list[RoutingEvent] = [] + adapter = RoutingModelAdapter( + ( + ModelRoute("primary", primary_model), + ModelRoute("fallback", fallback_model), + ), + observer=events.append, + ) + + response = adapter.complete(REQUEST) + + assert response.content == "primary" + assert len(primary_model.requests) == 1 + assert fallback_model.requests == () + assert events == [] + + +@pytest.mark.parametrize( + "failure", + [ + ModelTransportError("offline"), + ModelHTTPError(408, "timeout"), + ModelHTTPError(409, "conflict"), + ModelHTTPError(429, "busy"), + ModelHTTPError(503, "unavailable"), + ], +) +def test_transient_failure_routes_to_fallback(failure: Exception) -> None: + events: list[RoutingEvent] = [] + adapter = RoutingModelAdapter( + ( + route("primary", failure), + route("fallback", ModelResponse(content="recovered")), + ), + observer=events.append, + ) + + response = adapter.complete(REQUEST) + + assert response.content == "recovered" + assert events == [ + RoutingEvent( + route="primary", + attempt=1, + status="failed", + error_type=type(failure).__name__, + retrying=True, + ), + RoutingEvent(route="fallback", attempt=2, status="selected"), + ] + + +@pytest.mark.parametrize( + "failure", + [ + ModelHTTPError(400, "bad request"), + ModelHTTPError(401, "invalid key"), + ModelHTTPError(403, "forbidden"), + ModelResponseError("malformed response"), + ValueError("adapter bug"), + ], +) +def test_permanent_or_malformed_failure_does_not_fallback( + failure: Exception, +) -> None: + fallback = MockModel([ModelResponse(content="must not run")]) + adapter = RoutingModelAdapter( + ( + route("primary", failure), + ModelRoute("fallback", fallback), + ) + ) + + with pytest.raises(type(failure)): + adapter.complete(REQUEST) + + assert fallback.requests == () + + +def test_all_transient_routes_fail_with_redacted_summary() -> None: + adapter = RoutingModelAdapter( + ( + route("primary", ModelTransportError("secret-one")), + route("fallback", ModelHTTPError(503, "secret-two")), + ) + ) + + with pytest.raises(ModelRoutingError) as raised: + adapter.complete(REQUEST) + + assert raised.value.attempts == ( + "primary:ModelTransportError", + "fallback:ModelHTTPError", + ) + assert "secret-one" not in str(raised.value) + assert "secret-two" not in str(raised.value) + + +def test_fallback_receives_the_identical_normalized_request() -> None: + request = ModelRequest( + messages=REQUEST.messages, + max_output_tokens=123, + ) + primary = MockModel([ModelTransportError("offline")]) + fallback = MockModel([ModelResponse(content="recovered")]) + adapter = RoutingModelAdapter( + ( + ModelRoute("primary", primary), + ModelRoute("fallback", fallback), + ) + ) + + adapter.complete(request) + + assert primary.requests == (request,) + assert fallback.requests == (request,) + + +def test_route_observer_failure_does_not_break_recovery() -> None: + adapter = RoutingModelAdapter( + ( + route("primary", ModelTransportError("offline")), + route("fallback", ModelResponse(content="recovered")), + ), + observer=lambda _event: (_ for _ in ()).throw(RuntimeError("observer")), + ) + + assert adapter.complete(REQUEST).content == "recovered" + + +def test_control_flow_exception_propagates_without_fallback() -> None: + class InterruptingModel: + def complete(self, _request: ModelRequest) -> ModelResponse: + raise KeyboardInterrupt + + fallback = MockModel([ModelResponse(content="must not run")]) + adapter = RoutingModelAdapter( + ( + ModelRoute("primary", InterruptingModel()), + ModelRoute("fallback", fallback), + ) + ) + + with pytest.raises(KeyboardInterrupt): + adapter.complete(REQUEST) + + assert fallback.requests == () + + +@pytest.mark.parametrize( + "routes", + [ + (), + tuple(route(str(index), ModelResponse()) for index in range(6)), + (route("same", ModelResponse()), route("same", ModelResponse())), + ], +) +def test_router_rejects_invalid_route_sets(routes: tuple[ModelRoute, ...]) -> None: + with pytest.raises(ValueError): + RoutingModelAdapter(routes) + + +def test_model_route_validates_name_and_adapter() -> None: + with pytest.raises(ValueError, match="name"): + ModelRoute(" ", MockModel([])) + with pytest.raises(TypeError, match="adapter"): + ModelRoute("invalid", object()) # type: ignore[arg-type] + with pytest.raises(TypeError, match="name"): + ModelRoute(1, MockModel([])) # type: ignore[arg-type] + + +@pytest.mark.parametrize("name", ["primary\nforged", "primary\x1b[31m", "x" * 257]) +def test_model_route_rejects_unsafe_terminal_names(name: str) -> None: + with pytest.raises(ValueError, match="name"): + ModelRoute(name, MockModel([])) diff --git a/tests/test_openai_compatible.py b/tests/test_openai_compatible.py index 1b574cb..c7a42aa 100644 --- a/tests/test_openai_compatible.py +++ b/tests/test_openai_compatible.py @@ -18,7 +18,11 @@ TokenUsage, ToolCall, ) -from minicode_rebuild.models.errors import ModelResponseError, ModelTransportError +from minicode_rebuild.models.errors import ( + ModelHTTPError, + ModelResponseError, + ModelTransportError, +) from minicode_rebuild.models.openai_compatible import ( HttpResponse, OpenAICompatibleAdapter, @@ -277,6 +281,28 @@ def test_adapter_surfaces_http_error_message() -> None: adapter.complete(request(Message(role=MessageRole.USER, content="Hi"))) +@pytest.mark.parametrize( + ("status", "retryable"), + [(400, False), (401, False), (408, True), (429, True), (500, True), (599, True)], +) +def test_adapter_http_error_preserves_status_and_retryability( + status: int, + retryable: bool, +) -> None: + adapter = OpenAICompatibleAdapter( + settings(), + transport=FakeTransport( + [json_response({"error": {"message": "failed"}}, status=status)] + ), + ) + + with pytest.raises(ModelHTTPError) as raised: + adapter.complete(request(Message(role=MessageRole.USER, content="Hi"))) + + assert raised.value.status == status + assert raised.value.retryable is retryable + + def test_default_transport_wraps_network_error( monkeypatch: pytest.MonkeyPatch, ) -> None: diff --git a/tests/test_readiness.py b/tests/test_readiness.py index 908749f..c3f4366 100644 --- a/tests/test_readiness.py +++ b/tests/test_readiness.py @@ -18,12 +18,26 @@ def test_readiness_validates_provider_without_network(tmp_path: Path) -> None: assert report.ready is True output = format_readiness(report) - assert "model=demo-model" in output + assert "models=demo-model" in output assert "https://example.test/v1/chat/completions" in output assert "not-sent-anywhere" not in output assert "memory-store: 0 stored" in output +def test_readiness_validates_ordered_fallback_models(tmp_path: Path) -> None: + report = check_readiness( + tmp_path, + { + "OPENAI_API_KEY": "not-sent-anywhere", + "MINICODE_MODEL": "primary", + "MINICODE_FALLBACK_MODELS": "secondary,tertiary", + }, + ) + + assert report.ready is True + assert "models=primary,secondary,tertiary" in format_readiness(report) + + def test_readiness_reports_missing_provider_key(tmp_path: Path) -> None: report = check_readiness(tmp_path, {}, require_provider=True) @@ -34,6 +48,24 @@ def test_readiness_reports_missing_provider_key(tmp_path: Path) -> None: ) +def test_readiness_rejects_endpoint_credentials_without_displaying_them( + tmp_path: Path, +) -> None: + report = check_readiness( + tmp_path, + { + "OPENAI_API_KEY": "not-sent-anywhere", + "OPENAI_BASE_URL": "https://user:password@example.test/v1", + }, + ) + + output = format_readiness(report) + assert report.ready is False + assert "embedded credentials" in output + assert "user" not in output + assert "password" not in output + + def test_demo_readiness_does_not_require_provider_key(tmp_path: Path) -> None: report = check_readiness(tmp_path, {}, require_provider=False)