diff --git a/pyproject.toml b/pyproject.toml index e36c1e6..0965edc 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -4,7 +4,7 @@ build-backend = "uv_build" [project] name = "ucode" -version = "0.1.0" +version = "0.2.0" description = "Bootstrap Codex against a Databricks workspace with minimal setup." readme = "README.md" requires-python = ">=3.12" diff --git a/src/ucode/agents/_mlflow_proxy.py b/src/ucode/agents/_mlflow_proxy.py new file mode 100644 index 0000000..70e26ed --- /dev/null +++ b/src/ucode/agents/_mlflow_proxy.py @@ -0,0 +1,150 @@ +"""Local SSE-repair proxy for the MLflow chat-completions gateway. + +Some MLflow-served models (notably ``databricks-inkling``) stream a +spec-incomplete response: on natural completion the terminal chunk omits the +OpenAI-required ``finish_reason`` and the stream ends with only ``data: [DONE]``. +Strict clients — Pi's ``openai-completions`` parser among them — reject this +with "Stream ended without finish_reason", so the model is unusable. + +This proxy sits between Pi and the gateway. It forwards every request verbatim +and passes every response byte through unchanged, except that it repairs the +stream terminator: when a streaming response ends — cleanly at ``[DONE]`` OR +abnormally (an upstream drop / mid-stream 429, which otherwise reaches Pi as +"Stream ended without finish_reason") — after some data but without a +``finish_reason``, it synthesizes a ``finish_reason: "stop"`` chunk (and a +``[DONE]`` if the upstream never sent one). Models that already terminate +correctly (glm, llama, qwen, ...) never trigger the injection, so the proxy is +a no-op for them. + +Tracked upstream as the gateway bug that should make this unnecessary; once the +gateway emits ``finish_reason`` this module can be deleted. +""" + +from __future__ import annotations + +import json +import threading +import urllib.error +import urllib.request +from http.server import BaseHTTPRequestHandler, HTTPServer +from socketserver import ThreadingMixIn + +# Request headers we must not forward: hop-by-hop or ones urllib recomputes. +_SKIP_REQUEST_HEADERS = frozenset({"host", "content-length", "accept-encoding", "connection"}) + + +def _make_handler(upstream: str) -> type[BaseHTTPRequestHandler]: + class _Handler(BaseHTTPRequestHandler): + protocol_version = "HTTP/1.1" + + def do_POST(self) -> None: # noqa: N802 (BaseHTTPRequestHandler API) + length = int(self.headers.get("Content-Length", 0)) + body = self.rfile.read(length) + req = urllib.request.Request(upstream + self.path, data=body, method="POST") + for key, value in self.headers.items(): + if key.lower() not in _SKIP_REQUEST_HEADERS: + req.add_header(key, value) + try: + upstream_resp = urllib.request.urlopen(req) # noqa: S310 (trusted upstream) + except urllib.error.HTTPError as exc: + payload = exc.read() + self.send_response(exc.code) + self.send_header("Content-Length", str(len(payload))) + self.end_headers() + self.wfile.write(payload) + return + + self.send_response(upstream_resp.status) + self.send_header("Content-Type", "text/event-stream") + self.send_header("Cache-Control", "no-cache") + self.send_header("Connection", "close") + self.end_headers() + self._relay_stream(upstream_resp) + + def _relay_stream(self, upstream_resp) -> None: + saw_finish = False + saw_done = False + saw_data = False + last_id: str | None = None + try: + for raw in upstream_resp: # yields line-by-line as bytes arrive + stripped = raw.strip() + if stripped.startswith(b"data: ") and stripped[6:] != b"[DONE]": + saw_data = True + try: + obj = json.loads(stripped[6:]) + last_id = obj.get("id", last_id) + choice = (obj.get("choices") or [{}])[0] + if choice.get("finish_reason"): + saw_finish = True + except (ValueError, AttributeError, IndexError): + pass + self._write(raw) + elif stripped == b"data: [DONE]": + if not saw_finish: + self._write(b"data: " + _finish_chunk(last_id) + b"\n\n") + saw_finish = True + self._write(raw) + saw_done = True + else: + self._write(raw) + except OSError: + # Upstream stream dropped mid-flight (throttle/reset). Fall + # through to the terminator repair below so the client still + # sees a well-formed end instead of a truncated stream. + pass + # If the stream ended (cleanly or abnormally) after some data but + # without a finish_reason and/or [DONE], synthesize the terminator + # so strict clients (Pi) don't error on an incomplete stream. A + # gateway 429 mid-stream is the common trigger. + if saw_data and not saw_finish: + self._write(b"data: " + _finish_chunk(last_id) + b"\n\n") + if saw_data and not saw_done: + self._write(b"data: [DONE]\n\n") + + def _write(self, data: bytes) -> None: + self.wfile.write(data) + self.wfile.flush() + + def log_message(self, format: str, *args) -> None: # silence default stderr logging + pass + + return _Handler + + +def _finish_chunk(chunk_id: str | None) -> bytes: + return json.dumps( + { + "id": chunk_id, + "object": "chat.completion.chunk", + "choices": [{"delta": {}, "index": 0, "finish_reason": "stop"}], + } + ).encode() + + +class _Server(ThreadingMixIn, HTTPServer): + daemon_threads = True + allow_reuse_address = True + + +def start(upstream: str) -> tuple[str, threading.Event]: + """Start the repair proxy on a free localhost port. + + :param upstream: gateway origin to forward to, e.g. + ``https://workspace.databricks.com``. + :returns: ``(base_url, stop_event)`` where ``base_url`` is the local origin + Pi should target (``http://127.0.0.1:``) and setting + ``stop_event`` shuts the server down. + """ + server = _Server(("127.0.0.1", 0), _make_handler(upstream.rstrip("/"))) + port = server.server_address[1] + stop_event = threading.Event() + + threading.Thread(target=server.serve_forever, daemon=True).start() + + def _watch_stop() -> None: + stop_event.wait() + server.shutdown() + + threading.Thread(target=_watch_stop, daemon=True).start() + return f"http://127.0.0.1:{port}", stop_event diff --git a/src/ucode/agents/opencode.py b/src/ucode/agents/opencode.py index 7215388..c9f3508 100644 --- a/src/ucode/agents/opencode.py +++ b/src/ucode/agents/opencode.py @@ -1,4 +1,11 @@ -"""OpenCode agent: writes opencode.json with two Databricks-backed providers.""" +"""OpenCode agent: writes opencode.json with Databricks-backed providers. + +Providers: `databricks-anthropic` (@ai-sdk/anthropic), `databricks-google` +(@ai-sdk/google), and `databricks-oss` (@ai-sdk/openai, chat-completions mode) +for mlflow-only foundation models (inkling, glm, llama, qwen, ...). Unlike Pi, +opencode's @ai-sdk/openai tolerates the `finish_reason` the gateway omits for +inkling, so no SSE-repair proxy is needed here. +""" from __future__ import annotations @@ -41,6 +48,7 @@ PROVIDER_KEYS: list[list[str]] = [ ["provider", "databricks-anthropic"], ["provider", "databricks-google"], + ["provider", "databricks-oss"], ] @@ -50,7 +58,11 @@ def is_update_available() -> tuple[str, str] | None: def _resolve_model_selector(model: str, opencode_models: dict[str, list[str]]) -> str: """Return an OpenCode model selector in provider/model form when possible.""" - if model.startswith("databricks-anthropic/") or model.startswith("databricks-google/"): + if ( + model.startswith("databricks-anthropic/") + or model.startswith("databricks-google/") + or model.startswith("databricks-oss/") + ): return model anthropic_models = opencode_models.get("anthropic") or [] @@ -61,14 +73,37 @@ def _resolve_model_selector(model: str, opencode_models: dict[str, list[str]]) - if model in gemini_models: return f"databricks-google/{model}" + oss_models = opencode_models.get("oss") or [] + if model in oss_models: + return f"databricks-oss/{model}" + return model +def _oss_model_entry(ua_header: dict, spec: dict | None) -> dict: + """Build an opencode oss model entry. @ai-sdk/openai surfaces reasoning + automatically, so we only set `limit`: `context` from the gateway's stated + context window and `output` from its per-model max-output ceiling. Each is + included only when known; omit the whole `limit` if neither is.""" + entry: dict = {"headers": ua_header} + if not spec: + return entry + limit = {} + if spec.get("context_window"): + limit["context"] = spec["context_window"] + if spec.get("max_tokens"): + limit["output"] = spec["max_tokens"] + if limit: + entry["limit"] = limit + return entry + + def render_overlay( model: str, token: str, opencode_base_urls: dict[str, str], opencode_models: dict[str, list[str]], + oss_specs: list[dict] | None = None, ) -> tuple[dict, list[list[str]]]: """Return (overlay, managed_key_paths) for opencode.json.""" auth_headers = {"Authorization": f"Bearer {token}"} @@ -82,6 +117,7 @@ def render_overlay( anthropic_models = opencode_models.get("anthropic") or [] gemini_models = opencode_models.get("gemini") or [] + oss_models = opencode_models.get("oss") or [] providers: dict = {} keys: list[list[str]] = [["model"]] @@ -116,6 +152,23 @@ def render_overlay( "models": {m: {"headers": ua_header} for m in gemini_models}, } keys.append(["provider", "databricks-google"]) + if oss_models: + # mlflow chat-completions-only foundation models (inkling, glm, llama, + # qwen, ...) via @ai-sdk/openai in legacy chat-completions mode — NOT + # the Responses API (no useResponsesApi), unlike the codex provider. + # Unlike Pi, opencode's @ai-sdk/openai tolerates the finish_reason the + # gateway omits for reasoning models (inkling), so no proxy is needed. + oss_specs_by_id = {s["id"]: s for s in (oss_specs or [])} + providers["databricks-oss"] = { + "npm": "@ai-sdk/openai", + "options": { + "baseURL": opencode_base_urls["oss"], + "apiKey": token, + "headers": auth_headers, + }, + "models": {m: _oss_model_entry(ua_header, oss_specs_by_id.get(m)) for m in oss_models}, + } + keys.append(["provider", "databricks-oss"]) overlay: dict = {"model": _resolve_model_selector(model, opencode_models)} if providers: @@ -141,11 +194,17 @@ def write_tool_config( token, opencode_base_urls, state.get("opencode_models") or {}, + state.get("oss_model_specs") or [], ) existing = read_json_safe(OPENCODE_CONFIG_PATH) providers = existing.get("provider") if isinstance(providers, dict): - for stale in ("databricks-anthropic", "databricks-google", "databricks-openai"): + for stale in ( + "databricks-anthropic", + "databricks-google", + "databricks-openai", + "databricks-oss", + ): providers.pop(stale, None) merged = deep_merge_dict(existing, overlay) write_json_file(OPENCODE_CONFIG_PATH, merged) @@ -195,7 +254,10 @@ def default_model(state: dict) -> str | None: if anthropic: return anthropic[0] gemini = opencode_models.get("gemini") or [] - return gemini[0] if gemini else None + if gemini: + return gemini[0] + oss = opencode_models.get("oss") or [] + return oss[0] if oss else None def _refresh_token_once(state: dict, *, force_refresh: bool = False) -> str: diff --git a/src/ucode/agents/pi.py b/src/ucode/agents/pi.py index 995c1a5..12be6a8 100644 --- a/src/ucode/agents/pi.py +++ b/src/ucode/agents/pi.py @@ -1,12 +1,13 @@ """Pi coding agent: writes ~/.pi/agent/models.json with Databricks-backed providers. -Pi (https://pi.dev) is a multi-provider coding agent. We register three +Pi (https://pi.dev) is a multi-provider coding agent. We register four providers in its `models.json`, each speaking the API dialect best suited to that family's gateway path: - `databricks-claude` (api: anthropic-messages) → /ai-gateway/anthropic - `databricks-openai` (api: openai-responses) → /ai-gateway/codex/v1 - `databricks-gemini` (api: google-generative-ai) → /ai-gateway/gemini/v1beta +- `databricks-mlflow` (api: openai-completions) → /ai-gateway/mlflow/v1 Per-provider `compat` flags work around fields the gateway translators reject: @@ -15,11 +16,19 @@ pi uses for every request. With this flag pi omits the per-tool field and sends the legacy `anthropic-beta: fine-grained-tool-streaming-...` header instead, which the gateway accepts. +- mlflow: `supportsStore: false` and `supportsStrictMode: false` — the MLflow + chat-completions gateway rejects OpenAI's `store` field and + `tools[].function.strict`. -OSS / Databricks-foundation models (Llama, Qwen, etc.) are not exposed via -pi today — they live behind /ai-gateway/mlflow/v1 with per-model -`max_tokens` caps that pi has no global way to honor without per-model -config we don't currently maintain. +The `databricks-mlflow` provider carries chat-completions-only foundation +models (Llama, Qwen, GLM, inkling, ...) that have no native Anthropic / +OpenAI-responses / Gemini surface. Endpoints already served by one of the +other three providers are filtered out upstream so they aren't listed twice. + +At launch the mlflow provider is routed through a local SSE-repair proxy (see +`_mlflow_proxy`) that injects the `finish_reason` some models (inkling) omit on +the streaming terminal chunk, which Pi's parser otherwise rejects. The proxy is +a no-op for well-behaved models and can be removed once the gateway is fixed. The bearer token is baked into the file and refreshed by a background thread while the session runs (same pattern as OpenCode/Copilot). @@ -33,6 +42,7 @@ import threading from ucode.agent_updates import available_npm_package_update +from ucode.agents import _mlflow_proxy from ucode.config_io import ( APP_DIR, ToolSpec, @@ -66,6 +76,7 @@ "databricks-claude", "databricks-openai", "databricks-gemini", + "databricks-mlflow", ) PROVIDER_KEYS: list[list[str]] = [["providers", name] for name in PROVIDER_NAMES] @@ -84,6 +95,7 @@ def _resolve_model_selector( claude_models: dict[str, str], codex_models: list[str], gemini_models: list[str], + oss_models: list[str], ) -> str: """Return a Pi model selector in `/` form when possible.""" for name in PROVIDER_NAMES: @@ -95,9 +107,32 @@ def _resolve_model_selector( return f"databricks-openai/{model}" if model in gemini_models: return f"databricks-gemini/{model}" + if model in oss_models: + return f"databricks-mlflow/{model}" return model +def _pi_oss_model_entry(model_id: str, spec: dict | None) -> dict: + """Build a Pi mlflow model entry, enriched from the gateway capability spec. + + Sets `reasoning: true` when the model reports `openai_reasoning` (Pi renders + the gateway's reasoning_content as thinking), `contextWindow` when the + gateway states one, and `maxTokens` to the gateway's per-model output + ceiling (so requests don't 400). Fields are omitted when unknown so Pi + keeps its default. + """ + entry: dict = {"id": model_id} + if not spec: + return entry + if spec.get("reasoning"): + entry["reasoning"] = True + if spec.get("context_window"): + entry["contextWindow"] = spec["context_window"] + if spec.get("max_tokens"): + entry["maxTokens"] = spec["max_tokens"] + return entry + + def render_overlay( model: str, token: str, @@ -105,6 +140,8 @@ def render_overlay( claude_models: dict[str, str], codex_models: list[str], gemini_models: list[str], + oss_models: list[str], + oss_specs: list[dict] | None = None, ) -> tuple[dict, list[list[str]]]: """Return (overlay, managed_key_paths) for ~/.pi/agent/models.json.""" providers: dict = {} @@ -148,8 +185,24 @@ def render_overlay( "models": [{"id": m} for m in gemini_models], } keys.append(["providers", "databricks-gemini"]) + if oss_models: + specs_by_id = {s["id"]: s for s in (oss_specs or [])} + providers["databricks-mlflow"] = { + "baseUrl": pi_base_urls["oss"], + "api": "openai-completions", + "apiKey": token, + "authHeader": True, + # MLflow chat-completions gateway rejects OpenAI's `store` field + # and per-tool `strict`. Pi omits both when these are false. + "compat": {"supportsStore": False, "supportsStrictMode": False}, + "headers": ua_headers, + "models": [_pi_oss_model_entry(m, specs_by_id.get(m)) for m in oss_models], + } + keys.append(["providers", "databricks-mlflow"]) overlay: dict = { - "model": _resolve_model_selector(model, claude_models, codex_models, gemini_models), + "model": _resolve_model_selector( + model, claude_models, codex_models, gemini_models, oss_models + ), } if providers: overlay["providers"] = providers @@ -174,6 +227,8 @@ def write_tool_config( state.get("claude_models") or {}, state.get("codex_models") or [], state.get("gemini_models") or [], + state.get("oss_models") or [], + state.get("oss_model_specs") or [], ) existing = read_json_safe(PI_CONFIG_PATH) providers = existing.get("providers") @@ -188,7 +243,7 @@ def write_tool_config( def default_model(state: dict) -> str | None: - """Prefer Claude opus → sonnet → haiku; fall back to codex, gemini.""" + """Prefer Claude opus → sonnet → haiku; fall back to codex, gemini, oss.""" claude_models = state.get("claude_models") or {} for family in ("opus", "sonnet", "haiku"): if claude_models.get(family): @@ -197,7 +252,10 @@ def default_model(state: dict) -> str | None: if codex_models: return codex_models[0] gemini_models = state.get("gemini_models") or [] - return gemini_models[0] if gemini_models else None + if gemini_models: + return gemini_models[0] + oss_models = state.get("oss_models") or [] + return oss_models[0] if oss_models else None def _refresh_token_once(state: dict, *, force_refresh: bool = False) -> str: @@ -223,7 +281,31 @@ def build_runtime_env(token: str) -> dict[str, str]: return env +def _start_oss_proxy(state: dict) -> threading.Event | None: + """Route the mlflow provider through the local SSE-repair proxy. + + Rewrites ``state["base_urls"]["pi"]["oss"]`` in-memory to the proxy origin + so both the initial config write and every token-refresh rewrite point Pi + at the proxy. Returns the proxy's stop event, or None when no OSS models + are configured (nothing to proxy). See `_mlflow_proxy` for the why. + """ + if not (state.get("oss_models") or []): + return None + pi_urls = state.setdefault("base_urls", {}).setdefault( + "pi", build_pi_base_urls(state["workspace"]) + ) + # Always derive the gateway origin from the workspace, NOT from the + # persisted `oss` URL: a prior session may have baked a now-dead proxy port + # (http://127.0.0.1:) into state/config. Trusting it would proxy a + # dead proxy. build_pi_base_urls always yields the real gateway URL. + origin = build_pi_base_urls(state["workspace"])["oss"].split("/ai-gateway/", 1)[0] + proxy_base, stop_event = _mlflow_proxy.start(origin) + pi_urls["oss"] = f"{proxy_base}/ai-gateway/mlflow/v1" + return stop_event + + def launch(state: dict, tool_args: list[str]) -> None: + proxy_stop = _start_oss_proxy(state) token = _refresh_token_once(state) env = build_runtime_env(token) @@ -244,6 +326,8 @@ def launch(state: dict, tool_args: list[str]) -> None: finally: stop_event.set() refresher.join(timeout=1) + if proxy_stop is not None: + proxy_stop.set() raise SystemExit(returncode) diff --git a/src/ucode/cli.py b/src/ucode/cli.py index 6bc3f93..96bbd95 100644 --- a/src/ucode/cli.py +++ b/src/ucode/cli.py @@ -31,6 +31,7 @@ discover_claude_models, discover_codex_models, discover_gemini_models, + discover_oss_model_specs, ensure_ai_gateway_v2, ensure_databricks_auth, get_databricks_profiles, @@ -150,10 +151,12 @@ def configure_shared_state( ) want_gemini = fetch_all or "gemini" in tools or "opencode" in tools or "pi" in tools want_codex = fetch_all or "codex" in tools or "copilot" in tools or "pi" in tools + want_oss = fetch_all or "pi" in tools or "opencode" in tools claude_reason: str | None = None gemini_reason: str | None = None codex_reason: str | None = None + oss_reason: str | None = None with spinner("Fetching available models..."): if want_claude: claude_models, claude_reason = discover_claude_models(workspace, token) @@ -167,11 +170,19 @@ def configure_shared_state( codex_models, codex_reason = discover_codex_models(workspace, token) else: codex_models = [] + if want_oss: + oss_specs, oss_reason = discover_oss_model_specs(workspace, token) + oss_models = [s["id"] for s in oss_specs] + else: + oss_specs = [] + oss_models = [] opencode_models: dict[str, list[str]] = {} if claude_models: opencode_models["anthropic"] = list(claude_models.values()) if gemini_models: opencode_models["gemini"] = gemini_models + if oss_models: + opencode_models["oss"] = oss_models # Merge into existing workspace state so prior tool configs are preserved. state = load_state() @@ -183,6 +194,9 @@ def configure_shared_state( state["gemini_models"] = gemini_models if want_codex: state["codex_models"] = codex_models + if want_oss: + state["oss_models"] = oss_models + state["oss_model_specs"] = oss_specs if fetch_all or "opencode" in tools: state["opencode_models"] = opencode_models save_state(state) @@ -192,6 +206,7 @@ def configure_shared_state( "claude": claude_reason, "gemini": gemini_reason, "codex": codex_reason, + "oss": oss_reason, } return state diff --git a/src/ucode/databricks.py b/src/ucode/databricks.py index 35c7f31..731da63 100644 --- a/src/ucode/databricks.py +++ b/src/ucode/databricks.py @@ -809,6 +809,136 @@ def discover_codex_models(workspace: str, token: str) -> tuple[list[str], str | return discover_endpoints_with_api_type(workspace, token, "openai/v1/responses") +# api_types already served by the claude/openai/gemini Pi providers. Endpoints +# exposing any of these are reached through their native provider, so the OSS +# (mlflow chat-completions) provider skips them to avoid listing a model twice. +_NATIVE_PROVIDER_API_TYPES = frozenset( + {"anthropic/v1/messages", "openai/v1/responses", "gemini/v1/generateContent"} +) + + +def discover_oss_models(workspace: str, token: str) -> tuple[list[str], str | None]: + """Discover chat-completions-only endpoints (Llama, Qwen, GLM, inkling, ...). + + Returns (endpoints, reason). Keeps endpoints whose served entities expose + `mlflow/v1/chat/completions` but none of the native api_types already served + by the dedicated claude/openai/gemini Pi providers, and excludes embedding + endpoints. reason is None on success; otherwise it describes the empty list. + """ + hostname = workspace_hostname(workspace) + payload, reason = _http_get_json( + f"https://{hostname}/api/2.0/serving-endpoints:foundation-models", token + ) + if payload is None: + return [], reason + + data = cast(dict, payload) if isinstance(payload, dict) else {} + endpoints = data.get("endpoints", []) + out: list[str] = [] + for ep in endpoints: + name = ep.get("name", "") + api_types: set[str] = set() + for se in ep.get("config", {}).get("served_entities", []): + api_types.update(se.get("foundation_model", {}).get("api_types", [])) + if "mlflow/v1/chat/completions" not in api_types: + continue + if api_types & _NATIVE_PROVIDER_API_TYPES: + continue + out.append(name) + if out: + return sorted(out), None + if not endpoints: + return [], "foundation-models listing returned no endpoints" + return [], "no chat-completions-only endpoints (all served by native providers)" + + +# Matches a stated context length in a foundation-model description, e.g. +# "context length of 1M tokens" or "context window of 128K". The gateway only +# states the window in free text (there is no structured field), so this is +# best-effort: models without a stated length get no contextWindow override. +_CONTEXT_LENGTH_RE = re.compile(r"context (?:length|window) of ([\d.,]+)\s*([MK])", re.IGNORECASE) + + +def _parse_context_window(description: str) -> int | None: + """Extract a context window (in tokens) from a model description, or None.""" + match = _CONTEXT_LENGTH_RE.search(description or "") + if not match: + return None + value = float(match.group(1).replace(",", "")) + multiplier = 1_000_000 if match.group(2).upper() == "M" else 1_000 + return int(value * multiplier) + + +# Per-model max output-token ceilings enforced by the mlflow gateway. These are +# NOT exposed by any metadata API — the gateway only reveals them by 400ing an +# oversized request ("max_tokens (N) cannot exceed "). Values probed +# 2026-07-16; if the gateway raises a cap or adds a model, this may go stale, so +# unknown ids get NO override (client keeps its default) rather than a guess. +_OSS_MAX_OUTPUT_TOKENS: dict[str, int] = { + "databricks-glm-5-2": 65536, + "databricks-inkling": 65536, + "databricks-kimi-k2-7-code": 65536, + "databricks-gpt-oss-120b": 25000, + "databricks-gpt-oss-20b": 25000, + "databricks-qwen35-122b-a10b": 25000, + "databricks-qwen3-next-80b-a3b-instruct": 10000, + "databricks-llama-4-maverick": 8192, + "databricks-meta-llama-3-1-8b-instruct": 8192, + "databricks-meta-llama-3-3-70b-instruct": 8192, + "databricks-gemma-3-12b": 8192, +} + + +def discover_oss_model_specs(workspace: str, token: str) -> tuple[list[dict], str | None]: + """Discover OSS (mlflow chat-completions-only) models with capability specs. + + Same endpoint selection as `discover_oss_models`, but returns per-model + dicts ``{"id", "reasoning", "context_window", "max_tokens"}`` so adapters + can enrich Pi / OpenCode config. ``reasoning`` comes from the endpoint's + structured ``capabilities.openai_reasoning`` (reliable). ``context_window`` + is parsed from the free-text description and is ``None`` when not stated. + ``max_tokens`` is the gateway's per-model output ceiling from + ``_OSS_MAX_OUTPUT_TOKENS``, or ``None`` for models we haven't probed. We + never fabricate a window or cap we don't know. Returns (specs, reason). + """ + hostname = workspace_hostname(workspace) + payload, reason = _http_get_json( + f"https://{hostname}/api/2.0/serving-endpoints:foundation-models", token + ) + if payload is None: + return [], reason + + data = cast(dict, payload) if isinstance(payload, dict) else {} + endpoints = data.get("endpoints", []) + specs: list[dict] = [] + for ep in endpoints: + entities = ep.get("config", {}).get("served_entities", []) + api_types: set[str] = set() + description = "" + for se in entities: + fm = se.get("foundation_model", {}) + api_types.update(fm.get("api_types", [])) + description = description or fm.get("description", "") + if "mlflow/v1/chat/completions" not in api_types: + continue + if api_types & _NATIVE_PROVIDER_API_TYPES: + continue + name = ep.get("name", "") + specs.append( + { + "id": name, + "reasoning": bool(ep.get("capabilities", {}).get("openai_reasoning")), + "context_window": _parse_context_window(description), + "max_tokens": _OSS_MAX_OUTPUT_TOKENS.get(name), + } + ) + if specs: + return sorted(specs, key=lambda s: s["id"]), None + if not endpoints: + return [], "foundation-models listing returned no endpoints" + return [], "no chat-completions-only endpoints (all served by native providers)" + + def fetch_gemini_models(workspace: str, token: str) -> list[str]: models, _ = discover_gemini_models(workspace, token) return models @@ -819,6 +949,11 @@ def fetch_codex_models(workspace: str, token: str) -> list[str]: return models +def fetch_oss_models(workspace: str, token: str) -> list[str]: + models, _ = discover_oss_models(workspace, token) + return models + + def ensure_ai_gateway_v2(workspace: str, token: str) -> None: """Probe AI Gateway v2 and raise if unavailable. @@ -958,6 +1093,9 @@ def build_opencode_base_urls(workspace: str) -> dict[str, str]: return { "anthropic": build_tool_base_url("claude", workspace) + "/v1", "gemini": build_tool_base_url("gemini", workspace) + "/v1beta", + # @ai-sdk/openai (chat-completions mode) appends `/chat/completions`, so + # stop before that suffix. Carries mlflow-only models (inkling, glm, ...). + "oss": f"{workspace}/ai-gateway/mlflow/v1", } @@ -977,6 +1115,7 @@ def build_pi_base_urls(workspace: str) -> dict[str, str]: "claude": build_tool_base_url("claude", workspace), "openai": build_tool_base_url("codex", workspace), "gemini": build_tool_base_url("gemini", workspace) + "/v1beta", + "oss": f"{workspace}/ai-gateway/mlflow/v1", } diff --git a/tests/test_agent_opencode.py b/tests/test_agent_opencode.py index 0f32f4d..f28bada 100644 --- a/tests/test_agent_opencode.py +++ b/tests/test_agent_opencode.py @@ -14,6 +14,7 @@ def _base_urls() -> dict[str, str]: return { "anthropic": f"{WS}/ai-gateway/anthropic/v1", "gemini": f"{WS}/ai-gateway/gemini/v1beta", + "oss": f"{WS}/ai-gateway/mlflow/v1", } @@ -152,6 +153,97 @@ def test_prefixes_gemini_model_with_provider_id(self): assert overlay["model"] == "databricks-google/gemini-2" +class TestRenderOverlayOss: + """mlflow chat-completions-only foundation models (inkling, glm, ...) reach + OpenCode via a databricks-oss provider using @ai-sdk/openai in legacy + chat-completions mode. No SSE-repair proxy is needed (unlike Pi) because + @ai-sdk/openai tolerates the finish_reason inkling omits.""" + + def test_oss_provider_added_when_oss_models_present(self): + models = {"oss": ["databricks-inkling"]} + overlay, _ = opencode.render_overlay("databricks-inkling", "tok", _base_urls(), models) + assert "databricks-oss" in overlay["provider"] + + def test_oss_provider_uses_ai_sdk_openai_npm(self): + models = {"oss": ["databricks-inkling"]} + overlay, _ = opencode.render_overlay("databricks-inkling", "tok", _base_urls(), models) + assert overlay["provider"]["databricks-oss"]["npm"] == "@ai-sdk/openai" + + def test_oss_base_url_points_at_mlflow_gateway(self): + models = {"oss": ["databricks-inkling"]} + overlay, _ = opencode.render_overlay("databricks-inkling", "tok", _base_urls(), models) + options = overlay["provider"]["databricks-oss"]["options"] + assert options["baseURL"] == f"{WS}/ai-gateway/mlflow/v1" + + def test_oss_does_not_use_responses_api(self): + # Distinct from the codex provider: mlflow route is chat-completions, + # so useResponsesApi must NOT be set on oss models. + models = {"oss": ["databricks-inkling"]} + overlay, _ = opencode.render_overlay("databricks-inkling", "tok", _base_urls(), models) + entry = overlay["provider"]["databricks-oss"]["models"]["databricks-inkling"] + assert "options" not in entry or "useResponsesApi" not in entry.get("options", {}) + + def test_oss_authorization_header(self): + models = {"oss": ["databricks-inkling"]} + overlay, _ = opencode.render_overlay("databricks-inkling", "tok", _base_urls(), models) + headers = overlay["provider"]["databricks-oss"]["options"]["headers"] + assert headers["Authorization"] == "Bearer tok" + + def test_prefixes_oss_model_with_provider_id(self): + models = {"oss": ["databricks-inkling"]} + overlay, _ = opencode.render_overlay("databricks-inkling", "tok", _base_urls(), models) + assert overlay["model"] == "databricks-oss/databricks-inkling" + + def test_managed_keys_include_oss_provider(self): + models = {"oss": ["databricks-inkling"]} + _, keys = opencode.render_overlay("databricks-inkling", "tok", _base_urls(), models) + assert ["provider", "databricks-oss"] in keys + + def test_provider_keys_listed_in_module(self): + assert ["provider", "databricks-oss"] in opencode.PROVIDER_KEYS + + def test_oss_model_sets_limit_context_and_output_when_known(self): + models = {"oss": ["databricks-glm-5-2"]} + specs = [ + { + "id": "databricks-glm-5-2", + "reasoning": True, + "context_window": 1_000_000, + "max_tokens": 65536, + } + ] + overlay, _ = opencode.render_overlay( + "databricks-glm-5-2", "tok", _base_urls(), models, specs + ) + entry = overlay["provider"]["databricks-oss"]["models"]["databricks-glm-5-2"] + assert entry["limit"] == {"context": 1_000_000, "output": 65536} + + def test_oss_model_output_only_when_context_unstated(self): + models = {"oss": ["databricks-inkling"]} + specs = [ + { + "id": "databricks-inkling", + "reasoning": True, + "context_window": None, + "max_tokens": 65536, + } + ] + overlay, _ = opencode.render_overlay( + "databricks-inkling", "tok", _base_urls(), models, specs + ) + entry = overlay["provider"]["databricks-oss"]["models"]["databricks-inkling"] + assert entry["limit"] == {"output": 65536} + + def test_oss_model_omits_limit_when_nothing_known(self): + models = {"oss": ["databricks-x"]} + specs = [ + {"id": "databricks-x", "reasoning": False, "context_window": None, "max_tokens": None} + ] + overlay, _ = opencode.render_overlay("databricks-x", "tok", _base_urls(), models, specs) + entry = overlay["provider"]["databricks-oss"]["models"]["databricks-x"] + assert "limit" not in entry + + class TestMcpServerConfig: def test_builds_remote_server_entry_with_oauth_token_env_header(self): entry = opencode.build_mcp_server_entry(f"{WS}/api/2.0/mcp/external/github") @@ -268,6 +360,10 @@ def test_falls_back_to_gemini(self): state = {"opencode_models": {"anthropic": [], "gemini": ["gemini-2"]}} assert opencode.default_model(state) == "gemini-2" + def test_falls_back_to_oss_last(self): + state = {"opencode_models": {"anthropic": [], "gemini": [], "oss": ["databricks-inkling"]}} + assert opencode.default_model(state) == "databricks-inkling" + def test_returns_none_when_empty(self): assert opencode.default_model({}) is None assert opencode.default_model({"opencode_models": {}}) is None diff --git a/tests/test_agent_pi.py b/tests/test_agent_pi.py index 02ec6f1..2ddca0f 100644 --- a/tests/test_agent_pi.py +++ b/tests/test_agent_pi.py @@ -16,6 +16,7 @@ def _base_urls() -> dict[str, str]: "claude": f"{WS}/ai-gateway/anthropic", "openai": f"{WS}/ai-gateway/codex/v1", "gemini": f"{WS}/ai-gateway/gemini/v1beta", + "oss": f"{WS}/ai-gateway/mlflow/v1", } @@ -25,10 +26,11 @@ def _empty() -> dict: "claude_models": {}, "codex_models": [], "gemini_models": [], + "oss_models": [], } -def _overlay(model: str, token: str = "tok", **kwargs): +def _overlay(model: str, token: str = "tok", oss_specs=None, **kwargs): """Wrapper to call render_overlay with sensible defaults so tests stay terse.""" bundle = {**_empty(), **kwargs} return pi.render_overlay( @@ -38,6 +40,8 @@ def _overlay(model: str, token: str = "tok", **kwargs): bundle["claude_models"], bundle["codex_models"], bundle["gemini_models"], + bundle["oss_models"], + oss_specs, ) @@ -80,17 +84,84 @@ def test_gemini_provider_uses_google_generative_ai(self): assert provider["api"] == "google-generative-ai" assert provider["baseUrl"] == f"{WS}/ai-gateway/gemini/v1beta" - def test_all_three_providers_when_all_present(self): + def test_mlflow_provider_uses_openai_completions(self): + overlay, _ = _overlay("databricks-inkling", oss_models=["databricks-inkling"]) + provider = overlay["providers"]["databricks-mlflow"] + assert provider["api"] == "openai-completions" + assert provider["baseUrl"] == f"{WS}/ai-gateway/mlflow/v1" + assert provider["models"] == [{"id": "databricks-inkling"}] + + def test_no_mlflow_provider_when_no_oss_models(self): + overlay, _ = _overlay("gpt-5", codex_models=["gpt-5"]) + assert "databricks-mlflow" not in overlay.get("providers", {}) + + def test_mlflow_model_bare_without_specs(self): + overlay, _ = _overlay("databricks-inkling", oss_models=["databricks-inkling"]) + assert overlay["providers"]["databricks-mlflow"]["models"] == [{"id": "databricks-inkling"}] + + def test_mlflow_model_enriched_with_reasoning_context_and_max_tokens(self): + specs = [ + { + "id": "databricks-glm-5-2", + "reasoning": True, + "context_window": 1_000_000, + "max_tokens": 65536, + } + ] + overlay, _ = _overlay( + "databricks-glm-5-2", oss_models=["databricks-glm-5-2"], oss_specs=specs + ) + entry = overlay["providers"]["databricks-mlflow"]["models"][0] + assert entry["id"] == "databricks-glm-5-2" + assert entry["reasoning"] is True + assert entry["contextWindow"] == 1_000_000 + assert entry["maxTokens"] == 65536 + + def test_mlflow_model_omits_unknown_fields(self): + specs = [ + { + "id": "databricks-inkling", + "reasoning": True, + "context_window": None, + "max_tokens": None, + } + ] + overlay, _ = _overlay( + "databricks-inkling", oss_models=["databricks-inkling"], oss_specs=specs + ) + entry = overlay["providers"]["databricks-mlflow"]["models"][0] + assert entry["reasoning"] is True + assert "contextWindow" not in entry + assert "maxTokens" not in entry + + def test_mlflow_model_no_reasoning_key_when_false(self): + specs = [ + { + "id": "databricks-llama", + "reasoning": False, + "context_window": 128_000, + "max_tokens": 8192, + } + ] + overlay, _ = _overlay("databricks-llama", oss_models=["databricks-llama"], oss_specs=specs) + entry = overlay["providers"]["databricks-mlflow"]["models"][0] + assert "reasoning" not in entry + assert entry["contextWindow"] == 128_000 + assert entry["maxTokens"] == 8192 + + def test_all_four_providers_when_all_present(self): overlay, _ = _overlay( "claude-sonnet", claude_models={"sonnet": "claude-sonnet"}, codex_models=["gpt-5"], gemini_models=["gemini-2"], + oss_models=["databricks-inkling"], ) assert set(overlay["providers"].keys()) == { "databricks-claude", "databricks-openai", "databricks-gemini", + "databricks-mlflow", } @@ -128,6 +199,14 @@ def test_openai_and_gemini_have_no_compat_flags(self): assert "compat" not in overlay["providers"]["databricks-openai"] assert "compat" not in overlay["providers"]["databricks-gemini"] + def test_mlflow_disables_store_and_strict_mode(self): + # MLflow chat-completions gateway rejects OpenAI's `store` field and + # per-tool `strict`; these flags make pi omit both. + overlay, _ = _overlay("databricks-inkling", oss_models=["databricks-inkling"]) + compat = overlay["providers"]["databricks-mlflow"]["compat"] + assert compat["supportsStore"] is False + assert compat["supportsStrictMode"] is False + class TestRenderOverlayAuthAndModels: def test_token_in_api_key(self): @@ -192,6 +271,10 @@ def test_prefixes_gemini_model(self): overlay, _ = _overlay("gemini-2", gemini_models=["gemini-2"]) assert overlay["model"] == "databricks-gemini/gemini-2" + def test_prefixes_oss_model(self): + overlay, _ = _overlay("databricks-inkling", oss_models=["databricks-inkling"]) + assert overlay["model"] == "databricks-mlflow/databricks-inkling" + def test_preserves_already_prefixed_model(self): overlay, _ = _overlay( "databricks-claude/claude-sonnet", @@ -227,6 +310,15 @@ def test_falls_back_to_gemini(self): state = {"claude_models": {}, "codex_models": [], "gemini_models": ["gemini-2"]} assert pi.default_model(state) == "gemini-2" + def test_falls_back_to_oss_last(self): + state = { + "claude_models": {}, + "codex_models": [], + "gemini_models": [], + "oss_models": ["databricks-inkling"], + } + assert pi.default_model(state) == "databricks-inkling" + def test_returns_none_when_empty(self): assert pi.default_model({}) is None assert ( @@ -234,6 +326,44 @@ def test_returns_none_when_empty(self): ) +class TestStartOssProxy: + def test_none_when_no_oss_models(self): + state = {"workspace": WS, "oss_models": []} + assert pi._start_oss_proxy(state) is None + + def test_rewrites_oss_base_url_to_proxy(self): + state = { + "workspace": WS, + "oss_models": ["databricks-inkling"], + "base_urls": {"pi": {"oss": f"{WS}/ai-gateway/mlflow/v1"}}, + } + with patch.object( + pi._mlflow_proxy, "start", return_value=("http://127.0.0.1:5000", object()) + ) as started: + stop = pi._start_oss_proxy(state) + # Proxy is fed the bare gateway origin (path suffix stripped)... + started.assert_called_once_with(WS) + # ...and the mlflow provider now targets the local proxy. + assert state["base_urls"]["pi"]["oss"] == "http://127.0.0.1:5000/ai-gateway/mlflow/v1" + assert stop is not None + + def test_ignores_stale_persisted_proxy_url_as_origin(self): + # A prior session baked a now-dead proxy port into state["...]["oss"]. + # The proxy must forward to the real gateway, NOT the stale proxy URL, + # or it would proxy a dead port (the cross-session finish_reason bug). + state = { + "workspace": WS, + "oss_models": ["databricks-inkling"], + "base_urls": {"pi": {"oss": "http://127.0.0.1:58793/ai-gateway/mlflow/v1"}}, + } + with patch.object( + pi._mlflow_proxy, "start", return_value=("http://127.0.0.1:6000", object()) + ) as started: + pi._start_oss_proxy(state) + # Origin is the gateway derived from workspace, never the stale localhost. + started.assert_called_once_with(WS) + + class TestBuildRuntimeEnv: def test_sets_oauth_token(self): env = pi.build_runtime_env("tok") diff --git a/tests/test_cli.py b/tests/test_cli.py index 0664921..dcd643e 100644 --- a/tests/test_cli.py +++ b/tests/test_cli.py @@ -642,3 +642,57 @@ def fake_configure_shared_state(workspace, tools=None, force_login=False): ] assert saved == ["https://first.com"] assert configured_tools == [("https://first.com", ["codex"])] + + +class TestConfigureSharedStateOssWiring: + """configure_shared_state must fetch mlflow-only (oss) models and route + them into both state['oss_models'] (Pi) and opencode_models['oss'] + (OpenCode) when pi/opencode are requested. Guards the discovery->state seam + the mlflow-provider work depends on.""" + + def _run(self, monkeypatch, tools): + import ucode.cli as cli_mod + + saved = {} + monkeypatch.setattr(cli_mod, "ensure_databricks_auth", lambda ws: None) + monkeypatch.setattr(cli_mod, "get_databricks_token", lambda ws, **k: "tok") + monkeypatch.setattr(cli_mod, "ensure_ai_gateway_v2", lambda ws, tok: None) + monkeypatch.setattr(cli_mod, "discover_claude_models", lambda ws, tok: ({}, None)) + monkeypatch.setattr(cli_mod, "discover_gemini_models", lambda ws, tok: ([], None)) + monkeypatch.setattr(cli_mod, "discover_codex_models", lambda ws, tok: ([], None)) + monkeypatch.setattr( + cli_mod, + "discover_oss_model_specs", + lambda ws, tok: ( + [ + { + "id": "databricks-inkling", + "reasoning": True, + "context_window": None, + "max_tokens": 65536, + } + ], + None, + ), + ) + monkeypatch.setattr(cli_mod, "load_state", lambda: {}) + monkeypatch.setattr(cli_mod, "save_state", lambda s: saved.update(s)) + return cli_mod.configure_shared_state("https://ws.databricks.com", tools=tools) + + def test_oss_models_in_state_for_pi(self, monkeypatch): + state = self._run(monkeypatch, ["pi"]) + assert state["oss_models"] == ["databricks-inkling"] + + def test_oss_models_routed_to_opencode(self, monkeypatch): + state = self._run(monkeypatch, ["opencode"]) + assert state["oss_models"] == ["databricks-inkling"] + assert state["opencode_models"]["oss"] == ["databricks-inkling"] + + def test_oss_specs_stored_in_state(self, monkeypatch): + state = self._run(monkeypatch, ["pi"]) + assert state["oss_model_specs"][0]["reasoning"] is True + + def test_oss_not_fetched_for_unrelated_tool(self, monkeypatch): + # A claude-only configure should not populate oss_models. + state = self._run(monkeypatch, ["claude"]) + assert not state.get("oss_models") diff --git a/tests/test_databricks.py b/tests/test_databricks.py index dfcf17c..52a370c 100644 --- a/tests/test_databricks.py +++ b/tests/test_databricks.py @@ -18,8 +18,11 @@ build_auth_shell_command, build_databricks_cli_env, build_opencode_base_urls, + build_pi_base_urls, build_shared_base_urls, build_tool_base_url, + discover_oss_model_specs, + discover_oss_models, ensure_databricks_cli_version, get_databricks_token, list_databricks_apps, @@ -80,6 +83,10 @@ def test_returns_anthropic_and_gemini(self): assert urls["anthropic"] == f"{WS}/ai-gateway/anthropic/v1" assert urls["gemini"] == f"{WS}/ai-gateway/gemini/v1beta" + def test_oss_points_at_mlflow_gateway(self): + urls = build_opencode_base_urls(WS) + assert urls["oss"] == f"{WS}/ai-gateway/mlflow/v1" + class TestBuildSharedBaseUrls: def test_contains_all_tools(self): @@ -98,6 +105,229 @@ def test_codex_url_format(self): assert urls["codex"] == f"{WS}/ai-gateway/codex/v1" +class TestBuildPiBaseUrls: + def test_oss_points_at_mlflow_gateway(self): + urls = build_pi_base_urls(WS) + assert urls["oss"] == f"{WS}/ai-gateway/mlflow/v1" + + +class TestDiscoverOssModels: + """discover_oss_models keeps chat-completions-only endpoints and dedupes + anything already served by the native claude/openai/gemini providers.""" + + _PAYLOAD = { + "endpoints": [ + # Chat-completions only → kept. + { + "name": "databricks-inkling", + "config": { + "served_entities": [ + {"foundation_model": {"api_types": ["mlflow/v1/chat/completions"]}} + ] + }, + }, + { + "name": "databricks-llama-4-maverick", + "config": { + "served_entities": [ + {"foundation_model": {"api_types": ["mlflow/v1/chat/completions"]}} + ] + }, + }, + # Also served via openai-responses → skipped (native provider). + { + "name": "databricks-gpt-5", + "config": { + "served_entities": [ + { + "foundation_model": { + "api_types": [ + "mlflow/v1/chat/completions", + "openai/v1/responses", + ] + } + } + ] + }, + }, + # Also served via anthropic/messages → skipped (native provider). + { + "name": "databricks-claude-sonnet-5", + "config": { + "served_entities": [ + { + "foundation_model": { + "api_types": [ + "mlflow/v1/chat/completions", + "anthropic/v1/messages", + ] + } + } + ] + }, + }, + # Embeddings → skipped (no chat-completions). + { + "name": "databricks-bge-large-en", + "config": { + "served_entities": [ + {"foundation_model": {"api_types": ["mlflow/v1/embeddings"]}} + ] + }, + }, + ] + } + + def _patch(self, payload, reason=None): + from unittest.mock import patch + + return patch.object(db_mod, "_http_get_json", return_value=(payload, reason)) + + def test_keeps_only_chat_completions_only_endpoints(self): + with self._patch(self._PAYLOAD): + models, reason = discover_oss_models(WS, "tok") + assert models == ["databricks-inkling", "databricks-llama-4-maverick"] + assert reason is None + + def test_reports_reason_when_all_served_by_native_providers(self): + payload = {"endpoints": [self._PAYLOAD["endpoints"][2]]} # gpt-5 only + with self._patch(payload): + models, reason = discover_oss_models(WS, "tok") + assert models == [] + assert reason is not None + + def test_propagates_http_failure(self): + with self._patch(None, "HTTP 401"): + models, reason = discover_oss_models(WS, "tok") + assert models == [] + assert reason == "HTTP 401" + + +class TestParseContextWindow: + def test_millions(self): + assert db_mod._parse_context_window("supports a context length of 1M tokens") == 1_000_000 + + def test_thousands(self): + assert db_mod._parse_context_window("context window of 128K") == 128_000 + + def test_decimal_millions(self): + assert db_mod._parse_context_window("context length of 1.5M tokens") == 1_500_000 + + def test_unstated_returns_none(self): + assert db_mod._parse_context_window("a great model for coding") is None + + def test_empty(self): + assert db_mod._parse_context_window("") is None + + +class TestDiscoverOssModelSpecs: + """discover_oss_model_specs returns per-model reasoning (from structured + capabilities) and context_window (parsed from description, None when + unstated) for the same chat-completions-only cohort.""" + + _PAYLOAD = { + "endpoints": [ + { + "name": "databricks-glm-5-2", + "capabilities": {"openai_reasoning": True}, + "config": { + "served_entities": [ + { + "foundation_model": { + "api_types": ["mlflow/v1/chat/completions"], + "description": "MoE model with a context length of 1M tokens.", + } + } + ] + }, + }, + { + "name": "databricks-llama-4-maverick", + "capabilities": {"openai_reasoning": False}, + "config": { + "served_entities": [ + { + "foundation_model": { + "api_types": ["mlflow/v1/chat/completions"], + "description": "context window of 128K.", + } + } + ] + }, + }, + { + "name": "databricks-inkling", + "capabilities": {"openai_reasoning": True}, + "config": { + "served_entities": [ + { + "foundation_model": { + "api_types": ["mlflow/v1/chat/completions"], + "description": "a reasoning model (no window stated).", + } + } + ] + }, + }, + ] + } + + def _patch(self, payload, reason=None): + from unittest.mock import patch + + return patch.object(db_mod, "_http_get_json", return_value=(payload, reason)) + + def test_reasoning_from_capabilities(self): + with self._patch(self._PAYLOAD): + specs, _ = discover_oss_model_specs(WS, "tok") + by_id = {s["id"]: s for s in specs} + assert by_id["databricks-glm-5-2"]["reasoning"] is True + assert by_id["databricks-inkling"]["reasoning"] is True + assert by_id["databricks-llama-4-maverick"]["reasoning"] is False + + def test_context_window_parsed_or_none(self): + with self._patch(self._PAYLOAD): + specs, _ = discover_oss_model_specs(WS, "tok") + by_id = {s["id"]: s for s in specs} + assert by_id["databricks-glm-5-2"]["context_window"] == 1_000_000 + assert by_id["databricks-llama-4-maverick"]["context_window"] == 128_000 + # inkling states no window -> None (we never fabricate one). + assert by_id["databricks-inkling"]["context_window"] is None + + def test_max_tokens_from_known_caps(self): + with self._patch(self._PAYLOAD): + specs, _ = discover_oss_model_specs(WS, "tok") + by_id = {s["id"]: s for s in specs} + # Probed gateway ceilings baked into _OSS_MAX_OUTPUT_TOKENS. + assert by_id["databricks-glm-5-2"]["max_tokens"] == 65536 + assert by_id["databricks-inkling"]["max_tokens"] == 65536 + assert by_id["databricks-llama-4-maverick"]["max_tokens"] == 8192 + + def test_max_tokens_none_for_unprobed_model(self): + payload = { + "endpoints": [ + { + "name": "databricks-some-new-model", + "capabilities": {"openai_reasoning": False}, + "config": { + "served_entities": [ + {"foundation_model": {"api_types": ["mlflow/v1/chat/completions"]}} + ] + }, + } + ] + } + with self._patch(payload): + specs, _ = discover_oss_model_specs(WS, "tok") + assert specs[0]["max_tokens"] is None + + def test_propagates_http_failure(self): + with self._patch(None, "HTTP 401"): + specs, reason = discover_oss_model_specs(WS, "tok") + assert specs == [] + assert reason == "HTTP 401" + + class TestBuildAuthShellCommand: def test_contains_workspace(self): cmd = build_auth_shell_command(WS) diff --git a/tests/test_mlflow_proxy.py b/tests/test_mlflow_proxy.py new file mode 100644 index 0000000..f71c0da --- /dev/null +++ b/tests/test_mlflow_proxy.py @@ -0,0 +1,122 @@ +"""Tests for agents/_mlflow_proxy.py — the SSE finish_reason repair proxy.""" + +from __future__ import annotations + +import threading +import urllib.request +from http.server import BaseHTTPRequestHandler, HTTPServer + +from ucode.agents import _mlflow_proxy + +# An inkling-style stream: content deltas then [DONE], NO finish_reason anywhere. +_STREAM_NO_FINISH = ( + b'data: {"id":"c1","choices":[{"delta":{"content":"ok"},"index":0}]}\n\ndata: [DONE]\n\n' +) +# A well-behaved stream: terminal chunk carries finish_reason. +_STREAM_WITH_FINISH = ( + b'data: {"id":"c2","choices":[{"delta":{"content":"ok"},"index":0}]}\n\n' + b'data: {"id":"c2","choices":[{"delta":{},"finish_reason":"stop","index":0}]}\n\n' + b"data: [DONE]\n\n" +) +# An abnormally-terminated stream: content delta, then the upstream just stops — +# no finish_reason AND no [DONE] (what a mid-stream 429/drop looks like). +_STREAM_ABRUPT_END = b'data: {"id":"c3","choices":[{"delta":{"content":"ok"},"index":0}]}\n\n' + + +def _make_fake_gateway(payload: bytes) -> tuple[str, HTTPServer]: + class _Fake(BaseHTTPRequestHandler): + protocol_version = "HTTP/1.1" + + def do_POST(self): # noqa: N802 + self.send_response(200) + self.send_header("Content-Type", "text/event-stream") + self.send_header("Content-Length", str(len(payload))) + self.end_headers() + self.wfile.write(payload) + + def log_message(self, format: str, *args): + pass + + server = HTTPServer(("127.0.0.1", 0), _Fake) + threading.Thread(target=server.serve_forever, daemon=True).start() + return f"http://127.0.0.1:{server.server_address[1]}", server + + +def _post_through_proxy(upstream: str) -> str: + base, stop = _mlflow_proxy.start(upstream) + try: + req = urllib.request.Request( + f"{base}/ai-gateway/mlflow/v1/chat/completions", + data=b'{"model":"databricks-inkling","stream":true}', + headers={"Content-Type": "application/json"}, + method="POST", + ) + with urllib.request.urlopen(req, timeout=5) as resp: + return resp.read().decode("utf-8") + finally: + stop.set() + + +class TestFinishReasonInjection: + def test_injects_finish_reason_when_absent(self): + upstream, server = _make_fake_gateway(_STREAM_NO_FINISH) + try: + out = _post_through_proxy(upstream) + finally: + server.shutdown() + assert '"finish_reason": "stop"' in out or '"finish_reason":"stop"' in out + # The injected chunk must come before the terminal [DONE]. + assert out.index("finish_reason") < out.index("[DONE]") + assert out.rstrip().endswith("[DONE]") + + def test_passes_through_when_finish_reason_present(self): + upstream, server = _make_fake_gateway(_STREAM_WITH_FINISH) + try: + out = _post_through_proxy(upstream) + finally: + server.shutdown() + # Exactly one finish_reason — the proxy did not add a second. + assert out.count("finish_reason") == 1 + + def test_forwards_content_unchanged(self): + upstream, server = _make_fake_gateway(_STREAM_NO_FINISH) + try: + out = _post_through_proxy(upstream) + finally: + server.shutdown() + assert '"content":"ok"' in out + + def test_repairs_abrupt_end_with_finish_and_done(self): + # Upstream stopped after content with no finish_reason and no [DONE] + # (mid-stream 429/drop). Proxy must synthesize BOTH so a strict client + # doesn't error on a truncated stream. + upstream, server = _make_fake_gateway(_STREAM_ABRUPT_END) + try: + out = _post_through_proxy(upstream) + finally: + server.shutdown() + assert "finish_reason" in out + assert out.rstrip().endswith("[DONE]") + assert '"content":"ok"' in out + + +class TestFinishChunk: + def test_shape(self): + import json + + chunk = json.loads(_mlflow_proxy._finish_chunk("abc")) + assert chunk["id"] == "abc" + assert chunk["choices"][0]["finish_reason"] == "stop" + assert chunk["object"] == "chat.completion.chunk" + + +class TestStart: + def test_returns_local_base_url_and_stop_event(self): + upstream, server = _make_fake_gateway(_STREAM_NO_FINISH) + try: + base, stop = _mlflow_proxy.start(upstream) + assert base.startswith("http://127.0.0.1:") + assert isinstance(stop, threading.Event) + stop.set() + finally: + server.shutdown() diff --git a/uv.lock b/uv.lock index 15fd025..1e75a6c 100644 --- a/uv.lock +++ b/uv.lock @@ -602,7 +602,7 @@ wheels = [ [[package]] name = "ucode" -version = "0.1.0" +version = "0.2.0" source = { editable = "." } dependencies = [ { name = "databricks-sql-connector" },