From ea0e340e5104ff6aa337b7b2c5e01acd1937e397 Mon Sep 17 00:00:00 2001 From: Edgar Arakelyan Date: Thu, 19 Nov 2026 08:55:10 -0800 Subject: [PATCH 1/4] fix(agents): a model base URL may end in /v1 for every built-in agent and the direct judge LITELLM_BASE_URL reaches agents as configured. A client that appends its own versioned path now strips one trailing /v1 first: Claude Code's ANTHROPIC_BASE_URL, Codex's /openai/v1 route, Gemini's /gemini and /v1/chat/completions forwards, browser-use's Anthropic client, and the unit-test verifier's ANTHROPIC_BASE_URL, through core's api_root, which shares same_endpoint's rule. A base ending in /v1 no longer calls .../v1/v1/messages. The direct judge routes by what litellm makes of the model name: an Anthropic name gets the base without its /v1, an OpenAI one the URL as configured, and a name litellm can't place goes through the endpoint as openai/ instead of failing with "LLM Provider NOT provided". What it records keeps the name as given. Configs that already worked call the same paths as before. Co-Authored-By: Claude Opus 5.5 --- src/agent_env/agents/AGENTS.md | 4 +- .../agents/agents/browser-use/agent.py | 8 +- .../agents/claude-code/claude_bridge.py | 8 +- .../agents/agents/codex/codex_config.py | 8 +- .../agents/agents/gemini/a2a_server.py | 8 +- .../agents/agents/hermes/hermes_bridge.py | 2 +- .../agents/letta-code/letta_code_bridge.py | 2 +- .../agents/agents/opencode/opencode_bridge.py | 2 +- src/agent_env/config/model.py | 17 ++- .../task_steps/verifiers/rubrics_verifier.py | 25 +++- .../run_container_unit_tests_verifier.py | 4 +- tst/unit/agents/claude_code/test_cli_env.py | 8 ++ tst/unit/agents/codex/test_config_and_mcp.py | 15 +++ tst/unit/agents/gemini/test_litellm_proxy.py | 15 +++ tst/unit/config/test_model.py | 15 +++ .../task_step/local_container_urls_test.py | 4 +- tst/unit/task_step/test_model_key_scope.py | 8 +- .../test_rubrics_verifier_base_url.py | 116 ++++++++++++++++++ 18 files changed, 249 insertions(+), 20 deletions(-) create mode 100644 tst/unit/task_step/verifiers/test_rubrics_verifier_base_url.py diff --git a/src/agent_env/agents/AGENTS.md b/src/agent_env/agents/AGENTS.md index 68506afc1..78f9ecb06 100644 --- a/src/agent_env/agents/AGENTS.md +++ b/src/agent_env/agents/AGENTS.md @@ -229,7 +229,9 @@ Use your own URI, not `urn:agentenv:*`. Reference: `packages/agentenv-protocol/e `request.config.timeout_seconds` with `request.mcp_servers` connected, records the trajectory, and returns a `TaskResult` with a `session_ref`. Replace `claude_bridge.py` with your runtime's bridge. 4. **Map the model endpoint**: read `LITELLM_BASE_URL`, `LITELLM_API_KEY` and `LITELLM_EXTRA_HEADERS` and hand them - to your runtime, as `claude_bridge.py` `cli_env` does. + to your runtime, as `claude_bridge.py` `cli_env` does. `LITELLM_BASE_URL` comes as configured and may end in `/v1` + or not: a client that appends its own versioned path (Anthropic's `/v1/messages`) strips one trailing `/v1` first, + and one that wants the `/v1` root adds it unless it's there. 5. **Add extensions** (Extensions): `mcp-config` and `trajectory` first, then every other one your runtime can honour. Keep `AGENT_CARD = app.state.agentenv_a2a.card` at the end of `a2a_server.py`. 6. **Test it offline** with Starlette's `TestClient` and your runtime faked, as `tst/unit/agents/claude_code/` does: diff --git a/src/agent_env/agents/agents/browser-use/agent.py b/src/agent_env/agents/agents/browser-use/agent.py index 80b4dfa43..ac9127197 100644 --- a/src/agent_env/agents/agents/browser-use/agent.py +++ b/src/agent_env/agents/agents/browser-use/agent.py @@ -38,12 +38,18 @@ _CHECKPOINT = TypeAdapter(ModelRequest | None) +def _api_root(url: str) -> str: + """``LITELLM_BASE_URL`` without a trailing ``/`` and one trailing ``/v1``: it may be configured either way, and + Anthropic's client appends its own ``/v1/messages``.""" + return url.strip().rstrip("/").removesuffix("/v1").rstrip("/") + + def model(turn: dict): """The turn's model on agent-env's endpoint, with the endpoint's headers.""" name, base_url, key = turn["model"], os.environ["LITELLM_BASE_URL"], os.environ["LITELLM_API_KEY"] headers = json.loads(os.environ.get("LITELLM_EXTRA_HEADERS") or "{}") if "claude" in name.lower(): - client = AsyncAnthropic(auth_token=key, base_url=base_url, default_headers=headers) + client = AsyncAnthropic(auth_token=key, base_url=_api_root(base_url), default_headers=headers) return AnthropicModel(name, provider=AnthropicProvider(anthropic_client=client), profile=anthropic_model_profile(name.rsplit("/", 1)[-1])) client = AsyncOpenAI(api_key=key, base_url=base_url, default_headers=headers) diff --git a/src/agent_env/agents/agents/claude-code/claude_bridge.py b/src/agent_env/agents/agents/claude-code/claude_bridge.py index e1a30956d..229c88256 100644 --- a/src/agent_env/agents/agents/claude-code/claude_bridge.py +++ b/src/agent_env/agents/agents/claude-code/claude_bridge.py @@ -177,6 +177,12 @@ def _model_headers(raw: str | None) -> dict[str, str]: #: Set by the image (root.conf) where the egress proxy runs: the CLI and the model's commands go out through it #: while the agent follows an armed clock. +def _api_root(url: str) -> str: + """``LITELLM_BASE_URL`` without a trailing ``/`` and one trailing ``/v1``: it may be configured either way, and the + CLI appends its own ``/v1/messages``.""" + return url.strip().rstrip("/").removesuffix("/v1").rstrip("/") + + EGRESS_PROXY_ENV = "AGENTENV_EGRESS_PROXY" #: The clock page agent-env's libfaketime reads the CLI's time from (clock/clockpage.py, version 2): it keeps the last #: armed clock while the follower reconnects or restarts, which the follower's status doesn't say. @@ -251,7 +257,7 @@ def cli_env(environ: Mapping[str, str], *, model: str | None = None, project_id: if not env.get("ANTHROPIC_AUTH_TOKEN") and env.get("LITELLM_API_KEY"): env["ANTHROPIC_AUTH_TOKEN"] = env["LITELLM_API_KEY"] if not env.get("ANTHROPIC_BASE_URL") and env.get("LITELLM_BASE_URL"): - env["ANTHROPIC_BASE_URL"] = env["LITELLM_BASE_URL"] + env["ANTHROPIC_BASE_URL"] = _api_root(env["LITELLM_BASE_URL"]) if env.get("ANTHROPIC_AUTH_TOKEN"): env["ANTHROPIC_API_KEY"] = "" if model and env.get("ANTHROPIC_BASE_URL"): diff --git a/src/agent_env/agents/agents/codex/codex_config.py b/src/agent_env/agents/agents/codex/codex_config.py index ebb78a100..7622c09b7 100644 --- a/src/agent_env/agents/agents/codex/codex_config.py +++ b/src/agent_env/agents/agents/codex/codex_config.py @@ -194,6 +194,12 @@ def model_headers(raw: str | None) -> dict[str, str]: return {str(name): str(value) for name, value in headers.items()} +def _api_root(url: str) -> str: + """``LITELLM_BASE_URL`` without a trailing ``/`` and one trailing ``/v1``: it may be configured either way, and + codex is given LiteLLM's ``/openai/v1`` route under it.""" + return url.strip().rstrip("/").removesuffix("/v1").rstrip("/") + + def write_base_config(environ: Mapping[str, str]) -> None: """The settings every turn shares: the collaboration tools, and the model endpoint agent-env configured as a Responses API provider, with its headers and the run's attribution on each call.""" @@ -206,7 +212,7 @@ def write_base_config(environ: Mapping[str, str]) -> None: cfg["model_provider"] = "litellm" provider = { "name": "LiteLLM proxy", - "base_url": base_url.rstrip("/") + "/openai/v1", + "base_url": _api_root(base_url) + "/openai/v1", "env_key": "LITELLM_API_KEY", "wire_api": "responses", # Bound codex retry/idle behavior so a hung stream fails fast diff --git a/src/agent_env/agents/agents/gemini/a2a_server.py b/src/agent_env/agents/agents/gemini/a2a_server.py index fe4660edd..a6481b406 100644 --- a/src/agent_env/agents/agents/gemini/a2a_server.py +++ b/src/agent_env/agents/agents/gemini/a2a_server.py @@ -144,6 +144,12 @@ _LITELLM_ROUTE_HEADERS = ("x-litellm-model-id", "x-litellm-model-api-base", "x-litellm-model-group") +def _api_root(url: str) -> str: + """``LITELLM_BASE_URL`` without a trailing ``/`` and one trailing ``/v1``: it may be configured either way, and the + proxy appends LiteLLM's ``/gemini/...`` and ``/v1/chat/completions`` routes to it.""" + return url.strip().rstrip("/").removesuffix("/v1").rstrip("/") + + def _describe_transport_error(exc: Exception) -> str: """Name the exception class: many httpx errors (ReadTimeout, RemoteProtocolError) stringify to ''.""" text = str(exc) @@ -747,7 +753,7 @@ async def litellm_gemini_proxy(self, request: Request) -> Response: if route is None: raise HTTPException(status_code=404, detail="no running turn has this proxy route") rest = request.path_params["rest"] - base = (os.environ.get("LITELLM_BASE_URL") or "").rstrip("/") + base = _api_root(os.environ.get("LITELLM_BASE_URL") or "") if not base: raise HTTPException(status_code=503, detail="LITELLM_BASE_URL not set; cannot forward gemini model calls") # Provider-prefixed route models (vertex_ai/...) can't traverse the /gemini diff --git a/src/agent_env/agents/agents/hermes/hermes_bridge.py b/src/agent_env/agents/agents/hermes/hermes_bridge.py index c9c0c4b4d..acc5cd9ae 100644 --- a/src/agent_env/agents/agents/hermes/hermes_bridge.py +++ b/src/agent_env/agents/agents/hermes/hermes_bridge.py @@ -52,7 +52,7 @@ def _base_url(environ: Mapping[str, str]) -> str: - """The endpoint's chat-completions root: agent-env's LITELLM_BASE_URL has no /v1 suffix.""" + """The endpoint's chat-completions root: LITELLM_BASE_URL may end in /v1 or not.""" base = (environ.get("LITELLM_BASE_URL") or "").rstrip("/") return base if base.endswith("/v1") else f"{base}/v1" diff --git a/src/agent_env/agents/agents/letta-code/letta_code_bridge.py b/src/agent_env/agents/agents/letta-code/letta_code_bridge.py index e6aa5e174..f6c7b4ed9 100644 --- a/src/agent_env/agents/agents/letta-code/letta_code_bridge.py +++ b/src/agent_env/agents/agents/letta-code/letta_code_bridge.py @@ -45,7 +45,7 @@ def _base_url(environ: Mapping[str, str]) -> str: - """The endpoint's chat-completions root: agent-env's LITELLM_BASE_URL has no /v1 suffix.""" + """The endpoint's chat-completions root: LITELLM_BASE_URL may end in /v1 or not.""" base = (environ.get("LITELLM_BASE_URL") or "").rstrip("/") return base if base.endswith("/v1") else f"{base}/v1" diff --git a/src/agent_env/agents/agents/opencode/opencode_bridge.py b/src/agent_env/agents/agents/opencode/opencode_bridge.py index 247ca21b4..10f9b3c1d 100644 --- a/src/agent_env/agents/agents/opencode/opencode_bridge.py +++ b/src/agent_env/agents/agents/opencode/opencode_bridge.py @@ -75,7 +75,7 @@ def _one_of(self) -> OpenCodeTools: def _base_url(environ: Mapping[str, str]) -> str: - """The endpoint's chat-completions root: agent-env's LITELLM_BASE_URL has no /v1 suffix.""" + """The endpoint's chat-completions root: LITELLM_BASE_URL may end in /v1 or not.""" base = (environ.get("LITELLM_BASE_URL") or "").rstrip("/") return base if base.endswith("/v1") else f"{base}/v1" diff --git a/src/agent_env/config/model.py b/src/agent_env/config/model.py index 8d851957c..60106d4f5 100644 --- a/src/agent_env/config/model.py +++ b/src/agent_env/config/model.py @@ -10,7 +10,7 @@ from dataclasses import dataclass, field, fields from enum import StrEnum from typing import Optional -from urllib.parse import urlsplit +from urllib.parse import urlsplit, urlunsplit from agent_env.config.errors import ConfigError from agent_env.config.loader import ENV_REF_PREFIX, SECRET_REF_PREFIX, SecretResolver, interpolate @@ -284,6 +284,18 @@ def same_endpoint(a: Optional[str], b: Optional[str]) -> bool: return bool(a and b) and _endpoint(a) == _endpoint(b) +def api_root(url: str) -> str: + """``url`` without a trailing ``/`` and one trailing ``/v1``, for a client that appends its own versioned path + (Anthropic's ``/v1/messages``): the endpoint may be configured either way, and ``same_endpoint`` compares by the + same root.""" + parts = urlsplit(url.strip()) + return urlunsplit(parts._replace(path=_root_path(parts.path))) + + +def _root_path(path: str) -> str: + return path.rstrip("/").removesuffix("/v1").rstrip("/") + + def _endpoint(url: str) -> tuple: parts = urlsplit(url.strip()) scheme = parts.scheme.lower() @@ -291,8 +303,7 @@ def _endpoint(url: str) -> tuple: port = parts.port except ValueError: return (url,) - path = parts.path.rstrip("/") - path = path.removesuffix("/v1").rstrip("/") + path = _root_path(parts.path) return scheme, (parts.hostname or "").lower(), None if port == _DEFAULT_PORTS.get(scheme) else port, path, parts.query diff --git a/src/agent_env/task_step/task_steps/verifiers/rubrics_verifier.py b/src/agent_env/task_step/task_steps/verifiers/rubrics_verifier.py index ce2d04769..a7f482b14 100644 --- a/src/agent_env/task_step/task_steps/verifiers/rubrics_verifier.py +++ b/src/agent_env/task_step/task_steps/verifiers/rubrics_verifier.py @@ -3,11 +3,13 @@ from __future__ import annotations import asyncio +import contextlib +import io import json import logging import re import uuid -from collections.abc import Mapping, Sequence +from collections.abc import Callable, Mapping, Sequence from dataclasses import dataclass, field, replace from typing import Any, ClassVar, Optional @@ -20,7 +22,7 @@ from agent_env.a2a_agent.staging import transfer_store from agent_env.config import get_config from agent_env.config.errors import ConfigError -from agent_env.config.model import ModelParam, ModelRole, no_key_for +from agent_env.config.model import ModelParam, ModelRole, api_root, no_key_for from agent_env.providers.sandbox_providers.sandbox import into_container from agent_env.task_step.context import ( PromptResponse, @@ -281,6 +283,21 @@ def _caches_prompts(model: str) -> bool: return "claude" in model.lower() +def _judge_route(model: str, api_base: Optional[str], provider_of: Callable[[str], Any]) -> tuple[str, Optional[str]]: + """The model name and base a direct judge call hands litellm; what it records stays ``model``. Through an + endpoint, a name litellm can't place goes as ``openai/``, and an Anthropic one, whose client appends + ``/v1/messages``, gets the base without a trailing ``/v1``. ``provider_of`` is litellm's ``get_llm_provider``, + which prints a banner before refusing a name it can't place.""" + if api_base is None: + return model, None + try: + with contextlib.redirect_stdout(io.StringIO()): + provider = provider_of(model)[1] + except Exception: + return f"openai/{model}", api_base + return model, api_root(api_base) if provider == "anthropic" else api_base + + def _usage_and_cost(response: Any) -> dict: """A LiteLLM response's token counts and LiteLLM's price of it in USD, each None when it has none.""" usage = getattr(response, "usage", None) @@ -1389,6 +1406,8 @@ async def _prompt_llm_judge( if base and not api_key and not config.is_model_endpoint(base): raise ConfigError(f"{self.type} '{self.id}': {no_key_for(base, _JUDGE_KEY_HOW)}") call_config = config.resolve_model_call(model, default_api_key=api_key, base_override=base) + route_model, route_base = _judge_route(model, call_config.api_base, litellm.get_llm_provider) + call_config = replace(call_config, api_base=route_base) text_blocks: list[dict] = [{"type": "text", "text": eval_prompt}] cached = bool(cached_prefix) and eval_prompt.startswith(cached_prefix or "") and _caches_prompts(model) if cached: @@ -1414,7 +1433,7 @@ async def _prompt_llm_judge( for attempt in range(1, self.DEFAULT_MAX_RETRIES + 1): try: request_kwargs: dict = dict( - model=model, + model=route_model, messages=messages, timeout=self.judge_timeout_seconds, **call_config.client_kwargs(), diff --git a/src/agent_env/task_step/task_steps/verifiers/run_container_unit_tests_verifier.py b/src/agent_env/task_step/task_steps/verifiers/run_container_unit_tests_verifier.py index 9ebf72091..4f0cdd36e 100644 --- a/src/agent_env/task_step/task_steps/verifiers/run_container_unit_tests_verifier.py +++ b/src/agent_env/task_step/task_steps/verifiers/run_container_unit_tests_verifier.py @@ -39,7 +39,7 @@ from typing import Any, ClassVar, Optional from agent_env.config import ConfigError, get_config -from agent_env.config.model import MODEL_HEADERS_ENV, no_key_for +from agent_env.config.model import MODEL_HEADERS_ENV, api_root, no_key_for from agent_env.entity.refs import EntityRef from agent_env.env.env import require_sandbox from agent_env.env.envs.computer.utils import inside @@ -376,7 +376,7 @@ def _merged_env(self, config, sandbox_type: Optional[str], extra_env: dict[str, try: litellm_url = config.get_litellm_base_url() merged_env["LITELLM_BASE_URL"] = litellm_url - merged_env["ANTHROPIC_BASE_URL"] = litellm_url + merged_env["ANTHROPIC_BASE_URL"] = api_root(litellm_url) except (KeyError, ConfigError): logger.warning("LITELLM_BASE_URL unavailable; verifier won't have LLM base URL") merged_env.update(all_sandbox_container_env()) diff --git a/tst/unit/agents/claude_code/test_cli_env.py b/tst/unit/agents/claude_code/test_cli_env.py index c76b7d48d..9465beecf 100644 --- a/tst/unit/agents/claude_code/test_cli_env.py +++ b/tst/unit/agents/claude_code/test_cli_env.py @@ -26,6 +26,14 @@ def test_the_configured_endpoint_is_the_clis_with_its_key_as_a_bearer_token(): assert env["CLAUDE_CODE_ATTRIBUTION_HEADER"] == "0" +@pytest.mark.parametrize("base", ["https://gateway.example.com/v1", "https://gateway.example.com/v1/", + "https://gateway.example.com/"]) +def test_the_endpoint_may_end_in_v1_since_the_cli_appends_its_own(base): + env = claude_bridge.cli_env({**GATEWAY, "LITELLM_BASE_URL": base}) + + assert (env["ANTHROPIC_BASE_URL"], env["LITELLM_BASE_URL"]) == ("https://gateway.example.com", base) + + def test_a_script_the_model_runs_can_call_the_endpoint_with_the_litellm_variables(): headers = json.dumps({"X-Bill-To": "team-a"}) diff --git a/tst/unit/agents/codex/test_config_and_mcp.py b/tst/unit/agents/codex/test_config_and_mcp.py index 8045c0d5c..ded20a4cc 100644 --- a/tst/unit/agents/codex/test_config_and_mcp.py +++ b/tst/unit/agents/codex/test_config_and_mcp.py @@ -10,6 +10,7 @@ from pathlib import Path import httpx +import pytest from tst.unit.agents.codex.agent import a2a_server, codex_bridge @@ -271,3 +272,17 @@ def test_triggers_register_decide_and_state(codex): decided = _call(codex, "POST", "/ext/triggers/decide", {"turn": 1, "solver_message": "x"}).json() assert (decided["parts"], decided["done"], decided["fired"]) == ([{"kind": "text", "text": "hi"}], False, ["hello"]) assert _call(codex, "GET", "/ext/triggers").json()["firing_log"] + + +@pytest.mark.parametrize("base", ["https://llm.example.com/v1", "https://llm.example.com/v1/"]) +def test_an_endpoint_ending_in_v1_gets_litellms_openai_route_once(codex, monkeypatch, base): + monkeypatch.setenv("LITELLM_BASE_URL", base) + + async def exercise(): + async with codex.app.router.lifespan_context(codex.app): + pass + + asyncio.run(exercise()) + + config = tomllib.loads((Path(os.environ["CODEX_HOME"]) / "config.toml").read_text()) + assert config["model_providers"]["litellm"]["base_url"] == "https://llm.example.com/openai/v1" diff --git a/tst/unit/agents/gemini/test_litellm_proxy.py b/tst/unit/agents/gemini/test_litellm_proxy.py index 4d6969f4d..a2ed2dafc 100644 --- a/tst/unit/agents/gemini/test_litellm_proxy.py +++ b/tst/unit/agents/gemini/test_litellm_proxy.py @@ -322,3 +322,18 @@ def test_a_message_level_signature_goes_only_to_the_first_parallel_call(vertex, parts = client.post(GENERATE, json=_GREQ).json()["candidates"][0]["content"]["parts"] assert [p.get("thoughtSignature") for p in parts] == ["sig-a", None] + + +def test_an_endpoint_ending_in_v1_gets_litellms_routes_once(upstream, agent, client, monkeypatch): + monkeypatch.setenv("LITELLM_BASE_URL", BASE + "/v1/") + upstream.response = httpx.Response(200, json={"candidates": []}) + _route(agent, model="gemini-2.5-flash", project_id="p-1") + client.post(GENERATE + "?alt=json", content=b'{"contents":[]}', headers={"content-type": "application/json"}) + upstream.response = httpx.Response(200, json={ + "model": "gemini-3.1-pro-preview", "choices": [{"message": {"content": "hi"}, "finish_reason": "stop"}], + "usage": {"prompt_tokens": 3, "completion_tokens": 1, "total_tokens": 4}}) + _route(agent, model="vertex_ai/global/gemini-3.1-pro-preview", project_id="p-1") + client.post(GENERATE, json=_GREQ) + + assert [str(request.url) for request in upstream.requests] == [ + f"{BASE}/gemini/v1beta/models/gemini-3.1-pro-preview:generateContent?alt=json", f"{BASE}/v1/chat/completions"] diff --git a/tst/unit/config/test_model.py b/tst/unit/config/test_model.py index 880b68263..df72eef8c 100644 --- a/tst/unit/config/test_model.py +++ b/tst/unit/config/test_model.py @@ -12,6 +12,7 @@ ModelRole, ModelSource, ResolvedModel, + api_root, config_model, named_model, resolve_model, @@ -345,6 +346,20 @@ def test_same_endpoint_reads_one_endpoint_spelled_two_ways_as_one(a, b): assert same_endpoint(a, b) and same_endpoint(b, a) +@pytest.mark.parametrize("url, root", [ + ("https://gw.example/v1", "https://gw.example"), + ("https://gw.example/v1/", "https://gw.example"), + ("https://gw.example/", "https://gw.example"), + ("https://gw.example", "https://gw.example"), + ("https://gw.example/litellm/v1", "https://gw.example/litellm"), + ("https://gw.example/v1/v1", "https://gw.example/v1"), + ("http://localhost:4000/v1?team=a", "http://localhost:4000?team=a"), + ("HTTPS://GW.Example/v1", "https://GW.Example"), +], ids=["v1", "v1-slash", "slash", "bare", "path-prefix", "one-v1-only", "port-and-query", "case-kept"]) +def test_api_root_drops_a_trailing_slash_and_one_trailing_v1(url, root): + assert api_root(url) == root + + @pytest.mark.parametrize("a, b", [ ("https://gw.example", "http://gw.example"), ("https://gw.example", "https://gw.example:8443"), diff --git a/tst/unit/task_step/local_container_urls_test.py b/tst/unit/task_step/local_container_urls_test.py index f732f8cde..f054a3631 100644 --- a/tst/unit/task_step/local_container_urls_test.py +++ b/tst/unit/task_step/local_container_urls_test.py @@ -186,7 +186,9 @@ def test_a_unit_test_verifier_gets_the_model_endpoint_its_container_reaches( ): run = _run_verifier(monkeypatch, sandbox_type) - assert f"-e LITELLM_BASE_URL={expected} " in run and f"-e ANTHROPIC_BASE_URL={expected} " in run + # Anthropic's clients append their own /v1/messages, so their variable is the endpoint without its /v1. + anthropic = expected.removesuffix("/v1") + assert f"-e LITELLM_BASE_URL={expected} " in run and f"-e ANTHROPIC_BASE_URL={anthropic} " in run def test_a_model_endpoint_the_verifier_sets_itself_is_reached_through_the_host(monkeypatch, local_stores, model_endpoint): diff --git a/tst/unit/task_step/test_model_key_scope.py b/tst/unit/task_step/test_model_key_scope.py index c9b14b132..5de3d8f6b 100644 --- a/tst/unit/task_step/test_model_key_scope.py +++ b/tst/unit/task_step/test_model_key_scope.py @@ -27,6 +27,8 @@ ) MODEL = "http://localhost:4000/v1" +# Anthropic's clients append their own /v1/messages, so their variable is the endpoint without its /v1. +MODEL_ROOT = "http://localhost:4000" HEADERS = {"X-Bill-To": "acme"} ELSEWHERE = "http://127.0.0.1:5001" REFUSED = "refused" @@ -188,11 +190,11 @@ def _verifier_env(monkeypatch, env_vars: dict, step_params: dict | None = None) VERIFIER_CASES = { "configured": ({}, None, {"LITELLM_API_KEY": "k", "LITELLM_BASE_URL": MODEL, "ANTHROPIC_API_KEY": "k", - "ANTHROPIC_BASE_URL": MODEL}), + "ANTHROPIC_BASE_URL": MODEL_ROOT}), "elsewhere-is-refused": ({"LITELLM_BASE_URL": ELSEWHERE}, None, REFUSED), "elsewhere-with-its-key": ({"LITELLM_BASE_URL": ELSEWHERE, "LITELLM_API_KEY": "theirs"}, None, {"LITELLM_API_KEY": "theirs", "LITELLM_BASE_URL": ELSEWHERE, "ANTHROPIC_API_KEY": "k", - "ANTHROPIC_BASE_URL": MODEL}), + "ANTHROPIC_BASE_URL": MODEL_ROOT}), "anthropic-elsewhere-is-refused": ({"ANTHROPIC_BASE_URL": ELSEWHERE, "LITELLM_API_KEY": "theirs"}, None, REFUSED), "anthropic-elsewhere-with-its-key": ({"ANTHROPIC_BASE_URL": ELSEWHERE, "ANTHROPIC_API_KEY": "theirs"}, None, {"LITELLM_API_KEY": "k", "LITELLM_BASE_URL": MODEL, @@ -200,7 +202,7 @@ def _verifier_env(monkeypatch, env_vars: dict, step_params: dict | None = None) "elsewhere-by-step-params-is-refused": ({}, {"LITELLM_BASE_URL": ELSEWHERE}, REFUSED), "elsewhere-by-step-params-with-the-steps-key": ({"LITELLM_API_KEY": "theirs"}, {"LITELLM_BASE_URL": ELSEWHERE}, {"LITELLM_API_KEY": "theirs", "LITELLM_BASE_URL": ELSEWHERE, - "ANTHROPIC_API_KEY": "k", "ANTHROPIC_BASE_URL": MODEL}), + "ANTHROPIC_API_KEY": "k", "ANTHROPIC_BASE_URL": MODEL_ROOT}), } diff --git a/tst/unit/task_step/verifiers/test_rubrics_verifier_base_url.py b/tst/unit/task_step/verifiers/test_rubrics_verifier_base_url.py new file mode 100644 index 000000000..1aae15990 --- /dev/null +++ b/tst/unit/task_step/verifiers/test_rubrics_verifier_base_url.py @@ -0,0 +1,116 @@ +"""The direct judge reaches the configured endpoint whether its base URL ends in ``/v1`` or not, and with a model name +litellm can't place: the real litellm calls a loopback server that records each call's path and model name. A +Claude name keeps Anthropic's Messages route, its ``/v1`` not doubled; an OpenAI name gets the URL as configured; a +name litellm can't place goes through the endpoint as ``openai/``. The rows without ``/v1`` on Claude, and every +OpenAI row, called these same paths before.""" +from __future__ import annotations + +import json +import threading +from http.server import BaseHTTPRequestHandler, ThreadingHTTPServer + +import pytest +from pytest_socket import enable_socket + +import agent_env.config as cfg_mod +from agent_env.config.model import ModelConfig +from agent_env.task_step.context import TaskStepContext +from agent_env.task_step.task_steps.verifiers.rubrics_verifier import RubricsVerifierTaskStep, _judge_route + +_VERDICT = json.dumps({"results": [{"criterion_id": "c1", "passed": True}]}) + + +class _Endpoint(BaseHTTPRequestHandler): + calls: list = [] + + def log_message(self, *args) -> None: + pass + + def do_POST(self) -> None: + body = json.loads(self.rfile.read(int(self.headers.get("content-length") or 0)) or b"{}") + self.calls.append((self.path, body.get("model"))) + if self.path.endswith("/messages"): + answer = {"id": "m", "type": "message", "role": "assistant", "model": body.get("model"), + "content": [{"type": "text", "text": _VERDICT}], "stop_reason": "end_turn", + "usage": {"input_tokens": 1, "output_tokens": 1}} + else: + answer = {"id": "c", "object": "chat.completion", "created": 0, "model": body.get("model"), + "choices": [{"index": 0, "message": {"role": "assistant", "content": _VERDICT}, + "finish_reason": "stop"}], + "usage": {"prompt_tokens": 1, "completion_tokens": 1, "total_tokens": 2}} + data = json.dumps(answer).encode() + self.send_response(200 if self.path.endswith(("/messages", "/chat/completions")) else 404) + self.send_header("content-type", "application/json") + self.send_header("content-length", str(len(data))) + self.end_headers() + self.wfile.write(data) + + +@pytest.fixture(autouse=True) +def _disable_network(): + """Override the suite-wide socket block (tst/unit/conftest.py) for this module: the endpoint is on loopback.""" + enable_socket() + yield + + +@pytest.fixture +def endpoint(): + _Endpoint.calls = [] + server = ThreadingHTTPServer(("127.0.0.1", 0), _Endpoint) + threading.Thread(target=server.serve_forever, daemon=True).start() + try: + yield f"http://127.0.0.1:{server.server_address[1]}" + finally: + server.shutdown() + server.server_close() + + +def _judge(monkeypatch, base_url: str) -> RubricsVerifierTaskStep: + config = cfg_mod.Config() + monkeypatch.setattr(config, "_model_cfg", ModelConfig(base_url=base_url, api_key="k")) + monkeypatch.setattr(cfg_mod, "get_config", lambda: config) + monkeypatch.delenv("LITELLM_BASE_URL", raising=False) + monkeypatch.delenv("LITELLM_API_KEY", raising=False) + return RubricsVerifierTaskStep(id="v", version=1, criteria=[], prompt_id="p", use_agent_judge=False, + verifier_id="vt") + + +@pytest.mark.asyncio +@pytest.mark.parametrize("model, suffix, path, sent", [ + ("claude-sonnet-4-6", "", "/v1/messages", "claude-sonnet-4-6"), + ("claude-sonnet-4-6", "/v1", "/v1/messages", "claude-sonnet-4-6"), + ("claude-sonnet-4-6", "/v1/", "/v1/messages", "claude-sonnet-4-6"), + ("anthropic/claude-sonnet-4-6", "/v1", "/v1/messages", "claude-sonnet-4-6"), + ("gpt-4.1-mini", "", "/chat/completions", "gpt-4.1-mini"), + ("gpt-4.1-mini", "/v1", "/v1/chat/completions", "gpt-4.1-mini"), + ("openai/qwen3:8b", "/v1", "/v1/chat/completions", "qwen3:8b"), + ("qwen3:8b", "", "/chat/completions", "qwen3:8b"), + ("qwen3:8b", "/v1", "/v1/chat/completions", "qwen3:8b"), + ("Qwen/Qwen3-8B", "/v1", "/v1/chat/completions", "Qwen/Qwen3-8B"), +]) +async def test_the_direct_judge_reaches_the_endpoint_however_its_base_url_ends( + monkeypatch, endpoint, model, suffix, path, sent, +): + judge = _judge(monkeypatch, endpoint + suffix) + + out = await judge._prompt_llm_judge("grade it", model=model, context=TaskStepContext()) + + assert _Endpoint.calls == [(path, sent)] + assert json.loads(out["response"]) == json.loads(_VERDICT) + + +def _provider(name: str): + providers = {"claude-sonnet-4-6": "anthropic", "gpt-4.1-mini": "openai"} + if name not in providers: + print("Provider List") + raise ValueError(name) + return name, providers[name], None, None + + +def test_the_route_keeps_a_native_name_and_sends_an_unknown_one_as_openai(capsys): + assert _judge_route("claude-sonnet-4-6", "https://gw.example/v1", _provider) == ( + "claude-sonnet-4-6", "https://gw.example") + assert _judge_route("gpt-4.1-mini", "https://gw.example/v1", _provider) == ("gpt-4.1-mini", "https://gw.example/v1") + assert _judge_route("qwen3:8b", "https://gw.example", _provider) == ("openai/qwen3:8b", "https://gw.example") + assert _judge_route("qwen3:8b", None, _provider) == ("qwen3:8b", None) + assert capsys.readouterr().out == "" From ff075a9195166cd7f814e4b5072d5cd8abbaec2c Mon Sep 17 00:00:00 2001 From: Edgar Arakelyan Date: Thu, 19 Nov 2026 09:19:56 -0800 Subject: [PATCH 2/4] fix(verifiers): the direct judge's route follows a custom_llm_provider in its params [model] params can name litellm's provider outright; the judge now decides whether to drop the base's /v1 by that provider, as litellm does, so a Claude name sent through the OpenAI route keeps its /v1/chat/completions. Co-Authored-By: Claude Opus 5.5 --- .../task_steps/verifiers/rubrics_verifier.py | 13 ++++++----- .../test_rubrics_verifier_base_url.py | 22 ++++++++++++++++--- 2 files changed, 27 insertions(+), 8 deletions(-) diff --git a/src/agent_env/task_step/task_steps/verifiers/rubrics_verifier.py b/src/agent_env/task_step/task_steps/verifiers/rubrics_verifier.py index a7f482b14..7bf0b161c 100644 --- a/src/agent_env/task_step/task_steps/verifiers/rubrics_verifier.py +++ b/src/agent_env/task_step/task_steps/verifiers/rubrics_verifier.py @@ -283,16 +283,18 @@ def _caches_prompts(model: str) -> bool: return "claude" in model.lower() -def _judge_route(model: str, api_base: Optional[str], provider_of: Callable[[str], Any]) -> tuple[str, Optional[str]]: +def _judge_route(model: str, api_base: Optional[str], provider_of: Callable[..., Any], + custom_provider: Optional[str] = None) -> tuple[str, Optional[str]]: """The model name and base a direct judge call hands litellm; what it records stays ``model``. Through an endpoint, a name litellm can't place goes as ``openai/``, and an Anthropic one, whose client appends - ``/v1/messages``, gets the base without a trailing ``/v1``. ``provider_of`` is litellm's ``get_llm_provider``, - which prints a banner before refusing a name it can't place.""" + ``/v1/messages``, gets the base without a trailing ``/v1``. A ``custom_llm_provider`` among the call's params + decides the provider as it does for litellm. ``provider_of`` is litellm's ``get_llm_provider``, which prints a + banner before refusing a name it can't place.""" if api_base is None: return model, None try: with contextlib.redirect_stdout(io.StringIO()): - provider = provider_of(model)[1] + provider = provider_of(model, custom_llm_provider=custom_provider)[1] except Exception: return f"openai/{model}", api_base return model, api_root(api_base) if provider == "anthropic" else api_base @@ -1406,7 +1408,8 @@ async def _prompt_llm_judge( if base and not api_key and not config.is_model_endpoint(base): raise ConfigError(f"{self.type} '{self.id}': {no_key_for(base, _JUDGE_KEY_HOW)}") call_config = config.resolve_model_call(model, default_api_key=api_key, base_override=base) - route_model, route_base = _judge_route(model, call_config.api_base, litellm.get_llm_provider) + route_model, route_base = _judge_route(model, call_config.api_base, litellm.get_llm_provider, + call_config.params.get("custom_llm_provider")) call_config = replace(call_config, api_base=route_base) text_blocks: list[dict] = [{"type": "text", "text": eval_prompt}] cached = bool(cached_prefix) and eval_prompt.startswith(cached_prefix or "") and _caches_prompts(model) diff --git a/tst/unit/task_step/verifiers/test_rubrics_verifier_base_url.py b/tst/unit/task_step/verifiers/test_rubrics_verifier_base_url.py index 1aae15990..a1ed4f06d 100644 --- a/tst/unit/task_step/verifiers/test_rubrics_verifier_base_url.py +++ b/tst/unit/task_step/verifiers/test_rubrics_verifier_base_url.py @@ -65,9 +65,9 @@ def endpoint(): server.server_close() -def _judge(monkeypatch, base_url: str) -> RubricsVerifierTaskStep: +def _judge(monkeypatch, base_url: str, params: dict | None = None) -> RubricsVerifierTaskStep: config = cfg_mod.Config() - monkeypatch.setattr(config, "_model_cfg", ModelConfig(base_url=base_url, api_key="k")) + monkeypatch.setattr(config, "_model_cfg", ModelConfig(base_url=base_url, api_key="k", params=params or {})) monkeypatch.setattr(cfg_mod, "get_config", lambda: config) monkeypatch.delenv("LITELLM_BASE_URL", raising=False) monkeypatch.delenv("LITELLM_API_KEY", raising=False) @@ -99,8 +99,21 @@ async def test_the_direct_judge_reaches_the_endpoint_however_its_base_url_ends( assert json.loads(out["response"]) == json.loads(_VERDICT) -def _provider(name: str): +@pytest.mark.asyncio +async def test_a_judge_whose_params_name_its_provider_keeps_that_providers_url(monkeypatch, endpoint): + """``custom_llm_provider = "openai"`` in ``[model] params`` sends even a Claude name through the OpenAI route, + whose base keeps its ``/v1``.""" + judge = _judge(monkeypatch, endpoint + "/v1", params={"custom_llm_provider": "openai"}) + + await judge._prompt_llm_judge("grade it", model="claude-sonnet-4-6", context=TaskStepContext()) + + assert _Endpoint.calls == [("/v1/chat/completions", "claude-sonnet-4-6")] + + +def _provider(name: str, custom_llm_provider: str | None = None): providers = {"claude-sonnet-4-6": "anthropic", "gpt-4.1-mini": "openai"} + if custom_llm_provider: + return name, custom_llm_provider, None, None if name not in providers: print("Provider List") raise ValueError(name) @@ -113,4 +126,7 @@ def test_the_route_keeps_a_native_name_and_sends_an_unknown_one_as_openai(capsys assert _judge_route("gpt-4.1-mini", "https://gw.example/v1", _provider) == ("gpt-4.1-mini", "https://gw.example/v1") assert _judge_route("qwen3:8b", "https://gw.example", _provider) == ("openai/qwen3:8b", "https://gw.example") assert _judge_route("qwen3:8b", None, _provider) == ("qwen3:8b", None) + assert _judge_route("claude-sonnet-4-6", "https://gw.example/v1", _provider, "openai") == ( + "claude-sonnet-4-6", "https://gw.example/v1") + assert _judge_route("qwen3:8b", "https://gw.example", _provider, "hosted_vllm") == ("qwen3:8b", "https://gw.example") assert capsys.readouterr().out == "" From 5ca810453393d6313bc173767ac0ff884796c521 Mon Sep 17 00:00:00 2001 From: Edgar Arakelyan Date: Thu, 19 Nov 2026 10:26:38 -0800 Subject: [PATCH 3/4] fix(verifiers): the direct judge sends a bare name litellm would route to another provider's API through the endpoint A bare name litellm places with a provider other than Anthropic or OpenAI, such as gemini-* with Vertex or command-* with Cohere, made the judge call that provider's own API at the endpoint's base, which a LiteLLM proxy doesn't serve. Through an endpoint it now goes as openai/, as a name litellm can't place does. A name with a provider prefix, or a custom_llm_provider in the params, keeps the route it names. Co-Authored-By: Claude Opus 5.5 --- .../task_steps/verifiers/rubrics_verifier.py | 16 +++++++++---- .../test_rubrics_verifier_base_url.py | 24 ++++++++++++++----- 2 files changed, 29 insertions(+), 11 deletions(-) diff --git a/src/agent_env/task_step/task_steps/verifiers/rubrics_verifier.py b/src/agent_env/task_step/task_steps/verifiers/rubrics_verifier.py index 7bf0b161c..9efd02b3c 100644 --- a/src/agent_env/task_step/task_steps/verifiers/rubrics_verifier.py +++ b/src/agent_env/task_step/task_steps/verifiers/rubrics_verifier.py @@ -286,10 +286,12 @@ def _caches_prompts(model: str) -> bool: def _judge_route(model: str, api_base: Optional[str], provider_of: Callable[..., Any], custom_provider: Optional[str] = None) -> tuple[str, Optional[str]]: """The model name and base a direct judge call hands litellm; what it records stays ``model``. Through an - endpoint, a name litellm can't place goes as ``openai/``, and an Anthropic one, whose client appends - ``/v1/messages``, gets the base without a trailing ``/v1``. A ``custom_llm_provider`` among the call's params - decides the provider as it does for litellm. ``provider_of`` is litellm's ``get_llm_provider``, which prints a - banner before refusing a name it can't place.""" + endpoint, a bare name goes by what litellm makes of it: an Anthropic one keeps that route, with the base's + trailing ``/v1`` dropped since its client appends ``/v1/messages``; an OpenAI one goes as configured; any other, + which litellm would send to that provider's own API (``gemini-*`` to Vertex), or one it can't place, goes + through the endpoint as ``openai/``. A provider prefix in the name, or a ``custom_llm_provider`` among the + call's params, keeps the route it names. ``provider_of`` is litellm's ``get_llm_provider``, which prints a banner + before refusing a name it can't place.""" if api_base is None: return model, None try: @@ -297,7 +299,11 @@ def _judge_route(model: str, api_base: Optional[str], provider_of: Callable[..., provider = provider_of(model, custom_llm_provider=custom_provider)[1] except Exception: return f"openai/{model}", api_base - return model, api_root(api_base) if provider == "anthropic" else api_base + if provider == "anthropic": + return model, api_root(api_base) + if provider == "openai" or custom_provider or "/" in model: + return model, api_base + return f"openai/{model}", api_base def _usage_and_cost(response: Any) -> dict: diff --git a/tst/unit/task_step/verifiers/test_rubrics_verifier_base_url.py b/tst/unit/task_step/verifiers/test_rubrics_verifier_base_url.py index a1ed4f06d..fdc486cf9 100644 --- a/tst/unit/task_step/verifiers/test_rubrics_verifier_base_url.py +++ b/tst/unit/task_step/verifiers/test_rubrics_verifier_base_url.py @@ -1,8 +1,9 @@ -"""The direct judge reaches the configured endpoint whether its base URL ends in ``/v1`` or not, and with a model name -litellm can't place: the real litellm calls a loopback server that records each call's path and model name. A -Claude name keeps Anthropic's Messages route, its ``/v1`` not doubled; an OpenAI name gets the URL as configured; a -name litellm can't place goes through the endpoint as ``openai/``. The rows without ``/v1`` on Claude, and every -OpenAI row, called these same paths before.""" +"""The direct judge reaches the configured endpoint whether its base URL ends in ``/v1`` or not, whatever bare model +name it's given: the real litellm calls a loopback server that records each call's path and model name. A Claude +name keeps Anthropic's Messages route, its ``/v1`` not doubled; an OpenAI name gets the URL as configured; any other +bare name, one litellm can't place or one it would send to another provider's own API (``gemini-*`` to Vertex), goes +through the endpoint as ``openai/``. A prefixed name keeps its provider's route. The rows without ``/v1`` on +Claude, every OpenAI row and the prefixed DeepSeek row called these same paths before.""" from __future__ import annotations import json @@ -87,6 +88,10 @@ def _judge(monkeypatch, base_url: str, params: dict | None = None) -> RubricsVer ("qwen3:8b", "", "/chat/completions", "qwen3:8b"), ("qwen3:8b", "/v1", "/v1/chat/completions", "qwen3:8b"), ("Qwen/Qwen3-8B", "/v1", "/v1/chat/completions", "Qwen/Qwen3-8B"), + ("gemini-2.5-flash", "", "/chat/completions", "gemini-2.5-flash"), + ("gemini-2.5-flash", "/v1", "/v1/chat/completions", "gemini-2.5-flash"), + ("command-a-03-2025", "/v1", "/v1/chat/completions", "command-a-03-2025"), + ("deepseek/deepseek-chat", "/v1", "/v1/chat/completions", "deepseek-chat"), ]) async def test_the_direct_judge_reaches_the_endpoint_however_its_base_url_ends( monkeypatch, endpoint, model, suffix, path, sent, @@ -111,7 +116,8 @@ async def test_a_judge_whose_params_name_its_provider_keeps_that_providers_url(m def _provider(name: str, custom_llm_provider: str | None = None): - providers = {"claude-sonnet-4-6": "anthropic", "gpt-4.1-mini": "openai"} + providers = {"claude-sonnet-4-6": "anthropic", "gpt-4.1-mini": "openai", "gemini-2.5-flash": "vertex_ai", + "deepseek/deepseek-chat": "deepseek"} if custom_llm_provider: return name, custom_llm_provider, None, None if name not in providers: @@ -129,4 +135,10 @@ def test_the_route_keeps_a_native_name_and_sends_an_unknown_one_as_openai(capsys assert _judge_route("claude-sonnet-4-6", "https://gw.example/v1", _provider, "openai") == ( "claude-sonnet-4-6", "https://gw.example/v1") assert _judge_route("qwen3:8b", "https://gw.example", _provider, "hosted_vllm") == ("qwen3:8b", "https://gw.example") + assert _judge_route("gemini-2.5-flash", "https://gw.example/v1", _provider) == ( + "openai/gemini-2.5-flash", "https://gw.example/v1") + assert _judge_route("deepseek/deepseek-chat", "https://gw.example/v1", _provider) == ( + "deepseek/deepseek-chat", "https://gw.example/v1") + assert _judge_route("gemini-2.5-flash", "https://gw.example", _provider, "vertex_ai") == ( + "gemini-2.5-flash", "https://gw.example") assert capsys.readouterr().out == "" From a7eb1db21e95f70387238e9a929d303d705de7af Mon Sep 17 00:00:00 2001 From: Edgar Arakelyan Date: Thu, 19 Nov 2026 11:21:36 -0800 Subject: [PATCH 4/4] refactor(agents): the agents' copies of api_root parse the URL as core's does, and a test keeps them equal Claude Code, Codex, Gemini and browser-use each carry a copy of core's api_root, since their images can't import core. The copies now take the same urlsplit path as core's, so they agree on every base, a query string included, and a unit test runs each copy, read from its source, against core's over a table of URLs. Co-Authored-By: Claude Opus 5.5 --- .../agents/agents/browser-use/agent.py | 7 ++- .../agents/claude-code/claude_bridge.py | 10 +++-- .../agents/agents/codex/codex_config.py | 7 ++- .../agents/agents/gemini/a2a_server.py | 9 ++-- tst/unit/agents/test_api_root_copies.py | 45 +++++++++++++++++++ 5 files changed, 67 insertions(+), 11 deletions(-) create mode 100644 tst/unit/agents/test_api_root_copies.py diff --git a/src/agent_env/agents/agents/browser-use/agent.py b/src/agent_env/agents/agents/browser-use/agent.py index ac9127197..324e2c3ed 100644 --- a/src/agent_env/agents/agents/browser-use/agent.py +++ b/src/agent_env/agents/agents/browser-use/agent.py @@ -16,6 +16,7 @@ import re import sys from pathlib import Path +from urllib.parse import urlsplit, urlunsplit import httpx2 from anthropic import AsyncAnthropic @@ -40,8 +41,10 @@ def _api_root(url: str) -> str: """``LITELLM_BASE_URL`` without a trailing ``/`` and one trailing ``/v1``: it may be configured either way, and - Anthropic's client appends its own ``/v1/messages``.""" - return url.strip().rstrip("/").removesuffix("/v1").rstrip("/") + Anthropic's client appends its own ``/v1/messages``. Core's ``api_root``, copied since this image can't import + core; a unit test keeps the copies equal.""" + parts = urlsplit(url.strip()) + return urlunsplit(parts._replace(path=parts.path.rstrip("/").removesuffix("/v1").rstrip("/"))) def model(turn: dict): diff --git a/src/agent_env/agents/agents/claude-code/claude_bridge.py b/src/agent_env/agents/agents/claude-code/claude_bridge.py index 229c88256..46385b243 100644 --- a/src/agent_env/agents/agents/claude-code/claude_bridge.py +++ b/src/agent_env/agents/agents/claude-code/claude_bridge.py @@ -18,7 +18,7 @@ from functools import partial from pathlib import Path from typing import Literal -from urllib.parse import urlsplit +from urllib.parse import urlsplit, urlunsplit from pydantic import BaseModel, ConfigDict, model_validator @@ -178,9 +178,11 @@ def _model_headers(raw: str | None) -> dict[str, str]: #: Set by the image (root.conf) where the egress proxy runs: the CLI and the model's commands go out through it #: while the agent follows an armed clock. def _api_root(url: str) -> str: - """``LITELLM_BASE_URL`` without a trailing ``/`` and one trailing ``/v1``: it may be configured either way, and the - CLI appends its own ``/v1/messages``.""" - return url.strip().rstrip("/").removesuffix("/v1").rstrip("/") + """``LITELLM_BASE_URL`` without a trailing ``/`` and one trailing ``/v1``: it may be configured either way, and + the CLI appends its own ``/v1/messages``. Core's ``api_root``, copied since this image can't import core; a unit + test keeps the copies equal.""" + parts = urlsplit(url.strip()) + return urlunsplit(parts._replace(path=parts.path.rstrip("/").removesuffix("/v1").rstrip("/"))) EGRESS_PROXY_ENV = "AGENTENV_EGRESS_PROXY" diff --git a/src/agent_env/agents/agents/codex/codex_config.py b/src/agent_env/agents/agents/codex/codex_config.py index 7622c09b7..e60d0f358 100644 --- a/src/agent_env/agents/agents/codex/codex_config.py +++ b/src/agent_env/agents/agents/codex/codex_config.py @@ -15,6 +15,7 @@ from collections.abc import Mapping from pathlib import Path from typing import Any, Literal +from urllib.parse import urlsplit, urlunsplit import tomli_w from pydantic import BaseModel, ConfigDict, model_validator @@ -196,8 +197,10 @@ def model_headers(raw: str | None) -> dict[str, str]: def _api_root(url: str) -> str: """``LITELLM_BASE_URL`` without a trailing ``/`` and one trailing ``/v1``: it may be configured either way, and - codex is given LiteLLM's ``/openai/v1`` route under it.""" - return url.strip().rstrip("/").removesuffix("/v1").rstrip("/") + codex is given LiteLLM's ``/openai/v1`` route under it. Core's ``api_root``, copied since this image can't + import core; a unit test keeps the copies equal.""" + parts = urlsplit(url.strip()) + return urlunsplit(parts._replace(path=parts.path.rstrip("/").removesuffix("/v1").rstrip("/"))) def write_base_config(environ: Mapping[str, str]) -> None: diff --git a/src/agent_env/agents/agents/gemini/a2a_server.py b/src/agent_env/agents/agents/gemini/a2a_server.py index a6481b406..1021a70c9 100644 --- a/src/agent_env/agents/agents/gemini/a2a_server.py +++ b/src/agent_env/agents/agents/gemini/a2a_server.py @@ -19,6 +19,7 @@ from datetime import datetime, timezone from pathlib import Path from typing import Any +from urllib.parse import urlsplit, urlunsplit import httpx from a2a.types import AgentSkill @@ -145,9 +146,11 @@ def _api_root(url: str) -> str: - """``LITELLM_BASE_URL`` without a trailing ``/`` and one trailing ``/v1``: it may be configured either way, and the - proxy appends LiteLLM's ``/gemini/...`` and ``/v1/chat/completions`` routes to it.""" - return url.strip().rstrip("/").removesuffix("/v1").rstrip("/") + """``LITELLM_BASE_URL`` without a trailing ``/`` and one trailing ``/v1``: it may be configured either way, and + the proxy appends LiteLLM's ``/gemini/...`` and ``/v1/chat/completions`` routes to it. Core's ``api_root``, + copied since this image can't import core; a unit test keeps the copies equal.""" + parts = urlsplit(url.strip()) + return urlunsplit(parts._replace(path=parts.path.rstrip("/").removesuffix("/v1").rstrip("/"))) def _describe_transport_error(exc: Exception) -> str: diff --git a/tst/unit/agents/test_api_root_copies.py b/tst/unit/agents/test_api_root_copies.py new file mode 100644 index 000000000..5433f4216 --- /dev/null +++ b/tst/unit/agents/test_api_root_copies.py @@ -0,0 +1,45 @@ +"""Every agent's copy of core's ``api_root``, which an agent's image can't import, answers as core's does. Each copy is +read from its file's source, since some agents' modules import packages only their images install.""" + +import ast +from pathlib import Path +from urllib.parse import urlsplit, urlunsplit + +import pytest + +import agent_env +from agent_env.config.model import api_root + +AGENTS = Path(agent_env.__file__).parent / "agents" / "agents" +URLS = [ + "https://proxy.example", "https://proxy.example/", "https://proxy.example/v1", "https://proxy.example/v1/", + "https://proxy.example/v1/v1", "https://proxy.example/prefix", "https://proxy.example/prefix/v1", + "http://172.17.0.1:4000/v1", "https://proxy.example/v1?team=a", " https://proxy.example/v1 ", + "HTTPS://Proxy.Example/V1", "https://proxy.example/v10", "", +] + + +def _copies() -> dict[str, object]: + """Each agent file's ``_api_root``, run from its source with only what it imports from ``urllib.parse``.""" + copies = {} + for path in sorted(AGENTS.glob("*/*.py")): + tree = ast.parse(path.read_text()) + for node in tree.body: + if isinstance(node, ast.FunctionDef) and node.name == "_api_root": + namespace = {"urlsplit": urlsplit, "urlunsplit": urlunsplit} + exec(compile(ast.Module([node], type_ignores=[]), str(path), "exec"), namespace) + copies[str(path.relative_to(AGENTS))] = namespace["_api_root"] + return copies + + +COPIES = _copies() + + +def test_the_agents_that_append_their_own_path_each_have_a_copy(): + assert {"browser-use/agent.py", "claude-code/claude_bridge.py", "codex/codex_config.py", + "gemini/a2a_server.py"} <= set(COPIES) + + +@pytest.mark.parametrize("copy", sorted(COPIES)) +def test_a_copy_answers_as_core_does(copy): + assert [COPIES[copy](url) for url in URLS] == [api_root(url) for url in URLS]