From 11a0497b538356f13f136674f870b1d6afc45b64 Mon Sep 17 00:00:00 2001 From: Edgar Arakelyan Date: Wed, 18 Nov 2026 13:28:26 -0800 Subject: [PATCH 1/6] test(config): pin which key, URL and headers each place hands the model endpoint to today Rows named leaks-* are the configured key reaching a URL it wasn't issued for; the fix that follows flips them. Co-Authored-By: Claude Opus 5.5 --- tst/unit/task_step/test_model_key_scope.py | 175 +++++++++++++++++++++ 1 file changed, 175 insertions(+) create mode 100644 tst/unit/task_step/test_model_key_scope.py diff --git a/tst/unit/task_step/test_model_key_scope.py b/tst/unit/task_step/test_model_key_scope.py new file mode 100644 index 000000000..a43b9cb0d --- /dev/null +++ b/tst/unit/task_step/test_model_key_scope.py @@ -0,0 +1,175 @@ +"""Which key, URL and headers each place that hands out the model endpoint sends: an agent's deploy, core's own model +call (the direct judge's) and a unit-test verifier's container. The configured key goes only with the configured +endpoint; any other URL needs a key of its own.""" + +from __future__ import annotations + +import asyncio +import json +import shlex +import types + +import pytest + +from agent_env.a2a_agent.a2a_agent import TAKES_CREDENTIALS, A2AAgent +from agent_env.artifact.artifacts.docker_image import DockerImageArtifact +from agent_env.config.model import ModelConfig +from agent_env.providers.sandbox_providers import sandbox_provider +from agent_env.providers.sandbox_providers.sandbox_provider import SandboxProvider +from agent_env.task_step.context import DeployedSandbox, TaskStepContext +from agent_env.task_step.task_steps.verifiers.run_container_unit_tests_verifier import ( + RunContainerUnitTestsVerifierTaskStep, +) + +MODEL = "http://localhost:4000/v1" +HEADERS = {"X-Bill-To": "acme"} +ELSEWHERE = "http://127.0.0.1:5001" + + +@pytest.fixture +def configured(monkeypatch, local_stores): + monkeypatch.setenv("LITELLM_BASE_URL", MODEL) + monkeypatch.setenv("LITELLM_API_KEY", "k") + monkeypatch.setattr(local_stores, "_model_cfg", ModelConfig(headers=HEADERS)) + + +class _Started(Exception): + """Raised once the fake provider has seen the container's env.""" + + +class _Provider(SandboxProvider): + async def create_sandbox(self, *, env, **_): + self.env = env + raise _Started + + +def _deploy(monkeypatch, passed: dict | None, own: dict | None, **metadata) -> dict: + provider = _Provider() + monkeypatch.setattr(sandbox_provider, "build_sandbox_provider", lambda name: provider) + agent = A2AAgent(id="solver", version=1, metadata=metadata, default_env_vars=own, + docker_image_artifact=DockerImageArtifact(id="img", description="d", image_name="img")) + with pytest.raises(_Started): + asyncio.run(agent.deploy(sandbox_type="any", env_vars=passed)) + headers = provider.env.get("LITELLM_EXTRA_HEADERS") + return {"key": provider.env.get("LITELLM_API_KEY"), "url": provider.env.get("LITELLM_BASE_URL"), + "headers": json.loads(headers) if headers else None} + + +AGENT_CASES = { + "configured": (None, None, {"key": "k", "url": MODEL, "headers": HEADERS}), + "passed-configured": ({"LITELLM_BASE_URL": MODEL}, None, {"key": "k", "url": MODEL, "headers": HEADERS}), + "passed-configured-slash": ({"LITELLM_BASE_URL": MODEL + "/"}, None, + {"key": "k", "url": MODEL + "/", "headers": HEADERS}), + "passed-configured-without-v1": ({"LITELLM_BASE_URL": "http://localhost:4000"}, None, + {"key": "k", "url": "http://localhost:4000", "headers": None}), + "passed-configured-upper-case": ({"LITELLM_BASE_URL": "HTTP://LOCALHOST:4000/v1"}, None, + {"key": "k", "url": "HTTP://LOCALHOST:4000/v1", "headers": None}), + "passed-elsewhere-leaks-the-key": ({"LITELLM_BASE_URL": ELSEWHERE}, None, + {"key": "k", "url": ELSEWHERE, "headers": None}), + "passed-elsewhere-with-its-key": ({"LITELLM_BASE_URL": ELSEWHERE, "LITELLM_API_KEY": "theirs"}, None, + {"key": "theirs", "url": ELSEWHERE, "headers": None}), + "own-elsewhere-leaks-the-key": (None, {"LITELLM_BASE_URL": ELSEWHERE}, + {"key": "k", "url": ELSEWHERE, "headers": None}), + "own-elsewhere-with-its-key": (None, {"LITELLM_BASE_URL": ELSEWHERE, "LITELLM_API_KEY": "own"}, + {"key": "own", "url": ELSEWHERE, "headers": None}), + "passed-key-only": ({"LITELLM_API_KEY": "theirs"}, None, {"key": "theirs", "url": MODEL, "headers": HEADERS}), +} + + +@pytest.mark.parametrize("passed, own, expected", AGENT_CASES.values(), ids=AGENT_CASES.keys()) +def test_an_agent_deploy_sends(monkeypatch, configured, passed, own, expected): + assert _deploy(monkeypatch, passed, own) == expected + + +def test_an_agent_that_takes_no_credentials_gets_none(monkeypatch, configured): + sent = _deploy(monkeypatch, {"LITELLM_BASE_URL": ELSEWHERE, "LITELLM_API_KEY": "x"}, None, + **{TAKES_CREDENTIALS: False}) + assert sent == {"key": None, "url": None, "headers": None} + + +GATEWAY = "https://gw.example.com" +CALL_CASES = { + "configured": ({}, {}, ("file-key", GATEWAY, True)), + "configured-env-key": ({"LITELLM_API_KEY": "env-key"}, {}, ("env-key", GATEWAY, True)), + "elsewhere-env-key-leaks": ({"LITELLM_API_KEY": "env-key"}, {"base_override": "https://elsewhere.example.com"}, + ("env-key", "https://elsewhere.example.com", False)), + "elsewhere-no-key": ({}, {"base_override": "https://elsewhere.example.com"}, + (None, "https://elsewhere.example.com", False)), + "elsewhere-with-its-key": ({}, {"base_override": "https://elsewhere.example.com", "default_api_key": "theirs"}, + ("theirs", "https://elsewhere.example.com", False)), + "configured-with-v1": ({}, {"base_override": GATEWAY + "/v1"}, (None, GATEWAY + "/v1", False)), +} + + +@pytest.mark.parametrize("env, call, expected", CALL_CASES.values(), ids=CALL_CASES.keys()) +def test_a_model_call_core_makes_sends(monkeypatch, env, call, expected): + for name in ("LITELLM_BASE_URL", "LITELLM_API_KEY"): + monkeypatch.delenv(name, raising=False) + for name, value in env.items(): + monkeypatch.setenv(name, value) + config = ModelConfig(base_url=GATEWAY, api_key="file-key", headers={"X-Bill-To": "acme"}) + resolved = config.resolve_call("m", secret_resolver=lambda key: None, **call) + assert (resolved.api_key, resolved.api_base, "extra_headers" in resolved.params) == expected + + +def test_a_natively_routed_call_gets_the_configured_key(monkeypatch): + monkeypatch.delenv("LITELLM_BASE_URL", raising=False) + monkeypatch.delenv("LITELLM_API_KEY", raising=False) + resolved = ModelConfig(api_key="file-key").resolve_call("openai/m", secret_resolver=lambda key: None) + assert (resolved.api_key, resolved.api_base) == ("file-key", None) + + +class _VerifierSandbox: + def __init__(self): + self.type = "modal_vm" + self.commands: list[str] = [] + + def scoped_name(self, name): + return name + + async def exec_script(self, script: str) -> str: + self.commands.append(script) + return "" + + async def exec_with_output(self, *args) -> tuple[int, str, str]: + self.commands.append(args[-1]) + return 0, "ok", "" + + +def _verifier_env(monkeypatch, env_vars: dict) -> dict: + """The model variables a unit-test verifier's command ran with.""" + sandbox = _VerifierSandbox() + provider = types.SimpleNamespace(get_sandbox=lambda sandbox_id: asyncio.sleep(0, result=sandbox)) + monkeypatch.setattr(sandbox_provider, "build_sandbox_provider", lambda name: provider) + monkeypatch.setattr(RunContainerUnitTestsVerifierTaskStep, "_upload_text_artifact", staticmethod( + lambda text, artifact_id, description, object_url: types.SimpleNamespace( + id=artifact_id, version=1, object_url=object_url))) + context = TaskStepContext() + context.metadata["deployed_docker_containers"] = [{"container_name": "c", "sandbox_name": "h"}] + context.deployed_sandboxes.append( + DeployedSandbox(sandbox_name="h", sandbox_id="sb-1", sandbox_mode="vm", sandbox_type="modal_vm")) + asyncio.run(RunContainerUnitTestsVerifierTaskStep( + id="tests", version=None, sandbox_name="h", container_name="c", command="true", env_vars=env_vars, + ).execute(context)) + [run] = [c for c in sandbox.commands if "timeout --kill-after" in c] + words = shlex.split(run) + pairs = dict(words[i + 1].split("=", 1) for i, word in enumerate(words) if word == "-e") + return {name: pairs.get(name) for name in ("LITELLM_API_KEY", "LITELLM_BASE_URL", "ANTHROPIC_API_KEY", + "ANTHROPIC_BASE_URL")} + + +VERIFIER_CASES = { + "configured": ({}, {"LITELLM_API_KEY": "k", "LITELLM_BASE_URL": MODEL, "ANTHROPIC_API_KEY": "k", + "ANTHROPIC_BASE_URL": MODEL}), + "elsewhere-leaks-the-key": ({"LITELLM_BASE_URL": ELSEWHERE}, + {"LITELLM_API_KEY": "k", "LITELLM_BASE_URL": ELSEWHERE, "ANTHROPIC_API_KEY": "k", + "ANTHROPIC_BASE_URL": MODEL}), + "elsewhere-with-its-key": ({"LITELLM_BASE_URL": ELSEWHERE, "LITELLM_API_KEY": "theirs"}, + {"LITELLM_API_KEY": "theirs", "LITELLM_BASE_URL": ELSEWHERE, "ANTHROPIC_API_KEY": "k", + "ANTHROPIC_BASE_URL": MODEL}), +} + + +@pytest.mark.parametrize("env_vars, expected", VERIFIER_CASES.values(), ids=VERIFIER_CASES.keys()) +def test_a_unit_test_verifier_runs_its_tests_with(monkeypatch, configured, env_vars, expected): + assert _verifier_env(monkeypatch, env_vars) == expected From 8f33bd547b68022c014920454d5a3dfd66a378b9 Mon Sep 17 00:00:00 2001 From: Edgar Arakelyan Date: Wed, 18 Nov 2026 13:42:25 -0800 Subject: [PATCH 2/6] fix(config)!: a model key reaches only the URL it was issued for The configured key, from [model] api_key or LITELLM_API_KEY, now goes only to the configured endpoint. An agent, a judge or a unit-test verifier pointed at another URL with no key of its own is refused before anything starts, naming how to pass one; before, it was handed the configured key. Endpoints compare as one when they differ only in scheme or host case, a default port, a trailing slash or one trailing /v1, so the configured endpoint's headers reach it however it is spelled. Bundle preflight reports the refusal before the run writes anything. Co-Authored-By: Claude Opus 5.5 --- src/agent_env/a2a_agent/a2a_agent.py | 10 +- src/agent_env/bundle/preflight.py | 36 ++-- src/agent_env/config/model.py | 62 +++++-- src/agent_env/config/runtime.py | 3 + .../task_steps/verifiers/rubrics_verifier.py | 13 +- .../run_container_unit_tests_verifier.py | 18 +- .../task_step/test_model_key_scope_local.py | 161 ++++++++++++++++++ tst/unit/bundle/preflight_test.py | 36 ++++ tst/unit/config/test_model.py | 49 ++++-- .../task_step/local_container_urls_test.py | 12 +- tst/unit/task_step/test_model_key_scope.py | 142 +++++++++++---- 11 files changed, 465 insertions(+), 77 deletions(-) create mode 100644 tst/integration/task_step/test_model_key_scope_local.py diff --git a/src/agent_env/a2a_agent/a2a_agent.py b/src/agent_env/a2a_agent/a2a_agent.py index efdcfc4f5..0c1288697 100644 --- a/src/agent_env/a2a_agent/a2a_agent.py +++ b/src/agent_env/a2a_agent/a2a_agent.py @@ -15,7 +15,8 @@ from agentenv_protocol.types import CONTROL_TOKEN_ENV from agent_env.config import get_config -from agent_env.config.model import MODEL_HEADERS_ENV +from agent_env.config.errors import ConfigError +from agent_env.config.model import MODEL_HEADERS_ENV, no_key_for from agent_env.entity.refs import EntityRef from agent_env.plugins._bundles import installed_aliases from agent_env.store.base import NotFoundError @@ -624,11 +625,16 @@ def _refusal(self, image: DockerImageArtifact, link: SandboxProvider) -> str | N def _with_model_credentials(self, env_vars: dict[str, str], config) -> dict[str, str]: """``env_vars`` with the model endpoint, key and headers each taken from config when neither they nor - ``default_env_vars`` set it; for an agent that takes no credentials, with none of them, whatever set them.""" + ``default_env_vars`` set it; for an agent that takes no credentials, with none of them, whatever set them. The + configured key goes only with the configured endpoint: an agent sent elsewhere with no key of its own is + refused before anything starts.""" if not self.metadata.get(TAKES_CREDENTIALS, True): return {name: value for name, value in env_vars.items() if name not in MODEL_CREDENTIALS_ENV} resolved_env = dict(env_vars) if "LITELLM_API_KEY" not in resolved_env and "LITELLM_API_KEY" not in self.default_env_vars: + url = resolved_env.get("LITELLM_BASE_URL") or self.default_env_vars.get("LITELLM_BASE_URL") + if url and not config.is_model_endpoint(url): + raise ConfigError(f"Agent {self.id!r}: {no_key_for(url)}") resolved_env["LITELLM_API_KEY"] = config.get_litellm_api_key() if "LITELLM_BASE_URL" not in resolved_env and "LITELLM_BASE_URL" not in self.default_env_vars: resolved_env["LITELLM_BASE_URL"] = config.get_litellm_base_url() diff --git a/src/agent_env/bundle/preflight.py b/src/agent_env/bundle/preflight.py index 39c638467..4c78bf866 100644 --- a/src/agent_env/bundle/preflight.py +++ b/src/agent_env/bundle/preflight.py @@ -34,7 +34,7 @@ from agent_env.artifact.artifacts.docker_image import PUT_AGAIN, DockerImageArtifact, pull_problem from agent_env.config import get_config from agent_env.config.errors import ConfigError -from agent_env.config.model import ENV_API_KEY, ENV_BASE_URL +from agent_env.config.model import ENV_API_KEY, ENV_BASE_URL, no_key_for from agent_env.entity.refs import EntityKind, parse_toml_ref from agent_env.env.bootstrap import ( GATEWAY, @@ -77,6 +77,9 @@ from agent_env.task_step.task_steps.deploy_env import DeployEnvTaskStep from agent_env.task_step.task_steps.deploy_sandbox import DeploySandboxTaskStep from agent_env.task_step.task_steps.verifiers.rubrics_verifier import RubricsVerifierTaskStep +from agent_env.task_step.task_steps.verifiers.run_container_unit_tests_verifier import ( + RunContainerUnitTestsVerifierTaskStep, +) from agent_env.utils.docker.build import DEFAULT_BUILD_PLATFORM, docker_plugin_missing, docker_unreachable from .parse import BundleError, BundleKind @@ -170,6 +173,9 @@ def task(self, task: ResolvedEntry, skip: frozenset[str], given: frozenset[str] self._sandbox(where, step) elif isinstance(step, RubricsVerifierTaskStep): self._judge(where, step) + elif isinstance(step, RunContainerUnitTestsVerifierTaskStep): + if problem := step.model_key_problem(): + self._problem(where, problem) elif isinstance(step, InstallAgentTaskStep) and step.a2a_agent_id is None and override is None: self.default_agent_users.append((where, "names no agent")) except ValueError as e: # a provider that names no backend, or can't be built from its config @@ -311,7 +317,8 @@ def _agent(self, where: str, step: DeployAgentTaskStep, sandboxes: dict[str, Dep *(_MODEL_VARS if endpoint_given else ())}) agent_id, version = ((override.agent_id, override.version) if override is not None else (step.a2a_agent_id, step.a2a_agent_version)) - self._agent_deploy(where, provider, agent_id, version, "names no agent", accepts, given=given) + self._agent_deploy(where, provider, agent_id, version, "names no agent", accepts, given=given, + url=step.litellm_base_url or step.env_vars.get(ENV_BASE_URL)) def _judge(self, where: str, step: RubricsVerifierTaskStep) -> None: if not step.use_agent_judge or step.agent_name is not None: @@ -322,13 +329,22 @@ def _judge(self, where: str, step: RubricsVerifierTaskStep) -> None: def _agent_deploy(self, where: str, provider: SandboxProvider, agent_id: str | None, version: int | None, unnamed: str, accepts: Callable[[SandboxProvider], Accepts] = A2AAgent.accepts, *, - given: frozenset[str] = frozenset(), needs: frozenset[str] | None = None) -> None: - """Check an agent deploy, ``given`` the model variables its step sets, or ``needs`` those the config must.""" + given: frozenset[str] = frozenset(), needs: frozenset[str] | None = None, + url: str | None = None) -> None: + """Check an agent deploy, ``given`` the model variables its step sets and ``url`` the endpoint it names, or + ``needs`` those the config must.""" if agent_id is None: self.default_agent_users.append((where, unnamed)) agent_id = self._default_agent_id() - if needed := needs if needs is not None else _MODEL_VARS - given - self._agent_env_vars(agent_id, version): - self.model_users.append((where, needed)) + if needs is None: + own = self._agent_env_vars(agent_id, version) + url = url or own.get(ENV_BASE_URL) + if url and ENV_API_KEY not in given and ENV_API_KEY not in own and not get_config().is_model_endpoint(url): + self._problem(where, f"deploys agent {agent_id!r}, but {no_key_for(url)}") + elif needed := _MODEL_VARS - given - own.keys(): + self.model_users.append((where, needed)) + else: + self.model_users.append((where, needs)) image = self._agent_image(agent_id, version) taking = self._check(where, provider, accepts, [image] if image else [], vms_load=True) if any(link.ON_THIS_MACHINE for link in taking): @@ -352,14 +368,14 @@ def _sandbox(self, where: str, step: DeploySandboxTaskStep) -> None: # What a deploy runs - def _agent_env_vars(self, agent_id: str, version: int | None) -> frozenset[str]: + def _agent_env_vars(self, agent_id: str, version: int | None) -> dict[str, str]: """The variables the agent ``agent_id`` declares for itself, which a deploy doesn't take from the config.""" if (agent := self.agents.get(agent_id)) is not None: - return frozenset(agent.config.get("default_env_vars") or {}) + return dict(agent.config.get("default_env_vars") or {}) try: - return frozenset(A2AAgent.get(agent_id, version).default_env_vars or {}) + return dict(A2AAgent.get(agent_id, version).default_env_vars or {}) except (NotFoundError, ValueError, KeyError, TypeError): - return frozenset() + return {} def _agent_image(self, agent_id: str, version: int | None) -> _Image | None: """The image the agent ``agent_id`` runs, or None when the store doesn't hold the agent or can't read it, which diff --git a/src/agent_env/config/model.py b/src/agent_env/config/model.py index 5e2bef2ca..dfbeb5f43 100644 --- a/src/agent_env/config/model.py +++ b/src/agent_env/config/model.py @@ -10,7 +10,7 @@ from dataclasses import dataclass, field, fields from enum import StrEnum from typing import Optional -from urllib.parse import urlparse +from urllib.parse import urlsplit from agent_env.config.errors import ConfigError from agent_env.config.loader import ENV_REF_PREFIX, SECRET_REF_PREFIX, SecretResolver, interpolate @@ -170,15 +170,19 @@ def _resolved_api_key(self, secret_resolver: SecretResolver) -> Optional[str]: return None return interpolate(self.api_key, secret_resolver=secret_resolver) + def is_endpoint(self, url: Optional[str]) -> bool: + """Whether ``url`` names the configured endpoint (``LITELLM_BASE_URL``, else ``base_url``), by + ``same_endpoint``: the only URL the configured key and headers go to.""" + return same_endpoint(url, os.getenv(ENV_BASE_URL) or self.base_url) + def headers_for(self, base_url: Optional[str], *, secret_resolver: SecretResolver) -> dict[str, str]: """``[model.headers]``, resolved, for a call or an agent pointed at ``base_url``: none unless that is the configured endpoint, since a header may carry a credential.""" if not self.headers or not base_url: return {} - configured_base = os.getenv(ENV_BASE_URL) or self.base_url - if not configured_base or base_url.rstrip("/") != configured_base.rstrip("/"): + if not self.is_endpoint(base_url): logger.info("Not sending [model.headers] to %s, which is not the configured model endpoint", - urlparse(base_url).hostname or base_url) + _netloc(base_url)) return {} headers = interpolate(dict(self.headers), secret_resolver=secret_resolver) for name, value in headers.items(): @@ -194,8 +198,11 @@ def resolve_call( secret_resolver: SecretResolver, ) -> ModelCallConfig: """A provider-prefixed model routes natively when no endpoint is configured; a bare - model name needs one (override/env/config) and fails actionably without it.""" - key = default_api_key or os.getenv(ENV_API_KEY) + model name needs one (override/env/config) and fails actionably without it. The configured key, from + ``LITELLM_API_KEY`` or ``api_key``, goes only to the configured endpoint or a native route; a ``base_override`` + elsewhere needs ``default_api_key``, since without one ``litellm`` would fall back to provider keys in the + environment.""" + key = default_api_key configured_base = os.getenv(ENV_BASE_URL) or self.base_url if base_override: api_base = base_override @@ -209,11 +216,10 @@ def resolve_call( f".agentenv/config.toml or {ENV_BASE_URL}, or use a provider-prefixed " "model name (e.g. 'openai/gpt-4o') for native routing." ) - # The configured key is scoped to the configured endpoint (and native routing): - # a base_override pointing elsewhere must not inherit it, or a caller-supplied - # URL would receive the configured proxy's credential. - if key is None and (api_base is None or api_base.rstrip("/") == (configured_base or "").rstrip("/")): - key = self._resolved_api_key(secret_resolver) + if key is None: + if api_base is not None and not same_endpoint(api_base, configured_base): + raise ConfigError(no_key_for(api_base)) + key = os.getenv(ENV_API_KEY) or self._resolved_api_key(secret_resolver) params = dict(self.params) headers = self.headers_for(api_base, secret_resolver=secret_resolver) if headers: @@ -269,6 +275,40 @@ def config_model(role: ModelRole, config: Optional[ModelConfig]) -> ResolvedMode return ResolvedModel(role, None, None) +_DEFAULT_PORTS = {"http": 80, "https": 443} + + +def same_endpoint(a: Optional[str], b: Optional[str]) -> bool: + """Whether two URLs name one model endpoint: scheme and host in any case, a default port, a trailing ``/`` and one + trailing ``/v1`` don't tell them apart, so a client that wants ``/v1`` and one that doesn't reach the same one.""" + return bool(a and b) and _endpoint(a) == _endpoint(b) + + +def _endpoint(url: str) -> tuple: + parts = urlsplit(url.strip()) + scheme = parts.scheme.lower() + try: + port = parts.port + except ValueError: + return (url,) + path = parts.path.rstrip("/") + path = path.removesuffix("/v1").rstrip("/") + return scheme, (parts.hostname or "").lower(), None if port == _DEFAULT_PORTS.get(scheme) else port, path, parts.query + + +def _netloc(url: str) -> str: + """``url``'s host and port, without any user or password it carries.""" + return urlsplit(url).netloc.rpartition("@")[2] or url + + +def no_key_for(url: str) -> str: + """Why a model call or an agent pointed at ``url`` with no key of its own is refused, and how to give it one.""" + return (f"{_netloc(url)} isn't the configured model endpoint, so the configured key isn't sent there, and no key " + "was given with it. Pass one with the URL: litellm_api_key (judge_litellm_api_key for a judge) in the run's " + "overrides or a step's step_params, or LITELLM_API_KEY in the step's env_vars or the agent's " + "default_env_vars. Any non-empty value works for a server that needs no key.") + + def _check_roles(roles: object) -> None: """Refuse a ``[model.roles]`` that maps a role to anything but a model name, or names one role twice, under its name and its synonym.""" diff --git a/src/agent_env/config/runtime.py b/src/agent_env/config/runtime.py index 876e45b2f..c6e49565e 100644 --- a/src/agent_env/config/runtime.py +++ b/src/agent_env/config/runtime.py @@ -488,6 +488,9 @@ def get_model_params(self, overrides: dict | None = None) -> dict: def get_model_headers(self, base_url: str | None) -> dict[str, str]: return self._model_config().headers_for(base_url, secret_resolver=self._resolve_secret) + def is_model_endpoint(self, url: str | None) -> bool: + return self._model_config().is_endpoint(url) + def resolve_model_call( self, model: str, diff --git a/src/agent_env/task_step/task_steps/verifiers/rubrics_verifier.py b/src/agent_env/task_step/task_steps/verifiers/rubrics_verifier.py index 90527df3e..333b2ed03 100644 --- a/src/agent_env/task_step/task_steps/verifiers/rubrics_verifier.py +++ b/src/agent_env/task_step/task_steps/verifiers/rubrics_verifier.py @@ -19,7 +19,8 @@ from agent_env.a2a_agent.a2a_agent import unwritten_local_agent from agent_env.a2a_agent.staging import transfer_store from agent_env.config import get_config -from agent_env.config.model import ModelParam, ModelRole +from agent_env.config.errors import ConfigError +from agent_env.config.model import ModelParam, ModelRole, no_key_for from agent_env.providers.sandbox_providers.sandbox import into_container from agent_env.task_step.context import ( PromptResponse, @@ -906,15 +907,17 @@ async def _grade(self, context: TaskStepContext) -> TaskStepContext: from agent_env.config import get_config config = get_config() - api_key = ( - config.resolve_reference(overrides.get("judge_litellm_api_key") or overrides.get("litellm_api_key")) - or config.get_litellm_api_key() - ) + api_key = config.resolve_reference( + overrides.get("judge_litellm_api_key") or overrides.get("litellm_api_key")) litellm_base_url = ( overrides.get("judge_litellm_base_url") or overrides.get("litellm_base_url") or config.get_litellm_base_url() ) + if not api_key: + if not config.is_model_endpoint(litellm_base_url): + raise ConfigError(f"{self.type} '{self.id}': its judge agent: {no_key_for(litellm_base_url)}") + api_key = config.get_litellm_api_key() env_vars = {"LITELLM_API_KEY": api_key, "LITELLM_BASE_URL": litellm_base_url} resolved_judge_id = self.judge_a2a_agent_id or config.get_default_a2a_agent_id() try: diff --git a/src/agent_env/task_step/task_steps/verifiers/run_container_unit_tests_verifier.py b/src/agent_env/task_step/task_steps/verifiers/run_container_unit_tests_verifier.py index 1b15b316b..41b1a86ab 100644 --- a/src/agent_env/task_step/task_steps/verifiers/run_container_unit_tests_verifier.py +++ b/src/agent_env/task_step/task_steps/verifiers/run_container_unit_tests_verifier.py @@ -36,11 +36,11 @@ import tempfile import time import uuid -from collections.abc import Awaitable, Callable +from collections.abc import Awaitable, Callable, Mapping from typing import Any, ClassVar, Optional from agent_env.config import ConfigError, get_config -from agent_env.config.model import MODEL_HEADERS_ENV +from agent_env.config.model import MODEL_HEADERS_ENV, no_key_for from agent_env.entity.refs import EntityRef from agent_env.env.env import require_sandbox from agent_env.env.envs.computer.utils import inside @@ -344,10 +344,22 @@ async def execute(self, context: TaskStepContext) -> TaskStepContext: return context + def model_key_problem(self, extra_env: Optional[Mapping[str, str]] = None) -> Optional[str]: + """Why the commands can't be given the configured key: a model URL among ``env_vars`` and ``extra_env`` + that isn't the configured endpoint, with no key of its own beside it; None when there's none.""" + given = {**self.env_vars, **(extra_env or {})} + for url_name, key_name in (("LITELLM_BASE_URL", "LITELLM_API_KEY"), ("ANTHROPIC_BASE_URL", "ANTHROPIC_API_KEY")): + url = given.get(url_name) + if url and key_name not in given and not get_config().is_model_endpoint(url): + return f"{url_name}: {no_key_for(url)}" + return None + def _merged_env(self, config, sandbox_type: Optional[str], extra_env: dict[str, str]) -> dict[str, str]: """The commands' environment: the model endpoint's key and URL from agent-env's config, every sandbox provider's container env, ``env_vars`` and this run's ``extra_env``, then the endpoint's headers, its URL as - ``sandbox_type``'s containers reach it.""" + ``sandbox_type``'s containers reach it. A URL other than the configured endpoint needs its key beside it.""" + if problem := self.model_key_problem(extra_env): + raise ConfigError(f"{self.type} '{self.id}': {problem}") merged_env: dict[str, str] = {} try: litellm_key = config.get_litellm_api_key() diff --git a/tst/integration/task_step/test_model_key_scope_local.py b/tst/integration/task_step/test_model_key_scope_local.py new file mode 100644 index 000000000..9c56e0ce8 --- /dev/null +++ b/tst/integration/task_step/test_model_key_scope_local.py @@ -0,0 +1,161 @@ +"""The configured model key reaches only the configured endpoint. Two stand-in model endpoints, the configured one +and another, record the key each call brings: the direct judge calls them over HTTP, and an echo agent +(tst/data/a2a_agent) is deployed on a local Docker sandbox and its container's environment read back. A URL other +than the configured one with no key of its own is refused before anything is called or started. Builds the echo +agent; needs Docker.""" + +import json +import shutil +import subprocess +import threading +import uuid +from http.server import BaseHTTPRequestHandler, ThreadingHTTPServer + +import pytest + +from agent_env.a2a_agent import A2AAgent +from agent_env.artifact.store import reset_artifact_store +from agent_env.config import configure, reset_config +from agent_env.providers.sandbox_providers.sandbox import SANDBOX_LABEL +from agent_env.providers.sandbox_providers.sandbox_provider import ( + reset_agent_sandbox_provider, + reset_env_sandbox_provider, + reset_sandbox_provider, +) +from agent_env.task import Task +from agent_env.task.teardown import teardown_run +from agent_env.task_step.context import PromptResponse, TaskStepContext +from tst.util.a2a_test_agent import build_test_agent_image +from tst.util.capabilities import missing_capability_reason + +pytestmark = [pytest.mark.integration, pytest.mark.int_test_slow] + +CONFIGURED_KEY = "configured-key" +REFUSED = "isn't the configured model endpoint, so the configured key isn't sent there" +VERDICT = json.dumps({"results": [{"id": "c1", "score": 1.0, "justification": "it answered"}]}) + + +class _Endpoint(BaseHTTPRequestHandler): + """An OpenAI-style chat completions endpoint that records the Authorization each call brings.""" + + def do_POST(self): + self.rfile.read(int(self.headers["Content-Length"])) + self.server.calls.append(self.headers.get("Authorization")) + reply = json.dumps({ + "id": "chatcmpl-1", "object": "chat.completion", "created": 0, "model": "judge", + "choices": [{"index": 0, "message": {"role": "assistant", "content": VERDICT}, "finish_reason": "stop"}], + "usage": {"prompt_tokens": 1, "completion_tokens": 1, "total_tokens": 2}, + }).encode() + self.send_response(200) + self.send_header("Content-Type", "application/json") + self.send_header("Content-Length", str(len(reply))) + self.end_headers() + self.wfile.write(reply) + + def log_message(self, *args): + pass + + +def _serve() -> ThreadingHTTPServer: + server = ThreadingHTTPServer(("127.0.0.1", 0), _Endpoint) + server.calls = [] + threading.Thread(target=server.serve_forever, daemon=True).start() + return server + + +def _reset(): + for reset in (reset_config, reset_artifact_store, reset_sandbox_provider, reset_env_sandbox_provider, + reset_agent_sandbox_provider): + reset() + + +@pytest.fixture +def endpoints(monkeypatch, tmp_path): + """The configured endpoint and another, in local stores whose ``[model]`` is the first and its key.""" + configured, other = _serve(), _serve() + sandboxes = tmp_path / "sandboxes" + monkeypatch.chdir(tmp_path) + monkeypatch.setenv("XDG_STATE_HOME", str(tmp_path / "state")) + monkeypatch.setenv("AGENT_ENV_DOCUMENT_STORE", "local") + monkeypatch.setenv("AGENT_ENV_OBJECT_STORE", "local") + monkeypatch.setenv("AGENT_ENV_LOCAL_SANDBOX_DIR", str(sandboxes)) + monkeypatch.delenv("LITELLM_BASE_URL", raising=False) + monkeypatch.delenv("LITELLM_API_KEY", raising=False) + config = tmp_path / "config.toml" + config.write_text(f'[model]\nbase_url = "http://127.0.0.1:{configured.server_port}/v1"\n' + f'api_key = "{CONFIGURED_KEY}"\n') + monkeypatch.setenv("AGENT_ENV_CONFIG", str(config)) + _reset() + configure() + try: + yield configured, other, sandboxes + finally: + configured.shutdown() + other.shutdown() + _reset() + + +def _judge_run(overrides: dict) -> tuple[TaskStepContext, Task]: + task = Task.from_dict({"id": f"key-scope-{uuid.uuid4().hex[:8]}", "steps": [{ + "id": "judge", "type": "rubrics_verifier", "verifier_id": "judge", "prompt_id": "hello", + "use_agent_judge": False, "use_trajectory": False, "default_model": "openai/judge", + "criteria": [{"id": "c1", "description": "it says hello"}]}]}) + return TaskStepContext(prompt_responses=[PromptResponse(prompt_id="hello", response="hello", prompt_text="Hi.")], + metadata={"user_overrides": overrides}), task + + +@pytest.mark.asyncio +async def test_the_judge_sends_the_configured_key_only_to_the_configured_endpoint(endpoints): + configured, other, _ = endpoints + elsewhere = f"http://127.0.0.1:{other.server_port}" + + context, task = _judge_run({"judge_litellm_base_url": elsewhere}) + with pytest.raises(Exception, match=REFUSED): + await task.run(context=context) + assert configured.calls == [] and other.calls == [] + + context, task = _judge_run({"judge_litellm_base_url": elsewhere, "judge_litellm_api_key": "theirs"}) + await task.run(context=context) + assert other.calls == ["Bearer theirs"] and configured.calls == [] + + context, task = _judge_run({"judge_litellm_base_url": f"http://127.0.0.1:{configured.server_port}/"}) + await task.run(context=context) + assert configured.calls == [f"Bearer {CONFIGURED_KEY}"] and other.calls == ["Bearer theirs"] + + +def _container_env(sandbox_id: str) -> dict[str, str]: + [container] = subprocess.run(["docker", "ps", "-aq", "--filter", f"label={SANDBOX_LABEL}={sandbox_id}"], + capture_output=True, text=True, check=True).stdout.split() + inspected = subprocess.run(["docker", "inspect", "--format", "{{json .Config.Env}}", container], + capture_output=True, text=True, check=True).stdout + return dict(pair.split("=", 1) for pair in json.loads(inspected)) + + +@pytest.mark.asyncio +async def test_an_agent_sent_elsewhere_gets_its_own_key_or_is_refused_before_its_container_starts(endpoints): + if shutil.which("docker") is None or subprocess.run(["docker", "info"], capture_output=True).returncode != 0: + pytest.skip(missing_capability_reason("docker_daemon")) + _, other, sandboxes = endpoints + elsewhere = f"http://127.0.0.1:{other.server_port}" + agent_id = f"key-scope-{uuid.uuid4().hex[:8]}" + A2AAgent.put(id=agent_id, docker_image_artifact=build_test_agent_image(f"{agent_id}-image")) + task = Task.from_dict({"id": agent_id, "steps": [{ + "id": "solver", "type": "deploy_agent", "agent_name": "solver", "env_ids": [], "a2a_agent_id": agent_id, + "sandbox_type": "local", "litellm_base_url": elsewhere}]}) + + refused = TaskStepContext() + with pytest.raises(Exception, match=REFUSED): + await task.run(context=refused) + assert refused.deployed_agents == [] and not (sandboxes.exists() and any(sandboxes.iterdir())) + + context = TaskStepContext(metadata={"user_overrides": {"litellm_api_key": "theirs"}}) + try: + await task.run(context=context) + [agent] = context.deployed_agents + env = _container_env(agent.sandbox_id) + assert (env["LITELLM_API_KEY"], env["LITELLM_BASE_URL"]) == ( + "theirs", f"http://host.docker.internal:{other.server_port}") + assert CONFIGURED_KEY not in env.values() and "LITELLM_EXTRA_HEADERS" not in env + finally: + await teardown_run(context) + assert other.calls == [] diff --git a/tst/unit/bundle/preflight_test.py b/tst/unit/bundle/preflight_test.py index 0820eca28..276018d15 100644 --- a/tst/unit/bundle/preflight_test.py +++ b/tst/unit/bundle/preflight_test.py @@ -1231,6 +1231,42 @@ def test_a_model_table_that_doesn_t_parse_says_why_rather_than_reading_as_a_miss MODEL_ENDPOINT_MISSING(frozenset({ENV_BASE_URL})) +def test_a_deploy_sending_the_configured_key_to_another_url_is_refused_before_anything_runs(bundle_dir, monkeypatch): + monkeypatch.setenv("LITELLM_BASE_URL", "http://models:4000/v1") + monkeypatch.setenv("LITELLM_API_KEY", "sk-1") + _agent("solver") + with namespace_routing(): + A2AAgent.put(id="own-url", docker_image_artifact=_image("own-url-image"), + default_env_vars={"LITELLM_BASE_URL": "http://other:5001"}) + tests = {"id": "tests", "type": "run_container_unit_tests_verifier", "sandbox_name": "h", "container_name": "c", + "command": "true", "depends_on": [], "env_vars": {"ANTHROPIC_BASE_URL": "http://other:5001"}} + _task(bundle_dir, [{**AGENT, "litellm_base_url": "http://other:5001"}], name="step-url") + _task(bundle_dir, [{**AGENT, "env_vars": {"LITELLM_BASE_URL": "http://other:5001"}}], name="env-url") + _task(bundle_dir, [{**AGENT, "a2a_agent_id": "own-url"}], name="own-url") + _task(bundle_dir, [tests], name="unit-tests") + _task(bundle_dir, [{**AGENT, "litellm_base_url": "http://MODELS:4000/"}, + {**AGENT, "id": "keyed", "litellm_base_url": "http://other:5001", + "env_vars": {"LITELLM_API_KEY": "theirs"}}, + {**tests, "env_vars": {"ANTHROPIC_BASE_URL": "http://other:5001", "ANTHROPIC_API_KEY": "x"}}], + name="fine") + + refused = "other:5001 isn't the configured model endpoint, so the configured key isn't sent there" + problems = _problems(lambda: dry_run_bundle(bundle_dir, sandbox="local", + tasks=["step-url", "env-url", "own-url", "unit-tests", "fine"])) + assert sorted(problem.split(": ")[0] for problem in problems) == [ + "tasks/env-url.json", "tasks/own-url.json", "tasks/unit-tests.json"] + assert any(problem.startswith("tasks/env-url.json: step 'agent' (and 1 more deploy): deploys agent 'solver'") + for problem in problems) # step-url's, the same problem + assert all(refused in problem for problem in problems) + assert any("step 'agent': deploys agent 'own-url', but other:5001" in problem for problem in problems) + assert any("step 'tests': ANTHROPIC_BASE_URL: other:5001" in problem for problem in problems) + assert [entry.name for entry in dry_run_bundle(bundle_dir, tasks=["fine"], sandbox="local").runs] == ["fine"] + asks = {"id": "ask", "type": "prompt_agent", "prompt_id": "agent", "prompt": "go", "depends_on": ["agent"]} + _task(bundle_dir, [{**AGENT, "litellm_base_url": "http://other:5001"}, asks], name="trained") + with namespace_routing(): # a trainer gives the solver its own endpoint and key + assert prepare_bundle(bundle_dir, tasks=["trained"], sandbox="local", solver_endpoint=True) + + def test_an_agent_declaring_the_model_variables_itself_needs_no_endpoint_but_its_judge_does(bundle_dir, no_model): with namespace_routing(): A2AAgent.put(id="echo", docker_image_artifact=_image("echo-image"), diff --git a/tst/unit/config/test_model.py b/tst/unit/config/test_model.py index 5e5c96d93..880b68263 100644 --- a/tst/unit/config/test_model.py +++ b/tst/unit/config/test_model.py @@ -15,6 +15,7 @@ config_model, named_model, resolve_model, + same_endpoint, ) @@ -217,17 +218,16 @@ def test_resolve_call_api_key_ref_resolves_lazily(): assert call == ModelCallConfig(api_key="sk-secret", api_base="https://gw/v1", params={}) -def test_base_override_to_a_foreign_url_does_not_inherit_the_configured_key(): - """The configured key is scoped to the configured endpoint — a caller-supplied - override URL must not receive the configured proxy's credential.""" - call = ModelConfig(base_url="https://proxy.example", api_key="secret:k").resolve_call( - "m", base_override="https://elsewhere.example", secret_resolver=_secrets({"k": "sk-proxy"}) - ) - assert call.api_key is None - assert call.api_base == "https://elsewhere.example" +def test_base_override_to_a_foreign_url_without_a_key_is_refused(): + """The configured key is scoped to the configured endpoint, and a call there with no key would have litellm send + whatever provider key the environment holds.""" + with pytest.raises(ConfigError, match="elsewhere.example isn't the configured model endpoint"): + ModelConfig(base_url="https://proxy.example", api_key="secret:k").resolve_call( + "m", base_override="https://elsewhere.example", secret_resolver=_secrets({"k": "sk-proxy"})) -@pytest.mark.parametrize("override", ["https://proxy.example", "https://proxy.example/"]) +@pytest.mark.parametrize("override", ["https://proxy.example", "https://proxy.example/", "https://proxy.example/v1", + "HTTPS://Proxy.Example:443"]) def test_base_override_equal_to_the_configured_base_keeps_the_key(override): call = ModelConfig(base_url="https://proxy.example", api_key="secret:k").resolve_call( "m", base_override=override, secret_resolver=_secrets({"k": "sk-proxy"}) @@ -307,7 +307,7 @@ def test_a_header_ref_that_cant_be_sent_fails_loud(secrets, problem): @pytest.mark.parametrize("call, expected", [ ({}, {"extra_headers": _RESOLVED}), - ({"base_override": "https://elsewhere.example"}, {}), + ({"base_override": "https://elsewhere.example", "default_api_key": "theirs"}, {}), ], ids=["configured", "foreign-override"]) def test_resolve_call_sends_the_headers_only_to_the_configured_endpoint(call, expected): assert _WITH_HEADERS.resolve_call("m", secret_resolver=_secrets({"org": "acme", "k": "sk"}), **call).params == expected @@ -330,3 +330,32 @@ def test_no_named_layer_leaves_the_role_to_its_config(): assert named_model(ModelRole.SOLVER) is None assert config_model(ModelRole.SOLVER, ModelConfig(default="D")) == ResolvedModel( ModelRole.SOLVER, "D", ModelSource.CONFIG_DEFAULT) + + +@pytest.mark.parametrize("a, b", [ + ("https://gw.example/v1", "https://gw.example"), + ("https://gw.example/v1/", "https://gw.example/"), + ("HTTPS://GW.Example/v1", "https://gw.example/v1"), + ("https://gw.example:443/v1", "https://gw.example/v1"), + ("http://gw.example:80", "http://gw.example"), + ("https://gw.example/litellm/v1", "https://gw.example/litellm"), + (" https://gw.example ", "https://gw.example"), +], ids=["v1", "slashes", "case", "https-default-port", "http-default-port", "path-prefix", "whitespace"]) +def test_same_endpoint_reads_one_endpoint_spelled_two_ways_as_one(a, b): + assert same_endpoint(a, b) and same_endpoint(b, a) + + +@pytest.mark.parametrize("a, b", [ + ("https://gw.example", "http://gw.example"), + ("https://gw.example", "https://gw.example:8443"), + ("https://gw.example", "https://other.example"), + ("https://gw.example/team-a", "https://gw.example/team-b"), + ("https://gw.example/Team", "https://gw.example/team"), + ("https://gw.example/v1/v1", "https://gw.example"), + ("https://gw.example?team=a", "https://gw.example?team=b"), + ("https://gw.example", None), + (None, None), + ("", ""), +], ids=["scheme", "port", "host", "path", "path-case", "two-v1s", "query", "none", "both-none", "both-empty"]) +def test_same_endpoint_tells_two_endpoints_apart(a, b): + assert not same_endpoint(a, b) diff --git a/tst/unit/task_step/local_container_urls_test.py b/tst/unit/task_step/local_container_urls_test.py index 8320dfcd3..f732f8cde 100644 --- a/tst/unit/task_step/local_container_urls_test.py +++ b/tst/unit/task_step/local_container_urls_test.py @@ -91,8 +91,8 @@ async def test_an_agent_gets_the_model_endpoint_its_container_reaches( (None, None, HEADERS), ({"LITELLM_BASE_URL": MODEL}, None, HEADERS), (None, {"LITELLM_BASE_URL": MODEL}, HEADERS), - ({"LITELLM_BASE_URL": ELSEWHERE}, None, None), - (None, {"LITELLM_BASE_URL": ELSEWHERE}, None), + ({"LITELLM_BASE_URL": ELSEWHERE, "LITELLM_API_KEY": "theirs"}, None, None), + (None, {"LITELLM_BASE_URL": ELSEWHERE, "LITELLM_API_KEY": "theirs"}, None), ({"LITELLM_EXTRA_HEADERS": '{"X-Bill-To": "mine"}'}, None, {"X-Bill-To": "mine"}), ], ids=["configured", "passed-configured", "own-configured", "passed-elsewhere", "own-elsewhere", "given-its-own"]) async def test_an_agent_gets_the_headers_only_for_the_configured_endpoint( @@ -135,7 +135,8 @@ async def test_an_agents_own_model_endpoint_is_left_as_it_declares(monkeypatch, monkeypatch.setattr(sandbox_provider, "build_sandbox_provider", lambda name: provider) with pytest.raises(_Started): - await _agent(default_env_vars={"LITELLM_BASE_URL": "http://localhost:9000"}).deploy(sandbox_type="local") + await _agent(default_env_vars={"LITELLM_BASE_URL": "http://localhost:9000", "LITELLM_API_KEY": "own"}).deploy( + sandbox_type="local") assert provider.env["LITELLM_BASE_URL"] == "http://localhost:9000" @@ -189,14 +190,15 @@ def test_a_unit_test_verifier_gets_the_model_endpoint_its_container_reaches( def test_a_model_endpoint_the_verifier_sets_itself_is_reached_through_the_host(monkeypatch, local_stores, model_endpoint): - run = _run_verifier(monkeypatch, "local", env_vars={"LITELLM_BASE_URL": "http://127.0.0.1:5001"}) + run = _run_verifier(monkeypatch, "local", env_vars={"LITELLM_BASE_URL": "http://127.0.0.1:5001", + "LITELLM_API_KEY": "theirs"}) assert "-e LITELLM_BASE_URL=http://host.docker.internal:5001 " in run @pytest.mark.parametrize("step_env, seed, expected", [ ({}, None, HEADERS), - ({"LITELLM_BASE_URL": ELSEWHERE}, None, None), + ({"LITELLM_BASE_URL": ELSEWHERE, "LITELLM_API_KEY": "theirs"}, None, None), ({}, {"litellm_extra_headers": "{}"}, HEADERS), ], ids=["configured", "elsewhere", "a-seed-column-cant-replace-them"]) def test_a_unit_test_verifier_gets_the_headers_only_for_the_configured_endpoint( diff --git a/tst/unit/task_step/test_model_key_scope.py b/tst/unit/task_step/test_model_key_scope.py index a43b9cb0d..c9b14b132 100644 --- a/tst/unit/task_step/test_model_key_scope.py +++ b/tst/unit/task_step/test_model_key_scope.py @@ -9,14 +9,19 @@ import shlex import types +from unittest.mock import AsyncMock, MagicMock + import pytest from agent_env.a2a_agent.a2a_agent import TAKES_CREDENTIALS, A2AAgent from agent_env.artifact.artifacts.docker_image import DockerImageArtifact +from agent_env.config.errors import ConfigError from agent_env.config.model import ModelConfig from agent_env.providers.sandbox_providers import sandbox_provider from agent_env.providers.sandbox_providers.sandbox_provider import SandboxProvider -from agent_env.task_step.context import DeployedSandbox, TaskStepContext +from agent_env.task_step.context import DeployedSandbox, PromptResponse, TaskStepContext +from agent_env.task_step.task_steps.verifiers.judge_utils.judge_output_format import ResultRows +from agent_env.task_step.task_steps.verifiers.rubrics_verifier import RubricsVerifierTaskStep from agent_env.task_step.task_steps.verifiers.run_container_unit_tests_verifier import ( RunContainerUnitTestsVerifierTaskStep, ) @@ -24,6 +29,7 @@ MODEL = "http://localhost:4000/v1" HEADERS = {"X-Bill-To": "acme"} ELSEWHERE = "http://127.0.0.1:5001" +REFUSED = "refused" @pytest.fixture @@ -43,13 +49,19 @@ async def create_sandbox(self, *, env, **_): raise _Started -def _deploy(monkeypatch, passed: dict | None, own: dict | None, **metadata) -> dict: +def _deploy(monkeypatch, passed: dict | None, own: dict | None, **metadata) -> dict | str: + """What the agent's container is started with, or REFUSED when the deploy stops before starting one.""" provider = _Provider() monkeypatch.setattr(sandbox_provider, "build_sandbox_provider", lambda name: provider) agent = A2AAgent(id="solver", version=1, metadata=metadata, default_env_vars=own, docker_image_artifact=DockerImageArtifact(id="img", description="d", image_name="img")) - with pytest.raises(_Started): + try: asyncio.run(agent.deploy(sandbox_type="any", env_vars=passed)) + except _Started: + pass + except ConfigError as e: + assert "127.0.0.1:5001 isn't the configured model endpoint" in str(e) and not hasattr(provider, "env") + return REFUSED headers = provider.env.get("LITELLM_EXTRA_HEADERS") return {"key": provider.env.get("LITELLM_API_KEY"), "url": provider.env.get("LITELLM_BASE_URL"), "headers": json.loads(headers) if headers else None} @@ -61,17 +73,19 @@ def _deploy(monkeypatch, passed: dict | None, own: dict | None, **metadata) -> d "passed-configured-slash": ({"LITELLM_BASE_URL": MODEL + "/"}, None, {"key": "k", "url": MODEL + "/", "headers": HEADERS}), "passed-configured-without-v1": ({"LITELLM_BASE_URL": "http://localhost:4000"}, None, - {"key": "k", "url": "http://localhost:4000", "headers": None}), + {"key": "k", "url": "http://localhost:4000", "headers": HEADERS}), "passed-configured-upper-case": ({"LITELLM_BASE_URL": "HTTP://LOCALHOST:4000/v1"}, None, - {"key": "k", "url": "HTTP://LOCALHOST:4000/v1", "headers": None}), - "passed-elsewhere-leaks-the-key": ({"LITELLM_BASE_URL": ELSEWHERE}, None, - {"key": "k", "url": ELSEWHERE, "headers": None}), + {"key": "k", "url": "HTTP://LOCALHOST:4000/v1", "headers": HEADERS}), + "passed-elsewhere-is-refused": ({"LITELLM_BASE_URL": ELSEWHERE}, None, REFUSED), "passed-elsewhere-with-its-key": ({"LITELLM_BASE_URL": ELSEWHERE, "LITELLM_API_KEY": "theirs"}, None, {"key": "theirs", "url": ELSEWHERE, "headers": None}), - "own-elsewhere-leaks-the-key": (None, {"LITELLM_BASE_URL": ELSEWHERE}, - {"key": "k", "url": ELSEWHERE, "headers": None}), + "own-elsewhere-is-refused": (None, {"LITELLM_BASE_URL": ELSEWHERE}, REFUSED), "own-elsewhere-with-its-key": (None, {"LITELLM_BASE_URL": ELSEWHERE, "LITELLM_API_KEY": "own"}, {"key": "own", "url": ELSEWHERE, "headers": None}), + "own-elsewhere-with-a-passed-key": ({"LITELLM_API_KEY": "theirs"}, {"LITELLM_BASE_URL": ELSEWHERE}, + {"key": "theirs", "url": ELSEWHERE, "headers": None}), + "passed-configured-over-own-elsewhere": ({"LITELLM_BASE_URL": MODEL}, {"LITELLM_BASE_URL": ELSEWHERE}, + {"key": "k", "url": MODEL, "headers": HEADERS}), "passed-key-only": ({"LITELLM_API_KEY": "theirs"}, None, {"key": "theirs", "url": MODEL, "headers": HEADERS}), } @@ -88,16 +102,18 @@ def test_an_agent_that_takes_no_credentials_gets_none(monkeypatch, configured): GATEWAY = "https://gw.example.com" +OTHER = "https://elsewhere.example.com" CALL_CASES = { "configured": ({}, {}, ("file-key", GATEWAY, True)), "configured-env-key": ({"LITELLM_API_KEY": "env-key"}, {}, ("env-key", GATEWAY, True)), - "elsewhere-env-key-leaks": ({"LITELLM_API_KEY": "env-key"}, {"base_override": "https://elsewhere.example.com"}, - ("env-key", "https://elsewhere.example.com", False)), - "elsewhere-no-key": ({}, {"base_override": "https://elsewhere.example.com"}, - (None, "https://elsewhere.example.com", False)), - "elsewhere-with-its-key": ({}, {"base_override": "https://elsewhere.example.com", "default_api_key": "theirs"}, - ("theirs", "https://elsewhere.example.com", False)), - "configured-with-v1": ({}, {"base_override": GATEWAY + "/v1"}, (None, GATEWAY + "/v1", False)), + "elsewhere-env-key-is-refused": ({"LITELLM_API_KEY": "env-key"}, {"base_override": OTHER}, REFUSED), + "elsewhere-no-key-is-refused": ({}, {"base_override": OTHER}, REFUSED), + "elsewhere-with-its-key": ({}, {"base_override": OTHER, "default_api_key": "theirs"}, ("theirs", OTHER, False)), + "configured-with-v1": ({}, {"base_override": GATEWAY + "/v1"}, ("file-key", GATEWAY + "/v1", True)), + "configured-with-default-port": ({}, {"base_override": "https://GW.example.com:443/"}, + ("file-key", "https://GW.example.com:443/", True)), + "env-url-is-the-configured-one": ({"LITELLM_BASE_URL": OTHER}, {"base_override": OTHER}, ("file-key", OTHER, True)), + "file-url-under-an-env-url-is-refused": ({"LITELLM_BASE_URL": OTHER}, {"base_override": GATEWAY}, REFUSED), } @@ -108,7 +124,12 @@ def test_a_model_call_core_makes_sends(monkeypatch, env, call, expected): for name, value in env.items(): monkeypatch.setenv(name, value) config = ModelConfig(base_url=GATEWAY, api_key="file-key", headers={"X-Bill-To": "acme"}) - resolved = config.resolve_call("m", secret_resolver=lambda key: None, **call) + try: + resolved = config.resolve_call("m", secret_resolver=lambda key: None, **call) + except ConfigError as e: + assert "isn't the configured model endpoint" in str(e) + assert expected == REFUSED + return assert (resolved.api_key, resolved.api_base, "extra_headers" in resolved.params) == expected @@ -136,8 +157,8 @@ async def exec_with_output(self, *args) -> tuple[int, str, str]: return 0, "ok", "" -def _verifier_env(monkeypatch, env_vars: dict) -> dict: - """The model variables a unit-test verifier's command ran with.""" +def _verifier_env(monkeypatch, env_vars: dict, step_params: dict | None = None) -> dict | str: + """The model variables a unit-test verifier's command ran with, or REFUSED when it ran none.""" sandbox = _VerifierSandbox() provider = types.SimpleNamespace(get_sandbox=lambda sandbox_id: asyncio.sleep(0, result=sandbox)) monkeypatch.setattr(sandbox_provider, "build_sandbox_provider", lambda name: provider) @@ -146,11 +167,18 @@ def _verifier_env(monkeypatch, env_vars: dict) -> dict: id=artifact_id, version=1, object_url=object_url))) context = TaskStepContext() context.metadata["deployed_docker_containers"] = [{"container_name": "c", "sandbox_name": "h"}] + if step_params: + context.metadata["user_overrides"] = {"step_params": {"tests": {"env_vars": step_params}}} context.deployed_sandboxes.append( DeployedSandbox(sandbox_name="h", sandbox_id="sb-1", sandbox_mode="vm", sandbox_type="modal_vm")) - asyncio.run(RunContainerUnitTestsVerifierTaskStep( - id="tests", version=None, sandbox_name="h", container_name="c", command="true", env_vars=env_vars, - ).execute(context)) + step = RunContainerUnitTestsVerifierTaskStep( + id="tests", version=None, sandbox_name="h", container_name="c", command="true", env_vars=env_vars) + try: + asyncio.run(step.execute(context)) + except ConfigError as e: + assert "127.0.0.1:5001 isn't the configured model endpoint" in str(e) + assert not [c for c in sandbox.commands if "timeout --kill-after" in c] + return REFUSED [run] = [c for c in sandbox.commands if "timeout --kill-after" in c] words = shlex.split(run) pairs = dict(words[i + 1].split("=", 1) for i, word in enumerate(words) if word == "-e") @@ -159,17 +187,69 @@ def _verifier_env(monkeypatch, env_vars: dict) -> dict: VERIFIER_CASES = { - "configured": ({}, {"LITELLM_API_KEY": "k", "LITELLM_BASE_URL": MODEL, "ANTHROPIC_API_KEY": "k", - "ANTHROPIC_BASE_URL": MODEL}), - "elsewhere-leaks-the-key": ({"LITELLM_BASE_URL": ELSEWHERE}, - {"LITELLM_API_KEY": "k", "LITELLM_BASE_URL": ELSEWHERE, "ANTHROPIC_API_KEY": "k", - "ANTHROPIC_BASE_URL": MODEL}), - "elsewhere-with-its-key": ({"LITELLM_BASE_URL": ELSEWHERE, "LITELLM_API_KEY": "theirs"}, + "configured": ({}, None, {"LITELLM_API_KEY": "k", "LITELLM_BASE_URL": MODEL, "ANTHROPIC_API_KEY": "k", + "ANTHROPIC_BASE_URL": MODEL}), + "elsewhere-is-refused": ({"LITELLM_BASE_URL": ELSEWHERE}, None, REFUSED), + "elsewhere-with-its-key": ({"LITELLM_BASE_URL": ELSEWHERE, "LITELLM_API_KEY": "theirs"}, None, {"LITELLM_API_KEY": "theirs", "LITELLM_BASE_URL": ELSEWHERE, "ANTHROPIC_API_KEY": "k", "ANTHROPIC_BASE_URL": MODEL}), + "anthropic-elsewhere-is-refused": ({"ANTHROPIC_BASE_URL": ELSEWHERE, "LITELLM_API_KEY": "theirs"}, None, REFUSED), + "anthropic-elsewhere-with-its-key": ({"ANTHROPIC_BASE_URL": ELSEWHERE, "ANTHROPIC_API_KEY": "theirs"}, None, + {"LITELLM_API_KEY": "k", "LITELLM_BASE_URL": MODEL, + "ANTHROPIC_API_KEY": "theirs", "ANTHROPIC_BASE_URL": ELSEWHERE}), + "elsewhere-by-step-params-is-refused": ({}, {"LITELLM_BASE_URL": ELSEWHERE}, REFUSED), + "elsewhere-by-step-params-with-the-steps-key": ({"LITELLM_API_KEY": "theirs"}, {"LITELLM_BASE_URL": ELSEWHERE}, + {"LITELLM_API_KEY": "theirs", "LITELLM_BASE_URL": ELSEWHERE, + "ANTHROPIC_API_KEY": "k", "ANTHROPIC_BASE_URL": MODEL}), +} + + +@pytest.mark.parametrize("env_vars, step_params, expected", VERIFIER_CASES.values(), ids=VERIFIER_CASES.keys()) +def test_a_unit_test_verifier_runs_its_tests_with(monkeypatch, configured, env_vars, step_params, expected): + assert _verifier_env(monkeypatch, env_vars, step_params) == expected + + +def _judge_env(monkeypatch, overrides: dict) -> dict | str: + """The model variables an agent judge is deployed with, or REFUSED when none is deployed.""" + step = RubricsVerifierTaskStep(id="judge", version=1, criteria=[{"id": "done", "description": "It is done"}], + prompt_id="p1", verifier_id="v", use_agent_judge=True, use_trajectory=False, + judge_a2a_agent_id="judge-agent") + agent = MagicMock() + agent.deploy = AsyncMock(return_value=types.SimpleNamespace( + a2a_url="http://judge", sandbox_id="sb-judge", agent_card={}, sandbox_type="modal")) + monkeypatch.setattr(A2AAgent, "get", MagicMock(return_value=agent)) + + async def judged(*, criteria, **_): + return ResultRows([{**c, "justification": "", "score": 1.0, "result": True} for c in criteria], + reasoning="", checks=[]), 0, [], None + + monkeypatch.setattr(step, "_run_judge_with_output_retries", judged) + provider = MagicMock(get_sandbox=AsyncMock(return_value=AsyncMock())) + monkeypatch.setattr(sandbox_provider, "build_sandbox_provider", lambda name: provider) + context = TaskStepContext(prompt_responses=[PromptResponse(prompt_id="p1", response="done", prompt_text="do it")], + metadata={"user_overrides": overrides}) + try: + asyncio.run(step.execute(context)) + except ConfigError as e: + assert "127.0.0.1:5001 isn't the configured model endpoint" in str(e) and not agent.deploy.called + return REFUSED + env = agent.deploy.call_args.kwargs["env_vars"] + return {"key": env.get("LITELLM_API_KEY"), "url": env.get("LITELLM_BASE_URL")} + + +JUDGE_CASES = { + "configured": ({}, {"key": "k", "url": MODEL}), + "judge-elsewhere-is-refused": ({"judge_litellm_base_url": ELSEWHERE}, REFUSED), + "run-elsewhere-is-refused": ({"litellm_base_url": ELSEWHERE}, REFUSED), + "judge-elsewhere-with-its-key": ({"judge_litellm_base_url": ELSEWHERE, "judge_litellm_api_key": "theirs"}, + {"key": "theirs", "url": ELSEWHERE}), + "run-elsewhere-with-the-runs-key": ({"litellm_base_url": ELSEWHERE, "litellm_api_key": "theirs"}, + {"key": "theirs", "url": ELSEWHERE}), + "judge-configured-spelled-otherwise": ({"judge_litellm_base_url": "http://LOCALHOST:4000/"}, + {"key": "k", "url": "http://LOCALHOST:4000/"}), } -@pytest.mark.parametrize("env_vars, expected", VERIFIER_CASES.values(), ids=VERIFIER_CASES.keys()) -def test_a_unit_test_verifier_runs_its_tests_with(monkeypatch, configured, env_vars, expected): - assert _verifier_env(monkeypatch, env_vars) == expected +@pytest.mark.parametrize("overrides, expected", JUDGE_CASES.values(), ids=JUDGE_CASES.keys()) +def test_an_agent_judge_is_deployed_with(monkeypatch, configured, overrides, expected): + assert _judge_env(monkeypatch, overrides) == expected From da3daffad5ac1f3e24b8b6f607494616699c06cf Mon Sep 17 00:00:00 2001 From: Edgar Arakelyan Date: Wed, 18 Nov 2026 13:43:42 -0800 Subject: [PATCH 3/6] test(cli): the scopes test's source machine configures the endpoint its tasks name Co-Authored-By: Claude Opus 5.5 --- tst/integration/cli/bundle_init_scopes_test.py | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/tst/integration/cli/bundle_init_scopes_test.py b/tst/integration/cli/bundle_init_scopes_test.py index 6701f0510..2df031c83 100644 --- a/tst/integration/cli/bundle_init_scopes_test.py +++ b/tst/integration/cli/bundle_init_scopes_test.py @@ -83,17 +83,19 @@ def machines(monkeypatch, tmp_path): monkeypatch.setenv("AGENT_ENV_DOCUMENT_STORE", "local") monkeypatch.setenv("AGENT_ENV_OBJECT_STORE", "local") monkeypatch.setenv("AGENT_ENV_LOCAL_SANDBOX_DIR", str(sandboxes)) - monkeypatch.setenv("LITELLM_BASE_URL", "http://127.0.0.1:4000") # each machine's model endpoint, which no run calls monkeypatch.setenv("LITELLM_API_KEY", "sk-test") config = tmp_path / "source.toml" config.write_text(f'[sandbox.providers]\nacme_vm = "{_HERE}:_AcmeVm"\n\n[task_steps]\nimpls = ["{_HERE}:_AcmeNote"]\n') def on(machine): monkeypatch.setenv("XDG_STATE_HOME", str(tmp_path / machine)) + # Each machine's model endpoint, which no run calls: the source's is the one its tasks name. if machine == "source": monkeypatch.setenv("AGENT_ENV_CONFIG", str(config)) + monkeypatch.setenv("LITELLM_BASE_URL", "https://litellm.acme.example") else: monkeypatch.delenv("AGENT_ENV_CONFIG", raising=False) + monkeypatch.setenv("LITELLM_BASE_URL", "http://127.0.0.1:4000") _reset() configure() From be0f371352b4aa6ad1e4818a95f78efc4d029264 Mon Sep 17 00:00:00 2001 From: Edgar Arakelyan Date: Wed, 18 Nov 2026 13:49:56 -0800 Subject: [PATCH 4/6] test(task_step): find the agent's container by its name Co-Authored-By: Claude Opus 5.5 --- tst/integration/task_step/test_model_key_scope_local.py | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/tst/integration/task_step/test_model_key_scope_local.py b/tst/integration/task_step/test_model_key_scope_local.py index 9c56e0ce8..1e68355a2 100644 --- a/tst/integration/task_step/test_model_key_scope_local.py +++ b/tst/integration/task_step/test_model_key_scope_local.py @@ -16,7 +16,6 @@ from agent_env.a2a_agent import A2AAgent from agent_env.artifact.store import reset_artifact_store from agent_env.config import configure, reset_config -from agent_env.providers.sandbox_providers.sandbox import SANDBOX_LABEL from agent_env.providers.sandbox_providers.sandbox_provider import ( reset_agent_sandbox_provider, reset_env_sandbox_provider, @@ -124,8 +123,10 @@ async def test_the_judge_sends_the_configured_key_only_to_the_configured_endpoin def _container_env(sandbox_id: str) -> dict[str, str]: - [container] = subprocess.run(["docker", "ps", "-aq", "--filter", f"label={SANDBOX_LABEL}={sandbox_id}"], - capture_output=True, text=True, check=True).stdout.split() + """The environment the agent's container behind ``sandbox_id`` was started with: the one named after it.""" + names = subprocess.run(["docker", "ps", "-a", "--format", "{{.Names}}"], capture_output=True, text=True, + check=True).stdout.split() + [container] = [name for name in names if sandbox_id in name] inspected = subprocess.run(["docker", "inspect", "--format", "{{json .Config.Env}}", container], capture_output=True, text=True, check=True).stdout return dict(pair.split("=", 1) for pair in json.loads(inspected)) From 36555eae562d6bcd4d9928f225875e6ff149288a Mon Sep 17 00:00:00 2001 From: Edgar Arakelyan Date: Wed, 18 Nov 2026 13:51:53 -0800 Subject: [PATCH 5/6] fix(bundle): preflight asks no model endpoint or key of an agent that takes no credentials Co-Authored-By: Claude Opus 5.5 --- src/agent_env/bundle/preflight.py | 28 +++++++++++++++++----------- tst/unit/bundle/preflight_test.py | 12 ++++++++++++ 2 files changed, 29 insertions(+), 11 deletions(-) diff --git a/src/agent_env/bundle/preflight.py b/src/agent_env/bundle/preflight.py index 4c78bf866..b2cfee9f3 100644 --- a/src/agent_env/bundle/preflight.py +++ b/src/agent_env/bundle/preflight.py @@ -31,6 +31,7 @@ from typing import Any from agent_env.a2a_agent import A2AAgent +from agent_env.a2a_agent.a2a_agent import TAKES_CREDENTIALS from agent_env.artifact.artifacts.docker_image import PUT_AGAIN, DockerImageArtifact, pull_problem from agent_env.config import get_config from agent_env.config.errors import ConfigError @@ -336,15 +337,14 @@ def _agent_deploy(self, where: str, provider: SandboxProvider, agent_id: str | N if agent_id is None: self.default_agent_users.append((where, unnamed)) agent_id = self._default_agent_id() - if needs is None: - own = self._agent_env_vars(agent_id, version) + if needs is not None: + self.model_users.append((where, needs)) + elif (own := self._agent_model_vars(agent_id, version)) is not None: url = url or own.get(ENV_BASE_URL) if url and ENV_API_KEY not in given and ENV_API_KEY not in own and not get_config().is_model_endpoint(url): self._problem(where, f"deploys agent {agent_id!r}, but {no_key_for(url)}") elif needed := _MODEL_VARS - given - own.keys(): self.model_users.append((where, needed)) - else: - self.model_users.append((where, needs)) image = self._agent_image(agent_id, version) taking = self._check(where, provider, accepts, [image] if image else [], vms_load=True) if any(link.ON_THIS_MACHINE for link in taking): @@ -368,14 +368,20 @@ def _sandbox(self, where: str, step: DeploySandboxTaskStep) -> None: # What a deploy runs - def _agent_env_vars(self, agent_id: str, version: int | None) -> dict[str, str]: - """The variables the agent ``agent_id`` declares for itself, which a deploy doesn't take from the config.""" + def _agent_model_vars(self, agent_id: str, version: int | None) -> dict[str, str] | None: + """The model variables the agent ``agent_id`` declares for itself, which a deploy doesn't take from the config; + None for an agent that takes no model credentials, which a deploy hands none.""" if (agent := self.agents.get(agent_id)) is not None: - return dict(agent.config.get("default_env_vars") or {}) - try: - return dict(A2AAgent.get(agent_id, version).default_env_vars or {}) - except (NotFoundError, ValueError, KeyError, TypeError): - return {} + env_vars, metadata = agent.config.get("default_env_vars"), agent.config.get("metadata") + else: + try: + stored = A2AAgent.get(agent_id, version) + except (NotFoundError, ValueError, KeyError, TypeError): + return {} + env_vars, metadata = stored.default_env_vars, stored.metadata + if not (metadata or {}).get(TAKES_CREDENTIALS, True): + return None + return {name: value for name, value in (env_vars or {}).items() if name in _MODEL_VARS} def _agent_image(self, agent_id: str, version: int | None) -> _Image | None: """The image the agent ``agent_id`` runs, or None when the store doesn't hold the agent or can't read it, which diff --git a/tst/unit/bundle/preflight_test.py b/tst/unit/bundle/preflight_test.py index 276018d15..558a17efc 100644 --- a/tst/unit/bundle/preflight_test.py +++ b/tst/unit/bundle/preflight_test.py @@ -1267,6 +1267,18 @@ def test_a_deploy_sending_the_configured_key_to_another_url_is_refused_before_an assert prepare_bundle(bundle_dir, tasks=["trained"], sandbox="local", solver_endpoint=True) +def test_an_agent_that_takes_no_credentials_needs_no_endpoint_or_key_wherever_its_step_points_it(bundle_dir, no_model): + with namespace_routing(): + A2AAgent.put(id="stored-bare", docker_image_artifact=_image("stored-bare-image"), + metadata={"takes_credentials": False}) + layout(bundle_dir, {"agents/folder-bare/agent.toml": 'image = { ref = "docker.io/library/python:3.12-slim" }\n\n' + '[metadata]\ntakes_credentials = false\n'}) + _task(bundle_dir, [{**AGENT, "a2a_agent_id": "stored-bare", "litellm_base_url": "http://other:5001"}, + {**AGENT, "id": "folder", "a2a_agent_id": "folder-bare"}], name="bare") + + assert [entry.name for entry in dry_run_bundle(bundle_dir, tasks=["bare"], sandbox="local").runs] == ["bare"] + + def test_an_agent_declaring_the_model_variables_itself_needs_no_endpoint_but_its_judge_does(bundle_dir, no_model): with namespace_routing(): A2AAgent.put(id="echo", docker_image_artifact=_image("echo-image"), From e295cb004844eb2723bacbfe476673da2554c15e Mon Sep 17 00:00:00 2001 From: Edgar Arakelyan Date: Wed, 18 Nov 2026 14:11:04 -0800 Subject: [PATCH 6/6] fix(verifiers): a judge's own default_model_api_base takes the run's judge key, and each refusal says how to pass one The direct judge refuses a default_model_api_base, or a run override, other than the configured endpoint before it calls resolve_model_call, naming the run overrides a judge reads its key from; an agent, a unit-test verifier and preflight each name the fields theirs read. Co-Authored-By: Claude Opus 5.5 --- src/agent_env/a2a_agent/a2a_agent.py | 5 ++++- src/agent_env/bundle/preflight.py | 4 ++-- src/agent_env/config/model.py | 10 ++++------ .../task_steps/verifiers/rubrics_verifier.py | 19 +++++++++---------- .../run_container_unit_tests_verifier.py | 2 +- .../test_rubrics_verifier_litellm_contract.py | 2 +- .../test_rubrics_verifier_screenshots.py | 16 +++++++++++----- .../test_rubrics_verifier_staging_cleanup.py | 3 ++- 8 files changed, 34 insertions(+), 27 deletions(-) diff --git a/src/agent_env/a2a_agent/a2a_agent.py b/src/agent_env/a2a_agent/a2a_agent.py index 0c1288697..b9139a6af 100644 --- a/src/agent_env/a2a_agent/a2a_agent.py +++ b/src/agent_env/a2a_agent/a2a_agent.py @@ -72,6 +72,9 @@ # Metadata an agent whose commands its player runs sets False: it is given no model endpoint, key or headers, whatever # a deploy passes, and none of the object store's shared credentials. TAKES_CREDENTIALS = "takes_credentials" +# How an agent sent to a URL other than the configured endpoint is given that URL's key. +AGENT_KEY_HOW = ("LITELLM_API_KEY in the step's env_vars or the agent's default_env_vars, or litellm_api_key in the " + "run's overrides or the step's step_params") MODEL_CREDENTIALS_ENV = ("LITELLM_API_KEY", "LITELLM_BASE_URL", MODEL_HEADERS_ENV) @@ -634,7 +637,7 @@ def _with_model_credentials(self, env_vars: dict[str, str], config) -> dict[str, if "LITELLM_API_KEY" not in resolved_env and "LITELLM_API_KEY" not in self.default_env_vars: url = resolved_env.get("LITELLM_BASE_URL") or self.default_env_vars.get("LITELLM_BASE_URL") if url and not config.is_model_endpoint(url): - raise ConfigError(f"Agent {self.id!r}: {no_key_for(url)}") + raise ConfigError(f"Agent {self.id!r}: {no_key_for(url, AGENT_KEY_HOW)}") resolved_env["LITELLM_API_KEY"] = config.get_litellm_api_key() if "LITELLM_BASE_URL" not in resolved_env and "LITELLM_BASE_URL" not in self.default_env_vars: resolved_env["LITELLM_BASE_URL"] = config.get_litellm_base_url() diff --git a/src/agent_env/bundle/preflight.py b/src/agent_env/bundle/preflight.py index b2cfee9f3..90bb4f301 100644 --- a/src/agent_env/bundle/preflight.py +++ b/src/agent_env/bundle/preflight.py @@ -31,7 +31,7 @@ from typing import Any from agent_env.a2a_agent import A2AAgent -from agent_env.a2a_agent.a2a_agent import TAKES_CREDENTIALS +from agent_env.a2a_agent.a2a_agent import AGENT_KEY_HOW, TAKES_CREDENTIALS from agent_env.artifact.artifacts.docker_image import PUT_AGAIN, DockerImageArtifact, pull_problem from agent_env.config import get_config from agent_env.config.errors import ConfigError @@ -342,7 +342,7 @@ def _agent_deploy(self, where: str, provider: SandboxProvider, agent_id: str | N elif (own := self._agent_model_vars(agent_id, version)) is not None: url = url or own.get(ENV_BASE_URL) if url and ENV_API_KEY not in given and ENV_API_KEY not in own and not get_config().is_model_endpoint(url): - self._problem(where, f"deploys agent {agent_id!r}, but {no_key_for(url)}") + self._problem(where, f"deploys agent {agent_id!r}, but {no_key_for(url, AGENT_KEY_HOW)}") elif needed := _MODEL_VARS - given - own.keys(): self.model_users.append((where, needed)) image = self._agent_image(agent_id, version) diff --git a/src/agent_env/config/model.py b/src/agent_env/config/model.py index dfbeb5f43..8d851957c 100644 --- a/src/agent_env/config/model.py +++ b/src/agent_env/config/model.py @@ -218,7 +218,7 @@ def resolve_call( ) if key is None: if api_base is not None and not same_endpoint(api_base, configured_base): - raise ConfigError(no_key_for(api_base)) + raise ConfigError(no_key_for(api_base, "default_api_key")) key = os.getenv(ENV_API_KEY) or self._resolved_api_key(secret_resolver) params = dict(self.params) headers = self.headers_for(api_base, secret_resolver=secret_resolver) @@ -301,12 +301,10 @@ def _netloc(url: str) -> str: return urlsplit(url).netloc.rpartition("@")[2] or url -def no_key_for(url: str) -> str: - """Why a model call or an agent pointed at ``url`` with no key of its own is refused, and how to give it one.""" +def no_key_for(url: str, how: str) -> str: + """Why a model call or an agent pointed at ``url`` with no key of its own is refused, and ``how`` to give it one.""" return (f"{_netloc(url)} isn't the configured model endpoint, so the configured key isn't sent there, and no key " - "was given with it. Pass one with the URL: litellm_api_key (judge_litellm_api_key for a judge) in the run's " - "overrides or a step's step_params, or LITELLM_API_KEY in the step's env_vars or the agent's " - "default_env_vars. Any non-empty value works for a server that needs no key.") + f"was given with it. Pass one with it: {how}. Any non-empty value works for a server that needs no key.") def _check_roles(roles: object) -> None: diff --git a/src/agent_env/task_step/task_steps/verifiers/rubrics_verifier.py b/src/agent_env/task_step/task_steps/verifiers/rubrics_verifier.py index 333b2ed03..6a12088bb 100644 --- a/src/agent_env/task_step/task_steps/verifiers/rubrics_verifier.py +++ b/src/agent_env/task_step/task_steps/verifiers/rubrics_verifier.py @@ -74,6 +74,8 @@ ) logger = logging.getLogger(__name__) +# How a judge sent to a URL other than the configured endpoint is given that URL's key. +_JUDGE_KEY_HOW = "judge_litellm_api_key, or litellm_api_key, in the run's overrides" # The judge echoes each criterion's id back and results are matched by exact string. @@ -916,7 +918,8 @@ async def _grade(self, context: TaskStepContext) -> TaskStepContext: ) if not api_key: if not config.is_model_endpoint(litellm_base_url): - raise ConfigError(f"{self.type} '{self.id}': its judge agent: {no_key_for(litellm_base_url)}") + raise ConfigError(f"{self.type} '{self.id}': its judge agent: " + f"{no_key_for(litellm_base_url, _JUDGE_KEY_HOW)}") api_key = config.get_litellm_api_key() env_vars = {"LITELLM_API_KEY": api_key, "LITELLM_BASE_URL": litellm_base_url} resolved_judge_id = self.judge_a2a_agent_id or config.get_default_a2a_agent_id() @@ -1355,15 +1358,11 @@ async def _prompt_llm_judge( overrides = context.metadata.get("user_overrides", {}) config = get_config() - call_config = config.resolve_model_call( - model, - default_api_key=config.resolve_reference( - overrides.get("judge_litellm_api_key") or overrides.get("litellm_api_key") - ), - base_override=overrides.get("judge_litellm_base_url") - or overrides.get("litellm_base_url") - or self.default_model_api_base, - ) + api_key = config.resolve_reference(overrides.get("judge_litellm_api_key") or overrides.get("litellm_api_key")) + base = overrides.get("judge_litellm_base_url") or overrides.get("litellm_base_url") or self.default_model_api_base + if base and not api_key and not config.is_model_endpoint(base): + raise ConfigError(f"{self.type} '{self.id}': {no_key_for(base, _JUDGE_KEY_HOW)}") + call_config = config.resolve_model_call(model, default_api_key=api_key, base_override=base) text_blocks: list[dict] = [{"type": "text", "text": eval_prompt}] cached = bool(cached_prefix) and eval_prompt.startswith(cached_prefix or "") and _caches_prompts(model) if cached: diff --git a/src/agent_env/task_step/task_steps/verifiers/run_container_unit_tests_verifier.py b/src/agent_env/task_step/task_steps/verifiers/run_container_unit_tests_verifier.py index 41b1a86ab..a06203a84 100644 --- a/src/agent_env/task_step/task_steps/verifiers/run_container_unit_tests_verifier.py +++ b/src/agent_env/task_step/task_steps/verifiers/run_container_unit_tests_verifier.py @@ -351,7 +351,7 @@ def model_key_problem(self, extra_env: Optional[Mapping[str, str]] = None) -> Op for url_name, key_name in (("LITELLM_BASE_URL", "LITELLM_API_KEY"), ("ANTHROPIC_BASE_URL", "ANTHROPIC_API_KEY")): url = given.get(url_name) if url and key_name not in given and not get_config().is_model_endpoint(url): - return f"{url_name}: {no_key_for(url)}" + return f"{url_name}: {no_key_for(url, f'{key_name} in the env_vars of the step or its step_params')}" return None def _merged_env(self, config, sandbox_type: Optional[str], extra_env: dict[str, str]) -> dict[str, str]: diff --git a/tst/unit/task_step/verifiers/test_rubrics_verifier_litellm_contract.py b/tst/unit/task_step/verifiers/test_rubrics_verifier_litellm_contract.py index b1c38e2b2..56dc56d6f 100644 --- a/tst/unit/task_step/verifiers/test_rubrics_verifier_litellm_contract.py +++ b/tst/unit/task_step/verifiers/test_rubrics_verifier_litellm_contract.py @@ -28,7 +28,7 @@ def judge(monkeypatch): config = cfg_mod.Config() config.set_secret_store(LocalSecretStore(values={"litellm_api_key": "k"}, use_env=False)) - monkeypatch.setattr(config, "_model_cfg", ModelConfig()) + monkeypatch.setattr(config, "_model_cfg", ModelConfig(base_url="https://judge.invalid/v1")) monkeypatch.setattr(cfg_mod, "get_config", lambda: config) monkeypatch.delenv("LITELLM_BASE_URL", raising=False) monkeypatch.delenv("LITELLM_API_KEY", raising=False) diff --git a/tst/unit/task_step/verifiers/test_rubrics_verifier_screenshots.py b/tst/unit/task_step/verifiers/test_rubrics_verifier_screenshots.py index 9ffaf1e4c..0ec935838 100644 --- a/tst/unit/task_step/verifiers/test_rubrics_verifier_screenshots.py +++ b/tst/unit/task_step/verifiers/test_rubrics_verifier_screenshots.py @@ -11,6 +11,7 @@ import pytest +from agent_env.config.errors import ConfigError from agent_env.config.model import ModelConfig from agent_env.task_step.context import PromptResponse, TaskStepContext from agent_env.task_step.task_steps.verifiers.judge_utils.judge_output_format import SCREENSHOT_GROUNDING @@ -425,16 +426,21 @@ async def fake_acompletion(**kwargs): default_model_api_base="https://verifier-base.example/v1", verifier_id="vt", ) - # Model passed through verbatim; no inferred provider; base = default_model_api_base. - await v._prompt_llm_judge("eval", model="openai/gemini-pro-latest", - context=TaskStepContext(), image_blocks=None) + # Model passed through verbatim; no inferred provider; base = default_model_api_base, with the run's judge key. + await v._prompt_llm_judge("eval", model="openai/gemini-pro-latest", image_blocks=None, + context=TaskStepContext(metadata={"user_overrides": {"judge_litellm_api_key": "jk"}})) assert calls[-1]["model"] == "openai/gemini-pro-latest" assert "custom_llm_provider" not in calls[-1] - assert calls[-1]["api_base"] == "https://verifier-base.example/v1" + assert (calls[-1]["api_base"], calls[-1]["api_key"]) == ("https://verifier-base.example/v1", "jk") + + # Without a key of its own, the configured key isn't sent to default_model_api_base: the call is refused. + with pytest.raises(ConfigError, match="verifier-base.example isn't the configured model endpoint"): + await v._prompt_llm_judge("eval", model="openai/gemini-pro-latest", context=TaskStepContext(), + image_blocks=None) # A per-run override wins the base precedence. ctx_override = TaskStepContext( - metadata={"user_overrides": {"litellm_base_url": "https://override.example/v1"}}) + metadata={"user_overrides": {"litellm_base_url": "https://override.example/v1", "litellm_api_key": "rk"}}) await v._prompt_llm_judge("eval", model="openai/gemini-pro-latest", context=ctx_override, image_blocks=None) assert calls[-1]["api_base"] == "https://override.example/v1" diff --git a/tst/unit/task_step/verifiers/test_rubrics_verifier_staging_cleanup.py b/tst/unit/task_step/verifiers/test_rubrics_verifier_staging_cleanup.py index 4de0cad10..4e8f2c707 100644 --- a/tst/unit/task_step/verifiers/test_rubrics_verifier_staging_cleanup.py +++ b/tst/unit/task_step/verifiers/test_rubrics_verifier_staging_cleanup.py @@ -39,7 +39,8 @@ async def deploy(self, **kwargs): return SimpleNamespace(a2a_url="http://judge.test", sandbox_id="sb-judge", agent_card={}, sandbox_type=None) config = SimpleNamespace(resolve_reference=lambda v: v, get_litellm_api_key=lambda: "k", - get_litellm_base_url=lambda: "http://llm.test", get_default_a2a_agent_id=lambda: "judge") + get_litellm_base_url=lambda: "http://llm.test", get_default_a2a_agent_id=lambda: "judge", + is_model_endpoint=lambda url: url == "http://llm.test") monkeypatch.setattr(agent_env_config, "get_config", lambda: config) monkeypatch.setattr(A2AAgent, "get", staticmethod(lambda agent_id: Judge())) monkeypatch.setattr(sandbox_provider, "get_agent_sandbox_provider", lambda: Provider())