Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
13 changes: 11 additions & 2 deletions src/agent_env/a2a_agent/a2a_agent.py
Original file line number Diff line number Diff line change
Expand Up @@ -15,7 +15,8 @@
from agentenv_protocol.types import CONTROL_TOKEN_ENV

from agent_env.config import get_config
from agent_env.config.model import MODEL_HEADERS_ENV
from agent_env.config.errors import ConfigError
from agent_env.config.model import MODEL_HEADERS_ENV, no_key_for
from agent_env.entity.refs import EntityRef
from agent_env.plugins._bundles import installed_aliases
from agent_env.store.base import NotFoundError
Expand Down Expand Up @@ -71,6 +72,9 @@
# Metadata an agent whose commands its player runs sets False: it is given no model endpoint, key or headers, whatever
# a deploy passes, and none of the object store's shared credentials.
TAKES_CREDENTIALS = "takes_credentials"
# How an agent sent to a URL other than the configured endpoint is given that URL's key.
AGENT_KEY_HOW = ("LITELLM_API_KEY in the step's env_vars or the agent's default_env_vars, or litellm_api_key in the "
"run's overrides or the step's step_params")
MODEL_CREDENTIALS_ENV = ("LITELLM_API_KEY", "LITELLM_BASE_URL", MODEL_HEADERS_ENV)


Expand Down Expand Up @@ -624,11 +628,16 @@ def _refusal(self, image: DockerImageArtifact, link: SandboxProvider) -> str | N

def _with_model_credentials(self, env_vars: dict[str, str], config) -> dict[str, str]:
"""``env_vars`` with the model endpoint, key and headers each taken from config when neither they nor
``default_env_vars`` set it; for an agent that takes no credentials, with none of them, whatever set them."""
``default_env_vars`` set it; for an agent that takes no credentials, with none of them, whatever set them. The
configured key goes only with the configured endpoint: an agent sent elsewhere with no key of its own is
refused before anything starts."""
if not self.metadata.get(TAKES_CREDENTIALS, True):
return {name: value for name, value in env_vars.items() if name not in MODEL_CREDENTIALS_ENV}
resolved_env = dict(env_vars)
if "LITELLM_API_KEY" not in resolved_env and "LITELLM_API_KEY" not in self.default_env_vars:
url = resolved_env.get("LITELLM_BASE_URL") or self.default_env_vars.get("LITELLM_BASE_URL")
if url and not config.is_model_endpoint(url):
raise ConfigError(f"Agent {self.id!r}: {no_key_for(url, AGENT_KEY_HOW)}")
resolved_env["LITELLM_API_KEY"] = config.get_litellm_api_key()
if "LITELLM_BASE_URL" not in resolved_env and "LITELLM_BASE_URL" not in self.default_env_vars:
resolved_env["LITELLM_BASE_URL"] = config.get_litellm_base_url()
Expand Down
48 changes: 35 additions & 13 deletions src/agent_env/bundle/preflight.py
Original file line number Diff line number Diff line change
Expand Up @@ -31,10 +31,11 @@
from typing import Any

from agent_env.a2a_agent import A2AAgent
from agent_env.a2a_agent.a2a_agent import AGENT_KEY_HOW, TAKES_CREDENTIALS
from agent_env.artifact.artifacts.docker_image import PUT_AGAIN, DockerImageArtifact, pull_problem
from agent_env.config import get_config
from agent_env.config.errors import ConfigError
from agent_env.config.model import ENV_API_KEY, ENV_BASE_URL
from agent_env.config.model import ENV_API_KEY, ENV_BASE_URL, no_key_for
from agent_env.entity.refs import EntityKind, parse_toml_ref
from agent_env.env.bootstrap import (
GATEWAY,
Expand Down Expand Up @@ -77,6 +78,9 @@
from agent_env.task_step.task_steps.deploy_env import DeployEnvTaskStep
from agent_env.task_step.task_steps.deploy_sandbox import DeploySandboxTaskStep
from agent_env.task_step.task_steps.verifiers.rubrics_verifier import RubricsVerifierTaskStep
from agent_env.task_step.task_steps.verifiers.run_container_unit_tests_verifier import (
RunContainerUnitTestsVerifierTaskStep,
)
from agent_env.utils.docker.build import DEFAULT_BUILD_PLATFORM, docker_plugin_missing, docker_unreachable

from .parse import BundleError, BundleKind
Expand Down Expand Up @@ -170,6 +174,9 @@ def task(self, task: ResolvedEntry, skip: frozenset[str], given: frozenset[str]
self._sandbox(where, step)
elif isinstance(step, RubricsVerifierTaskStep):
self._judge(where, step)
elif isinstance(step, RunContainerUnitTestsVerifierTaskStep):
if problem := step.model_key_problem():
self._problem(where, problem)
elif isinstance(step, InstallAgentTaskStep) and step.a2a_agent_id is None and override is None:
self.default_agent_users.append((where, "names no agent"))
except ValueError as e: # a provider that names no backend, or can't be built from its config
Expand Down Expand Up @@ -311,7 +318,8 @@ def _agent(self, where: str, step: DeployAgentTaskStep, sandboxes: dict[str, Dep
*(_MODEL_VARS if endpoint_given else ())})
agent_id, version = ((override.agent_id, override.version) if override is not None
else (step.a2a_agent_id, step.a2a_agent_version))
self._agent_deploy(where, provider, agent_id, version, "names no agent", accepts, given=given)
self._agent_deploy(where, provider, agent_id, version, "names no agent", accepts, given=given,
url=step.litellm_base_url or step.env_vars.get(ENV_BASE_URL))

def _judge(self, where: str, step: RubricsVerifierTaskStep) -> None:
if not step.use_agent_judge or step.agent_name is not None:
Expand All @@ -322,13 +330,21 @@ def _judge(self, where: str, step: RubricsVerifierTaskStep) -> None:

def _agent_deploy(self, where: str, provider: SandboxProvider, agent_id: str | None, version: int | None,
unnamed: str, accepts: Callable[[SandboxProvider], Accepts] = A2AAgent.accepts, *,
given: frozenset[str] = frozenset(), needs: frozenset[str] | None = None) -> None:
"""Check an agent deploy, ``given`` the model variables its step sets, or ``needs`` those the config must."""
given: frozenset[str] = frozenset(), needs: frozenset[str] | None = None,
url: str | None = None) -> None:
"""Check an agent deploy, ``given`` the model variables its step sets and ``url`` the endpoint it names, or
``needs`` those the config must."""
if agent_id is None:
self.default_agent_users.append((where, unnamed))
agent_id = self._default_agent_id()
if needed := needs if needs is not None else _MODEL_VARS - given - self._agent_env_vars(agent_id, version):
self.model_users.append((where, needed))
if needs is not None:
self.model_users.append((where, needs))
elif (own := self._agent_model_vars(agent_id, version)) is not None:
url = url or own.get(ENV_BASE_URL)
if url and ENV_API_KEY not in given and ENV_API_KEY not in own and not get_config().is_model_endpoint(url):
self._problem(where, f"deploys agent {agent_id!r}, but {no_key_for(url, AGENT_KEY_HOW)}")
elif needed := _MODEL_VARS - given - own.keys():
self.model_users.append((where, needed))
image = self._agent_image(agent_id, version)
taking = self._check(where, provider, accepts, [image] if image else [], vms_load=True)
if any(link.ON_THIS_MACHINE for link in taking):
Expand All @@ -352,14 +368,20 @@ def _sandbox(self, where: str, step: DeploySandboxTaskStep) -> None:

# What a deploy runs

def _agent_env_vars(self, agent_id: str, version: int | None) -> frozenset[str]:
"""The variables the agent ``agent_id`` declares for itself, which a deploy doesn't take from the config."""
def _agent_model_vars(self, agent_id: str, version: int | None) -> dict[str, str] | None:
"""The model variables the agent ``agent_id`` declares for itself, which a deploy doesn't take from the config;
None for an agent that takes no model credentials, which a deploy hands none."""
if (agent := self.agents.get(agent_id)) is not None:
return frozenset(agent.config.get("default_env_vars") or {})
try:
return frozenset(A2AAgent.get(agent_id, version).default_env_vars or {})
except (NotFoundError, ValueError, KeyError, TypeError):
return frozenset()
env_vars, metadata = agent.config.get("default_env_vars"), agent.config.get("metadata")
else:
try:
stored = A2AAgent.get(agent_id, version)
except (NotFoundError, ValueError, KeyError, TypeError):
return {}
env_vars, metadata = stored.default_env_vars, stored.metadata
if not (metadata or {}).get(TAKES_CREDENTIALS, True):
return None
return {name: value for name, value in (env_vars or {}).items() if name in _MODEL_VARS}

def _agent_image(self, agent_id: str, version: int | None) -> _Image | None:
"""The image the agent ``agent_id`` runs, or None when the store doesn't hold the agent or can't read it, which
Expand Down
60 changes: 49 additions & 11 deletions src/agent_env/config/model.py
Original file line number Diff line number Diff line change
Expand Up @@ -10,7 +10,7 @@
from dataclasses import dataclass, field, fields
from enum import StrEnum
from typing import Optional
from urllib.parse import urlparse
from urllib.parse import urlsplit

from agent_env.config.errors import ConfigError
from agent_env.config.loader import ENV_REF_PREFIX, SECRET_REF_PREFIX, SecretResolver, interpolate
Expand Down Expand Up @@ -170,15 +170,19 @@ def _resolved_api_key(self, secret_resolver: SecretResolver) -> Optional[str]:
return None
return interpolate(self.api_key, secret_resolver=secret_resolver)

def is_endpoint(self, url: Optional[str]) -> bool:
"""Whether ``url`` names the configured endpoint (``LITELLM_BASE_URL``, else ``base_url``), by
``same_endpoint``: the only URL the configured key and headers go to."""
return same_endpoint(url, os.getenv(ENV_BASE_URL) or self.base_url)

def headers_for(self, base_url: Optional[str], *, secret_resolver: SecretResolver) -> dict[str, str]:
"""``[model.headers]``, resolved, for a call or an agent pointed at ``base_url``: none unless
that is the configured endpoint, since a header may carry a credential."""
if not self.headers or not base_url:
return {}
configured_base = os.getenv(ENV_BASE_URL) or self.base_url
if not configured_base or base_url.rstrip("/") != configured_base.rstrip("/"):
if not self.is_endpoint(base_url):
logger.info("Not sending [model.headers] to %s, which is not the configured model endpoint",
urlparse(base_url).hostname or base_url)
_netloc(base_url))
return {}
headers = interpolate(dict(self.headers), secret_resolver=secret_resolver)
for name, value in headers.items():
Expand All @@ -194,8 +198,11 @@ def resolve_call(
secret_resolver: SecretResolver,
) -> ModelCallConfig:
"""A provider-prefixed model routes natively when no endpoint is configured; a bare
model name needs one (override/env/config) and fails actionably without it."""
key = default_api_key or os.getenv(ENV_API_KEY)
model name needs one (override/env/config) and fails actionably without it. The configured key, from
``LITELLM_API_KEY`` or ``api_key``, goes only to the configured endpoint or a native route; a ``base_override``
elsewhere needs ``default_api_key``, since without one ``litellm`` would fall back to provider keys in the
environment."""
key = default_api_key
configured_base = os.getenv(ENV_BASE_URL) or self.base_url
if base_override:
api_base = base_override
Expand All @@ -209,11 +216,10 @@ def resolve_call(
f".agentenv/config.toml or {ENV_BASE_URL}, or use a provider-prefixed "
"model name (e.g. 'openai/gpt-4o') for native routing."
)
# The configured key is scoped to the configured endpoint (and native routing):
# a base_override pointing elsewhere must not inherit it, or a caller-supplied
# URL would receive the configured proxy's credential.
if key is None and (api_base is None or api_base.rstrip("/") == (configured_base or "").rstrip("/")):
key = self._resolved_api_key(secret_resolver)
if key is None:
if api_base is not None and not same_endpoint(api_base, configured_base):
raise ConfigError(no_key_for(api_base, "default_api_key"))
key = os.getenv(ENV_API_KEY) or self._resolved_api_key(secret_resolver)
params = dict(self.params)
headers = self.headers_for(api_base, secret_resolver=secret_resolver)
if headers:
Expand Down Expand Up @@ -269,6 +275,38 @@ def config_model(role: ModelRole, config: Optional[ModelConfig]) -> ResolvedMode
return ResolvedModel(role, None, None)


_DEFAULT_PORTS = {"http": 80, "https": 443}


def same_endpoint(a: Optional[str], b: Optional[str]) -> bool:
"""Whether two URLs name one model endpoint: scheme and host in any case, a default port, a trailing ``/`` and one
trailing ``/v1`` don't tell them apart, so a client that wants ``/v1`` and one that doesn't reach the same one."""
return bool(a and b) and _endpoint(a) == _endpoint(b)


def _endpoint(url: str) -> tuple:
parts = urlsplit(url.strip())
scheme = parts.scheme.lower()
try:
port = parts.port
except ValueError:
return (url,)
path = parts.path.rstrip("/")
path = path.removesuffix("/v1").rstrip("/")
return scheme, (parts.hostname or "").lower(), None if port == _DEFAULT_PORTS.get(scheme) else port, path, parts.query


def _netloc(url: str) -> str:
"""``url``'s host and port, without any user or password it carries."""
return urlsplit(url).netloc.rpartition("@")[2] or url


def no_key_for(url: str, how: str) -> str:
"""Why a model call or an agent pointed at ``url`` with no key of its own is refused, and ``how`` to give it one."""
return (f"{_netloc(url)} isn't the configured model endpoint, so the configured key isn't sent there, and no key "
f"was given with it. Pass one with it: {how}. Any non-empty value works for a server that needs no key.")


def _check_roles(roles: object) -> None:
"""Refuse a ``[model.roles]`` that maps a role to anything but a model name, or names one role twice, under its
name and its synonym."""
Expand Down
3 changes: 3 additions & 0 deletions src/agent_env/config/runtime.py
Original file line number Diff line number Diff line change
Expand Up @@ -488,6 +488,9 @@ def get_model_params(self, overrides: dict | None = None) -> dict:
def get_model_headers(self, base_url: str | None) -> dict[str, str]:
return self._model_config().headers_for(base_url, secret_resolver=self._resolve_secret)

def is_model_endpoint(self, url: str | None) -> bool:
return self._model_config().is_endpoint(url)

def resolve_model_call(
self,
model: str,
Expand Down
30 changes: 16 additions & 14 deletions src/agent_env/task_step/task_steps/verifiers/rubrics_verifier.py
Original file line number Diff line number Diff line change
Expand Up @@ -19,7 +19,8 @@
from agent_env.a2a_agent.a2a_agent import unwritten_local_agent
from agent_env.a2a_agent.staging import transfer_store
from agent_env.config import get_config
from agent_env.config.model import ModelParam, ModelRole
from agent_env.config.errors import ConfigError
from agent_env.config.model import ModelParam, ModelRole, no_key_for
from agent_env.providers.sandbox_providers.sandbox import into_container
from agent_env.task_step.context import (
PromptResponse,
Expand Down Expand Up @@ -73,6 +74,8 @@
)

logger = logging.getLogger(__name__)
# How a judge sent to a URL other than the configured endpoint is given that URL's key.
_JUDGE_KEY_HOW = "judge_litellm_api_key, or litellm_api_key, in the run's overrides"


# The judge echoes each criterion's id back and results are matched by exact string.
Expand Down Expand Up @@ -906,15 +909,18 @@ async def _grade(self, context: TaskStepContext) -> TaskStepContext:
from agent_env.config import get_config

config = get_config()
api_key = (
config.resolve_reference(overrides.get("judge_litellm_api_key") or overrides.get("litellm_api_key"))
or config.get_litellm_api_key()
)
api_key = config.resolve_reference(
overrides.get("judge_litellm_api_key") or overrides.get("litellm_api_key"))
litellm_base_url = (
overrides.get("judge_litellm_base_url")
or overrides.get("litellm_base_url")
or config.get_litellm_base_url()
)
if not api_key:
if not config.is_model_endpoint(litellm_base_url):
raise ConfigError(f"{self.type} '{self.id}': its judge agent: "
f"{no_key_for(litellm_base_url, _JUDGE_KEY_HOW)}")
api_key = config.get_litellm_api_key()
env_vars = {"LITELLM_API_KEY": api_key, "LITELLM_BASE_URL": litellm_base_url}
resolved_judge_id = self.judge_a2a_agent_id or config.get_default_a2a_agent_id()
try:
Expand Down Expand Up @@ -1352,15 +1358,11 @@ async def _prompt_llm_judge(

overrides = context.metadata.get("user_overrides", {})
config = get_config()
call_config = config.resolve_model_call(
model,
default_api_key=config.resolve_reference(
overrides.get("judge_litellm_api_key") or overrides.get("litellm_api_key")
),
base_override=overrides.get("judge_litellm_base_url")
or overrides.get("litellm_base_url")
or self.default_model_api_base,
)
api_key = config.resolve_reference(overrides.get("judge_litellm_api_key") or overrides.get("litellm_api_key"))
base = overrides.get("judge_litellm_base_url") or overrides.get("litellm_base_url") or self.default_model_api_base
if base and not api_key and not config.is_model_endpoint(base):
raise ConfigError(f"{self.type} '{self.id}': {no_key_for(base, _JUDGE_KEY_HOW)}")
call_config = config.resolve_model_call(model, default_api_key=api_key, base_override=base)
text_blocks: list[dict] = [{"type": "text", "text": eval_prompt}]
cached = bool(cached_prefix) and eval_prompt.startswith(cached_prefix or "") and _caches_prompts(model)
if cached:
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -36,11 +36,11 @@
import tempfile
import time
import uuid
from collections.abc import Awaitable, Callable
from collections.abc import Awaitable, Callable, Mapping
from typing import Any, ClassVar, Optional

from agent_env.config import ConfigError, get_config
from agent_env.config.model import MODEL_HEADERS_ENV
from agent_env.config.model import MODEL_HEADERS_ENV, no_key_for
from agent_env.entity.refs import EntityRef
from agent_env.env.env import require_sandbox
from agent_env.env.envs.computer.utils import inside
Expand Down Expand Up @@ -344,10 +344,22 @@ async def execute(self, context: TaskStepContext) -> TaskStepContext:

return context

def model_key_problem(self, extra_env: Optional[Mapping[str, str]] = None) -> Optional[str]:
"""Why the commands can't be given the configured key: a model URL among ``env_vars`` and ``extra_env``
that isn't the configured endpoint, with no key of its own beside it; None when there's none."""
given = {**self.env_vars, **(extra_env or {})}
for url_name, key_name in (("LITELLM_BASE_URL", "LITELLM_API_KEY"), ("ANTHROPIC_BASE_URL", "ANTHROPIC_API_KEY")):
url = given.get(url_name)
if url and key_name not in given and not get_config().is_model_endpoint(url):
return f"{url_name}: {no_key_for(url, f'{key_name} in the env_vars of the step or its step_params')}"
return None

def _merged_env(self, config, sandbox_type: Optional[str], extra_env: dict[str, str]) -> dict[str, str]:
"""The commands' environment: the model endpoint's key and URL from agent-env's config, every sandbox
provider's container env, ``env_vars`` and this run's ``extra_env``, then the endpoint's headers, its URL as
``sandbox_type``'s containers reach it."""
``sandbox_type``'s containers reach it. A URL other than the configured endpoint needs its key beside it."""
if problem := self.model_key_problem(extra_env):
raise ConfigError(f"{self.type} '{self.id}': {problem}")
merged_env: dict[str, str] = {}
try:
litellm_key = config.get_litellm_api_key()
Expand Down
Loading
Loading