Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
4 changes: 3 additions & 1 deletion src/agent_env/bundle/preflight.py
Original file line number Diff line number Diff line change
Expand Up @@ -588,8 +588,10 @@ def _provider(spec: str | None, default: Callable[[], SandboxProvider]) -> Sandb


def model_endpoint_missing(needed: frozenset[str]) -> tuple[str, str] | None:
"""The first of the ``needed`` model variables the config can't give, and why; None when it gives them all."""
"""The first of the ``needed`` model variables the config can't give, and why; None when it gives them all. A
``[model]`` table that doesn't parse raises its own error, which would otherwise read as a missing endpoint."""
config = get_config()
config.get_default_model()
if ENV_BASE_URL in needed:
try:
config.get_litellm_base_url()
Expand Down
124 changes: 106 additions & 18 deletions src/agent_env/config/model.py
Original file line number Diff line number Diff line change
Expand Up @@ -42,6 +42,45 @@ class ModelParam(StrEnum):
MODEL_PARAMS_RESERVED = frozenset(p.value for p in ModelParam)


class ModelRole(StrEnum):
"""A role a model plays in a run. ``[model.roles]`` also takes the names it had first: ``agent`` for the solver
and ``user`` for the user simulator."""

SOLVER = "solver"
USER_SIMULATOR = "user_simulator"
JUDGE = "judge"

@property
def record_key(self) -> str:
"""The role's key in ``metadata.models``, which its readers have always seen."""
return "user simulator" if self is ModelRole.USER_SIMULATOR else self.value


ROLE_SYNONYMS: dict[str, ModelRole] = {"agent": ModelRole.SOLVER, "user": ModelRole.USER_SIMULATOR}
JUDGE_FALLBACK_MODEL = "claude-sonnet-4-6"


class ModelSource(StrEnum):
"""The layer a role's model came from: the first of them, in this order, that names one."""

STEP_PARAMS = "step_params"
RUN = "run"
STEP = "step"
AGENT = "agent"
CONFIG_ROLE = "config_role"
CONFIG_DEFAULT = "config_default"
FALLBACK = "fallback"


@dataclass(frozen=True)
class ResolvedModel:
"""A role's model for one step and the layer it came from; both None when no layer names one."""

role: ModelRole
model: Optional[str]
source: Optional[ModelSource]


@dataclass(frozen=True)
class ModelCallConfig:
"""Resolved api_base / api_key / params for one model call."""
Expand Down Expand Up @@ -94,6 +133,7 @@ def from_section(cls, section: dict, *, secret_resolver: SecretResolver) -> Mode
{k: v for k, v in section.items() if k not in ("api_key", "headers")},
secret_resolver=secret_resolver,
)
_check_roles(cfg.get("roles", {}))
return cls(
base_url=cfg.get("base_url"),
api_key=section.get("api_key"),
Expand All @@ -103,9 +143,16 @@ def from_section(cls, section: dict, *, secret_resolver: SecretResolver) -> Mode
headers=dict(headers),
)

def role_model(self, role: str) -> Optional[str]:
"""``[model.roles]``'s model for ``role``, under its name or the synonym it had first; the map is open, so a
plugin's own role is looked up by its name."""
canonical = ROLE_SYNONYMS.get(role, role)
names = (canonical, *(synonym for synonym, named in ROLE_SYNONYMS.items() if named == canonical))
return next((self.roles[name] for name in names if self.roles.get(name)), None)

def model_for_role(self, role: str, *, or_default: bool = True) -> Optional[str]:
"""The ``[model.roles].<role>`` model, else ``default`` unless ``or_default`` is False (open map)."""
return self.roles.get(role) or (self.default if or_default else None)
"""The ``[model.roles].<role>`` model, else ``default`` unless ``or_default`` is False."""
return self.role_model(role) or (self.default if or_default else None)

def resolve_api_key(self, *, secret_resolver: SecretResolver) -> str:
"""Key: env ``LITELLM_API_KEY`` > ``api_key`` (resolving a reference lazily);
Expand Down Expand Up @@ -144,26 +191,10 @@ def resolve_call(
*,
default_api_key: Optional[str] = None,
base_override: Optional[str] = None,
model_overrides: Optional[dict[str, dict[str, str]]] = None,
secret_resolver: SecretResolver,
) -> ModelCallConfig:
"""A provider-prefixed model routes natively when no endpoint is configured; a bare
model name needs one (override/env/config) and fails actionably without it."""
if model_overrides:
override = model_overrides.get(model)
if override:
key = secret_resolver(override["secret_key"])
if key is None:
raise ConfigError(
f"Model override for {model!r} names secret {override['secret_key']!r}, "
"which the configured secret store has no value for"
)
return ModelCallConfig(
api_key=key,
api_base=base_override or override["base_url"],
params={},
)

key = default_api_key or os.getenv(ENV_API_KEY)
configured_base = os.getenv(ENV_BASE_URL) or self.base_url
if base_override:
Expand Down Expand Up @@ -192,6 +223,63 @@ def resolve_call(
return ModelCallConfig(api_key=key, api_base=api_base, params=params)


def resolve_model(
role: ModelRole,
*,
step_params: Optional[str] = None,
run: Optional[str] = None,
step: Optional[str] = None,
agent: Optional[str] = None,
config: Optional[ModelConfig] = None,
) -> ResolvedModel:
"""``role``'s model for one step: the first of the layers its call site has that names one, else ``config``'s
(``named_model``, then ``config_model``). It reads no environment, key or secret, so a plan can resolve every role
before its run starts."""
return (named_model(role, step_params=step_params, run=run, step=step, agent=agent)
or config_model(role, config))


def named_model(
role: ModelRole,
*,
step_params: Optional[str] = None,
run: Optional[str] = None,
step: Optional[str] = None,
agent: Optional[str] = None,
) -> Optional[ResolvedModel]:
"""The first of these layers that names a model: this step's ``step_params``, the run's override, the step's own
field, the deployed agent's default; None when none does."""
for source, model in ((ModelSource.STEP_PARAMS, step_params), (ModelSource.RUN, run),
(ModelSource.STEP, step), (ModelSource.AGENT, agent)):
if model:
return ResolvedModel(role, model, source)
return None


def config_model(role: ModelRole, config: Optional[ModelConfig]) -> ResolvedModel:
"""``role``'s model when no layer names one: ``[model.roles]``, then for the solver and the judge ``[model]
default``; a judge always has one, claude-sonnet-4-6 last."""
if config is not None:
if model := config.role_model(role):
return ResolvedModel(role, model, ModelSource.CONFIG_ROLE)
if role is not ModelRole.USER_SIMULATOR and config.default:
return ResolvedModel(role, config.default, ModelSource.CONFIG_DEFAULT)
if role is ModelRole.JUDGE:
return ResolvedModel(role, JUDGE_FALLBACK_MODEL, ModelSource.FALLBACK)
return ResolvedModel(role, None, None)


def _check_roles(roles: object) -> None:
"""Refuse a ``[model.roles]`` that maps a role to anything but a model name, or names one role twice, under its
name and its synonym."""
if not isinstance(roles, dict) or not all(isinstance(model, str) for model in roles.values()):
raise ConfigError("[model.roles] maps each role to a model name, such as judge = \"claude-sonnet-4-6\"")
for synonym, role in ROLE_SYNONYMS.items():
if synonym in roles and role.value in roles:
raise ConfigError(f"[model.roles] sets both {role.value!r} and {synonym!r}, which name the same role; "
f"keep {role.value!r}")


def _check_header(name: str, value: Optional[str]) -> None:
"""Refuse a ``[model.headers]`` entry HTTP can't carry (``value`` None checks the name alone): a non-ASCII
value would fail inside an agent's client, and a CR or LF would split one header into two."""
Expand Down
17 changes: 15 additions & 2 deletions src/agent_env/config/runtime.py
Original file line number Diff line number Diff line change
Expand Up @@ -464,6 +464,21 @@ def get_default_model(self) -> str | None:
def get_model_for_role(self, role: str, *, or_default: bool = True) -> str | None:
return self._model_config().model_for_role(role, or_default=or_default)

def resolve_model(
self,
role: model_config.ModelRole,
*,
step_params: str | None = None,
run: str | None = None,
step: str | None = None,
agent: str | None = None,
) -> model_config.ResolvedModel:
"""``role``'s model for one step, falling back to this config's ``[model.roles]`` and default. ``[model]`` is
read only when no layer names a model, so a fallback the step doesn't use, such as an unset ``env:``
reference, can't fail it."""
return (model_config.named_model(role, step_params=step_params, run=run, step=step, agent=agent)
or model_config.config_model(role, self._model_config()))

def get_model_params(self, overrides: dict | None = None) -> dict:
params = dict(self._model_config().params)
if overrides:
Expand All @@ -479,13 +494,11 @@ def resolve_model_call(
*,
default_api_key: str | None = None,
base_override: str | None = None,
model_overrides: dict[str, dict[str, str]] | None = None,
) -> model_config.ModelCallConfig:
return self._model_config().resolve_call(
model,
default_api_key=default_api_key,
base_override=base_override,
model_overrides=model_overrides,
secret_resolver=self._resolve_secret,
)

Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -101,6 +101,7 @@

from agent_env.a2a_agent import protocol
from agent_env.config import get_config
from agent_env.config.model import ModelRole, ModelSource, resolve_model
from agent_env.entity.scopes import FieldScope
from agent_env.task_step.context import TaskStepContext
from agent_env.task_step.task_steps.agent import user_sim_versions
Expand Down Expand Up @@ -216,6 +217,16 @@ def recorded_person(us: dict, seed: dict, multi_day: Optional[bool] = None) -> d
us["multi_day"] if us.get("multi_day") is not None else multi_day)


_USER_SIM_MODEL_SOURCES = {ModelSource.STEP_PARAMS: "user_sim_step_params", ModelSource.RUN: "run_user_model"}


def user_sim_model(*, own: Optional[str], run: Optional[str], step: Optional[str]) -> tuple[Optional[str], str]:
"""A user sim's model, and where it came from as its record names it: its deploy step's own ``step_params``, the
run's ``user_model``, then the step's ``model``. ``[model.roles]`` isn't read: the step names its model."""
resolved = resolve_model(ModelRole.USER_SIMULATOR, step_params=own, run=run, step=step)
return resolved.model, _USER_SIM_MODEL_SOURCES.get(resolved.source, "user_sim_step")


class DeployUserSimAgentTaskStep(DeployAgentTaskStep):
"""Deploys the agent that plays the task's user, its persona set at deploy, and records the user under
``agent_name`` in ``metadata["user_sims"]``."""
Expand Down Expand Up @@ -297,13 +308,8 @@ def _run_system_prompt(self, context: TaskStepContext) -> Optional[str]:

def _user_model(self, context: TaskStepContext) -> tuple[str, str]:
"""The model for this run, and where it came from."""
own = self.step_param_overrides(context).get("model")
if own:
return own, "user_sim_step_params"
run = (context.metadata.get("user_overrides") or {}).get("user_model")
if run:
return run, "run_user_model"
return self.model, "user_sim_step"
return user_sim_model(own=self.step_param_overrides(context).get("model"),
run=(context.metadata.get("user_overrides") or {}).get("user_model"), step=self.model)

async def _execute(self, context: TaskStepContext, deployed_agents: list) -> TaskStepContext:
# Inside deploy_agent's execute, so an agent whose model can't be set is closed like any failed deploy.
Expand All @@ -327,7 +333,7 @@ async def _set_user(self, context: TaskStepContext, agent, setup: dict) -> TaskS
f"{A2AAgent.EXT_AGENT_CONFIG}, so it would play the user on a model of its own")
await protocol.post_agent_config((agent.a2a_url or agent.api_url) + negotiated.endpoint, negotiated.fields,
headers=agent.control_headers)
context.record_model("user simulator", self.id, model)
context.record_model(ModelRole.USER_SIMULATOR.record_key, self.id, model)
prompt = setup["system_prompt"]
persona = self.persona
context.metadata.setdefault(USER_SIMS_KEY, {})[self.agent_name] = {
Expand Down
13 changes: 5 additions & 8 deletions src/agent_env/task_step/task_steps/always_on.py
Original file line number Diff line number Diff line change
Expand Up @@ -110,11 +110,12 @@
EXT_TRIGGERS_URI,
EXT_USER_CHAT_URI,
)
from agent_env.config.model import ModelRole
from agent_env.store import DuplicateKeyError, get_config
from agent_env.task_step.context import PromptResponse, StepContextError, TaskStepContext
from agent_env.task_step.task_step import TaskStep
from agent_env.task_step.task_steps.agent import user_sim_versions
from agent_env.task_step.task_steps.agent.deploy_user_sim_agent import USER_SIMS_KEY
from agent_env.task_step.task_steps.agent.deploy_user_sim_agent import USER_SIMS_KEY, user_sim_model
from agent_env.task_step.task_steps.notification_delivery import delivered_parts, label_time, take_due
from agent_env.task_step.task_steps.prompt_agent import PromptAgentTaskStep
from agent_env.task_step.task_steps.agent.user_sim_common import iso as _iso, person_of, task_zone, utc as _utc
Expand Down Expand Up @@ -593,7 +594,7 @@ async def _configure_agents(self) -> None:
if negotiated:
await protocol.post_agent_config(self.target_url + negotiated.endpoint, negotiated.fields,
headers=self.agent.control_headers)
ctx.record_model("solver", step.id, negotiated.fields.get("model") if negotiated else None)
ctx.record_model(ModelRole.SOLVER.record_key, step.id, negotiated.fields.get("model") if negotiated else None)
self.rules_in_system_prompt = bool(desired.get("system_prompt") and negotiated
and "system_prompt" in negotiated.fields)
if desired.get("system_prompt") and not self.rules_in_system_prompt:
Expand All @@ -614,7 +615,7 @@ async def _configure_agents(self) -> None:
await protocol.post_agent_config(self.user_url + negotiated.endpoint, negotiated.fields,
headers=self.user_agent.control_headers)
accepted = sorted(negotiated.fields) if negotiated else []
ctx.record_model("user simulator", step.id, negotiated.fields.get("model") if negotiated else None)
ctx.record_model(ModelRole.USER_SIMULATOR.record_key, step.id, negotiated.fields.get("model") if negotiated else None)
self.user_setup = {"model": self.user_model, "model_from": self.user_model_from,
"user_sim_step_model": self.us.get("model"),
"model_set": "model" in accepted, # else the agent ran its own
Expand Down Expand Up @@ -706,11 +707,7 @@ def _user_model(self) -> tuple[str, str]:
prompt_agent's ``_resolve_user_model`` honours it), else the user_sim step's ``model``."""
overrides = self.ctx.metadata.get("user_overrides") or {}
own = ((overrides.get("step_params") or {}).get(self.us.get("step_id")) or {}).get("model")
if own:
return own, "user_sim_step_params"
if overrides.get("user_model"):
return overrides["user_model"], "run_user_model"
return self.us["model"], "user_sim_step"
return user_sim_model(own=own, run=overrides.get("user_model"), step=self.us["model"])

async def _configure_checks(self) -> None:
"""Give the agent's role scheduled checks (notifications/v1) on each env whose clock is set: by default when
Expand Down
3 changes: 2 additions & 1 deletion src/agent_env/task_step/task_steps/deploy_agent.py
Original file line number Diff line number Diff line change
Expand Up @@ -30,6 +30,7 @@
)
from agent_env.a2a_agent.staging import StagedObjectStore, transfer_store
from agent_env.config import get_config
from agent_env.config.model import ModelRole
from agent_env.env.env import DeployedEnv, DeployedSandboxEnv, Env
from agent_env.env.gateway import AGENT_ENV_ROLE_HEADER
from agent_env.providers.sandbox_providers.local_docker.provider import host_url_for
Expand Down Expand Up @@ -520,7 +521,7 @@ async def _execute(self, context: TaskStepContext, deployed_agents: list) -> Tas
card.update(identity.fields)
logger.info(f"Stamped card identity on '{self.agent_name}': {sorted(identity.fields)}")

default_model = config.get_model_for_role("agent")
default_model = config.resolve_model(ModelRole.SOLVER).model
if default_model and not context.default_agent_model:
context.default_agent_model = default_model
logger.info(f"Set context.default_agent_model to '{default_model}'")
Expand Down
3 changes: 2 additions & 1 deletion src/agent_env/task_step/task_steps/install_agent.py
Original file line number Diff line number Diff line change
Expand Up @@ -24,6 +24,7 @@
from agent_env.a2a_agent import A2AAgent
from agent_env.artifact.artifacts.docker_image import DockerImageArtifact
from agent_env.config import get_config
from agent_env.config.model import ModelRole
from agent_env.entity.scopes import FieldScope
from agent_env.env.envs.computer.utils.boot import control_token as new_control_token
from agent_env.providers.sandbox_providers.local_docker.provider import (
Expand Down Expand Up @@ -377,7 +378,7 @@ async def execute(self, context: TaskStepContext) -> TaskStepContext:
control_token=deployed.control_token,
))

default_model = agent.metadata.get("default_model") or get_config().get_model_for_role("agent")
default_model = get_config().resolve_model(ModelRole.SOLVER, agent=agent.metadata.get("default_model")).model
Comment thread
earakely-scale marked this conversation as resolved.
Resolved
if default_model and not context.default_agent_model:
context.default_agent_model = default_model
logger.info(f"Set context.default_agent_model to '{default_model}'")
Expand Down
Loading
Loading