diff --git a/src/agent_env/bundle/preflight.py b/src/agent_env/bundle/preflight.py index dc60fd407..39c638467 100644 --- a/src/agent_env/bundle/preflight.py +++ b/src/agent_env/bundle/preflight.py @@ -588,8 +588,10 @@ def _provider(spec: str | None, default: Callable[[], SandboxProvider]) -> Sandb def model_endpoint_missing(needed: frozenset[str]) -> tuple[str, str] | None: - """The first of the ``needed`` model variables the config can't give, and why; None when it gives them all.""" + """The first of the ``needed`` model variables the config can't give, and why; None when it gives them all. A + ``[model]`` table that doesn't parse raises its own error, which would otherwise read as a missing endpoint.""" config = get_config() + config.get_default_model() if ENV_BASE_URL in needed: try: config.get_litellm_base_url() diff --git a/src/agent_env/config/model.py b/src/agent_env/config/model.py index b32d54e96..5e2bef2ca 100644 --- a/src/agent_env/config/model.py +++ b/src/agent_env/config/model.py @@ -42,6 +42,45 @@ class ModelParam(StrEnum): MODEL_PARAMS_RESERVED = frozenset(p.value for p in ModelParam) +class ModelRole(StrEnum): + """A role a model plays in a run. ``[model.roles]`` also takes the names it had first: ``agent`` for the solver + and ``user`` for the user simulator.""" + + SOLVER = "solver" + USER_SIMULATOR = "user_simulator" + JUDGE = "judge" + + @property + def record_key(self) -> str: + """The role's key in ``metadata.models``, which its readers have always seen.""" + return "user simulator" if self is ModelRole.USER_SIMULATOR else self.value + + +ROLE_SYNONYMS: dict[str, ModelRole] = {"agent": ModelRole.SOLVER, "user": ModelRole.USER_SIMULATOR} +JUDGE_FALLBACK_MODEL = "claude-sonnet-4-6" + + +class ModelSource(StrEnum): + """The layer a role's model came from: the first of them, in this order, that names one.""" + + STEP_PARAMS = "step_params" + RUN = "run" + STEP = "step" + AGENT = "agent" + CONFIG_ROLE = "config_role" + CONFIG_DEFAULT = "config_default" + FALLBACK = "fallback" + + +@dataclass(frozen=True) +class ResolvedModel: + """A role's model for one step and the layer it came from; both None when no layer names one.""" + + role: ModelRole + model: Optional[str] + source: Optional[ModelSource] + + @dataclass(frozen=True) class ModelCallConfig: """Resolved api_base / api_key / params for one model call.""" @@ -94,6 +133,7 @@ def from_section(cls, section: dict, *, secret_resolver: SecretResolver) -> Mode {k: v for k, v in section.items() if k not in ("api_key", "headers")}, secret_resolver=secret_resolver, ) + _check_roles(cfg.get("roles", {})) return cls( base_url=cfg.get("base_url"), api_key=section.get("api_key"), @@ -103,9 +143,16 @@ def from_section(cls, section: dict, *, secret_resolver: SecretResolver) -> Mode headers=dict(headers), ) + def role_model(self, role: str) -> Optional[str]: + """``[model.roles]``'s model for ``role``, under its name or the synonym it had first; the map is open, so a + plugin's own role is looked up by its name.""" + canonical = ROLE_SYNONYMS.get(role, role) + names = (canonical, *(synonym for synonym, named in ROLE_SYNONYMS.items() if named == canonical)) + return next((self.roles[name] for name in names if self.roles.get(name)), None) + def model_for_role(self, role: str, *, or_default: bool = True) -> Optional[str]: - """The ``[model.roles].`` model, else ``default`` unless ``or_default`` is False (open map).""" - return self.roles.get(role) or (self.default if or_default else None) + """The ``[model.roles].`` model, else ``default`` unless ``or_default`` is False.""" + return self.role_model(role) or (self.default if or_default else None) def resolve_api_key(self, *, secret_resolver: SecretResolver) -> str: """Key: env ``LITELLM_API_KEY`` > ``api_key`` (resolving a reference lazily); @@ -144,26 +191,10 @@ def resolve_call( *, default_api_key: Optional[str] = None, base_override: Optional[str] = None, - model_overrides: Optional[dict[str, dict[str, str]]] = None, secret_resolver: SecretResolver, ) -> ModelCallConfig: """A provider-prefixed model routes natively when no endpoint is configured; a bare model name needs one (override/env/config) and fails actionably without it.""" - if model_overrides: - override = model_overrides.get(model) - if override: - key = secret_resolver(override["secret_key"]) - if key is None: - raise ConfigError( - f"Model override for {model!r} names secret {override['secret_key']!r}, " - "which the configured secret store has no value for" - ) - return ModelCallConfig( - api_key=key, - api_base=base_override or override["base_url"], - params={}, - ) - key = default_api_key or os.getenv(ENV_API_KEY) configured_base = os.getenv(ENV_BASE_URL) or self.base_url if base_override: @@ -192,6 +223,63 @@ def resolve_call( return ModelCallConfig(api_key=key, api_base=api_base, params=params) +def resolve_model( + role: ModelRole, + *, + step_params: Optional[str] = None, + run: Optional[str] = None, + step: Optional[str] = None, + agent: Optional[str] = None, + config: Optional[ModelConfig] = None, +) -> ResolvedModel: + """``role``'s model for one step: the first of the layers its call site has that names one, else ``config``'s + (``named_model``, then ``config_model``). It reads no environment, key or secret, so a plan can resolve every role + before its run starts.""" + return (named_model(role, step_params=step_params, run=run, step=step, agent=agent) + or config_model(role, config)) + + +def named_model( + role: ModelRole, + *, + step_params: Optional[str] = None, + run: Optional[str] = None, + step: Optional[str] = None, + agent: Optional[str] = None, +) -> Optional[ResolvedModel]: + """The first of these layers that names a model: this step's ``step_params``, the run's override, the step's own + field, the deployed agent's default; None when none does.""" + for source, model in ((ModelSource.STEP_PARAMS, step_params), (ModelSource.RUN, run), + (ModelSource.STEP, step), (ModelSource.AGENT, agent)): + if model: + return ResolvedModel(role, model, source) + return None + + +def config_model(role: ModelRole, config: Optional[ModelConfig]) -> ResolvedModel: + """``role``'s model when no layer names one: ``[model.roles]``, then for the solver and the judge ``[model] + default``; a judge always has one, claude-sonnet-4-6 last.""" + if config is not None: + if model := config.role_model(role): + return ResolvedModel(role, model, ModelSource.CONFIG_ROLE) + if role is not ModelRole.USER_SIMULATOR and config.default: + return ResolvedModel(role, config.default, ModelSource.CONFIG_DEFAULT) + if role is ModelRole.JUDGE: + return ResolvedModel(role, JUDGE_FALLBACK_MODEL, ModelSource.FALLBACK) + return ResolvedModel(role, None, None) + + +def _check_roles(roles: object) -> None: + """Refuse a ``[model.roles]`` that maps a role to anything but a model name, or names one role twice, under its + name and its synonym.""" + if not isinstance(roles, dict) or not all(isinstance(model, str) for model in roles.values()): + raise ConfigError("[model.roles] maps each role to a model name, such as judge = \"claude-sonnet-4-6\"") + for synonym, role in ROLE_SYNONYMS.items(): + if synonym in roles and role.value in roles: + raise ConfigError(f"[model.roles] sets both {role.value!r} and {synonym!r}, which name the same role; " + f"keep {role.value!r}") + + def _check_header(name: str, value: Optional[str]) -> None: """Refuse a ``[model.headers]`` entry HTTP can't carry (``value`` None checks the name alone): a non-ASCII value would fail inside an agent's client, and a CR or LF would split one header into two.""" diff --git a/src/agent_env/config/runtime.py b/src/agent_env/config/runtime.py index 66326e090..876e45b2f 100644 --- a/src/agent_env/config/runtime.py +++ b/src/agent_env/config/runtime.py @@ -464,6 +464,21 @@ def get_default_model(self) -> str | None: def get_model_for_role(self, role: str, *, or_default: bool = True) -> str | None: return self._model_config().model_for_role(role, or_default=or_default) + def resolve_model( + self, + role: model_config.ModelRole, + *, + step_params: str | None = None, + run: str | None = None, + step: str | None = None, + agent: str | None = None, + ) -> model_config.ResolvedModel: + """``role``'s model for one step, falling back to this config's ``[model.roles]`` and default. ``[model]`` is + read only when no layer names a model, so a fallback the step doesn't use, such as an unset ``env:`` + reference, can't fail it.""" + return (model_config.named_model(role, step_params=step_params, run=run, step=step, agent=agent) + or model_config.config_model(role, self._model_config())) + def get_model_params(self, overrides: dict | None = None) -> dict: params = dict(self._model_config().params) if overrides: @@ -479,13 +494,11 @@ def resolve_model_call( *, default_api_key: str | None = None, base_override: str | None = None, - model_overrides: dict[str, dict[str, str]] | None = None, ) -> model_config.ModelCallConfig: return self._model_config().resolve_call( model, default_api_key=default_api_key, base_override=base_override, - model_overrides=model_overrides, secret_resolver=self._resolve_secret, ) diff --git a/src/agent_env/task_step/task_steps/agent/deploy_user_sim_agent.py b/src/agent_env/task_step/task_steps/agent/deploy_user_sim_agent.py index dd31e1f2a..756f46835 100644 --- a/src/agent_env/task_step/task_steps/agent/deploy_user_sim_agent.py +++ b/src/agent_env/task_step/task_steps/agent/deploy_user_sim_agent.py @@ -101,6 +101,7 @@ from agent_env.a2a_agent import protocol from agent_env.config import get_config +from agent_env.config.model import ModelRole, ModelSource, resolve_model from agent_env.entity.scopes import FieldScope from agent_env.task_step.context import TaskStepContext from agent_env.task_step.task_steps.agent import user_sim_versions @@ -216,6 +217,16 @@ def recorded_person(us: dict, seed: dict, multi_day: Optional[bool] = None) -> d us["multi_day"] if us.get("multi_day") is not None else multi_day) +_USER_SIM_MODEL_SOURCES = {ModelSource.STEP_PARAMS: "user_sim_step_params", ModelSource.RUN: "run_user_model"} + + +def user_sim_model(*, own: Optional[str], run: Optional[str], step: Optional[str]) -> tuple[Optional[str], str]: + """A user sim's model, and where it came from as its record names it: its deploy step's own ``step_params``, the + run's ``user_model``, then the step's ``model``. ``[model.roles]`` isn't read: the step names its model.""" + resolved = resolve_model(ModelRole.USER_SIMULATOR, step_params=own, run=run, step=step) + return resolved.model, _USER_SIM_MODEL_SOURCES.get(resolved.source, "user_sim_step") + + class DeployUserSimAgentTaskStep(DeployAgentTaskStep): """Deploys the agent that plays the task's user, its persona set at deploy, and records the user under ``agent_name`` in ``metadata["user_sims"]``.""" @@ -297,13 +308,8 @@ def _run_system_prompt(self, context: TaskStepContext) -> Optional[str]: def _user_model(self, context: TaskStepContext) -> tuple[str, str]: """The model for this run, and where it came from.""" - own = self.step_param_overrides(context).get("model") - if own: - return own, "user_sim_step_params" - run = (context.metadata.get("user_overrides") or {}).get("user_model") - if run: - return run, "run_user_model" - return self.model, "user_sim_step" + return user_sim_model(own=self.step_param_overrides(context).get("model"), + run=(context.metadata.get("user_overrides") or {}).get("user_model"), step=self.model) async def _execute(self, context: TaskStepContext, deployed_agents: list) -> TaskStepContext: # Inside deploy_agent's execute, so an agent whose model can't be set is closed like any failed deploy. @@ -327,7 +333,7 @@ async def _set_user(self, context: TaskStepContext, agent, setup: dict) -> TaskS f"{A2AAgent.EXT_AGENT_CONFIG}, so it would play the user on a model of its own") await protocol.post_agent_config((agent.a2a_url or agent.api_url) + negotiated.endpoint, negotiated.fields, headers=agent.control_headers) - context.record_model("user simulator", self.id, model) + context.record_model(ModelRole.USER_SIMULATOR.record_key, self.id, model) prompt = setup["system_prompt"] persona = self.persona context.metadata.setdefault(USER_SIMS_KEY, {})[self.agent_name] = { diff --git a/src/agent_env/task_step/task_steps/always_on.py b/src/agent_env/task_step/task_steps/always_on.py index 1063c8779..09d99bedc 100644 --- a/src/agent_env/task_step/task_steps/always_on.py +++ b/src/agent_env/task_step/task_steps/always_on.py @@ -110,11 +110,12 @@ EXT_TRIGGERS_URI, EXT_USER_CHAT_URI, ) +from agent_env.config.model import ModelRole from agent_env.store import DuplicateKeyError, get_config from agent_env.task_step.context import PromptResponse, StepContextError, TaskStepContext from agent_env.task_step.task_step import TaskStep from agent_env.task_step.task_steps.agent import user_sim_versions -from agent_env.task_step.task_steps.agent.deploy_user_sim_agent import USER_SIMS_KEY +from agent_env.task_step.task_steps.agent.deploy_user_sim_agent import USER_SIMS_KEY, user_sim_model from agent_env.task_step.task_steps.notification_delivery import delivered_parts, label_time, take_due from agent_env.task_step.task_steps.prompt_agent import PromptAgentTaskStep from agent_env.task_step.task_steps.agent.user_sim_common import iso as _iso, person_of, task_zone, utc as _utc @@ -593,7 +594,7 @@ async def _configure_agents(self) -> None: if negotiated: await protocol.post_agent_config(self.target_url + negotiated.endpoint, negotiated.fields, headers=self.agent.control_headers) - ctx.record_model("solver", step.id, negotiated.fields.get("model") if negotiated else None) + ctx.record_model(ModelRole.SOLVER.record_key, step.id, negotiated.fields.get("model") if negotiated else None) self.rules_in_system_prompt = bool(desired.get("system_prompt") and negotiated and "system_prompt" in negotiated.fields) if desired.get("system_prompt") and not self.rules_in_system_prompt: @@ -614,7 +615,7 @@ async def _configure_agents(self) -> None: await protocol.post_agent_config(self.user_url + negotiated.endpoint, negotiated.fields, headers=self.user_agent.control_headers) accepted = sorted(negotiated.fields) if negotiated else [] - ctx.record_model("user simulator", step.id, negotiated.fields.get("model") if negotiated else None) + ctx.record_model(ModelRole.USER_SIMULATOR.record_key, step.id, negotiated.fields.get("model") if negotiated else None) self.user_setup = {"model": self.user_model, "model_from": self.user_model_from, "user_sim_step_model": self.us.get("model"), "model_set": "model" in accepted, # else the agent ran its own @@ -706,11 +707,7 @@ def _user_model(self) -> tuple[str, str]: prompt_agent's ``_resolve_user_model`` honours it), else the user_sim step's ``model``.""" overrides = self.ctx.metadata.get("user_overrides") or {} own = ((overrides.get("step_params") or {}).get(self.us.get("step_id")) or {}).get("model") - if own: - return own, "user_sim_step_params" - if overrides.get("user_model"): - return overrides["user_model"], "run_user_model" - return self.us["model"], "user_sim_step" + return user_sim_model(own=own, run=overrides.get("user_model"), step=self.us["model"]) async def _configure_checks(self) -> None: """Give the agent's role scheduled checks (notifications/v1) on each env whose clock is set: by default when diff --git a/src/agent_env/task_step/task_steps/deploy_agent.py b/src/agent_env/task_step/task_steps/deploy_agent.py index 7aa515d06..6e7b513c5 100644 --- a/src/agent_env/task_step/task_steps/deploy_agent.py +++ b/src/agent_env/task_step/task_steps/deploy_agent.py @@ -30,6 +30,7 @@ ) from agent_env.a2a_agent.staging import StagedObjectStore, transfer_store from agent_env.config import get_config +from agent_env.config.model import ModelRole from agent_env.env.env import DeployedEnv, DeployedSandboxEnv, Env from agent_env.env.gateway import AGENT_ENV_ROLE_HEADER from agent_env.providers.sandbox_providers.local_docker.provider import host_url_for @@ -520,7 +521,7 @@ async def _execute(self, context: TaskStepContext, deployed_agents: list) -> Tas card.update(identity.fields) logger.info(f"Stamped card identity on '{self.agent_name}': {sorted(identity.fields)}") - default_model = config.get_model_for_role("agent") + default_model = config.resolve_model(ModelRole.SOLVER).model if default_model and not context.default_agent_model: context.default_agent_model = default_model logger.info(f"Set context.default_agent_model to '{default_model}'") diff --git a/src/agent_env/task_step/task_steps/install_agent.py b/src/agent_env/task_step/task_steps/install_agent.py index c72b74d03..9825d3108 100644 --- a/src/agent_env/task_step/task_steps/install_agent.py +++ b/src/agent_env/task_step/task_steps/install_agent.py @@ -24,6 +24,7 @@ from agent_env.a2a_agent import A2AAgent from agent_env.artifact.artifacts.docker_image import DockerImageArtifact from agent_env.config import get_config +from agent_env.config.model import ModelRole from agent_env.entity.scopes import FieldScope from agent_env.env.envs.computer.utils.boot import control_token as new_control_token from agent_env.providers.sandbox_providers.local_docker.provider import ( @@ -377,7 +378,7 @@ async def execute(self, context: TaskStepContext) -> TaskStepContext: control_token=deployed.control_token, )) - default_model = agent.metadata.get("default_model") or get_config().get_model_for_role("agent") + default_model = get_config().resolve_model(ModelRole.SOLVER, agent=agent.metadata.get("default_model")).model if default_model and not context.default_agent_model: context.default_agent_model = default_model logger.info(f"Set context.default_agent_model to '{default_model}'") diff --git a/src/agent_env/task_step/task_steps/prompt_agent.py b/src/agent_env/task_step/task_steps/prompt_agent.py index 218e5e282..9859093ab 100644 --- a/src/agent_env/task_step/task_steps/prompt_agent.py +++ b/src/agent_env/task_step/task_steps/prompt_agent.py @@ -26,7 +26,7 @@ ) from agent_env.a2a_agent.staging import draining, staged_changelogs, transfer_store from agent_env.a2a_agent.trajectory_follower import live_trajectory_endpoint -from agent_env.config.model import MODEL_PARAMS_RESERVED +from agent_env.config.model import MODEL_PARAMS_RESERVED, ModelRole, resolve_model from agent_env.env.gateway.constants import ( DEFAULT_ROLE, EXT_CLOCK_URI, @@ -392,16 +392,17 @@ def _resolve_model_and_effort(self, context: TaskStepContext) -> tuple[Optional[ """Model and effort for this run: this step's ``step_overrides`` win over the run-wide ``agent_model`` / ``agent_effort`` and the stored values.""" step_overrides = self.step_param_overrides(context) overrides = context.metadata.get("user_overrides", {}) - model = step_overrides.get("model") or context.agent_model or self.model or context.default_agent_model + model = resolve_model(ModelRole.SOLVER, step_params=step_overrides.get("model"), run=context.agent_model, + step=self.model, agent=context.default_agent_model).model effort = step_overrides.get("effort") or overrides.get("agent_effort") or self.effort return model, effort def _resolve_user_model(self, context: TaskStepContext) -> Optional[str]: """The user simulator's model for this run: this step's ``step_overrides``, then the run's ``user_model``, the stored value, and the config's ``user`` role when it names one.""" - return (self.step_param_overrides(context).get("user_model") - or context.metadata.get("user_overrides", {}).get("user_model") or self.user_model - or get_config().get_model_for_role("user", or_default=False)) + return get_config().resolve_model( + ModelRole.USER_SIMULATOR, step_params=self.step_param_overrides(context).get("user_model"), + run=context.metadata.get("user_overrides", {}).get("user_model"), step=self.user_model).model def _built_in_user_sim(self, context: TaskStepContext) -> Optional[tuple[dict, str]]: """The user deploy_user_sim_agent deployed, when it is this step's user, and their time zone; None for any other @@ -558,11 +559,11 @@ async def _converse( await protocol.post_agent_config(target_url + negotiated.endpoint, negotiated.fields, headers=agent.control_headers) # A model the agent's card doesn't accept is never sent, and the agent runs its own. - context.record_model("solver", self.id, negotiated.fields.get("model") if negotiated else None) + context.record_model(ModelRole.SOLVER.record_key, self.id, negotiated.fields.get("model") if negotiated else None) is_user_sim = user_sim is not None and self.user_a2a_url is None if built_in is not None: # deploy_user_sim_agent set its model and answer format - context.record_model("user simulator", self.id, built_in[0].get("model")) + context.record_model(ModelRole.USER_SIMULATOR.record_key, self.id, built_in[0].get("model")) elif is_user_sim: user_output_format = self.user_output_format or _DEFAULT_USER_SIM_OUTPUT_FORMAT user_model = self._resolve_user_model(context) @@ -572,7 +573,7 @@ async def _converse( await protocol.post_agent_config(user_url + negotiated.endpoint, negotiated.fields, headers=user_sim.control_headers) logger.info(f"Configured user-sim '{self.user_agent_name}' with structured output_format for done-signal") - context.record_model("user simulator", self.id, negotiated.fields.get("model") if negotiated else None) + context.record_model(ModelRole.USER_SIMULATOR.record_key, self.id, negotiated.fields.get("model") if negotiated else None) triggers_registered = is_user_sim and any( r.get("agent_name") == self.user_agent_name diff --git a/src/agent_env/task_step/task_steps/verifiers/rubrics_verifier.py b/src/agent_env/task_step/task_steps/verifiers/rubrics_verifier.py index 1e70514ad..44a3c7c21 100644 --- a/src/agent_env/task_step/task_steps/verifiers/rubrics_verifier.py +++ b/src/agent_env/task_step/task_steps/verifiers/rubrics_verifier.py @@ -19,7 +19,7 @@ from agent_env.a2a_agent.a2a_agent import unwritten_local_agent from agent_env.a2a_agent.staging import transfer_store from agent_env.config import get_config -from agent_env.config.model import ModelParam +from agent_env.config.model import ModelParam, ModelRole from agent_env.providers.sandbox_providers.sandbox import into_container from agent_env.task_step.context import ( PromptResponse, @@ -563,9 +563,9 @@ def __init__( self.verifier_id = verifier_id or uuid.uuid4().hex self.criteria = criteria self.prompt_id = prompt_id - # A stored judge always holds a model: its author's, or the one the config names for judges, as for a step - # built in code, which the A2A validator's are. - self.default_model = default_model or get_config().get_model_for_role("judge") or "claude-sonnet-4-6" + # The author's model, or None to judge with the run's config when the step runs (``_judge_model``), so a + # stored task never carries the config of whoever wrote it. + self.default_model = default_model self.default_model_api_base = default_model_api_base if isinstance(score_aggregator, str): score_aggregator = ScoreAggregator(score_aggregator) @@ -637,6 +637,12 @@ def skipped(fields: dict) -> dict: if self._writes_mistakes_entry: verifications[trajectory_mistakes_key(self.verifier_id)] = skipped(self._mistakes_entry_fields()) + def _judge_model(self, context: TaskStepContext) -> str: + """The model this judge asks for in this run: the run's ``judge_model``, the step's own, then the config's judge + role and default, and claude-sonnet-4-6 last.""" + overrides = context.metadata.get("user_overrides") or {} + return get_config().resolve_model(ModelRole.JUDGE, run=overrides.get("judge_model"), step=self.default_model).model + def to_dict(self) -> dict: base = super().to_dict() base["verifier_id"] = self.verifier_id @@ -949,8 +955,8 @@ async def _grade(self, context: TaskStepContext) -> TaskStepContext: raise try: - model = overrides.get("judge_model") or self.default_model - context.record_model("judge", self.id, model) + model = self._judge_model(context) + context.record_model(ModelRole.JUDGE.record_key, self.id, model) agent_prompt_text = prompt_response.prompt_text or "" judge_effort = overrides.get("judge_effort") or self.effort judge_max_thinking_tokens = overrides.get("judge_max_thinking_tokens") or self.max_thinking_tokens @@ -1444,7 +1450,7 @@ async def _configure_judge_a2a( await protocol.post_agent_config(judge_a2a_url + endpoint, config_payload, headers=headers) logger.info(f"Set judge agent config: {list(config_payload.keys())}") # What the judge agent took; one that doesn't accept a model grades on its own. - context.record_model("judge", self.id, config_payload.get("model")) + context.record_model(ModelRole.JUDGE.record_key, self.id, config_payload.get("model")) async def _invoke_judge_a2a( self, diff --git a/src/agent_env/task_step/task_steps/verifiers/user_sim_judge.py b/src/agent_env/task_step/task_steps/verifiers/user_sim_judge.py index 7cf7d1453..944a08400 100644 --- a/src/agent_env/task_step/task_steps/verifiers/user_sim_judge.py +++ b/src/agent_env/task_step/task_steps/verifiers/user_sim_judge.py @@ -332,7 +332,7 @@ async def _grade(self, context: TaskStepContext) -> TaskStepContext: items = [core.RubricItem.from_dict(i) for i in self.rubric_items] kind, step_id = self._find_run(context) response = _prompt_response(context, step_id) - model = (context.metadata.get("user_overrides") or {}).get("judge_model") or self.default_model + model = self._judge_model(context) entry: dict[str, Any] = {"run_step_id": step_id, "run_kind": kind, "judge_model": model} record = (context.metadata.get(ALWAYS_ON_KEY) or {}).get(step_id) if kind == ALWAYS_ON else None error_type = (record or {}).get("error_type") or (response.error_type if response else None) diff --git a/tst/unit/bundle/importer_test.py b/tst/unit/bundle/importer_test.py index f2a2e0789..b592024da 100644 --- a/tst/unit/bundle/importer_test.py +++ b/tst/unit/bundle/importer_test.py @@ -594,8 +594,8 @@ def test_reading_the_store_writes_nothing_to_it(store): SCOPED_FIELDS = {"env": ["sandbox_type", "metadata.attribution"], "agent": ["a2a_agent_id", "sandbox_type", "litellm_base_url", "metadata.attribution"], "prompt": ["trajectory_output_prefix", "max_thinking_tokens"], # a set zero is set - # A stored judge always holds a model: its author's, or the one its store's config named for judges. - "judge": ["default_model", "judge_sandbox_type"], + # The judge's author named no model, so none was stored and none is dropped. + "judge": ["judge_sandbox_type"], "install": ["a2a_agent_id"], "fetch": ["urls"], "build": ["docker_context_url"], diff --git a/tst/unit/bundle/preflight_test.py b/tst/unit/bundle/preflight_test.py index 5896fe17e..0820eca28 100644 --- a/tst/unit/bundle/preflight_test.py +++ b/tst/unit/bundle/preflight_test.py @@ -22,6 +22,8 @@ from agent_env.bundle.resolve import build_step from agent_env.cli import cli from agent_env.config import get_config +from agent_env.config.errors import ConfigError +from agent_env.config.model import ENV_BASE_URL from agent_env.config.runtime import Config from agent_env.env import Env, GatewayEnv, MCPServerEnv, MultiEnv from agent_env.env import bootstrap @@ -40,6 +42,7 @@ from agent_env.task_step.task_steps.deploy_sandbox import DeploySandboxTaskStep from agent_env.task_step.task_steps.verifiers.rubrics_verifier import RubricsVerifierTaskStep from tst.unit.bundle._support import layout, local_store, shipped_agent +from tst.util.config import config_with_document LOCAL = ("localhost:5000/local/img:v1", "file:///state/img-v1.tar.gz") REMOTE = ("123456789012.dkr.ecr.us-west-2.amazonaws.com/img:v1", "s3://bucket/img-v1.tar.gz") @@ -1219,6 +1222,15 @@ def test_an_agent_or_agent_judge_needs_a_model_endpoint_unless_its_step_gives_on assert len(dry_run_bundle(bundle_dir, sandbox="local").runs) == 2 +@pytest.mark.parametrize("model", [{"base_url": "http://models:4000", "roles": {"agent": "a", "solver": "s"}}, + {"base_url": "http://models:4000", "endpoints": {}}], ids=["role-twice", "unknown-key"]) +def test_a_model_table_that_doesn_t_parse_says_why_rather_than_reading_as_a_missing_endpoint(no_model, monkeypatch, + model): + monkeypatch.setattr(preflight_module, "get_config", lambda: config_with_document({"model": model})) + with pytest.raises(ConfigError, match="sets both|unknown keys"): + MODEL_ENDPOINT_MISSING(frozenset({ENV_BASE_URL})) + + def test_an_agent_declaring_the_model_variables_itself_needs_no_endpoint_but_its_judge_does(bundle_dir, no_model): with namespace_routing(): A2AAgent.put(id="echo", docker_image_artifact=_image("echo-image"), diff --git a/tst/unit/config/test_model.py b/tst/unit/config/test_model.py index 54554070a..5e5c96d93 100644 --- a/tst/unit/config/test_model.py +++ b/tst/unit/config/test_model.py @@ -4,10 +4,17 @@ from agent_env.config.errors import ConfigError from agent_env.config.model import ( + JUDGE_FALLBACK_MODEL, MODEL_PARAMS_RESERVED, ModelCallConfig, ModelConfig, ModelParam, + ModelRole, + ModelSource, + ResolvedModel, + config_model, + named_model, + resolve_model, ) @@ -98,6 +105,72 @@ def test_a_role_asked_for_without_the_default_is_only_the_one_the_config_names() assert cfg.model_for_role("judge", or_default=False) is None +@pytest.mark.parametrize("roles", [{"solver": "m-s", "user_simulator": "m-u"}, {"agent": "m-s", "user": "m-u"}], + ids=["canonical", "synonyms"]) +def test_a_role_is_found_under_its_name_or_its_synonym_whichever_is_asked(roles): + cfg = ModelConfig.from_section({"roles": roles}, secret_resolver=_secrets({})) + for asked in ("solver", "agent", ModelRole.SOLVER): + assert cfg.role_model(asked) == "m-s" + for asked in ("user_simulator", "user", ModelRole.USER_SIMULATOR): + assert cfg.role_model(asked) == "m-u" + + +def test_a_plugin_s_own_role_is_looked_up_by_its_name(): + assert ModelConfig(roles={"reviewer": "m-r"}).model_for_role("reviewer") == "m-r" + + +@pytest.mark.parametrize("roles, problem", [ + ({"agent": "a", "solver": "s"}, "sets both 'solver' and 'agent'"), + ({"user": "a", "user_simulator": "s"}, "sets both 'user_simulator' and 'user'"), + ({"judge": {"endpoint": "local", "model": "fast"}}, "maps each role to a model name"), + ("claude", "maps each role to a model name"), +], ids=["solver-twice", "user-simulator-twice", "table", "not-a-table"]) +def test_from_section_refuses_a_role_named_twice_or_mapped_to_anything_but_a_model(roles, problem): + with pytest.raises(ConfigError, match=problem): + ModelConfig.from_section({"roles": roles}, secret_resolver=_secrets({})) + + +# --- resolve_model ------------------------------------------------------------ + +_LAYERS = ("step_params", "run", "step", "agent") + + +@pytest.mark.parametrize("first", range(len(_LAYERS)), ids=_LAYERS) +@pytest.mark.parametrize("role", list(ModelRole)) +def test_the_first_layer_that_names_a_model_wins(role, first): + given = {layer: f"m-{layer}" for layer in _LAYERS[first:]} + resolved = resolve_model(role, config=ModelConfig(default="m-default", roles={role.value: "m-role"}), **given) + assert resolved == ResolvedModel(role, f"m-{_LAYERS[first]}", ModelSource(_LAYERS[first])) + + +@pytest.mark.parametrize("role, config, expected", [ + (ModelRole.SOLVER, ModelConfig(default="D", roles={"agent": "A"}), ("A", ModelSource.CONFIG_ROLE)), + (ModelRole.SOLVER, ModelConfig(default="D"), ("D", ModelSource.CONFIG_DEFAULT)), + (ModelRole.SOLVER, ModelConfig(), (None, None)), + (ModelRole.USER_SIMULATOR, ModelConfig(default="D", roles={"user": "U"}), ("U", ModelSource.CONFIG_ROLE)), + (ModelRole.USER_SIMULATOR, ModelConfig(default="D"), (None, None)), + (ModelRole.JUDGE, ModelConfig(default="D", roles={"judge": "J"}), ("J", ModelSource.CONFIG_ROLE)), + (ModelRole.JUDGE, ModelConfig(default="D"), ("D", ModelSource.CONFIG_DEFAULT)), + (ModelRole.JUDGE, ModelConfig(), (JUDGE_FALLBACK_MODEL, ModelSource.FALLBACK)), + (ModelRole.JUDGE, None, (JUDGE_FALLBACK_MODEL, ModelSource.FALLBACK)), + (ModelRole.SOLVER, None, (None, None)), +], ids=["solver-role", "solver-default", "solver-none", "user-role", "user-never-the-default", "judge-role", + "judge-default", "judge-fallback", "judge-without-config", "solver-without-config"]) +def test_with_no_layer_the_config_decides_per_role(role, config, expected): + resolved = resolve_model(role, config=config) + assert (resolved.model, resolved.source) == expected + + +def test_resolving_reads_no_environment_and_no_secret(monkeypatch): + def refuse(key): + raise AssertionError(f"resolution read the secret {key!r}") + + monkeypatch.setattr("os.environ", {}) + cfg = ModelConfig.from_section({"api_key": "secret:litellm_api_key", "headers": {"X-Org": "secret:org"}, + "default": "D", "roles": {"judge": "J"}}, secret_resolver=refuse) + assert [resolve_model(role, config=cfg).model for role in ModelRole] == ["D", None, "J"] + + # --- resolve_api_key ---------------------------------------------------------- def test_api_key_env_wins_and_refs_stay_lazy(monkeypatch): @@ -125,24 +198,6 @@ def test_api_key_unconfigured_raises_actionable(): # --- resolve_call ------------------------------------------------------------- -def test_resolve_call_inline_override_wins(): - call = ModelConfig().resolve_call( - "m", - model_overrides={"m": {"secret_key": "k", "base_url": "https://ov"}}, - secret_resolver=_secrets({"k": "sk-k"}), - ) - assert call == ModelCallConfig(api_key="sk-k", api_base="https://ov", params={}) - - -def test_resolve_call_override_with_missing_secret_raises(): - with pytest.raises(ConfigError, match="no value"): - ModelConfig().resolve_call( - "m", - model_overrides={"m": {"secret_key": "absent", "base_url": "https://ov"}}, - secret_resolver=_secrets({}), - ) - - def test_resolve_call_bare_model_without_endpoint_raises_actionable(): with pytest.raises(ConfigError, match=r"\[model\] base_url"): ModelConfig().resolve_call("claude-sonnet-4-6", secret_resolver=_secrets({})) @@ -253,8 +308,7 @@ def test_a_header_ref_that_cant_be_sent_fails_loud(secrets, problem): @pytest.mark.parametrize("call, expected", [ ({}, {"extra_headers": _RESOLVED}), ({"base_override": "https://elsewhere.example"}, {}), - ({"model_overrides": {"m": {"secret_key": "k", "base_url": "https://gw/v1"}}}, {}), -], ids=["configured", "foreign-override", "model-override"]) +], ids=["configured", "foreign-override"]) def test_resolve_call_sends_the_headers_only_to_the_configured_endpoint(call, expected): assert _WITH_HEADERS.resolve_call("m", secret_resolver=_secrets({"org": "acme", "k": "sk"}), **call).params == expected @@ -265,3 +319,14 @@ def test_params_extra_headers_win_per_header_whatever_its_case(): call = cfg.resolve_call("m", secret_resolver=_secrets({})) assert call.params == {"temperature": 0, "extra_headers": {"X-Plain": "v", "x-bill-to": "other"}} assert cfg.params == {"temperature": 0, "extra_headers": {"x-bill-to": "other"}} + + +def test_each_role_records_under_the_key_metadata_models_has_always_used(): + assert {role: role.record_key for role in ModelRole} == { + ModelRole.SOLVER: "solver", ModelRole.USER_SIMULATOR: "user simulator", ModelRole.JUDGE: "judge"} + + +def test_no_named_layer_leaves_the_role_to_its_config(): + assert named_model(ModelRole.SOLVER) is None + assert config_model(ModelRole.SOLVER, ModelConfig(default="D")) == ResolvedModel( + ModelRole.SOLVER, "D", ModelSource.CONFIG_DEFAULT) diff --git a/tst/unit/store/test_config_model.py b/tst/unit/store/test_config_model.py index 974a08cc2..2575fc37d 100644 --- a/tst/unit/store/test_config_model.py +++ b/tst/unit/store/test_config_model.py @@ -8,7 +8,7 @@ import pytest -from agent_env.config.model import ModelCallConfig +from agent_env.config.model import ModelCallConfig, ModelRole from agent_env.store import ConfigError, LocalSecretStore from agent_env.config import Config @@ -160,16 +160,6 @@ def test_arbitrary_role_falls_back_to_default(tmp_path, monkeypatch): assert _cfg(tmp_path, monkeypatch, _ROLES).get_model_for_role("some_custom_role") == "m-default" -def test_model_overrides_tier(tmp_path, monkeypatch): - cfg = _cfg(tmp_path, monkeypatch, secrets={"ov_key": "sk-ov", "litellm_api_key": "sk-bundle"}) - overrides = {"m": {"secret_key": "ov_key", "base_url": "https://ov.example"}} - assert cfg.resolve_model_call("m", model_overrides=overrides) == ModelCallConfig( - api_key="sk-ov", - api_base="https://ov.example", - params={}, - ) - - def test_base_override_wins(tmp_path, monkeypatch): cfg = _cfg(tmp_path, monkeypatch, _MODEL) call = cfg.resolve_model_call("claude-sonnet-4-6", default_api_key="k", base_override="https://ovr") @@ -193,3 +183,16 @@ def test_reserved_param_key_fails_loud(tmp_path, monkeypatch, reserved): cfg = _cfg(tmp_path, monkeypatch, body) with pytest.raises(ConfigError, match="reserved"): cfg.get_default_model() + + +_UNSET_FALLBACK = '[model]\ndefault = "env:FD5976_UNSET_MODEL"\n' + + +@pytest.mark.parametrize("layer", ["step_params", "run", "step", "agent"]) +def test_a_model_a_layer_names_doesn_t_read_a_fallback_it_doesn_t_use(tmp_path, monkeypatch, layer): + monkeypatch.delenv("FD5976_UNSET_MODEL", raising=False) + cfg = _cfg(tmp_path, monkeypatch, _UNSET_FALLBACK) + + assert cfg.resolve_model(ModelRole.SOLVER, **{layer: "m-given"}).model == "m-given" + with pytest.raises(ConfigError, match="FD5976_UNSET_MODEL"): + cfg.resolve_model(ModelRole.SOLVER) diff --git a/tst/unit/task_step/test_prompt_agent_snapshots.py b/tst/unit/task_step/test_prompt_agent_snapshots.py index b13829063..879c6c1e4 100644 --- a/tst/unit/task_step/test_prompt_agent_snapshots.py +++ b/tst/unit/task_step/test_prompt_agent_snapshots.py @@ -14,6 +14,7 @@ import pytest from agent_env.config import set_object_store +from agent_env.config.model import resolve_model from agent_env.task_step.context import PromptResponse, TaskStepContext from agent_env.task_step.snapshot_utils import agent_state_capture as mod from agent_env.task_step.snapshot_utils import snapshot_series as ps @@ -1292,8 +1293,8 @@ def get_model_params(self, overrides=None): def get_default_human_a2a_url(self): return _Conversation.USER_URL - def get_model_for_role(self, role, *, or_default=True): - return None + def resolve_model(self, role, **layers): + return resolve_model(role, **layers) monkeypatch.setattr(pa_mod, "get_config", lambda: _Cfg()) diff --git a/tst/unit/task_step/test_recorded_models.py b/tst/unit/task_step/test_recorded_models.py index 673c4cbe6..792dbc9e0 100644 --- a/tst/unit/task_step/test_recorded_models.py +++ b/tst/unit/task_step/test_recorded_models.py @@ -8,6 +8,7 @@ from agent_env.a2a_agent import A2AAgent from agent_env.a2a_agent import protocol +from agent_env.config.model import resolve_model from agent_env.task_step.context import DeployedAgent, TaskStepContext from agent_env.task_step.task_steps import prompt_agent as pa_mod from agent_env.task_step.task_steps.prompt_agent import PromptAgentTaskStep @@ -24,7 +25,7 @@ def posted(monkeypatch): """Mocks the conversation's HTTP and store boundary, and collects each agent config posted, by URL.""" cfg = MagicMock() cfg.get_model_params.return_value = {} - cfg.get_model_for_role.return_value = None + cfg.resolve_model.side_effect = lambda role, **layers: resolve_model(role, **layers) monkeypatch.setattr(pa_mod, "get_config", lambda: cfg) for fn in ("create_conversation", "add_a2a_task", "complete_a2a_task", "mark_closed", "get_conversation"): monkeypatch.setattr(pa_mod.conversation_store, fn, lambda *a, **kw: None) diff --git a/tst/unit/task_step/test_role_models_golden.py b/tst/unit/task_step/test_role_models_golden.py new file mode 100644 index 000000000..b0cdcb3e4 --- /dev/null +++ b/tst/unit/task_step/test_role_models_golden.py @@ -0,0 +1,165 @@ +"""Each role's model at each call site that resolves one, across the configs and overrides in use: the solver's +default a deploy sets, the solver's model a prompt step asks for, the user simulator's at prompt_agent, +deploy_user_sim_agent and always_on, and the judge's, a model stored in a task included.""" + +from __future__ import annotations + +import itertools +import json +from pathlib import Path + +import pytest + +from agent_env.config import get_config, reset_config +from agent_env.config.model import ModelRole +from agent_env.task_step.context import TaskStepContext +from agent_env.task_step.task_steps import always_on as ao +from agent_env.task_step.task_steps.agent.deploy_user_sim_agent import DeployUserSimAgentTaskStep +from agent_env.task_step.task_steps.prompt_agent import PromptAgentTaskStep +from agent_env.task_step.task_steps.verifiers.rubrics_verifier import RubricsVerifierTaskStep + +PERSON = json.loads((Path(__file__).parents[2] / "data" / "user_sim" / "hannah_person.json").read_text()) +HANNAH = {"display_name": PERSON["name"], "first_name": PERSON["first"], "description": PERSON["description"], + "permissions": PERSON["permissions"], "keep_private": PERSON["keep_private"]} +SDK_PROD = ('base_url = "https://litellm-proxy.example.com"\napi_key = "secret:litellm_api_key"\n\n' + '[model.headers]\nx-litellm-tags = "useCase:agent-env"\n') +CONFIGS = { + "none": "", + "sdk_prod": SDK_PROD, + "default_only": 'default = "D"\n', + "every_role": 'default = "D"\n\n[model.roles]\nagent = "A"\nuser = "U"\njudge = "J"\n', + "agent_role": 'default = "D"\n\n[model.roles]\nagent = "A"\n', + "user_role": '[model.roles]\nuser = "U"\n', + "judge_role": '[model.roles]\njudge = "J"\n', +} +SOLVER_DEFAULT = {"none": None, "sdk_prod": None, "default_only": "D", "every_role": "A", "agent_role": "A", + "user_role": None, "judge_role": None} +USER_CONFIG = {"none": None, "sdk_prod": None, "default_only": None, "every_role": "U", "agent_role": None, + "user_role": "U", "judge_role": None} +JUDGE = {"none": "claude-sonnet-4-6", "sdk_prod": "claude-sonnet-4-6", "default_only": "D", "every_role": "J", + "agent_role": "D", "user_role": "claude-sonnet-4-6", "judge_role": "J"} + + +@pytest.fixture +def model_config(tmp_path, monkeypatch): + """Install a config file whose whole [model] table is the given text, so no developer config is read.""" + + def install(model: str) -> None: + path = tmp_path / "config.toml" + path.write_text(f"[model]\n{model}") + monkeypatch.setenv("AGENT_ENV_CONFIG", str(path)) + reset_config() + + yield install + reset_config() + + +def _ctx(step_params: dict | None = None, run: dict | None = None, **fields) -> TaskStepContext: + overrides = dict(run or {}) + if step_params: + overrides["step_params"] = step_params + return TaskStepContext(metadata={"user_overrides": overrides} if overrides else {}, **fields) + + +def _first(*layers): + return next((value for value in layers if value), None) + + +def _subsets(names: tuple[str, ...]): + return [dict(zip(names, chosen)) for chosen in itertools.product((False, True), repeat=len(names))] + + +def _deploy_default() -> str | None: + """The solver's default a deploy_agent step sets on the context.""" + return get_config().resolve_model(ModelRole.SOLVER).model + + +def _judge_model(verifier: RubricsVerifierTaskStep, context: TaskStepContext) -> str | None: + """The model a rubrics judge asks for when it runs.""" + return verifier._judge_model(context) + + +@pytest.mark.parametrize("name", CONFIGS) +def test_the_solver_default_a_deploy_sets(model_config, name): + model_config(CONFIGS[name]) + assert _deploy_default() == SOLVER_DEFAULT[name] + + +@pytest.mark.parametrize("layers", _subsets(("step_params", "run", "step", "deployed")), ids=str) +def test_the_solver_model_a_prompt_step_asks_for(layers): + step = PromptAgentTaskStep(id="prompt", version=None, agent_name="solver", prompt="go", + model="step-model" if layers["step"] else None) + context = _ctx({"prompt": {"model": "step-params-model"}} if layers["step_params"] else None, + agent_model="run-model" if layers["run"] else None, + default_agent_model="deployed-model" if layers["deployed"] else None) + expected = _first(layers["step_params"] and "step-params-model", layers["run"] and "run-model", + layers["step"] and "step-model", layers["deployed"] and "deployed-model") + assert step._resolve_model_and_effort(context)[0] == expected + + +@pytest.mark.parametrize("name", CONFIGS) +@pytest.mark.parametrize("layers", _subsets(("step_params", "run", "step")), ids=str) +def test_the_user_simulator_model_a_prompt_step_asks_for(model_config, name, layers): + model_config(CONFIGS[name]) + step = PromptAgentTaskStep(id="prompt", version=None, agent_name="solver", prompt="go", + user_model="step-user" if layers["step"] else None) + context = _ctx({"prompt": {"user_model": "step-params-user"}} if layers["step_params"] else None, + {"user_model": "run-user", "agent_model": "run-solver"} if layers["run"] else None, + agent_model="run-solver") + expected = _first(layers["step_params"] and "step-params-user", layers["run"] and "run-user", + layers["step"] and "step-user", USER_CONFIG[name]) + assert step._resolve_user_model(context) == expected + + +USER_SIM_CASES = [ + ({}, ("sim-model", "user_sim_step")), + ({"run": {"user_model": "run-model"}}, ("run-model", "run_user_model")), + ({"step_params": {"user": {"model": "own-model"}}}, ("own-model", "user_sim_step_params")), + ({"run": {"user_model": "run-model"}, "step_params": {"user": {"model": "own-model"}}}, + ("own-model", "user_sim_step_params")), +] + + +@pytest.mark.parametrize("name", CONFIGS) +@pytest.mark.parametrize("overrides, expected", USER_SIM_CASES, ids=["step", "run", "own", "own_over_run"]) +def test_the_user_simulator_model_its_deploy_sets_ignores_model_roles(model_config, name, overrides, expected): + model_config(CONFIGS[name]) + step = DeployUserSimAgentTaskStep.from_dict({ + "id": "user", "type": "deploy_user_sim_agent", "a2a_agent_id": "sim-image", "agent_name": "hannah", + "model": "sim-model", "multi_day": True, "opening_prompt": PERSON["opening"]["text"], "persona": HANNAH}) + assert step._user_model(_ctx(overrides.get("step_params"), overrides.get("run"))) == expected + + +@pytest.mark.parametrize("name", CONFIGS) +@pytest.mark.parametrize("overrides, expected", USER_SIM_CASES, ids=["step", "run", "own", "own_over_run"]) +def test_the_user_simulator_model_always_on_sets_ignores_model_roles(model_config, name, overrides, expected): + model_config(CONFIGS[name]) + run = object.__new__(ao._Run) + run.ctx = _ctx(overrides.get("step_params"), overrides.get("run")) + run.us = {"step_id": "user", "model": "sim-model"} + assert run._user_model() == expected + + +def _judge(data_model: str | None = None) -> RubricsVerifierTaskStep: + data = RubricsVerifierTaskStep(id="judge", version=None, criteria=[{"id": "c1", "description": "says hi"}], + prompt_id="prompt").to_dict() + data["default_model"] = data_model + return RubricsVerifierTaskStep.from_dict(data) + + +@pytest.mark.parametrize("name", CONFIGS) +@pytest.mark.parametrize("layers", _subsets(("run", "step")), ids=str) +def test_the_judge_model(model_config, name, layers): + model_config(CONFIGS[name]) + verifier = _judge("step-judge" if layers["step"] else None) + context = _ctx(run={"judge_model": "run-judge"} if layers["run"] else None) + expected = _first(layers["run"] and "run-judge", layers["step"] and "step-judge", JUDGE[name]) + assert _judge_model(verifier, context) == expected + + +def test_a_judge_model_stored_in_a_task_wins_over_the_reader_s_config(model_config): + model_config(CONFIGS["judge_role"]) + stored = RubricsVerifierTaskStep(id="judge", version=None, criteria=[{"id": "c1", "description": "says hi"}], + prompt_id="prompt", default_model="writer-judge").to_dict() + model_config(CONFIGS["every_role"]) + assert _judge_model(RubricsVerifierTaskStep.from_dict(stored), _ctx()) == "writer-judge" diff --git a/tst/unit/task_step/test_step_overrides_agent_and_model.py b/tst/unit/task_step/test_step_overrides_agent_and_model.py index d9eb11de0..9403bc486 100644 --- a/tst/unit/task_step/test_step_overrides_agent_and_model.py +++ b/tst/unit/task_step/test_step_overrides_agent_and_model.py @@ -8,7 +8,7 @@ import pytest -from agent_env.config.model import ModelConfig +from agent_env.config.model import ModelConfig, resolve_model from agent_env.task_step.context import TaskStepContext from agent_env.task_step.task_steps import prompt_agent from agent_env.task_step.task_steps.deploy_agent import DeployAgentTaskStep @@ -119,7 +119,7 @@ class TestUserSimModel: def config(self, monkeypatch): roles = ModelConfig(default="config-default", roles={}) monkeypatch.setattr(prompt_agent, "get_config", lambda: type("C", (), { - "get_model_for_role": staticmethod(roles.model_for_role)})()) + "resolve_model": staticmethod(lambda role, **layers: resolve_model(role, config=roles, **layers))})()) return roles def test_stored_value_without_overrides(self): diff --git a/tst/unit/task_step/verifiers/test_rubrics_verifier_judge_model.py b/tst/unit/task_step/verifiers/test_rubrics_verifier_judge_model.py index 643cfb5d8..458075796 100644 --- a/tst/unit/task_step/verifiers/test_rubrics_verifier_judge_model.py +++ b/tst/unit/task_step/verifiers/test_rubrics_verifier_judge_model.py @@ -1,9 +1,11 @@ -"""A rubrics verifier built in code, as the A2A validator builds its own, judges with the configured ``judge`` model, -as one read from a task does.""" +"""A rubrics judge resolves its model when it runs: the run's ``judge_model``, the model its author named, then the +config's ``judge`` role and default, and claude-sonnet-4-6. A verifier built in code, as the A2A validator builds its +own, judges like one read from a task, and a stored task keeps only its author's model, never its writer's config.""" import pytest from agent_env.config import reset_config +from agent_env.task_step.context import TaskStepContext from agent_env.task_step.task_steps.verifiers.rubrics_verifier import RubricsVerifierTaskStep CRITERIA = [{"id": "c1", "description": "Response says hello"}] @@ -32,28 +34,50 @@ def _built(**kwargs) -> RubricsVerifierTaskStep: return RubricsVerifierTaskStep(id="verify", version=None, criteria=CRITERIA, prompt_id="p1", **kwargs) +def _run(**overrides) -> TaskStepContext: + return TaskStepContext(metadata={"user_overrides": overrides} if overrides else {}) + + def test_a_verifier_built_in_code_judges_with_the_configured_judge_model(judge_role): - assert _built().default_model == "judge-model" + assert _built()._judge_model(_run()) == "judge-model" def test_a_verifier_read_from_a_task_judges_with_the_configured_judge_model(judge_role): data = _built().to_dict() del data["default_model"] - assert RubricsVerifierTaskStep.from_dict(data).default_model == "judge-model" + assert RubricsVerifierTaskStep.from_dict(data)._judge_model(_run()) == "judge-model" def test_a_model_the_step_names_wins_over_the_judge_role(judge_role): - assert _built(default_model="named-model").default_model == "named-model" + assert _built(default_model="named-model")._judge_model(_run()) == "named-model" + + +def test_the_run_s_judge_model_wins_over_the_step_s(judge_role): + assert _built(default_model="named-model")._judge_model(_run(judge_model="run-model")) == "run-model" def test_without_a_judge_role_the_judge_is_the_configured_default_model(model_config): model_config('default = "agent-model"\n') - assert _built().default_model == "agent-model" + assert _built()._judge_model(_run()) == "agent-model" def test_with_no_model_configured_the_judge_is_claude_sonnet(model_config): model_config("") - assert _built().default_model == "claude-sonnet-4-6" + assert _built()._judge_model(_run()) == "claude-sonnet-4-6" + + +def test_a_stored_task_carries_no_model_from_its_writer_s_config(model_config, judge_role): + stored = _built().to_dict() + assert stored["default_model"] is None + + model_config('[model.roles]\njudge = "reader-judge"\n') + assert RubricsVerifierTaskStep.from_dict(stored)._judge_model(_run()) == "reader-judge" + + +def test_a_model_frozen_into_a_task_stored_before_keeps_judging(judge_role): + stored = {**_built().to_dict(), "default_model": "frozen-model"} + + assert RubricsVerifierTaskStep.from_dict(stored)._judge_model(_run()) == "frozen-model" diff --git a/tst/unit/task_step/verifiers/test_user_sim_judge.py b/tst/unit/task_step/verifiers/test_user_sim_judge.py index 65c3942e7..7b7d86e56 100644 --- a/tst/unit/task_step/verifiers/test_user_sim_judge.py +++ b/tst/unit/task_step/verifiers/test_user_sim_judge.py @@ -13,7 +13,7 @@ import litellm import pytest -from agent_env.config.model import ModelConfig +from agent_env.config.model import ModelConfig, resolve_model from agent_env.openenv.reward import reward_of from agent_env.store import LocalSecretStore from agent_env.task import Task @@ -964,17 +964,19 @@ def test_by_default_it_fails_the_task_on_a_slip_and_judges_with_the_verifiers_de assert step.to_dict()["judge_model"] is None # the task doesn't pin it class Config: - role = None + roles: dict = {} - def get_model_for_role(self, role): + def resolve_model(self, role, **layers): assert role == "judge" - return self.role + return resolve_model(role, config=ModelConfig(roles=self.roles), **layers) fake = Config() monkeypatch.setattr("agent_env.task_step.task_steps.verifiers.rubrics_verifier.get_config", lambda: fake) - assert usj.UserSimJudge("s", None, run_step_id="week").default_model == "claude-sonnet-4-6" - fake.role = "anthropic/claude-opus-4-8" - assert usj.UserSimJudge("s", None, run_step_id="week").default_model == "anthropic/claude-opus-4-8" + judge = usj.UserSimJudge("s", None, run_step_id="week") + assert judge.default_model is None + assert judge._judge_model(TaskStepContext()) == "claude-sonnet-4-6" + fake.roles = {"judge": "anthropic/claude-opus-4-8"} + assert judge._judge_model(TaskStepContext()) == "anthropic/claude-opus-4-8" # ---- the load checks ----------------------------------------------------------------------------------------------