From 6968d74b943a318f9b2db2e555104a65081895b2 Mon Sep 17 00:00:00 2001 From: Edgar Arakelyan Date: Thu, 19 Nov 2026 15:19:03 -0800 Subject: [PATCH 1/4] feat(task-steps): a prompt step keeps each role's usage, with the endpoint host it called The host read only tool_call_count from an agent's reply and dropped its tokens and cost. It now reads the turn's whole Usage with read_usage, which never raises: a value Usage refuses is kept aside, an unknown key goes to provider_details, and an agent from before the typed cache fields has its provider_details cache counts read. prompt_agent and always_on sum each role's turns into one entry, solver and user_simulator, with the model, the endpoint host (no key or URL userinfo) and the attempt epoch the scheduler now sets for each step. A count nobody reported is None, never 0, and partial marks a lower bound: turns that reported nothing, or calls a turn says went unreported. Entries are written to metadata.step_usage as each turn ends, so a step that times out keeps its finished turns, and to the step's PromptResponse.usage. metadata.model_endpoints records each role's model and host beside metadata.models, which keeps its shape. Co-Authored-By: Claude Opus 5.5 --- src/agent_env/a2a_agent/a2a_agent.py | 14 ++ src/agent_env/a2a_agent/protocol.py | 59 +++++- src/agent_env/config/model.py | 12 ++ src/agent_env/task/task.py | 10 +- src/agent_env/task_step/context.py | 16 ++ .../task_step/task_steps/always_on.py | 33 +++- .../task_step/task_steps/deploy_agent.py | 3 +- .../task_step/task_steps/install_agent.py | 17 +- .../task_step/task_steps/prompt_agent.py | 44 ++++- src/agent_env/task_step/usage.py | 81 +++++++++ tst/unit/a2a_agent/protocol_test.py | 37 ++++ tst/unit/task/attempt_epoch_test.py | 55 ++++++ tst/unit/task_step/agent_clock_test.py | 3 + tst/unit/task_step/always_on_harness.py | 12 +- tst/unit/task_step/capture_stubs.py | 1 + .../task_step/deploy_agent_cleanup_test.py | 3 + tst/unit/task_step/test_always_on_usage.py | 49 +++++ tst/unit/task_step/test_model_key_scope.py | 20 +- tst/unit/task_step/test_prompt_agent_usage.py | 171 ++++++++++++++++++ tst/unit/task_step/usage_test.py | 45 +++++ 20 files changed, 663 insertions(+), 22 deletions(-) create mode 100644 src/agent_env/task_step/usage.py create mode 100644 tst/unit/task/attempt_epoch_test.py create mode 100644 tst/unit/task_step/test_always_on_usage.py create mode 100644 tst/unit/task_step/test_prompt_agent_usage.py create mode 100644 tst/unit/task_step/usage_test.py diff --git a/src/agent_env/a2a_agent/a2a_agent.py b/src/agent_env/a2a_agent/a2a_agent.py index 915375304..d26f3fdd2 100644 --- a/src/agent_env/a2a_agent/a2a_agent.py +++ b/src/agent_env/a2a_agent/a2a_agent.py @@ -633,6 +633,20 @@ def _refusal(self, image: DockerImageArtifact, link: SandboxProvider) -> str | N return f"{how}: {problem}" return None + def model_base_url(self, env_vars: dict[str, str], config) -> Optional[str]: + """The model endpoint a deploy with ``env_vars`` gives this agent, by ``_with_model_credentials``'s rule: + theirs, else its ``default_env_vars``', else the configured one; None for an agent that takes no credentials, + which gets none, or with no endpoint configured.""" + if not self.metadata.get(TAKES_CREDENTIALS, True): + return None + for given in (env_vars, self.default_env_vars): + if "LITELLM_BASE_URL" in given: + return given["LITELLM_BASE_URL"] + try: + return config.get_litellm_base_url() + except ConfigError: + return None + def _with_model_credentials(self, env_vars: dict[str, str], config) -> dict[str, str]: """``env_vars`` with the model endpoint, key and headers each taken from config when neither they nor ``default_env_vars`` set it; for an agent that takes no credentials, with none of them, whatever set them. The diff --git a/src/agent_env/a2a_agent/protocol.py b/src/agent_env/a2a_agent/protocol.py index 5b458b83e..3f2f7d959 100644 --- a/src/agent_env/a2a_agent/protocol.py +++ b/src/agent_env/a2a_agent/protocol.py @@ -7,11 +7,12 @@ import re import time from collections.abc import Mapping -from dataclasses import dataclass +from dataclasses import dataclass, fields from typing import Any, Optional import httpx from a2a.types import TaskState +from agentenv_protocol.a2a_agent import Usage logger = logging.getLogger(__name__) @@ -268,10 +269,51 @@ async def cancel_a2a_task(a2a_url: str, task_id: str) -> None: logger.info(f"Canceled A2A task {task_id}") +_USAGE_FIELDS = frozenset(f.name for f in fields(Usage)) - {"provider_details"} +# Where agents before Usage had cache fields put their cache counts. +_LEGACY_CACHE_KEYS = ("cache_read_tokens", "cache_write_tokens") + + +def read_usage(raw: Any) -> tuple[Optional[Usage], Optional[dict]]: + """An agent's reported usage and what of it couldn't be read, never raising: each field ``Usage`` has, checked on + its own, a value it refuses kept under its name in the second; a key it doesn't have kept in ``provider_details``; + the cache counts an agent from before the typed fields put in ``provider_details`` read from there.""" + if raw is None: + return None, None + if not isinstance(raw, Mapping): + return None, {"usage": raw} + unparsed: dict = {} + details = raw.get("provider_details") + if details is not None and not isinstance(details, Mapping): + unparsed["provider_details"] = details + details = None + details = dict(details or {}) + given = {key: value for key, value in raw.items() if key in _USAGE_FIELDS} + details.update({key: value for key, value in raw.items() if key not in _USAGE_FIELDS and key != "provider_details"}) + for name in _LEGACY_CACHE_KEYS: + if given.get(name) is None and name in details: + given[name] = details.pop(name) + valid = {} + for key, value in given.items(): + try: + Usage(**{key: value}) + except (TypeError, ValueError): + unparsed[key] = value + else: + valid[key] = value + try: + usage = Usage(**valid, provider_details=details) + except (TypeError, ValueError): + unparsed["provider_details"] = details + usage = Usage(**valid) + return usage, unparsed or None + + @dataclass(frozen=True) class TerminalResponse: """What's pulled from an A2A terminal status message: the text reply, the typed - ``structured_output`` (when ``output_format`` was set), and tool/error telemetry.""" + ``structured_output`` (when ``output_format`` was set), tool/error telemetry, and the turn's ``usage`` as + ``read_usage`` reads it, with what of it couldn't be read in ``usage_unparsed``.""" response_text: str tool_call_count: Optional[int] = None @@ -280,6 +322,8 @@ class TerminalResponse: error_class: Optional[str] = None error_message: Optional[str] = None structured_output: Optional[Any] = None + usage: Optional[Usage] = None + usage_unparsed: Optional[dict] = None @classmethod def from_message(cls, status_message: dict) -> "TerminalResponse": @@ -292,6 +336,8 @@ def from_message(cls, status_message: dict) -> "TerminalResponse": error_class: Optional[str] = None error_message: Optional[str] = None structured_output: Optional[Any] = None + usage: Optional[Usage] = None + usage_unparsed: Optional[dict] = None for part in (status_message or {}).get("parts", []): kind = part.get("kind") if kind == "text": @@ -300,9 +346,10 @@ def from_message(cls, status_message: dict) -> "TerminalResponse": data = part.get("data", {}) if "tool_call_count" in data: tool_call_count = data["tool_call_count"] - usage = data.get("usage") - if isinstance(usage, dict) and "tool_call_count" in usage: - tool_call_count = usage["tool_call_count"] + if "usage" in data: + usage, usage_unparsed = read_usage(data["usage"]) + if isinstance(data["usage"], dict) and "tool_call_count" in data["usage"]: + tool_call_count = data["usage"]["tool_call_count"] if "error_type" in data: error_type = data["error_type"] if "error_code" in data: @@ -321,6 +368,8 @@ def from_message(cls, status_message: dict) -> "TerminalResponse": error_class=error_class, error_message=error_message, structured_output=structured_output, + usage=usage, + usage_unparsed=usage_unparsed, ) diff --git a/src/agent_env/config/model.py b/src/agent_env/config/model.py index 60106d4f5..457c7ef11 100644 --- a/src/agent_env/config/model.py +++ b/src/agent_env/config/model.py @@ -312,6 +312,18 @@ def _netloc(url: str) -> str: return urlsplit(url).netloc.rpartition("@")[2] or url +def endpoint_host(url: object) -> Optional[str]: + """The host, and port when it names one, a model call to ``url`` reaches, for a record: never the user or + password a URL may carry. None for anything but a URL with a host, never raising, so a record can't fail what it + records.""" + if not isinstance(url, str): + return None + try: + return urlsplit(url.strip()).netloc.rpartition("@")[2] or None + except ValueError: + return None + + def no_key_for(url: str, how: str) -> str: """Why a model call or an agent pointed at ``url`` with no key of its own is refused, and ``how`` to give it one.""" return (f"{_netloc(url)} isn't the configured model endpoint, so the configured key isn't sent there, and no key " diff --git a/src/agent_env/task/task.py b/src/agent_env/task/task.py index 8b761d893..506886890 100644 --- a/src/agent_env/task/task.py +++ b/src/agent_env/task/task.py @@ -26,7 +26,13 @@ from agent_env.task_step.preflight import pinned from agent_env.task.outcome import FailedStep from agent_env.task.teardown import env_record_ids -from agent_env.task_step.context import ENTITY_IDS_KEY, TaskStepContext, VerifierNotPassingError, regraft_redacted_keys +from agent_env.task_step.context import ( + ATTEMPT_EPOCH, + ENTITY_IDS_KEY, + TaskStepContext, + VerifierNotPassingError, + regraft_redacted_keys, +) from agent_env.task_step.context_ops import ContextUpdateOps, build_context_update_ops from agent_env.task_step.task_step import RetryConfig, TaskStep @@ -699,6 +705,8 @@ async def _run_one(step: TaskStep, idx: int, dispatch_epoch: int) -> None: pre_context = copy.deepcopy(context) started_mono = time.monotonic() started_utc = datetime.now(timezone.utc).isoformat() + # This step's own asyncio task, so a sibling dispatched alongside it doesn't see it. + ATTEMPT_EPOCH.set(dispatch_epoch) try: with protocol_v1.authorized({d.environment_url: d.control_headers for d in context.deployed_envs}): await step.execute(context) diff --git a/src/agent_env/task_step/context.py b/src/agent_env/task_step/context.py index 002a68d71..8c41461bb 100644 --- a/src/agent_env/task_step/context.py +++ b/src/agent_env/task_step/context.py @@ -6,6 +6,7 @@ import dataclasses import re from collections.abc import Callable +from contextvars import ContextVar from dataclasses import dataclass, field from typing import Any @@ -17,6 +18,9 @@ from agent_env.providers.sandbox_providers.sandbox_provider import reachable_url HEARTBEAT_KEY = "_heartbeat_fn" +# The attempt epoch the step running in this task was dispatched under, which ``Task.run`` sets for it; None for a +# step run outside it. +ATTEMPT_EPOCH: ContextVar[int | None] = ContextVar("agentenv_attempt_epoch", default=None) # context.metadata[ENTITY_IDS_KEY]: the run's task's Task.entity_ids, set by Task.run. ENTITY_IDS_KEY = "entity_ids" # The context.metadata keys whose entries name a deployed env by its id, each with the index of the path segment that @@ -134,6 +138,8 @@ class DeployedAgent: env_bindings: dict[str, str | None] = field(default_factory=dict) # The token its extension routes take, from the deploy that started it; None for an agent agent-env didn't start. control_token: str | None = None + # The host (and port) of the model endpoint it was given, never a key or a URL's userinfo; None when unknown. + model_endpoint_host: str | None = None @classmethod def from_dict(cls, data: dict) -> DeployedAgent: @@ -152,6 +158,7 @@ def from_dict(cls, data: dict) -> DeployedAgent: env_ids=list(data.get("env_ids") or []), env_bindings=dict(data.get("env_bindings") or {}), control_token=data.get("control_token"), + model_endpoint_host=data.get("model_endpoint_host"), ) @property @@ -248,6 +255,8 @@ class PromptResponse: agent_trajectory_object_url: str | None = None agent_trajectory_object_prefix: str | None = None target_agent_per_turn_trajectory_object_urls: list[str | None] | None = None + # The step's usage, one entry per role that ran turns in it (``task_step.usage``); None before it was recorded. + usage: list[dict] | None = None def to_dict(self) -> dict[str, Any]: """The stored form: every field, plus the legacy key of each renamed one.""" @@ -282,6 +291,7 @@ def from_dict(cls, data: dict) -> PromptResponse: agent_name=data.get("agent_name"), step_id=data.get("step_id"), structured_output=data.get("structured_output"), + usage=data.get("usage"), ) @@ -316,6 +326,12 @@ def record_model(self, role: str, step_id: str, model: str | None) -> None: """Record the model ``step_id`` ran ``role`` on, None when the agent chose its own, for the run's summary.""" self.metadata.setdefault("models", {}).setdefault(role, {})[step_id] = model + def record_model_endpoint(self, role: str, step_id: str, model: str | None, endpoint_host: str | None) -> None: + """Record the model ``step_id`` ran ``role`` on and the host of the endpoint it called (``endpoint_host``, + never a key or a URL's userinfo), beside ``models`` under the role's own name.""" + self.metadata.setdefault("model_endpoints", {}).setdefault(role, {})[step_id] = { + "model": model, "endpoint_host": endpoint_host} + def to_safe_dict(self) -> dict[str, Any]: """Return a dict representation with sensitive keys recursively removed.""" d = self.to_dict() diff --git a/src/agent_env/task_step/task_steps/always_on.py b/src/agent_env/task_step/task_steps/always_on.py index 09d99bedc..2c89099b2 100644 --- a/src/agent_env/task_step/task_steps/always_on.py +++ b/src/agent_env/task_step/task_steps/always_on.py @@ -113,6 +113,7 @@ from agent_env.config.model import ModelRole from agent_env.store import DuplicateKeyError, get_config from agent_env.task_step.context import PromptResponse, StepContextError, TaskStepContext +from agent_env.task_step.usage import RoleUsage, record_step_usage from agent_env.task_step.task_step import TaskStep from agent_env.task_step.task_steps.agent import user_sim_versions from agent_env.task_step.task_steps.agent.deploy_user_sim_agent import USER_SIMS_KEY, user_sim_model @@ -379,6 +380,9 @@ def __init__(self, step: AlwaysOnTaskStep, context: TaskStepContext): self.per_turn_trajectory_urls: list[Optional[str]] = [] self.per_turn_parts: list[Optional[list[dict]]] = [] self.tool_calls: Optional[int] = None + # Each role's turns' usage, recorded as each turn ends. + self.solver_usage: Optional[RoleUsage] = None + self.user_usage: Optional[RoleUsage] = None self.all_messages: list[dict] = [] self.final_task_id = "" self.error: Optional[str] = None @@ -594,7 +598,10 @@ async def _configure_agents(self) -> None: if negotiated: await protocol.post_agent_config(self.target_url + negotiated.endpoint, negotiated.fields, headers=self.agent.control_headers) - ctx.record_model(ModelRole.SOLVER.record_key, step.id, negotiated.fields.get("model") if negotiated else None) + solver_model = negotiated.fields.get("model") if negotiated else None + ctx.record_model(ModelRole.SOLVER.record_key, step.id, solver_model) + ctx.record_model_endpoint(ModelRole.SOLVER.value, step.id, solver_model, self.agent.model_endpoint_host) + self.solver_usage = RoleUsage(ModelRole.SOLVER.value, solver_model, self.agent.model_endpoint_host) self.rules_in_system_prompt = bool(desired.get("system_prompt") and negotiated and "system_prompt" in negotiated.fields) if desired.get("system_prompt") and not self.rules_in_system_prompt: @@ -604,6 +611,7 @@ async def _configure_agents(self) -> None: if self.us.get("prompt_set"): # deploy_user_sim_agent set the user sim's prompt and model at deploy self.user_model, self.user_model_from = self.us["model"], self.us["model_source"] self.user_setup = dict(self.us["setup"]) + self._start_user_usage(self.user_model) return setup = self.plugin.setup(self.version, self.person) self.user_model, self.user_model_from = self._user_model() @@ -616,6 +624,7 @@ async def _configure_agents(self) -> None: headers=self.user_agent.control_headers) accepted = sorted(negotiated.fields) if negotiated else [] ctx.record_model(ModelRole.USER_SIMULATOR.record_key, step.id, negotiated.fields.get("model") if negotiated else None) + self._start_user_usage(negotiated.fields.get("model") if negotiated else None) self.user_setup = {"model": self.user_model, "model_from": self.user_model_from, "user_sim_step_model": self.us.get("model"), "model_set": "model" in accepted, # else the agent ran its own @@ -1271,6 +1280,7 @@ def record() -> None: term = protocol.TerminalResponse.from_message(status_msg) if term.tool_call_count is not None: self.tool_calls = (self.tool_calls or 0) + term.tool_call_count + self._record_usage(self.solver_usage, term) messages, quiet = await self._take_messages() await self._read_changes("turn") traj = None @@ -1351,6 +1361,7 @@ def record() -> None: term = protocol.TerminalResponse.from_message(status_msg) if term.tool_call_count is not None: self.tool_calls = (self.tool_calls or 0) + term.tool_call_count + self._record_usage(self.solver_usage, term) messages, quiet = await self._take_messages() await self._read_changes("nudge") traj = None @@ -1557,10 +1568,11 @@ async def _ask_user_sim(self, text: str, deadline: float) -> dict: except TimeoutError as e: return {"text": None, "structured": None, "error": f"timeout: {e}"} status = (result.get("status") or {}) - if status.get("state") != TaskState.completed and str(status.get("state")) != "completed": - return {"text": None, "structured": None, "error": f"state {status.get('state')}"} msg = status.get("message") or {} term = protocol.TerminalResponse.from_message(msg) + self._record_usage(self.user_usage, term) + if status.get("state") != TaskState.completed and str(status.get("state")) != "completed": + return {"text": None, "structured": None, "error": f"state {status.get('state')}"} raw = next((p["text"] for p in msg.get("parts") or [] if p.get("kind") == "text"), term.response_text) structured = term.structured_output if structured is None: @@ -1736,7 +1748,20 @@ def _append_prompt_response(self, final_text: str) -> None: source_agent_per_turn_prompt_parts=self.per_turn_parts or None, tool_call_count=self.tool_calls, model=self.model, error_type=self.error_type, error_code=self.error_code, error_message=self.error, agent_session_id=self.final_task_id, - a2a_context_id=self.solver_context_id, agent_name=step.agent_name, step_id=step.id)) + a2a_context_id=self.solver_context_id, agent_name=step.agent_name, step_id=step.id, + usage=record_step_usage(ctx, step.id, (self.solver_usage, self.user_usage)))) + + def _start_user_usage(self, model: Optional[str]) -> None: + """The user sim's usage in this step, its model and endpoint host recorded beside the solver's.""" + host = self.user_agent.model_endpoint_host + self.ctx.record_model_endpoint(ModelRole.USER_SIMULATOR.value, self.step.id, model, host) + self.user_usage = RoleUsage(ModelRole.USER_SIMULATOR.value, model, host) + + def _record_usage(self, role: Optional[RoleUsage], term: protocol.TerminalResponse) -> None: + """One finished turn's usage, recorded as it ends, so a step that raises later keeps it.""" + if role is not None: + role.add(term.usage, term.usage_unparsed) + record_step_usage(self.ctx, self.step.id, (self.solver_usage, self.user_usage)) # ================================================================================================================== diff --git a/src/agent_env/task_step/task_steps/deploy_agent.py b/src/agent_env/task_step/task_steps/deploy_agent.py index cbb4adb95..a4d8065cc 100644 --- a/src/agent_env/task_step/task_steps/deploy_agent.py +++ b/src/agent_env/task_step/task_steps/deploy_agent.py @@ -30,7 +30,7 @@ ) from agent_env.a2a_agent.staging import StagedObjectStore, transfer_store from agent_env.config import get_config -from agent_env.config.model import ModelRole +from agent_env.config.model import ModelRole, endpoint_host from agent_env.env.env import DeployedEnv, DeployedSandboxEnv, Env from agent_env.env.gateway import AGENT_ENV_ROLE_HEADER from agent_env.providers.sandbox_providers.local_docker.provider import host_url_for @@ -554,6 +554,7 @@ async def _execute(self, context: TaskStepContext, deployed_agents: list) -> Tas env_bindings={env_id: (d.metadata or {}).get("deploy_step_id") for env_id, d in env_deployments.items() if d is not None}, control_token=deployed.control_token, + model_endpoint_host=endpoint_host(agent.model_base_url(env_vars, get_config())), )) # An agent deployed while a clock is armed follows it, as sync_env_clock had every agent then. from agent_env.a2a_agent.clock import follow_armed_clocks diff --git a/src/agent_env/task_step/task_steps/install_agent.py b/src/agent_env/task_step/task_steps/install_agent.py index f7808ee16..13caa6499 100644 --- a/src/agent_env/task_step/task_steps/install_agent.py +++ b/src/agent_env/task_step/task_steps/install_agent.py @@ -27,9 +27,10 @@ import httpx from agent_env.a2a_agent import A2AAgent +from agent_env.a2a_agent.a2a_agent import TAKES_CREDENTIALS from agent_env.artifact.artifacts.docker_image import DockerImageArtifact -from agent_env.config import get_config -from agent_env.config.model import ModelRole +from agent_env.config import ConfigError, get_config +from agent_env.config.model import ModelRole, endpoint_host from agent_env.entity.scopes import FieldScope from agent_env.env.envs.computer.utils.boot import control_token as new_control_token from agent_env.providers.sandbox_providers.local_docker.provider import ( @@ -151,6 +152,17 @@ def _resolve_container() -> str: } +def _installed_model_host(agent: A2AAgent) -> Optional[str]: + """The host of the model endpoint an installed agent calls: the configured one, which its install commands take, + and none for an agent that takes no credentials.""" + if not agent.metadata.get(TAKES_CREDENTIALS, True): + return None + try: + return endpoint_host(get_config().get_litellm_base_url()) + except ConfigError: + return None + + class InstallAgentTaskStep(TaskStep): type: ClassVar[str] = "install_agent" entity_refs = (EntityRef.agent("a2a_agent_id", version_field="a2a_agent_version"),) @@ -390,6 +402,7 @@ async def execute(self, context: TaskStepContext) -> TaskStepContext: on_host=host_mode, installed_container=container, control_token=deployed.control_token, + model_endpoint_host=_installed_model_host(agent), )) default_model = get_config().resolve_model(ModelRole.SOLVER, agent=agent.metadata.get("default_model")).model diff --git a/src/agent_env/task_step/task_steps/prompt_agent.py b/src/agent_env/task_step/task_steps/prompt_agent.py index 64f862c81..4469921db 100644 --- a/src/agent_env/task_step/task_steps/prompt_agent.py +++ b/src/agent_env/task_step/task_steps/prompt_agent.py @@ -10,7 +10,7 @@ import uuid from contextlib import nullcontext from datetime import datetime, timezone -from collections.abc import Mapping +from collections.abc import Callable, Mapping from typing import Any, ClassVar, Optional import httpx from a2a.types import TaskState @@ -37,6 +37,7 @@ from agent_env.store import DuplicateKeyError, get_config from agent_env.store.object_store import ObjectStore from agent_env.task_step.context import DeployedAgent, PromptResponse, StepContextError, TaskStepContext +from agent_env.task_step.usage import RoleUsage, record_step_usage from agent_env.entity.refs import EntityRef from agent_env.entity.scopes import FieldScope from agent_env.task_step.task_step import RetryConfig, StepRole, TaskStep, TaskStepDependency @@ -398,6 +399,13 @@ def _resolve_model_and_effort(self, context: TaskStepContext) -> tuple[Optional[ effort = step_overrides.get("effort") or overrides.get("agent_effort") or self.effort return model, effort + def _user_usage(self, context: TaskStepContext, model: Optional[str], + user_sim: Optional[DeployedAgent]) -> RoleUsage: + """The user simulator's usage in this step, its model and endpoint host recorded beside the solver's.""" + host = user_sim.model_endpoint_host if user_sim else None + context.record_model_endpoint(ModelRole.USER_SIMULATOR.value, self.id, model, host) + return RoleUsage(ModelRole.USER_SIMULATOR.value, model, host) + def _resolve_user_model(self, context: TaskStepContext) -> Optional[str]: """The user simulator's model for this run: this step's ``step_overrides``, then the run's ``user_model``, the stored value, and the config's ``user`` role when it names one.""" @@ -560,11 +568,16 @@ async def _converse( await protocol.post_agent_config(target_url + negotiated.endpoint, negotiated.fields, headers=agent.control_headers) # A model the agent's card doesn't accept is never sent, and the agent runs its own. - context.record_model(ModelRole.SOLVER.record_key, self.id, negotiated.fields.get("model") if negotiated else None) + solver_model = negotiated.fields.get("model") if negotiated else None + context.record_model(ModelRole.SOLVER.record_key, self.id, solver_model) + context.record_model_endpoint(ModelRole.SOLVER.value, self.id, solver_model, agent.model_endpoint_host) + solver_usage = RoleUsage(ModelRole.SOLVER.value, solver_model, agent.model_endpoint_host) + user_usage: Optional[RoleUsage] = None is_user_sim = user_sim is not None and self.user_a2a_url is None if built_in is not None: # deploy_user_sim_agent set its model and answer format context.record_model(ModelRole.USER_SIMULATOR.record_key, self.id, built_in[0].get("model")) + user_usage = self._user_usage(context, built_in[0].get("model"), user_sim) elif is_user_sim: user_output_format = self.user_output_format or _DEFAULT_USER_SIM_OUTPUT_FORMAT user_model = self._resolve_user_model(context) @@ -575,6 +588,7 @@ async def _converse( headers=user_sim.control_headers) logger.info(f"Configured user-sim '{self.user_agent_name}' with structured output_format for done-signal") context.record_model(ModelRole.USER_SIMULATOR.record_key, self.id, negotiated.fields.get("model") if negotiated else None) + user_usage = self._user_usage(context, negotiated.fields.get("model") if negotiated else None, user_sim) triggers_registered = is_user_sim and any( r.get("agent_name") == self.user_agent_name @@ -643,7 +657,8 @@ async def _converse( self, context, url=user_url, user_sim=user_sim, is_user_sim=is_user_sim, conversation_id=conversation_id, decide_url=decide_url, shareable=sent_to_target, decides_quietly=card_request_accepts(decide_method.get("request") or {}, _QUIET_DECIDE), - built_in=sim, + built_in=sim, usage=user_usage, + on_usage=lambda: record_step_usage(context, self.id, (solver_usage, user_usage)), ) # Set by a user's done with a message: the agent answers it, then the conversation ends for this reason. end_after: Optional[str] = None @@ -692,6 +707,9 @@ def record_turn() -> None: final_terminal = protocol.TerminalResponse.from_message(status_msg) if final_terminal.tool_call_count is not None: conversation_tool_call_count = (conversation_tool_call_count or 0) + final_terminal.tool_call_count + # As each turn ends, so a step that raises later keeps it. + solver_usage.add(final_terminal.usage, final_terminal.usage_unparsed) + record_step_usage(context, self.id, (solver_usage, user_usage)) agent_response_parts = status_msg.get("parts") or [{"kind": "text", "text": final_terminal.response_text}] conversation_store.complete_a2a_task( @@ -844,6 +862,7 @@ def record_turn() -> None: agent_name=self.agent_name, step_id=self.id, structured_output=structured_output, + usage=record_step_usage(context, self.id, (solver_usage, user_usage)), )) if final_state == TaskState.failed: @@ -1115,9 +1134,12 @@ class _UserSide: def __init__(self, step: PromptAgentTaskStep, context: TaskStepContext, *, url: Optional[str], user_sim: Optional[DeployedAgent], is_user_sim: bool, conversation_id: str, decide_url: Optional[str], decides_quietly: bool, shareable: set[str], - built_in: Optional[_BuiltInUserSim] = None) -> None: + built_in: Optional[_BuiltInUserSim] = None, usage: Optional[RoleUsage] = None, + on_usage: Callable[[], Any] = lambda: None) -> None: self._step = step self._built_in = built_in + self._usage = usage + self._on_usage = on_usage self._context = context self._url = url self._user_sim = user_sim @@ -1166,10 +1188,10 @@ async def _ask_built_in(self, text: str) -> dict: except TimeoutError as e: return {"text": None, "error": f"timeout: {e}"} status = result.get("status") or {} + msg = status.get("message") or {} + term = self._record_usage(msg) if status.get("state") != TaskState.completed: return {"text": None, "error": f"state {status.get('state')}"} - msg = status.get("message") or {} - term = protocol.TerminalResponse.from_message(msg) raw = next((p["text"] for p in msg.get("parts") or [] if p.get("kind") == "text"), term.response_text) return {"text": raw, "structured": term.structured_output if isinstance(term.structured_output, dict) else None, "trajectory_url": await self._user_sim_trajectory(task_id, message_id)} @@ -1229,6 +1251,8 @@ async def _ask_peer(self, number: int, agent_parts: list[dict]) -> dict: return {"end": "user_timeout"} user_state = (user_result.get("status") or {}).get("state") + if self._is_user_sim: + self._record_usage((user_result.get("status") or {}).get("message") or {}) if user_state != TaskState.completed: logger.info(f"user_a2a_url returned state={user_state}; ending multi-turn") return {"end": "user_not_completed"} @@ -1260,6 +1284,14 @@ async def _ask_peer(self, number: int, agent_parts: list[dict]) -> dict: def _record(self, key: str, entry: dict) -> None: self._context.metadata.setdefault(key, {}).setdefault(self._step.id, []).append(entry) + def _record_usage(self, message: dict) -> protocol.TerminalResponse: + """The user sim's reply, its turn's usage recorded as the turn ends.""" + term = protocol.TerminalResponse.from_message(message) + if self._usage is not None: + self._usage.add(term.usage, term.usage_unparsed) + self._on_usage() + return term + class _BuiltInUserSim: """The user deployed by deploy_user_sim_agent, read as always_on reads it, in one sitting: each agent reply is one diff --git a/src/agent_env/task_step/usage.py b/src/agent_env/task_step/usage.py new file mode 100644 index 000000000..3324a5d27 --- /dev/null +++ b/src/agent_env/task_step/usage.py @@ -0,0 +1,81 @@ +"""The usage a prompt step keeps for each role it ran turns of: the turns' reported usage summed into one entry per +role, in the step's ``PromptResponse.usage`` and, as each turn ends, in ``metadata["step_usage"]``, so a step that +raises keeps its finished turns. An entry's counts and cost have ``Usage``'s keys, so a price reads it as a Usage.""" + +from __future__ import annotations + +from collections.abc import Iterable +from dataclasses import dataclass, field +from typing import Any, Optional + +from agentenv_protocol.a2a_agent import Usage + +from agent_env.task_step.context import ATTEMPT_EPOCH, TaskStepContext + +STEP_USAGE_KEY = "step_usage" +_COUNTERS = ("tool_call_count", "input_tokens", "output_tokens", "total_tokens", "cache_read_tokens", + "cache_write_tokens") + + +@dataclass +class RoleUsage: + """One role's turns in one step: the model it ran, the host of the endpoint it called, the attempt epoch the step + ran under, and each turn's usage, None for a turn that reported none.""" + + role: str + model: Optional[str] = None + endpoint_host: Optional[str] = None + attempt: Optional[int] = field(default_factory=ATTEMPT_EPOCH.get) + turns: list[Optional[Usage]] = field(default_factory=list) + unparsed: list[dict] = field(default_factory=list) + + def add(self, usage: Optional[Usage], unparsed: Optional[dict] = None) -> None: + """One finished turn's usage, and what of it couldn't be read.""" + self.turns.append(usage) + if unparsed: + self.unparsed.append(unparsed) + + def entry(self) -> dict[str, Any]: + """The turns summed: each count and the cost the sum of the turns that reported it, None when none did, never + 0. ``input_includes_cache`` is the turns' flag when they agree, else None. ``partial`` says the sum is a lower + bound: some turns reported tokens and others didn't, or a turn said some of its calls reported none + (``unreported_calls``, which the entry sums, or a proxy's ``unreported``).""" + reported = [usage for usage in self.turns if usage is not None] + + def total(name: str) -> Any: + values = [getattr(usage, name) for usage in reported if getattr(usage, name) is not None] + return sum(values) if values else None + + counted = [usage for usage in reported if usage.input_tokens is not None or usage.output_tokens is not None] + flags = {usage.input_includes_cache for usage in counted} + unreported = [n for usage in reported if (n := _unreported_calls(usage))] + partial = (0 < len(counted) < len(self.turns) or bool(unreported) + or any(usage.provider_details.get("unreported") for usage in reported)) + entry: dict[str, Any] = { + "role": self.role, "model": self.model, "endpoint_host": self.endpoint_host, "attempt": self.attempt, + "turns": len(self.turns), **{name: total(name) for name in _COUNTERS}, + "input_includes_cache": flags.pop() if len(flags) == 1 else None, "cost_usd": total("cost_usd"), + "partial": partial, + } + if unreported: + entry["unreported_calls"] = sum(unreported) + if self.unparsed: + entry["unparsed"] = list(self.unparsed) + return entry + + +def _unreported_calls(usage: Usage) -> int: + """The calls a turn's usage says reported none, as the shipped agents put it in ``provider_details``.""" + value = usage.provider_details.get("unreported_calls") + return value if isinstance(value, int) and not isinstance(value, bool) and value > 0 else 0 + + +def record_step_usage(context: TaskStepContext, step_id: str, + roles: Iterable[Optional[RoleUsage]]) -> Optional[list[dict]]: + """The entries of ``roles`` that ran turns, written to ``metadata["step_usage"][step_id]`` and returned; None, and + nothing written, when none did.""" + entries = [role.entry() for role in roles if role is not None and role.turns] + if not entries: + return None + context.metadata.setdefault(STEP_USAGE_KEY, {})[step_id] = entries + return entries diff --git a/tst/unit/a2a_agent/protocol_test.py b/tst/unit/a2a_agent/protocol_test.py index 05ad0728f..cedfe8cfd 100644 --- a/tst/unit/a2a_agent/protocol_test.py +++ b/tst/unit/a2a_agent/protocol_test.py @@ -20,6 +20,7 @@ TerminalResponse, extract_terminal_response, poll_a2a_task, + read_usage, raise_for_extension_status, wake_poll, ) @@ -181,6 +182,42 @@ def test_a_terminal_message_with_cache_usage_parses_and_its_usage_reads_back_who assert Usage.from_dict(sent) == usage +def test_the_host_reads_a_turns_whole_usage_from_the_sdk(): + usage = Usage(tool_call_count=2, input_tokens=120, output_tokens=7, total_tokens=127, cache_read_tokens=100, + input_includes_cache=True, cost_usd=0.004, provider_details={"unreported_calls": 1}) + + parsed = TerminalResponse.from_message(_sdk_terminal_message(TaskResult.success(parts=(TextPart(text="ok"),), + usage=usage))) + + assert (parsed.usage, parsed.usage_unparsed) == (usage, None) + + +@pytest.mark.parametrize("raw, usage, unparsed", [ + ({"tool_call_count": 3}, Usage(tool_call_count=3), None), + ({"input_tokens": 9, "vendor_tokens": 4}, Usage(input_tokens=9, provider_details={"vendor_tokens": 4}), None), + ({"input_tokens": 9, "provider_details": {"cache_read_tokens": 5, "cache_write_tokens": 2, "requests": 1}}, + Usage(input_tokens=9, cache_read_tokens=5, cache_write_tokens=2, provider_details={"requests": 1}), None), + ({"input_tokens": -1, "output_tokens": "many", "input_includes_cache": "yes", "cost_usd": True}, + Usage(), {"input_tokens": -1, "output_tokens": "many", "input_includes_cache": "yes", "cost_usd": True}), + ({"input_tokens": 1, "provider_details": ["not", "a", "mapping"]}, + Usage(input_tokens=1), {"provider_details": ["not", "a", "mapping"]}), + ("garbage", None, {"usage": "garbage"}), + (None, None, None), +], ids=["tool-calls-only", "unknown-key", "legacy-cache-keys", "bad-values", "bad-details", "not-a-mapping", "none"]) +def test_an_agents_usage_is_read_as_far_as_it_can_be_and_never_raises(raw, usage, unparsed): + assert read_usage(raw) == (usage, unparsed) + + +def test_the_tool_call_count_is_read_as_it_always_was_whatever_the_rest_of_the_usage(): + message = {"parts": [{"kind": "data", "data": {"usage": {"tool_call_count": "7", "input_tokens": 3}}}]} + + parsed = TerminalResponse.from_message(message) + + assert parsed.tool_call_count == "7" + assert parsed.usage == Usage(input_tokens=3) and parsed.usage_unparsed == {"tool_call_count": "7"} + assert extract_terminal_response(message)[1] == "7" + + def test_a_failed_turn_sends_its_usage(): usage = Usage(input_tokens=5, output_tokens=2, input_includes_cache=False) message = _sdk_terminal_message(TaskResult.failure("agent.boom", "it broke", usage=usage)) diff --git a/tst/unit/task/attempt_epoch_test.py b/tst/unit/task/attempt_epoch_test.py new file mode 100644 index 000000000..030a609e7 --- /dev/null +++ b/tst/unit/task/attempt_epoch_test.py @@ -0,0 +1,55 @@ +"""A step sees the attempt epoch it was dispatched under, which a rolled-back and retried span moves on, so the usage +it records says which attempt spent it.""" + +import pytest + +from agent_env.config.runtime import Config +from agent_env.task import Task +from agent_env.task_step.context import ATTEMPT_EPOCH, TaskStepContext +from agent_env.task_step.task_step import RetryConfig, TaskStep + + +class _Step(TaskStep): + """Records the epoch it runs under; fails its first run when ``flaky``.""" + + type = "attempt_epoch_test" + entity_refs = () + seen: list[tuple[str, int | None]] = [] + failed: set[str] = set() + + def __init__(self, *, flaky: bool = False, **kwargs): + super().__init__(**kwargs) + self.flaky = flaky + + @classmethod + def from_dict(cls, data): + return cls(**cls._base_from_dict(data)) + + async def execute(self, context): + _Step.seen.append((self.id, ATTEMPT_EPOCH.get())) + if self.flaky and self.id not in _Step.failed: + _Step.failed.add(self.id) + raise RuntimeError("first attempt fails") + return context + + +@pytest.fixture(autouse=True) +def registries(monkeypatch): + steps = {**Config().task_step_registry(), _Step.type: _Step} + monkeypatch.setattr(Config, "task_step_registry", lambda self: steps) + monkeypatch.setattr(_Step, "seen", []) + monkeypatch.setattr(_Step, "failed", set()) + + +@pytest.mark.asyncio +async def test_a_retried_span_runs_under_the_next_epoch(local_stores): + task = Task(id="t", version=None, steps=[ + _Step(id="a", version=None), + _Step(id="b", version=None, flaky=True, depends_on=["a"], + retry_config=RetryConfig(retry_from_step_id="b", max_retries=1)), + ]) + + await task.run(context=TaskStepContext()) + + assert _Step.seen == [("a", 0), ("b", 0), ("b", 1)] + assert ATTEMPT_EPOCH.get() is None diff --git a/tst/unit/task_step/agent_clock_test.py b/tst/unit/task_step/agent_clock_test.py index 551ba0cd0..22c3447ba 100644 --- a/tst/unit/task_step/agent_clock_test.py +++ b/tst/unit/task_step/agent_clock_test.py @@ -417,6 +417,9 @@ async def deploy(self, **_): return SimpleNamespace(a2a_url=AGENT_URL, agent_card=card, sandbox_id="sb-agent", sandbox_type="fake", instance_id="i-1", network_policy=None, control_token=None, control_headers={}) + def model_base_url(self, env_vars, config): + return None + monkeypatch.setattr(a2a_agent_package.A2AAgent, "get", classmethod(lambda cls, *a, **k: _Agent())) sent = _mock_http(monkeypatch, state={"env_get_time_url": CLOCK_URL}) _no_servers(monkeypatch) diff --git a/tst/unit/task_step/always_on_harness.py b/tst/unit/task_step/always_on_harness.py index 15c0d30f3..8e40c3bcb 100644 --- a/tst/unit/task_step/always_on_harness.py +++ b/tst/unit/task_step/always_on_harness.py @@ -54,7 +54,8 @@ class Rig: user sim got in ``reads``. A ``plugin`` replaces the scripted one; what it sends the user sim agent with ``ask`` is in ``user_calls``, answered in turn by ``user_texts`` (then no_reply). ``during`` has, for each of the agent's turns, the gateway's settings and the agent's tools as the agent saw them. With ``takes_config`` the agent's card - takes its config (``CONFIG_CARD``), whose posts land in ``configs``.""" + takes its config (``CONFIG_CARD``), whose posts land in ``configs``. ``solver_usage`` and ``user_usage``, when set, + are the usage each of the agent's and the user sim agent's replies reports.""" def __init__(self, monkeypatch, tmp_path, *, agent: Optional[Turn] = None, answers: tuple = (), rate: float = 0, t0: str = T0, plugin: Any = None, user_texts: tuple = (), takes_config: bool = False): @@ -80,6 +81,8 @@ def __init__(self, monkeypatch, tmp_path, *, agent: Optional[Turn] = None, answe self.solver_contexts: list[Optional[str]] = [] self.takes_config = takes_config self.configs: list[tuple[str, dict]] = [] + self.solver_usage: Optional[dict] = None + self.user_usage: Optional[dict] = None self._contacts: dict[str, dict] = {} self._monkeypatch = monkeypatch monkeypatch.setattr(ao, "POLL_SECONDS", 0) @@ -249,7 +252,7 @@ async def _send_and_wait(self, url, parts, *, before_send=None, **kw): self.user_calls.append({"parts": list(parts), "context_id": kw.get("context_id")}) text = self.user_texts.pop(0) if self.user_texts else '{"outcome": "no_reply", "message": ""}' return f"user-{len(self.user_calls)}", {"status": {"state": "completed", "message": { - "parts": [{"kind": "text", "text": text}]}}} + "parts": [{"kind": "text", "text": text}, *self._usage_part(self.user_usage)]}}} assert url == SOLVER, url self.solver_contexts.append(kw.get("context_id")) if before_send: @@ -269,8 +272,13 @@ async def _send_and_wait(self, url, parts, *, before_send=None, **kw): return f"task-{len(self.sent)}", {"status": {"state": closing.state, "message": {"parts": [ {"kind": "text", "text": closing.message}, {"kind": "data", "data": data}]}}} reply = [{"kind": "text", "text": closing}] if closing else [] + reply += self._usage_part(self.solver_usage) return f"task-{len(self.sent)}", {"status": {"state": "completed", "message": {"parts": reply}}} + @staticmethod + def _usage_part(usage: Optional[dict]) -> list[dict]: + return [{"kind": "data", "data": {"usage": usage}}] if usage is not None else [] + async def _load_plugin(self, spec): if self.plugin is not None: return self.plugin diff --git a/tst/unit/task_step/capture_stubs.py b/tst/unit/task_step/capture_stubs.py index c1bc73954..746e0f1a7 100644 --- a/tst/unit/task_step/capture_stubs.py +++ b/tst/unit/task_step/capture_stubs.py @@ -197,6 +197,7 @@ def __init__(self, name="solver", a2a_url="https://agent", card=None): self.sandbox_id = "sb-agent" self.sandbox_type = None self.role = None + self.model_endpoint_host = None self.control_headers = {} diff --git a/tst/unit/task_step/deploy_agent_cleanup_test.py b/tst/unit/task_step/deploy_agent_cleanup_test.py index 66b9eb1e5..b29b3b463 100644 --- a/tst/unit/task_step/deploy_agent_cleanup_test.py +++ b/tst/unit/task_step/deploy_agent_cleanup_test.py @@ -22,6 +22,9 @@ async def deploy(self, **_): instance_id="i-1", network_policy=None, control_token=None, control_headers={}, ) + def model_base_url(self, env_vars, config): + return None + async def close(self) -> None: self.closed += 1 diff --git a/tst/unit/task_step/test_always_on_usage.py b/tst/unit/task_step/test_always_on_usage.py new file mode 100644 index 000000000..7f6099967 --- /dev/null +++ b/tst/unit/task_step/test_always_on_usage.py @@ -0,0 +1,49 @@ +"""An always_on step keeps its agents' usage as prompt_agent does: one entry for the agent's wakes and nudges and one +for the user sim agent's reads, each summed, with the endpoint each called.""" + +from __future__ import annotations + +import pytest + +from tst.unit.task_step.always_on_harness import Rig + +pytestmark = pytest.mark.usefixtures("local_stores") + + +class _AskingPlugin: + @staticmethod + def setup(version, person): + return {"system_prompt": "you are Linda", "output_format": None} + + @staticmethod + async def answer(version, person, history, read, ask): + got = await ask(f"read at {read['time']}") + return {**(got["structured"] or {}), "raw": got["text"]} + + +def _agent(n, texts, rig): + rig.message("blue or green?") if n == 1 else rig.quiet() + return "" + + +@pytest.mark.asyncio +async def test_each_role_s_turns_are_summed_with_the_endpoint_it_called(monkeypatch, tmp_path): + rig = Rig(monkeypatch, tmp_path, agent=_agent, plugin=_AskingPlugin, + user_texts=['{"outcome": "reply", "message": "blue"}']) + rig.solver_usage = {"tool_call_count": 1, "input_tokens": 100, "output_tokens": 10, "input_includes_cache": True} + rig.user_usage = {"input_tokens": 30, "output_tokens": 3, "input_includes_cache": True, "cost_usd": 0.001} + ctx = rig.context() + ctx.deployed_agents[0].model_endpoint_host = "proxy.example" + ctx.deployed_agents[1].model_endpoint_host = "sim-proxy.example" + await rig.arm() + + await rig.run(rig.step(), ctx) + + solver, user = ctx.prompt_responses[0].usage + wakes, reads = len(rig.sent), len(rig.user_calls) + assert (solver["role"], solver["endpoint_host"], solver["turns"], solver["input_tokens"]) == ( + "solver", "proxy.example", wakes, 100 * wakes) + assert (user["role"], user["model"], user["endpoint_host"], user["turns"], user["cost_usd"]) == ( + "user_simulator", None, "sim-proxy.example", reads, pytest.approx(0.001 * reads)) + assert ctx.metadata["step_usage"]["week"] == [solver, user] + assert ctx.metadata["model_endpoints"]["user_simulator"]["week"]["endpoint_host"] == "sim-proxy.example" diff --git a/tst/unit/task_step/test_model_key_scope.py b/tst/unit/task_step/test_model_key_scope.py index 5de3d8f6b..4f761794c 100644 --- a/tst/unit/task_step/test_model_key_scope.py +++ b/tst/unit/task_step/test_model_key_scope.py @@ -15,8 +15,9 @@ from agent_env.a2a_agent.a2a_agent import TAKES_CREDENTIALS, A2AAgent from agent_env.artifact.artifacts.docker_image import DockerImageArtifact +from agent_env.config import get_config from agent_env.config.errors import ConfigError -from agent_env.config.model import ModelConfig +from agent_env.config.model import ModelConfig, endpoint_host from agent_env.providers.sandbox_providers import sandbox_provider from agent_env.providers.sandbox_providers.sandbox_provider import SandboxProvider from agent_env.task_step.context import DeployedSandbox, PromptResponse, TaskStepContext @@ -97,6 +98,23 @@ def test_an_agent_deploy_sends(monkeypatch, configured, passed, own, expected): assert _deploy(monkeypatch, passed, own) == expected +@pytest.mark.parametrize("passed, own, expected", [case for case in AGENT_CASES.values() if case[2] != REFUSED], + ids=[name for name, case in AGENT_CASES.items() if case[2] != REFUSED]) +def test_the_endpoint_an_agent_records_is_the_one_its_deploy_sends(configured, passed, own, expected): + agent = A2AAgent(id="solver", version=1, default_env_vars=own, + docker_image_artifact=DockerImageArtifact(id="img", description="d", image_name="img")) + + assert agent.model_base_url(dict(passed or {}), get_config()) == expected["url"] + + +def test_an_endpoints_recorded_host_carries_no_userinfo_and_an_agent_without_credentials_none(configured): + agent = A2AAgent(id="solver", version=1, metadata={TAKES_CREDENTIALS: False}, + docker_image_artifact=DockerImageArtifact(id="img", description="d", image_name="img")) + + assert endpoint_host("https://user:secret@gw.example.com:8443/v1") == "gw.example.com:8443" + assert agent.model_base_url({"LITELLM_BASE_URL": ELSEWHERE}, get_config()) is None + + def test_an_agent_that_takes_no_credentials_gets_none(monkeypatch, configured): sent = _deploy(monkeypatch, {"LITELLM_BASE_URL": ELSEWHERE, "LITELLM_API_KEY": "x"}, None, **{TAKES_CREDENTIALS: False}) diff --git a/tst/unit/task_step/test_prompt_agent_usage.py b/tst/unit/task_step/test_prompt_agent_usage.py new file mode 100644 index 000000000..c8deb3d75 --- /dev/null +++ b/tst/unit/task_step/test_prompt_agent_usage.py @@ -0,0 +1,171 @@ +"""A prompt step keeps the usage its agents report: one entry per role, the solver's turns and the user sim's each +summed, with the model, the endpoint's host and the attempt epoch, in the step's prompt response and, as each turn ends, +in ``metadata["step_usage"]``, so a step that times out keeps the turns before it.""" + +import json +from types import SimpleNamespace +from unittest.mock import MagicMock + +import pytest +from a2a.types import TaskState + +import agent_env.a2a_agent as a2a_agent_package +from agent_env.task_step.context import ATTEMPT_EPOCH, DeployedAgent, TaskStepContext +from agent_env.task_step.task_steps import prompt_agent as pa_mod +from agent_env.task_step.task_steps.deploy_agent import DeployAgentTaskStep +from agent_env.task_step.task_steps.prompt_agent import PromptAgentTaskStep + + +def _turn(input_tokens, output_tokens, **more): + return {"tool_call_count": 1, "input_tokens": input_tokens, "output_tokens": output_tokens, + "total_tokens": input_tokens + output_tokens, **more} + + +def _wire(monkeypatch, solver_usage: list, user_usage=None): + """The conversation's HTTP and store boundary. The solver's n-th reply reports ``solver_usage[n]`` (no usage when + None; a TimeoutError raised instead when it is one); the user sim reports ``user_usage`` and never says done.""" + cfg = MagicMock() + cfg.get_model_params.return_value = {} + monkeypatch.setattr(pa_mod, "get_config", lambda: cfg) + for fn in ("create_conversation", "add_a2a_task", "complete_a2a_task", "mark_closed", "get_conversation"): + monkeypatch.setattr(pa_mod.conversation_store, fn, lambda *a, **kw: None) + solver_turns = iter(solver_usage) + + async def post_agent_config(url, fields, headers=None): + return None + + async def send(url, parts, message_id, context_id, timeout): + return f"task-{message_id}", None + + async def poll(url, task_id, timeout, interval, **_): + if "user-sim" in url: + parts = [{"kind": "text", "text": '{"message": "keep going", "done": false}'}] + usage = user_usage + else: + usage = next(solver_turns) + if isinstance(usage, Exception): + raise usage + parts = [{"kind": "text", "text": "done"}] + if usage is not None: + parts.append({"kind": "data", "data": {"usage": usage}}) + return {"status": {"state": TaskState.completed, "message": {"parts": parts}}} + + monkeypatch.setattr(pa_mod.protocol, "post_agent_config", post_agent_config) + monkeypatch.setattr(pa_mod.protocol, "send_a2a_message", send) + monkeypatch.setattr(pa_mod.protocol, "poll_a2a_task", poll) + + +def _context() -> TaskStepContext: + ctx = TaskStepContext(instance_id="ti-1") + ctx.deployed_agents.append(DeployedAgent(agent_name="solver", api_url="http://solver", a2a_url="http://solver", + a2a_card={}, model_endpoint_host="proxy.example")) + ctx.deployed_agents.append(DeployedAgent(agent_name="usersim", api_url="http://user-sim", + a2a_url="http://user-sim", a2a_card={}, + model_endpoint_host="sim-proxy.example:4000")) + return ctx + + +def _step(turns: int) -> PromptAgentTaskStep: + return PromptAgentTaskStep(id="solve", version=None, prompt="hi", agent_name="solver", + max_conversation_turns=turns, user_agent_name="usersim") + + +@pytest.mark.asyncio +async def test_each_roles_turns_are_summed_into_one_entry(monkeypatch): + _wire(monkeypatch, [_turn(100, 10, cache_read_tokens=60, input_includes_cache=True, cost_usd=0.01), + _turn(200, 20, cache_read_tokens=150, input_includes_cache=True, cost_usd=0.02)], + user_usage=_turn(30, 5, input_includes_cache=True)) + token = ATTEMPT_EPOCH.set(2) + try: + ctx = await _step(2).execute(_context()) + finally: + ATTEMPT_EPOCH.reset(token) + + solver, user = ctx.prompt_responses[-1].usage + assert solver == { + "role": "solver", "model": None, "endpoint_host": "proxy.example", "attempt": 2, "turns": 2, + "tool_call_count": 2, "input_tokens": 300, "output_tokens": 30, "total_tokens": 330, + "cache_read_tokens": 210, "cache_write_tokens": None, "input_includes_cache": True, "cost_usd": 0.03, + "partial": False} + assert (user["role"], user["endpoint_host"], user["turns"], user["input_tokens"], user["partial"]) == ( + "user_simulator", "sim-proxy.example:4000", 1, 30, False) + assert ctx.metadata["step_usage"]["solve"] == [solver, user] + assert ctx.metadata["model_endpoints"] == { + "solver": {"solve": {"model": None, "endpoint_host": "proxy.example"}}, + "user_simulator": {"solve": {"model": None, "endpoint_host": "sim-proxy.example:4000"}}} + + +@pytest.mark.asyncio +async def test_an_agent_that_reports_only_tool_calls_gets_no_token_counts(monkeypatch): + _wire(monkeypatch, [{"tool_call_count": 3}]) + + ctx = await _step(1).execute(_context()) + + [solver] = ctx.prompt_responses[-1].usage + assert solver["tool_call_count"] == 3 and ctx.prompt_responses[-1].tool_call_count == 3 + assert [solver[name] for name in ("input_tokens", "output_tokens", "cost_usd")] == [None, None, None] + assert solver["partial"] is False + + +@pytest.mark.asyncio +async def test_a_sum_some_turns_or_calls_left_out_is_partial(monkeypatch): + _wire(monkeypatch, [_turn(100, 10), None, _turn(50, 5, provider_details={"unreported_calls": 2})]) + + ctx = await _step(3).execute(_context()) + + solver = ctx.prompt_responses[-1].usage[0] + assert (solver["turns"], solver["input_tokens"], solver["partial"], solver["unreported_calls"]) == (3, 150, True, 2) + + +@pytest.mark.asyncio +async def test_a_step_that_times_out_keeps_the_turns_before_it(monkeypatch): + _wire(monkeypatch, [_turn(100, 10), _turn(200, 20), TimeoutError("no answer")], user_usage=_turn(30, 5)) + ctx = _context() + + with pytest.raises(TimeoutError): + await _step(3).execute(ctx) + + assert not ctx.prompt_responses + solver, user = ctx.metadata["step_usage"]["solve"] + assert (solver["turns"], solver["input_tokens"], user["turns"]) == (2, 300, 2) + + +@pytest.mark.asyncio +async def test_malformed_usage_is_kept_aside_and_the_step_runs(monkeypatch): + _wire(monkeypatch, [{"tool_call_count": 1, "input_tokens": -5, "output_tokens": 7}, "garbage"]) + + ctx = await _step(2).execute(_context()) + + solver = ctx.prompt_responses[-1].usage[0] + assert (solver["input_tokens"], solver["output_tokens"], solver["partial"]) == (None, 7, True) + assert solver["unparsed"] == [{"input_tokens": -5}, {"usage": "garbage"}] + + +class _Agent: + """An agent whose deploy is faked; the endpoint it records is the one the deploy is given, as the real rule says.""" + + metadata: dict = {} + + async def deploy(self, **_): + return SimpleNamespace(a2a_url="http://solver", agent_card={}, sandbox_id="sb-1", sandbox_type="fake", + instance_id="i-1", network_policy=None, control_token=None, control_headers={}) + + def model_base_url(self, env_vars, config): + return env_vars.get("LITELLM_BASE_URL") + + +@pytest.mark.asyncio +async def test_no_key_or_userinfo_reaches_what_a_run_stores(monkeypatch): + monkeypatch.setattr(a2a_agent_package.A2AAgent, "get", classmethod(lambda cls, *a, **k: _Agent())) + context = TaskStepContext(instance_id="ti-1") + context.metadata["user_overrides"] = {"litellm_api_key": "sk-live-secret"} + await DeployAgentTaskStep(id="deploy", version=None, a2a_agent_id="a", agent_name="solver", + litellm_base_url="https://alice:hunter2@gw.example:8443/v1").execute(context) + _wire(monkeypatch, [_turn(10, 1)]) + + await PromptAgentTaskStep(id="solve", version=None, prompt="hi", agent_name="solver").execute(context) + + assert context.prompt_responses[-1].usage[0]["endpoint_host"] == "gw.example:8443" + stored = json.dumps(context.to_safe_dict(), default=str) + assert "gw.example:8443" in stored + assert not [secret for secret in ("alice", "hunter2", "sk-live-secret") if secret in stored] diff --git a/tst/unit/task_step/usage_test.py b/tst/unit/task_step/usage_test.py new file mode 100644 index 000000000..50df8fec4 --- /dev/null +++ b/tst/unit/task_step/usage_test.py @@ -0,0 +1,45 @@ +"""A role's step usage entry: the sums, what they leave out, and where the entry is kept.""" + +from agentenv_protocol.a2a_agent import Usage + +from agent_env.task_step.context import ATTEMPT_EPOCH, TaskStepContext +from agent_env.task_step.usage import RoleUsage, record_step_usage + + +def test_turns_that_disagree_on_whether_input_counts_the_cache_leave_the_flag_unknown(): + role = RoleUsage("solver") + role.add(Usage(input_tokens=10, output_tokens=1, input_includes_cache=True)) + role.add(Usage(input_tokens=20, output_tokens=2)) + + entry = role.entry() + + assert (entry["input_tokens"], entry["input_includes_cache"], entry["partial"]) == (30, None, False) + + +def test_a_proxy_turn_short_of_some_calls_counts_is_partial(): + role = RoleUsage("solver") + role.add(Usage(input_tokens=10, output_tokens=1, provider_details={"unreported": {"cost_usd": 2}})) + + entry = role.entry() + + assert entry["partial"] is True and "unreported_calls" not in entry + + +def test_the_entry_carries_the_attempt_its_step_ran_under(): + token = ATTEMPT_EPOCH.set(3) + try: + role = RoleUsage("user_simulator", "claude-haiku-4-5", "proxy.example") + finally: + ATTEMPT_EPOCH.reset(token) + role.add(None) + + entry = role.entry() + + assert (entry["attempt"], entry["model"], entry["turns"], entry["tool_call_count"]) == (3, "claude-haiku-4-5", 1, None) + + +def test_a_step_whose_roles_ran_no_turns_records_nothing(): + context = TaskStepContext() + + assert record_step_usage(context, "solve", (RoleUsage("solver"), None)) is None + assert "step_usage" not in context.metadata From da3768058bbcb35345844b0289b2028b20cfd983 Mon Sep 17 00:00:00 2001 From: Edgar Arakelyan Date: Thu, 19 Nov 2026 15:35:16 -0800 Subject: [PATCH 2/4] fix(task-steps): a step's usage is partial when a turn that reported any token count sits beside one that reported none A turn reporting only total_tokens didn't count as reporting tokens, so beside a turn that reported nothing the entry read as complete. Any token count now counts. DeployedAgent.model_endpoint_host says it's the endpoint's host as configured, before a sandbox's rewrite to reach it. Co-Authored-By: Claude Opus 5.5 --- src/agent_env/task_step/context.py | 4 +++- src/agent_env/task_step/usage.py | 5 +++-- tst/unit/task_step/usage_test.py | 10 ++++++++++ 3 files changed, 16 insertions(+), 3 deletions(-) diff --git a/src/agent_env/task_step/context.py b/src/agent_env/task_step/context.py index 8c41461bb..67445e1e8 100644 --- a/src/agent_env/task_step/context.py +++ b/src/agent_env/task_step/context.py @@ -138,7 +138,9 @@ class DeployedAgent: env_bindings: dict[str, str | None] = field(default_factory=dict) # The token its extension routes take, from the deploy that started it; None for an agent agent-env didn't start. control_token: str | None = None - # The host (and port) of the model endpoint it was given, never a key or a URL's userinfo; None when unknown. + # The host (and port) of the model endpoint it was given, as configured, before a sandbox's rewrite to reach it + # (localhost to host.docker.internal), so one endpoint has one name; never a key or a URL's userinfo. None when + # unknown. model_endpoint_host: str | None = None @classmethod diff --git a/src/agent_env/task_step/usage.py b/src/agent_env/task_step/usage.py index 3324a5d27..9bf109414 100644 --- a/src/agent_env/task_step/usage.py +++ b/src/agent_env/task_step/usage.py @@ -15,6 +15,7 @@ STEP_USAGE_KEY = "step_usage" _COUNTERS = ("tool_call_count", "input_tokens", "output_tokens", "total_tokens", "cache_read_tokens", "cache_write_tokens") +_TOKEN_COUNTS = _COUNTERS[1:] @dataclass @@ -38,7 +39,7 @@ def add(self, usage: Optional[Usage], unparsed: Optional[dict] = None) -> None: def entry(self) -> dict[str, Any]: """The turns summed: each count and the cost the sum of the turns that reported it, None when none did, never 0. ``input_includes_cache`` is the turns' flag when they agree, else None. ``partial`` says the sum is a lower - bound: some turns reported tokens and others didn't, or a turn said some of its calls reported none + bound: some turns reported a token count and others none, or a turn said some of its calls reported none (``unreported_calls``, which the entry sums, or a proxy's ``unreported``).""" reported = [usage for usage in self.turns if usage is not None] @@ -46,7 +47,7 @@ def total(name: str) -> Any: values = [getattr(usage, name) for usage in reported if getattr(usage, name) is not None] return sum(values) if values else None - counted = [usage for usage in reported if usage.input_tokens is not None or usage.output_tokens is not None] + counted = [usage for usage in reported if any(getattr(usage, name) is not None for name in _TOKEN_COUNTS)] flags = {usage.input_includes_cache for usage in counted} unreported = [n for usage in reported if (n := _unreported_calls(usage))] partial = (0 < len(counted) < len(self.turns) or bool(unreported) diff --git a/tst/unit/task_step/usage_test.py b/tst/unit/task_step/usage_test.py index 50df8fec4..4beb78162 100644 --- a/tst/unit/task_step/usage_test.py +++ b/tst/unit/task_step/usage_test.py @@ -43,3 +43,13 @@ def test_a_step_whose_roles_ran_no_turns_records_nothing(): assert record_step_usage(context, "solve", (RoleUsage("solver"), None)) is None assert "step_usage" not in context.metadata + + +def test_a_turn_that_reported_any_token_count_makes_a_turn_that_reported_none_partial(): + role = RoleUsage("solver") + role.add(Usage(total_tokens=100)) + role.add(None) + + entry = role.entry() + + assert (entry["total_tokens"], entry["input_tokens"], entry["partial"]) == (100, None, True) From 504d6f1cbca33fd0de1a2342ea267ddb17d61d66 Mon Sep 17 00:00:00 2001 From: Edgar Arakelyan Date: Thu, 19 Nov 2026 16:43:12 -0800 Subject: [PATCH 3/4] fix(task-steps): usage keeps only the start of a value it couldn't read, and runners that execute steps themselves set the attempt read_usage kept a malformed usage value verbatim, and an entry kept one such record per turn, so a broken agent could bloat a run's stored context past a store's item limit. A long value is now kept as the first 200 characters of its JSON, and an entry keeps the first five turns' records with unparsed_turns counting them all. AGENTS.md's contract for embedders says a runner that calls step.execute() itself, not through Task.run(), sets ATTEMPT_EPOCH, or its usage entries carry no attempt. Co-Authored-By: Claude Opus 5.5 --- AGENTS.md | 4 ++++ src/agent_env/a2a_agent/protocol.py | 19 +++++++++++++++++-- src/agent_env/task_step/usage.py | 5 ++++- tst/unit/a2a_agent/protocol_test.py | 8 ++++++++ tst/unit/task_step/test_prompt_agent_usage.py | 2 +- tst/unit/task_step/usage_test.py | 10 ++++++++++ 6 files changed, 44 insertions(+), 4 deletions(-) diff --git a/AGENTS.md b/AGENTS.md index 04ed36bc6..b4d0b7376 100644 --- a/AGENTS.md +++ b/AGENTS.md @@ -346,6 +346,10 @@ orchestrators call it the same way. What callers rely on: - Each step runs under `protocol_v1.authorized` for the deployed envs in its context: a protocol-client call the step makes to a deployed env's address, naming no headers of its own, sends that env's `DeployedEnv.control_headers`. +- Each step runs with `task_step.context.ATTEMPT_EPOCH` set to the attempt epoch it was dispatched + under, in its own asyncio task, so the usage a prompt step records (`metadata["step_usage"]`, + `PromptResponse.usage`) says which attempt spent it. A runner that calls `step.execute()` itself, not + through `Task.run()`, sets it the same way, or its entries carry no attempt. - A runner records every run it accepts in the run index (`agent_env.runner.store`): `insert_run` at submit, `set_running` when it starts, `mark_terminal` when it ends. The explorer lists, counts and groups runs from that collection and never calls the runner to ask, so a run missing there is diff --git a/src/agent_env/a2a_agent/protocol.py b/src/agent_env/a2a_agent/protocol.py index 3f2f7d959..74e9305f7 100644 --- a/src/agent_env/a2a_agent/protocol.py +++ b/src/agent_env/a2a_agent/protocol.py @@ -3,6 +3,7 @@ import asyncio import contextlib +import json import logging import re import time @@ -270,14 +271,28 @@ async def cancel_a2a_task(a2a_url: str, task_id: str) -> None: _USAGE_FIELDS = frozenset(f.name for f in fields(Usage)) - {"provider_details"} +# How much of a value ``read_usage`` couldn't read it keeps, so a broken agent can't bloat a run's stored context. +_UNPARSED_PREVIEW_CHARS = 200 # Where agents before Usage had cache fields put their cache counts. _LEGACY_CACHE_KEYS = ("cache_read_tokens", "cache_write_tokens") +def _preview(value: Any) -> Any: + """``value`` as it came when it's short, else the start of its JSON as a string.""" + text = json.dumps(value, default=str) + return value if len(text) <= _UNPARSED_PREVIEW_CHARS else text[:_UNPARSED_PREVIEW_CHARS] + "..." + + def read_usage(raw: Any) -> tuple[Optional[Usage], Optional[dict]]: """An agent's reported usage and what of it couldn't be read, never raising: each field ``Usage`` has, checked on - its own, a value it refuses kept under its name in the second; a key it doesn't have kept in ``provider_details``; - the cache counts an agent from before the typed fields put in ``provider_details`` read from there.""" + its own, a value it refuses kept under its name in the second (a long one cut to its start); a key it doesn't have + kept in ``provider_details``; the cache counts an agent from before the typed fields put in ``provider_details`` + read from there.""" + usage, unparsed = _read_usage(raw) + return usage, ({key: _preview(value) for key, value in unparsed.items()} if unparsed else None) + + +def _read_usage(raw: Any) -> tuple[Optional[Usage], Optional[dict]]: if raw is None: return None, None if not isinstance(raw, Mapping): diff --git a/src/agent_env/task_step/usage.py b/src/agent_env/task_step/usage.py index 9bf109414..7e60d6bc8 100644 --- a/src/agent_env/task_step/usage.py +++ b/src/agent_env/task_step/usage.py @@ -16,6 +16,8 @@ _COUNTERS = ("tool_call_count", "input_tokens", "output_tokens", "total_tokens", "cache_read_tokens", "cache_write_tokens") _TOKEN_COUNTS = _COUNTERS[1:] +# How many turns' unreadable values an entry keeps; ``unparsed_turns`` counts them all. +_UNPARSED_TURNS_KEPT = 5 @dataclass @@ -61,7 +63,8 @@ def total(name: str) -> Any: if unreported: entry["unreported_calls"] = sum(unreported) if self.unparsed: - entry["unparsed"] = list(self.unparsed) + entry["unparsed"] = self.unparsed[:_UNPARSED_TURNS_KEPT] + entry["unparsed_turns"] = len(self.unparsed) return entry diff --git a/tst/unit/a2a_agent/protocol_test.py b/tst/unit/a2a_agent/protocol_test.py index cedfe8cfd..1509387cb 100644 --- a/tst/unit/a2a_agent/protocol_test.py +++ b/tst/unit/a2a_agent/protocol_test.py @@ -208,6 +208,14 @@ def test_an_agents_usage_is_read_as_far_as_it_can_be_and_never_raises(raw, usage assert read_usage(raw) == (usage, unparsed) +def test_a_long_unreadable_value_is_kept_only_as_its_start(): + usage, unparsed = read_usage({"input_tokens": 5, "output_tokens": "x" * 5000, "provider_details": ["y"] * 2000}) + + assert usage == Usage(input_tokens=5) + assert unparsed["output_tokens"].startswith('"xxx') and unparsed["output_tokens"].endswith("...") + assert all(len(value) == 203 for value in unparsed.values()) + + def test_the_tool_call_count_is_read_as_it_always_was_whatever_the_rest_of_the_usage(): message = {"parts": [{"kind": "data", "data": {"usage": {"tool_call_count": "7", "input_tokens": 3}}}]} diff --git a/tst/unit/task_step/test_prompt_agent_usage.py b/tst/unit/task_step/test_prompt_agent_usage.py index c8deb3d75..5fb7587e0 100644 --- a/tst/unit/task_step/test_prompt_agent_usage.py +++ b/tst/unit/task_step/test_prompt_agent_usage.py @@ -138,7 +138,7 @@ async def test_malformed_usage_is_kept_aside_and_the_step_runs(monkeypatch): solver = ctx.prompt_responses[-1].usage[0] assert (solver["input_tokens"], solver["output_tokens"], solver["partial"]) == (None, 7, True) - assert solver["unparsed"] == [{"input_tokens": -5}, {"usage": "garbage"}] + assert (solver["unparsed"], solver["unparsed_turns"]) == ([{"input_tokens": -5}, {"usage": "garbage"}], 2) class _Agent: diff --git a/tst/unit/task_step/usage_test.py b/tst/unit/task_step/usage_test.py index 4beb78162..adf60a49a 100644 --- a/tst/unit/task_step/usage_test.py +++ b/tst/unit/task_step/usage_test.py @@ -53,3 +53,13 @@ def test_a_turn_that_reported_any_token_count_makes_a_turn_that_reported_none_pa entry = role.entry() assert (entry["total_tokens"], entry["input_tokens"], entry["partial"]) == (100, None, True) + + +def test_an_entry_keeps_the_first_turns_unreadable_values_and_counts_them_all(): + role = RoleUsage("solver") + for n in range(8): + role.add(None, {"input_tokens": -n}) + + entry = role.entry() + + assert entry["unparsed"] == [{"input_tokens": -n} for n in range(5)] and entry["unparsed_turns"] == 8 From 1d5799a13ddb9b58a7d804352930ae00854d5a72 Mon Sep 17 00:00:00 2001 From: Edgar Arakelyan Date: Thu, 19 Nov 2026 17:15:42 -0800 Subject: [PATCH 4/4] refactor(task-steps): one StepUsage per step records its roles' usage, and the entry has a type prompt_agent and always_on each started a RoleUsage per role, recorded its model endpoint and rewrote metadata.step_usage after each turn at four call sites, choosing the role's spelling each time, and _UserSide reached the step's usage through a callback. A StepUsage per step now does it: start(role, model, agent), turn(role, reply), entries(). The entry is a UsageEntry TypedDict, so P10, P11 and E13 read a declared shape. Nothing a run records changes. Co-Authored-By: Claude Opus 5.5 --- .../task_step/task_steps/always_on.py | 35 +++----- .../task_step/task_steps/prompt_agent.py | 44 ++++----- src/agent_env/task_step/usage.py | 89 +++++++++++++++---- tst/unit/task_step/usage_test.py | 23 ++++- 4 files changed, 116 insertions(+), 75 deletions(-) diff --git a/src/agent_env/task_step/task_steps/always_on.py b/src/agent_env/task_step/task_steps/always_on.py index 2c89099b2..f32c60b46 100644 --- a/src/agent_env/task_step/task_steps/always_on.py +++ b/src/agent_env/task_step/task_steps/always_on.py @@ -113,7 +113,7 @@ from agent_env.config.model import ModelRole from agent_env.store import DuplicateKeyError, get_config from agent_env.task_step.context import PromptResponse, StepContextError, TaskStepContext -from agent_env.task_step.usage import RoleUsage, record_step_usage +from agent_env.task_step.usage import StepUsage from agent_env.task_step.task_step import TaskStep from agent_env.task_step.task_steps.agent import user_sim_versions from agent_env.task_step.task_steps.agent.deploy_user_sim_agent import USER_SIMS_KEY, user_sim_model @@ -381,8 +381,7 @@ def __init__(self, step: AlwaysOnTaskStep, context: TaskStepContext): self.per_turn_parts: list[Optional[list[dict]]] = [] self.tool_calls: Optional[int] = None # Each role's turns' usage, recorded as each turn ends. - self.solver_usage: Optional[RoleUsage] = None - self.user_usage: Optional[RoleUsage] = None + self.usage = StepUsage(context, step.id) self.all_messages: list[dict] = [] self.final_task_id = "" self.error: Optional[str] = None @@ -600,8 +599,7 @@ async def _configure_agents(self) -> None: headers=self.agent.control_headers) solver_model = negotiated.fields.get("model") if negotiated else None ctx.record_model(ModelRole.SOLVER.record_key, step.id, solver_model) - ctx.record_model_endpoint(ModelRole.SOLVER.value, step.id, solver_model, self.agent.model_endpoint_host) - self.solver_usage = RoleUsage(ModelRole.SOLVER.value, solver_model, self.agent.model_endpoint_host) + self.usage.start(ModelRole.SOLVER, solver_model, self.agent) self.rules_in_system_prompt = bool(desired.get("system_prompt") and negotiated and "system_prompt" in negotiated.fields) if desired.get("system_prompt") and not self.rules_in_system_prompt: @@ -611,7 +609,7 @@ async def _configure_agents(self) -> None: if self.us.get("prompt_set"): # deploy_user_sim_agent set the user sim's prompt and model at deploy self.user_model, self.user_model_from = self.us["model"], self.us["model_source"] self.user_setup = dict(self.us["setup"]) - self._start_user_usage(self.user_model) + self.usage.start(ModelRole.USER_SIMULATOR, self.user_model, self.user_agent) return setup = self.plugin.setup(self.version, self.person) self.user_model, self.user_model_from = self._user_model() @@ -623,8 +621,9 @@ async def _configure_agents(self) -> None: await protocol.post_agent_config(self.user_url + negotiated.endpoint, negotiated.fields, headers=self.user_agent.control_headers) accepted = sorted(negotiated.fields) if negotiated else [] - ctx.record_model(ModelRole.USER_SIMULATOR.record_key, step.id, negotiated.fields.get("model") if negotiated else None) - self._start_user_usage(negotiated.fields.get("model") if negotiated else None) + user_model = negotiated.fields.get("model") if negotiated else None + ctx.record_model(ModelRole.USER_SIMULATOR.record_key, step.id, user_model) + self.usage.start(ModelRole.USER_SIMULATOR, user_model, self.user_agent) self.user_setup = {"model": self.user_model, "model_from": self.user_model_from, "user_sim_step_model": self.us.get("model"), "model_set": "model" in accepted, # else the agent ran its own @@ -1280,7 +1279,7 @@ def record() -> None: term = protocol.TerminalResponse.from_message(status_msg) if term.tool_call_count is not None: self.tool_calls = (self.tool_calls or 0) + term.tool_call_count - self._record_usage(self.solver_usage, term) + self.usage.turn(ModelRole.SOLVER, term) messages, quiet = await self._take_messages() await self._read_changes("turn") traj = None @@ -1361,7 +1360,7 @@ def record() -> None: term = protocol.TerminalResponse.from_message(status_msg) if term.tool_call_count is not None: self.tool_calls = (self.tool_calls or 0) + term.tool_call_count - self._record_usage(self.solver_usage, term) + self.usage.turn(ModelRole.SOLVER, term) messages, quiet = await self._take_messages() await self._read_changes("nudge") traj = None @@ -1570,7 +1569,7 @@ async def _ask_user_sim(self, text: str, deadline: float) -> dict: status = (result.get("status") or {}) msg = status.get("message") or {} term = protocol.TerminalResponse.from_message(msg) - self._record_usage(self.user_usage, term) + self.usage.turn(ModelRole.USER_SIMULATOR, term) if status.get("state") != TaskState.completed and str(status.get("state")) != "completed": return {"text": None, "structured": None, "error": f"state {status.get('state')}"} raw = next((p["text"] for p in msg.get("parts") or [] if p.get("kind") == "text"), term.response_text) @@ -1749,19 +1748,7 @@ def _append_prompt_response(self, final_text: str) -> None: model=self.model, error_type=self.error_type, error_code=self.error_code, error_message=self.error, agent_session_id=self.final_task_id, a2a_context_id=self.solver_context_id, agent_name=step.agent_name, step_id=step.id, - usage=record_step_usage(ctx, step.id, (self.solver_usage, self.user_usage)))) - - def _start_user_usage(self, model: Optional[str]) -> None: - """The user sim's usage in this step, its model and endpoint host recorded beside the solver's.""" - host = self.user_agent.model_endpoint_host - self.ctx.record_model_endpoint(ModelRole.USER_SIMULATOR.value, self.step.id, model, host) - self.user_usage = RoleUsage(ModelRole.USER_SIMULATOR.value, model, host) - - def _record_usage(self, role: Optional[RoleUsage], term: protocol.TerminalResponse) -> None: - """One finished turn's usage, recorded as it ends, so a step that raises later keeps it.""" - if role is not None: - role.add(term.usage, term.usage_unparsed) - record_step_usage(self.ctx, self.step.id, (self.solver_usage, self.user_usage)) + usage=self.usage.entries())) # ================================================================================================================== diff --git a/src/agent_env/task_step/task_steps/prompt_agent.py b/src/agent_env/task_step/task_steps/prompt_agent.py index 4469921db..24cf395fd 100644 --- a/src/agent_env/task_step/task_steps/prompt_agent.py +++ b/src/agent_env/task_step/task_steps/prompt_agent.py @@ -10,7 +10,7 @@ import uuid from contextlib import nullcontext from datetime import datetime, timezone -from collections.abc import Callable, Mapping +from collections.abc import Mapping from typing import Any, ClassVar, Optional import httpx from a2a.types import TaskState @@ -37,7 +37,7 @@ from agent_env.store import DuplicateKeyError, get_config from agent_env.store.object_store import ObjectStore from agent_env.task_step.context import DeployedAgent, PromptResponse, StepContextError, TaskStepContext -from agent_env.task_step.usage import RoleUsage, record_step_usage +from agent_env.task_step.usage import StepUsage from agent_env.entity.refs import EntityRef from agent_env.entity.scopes import FieldScope from agent_env.task_step.task_step import RetryConfig, StepRole, TaskStep, TaskStepDependency @@ -399,13 +399,6 @@ def _resolve_model_and_effort(self, context: TaskStepContext) -> tuple[Optional[ effort = step_overrides.get("effort") or overrides.get("agent_effort") or self.effort return model, effort - def _user_usage(self, context: TaskStepContext, model: Optional[str], - user_sim: Optional[DeployedAgent]) -> RoleUsage: - """The user simulator's usage in this step, its model and endpoint host recorded beside the solver's.""" - host = user_sim.model_endpoint_host if user_sim else None - context.record_model_endpoint(ModelRole.USER_SIMULATOR.value, self.id, model, host) - return RoleUsage(ModelRole.USER_SIMULATOR.value, model, host) - def _resolve_user_model(self, context: TaskStepContext) -> Optional[str]: """The user simulator's model for this run: this step's ``step_overrides``, then the run's ``user_model``, the stored value, and the config's ``user`` role when it names one.""" @@ -570,14 +563,13 @@ async def _converse( # A model the agent's card doesn't accept is never sent, and the agent runs its own. solver_model = negotiated.fields.get("model") if negotiated else None context.record_model(ModelRole.SOLVER.record_key, self.id, solver_model) - context.record_model_endpoint(ModelRole.SOLVER.value, self.id, solver_model, agent.model_endpoint_host) - solver_usage = RoleUsage(ModelRole.SOLVER.value, solver_model, agent.model_endpoint_host) - user_usage: Optional[RoleUsage] = None + usage = StepUsage(context, self.id) + usage.start(ModelRole.SOLVER, solver_model, agent) is_user_sim = user_sim is not None and self.user_a2a_url is None if built_in is not None: # deploy_user_sim_agent set its model and answer format context.record_model(ModelRole.USER_SIMULATOR.record_key, self.id, built_in[0].get("model")) - user_usage = self._user_usage(context, built_in[0].get("model"), user_sim) + usage.start(ModelRole.USER_SIMULATOR, built_in[0].get("model"), user_sim) elif is_user_sim: user_output_format = self.user_output_format or _DEFAULT_USER_SIM_OUTPUT_FORMAT user_model = self._resolve_user_model(context) @@ -587,8 +579,9 @@ async def _converse( await protocol.post_agent_config(user_url + negotiated.endpoint, negotiated.fields, headers=user_sim.control_headers) logger.info(f"Configured user-sim '{self.user_agent_name}' with structured output_format for done-signal") - context.record_model(ModelRole.USER_SIMULATOR.record_key, self.id, negotiated.fields.get("model") if negotiated else None) - user_usage = self._user_usage(context, negotiated.fields.get("model") if negotiated else None, user_sim) + user_model = negotiated.fields.get("model") if negotiated else None + context.record_model(ModelRole.USER_SIMULATOR.record_key, self.id, user_model) + usage.start(ModelRole.USER_SIMULATOR, user_model, user_sim) triggers_registered = is_user_sim and any( r.get("agent_name") == self.user_agent_name @@ -657,8 +650,7 @@ async def _converse( self, context, url=user_url, user_sim=user_sim, is_user_sim=is_user_sim, conversation_id=conversation_id, decide_url=decide_url, shareable=sent_to_target, decides_quietly=card_request_accepts(decide_method.get("request") or {}, _QUIET_DECIDE), - built_in=sim, usage=user_usage, - on_usage=lambda: record_step_usage(context, self.id, (solver_usage, user_usage)), + built_in=sim, usage=usage, ) # Set by a user's done with a message: the agent answers it, then the conversation ends for this reason. end_after: Optional[str] = None @@ -707,9 +699,7 @@ def record_turn() -> None: final_terminal = protocol.TerminalResponse.from_message(status_msg) if final_terminal.tool_call_count is not None: conversation_tool_call_count = (conversation_tool_call_count or 0) + final_terminal.tool_call_count - # As each turn ends, so a step that raises later keeps it. - solver_usage.add(final_terminal.usage, final_terminal.usage_unparsed) - record_step_usage(context, self.id, (solver_usage, user_usage)) + usage.turn(ModelRole.SOLVER, final_terminal) agent_response_parts = status_msg.get("parts") or [{"kind": "text", "text": final_terminal.response_text}] conversation_store.complete_a2a_task( @@ -862,7 +852,7 @@ def record_turn() -> None: agent_name=self.agent_name, step_id=self.id, structured_output=structured_output, - usage=record_step_usage(context, self.id, (solver_usage, user_usage)), + usage=usage.entries(), )) if final_state == TaskState.failed: @@ -1134,12 +1124,10 @@ class _UserSide: def __init__(self, step: PromptAgentTaskStep, context: TaskStepContext, *, url: Optional[str], user_sim: Optional[DeployedAgent], is_user_sim: bool, conversation_id: str, decide_url: Optional[str], decides_quietly: bool, shareable: set[str], - built_in: Optional[_BuiltInUserSim] = None, usage: Optional[RoleUsage] = None, - on_usage: Callable[[], Any] = lambda: None) -> None: + built_in: Optional[_BuiltInUserSim] = None, usage: Optional[StepUsage] = None) -> None: self._step = step self._built_in = built_in self._usage = usage - self._on_usage = on_usage self._context = context self._url = url self._user_sim = user_sim @@ -1251,8 +1239,7 @@ async def _ask_peer(self, number: int, agent_parts: list[dict]) -> dict: return {"end": "user_timeout"} user_state = (user_result.get("status") or {}).get("state") - if self._is_user_sim: - self._record_usage((user_result.get("status") or {}).get("message") or {}) + self._record_usage((user_result.get("status") or {}).get("message") or {}) if user_state != TaskState.completed: logger.info(f"user_a2a_url returned state={user_state}; ending multi-turn") return {"end": "user_not_completed"} @@ -1285,11 +1272,10 @@ def _record(self, key: str, entry: dict) -> None: self._context.metadata.setdefault(key, {}).setdefault(self._step.id, []).append(entry) def _record_usage(self, message: dict) -> protocol.TerminalResponse: - """The user sim's reply, its turn's usage recorded as the turn ends.""" + """The user side's reply, a user sim's turn recorded as it ends; a human peer's has none to record.""" term = protocol.TerminalResponse.from_message(message) if self._usage is not None: - self._usage.add(term.usage, term.usage_unparsed) - self._on_usage() + self._usage.turn(ModelRole.USER_SIMULATOR, term) return term diff --git a/src/agent_env/task_step/usage.py b/src/agent_env/task_step/usage.py index 7e60d6bc8..451da992f 100644 --- a/src/agent_env/task_step/usage.py +++ b/src/agent_env/task_step/usage.py @@ -4,13 +4,16 @@ from __future__ import annotations -from collections.abc import Iterable from dataclasses import dataclass, field -from typing import Any, Optional +from typing import TYPE_CHECKING, Any, NotRequired, Optional, TypedDict from agentenv_protocol.a2a_agent import Usage -from agent_env.task_step.context import ATTEMPT_EPOCH, TaskStepContext +from agent_env.config.model import ModelRole +from agent_env.task_step.context import ATTEMPT_EPOCH, DeployedAgent, TaskStepContext + +if TYPE_CHECKING: + from agent_env.a2a_agent.protocol import TerminalResponse STEP_USAGE_KEY = "step_usage" _COUNTERS = ("tool_call_count", "input_tokens", "output_tokens", "total_tokens", "cache_read_tokens", @@ -20,6 +23,29 @@ _UNPARSED_TURNS_KEPT = 5 +class UsageEntry(TypedDict): + """One role's usage in one step, as ``PromptResponse.usage`` and ``metadata["step_usage"]`` hold it. A count or + the cost is None when no turn reported it, never 0.""" + + role: str + model: Optional[str] + endpoint_host: Optional[str] + attempt: Optional[int] + turns: int + tool_call_count: Optional[int] + input_tokens: Optional[int] + output_tokens: Optional[int] + total_tokens: Optional[int] + cache_read_tokens: Optional[int] + cache_write_tokens: Optional[int] + input_includes_cache: Optional[bool] + cost_usd: Optional[float] + partial: bool + unreported_calls: NotRequired[int] + unparsed: NotRequired[list[dict]] + unparsed_turns: NotRequired[int] + + @dataclass class RoleUsage: """One role's turns in one step: the model it ran, the host of the endpoint it called, the attempt epoch the step @@ -38,7 +64,7 @@ def add(self, usage: Optional[Usage], unparsed: Optional[dict] = None) -> None: if unparsed: self.unparsed.append(unparsed) - def entry(self) -> dict[str, Any]: + def entry(self) -> UsageEntry: """The turns summed: each count and the cost the sum of the turns that reported it, None when none did, never 0. ``input_includes_cache`` is the turns' flag when they agree, else None. ``partial`` says the sum is a lower bound: some turns reported a token count and others none, or a turn said some of its calls reported none @@ -54,12 +80,14 @@ def total(name: str) -> Any: unreported = [n for usage in reported if (n := _unreported_calls(usage))] partial = (0 < len(counted) < len(self.turns) or bool(unreported) or any(usage.provider_details.get("unreported") for usage in reported)) - entry: dict[str, Any] = { - "role": self.role, "model": self.model, "endpoint_host": self.endpoint_host, "attempt": self.attempt, - "turns": len(self.turns), **{name: total(name) for name in _COUNTERS}, - "input_includes_cache": flags.pop() if len(flags) == 1 else None, "cost_usd": total("cost_usd"), - "partial": partial, - } + entry = UsageEntry( + role=self.role, model=self.model, endpoint_host=self.endpoint_host, attempt=self.attempt, + turns=len(self.turns), tool_call_count=total("tool_call_count"), input_tokens=total("input_tokens"), + output_tokens=total("output_tokens"), total_tokens=total("total_tokens"), + cache_read_tokens=total("cache_read_tokens"), cache_write_tokens=total("cache_write_tokens"), + input_includes_cache=flags.pop() if len(flags) == 1 else None, cost_usd=total("cost_usd"), + partial=partial, + ) if unreported: entry["unreported_calls"] = sum(unreported) if self.unparsed: @@ -74,12 +102,35 @@ def _unreported_calls(usage: Usage) -> int: return value if isinstance(value, int) and not isinstance(value, bool) and value > 0 else 0 -def record_step_usage(context: TaskStepContext, step_id: str, - roles: Iterable[Optional[RoleUsage]]) -> Optional[list[dict]]: - """The entries of ``roles`` that ran turns, written to ``metadata["step_usage"][step_id]`` and returned; None, and - nothing written, when none did.""" - entries = [role.entry() for role in roles if role is not None and role.turns] - if not entries: - return None - context.metadata.setdefault(STEP_USAGE_KEY, {})[step_id] = entries - return entries +class StepUsage: + """A prompt step's usage, role by role: each role ``start``ed with its model and agent, each of its finished turns + recorded with ``turn``, and the entries written to ``metadata["step_usage"][step_id]`` after every turn.""" + + def __init__(self, context: TaskStepContext, step_id: str) -> None: + self._context = context + self._step_id = step_id + self._roles: dict[ModelRole, RoleUsage] = {} + + def start(self, role: ModelRole, model: Optional[str], agent: Optional[DeployedAgent]) -> None: + """Count ``role``'s turns on ``model``, recording the model and the host of the endpoint ``agent`` was given in + ``metadata.model_endpoints``, under the role's own name.""" + host = agent.model_endpoint_host if agent else None + self._context.record_model_endpoint(role.value, self._step_id, model, host) + self._roles[role] = RoleUsage(role.value, model, host) + + def turn(self, role: ModelRole, reply: TerminalResponse) -> None: + """One finished turn of ``role``, written at once, so a step that raises later keeps it. A role that wasn't + started, a human peer's, is skipped.""" + usage = self._roles.get(role) + if usage is not None: + usage.add(reply.usage, reply.usage_unparsed) + self.entries() + + def entries(self) -> Optional[list[UsageEntry]]: + """The entries of the roles that ran turns, written to ``metadata["step_usage"]``; None, and nothing written, + when none did.""" + entries = [usage.entry() for usage in self._roles.values() if usage.turns] + if not entries: + return None + self._context.metadata.setdefault(STEP_USAGE_KEY, {})[self._step_id] = entries + return entries diff --git a/tst/unit/task_step/usage_test.py b/tst/unit/task_step/usage_test.py index adf60a49a..6d66d142a 100644 --- a/tst/unit/task_step/usage_test.py +++ b/tst/unit/task_step/usage_test.py @@ -2,8 +2,10 @@ from agentenv_protocol.a2a_agent import Usage -from agent_env.task_step.context import ATTEMPT_EPOCH, TaskStepContext -from agent_env.task_step.usage import RoleUsage, record_step_usage +from agent_env.a2a_agent.protocol import TerminalResponse +from agent_env.config.model import ModelRole +from agent_env.task_step.context import ATTEMPT_EPOCH, DeployedAgent, TaskStepContext +from agent_env.task_step.usage import RoleUsage, StepUsage def test_turns_that_disagree_on_whether_input_counts_the_cache_leave_the_flag_unknown(): @@ -40,11 +42,26 @@ def test_the_entry_carries_the_attempt_its_step_ran_under(): def test_a_step_whose_roles_ran_no_turns_records_nothing(): context = TaskStepContext() + usage = StepUsage(context, "solve") + usage.start(ModelRole.SOLVER, "m", None) - assert record_step_usage(context, "solve", (RoleUsage("solver"), None)) is None + assert usage.entries() is None assert "step_usage" not in context.metadata +def test_a_step_records_each_started_role_as_each_turn_ends_and_skips_one_never_started(): + context = TaskStepContext() + usage = StepUsage(context, "solve") + usage.start(ModelRole.SOLVER, "m", DeployedAgent(agent_name="a", api_url="u", model_endpoint_host="gw:443")) + + usage.turn(ModelRole.SOLVER, TerminalResponse(response_text="", usage=Usage(input_tokens=3, output_tokens=1))) + usage.turn(ModelRole.USER_SIMULATOR, TerminalResponse(response_text="", usage=Usage(input_tokens=99))) + + [entry] = context.metadata["step_usage"]["solve"] + assert (entry["role"], entry["model"], entry["endpoint_host"], entry["input_tokens"]) == ("solver", "m", "gw:443", 3) + assert context.metadata["model_endpoints"] == {"solver": {"solve": {"model": "m", "endpoint_host": "gw:443"}}} + + def test_a_turn_that_reported_any_token_count_makes_a_turn_that_reported_none_partial(): role = RoleUsage("solver") role.add(Usage(total_tokens=100))