Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
4 changes: 4 additions & 0 deletions AGENTS.md
Original file line number Diff line number Diff line change
Expand Up @@ -346,6 +346,10 @@ orchestrators call it the same way. What callers rely on:
- Each step runs under `protocol_v1.authorized` for the deployed envs in its context: a protocol-client
call the step makes to a deployed env's address, naming no headers of its own, sends that env's
`DeployedEnv.control_headers`.
- Each step runs with `task_step.context.ATTEMPT_EPOCH` set to the attempt epoch it was dispatched
under, in its own asyncio task, so the usage a prompt step records (`metadata["step_usage"]`,
`PromptResponse.usage`) says which attempt spent it. A runner that calls `step.execute()` itself, not
through `Task.run()`, sets it the same way, or its entries carry no attempt.
- A runner records every run it accepts in the run index (`agent_env.runner.store`): `insert_run` at
submit, `set_running` when it starts, `mark_terminal` when it ends. The explorer lists, counts and
groups runs from that collection and never calls the runner to ask, so a run missing there is
Expand Down
14 changes: 14 additions & 0 deletions src/agent_env/a2a_agent/a2a_agent.py
Original file line number Diff line number Diff line change
Expand Up @@ -633,6 +633,20 @@ def _refusal(self, image: DockerImageArtifact, link: SandboxProvider) -> str | N
return f"{how}: {problem}"
return None

def model_base_url(self, env_vars: dict[str, str], config) -> Optional[str]:
"""The model endpoint a deploy with ``env_vars`` gives this agent, by ``_with_model_credentials``'s rule:
theirs, else its ``default_env_vars``', else the configured one; None for an agent that takes no credentials,
which gets none, or with no endpoint configured."""
if not self.metadata.get(TAKES_CREDENTIALS, True):
return None
for given in (env_vars, self.default_env_vars):
if "LITELLM_BASE_URL" in given:
return given["LITELLM_BASE_URL"]
try:
return config.get_litellm_base_url()
except ConfigError:
return None

def _with_model_credentials(self, env_vars: dict[str, str], config) -> dict[str, str]:
"""``env_vars`` with the model endpoint, key and headers each taken from config when neither they nor
``default_env_vars`` set it; for an agent that takes no credentials, with none of them, whatever set them. The
Expand Down
74 changes: 69 additions & 5 deletions src/agent_env/a2a_agent/protocol.py
Original file line number Diff line number Diff line change
Expand Up @@ -3,15 +3,17 @@

import asyncio
import contextlib
import json
import logging
import re
import time
from collections.abc import Mapping
from dataclasses import dataclass
from dataclasses import dataclass, fields
from typing import Any, Optional

import httpx
from a2a.types import TaskState
from agentenv_protocol.a2a_agent import Usage

logger = logging.getLogger(__name__)

Expand Down Expand Up @@ -268,10 +270,65 @@ async def cancel_a2a_task(a2a_url: str, task_id: str) -> None:
logger.info(f"Canceled A2A task {task_id}")


_USAGE_FIELDS = frozenset(f.name for f in fields(Usage)) - {"provider_details"}
# How much of a value ``read_usage`` couldn't read it keeps, so a broken agent can't bloat a run's stored context.
_UNPARSED_PREVIEW_CHARS = 200
# Where agents before Usage had cache fields put their cache counts.
_LEGACY_CACHE_KEYS = ("cache_read_tokens", "cache_write_tokens")


def _preview(value: Any) -> Any:
"""``value`` as it came when it's short, else the start of its JSON as a string."""
text = json.dumps(value, default=str)
return value if len(text) <= _UNPARSED_PREVIEW_CHARS else text[:_UNPARSED_PREVIEW_CHARS] + "..."


def read_usage(raw: Any) -> tuple[Optional[Usage], Optional[dict]]:
"""An agent's reported usage and what of it couldn't be read, never raising: each field ``Usage`` has, checked on
its own, a value it refuses kept under its name in the second (a long one cut to its start); a key it doesn't have
kept in ``provider_details``; the cache counts an agent from before the typed fields put in ``provider_details``
read from there."""
usage, unparsed = _read_usage(raw)
return usage, ({key: _preview(value) for key, value in unparsed.items()} if unparsed else None)


def _read_usage(raw: Any) -> tuple[Optional[Usage], Optional[dict]]:
if raw is None:
return None, None
if not isinstance(raw, Mapping):
return None, {"usage": raw}
unparsed: dict = {}
details = raw.get("provider_details")
if details is not None and not isinstance(details, Mapping):
unparsed["provider_details"] = details
details = None
details = dict(details or {})
given = {key: value for key, value in raw.items() if key in _USAGE_FIELDS}
details.update({key: value for key, value in raw.items() if key not in _USAGE_FIELDS and key != "provider_details"})
for name in _LEGACY_CACHE_KEYS:
if given.get(name) is None and name in details:
given[name] = details.pop(name)
valid = {}
for key, value in given.items():
try:
Usage(**{key: value})
except (TypeError, ValueError):
unparsed[key] = value
else:
valid[key] = value
try:
usage = Usage(**valid, provider_details=details)
except (TypeError, ValueError):
unparsed["provider_details"] = details
usage = Usage(**valid)
return usage, unparsed or None


@dataclass(frozen=True)
class TerminalResponse:
"""What's pulled from an A2A terminal status message: the text reply, the typed
``structured_output`` (when ``output_format`` was set), and tool/error telemetry."""
``structured_output`` (when ``output_format`` was set), tool/error telemetry, and the turn's ``usage`` as
``read_usage`` reads it, with what of it couldn't be read in ``usage_unparsed``."""

response_text: str
tool_call_count: Optional[int] = None
Expand All @@ -280,6 +337,8 @@ class TerminalResponse:
error_class: Optional[str] = None
error_message: Optional[str] = None
structured_output: Optional[Any] = None
usage: Optional[Usage] = None
usage_unparsed: Optional[dict] = None

@classmethod
def from_message(cls, status_message: dict) -> "TerminalResponse":
Expand All @@ -292,6 +351,8 @@ def from_message(cls, status_message: dict) -> "TerminalResponse":
error_class: Optional[str] = None
error_message: Optional[str] = None
structured_output: Optional[Any] = None
usage: Optional[Usage] = None
usage_unparsed: Optional[dict] = None
for part in (status_message or {}).get("parts", []):
kind = part.get("kind")
if kind == "text":
Expand All @@ -300,9 +361,10 @@ def from_message(cls, status_message: dict) -> "TerminalResponse":
data = part.get("data", {})
if "tool_call_count" in data:
tool_call_count = data["tool_call_count"]
usage = data.get("usage")
if isinstance(usage, dict) and "tool_call_count" in usage:
tool_call_count = usage["tool_call_count"]
if "usage" in data:
usage, usage_unparsed = read_usage(data["usage"])
if isinstance(data["usage"], dict) and "tool_call_count" in data["usage"]:
tool_call_count = data["usage"]["tool_call_count"]
if "error_type" in data:
error_type = data["error_type"]
if "error_code" in data:
Expand All @@ -321,6 +383,8 @@ def from_message(cls, status_message: dict) -> "TerminalResponse":
error_class=error_class,
error_message=error_message,
structured_output=structured_output,
usage=usage,
usage_unparsed=usage_unparsed,
)


Expand Down
12 changes: 12 additions & 0 deletions src/agent_env/config/model.py
Original file line number Diff line number Diff line change
Expand Up @@ -312,6 +312,18 @@ def _netloc(url: str) -> str:
return urlsplit(url).netloc.rpartition("@")[2] or url


def endpoint_host(url: object) -> Optional[str]:
"""The host, and port when it names one, a model call to ``url`` reaches, for a record: never the user or
password a URL may carry. None for anything but a URL with a host, never raising, so a record can't fail what it
records."""
if not isinstance(url, str):
return None
try:
return urlsplit(url.strip()).netloc.rpartition("@")[2] or None
except ValueError:
return None


def no_key_for(url: str, how: str) -> str:
"""Why a model call or an agent pointed at ``url`` with no key of its own is refused, and ``how`` to give it one."""
return (f"{_netloc(url)} isn't the configured model endpoint, so the configured key isn't sent there, and no key "
Expand Down
10 changes: 9 additions & 1 deletion src/agent_env/task/task.py
Original file line number Diff line number Diff line change
Expand Up @@ -26,7 +26,13 @@
from agent_env.task_step.preflight import pinned
from agent_env.task.outcome import FailedStep
from agent_env.task.teardown import env_record_ids
from agent_env.task_step.context import ENTITY_IDS_KEY, TaskStepContext, VerifierNotPassingError, regraft_redacted_keys
from agent_env.task_step.context import (
ATTEMPT_EPOCH,
ENTITY_IDS_KEY,
TaskStepContext,
VerifierNotPassingError,
regraft_redacted_keys,
)
from agent_env.task_step.context_ops import ContextUpdateOps, build_context_update_ops
from agent_env.task_step.task_step import RetryConfig, TaskStep

Expand Down Expand Up @@ -699,6 +705,8 @@ async def _run_one(step: TaskStep, idx: int, dispatch_epoch: int) -> None:
pre_context = copy.deepcopy(context)
started_mono = time.monotonic()
started_utc = datetime.now(timezone.utc).isoformat()
# This step's own asyncio task, so a sibling dispatched alongside it doesn't see it.
ATTEMPT_EPOCH.set(dispatch_epoch)
try:
with protocol_v1.authorized({d.environment_url: d.control_headers for d in context.deployed_envs}):
await step.execute(context)
Expand Down
18 changes: 18 additions & 0 deletions src/agent_env/task_step/context.py
Original file line number Diff line number Diff line change
Expand Up @@ -6,6 +6,7 @@
import dataclasses
import re
from collections.abc import Callable
from contextvars import ContextVar
from dataclasses import dataclass, field
from typing import Any

Expand All @@ -17,6 +18,9 @@
from agent_env.providers.sandbox_providers.sandbox_provider import reachable_url

HEARTBEAT_KEY = "_heartbeat_fn"
# The attempt epoch the step running in this task was dispatched under, which ``Task.run`` sets for it; None for a
# step run outside it.
ATTEMPT_EPOCH: ContextVar[int | None] = ContextVar("agentenv_attempt_epoch", default=None)
# context.metadata[ENTITY_IDS_KEY]: the run's task's Task.entity_ids, set by Task.run.
ENTITY_IDS_KEY = "entity_ids"
# The context.metadata keys whose entries name a deployed env by its id, each with the index of the path segment that
Expand Down Expand Up @@ -134,6 +138,10 @@ class DeployedAgent:
env_bindings: dict[str, str | None] = field(default_factory=dict)
# The token its extension routes take, from the deploy that started it; None for an agent agent-env didn't start.
control_token: str | None = None
# The host (and port) of the model endpoint it was given, as configured, before a sandbox's rewrite to reach it
# (localhost to host.docker.internal), so one endpoint has one name; never a key or a URL's userinfo. None when
# unknown.
model_endpoint_host: str | None = None

@classmethod
def from_dict(cls, data: dict) -> DeployedAgent:
Expand All @@ -152,6 +160,7 @@ def from_dict(cls, data: dict) -> DeployedAgent:
env_ids=list(data.get("env_ids") or []),
env_bindings=dict(data.get("env_bindings") or {}),
control_token=data.get("control_token"),
model_endpoint_host=data.get("model_endpoint_host"),
)

@property
Expand Down Expand Up @@ -248,6 +257,8 @@ class PromptResponse:
agent_trajectory_object_url: str | None = None
agent_trajectory_object_prefix: str | None = None
target_agent_per_turn_trajectory_object_urls: list[str | None] | None = None
# The step's usage, one entry per role that ran turns in it (``task_step.usage``); None before it was recorded.
usage: list[dict] | None = None

def to_dict(self) -> dict[str, Any]:
"""The stored form: every field, plus the legacy key of each renamed one."""
Expand Down Expand Up @@ -282,6 +293,7 @@ def from_dict(cls, data: dict) -> PromptResponse:
agent_name=data.get("agent_name"),
step_id=data.get("step_id"),
structured_output=data.get("structured_output"),
usage=data.get("usage"),
)


Expand Down Expand Up @@ -316,6 +328,12 @@ def record_model(self, role: str, step_id: str, model: str | None) -> None:
"""Record the model ``step_id`` ran ``role`` on, None when the agent chose its own, for the run's summary."""
self.metadata.setdefault("models", {}).setdefault(role, {})[step_id] = model

def record_model_endpoint(self, role: str, step_id: str, model: str | None, endpoint_host: str | None) -> None:
"""Record the model ``step_id`` ran ``role`` on and the host of the endpoint it called (``endpoint_host``,
never a key or a URL's userinfo), beside ``models`` under the role's own name."""
self.metadata.setdefault("model_endpoints", {}).setdefault(role, {})[step_id] = {
"model": model, "endpoint_host": endpoint_host}

def to_safe_dict(self) -> dict[str, Any]:
"""Return a dict representation with sensitive keys recursively removed."""
d = self.to_dict()
Expand Down
22 changes: 17 additions & 5 deletions src/agent_env/task_step/task_steps/always_on.py
Original file line number Diff line number Diff line change
Expand Up @@ -113,6 +113,7 @@
from agent_env.config.model import ModelRole
from agent_env.store import DuplicateKeyError, get_config
from agent_env.task_step.context import PromptResponse, StepContextError, TaskStepContext
from agent_env.task_step.usage import StepUsage
from agent_env.task_step.task_step import TaskStep
from agent_env.task_step.task_steps.agent import user_sim_versions
from agent_env.task_step.task_steps.agent.deploy_user_sim_agent import USER_SIMS_KEY, user_sim_model
Expand Down Expand Up @@ -379,6 +380,8 @@ def __init__(self, step: AlwaysOnTaskStep, context: TaskStepContext):
self.per_turn_trajectory_urls: list[Optional[str]] = []
self.per_turn_parts: list[Optional[list[dict]]] = []
self.tool_calls: Optional[int] = None
# Each role's turns' usage, recorded as each turn ends.
self.usage = StepUsage(context, step.id)
self.all_messages: list[dict] = []
self.final_task_id = ""
self.error: Optional[str] = None
Expand Down Expand Up @@ -594,7 +597,9 @@ async def _configure_agents(self) -> None:
if negotiated:
await protocol.post_agent_config(self.target_url + negotiated.endpoint, negotiated.fields,
headers=self.agent.control_headers)
ctx.record_model(ModelRole.SOLVER.record_key, step.id, negotiated.fields.get("model") if negotiated else None)
solver_model = negotiated.fields.get("model") if negotiated else None
ctx.record_model(ModelRole.SOLVER.record_key, step.id, solver_model)
self.usage.start(ModelRole.SOLVER, solver_model, self.agent)
self.rules_in_system_prompt = bool(desired.get("system_prompt") and negotiated
and "system_prompt" in negotiated.fields)
if desired.get("system_prompt") and not self.rules_in_system_prompt:
Expand All @@ -604,6 +609,7 @@ async def _configure_agents(self) -> None:
if self.us.get("prompt_set"): # deploy_user_sim_agent set the user sim's prompt and model at deploy
self.user_model, self.user_model_from = self.us["model"], self.us["model_source"]
self.user_setup = dict(self.us["setup"])
self.usage.start(ModelRole.USER_SIMULATOR, self.user_model, self.user_agent)
return
setup = self.plugin.setup(self.version, self.person)
self.user_model, self.user_model_from = self._user_model()
Expand All @@ -615,7 +621,9 @@ async def _configure_agents(self) -> None:
await protocol.post_agent_config(self.user_url + negotiated.endpoint, negotiated.fields,
headers=self.user_agent.control_headers)
accepted = sorted(negotiated.fields) if negotiated else []
ctx.record_model(ModelRole.USER_SIMULATOR.record_key, step.id, negotiated.fields.get("model") if negotiated else None)
user_model = negotiated.fields.get("model") if negotiated else None
ctx.record_model(ModelRole.USER_SIMULATOR.record_key, step.id, user_model)
self.usage.start(ModelRole.USER_SIMULATOR, user_model, self.user_agent)
self.user_setup = {"model": self.user_model, "model_from": self.user_model_from,
"user_sim_step_model": self.us.get("model"),
"model_set": "model" in accepted, # else the agent ran its own
Expand Down Expand Up @@ -1271,6 +1279,7 @@ def record() -> None:
term = protocol.TerminalResponse.from_message(status_msg)
if term.tool_call_count is not None:
self.tool_calls = (self.tool_calls or 0) + term.tool_call_count
self.usage.turn(ModelRole.SOLVER, term)
messages, quiet = await self._take_messages()
await self._read_changes("turn")
traj = None
Expand Down Expand Up @@ -1351,6 +1360,7 @@ def record() -> None:
term = protocol.TerminalResponse.from_message(status_msg)
if term.tool_call_count is not None:
self.tool_calls = (self.tool_calls or 0) + term.tool_call_count
self.usage.turn(ModelRole.SOLVER, term)
messages, quiet = await self._take_messages()
await self._read_changes("nudge")
traj = None
Expand Down Expand Up @@ -1557,10 +1567,11 @@ async def _ask_user_sim(self, text: str, deadline: float) -> dict:
except TimeoutError as e:
return {"text": None, "structured": None, "error": f"timeout: {e}"}
status = (result.get("status") or {})
if status.get("state") != TaskState.completed and str(status.get("state")) != "completed":
return {"text": None, "structured": None, "error": f"state {status.get('state')}"}
msg = status.get("message") or {}
term = protocol.TerminalResponse.from_message(msg)
self.usage.turn(ModelRole.USER_SIMULATOR, term)
if status.get("state") != TaskState.completed and str(status.get("state")) != "completed":
return {"text": None, "structured": None, "error": f"state {status.get('state')}"}
raw = next((p["text"] for p in msg.get("parts") or [] if p.get("kind") == "text"), term.response_text)
structured = term.structured_output
if structured is None:
Expand Down Expand Up @@ -1736,7 +1747,8 @@ def _append_prompt_response(self, final_text: str) -> None:
source_agent_per_turn_prompt_parts=self.per_turn_parts or None, tool_call_count=self.tool_calls,
model=self.model, error_type=self.error_type, error_code=self.error_code, error_message=self.error,
agent_session_id=self.final_task_id,
a2a_context_id=self.solver_context_id, agent_name=step.agent_name, step_id=step.id))
a2a_context_id=self.solver_context_id, agent_name=step.agent_name, step_id=step.id,
usage=self.usage.entries()))


# ==================================================================================================================
Expand Down
3 changes: 2 additions & 1 deletion src/agent_env/task_step/task_steps/deploy_agent.py
Original file line number Diff line number Diff line change
Expand Up @@ -30,7 +30,7 @@
)
from agent_env.a2a_agent.staging import StagedObjectStore, transfer_store
from agent_env.config import get_config
from agent_env.config.model import ModelRole
from agent_env.config.model import ModelRole, endpoint_host
from agent_env.env.env import DeployedEnv, DeployedSandboxEnv, Env
from agent_env.env.gateway import AGENT_ENV_ROLE_HEADER
from agent_env.providers.sandbox_providers.local_docker.provider import host_url_for
Expand Down Expand Up @@ -554,6 +554,7 @@ async def _execute(self, context: TaskStepContext, deployed_agents: list) -> Tas
env_bindings={env_id: (d.metadata or {}).get("deploy_step_id")
for env_id, d in env_deployments.items() if d is not None},
control_token=deployed.control_token,
model_endpoint_host=endpoint_host(agent.model_base_url(env_vars, get_config())),
Comment thread
earakely-scale marked this conversation as resolved.
Resolved
))
# An agent deployed while a clock is armed follows it, as sync_env_clock had every agent then.
from agent_env.a2a_agent.clock import follow_armed_clocks
Expand Down
Loading
Loading