Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension


Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
4 changes: 3 additions & 1 deletion src/agent_env/agents/AGENTS.md
Original file line number Diff line number Diff line change
Expand Up @@ -55,7 +55,9 @@ terminal state, `completed`, `failed` or `canceled`. Your `run()` is one turn.
- **Usage:** when the runtime counts prompt-cache tokens, put them in `Usage`'s `cache_read_tokens` and
`cache_write_tokens`, not `provider_details`, and set `input_includes_cache` to whether `input_tokens` counts them
(`None` when it can't tell). Set it per call: through a gateway it depends on the model behind it. Where the runtime
hands back a model response's own usage, `Usage.from_model_response` reads it. Code that sets these fields needs a
hands back a model response's own usage, `Usage.from_model_response` reads it. A call that came back with no usage,
as a streamed answer through some gateways does, leaves the counts unset rather than zero and is counted in
`provider_details.unreported_calls`. Code that sets these fields needs a
`requirements.txt` protocol floor at the first release that has them.
- **Errors:** `agent_error`, the default, means the agent tried and failed; `infra_error` means the run couldn't
happen, and agent-env may retry it. An exception that escapes `run()` becomes an opaque `infra_error`, its text
Expand Down
9 changes: 7 additions & 2 deletions src/agent_env/agents/agents/browser-use/a2a_server.py
Original file line number Diff line number Diff line change
Expand Up @@ -161,9 +161,14 @@ async def run(self, request: TaskRequest[BrowserUseConfig]) -> TaskResult:
input_tokens=usage["input"],
output_tokens=usage["output"],
total_tokens=usage["input"] + usage["output"],
# browser-use counts the cache in its input on both of its routes: Anthropic's Messages, whose cache
# it adds in, and OpenAI's Chat Completions, whose prompt already holds it.
cache_read_tokens=usage["cache_read"],
cache_write_tokens=usage["cache_write"],
input_includes_cache=True,
cost_usd=result.get("cost_usd"),
provider_details={"steps": result["steps"], "model": result["model"], "requests": usage["requests"],
"cache_read_tokens": usage["cache_read"], "cache_write_tokens": usage["cache_write"],
"cost_usd": result.get("cost_usd"), "success": result.get("success")},
"success": result.get("success")},
))
.build()
)
Expand Down
6 changes: 3 additions & 3 deletions src/agent_env/agents/agents/browser-use/requirements.txt
Original file line number Diff line number Diff line change
@@ -1,4 +1,4 @@
# The image installs the newest agentenv-framework-protocol at or above this floor; a release doesn't pin it, so the
# image is built again only when this folder changes. The floor is the first that takes the native trajectory format
# the agent declares (enable(TRAJECTORY_V1, native=...)); raise it when the agent needs a newer protocol.
agentenv-framework-protocol[agent]>=0.1.588
# image is built again only when this folder changes. The floor is the first whose Usage takes the prompt cache's
# counts (cache_read_tokens, cache_write_tokens, input_includes_cache); raise it when the agent needs a newer protocol.
agentenv-framework-protocol[agent]>=0.1.597
35 changes: 29 additions & 6 deletions src/agent_env/agents/agents/hermes/a2a_server.py
Original file line number Diff line number Diff line change
Expand Up @@ -519,21 +519,44 @@ async def swap() -> None:


def _usage(result) -> Usage:
"""The turn's tool calls and the token counts and cost Hermes reported for its main model calls."""
"""The turn's tool calls and the token counts and cost Hermes reported for its main model calls, with
``input_tokens`` counting the cache. Hermes' ``input_tokens`` is the prompt less its cache reads and writes, and its
total counts them, so they're added back; a total of the input and output alone says the input already holds them,
and a total that says neither keeps Hermes' rule with the flag unknown. Calls that reported no counts at all got
none from their endpoint, as streamed Claude answers through a LiteLLM proxy, so the turn's counts are unknown
rather than nothing."""
report = result.usage or {}

def count(name: str) -> int | None:
value = report.get(name)
return value if isinstance(value, int) and not isinstance(value, bool) and value >= 0 else None

cost = report.get("estimated_cost_usd")
details = {key: report[key] for key in ("api_calls", "cache_read_tokens", "cache_write_tokens", "reasoning_tokens",
"cost_status", "turn_exit_reason") if report.get(key) is not None}
details = {key: report[key] for key in ("api_calls", "reasoning_tokens", "cost_status", "turn_exit_reason")
if report.get(key) is not None}
counts = {name: count(name) for name in ("input_tokens", "output_tokens", "total_tokens", "cache_read_tokens",
"cache_write_tokens")}
if (count("api_calls") or 0) > 0 and not any(counts.values()):
return Usage(tool_call_count=result.tool_call_count,
provider_details={**details, "unreported_calls": details["api_calls"]})
read, write = counts["cache_read_tokens"], counts["cache_write_tokens"]
input_tokens, output, total = counts["input_tokens"], counts["output_tokens"] or 0, counts["total_tokens"]
cached = (read or 0) + (write or 0)
includes_cache = None
if input_tokens is not None:
if cached == 0 or total == input_tokens + output:
includes_cache = True
else:
includes_cache = True if total == input_tokens + output + cached else None
input_tokens += cached
return Usage(
tool_call_count=result.tool_call_count,
input_tokens=count("input_tokens"),
output_tokens=count("output_tokens"),
total_tokens=count("total_tokens"),
input_tokens=input_tokens,
output_tokens=counts["output_tokens"],
total_tokens=total,
cache_read_tokens=read,
cache_write_tokens=write,
input_includes_cache=includes_cache,
cost_usd=cost if isinstance(cost, (int, float)) and not isinstance(cost, bool) and cost >= 0 else None,
provider_details=details,
)
Expand Down
6 changes: 3 additions & 3 deletions src/agent_env/agents/agents/hermes/requirements.txt
Original file line number Diff line number Diff line change
@@ -1,5 +1,5 @@
# The image installs the newest agentenv-framework-protocol at or above this floor; a release doesn't pin it, so the
# image is built again only when this folder changes. The floor is the first that takes the native trajectory format
# the agent declares (enable(TRAJECTORY_V1, native=...)); raise it when the agent needs a newer protocol.
agentenv-framework-protocol[agent]>=0.1.588
# image is built again only when this folder changes. The floor is the first whose Usage takes the prompt cache's
# counts (cache_read_tokens, cache_write_tokens, input_includes_cache); raise it when the agent needs a newer protocol.
agentenv-framework-protocol[agent]>=0.1.597
jsonschema>=4.18,<5
38 changes: 37 additions & 1 deletion src/agent_env/agents/agents/openhands/openhands_agent.py
Original file line number Diff line number Diff line change
Expand Up @@ -17,7 +17,7 @@
import uuid
from collections.abc import AsyncIterator, Callable, Mapping
from contextlib import asynccontextmanager, suppress
from dataclasses import dataclass, field
from dataclasses import dataclass, field, replace
from pathlib import Path
from typing import Any, Final, Literal, Protocol, Self
from urllib.parse import urlsplit
Expand Down Expand Up @@ -157,6 +157,42 @@ class Outcome:
error_type: Literal["agent_error", "infra_error"] = "agent_error"


def turn_usage(calls: list[tuple[Any, bool]], *, responses: int, tool_calls: int, cost: float | None) -> Usage:
"""A turn's usage from the ``TokenUsage`` OpenHands recorded for each model call, beside whether its LLM speaks
OpenAI's Responses API, with ``input_tokens`` counting the cache. A Responses call's input counts it. A Chat
Completions call's may or may not: LiteLLM reads cached tokens from OpenAI's ``prompt_tokens_details``, whose
prompt counts them, and from Anthropic's keys beside a prompt that doesn't, as a LiteLLM proxy sends Claude's.
The cache is added when the prompt is smaller than it, and the turn's flag is left unknown when the counts can't
say. OpenHands records no ``TokenUsage`` for a response without tokens, so the turn's ``responses`` beyond its
calls are counted in ``unreported_calls``."""
counted = [_counted(usage.prompt_tokens, usage.completion_tokens, usage.cache_read_tokens,
usage.cache_write_tokens, True if responses_api else None) for usage, responses_api in calls]
unreported = responses - len(calls)
details = {"reasoning_tokens": sum(usage.reasoning_tokens or 0 for usage, _ in calls),
**({"unreported_calls": unreported} if unreported else {})}
return replace(_summed(counted, tool_calls, details), cost_usd=cost)


def _counted(prompt: int, output: int, read: int, write: int, counts_cache: bool | None) -> tuple[int, ...]:
"""A call's input counting its cache, its output, cache read, cache write, and 1 when whether its prompt held the
cache is known (said so, ``counts_cache``; nothing cached; or a prompt smaller than the cache), else 0."""
if counts_cache or read + write == 0:
return prompt, output, read, write, 1
if prompt < read + write:
return prompt + read + write, output, read, write, 1
return prompt, output, read, write, 0


def _summed(calls: list[tuple[int, ...]], tool_calls: int, details: dict[str, Any]) -> Usage:
"""The calls' counts summed: none at all when no call reported usage, the flag unknown when a call's can't be."""
if not calls:
return Usage(tool_call_count=tool_calls, provider_details=details)
input_tokens, output_tokens, read, write, settled = (sum(column) for column in zip(*calls))
return Usage(tool_call_count=tool_calls, input_tokens=input_tokens, output_tokens=output_tokens,
total_tokens=input_tokens + output_tokens, cache_read_tokens=read, cache_write_tokens=write,
input_includes_cache=True if settled == len(calls) else None, provider_details=details)


@dataclass(slots=True)
class _ConversationLock:
lock: asyncio.Lock = field(default_factory=asyncio.Lock)
Expand Down
33 changes: 8 additions & 25 deletions src/agent_env/agents/agents/openhands/openhands_runtime.py
Original file line number Diff line number Diff line change
Expand Up @@ -15,14 +15,13 @@
from openhands.sdk.event import ActionEvent, Event
from openhands.sdk.event.conversation_error import ConversationErrorEvent
from openhands.sdk.event.error_classification import FailureKind
from openhands.sdk.llm.utils.metrics import Metrics
from openhands.sdk.mcp import MCPServer
from openhands.sdk.skills.skill import load_skills_from_dir
from openhands.tools.preset.default import get_default_condenser, get_default_tools
from pydantic import PrivateAttr

from conversation_run import ConversationRun
from openhands_agent import Outcome, Turn
from openhands_agent import Outcome, Turn, turn_usage

# The AgentEnv endpoint is a LiteLLM gateway; this provider prefix sends LiteLLM's requests to it as is.
_PROVIDER: Final[str] = "litellm_proxy"
Expand Down Expand Up @@ -88,29 +87,13 @@ def _agent(turn: Turn) -> Agent:


def _usage(agent: Agent, tool_calls: int) -> Usage:
"""What the turn's LLMs have spent: they are made for the turn, so they start at nothing."""
llms = (agent.llm, getattr(agent.condenser, "llm", None))
metrics = Metrics()
for each in {id(llm.metrics): llm.metrics for llm in llms if llm is not None}.values():
metrics.merge(each)

def spent(name: str) -> int:
return getattr(metrics.accumulated_token_usage, name, 0) or 0

input_tokens, output_tokens = spent("prompt_tokens"), spent("completion_tokens")
cost = metrics.accumulated_cost
return Usage(
tool_call_count=tool_calls,
input_tokens=input_tokens,
output_tokens=output_tokens,
total_tokens=input_tokens + output_tokens,
cost_usd=cost or None,
provider_details={
"cache_read_tokens": spent("cache_read_tokens"),
"cache_write_tokens": spent("cache_write_tokens"),
"reasoning_tokens": spent("reasoning_tokens"),
},
)
"""What the turn's LLMs have spent, call by call: they are made for the turn, so they start at nothing. Each
response they got has a latency recorded, with or without usage."""
llms = {id(llm.metrics): llm for llm in (agent.llm, getattr(agent.condenser, "llm", None)) if llm is not None}
calls = [(usage, llm.uses_responses_api()) for llm in llms.values() for usage in llm.metrics.token_usages]
responses = sum(len(llm.metrics.response_latencies) for llm in llms.values())
cost = sum(llm.metrics.accumulated_cost for llm in llms.values())
return turn_usage(calls, responses=responses, tool_calls=tool_calls, cost=cost or None)


def _unstarted(exc: Exception) -> Outcome:
Expand Down
6 changes: 3 additions & 3 deletions src/agent_env/agents/agents/openhands/requirements.txt
Original file line number Diff line number Diff line change
@@ -1,6 +1,6 @@
# The image installs the newest agentenv-framework-protocol at or above this floor; a release doesn't pin it, so the
# image is built again only when this folder changes. The floor is the first that takes the native trajectory format
# the agent declares (enable(TRAJECTORY_V1, native=...)); raise it when the agent needs a newer protocol.
agentenv-framework-protocol[agent]>=0.1.588
# image is built again only when this folder changes. The floor is the first whose Usage takes the prompt cache's
# counts (cache_read_tokens, cache_write_tokens, input_includes_cache); raise it when the agent needs a newer protocol.
agentenv-framework-protocol[agent]>=0.1.597
openhands-sdk==1.58.2
openhands-tools==1.58.2
38 changes: 30 additions & 8 deletions src/agent_env/agents/agents/pi/pi_agent.py
Original file line number Diff line number Diff line change
Expand Up @@ -255,23 +255,45 @@ def _record(run: _Run, event: dict[str, Any]) -> None:


def _usage(run: _Run) -> Usage:
totals = {"input": 0, "output": 0, "totalTokens": 0, "cacheRead": 0, "cacheWrite": 0}
cost = 0.0
"""The turn's usage, summed over its assistant messages, with ``input_tokens`` counting the cache. A message whose
usage is all zero got none from its endpoint, as a streamed Claude answer through a LiteLLM proxy, and is counted
in ``unreported_calls`` instead of as nothing used."""
counted, cost, unreported = [], 0.0, 0
for message in run.assistant_messages:
usage = message.get("usage") or {}
for key in totals:
totals[key] += int(usage.get(key) or 0)
counts = [int(usage.get(key) or 0) for key in ("input", "output", "cacheRead", "cacheWrite", "totalTokens")]
if not any(counts):
unreported += 1
continue
counted.append(_counted(*counts))
cost += float((usage.get("cost") or {}).get("total") or 0)
details = {"unreported_calls": unreported} if unreported else {}
if not counted:
return Usage(tool_call_count=run.tool_calls, provider_details=details)
input_tokens, output_tokens, read, write, total, settled = (sum(column) for column in zip(*counted))
return Usage(
tool_call_count=run.tool_calls,
input_tokens=totals["input"],
output_tokens=totals["output"],
total_tokens=totals["totalTokens"],
input_tokens=input_tokens,
output_tokens=output_tokens,
total_tokens=total,
cache_read_tokens=read,
cache_write_tokens=write,
input_includes_cache=True if settled == len(counted) else None,
cost_usd=cost or None,
provider_details={"cache_read_tokens": totals["cacheRead"], "cache_write_tokens": totals["cacheWrite"]},
provider_details=details,
)


def _counted(input: int, output: int, read: int, write: int, total: int) -> tuple[int, ...]:
"""A message's input counting its cache, its output, cache read, cache write and total, and 1 when its total
confirms which its input was, else 0. pi's ``input`` is the prompt less its cache reads and writes, and its total
counts them, so they're added back; a total of the input and output alone says the input already holds them. A
total that says neither keeps pi's rule, unconfirmed."""
if read + write == 0 or total == input + output:
return input, output, read, write, total, 1
return input + read + write, output, read, write, total, int(total == input + output + read + write)


def _text(message: Mapping[str, Any]) -> str:
content = message.get("content")
if isinstance(content, str):
Expand Down
6 changes: 3 additions & 3 deletions src/agent_env/agents/agents/pi/requirements.txt
Original file line number Diff line number Diff line change
@@ -1,4 +1,4 @@
# The image installs the newest agentenv-framework-protocol at or above this floor; a release doesn't pin it, so the
# image is built again only when this folder changes. The floor is the first that takes the native trajectory format
# the agent declares (enable(TRAJECTORY_V1, native=...)); raise it when the agent needs a newer protocol.
agentenv-framework-protocol[agent]>=0.1.588
# image is built again only when this folder changes. The floor is the first whose Usage takes the prompt cache's
# counts (cache_read_tokens, cache_write_tokens, input_includes_cache); raise it when the agent needs a newer protocol.
agentenv-framework-protocol[agent]>=0.1.597
6 changes: 3 additions & 3 deletions src/agent_env/agents/agents/proxy/requirements.txt
Original file line number Diff line number Diff line change
@@ -1,4 +1,4 @@
# The image installs the newest agentenv-framework-protocol at or above this floor; a release doesn't pin it, so the
# image is built again only when this folder changes. The floor is the first that takes the native trajectory format
# the agent declares (enable(TRAJECTORY_V1, native=...)); raise it when the agent needs a newer protocol.
agentenv-framework-protocol[agent]>=0.1.588
# image is built again only when this folder changes. The floor is the first whose proxy reports each turn's model usage,
# as the driver records it; raise it when the agent needs a newer protocol.
agentenv-framework-protocol[agent]>=0.1.597
Loading
Loading