From 9970c150f04d4a27bf49817c62e775b7b365f4f9 Mon Sep 17 00:00:00 2001 From: Edgar Arakelyan Date: Wed, 18 Nov 2026 23:56:32 -0800 Subject: [PATCH 1/3] feat(agents): shipped agents report cache reads and writes as Usage fields, not provider_details Each shipped agent (Pydantic AI, OpenHands, pi, Hermes, browser-use) now fills Usage's cache_read_tokens, cache_write_tokens and input_includes_cache, and its input_tokens counts the cache where the agent can tell: pi and Hermes add back the cache their runtimes take out of the prompt; Pydantic AI and OpenHands decide call by call, adding Anthropic's cache keys to a prompt smaller than them and leaving the flag unknown when the counts can't say. A model call that returned no usage, as Claude's streamed answers through a LiteLLM proxy, leaves the counts unknown and is counted in provider_details.unreported_calls instead of reading as zero tokens. provider_details.cache_read_tokens and cache_write_tokens move to the Usage fields for all five agents, and browser-use's cost to Usage.cost_usd. The agents' and the proxy's protocol floor is 0.1.597, the first whose Usage has the cache fields. Co-Authored-By: Claude Opus 5.5 --- .../agents/agents/browser-use/a2a_server.py | 9 +++- .../agents/browser-use/requirements.txt | 6 +-- .../agents/agents/hermes/a2a_server.py | 25 +++++++--- .../agents/agents/hermes/requirements.txt | 6 +-- .../agents/openhands/openhands_agent.py | 38 ++++++++++++++- .../agents/openhands/openhands_runtime.py | 33 ++++--------- .../agents/agents/openhands/requirements.txt | 6 +-- src/agent_env/agents/agents/pi/pi_agent.py | 23 +++++++-- .../agents/agents/pi/requirements.txt | 6 +-- .../agents/agents/proxy/requirements.txt | 6 +-- .../agents/pydantic-ai/pydantic_ai_agent.py | 46 +++++++++++++++++- .../agents/pydantic-ai/pydantic_ai_runtime.py | 30 +++++------- .../agents/pydantic-ai/requirements.txt | 6 +-- .../cli/run_bundle_openhands_test.py | 47 +++++++++++++++++++ .../cli/run_bundle_pydantic_ai_test.py | 42 +++++++++++++++++ .../browser_use/test_browser_use_agent.py | 3 +- .../agents/hermes/test_message_contract.py | 22 +++++++++ .../agents/openhands/test_openhands_agent.py | 35 ++++++++++++++ tst/unit/agents/pi/test_pi_agent.py | 32 ++++++++++++- .../pydantic_ai/test_pydantic_ai_agent.py | 36 ++++++++++++++ tst/util/fake_openai.py | 17 ++++--- 21 files changed, 389 insertions(+), 85 deletions(-) diff --git a/src/agent_env/agents/agents/browser-use/a2a_server.py b/src/agent_env/agents/agents/browser-use/a2a_server.py index eaef8ea8f..3146f8910 100644 --- a/src/agent_env/agents/agents/browser-use/a2a_server.py +++ b/src/agent_env/agents/agents/browser-use/a2a_server.py @@ -161,9 +161,14 @@ async def run(self, request: TaskRequest[BrowserUseConfig]) -> TaskResult: input_tokens=usage["input"], output_tokens=usage["output"], total_tokens=usage["input"] + usage["output"], + # browser-use counts the cache in its input on both of its routes: Anthropic's Messages, whose cache + # it adds in, and OpenAI's Chat Completions, whose prompt already holds it. + cache_read_tokens=usage["cache_read"], + cache_write_tokens=usage["cache_write"], + input_includes_cache=True, + cost_usd=result.get("cost_usd"), provider_details={"steps": result["steps"], "model": result["model"], "requests": usage["requests"], - "cache_read_tokens": usage["cache_read"], "cache_write_tokens": usage["cache_write"], - "cost_usd": result.get("cost_usd"), "success": result.get("success")}, + "success": result.get("success")}, )) .build() ) diff --git a/src/agent_env/agents/agents/browser-use/requirements.txt b/src/agent_env/agents/agents/browser-use/requirements.txt index 91e032ab6..dd31306bb 100644 --- a/src/agent_env/agents/agents/browser-use/requirements.txt +++ b/src/agent_env/agents/agents/browser-use/requirements.txt @@ -1,4 +1,4 @@ # The image installs the newest agentenv-framework-protocol at or above this floor; a release doesn't pin it, so the -# image is built again only when this folder changes. The floor is the first that takes the native trajectory format -# the agent declares (enable(TRAJECTORY_V1, native=...)); raise it when the agent needs a newer protocol. -agentenv-framework-protocol[agent]>=0.1.588 +# image is built again only when this folder changes. The floor is the first whose Usage takes the prompt cache's +# counts (cache_read_tokens, cache_write_tokens, input_includes_cache); raise it when the agent needs a newer protocol. +agentenv-framework-protocol[agent]>=0.1.597 diff --git a/src/agent_env/agents/agents/hermes/a2a_server.py b/src/agent_env/agents/agents/hermes/a2a_server.py index ee0565539..4f994a7d7 100644 --- a/src/agent_env/agents/agents/hermes/a2a_server.py +++ b/src/agent_env/agents/agents/hermes/a2a_server.py @@ -519,7 +519,10 @@ async def swap() -> None: def _usage(result) -> Usage: - """The turn's tool calls and the token counts and cost Hermes reported for its main model calls.""" + """The turn's tool calls and the token counts and cost Hermes reported for its main model calls, with + ``input_tokens`` counting the cache: Hermes' ``input_tokens`` is the prompt less its cache reads and writes, so + they're added back. Calls that reported no counts at all got none from their endpoint, as streamed Claude answers + through a LiteLLM proxy, so the turn's counts are unknown rather than nothing.""" report = result.usage or {} def count(name: str) -> int | None: @@ -527,13 +530,23 @@ def count(name: str) -> int | None: return value if isinstance(value, int) and not isinstance(value, bool) and value >= 0 else None cost = report.get("estimated_cost_usd") - details = {key: report[key] for key in ("api_calls", "cache_read_tokens", "cache_write_tokens", "reasoning_tokens", - "cost_status", "turn_exit_reason") if report.get(key) is not None} + details = {key: report[key] for key in ("api_calls", "reasoning_tokens", "cost_status", "turn_exit_reason") + if report.get(key) is not None} + counts = {name: count(name) for name in ("input_tokens", "output_tokens", "total_tokens", "cache_read_tokens", + "cache_write_tokens")} + if (count("api_calls") or 0) > 0 and not any(counts.values()): + return Usage(tool_call_count=result.tool_call_count, + provider_details={**details, "unreported_calls": details["api_calls"]}) + read, write = counts["cache_read_tokens"], counts["cache_write_tokens"] + input_tokens = counts["input_tokens"] return Usage( tool_call_count=result.tool_call_count, - input_tokens=count("input_tokens"), - output_tokens=count("output_tokens"), - total_tokens=count("total_tokens"), + input_tokens=None if input_tokens is None else input_tokens + (read or 0) + (write or 0), + output_tokens=counts["output_tokens"], + total_tokens=counts["total_tokens"], + cache_read_tokens=read, + cache_write_tokens=write, + input_includes_cache=None if input_tokens is None else True, cost_usd=cost if isinstance(cost, (int, float)) and not isinstance(cost, bool) and cost >= 0 else None, provider_details=details, ) diff --git a/src/agent_env/agents/agents/hermes/requirements.txt b/src/agent_env/agents/agents/hermes/requirements.txt index b0dd0ae3f..219eb7912 100644 --- a/src/agent_env/agents/agents/hermes/requirements.txt +++ b/src/agent_env/agents/agents/hermes/requirements.txt @@ -1,5 +1,5 @@ # The image installs the newest agentenv-framework-protocol at or above this floor; a release doesn't pin it, so the -# image is built again only when this folder changes. The floor is the first that takes the native trajectory format -# the agent declares (enable(TRAJECTORY_V1, native=...)); raise it when the agent needs a newer protocol. -agentenv-framework-protocol[agent]>=0.1.588 +# image is built again only when this folder changes. The floor is the first whose Usage takes the prompt cache's +# counts (cache_read_tokens, cache_write_tokens, input_includes_cache); raise it when the agent needs a newer protocol. +agentenv-framework-protocol[agent]>=0.1.597 jsonschema>=4.18,<5 diff --git a/src/agent_env/agents/agents/openhands/openhands_agent.py b/src/agent_env/agents/agents/openhands/openhands_agent.py index fecf02c4b..43d4f5bdb 100644 --- a/src/agent_env/agents/agents/openhands/openhands_agent.py +++ b/src/agent_env/agents/agents/openhands/openhands_agent.py @@ -17,7 +17,7 @@ import uuid from collections.abc import AsyncIterator, Callable, Mapping from contextlib import asynccontextmanager, suppress -from dataclasses import dataclass, field +from dataclasses import dataclass, field, replace from pathlib import Path from typing import Any, Final, Literal, Protocol, Self from urllib.parse import urlsplit @@ -157,6 +157,42 @@ class Outcome: error_type: Literal["agent_error", "infra_error"] = "agent_error" +def turn_usage(calls: list[tuple[Any, bool]], *, responses: int, tool_calls: int, cost: float | None) -> Usage: + """A turn's usage from the ``TokenUsage`` OpenHands recorded for each model call, beside whether its LLM speaks + OpenAI's Responses API, with ``input_tokens`` counting the cache. A Responses call's input counts it. A Chat + Completions call's may or may not: LiteLLM reads cached tokens from OpenAI's ``prompt_tokens_details``, whose + prompt counts them, and from Anthropic's keys beside a prompt that doesn't, as a LiteLLM proxy sends Claude's. + The cache is added when the prompt is smaller than it, and the turn's flag is left unknown when the counts can't + say. OpenHands records no ``TokenUsage`` for a response without tokens, so the turn's ``responses`` beyond its + calls are counted in ``unreported_calls``.""" + counted = [_counted(usage.prompt_tokens, usage.completion_tokens, usage.cache_read_tokens, + usage.cache_write_tokens, True if responses_api else None) for usage, responses_api in calls] + unreported = responses - len(calls) + details = {"reasoning_tokens": sum(usage.reasoning_tokens or 0 for usage, _ in calls), + **({"unreported_calls": unreported} if unreported else {})} + return replace(_summed(counted, tool_calls, details), cost_usd=cost) + + +def _counted(prompt: int, output: int, read: int, write: int, counts_cache: bool | None) -> tuple[int, ...]: + """A call's input counting its cache, its output, cache read, cache write, and 1 when whether its prompt held the + cache is known (said so, ``counts_cache``; nothing cached; or a prompt smaller than the cache), else 0.""" + if counts_cache or read + write == 0: + return prompt, output, read, write, 1 + if prompt < read + write: + return prompt + read + write, output, read, write, 1 + return prompt, output, read, write, 0 + + +def _summed(calls: list[tuple[int, ...]], tool_calls: int, details: dict[str, Any]) -> Usage: + """The calls' counts summed: none at all when no call reported usage, the flag unknown when a call's can't be.""" + if not calls: + return Usage(tool_call_count=tool_calls, provider_details=details) + input_tokens, output_tokens, read, write, settled = (sum(column) for column in zip(*calls)) + return Usage(tool_call_count=tool_calls, input_tokens=input_tokens, output_tokens=output_tokens, + total_tokens=input_tokens + output_tokens, cache_read_tokens=read, cache_write_tokens=write, + input_includes_cache=True if settled == len(calls) else None, provider_details=details) + + @dataclass(slots=True) class _ConversationLock: lock: asyncio.Lock = field(default_factory=asyncio.Lock) diff --git a/src/agent_env/agents/agents/openhands/openhands_runtime.py b/src/agent_env/agents/agents/openhands/openhands_runtime.py index 9add55f5b..b99d3a1dc 100644 --- a/src/agent_env/agents/agents/openhands/openhands_runtime.py +++ b/src/agent_env/agents/agents/openhands/openhands_runtime.py @@ -15,14 +15,13 @@ from openhands.sdk.event import ActionEvent, Event from openhands.sdk.event.conversation_error import ConversationErrorEvent from openhands.sdk.event.error_classification import FailureKind -from openhands.sdk.llm.utils.metrics import Metrics from openhands.sdk.mcp import MCPServer from openhands.sdk.skills.skill import load_skills_from_dir from openhands.tools.preset.default import get_default_condenser, get_default_tools from pydantic import PrivateAttr from conversation_run import ConversationRun -from openhands_agent import Outcome, Turn +from openhands_agent import Outcome, Turn, turn_usage # The AgentEnv endpoint is a LiteLLM gateway; this provider prefix sends LiteLLM's requests to it as is. _PROVIDER: Final[str] = "litellm_proxy" @@ -88,29 +87,13 @@ def _agent(turn: Turn) -> Agent: def _usage(agent: Agent, tool_calls: int) -> Usage: - """What the turn's LLMs have spent: they are made for the turn, so they start at nothing.""" - llms = (agent.llm, getattr(agent.condenser, "llm", None)) - metrics = Metrics() - for each in {id(llm.metrics): llm.metrics for llm in llms if llm is not None}.values(): - metrics.merge(each) - - def spent(name: str) -> int: - return getattr(metrics.accumulated_token_usage, name, 0) or 0 - - input_tokens, output_tokens = spent("prompt_tokens"), spent("completion_tokens") - cost = metrics.accumulated_cost - return Usage( - tool_call_count=tool_calls, - input_tokens=input_tokens, - output_tokens=output_tokens, - total_tokens=input_tokens + output_tokens, - cost_usd=cost or None, - provider_details={ - "cache_read_tokens": spent("cache_read_tokens"), - "cache_write_tokens": spent("cache_write_tokens"), - "reasoning_tokens": spent("reasoning_tokens"), - }, - ) + """What the turn's LLMs have spent, call by call: they are made for the turn, so they start at nothing. Each + response they got has a latency recorded, with or without usage.""" + llms = {id(llm.metrics): llm for llm in (agent.llm, getattr(agent.condenser, "llm", None)) if llm is not None} + calls = [(usage, llm.uses_responses_api()) for llm in llms.values() for usage in llm.metrics.token_usages] + responses = sum(len(llm.metrics.response_latencies) for llm in llms.values()) + cost = sum(llm.metrics.accumulated_cost for llm in llms.values()) + return turn_usage(calls, responses=responses, tool_calls=tool_calls, cost=cost or None) def _unstarted(exc: Exception) -> Outcome: diff --git a/src/agent_env/agents/agents/openhands/requirements.txt b/src/agent_env/agents/agents/openhands/requirements.txt index de2a01407..4a3a05316 100644 --- a/src/agent_env/agents/agents/openhands/requirements.txt +++ b/src/agent_env/agents/agents/openhands/requirements.txt @@ -1,6 +1,6 @@ # The image installs the newest agentenv-framework-protocol at or above this floor; a release doesn't pin it, so the -# image is built again only when this folder changes. The floor is the first that takes the native trajectory format -# the agent declares (enable(TRAJECTORY_V1, native=...)); raise it when the agent needs a newer protocol. -agentenv-framework-protocol[agent]>=0.1.588 +# image is built again only when this folder changes. The floor is the first whose Usage takes the prompt cache's +# counts (cache_read_tokens, cache_write_tokens, input_includes_cache); raise it when the agent needs a newer protocol. +agentenv-framework-protocol[agent]>=0.1.597 openhands-sdk==1.58.2 openhands-tools==1.58.2 diff --git a/src/agent_env/agents/agents/pi/pi_agent.py b/src/agent_env/agents/agents/pi/pi_agent.py index 46e550d8c..cf11b2ede 100644 --- a/src/agent_env/agents/agents/pi/pi_agent.py +++ b/src/agent_env/agents/agents/pi/pi_agent.py @@ -255,20 +255,35 @@ def _record(run: _Run, event: dict[str, Any]) -> None: def _usage(run: _Run) -> Usage: + """The turn's usage, summed over its assistant messages, with ``input_tokens`` counting the cache: pi's ``input`` + is the prompt less its cache reads and writes, so they're added back. A message whose usage is all zero got none + from its endpoint, as a streamed Claude answer through a LiteLLM proxy, and is counted in ``unreported_calls`` + instead of as nothing used.""" totals = {"input": 0, "output": 0, "totalTokens": 0, "cacheRead": 0, "cacheWrite": 0} - cost = 0.0 + cost, reported, unreported = 0.0, 0, 0 for message in run.assistant_messages: usage = message.get("usage") or {} + counts = {key: int(usage.get(key) or 0) for key in totals} + if not any(counts.values()): + unreported += 1 + continue + reported += 1 for key in totals: - totals[key] += int(usage.get(key) or 0) + totals[key] += counts[key] cost += float((usage.get("cost") or {}).get("total") or 0) + details = {"unreported_calls": unreported} if unreported else {} + if not reported: + return Usage(tool_call_count=run.tool_calls, provider_details=details) return Usage( tool_call_count=run.tool_calls, - input_tokens=totals["input"], + input_tokens=totals["input"] + totals["cacheRead"] + totals["cacheWrite"], output_tokens=totals["output"], total_tokens=totals["totalTokens"], + cache_read_tokens=totals["cacheRead"], + cache_write_tokens=totals["cacheWrite"], + input_includes_cache=True, cost_usd=cost or None, - provider_details={"cache_read_tokens": totals["cacheRead"], "cache_write_tokens": totals["cacheWrite"]}, + provider_details=details, ) diff --git a/src/agent_env/agents/agents/pi/requirements.txt b/src/agent_env/agents/agents/pi/requirements.txt index 91e032ab6..dd31306bb 100644 --- a/src/agent_env/agents/agents/pi/requirements.txt +++ b/src/agent_env/agents/agents/pi/requirements.txt @@ -1,4 +1,4 @@ # The image installs the newest agentenv-framework-protocol at or above this floor; a release doesn't pin it, so the -# image is built again only when this folder changes. The floor is the first that takes the native trajectory format -# the agent declares (enable(TRAJECTORY_V1, native=...)); raise it when the agent needs a newer protocol. -agentenv-framework-protocol[agent]>=0.1.588 +# image is built again only when this folder changes. The floor is the first whose Usage takes the prompt cache's +# counts (cache_read_tokens, cache_write_tokens, input_includes_cache); raise it when the agent needs a newer protocol. +agentenv-framework-protocol[agent]>=0.1.597 diff --git a/src/agent_env/agents/agents/proxy/requirements.txt b/src/agent_env/agents/agents/proxy/requirements.txt index 91e032ab6..cbb1ff189 100644 --- a/src/agent_env/agents/agents/proxy/requirements.txt +++ b/src/agent_env/agents/agents/proxy/requirements.txt @@ -1,4 +1,4 @@ # The image installs the newest agentenv-framework-protocol at or above this floor; a release doesn't pin it, so the -# image is built again only when this folder changes. The floor is the first that takes the native trajectory format -# the agent declares (enable(TRAJECTORY_V1, native=...)); raise it when the agent needs a newer protocol. -agentenv-framework-protocol[agent]>=0.1.588 +# image is built again only when this folder changes. The floor is the first whose proxy reports each turn's model usage, +# as the driver records it; raise it when the agent needs a newer protocol. +agentenv-framework-protocol[agent]>=0.1.597 diff --git a/src/agent_env/agents/agents/pydantic-ai/pydantic_ai_agent.py b/src/agent_env/agents/agents/pydantic-ai/pydantic_ai_agent.py index 6596f606a..d14b2831f 100644 --- a/src/agent_env/agents/agents/pydantic-ai/pydantic_ai_agent.py +++ b/src/agent_env/agents/agents/pydantic-ai/pydantic_ai_agent.py @@ -14,7 +14,7 @@ import tarfile import tempfile import uuid -from collections.abc import AsyncIterator, Callable, Mapping +from collections.abc import AsyncIterator, Callable, Iterable, Mapping from contextlib import asynccontextmanager, suppress from dataclasses import dataclass, field from pathlib import Path @@ -165,6 +165,50 @@ class Outcome: error_type: Literal["agent_error", "infra_error"] = "agent_error" +def turn_usage(responses: Iterable[Any], *, tool_calls: int, requests: int) -> Usage: + """A turn's usage from the usage Pydantic AI kept of each model response (its ``RequestUsage``), with + ``input_tokens`` counting the cache. Pydantic AI reads its cache counts from OpenAI's ``prompt_tokens_details``, + whose prompt counts them; a LiteLLM proxy's Claude answers send Anthropic's ``cache_read_input_tokens`` and + ``cache_creation_input_tokens`` beside a prompt that doesn't, which Pydantic AI keeps in ``details``. Those are + added when the prompt is smaller than them, and leave the turn's flag unknown otherwise, since the counts can't say. + A response that kept no usage at all, as a streamed Claude answer through that proxy, is counted in + ``unreported_calls`` instead of as nothing used.""" + calls, unreported = [], 0 + for usage in responses: + if not (usage.input_tokens or usage.output_tokens): + unreported += 1 + continue + details = usage.details or {} + if usage.cache_read_tokens or usage.cache_write_tokens: + read, write, counts_cache = usage.cache_read_tokens, usage.cache_write_tokens, True + else: + read = details.get("cache_read_input_tokens", 0) + write = details.get("cache_creation_input_tokens", 0) + counts_cache = None + calls.append(_counted(usage.input_tokens, usage.output_tokens, read, write, counts_cache)) + return _summed(calls, tool_calls, {"requests": requests, **({"unreported_calls": unreported} if unreported else {})}) + + +def _counted(prompt: int, output: int, read: int, write: int, counts_cache: bool | None) -> tuple[int, ...]: + """A call's input counting its cache, its output, cache read, cache write, and 1 when whether its prompt held the + cache is known (said so, ``counts_cache``; nothing cached; or a prompt smaller than the cache), else 0.""" + if counts_cache or read + write == 0: + return prompt, output, read, write, 1 + if prompt < read + write: + return prompt + read + write, output, read, write, 1 + return prompt, output, read, write, 0 + + +def _summed(calls: list[tuple[int, ...]], tool_calls: int, details: dict[str, Any]) -> Usage: + """The calls' counts summed: none at all when no call reported usage, the flag unknown when a call's can't be.""" + if not calls: + return Usage(tool_call_count=tool_calls, provider_details=details) + input_tokens, output_tokens, read, write, settled = (sum(column) for column in zip(*calls)) + return Usage(tool_call_count=tool_calls, input_tokens=input_tokens, output_tokens=output_tokens, + total_tokens=input_tokens + output_tokens, cache_read_tokens=read, cache_write_tokens=write, + input_includes_cache=True if settled == len(calls) else None, provider_details=details) + + @dataclass(slots=True) class _ConversationLock: lock: asyncio.Lock = field(default_factory=asyncio.Lock) diff --git a/src/agent_env/agents/agents/pydantic-ai/pydantic_ai_runtime.py b/src/agent_env/agents/agents/pydantic-ai/pydantic_ai_runtime.py index d96e065f7..0f256a89d 100644 --- a/src/agent_env/agents/agents/pydantic-ai/pydantic_ai_runtime.py +++ b/src/agent_env/agents/agents/pydantic-ai/pydantic_ai_runtime.py @@ -11,7 +11,7 @@ from pydantic_ai.capabilities import LocalWorkspace from pydantic_ai.exceptions import ModelAPIError, ModelHTTPError, UnexpectedModelBehavior, UsageLimitExceeded from pydantic_ai.mcp import MCPToolset -from pydantic_ai.messages import BinaryContent, ModelMessage, ModelMessagesTypeAdapter, UserContent +from pydantic_ai.messages import BinaryContent, ModelMessage, ModelMessagesTypeAdapter, ModelResponse, UserContent from pydantic_ai.models.openai import OpenAIChatModel from pydantic_ai.providers.openai import OpenAIProvider from pydantic_ai.providers.vllm import VLLMProvider @@ -20,7 +20,7 @@ from pydantic_ai_harness import Skills from pydantic_ai_harness.coder import Coder -from pydantic_ai_agent import MESSAGES, Outcome, Turn, to_completion +from pydantic_ai_agent import MESSAGES, Outcome, Turn, to_completion, turn_usage # Consecutive retries a tool gets for arguments it refuses, as many as Coder gives its file tools, so a model that # fumbles a command twice doesn't end the task. @@ -85,18 +85,10 @@ def _events(messages: Sequence[ModelMessage]) -> list[dict[str, Any]]: return ModelMessagesTypeAdapter.dump_python(list(messages), mode="json") -def _usage(usage: RunUsage) -> Usage: - return Usage( - tool_call_count=usage.tool_calls, - input_tokens=usage.input_tokens, - output_tokens=usage.output_tokens, - total_tokens=usage.input_tokens + usage.output_tokens, - provider_details={ - "requests": usage.requests, - "cache_read_tokens": usage.cache_read_tokens, - "cache_write_tokens": usage.cache_write_tokens, - }, - ) +def _usage(messages: Sequence[ModelMessage], usage: RunUsage) -> Usage: + """The turn's usage from each of its model responses', and its tool calls and requests from the run's.""" + return turn_usage((message.usage for message in messages if isinstance(message, ModelResponse)), + tool_calls=usage.tool_calls, requests=usage.requests) def _failure(exc: Exception, usage: Usage) -> Outcome: @@ -145,7 +137,8 @@ async def _run(self, turn: Turn) -> Outcome: for image in turn.images: prompt.append(BinaryContent(data=await asyncio.to_thread(image.path.read_bytes), media_type=image.media_type)) agent = _agent(turn) - recorded = len(history) + started = recorded = len(history) + messages: list[ModelMessage] = list(history) usage = RunUsage() try: async with agent.iter(prompt, message_history=history, @@ -158,13 +151,14 @@ async def _run(self, turn: Turn) -> Outcome: usage = run.usage result = run.result except asyncio.CancelledError: - turn.on_usage(_usage(usage)) + turn.on_usage(_usage(messages[started:], usage)) raise except Exception as exc: # every failure is the task's, typed by whose it was - return _failure(exc, _usage(usage)) + return _failure(exc, _usage(messages[started:], usage)) messages = result.all_messages() for event in _events(messages[recorded:]): turn.on_event(event) # A turn whose run finished is saved whole even if it is cancelled meanwhile, before its locks are let go. await to_completion(_save, turn, messages) - return Outcome(result.output.strip() or "The agent finished without a reply.", _usage(result.usage)) + return Outcome(result.output.strip() or "The agent finished without a reply.", + _usage(messages[started:], result.usage)) diff --git a/src/agent_env/agents/agents/pydantic-ai/requirements.txt b/src/agent_env/agents/agents/pydantic-ai/requirements.txt index 9d88ef579..1f44eca03 100644 --- a/src/agent_env/agents/agents/pydantic-ai/requirements.txt +++ b/src/agent_env/agents/agents/pydantic-ai/requirements.txt @@ -1,6 +1,6 @@ # The image installs the newest agentenv-framework-protocol at or above this floor; a release doesn't pin it, so the -# image is built again only when this folder changes. The floor is the first that takes the native trajectory format -# the agent declares (enable(TRAJECTORY_V1, native=...)); raise it when the agent needs a newer protocol. -agentenv-framework-protocol[agent]>=0.1.588 +# image is built again only when this folder changes. The floor is the first whose Usage takes the prompt cache's +# counts (cache_read_tokens, cache_write_tokens, input_includes_cache); raise it when the agent needs a newer protocol. +agentenv-framework-protocol[agent]>=0.1.597 pydantic-ai-slim[mcp,openai]==2.61.0 pydantic-ai-harness[coder,skills]==0.61.0 diff --git a/tst/integration/cli/run_bundle_openhands_test.py b/tst/integration/cli/run_bundle_openhands_test.py index e1e260e49..6957cf1f1 100644 --- a/tst/integration/cli/run_bundle_openhands_test.py +++ b/tst/integration/cli/run_bundle_openhands_test.py @@ -8,12 +8,15 @@ import subprocess import time import uuid +from collections import deque from typing import Final import httpx import pytest from click.testing import CliRunner +from agentenv_protocol.a2a_agent import Usage + from agent_env.a2a_agent import protocol from agent_env.artifact.store import reset_artifact_store from agent_env.cli import cli @@ -32,6 +35,16 @@ AGENT: Final[str] = "@local/agentenv-framework/agents/openhands" MODEL: Final[str] = "fake-model-1" ECHO_ENV: Final[str] = "@local/agentenv-framework/echo-mcp/echo" +# A tool call and an answer from Claude through a LiteLLM proxy, whose prompt leaves out the cache it wrote and read, +# and an answer from GPT through its Chat Completions, whose prompt counts the cache. +CLAUDE_CALLS: Final[list[dict]] = [ + {"prompt_tokens": 10, "completion_tokens": 129, "total_tokens": 139, "cache_creation_input_tokens": 10363, + "cache_read_input_tokens": 0}, + {"prompt_tokens": 13, "completion_tokens": 43, "total_tokens": 56, "cache_creation_input_tokens": 199, + "cache_read_input_tokens": 10363}, +] +GPT_CALL: Final[dict] = {"prompt_tokens": 2303, "completion_tokens": 2, "total_tokens": 2305, + "prompt_tokens_details": {"cached_tokens": 2048}} @pytest.fixture @@ -224,6 +237,40 @@ async def steps(context): assert second["text"].startswith("OpenHands could not start the turn: "), outcomes +def test_a_turns_usage_counts_the_cache_in_its_input_and_is_unknown_where_no_usage_came_back(state, endpoint): + outcomes = {} + endpoint.respond = answer("pong") + + async def steps(context): + await DeployAgentTaskStep(id="agent", version=None, sandbox_type="local", a2a_agent_id=AGENT).execute(context) + (agent,) = context.deployed_agents + base_url = agent.a2a_url or agent.api_url + await protocol.post_agent_config(f"{base_url}/ext/agent-config", {"model": MODEL}, headers=agent.control_headers) + claude = deque(CLAUDE_CALLS) + endpoint.respond = tool_then_answer("terminal", {"command": "echo hi", "security_risk": "LOW"}, + prefix="the shell said: ") + endpoint.usage = lambda body: claude.popleft() if claude else None + outcomes["claude"] = await _send(base_url, "ctx", "Say hi with the shell.") + endpoint.respond, endpoint.usage = answer("pong"), lambda body: GPT_CALL + outcomes["gpt"] = await _send(base_url, "ctx", "Say pong.") + endpoint.usage = lambda body: None + outcomes["none"] = await _send(base_url, "ctx", "Say pong again.") + + _with_agent(state, steps) + + assert {outcome["state"] for outcome in outcomes.values()} == {"completed"}, outcomes + usage = {name: Usage.from_dict(outcome["data"]["usage"]) for name, outcome in outcomes.items()} + # GPT's prompt is as large as its cache, so whether it counts it is left unknown. + assert usage == { + "claude": Usage(tool_call_count=1, input_tokens=20948, output_tokens=172, total_tokens=21120, + cache_read_tokens=10363, cache_write_tokens=10562, input_includes_cache=True, + provider_details={"reasoning_tokens": 0}), + "gpt": Usage(tool_call_count=0, input_tokens=2303, output_tokens=2, total_tokens=2305, cache_read_tokens=2048, + cache_write_tokens=0, input_includes_cache=None, provider_details={"reasoning_tokens": 0}), + "none": Usage(tool_call_count=0, provider_details={"reasoning_tokens": 0, "unreported_calls": 1}), + }, outcomes + + def test_a_model_endpoint_that_refuses_the_key_fails_the_task_as_an_infra_error(state, endpoint): endpoint.respond = answer("pong") diff --git a/tst/integration/cli/run_bundle_pydantic_ai_test.py b/tst/integration/cli/run_bundle_pydantic_ai_test.py index d57c62aa2..3829751e1 100644 --- a/tst/integration/cli/run_bundle_pydantic_ai_test.py +++ b/tst/integration/cli/run_bundle_pydantic_ai_test.py @@ -7,12 +7,15 @@ import logging import time import uuid +from collections import deque from typing import Final import httpx import pytest from click.testing import CliRunner +from agentenv_protocol.a2a_agent import Usage + from agent_env.a2a_agent import protocol from agent_env.artifact.store import reset_artifact_store from agent_env.cli import cli @@ -31,6 +34,16 @@ AGENT: Final[str] = "@local/agentenv-framework/agents/pydantic-ai" MODEL: Final[str] = "fake-model-1" ECHO_ENV: Final[str] = "@local/agentenv-framework/echo-mcp/echo" +# A tool call and an answer from Claude through a LiteLLM proxy, whose prompt leaves out the cache it wrote and read, +# and an answer from GPT through it, whose prompt counts the cache. +CLAUDE_CALLS: Final[list[dict]] = [ + {"prompt_tokens": 10, "completion_tokens": 129, "total_tokens": 139, "cache_creation_input_tokens": 10363, + "cache_read_input_tokens": 0}, + {"prompt_tokens": 13, "completion_tokens": 43, "total_tokens": 56, "cache_creation_input_tokens": 199, + "cache_read_input_tokens": 10363}, +] +GPT_CALL: Final[dict] = {"prompt_tokens": 2303, "completion_tokens": 2, "total_tokens": 2305, + "prompt_tokens_details": {"cached_tokens": 2048}} @pytest.fixture @@ -202,6 +215,35 @@ async def steps(context): assert {call.headers.get("x-agentenv-test") for call in endpoint.completions()} == {"header-check"} +def test_a_turns_usage_counts_the_cache_in_its_input_and_is_unknown_where_no_usage_came_back(state, endpoint): + outcomes = {} + endpoint.respond = answer("pong") + + async def steps(context): + base_url, _ = await _deployed(context) + claude = deque(CLAUDE_CALLS) + endpoint.respond = tool_then_answer("shell", {"command": "echo hi"}, prefix="the shell said: ") + endpoint.usage = lambda body: claude.popleft() if claude else None + outcomes["claude"] = await _send(base_url, "ctx", "Say hi with the shell.") + endpoint.respond, endpoint.usage = answer("pong"), lambda body: GPT_CALL + outcomes["gpt"] = await _send(base_url, "ctx", "Say pong.") + endpoint.usage = lambda body: None + outcomes["none"] = await _send(base_url, "ctx", "Say pong again.") + + _with_agent(state, steps) + + assert {outcome["state"] for outcome in outcomes.values()} == {"completed"}, outcomes + usage = {name: Usage.from_dict(outcome["data"]["usage"]) for name, outcome in outcomes.items()} + assert usage == { + "claude": Usage(tool_call_count=1, input_tokens=20948, output_tokens=172, total_tokens=21120, + cache_read_tokens=10363, cache_write_tokens=10562, input_includes_cache=True, + provider_details={"requests": 2}), + "gpt": Usage(tool_call_count=0, input_tokens=2303, output_tokens=2, total_tokens=2305, cache_read_tokens=2048, + cache_write_tokens=0, input_includes_cache=True, provider_details={"requests": 1}), + "none": Usage(tool_call_count=0, provider_details={"requests": 1, "unreported_calls": 1}), + }, outcomes + + def test_a_model_endpoint_that_refuses_the_key_fails_the_task_as_an_infra_error(state, endpoint): endpoint.respond = answer("pong") diff --git a/tst/unit/agents/browser_use/test_browser_use_agent.py b/tst/unit/agents/browser_use/test_browser_use_agent.py index 881f277b5..848f2df85 100644 --- a/tst/unit/agents/browser_use/test_browser_use_agent.py +++ b/tst/unit/agents/browser_use/test_browser_use_agent.py @@ -58,7 +58,8 @@ def test_a_turn_returns_the_answer_and_a_context_keeps_one_conversation(client: assert (task["status"]["state"], text) == ("completed", "The top story.") usage = data["usage"] assert (usage["tool_call_count"], usage["input_tokens"], usage["output_tokens"], usage["total_tokens"]) == (2, 100, 20, 120) - assert usage["provider_details"]["cache_read_tokens"] == 60 and usage["provider_details"]["requests"] == 3 + assert (usage["cache_read_tokens"], usage["cache_write_tokens"], usage["input_includes_cache"]) == (60, 10, True) + assert usage["provider_details"]["requests"] == 3 and "cost_usd" not in usage turn = json.loads(record.read_text()) assert (turn["prompt"], turn["model"], turn["effort"], turn["systemPrompt"]) == ("top story?", "gpt-test", "low", "Be brief.") # Unset, browser-use's own step budget and mode; its run ends before the turn's deadline, leaving it to deliver. diff --git a/tst/unit/agents/hermes/test_message_contract.py b/tst/unit/agents/hermes/test_message_contract.py index beb946a3b..1b5986225 100644 --- a/tst/unit/agents/hermes/test_message_contract.py +++ b/tst/unit/agents/hermes/test_message_contract.py @@ -240,3 +240,25 @@ async def download(source, destination): assert not (target / "config.yaml").exists() assert (folders / "result.txt").read_text() == "42" assert turns.calls[-1]["session_id"] == "s-ctx-src" + + +def test_a_turns_usage_adds_the_cache_back_to_the_input(): + """GPT's two calls of a smoke run through a LiteLLM proxy, as Hermes reported them: its input leaves out the 10,496 + cached tokens the second call read.""" + report = {"input_tokens": 10750, "output_tokens": 48, "total_tokens": 21294, "cache_read_tokens": 10496, + "cache_write_tokens": 0, "reasoning_tokens": 0, "api_calls": 2, "estimated_cost_usd": 0.0, + "cost_status": "unknown"} + + assert a2a_server._usage(result(usage=report, tool_call_count=1)).to_dict() == { + "tool_call_count": 1, "input_tokens": 21246, "output_tokens": 48, "total_tokens": 21294, + "cache_read_tokens": 10496, "cache_write_tokens": 0, "input_includes_cache": True, "cost_usd": 0.0, + "provider_details": {"api_calls": 2, "reasoning_tokens": 0, "cost_status": "unknown"}} + + +def test_a_turn_whose_calls_reported_no_usage_has_unknown_counts(): + """Claude's streamed answers through a LiteLLM proxy carry no usage, so Hermes reports zeros for them.""" + report = {"input_tokens": 0, "output_tokens": 0, "total_tokens": 0, "cache_read_tokens": 0, + "cache_write_tokens": 0, "api_calls": 3} + + assert a2a_server._usage(result(usage=report, tool_call_count=2)).to_dict() == { + "tool_call_count": 2, "provider_details": {"api_calls": 3, "unreported_calls": 3}} diff --git a/tst/unit/agents/openhands/test_openhands_agent.py b/tst/unit/agents/openhands/test_openhands_agent.py index aa911af7d..a686a44a0 100644 --- a/tst/unit/agents/openhands/test_openhands_agent.py +++ b/tst/unit/agents/openhands/test_openhands_agent.py @@ -9,6 +9,7 @@ from collections import Counter from dataclasses import dataclass, field from pathlib import Path +from types import SimpleNamespace from typing import Any, Final import httpx @@ -676,3 +677,37 @@ def test_the_card_version_is_the_pinned_sdk() -> None: assert f"openhands-sdk=={openhands_agent.OPENHANDS_VERSION}" in requirements assert f"openhands-tools=={openhands_agent.OPENHANDS_VERSION}" in requirements + + +def _call(prompt: int, completion: int, read: int = 0, write: int = 0) -> SimpleNamespace: + """The ``TokenUsage`` OpenHands keeps for a model call.""" + return SimpleNamespace(prompt_tokens=prompt, completion_tokens=completion, cache_read_tokens=read, + cache_write_tokens=write, reasoning_tokens=0) + + +def test_a_turns_usage_counts_the_cache_in_its_input() -> None: + """A smoke run's two calls through a LiteLLM proxy: Claude's through Chat Completions, whose prompt leaves the cache + out, and GPT's through the Responses API, whose input counts it.""" + claude = openhands_agent.turn_usage([(_call(10, 129, write=10363), False), (_call(13, 43, 10363, 199), False)], + responses=2, tool_calls=1, cost=0.0151) + gpt = openhands_agent.turn_usage([(_call(7803, 22), True), (_call(7878, 3, read=7680), True)], responses=2, + tool_calls=1, cost=0.004) + + assert claude == Usage(tool_call_count=1, input_tokens=20948, output_tokens=172, total_tokens=21120, + cache_read_tokens=10363, cache_write_tokens=10562, input_includes_cache=True, cost_usd=0.0151, + provider_details={"reasoning_tokens": 0}) + assert gpt == Usage(tool_call_count=1, input_tokens=15681, output_tokens=25, total_tokens=15706, + cache_read_tokens=7680, cache_write_tokens=0, input_includes_cache=True, cost_usd=0.004, + provider_details={"reasoning_tokens": 0}) + + +def test_a_turns_usage_is_unknown_where_no_usage_came_back() -> None: + """A response that carried no usage, which OpenHands keeps no ``TokenUsage`` for, is counted apart, and a Chat + Completions prompt as large as its cache may or may not have counted it.""" + unreported = openhands_agent.turn_usage([], responses=1, tool_calls=0, cost=None) + ambiguous = openhands_agent.turn_usage([(_call(5000, 10, read=4000), False)], responses=2, tool_calls=2, cost=None) + + assert unreported == Usage(tool_call_count=0, provider_details={"reasoning_tokens": 0, "unreported_calls": 1}) + assert ambiguous == Usage(tool_call_count=2, input_tokens=5000, output_tokens=10, total_tokens=5010, + cache_read_tokens=4000, cache_write_tokens=0, input_includes_cache=None, + provider_details={"reasoning_tokens": 0, "unreported_calls": 1}) diff --git a/tst/unit/agents/pi/test_pi_agent.py b/tst/unit/agents/pi/test_pi_agent.py index 641c5bd3a..2ae8b6820 100644 --- a/tst/unit/agents/pi/test_pi_agent.py +++ b/tst/unit/agents/pi/test_pi_agent.py @@ -69,8 +69,8 @@ def test_run_returns_final_text_usage_and_trajectory(client: TestClient, record: assert task["status"]["state"] == "completed" assert text == "Done." assert data["usage"] == { - "tool_call_count": 1, "input_tokens": 200, "output_tokens": 40, "total_tokens": 252, "cost_usd": 0.5, - "provider_details": {"cache_read_tokens": 10, "cache_write_tokens": 2}, + "tool_call_count": 1, "input_tokens": 212, "output_tokens": 40, "total_tokens": 252, "cache_read_tokens": 10, + "cache_write_tokens": 2, "input_includes_cache": True, "cost_usd": 0.5, } invocation = json.loads(record.read_text()) assert invocation["stdin"] == 'fix the bug\n\n{\n "k": 1\n}' @@ -92,6 +92,34 @@ def test_run_returns_final_text_usage_and_trajectory(client: TestClient, record: assert "message_update" not in kinds +def _message(input: int, output: int, read: int, total: int) -> dict[str, Any]: + return {"role": "assistant", "usage": {"input": input, "output": output, "cacheRead": read, "cacheWrite": 0, + "totalTokens": total, "cost": {"total": 0}}} + + +def test_usage_adds_the_cache_back_to_the_input() -> None: + """GPT's two calls of a smoke run through a LiteLLM proxy, as pi recorded them: the second read 1,024 cached tokens + of its 1,139-token prompt, which pi's ``input`` leaves out.""" + run = pi_agent._Run(assistant_messages=[_message(1109, 21, 0, 1130), _message(115, 2, 1024, 1141)], tool_calls=1) + + assert pi_agent._usage(run).to_dict() == { + "tool_call_count": 1, "input_tokens": 2248, "output_tokens": 23, "total_tokens": 2271, + "cache_read_tokens": 1024, "cache_write_tokens": 0, "input_includes_cache": True, + } + + +def test_a_call_that_reported_no_usage_is_unknown_not_zero() -> None: + """Claude's streamed answers through a LiteLLM proxy carry no usage, so pi records zeros for them.""" + unreported = pi_agent._Run(assistant_messages=[_message(0, 0, 0, 0), _message(0, 0, 0, 0)], tool_calls=1) + partly = pi_agent._Run(assistant_messages=[_message(0, 0, 0, 0), _message(115, 2, 1024, 1141)]) + + assert pi_agent._usage(unreported).to_dict() == { + "tool_call_count": 1, "provider_details": {"unreported_calls": 2}} + assert pi_agent._usage(partly).to_dict() == { + "tool_call_count": 0, "input_tokens": 1139, "output_tokens": 2, "total_tokens": 1141, "cache_read_tokens": 1024, + "cache_write_tokens": 0, "input_includes_cache": True, "provider_details": {"unreported_calls": 1}} + + def test_a_context_keeps_one_pi_session(client: TestClient, record: Path) -> None: _send(client, [{"kind": "text", "text": "one"}], context_id="ctx/a b") first = json.loads(record.read_text())["argv"] diff --git a/tst/unit/agents/pydantic_ai/test_pydantic_ai_agent.py b/tst/unit/agents/pydantic_ai/test_pydantic_ai_agent.py index 39313b428..5d90797e6 100644 --- a/tst/unit/agents/pydantic_ai/test_pydantic_ai_agent.py +++ b/tst/unit/agents/pydantic_ai/test_pydantic_ai_agent.py @@ -13,6 +13,7 @@ from collections import Counter from dataclasses import dataclass, field from pathlib import Path +from types import SimpleNamespace from typing import Any, Final import httpx @@ -697,3 +698,38 @@ async def cancel_midway() -> bool: assert asyncio.run(cancel_midway()) assert finished.is_set() + + +def _response(input: int, output: int, read: int = 0, write: int = 0, **details: int) -> SimpleNamespace: + """A ``ModelResponse``'s ``RequestUsage``: its cache counts, and the usage keys it doesn't read in ``details``.""" + return SimpleNamespace(input_tokens=input, output_tokens=output, cache_read_tokens=read, cache_write_tokens=write, + details=details) + + +def test_a_turns_usage_counts_the_cache_in_its_input() -> None: + """GPT's two calls of a smoke run through a LiteLLM proxy, whose prompt counts the cache, and Claude's, whose prompt + a LiteLLM proxy sends without it, with Anthropic's keys beside it, which Pydantic AI keeps in ``details``.""" + gpt = pydantic_ai_agent.turn_usage([_response(2169, 20), _response(2303, 2, read=2048)], tool_calls=1, requests=2) + claude = pydantic_ai_agent.turn_usage( + [_response(10, 129, cache_creation_input_tokens=10363, cache_read_input_tokens=0), + _response(13, 43, cache_creation_input_tokens=199, cache_read_input_tokens=10363)], tool_calls=1, requests=2) + + assert gpt == Usage(tool_call_count=1, input_tokens=4472, output_tokens=22, total_tokens=4494, + cache_read_tokens=2048, cache_write_tokens=0, input_includes_cache=True, + provider_details={"requests": 2}) + assert claude == Usage(tool_call_count=1, input_tokens=20948, output_tokens=172, total_tokens=21120, + cache_read_tokens=10363, cache_write_tokens=10562, input_includes_cache=True, + provider_details={"requests": 2}) + + +def test_a_turns_usage_is_unknown_where_no_usage_came_back() -> None: + """Claude's streamed answers through a LiteLLM proxy carry no usage, which Pydantic AI records as zeros; and a + prompt as large as the cache beside it may or may not have counted it.""" + unreported = pydantic_ai_agent.turn_usage([_response(0, 0), _response(0, 0)], tool_calls=1, requests=2) + ambiguous = pydantic_ai_agent.turn_usage( + [_response(0, 0), _response(5000, 10, cache_read_input_tokens=4000)], tool_calls=0, requests=2) + + assert unreported == Usage(tool_call_count=1, provider_details={"requests": 2, "unreported_calls": 2}) + assert ambiguous == Usage(tool_call_count=0, input_tokens=5000, output_tokens=10, total_tokens=5010, + cache_read_tokens=4000, cache_write_tokens=0, input_includes_cache=None, + provider_details={"requests": 2, "unreported_calls": 1}) diff --git a/tst/util/fake_openai.py b/tst/util/fake_openai.py index 4f47fbc6a..00b012ac1 100644 --- a/tst/util/fake_openai.py +++ b/tst/util/fake_openai.py @@ -1,7 +1,8 @@ """A fake OpenAI Chat Completions endpoint, so an agent that calls the model through LiteLLM runs with no model key: ``POST .../chat/completions`` is answered by a responder, as one JSON completion or, to a request that asks for a -stream, as server-sent chunks, and every request is recorded. A fault queued with ``fail_next`` replaces the next -answer: an HTTP error, or an answer that stalls.""" +stream, as server-sent chunks, and every request is recorded. Each answer carries the usage ``usage`` gives for its +request, none when it gives None. A fault queued with ``fail_next`` replaces the next answer: an HTTP error, or an +answer that stalls.""" import json import threading @@ -81,6 +82,7 @@ def tool_result_text(body: dict) -> str | None: class FakeOpenAI: host: str = "127.0.0.1" respond: Responder = field(default_factory=lambda: answer("ok")) + usage: Callable[[dict], dict | None] = lambda body: USAGE seen: list[Seen] = field(default_factory=list) _faults: deque = field(default_factory=deque) _server: ThreadingHTTPServer | None = None @@ -118,7 +120,7 @@ def do_POST(self) -> None: return if fault is not None and fault.kind == "stall": time.sleep(fault.seconds) - completion = _completion(body, fake.respond(body)) + completion = _completion(body, fake.respond(body), fake.usage(body)) if body.get("stream"): self._stream(completion) else: @@ -157,7 +159,8 @@ def __exit__(self, *exc) -> None: def _chunks(completion: dict) -> list[dict]: - """``completion`` as the chunks a stream sends: its message whole as one delta, its finish, then its usage.""" + """``completion`` as the chunks a stream sends: its message whole as one delta, its finish, then its usage if it + has one.""" (choice,) = completion["choices"] message = choice["message"] delta = {"role": "assistant", "content": message["content"]} @@ -167,11 +170,11 @@ def _chunks(completion: dict) -> list[dict]: return [ {**head, "choices": [{"index": 0, "delta": delta, "finish_reason": None}]}, {**head, "choices": [{"index": 0, "delta": {}, "finish_reason": choice["finish_reason"]}]}, - {**head, "choices": [], "usage": completion["usage"]}, + *([{**head, "choices": [], "usage": completion["usage"]}] if "usage" in completion else []), ] -def _completion(body: dict, reply: Text | ToolCall) -> dict: +def _completion(body: dict, reply: Text | ToolCall, usage: dict | None) -> dict: if isinstance(reply, Text): message, finish = {"role": "assistant", "content": reply.text}, "stop" else: @@ -181,4 +184,4 @@ def _completion(body: dict, reply: Text | ToolCall) -> dict: finish = "tool_calls" return {"id": f"chatcmpl-{uuid.uuid4().hex[:12]}", "object": "chat.completion", "created": int(time.time()), "model": body.get("model", ""), "choices": [{"index": 0, "message": message, "finish_reason": finish}], - "usage": USAGE} + **({"usage": usage} if usage is not None else {})} From ed3094ab37cabc34520063816b5d00642464824e Mon Sep 17 00:00:00 2001 From: Edgar Arakelyan Date: Thu, 19 Nov 2026 08:26:24 -0800 Subject: [PATCH 2/3] fix(agents): pi and Hermes check their cache rule against the reported total pi's and Hermes' input leaves the prompt cache out and their total counts it, so the adapters add the cache back; that rested on each runtime's current counting, and pi's library floats within 1.x at each image build. Each now checks it against the total the runtime reported: a total of the input, output and cache confirms the rule, one of the input and output alone says the input already holds the cache, and one that is neither keeps the rule with input_includes_cache unknown. The agents guide says how a call without usage is reported. Co-Authored-By: Claude Opus 5.5 --- .../agents/agents/hermes/a2a_server.py | 24 ++++++++--- src/agent_env/agents/agents/pi/pi_agent.py | 43 +++++++++++-------- .../agents/hermes/test_message_contract.py | 16 +++++++ tst/unit/agents/pi/test_pi_agent.py | 14 ++++++ 4 files changed, 72 insertions(+), 25 deletions(-) diff --git a/src/agent_env/agents/agents/hermes/a2a_server.py b/src/agent_env/agents/agents/hermes/a2a_server.py index 4f994a7d7..e250f708d 100644 --- a/src/agent_env/agents/agents/hermes/a2a_server.py +++ b/src/agent_env/agents/agents/hermes/a2a_server.py @@ -520,9 +520,11 @@ async def swap() -> None: def _usage(result) -> Usage: """The turn's tool calls and the token counts and cost Hermes reported for its main model calls, with - ``input_tokens`` counting the cache: Hermes' ``input_tokens`` is the prompt less its cache reads and writes, so - they're added back. Calls that reported no counts at all got none from their endpoint, as streamed Claude answers - through a LiteLLM proxy, so the turn's counts are unknown rather than nothing.""" + ``input_tokens`` counting the cache. Hermes' ``input_tokens`` is the prompt less its cache reads and writes, and its + total counts them, so they're added back; a total of the input and output alone says the input already holds them, + and a total that says neither keeps Hermes' rule with the flag unknown. Calls that reported no counts at all got + none from their endpoint, as streamed Claude answers through a LiteLLM proxy, so the turn's counts are unknown + rather than nothing.""" report = result.usage or {} def count(name: str) -> int | None: @@ -538,15 +540,23 @@ def count(name: str) -> int | None: return Usage(tool_call_count=result.tool_call_count, provider_details={**details, "unreported_calls": details["api_calls"]}) read, write = counts["cache_read_tokens"], counts["cache_write_tokens"] - input_tokens = counts["input_tokens"] + input_tokens, output, total = counts["input_tokens"], counts["output_tokens"] or 0, counts["total_tokens"] + cached = (read or 0) + (write or 0) + includes_cache = None + if input_tokens is not None: + if cached == 0 or total == input_tokens + output: + includes_cache = True + else: + includes_cache = True if total == input_tokens + output + cached else None + input_tokens += cached return Usage( tool_call_count=result.tool_call_count, - input_tokens=None if input_tokens is None else input_tokens + (read or 0) + (write or 0), + input_tokens=input_tokens, output_tokens=counts["output_tokens"], - total_tokens=counts["total_tokens"], + total_tokens=total, cache_read_tokens=read, cache_write_tokens=write, - input_includes_cache=None if input_tokens is None else True, + input_includes_cache=includes_cache, cost_usd=cost if isinstance(cost, (int, float)) and not isinstance(cost, bool) and cost >= 0 else None, provider_details=details, ) diff --git a/src/agent_env/agents/agents/pi/pi_agent.py b/src/agent_env/agents/agents/pi/pi_agent.py index cf11b2ede..158312c5d 100644 --- a/src/agent_env/agents/agents/pi/pi_agent.py +++ b/src/agent_env/agents/agents/pi/pi_agent.py @@ -255,38 +255,45 @@ def _record(run: _Run, event: dict[str, Any]) -> None: def _usage(run: _Run) -> Usage: - """The turn's usage, summed over its assistant messages, with ``input_tokens`` counting the cache: pi's ``input`` - is the prompt less its cache reads and writes, so they're added back. A message whose usage is all zero got none - from its endpoint, as a streamed Claude answer through a LiteLLM proxy, and is counted in ``unreported_calls`` - instead of as nothing used.""" - totals = {"input": 0, "output": 0, "totalTokens": 0, "cacheRead": 0, "cacheWrite": 0} - cost, reported, unreported = 0.0, 0, 0 + """The turn's usage, summed over its assistant messages, with ``input_tokens`` counting the cache. A message whose + usage is all zero got none from its endpoint, as a streamed Claude answer through a LiteLLM proxy, and is counted + in ``unreported_calls`` instead of as nothing used.""" + counted, cost, unreported = [], 0.0, 0 for message in run.assistant_messages: usage = message.get("usage") or {} - counts = {key: int(usage.get(key) or 0) for key in totals} - if not any(counts.values()): + counts = [int(usage.get(key) or 0) for key in ("input", "output", "cacheRead", "cacheWrite", "totalTokens")] + if not any(counts): unreported += 1 continue - reported += 1 - for key in totals: - totals[key] += counts[key] + counted.append(_counted(*counts)) cost += float((usage.get("cost") or {}).get("total") or 0) details = {"unreported_calls": unreported} if unreported else {} - if not reported: + if not counted: return Usage(tool_call_count=run.tool_calls, provider_details=details) + input_tokens, output_tokens, read, write, total, settled = (sum(column) for column in zip(*counted)) return Usage( tool_call_count=run.tool_calls, - input_tokens=totals["input"] + totals["cacheRead"] + totals["cacheWrite"], - output_tokens=totals["output"], - total_tokens=totals["totalTokens"], - cache_read_tokens=totals["cacheRead"], - cache_write_tokens=totals["cacheWrite"], - input_includes_cache=True, + input_tokens=input_tokens, + output_tokens=output_tokens, + total_tokens=total, + cache_read_tokens=read, + cache_write_tokens=write, + input_includes_cache=True if settled == len(counted) else None, cost_usd=cost or None, provider_details=details, ) +def _counted(input: int, output: int, read: int, write: int, total: int) -> tuple[int, ...]: + """A message's input counting its cache, its output, cache read, cache write and total, and 1 when its total + confirms which its input was, else 0. pi's ``input`` is the prompt less its cache reads and writes, and its total + counts them, so they're added back; a total of the input and output alone says the input already holds them. A + total that says neither keeps pi's rule, unconfirmed.""" + if read + write == 0 or total == input + output: + return input, output, read, write, total, 1 + return input + read + write, output, read, write, total, int(total == input + output + read + write) + + def _text(message: Mapping[str, Any]) -> str: content = message.get("content") if isinstance(content, str): diff --git a/tst/unit/agents/hermes/test_message_contract.py b/tst/unit/agents/hermes/test_message_contract.py index 1b5986225..5501d0963 100644 --- a/tst/unit/agents/hermes/test_message_contract.py +++ b/tst/unit/agents/hermes/test_message_contract.py @@ -262,3 +262,19 @@ def test_a_turn_whose_calls_reported_no_usage_has_unknown_counts(): assert a2a_server._usage(result(usage=report, tool_call_count=2)).to_dict() == { "tool_call_count": 2, "provider_details": {"api_calls": 3, "unreported_calls": 3}} + + +def test_a_turns_usage_follows_what_its_total_says_its_input_holds(): + """A total of the input and output alone says Hermes counted the cache in the input already; one that is neither + sum keeps Hermes' rule, unconfirmed.""" + held = {"input_tokens": 21246, "output_tokens": 48, "total_tokens": 21294, "cache_read_tokens": 10496, + "cache_write_tokens": 0, "api_calls": 2} + neither = {**held, "input_tokens": 10750, "total_tokens": 30000} + + assert a2a_server._usage(result(usage=held, tool_call_count=1)).to_dict() == { + "tool_call_count": 1, "input_tokens": 21246, "output_tokens": 48, "total_tokens": 21294, + "cache_read_tokens": 10496, "cache_write_tokens": 0, "input_includes_cache": True, + "provider_details": {"api_calls": 2}} + assert a2a_server._usage(result(usage=neither, tool_call_count=1)).to_dict() == { + "tool_call_count": 1, "input_tokens": 21246, "output_tokens": 48, "total_tokens": 30000, + "cache_read_tokens": 10496, "cache_write_tokens": 0, "provider_details": {"api_calls": 2}} diff --git a/tst/unit/agents/pi/test_pi_agent.py b/tst/unit/agents/pi/test_pi_agent.py index 2ae8b6820..21c92389d 100644 --- a/tst/unit/agents/pi/test_pi_agent.py +++ b/tst/unit/agents/pi/test_pi_agent.py @@ -108,6 +108,20 @@ def test_usage_adds_the_cache_back_to_the_input() -> None: } +def test_usage_follows_what_each_messages_total_says_its_input_holds() -> None: + """A total of the input and output alone says pi counted the cache in the input already; one that is neither sum + keeps pi's rule, unconfirmed.""" + held = pi_agent._Run(assistant_messages=[_message(1139, 2, 1024, 1141)]) + neither = pi_agent._Run(assistant_messages=[_message(1109, 21, 0, 1130), _message(115, 2, 1024, 0)]) + + assert pi_agent._usage(held).to_dict() == { + "tool_call_count": 0, "input_tokens": 1139, "output_tokens": 2, "total_tokens": 1141, + "cache_read_tokens": 1024, "cache_write_tokens": 0, "input_includes_cache": True} + assert pi_agent._usage(neither).to_dict() == { + "tool_call_count": 0, "input_tokens": 2248, "output_tokens": 23, "total_tokens": 1130, + "cache_read_tokens": 1024, "cache_write_tokens": 0} + + def test_a_call_that_reported_no_usage_is_unknown_not_zero() -> None: """Claude's streamed answers through a LiteLLM proxy carry no usage, so pi records zeros for them.""" unreported = pi_agent._Run(assistant_messages=[_message(0, 0, 0, 0), _message(0, 0, 0, 0)], tool_calls=1) From 993d26f0770d5f1a8d30dfa4f20d8cce96217930 Mon Sep 17 00:00:00 2001 From: Edgar Arakelyan Date: Thu, 19 Nov 2026 08:26:39 -0800 Subject: [PATCH 3/3] docs(agents): the agents guide says how a call without usage is reported Co-Authored-By: Claude Opus 5.5 --- src/agent_env/agents/AGENTS.md | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/src/agent_env/agents/AGENTS.md b/src/agent_env/agents/AGENTS.md index a25b5086d..68506afc1 100644 --- a/src/agent_env/agents/AGENTS.md +++ b/src/agent_env/agents/AGENTS.md @@ -55,7 +55,9 @@ terminal state, `completed`, `failed` or `canceled`. Your `run()` is one turn. - **Usage:** when the runtime counts prompt-cache tokens, put them in `Usage`'s `cache_read_tokens` and `cache_write_tokens`, not `provider_details`, and set `input_includes_cache` to whether `input_tokens` counts them (`None` when it can't tell). Set it per call: through a gateway it depends on the model behind it. Where the runtime - hands back a model response's own usage, `Usage.from_model_response` reads it. Code that sets these fields needs a + hands back a model response's own usage, `Usage.from_model_response` reads it. A call that came back with no usage, + as a streamed answer through some gateways does, leaves the counts unset rather than zero and is counted in + `provider_details.unreported_calls`. Code that sets these fields needs a `requirements.txt` protocol floor at the first release that has them. - **Errors:** `agent_error`, the default, means the agent tried and failed; `infra_error` means the run couldn't happen, and agent-env may retry it. An exception that escapes `run()` becomes an opaque `infra_error`, its text