From c48664d72c21000aee1d9075b2e17de6ff8f8dad Mon Sep 17 00:00:00 2001 From: Aymen Hammouda Date: Wed, 8 Jul 2026 19:30:04 +0100 Subject: [PATCH 1/6] agent: register anthropic in the live-provider guard (issue #89 D6b) Registers "anthropic": "ANTHROPIC_API_KEY" in guard.PROVIDER_API_KEY_ENV so the upcoming Claude count-tokens caller (issue #89) can reuse the same BENCHMARK_LIVE_PROVIDERS_ENABLED + per-provider-key guard as the OpenAI/ Google adapters, rather than inventing a second gate. D6b-sanctioned re-point (issue #89 pre-flight amendment, 2026-07-08): this invalidates the merged test asserting "anthropic" is an unknown provider, since it is now registered. Re-points that one assertion at a still- unregistered provider string; nothing else about the test changes. Refs #63. --- benchmarks/adapters/guard.py | 26 ++++++++++++++++++++------ tests/benchmarks/test_adapters.py | 7 ++++++- 2 files changed, 26 insertions(+), 7 deletions(-) diff --git a/benchmarks/adapters/guard.py b/benchmarks/adapters/guard.py index f67b13d..36eecd8 100644 --- a/benchmarks/adapters/guard.py +++ b/benchmarks/adapters/guard.py @@ -6,12 +6,21 @@ ``google_adapter.LiveGoogleAdapter``) must call :func:`require_live_environment` before doing anything else. -This package never makes a live or paid API call, with or without the -guard passing -- the live adapters in this release are stubs that, after -the guard passes, still refuse via ``LiveExecutionNotImplementedError``. -Actually implementing a live call is out of scope for issue #73's mocked -plumbing and is reserved for a maintainer-run, human-supervised phase (see -PLAN.md Amendment 2026-07-08). +The OpenAI/Google adapters in this package never make a live or paid API +call, with or without the guard passing -- they are stubs that, after the +guard passes, still refuse via ``LiveExecutionNotImplementedError``. +Actually implementing a live call for those providers is out of scope for +issue #73's mocked plumbing and is reserved for a maintainer-run, +human-supervised phase (see PLAN.md Amendment 2026-07-08). + +Issue #89 is the one exception: ``benchmarks.adapters.claude_tokens``'s +Anthropic count-tokens caller *does* perform a real HTTP call once this +guard passes, because the maintainer-run live phase is exactly what that +call is for (Claude token counting after client-side rewrap, roadmap +decision 5.8). It still never runs in CI or unit tests -- tests use a fake +counter -- and it is structurally incapable of running without both +``BENCHMARK_LIVE_PROVIDERS_ENABLED`` and ``ANTHROPIC_API_KEY`` set, exactly +like every other provider gated here. """ from __future__ import annotations @@ -25,9 +34,14 @@ LIVE_PROVIDERS_ENABLED_ENV = "BENCHMARK_LIVE_PROVIDERS_ENABLED" #: Per-provider environment variable that must also be non-empty. +#: +#: ``anthropic`` (issue #89) is not a competitor/model-matrix provider -- +#: it gates the Claude count-tokens methodology caller in +#: ``benchmarks.adapters.claude_tokens``, not a competitor answer adapter. PROVIDER_API_KEY_ENV = { "openai": "OPENAI_API_KEY", "google": "GOOGLE_API_KEY", + "anthropic": "ANTHROPIC_API_KEY", } _TRUTHY = {"1", "true", "yes"} diff --git a/tests/benchmarks/test_adapters.py b/tests/benchmarks/test_adapters.py index f54b12a..89fd607 100644 --- a/tests/benchmarks/test_adapters.py +++ b/tests/benchmarks/test_adapters.py @@ -143,8 +143,13 @@ def test_require_live_environment_rejects_unknown_provider( ) -> None: monkeypatch.setenv(LIVE_PROVIDERS_ENABLED_ENV, "1") + # D6b sanction (issue #89 pre-flight amendment, 2026-07-08): registering + # "anthropic" in PROVIDER_API_KEY_ENV for the guarded Claude count-tokens + # caller means "anthropic" is no longer an unknown provider here, so + # this assertion is re-pointed at a still-unregistered provider string. + # This is the sole merged-test change the sanction covers. with pytest.raises(LiveProviderDisabledError, match="unknown provider"): - require_live_environment("anthropic") + require_live_environment("not-a-registered-provider") # --- Live adapter stubs: guard enforced, no network ever possible ----------- From 12a5f3796e5657cee527905aa0b1d3604a867b09 Mon Sep 17 00:00:00 2001 From: Aymen Hammouda Date: Wed, 8 Jul 2026 19:30:19 +0100 Subject: [PATCH 2/6] agent: add guarded Claude count-tokens caller (issue #89) Adds benchmarks/adapters/claude_tokens.py: the Anthropic count-tokens API caller for the "Token Measurement" methodology (roadmap decision 5.8), confined to the maintainer-run live phase. - LiveClaudeTokenCounter calls require_live_environment("anthropic") on every count() call, then POSTs to the count-tokens endpoint via urllib.request only -- no SDK, no new dependency (binding pre-flight amendment, 2026-07-08). - build_client_wrapped_envelope implements methodology steps 1-3: wrap the prompt (and any recorded tool_result payloads) the same way for every adapter, marking approximation=True only when handed a provider-adapter mock/live-stub payload (MockOpenAIAdapter/MockGoogleAdapter and their live-stub counterparts never call a real client SDK, so there is no real wrapped envelope to recover). - count_cell_tokens ties counting + approximation marking + serialization- latency capture (decision 5.8) into one path shared by fake and live counters. - FakeTokenCounter is the only counter every test in this repo uses; CI and unit tests never reach the network (see tests/benchmarks/test_claude_ tokens.py, including guard-refusal tests that assert urlopen is never called). Also fixes a stale docs/benchmarks/model-matrix.yml comment that claimed nothing under benchmarks/adapters/ calls the count-tokens API -- this module now does, strictly behind the guard. Refs #63. --- benchmarks/adapters/__init__.py | 26 ++- benchmarks/adapters/claude_tokens.py | 277 +++++++++++++++++++++++++ docs/benchmarks/model-matrix.yml | 10 +- tests/benchmarks/test_claude_tokens.py | 225 ++++++++++++++++++++ 4 files changed, 532 insertions(+), 6 deletions(-) create mode 100644 benchmarks/adapters/claude_tokens.py create mode 100644 tests/benchmarks/test_claude_tokens.py diff --git a/benchmarks/adapters/__init__.py b/benchmarks/adapters/__init__.py index e090062..7d6b414 100644 --- a/benchmarks/adapters/__init__.py +++ b/benchmarks/adapters/__init__.py @@ -1,10 +1,16 @@ """Provider adapter interfaces for the v0.5.0 public benchmark (issue #73). -Mocked plumbing only: no adapter in this package ever performs network I/O. -See ``benchmarks/adapters/base.py`` for the shared request/response contract, -``benchmarks/adapters/guard.py`` for the live-provider refusal guardrail, and +Mocked plumbing only: no OpenAI/Google adapter in this package ever +performs network I/O. See ``benchmarks/adapters/base.py`` for the shared +request/response contract, ``benchmarks/adapters/guard.py`` for the +live-provider refusal guardrail, and ``benchmarks/adapters/openai_adapter.py`` / ``google_adapter.py`` for the mock and guarded live-stub implementations. + +``benchmarks/adapters/claude_tokens.py`` (issue #89) is the one exception: +its ``LiveClaudeTokenCounter`` does perform a real, guarded HTTP call to the +Anthropic count-tokens API, confined to the maintainer-run live phase (see +that module's docstring). """ from __future__ import annotations @@ -15,6 +21,14 @@ ProviderAdapter, TokenMetadata, ) +from benchmarks.adapters.claude_tokens import ( + FakeTokenCounter, + LiveClaudeTokenCounter, + TokenCounter, + TokenCountResult, + build_client_wrapped_envelope, + count_cell_tokens, +) from benchmarks.adapters.google_adapter import LiveGoogleAdapter, MockGoogleAdapter from benchmarks.adapters.guard import ( LiveExecutionNotImplementedError, @@ -37,4 +51,10 @@ "LiveOpenAIAdapter", "MockGoogleAdapter", "LiveGoogleAdapter", + "TokenCounter", + "FakeTokenCounter", + "LiveClaudeTokenCounter", + "TokenCountResult", + "build_client_wrapped_envelope", + "count_cell_tokens", ] diff --git a/benchmarks/adapters/claude_tokens.py b/benchmarks/adapters/claude_tokens.py new file mode 100644 index 0000000..b49d555 --- /dev/null +++ b/benchmarks/adapters/claude_tokens.py @@ -0,0 +1,277 @@ +"""Claude token counting via the Anthropic count-tokens API (issue #89). + +Implements the "Token Measurement" section of +``docs/benchmarks/PUBLIC-BENCHMARK-METHODOLOGY.md`` (roadmap decision 5.8): +count the client-rewrapped message envelope with Claude token counting, +record raw payload tokens separately as diagnostic data, and mark a result +``approximation: true`` when the exact client-wrapped envelope cannot be +recovered. + +**Binding pre-flight amendment (issue #89, 2026-07-08):** no third-party +dependency may be added for this -- not the ``anthropic`` SDK, not an +optional extra, not a dev/benchmark dependency group. The live HTTP call +below uses only :mod:`urllib.request` (stdlib). If a genuine SDK need is +ever found, that is a stop-and-comment per the pipeline, not a decision this +module makes unilaterally. + +**Live-phase guard:** :class:`LiveClaudeTokenCounter.count` calls +:func:`benchmarks.adapters.guard.require_live_environment` itself, on every +call, regardless of caller -- so this module is structurally incapable of a +network call unless ``BENCHMARK_LIVE_PROVIDERS_ENABLED`` and +``ANTHROPIC_API_KEY`` are both set. CI and unit tests never set those, so +they never reach the network; tests use :class:`FakeTokenCounter` instead. + +**Envelope exactness:** :func:`build_client_wrapped_envelope` returns +``approximation=True`` only when handed a provider-adapter mock/live-stub +payload (``benchmarks.adapters.base.AdapterResponse.raw`` from +``MockOpenAIAdapter`` / ``MockGoogleAdapter`` / their live-stub +counterparts). Those adapters never call a real ``openai-python`` / +``google-genai`` client, so there is no real wrapped request body to +recover -- the methodology requires marking that gap honestly rather than +guessing. Cells produced by adapters that do not go through a provider SDK +at all (the no-MCP baseline, or the offline +``benchmarks.adapters.python_docs_mcp_adapter`` retrieval adapter) hand this +module real prompt/tool-call data directly, so this module's own +fixed wrapping of that data is exact, not an approximation. +""" + +from __future__ import annotations + +import json +import os +import time +import urllib.error +import urllib.request +from dataclasses import dataclass, field +from typing import Any, Protocol + +from benchmarks.adapters.guard import PROVIDER_API_KEY_ENV, require_live_environment +from benchmarks.runner import BenchmarkCellFailure + +#: Anthropic Messages API count-tokens endpoint. Counts a message envelope +#: without generating a response and without being billed -- see +#: https://docs.claude.com/en/api/messages-count-tokens (Anthropic API +#: docs). Called only via ``urllib.request``; no SDK involved. +COUNT_TOKENS_URL = "https://api.anthropic.com/v1/messages/count_tokens" + +#: Required API version header for the Anthropic Messages API. +ANTHROPIC_API_VERSION = "2023-06-01" + +#: Model id used solely to select a tokenizer for the count-tokens call. +#: This never generates text and is never billed as a completion -- it only +#: selects which Claude tokenizer counts the envelope. +DEFAULT_COUNT_MODEL = "claude-3-5-sonnet-20241022" + +#: Per-call HTTP timeout for the live count-tokens request. +REQUEST_TIMEOUT_SECONDS = 30.0 + +#: Provider id registered in ``benchmarks.adapters.guard.PROVIDER_API_KEY_ENV`` +#: for this caller. Not a competitor/model-matrix provider (see that +#: module's docstring). +PROVIDER = "anthropic" + +__all__ = [ + "TokenCounter", + "FakeTokenCounter", + "LiveClaudeTokenCounter", + "TokenCountResult", + "build_client_wrapped_envelope", + "count_cell_tokens", +] + + +class TokenCounter(Protocol): + """Anything that can count tokens for a Claude-format message envelope. + + :func:`count_cell_tokens` depends only on this protocol, so it never + has to know whether it is talking to :class:`FakeTokenCounter` (tests) + or :class:`LiveClaudeTokenCounter` (the guarded maintainer-run phase). + """ + + def count(self, messages: list[dict[str, Any]]) -> int: + """Return the token count for ``messages`` (an Anthropic-shaped envelope).""" + raise NotImplementedError + + +@dataclass +class FakeTokenCounter: + """Deterministic test double for :class:`TokenCounter`. No network I/O. + + Every test in this codebase for Claude token counting must use this + (or an equivalent fake) -- CI and unit tests never call the real + Anthropic API. Defaults to counting whitespace-separated words in the + JSON-serialized envelope, which is deterministic and dependency-free; + ``tokens_per_call`` overrides that with a fixed count when a test wants + an exact, predictable number. + """ + + tokens_per_call: int | None = None + calls: list[list[dict[str, Any]]] = field(default_factory=list) + + def count(self, messages: list[dict[str, Any]]) -> int: + self.calls.append(messages) + if self.tokens_per_call is not None: + return self.tokens_per_call + return len(json.dumps(messages, sort_keys=True).split()) + + +class LiveClaudeTokenCounter: + """Real Anthropic count-tokens API caller. Stdlib HTTP only, no SDK. + + Confined to the maintainer-run live phase (PLAN.md Amendment + 2026-07-08): :meth:`count` calls :func:`require_live_environment` + before doing anything else, so it is structurally incapable of a + network call without both ``BENCHMARK_LIVE_PROVIDERS_ENABLED`` and + ``ANTHROPIC_API_KEY`` set. Raises + :class:`benchmarks.adapters.guard.LiveProviderDisabledError` (a + ``BenchmarkCellFailure`` subclass) when the guard fails, and + ``BenchmarkCellFailure`` (category ``tool_failure``) on any HTTP or + response-shape failure, matching every other adapter's failure-capture + contract. + """ + + provider = PROVIDER + + def __init__(self, *, model: str = DEFAULT_COUNT_MODEL) -> None: + self._model = model + + def count(self, messages: list[dict[str, Any]]) -> int: + require_live_environment(self.provider) + api_key = os.environ.get(PROVIDER_API_KEY_ENV[self.provider], "").strip() + + body = json.dumps({"model": self._model, "messages": messages}).encode("utf-8") + request = urllib.request.Request( + COUNT_TOKENS_URL, + data=body, + method="POST", + headers={ + "content-type": "application/json", + "x-api-key": api_key, + "anthropic-version": ANTHROPIC_API_VERSION, + }, + ) + try: + with urllib.request.urlopen(request, timeout=REQUEST_TIMEOUT_SECONDS) as response: + payload = json.loads(response.read().decode("utf-8")) + except (urllib.error.URLError, TimeoutError, OSError) as exc: + raise BenchmarkCellFailure( + "tool_failure", f"Anthropic count-tokens call failed: {exc}" + ) from exc + input_tokens = payload.get("input_tokens") if isinstance(payload, dict) else None + if not isinstance(input_tokens, int): + raise BenchmarkCellFailure( + "tool_failure", + f"Anthropic count-tokens response missing integer 'input_tokens': {payload!r}", + ) + return input_tokens + + +@dataclass(frozen=True) +class TokenCountResult: + """Result of counting one benchmark cell's tokens (fake or live counter).""" + + client_wrapped_tokens: int | None + raw_payload_tokens: int | None + approximation: bool + notes: str + serialization_latency_ms: float + """Wall-clock time (ms) spent building the envelope(s) and counting them + (decision 5.8: report serialization latency alongside token counts).""" + + +def build_client_wrapped_envelope( + prompt: str, + *, + tool_calls: list[dict[str, Any]] | None = None, + provider_mock_payload: dict[str, Any] | None = None, +) -> tuple[list[dict[str, Any]], bool]: + """Build the Claude-format message envelope for one benchmark cell. + + Implements methodology "Token Measurement" steps 1-3: capture the MCP + tool response or baseline prompt context, pass it through this + benchmark's client-side wrapping (a user message carrying the corpus + prompt, plus one ``tool_result`` content block per recorded tool call), + and return the resulting envelope for counting. + + ``provider_mock_payload`` is the raw payload from a provider-adapter + mock/live-stub (``benchmarks.adapters.base.AdapterResponse.raw`` for + ``MockOpenAIAdapter`` / ``MockGoogleAdapter`` and their live-stub + counterparts, identifiable by their ``"mock": True`` marker). Those + adapters never call a real client SDK, so there is no real wrapped + request body to recover; per the methodology, this function then wraps + only the prompt text and reports ``approximation=True`` rather than + guessing at a shape the real SDK would have produced. + + Returns ``(envelope, approximation)``. + """ + if provider_mock_payload is not None: + return [{"role": "user", "content": prompt}], True + + messages: list[dict[str, Any]] = [{"role": "user", "content": prompt}] + if tool_calls: + content_blocks = [ + { + "type": "tool_result", + "tool_use_id": f"benchmark-cell-{index}-{call.get('tool', 'tool')}", + "content": json.dumps(call.get("result"), sort_keys=True, default=str), + "is_error": bool(call.get("is_error", False)), + } + for index, call in enumerate(tool_calls) + ] + messages.append({"role": "user", "content": content_blocks}) + return messages, False + + +def _raw_payload_envelope( + prompt: str, tool_calls: list[dict[str, Any]] | None +) -> list[dict[str, Any]]: + """Build the diagnostic-only raw-payload envelope (unwrapped tool bytes). + + Never reported under ``METHODOLOGY_TOKEN_LABEL`` on its own merits as a + cross-model metric -- see ``benchmarks.runner``'s token record, which + keeps ``raw_payload_tokens`` clearly separate from + ``client_wrapped_tokens``. + """ + if not tool_calls: + return [{"role": "user", "content": prompt}] + raw_results = [call.get("result") for call in tool_calls] + return [{"role": "user", "content": json.dumps(raw_results, sort_keys=True, default=str)}] + + +def count_cell_tokens( + *, + prompt: str, + tool_calls: list[dict[str, Any]] | None, + counter: TokenCounter, + provider_mock_payload: dict[str, Any] | None = None, +) -> TokenCountResult: + """Count one benchmark cell's tokens with ``counter`` (fake or live). + + ``counter`` is injected so callers (the runner's live-phase path, or a + test with :class:`FakeTokenCounter`) share this one code path for + envelope construction, approximation marking, and serialization-latency + capture -- only the counter implementation differs. + """ + started = time.perf_counter() + envelope, approximation = build_client_wrapped_envelope( + prompt, tool_calls=tool_calls, provider_mock_payload=provider_mock_payload + ) + client_wrapped_tokens = counter.count(envelope) + raw_envelope = _raw_payload_envelope(prompt, tool_calls) + raw_payload_tokens = counter.count(raw_envelope) + serialization_latency_ms = round((time.perf_counter() - started) * 1000, 3) + + notes = ( + "approximation: provider-adapter mock/live-stub cannot expose its exact " + "client-wrapped envelope (no real client SDK call was made); wrapping the " + "raw prompt only, per PUBLIC-BENCHMARK-METHODOLOGY.md 'Token Measurement'" + if approximation + else "counted from this benchmark's client-side wrapped message envelope" + ) + return TokenCountResult( + client_wrapped_tokens=client_wrapped_tokens, + raw_payload_tokens=raw_payload_tokens, + approximation=approximation, + notes=notes, + serialization_latency_ms=serialization_latency_ms, + ) diff --git a/docs/benchmarks/model-matrix.yml b/docs/benchmarks/model-matrix.yml index 6ec06f8..4bfc219 100644 --- a/docs/benchmarks/model-matrix.yml +++ b/docs/benchmarks/model-matrix.yml @@ -29,9 +29,13 @@ schema_version: 1 # # Maintainer decision (PLAN.md Amendment 2026-07-08): real Claude token # counting via the Anthropic count-tokens API is confined to the -# maintainer-run live phase. Nothing in benchmarks/model_matrix.py or -# benchmarks/adapters/ calls that API; this file and the mocked adapters -# only record where/how that count would be produced. +# maintainer-run live phase. Nothing in benchmarks/model_matrix.py calls +# that API. benchmarks/adapters/claude_tokens.py (issue #89) is the one +# module that does, and only after benchmarks.adapters.guard's live-phase +# guard passes (BENCHMARK_LIVE_PROVIDERS_ENABLED + ANTHROPIC_API_KEY, both +# maintainer-set) -- CI and unit tests never reach that call. This file and +# the mocked OpenAI/Google adapters only record where/how that count would +# be produced for those model families. methodology_token_label: "Claude Tokens (Normalized Payload)" model_families: diff --git a/tests/benchmarks/test_claude_tokens.py b/tests/benchmarks/test_claude_tokens.py new file mode 100644 index 0000000..c772622 --- /dev/null +++ b/tests/benchmarks/test_claude_tokens.py @@ -0,0 +1,225 @@ +"""Tests for the Claude count-tokens integration (issue #89). + +Every test here uses :class:`~benchmarks.adapters.claude_tokens.FakeTokenCounter` +or a stubbed ``urllib.request.urlopen`` -- none ever calls the real Anthropic +API. The guard-refusal tests additionally monkeypatch ``urlopen`` to raise if +it is ever invoked, proving this module is structurally incapable of a +network call when the live-phase guard is not satisfied. +""" + +from __future__ import annotations + +import json +from typing import Any + +import pytest + +from benchmarks.adapters.claude_tokens import ( + FakeTokenCounter, + LiveClaudeTokenCounter, + build_client_wrapped_envelope, + count_cell_tokens, +) +from benchmarks.adapters.guard import ( + LIVE_PROVIDERS_ENABLED_ENV, + PROVIDER_API_KEY_ENV, + LiveProviderDisabledError, +) +from benchmarks.runner import BenchmarkCellFailure + +ANTHROPIC_KEY_ENV = PROVIDER_API_KEY_ENV["anthropic"] + + +# --- FakeTokenCounter -------------------------------------------------------- + + +def test_fake_token_counter_returns_configured_fixed_count() -> None: + counter = FakeTokenCounter(tokens_per_call=17) + + assert counter.count([{"role": "user", "content": "hi"}]) == 17 + assert counter.count([{"role": "user", "content": "something else entirely"}]) == 17 + assert len(counter.calls) == 2 + + +def test_fake_token_counter_default_count_is_deterministic() -> None: + counter = FakeTokenCounter() + messages = [{"role": "user", "content": "one two three"}] + + assert counter.count(messages) == counter.count(messages) + assert counter.count(messages) > 0 + + +# --- Envelope construction / approximation marking -------------------------- + + +def test_envelope_is_exact_for_a_plain_prompt_with_no_tool_calls() -> None: + envelope, approximation = build_client_wrapped_envelope("what is pathlib?", tool_calls=None) + + assert approximation is False + assert envelope == [{"role": "user", "content": "what is pathlib?"}] + + +def test_envelope_wraps_tool_call_results_as_tool_result_blocks() -> None: + tool_calls = [ + {"tool": "search_docs", "result": {"hits": [{"slug": "x"}]}, "is_error": False}, + {"tool": "get_docs", "result": {"content": "docs"}, "is_error": False}, + ] + + envelope, approximation = build_client_wrapped_envelope("prompt", tool_calls=tool_calls) + + assert approximation is False + assert envelope[0] == {"role": "user", "content": "prompt"} + tool_result_blocks = envelope[1]["content"] + assert len(tool_result_blocks) == 2 + assert all(block["type"] == "tool_result" for block in tool_result_blocks) + assert "search_docs" in tool_result_blocks[0]["tool_use_id"] + assert "get_docs" in tool_result_blocks[1]["tool_use_id"] + + +def test_envelope_is_approximation_when_built_from_a_provider_mock_payload() -> None: + # MockOpenAIAdapter/MockGoogleAdapter (and their live-stub counterparts) + # never call a real client SDK, so there is no real wrapped envelope to + # recover -- the methodology requires marking that gap, not guessing. + envelope, approximation = build_client_wrapped_envelope( + "prompt", provider_mock_payload={"provider": "openai", "mock": True} + ) + + assert approximation is True + assert envelope == [{"role": "user", "content": "prompt"}] + + +# --- count_cell_tokens: fake-counter records, approximation, latency ------- + + +def test_count_cell_tokens_fills_client_wrapped_and_raw_payload_tokens() -> None: + counter = FakeTokenCounter(tokens_per_call=25) + + result = count_cell_tokens(prompt="prompt", tool_calls=None, counter=counter) + + assert result.client_wrapped_tokens == 25 + assert result.raw_payload_tokens == 25 + assert result.approximation is False + # One counter call for the client-wrapped envelope, one for the raw + # payload envelope -- they are always counted (and reported) separately. + assert len(counter.calls) == 2 + + +def test_count_cell_tokens_marks_approximation_for_provider_mock_payload() -> None: + counter = FakeTokenCounter(tokens_per_call=9) + + result = count_cell_tokens( + prompt="prompt", + tool_calls=None, + counter=counter, + provider_mock_payload={"provider": "google", "mock": True}, + ) + + assert result.approximation is True + assert "approximation" in result.notes + + +def test_count_cell_tokens_captures_serialization_latency_alongside_counts() -> None: + # Roadmap decision 5.8: report tokens AND serialization latency, not + # tokens alone. + counter = FakeTokenCounter(tokens_per_call=3) + + result = count_cell_tokens(prompt="prompt", tool_calls=None, counter=counter) + + assert isinstance(result.serialization_latency_ms, float) + assert result.serialization_latency_ms >= 0 + + +# --- LiveClaudeTokenCounter: guarded, stdlib-only, never in CI -------------- + + +def test_live_claude_token_counter_refuses_without_config_and_never_touches_network( + monkeypatch: pytest.MonkeyPatch, +) -> None: + monkeypatch.delenv(LIVE_PROVIDERS_ENABLED_ENV, raising=False) + monkeypatch.delenv(ANTHROPIC_KEY_ENV, raising=False) + + def _network_must_not_be_called(*args: Any, **kwargs: Any) -> Any: + raise AssertionError("urlopen must never be called when the live guard is disabled") + + monkeypatch.setattr("urllib.request.urlopen", _network_must_not_be_called) + + with pytest.raises(LiveProviderDisabledError): + LiveClaudeTokenCounter().count([{"role": "user", "content": "prompt"}]) + + +def test_live_claude_token_counter_refuses_with_flag_but_no_key( + monkeypatch: pytest.MonkeyPatch, +) -> None: + monkeypatch.setenv(LIVE_PROVIDERS_ENABLED_ENV, "1") + monkeypatch.delenv(ANTHROPIC_KEY_ENV, raising=False) + + with pytest.raises(LiveProviderDisabledError, match=ANTHROPIC_KEY_ENV): + LiveClaudeTokenCounter().count([{"role": "user", "content": "prompt"}]) + + +def test_live_claude_token_counter_refuses_with_key_but_no_flag( + monkeypatch: pytest.MonkeyPatch, +) -> None: + monkeypatch.delenv(LIVE_PROVIDERS_ENABLED_ENV, raising=False) + monkeypatch.setenv(ANTHROPIC_KEY_ENV, "fake-not-a-real-key") + + with pytest.raises(LiveProviderDisabledError, match=LIVE_PROVIDERS_ENABLED_ENV): + LiveClaudeTokenCounter().count([{"role": "user", "content": "prompt"}]) + + +class _FakeHttpResponse: + """Stand-in for the context manager ``urllib.request.urlopen`` returns.""" + + def __init__(self, payload: dict[str, Any]) -> None: + self._body = json.dumps(payload).encode("utf-8") + + def __enter__(self) -> "_FakeHttpResponse": + return self + + def __exit__(self, *exc_info: object) -> None: + return None + + def read(self) -> bytes: + return self._body + + +def test_live_claude_token_counter_calls_count_tokens_endpoint_when_guard_passes( + monkeypatch: pytest.MonkeyPatch, +) -> None: + # The guard is satisfied (simulating a maintainer-run live phase), but + # the HTTP layer is stubbed -- this test still never reaches the real + # network or needs a real key. It proves the request is built correctly + # via stdlib urllib only (no SDK) once the guard passes. + monkeypatch.setenv(LIVE_PROVIDERS_ENABLED_ENV, "1") + monkeypatch.setenv(ANTHROPIC_KEY_ENV, "fake-not-a-real-key") + captured: dict[str, Any] = {} + + def _fake_urlopen(request: Any, timeout: float) -> _FakeHttpResponse: + captured["url"] = request.full_url + captured["headers"] = {k.lower(): v for k, v in request.header_items()} + captured["body"] = json.loads(request.data.decode("utf-8")) + return _FakeHttpResponse({"input_tokens": 123}) + + monkeypatch.setattr("urllib.request.urlopen", _fake_urlopen) + + tokens = LiveClaudeTokenCounter().count([{"role": "user", "content": "prompt"}]) + + assert tokens == 123 + assert captured["url"] == "https://api.anthropic.com/v1/messages/count_tokens" + assert captured["headers"]["x-api-key"] == "fake-not-a-real-key" + assert captured["headers"]["anthropic-version"] + assert captured["body"]["messages"] == [{"role": "user", "content": "prompt"}] + + +def test_live_claude_token_counter_raises_benchmark_cell_failure_on_malformed_response( + monkeypatch: pytest.MonkeyPatch, +) -> None: + monkeypatch.setenv(LIVE_PROVIDERS_ENABLED_ENV, "1") + monkeypatch.setenv(ANTHROPIC_KEY_ENV, "fake-not-a-real-key") + monkeypatch.setattr( + "urllib.request.urlopen", + lambda request, timeout: _FakeHttpResponse({"unexpected": "shape"}), + ) + + with pytest.raises(BenchmarkCellFailure, match="input_tokens"): + LiveClaudeTokenCounter().count([{"role": "user", "content": "prompt"}]) From 942cff655d3f2f875c87c8ba3a7a81ca922acce3 Mon Sep 17 00:00:00 2001 From: Aymen Hammouda Date: Wed, 8 Jul 2026 19:30:34 +0100 Subject: [PATCH 3/6] agent: fill runner token records via guarded Claude count-tokens integration benchmarks.runner._execute_cell now builds each cell's token record via a new _build_token_record helper instead of an always-None placeholder dict: - Outside a maintainer-run live phase (the default for every CI/unit-test run, since BENCHMARK_LIVE_PROVIDERS_ENABLED + ANTHROPIC_API_KEY are never both set there), require_live_environment("anthropic") fails and the record falls back to the same honest-placeholder shape as before, plus approximation/token_label fields. - When the live-phase guard passes, client_wrapped_tokens/raw_payload_tokens are filled via benchmarks.adapters.claude_tokens.count_cell_tokens using LiveClaudeTokenCounter, labeled under the exported benchmarks.model_matrix.METHODOLOGY_TOKEN_LABEL constant (never a string literal). A count-tokens failure is captured as this cell's own failed token status rather than propagated, so it can never crash the whole run or mask that cell's actual answer/latency/scoring results. Both branches are imported lazily inside the function, matching the existing _python_docs_mcp_stdio_answer convention: benchmarks.adapters.guard and benchmarks.model_matrix both import from this module at their own module scope, so this module cannot import them at module scope. Tests cover the guard-disabled default (placeholder shape unchanged) and the guard-enabled path with LiveClaudeTokenCounter substituted for a fake counter -- never the real Anthropic API. Refs #63. --- benchmarks/runner.py | 97 +++++++++++++++++++++++++++++---- tests/benchmarks/test_runner.py | 78 ++++++++++++++++++++++++++ 2 files changed, 164 insertions(+), 11 deletions(-) diff --git a/benchmarks/runner.py b/benchmarks/runner.py index 2da4879..c9054dd 100644 --- a/benchmarks/runner.py +++ b/benchmarks/runner.py @@ -263,17 +263,7 @@ def _execute_cell(cell: BenchmarkCell) -> dict[str, Any]: "error": error, "external_provider_calls": False, } - token_record = { - "competitor_id": cell.competitor.id, - "tool_model_key": tool_model_key, - "corpus_id": cell.question.id, - "status": "placeholder", - "input_characters": len(cell.question.prompt), - "output_characters": len(answer), - "client_wrapped_tokens": None, - "raw_payload_tokens": None, - "notes": "Token counting integration is intentionally out of scope for issue #72.", - } + token_record = _build_token_record(cell, tool_model_key, answer, tool_calls) latency_record = { "competitor_id": cell.competitor.id, "tool_model_key": tool_model_key, @@ -300,6 +290,91 @@ def _execute_cell(cell: BenchmarkCell) -> dict[str, Any]: } +def _build_token_record( + cell: BenchmarkCell, + tool_model_key: str, + answer: str, + tool_calls: list[dict[str, Any]] | None, +) -> dict[str, Any]: + """Build one cell's token record (issue #89). + + Fills ``client_wrapped_tokens`` / ``raw_payload_tokens`` via the + Anthropic count-tokens API only when + ``benchmarks.adapters.guard.require_live_environment("anthropic")`` + passes -- i.e. only in a maintainer-run live phase with + ``BENCHMARK_LIVE_PROVIDERS_ENABLED`` and ``ANTHROPIC_API_KEY`` both set. + CI and unit tests never set those, so this always returns the honest + ``None`` placeholder there, matching this record's pre-#89 shape (see + ``tests/benchmarks/test_runner.py``, which only asserts the tokens file + exists, not its content). + + A count-tokens call failure (e.g. a live-phase network error) is + recorded as this cell's own ``failed`` token status rather than + propagated -- a token-counting problem must never crash the whole + benchmark run or mask a competitor's actual answer/latency/scoring + results for the same cell. + + Imported lazily, for the same reason ``_python_docs_mcp_stdio_answer`` + imports its adapter lazily: ``benchmarks.adapters.guard`` and + ``benchmarks.model_matrix`` both import from this module at their own + module scope, so this module cannot import from them at module scope + without a circular import. + """ + from benchmarks.adapters.claude_tokens import LiveClaudeTokenCounter, count_cell_tokens + from benchmarks.adapters.guard import LiveProviderDisabledError, require_live_environment + from benchmarks.model_matrix import METHODOLOGY_TOKEN_LABEL + + base_fields = { + "competitor_id": cell.competitor.id, + "tool_model_key": tool_model_key, + "corpus_id": cell.question.id, + "token_label": METHODOLOGY_TOKEN_LABEL, + "input_characters": len(cell.question.prompt), + "output_characters": len(answer), + } + try: + require_live_environment("anthropic") + except LiveProviderDisabledError as exc: + return { + **base_fields, + "status": "placeholder", + "client_wrapped_tokens": None, + "raw_payload_tokens": None, + "approximation": False, + "notes": ( + "Claude token-count integration (issue #89) is gated behind " + "BENCHMARK_LIVE_PROVIDERS_ENABLED + ANTHROPIC_API_KEY for a " + f"maintainer-run live phase; disabled here: {exc}" + ), + } + + try: + result = count_cell_tokens( + prompt=cell.question.prompt, + tool_calls=tool_calls, + counter=LiveClaudeTokenCounter(), + ) + except BenchmarkCellFailure as exc: + return { + **base_fields, + "status": "failed", + "client_wrapped_tokens": None, + "raw_payload_tokens": None, + "approximation": False, + "notes": f"Claude count-tokens call failed: {exc}", + } + + return { + **base_fields, + "status": "counted", + "client_wrapped_tokens": result.client_wrapped_tokens, + "raw_payload_tokens": result.raw_payload_tokens, + "approximation": result.approximation, + "serialization_latency_ms": result.serialization_latency_ms, + "notes": result.notes, + } + + @dataclass(frozen=True) class _CellDispatchResult: """Return value of one adapter dispatch call (see ``_ADAPTER_DISPATCH``). diff --git a/tests/benchmarks/test_runner.py b/tests/benchmarks/test_runner.py index 5ae8d98..8d427b2 100644 --- a/tests/benchmarks/test_runner.py +++ b/tests/benchmarks/test_runner.py @@ -496,3 +496,81 @@ def test_unrecognized_adapter_id_fails_cleanly_without_crashing(tmp_path: Path) ) assert failure["error"]["category"] == "tool_failure" assert "not implemented in this runner" in failure["error"]["message"] + + +# --- Claude token-count integration (issue #89) ------------------------------ +# +# The count-tokens call itself is unit-tested in isolation, with a fake +# counter, in tests/benchmarks/test_claude_tokens.py. These tests only cover +# how benchmarks.runner wires that module in: the guard-disabled default +# (every CI/unit-test run, since BENCHMARK_LIVE_PROVIDERS_ENABLED + +# ANTHROPIC_API_KEY are never both set here) and the guard-enabled path with +# LiveClaudeTokenCounter substituted for a fake counter -- never the real +# Anthropic API. + + +def test_token_record_is_the_honest_placeholder_when_live_guard_is_disabled( + tmp_path: Path, monkeypatch: pytest.MonkeyPatch +) -> None: + from benchmarks.adapters.guard import LIVE_PROVIDERS_ENABLED_ENV, PROVIDER_API_KEY_ENV + from benchmarks.model_matrix import METHODOLOGY_TOKEN_LABEL + + monkeypatch.delenv(LIVE_PROVIDERS_ENABLED_ENV, raising=False) + monkeypatch.delenv(PROVIDER_API_KEY_ENV["anthropic"], raising=False) + out_dir = tmp_path / "results" / "token-guard-disabled" + + run_benchmark( + BenchmarkConfig( + corpus_path=_corpus(tmp_path), + manifest_path=_manifest(tmp_path), + out_dir=out_dir, + run_id="token-guard-disabled", + ) + ) + + token_record = json.loads( + (out_dir / "tokens" / "no-mcp" / "q001.json").read_text(encoding="utf-8") + ) + assert token_record["status"] == "placeholder" + assert token_record["client_wrapped_tokens"] is None + assert token_record["raw_payload_tokens"] is None + assert token_record["approximation"] is False + assert token_record["token_label"] == METHODOLOGY_TOKEN_LABEL + + +def test_token_record_is_filled_when_live_guard_passes_with_a_fake_counter( + tmp_path: Path, monkeypatch: pytest.MonkeyPatch +) -> None: + from benchmarks.adapters.claude_tokens import FakeTokenCounter + from benchmarks.adapters.guard import LIVE_PROVIDERS_ENABLED_ENV, PROVIDER_API_KEY_ENV + from benchmarks.model_matrix import METHODOLOGY_TOKEN_LABEL + + # Simulate a maintainer-run live phase's environment, but substitute a + # fake counter for LiveClaudeTokenCounter so this test -- like every + # other test in this suite -- never calls the real Anthropic API. + monkeypatch.setenv(LIVE_PROVIDERS_ENABLED_ENV, "1") + monkeypatch.setenv(PROVIDER_API_KEY_ENV["anthropic"], "fake-not-a-real-key") + monkeypatch.setattr( + "benchmarks.adapters.claude_tokens.LiveClaudeTokenCounter", + lambda: FakeTokenCounter(tokens_per_call=55), + ) + out_dir = tmp_path / "results" / "token-guard-enabled" + + run_benchmark( + BenchmarkConfig( + corpus_path=_corpus(tmp_path), + manifest_path=_manifest(tmp_path), + out_dir=out_dir, + run_id="token-guard-enabled", + ) + ) + + token_record = json.loads( + (out_dir / "tokens" / "no-mcp" / "q001.json").read_text(encoding="utf-8") + ) + assert token_record["status"] == "counted" + assert token_record["client_wrapped_tokens"] == 55 + assert token_record["raw_payload_tokens"] == 55 + assert token_record["approximation"] is False + assert token_record["token_label"] == METHODOLOGY_TOKEN_LABEL + assert token_record["serialization_latency_ms"] >= 0 From 53c3f10fa4c0f0b377add0b0125a38a0a921eb5d Mon Sep 17 00:00:00 2001 From: Aymen Hammouda Date: Wed, 8 Jul 2026 19:30:47 +0100 Subject: [PATCH 4/6] agent: exclude approximation:true token records from headline report tables Additive report.py change (issue #89): CellRecord gains a token_approximation field parsed from each cell's tokens/*.json record. _basic_stats now excludes approximation:true records from the headline-eligible client-wrapped-token list (median, in both REPORT.md's "Token Counts" table and the README-safe summary's results table) instead of folding every present count together -- methodology "Token Measurement" is explicit that approximate counts must never be used for headline claims. The excluded count is surfaced, not silently dropped: _fmt_tokens gains an `approximated` parameter and both call sites report e.g. "100, 1 approximation:true excluded" rather than only the filtered median. No existing field is removed or renamed; existing tests already only assert `None`-placeholder token behavior, so nothing here narrows prior coverage. Refs #63. --- benchmarks/report.py | 48 ++++++++++-- tests/benchmarks/test_report.py | 131 ++++++++++++++++++++++++++++++++ 2 files changed, 173 insertions(+), 6 deletions(-) diff --git a/benchmarks/report.py b/benchmarks/report.py index 3941777..b425b97 100644 --- a/benchmarks/report.py +++ b/benchmarks/report.py @@ -9,7 +9,11 @@ - A full raw report (``REPORT.md``): methodology link, corpus hash, repo commit, model/client matrix, competitor manifest, correctness by category, token counts after client rewrap (with honest ``None`` placeholders), - latency median/p95, failures/exclusions, and environment metadata. + latency median/p95, failures/exclusions, and environment metadata. Token + records marked ``approximation: true`` (issue #89 -- the client could not + expose its exact client-wrapped message envelope; see + ``benchmarks.adapters.claude_tokens``) are counted and shown separately, + never folded into the headline-eligible median (see ``_basic_stats``). - A compact README-safe summary template: strict tool+model (``tool_model_key``) pairings only -- never tool-only rows -- with an error/timeout-rate column, plus links to the methodology and the raw @@ -108,6 +112,11 @@ class CellRecord: latency_ms: float | None client_wrapped_tokens: int | None raw_payload_tokens: int | None + token_approximation: bool + """True when the token record's ``client_wrapped_tokens`` count (if any) is an + approximation -- issue #89 / methodology "Token Measurement": the client + could not expose its exact wrapped message envelope. Approximate counts must + never be used for headline claims (see ``_basic_stats``, which excludes them).""" @dataclass(frozen=True) @@ -229,6 +238,7 @@ def _load_run_bundle(run_dir: Path) -> RunBundle: latency_ms=latency_record.get("latency_ms"), client_wrapped_tokens=token_record.get("client_wrapped_tokens"), raw_payload_tokens=token_record.get("raw_payload_tokens"), + token_approximation=bool(token_record.get("approximation", False)), ) ) @@ -366,9 +376,20 @@ def _basic_stats(cells: list[CellRecord]) -> dict[str, Any]: scored = [cell.score for cell in cells if cell.score is not None] pending = sum(1 for cell in cells if cell.requires_manual_scoring) latencies = [cell.latency_ms for cell in cells if cell.latency_ms is not None] + # Headline-eligible token counts exclude approximations (issue #89 / + # methodology "Token Measurement": approximate counts must never be used + # for headline claims). `tokens_approximated` surfaces how many were + # excluded so a reader can see the gap isn't silently dropped. tokens = [ - cell.client_wrapped_tokens for cell in cells if cell.client_wrapped_tokens is not None + cell.client_wrapped_tokens + for cell in cells + if cell.client_wrapped_tokens is not None and not cell.token_approximation ] + tokens_approximated = sum( + 1 + for cell in cells + if cell.client_wrapped_tokens is not None and cell.token_approximation + ) return { "total_cells": total, "succeeded_cells": total - failed, @@ -382,7 +403,8 @@ def _basic_stats(cells: list[CellRecord]) -> dict[str, Any]: "latency_median_ms": statistics.median(latencies) if latencies else None, "latency_p95_ms": _percentile(latencies, 0.95) if latencies else None, "tokens_present": len(tokens), - "tokens_placeholder": total - len(tokens), + "tokens_approximated": tokens_approximated, + "tokens_placeholder": total - len(tokens) - tokens_approximated, "median_client_wrapped_tokens": statistics.median(tokens) if tokens else None, } @@ -401,10 +423,20 @@ def _fmt_score(scored: int, total: int, mean: float | None) -> str: return f"{mean:.2f} ({scored}/{total} scored)" -def _fmt_tokens(present: int, placeholder: int, median: float | None) -> str: +def _fmt_tokens(present: int, placeholder: int, median: float | None, approximated: int = 0) -> str: if present == 0: + if approximated: + return ( + f"None headline-eligible (approximation:true excludes {approximated} cell(s); " + f"{placeholder} placeholder)" + ) return f"None (placeholder -- {placeholder} cell(s) pending token-count integration)" - suffix = f", {placeholder} placeholder" if placeholder else "" + suffixes = [] + if placeholder: + suffixes.append(f"{placeholder} placeholder") + if approximated: + suffixes.append(f"{approximated} approximation:true excluded") + suffix = f", {', '.join(suffixes)}" if suffixes else "" return f"{median:.0f}{suffix}" @@ -567,7 +599,9 @@ def _render_report( lines.append( f"Primary metric label: `{METHODOLOGY_TOKEN_LABEL}` (see the methodology's 'Token " "Measurement' section). `None` placeholders are surfaced honestly below, never " - "reported as zero." + "reported as zero. Records marked `approximation: true` (issue #89 -- the client " + "could not expose its exact wrapped message envelope) are excluded from this " + "headline-eligible median; the excluded count is shown alongside it." ) lines.append("") lines.extend( @@ -580,6 +614,7 @@ def _render_report( (stats := _basic_stats(by_tool_model[tool_model_key]))["tokens_present"], stats["tokens_placeholder"], stats["median_client_wrapped_tokens"], + stats["tokens_approximated"], ), ] for tool_model_key in sorted(by_tool_model) @@ -781,6 +816,7 @@ def _render_readme_summary(bundle: RunBundle, methodology_path: Path, summary_pa stats["tokens_present"], stats["tokens_placeholder"], stats["median_client_wrapped_tokens"], + stats["tokens_approximated"], ), ] for tool_model_key in sorted(by_tool_model) diff --git a/tests/benchmarks/test_report.py b/tests/benchmarks/test_report.py index ac8a0f4..f9515b3 100644 --- a/tests/benchmarks/test_report.py +++ b/tests/benchmarks/test_report.py @@ -536,3 +536,134 @@ def test_no_aggregate_section_claims_when_each_tool_has_one_model(tmp_path: Path ).read_text(encoding="utf-8") assert "No cross-model aggregation applies to this run" in text + + +# --- Token approximation exclusion from headline-eligible tables (#89) ------ +# +# The count-tokens integration itself lives in benchmarks/adapters/claude_ +# tokens.py (see tests/benchmarks/test_claude_tokens.py). These tests only +# cover the additive report.py contract: a token record marked +# ``approximation: true`` is counted and disclosed, but never folded into a +# headline-eligible median (methodology "Token Measurement": "Approximate +# counts must not be used for headline claims"). + + +def _write_run_dir_with_one_approximated_and_one_exact_token_record(tmp_path: Path) -> Path: + """One competitor, two corpus questions: one exact count, one approximation. + + Both cells share a ``tool_model_key`` ("tool-a") so they land in the same + headline-eligible aggregation group, isolating the approximation filter + as the only variable. + """ + run_dir = tmp_path / "run" + _write_json( + run_dir / "run-summary.json", + { + "run_id": "token-approximation", + "repo_commit": "deadbeef", + "artifact_root": str(run_dir), + "competitors": ["tool-a"], + "corpus_ids": ["q001", "q002"], + "planned_cells": 2, + "scored_cells": 2, + "succeeded_cells": 2, + "failed_cells": 0, + }, + ) + _write_json( + run_dir / "environment.json", + { + "run_id": "token-approximation", + "created_at": "2026-07-08T00:00:00Z", + "repo_commit": "deadbeef", + "python_version": "3.12.0", + "platform": "test-platform", + "system": "Test", + "machine": "x86_64", + }, + ) + _write_yaml( + run_dir / "snapshots" / "corpus.yml", + { + "questions": [ + {"id": "q001", "category": "exact-symbol", "prompt": "p1"}, + {"id": "q002", "category": "exact-symbol", "prompt": "p2"}, + ] + }, + ) + _write_yaml( + run_dir / "snapshots" / "competitor-manifest.yml", + {"competitors": [{"id": "tool-a", "adapter": "no-mcp-baseline"}]}, + ) + for corpus_id in ("q001", "q002"): + _write_json( + run_dir / "scoring" / "tool-a" / f"{corpus_id}.json", + { + "competitor_id": "tool-a", + "corpus_id": corpus_id, + "tool_model_key": "tool-a", + "status": "succeeded", + "score": None, + "requires_manual_scoring": True, + "included_in_correctness_denominator": True, + "error_category": None, + }, + ) + _write_json( + run_dir / "latency" / "tool-a" / f"{corpus_id}.json", + { + "competitor_id": "tool-a", + "corpus_id": corpus_id, + "latency_ms": 10.0, + "status": "succeeded", + }, + ) + _write_json( + run_dir / "tokens" / "tool-a" / "q001.json", + { + "competitor_id": "tool-a", + "corpus_id": "q001", + "status": "counted", + "client_wrapped_tokens": 100, + "raw_payload_tokens": 100, + "approximation": False, + }, + ) + _write_json( + run_dir / "tokens" / "tool-a" / "q002.json", + { + "competitor_id": "tool-a", + "corpus_id": "q002", + "status": "counted", + # An outlier value: if this were folded into the headline median + # alongside q001's 100, the median would move. It must not be. + "client_wrapped_tokens": 999999, + "raw_payload_tokens": 999999, + "approximation": True, + }, + ) + return run_dir + + +def test_report_excludes_approximated_tokens_from_headline_median(tmp_path: Path) -> None: + run_dir = _write_run_dir_with_one_approximated_and_one_exact_token_record(tmp_path) + + text = generate_report( + run_dir, model_matrix_path=MODEL_MATRIX_PATH, methodology_path=METHODOLOGY_PATH + ).read_text(encoding="utf-8") + + # Only the exact count (100) drives the headline-eligible median -- the + # approximation (999999) must never move it. + assert "| tool-a | 100, 1 approximation:true excluded |" in text + assert "999999" not in text + + +def test_readme_summary_excludes_approximated_tokens_from_headline_row(tmp_path: Path) -> None: + run_dir = _write_run_dir_with_one_approximated_and_one_exact_token_record(tmp_path) + + text = generate_readme_summary( + run_dir, methodology_path=METHODOLOGY_PATH + ).read_text(encoding="utf-8") + + assert "100, 1 approximation:true excluded" in text + assert "999999" not in text From 9e64066bd34cca1e5362537157ec2d020851c042 Mon Sep 17 00:00:00 2001 From: Aymen Hammouda Date: Wed, 8 Jul 2026 19:35:30 +0100 Subject: [PATCH 5/6] agent: note the token-count integration in runner.py's module docstring Small documentation-only follow-up to the prior commit: runner.py's module docstring enumerated the adjacent adapter/report modules but didn't mention that _build_token_record (issue #89) is the one place this module reaches outside itself for network I/O, and only under the live-phase guard. Refs #63. --- benchmarks/runner.py | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/benchmarks/runner.py b/benchmarks/runner.py index c9054dd..3846e49 100644 --- a/benchmarks/runner.py +++ b/benchmarks/runner.py @@ -11,6 +11,12 @@ dispatch remains out of scope -- cell composition stays competitor x question (see the ``benchmarks.model_matrix`` module docstring for the confirmed composition decision). + +Each cell's token record (``_build_token_record``, issue #89) is the one +exception to "no network I/O": it fills real counts via the guarded +Anthropic count-tokens caller (``benchmarks.adapters.claude_tokens``) when +a maintainer-run live phase is active, and otherwise stays the honest +``None`` placeholder every CI/unit-test run produces. """ from __future__ import annotations From 417b330ed025a42332a9aa97c33cdcf2a7a4f7a8 Mon Sep 17 00:00:00 2001 From: Aymen Hammouda Date: Wed, 8 Jul 2026 19:56:28 +0100 Subject: [PATCH 6/6] agent: fail-closed urlopen stub in remaining guard-refusal tests CodeRabbit (PR #101 review): the two guard-refusal tests exercising "flag but no key" and "key but no flag" did not stub urllib.request.urlopen, unlike the no-config refusal test above them. If the guard order in LiveClaudeTokenCounter.count() ever regressed, those tests could reach the real network instead of failing deterministically. Both now monkeypatch urlopen to raise AssertionError("network reached") before exercising the refusal, matching the existing fail-closed pattern. Defensive change to this PR's own new test file only -- no merged test touched, no assertion weakened. Refs #63. --- tests/benchmarks/test_claude_tokens.py | 14 ++++++++++++++ 1 file changed, 14 insertions(+) diff --git a/tests/benchmarks/test_claude_tokens.py b/tests/benchmarks/test_claude_tokens.py index c772622..d22ba87 100644 --- a/tests/benchmarks/test_claude_tokens.py +++ b/tests/benchmarks/test_claude_tokens.py @@ -153,6 +153,13 @@ def test_live_claude_token_counter_refuses_with_flag_but_no_key( monkeypatch.setenv(LIVE_PROVIDERS_ENABLED_ENV, "1") monkeypatch.delenv(ANTHROPIC_KEY_ENV, raising=False) + def _network_must_not_be_called(*args: Any, **kwargs: Any) -> Any: + raise AssertionError("network reached") + + # Fail closed: if the guard order ever regresses, this test must fail on + # the stubbed network call, never actually reach the network. + monkeypatch.setattr("urllib.request.urlopen", _network_must_not_be_called) + with pytest.raises(LiveProviderDisabledError, match=ANTHROPIC_KEY_ENV): LiveClaudeTokenCounter().count([{"role": "user", "content": "prompt"}]) @@ -163,6 +170,13 @@ def test_live_claude_token_counter_refuses_with_key_but_no_flag( monkeypatch.delenv(LIVE_PROVIDERS_ENABLED_ENV, raising=False) monkeypatch.setenv(ANTHROPIC_KEY_ENV, "fake-not-a-real-key") + def _network_must_not_be_called(*args: Any, **kwargs: Any) -> Any: + raise AssertionError("network reached") + + # Fail closed: if the guard order ever regresses, this test must fail on + # the stubbed network call, never actually reach the network. + monkeypatch.setattr("urllib.request.urlopen", _network_must_not_be_called) + with pytest.raises(LiveProviderDisabledError, match=LIVE_PROVIDERS_ENABLED_ENV): LiveClaudeTokenCounter().count([{"role": "user", "content": "prompt"}])