diff --git a/benchmarks/LANGFUSE_TURN_BENCHMARKS.md b/benchmarks/LANGFUSE_TURN_BENCHMARKS.md new file mode 100644 index 0000000..d9e84e1 --- /dev/null +++ b/benchmarks/LANGFUSE_TURN_BENCHMARKS.md @@ -0,0 +1,73 @@ +# Langfuse per-turn payload baseline + +The `langfuse-turns-v1` benchmark exercises the same metadata generation used +by TCT's Langfuse tool observations. It groups tool calls into representative +agent turns and compares one-call-per-identifier behavior with a batched call. + +The fixture is deterministic, makes no network requests, and sends nothing to +Langfuse. Regenerate it with: + +```bash +python -m benchmarks.langfuse_turns +``` + +| Metric | One by one | Batched | Duplicate batch | +| --- | ---: | ---: | ---: | +| Tool calls | 7 | 1 | 2 | +| Unique inputs | 7 | 1 | 1 | +| Repeated input calls | 0 | 0 | 1 | +| Input bytes | 10,269 | 1,563 | 3,126 | +| Output bytes | 679 | 457 | 914 | +| Total payload bytes | 10,948 | 2,020 | 4,040 | +| Repeated provider metadata bytes | 8,372 | 1,196 | 2,392 | + +For seven identifiers, batching avoids six tool calls, 8,706 input bytes +(84.8%), and 8,928 total payload bytes (81.5%). It also avoids 7,176 bytes of +provider metadata that would otherwise be repeated within the turn. + +These are tool-boundary payload measurements, not model token counts. Model +input/output tokens and price belong to the parent generation observation. +When the MCP client propagates the parent W3C trace context, Langfuse can join +those generation costs with TCT's tool-call counts, hashes, and payload sizes +for a complete per-turn view. + +## Live acceptance: conversational turns returned by Langfuse + +The offline benchmark is not proof that a Langfuse deployment accepted the +observations. The live acceptance probe creates two parent `agent` +observations, invokes the TCT fixture tools beneath them, flushes the SDK, and +then queries those trace IDs through the Langfuse observations API. Its JSON +output is computed only from observations returned by Langfuse. + +Set `LANGFUSE_PUBLIC_KEY` and `LANGFUSE_SECRET_KEY`; set +`LANGFUSE_BASE_URL` as well for a self-hosted deployment. Then run: + +```bash +uv run --extra langfuse python -m benchmarks.langfuse_conversation_acceptance +``` + +Exit status zero and `"acceptance": "passed"` mean that both conversational +turns and their TCT child observations were returned with metrics. Credentials +are required but are never included in the report or configuration errors. +The latest sanitized live result is checked in as +`langfuse_conversation_acceptance_result.json` so reviewers can inspect the +server-returned trace IDs and metrics directly in the pull request. + +The complementary `langfuse_codex_turn_result.json` records a real Codex turn +that successfully called the TCT MCP server. It includes the parent turn's +actual model token, cost, latency, generation, and tool-call rollups returned +by Langfuse. + + +## Optional Codex development setup + +The helper below reads a trusted shell environment file (default: `.env`) +and writes local `.codex/langfuse.json` with mode 600. It requires +`LANGFUSE_PUBLIC_KEY`, `LANGFUSE_SECRET_KEY`, and `LANGFUSE_BASE_URL`. + +```bash +sh scripts/setup-langfuse-codex.sh +``` + +This configures the developer's Codex integration, separately from TCT's +runtime opt-in flag. It is not required to import TCT or run the CLI/MCP server. diff --git a/benchmarks/__init__.py b/benchmarks/__init__.py new file mode 100644 index 0000000..d76d7c8 --- /dev/null +++ b/benchmarks/__init__.py @@ -0,0 +1 @@ +"""Deterministic benchmark fixtures for TCT.""" diff --git a/benchmarks/langfuse_codex_turn_result.json b/benchmarks/langfuse_codex_turn_result.json new file mode 100644 index 0000000..74e819b --- /dev/null +++ b/benchmarks/langfuse_codex_turn_result.json @@ -0,0 +1,18 @@ +{ + "acceptance": "passed", + "dashboard_url": "https://us.cloud.langfuse.com/project/cmthc70qs00tmad0dr7hd1ail/traces/94ac7fb1c9a153df10cca766e4d50a66", + "known_plugin_limitation": "Codex 0.152 records MCP calls through the unified exec wrapper, so tracing plugin 0.1.0 displays the tool observation as exec instead of translator-component-toolkit.optimize_query_for_api.", + "source": "Langfuse trace and observations APIs", + "trace": { + "latency_seconds": 13.463, + "llm_calls": 2, + "model": "gpt-5.6-sol", + "name": "Codex Turn", + "prompt_tokens": 34354, + "completion_tokens": 403, + "total_cost_usd": 0.0450216, + "total_tokens": 34757, + "tool_calls": 1 + }, + "trace_id": "94ac7fb1c9a153df10cca766e4d50a66" +} diff --git a/benchmarks/langfuse_conversation_acceptance.py b/benchmarks/langfuse_conversation_acceptance.py new file mode 100644 index 0000000..af69ee0 --- /dev/null +++ b/benchmarks/langfuse_conversation_acceptance.py @@ -0,0 +1,258 @@ +"""Live Langfuse acceptance probe for conversational-turn metrics. + +Unlike ``langfuse_turns.py``, this uploads two agent turns to a real Langfuse +project and builds its report from observations read back through the API. +""" + +from __future__ import annotations + +import argparse +import json +import os +import time +import uuid +from collections import Counter +from collections.abc import Mapping +from datetime import datetime, timezone +from typing import Any + +from benchmarks.langfuse_turns import ( + API_NAMES, + API_PREDICATES, + IDENTIFIERS, + _fixture_query_provider, + _query, +) + + +TURN_NAME = "TCT acceptance conversational turn" +REQUIRED_ENVIRONMENT = ("LANGFUSE_PUBLIC_KEY", "LANGFUSE_SECRET_KEY") +EXPECTED_METRICS = { + "one_call_per_identifier": { + "agent_turns_returned": 1, + "tool_observations_returned": 7, + "query_identifiers": 14, + }, + "batched_identifiers": { + "agent_turns_returned": 1, + "tool_observations_returned": 1, + "query_identifiers": 8, + }, +} + + +class AcceptanceConfigurationError(RuntimeError): + """Report missing configuration needed for a live acceptance run.""" + + +def validate_environment(environ: Mapping[str, str]) -> None: + """Require credentials without ever including their values in errors.""" + missing = [name for name in REQUIRED_ENVIRONMENT if not environ.get(name)] + if missing: + raise AcceptanceConfigurationError( + "Live Langfuse acceptance requires: " + ", ".join(missing) + ) + + +def _as_dict(value: Any) -> dict[str, Any]: + if isinstance(value, Mapping): + return dict(value) + model_dump = getattr(value, "model_dump", None) + if callable(model_dump): + return model_dump(mode="json") + to_dict = getattr(value, "dict", None) + if callable(to_dict): + return to_dict() + raise TypeError(f"Cannot normalize Langfuse response type {type(value).__name__}") + + +def _observation_data(response: Any) -> list[dict[str, Any]]: + data = response.get("data", []) if isinstance(response, Mapping) else response.data + return [_as_dict(item) for item in data] + + +def summarize_returned_turn( + observations: list[dict[str, Any]], + *, + trace_id: str, + scenario: str, +) -> dict[str, Any]: + """Derive acceptance metrics exclusively from API-returned observations.""" + roots = [item for item in observations if item.get("name") == TURN_NAME] + tools = [ + item + for item in observations + if item.get("name") == "tct.tool._fixture_query_provider" + ] + hashes = Counter( + item.get("metadata", {}).get("tct.input.sha256") + for item in tools + if item.get("metadata", {}).get("tct.input.sha256") + ) + return { + "scenario": scenario, + "trace_id": trace_id, + "agent_turns_returned": len(roots), + "tool_observations_returned": len(tools), + "input_bytes": sum( + item.get("metadata", {}).get("tct.input.bytes", 0) for item in tools + ), + "output_bytes": sum( + item.get("metadata", {}).get("tct.output.bytes", 0) for item in tools + ), + "query_identifiers": sum( + item.get("metadata", {}).get("tct.query.identifier_count", 0) + for item in tools + ), + "unique_tool_inputs": len(hashes), + "repeated_tool_inputs": sum(count - 1 for count in hashes.values()), + } + + +def _run_turn(client: Any, run_id: str, scenario: str, calls: list[list[str]]) -> str: + from TCT.interfaces import invocation + + with client.start_as_current_observation( + as_type="agent", + name=TURN_NAME, + input={"scenario": scenario, "identifier_count": len(IDENTIFIERS)}, + metadata={ + "tct.acceptance": True, + "tct.acceptance.run_id": run_id, + "tct.acceptance.scenario": scenario, + }, + ) as turn: + trace_id = turn.trace_id + results = [] + for identifiers in calls: + results.append( + invocation.invoke( + _fixture_query_provider, + "Fixture KP 0", + _query(identifiers), + API_NAMES, + API_PREDICATES, + _interface="acceptance", + ) + ) + turn.update(output={"tool_calls": len(results), "status": "complete"}) + return trace_id + + +def _read_back( + client: Any, + trace_ids: Mapping[str, str], + *, + timeout_seconds: float, +) -> dict[str, list[dict[str, Any]]]: + deadline = time.monotonic() + timeout_seconds + pending = dict(trace_ids) + returned: dict[str, list[dict[str, Any]]] = {} + while pending and time.monotonic() < deadline: + for scenario, trace_id in list(pending.items()): + response = client.api.observations.get_many( + trace_id=trace_id, + fields="basic,metadata,metrics", + limit=100, + ) + observations = _observation_data(response) + has_turn = any(item.get("name") == TURN_NAME for item in observations) + has_tool = any( + item.get("name") == "tct.tool._fixture_query_provider" + for item in observations + ) + if has_turn and has_tool: + returned[scenario] = observations + del pending[scenario] + if pending: + time.sleep(1) + if pending: + names = ", ".join(sorted(pending)) + raise TimeoutError( + f"Langfuse did not return complete observations within " + f"{timeout_seconds:g}s for: {names}" + ) + return returned + + +def validate_returned_metrics(turns: Mapping[str, Mapping[str, Any]]) -> None: + """Reject partial read-backs and observations missing payload metrics.""" + problems = [] + for scenario, expected in EXPECTED_METRICS.items(): + actual = turns[scenario] + for metric, expected_value in expected.items(): + if actual[metric] != expected_value: + problems.append( + f"{scenario}.{metric}={actual[metric]!r} " + f"(expected {expected_value!r})" + ) + for metric in ("input_bytes", "output_bytes", "unique_tool_inputs"): + if actual[metric] <= 0: + problems.append(f"{scenario}.{metric} is not positive") + if problems: + raise RuntimeError("Invalid Langfuse metric read-back: " + "; ".join(problems)) + + +def run_acceptance(timeout_seconds: float = 30) -> dict[str, Any]: + """Upload conversational turns, read them back, and return their metrics.""" + validate_environment(os.environ) + os.environ["TCT_LANGFUSE_ENABLED"] = "true" + + from langfuse import get_client + + client = get_client() + run_id = str(uuid.uuid4()) + scenarios = { + "one_call_per_identifier": [[identifier] for identifier in IDENTIFIERS], + "batched_identifiers": [IDENTIFIERS], + } + trace_ids = { + scenario: _run_turn(client, run_id, scenario, calls) + for scenario, calls in scenarios.items() + } + client.flush() + returned = _read_back(client, trace_ids, timeout_seconds=timeout_seconds) + turns = { + scenario: summarize_returned_turn( + returned[scenario], trace_id=trace_id, scenario=scenario + ) + for scenario, trace_id in trace_ids.items() + } + validate_returned_metrics(turns) + one_by_one = turns["one_call_per_identifier"] + batched = turns["batched_identifiers"] + return { + "acceptance": "passed", + "source": "Langfuse observations API", + "run_id": run_id, + "read_back_at": datetime.now(timezone.utc).isoformat(), + "turns": turns, + "comparison": { + "tool_calls_avoided": ( + one_by_one["tool_observations_returned"] + - batched["tool_observations_returned"] + ), + "input_bytes_avoided": ( + one_by_one["input_bytes"] - batched["input_bytes"] + ), + "output_bytes_avoided": ( + one_by_one["output_bytes"] - batched["output_bytes"] + ), + }, + } + + +def main() -> int: + parser = argparse.ArgumentParser(description=__doc__) + parser.add_argument("--timeout", type=float, default=30) + arguments = parser.parse_args() + try: + report = run_acceptance(arguments.timeout) + except (AcceptanceConfigurationError, RuntimeError, TimeoutError) as error: + parser.exit(2, f"acceptance failed: {error}\n") + print(json.dumps(report, indent=2, sort_keys=True)) + return 0 + + +if __name__ == "__main__": + raise SystemExit(main()) diff --git a/benchmarks/langfuse_conversation_acceptance_result.json b/benchmarks/langfuse_conversation_acceptance_result.json new file mode 100644 index 0000000..3f2dcb8 --- /dev/null +++ b/benchmarks/langfuse_conversation_acceptance_result.json @@ -0,0 +1,35 @@ +{ + "acceptance": "passed", + "comparison": { + "input_bytes_avoided": 8706, + "output_bytes_avoided": 222, + "tool_calls_avoided": 6 + }, + "read_back_at": "2026-09-02T16:54:05.941518+00:00", + "run_id": "3ec4fa9e-fa5a-44f6-bf48-9e284553818c", + "source": "Langfuse observations API", + "turns": { + "batched_identifiers": { + "agent_turns_returned": 1, + "input_bytes": 1563, + "output_bytes": 457, + "query_identifiers": 8, + "repeated_tool_inputs": 0, + "scenario": "batched_identifiers", + "tool_observations_returned": 1, + "trace_id": "461d85e03201fd38f80d9afd031d32c9", + "unique_tool_inputs": 1 + }, + "one_call_per_identifier": { + "agent_turns_returned": 1, + "input_bytes": 10269, + "output_bytes": 679, + "query_identifiers": 14, + "repeated_tool_inputs": 0, + "scenario": "one_call_per_identifier", + "tool_observations_returned": 7, + "trace_id": "811c2a8882a494120e1edb978e91ab29", + "unique_tool_inputs": 7 + } + } +} diff --git a/benchmarks/langfuse_turns.py b/benchmarks/langfuse_turns.py new file mode 100644 index 0000000..611d929 --- /dev/null +++ b/benchmarks/langfuse_turns.py @@ -0,0 +1,165 @@ +"""Deterministic per-turn benchmarks for TCT's Langfuse telemetry contract. + +The scenarios use in-memory tool functions and observations. They make no +network requests and send no data to Langfuse. +""" + +from __future__ import annotations + +import json +from collections import Counter +from contextlib import contextmanager +from typing import Any + +from TCT.interfaces import invocation, telemetry + + +IDENTIFIERS = [f"NCBIGene:{1000 + index}" for index in range(7)] +API_NAMES = { + f"Fixture KP {index}": f"https://kp-{index}.example/query" for index in range(12) +} +API_PREDICATES = {name: ["biolink:related_to", "biolink:treats"] for name in API_NAMES} + + +def _query(identifier_values: list[str]) -> dict[str, Any]: + return { + "message": { + "query_graph": { + "nodes": { + "genes": {"ids": identifier_values}, + "disease": {"ids": ["MONDO:0005148"]}, + }, + "edges": { + "e0": { + "subject": "disease", + "object": "genes", + "predicates": ["biolink:related_to"], + } + }, + } + } + } + + +def _fixture_query_provider( + api_name: str, + query_json: dict[str, Any], + api_names: dict[str, str], + api_predicates: dict[str, list[str]], +) -> dict[str, Any]: + identifiers = query_json["message"]["query_graph"]["nodes"]["genes"]["ids"] + return { + "provider": api_name, + "nodes": { + identifier: {"name": f"fixture result for {identifier}"} + for identifier in identifiers + }, + } + + +def _collect_turn(calls: list[list[str]]) -> dict[str, Any]: + records: list[dict[str, Any]] = [] + + class Observation: + def __init__(self, record: dict[str, Any]) -> None: + self.record = record + + def update(self, **values: Any) -> None: + self.record["update"] = values + + @contextmanager + def collect_observation(*, name, input_factory, metadata): + input_value = input_factory() + record = { + "name": name, + "input": input_value, + "metadata": dict(metadata), + } + records.append(record) + yield Observation(record) + + original_observer = telemetry.observe_tool + telemetry.observe_tool = collect_observation + try: + for identifiers in calls: + invocation.invoke( + _fixture_query_provider, + "Fixture KP 0", + _query(identifiers), + API_NAMES, + API_PREDICATES, + _interface="mcp", + ) + finally: + telemetry.observe_tool = original_observer + + hashes = Counter(record["metadata"]["tct.input.sha256"] for record in records) + input_bytes = sum(record["metadata"]["tct.input.bytes"] for record in records) + output_bytes = sum( + record["update"]["metadata"]["tct.output.bytes"] for record in records + ) + return { + "tool_calls": len(records), + "unique_inputs": len(hashes), + "repeated_input_calls": sum(count - 1 for count in hashes.values()), + "input_bytes": input_bytes, + "output_bytes": output_bytes, + "total_payload_bytes": input_bytes + output_bytes, + "query_identifiers": sum( + record["metadata"]["tct.query.identifier_count"] for record in records + ), + "provider_metadata_bytes": sum( + record["metadata"]["tct.input.argument.api_names.bytes"] + + record["metadata"]["tct.input.argument.api_predicates.bytes"] + for record in records + ), + } + + +def benchmark_report() -> dict[str, Any]: + """Compare representative agent-turn tool-call shapes.""" + one_by_one = _collect_turn([[identifier] for identifier in IDENTIFIERS]) + batched = _collect_turn([IDENTIFIERS]) + duplicate = _collect_turn([IDENTIFIERS, IDENTIFIERS]) + input_bytes_avoided = one_by_one["input_bytes"] - batched["input_bytes"] + total_bytes_avoided = ( + one_by_one["total_payload_bytes"] - batched["total_payload_bytes"] + ) + + return { + "fixture": "langfuse-turns-v1", + "notes": { + "network_requests": 0, + "langfuse_uploads": 0, + "token_counts": ( + "Model tokens and price belong to the parent Langfuse generation; " + "this report measures TCT tool payloads within each turn." + ), + }, + "turns": { + "one_call_per_identifier": one_by_one, + "batched_identifiers": batched, + "duplicate_batched_calls": duplicate, + }, + "batching_comparison": { + "tool_calls_avoided": one_by_one["tool_calls"] - batched["tool_calls"], + "input_bytes_avoided": input_bytes_avoided, + "input_bytes_reduction_percent": round( + input_bytes_avoided / one_by_one["input_bytes"] * 100, + 1, + ), + "total_payload_bytes_avoided": total_bytes_avoided, + "total_payload_bytes_reduction_percent": round( + total_bytes_avoided / one_by_one["total_payload_bytes"] * 100, + 1, + ), + "provider_metadata_bytes_avoided": ( + one_by_one["provider_metadata_bytes"] + - batched["provider_metadata_bytes"] + ), + }, + } + + +if __name__ == "__main__": + print(json.dumps(benchmark_report(), indent=2, sort_keys=True)) diff --git a/benchmarks/langfuse_turns_baseline.json b/benchmarks/langfuse_turns_baseline.json new file mode 100644 index 0000000..2e45280 --- /dev/null +++ b/benchmarks/langfuse_turns_baseline.json @@ -0,0 +1,48 @@ +{ + "batching_comparison": { + "input_bytes_avoided": 8706, + "input_bytes_reduction_percent": 84.8, + "provider_metadata_bytes_avoided": 7176, + "tool_calls_avoided": 6, + "total_payload_bytes_avoided": 8928, + "total_payload_bytes_reduction_percent": 81.5 + }, + "fixture": "langfuse-turns-v1", + "notes": { + "langfuse_uploads": 0, + "network_requests": 0, + "token_counts": "Model tokens and price belong to the parent Langfuse generation; this report measures TCT tool payloads within each turn." + }, + "turns": { + "batched_identifiers": { + "input_bytes": 1563, + "output_bytes": 457, + "provider_metadata_bytes": 1196, + "query_identifiers": 8, + "repeated_input_calls": 0, + "tool_calls": 1, + "total_payload_bytes": 2020, + "unique_inputs": 1 + }, + "duplicate_batched_calls": { + "input_bytes": 3126, + "output_bytes": 914, + "provider_metadata_bytes": 2392, + "query_identifiers": 16, + "repeated_input_calls": 1, + "tool_calls": 2, + "total_payload_bytes": 4040, + "unique_inputs": 1 + }, + "one_call_per_identifier": { + "input_bytes": 10269, + "output_bytes": 679, + "provider_metadata_bytes": 8372, + "query_identifiers": 14, + "repeated_input_calls": 0, + "tool_calls": 7, + "total_payload_bytes": 10948, + "unique_inputs": 7 + } + } +} diff --git a/scripts/setup-langfuse-codex.sh b/scripts/setup-langfuse-codex.sh new file mode 100755 index 0000000..bb49b06 --- /dev/null +++ b/scripts/setup-langfuse-codex.sh @@ -0,0 +1,46 @@ +#!/bin/sh +set -eu + +env_file=${1:-./.env} +case "$env_file" in + */*) ;; + *) env_file="./$env_file" ;; +esac + +if [ ! -f "$env_file" ]; then + echo "Langfuse environment file not found: $env_file" >&2 + exit 2 +fi + +set -a +. "$env_file" +set +a + +: "${LANGFUSE_PUBLIC_KEY:?LANGFUSE_PUBLIC_KEY is required}" +: "${LANGFUSE_SECRET_KEY:?LANGFUSE_SECRET_KEY is required}" +: "${LANGFUSE_BASE_URL:?LANGFUSE_BASE_URL is required}" + +mkdir -p .codex +umask 077 +temporary_file=$(mktemp .codex/langfuse.json.XXXXXX) +trap 'rm -f "$temporary_file"' EXIT HUP INT TERM + +jq -n \ + --arg public_key "$LANGFUSE_PUBLIC_KEY" \ + --arg secret_key "$LANGFUSE_SECRET_KEY" \ + --arg base_url "$LANGFUSE_BASE_URL" \ + '{ + enabled: true, + public_key: $public_key, + secret_key: $secret_key, + base_url: $base_url, + environment: "development", + tags: ["tct", "codex"] + }' > "$temporary_file" + +mv "$temporary_file" .codex/langfuse.json +chmod 600 .codex/langfuse.json +trap - EXIT HUP INT TERM + +echo "Configured Codex Langfuse tracing in .codex/langfuse.json (mode 600)." +echo "The file is ignored by git; credential values were not printed." diff --git a/tests/test_langfuse_conversation_acceptance.py b/tests/test_langfuse_conversation_acceptance.py new file mode 100644 index 0000000..1b0e6aa --- /dev/null +++ b/tests/test_langfuse_conversation_acceptance.py @@ -0,0 +1,89 @@ +from benchmarks.langfuse_conversation_acceptance import ( + AcceptanceConfigurationError, + TURN_NAME, + summarize_returned_turn, + validate_environment, + validate_returned_metrics, +) + + +def test_live_acceptance_requires_credentials_without_disclosing_values(): + try: + validate_environment({"LANGFUSE_PUBLIC_KEY": "pk-secret-value"}) + except AcceptanceConfigurationError as error: + message = str(error) + else: + raise AssertionError("Expected missing secret key to fail") + + assert "LANGFUSE_SECRET_KEY" in message + assert "pk-secret-value" not in message + + +def test_summarizes_metrics_returned_by_langfuse(): + observations = [ + {"name": TURN_NAME, "metadata": {"tct.acceptance": True}}, + { + "name": "tct.tool._fixture_query_provider", + "metadata": { + "tct.input.bytes": 100, + "tct.output.bytes": 20, + "tct.query.identifier_count": 3, + "tct.input.sha256": "same", + }, + }, + { + "name": "tct.tool._fixture_query_provider", + "metadata": { + "tct.input.bytes": 100, + "tct.output.bytes": 20, + "tct.query.identifier_count": 3, + "tct.input.sha256": "same", + }, + }, + ] + + metrics = summarize_returned_turn( + observations, + trace_id="abc123", + scenario="duplicate", + ) + + assert metrics == { + "scenario": "duplicate", + "trace_id": "abc123", + "agent_turns_returned": 1, + "tool_observations_returned": 2, + "input_bytes": 200, + "output_bytes": 40, + "query_identifiers": 6, + "unique_tool_inputs": 1, + "repeated_tool_inputs": 1, + } + + +def test_rejects_a_read_back_without_payload_metrics(): + turns = { + "one_call_per_identifier": { + "agent_turns_returned": 1, + "tool_observations_returned": 7, + "query_identifiers": 14, + "input_bytes": 0, + "output_bytes": 0, + "unique_tool_inputs": 0, + }, + "batched_identifiers": { + "agent_turns_returned": 1, + "tool_observations_returned": 1, + "query_identifiers": 8, + "input_bytes": 0, + "output_bytes": 0, + "unique_tool_inputs": 0, + }, + } + + try: + validate_returned_metrics(turns) + except RuntimeError as error: + assert "input_bytes is not positive" in str(error) + else: + raise AssertionError("Expected missing returned metrics to fail acceptance") diff --git a/tests/test_langfuse_turn_benchmark.py b/tests/test_langfuse_turn_benchmark.py new file mode 100644 index 0000000..cc035c5 --- /dev/null +++ b/tests/test_langfuse_turn_benchmark.py @@ -0,0 +1,26 @@ +"""Regression coverage for deterministic per-turn telemetry benchmarks.""" + +import json +from pathlib import Path + +from benchmarks.langfuse_turns import benchmark_report + + +def test_turn_benchmark_exposes_batching_and_duplicate_calls(): + report = benchmark_report() + turns = report["turns"] + + assert turns["one_call_per_identifier"]["tool_calls"] == 7 + assert turns["batched_identifiers"]["tool_calls"] == 1 + assert turns["duplicate_batched_calls"]["repeated_input_calls"] == 1 + assert report["batching_comparison"]["tool_calls_avoided"] == 6 + assert report["batching_comparison"]["input_bytes_avoided"] > 0 + assert report["batching_comparison"]["provider_metadata_bytes_avoided"] > 0 + + +def test_checked_in_turn_baseline_matches_the_benchmark(): + baseline_path = ( + Path(__file__).parents[1] / "benchmarks" / "langfuse_turns_baseline.json" + ) + + assert json.loads(baseline_path.read_text()) == benchmark_report()