From 80e8c06eec3b617be2aa1c1fe46d4bdc9e16d7f8 Mon Sep 17 00:00:00 2001 From: Markus Neusinger <2921697+MarkusNeusinger@users.noreply.github.com> Date: Fri, 16 Oct 2026 15:14:11 +0200 Subject: [PATCH 1/6] feat(agents): content-free usage telemetry for the agent chat Four events (data_input, chat_turn, pipeline_run, outcome) as strict pydantic models on their own logger with log_type=agent_telemetry; closed warning codes; salted per-process session hashes (the attribution lines' too); the outcome beacon route on the agents service and the BFF; BigQuery views (dedup, sessions, usage, failures, cost) and the admin GET /debug/agent-usage via the BigQuery REST API; docs with the one-time setup. Co-Authored-By: Claude Opus 5.5 Claude-Session: https://claude.ai/code/session_01GdMSqLR5ww4ji74EUSmk9M --- agents/anyplot/data/warning_codes.py | 243 +++++ agents/anyplot/dataset_guard.py | 1 + agents/anyplot/pipeline.py | 84 ++ agents/anyplot/plugins/budget.py | 2 + agents/anyplot/plugins/ledger.py | 92 +- agents/anyplot/plugins/scope_guard.py | 1 + agents/anyplot/plugins/tool_safety.py | 2 + agents/anyplot/telemetry.py | 922 ++++++++++++++++++ agents/anyplot/tools/session.py | 3 + agents/evals/matrix.py | 2 + agents/evals/prep.py | 2 + agents/main.py | 264 ++++- agents/stream.py | 4 + agents/telemetry/views/01_events.sql | 26 + agents/telemetry/views/02_event_views.sql | 97 ++ agents/telemetry/views/03_sessions.sql | 98 ++ agents/telemetry/views/04_usage_failures.sql | 71 ++ agents/telemetry/views/05_prices_cost.sql | 52 + api/agent_usage.py | 240 +++++ api/routers/agent.py | 35 + api/routers/debug.py | 26 +- changelog.d/agent-usage-telemetry.md | 17 + core/config.py | 8 + docs/concepts/agent-network.md | 11 + docs/index.md | 3 + docs/reference/agent-telemetry.md | 223 +++++ docs/reference/api.md | 29 + docs/reference/plausible.md | 8 + tests/unit/agents/data/test_warning_codes.py | 189 ++++ .../agents/evals/test_telemetry_prices.py | 52 + .../agents/runtime/test_data_code_tool.py | 6 +- tests/unit/agents/runtime/test_telemetry.py | 366 +++++++ tests/unit/api/test_agent_router.py | 46 + tests/unit/api/test_agent_usage.py | 188 ++++ 34 files changed, 3370 insertions(+), 43 deletions(-) create mode 100644 agents/anyplot/data/warning_codes.py create mode 100644 agents/anyplot/telemetry.py create mode 100644 agents/telemetry/views/01_events.sql create mode 100644 agents/telemetry/views/02_event_views.sql create mode 100644 agents/telemetry/views/03_sessions.sql create mode 100644 agents/telemetry/views/04_usage_failures.sql create mode 100644 agents/telemetry/views/05_prices_cost.sql create mode 100644 api/agent_usage.py create mode 100644 changelog.d/agent-usage-telemetry.md create mode 100644 docs/reference/agent-telemetry.md create mode 100644 tests/unit/agents/data/test_warning_codes.py create mode 100644 tests/unit/agents/evals/test_telemetry_prices.py create mode 100644 tests/unit/agents/runtime/test_telemetry.py create mode 100644 tests/unit/api/test_agent_usage.py diff --git a/agents/anyplot/data/warning_codes.py b/agents/anyplot/data/warning_codes.py new file mode 100644 index 00000000000..0f5789dab20 --- /dev/null +++ b/agents/anyplot/data/warning_codes.py @@ -0,0 +1,243 @@ +"""Closed codes for the parser's and the binding check's warning sentences, for the usage telemetry. + +The parser (`parse.py`), the generator (`generate.py`), the dataset route (`opening.py`) and the +binding check (`bindings.py`) say what they did in sentences written for the user, and many of them +name a column or quote a count. The usage telemetry (`telemetry.py`) never logs a sentence: it logs +the code `warning_code` maps a sentence to, from the closed set `WarningCode`, and `other` for a +sentence no rule knows. The rules match the fixed parts of each sentence only, anchored where a +column name could sit in between, so a crafted column name can at worst pick a wrong code from the +set, never put its own text into a log line. + +The table lives here, next to the modules that write the sentences, so a new warning and its code +change together: `tests/unit/agents/data/test_warning_codes.py` feeds every sentence the parser +writes for its fixtures through `warning_code` and fails on `other`. +""" + +import re +from typing import Literal, cast, get_args + + +WarningCode = Literal[ + "notes_instruction_like", + "notes_refused", + "notes_unchecked", + "notes_cut", + "nfc_normalised", + "single_space_separator", + "lines_above_left_out", + "lines_below_left_out", + "summary_rows_left_out", + "short_rows_positional", + "short_rows_padded", + "trailing_empty_fields", + "multiple_tables", + "headerless_list", + "category_header", + "repeated_keys", + "nan_infinity_missing", + "object_as_pairs", + "fields_beside_records_to_notes", + "list_fields_beside_left_out", + "nested_fields_dotted", + "deep_fields_left_out", + "list_fields_expanded", + "record_id_added", + "list_values_thinned", + "records_missing_keys", + "jsonstat_dimension_renamed", + "jsonstat_labels_with_codes", + "jsonstat_status_ignored", + "jsonstat_single_category_dims", + "empty_header_named", + "long_header_shortened", + "header_shortened_for_unit", + "duplicate_header_renamed", + "empty_columns_dropped", + "missing_marks_emptied", + "header_whitespace_replaced", + "headers_look_like_values", + "invisible_chars_removed", + "unit_removed_from_values", + "unit_moved_to_name", + "numbers_respelled", + "valueless_columns_dropped", + "row_number_column_dropped", + "constant_columns_to_notes", + "leading_zeros_text", + "big_integers_text", + "decimal_commas_ambiguous", + "thousands_separators", + "utc_offsets_normalised", + "sub_microsecond_text", + "date_order_ambiguous", + "months_read", + "na_as_code", + "digits_rounded", + "cells_trimmed", + "cells_read_missing", + "series_undefined", + "more_omitted", + "other", +] +"""Every warning code; `other` is a sentence no rule in `_RULES` knows (a new warning without its code).""" + +WARNING_CODES: frozenset[str] = frozenset(get_args(WarningCode)) + +_COLUMN = r"column '.*': " +"""A column-scoped sentence's lead; greedy, so the rule's own text is matched at the end.""" + +_RULES: tuple[tuple[re.Pattern[str], WarningCode], ...] = tuple( + (re.compile(pattern, re.DOTALL), cast(WarningCode, code)) # each code is in the set: test_warning_codes.py + for pattern, code in ( + ( + r"^the notes above or below the table were left out: they read like instructions to an AI$", + "notes_instruction_like", + ), + ( + r"^the notes above or below the table were left out: they are not about the data or the plot$", + "notes_refused", + ), + (r"^the notes above or below the table were left out: they could not be checked right now$", "notes_unchecked"), + ( + r"^the (?:notes around the table|notes from the fields beside the records|dataset's notes) were cut at [0-9,]+ characters$", + "notes_cut", + ), + (r"^[0-9,]+ lines? held decomposed characters .* \(NFC\); the text reads the same$", "nfc_normalised"), + ( + r"^single spaces read as the column separator; check that no value was split in two$", + "single_space_separator", + ), + (r"^left out [0-9,]+ lines? above the table \(a title, notes or metadata\)$", "lines_above_left_out"), + (r"^left out [0-9,]+ lines? below the table \(notes or a source line\)$", "lines_below_left_out"), + (r"^left out [0-9,]+ summary rows? at the end of the table .*$", "summary_rows_left_out"), + ( + r"^[0-9,]+ rows? had fewer fields than the header; their cells were placed under the header's columns by their position.*$", + "short_rows_positional", + ), + (r"^[0-9,]+ rows? had fewer fields than the header; the missing cells are empty.*$", "short_rows_padded"), + ( + r"^[0-9,]+ rows? had empty fields past the header's [0-9]+ columns; those empty fields were left out$", + "trailing_empty_fields", + ), + ( + r"^the data holds [0-9]+ tables, each under its own header; .*the table picker in the data panel reads another one$", + "multiple_tables", + ), + ( + r"^no header line: the first line is a value, kept as the first row, and the column is named '.*'$", + "headerless_list", + ), + (r"^read '.*' as the column name$", "category_header"), + (r"^[0-9,]+ keys? repeated; only the last value of a repeated key is kept$", "repeated_keys"), + (r"^[0-9,]+ values? NaN, Infinity or -Infinity read as missing$", "nan_infinity_missing"), + (r"^the object was read as [0-9,]+ category/value pairs?, one row per key; .*$", "object_as_pairs"), + ( + r"^[0-9,]+ fields? beside the records \(such as paging or a timestamp\) moved to the notes$", + "fields_beside_records_to_notes", + ), + (r"^[0-9,]+ fields? beside the records that hold lists left out$", "list_fields_beside_left_out"), + (r"^fields of nested objects became columns named with a dot, such as parent\.field$", "nested_fields_dotted"), + ( + r"^[0-9,]+ fields? holding a list of objects or lists, or objects nested more than [0-9]+ levels deep, left out: .*$", + "deep_fields_left_out", + ), + (r"^list fields expanded into one row per value.*$", "list_fields_expanded"), + (r"^column '.*' added: it numbers the records, so the rows of each stay together$", "record_id_added"), + (r"^the lists hold [0-9,]+ values, more than the [0-9,]+ rows a table may have: .*$", "list_values_thinned"), + (r"^[0-9,]+ records? [a-z ]+ some keys; those cells are empty$", "records_missing_keys"), + (_COLUMN + r"a dimension named like the value column renamed$", "jsonstat_dimension_renamed"), + (_COLUMN + r"labels shared by two categories carry their codes$", "jsonstat_labels_with_codes"), + (r"^the dataset's status symbols \(such as provisional or estimated\) are ignored$", "jsonstat_status_ignored"), + (r"^[0-9,]+ dimensions? with a single category left out of the columns; .*$", "jsonstat_single_category_dims"), + (r"^column [0-9]+: empty header named '.*'$", "empty_header_named"), + (r"^column [0-9]+: header longer than [0-9]+ characters shortened to '.*'$", "long_header_shortened"), + ( + r"^column [0-9]+: header shortened to '.*' so the unit fits within [0-9]+ characters$", + "header_shortened_for_unit", + ), + (r"^column [0-9]+: duplicate header '.*' renamed to '.*'$", "duplicate_header_renamed"), + (r"^[0-9,]+ empty columns? without a header dropped$", "empty_columns_dropped"), + ( + r"^[0-9,]+ cells? of number or date columns matched the missing-value mark the notes declare and are empty$", + "missing_marks_emptied", + ), + (_COLUMN + r"tabs or line breaks in the header replaced with spaces$", "header_whitespace_replaced"), + ( + r"^every header looks like a value; the first row may be data rather than column names$", + "headers_look_like_values", + ), + (_COLUMN + r"invisible formatting characters \(zero-width or bidi\) removed$", "invisible_chars_removed"), + (_COLUMN + r"unit '.*' removed from the values; the name already carries it$", "unit_removed_from_values"), + (_COLUMN + r"unit '.*' moved from the values into the name '.*'$", "unit_moved_to_name"), + (r"^[0-9,]+ numbers? written in another form with the same value .*$", "numbers_respelled"), + (r"^[0-9,]+ columns? without any value dropped: .*$", "valueless_columns_dropped"), + (r"^left out the column .*: its [0-9,]+ cells? only number the rows .*$", "row_number_column_dropped"), + (r"^[0-9,]+ columns? with the same text in every row moved to the notes: .*$", "constant_columns_to_notes"), + (_COLUMN + r"numbers with leading zeros kept as text$", "leading_zeros_text"), + (_COLUMN + r"whole numbers beyond the 64-bit range kept as text, so no digit is lost$", "big_integers_text"), + ( + _COLUMN + r"commas read as decimal commas; remove them first if they separate thousands$", + "decimal_commas_ambiguous", + ), + (_COLUMN + r".* read as thousands separators(?:, as in the file's other columns)?$", "thousands_separators"), + ( + _COLUMN + r"times with the UTC offsets .* written in UTC \(\+00:00\); the instants are the same$", + "utc_offsets_normalised", + ), + (_COLUMN + r"kept as text: times with fractions below a microsecond .*$", "sub_microsecond_text"), + (_COLUMN + r"day and month order is ambiguous; read as (?:DD/MM/YYYY|MM/DD/YYYY)$", "date_order_ambiguous"), + (_COLUMN + r"[0-9,]+ cells? read as months \(YYYY-MM, each as the first day of the month\)$", "months_read"), + (_COLUMN + r"NA read as a code \(such as Namibia or North America\), not as a missing cell$", "na_as_code"), + (_COLUMN + r"[0-9,]+ values? with more significant digits than a number holds .*$", "digits_rounded"), + (_COLUMN + r"spaces around the text removed in [0-9,]+ cells?$", "cells_trimmed"), + (_COLUMN + r"[0-9,]+ cells? read as missing \(.*\)$", "cells_read_missing"), + (r"^.*: [0-9]+ of [0-9]+ values (?:is|are) undefined and left empty$", "series_undefined"), + (r"^[0-9,]+ more warnings omitted$", "more_omitted"), + ) +) +"""The rules in order, the first that matches wins: the column-scoped rules come after every rule that starts with +fixed text, so a column name cannot shadow a sentence that has none.""" + + +def warning_code(sentence: str) -> WarningCode: + """The closed code of one warning sentence; `other` when no rule matches.""" + for pattern, code in _RULES: + if pattern.match(sentence): + return code + return "other" + + +def warning_codes(sentences: list[str]) -> list[WarningCode]: + """The distinct codes of `sentences`, sorted, so a line carries each code once.""" + return sorted({warning_code(sentence) for sentence in sentences}) + + +BindingWarningCode = Literal["negative_values", "negative_unbound", "label_sibling", "window_mismatch", "other"] +"""The binding check's warnings (`bindings.check_bindings`) as closed codes.""" + +_BINDING_RULES: tuple[tuple[re.Pattern[str], BindingWarningCode], ...] = tuple( + (re.compile(pattern, re.DOTALL), cast(BindingWarningCode, code)) + for pattern, code in ( + ( + r"^.*: not bound, because every column that fits it \(.*\) has negative values; this role needs values of 0 or more$", + "negative_unbound", + ), + ( + r"^.*: column '.*' has negative values \(down to .*\); this role needs values of 0 or more$", + "negative_values", + ), + ( + r"^.*: bound to '.*', one of [0-9]+ number columns .* turn the columns into rows first \(\.to_long\(\)\)$", + "label_sibling", + ), + (r"^.*: column '.*' holds a .*-period value; the spec's convention is .*$", "window_mismatch"), + ) +) + + +def binding_warning_codes(sentences: list[str]) -> list[BindingWarningCode]: + """The distinct closed codes of the binding check's warnings, sorted; `other` for a sentence no rule knows.""" + codes: set[BindingWarningCode] = set() + for sentence in sentences: + codes.add(next((code for pattern, code in _BINDING_RULES if pattern.match(sentence)), "other")) + return sorted(codes) diff --git a/agents/anyplot/dataset_guard.py b/agents/anyplot/dataset_guard.py index 2a538e99322..5ea9130e2a4 100644 --- a/agents/anyplot/dataset_guard.py +++ b/agents/anyplot/dataset_guard.py @@ -80,6 +80,7 @@ async def judge_dataset(services: Services, ledger: RequestLedger, parts: list[s # Into the request too, like the scope guard: inside a chat turn (`run_data_code`) the root's next # budget check must count these tokens (#12172 review). ledger.judge_tokens += answer.tokens + ledger.book_judge(answer.input_tokens, answer.output_tokens, answer.tokens) ledger.budget_tokens += weighted services.usage.add_tokens(ledger.user_id, weighted) attribution( diff --git a/agents/anyplot/pipeline.py b/agents/anyplot/pipeline.py index 6f9666ffe8f..f0cdb048509 100644 --- a/agents/anyplot/pipeline.py +++ b/agents/anyplot/pipeline.py @@ -114,6 +114,7 @@ from core.defects import DEFECT_RE +from . import telemetry from .briefs import adapter_profile, profile_summary, spec_brief from .code.edits import ( MAX_CHANGE_LITERAL_CHARS, @@ -457,6 +458,22 @@ class Run: """The `changes` notes of every adapter call so far whose plan changed the code, in order (`note_changes`).""" expected_texts: list[str] = field(default_factory=list) """The texts and number formats the user asked to see (`PipelineArgs.expected_texts`), checked per render.""" + base: Literal["catalogue", "previous"] = "catalogue" + round_scores: dict[int, int] = field(default_factory=dict) + """The catalogue review score of each reviewed round's latest review, by round: the telemetry's score buckets.""" + forced_ids: set[str] = field(default_factory=set) + """Review ids under `VISIBLE_DEFECT_IDS` that forced a repair round.""" + review_ids: set[str] = field(default_factory=set) + """Every review id any review of the run named.""" + gate_ids: set[str] = field(default_factory=set) + """Every gate id any render of the run failed (`GateReport.failed_gates`).""" + sandbox_failures: set[str] = field(default_factory=set) + """The closed classes of the renders that failed in the sandbox (`telemetry.sandbox_failure`).""" + stage_s: dict[str, float] = field(default_factory=dict) + """Seconds spent in `adapt`, `check`, `render` and `review`, summed over the attempts.""" + + def timed(self, stage: str, seconds: float) -> None: + self.stage_s[stage] = self.stage_s.get(stage, 0.0) + seconds ROUND_OUTCOMES: dict[Stage, RoundOutcome] = { @@ -1287,7 +1304,9 @@ async def run_pipeline(ctx: Context, node_input: PipelineArgs) -> AsyncGenerator language=view.language, change_request=node_input.change_request, expected_texts=list(node_input.expected_texts), + base=node_input.base, ) + started = time.monotonic() scope = f"pipeline-{secrets.token_hex(6)}" try: async for event in _attempts(ctx, services, settings, run, node_input, scope): @@ -1318,6 +1337,7 @@ async def run_pipeline(ctx: Context, node_input: PipelineArgs) -> AsyncGenerator ledger.plot_shipped = result.status in ("ok", "needs_attention") settle_pending_request(ctx.state, node_input, ledger.plot_shipped) _attribute_result(ledger, run, result) + _telemetry_run(ledger, ctx.session.id, ctx.invocation_id, run, result, time.monotonic() - started) yield close_scope(scope) yield _result_event(result) @@ -1356,6 +1376,51 @@ def _attribute_result(ledger: RequestLedger, run: Run, result: PlotResult) -> No ) +def _defect_ids(lines: Sequence[str]) -> list[str]: + """The review ids of defect lines (`DEFECT_RE`); a line without the grammar names none.""" + return sorted({found for line in lines if (m := DEFECT_RE.match(line)) for found in m["ids"].split(", ")}) + + +def _telemetry_run( + ledger: RequestLedger, session_id: str, invocation_id: str, run: Run, result: PlotResult, seconds: float +) -> None: + """The usage telemetry's `pipeline_run` line (`telemetry.py`): ids, buckets, counts and times, never text. + + One line per run: `run_index` is the session's run number for this invocation, so a node ADK re-runs + writes the same idempotency key and the BigQuery base view keeps one row. + """ + shipped = shipped_candidate(run) if result.status in ("ok", "needs_attention") else None + index = telemetry.run_index(session_id, invocation_id) + telemetry.write( + telemetry.PipelineRun, + ledger, + attempt=f"run{index}", + run_index=index, + base=run.base, + status=result.status, + reason=result.reason, + stage=run.stage, + attempts=result.attempts, + rounds=len(run.rounds), + reviews=run.reviews, + score_round1=telemetry.score_bucket(run.round_scores.get(1)), + score_round2=telemetry.score_bucket(run.round_scores.get(2)), + shipped_attempt=shipped.attempt if shipped else None, + shipped_round=round_number(run, shipped.attempt) if shipped else None, + forced_repair=sorted(run.forced_ids), + gates_failed=sorted(run.gate_ids), + reviewer_defects=sorted(run.review_ids), + residual_defects=_defect_ids(result.residual_defects), + sandbox_failures=sorted(run.sandbox_failures), + adapt_ms=telemetry.ms(run.stage_s.get("adapt", 0.0)), + check_ms=telemetry.ms(run.stage_s.get("check", 0.0)), + render_ms=telemetry.ms(run.stage_s.get("render", 0.0)), + review_ms=telemetry.ms(run.stage_s.get("review", 0.0)), + total_ms=telemetry.ms(seconds), + steps=telemetry.step_tokens(ledger.steps, steps=("adapter", "reviewer")), + ) + + def close_scope(scope: str) -> Event: """The event that ends the sub-agents' isolation scope. @@ -1459,7 +1524,9 @@ async def _attempts( file_name=file_name, ) ledger.adapter_allow_full = request.allow_full + started = time.monotonic() answer = await _adapt(ctx, scope, render_adapt_request(request, view), view.library) + run.timed("adapt", time.monotonic() - started) call = _call_fields(ledger, adapter) if isinstance(answer, AdaptMiss): attribution("pipeline_adapt", ledger, attempt=attempt, outcome=answer.outcome, **call) @@ -1483,8 +1550,10 @@ async def _attempts( ) yield _status(run, "checking", attempt) + started = time.monotonic() applied = apply_plan(working, plan) if applied.code is None: + run.timed("check", time.monotonic() - started) attribution( "pipeline_check", ledger, @@ -1507,6 +1576,7 @@ async def _attempts( origin=view.normalised, data_texts=data_texts, ) + run.timed("check", time.monotonic() - started) adaptation_lines = check.defects attribution( "pipeline_check", @@ -1551,7 +1621,13 @@ async def _attempts( started = time.monotonic() rendered = await services.backend.render(job) render_s = time.monotonic() - started + run.timed("render", render_s) report = evaluate(rendered, themes=job.themes, library=view.library, rows=rows, plot_language=run.language) + run.gate_ids.update(report.failed_gates) + for theme_output in rendered.outputs.values(): + failure = telemetry.sandbox_failure(theme_output) + if failure is not None: + run.sandbox_failures.add(failure) if report.passed_host_gates: run.repair_hints = list(report.repair_hints) attribution( @@ -1669,7 +1745,9 @@ async def _attempts( dataset_context=context, file_name=file_name, ) + started = time.monotonic() verdict = await _review(ctx, scope, review_request, run.theme) + run.timed("review", time.monotonic() - started) if verdict is None and settings.check_retry and deadline.allows(REVIEWER_P95_S): # An unreadable review is asked once more before the render ships unreviewed (owner rule: # never unfinished for a formal miss; one in 26 Vertex runs on 2026-10-14). @@ -1680,7 +1758,9 @@ async def _attempts( verdict="unreadable_retry", **_call_fields(ledger, REVIEWER_NAME), ) + started = time.monotonic() verdict = await _review(ctx, scope, review_request, run.theme) + run.timed("review", time.monotonic() - started) review_call = _call_fields(ledger, REVIEWER_NAME) if verdict is None: attribution("pipeline_review", ledger, attempt=attempt, verdict="unreadable", **review_call) @@ -1697,6 +1777,10 @@ async def _attempts( **review_call, ) run.reviewed = True + run.review_ids.update(verdict.ids) + run.forced_ids.update(set(_defect_ids(verdict.visible)) & VISIBLE_DEFECT_IDS) + if verdict.score is not None: + run.round_scores[round_number(run, attempt)] = verdict.score candidate.reviewed_ok = verdict.ok candidate.status_ok = verdict.ok if verdict.status_ok is None else verdict.status_ok candidate.review_score = verdict.score diff --git a/agents/anyplot/plugins/budget.py b/agents/anyplot/plugins/budget.py index e3a6f08436d..8e202d8bc90 100644 --- a/agents/anyplot/plugins/budget.py +++ b/agents/anyplot/plugins/budget.py @@ -94,6 +94,8 @@ def book_response(ledger: RequestLedger, agent_name: str, session_user: str, llm breakdown = usage_breakdown(llm_response.usage_metadata) finish = finish_name(llm_response.finish_reason) ledger.last_calls[agent_name] = CallFacts(finish, breakdown["candidates"], breakdown["thoughts"]) + # The usage telemetry's tokens per step (`telemetry.step_tokens`): counts and the model id only. + ledger.book_step(agent_name, model=llm_response.model_version, usage=breakdown, cached=cached) attribution( "model", ledger, diff --git a/agents/anyplot/plugins/ledger.py b/agents/anyplot/plugins/ledger.py index c9b8c01c8b3..5b8464690d3 100644 --- a/agents/anyplot/plugins/ledger.py +++ b/agents/anyplot/plugins/ledger.py @@ -32,8 +32,10 @@ """ import hashlib +import hmac import json import logging +import secrets import time from collections import OrderedDict, deque from collections.abc import Awaitable, Callable @@ -45,12 +47,29 @@ from ..settings import AgentSettings -logger = logging.getLogger("anyplot.agents.attribution") +ATTRIBUTION_LOGGER = "anyplot.agents.attribution" +"""The logger of the attribution lines and of the usage telemetry (`telemetry.py`), shipped to stdout by +`agents/main.py` `enable_attribution_log`.""" +logger = logging.getLogger(ATTRIBUTION_LOGGER) RequestKind = Literal["action", "text"] MAX_ADHOC_LEDGERS = 64 +@dataclass +class StepUsage: + """What one agent (or the judges, under `judge`) spent in a request: calls and tokens by kind, and the model.""" + + calls: int = 0 + input: int = 0 + """Prompt tokens, cached and cache-write ones included, plus the Gemini tool-use prompt.""" + output: int = 0 + """Candidates plus thoughts.""" + cached: int = 0 + cache_write: int = 0 + model: str | None = None + + @dataclass(frozen=True, slots=True) class CallFacts: """The content-free facts of one model call: how it finished and the output tokens it spent.""" @@ -121,6 +140,40 @@ class RequestLedger: cancelled, after the route failed the turn with `capacity` (or ended it). None without a route (`adk web`), whose runs bypass the queue. The one helper for every sandbox run of a turn: `plot_pipeline` and `run_data_code` await it before they render or run code.""" + spec_id: str = "" + library: str = "" + turn: int = 0 + """The session's spec, library and user turn, set once by the `/v1` route from session state, for the + usage telemetry's envelope (`telemetry.py`); never set per call site.""" + steps: dict[str, StepUsage] = field(default_factory=dict) + """Calls and tokens by agent name (`book_step`), and the judges' under `judge` (`book_judge`).""" + tools: list[str] = field(default_factory=list) + """The tools the root called and ToolSafety allowed, in order (closed names only: `telemetry.TOOL_NAMES`).""" + prep_ops: list[str] = field(default_factory=list) + """The operation names of every `prepare_data` call that changed the data (closed: `telemetry.PREP_OPS`).""" + reply_question: bool = False + """The root's last reply sent to the page ends with a question mark (`stream.Translator`); the text is never kept.""" + + def book_step(self, agent: str, *, model: str | None, usage: dict[str, int], cached: int) -> None: + """Add one model call to the agent's step usage (`usage` as `usage_breakdown` counts it).""" + step = self.steps.setdefault(agent, StepUsage()) + step.calls += 1 + step.input += usage.get("prompt", 0) + usage.get("tool_use_prompt", 0) + step.output += usage.get("candidates", 0) + usage.get("thoughts", 0) + step.cached += cached + step.cache_write += usage.get("cache_write", 0) + if model: + step.model = model + + def book_judge(self, input_tokens: int, output_tokens: int, tokens: int = 0) -> None: + """Add one judge verdict (scope, dataset, notes, file name, transcriber) to the `judge` step.""" + step = self.steps.setdefault("judge", StepUsage()) + step.calls += 1 + if input_tokens or output_tokens: + step.input += input_tokens + step.output += output_tokens + else: + step.input += tokens # no split reported: the plain total counts as input def refuse(self, code: str, text: str) -> None: if self.refusal is None and self.error is None: @@ -170,13 +223,19 @@ class _Day: user_messages: dict[str, int] = field(default_factory=dict) user_pastes: dict[str, int] = field(default_factory=dict) user_conversions: dict[str, int] = field(default_factory=dict) + user_outcomes: dict[str, int] = field(default_factory=dict) MESSAGE_WINDOW_S = 60.0 """The sliding window of `AGENT_USER_MESSAGES_PER_MINUTE` and `AGENT_USER_PASTES_PER_MINUTE`.""" MessageLimit = Literal["rate_limited", "daily_limit"] """Why `message_limit` or `paste_limit` refuses: the minute's count, or the day's.""" -WindowKind = Literal["message", "paste", "convert"] +WindowKind = Literal["message", "paste", "convert", "outcome"] +OUTCOMES_PER_MINUTE = 30 +OUTCOMES_PER_DAY = 2000 +"""The page's outcome beacons per user (`POST /v1/sessions/{sid}/outcome`): a copy or a download is one click, so +30 in a minute is far beyond use; more get `429 rate_limited`, which the page ignores. Constants, not settings: the +beacon spends no model call and no sandbox time, the cap only bounds the telemetry lines one user can write.""" class UsageBook: @@ -313,6 +372,16 @@ def add_conversion(self, user_id: str) -> None: day.user_conversions[user_id] = day.user_conversions.get(user_id, 0) + 1 self._stamp("convert", user_id) + def outcome_limit(self, user_id: str) -> MessageLimit | None: + """Why the user's page may not write another outcome beacon now, or None (`OUTCOMES_PER_MINUTE`, `_PER_DAY`).""" + count = self._current().user_outcomes.get(user_id, 0) + return self._limit("outcome", user_id, count, OUTCOMES_PER_MINUTE, OUTCOMES_PER_DAY) + + def add_outcome(self, user_id: str) -> None: + day = self._current() + day.user_outcomes[user_id] = day.user_outcomes.get(user_id, 0) + 1 + self._stamp("outcome", user_id) + def _limit( self, kind: WindowKind, user_id: str, today_count: int, per_minute: int, per_day: int ) -> MessageLimit | None: @@ -492,8 +561,25 @@ def argument_hash(arguments: Any) -> str: return hashlib.sha256(text.encode()).hexdigest()[:12] +SESSION_SALT = secrets.token_bytes(32) +"""The per-process salt of every session hash a log line carries: the attribution lines' `session` and the usage +telemetry's (`telemetry.session_key`). The session id travels in every `/v1` request URL Cloud Run logs, so a plain +hash of it could be recomputed and joined to the request log; sessions live in memory and end with the instance, +so a salt that ends with the process keeps a session's lines together for exactly as long as it exists. Never +logged, never configured.""" + + +def salted(session_id: str, domain: str, length: int) -> str: + """HMAC-SHA256 of the session id under `SESSION_SALT`, separated by `domain`, so the attribution hash and the + telemetry hash of one session differ and the two kinds of lines cannot be joined by it.""" + if not session_id: + return "" + return hmac.new(SESSION_SALT, f"{domain}:{session_id}".encode(), hashlib.sha256).hexdigest()[:length] + + def session_hash(session_id: str) -> str: - return hashlib.sha256(session_id.encode()).hexdigest()[:12] if session_id else "" + """The attribution lines' `session`: 12 hex digits, salted (`salted`).""" + return salted(session_id, "attribution", 12) def attribution(hook: str, ledger: RequestLedger, **fields: Any) -> None: diff --git a/agents/anyplot/plugins/scope_guard.py b/agents/anyplot/plugins/scope_guard.py index 9a2fd3de53f..4e65559dd60 100644 --- a/agents/anyplot/plugins/scope_guard.py +++ b/agents/anyplot/plugins/scope_guard.py @@ -304,6 +304,7 @@ async def _judge( verdict.tokens, verdict.input_tokens, verdict.output_tokens, settings.judge_model ) ledger.judge_tokens += verdict.tokens + ledger.book_judge(verdict.input_tokens, verdict.output_tokens, verdict.tokens) ledger.budget_tokens += weighted services.usage.add_tokens(user, weighted) ledger.lang = verdict.lang diff --git a/agents/anyplot/plugins/tool_safety.py b/agents/anyplot/plugins/tool_safety.py index 1b53ba3f96f..d95961997e9 100644 --- a/agents/anyplot/plugins/tool_safety.py +++ b/agents/anyplot/plugins/tool_safety.py @@ -319,6 +319,8 @@ async def before_tool_callback( "tool", ledger, agent=agent, tool=tool.name, args=argument_hash(tool_args), verdict=verdict or "allowed" ) if verdict is None: + # The usage telemetry's `tools` (`telemetry.TOOL_NAMES` keeps only the root's known tools). + ledger.tools.append(tool.name) return None refusal = tool_error(verdict) if verdict == "invalid_arguments" and tool.name in ARGUMENT_ERROR_TOOLS: diff --git a/agents/anyplot/telemetry.py b/agents/anyplot/telemetry.py new file mode 100644 index 00000000000..7b0cdfb9861 --- /dev/null +++ b/agents/anyplot/telemetry.py @@ -0,0 +1,922 @@ +"""Content-free usage telemetry: four canonical events, one wide JSON line each. + +The admin-only chat ("Use with my data") writes one line per event to stdout on its own +logger (`TELEMETRY_LOGGER`, shipped by `enable_attribution_log` in `agents/main.py`), and a +Cloud Logging sink routes the lines with `jsonPayload.log_type="agent_telemetry"` into the +`agent-telemetry` log bucket, which BigQuery reads through a linked dataset +(`docs/reference/agent-telemetry.md`). The per-hook attribution lines (`plugins/ledger.py`) +stay on their logger for debugging; the sink never takes them, because they carry the HMAC +user id. + +Owner rule of 2026-10-16: no line ever holds a user input, question, reply, data value, +column name, file or sheet name, or cell, only enums, coarse buckets, counts and ids from +closed sets. Every event is a pydantic model with `extra="forbid"` whose fields are +`Literal` sets, bounded integers, booleans or ids that match a fixed pattern (`SpecId`, +`ModelId`, `DefectId`, `GateId`), so text from the user cannot reach a line even by +mistake: a value outside its set fails validation and the line is dropped with a warning +that names the event and the field only. Content reaches anyone only through the explicit +feedback-case submission (`api/agent_cases.py`). + +| Event | When | One line per | +|---|---|---| +| `data_input` | a paste or a dropped file reached the dataset route (or the conversion refused the file) | input | +| `chat_turn` | a `/messages` turn ended | turn | +| `pipeline_run` | a `plot_pipeline` run ended | run (`run_index` 1 is the session's first plot) | +| `outcome` | the page's beacon: copied code or image, downloaded a file, saved the session, sent feedback | action and version | + +Envelope (every line): `log_type`, `schema`, `event`, `service` and `revision` (Cloud +Run's `K_SERVICE` and `K_REVISION`), `source` (`live`, or `harness` for the eval +harnesses, `set_source`), `session` (a salted hash), `spec_id`, `library`, `turn` (the +session's user turns so far, from the request ledger), and `idempotency_key`, a hash of +session, turn, attempt and event: the BigQuery base view keeps one row per key, so a +line Cloud Logging delivers twice or a pipeline node ADK re-runs counts once. + +The session hash is an HMAC-SHA256 of the session id under a random per-process salt +(`plugins/ledger.SESSION_SALT`): the id travels in every `/v1` request URL that Cloud Run +logs, and a plain hash of it could be recomputed from those. Sessions live in memory and +die with the instance (`max-instances=1`), so a salt that dies with the process keeps one +session's lines together and still unlinkable to the request log. The attribution lines' +`session` uses the same salt under another domain (`ledger.salted`), so telemetry rows +and debug lines cannot be joined by it either. +""" + +from __future__ import annotations + +import hashlib +import json +import logging +import os +import re +import time +from collections import OrderedDict +from collections.abc import Iterable +from dataclasses import dataclass, field +from typing import TYPE_CHECKING, Annotated, Any, Literal, cast, get_args + +from pydantic import AfterValidator, BaseModel, ConfigDict, Field, ValidationError + +from core.constants import SUPPORTED_LIBRARIES + +from .data.warning_codes import BindingWarningCode, WarningCode, binding_warning_codes, warning_codes +from .plugins.ledger import RequestLedger, StepUsage, salted + + +if TYPE_CHECKING: + from google.adk.events.event import Event + + from .data.parse import ParsedDataset + from .render.contract import ThemeOutput + from .schemas import DatasetProfile + + +TELEMETRY_LOGGER = "anyplot.agents.telemetry" +"""The logger of the telemetry lines; `agents/main.py` `enable_attribution_log` ships it to stdout next to the +attribution logger, each with a handler of its own.""" +logger = logging.getLogger(TELEMETRY_LOGGER) +warn = logging.getLogger(__name__) + +LOG_TYPE: Literal["agent_telemetry"] = "agent_telemetry" +SCHEMA: Literal[1] = 1 +"""The line format's version: a field renamed or removed is a new schema; a field added is not.""" + +Source = Literal["live", "harness"] +_source: Source = "live" + + +def set_source(source: Source) -> None: + """Tag every later line with `source`: the eval harnesses (`evals/matrix.py`, `evals/prep.py`) call it with + `harness` before their first request, so the views can leave their runs out.""" + global _source + _source = source + + +def session_key(session_id: str) -> str: + """The salted hash of a session id: 16 hex characters, stable for this process only.""" + return salted(session_id, "telemetry", 16) + + +def idempotency_key(session: str, turn: int, attempt: str, event: str) -> str: + return hashlib.sha256(f"{session}|{turn}|{attempt}|{event}".encode()).hexdigest()[:24] + + +# --- Closed sets -------------------------------------------------------------------------- + + +def _library(value: str) -> str: + if value not in SUPPORTED_LIBRARIES: + raise ValueError("not a supported library") + return value + + +Library = Annotated[str, AfterValidator(_library)] +"""One of `core.constants.SUPPORTED_LIBRARIES`.""" +SpecId = Annotated[str, Field(pattern=r"^[a-z0-9-]{1,100}$")] +ModelId = Annotated[str, Field(pattern=r"^[a-z0-9][a-z0-9.@_-]{0,79}$")] +"""A model id as the provider names it (`claude-haiku-5-5`, `gemini-3.8-flash`, `...@20261001`).""" +DefectId = Annotated[str, Field(pattern=r"^[A-Z]{2}-[0-9]{2}$")] +GateId = Annotated[str, Field(pattern=r"^(?:R[0-9](?:-[a-z_]{1,24})?|G[0-9]{1,2})$")] + +InputFormat = Literal[ + "csv", + "tsv", + "semicolon", + "pipe", + "whitespace", + "json", + "json_lines", + "json_stat", + "api_json", + "xlsx", + "xlsm", + "xlsb", + "xls", + "ods", + "parquet", + "feather", + "arrow", + "npy", + "npz", + "transcribed", + "formula", + "code", + "other", +] +Entry = Literal["paste", "file"] +DataOutcome = Literal[ + "ok", + "transcribed", + "unparseable", + "too_large", + "code", + "formula", + "data_refused", + "guard_unavailable", + "capacity", + "rate_limited", + "daily_limit", + "locked_out", + "file_refused", + "file_failed", + "convert_unavailable", + "other", +] +DataCause = Literal[ + # the parser's refusals (`parse.ParseErrorCode`) + "too_long", + "unparseable", + "no_table", + "too_many_columns", + "too_many_rows", + "cell_too_long", + "control_chars", + # the paste route's own + "free_text_one_column", + # the transcriber's + "prefilter", + "cut_off", + "timeout", + "transcribed_unparseable", + "transcribed_no_table", + # the file conversion's (`wire.ConvertRefusal` and `main.CONVERT_FAILURES`) + "unsupported", + "encrypted", + "pickle", + "hdf5", + "corrupt", + "archive_limits", + "too_large", + "empty", + "sheet_not_found", + "array_shape", + "object_array", + "formulas_only", + "too_complex", + "unreadable", + "other", +] +SizeBucket = Literal["lt1k", "1k_10k", "10k_100k", "100k_1m", "ge1m"] +RowsBucket = Literal["1_10", "11_100", "101_1k", "1k_10k", "gt10k"] +ColsBucket = Literal["1", "2", "3_5", "6_10", "11_20", "gt20"] +CategoriesBucket = Literal["le5", "6_12", "13_50", "gt50"] +MissingBucket = Literal["none", "lt5", "5_30", "gt30"] +SpecFit = Literal["direct", "needs_reshape", "missing_role", "incompatible"] + +TurnKind = Literal["plot", "prep", "formula", "code", "talk", "refused", "error"] +TurnAction = Literal["text", "create_plot", "summarize_data"] +ToolName = Literal[ + "get_dataset_profile", + "get_spec_brief", + "get_current_code", + "set_bindings", + "set_language", + "prepare_data", + "query_data", + "generate_data", + "run_data_code", + "plot_pipeline", +] +TOOL_NAMES: frozenset[str] = frozenset(get_args(ToolName)) +PrepOp = Literal[ + "rename", + "drop_columns", + "drop_rows", + "set_type", + "aggregate", + "sort", + "pivot_longer", + "pivot_wider", + "classify", + "derive", + "make_date", +] +PREP_OPS: frozenset[str] = frozenset(get_args(PrepOp)) +RefusalCode = Literal[ + "out_of_scope", + "needs_data", + "other_library", + "unsupported_content", + "locked_out", + "session_closed", + "budget", + "rate_limited", + "daily_limit", + "other", +] +ErrorCode = Literal["guard_unavailable", "capacity", "deadline", "internal", "other"] +LanguageSource = Literal["default", "pasted", "detected", "chosen"] +Step = Literal["root", "adapter", "reviewer", "judge", "other"] + +PlotStatus = Literal["ok", "needs_attention", "failed", "not_ready"] +FailureReason = Literal["validation", "render", "deadline", "budget", "error"] +ScoreBucket = Literal["lt50", "50_69", "70_79", "80_89", "90_plus"] +SandboxFailure = Literal["syntax", "name", "key", "value", "type", "timeout", "memory", "other"] +PipelineStage = Literal[ + "budget", + "deadline", + "error", + "adapter_schema", + "adapter_truncated", + "adapter_full_code", + "edit_apply", + "validator", + "loader", + "render", + "gates", + "not_rereviewed", + "reviewer_unreadable", + "reviewer_defects", + "reviewer_ok", + "self_review", +] + +OutcomeAction = Literal[ + "copied_code", "copied_image", "downloaded_png", "downloaded_code", "downloaded_data", "saved_session", "feedback" +] +OUTCOME_ACTIONS: tuple[str, ...] = get_args(OutcomeAction) +Reaction = Literal["thumbs_up", "thumbs_down", "bug", "idea"] +"""The feedback reactions (`core/database/models.py` `FEEDBACK_REACTIONS`; pinned by a test).""" + +Count = Annotated[int, Field(ge=0, le=10_000_000)] +Ms = Annotated[int, Field(ge=0, le=86_400_000)] + + +def _list() -> Any: + """A list field of at most 64 items, empty by default (each field gets its own `FieldInfo`).""" + return Field(default_factory=list, max_length=64) + + +class _Event(BaseModel): + model_config = ConfigDict(extra="forbid", frozen=True) + + +class StepTokens(_Event): + """What one kind of agent spent in a turn or a run: calls and tokens by kind, and its model.""" + + step: Step + model: ModelId | None = None + calls: Count = 0 + input: Count = 0 + output: Count = 0 + cached: Count = 0 + cache_write: Count = 0 + + +class DataInput(_Event): + event: Literal["data_input"] = "data_input" + format: InputFormat + entry: Entry + outcome: DataOutcome + cause: DataCause | None = None + size_bucket: SizeBucket + rows_bucket: RowsBucket | None = None + cols_bucket: ColsBucket | None = None + numeric_cols: Count = 0 + integer_cols: Count = 0 + datetime_cols: Count = 0 + text_cols: Count = 0 + boolean_cols: Count = 0 + max_categories_bucket: CategoriesBucket | None = None + missing_bucket: MissingBucket | None = None + warnings: list[WarningCode] = _list() + tables: Count = 0 + sheets: Count = 0 + notes: bool = False + """Notes pasted around the table passed the screen and were kept (their text never is).""" + file_name: bool = False + """The input came with a file name (never the name).""" + spec_fit: SpecFit | None = None + judge_input: Count = 0 + judge_output: Count = 0 + judge_model: ModelId | None = None + latency_ms: Ms = 0 + + +class ChatTurn(_Event): + event: Literal["chat_turn"] = "chat_turn" + kind: TurnKind + action: TurnAction + asked_question: bool = False + """The root's final reply ends with a question mark: computed on the server, the reply itself never logged.""" + tools: list[ToolName] = _list() + prep_ops: list[PrepOp] = _list() + prep_calls: Count = 0 + undo: bool = False + to_long: bool = False + manual_binding_changed: bool = False + binding_warnings: list[BindingWarningCode] = _list() + refusal: RefusalCode | None = None + error: ErrorCode | None = None + reply_reasked: bool = False + language_source: LanguageSource | None = None + llm_calls: Count = 0 + steps: list[StepTokens] = _list() + latency_ms: Ms = 0 + + +class PipelineRun(_Event): + event: Literal["pipeline_run"] = "pipeline_run" + run_index: Annotated[int, Field(ge=1, le=10_000)] + base: Literal["catalogue", "previous"] + status: PlotStatus + reason: FailureReason | None = None + stage: PipelineStage | None = None + attempts: Count = 0 + rounds: Count = 0 + reviews: Count = 0 + score_round1: ScoreBucket | None = None + score_round2: ScoreBucket | None = None + shipped_attempt: Count | None = None + shipped_round: Count | None = None + forced_repair: list[DefectId] = _list() + """Review ids that forced a repair whatever the score (`pipeline.VISIBLE_DEFECT_IDS`).""" + gates_failed: list[GateId] = _list() + reviewer_defects: list[DefectId] = _list() + residual_defects: list[DefectId] = _list() + sandbox_failures: list[SandboxFailure] = _list() + adapt_ms: Ms = 0 + check_ms: Ms = 0 + render_ms: Ms = 0 + review_ms: Ms = 0 + total_ms: Ms = 0 + steps: list[StepTokens] = _list() + + +class Outcome(_Event): + event: Literal["outcome"] = "outcome" + action: OutcomeAction + version: Annotated[int, Field(ge=0, le=999)] = 0 + reaction: Reaction | None = None + + +AnyEvent = DataInput | ChatTurn | PipelineRun | Outcome +EVENTS: tuple[str, ...] = ("data_input", "chat_turn", "pipeline_run", "outcome") + + +class _Envelope(_Event): + severity: Literal["INFO"] = "INFO" + log_type: Literal["agent_telemetry"] = LOG_TYPE + schema_: Literal[1] = Field(default=SCHEMA, alias="schema") + service: Annotated[str, Field(pattern=r"^[a-z0-9-]{1,63}$")] + revision: Annotated[str, Field(pattern=r"^[a-z0-9-]{1,100}$")] + source: Source + session: Annotated[str, Field(pattern=r"^[0-9a-f]{0,16}$")] + spec_id: SpecId | None = None + library: Library | None = None + turn: Annotated[int, Field(ge=0, le=100_000)] = 0 + idempotency_key: Annotated[str, Field(pattern=r"^[0-9a-f]{24}$")] + + model_config = ConfigDict(extra="forbid", frozen=True, populate_by_name=True) + + +def _service() -> tuple[str, str]: + service = os.environ.get("K_SERVICE") or "anyplot-agents" + revision = os.environ.get("K_REVISION") or "local" + clean = re.compile(r"[^a-z0-9-]") + return clean.sub("-", service.lower())[:63] or "anyplot-agents", clean.sub("-", revision.lower())[:100] or "local" + + +def emit(event: AnyEvent, ledger: RequestLedger, *, attempt: str = "0") -> dict[str, Any] | None: + """Write one telemetry line for `event` with the ledger's envelope; the record, or None when it failed validation. + + Never raises: telemetry must not break a turn. `attempt` separates two lines of the same + event in one turn (a second paste, the version of an outcome) in the idempotency key. + """ + try: + service, revision = _service() + session = session_key(ledger.session_id) + envelope = _Envelope( + service=service, + revision=revision, + source=_source, + session=session, + spec_id=ledger.spec_id or None, + library=ledger.library or None, + turn=ledger.turn, + idempotency_key=idempotency_key(session, ledger.turn, attempt, event.event), + ) + record = {**envelope.model_dump(mode="json", by_alias=True), **event.model_dump(mode="json")} + except ValidationError as exc: + fields = sorted({str(error["loc"][0]) for error in exc.errors() if error.get("loc")}) + warn.warning("telemetry line dropped: %s fails %s", event.event, ", ".join(fields)) + return None + logger.info(json.dumps(record, sort_keys=True)) + return record + + +def build(model: type[AnyEvent], **fields: Any) -> AnyEvent | None: + """`model(**fields)`, or None (with a warning that names the fields only) when a value is outside its set.""" + try: + return model(**fields) + except ValidationError as exc: + names = sorted({str(error["loc"][0]) for error in exc.errors() if error.get("loc")}) + warn.warning("telemetry event invalid: %s fails %s", model.__name__, ", ".join(names)) + return None + + +def write(model: type[AnyEvent], ledger: RequestLedger, *, attempt: str = "0", **fields: Any) -> dict[str, Any] | None: + """`build` then `emit`; never raises.""" + try: + event = build(model, **fields) + return emit(event, ledger, attempt=attempt) if event is not None else None + except Exception as exc: # telemetry never breaks a request + warn.warning("telemetry failed: %s", type(exc).__name__) + return None + + +# --- Buckets ------------------------------------------------------------------------------ + + +def size_bucket(size: int) -> SizeBucket: + if size < 1024: + return "lt1k" + if size < 10 * 1024: + return "1k_10k" + if size < 100 * 1024: + return "10k_100k" + if size < 1024 * 1024: + return "100k_1m" + return "ge1m" + + +def rows_bucket(rows: int) -> RowsBucket: + if rows <= 10: + return "1_10" + if rows <= 100: + return "11_100" + if rows <= 1000: + return "101_1k" + if rows <= 10_000: + return "1k_10k" + return "gt10k" + + +def cols_bucket(columns: int) -> ColsBucket: + if columns <= 1: + return "1" + if columns == 2: + return "2" + if columns <= 5: + return "3_5" + if columns <= 10: + return "6_10" + if columns <= 20: + return "11_20" + return "gt20" + + +def categories_bucket(categories: int) -> CategoriesBucket: + if categories <= 5: + return "le5" + if categories <= 12: + return "6_12" + if categories <= 50: + return "13_50" + return "gt50" + + +def missing_bucket(missing: int, cells: int) -> MissingBucket: + if missing <= 0 or cells <= 0: + return "none" + share = missing / cells + if share < 0.05: + return "lt5" + if share <= 0.30: + return "5_30" + return "gt30" + + +def score_bucket(score: int | None) -> ScoreBucket | None: + if score is None: + return None + if score < 50: + return "lt50" + if score < 70: + return "50_69" + if score < 80: + return "70_79" + if score < 90: + return "80_89" + return "90_plus" + + +def ms(seconds: float) -> int: + return max(0, min(86_400_000, round(seconds * 1000))) + + +# --- Facts from the code's own objects --------------------------------------------------- + + +JSON_FORMATS: dict[str, InputFormat] = { + "json_records": "json", + "json_columns": "json", + "json_pairs": "json", + "json_lines": "json_lines", + "json_stat": "json_stat", +} +API_JSON_WARNINGS = frozenset({"fields_beside_records_to_notes", "list_fields_beside_left_out"}) +"""Warnings the parser writes only when it found the records inside an API answer (`parse._api_records`).""" + + +def input_format( + source_format: str | None, *, convert_kind: str | None = None, warnings: Iterable[str] = () +) -> InputFormat: + """The closed input format of a parsed table: the dropped file's kind, else the parser's `source_format`.""" + if convert_kind is not None: + return cast(InputFormat, closed(convert_kind, get_args(InputFormat))) + if source_format is None: + return "other" + if source_format in JSON_FORMATS: + if source_format == "json_records" and API_JSON_WARNINGS & set(warnings): + return "api_json" + return JSON_FORMATS[source_format] + return cast(InputFormat, closed(source_format, get_args(InputFormat))) + + +def profile_fields(profile: DatasetProfile) -> dict[str, Any]: + """The buckets and counts of a parsed table's profile: no name, no value.""" + dtypes = [column.dtype for column in profile.columns] + text_unique = [column.unique for column in profile.columns if column.dtype == "text"] + missing = sum(column.missing for column in profile.columns) + return { + "rows_bucket": rows_bucket(profile.rows), + "cols_bucket": cols_bucket(len(profile.columns)), + "numeric_cols": dtypes.count("number"), + "integer_cols": dtypes.count("integer"), + "datetime_cols": dtypes.count("datetime"), + "text_cols": dtypes.count("text"), + "boolean_cols": dtypes.count("boolean"), + "max_categories_bucket": categories_bucket(max(text_unique)) if text_unique else None, + "missing_bucket": missing_bucket(missing, profile.rows * len(profile.columns)), + } + + +def parsed_fields(parsed: ParsedDataset) -> dict[str, Any]: + """`profile_fields` plus the warning codes, the table count and whether notes were kept.""" + codes = warning_codes(parsed.warnings) + return { + **profile_fields(parsed.profile), + "warnings": codes, + "tables": len(parsed.tables) or 1, + "notes": bool(parsed.context_lines), + } + + +def spec_fit(*, complete: bool, bound: int, reshape: bool) -> SpecFit: + """How the table fits the spec at the paste: a wide-table suggestion first (`data/shape.py`), then the bindings.""" + if reshape: + return "needs_reshape" + if complete: + return "direct" + return "missing_role" if bound else "incompatible" + + +_EXCEPTION_CLASSES: dict[str, SandboxFailure] = { + "SyntaxError": "syntax", + "IndentationError": "syntax", + "TabError": "syntax", + "NameError": "name", + "UnboundLocalError": "name", + "KeyError": "key", + "ValueError": "value", + "TypeError": "type", + "MemoryError": "memory", + "TimeoutError": "timeout", +} + + +def sandbox_failure(output: ThemeOutput) -> SandboxFailure | None: + """The closed class of a render that failed in the sandbox, from the exit, the renderer's reason and the + exception class of the traceback (`gates.error_summary`, never the message); None for a run that exited 0.""" + from .render.gates import error_summary + + if output.timed_out or output.reason == "timeout": + return "timeout" + if output.reason == "memory": + return "memory" + if output.exit_code == 0 and output.png is not None: + return None + name = error_summary(output.stderr_tail).split(" ", 1)[0] + return _EXCEPTION_CLASSES.get(name, "other") + + +def step_of(agent: str) -> Step: + if agent == "anyplot": + return "root" + if agent.startswith("adapter"): + return "adapter" + if agent == "reviewer": + return "reviewer" + return "other" + + +def step_tokens( + usage: dict[str, StepUsage], *, steps: Iterable[Step], judge_model: str | None = None +) -> list[dict[str, Any]]: + """The ledger's per-agent usage folded by step, for the steps asked for, in a fixed order.""" + wanted = list(steps) + folded: dict[Step, dict[str, Any]] = {} + for agent, spent in usage.items(): + step: Step = "judge" if agent == "judge" else step_of(agent) + if step not in wanted: + continue + item = folded.setdefault( + step, {"step": step, "model": None, "calls": 0, "input": 0, "output": 0, "cached": 0, "cache_write": 0} + ) + item["calls"] += spent.calls + item["input"] += spent.input + item["output"] += spent.output + item["cached"] += spent.cached + item["cache_write"] += spent.cache_write + item["model"] = item["model"] or _model_id(spent.model or (judge_model if step == "judge" else None)) + return [folded[step] for step in wanted if step in folded] + + +_MODEL_ID = re.compile(r"^[a-z0-9][a-z0-9.@_-]{0,79}$") + + +def _model_id(value: str | None) -> str | None: + """A model id that fits `ModelId`, else None: a provider's odd version string never fails the whole line.""" + if not value: + return None + text = value.strip().lower().removeprefix("models/") + return text if _MODEL_ID.fullmatch(text) else None + + +def ends_with_question(text: str) -> bool: + """The reply's last visible character is a question mark (Latin, full-width or Arabic), past closing quotes, + brackets and Markdown emphasis.""" + stripped = text.rstrip().rstrip(")\"'»”’*_`").rstrip() + return stripped.endswith(("?", "？", "؟")) + + +def user_turns(events: Iterable[Event]) -> int: + """The session's user turns so far: events by the user with text, not the routes' state-only events.""" + count = 0 + for event in events: + if event.author != "user" or not event.content or not event.content.parts: + continue + if any(part.text for part in event.content.parts): + count += 1 + return count + + +def turn_kind(ledger: RequestLedger) -> TurnKind: + """What a turn did, in order of weight: a refusal, an error, a plot, data from code or a formula, a prep, talk.""" + if ledger.refusal is not None: + return "refused" + if ledger.error is not None: + return "error" + if "plot_pipeline" in ledger.tools: + return "plot" + if "run_data_code" in ledger.tools: + return "code" + if "generate_data" in ledger.tools: + return "formula" + if "prepare_data" in ledger.tools: + return "prep" + return "talk" + + +def closed(value: str | None, allowed: Iterable[str], fallback: str = "other") -> str | None: + """`value` when it is in `allowed`, `fallback` for any other value, None for None.""" + if value is None: + return None + return value if value in set(allowed) else fallback + + +# --- Per-session marks between turns ------------------------------------------------------ + +MAX_MARKED_SESSIONS = 4096 + + +@dataclass +class SessionMarks: + """What happened in a session between two telemetry lines, in memory only (the session dies with the instance). + + The data routes set the flags a `chat_turn` reports (`undo`, `to_long`, `manual_binding_changed`, + `binding_warnings`) and the next turn takes them; the conversion route keeps the dropped file's kind + for the paste that follows; the pipeline numbers its runs per session, once per invocation (`run_index`). + """ + + inputs: int = 0 + convert_kind: str | None = None + convert_sheets: int = 0 + undo: bool = False + to_long: bool = False + manual_binding_changed: bool = False + binding_warnings: set[str] = field(default_factory=set) + runs: dict[str, int] = field(default_factory=dict) + + +_MARKS: OrderedDict[str, SessionMarks] = OrderedDict() + + +def marks(session_id: str) -> SessionMarks: + found = _MARKS.get(session_id) + if found is None: + found = _MARKS[session_id] = SessionMarks() + while len(_MARKS) > MAX_MARKED_SESSIONS: + _MARKS.popitem(last=False) + else: + _MARKS.move_to_end(session_id) + return found + + +def forget(session_id: str) -> None: + _MARKS.pop(session_id, None) + + +def take_turn_marks(session_id: str) -> dict[str, Any]: + """The flags the data routes set since the last turn, cleared as they are read.""" + mark = marks(session_id) + taken = { + "undo": mark.undo, + "to_long": mark.to_long, + "manual_binding_changed": mark.manual_binding_changed, + "binding_warnings": sorted(mark.binding_warnings), + } + mark.undo = mark.to_long = mark.manual_binding_changed = False + mark.binding_warnings = set() + return taken + + +def note_binding_warnings(session_id: str, sentences: list[str]) -> None: + if sentences: + marks(session_id).binding_warnings.update(binding_warning_codes(sentences)) + + +def run_index(session_id: str, invocation_id: str | None) -> int: + """The session's run number for this invocation: 1 for its first plot run; the same number when ADK re-runs + the node in the same invocation, so the line keeps its idempotency key.""" + mark = marks(session_id) + key = invocation_id or f"run-{len(mark.runs) + 1}" + if key not in mark.runs: + mark.runs[key] = len(mark.runs) + 1 + return mark.runs[key] + + +def next_input(session_id: str) -> int: + mark = marks(session_id) + mark.inputs += 1 + return mark.inputs + + +REFUSAL_OUTCOMES: dict[str, DataOutcome] = { + "too_long": "too_large", + "unparseable": "unparseable", + "code": "code", + "formula": "formula", + "data_refused": "data_refused", + "guard_unavailable": "guard_unavailable", + "capacity": "capacity", + "rate_limited": "rate_limited", + "daily_limit": "daily_limit", + "locked_out": "locked_out", + "convert_unavailable": "convert_unavailable", +} +"""The dataset routes' refusal codes as a `data_input` outcome; any other code is `unparseable`'s sibling `other`.""" + + +@dataclass +class InputFacts: + """The content-free facts of one dataset input, collected along the route and written once (`write`). + + `format` is fixed up front for a dropped binary file (its kind) and a confirmed transcription, set by the + parse for every other paste (`parsed`), and by the refusal for code and formulas (`refused`). + """ + + session_id: str + entry: Entry + size: int + format: str | None = None + sheets: int = 0 + fields: dict[str, Any] = field(default_factory=dict) + outcome: str | None = None + cause: str | None = None + + @classmethod + def for_paste(cls, session_id: str, *, size: int, source: bool, confirmed: bool) -> InputFacts: + """A paste's facts; a paste that names a file after a conversion takes the converted file's kind, once.""" + mark = marks(session_id) + kind, sheets = mark.convert_kind, mark.convert_sheets + mark.convert_kind, mark.convert_sheets = None, 0 + fmt: str | None = None + if source and kind is not None: + fmt = kind + elif confirmed: + fmt = "transcribed" + return cls(session_id, "file" if source else "paste", size, fmt, sheets if fmt == kind else 0) + + def parsed(self, parsed: ParsedDataset) -> None: + self.fields.update(parsed_fields(parsed)) + if self.format is None: + self.format = input_format(parsed.profile.source_format, warnings=self.fields["warnings"]) + + def transcribed(self, parsed: ParsedDataset) -> None: + self.fields.update(parsed_fields(parsed)) + self.outcome = "transcribed" + + def stored(self, parsed: ParsedDataset, *, complete: bool, bound: int, reshape: bool, file_name: bool) -> None: + self.parsed(parsed) + self.fields.update(spec_fit=spec_fit(complete=complete, bound=bound, reshape=reshape), file_name=file_name) + self.outcome = "ok" + + def refused(self, code: str) -> None: + if code in ("code", "formula"): + self.format = code + if self.outcome is None: + self.outcome = REFUSAL_OUTCOMES.get(code, "other") + + def write(self, ledger: RequestLedger, clock: Clock, judge_model: str | None = None) -> dict[str, Any] | None: + """The `data_input` line; an input that neither stored nor refused (an exception) writes `other`.""" + judge = ledger.steps.get("judge") + index = next_input(self.session_id) + return write( + DataInput, + ledger, + attempt=f"in{index}", + format=closed(self.format, get_args(InputFormat)) or "other", + entry=self.entry, + outcome=self.outcome or "other", + cause=closed(self.cause, get_args(DataCause)), + size_bucket=size_bucket(self.size), + sheets=self.sheets, + judge_input=judge.input if judge else 0, + judge_output=judge.output if judge else 0, + judge_model=_model_id(judge_model) if judge else None, + latency_ms=clock.ms(), + **self.fields, + ) + + +def file_refusal( + session_id: str, ledger: RequestLedger, *, size: int, kind: str | None, code: str, clock: Clock +) -> dict[str, Any] | None: + """The `data_input` line of a dropped file the conversion refused or failed on (`/dataset/convert`).""" + if code.startswith("file_"): + cause: str | None = code.removeprefix("file_") + outcome = ( + "file_refused" + if cause in get_args(DataCause) and cause not in ("timeout", "too_complex", "unreadable") + else "file_failed" + ) + else: + cause, outcome = None, REFUSAL_OUTCOMES.get(code, "other") + facts = InputFacts(session_id, "file", size, kind or "other", outcome=outcome, cause=cause) + return facts.write(ledger, clock) + + +def converted(session_id: str, *, kind: str, sheets: int) -> None: + """A conversion succeeded: the paste that follows it with the file's name is that kind of file.""" + mark = marks(session_id) + mark.convert_kind, mark.convert_sheets = kind, sheets + + +class Clock: + """A monotonic stopwatch for `latency_ms`.""" + + def __init__(self) -> None: + self.started = time.monotonic() + + def ms(self) -> int: + return ms(time.monotonic() - self.started) diff --git a/agents/anyplot/tools/session.py b/agents/anyplot/tools/session.py index 40dfd3d7104..b22e9beecf0 100644 --- a/agents/anyplot/tools/session.py +++ b/agents/anyplot/tools/session.py @@ -66,6 +66,7 @@ from google.adk.tools.tool_context import ToolContext from pydantic import TypeAdapter, ValidationError +from .. import telemetry from ..briefs import bindings_text, spec_brief, trimmed_profiles from ..code.data_code import check_data_code from ..data.generate import Generate, GenerateError, generate @@ -243,6 +244,7 @@ async def set_bindings(bindings: list[dict[str, str]], tool_context: ToolContext "data_confirmed": bool(confirmed and confirmed.data_confirmed), } if check.warnings: # valid, but a role that needs values of 0 or more got negative ones; they name columns + telemetry.note_binding_warnings(tool_context.session.id, check.warnings) # closed codes for `chat_turn` result["warnings"] = DATA_PREAMBLE + "\n" + fence("user_data", json.dumps(check.warnings, ensure_ascii=False)) return result @@ -333,6 +335,7 @@ async def prepare_data(operations: list[dict[str, Any]], tool_context: ToolConte for key, value in version.delta.items(): tool_context.state[key] = value ledger.dataset_changes.extend(prepared.changes) + ledger.prep_ops.extend(operation.op for operation in parsed_operations) # the names only, for the telemetry base = { "status": "ok", "rows": parsed.profile.rows, diff --git a/agents/evals/matrix.py b/agents/evals/matrix.py index d0a56314aa0..9abc81058a3 100644 --- a/agents/evals/matrix.py +++ b/agents/evals/matrix.py @@ -1301,9 +1301,11 @@ async def run_matrix( ) -> MatrixResult: """Run every case `config.repeats` times, write the report, the summary and the galleries, compare and decide.""" from agents import main as service + from agents.anyplot import telemetry from agents.anyplot.services import Services, set_services from agents.anyplot.settings import get_settings + telemetry.set_source("harness") # the usage views count live sessions only (docs/reference/agent-telemetry.md) settings = get_settings() services = services or Services() recorder: AttemptRecorder | None = None diff --git a/agents/evals/prep.py b/agents/evals/prep.py index 812a3f4aa2d..925ba1f705f 100644 --- a/agents/evals/prep.py +++ b/agents/evals/prep.py @@ -588,9 +588,11 @@ async def run_cases( ) -> list[CaseResult]: """Every case through the in-process app, one at a time and paced by `pacer`, in the case order.""" from agents import main as service + from agents.anyplot import telemetry from agents.anyplot.services import set_services from agents.anyplot.settings import get_settings + telemetry.set_source("harness") # the usage views count live sessions only (docs/reference/agent-telemetry.md) settings = get_settings() set_services(services) saved_overrides = dict(service.app.dependency_overrides) diff --git a/agents/main.py b/agents/main.py index c78c5211fec..04db307a5c9 100644 --- a/agents/main.py +++ b/agents/main.py @@ -21,6 +21,7 @@ | `POST /v1/sessions/{sid}/dataset/to_long` | | The undo route's shape plus `changes`: the page's `.to_long()` applies the wide-table suggestion, recomputed on the server, as the next dataset version with the default bindings for the new columns; no model call | `409 run_active`, `422 no_suggestion` | | `POST /v1/sessions/{sid}/messages` | `{text}` or `{action}` (`create_plot`, `summarize_data`) | SSE `anyplot/1`; the `done` event carries `dataset` (with `roles` and `changes`) after a turn whose `prepare_data` or `generate_data` changed the data, version 1 when `generate_data` made the session's first data, and `language` (`{language, source}`) after every run | `413 too_long`, `409 run_active` | | `POST /v1/sessions/{sid}/cancel` | | `204` | | +| `POST /v1/sessions/{sid}/outcome` | `{action, version?, reaction?}` (closed sets: `telemetry.OutcomeAction`, `telemetry.Reaction`, the latter with `feedback` only) | `204`: the content-free `outcome` telemetry line (`anyplot/telemetry.py`) | `422`, `429 rate_limited` / `429 daily_limit` (`OUTCOMES_PER_MINUTE` / `_PER_DAY`) | | `POST /v1/sessions/{sid}/versions/{version}/render` | `{theme}` | `{status, reason?, artifacts}` | `404 not_found`, `409 run_active`, `503 capacity` (no render slot in time, or the render store is full) | | `GET /v1/sessions/{sid}/artifacts/{name}?v=` | | the file | `404` | | `GET /v1/sessions/{sid}/bundle?version=&include_data=` | | the feedback case bundle: every version, or the one numbered `version`, or with `version=0` the conversation alone (no version; `data_csv`, the current data, with `include_data`); with `AGENT_KEEP_REFUSALS` also `refusals`, the session's refused messages `[{verdict, lang, text, at}]` | `404` | @@ -108,7 +109,7 @@ import time from collections.abc import AsyncIterator, Callable from dataclasses import asdict, dataclass, field -from typing import Annotated, Any, Literal, TextIO +from typing import Annotated, Any, Literal, TextIO, get_args from fastapi import Body, Depends, FastAPI, Header, Path, Query, Request from fastapi.responses import JSONResponse, Response, StreamingResponse @@ -123,8 +124,9 @@ from starlette.background import BackgroundTask from agents.anyplot import agent as agent_module +from agents.anyplot import telemetry from agents.anyplot.code.readiness import MAP_SPECS -from agents.anyplot.data.bindings import label_siblings +from agents.anyplot.data.bindings import check_bindings, label_siblings from agents.anyplot.data.classify import classify_paste from agents.anyplot.data.parse import ( MAX_COMPUTED_BYTES, @@ -177,7 +179,14 @@ source_name, store_dataset, ) -from agents.anyplot.plugins.ledger import CURRENT_LEDGER, RequestLedger, attribution, budget_allows, judge_budget_tokens +from agents.anyplot.plugins.ledger import ( + ATTRIBUTION_LOGGER, + CURRENT_LEDGER, + RequestLedger, + attribution, + budget_allows, + judge_budget_tokens, +) from agents.anyplot.plugins.scope_guard import LOCKED_OUT, SESSION_CLOSED, scope_refusals from agents.anyplot.policy import fence, file_name_block, refusal, scope_rubric, transcribe_rubric, user_notes_block from agents.anyplot.render.contract import ConvertJob, RendererUnavailable @@ -269,9 +278,6 @@ def disable_content_capture() -> None: disable_content_capture() -ATTRIBUTION_LOGGER = "anyplot.agents.attribution" - - class _AttributionHandler(logging.StreamHandler[TextIO]): """The stdout handler of the attribution lines, a class of its own so `enable_attribution_log` finds it.""" @@ -285,13 +291,18 @@ def enable_attribution_log() -> None: model SDKs stay at WARNING (`disable_content_capture`). The lines carry ids, counts, verdicts and rule ids, never text, code or data. It still propagates, so a test's log capture sees it; the root has no handler of its own in the service. Idempotent: a second call adds no second handler. + + The usage telemetry's logger (`telemetry.TELEMETRY_LOGGER`) gets the same treatment with a handler of its + own: its lines carry `log_type: "agent_telemetry"`, which the Cloud Logging sink selects + (`docs/reference/agent-telemetry.md`), and never the user id the attribution lines carry. """ - attribution_logger = logging.getLogger(ATTRIBUTION_LOGGER) - attribution_logger.setLevel(logging.INFO) - if not any(isinstance(handler, _AttributionHandler) for handler in attribution_logger.handlers): - handler = _AttributionHandler(sys.stdout) - handler.setFormatter(logging.Formatter("%(message)s")) - attribution_logger.addHandler(handler) + for name in (ATTRIBUTION_LOGGER, telemetry.TELEMETRY_LOGGER): + named = logging.getLogger(name) + named.setLevel(logging.INFO) + if not any(isinstance(handler, _AttributionHandler) for handler in named.handlers): + handler = _AttributionHandler(sys.stdout) + handler.setFormatter(logging.Formatter("%(message)s")) + named.addHandler(handler) enable_attribution_log() @@ -527,6 +538,7 @@ async def purge(self, user: str, sid: str, services: Services) -> None: self.stop(run) await self.session_service.delete_session(app_name=APP_NAME, user_id=user, session_id=sid) services.purge_session(sid) + telemetry.forget(sid) def stop(self, run: ActiveRun) -> None: """Abort a run and take it out of the queue if it still waits; a running one keeps its slot until it ends.""" @@ -740,6 +752,15 @@ def _library(library: str, settings: AgentSettings) -> str: return library +def _stamp(ledger: RequestLedger, view: SessionView | None, session: Session) -> RequestLedger: + """The ledger with the session's spec, library and user turns so far: the usage telemetry's envelope + (`telemetry.py`), set once per request from session state, never per call site.""" + if view is not None: + ledger.spec_id, ledger.library = view.spec_id, view.library + ledger.turn = telemetry.user_turns(session.events) + return ledger + + # --- Routes ------------------------------------------------------------------------------ @@ -890,18 +911,28 @@ async def upload_dataset( view = read_session(session.state) if view is None: raise AgentsError(404, "session_expired") - if len(body.text.encode("utf-8", errors="surrogatepass")) > MAX_INPUT_BYTES: - raise AgentsError(413, "too_long") - _paste_budget(services, user, settings) - limit = services.usage.paste_limit(user, settings) - if limit is not None: - attribution("paste_limit", RequestLedger(request_id=rid, user_id=user, session_id=sid), verdict=limit) - raise AgentsError(429, limit) - runtime.pasting.add(user) # no await since the check above, so no second paste slipped in between + ledger = _stamp(RequestLedger(request_id=rid, user_id=user, session_id=sid, kind="action"), view, session) + size = len(body.text.encode("utf-8", errors="surrogatepass")) + facts = telemetry.InputFacts.for_paste(sid, size=size, source=body.source is not None, confirmed=body.confirmed) + clock = telemetry.Clock() try: - return await _ingest_paste(services, runtime, body, user, sid, rid, view) + if size > MAX_INPUT_BYTES: + raise AgentsError(413, "too_long") + _paste_budget(services, user, settings) + limit = services.usage.paste_limit(user, settings) + if limit is not None: + attribution("paste_limit", ledger, verdict=limit) + raise AgentsError(429, limit) + runtime.pasting.add(user) # no await since the check above, so no second paste slipped in between + try: + return await _ingest_paste(services, runtime, body, user, sid, ledger, view, facts) + finally: + runtime.pasting.discard(user) + except AgentsError as exc: + facts.refused(exc.code) + raise finally: - runtime.pasting.discard(user) + facts.write(ledger, clock, settings.judge_model) CONVERT_FAILURES: dict[str, str] = { @@ -945,10 +976,33 @@ async def convert_dataset_file( raise AgentsError(422, "invalid") from None if not data: raise AgentsError(422, "invalid") + ledger = _stamp( + RequestLedger(request_id=rid, user_id=user, session_id=sid, kind="action"), read_session(session.state), session + ) + clock = telemetry.Clock() + try: + return await _convert(services, settings, runtime, body, data, user, sid, ledger, clock) + except AgentsError as exc: + telemetry.file_refusal(sid, ledger, size=len(data), kind=None, code=exc.code, clock=clock) + raise + + +async def _convert( + services: Services, + settings: AgentSettings, + runtime: Runtime, + body: ConvertBody, + data: bytes, + user: str, + sid: str, + ledger: RequestLedger, + clock: telemetry.Clock, +) -> JSONResponse: + """The conversion route past its decode: its limits, the sandbox run, the answer; the route writes the + `data_input` line of a refusal raised here, this function the one of a file the converter refused.""" if len(data) > MAX_CONVERT_INPUT_BYTES: raise AgentsError(413, "too_long") _paste_budget(services, user, settings) - ledger = RequestLedger(request_id=rid, user_id=user, session_id=sid, kind="action") limit = services.usage.convert_limit(user, settings) if limit is not None: attribution("convert_limit", ledger, verdict=limit) @@ -984,6 +1038,8 @@ async def convert_dataset_file( bytes=len(data), wall_s=round(output.wall_s, 3), ) + # The paste that follows with the file's name is this kind of file: its `data_input` line says so. + telemetry.converted(sid, kind=meta.kind, sheets=meta.sheet_count) return JSONResponse( { "text": output.text, @@ -1021,6 +1077,7 @@ async def convert_dataset_file( bytes=len(data), wall_s=round(output.wall_s, 3), ) + telemetry.file_refusal(sid, ledger, size=len(data), kind=kind, code=code, clock=clock) content: dict[str, Any] = {"detail": code} if output.refusal is not None: content.update( @@ -1039,10 +1096,19 @@ def _paste_budget(services: Services, user: str, settings: AgentSettings) -> Non async def _ingest_paste( - services: Services, runtime: Runtime, body: DatasetBody, user: str, sid: str, rid: str, view: SessionView + services: Services, + runtime: Runtime, + body: DatasetBody, + user: str, + sid: str, + ledger: RequestLedger, + view: SessionView, + facts: telemetry.InputFacts, ) -> JSONResponse: - """The paste route past its limits: classify, parse or transcribe, judge, screen the notes, store.""" - ledger = RequestLedger(request_id=rid, user_id=user, session_id=sid, kind="action") + """The paste route past its limits: classify, parse or transcribe, judge, screen the notes, store. + + `facts` collects the paste's content-free facts for its `data_input` telemetry line, which the route writes. + """ if not body.confirmed: # a confirmed transcription is the service's own CSV _refuse_code_or_formula(ledger, body.text) try: @@ -1051,6 +1117,7 @@ async def _ingest_paste( json_lines = body.source is not None and body.source.lower().rstrip().endswith((".jsonl", ".ndjson")) parsed = parse_dataset(body.text, find_table=not body.confirmed, table=body.table or 1, json_lines=json_lines) except ParseError as exc: + facts.cause = exc.code if exc.code == "too_long": raise AgentsError(413, "too_long", exc.message) from None # Text with no table in it (prose, a list on one line) is the transcriber's; a broken @@ -1058,17 +1125,19 @@ async def _ingest_paste( # which names its cause (owner rule of 2026-10-16). refusal = transcribe_refusal(body.text) if exc.code == "no_table" and refusal is None: - return await _transcribe(services, ledger, body.text) + return await _transcribe(services, ledger, body.text, facts) cause = exc.message if exc.code == "no_table": cause = f"{cause}, and {refusal}" raise AgentsError(422, "unparseable", cause) from None + facts.parsed(parsed) if not body.confirmed and looks_like_free_text(parsed): # One column of prose, `label value` lines or names: the transcriber's, or a refusal that says why # not. Never stored as it parsed: its first line would be a column name nobody wrote (review of #12202). refusal = transcribe_refusal(body.text) if refusal is None: - return await _transcribe(services, ledger, body.text) + return await _transcribe(services, ledger, body.text, facts) + facts.cause = "free_text_one_column" attribution("data_transcribe", ledger, outcome="too_large") raise AgentsError( 422, @@ -1078,6 +1147,7 @@ async def _ingest_paste( ) if not body.confirmed and (header_warning := category_header_warning(parsed)) is not None: parsed = with_warning(parsed, header_warning) # its first line may be a category: said, never silent + facts.parsed(parsed) services.usage.add_paste(user) # the first model call of this paste: only a paste that reaches one counts try: header_lang = await check_dataset(services, ledger, parsed) @@ -1103,6 +1173,14 @@ async def _ingest_paste( ) except StoreFull: raise AgentsError(503, "capacity") from None + roles = view.snapshot.roles() + facts.stored( + ingested.parsed, + complete=check_bindings(ingested.bindings, roles, ingested.parsed.profile).complete, + bound=len(ingested.bindings), + reshape=long_suggestion(roles, ingested.parsed.profile) is not None, + file_name=file_name is not None, + ) delta = ingested.state_delta() # Read again: the chip's choice may have landed while the judges ran, and a choice is never overwritten. # From this read to the append below nothing suspends: `InMemorySessionService.get_session` and @@ -1164,6 +1242,7 @@ async def _screen_file_name(services: Services, ledger: RequestLedger, raw: str) except JudgeUnavailable: attribution("file_name_judge", ledger, verdict="guard_unavailable", chars=chars) return FILE_NAME_WITHHELD + ledger.book_judge(answer.input_tokens, answer.output_tokens, answer.tokens) services.usage.add_tokens( ledger.user_id, judge_budget_tokens(answer.tokens, answer.input_tokens, answer.output_tokens, get_settings().judge_model), @@ -1202,6 +1281,7 @@ async def _screen_notes( except JudgeUnavailable: attribution("notes_judge", ledger, verdict="guard_unavailable", **counts) return without_context(parsed, NOTES_UNCHECKED_WARNING), None + ledger.book_judge(answer.input_tokens, answer.output_tokens, answer.tokens) services.usage.add_tokens( ledger.user_id, judge_budget_tokens(answer.tokens, answer.input_tokens, answer.output_tokens, get_settings().judge_model), @@ -1254,7 +1334,9 @@ def _transcribe_failure_cause(failure: JudgeFailure, text: str) -> str: ) -async def _transcribe(services: Services, ledger: RequestLedger, text: str) -> JSONResponse: +async def _transcribe( + services: Services, ledger: RequestLedger, text: str, facts: telemetry.InputFacts +) -> JSONResponse: """Read free text as a table with the transcriber and answer with it, storing nothing. The answer carries `transcribed: true` and the table's CSV in `text`; the user @@ -1265,8 +1347,10 @@ async def _transcribe(services: Services, ledger: RequestLedger, text: str) -> J call books its tokens like a judge call and writes a `data_transcribe` attribution line. """ user = ledger.user_id + facts.format = "transcribed" # free text the transcriber reads: the input's format, whatever it parsed as services.usage.add_paste(user) # a paste counts against `paste_limit` when it reaches a model (or its pre-filter) if INSTRUCTION_LIKE.search(text): + facts.cause = "prefilter" attribution("data_transcribe", ledger, outcome="prefilter") raise AgentsError(403, "data_refused") try: @@ -1276,11 +1360,13 @@ async def _transcribe(services: Services, ledger: RequestLedger, text: str) -> J # the transcriber's limits; the user can change either. A short text that times out met a # stalled endpoint, and an error, a quota or an unparseable answer is an outage too. sized = exc.failure == "cut_off" or (exc.failure == "timeout" and near_transcribe_cap(text)) + facts.cause = exc.failure if exc.failure in ("cut_off", "timeout") else "other" if not sized: attribution("data_transcribe", ledger, outcome="guard_unavailable", failure=exc.failure) raise AgentsError(503, "guard_unavailable") from None attribution("data_transcribe", ledger, outcome=exc.failure) raise AgentsError(422, "unparseable", _transcribe_failure_cause(exc.failure, text)) from None + ledger.book_judge(reading.input_tokens, reading.output_tokens, reading.tokens) services.usage.add_tokens( user, judge_budget_tokens(reading.tokens, reading.input_tokens, reading.output_tokens, get_settings().judge_model), @@ -1289,12 +1375,14 @@ async def _transcribe(services: Services, ledger: RequestLedger, text: str) -> J try: parsed = parse_dataset(csv_text, find_table=False) except ParseError as exc: + facts.cause = "transcribed_unparseable" attribution("data_transcribe", ledger, outcome="unparseable", judge_tokens=reading.tokens) raise AgentsError( 422, "unparseable", f"the model read the text as a table, but that table was refused: {exc.message}" ) from None if looks_like_free_text(parsed) and parsed.profile.columns[0].name == "text": # The rubric's answer for text that holds no table: one column named `text`. + facts.cause = "transcribed_no_table" attribution("data_transcribe", ledger, outcome="no_table", judge_tokens=reading.tokens) raise AgentsError( 422, @@ -1312,6 +1400,7 @@ async def _transcribe(services: Services, ledger: RequestLedger, text: str) -> J judge_input=reading.input_tokens, judge_output=reading.output_tokens, ) + facts.transcribed(parsed) return JSONResponse( { "transcribed": True, @@ -1353,6 +1442,11 @@ def _page_roles(roles: list[DataRole]) -> list[dict[str, Any]]: return [_role_body(role) for role in roles] +def _binding_pairs(bindings: list[dict[str, Any]]) -> set[tuple[Any, Any]]: + """Bindings as a set of (role, column) pairs, so their order never reads as a change.""" + return {(item.get("role"), item.get("column")) for item in bindings} + + @app.put("/v1/sessions/{sid}/bindings", dependencies=v1_dependencies) async def put_bindings( sid: SessionId, @@ -1380,6 +1474,12 @@ async def put_bindings( ) # Other bindings than the user confirmed leave the confirmation's fingerprint behind: step 1 again. await runtime.write_state(session, delta) + # The next `chat_turn` line says the user changed a binding by hand, and the warnings' closed codes. The page + # also PUTs the parse's own defaults after every paste (to learn `complete`), which changes nothing: only a PUT + # whose bindings differ from the ones the session held counts, so that echo never reads as a manual change. + if _binding_pairs(delta[BINDINGS]) != _binding_pairs([binding.model_dump() for binding in view.bindings]): + telemetry.marks(sid).manual_binding_changed = True + telemetry.note_binding_warnings(sid, check.warnings) return JSONResponse( { "bindings": delta[BINDINGS], @@ -1456,6 +1556,7 @@ async def undo_dataset(sid: SessionId, user: User, runtime: RuntimeDep) -> JSONR if restored is None: raise AgentsError(422, "nothing_to_undo") await runtime.write_state(session, restored.delta) + telemetry.marks(sid).undo = True # the next `chat_turn` line's `undo` return JSONResponse( { **restored.payload(), @@ -1505,6 +1606,7 @@ async def dataset_to_long(sid: SessionId, user: User, runtime: RuntimeDep) -> JS except StoreFull: raise AgentsError(503, "capacity") from None await runtime.write_state(session, version.delta) + telemetry.marks(sid).to_long = True # the next `chat_turn` line's `to_long` return JSONResponse( { **version.payload(), @@ -1571,23 +1673,66 @@ async def _session_sync( return payload -async def _refused(translator: Translator, ledger: RequestLedger) -> AsyncIterator[str]: +@dataclass(frozen=True) +class TurnFacts: + """What the `chat_turn` telemetry line needs from the route besides the ledger: the action, the session + language's source before the turn, and the turn's stopwatch.""" + + action: Action | None + language_source: str | None + clock: telemetry.Clock = field(default_factory=telemetry.Clock) + + +def _telemetry_turn(ledger: RequestLedger, facts: TurnFacts, failure: str | None = None) -> None: + """The usage telemetry's `chat_turn` line (`telemetry.py`): kinds, flags, closed names and counts, never text. + + The request id goes into the idempotency key only (hashed), so two turns of the same number, such as a turn + refused before the judge and the turn sent after it, stay two rows. + """ + error = failure or ledger.error + telemetry.write( + telemetry.ChatTurn, + ledger, + attempt=f"turn:{ledger.request_id}", + kind="error" if error is not None and ledger.refusal is None else telemetry.turn_kind(ledger), + action=facts.action or "text", + asked_question=ledger.reply_question and ledger.refusal is None, + tools=sorted({tool for tool in ledger.tools if tool in telemetry.TOOL_NAMES}), + prep_ops=sorted({op for op in ledger.prep_ops if op in telemetry.PREP_OPS}), + prep_calls=ledger.tools.count("prepare_data"), + refusal=telemetry.closed(ledger.refusal[0] if ledger.refusal else None, get_args(telemetry.RefusalCode)), + error=telemetry.closed(error, get_args(telemetry.ErrorCode)), + reply_reasked=ledger.reply_reasked, + language_source=telemetry.closed(facts.language_source, get_args(telemetry.LanguageSource), "default"), + llm_calls=ledger.llm_calls, + steps=telemetry.step_tokens( + ledger.steps, steps=("root", "adapter", "reviewer", "judge"), judge_model=get_settings().judge_model + ), + latency_ms=facts.clock.ms(), + **telemetry.take_turn_marks(ledger.session_id), + ) + + +async def _refused(translator: Translator, ledger: RequestLedger, facts: TurnFacts) -> AsyncIterator[str]: """A turn the ledger refused before the scope judge and any model call: `ready`, the refusal, `done`.""" yield translator.ready() for chunk in translator.refusal(): yield chunk attribution("run", ledger, model_versions=[], kind=ledger.kind) + _telemetry_turn(ledger, facts) yield translator.done() def _refuse_now( - translator: Translator, ledger: RequestLedger, code: str, hook: str, **fields: Any + translator: Translator, ledger: RequestLedger, facts: TurnFacts, code: str, hook: str, **fields: Any ) -> StreamingResponse: """The fixed refusal `code` as the whole turn, sent before the judge: no model call, no queue, no message counted.""" ledger.refuse(code, refusal(code, ledger.lang)) attribution(hook, ledger, verdict=code, **fields) return StreamingResponse( - _refused(translator, ledger), media_type="text/event-stream", headers={**_NO_STORE, "X-Accel-Buffering": "no"} + _refused(translator, ledger, facts), + media_type="text/event-stream", + headers={**_NO_STORE, "X-Accel-Buffering": "no"}, ) @@ -1645,22 +1790,25 @@ async def post_message( # session language, never the browser's locale; after the judge, the message's. lang=refusal_lang(view.language) if view else DEFAULT_LANGUAGE, ) + _stamp(ledger, view, session) + ledger.turn += 1 # this turn's number: the user turns before it, plus one + facts = TurnFacts(action=body.action, language_source=view.language_source if view else None) translator = Translator(ledger, run_id=secrets.token_hex(8), spec_id=view.spec_id if view else None) # The fixed refusals sent before the scope judge, so none of these turns costs a model call. # A closed session answers `session_closed` first: its third `attack` refusal also reaches the strike lockout, # which would otherwise mask the closure behind `budget` here (the lockout still covers the user's other sessions). refused = scope_refusals(session.events) if refused >= settings.session_refusals: - return _refuse_now(translator, ledger, SESSION_CLOSED, "session_closed", refusals=refused) + return _refuse_now(translator, ledger, facts, SESSION_CLOSED, "session_closed", refusals=refused) if services.usage.locked_out(user, settings): # Past the day's `attack` strikes: its own refusal, never the misleading "usage limit" text (2026-10-14). - return _refuse_now(translator, ledger, LOCKED_OUT, "budget_halt", agent="route") + return _refuse_now(translator, ledger, facts, LOCKED_OUT, "budget_halt", agent="route") if not budget_allows(ledger, services.usage, settings): # The refusal the root would send, sent now: a user over the daily budget spends nothing. - return _refuse_now(translator, ledger, "budget", "budget_halt", agent="route") + return _refuse_now(translator, ledger, facts, "budget", "budget_halt", agent="route") limit = services.usage.message_limit(user, settings) if limit is not None: - return _refuse_now(translator, ledger, limit, "message_limit") + return _refuse_now(translator, ledger, facts, limit, "message_limit") services.usage.add_message(user) run = ActiveRun(abort=asyncio.Event(), user=user, started=runtime.now()) # Registered here so a second request cannot slip in before the stream starts; the @@ -1733,6 +1881,7 @@ async def stream() -> AsyncIterator[str]: for chunk in translator.error(failure): yield chunk attribution("run", ledger, model_versions=sorted(ledger.model_versions), kind=ledger.kind) + _telemetry_turn(ledger, facts, failure) # Read while the run is still active: no undo or paste can slip in between the run's end # and the `done` that reports its dataset, so the version and the change lines agree. sync = await _session_sync(runtime, user, sid, dataset_before=dataset_before, ledger=ledger) @@ -1950,6 +2099,49 @@ async def bundle( return JSONResponse(payload, headers=_NO_STORE) +class OutcomeBody(_Body): + """The page's outcome beacon: what the user did with a result, never what it held.""" + + action: telemetry.OutcomeAction + version: int = Field(default=0, strict=True, ge=0, le=999) + """The result version the action was on; 0 for a session-level action (`saved_session`, feedback on the chat).""" + reaction: telemetry.Reaction | None = None + + @model_validator(mode="after") + def _reaction_with_feedback(self) -> "OutcomeBody": + if self.reaction is not None and self.action != "feedback": + raise ValueError("reaction goes with the feedback action only") + return self + + +@app.post("/v1/sessions/{sid}/outcome", status_code=204, dependencies=v1_dependencies) +async def post_outcome(sid: SessionId, body: OutcomeBody, user: User, runtime: RuntimeDep, rid: RequestId) -> Response: + """The usage telemetry's `outcome` line for one beacon of the page (`telemetry.py`). + + The page sends it after a copy, a download, a session save or a feedback send; the BFF forwards it, so every + telemetry line comes from this one writer with the same salted session hash. Content-free by its schema + (closed `action` and `reaction`, a version number). `429 rate_limited` / `daily_limit` past + `OUTCOMES_PER_MINUTE` / `_PER_DAY` per user, which the page ignores; `404 session_expired` as everywhere. + No model call, no state write: the session's events are untouched. + """ + session = await runtime.session(user, sid) + services = get_services() + limit = services.usage.outcome_limit(user) + if limit is not None: + raise AgentsError(429, limit) + services.usage.add_outcome(user) + ledger = _stamp(RequestLedger(request_id=rid, user_id=user, session_id=sid), read_session(session.state), session) + telemetry.write( + telemetry.Outcome, + ledger, + attempt=f"{body.action}:{body.version}:{body.reaction or ''}", + action=body.action, + version=body.version, + reaction=body.reaction, + ) + return Response(status_code=204, headers=_NO_STORE) + + @app.delete("/v1/sessions/{sid}", status_code=204, dependencies=v1_dependencies) async def delete_session(sid: SessionId, user: User, runtime: RuntimeDep) -> Response: await runtime.session(user, sid) diff --git a/agents/stream.py b/agents/stream.py index a2986e5396d..ae05de5f87c 100644 --- a/agents/stream.py +++ b/agents/stream.py @@ -37,6 +37,7 @@ from agents.anyplot.pipeline import STATUS_KEY from agents.anyplot.plugins.ledger import RequestLedger from agents.anyplot.schemas import RoundOutcome +from agents.anyplot.telemetry import ends_with_question PROTOCOL = "anyplot/1" @@ -146,6 +147,9 @@ def _reply(self, text: str) -> list[str]: if self.ledger.error is not None: return self.error(self.ledger.error) clean = sanitize(text, spec_id=self.spec_id) + if clean: + # The usage telemetry's `asked_question`: a bool computed here, the reply itself is never kept. + self.ledger.reply_question = ends_with_question(clean) return [sse("message", {"text": clean})] if clean else [] def error(self, code: str) -> list[str]: diff --git a/agents/telemetry/views/01_events.sql b/agents/telemetry/views/01_events.sql new file mode 100644 index 00000000000..9172ae220d5 --- /dev/null +++ b/agents/telemetry/views/01_events.sql @@ -0,0 +1,26 @@ +-- Base view of the agent usage telemetry: one row per telemetry line, deduplicated by its idempotency key. +-- +-- Reads the Log Analytics linked dataset `agent_telemetry_logs` (the `agent-telemetry` log bucket, which the +-- sink fills with `jsonPayload.log_type="agent_telemetry"` lines of `anyplot-agents`). Cloud Logging may deliver +-- a line twice, and ADK may re-run a pipeline node; both repeat the line's `idempotency_key` +-- (agents/anyplot/telemetry.py), so the earliest row per key counts. +-- +-- The lines are content-free by construction: enums, buckets, counts and ids from closed sets only +-- (docs/reference/agent-telemetry.md). `payload` keeps the whole line for the event views below. +CREATE OR REPLACE VIEW `anyplot.agent_telemetry.events` AS +SELECT + timestamp AS ts, + JSON_VALUE(json_payload.event) AS event, + LAX_INT64(json_payload.schema) AS schema_version, + JSON_VALUE(json_payload.service) AS service, + JSON_VALUE(json_payload.revision) AS revision, + JSON_VALUE(json_payload.source) AS source, + JSON_VALUE(json_payload.session) AS session, + JSON_VALUE(json_payload.spec_id) AS spec_id, + JSON_VALUE(json_payload.library) AS library, + LAX_INT64(json_payload.turn) AS turn, + JSON_VALUE(json_payload.idempotency_key) AS idempotency_key, + json_payload AS payload +FROM `anyplot.agent_telemetry_logs._AllLogs` +WHERE JSON_VALUE(json_payload.log_type) = 'agent_telemetry' +QUALIFY ROW_NUMBER() OVER (PARTITION BY JSON_VALUE(json_payload.idempotency_key) ORDER BY timestamp) = 1; diff --git a/agents/telemetry/views/02_event_views.sql b/agents/telemetry/views/02_event_views.sql new file mode 100644 index 00000000000..4024d7f4bf6 --- /dev/null +++ b/agents/telemetry/views/02_event_views.sql @@ -0,0 +1,97 @@ +-- Typed views of the four telemetry events, one column per field of `agents/anyplot/telemetry.py`. +-- Run after 01_events.sql. Each statement is its own view; `bq query` runs the script as one job. + +CREATE OR REPLACE VIEW `anyplot.agent_telemetry.data_inputs` AS +SELECT + ts, source, session, spec_id, library, turn, idempotency_key, revision, + JSON_VALUE(payload.format) AS format, + JSON_VALUE(payload.entry) AS entry, + JSON_VALUE(payload.outcome) AS outcome, + JSON_VALUE(payload.cause) AS cause, + JSON_VALUE(payload.size_bucket) AS size_bucket, + JSON_VALUE(payload.rows_bucket) AS rows_bucket, + JSON_VALUE(payload.cols_bucket) AS cols_bucket, + LAX_INT64(payload.numeric_cols) AS numeric_cols, + LAX_INT64(payload.integer_cols) AS integer_cols, + LAX_INT64(payload.datetime_cols) AS datetime_cols, + LAX_INT64(payload.text_cols) AS text_cols, + LAX_INT64(payload.boolean_cols) AS boolean_cols, + JSON_VALUE(payload.max_categories_bucket) AS max_categories_bucket, + JSON_VALUE(payload.missing_bucket) AS missing_bucket, + JSON_VALUE_ARRAY(payload.warnings) AS warnings, + LAX_INT64(payload.tables) AS tables, + LAX_INT64(payload.sheets) AS sheets, + LAX_BOOL(payload.notes) AS notes, + LAX_BOOL(payload.file_name) AS has_file_name, + JSON_VALUE(payload.spec_fit) AS spec_fit, + LAX_INT64(payload.judge_input) AS judge_input, + LAX_INT64(payload.judge_output) AS judge_output, + JSON_VALUE(payload.judge_model) AS judge_model, + LAX_INT64(payload.latency_ms) AS latency_ms +FROM `anyplot.agent_telemetry.events` +WHERE event = 'data_input'; + +CREATE OR REPLACE VIEW `anyplot.agent_telemetry.chat_turns` AS +SELECT + ts, source, session, spec_id, library, turn, idempotency_key, revision, + JSON_VALUE(payload.kind) AS kind, + JSON_VALUE(payload.action) AS action, + LAX_BOOL(payload.asked_question) AS asked_question, + JSON_VALUE_ARRAY(payload.tools) AS tools, + JSON_VALUE_ARRAY(payload.prep_ops) AS prep_ops, + LAX_INT64(payload.prep_calls) AS prep_calls, + LAX_BOOL(payload.undo) AS undo, + LAX_BOOL(payload.to_long) AS to_long, + LAX_BOOL(payload.manual_binding_changed) AS manual_binding_changed, + JSON_VALUE_ARRAY(payload.binding_warnings) AS binding_warnings, + JSON_VALUE(payload.refusal) AS refusal, + JSON_VALUE(payload.error) AS error, + LAX_BOOL(payload.reply_reasked) AS reply_reasked, + JSON_VALUE(payload.language_source) AS language_source, + LAX_INT64(payload.llm_calls) AS llm_calls, + payload.steps AS steps, + LAX_INT64(payload.latency_ms) AS latency_ms +FROM `anyplot.agent_telemetry.events` +WHERE event = 'chat_turn'; + +CREATE OR REPLACE VIEW `anyplot.agent_telemetry.pipeline_runs` AS +SELECT + ts, source, session, spec_id, library, turn, idempotency_key, revision, + LAX_INT64(payload.run_index) AS run_index, + JSON_VALUE(payload.base) AS base, + JSON_VALUE(payload.status) AS status, + JSON_VALUE(payload.reason) AS reason, + JSON_VALUE(payload.stage) AS stage, + LAX_INT64(payload.attempts) AS attempts, + LAX_INT64(payload.rounds) AS rounds, + LAX_INT64(payload.reviews) AS reviews, + JSON_VALUE(payload.score_round1) AS score_round1, + JSON_VALUE(payload.score_round2) AS score_round2, + LAX_INT64(payload.shipped_attempt) AS shipped_attempt, + LAX_INT64(payload.shipped_round) AS shipped_round, + JSON_VALUE_ARRAY(payload.forced_repair) AS forced_repair, + JSON_VALUE_ARRAY(payload.gates_failed) AS gates_failed, + JSON_VALUE_ARRAY(payload.reviewer_defects) AS reviewer_defects, + JSON_VALUE_ARRAY(payload.residual_defects) AS residual_defects, + JSON_VALUE_ARRAY(payload.sandbox_failures) AS sandbox_failures, + LAX_INT64(payload.adapt_ms) AS adapt_ms, + LAX_INT64(payload.check_ms) AS check_ms, + LAX_INT64(payload.render_ms) AS render_ms, + LAX_INT64(payload.review_ms) AS review_ms, + LAX_INT64(payload.total_ms) AS total_ms, + payload.steps AS steps, + JSON_VALUE(payload.status) IN ('ok', 'needs_attention') AS shipped +FROM `anyplot.agent_telemetry.events` +WHERE event = 'pipeline_run'; + +CREATE OR REPLACE VIEW `anyplot.agent_telemetry.outcomes` AS +SELECT + ts, source, session, spec_id, library, turn, idempotency_key, revision, + JSON_VALUE(payload.action) AS action, + LAX_INT64(payload.version) AS version, + JSON_VALUE(payload.reaction) AS reaction, + JSON_VALUE(payload.action) IN ( + 'copied_code', 'copied_image', 'downloaded_png', 'downloaded_code', 'downloaded_data', 'saved_session' + ) AS keeps +FROM `anyplot.agent_telemetry.events` +WHERE event = 'outcome'; diff --git a/agents/telemetry/views/03_sessions.sql b/agents/telemetry/views/03_sessions.sql new file mode 100644 index 00000000000..ba97dac423f --- /dev/null +++ b/agents/telemetry/views/03_sessions.sql @@ -0,0 +1,98 @@ +-- One row per chat session (salted session hash and source): its funnel, its plots and how it ended. +-- Run after 02_event_views.sql. +-- +-- `funnel_last_step` is the furthest step the session reached: +-- data_refused every data input was refused or failed +-- data_ok a dataset was stored +-- chatted a chat turn ran +-- plot_run a pipeline run ran +-- plot_shipped a pipeline run shipped a plot (status ok or needs_attention) +-- kept the user copied or downloaded a result or saved the session +-- A session that never sent a data input or a turn writes no line, so the funnel starts at the first input. +-- +-- `state` applies a 30-minute inactivity rule: a session whose last line is younger than 30 minutes is +-- `active`; any other session ended, as +-- kept the user kept a result (copied, downloaded or saved) +-- refined no keep, but two or more shipped plots (the user iterated on a result) +-- abandoned_after_plot a shipped plot, no keep and no refinement +-- abandoned_before_plot no shipped plot +CREATE OR REPLACE VIEW `anyplot.agent_telemetry.sessions` AS +WITH lines AS ( + SELECT session, source, ts, spec_id, library FROM `anyplot.agent_telemetry.events` +), +spans AS ( + SELECT + session, + source, + MIN(ts) AS first_at, + MAX(ts) AS last_at, + ARRAY_AGG(spec_id IGNORE NULLS ORDER BY ts LIMIT 1)[SAFE_OFFSET(0)] AS spec_id, + ARRAY_AGG(library IGNORE NULLS ORDER BY ts LIMIT 1)[SAFE_OFFSET(0)] AS first_library, + COUNT(DISTINCT library) AS libraries + FROM lines + GROUP BY session, source +), +inputs AS ( + SELECT session, source, COUNT(*) AS data_inputs, COUNTIF(outcome = 'ok') AS data_ok + FROM `anyplot.agent_telemetry.data_inputs` + GROUP BY session, source +), +turns AS ( + SELECT session, source, COUNT(*) AS turns, MAX(turn) AS last_turn + FROM `anyplot.agent_telemetry.chat_turns` + GROUP BY session, source +), +runs AS ( + SELECT + session, + source, + COUNT(*) AS runs, + COUNTIF(shipped) AS plots_shipped, + MIN(IF(shipped, turn, NULL)) AS first_plot_turn, + COUNTIF(rounds > 1) AS repaired_runs + FROM `anyplot.agent_telemetry.pipeline_runs` + GROUP BY session, source +), +kept AS ( + SELECT session, source, COUNTIF(keeps) AS keeps, COUNTIF(action = 'feedback') AS feedback + FROM `anyplot.agent_telemetry.outcomes` + GROUP BY session, source +) +SELECT + spans.session, + spans.source, + spans.spec_id, + spans.first_library AS library, + spans.libraries, + spans.first_at, + spans.last_at, + TIMESTAMP_DIFF(spans.last_at, spans.first_at, SECOND) AS duration_s, + IFNULL(inputs.data_inputs, 0) AS data_inputs, + IFNULL(inputs.data_ok, 0) AS data_ok, + IFNULL(turns.turns, 0) AS turns, + IFNULL(runs.runs, 0) AS runs, + IFNULL(runs.plots_shipped, 0) AS plots_shipped, + IFNULL(runs.repaired_runs, 0) AS repaired_runs, + runs.first_plot_turn AS turns_to_first_plot, + IFNULL(kept.keeps, 0) AS keeps, + IFNULL(kept.feedback, 0) AS feedback, + CASE + WHEN IFNULL(kept.keeps, 0) > 0 THEN 'kept' + WHEN IFNULL(runs.plots_shipped, 0) > 0 THEN 'plot_shipped' + WHEN IFNULL(runs.runs, 0) > 0 THEN 'plot_run' + WHEN IFNULL(turns.turns, 0) > 0 THEN 'chatted' + WHEN IFNULL(inputs.data_ok, 0) > 0 THEN 'data_ok' + ELSE 'data_refused' + END AS funnel_last_step, + CASE + WHEN spans.last_at > TIMESTAMP_SUB(CURRENT_TIMESTAMP(), INTERVAL 30 MINUTE) THEN 'active' + WHEN IFNULL(kept.keeps, 0) > 0 THEN 'kept' + WHEN IFNULL(runs.plots_shipped, 0) >= 2 THEN 'refined' + WHEN IFNULL(runs.plots_shipped, 0) > 0 THEN 'abandoned_after_plot' + ELSE 'abandoned_before_plot' + END AS state +FROM spans +LEFT JOIN inputs USING (session, source) +LEFT JOIN turns USING (session, source) +LEFT JOIN runs USING (session, source) +LEFT JOIN kept USING (session, source); diff --git a/agents/telemetry/views/04_usage_failures.sql b/agents/telemetry/views/04_usage_failures.sql new file mode 100644 index 00000000000..27188b7906e --- /dev/null +++ b/agents/telemetry/views/04_usage_failures.sql @@ -0,0 +1,71 @@ +-- Usage by spec, library and input format, and the failures by kind and code. Run after 03_sessions.sql. + +-- One row per day, spec, library and input format: what was pasted, what ran, what shipped, what was kept. +-- `format` is the format of the session's stored data inputs on that day (`(none)` for sessions without one, +-- such as data computed in the chat from a formula or code). +CREATE OR REPLACE VIEW `anyplot.agent_telemetry.usage` AS +WITH formats AS ( + SELECT DISTINCT DATE(ts) AS day, session, source, format + FROM `anyplot.agent_telemetry.data_inputs` + WHERE outcome = 'ok' +), +daily AS ( + SELECT + DATE(ts) AS day, + session, + source, + ANY_VALUE(spec_id) AS spec_id, + ANY_VALUE(library) AS library, + COUNTIF(event = 'data_input') AS data_inputs, + COUNTIF(event = 'chat_turn') AS turns, + COUNTIF(event = 'pipeline_run') AS runs, + COUNTIF(event = 'pipeline_run' AND JSON_VALUE(payload.status) IN ('ok', 'needs_attention')) AS plots_shipped, + COUNTIF( + event = 'outcome' + AND JSON_VALUE(payload.action) IN ( + 'copied_code', 'copied_image', 'downloaded_png', 'downloaded_code', 'downloaded_data', 'saved_session' + ) + ) AS keeps + FROM `anyplot.agent_telemetry.events` + GROUP BY day, session, source +) +SELECT + daily.day, + daily.source, + daily.spec_id, + daily.library, + IFNULL(formats.format, '(none)') AS format, + COUNT(DISTINCT daily.session) AS sessions, + SUM(daily.data_inputs) AS data_inputs, + SUM(daily.turns) AS turns, + SUM(daily.runs) AS runs, + SUM(daily.plots_shipped) AS plots_shipped, + COUNT(DISTINCT IF(daily.keeps > 0, daily.session, NULL)) AS sessions_kept +FROM daily +LEFT JOIN formats USING (day, session, source) +GROUP BY day, source, spec_id, library, format; + +-- One row per failure: a refused or failed data input, a refused or failed turn, a failed pipeline run, and +-- every sandbox failure class and failed gate of a run. `kind` says which, `code` the closed code. +CREATE OR REPLACE VIEW `anyplot.agent_telemetry.failures` AS +SELECT ts, source, session, spec_id, library, 'data_input' AS kind, outcome AS code, cause AS detail +FROM `anyplot.agent_telemetry.data_inputs` +WHERE outcome NOT IN ('ok', 'transcribed') +UNION ALL +SELECT ts, source, session, spec_id, library, 'turn_refused', refusal, NULL +FROM `anyplot.agent_telemetry.chat_turns` +WHERE refusal IS NOT NULL +UNION ALL +SELECT ts, source, session, spec_id, library, 'turn_error', error, NULL +FROM `anyplot.agent_telemetry.chat_turns` +WHERE error IS NOT NULL +UNION ALL +SELECT ts, source, session, spec_id, library, 'run_failed', IFNULL(reason, status), stage +FROM `anyplot.agent_telemetry.pipeline_runs` +WHERE status IN ('failed', 'not_ready') +UNION ALL +SELECT ts, source, session, spec_id, library, 'sandbox', failure, NULL +FROM `anyplot.agent_telemetry.pipeline_runs`, UNNEST(sandbox_failures) AS failure +UNION ALL +SELECT ts, source, session, spec_id, library, 'gate', gate, NULL +FROM `anyplot.agent_telemetry.pipeline_runs`, UNNEST(gates_failed) AS gate; diff --git a/agents/telemetry/views/05_prices_cost.sql b/agents/telemetry/views/05_prices_cost.sql new file mode 100644 index 00000000000..4b9f8eae312 --- /dev/null +++ b/agents/telemetry/views/05_prices_cost.sql @@ -0,0 +1,52 @@ +-- Model list prices and the cost of every turn and data input. Run after 04_usage_failures.sql. +-- +-- `model_prices` holds USD per million tokens at the `eu` multi-region (the global list price plus 10 %), as +-- `agents/evals/pricing.py` records them (2026-10-10); `tests/unit/agents/evals/test_telemetry_prices.py` fails +-- when the two disagree, so a price change there names this file. List price, never a promotion; Vertex AI's +-- partner pricing for Claude may differ: re-check before a spend decision. +CREATE OR REPLACE VIEW `anyplot.agent_telemetry.model_prices` AS +SELECT * FROM UNNEST([ + STRUCT('claude-haiku-5-5' AS model, 0.11 AS input_usd_per_m, 0.55 AS output_usd_per_m), + STRUCT('claude-sonnet-5-5' AS model, 2.20 AS input_usd_per_m, 11.00 AS output_usd_per_m), + STRUCT('claude-opus-5-5' AS model, 4.40 AS input_usd_per_m, 22.00 AS output_usd_per_m), + STRUCT('gemini-3.8-flash' AS model, 1.65 AS input_usd_per_m, 8.25 AS output_usd_per_m), + STRUCT('gemini-3.5-flash-lite' AS model, 0.33 AS input_usd_per_m, 2.75 AS output_usd_per_m) +]); + +-- One row per step of a turn (root, adapter, reviewer, judge) and per data input's judge calls, priced. +-- `input` counts cached and cache-write tokens too (agents/anyplot/plugins/ledger.py `StepUsage`): cached ones +-- cost 0.1 and cache writes 1.25 of the input price, as `pricing.py` prices them. The cost comes from the +-- `chat_turn` and `data_input` lines only: `pipeline_run.steps` repeats the adapter's and the reviewer's share +-- of its turn for per-run analysis and is left out here, so no token is priced twice. A model without a price +-- row has a NULL cost (add the row above). +CREATE OR REPLACE VIEW `anyplot.agent_telemetry.cost` AS +WITH steps AS ( + SELECT + turns.ts, turns.source, turns.session, turns.spec_id, turns.library, 'chat_turn' AS origin, + JSON_VALUE(step.step) AS step, + REGEXP_EXTRACT(JSON_VALUE(step.model), r'^[^@]+') AS model, + IFNULL(LAX_INT64(step.calls), 0) AS calls, + IFNULL(LAX_INT64(step.input), 0) AS input_tokens, + IFNULL(LAX_INT64(step.output), 0) AS output_tokens, + IFNULL(LAX_INT64(step.cached), 0) AS cached_tokens, + IFNULL(LAX_INT64(step.cache_write), 0) AS cache_write_tokens + FROM `anyplot.agent_telemetry.chat_turns` AS turns, UNNEST(JSON_QUERY_ARRAY(turns.steps)) AS step + UNION ALL + SELECT + ts, source, session, spec_id, library, 'data_input', 'judge', + REGEXP_EXTRACT(judge_model, r'^[^@]+'), + IF(judge_input + judge_output > 0, 1, 0), + judge_input, judge_output, 0, 0 + FROM `anyplot.agent_telemetry.data_inputs` + WHERE judge_input + judge_output > 0 +) +SELECT + steps.*, + ( + GREATEST(steps.input_tokens - steps.cached_tokens - steps.cache_write_tokens, 0) * prices.input_usd_per_m + + steps.cached_tokens * prices.input_usd_per_m * 0.1 + + steps.cache_write_tokens * prices.input_usd_per_m * 1.25 + + steps.output_tokens * prices.output_usd_per_m + ) / 1e6 AS cost_usd +FROM steps +LEFT JOIN `anyplot.agent_telemetry.model_prices` AS prices USING (model); diff --git a/api/agent_usage.py b/api/agent_usage.py new file mode 100644 index 00000000000..ee0469b27ab --- /dev/null +++ b/api/agent_usage.py @@ -0,0 +1,240 @@ +"""Aggregates of the agent usage telemetry for the admin debug page (`GET /debug/agent-usage`). + +The agents service writes content-free telemetry lines (`agents/anyplot/telemetry.py`); a +Cloud Logging sink routes them into a log bucket that BigQuery reads through a linked dataset, +and the views in `agents/telemetry/views/*.sql` shape them (`docs/reference/agent-telemetry.md`). +This module runs one query against those views and answers aggregates only: counts, two rates +and the five most frequent specs, input formats and refusal codes, every key an id or a code +from a closed set. Live sessions only (`source = 'live'`): the eval harnesses tag theirs +`harness`. + +`AGENT_TELEMETRY_DATASET` unset answers `not_configured`; a query that fails (no table yet, no +permission, a timeout) answers `unavailable`, logged with the exception class only. Both carry +zeros, so the page shows its empty state and never an error. +""" + +from __future__ import annotations + +import logging +import re +from datetime import UTC, datetime +from functools import lru_cache +from typing import Any, Literal, Protocol + +from pydantic import BaseModel, Field + + +logger = logging.getLogger(__name__) + +UsageStatus = Literal["ok", "not_configured", "unavailable"] +USAGE_WINDOWS = (7, 30) +TOP_ITEMS = 5 +QUERY_TIMEOUT_S = 20.0 +DATASET_PATTERN = re.compile(r"^[a-z][a-z0-9-]{4,28}[a-z0-9]\.[A-Za-z_][A-Za-z0-9_]{0,1023}$") +"""`.`, checked before it goes into the SQL text: a dataset name cannot be a query parameter.""" + +KEEP_ACTIONS = ("copied_code", "copied_image", "downloaded_png", "downloaded_code", "downloaded_data", "saved_session") + + +class UsageItem(BaseModel): + key: str + count: int = Field(ge=0) + + +class AgentUsage(BaseModel): + status: UsageStatus + days: int + sessions: int = 0 + plots_shipped: int = 0 + kept_rate: float | None = None + """Sessions whose user kept a result (copied, downloaded, saved) among the sessions with a shipped plot.""" + repair_rate: float | None = None + """Pipeline runs that needed more than one round among all runs.""" + top_specs: list[UsageItem] = Field(default_factory=list) + top_formats: list[UsageItem] = Field(default_factory=list) + top_refusals: list[UsageItem] = Field(default_factory=list) + generated_at: str + + +class UsageQuery(Protocol): + """Runs one standard-SQL query with an INT64 `@days` parameter and returns its rows as dicts.""" + + def rows(self, sql: str, days: int) -> list[dict[str, Any]]: ... + + +BIGQUERY_SCOPE = "https://www.googleapis.com/auth/bigquery" +QUERY_URL = "https://bigquery.googleapis.com/bigquery/v2/projects/{project}/queries" + + +class QueryIncomplete(Exception): + """BigQuery did not finish the query within `QUERY_TIMEOUT_S` (`jobComplete: false`).""" + + +class BigQueryUsage: + """`UsageQuery` on BigQuery's REST API (`jobs.query`), billed to the dataset's project. + + It runs through google-auth's `AuthorizedSession`, which the API already ships for the + agent BFF's ID tokens, so the API image carries no BigQuery client library (and no gRPC). + One synchronous call: the query is small, and an answer that is not complete within + `QUERY_TIMEOUT_S` counts as a failure (`QueryIncomplete`), never a poll. + """ + + def __init__(self, project: str) -> None: + import google.auth + from google.auth.transport.requests import AuthorizedSession + + credentials, _ = google.auth.default(scopes=[BIGQUERY_SCOPE]) + self._session = AuthorizedSession(credentials) + self._url = QUERY_URL.format(project=project) + + def rows(self, sql: str, days: int) -> list[dict[str, Any]]: + body = { + "query": sql, + "useLegacySql": False, + "useQueryCache": True, + "timeoutMs": int(QUERY_TIMEOUT_S * 1000), + "parameterMode": "NAMED", + "queryParameters": [ + {"name": "days", "parameterType": {"type": "INT64"}, "parameterValue": {"value": str(days)}} + ], + } + response = self._session.post(self._url, json=body, timeout=QUERY_TIMEOUT_S + 10) + response.raise_for_status() + return decode_rows(response.json()) + + +def decode_rows(answer: dict[str, Any]) -> list[dict[str, Any]]: + """The rows of a `jobs.query` answer as dicts, typed by its schema (REST sends every scalar as a string).""" + if not answer.get("jobComplete", False): + raise QueryIncomplete + fields = (answer.get("schema") or {}).get("fields") or [] + return [ + {field["name"]: _cell(field, cell) for field, cell in zip(fields, row.get("f") or [], strict=False)} + for row in answer.get("rows") or [] + ] + + +def _cell(field: dict[str, Any], cell: dict[str, Any] | None) -> Any: + value = (cell or {}).get("v") + if field.get("mode") == "REPEATED": + return [_scalar(field, item.get("v") if isinstance(item, dict) else None) for item in value or []] + return _scalar(field, value) + + +def _scalar(field: dict[str, Any], value: Any) -> Any: + if value is None: + return None + kind = field.get("type") + if kind in ("RECORD", "STRUCT"): + subfields = field.get("fields") or [] + return {sub["name"]: _cell(sub, cell) for sub, cell in zip(subfields, value.get("f") or [], strict=False)} + if kind in ("INTEGER", "INT64"): + return int(value) + if kind in ("FLOAT", "FLOAT64", "NUMERIC", "BIGNUMERIC"): + return float(value) + if kind in ("BOOLEAN", "BOOL"): + return value == "true" + return value + + +def usage_sql(dataset: str) -> str: + """The one query behind the card, over the views of `dataset` (`.`).""" + keep = ", ".join(f"'{action}'" for action in KEEP_ACTIONS) + return f""" +WITH ev AS ( + SELECT * FROM `{dataset}.events` + WHERE source = 'live' AND ts >= TIMESTAMP_SUB(CURRENT_TIMESTAMP(), INTERVAL @days DAY) +), +runs AS ( + SELECT JSON_VALUE(payload.status) AS status, LAX_INT64(payload.rounds) AS rounds, session + FROM ev WHERE event = 'pipeline_run' +), +shipped_sessions AS ( + SELECT DISTINCT session FROM runs WHERE status IN ('ok', 'needs_attention') +), +kept_sessions AS ( + SELECT DISTINCT session FROM ev WHERE event = 'outcome' AND JSON_VALUE(payload.action) IN ({keep}) +), +refusals AS ( + SELECT IFNULL(JSON_VALUE(payload.cause), JSON_VALUE(payload.outcome)) AS key + FROM ev WHERE event = 'data_input' AND JSON_VALUE(payload.outcome) NOT IN ('ok', 'transcribed') + UNION ALL + SELECT JSON_VALUE(payload.refusal) FROM ev WHERE event = 'chat_turn' AND JSON_VALUE(payload.refusal) IS NOT NULL +) +SELECT + (SELECT COUNT(DISTINCT session) FROM ev) AS sessions, + (SELECT COUNTIF(status IN ('ok', 'needs_attention')) FROM runs) AS plots_shipped, + (SELECT SAFE_DIVIDE(COUNTIF(session IN (SELECT session FROM kept_sessions)), COUNT(*)) FROM shipped_sessions) + AS kept_rate, + (SELECT SAFE_DIVIDE(COUNTIF(rounds > 1), COUNT(*)) FROM runs) AS repair_rate, + ARRAY( + SELECT AS STRUCT spec_id AS key, COUNT(DISTINCT session) AS count FROM ev + WHERE spec_id IS NOT NULL GROUP BY spec_id ORDER BY count DESC, key LIMIT {TOP_ITEMS} + ) AS top_specs, + ARRAY( + SELECT AS STRUCT JSON_VALUE(payload.format) AS key, COUNT(*) AS count FROM ev + WHERE event = 'data_input' GROUP BY key ORDER BY count DESC, key LIMIT {TOP_ITEMS} + ) AS top_formats, + ARRAY( + SELECT AS STRUCT key, COUNT(*) AS count FROM refusals + WHERE key IS NOT NULL GROUP BY key ORDER BY count DESC, key LIMIT {TOP_ITEMS} + ) AS top_refusals +""" + + +def _now() -> str: + return datetime.now(UTC).isoformat(timespec="seconds").replace("+00:00", "Z") + + +def _items(value: Any) -> list[UsageItem]: + if not isinstance(value, list): + return [] + items = [] + for entry in value[:TOP_ITEMS]: + key, count = (entry.get("key"), entry.get("count")) if isinstance(entry, dict) else (None, None) + if isinstance(key, str) and isinstance(count, int): + items.append(UsageItem(key=key, count=count)) + return items + + +def _rate(value: Any) -> float | None: + return round(float(value), 4) if isinstance(value, int | float) else None + + +def read_usage(query: UsageQuery | None, dataset: str | None, days: int) -> AgentUsage: + """The card's aggregates for the last `days` days; `not_configured` or `unavailable` with zeros otherwise.""" + if not dataset or not DATASET_PATTERN.fullmatch(dataset): + return AgentUsage(status="not_configured", days=days, generated_at=_now()) + if query is None: # configured, but the client could not start (no credentials) + return AgentUsage(status="unavailable", days=days, generated_at=_now()) + try: + rows = query.rows(usage_sql(dataset), days) + except Exception as exc: # no table yet, no permission, a timeout: the page shows its empty state + logger.warning("agent usage query failed: %s", type(exc).__name__) + return AgentUsage(status="unavailable", days=days, generated_at=_now()) + row = rows[0] if rows else {} + return AgentUsage( + status="ok", + days=days, + sessions=int(row.get("sessions") or 0), + plots_shipped=int(row.get("plots_shipped") or 0), + kept_rate=_rate(row.get("kept_rate")), + repair_rate=_rate(row.get("repair_rate")), + top_specs=_items(row.get("top_specs")), + top_formats=_items(row.get("top_formats")), + top_refusals=_items(row.get("top_refusals")), + generated_at=_now(), + ) + + +@lru_cache(maxsize=4) +def usage_query(dataset: str | None) -> UsageQuery | None: + """The BigQuery client for `dataset`'s project, one per process, or None when the dataset is unset or the + client cannot start.""" + if not dataset or not DATASET_PATTERN.fullmatch(dataset): + return None + try: + return BigQueryUsage(project=dataset.split(".", 1)[0]) + except Exception as exc: # no credentials (a local run): the card's `unavailable` state + logger.warning("BigQuery client unavailable: %s", type(exc).__name__) + return None diff --git a/api/routers/agent.py b/api/routers/agent.py index f6429d916cb..59c49df8389 100644 --- a/api/routers/agent.py +++ b/api/routers/agent.py @@ -565,12 +565,33 @@ class RenderThemeBody(_StrictBody): theme: Literal["light", "dark"] +OutcomeAction = Literal[ + "copied_code", "copied_image", "downloaded_png", "downloaded_code", "downloaded_data", "saved_session", "feedback" +] +"""What the page reports a user did with a result (`agents/anyplot/telemetry.py` `OutcomeAction`; copied, because +the API image does not ship the agents package).""" + + def _check_reaction(value: str) -> str: if value not in FEEDBACK_REACTIONS: raise ValueError(f"reaction must be one of {', '.join(FEEDBACK_REACTIONS)}") return value +class OutcomeBody(_StrictBody): + # The page's content-free outcome beacon: a closed action, the version it was on (0 for the session), + # and with `feedback` the reaction. Nothing the user wrote or the result held ever travels here. + action: OutcomeAction + version: Annotated[int, Field(strict=True, ge=0, le=999)] = 0 + reaction: Annotated[str, AfterValidator(_check_reaction)] | None = None + + @model_validator(mode="after") + def _reaction_with_feedback(self) -> OutcomeBody: + if self.reaction is not None and self.action != "feedback": + raise ValueError("reaction goes with the feedback action only") + return self + + def _feedback_message(value: str | None) -> str | None: """The message stripped, None when nothing is left; over `MAX_FEEDBACK_CHARS` after the strip is refused.""" if value is None: @@ -1251,6 +1272,20 @@ async def cancel_run(sid: SessionId, ctx: Ctx, client: Client) -> Response: return _no_content(ctx) +@router.post("/sessions/{sid}/outcome", status_code=204) +async def post_outcome(sid: SessionId, body: OutcomeBody, ctx: Ctx, client: Client) -> Response: + """The page's outcome beacon (a copy, a download, a session save, a feedback send), forwarded to the agents + service, which writes the content-free `outcome` telemetry line (`docs/reference/agent-telemetry.md`). + + Fire and forget for the page: `429 rate_limited` / `daily_limit` past the agents service's per-user beacon + limits, `404 session_expired`, and an unreachable service are answered as for any route and ignored there. + """ + await _call_upstream( + client, ctx, "POST", f"/sessions/{sid}/outcome", json_body=body.model_dump(mode="json", exclude_none=True) + ) + return _no_content(ctx) + + @router.post("/sessions/{sid}/versions/{version}/render") async def render_version_theme( sid: SessionId, version: Annotated[int, Path(ge=0, le=999)], body: RenderThemeBody, ctx: Ctx, client: Client diff --git a/api/routers/debug.py b/api/routers/debug.py index d580cbf70d0..9069847a883 100644 --- a/api/routers/debug.py +++ b/api/routers/debug.py @@ -7,14 +7,16 @@ from dataclasses import dataclass from datetime import datetime, timedelta, timezone from functools import lru_cache -from typing import Literal +from typing import Annotated, Literal import jwt as pyjwt -from fastapi import APIRouter, Depends, Header, HTTPException, Request, status +from fastapi import APIRouter, Depends, Header, HTTPException, Query, Request, status +from fastapi.concurrency import run_in_threadpool from pydantic import BaseModel from sqlalchemy import text from sqlalchemy.ext.asyncio import AsyncSession +from api.agent_usage import USAGE_WINDOWS, AgentUsage, UsageQuery, read_usage, usage_query from api.cache import clear_cache, get_cache_stats from api.dependencies import require_db from api.exceptions import raise_validation_error @@ -509,6 +511,26 @@ async def ping(db: AsyncSession = Depends(require_db)) -> PingResponse: ) +def get_usage_query() -> UsageQuery | None: + """The BigQuery reader of the agent usage views; tests override it with a fake.""" + return usage_query(settings.agent_telemetry_dataset) + + +@router.get("/agent-usage", response_model=AgentUsage, dependencies=[Depends(require_admin)]) +async def agent_usage( + days: Annotated[int, Query()] = 7, query: UsageQuery | None = Depends(get_usage_query) +) -> AgentUsage: + """The agent chat's usage highlights for the last 7 or 30 days, read-only from the BigQuery telemetry views. + + Aggregates only (counts, two rates, the top five specs, input formats and refusal codes); `status` + `not_configured` while `AGENT_TELEMETRY_DATASET` is unset and `unavailable` when the query fails, both + with zeros (`api/agent_usage.py`, `docs/reference/agent-telemetry.md`). `days` other than 7 or 30 is a 400. + """ + if days not in USAGE_WINDOWS: + raise_validation_error(f"days must be one of {', '.join(map(str, USAGE_WINDOWS))}") + return await run_in_threadpool(read_usage, query, settings.agent_telemetry_dataset, days) + + class CacheInvalidateResponse(BaseModel): cleared: int maxsize: int diff --git a/changelog.d/agent-usage-telemetry.md b/changelog.d/agent-usage-telemetry.md new file mode 100644 index 00000000000..e5ef4e27e98 --- /dev/null +++ b/changelog.d/agent-usage-telemetry.md @@ -0,0 +1,17 @@ +### Added + +- **Content-free usage telemetry for the agent chat.** The agents service writes four events, + `data_input`, `chat_turn`, `pipeline_run` and `outcome`, one wide JSON line each, on a logger of + their own with `log_type: "agent_telemetry"` for a Cloud Logging sink into a 180-day log bucket + that BigQuery reads. Every field is a closed enum, a bucket, a count or an id from a closed set + (strict pydantic models, parser warnings mapped to closed codes in + `agents/anyplot/data/warning_codes.py`), the session is a salted per-process hash, no line carries + the user id, and a canary test fails if a column name, cell, note, file name or message reaches a + line. The page reports copies, downloads, saves and feedback through a new beacon + (`POST /debug/agent/sessions/{sid}/outcome`), and the eval harnesses tag their lines `harness`. + Committed BigQuery views (`agents/telemetry/views/`) deduplicate by idempotency key and add + sessions with a 30-minute inactivity rule, usage, failures and cost; `docs/reference/agent-telemetry.md` + lists the one-time setup. +- **Agent usage card on `/debug`.** Sessions, plots shipped, kept and repair rates, and the top specs, + input formats and refusal codes of the last 7 or 30 days, read through the new admin-only + `GET /debug/agent-usage` (aggregates only; an empty state while `AGENT_TELEMETRY_DATASET` is unset). diff --git a/core/config.py b/core/config.py index bde8d4e12af..435d4af72f7 100644 --- a/core/config.py +++ b/core/config.py @@ -305,6 +305,14 @@ def _parse_admin_allowed_emails(cls, value: Any) -> Any: public `gcs_bucket`. The API's runtime identity needs object create rights on it; the route writes nothing else.""" + agent_telemetry_dataset: str | None = None + """`.` of the agent usage telemetry's BigQuery views + (`agents/telemetry/views/*.sql`, `docs/reference/agent-telemetry.md`), such as + `anyplot.agent_telemetry`. Unset, `GET /debug/agent-usage` answers + `status: not_configured` with empty numbers. The API's runtime identity + needs `roles/bigquery.dataViewer` on that dataset and on the linked log + dataset the views read, and `roles/bigquery.jobUser` on the project.""" + # ============================================================================= # CORS # ============================================================================= diff --git a/docs/concepts/agent-network.md b/docs/concepts/agent-network.md index 7f51652de15..ae70ffcfb53 100644 --- a/docs/concepts/agent-network.md +++ b/docs/concepts/agent-network.md @@ -399,6 +399,17 @@ Built since 2026-10-12, the capture half. Every result card carries the control - **Analytics.** `agent_result_feedback{reaction, include_data, subject}` with enum properties only; `subject` is `result` or `chat`. - **Privacy.** Phase 1 holds owner data only. Before real users can submit: consent text on the control, a legal-page entry for the case store (purpose, retention of 180 days, EU bucket), `include_data` off by default, and a delete-my-case route keyed by the case id shown to the user. +## Usage telemetry + +Built since 2026-10-16 in the repository; the Google Cloud side is a one-time setup ([Agent usage telemetry](../reference/agent-telemetry.md)). Feedback cases answer "what went wrong in this session"; the telemetry answers "how is the chat used": which inputs arrive and which are refused, how many turns a first plot takes, how often a run needs a repair, which sessions end with a kept result, and what each step costs. + +- **Content-free by construction.** Owner rule of 2026-10-16: no user input, question, reply, data value, column name, file or sheet name, or cell, ever; only enums, coarse buckets, counts, durations and ids from closed sets. Each event is a strict pydantic model (`agents/anyplot/telemetry.py`), warning sentences become closed codes before they reach a line (`agents/anyplot/data/warning_codes.py`), and a canary test runs a whole session with distinctive content and fails if any of it appears in a line. Content still reaches the owner only through the explicit feedback case above. +- **Four events, one writer.** `data_input`, `chat_turn`, `pipeline_run` and `outcome`, one wide JSON line each, all written by anyplot-agents on its own logger; the page's outcome beacon (copy, download, save, feedback) goes through the BFF to `POST /v1/sessions/{sid}/outcome`, so every line carries the same salted session hash. The per-hook attribution lines stay for debugging and keep the HMAC user id; telemetry lines never carry it. +- **Session hash.** HMAC-SHA256 of the session id under a random per-process salt: the id is in every request URL Cloud Run logs, so a plain hash could be joined to the request log. Sessions die with the instance, so the salt lives exactly as long as they do. +- **Storage and analysis.** A sink routes `jsonPayload.log_type="agent_telemetry"` from anyplot-agents into the log bucket `agent-telemetry` (europe-west4, 180 days, owner decision of 2026-10-16), which BigQuery reads through a linked dataset. The committed views (`agents/telemetry/views/*.sql`) deduplicate by idempotency key and add sessions with a 30-minute inactivity rule, usage, failures and cost; Looker Studio reads them, and the admin `/debug` page shows a small "agent usage" card through `GET /debug/agent-usage` (aggregates only). +- **Harness runs.** The eval harnesses tag their lines `source: harness`; the views and the card count `live` sessions. +- **Privacy.** Before the chat opens to the public, the legal page needs an entry for the telemetry (purpose, the fields in plain words, 180 days, EU location, no content). The chat is admin-only on 2026-10-16, so the legal page is unchanged. + ## Model-version regression harness Testing a newer model version, the other provider's arm, a different judge model, or a prompt or validator change is one command locally or one workflow dispatch in CI, and the answer is a diff against the pinned baseline. diff --git a/docs/index.md b/docs/index.md index 2ad8d8feee4..92484f01b2d 100644 --- a/docs/index.md +++ b/docs/index.md @@ -11,6 +11,7 @@ Welcome to the anyplot documentation. Start here to find what you're looking for | Understand the project | [Vision](concepts/vision.md) | | Read the "Use with my data" agent design | [Agent network](concepts/agent-network.md) | | Read how catalogue code adapts to other data | [Adaptable code](concepts/adaptable-code.md) | +| Read or set up the agent chat's usage telemetry | [Agent usage telemetry](reference/agent-telemetry.md) | | Contribute plot ideas | [Contributing](contributing.md) | | Set up local development | [Development Guide](development.md) | | See how automation works | [Workflows](workflows/overview.md) | @@ -52,6 +53,7 @@ docs/ ├── seo.md # SEO configuration ├── style-guide.md # Brand, frontend, plot design system ├── gcp-budget-guard.md # Function that disables Vertex AI when its budget is reached + ├── agent-telemetry.md # Content-free usage telemetry of the agent chat, BigQuery setup └── performance.md # API latency measurements ``` @@ -91,6 +93,7 @@ Technical details for development and integration. - **[SEO](reference/seo.md)** - Search engine optimization setup - **[Style Guide](reference/style-guide.md)** - Brand, frontend, and plot design system - **[GCP Budget Guard](reference/gcp-budget-guard.md)** - Cloud Run function that disables the Vertex AI API when its Cloud Billing budget is reached: decision rules, logging, retries, permissions, limits (as of 2026-10-09, in the repository and not yet deployed) +- **[Agent usage telemetry](reference/agent-telemetry.md)** - The agent chat's four content-free events, the log bucket, sink and BigQuery views, the `/debug` card, and the one-time setup (as of 2026-10-16, in the repository; the Google Cloud resources are not created yet) - **[Performance](reference/performance.md)** - Backend API response-time measurements --- diff --git a/docs/reference/agent-telemetry.md b/docs/reference/agent-telemetry.md new file mode 100644 index 00000000000..0f439671f99 --- /dev/null +++ b/docs/reference/agent-telemetry.md @@ -0,0 +1,223 @@ +# Agent usage telemetry + +> **Status (2026-10-16):** the agents service writes the four events, the BFF forwards the page's outcome +> beacon, the views and the `/debug` card are in the repository. The log bucket, the sink, the linked dataset, +> the views dataset and the IAM grants are not created yet: run [Set up the BigQuery side](#set-up-the-bigquery-side) +> once. Until then the card shows `not configured`. + +The admin-only agent chat ("Use with my data") records how it is used, without recording what anyone +used it for. This page lists what is recorded, where it goes, how long it stays, and how to read it. + +## What is recorded, and what never is + +Owner rule of 2026-10-16: no line holds a user input, a question, a reply, a data value, a column +name, a file or sheet name, or a cell. A line holds only enums, coarse buckets, counts, durations, and ids +from closed sets (spec ids, library ids, model ids, review defect ids such as `VQ-01`, gate ids such as +`G7`). Content reaches anyone only through the explicit feedback-case submission +([Feedback capture](../concepts/agent-network.md#feedback-capture)). + +The guarantee is structural. Every event is a pydantic model in `agents/anyplot/telemetry.py` with +`extra="forbid"`, and every field is a `Literal` set, a bounded integer, a boolean, or an id that must match +a fixed pattern. A value outside its set fails validation, and the line is dropped with a warning that names +the event and the field, never the value. Warning sentences of the parser and the binding check are mapped +to closed codes before they reach a line (`agents/anyplot/data/warning_codes.py`), and a sentence no rule +knows becomes `other`. Tests run a whole session with distinctive column names, cells, notes, a file name and +a message, and fail if any of them reaches a telemetry line (`tests/unit/agents/runtime/test_telemetry.py`). + +## Events + +The agents service writes one wide JSON line per event to stdout on its own logger, +`anyplot.agents.telemetry` (`agents/main.py` `enable_attribution_log`). Cloud Run turns each line into a +`jsonPayload`. + +| Event | When | Main fields | +|---|---|---| +| `data_input` | A paste or a dropped file reaches the dataset route, or the file conversion refuses a file | `format`, `entry` (`paste` or `file`), `outcome`, `cause`, `size_bucket`, `rows_bucket`, `cols_bucket`, column counts by type, `max_categories_bucket`, `missing_bucket`, `warnings` (codes), `tables`, `sheets`, `notes` and `file_name` (booleans), `spec_fit`, judge tokens, `latency_ms` | +| `chat_turn` | A `/messages` turn ends, refused ones included; `manual_binding_changed` counts only a `PUT /bindings` whose bindings differ from the ones the session held, because the page echoes the parse's defaults after every paste | `kind` (`plot`, `prep`, `formula`, `code`, `talk`, `refused`, `error`), `action`, `asked_question`, `tools`, `prep_ops`, `undo`, `to_long`, `manual_binding_changed`, `binding_warnings`, `refusal`, `error`, `reply_reasked`, `language_source`, `steps` (calls, tokens and model per agent), `latency_ms` | +| `pipeline_run` | A `plot_pipeline` run ends | `run_index` (1 is the session's first plot), `base`, `status`, `reason`, `stage`, `attempts`, `rounds`, `score_round1`, `score_round2` (buckets), `shipped_attempt`, `forced_repair`, `gates_failed`, `reviewer_defects`, `residual_defects`, `sandbox_failures` (`syntax`, `name`, `key`, `value`, `type`, `timeout`, `memory`, `other`), time per stage, `steps` | +| `outcome` | The page's beacon after a copy, a download, a session save, or a feedback send | `action` (`copied_code`, `copied_image`, `downloaded_png`, `downloaded_code`, `downloaded_data`, `saved_session`, `feedback`), `version`, `reaction` | + +Every line carries the same envelope: + +| Field | Meaning | +|---|---| +| `log_type` | Always `agent_telemetry`: the sink selects on it | +| `schema` | The line format's version, `1`. A renamed or removed field is a new schema; an added field is not | +| `event` | One of the four events | +| `service`, `revision` | Cloud Run's `K_SERVICE` and `K_REVISION` | +| `source` | `live`, or `harness` for the eval harnesses (`agents/evals/matrix.py`, `agents/evals/prep.py`); the card counts `live` only | +| `session` | An HMAC-SHA256 of the session id under a random per-process salt, 16 hex characters | +| `spec_id`, `library` | The session's spec and library, set once per request from session state | +| `turn` | The session's user turns so far; a `chat_turn` carries its own number | +| `idempotency_key` | A hash of session, turn, attempt and event: a line Cloud Logging delivers twice, or a pipeline node that ADK runs again, keeps its key, and the base view keeps one row per key | + +### Session hash + +Every `/v1` request URL that Cloud Run logs holds the session id, so a plain hash of it could be recomputed +and joined to the request log. The telemetry hash uses a salt that the process draws at start and never +logs. Sessions live in memory and end with the instance (`max-instances=1`), so the salt keeps one session's +lines together for exactly as long as the session exists. Since 2026-10-16 the attribution lines' `session` +is salted with the same per-process salt under another domain (`agents/anyplot/plugins/ledger.py` `salted`), +so it cannot be joined to the request log either, and the two hashes of one session differ: telemetry rows +cannot be joined to the attribution lines, which carry the user id. + +### No user id + +The attribution lines (one per plugin hook, for debugging) carry the HMAC user id for the rate limits and +the security review. The telemetry lines never do: they come from a different logger, and the sink filter +takes only `log_type="agent_telemetry"`, so no attribution line reaches the telemetry bucket. + +### Outcome beacon + +The page posts `POST /debug/agent/sessions/{sid}/outcome` with `{action, version, reaction?}` after the +action succeeded. The BFF accepts only the closed fields and forwards them to +`POST /v1/sessions/{sid}/outcome`, which writes the line, so every line comes from one writer with one +session hash. The agents service limits the beacon to 30 per user per minute and 2,000 per day +(`OUTCOMES_PER_MINUTE`, `OUTCOMES_PER_DAY` in `agents/anyplot/plugins/ledger.py`); the page ignores every +error the beacon gets. + +## Where the lines go + +| Resource | Name | Notes | +|---|---|---| +| Log bucket | `agent-telemetry` in `europe-west4` | Retention 180 days (owner decision of 2026-10-16), Log Analytics on | +| Sink | `agent-telemetry` | Routes the filter below from `anyplot-agents` into the bucket | +| Linked dataset | `anyplot.agent_telemetry_logs` | Read-only view of the bucket in BigQuery (`_AllLogs`) | +| Views dataset | `anyplot.agent_telemetry` in `europe-west4` | The views of `agents/telemetry/views/*.sql`, for Looker Studio and the card | + +The sink filter: + +```text +resource.type="cloud_run_revision" +resource.labels.service_name="anyplot-agents" +jsonPayload.log_type="agent_telemetry" +``` + +An exclusion on the `_Default` sink (step 3 below) keeps the same lines out of the project's `_Default` +bucket, so they live in one place with one retention. + +## Views + +Create them in file order. Each file is a script of `CREATE OR REPLACE VIEW` statements with the project +and dataset names written out. + +| File | Views | Content | +|---|---|---| +| `01_events.sql` | `events` | One row per line, deduplicated by `idempotency_key` (`QUALIFY ROW_NUMBER()`), with the envelope as columns and the line as `payload` | +| `02_event_views.sql` | `data_inputs`, `chat_turns`, `pipeline_runs`, `outcomes` | One typed column per field | +| `03_sessions.sql` | `sessions` | One row per session: data inputs, turns, runs, shipped plots, `turns_to_first_plot`, keeps, `funnel_last_step`, and `state` under a 30-minute inactivity rule (`active`, `kept`, `refined`, `abandoned_after_plot`, `abandoned_before_plot`) | +| `04_usage_failures.sql` | `usage`, `failures` | Usage by day, spec, library and input format; one row per failure with its kind and closed code | +| `05_prices_cost.sql` | `model_prices`, `cost` | The `eu` list prices of `agents/evals/pricing.py` (a test keeps both equal) and the cost of every turn step and data-input judge call | + +The cost view prices the `chat_turn` and `data_input` lines only: `pipeline_run.steps` repeats the adapter's +and the reviewer's share of its turn for per-run analysis. + +## Set up the BigQuery side + +Run these steps once, as a project owner. They write nothing but the resources named. + +1. Create the log bucket: + + ```bash + gcloud logging buckets create agent-telemetry --project=anyplot --location=europe-west4 \ + --retention-days=180 --enable-analytics \ + --description="Content-free usage telemetry of the agent chat (docs/reference/agent-telemetry.md)" + ``` + +2. Create the sink: + + ```bash + gcloud logging sinks create agent-telemetry \ + logging.googleapis.com/projects/anyplot/locations/europe-west4/buckets/agent-telemetry \ + --project=anyplot \ + --log-filter='resource.type="cloud_run_revision" AND resource.labels.service_name="anyplot-agents" AND jsonPayload.log_type="agent_telemetry"' + ``` + + A sink into a log bucket of the same project needs no grant for its writer identity. + +3. Keep the lines out of the `_Default` bucket: + + ```bash + gcloud logging sinks update _Default --project=anyplot \ + --add-exclusion=name=agent-telemetry,filter='resource.type="cloud_run_revision" AND resource.labels.service_name="anyplot-agents" AND jsonPayload.log_type="agent_telemetry"' + ``` + + Run it after step 2: an exclusion on `_Default` does not affect the `agent-telemetry` sink. + +4. Link the bucket to BigQuery: + + ```bash + gcloud logging links create agent_telemetry_logs --bucket=agent-telemetry \ + --location=europe-west4 --project=anyplot + ``` + +5. Create the views dataset in the same location: + + ```bash + bq --location=europe-west4 mk --dataset \ + --description="Views over the agent usage telemetry (agents/telemetry/views)" anyplot:agent_telemetry + ``` + +6. Create the views, in file order, from the repository root: + + ```bash + for file in agents/telemetry/views/0*.sql; do + bq query --project_id=anyplot --location=europe-west4 --use_legacy_sql=false < "$file" + done + ``` + +7. Let the API read them. Find the API's runtime identity: + + ```bash + gcloud run services describe anyplot-api --region=europe-west4 \ + --format='value(spec.template.spec.serviceAccountName)' + ``` + + An empty answer means the Compute Engine default service account, + `239660669828-compute@developer.gserviceaccount.com`. + +8. Grant that identity `roles/bigquery.jobUser` on the project: + + ```bash + gcloud projects add-iam-policy-binding anyplot \ + --member=serviceAccount:API_SERVICE_ACCOUNT --role=roles/bigquery.jobUser + ``` + +9. Grant it `roles/bigquery.dataViewer` on both datasets: the views read the linked dataset with the + caller's rights. In the console, open each dataset (`agent_telemetry`, `agent_telemetry_logs`) and choose + **Sharing > Permissions > Add principal**, or use `bq add-iam-policy-binding`: + + ```bash + bq add-iam-policy-binding --member=serviceAccount:API_SERVICE_ACCOUNT \ + --role=roles/bigquery.dataViewer anyplot:agent_telemetry + bq add-iam-policy-binding --member=serviceAccount:API_SERVICE_ACCOUNT \ + --role=roles/bigquery.dataViewer anyplot:agent_telemetry_logs + ``` + +10. Point the API at the views. `--update-env-vars` adds the variable, and the next deploy keeps it + (`api/cloudbuild.yaml` updates its variables additively): + + ```bash + gcloud run services update anyplot-api --region=europe-west4 \ + --update-env-vars=AGENT_TELEMETRY_DATASET=anyplot.agent_telemetry + ``` + +11. Connect Looker Studio to the `anyplot.agent_telemetry` views with your own account, which needs the same + two roles. + +## The /debug card + +The admin page `/debug` shows an "agent usage" card for the last 7 or 30 days: sessions, plots shipped, +the kept rate (sessions whose user kept a result, among the sessions with a shipped plot), the repair rate +(runs that needed more than one round), and the five most frequent specs, input formats and refusal codes. +It reads `GET /debug/agent-usage?days=7|30` (`api/agent_usage.py`), one query over the `events` view of the +dataset that `AGENT_TELEMETRY_DATASET` names. The answer holds aggregates only. `status` is +`not_configured` while the variable is unset, and `unavailable` when the query fails (no table yet, no +permission, a timeout); both come with zeros, and the card shows its empty state. + +## Before the chat opens to the public + +The telemetry is content-free and keyed by a salted, per-instance session hash, but it is still a record of +use. Before real users reach the chat, the legal page (`app/src/pages/LegalPage.tsx`) needs an entry for it: +purpose (improving the chat), the fields in plain words, the 180-day retention, the EU location, and that +no content is recorded. The chat is admin-only on 2026-10-16, so the page stays unchanged for now. diff --git a/docs/reference/api.md b/docs/reference/api.md index cd25d91c2b4..74eaafcd4e2 100644 --- a/docs/reference/api.md +++ b/docs/reference/api.md @@ -495,6 +495,7 @@ The routes mirror the agents service's `/v1` API. All paths below start with | `POST /sessions/{sid}/dataset/to_long` | none | Make a wide table long for the plot's category and value roles (the page's `.to_long()`, see [Data changes](#data-changes)): the service recomputes its `suggestion` and applies that `pivot_longer` without a model call, as the next version with the default bindings for the new columns, answered like the undo route plus `changes`, unconfirmed. `422 no_suggestion` when the table is not wide for this plot, `409 run_active` while a turn or a render is in flight | | `POST /sessions/{sid}/messages` | `{text}` (at most 2,000 characters), `{"action": "create_plot"}` or `{"action": "summarize_data"}` | An SSE stream in protocol `anyplot/1`; `413 too_long` above the limit, `409 run_active` while you have a turn or a theme render in flight in any session; a full run queue ends the turn's stream with `error {"code": "capacity"}` (see [SSE protocol](#sse-protocol-anyplot1)). `summarize_data` asks the assistant for the step-1 summary of the data (rows, columns with types and units, missing values, percentages, the proposed bindings, one question when something is ambiguous) without the scope judge, like `create_plot`; when the stored dataset kept notes pasted with the table (`context_lines`, judged when they were pasted), the turn carries them to the assistant fenced as ``, so it can act on a request in them, such as "make it a line chart" | | `POST /sessions/{sid}/cancel` | None | `204`; a turn that still waits leaves the run queue | +| `POST /sessions/{sid}/outcome` | `{action, version?, reaction?}`: `action` is `copied_code`, `copied_image`, `downloaded_png`, `downloaded_code`, `downloaded_data`, `saved_session` or `feedback`; `version` a JSON integer from 0 to 999 (0 for the session); `reaction` only with `feedback`, one of the four reactions; nothing else | `204`: the page's content-free outcome beacon, forwarded to the agents service, which writes the `outcome` telemetry line ([Agent usage telemetry](agent-telemetry.md)); `429 rate_limited` / `daily_limit` past 30 a minute or 2,000 a day per user. The page ignores every error | | `POST /sessions/{sid}/versions/{version}/render` | `{"theme": "light"}` or `{"theme": "dark"}`; `version` is 0 to 999, where 0 is the latest version | `{status, reason?, artifacts}` once the render is done (see [Theme toggle](#theme-toggle)) | | `GET /sessions/{sid}/artifacts/{name}?v=` | `name` is one of `plot-light.png`, `plot-dark.png`, `plot.py`, `data.csv` | The file, with `Cache-Control: private, no-store` and `X-Content-Type-Options: nosniff`; any other name is `404`, and so is the PNG of a theme the version has not rendered. A PNG carries the footer strip (see [Footer strip](#footer-strip)) | | `GET /sessions/{sid}/bundle?include_data=&version=` | `include_data` is `true` or `false` (the default); `version` is 0 to 999, where 0 or none means every version (the BFF then asks the agents service without one, since there `version=0` is the chat's feedback case) | The session as one JSON file to save (the chat page's `.save()`), with `Content-Disposition: attachment; filename="anyplot-session-.json"`, `Cache-Control: private, no-store` and `X-Content-Type-Options: nosniff`. `404 not_found` for a version the session does not have; a bundle over 32 MiB can't be saved (see [Session bundle](#session-bundle)) | @@ -858,6 +859,33 @@ table (`alembic/versions/b4e7a9c2d6f1_add_agent_feedback_table.py`). Until it has run, the feedback route stores the case and answers `502 row_missing`, and the case routes answer `500`. +### Usage highlights + +`GET /debug/agent-usage?days=7` (also `days=30`; any other value is `400`) is +the admin-gated, read-only source of the "agent usage" card on `/debug` +(`api/routers/debug.py`, `api/agent_usage.py`). It never calls the agents +service: it runs one query over the BigQuery telemetry views of +`AGENT_TELEMETRY_DATASET` and answers aggregates of live sessions only: + +```json +{ + "status": "ok", + "days": 7, + "sessions": 12, + "plots_shipped": 30, + "kept_rate": 0.4167, + "repair_rate": 0.3125, + "top_specs": [{"key": "scatter-basic", "count": 5}], + "top_formats": [{"key": "csv", "count": 9}], + "top_refusals": [{"key": "unparseable", "count": 3}], + "generated_at": "2026-10-16T12:00:00Z" +} +``` + +`status` is `not_configured` while the variable is unset and `unavailable` +when the query fails; both come with zeros, `null` rates and empty lists. See +[Agent usage telemetry](agent-telemetry.md). + ### SSE protocol `anyplot/1` | Event | Data | @@ -988,6 +1016,7 @@ could not be read; `PATCH /cases/{id}` answers `422 no_data` and | `AGENT_USER_ID_KEY` | Unset | HMAC key for the user id; from Secret Manager in production | | `AGENT_REQUEST_TIMEOUT_S` | `550` | Upstream timeout, and the cap on one chat stream outside the run queue; above the agents service's 540-second deadline | | `AGENT_TURN_MAX_S` | `1190` | Hard cap on one chat turn, queue wait included; keep it below anyplot-api's Cloud Run `--timeout` (1,200 seconds in `api/cloudbuild.yaml`) | +| `AGENT_TELEMETRY_DATASET` | Unset | `.` of the usage telemetry's BigQuery views (`anyplot.agent_telemetry`), which `GET /debug/agent-usage` reads; unset, that route answers `status: not_configured` ([Agent usage telemetry](agent-telemetry.md)) | | `AGENT_CASES_BUCKET` | `anyplot-agent-cases` | The private EU bucket the feedback cases go to and the case routes read from (see [Feedback case](#feedback-case)); never the public image bucket. The default is the production bucket, so a local API with Application Default Credentials stores its cases there too; set a bucket of your own for local runs | In production, `AGENT_ENABLED` and `AGENT_SERVICE_URL` come from the diff --git a/docs/reference/plausible.md b/docs/reference/plausible.md index 269b02723ba..985e5c974e4 100644 --- a/docs/reference/plausible.md +++ b/docs/reference/plausible.md @@ -152,6 +152,14 @@ with `VITE_ENABLE_AGENT_CHAT=true`, and like every event they fire only on `copy_code` on a result card carries `method: agent` and `page: agent_chat` plus the `spec` and `library` of the version. +These Plausible events count page interactions in the browser. Since 2026-10-16 +the chat also has a server-side usage telemetry in BigQuery, with its own +content-free events (`data_input`, `chat_turn`, `pipeline_run`, `outcome`): the +copy, download, save and feedback actions above additionally send an outcome +beacon (`POST /debug/agent/sessions/{sid}/outcome`) that the agents service writes +as an `outcome` line. That beacon is not a Plausible event and fires on every host, +not only `anyplot.ai`. See [Agent usage telemetry](agent-telemetry.md). + ### Discovery | Event Name | Properties | Where | Description | diff --git a/tests/unit/agents/data/test_warning_codes.py b/tests/unit/agents/data/test_warning_codes.py new file mode 100644 index 00000000000..4837eef6fae --- /dev/null +++ b/tests/unit/agents/data/test_warning_codes.py @@ -0,0 +1,189 @@ +"""The warning sentences map to closed codes (`data/warning_codes.py`), never to their own text.""" + +from pathlib import Path +from typing import get_args + +import pytest + +from agents.anyplot.data.parse import ( + AMBIGUOUS_DATE_ORDER, + HEADERLESS_LIST_WARNING, + NS_OUT_OF_RANGE, + ParseError, + parse_dataset, + rounded_note, +) +from agents.anyplot.data.prepare import AMBIGUOUS_MONTH_FIRST +from agents.anyplot.data.signs import sign_line +from agents.anyplot.data.warning_codes import WARNING_CODES, binding_warning_codes, warning_code, warning_codes +from agents.anyplot.opening import NOTES_INSTRUCTION_WARNING, NOTES_REFUSED_WARNING, NOTES_UNCHECKED_WARNING +from agents.anyplot.schemas import ColumnProfile + + +REPO_ROOT = Path(__file__).resolve().parents[4] + +SAMPLES: dict[str, str] = { + NOTES_INSTRUCTION_WARNING: "notes_instruction_like", + NOTES_REFUSED_WARNING: "notes_refused", + NOTES_UNCHECKED_WARNING: "notes_unchecked", + "the notes around the table were cut at 2,000 characters": "notes_cut", + "the dataset's notes were cut at 2,000 characters": "notes_cut", + "3 lines held decomposed characters (such as e and a separate accent), now written in Unicode's composed form " + "(NFC); the text reads the same": "nfc_normalised", + "single spaces read as the column separator; check that no value was split in two": "single_space_separator", + "left out 2 lines above the table (a title, notes or metadata)": "lines_above_left_out", + "left out 1 line below the table (notes or a source line)": "lines_below_left_out", + "left out 1 summary row at the end of the table (a total, sum, mean or similar)": "summary_rows_left_out", + "2 rows had fewer fields than the header; their cells were placed under the header's columns by their " + "position, and the cells left over are empty (lines 3, 4)": "short_rows_positional", + "1 row had fewer fields than the header; the missing cells are empty (line 7)": "short_rows_padded", + "4 rows had empty fields past the header's 3 columns; those empty fields were left out": "trailing_empty_fields", + "the data holds 2 tables, each under its own header; read table 1 (a, b: 3 rows) and left out table 2; the " + "table picker in the data panel reads another one": "multiple_tables", + HEADERLESS_LIST_WARNING: "headerless_list", + "read 'Apples' as the column name": "category_header", + "2 keys repeated; only the last value of a repeated key is kept": "repeated_keys", + "1 value NaN, Infinity or -Infinity read as missing": "nan_infinity_missing", + "the object was read as 3 category/value pairs, one row per key; to read it as one row with the keys as " + "columns, put it in brackets [ … ]": "object_as_pairs", + "2 fields beside the records (such as paging or a timestamp) moved to the notes": "fields_beside_records_to_notes", + "1 field beside the records that hold lists left out": "list_fields_beside_left_out", + "fields of nested objects became columns named with a dot, such as parent.field": "nested_fields_dotted", + "2 fields holding a list of objects or lists, or objects nested more than 3 levels deep, left out: a, b": ( + "deep_fields_left_out" + ), + "list fields expanded into one row per value: x, y": "list_fields_expanded", + "column 'record' added: it numbers the records, so the rows of each stay together": "record_id_added", + "the lists hold 30,000 values, more than the 20,000 rows a table may have: 20,000 evenly spaced values are " + "kept, the first and last of each record among them; paste fewer points to keep every value": "list_values_thinned", + "3 records lack some keys; those cells are empty": "records_missing_keys", + "column 'value': a dimension named like the value column renamed": "jsonstat_dimension_renamed", + "column 'Region': labels shared by two categories carry their codes": "jsonstat_labels_with_codes", + "the dataset's status symbols (such as provisional or estimated) are ignored": "jsonstat_status_ignored", + "2 dimensions with a single category left out of the columns; the notes name each one with its category": ( + "jsonstat_single_category_dims" + ), + "column 3: empty header named 'column_3'": "empty_header_named", + "column 2: header longer than 64 characters shortened to 'A very long name'": "long_header_shortened", + "column 2: header shortened to 'Name (kg)' so the unit fits within 64 characters": "header_shortened_for_unit", + "column 4: duplicate header 'x' renamed to 'x_2'": "duplicate_header_renamed", + "2 empty columns without a header dropped": "empty_columns_dropped", + "5 cells of number or date columns matched the missing-value mark the notes declare and are empty": ( + "missing_marks_emptied" + ), + "column 'a b': tabs or line breaks in the header replaced with spaces": "header_whitespace_replaced", + "every header looks like a value; the first row may be data rather than column names": "headers_look_like_values", + "column 'x': invisible formatting characters (zero-width or bidi) removed": "invisible_chars_removed", + "column 'Weight (kg)': unit 'kg' removed from the values; the name already carries it": "unit_removed_from_values", + "column 'Weight': unit 'kg' moved from the values into the name 'Weight (kg)'": "unit_moved_to_name", + "12 numbers written in another form with the same value (trailing zeros, exponent or plus sign dropped, such as " + "1.50 as 1.5 or 1e3 as 1000) in 'a'": "numbers_respelled", + "2 columns without any value dropped: 'a', 'b'": "valueless_columns_dropped", + "left out the column '': its 24 cells only number the rows 1 to 24 in order, as R's write.csv or pandas' index " + "write row names": "row_number_column_dropped", + "1 column with the same text in every row moved to the notes: 'unit'": "constant_columns_to_notes", + "column 'zip': numbers with leading zeros kept as text": "leading_zeros_text", + "column 'id': whole numbers beyond the 64-bit range kept as text, so no digit is lost": "big_integers_text", + "column 'x': commas read as decimal commas; remove them first if they separate thousands": ( + "decimal_commas_ambiguous" + ), + "column 'x': dots read as thousands separators, as in the file's other columns": "thousands_separators", + "column 'x': commas read as thousands separators": "thousands_separators", + "column 't': times with the UTC offsets +01:00, +02:00 written in UTC (+00:00); the instants are the same": ( + "utc_offsets_normalised" + ), + f"column 't': {NS_OUT_OF_RANGE}": "sub_microsecond_text", + f"column 'd': {AMBIGUOUS_DATE_ORDER}": "date_order_ambiguous", + f"column 'd': {AMBIGUOUS_MONTH_FIRST}": "date_order_ambiguous", + "column 'm': 12 cells read as months (YYYY-MM, each as the first day of the month)": "months_read", + "column 'country': NA read as a code (such as Namibia or North America), not as a missing cell": "na_as_code", + f"column 'v': {rounded_note(3)}": "digits_rounded", + "column 'name': spaces around the text removed in 4 cells": "cells_trimmed", + "column 'v': 3 cells read as missing ('n/a', '-')": "cells_read_missing", + "y: 2 of 200 values are undefined and left empty": "series_undefined", + "4 more warnings omitted": "more_omitted", +} + + +@pytest.mark.parametrize(("sentence", "code"), SAMPLES.items()) +def test_every_sentence_maps_to_its_code(sentence: str, code: str) -> None: + assert warning_code(sentence) == code + + +def test_every_code_has_a_sample() -> None: + """A code without a sample sentence is a rule nobody checked.""" + assert set(SAMPLES.values()) | {"other"} == WARNING_CODES + + +def test_every_rule_names_a_code_of_the_set() -> None: + from agents.anyplot.data.warning_codes import _BINDING_RULES, _RULES, BindingWarningCode + + assert {code for _, code in _RULES} <= WARNING_CODES + assert {code for _, code in _BINDING_RULES} <= set(get_args(BindingWarningCode)) + + +def test_an_unknown_sentence_is_other() -> None: + assert warning_code("something the parser never wrote, with a value 42") == "other" + + +def test_a_column_name_cannot_shadow_a_fixed_sentence() -> None: + """A crafted column name picks a code from the set at worst; the code is never its text.""" + crafted = "column 'left out 9 lines above the table (a title, notes or metadata)': spaces around the text removed in 1 cell" + assert warning_code(crafted) == "cells_trimmed" + assert set(warning_codes([crafted, "CANARY free text"])) <= WARNING_CODES + + +def test_codes_are_distinct_and_sorted() -> None: + sentences = [ + "4 more warnings omitted", + "4 more warnings omitted", + "left out 1 line below the table (notes or a source line)", + ] + assert warning_codes(sentences) == ["lines_below_left_out", "more_omitted"] + + +def test_the_fixture_corpus_has_no_unknown_warning() -> None: + """Every warning the parser writes for the eval fixtures has a code: a new warning needs its rule.""" + files = sorted((REPO_ROOT / "agents" / "evals" / "fixtures").rglob("data.*")) + assert files + unknown = [] + for path in files: + try: + parsed = parse_dataset(path.read_text(encoding="utf-8")) + except (ParseError, UnicodeDecodeError): + continue + unknown += [line for line in parsed.warnings if warning_code(line) == "other"] + assert unknown == [] + + +def test_parser_warnings_of_real_pastes_have_codes() -> None: + pastes = [ + "Title of the table\n\nx,y\n1,2\n3,4\nTotal,6\n", + "x;y\n1,5;2\n3,5;4\n", + "a,b,c\n1,2\n3,4,5\n", + "x y\n1 2\n3 4\n", + '[{"a": 1, "b": [1, 2]}, {"a": 2, "b": [3]}]', + "name,value\n Alice ,1\nBob,n/a\n", + ] + for text in pastes: + try: + parsed = parse_dataset(text) + except ParseError: + continue + assert "other" not in warning_codes(parsed.warnings), parsed.warnings + + +def test_binding_warnings_map_to_codes() -> None: + column = ColumnProfile(name="Change", dtype="number", missing=0, unique=3, min=-2.5, max=4.0) + negative = sign_line("y", column) + unbound = "y: not bound, because every column that fits it ('a', 'b') has negative values; this role needs values of 0 or more" + sibling = ( + "y: bound to 'Mo', one of 7 number columns ('Mo', 'Tu', 'We', 'Th', ...) that hold one measure side by side; " + "y wants labels, so each number would become a category of its own: turn the columns into rows first (.to_long())" + ) + assert binding_warning_codes([negative, unbound, sibling, "CANARY"]) == [ + "label_sibling", + "negative_unbound", + "negative_values", + "other", + ] diff --git a/tests/unit/agents/evals/test_telemetry_prices.py b/tests/unit/agents/evals/test_telemetry_prices.py new file mode 100644 index 00000000000..f36eb2a3f2e --- /dev/null +++ b/tests/unit/agents/evals/test_telemetry_prices.py @@ -0,0 +1,52 @@ +"""The BigQuery price table of the usage telemetry (`agents/telemetry/views/05_prices_cost.sql`) is `pricing.py`'s.""" + +import re +from pathlib import Path + +import pytest + +from agents.evals.pricing import CACHE_READ_FACTOR, CACHE_WRITE_FACTOR, PRICES, surcharge + + +VIEWS = Path(__file__).resolve().parents[4] / "agents" / "telemetry" / "views" +ROW = re.compile( + r"STRUCT\('(?P[a-z0-9.-]+)' AS model, (?P[0-9.]+) AS input_usd_per_m, " + r"(?P[0-9.]+) AS output_usd_per_m\)" +) + + +def price_rows() -> dict[str, tuple[float, float]]: + text = (VIEWS / "05_prices_cost.sql").read_text() + return {m["model"]: (float(m["input"]), float(m["output"])) for m in ROW.finditer(text)} + + +def test_every_listed_model_has_its_eu_price() -> None: + rows = price_rows() + assert set(rows) == set(PRICES) + for model, price in PRICES.items(): + assert rows[model] == ( + pytest.approx(price.input * surcharge("eu")), + pytest.approx(price.output * surcharge("eu")), + ), model + + +def test_the_cost_view_uses_the_cache_factors() -> None: + text = (VIEWS / "05_prices_cost.sql").read_text() + assert f"input_usd_per_m * {CACHE_READ_FACTOR}" in text + assert f"input_usd_per_m * {CACHE_WRITE_FACTOR}" in text + + +def test_the_settings_models_are_priced() -> None: + from agents.anyplot.settings import AgentSettings + + fields = AgentSettings.model_fields + for name in ("model", "judge_model"): + assert fields[name].default in price_rows(), name + + +def test_the_views_name_one_project_and_dataset() -> None: + """The views are created as written (`docs/reference/agent-telemetry.md`): one project, two datasets.""" + names = set() + for path in sorted(VIEWS.glob("*.sql")): + names |= set(re.findall(r"`(anyplot\.[a-z_]+)\.[A-Za-z_]+`", path.read_text())) + assert names == {"anyplot.agent_telemetry", "anyplot.agent_telemetry_logs"} diff --git a/tests/unit/agents/runtime/test_data_code_tool.py b/tests/unit/agents/runtime/test_data_code_tool.py index 09a9e99350b..772b05721d5 100644 --- a/tests/unit/agents/runtime/test_data_code_tool.py +++ b/tests/unit/agents/runtime/test_data_code_tool.py @@ -413,7 +413,9 @@ async def test_a_turn_that_runs_code_reports_the_version_with_its_code( dataset: dict[str, Any] = done["dataset"] assert dataset["version"] == 1 and dataset["code"] == SNIPPET assert dataset["changes"] == [f"{DATA_CODE_CHANGE}: 4 rows, 2 columns"] - lines = [json.loads(record.getMessage()) for record in caplog.records] + lines = [ + json.loads(record.getMessage()) for record in caplog.records if record.name == "anyplot.agents.attribution" + ] data_code = [line for line in lines if line["hook"] == "data_code"] assert data_code and data_code[0]["outcome"] == "ran" and data_code[0]["language"] == "javascript" assert "arange" not in caplog.text @@ -441,7 +443,7 @@ async def test_code_in_the_data_field_is_refused_before_any_parser( outcomes = [ json.loads(record.getMessage())["outcome"] for record in caplog.records - if json.loads(record.getMessage())["hook"] == "data_transcribe" + if record.name == "anyplot.agents.attribution" and json.loads(record.getMessage())["hook"] == "data_transcribe" ] assert outcomes == ["code"] * len(snippets) + ["formula"] diff --git a/tests/unit/agents/runtime/test_telemetry.py b/tests/unit/agents/runtime/test_telemetry.py new file mode 100644 index 00000000000..8d0b2dbc400 --- /dev/null +++ b/tests/unit/agents/runtime/test_telemetry.py @@ -0,0 +1,366 @@ +"""The usage telemetry (`agents/anyplot/telemetry.py`): content-free lines, closed sets, dedupe keys, the beacon.""" + +import json +import logging +from collections.abc import AsyncIterator, Iterator +from typing import Any, get_args + +import httpx +import pytest +from pydantic import ValidationError + +from agents.anyplot import telemetry +from agents.anyplot.dev_fixture import snapshot_from_repo +from agents.anyplot.plugins.ledger import OUTCOMES_PER_MINUTE, RequestLedger +from agents.anyplot.render.contract import ThemeOutput +from agents.anyplot.run_queue import RunQueue +from agents.anyplot.services import Services +from agents.main import Runtime, app, get_runtime +from core.database.models import FEEDBACK_REACTIONS + +from .fakes import default_script + + +USER = "adm_telemetry0000001" +HEADERS = {"X-Anyplot-User": USER, "X-Request-Id": "req-telemetry"} +COLUMN_CANARY = "CanaryColQ7" +CELL_CANARY = "CANARY-CELL-e41a" +NOTE_CANARY = "CANARY-NOTE-90bd" +FILE_CANARY = "CANARY-FILE-77c3" +MESSAGE_CANARY = "CANARY-MSG-1f0e" +CANARY_DATA = f"Source: survey {NOTE_CANARY}\n\n{COLUMN_CANARY},Exam Score,Student\n" + "".join( + f"{hours},{50 + 2 * hours},{CELL_CANARY if hours == 3 else f'S{hours:02d}'}\n" for hours in range(1, 13) +) + + +@pytest.fixture +def runtime() -> Iterator[Runtime]: + fresh = Runtime(queue=RunQueue(concurrency=1, per_minute=1_000, max_wait_s=600)) + app.dependency_overrides[get_runtime] = lambda: fresh + yield fresh + app.dependency_overrides.clear() + + +@pytest.fixture +async def client(runtime: Runtime, services: Services) -> AsyncIterator[httpx.AsyncClient]: + transport = httpx.ASGITransport(app=app) + async with httpx.AsyncClient(transport=transport, base_url="http://agents") as http: + yield http + + +def telemetry_lines(caplog: pytest.LogCaptureFixture) -> list[dict[str, Any]]: + return [json.loads(r.getMessage()) for r in caplog.records if r.name == telemetry.TELEMETRY_LOGGER] + + +async def open_session(client: httpx.AsyncClient) -> str: + response = await client.post( + "/v1/sessions", + headers=HEADERS, + json={ + "user": USER, + "spec_id": "scatter-basic", + "library": "matplotlib", + "locale": "en", + "snapshot": snapshot_from_repo("scatter-basic", "matplotlib").model_dump(), + }, + ) + assert response.status_code == 200, response.text + return str(response.json()["session_id"]) + + +async def test_a_whole_session_writes_the_four_events_and_none_of_its_content( + client: httpx.AsyncClient, swap_models, caplog: pytest.LogCaptureFixture +) -> None: + """The canary: distinctive column names, cells, notes, file name and message never reach a telemetry line.""" + caplog.set_level(logging.INFO) + swap_models("gemini", default_script()) + sid = await open_session(client) + pasted = await client.post( + f"/v1/sessions/{sid}/dataset", headers=HEADERS, json={"text": CANARY_DATA, "source": f"{FILE_CANARY}.csv"} + ) + assert pasted.status_code == 200, pasted.text + defaults = pasted.json()["bindings"] + # The page echoes the defaults after every paste: no manual change. + assert (await client.put(f"/v1/sessions/{sid}/bindings", headers=HEADERS, json=defaults)).status_code == 200 + assert telemetry.marks(sid).manual_binding_changed is False + # The user's own choice: x and y swapped. + bindings = [{"role": "x", "column": "Exam Score"}, {"role": "y", "column": COLUMN_CANARY}] + assert (await client.put(f"/v1/sessions/{sid}/bindings", headers=HEADERS, json=bindings)).status_code == 200 + assert telemetry.marks(sid).manual_binding_changed is True + assert (await client.post(f"/v1/sessions/{sid}/dataset/confirm", headers=HEADERS)).status_code == 200 + turn = await client.post( + f"/v1/sessions/{sid}/messages", headers=HEADERS, json={"text": f"Please plot it {MESSAGE_CANARY}"} + ) + assert "event: plot" in turn.text + beacon = await client.post( + f"/v1/sessions/{sid}/outcome", headers=HEADERS, json={"action": "copied_code", "version": 1} + ) + assert beacon.status_code == 204 + + lines = telemetry_lines(caplog) + assert [line["event"] for line in lines] == ["data_input", "pipeline_run", "chat_turn", "outcome"] + raw = json.dumps(lines) + for canary in (COLUMN_CANARY, CELL_CANARY, NOTE_CANARY, FILE_CANARY, MESSAGE_CANARY, USER, sid, "Exam Score"): + assert canary not in raw, canary + for line in lines: + assert line["log_type"] == "agent_telemetry" and line["schema"] == 1 and line["source"] == "live" + assert "user" not in line and "request_id" not in line + assert line["spec_id"] == "scatter-basic" and line["library"] == "matplotlib" + assert len(line["session"]) == 16 and line["session"] == lines[0]["session"] + + data_input, run, chat_turn, outcome = lines + assert data_input["outcome"] == "ok" and data_input["format"] == "csv" and data_input["entry"] == "file" + assert data_input["file_name"] is True and data_input["notes"] is True and data_input["text_cols"] == 1 + assert data_input["rows_bucket"] == "11_100" and data_input["cols_bucket"] == "3_5" + assert data_input["spec_fit"] == "direct" and data_input["turn"] == 0 + assert run["run_index"] == 1 and run["status"] == "ok" and run["turn"] == 1 + assert [step["step"] for step in run["steps"]] == ["adapter", "reviewer"] + assert chat_turn["kind"] == "plot" and chat_turn["action"] == "text" and chat_turn["turn"] == 1 + assert chat_turn["tools"] == ["plot_pipeline"] and chat_turn["manual_binding_changed"] is True + assert {step["step"] for step in chat_turn["steps"]} >= {"root", "adapter", "reviewer"} + assert outcome["action"] == "copied_code" and outcome["version"] == 1 and outcome["turn"] == 1 + assert len({line["idempotency_key"] for line in lines}) == 4 + + +async def test_the_attribution_lines_still_carry_the_user_and_the_telemetry_lines_never( + client: httpx.AsyncClient, swap_models, caplog: pytest.LogCaptureFixture +) -> None: + """The sink takes `log_type="agent_telemetry"` only: the user id stays in the debug lines.""" + caplog.set_level(logging.INFO) + swap_models("gemini", default_script()) + sid = await open_session(client) + await client.post(f"/v1/sessions/{sid}/dataset", headers=HEADERS, json={"text": CANARY_DATA}) + attribution = [json.loads(r.getMessage()) for r in caplog.records if r.name == "anyplot.agents.attribution"] + assert attribution and all("log_type" not in line for line in attribution) + assert any(line.get("user") == USER for line in attribution) + assert all("user" not in line for line in telemetry_lines(caplog)) + + +async def test_refused_pastes_write_their_outcome_and_cause( + client: httpx.AsyncClient, caplog: pytest.LogCaptureFixture +) -> None: + caplog.set_level(logging.INFO) + sid = await open_session(client) + ragged = f"a,b\n1,2\n3,4,{CELL_CANARY}\n" + assert (await client.post(f"/v1/sessions/{sid}/dataset", headers=HEADERS, json={"text": ragged})).status_code == 422 + code = "import numpy as np\nx = np.linspace(0, 10, 100)\ny = np.sin(x)\n" + assert (await client.post(f"/v1/sessions/{sid}/dataset", headers=HEADERS, json={"text": code})).status_code == 422 + + first, second = telemetry_lines(caplog) + assert (first["outcome"], first["cause"], first["format"]) == ("unparseable", "unparseable", "other") + assert (second["outcome"], second["format"]) == ("code", "code") + assert first["idempotency_key"] != second["idempotency_key"] + assert CELL_CANARY not in json.dumps([first, second]) + + +async def test_the_beacon_takes_closed_values_only(client: httpx.AsyncClient) -> None: + sid = await open_session(client) + url = f"/v1/sessions/{sid}/outcome" + assert (await client.post(url, headers=HEADERS, json={"action": "feedback", "reaction": "bug"})).status_code == 204 + for body in ( + {"action": "copied_code", "reaction": "bug"}, # a reaction goes with feedback only + {"action": "shared_on_twitter"}, + {"action": "feedback", "reaction": "love"}, + {"action": "copied_code", "version": "1"}, + {"action": "copied_code", "note": MESSAGE_CANARY}, + {"action": "copied_code", "version": 1000}, + ): + assert (await client.post(url, headers=HEADERS, json=body)).status_code == 422, body + gone = await client.post("/v1/sessions/nope/outcome", headers=HEADERS, json={"action": "copied_code"}) + assert gone.status_code == 404 + + +async def test_the_beacon_is_rate_limited_per_user(client: httpx.AsyncClient) -> None: + sid = await open_session(client) + url = f"/v1/sessions/{sid}/outcome" + codes = [(await client.post(url, headers=HEADERS, json={"action": "downloaded_png"})).status_code] + for _ in range(OUTCOMES_PER_MINUTE): + codes.append((await client.post(url, headers=HEADERS, json={"action": "downloaded_png"})).status_code) + assert codes[:OUTCOMES_PER_MINUTE] == [204] * OUTCOMES_PER_MINUTE + assert codes[-1] == 429 + + +# --- Closed sets ---------------------------------------------------------------------------- + +BASELINES: dict[type[telemetry._Event], dict[str, Any]] = { + telemetry.DataInput: {"format": "csv", "entry": "paste", "outcome": "ok", "size_bucket": "lt1k"}, + telemetry.ChatTurn: {"kind": "talk", "action": "text"}, + telemetry.PipelineRun: {"run_index": 1, "base": "catalogue", "status": "ok"}, + telemetry.Outcome: {"action": "copied_code"}, + telemetry.StepTokens: {"step": "root"}, +} + + +@pytest.mark.parametrize("model", list(BASELINES)) +def test_no_field_takes_free_text(model: type[telemetry._Event]) -> None: + """Enum closure: every field of every event refuses a free text, as a value and inside a list.""" + model(**BASELINES[model]) + for name in model.model_fields: + if name == "event": + continue + for value in (f"free text {MESSAGE_CANARY}", [f"free text {MESSAGE_CANARY}"], {"text": MESSAGE_CANARY}): + with pytest.raises(ValidationError): + model(**{**BASELINES[model], name: value}) + + +def test_an_unknown_field_is_refused() -> None: + with pytest.raises(ValidationError): + telemetry.ChatTurn.model_validate({"kind": "talk", "action": "text", "message": MESSAGE_CANARY}) + + +def test_an_invalid_event_writes_no_line(caplog: pytest.LogCaptureFixture) -> None: + caplog.set_level(logging.INFO) + ledger = RequestLedger(session_id="s1", spec_id="scatter-basic", library="matplotlib", turn=1) + assert telemetry.write(telemetry.ChatTurn, ledger, kind=MESSAGE_CANARY, action="text") is None + assert telemetry_lines(caplog) == [] + assert MESSAGE_CANARY not in caplog.text + + +def test_an_envelope_value_outside_its_set_writes_no_line(caplog: pytest.LogCaptureFixture) -> None: + caplog.set_level(logging.INFO) + ledger = RequestLedger(session_id="s1", spec_id=f"Not A Spec {MESSAGE_CANARY}", library="matplotlib") + assert telemetry.write(telemetry.Outcome, ledger, action="copied_code") is None + assert MESSAGE_CANARY not in caplog.text + + +def test_the_reactions_are_the_feedback_reactions() -> None: + assert get_args(telemetry.Reaction) == FEEDBACK_REACTIONS + + +def test_the_prep_ops_are_the_operations() -> None: + from agents.anyplot.data.prepare import Operation + + union = get_args(get_args(Operation)[0]) + assert {get_args(member.model_fields["op"].annotation)[0] for member in union} == telemetry.PREP_OPS + + +def test_the_tool_names_are_the_root_tools() -> None: + from agents.anyplot.agent import ROOT_TOOLS + + names = {getattr(tool, "name", None) or getattr(tool, "__name__", "") for tool in ROOT_TOOLS} + assert names == telemetry.TOOL_NAMES + + +# --- Keys, buckets, helpers ----------------------------------------------------------------- + + +def test_the_session_hash_is_salted() -> None: + import hashlib + + key = telemetry.session_key("abc") + assert key == telemetry.session_key("abc") and len(key) == 16 + assert key not in (hashlib.sha256(b"abc").hexdigest()[:16], hashlib.sha256(b"abc").hexdigest()[:12]) + + +def test_the_attribution_hash_is_salted_and_not_the_telemetry_hash() -> None: + import hashlib + + from agents.anyplot.plugins.ledger import session_hash + + plain = hashlib.sha256(b"abc").hexdigest() + assert len(session_hash("abc")) == 12 and session_hash("abc") == session_hash("abc") + assert session_hash("abc") not in (plain[:12], telemetry.session_key("abc")[:12]) + assert session_hash("") == "" + + +def test_idempotency_keys_repeat_for_the_same_line_only() -> None: + first = telemetry.idempotency_key("s", 1, "run1", "pipeline_run") + assert first == telemetry.idempotency_key("s", 1, "run1", "pipeline_run") + assert first != telemetry.idempotency_key("s", 1, "run2", "pipeline_run") + assert first != telemetry.idempotency_key("s", 2, "run1", "pipeline_run") + + +def test_a_rerun_node_keeps_its_run_index() -> None: + telemetry.forget("sess-x") + assert telemetry.run_index("sess-x", "inv-1") == 1 + assert telemetry.run_index("sess-x", "inv-1") == 1 # ADK re-ran the node: the same line, the same key + assert telemetry.run_index("sess-x", "inv-2") == 2 + telemetry.forget("sess-x") + + +def test_the_turn_marks_are_taken_once() -> None: + telemetry.forget("sess-y") + telemetry.marks("sess-y").undo = True + telemetry.note_binding_warnings("sess-y", ["CANARY: not a known sentence"]) + assert telemetry.take_turn_marks("sess-y") == { + "undo": True, + "to_long": False, + "manual_binding_changed": False, + "binding_warnings": ["other"], + } + assert telemetry.take_turn_marks("sess-y")["undo"] is False + telemetry.forget("sess-y") + + +@pytest.mark.parametrize( + ("reply", "asked"), + [("Shall I make it dark?", True), ("Fertig. Soll ich die Achsen tauschen?**", True), ("Done.", False), ("", False)], +) +def test_a_question_is_a_bool(reply: str, asked: bool) -> None: + assert telemetry.ends_with_question(reply) is asked + + +def test_buckets() -> None: + assert [telemetry.size_bucket(n) for n in (10, 5_000, 50_000, 500_000, 2_000_000)] == [ + "lt1k", + "1k_10k", + "10k_100k", + "100k_1m", + "ge1m", + ] + assert [telemetry.missing_bucket(m, 100) for m in (0, 4, 30, 31)] == ["none", "lt5", "5_30", "gt30"] + assert [telemetry.categories_bucket(n) for n in (5, 6, 50, 51)] == ["le5", "6_12", "13_50", "gt50"] + assert [telemetry.score_bucket(s) for s in (None, 49, 69, 75, 89, 90)] == [ + None, + "lt50", + "50_69", + "70_79", + "80_89", + "90_plus", + ] + + +@pytest.mark.parametrize( + ("stderr", "kwargs", "expected"), + [ + (' File "plot.py", line 3\nSyntaxError: invalid syntax', {}, "syntax"), + (f"Traceback\nKeyError: '{CELL_CANARY}'", {}, "key"), + ("Traceback\nNameError: name 'x' is not defined", {}, "name"), + ("", {"timed_out": True}, "timeout"), + ("", {"reason": "memory"}, "memory"), + ("Traceback\nZeroDivisionError: division by zero", {}, "other"), + ], +) +def test_sandbox_failures_are_classes(stderr: str, kwargs: dict[str, Any], expected: str) -> None: + output = ThemeOutput(theme="light", exit_code=1, stderr_tail=stderr, **kwargs) + assert telemetry.sandbox_failure(output) == expected + + +def test_a_clean_render_is_no_failure() -> None: + assert telemetry.sandbox_failure(ThemeOutput(theme="light", exit_code=0, png=b"png")) is None + + +def test_input_formats() -> None: + assert telemetry.input_format("json_records") == "json" + assert telemetry.input_format("json_records", warnings=["fields_beside_records_to_notes"]) == "api_json" + assert telemetry.input_format("semicolon") == "semicolon" + assert telemetry.input_format("csv", convert_kind="xlsx") == "xlsx" + assert telemetry.input_format(None) == "other" + + +def test_spec_fit() -> None: + assert telemetry.spec_fit(complete=True, bound=2, reshape=False) == "direct" + assert telemetry.spec_fit(complete=True, bound=2, reshape=True) == "needs_reshape" + assert telemetry.spec_fit(complete=False, bound=1, reshape=False) == "missing_role" + assert telemetry.spec_fit(complete=False, bound=0, reshape=False) == "incompatible" + + +def test_the_harness_tags_its_lines(caplog: pytest.LogCaptureFixture) -> None: + caplog.set_level(logging.INFO) + ledger = RequestLedger(session_id="s1", spec_id="scatter-basic", library="matplotlib") + try: + telemetry.set_source("harness") + record = telemetry.write(telemetry.Outcome, ledger, action="saved_session") + finally: + telemetry.set_source("live") + assert record is not None and record["source"] == "harness" diff --git a/tests/unit/api/test_agent_router.py b/tests/unit/api/test_agent_router.py index 86afb275b47..ed3ea765fbf 100644 --- a/tests/unit/api/test_agent_router.py +++ b/tests/unit/api/test_agent_router.py @@ -333,6 +333,7 @@ def _sse_events(body: str) -> list[tuple[str, dict]]: ("POST", "/debug/agent/sessions/s1/dataset/to_long", None), ("POST", "/debug/agent/sessions/s1/messages", {"action": "create_plot"}), ("POST", "/debug/agent/sessions/s1/cancel", None), + ("POST", "/debug/agent/sessions/s1/outcome", {"action": "copied_code", "version": 1}), ("POST", "/debug/agent/sessions/s1/versions/1/render", {"theme": "dark"}), ("GET", "/debug/agent/sessions/s1/artifacts/plot-light.png", None), ("GET", "/debug/agent/sessions/s1/bundle?include_data=true", None), @@ -2058,6 +2059,51 @@ def test_cancel_is_204(self, client, upstream) -> None: assert str(upstream.requests[0].url) == f"{SERVICE_URL}/v1/sessions/s1/cancel" +class TestOutcomeBeacon: + """`POST /sessions/{sid}/outcome`: the page's content-free beacon, forwarded to the agents service.""" + + def test_forwards_the_closed_fields_only(self, client, upstream) -> None: + upstream.handler = lambda request: httpx.Response(204) + body = {"action": "feedback", "version": 2, "reaction": "thumbs_up"} + response = client.post("/debug/agent/sessions/s1/outcome", headers=CLIENT_HEADERS, json=body) + assert response.status_code == 204 and response.headers["X-Request-Id"] + assert str(upstream.requests[0].url) == f"{SERVICE_URL}/v1/sessions/s1/outcome" + assert upstream.last_json == body + + def test_leaves_out_an_absent_reaction(self, client, upstream) -> None: + upstream.handler = lambda request: httpx.Response(204) + client.post("/debug/agent/sessions/s1/outcome", headers=CLIENT_HEADERS, json={"action": "saved_session"}) + assert upstream.last_json == {"action": "saved_session", "version": 0} + + @pytest.mark.parametrize( + "body", + [ + {"action": "copied_code", "reaction": "bug"}, + {"action": "pasted_into_slack"}, + {"action": "feedback", "reaction": "love"}, + {"action": "copied_code", "version": "1"}, + {"action": "copied_code", "version": 1000}, + {"action": "copied_code", "message": "the legend hides the points"}, + ], + ) + def test_refuses_anything_outside_the_closed_sets(self, client, upstream, body) -> None: + response = client.post("/debug/agent/sessions/s1/outcome", headers=CLIENT_HEADERS, json=body) + assert response.status_code == 422 + assert upstream.requests == [] + + def test_relays_the_rate_limit(self, client, upstream) -> None: + upstream.handler = lambda request: httpx.Response(429, json={"detail": "rate_limited"}) + response = client.post( + "/debug/agent/sessions/s1/outcome", headers=CLIENT_HEADERS, json={"action": "copied_image"} + ) + assert response.status_code == 429 and response.json()["detail"] == "rate_limited" + + def test_needs_the_client_header(self, client, upstream) -> None: + response = client.post("/debug/agent/sessions/s1/outcome", json={"action": "copied_image"}) + assert response.status_code == 403 + assert upstream.requests == [] + + def _seed(cases: FakeBucket, rows: FakeRows, *, include_data: bool = True, **row_overrides: Any) -> AgentFeedback: """A stored case: its files in the fake bucket through the real `build_case`, its row in the fake table.""" case = build_case( diff --git a/tests/unit/api/test_agent_usage.py b/tests/unit/api/test_agent_usage.py new file mode 100644 index 00000000000..276797ea8b8 --- /dev/null +++ b/tests/unit/api/test_agent_usage.py @@ -0,0 +1,188 @@ +""" +Tests for GET /debug/agent-usage and api/agent_usage.py — the agent usage highlights read from BigQuery. + +No test reaches BigQuery: `get_usage_query` is overridden with a fake that records the SQL and answers rows. +""" + +from typing import Any +from unittest.mock import patch + +import pytest +from fastapi.testclient import TestClient + +from api.agent_usage import DATASET_PATTERN, decode_rows, read_usage, usage_sql +from api.main import app, fastapi_app +from api.routers.debug import get_usage_query, require_admin +from core.config import settings + + +DATASET = "anyplot.agent_telemetry" +ROW: dict[str, Any] = { + "sessions": 12, + "plots_shipped": 30, + "kept_rate": 0.4166666, + "repair_rate": 0.3125, + "top_specs": [{"key": "scatter-basic", "count": 5}, {"key": "bar-basic", "count": 3}], + "top_formats": [{"key": "csv", "count": 9}], + "top_refusals": [{"key": "unparseable", "count": 3}, {"key": None, "count": 1}], +} + + +class FakeQuery: + def __init__(self, rows: list[dict[str, Any]] | None = None, error: Exception | None = None) -> None: + self.calls: list[tuple[str, int]] = [] + self._rows = rows if rows is not None else [ROW] + self._error = error + + def rows(self, sql: str, days: int) -> list[dict[str, Any]]: + self.calls.append((sql, days)) + if self._error is not None: + raise self._error + return self._rows + + +@pytest.fixture +def fake() -> FakeQuery: + return FakeQuery() + + +@pytest.fixture +def client(fake: FakeQuery): + fastapi_app.dependency_overrides[require_admin] = lambda: None + fastapi_app.dependency_overrides[get_usage_query] = lambda: fake + with patch.object(settings, "agent_telemetry_dataset", DATASET): + yield TestClient(app) + fastapi_app.dependency_overrides.clear() + + +class TestEndpoint: + def test_answers_the_aggregates(self, client, fake) -> None: + response = client.get("/debug/agent-usage?days=30") + assert response.status_code == 200 + body = response.json() + assert body["status"] == "ok" and body["days"] == 30 + assert (body["sessions"], body["plots_shipped"]) == (12, 30) + assert body["kept_rate"] == 0.4167 and body["repair_rate"] == 0.3125 + assert body["top_specs"] == [{"key": "scatter-basic", "count": 5}, {"key": "bar-basic", "count": 3}] + assert body["top_refusals"] == [{"key": "unparseable", "count": 3}] # a null key is dropped + assert body["generated_at"].endswith("Z") + assert fake.calls[0][1] == 30 and f"`{DATASET}.events`" in fake.calls[0][0] + + def test_defaults_to_seven_days(self, client, fake) -> None: + assert client.get("/debug/agent-usage").json()["days"] == 7 + assert fake.calls[0][1] == 7 + + @pytest.mark.parametrize("days", ["8", "0", "365", "seven"]) + def test_other_windows_are_refused(self, client, fake, days) -> None: + assert client.get(f"/debug/agent-usage?days={days}").status_code in (400, 422) + assert fake.calls == [] + + def test_not_configured_is_an_empty_state(self, client, fake) -> None: + with patch.object(settings, "agent_telemetry_dataset", None): + body = client.get("/debug/agent-usage").json() + assert body["status"] == "not_configured" and body["sessions"] == 0 and body["top_specs"] == [] + assert fake.calls == [] + + def test_a_failed_query_is_unavailable(self, client) -> None: + fastapi_app.dependency_overrides[get_usage_query] = lambda: FakeQuery(error=RuntimeError("403 Access Denied")) + body = client.get("/debug/agent-usage").json() + assert body["status"] == "unavailable" and body["kept_rate"] is None and body["top_formats"] == [] + + def test_no_client_is_unavailable(self, client) -> None: + fastapi_app.dependency_overrides[get_usage_query] = lambda: None + assert client.get("/debug/agent-usage").json()["status"] == "unavailable" + + def test_needs_the_admin_gate(self) -> None: + fastapi_app.dependency_overrides[get_usage_query] = lambda: FakeQuery() + try: + with patch.object(settings, "admin_token", "supersecret"): + response = TestClient(app).get("/debug/agent-usage") + finally: + fastapi_app.dependency_overrides.clear() + assert response.status_code == 401 + + +class TestRestAnswer: + """`decode_rows` reads BigQuery's `jobs.query` REST answer, where every scalar is a string.""" + + ANSWER: dict[str, Any] = { + "jobComplete": True, + "schema": { + "fields": [ + {"name": "sessions", "type": "INTEGER", "mode": "NULLABLE"}, + {"name": "kept_rate", "type": "FLOAT", "mode": "NULLABLE"}, + {"name": "repair_rate", "type": "FLOAT", "mode": "NULLABLE"}, + { + "name": "top_specs", + "type": "RECORD", + "mode": "REPEATED", + "fields": [ + {"name": "key", "type": "STRING", "mode": "NULLABLE"}, + {"name": "count", "type": "INTEGER", "mode": "NULLABLE"}, + ], + }, + ] + }, + "rows": [ + { + "f": [ + {"v": "12"}, + {"v": "0.25"}, + {"v": None}, + {"v": [{"v": {"f": [{"v": "scatter-basic"}, {"v": "5"}]}}]}, + ] + } + ], + } + + def test_types_the_cells_by_the_schema(self) -> None: + assert decode_rows(self.ANSWER) == [ + { + "sessions": 12, + "kept_rate": 0.25, + "repair_rate": None, + "top_specs": [{"key": "scatter-basic", "count": 5}], + } + ] + + def test_feeds_the_card(self) -> None: + class Rest: + def rows(self, sql: str, days: int) -> list[dict[str, Any]]: + return decode_rows(TestRestAnswer.ANSWER) + + usage = read_usage(Rest(), DATASET, 7) + assert (usage.status, usage.sessions, usage.kept_rate) == ("ok", 12, 0.25) + assert usage.top_specs[0].key == "scatter-basic" and usage.top_specs[0].count == 5 + + def test_an_incomplete_job_is_unavailable(self) -> None: + class Slow: + def rows(self, sql: str, days: int) -> list[dict[str, Any]]: + return decode_rows({"jobComplete": False}) + + assert read_usage(Slow(), DATASET, 7).status == "unavailable" + + def test_no_rows_is_empty(self) -> None: + assert decode_rows({"jobComplete": True, "schema": {"fields": []}}) == [] + + +class TestQuery: + def test_counts_live_sessions_only_and_takes_days_as_a_parameter(self) -> None: + sql = usage_sql(DATASET) + assert "source = 'live'" in sql and "INTERVAL @days DAY" in sql + assert "LIMIT 5" in sql + + @pytest.mark.parametrize("dataset", ["anyplot.agent_telemetry", "my-project-1.usage_views"]) + def test_accepts_a_dataset_name(self, dataset) -> None: + assert DATASET_PATTERN.fullmatch(dataset) + + @pytest.mark.parametrize( + "dataset", ["agent_telemetry", "anyplot.agent`; DROP TABLE x; --", "anyplot.agent telemetry", "Anyplot.x"] + ) + def test_a_dataset_that_could_break_the_sql_is_not_configured(self, dataset) -> None: + fake = FakeQuery() + assert read_usage(fake, dataset, 7).status == "not_configured" + assert fake.calls == [] + + def test_an_empty_answer_is_zero(self) -> None: + usage = read_usage(FakeQuery(rows=[]), DATASET, 7) + assert usage.status == "ok" and usage.sessions == 0 and usage.kept_rate is None From 7b8c3a865744870c669d1fe84d0ed4fe182365c0 Mon Sep 17 00:00:00 2001 From: Markus Neusinger <2921697+MarkusNeusinger@users.noreply.github.com> Date: Fri, 16 Oct 2026 15:16:43 +0200 Subject: [PATCH 2/6] feat(app): outcome beacon and agent usage card on /debug The result card, the session save and the feedback send post the content-free outcome beacon (action, version, reaction only; fire and forget, keepalive). DebugPage gets an "agent usage" section (7d/30d, KPI tiles, top specs, formats and refusal codes, empty states for not_configured/unavailable). Co-Authored-By: Claude Opus 5.5 Claude-Session: https://claude.ai/code/session_01GdMSqLR5ww4ji74EUSmk9M --- app/src/hooks/useAgentSession.save.test.ts | 24 ++ app/src/hooks/useAgentSession.ts | 28 ++ app/src/lib/agent.test.ts | 45 +++ app/src/lib/agent.ts | 42 +++ app/src/pages/AgentChatPage.feedback.test.tsx | 102 ++++++ app/src/pages/AgentChatPage.tsx | 1 + app/src/pages/DebugPage.test.tsx | 135 +++++++- app/src/pages/DebugPage.tsx | 4 + app/src/sections/agent-chat/ChatThread.tsx | 5 +- .../sections/agent-chat/ResultCard.test.tsx | 41 ++- app/src/sections/agent-chat/ResultCard.tsx | 23 +- app/src/sections/debug/AgentUsageSection.tsx | 314 ++++++++++++++++++ 12 files changed, 754 insertions(+), 10 deletions(-) create mode 100644 app/src/sections/debug/AgentUsageSection.tsx diff --git a/app/src/hooks/useAgentSession.save.test.ts b/app/src/hooks/useAgentSession.save.test.ts index ccba4fb0cd9..07dd2afd9a7 100644 --- a/app/src/hooks/useAgentSession.save.test.ts +++ b/app/src/hooks/useAgentSession.save.test.ts @@ -31,9 +31,13 @@ interface Call { let calls: Call[]; let bundle: (call: Call) => Response | Promise; let dataset: (call: Call) => Response | Promise; +/** The usage beacons sent, and the answer the next one gets. */ +let outcomes: { body: unknown; keepalive?: boolean }[]; +let outcome: () => Response | Promise; function mockFetch() { calls = []; + outcomes = []; vi.stubGlobal( 'fetch', vi.fn(async (input: string, init: RequestInit = {}) => { @@ -44,6 +48,10 @@ function mockFetch() { search: url.search, }; calls.push(call); + if (call.method === 'POST' && call.path === '/sessions/S1abcdefgh/outcome') { + outcomes.push({ body: JSON.parse(String(init.body)), keepalive: init.keepalive }); + return outcome(); + } if (call.method === 'POST' && call.path === '/sessions') { return json(200, { session_id: 'S1abcdefgh', @@ -84,6 +92,7 @@ beforeEach(() => { 'Content-Disposition': 'attachment; filename="anyplot-session-S1abcdef.json"', }); dataset = () => json(422, { detail: 'unparseable', ref: 'ref-u' }); + outcome = () => new Response(null, { status: 204 }); URL.createObjectURL = vi.fn(() => 'blob:bundle'); URL.revokeObjectURL = vi.fn(); vi.spyOn(HTMLAnchorElement.prototype, 'click').mockImplementation(function ( @@ -107,6 +116,20 @@ describe('useAgentSession: .save()', () => { expect(call?.search).toBe('?include_data=true'); expect(downloads).toEqual([{ href: 'blob:bundle', download: 'anyplot-session-S1abcdef.json' }]); expect(trackEvent).toHaveBeenCalledWith('agent_session_saved', { include_data: 'yes' }); + // The usage beacon: the session-level action as version 0, nothing else. + await waitFor(() => + expect(outcomes).toEqual([{ body: { action: 'saved_session', version: 0 }, keepalive: true }]) + ); + }); + + it('keeps a save that worked when its usage beacon fails', async () => { + outcome = () => Promise.reject(new TypeError('Failed to fetch')); + const { result } = await openSession(); + await act(() => result.current.saveSession(true)); + + await waitFor(() => expect(outcomes).toHaveLength(1)); + expect(downloads).toHaveLength(1); + expect(result.current.state.phase).toBe('ready'); }); it('asks without the data and names the file itself where the header is unreadable', async () => { @@ -134,6 +157,7 @@ describe('useAgentSession: .save()', () => { expect(error).toMatchObject({ status: 502, code: 'too_large', ref: 'ref-big' }); expect(downloads).toEqual([]); expect(trackEvent).not.toHaveBeenCalledWith('agent_session_saved', expect.anything()); + expect(outcomes).toEqual([]); expect(result.current.state.phase).toBe('ready'); }); diff --git a/app/src/hooks/useAgentSession.ts b/app/src/hooks/useAgentSession.ts index ac76634b0e0..06a3224f075 100644 --- a/app/src/hooks/useAgentSession.ts +++ b/app/src/hooks/useAgentSession.ts @@ -128,6 +128,12 @@ * `agent_plot_rendered{library, status, repaired, spec}`, * `agent_guardrail_block{reason}`, `agent_session_saved{include_data}` and * `agent_result_feedback{reaction, include_data, subject}`. + * + * Usage telemetry: after a copy, a download, a save or a sent feedback, a + * fire-and-forget beacon (`POST .../outcome`) tells the BFF which action it + * was and on which version (with the reaction for feedback), and nothing else: + * no text, no file or column name, no code, no data. `reportOutcome` is the + * result card's way in; the save and the feedback send it themselves. */ import { useCallback, useEffect, useReducer, useRef } from 'react'; @@ -151,6 +157,7 @@ import { MAX_DATASET_BYTES, MAX_FEEDBACK_CHARS, MAX_MESSAGE_CHARS, + type OutcomeAction, parseAgentEvent, type PipelineStep, type PlotLanguage, @@ -1277,6 +1284,20 @@ export function useAgentSession({ specId, library, locale, token }: UseAgentSess [token] ); + /** + * The usage beacon for what the user did with a result (`copied_image`, + * `downloaded_code`, ...): the action and the version only, sent after the + * action succeeded. Fire and forget: without a session it sends nothing, and + * a failure never reaches the page (`agentApi.reportOutcome`). + */ + const reportOutcome = useCallback( + (action: Exclude, number: number): void => { + const sid = stateRef.current.sessionId; + if (sid) void agentApi.reportOutcome(token, sid, { action, version: number }); + }, + [token] + ); + // A dropped binary file read on the server; the data card keeps its own state for it (nothing is stored). const convertFile = useCallback( async (dataBase64: string, sheet?: string): Promise => { @@ -1722,6 +1743,7 @@ export function useAgentSession({ specId, library, locale, token }: UseAgentSess if (!mounted.current) return; downloadBlob(blob, filename); trackEvent('agent_session_saved', { include_data: includeData ? 'yes' : 'no' }); + void agentApi.reportOutcome(token, sid, { action: 'saved_session', version: 0 }); } catch (err) { const failure = handleFailure(err); throw err instanceof AgentApiError ? err : new AgentApiError(0, failure.code); @@ -1819,6 +1841,11 @@ export function useAgentSession({ specId, library, locale, token }: UseAgentSess include_data: input.includeData ? 'yes' : 'no', subject: chat ? 'chat' : 'result', }); + void agentApi.reportOutcome(token, sid, { + action: 'feedback', + version: number, + reaction: input.reaction, + }); } catch (err) { // `502 row_missing`: the case IS in the bucket, only its triage row is // missing. Record it, so the next edit names it in `replaces` and the @@ -1864,6 +1891,7 @@ export function useAgentSession({ specId, library, locale, token }: UseAgentSess switchLibrary, requestTheme, fetchArtifact, + reportOutcome, convertFile, saveSession, setLanguage, diff --git a/app/src/lib/agent.test.ts b/app/src/lib/agent.test.ts index 67510a129d8..687f806f4cc 100644 --- a/app/src/lib/agent.test.ts +++ b/app/src/lib/agent.test.ts @@ -783,6 +783,51 @@ describe('agentFetch', () => { }); }); +describe('agentApi.reportOutcome', () => { + afterEach(() => { + vi.unstubAllGlobals(); + }); + + it('posts the beacon with keepalive, the CSRF header and the three fields only', async () => { + const fetchMock = vi.fn(async () => new Response(null, { status: 204 })); + vi.stubGlobal('fetch', fetchMock); + // A caller that slips in more than the contract allows: the client rebuilds the body. + const smuggled = { + action: 'feedback', + version: 3, + reaction: 'bug', + message: 'the legend hides points', + columns: ['revenue'], + } as unknown as Parameters[2]; + await agentApi.reportOutcome('', 'S/1', smuggled); + await agentApi.reportOutcome('', 'S/1', { action: 'downloaded_data', version: 3 }); + + const [url, init] = fetchMock.mock.calls[0] as unknown as [string, RequestInit]; + expect(url).toMatch(/\/debug\/agent\/sessions\/S%2F1\/outcome$/); + expect(init).toMatchObject({ method: 'POST', keepalive: true }); + expect((init.headers as Record)['X-Anyplot-Client']).toBe('agent-chat/1'); + expect(JSON.parse(init.body as string)).toEqual({ + action: 'feedback', + version: 3, + reaction: 'bug', + }); + const [, second] = fetchMock.mock.calls[1] as unknown as [string, RequestInit]; + expect(JSON.parse(second.body as string)).toEqual({ action: 'downloaded_data', version: 3 }); + }); + + it.each([ + ['a 429', () => Promise.resolve(new Response('{}', { status: 429 }))], + ['a 404', () => Promise.resolve(new Response('{"detail":"not_found"}', { status: 404 }))], + ['a 502', () => Promise.resolve(new Response('', { status: 502 }))], + ['no network', () => Promise.reject(new TypeError('Failed to fetch'))], + ])('swallows %s', async (_name, answer) => { + vi.stubGlobal('fetch', vi.fn(answer)); + await expect( + agentApi.reportOutcome('', 'S1', { action: 'saved_session', version: 0 }) + ).resolves.toBeUndefined(); + }); +}); + describe('helpers', () => { it('buckets sizes without exposing them', () => { expect(sizeBucket(10)).toBe('lt_1kb'); diff --git a/app/src/lib/agent.ts b/app/src/lib/agent.ts index 51381464093..db464b416f2 100644 --- a/app/src/lib/agent.ts +++ b/app/src/lib/agent.ts @@ -371,6 +371,30 @@ export interface FeedbackBody { replaces?: string; } +/** + * What the user did with a result, for the content-free usage telemetry (`POST + * .../outcome`, `api/routers/agent.py`): a closed set, never a name or a text. + */ +export const OUTCOME_ACTIONS = [ + 'copied_code', + 'copied_image', + 'downloaded_png', + 'downloaded_code', + 'downloaded_data', + 'saved_session', + 'feedback', +] as const; +export type OutcomeAction = (typeof OUTCOME_ACTIONS)[number]; + +/** + * The body of `POST .../outcome`: the action, the result version it was on (0 + * for the session itself: `saved_session`, feedback on the chat) and, only with + * `feedback`, the reaction. Nothing else ever travels. + */ +export type OutcomeBody = + | { action: Exclude; version: number } + | { action: 'feedback'; version: number; reaction: FeedbackReaction }; + /** A case's version as the triage shows it: `v2`, or `chat, no plot` for version 0. */ export function caseVersionLabel(resultVersion: number): string { return resultVersion === 0 ? 'chat, no plot' : `v${resultVersion}`; @@ -965,6 +989,24 @@ export const agentApi = { */ sendFeedback: (token: string, sid: string, body: FeedbackBody) => agentJson(`${sessionPath(sid)}/feedback`, token, post(body)), + /** + * The usage beacon (`POST .../outcome`, `204`): fire and forget. The body is + * rebuilt from its three fields, so nothing else can ride along; `keepalive` + * lets it outlive a page that closes right after a download. Every failure + * (a refused body, `429`, an expired session, no network) is swallowed: the + * beacon never shows an error and never blocks the page. + */ + reportOutcome: async (token: string, sid: string, body: OutcomeBody): Promise => { + const payload = + body.action === 'feedback' + ? { action: body.action, version: body.version, reaction: body.reaction } + : { action: body.action, version: body.version }; + try { + await agentFetch(`${sessionPath(sid)}/outcome`, token, { ...post(payload), keepalive: true }); + } catch { + /* best effort: telemetry never surfaces */ + } + }, /** * The feedback cases, newest first (`GET /cases`). An answer without an * `items` list reads as an empty page. diff --git a/app/src/pages/AgentChatPage.feedback.test.tsx b/app/src/pages/AgentChatPage.feedback.test.tsx index 761ce4602ad..2498136ada3 100644 --- a/app/src/pages/AgentChatPage.feedback.test.tsx +++ b/app/src/pages/AgentChatPage.feedback.test.tsx @@ -57,6 +57,9 @@ const PAGE = '/debug/agent?spec=scatter-basic&library=matplotlib&language=python let feedbackBodies: unknown[]; let feedbackAnswer: () => Response; +/** The usage beacons the page sent, with the request options that matter. */ +let outcomes: { body: unknown; keepalive?: boolean; client?: string }[]; +let outcomeAnswer: () => Response | Promise; /** A turn that answers in words only: the session keeps no plot. */ let wordsOnly: boolean; /** Set to hold the next turn's answer until the test resolves it. */ @@ -64,6 +67,7 @@ let hold: Promise | null; function stubFetch() { feedbackBodies = []; + outcomes = []; vi.stubGlobal( 'fetch', vi.fn(async (input: string, init: RequestInit = {}) => { @@ -99,6 +103,14 @@ function stubFetch() { feedbackBodies.push(JSON.parse(String(init.body))); return feedbackAnswer(); } + if (url.pathname.endsWith('/sessions/S1/outcome') && method === 'POST') { + outcomes.push({ + body: JSON.parse(String(init.body)), + keepalive: init.keepalive, + client: (init.headers as Record)['X-Anyplot-Client'], + }); + return outcomeAnswer(); + } if (url.pathname.endsWith('/debug/agent/status')) { return json(200, { libraries: ['matplotlib', 'seaborn'], enabled: true }); } @@ -137,6 +149,7 @@ beforeEach(() => { case_id: `20261012T14300${feedbackBodies.length}Z-1a2b3c4d`, stored: ['plot.py', 'manifest.json'], }); + outcomeAnswer = () => new Response(null, { status: 204 }); URL.createObjectURL = vi.fn(() => 'blob:png'); URL.revokeObjectURL = vi.fn(); stubFetch(); @@ -180,6 +193,12 @@ describe('AgentChatPage quick feedback', () => { expect(feedbackBodies).toEqual([ { reaction: 'idea', message: null, include_data: false, result_version: 0 }, ]); + // The beacon: the chat is version 0, and only the reaction travels with it. + await waitFor(() => + expect(outcomes.map(item => item.body)).toEqual([ + { action: 'feedback', version: 0, reaction: 'idea' }, + ]) + ); }); it('sends feedback on a version, shows it as sent, and sends an edit', async () => { @@ -209,6 +228,16 @@ describe('AgentChatPage quick feedback', () => { include_data: 'yes', subject: 'result', }); + // The beacon carries the action, the version and the reaction; never the message. + await waitFor(() => + expect(outcomes).toEqual([ + { + body: { action: 'feedback', version: 1, reaction: 'thumbs_down' }, + keepalive: true, + client: 'agent-chat/1', + }, + ]) + ); // Once a plot exists, its card takes the feedback: the chat's own control is gone. expect(screen.queryByTestId('chat-feedback')).not.toBeInTheDocument(); @@ -247,6 +276,8 @@ describe('AgentChatPage quick feedback', () => { 'agent_result_feedback', expect.anything() ); + // Nothing was stored, so nothing is reported. + expect(outcomes).toEqual([]); }); it('records a case stored without its row and names it in the next send', async () => { @@ -288,3 +319,74 @@ describe('AgentChatPage quick feedback', () => { ); }); }); + +describe('AgentChatPage usage beacon', () => { + let clicked: string[]; + + beforeEach(() => { + clicked = []; + vi.spyOn(HTMLAnchorElement.prototype, 'click').mockImplementation(function ( + this: HTMLAnchorElement + ) { + clicked.push(this.download); + }); + }); + + it('reports a plot.py download as the action and the version, nothing else', async () => { + const user = userEvent.setup(); + renderPage(); + await plotVersionOne(); + await user.click(await screen.findByRole('button', { name: 'Download plot.py' })); + + await waitFor(() => expect(outcomes).toHaveLength(1)); + expect(clicked).toEqual(['plot.py']); + expect(outcomes[0]).toEqual({ + body: { action: 'downloaded_code', version: 1 }, + keepalive: true, + client: 'agent-chat/1', + }); + expect(Object.keys(outcomes[0].body as object).sort()).toEqual(['action', 'version']); + }); + + it('reports a copied image', async () => { + const user = userEvent.setup(); + vi.stubGlobal( + 'ClipboardItem', + class { + constructor(readonly items: Record) {} + } + ); + Object.defineProperty(navigator, 'clipboard', { + value: { write: vi.fn(async () => undefined), writeText: vi.fn() }, + configurable: true, + }); + renderPage(); + await plotVersionOne(); + const copy = await screen.findByRole('button', { name: 'Copy image' }); + await waitFor(() => expect(copy).toBeEnabled()); + await user.click(copy); + + await waitFor(() => + expect(outcomes.map(item => item.body)).toEqual([{ action: 'copied_image', version: 1 }]) + ); + }); + + it.each([ + ['a 500', () => json(500, { detail: 'boom', ref: 'ref-500' })], + ['a 429', () => json(429, { detail: 'rate_limited' })], + ['a 401', () => json(401, { status: 401, message: 'admin required' })], + ['no network', () => Promise.reject(new TypeError('Failed to fetch'))], + ])('shows nothing when the beacon meets %s', async (_name, answer) => { + const user = userEvent.setup(); + outcomeAnswer = answer; + renderPage(); + await plotVersionOne(); + await user.click(await screen.findByRole('button', { name: 'Download plot.py' })); + + await waitFor(() => expect(outcomes).toHaveLength(1)); + expect(clicked).toEqual(['plot.py']); + expect(screen.queryByRole('alert')).not.toBeInTheDocument(); + // The page carries on: no sign-in notice, the composer is still there. + expect(screen.getByLabelText('Message')).toBeInTheDocument(); + }); +}); diff --git a/app/src/pages/AgentChatPage.tsx b/app/src/pages/AgentChatPage.tsx index d08ddcb5258..1fdb9745129 100644 --- a/app/src/pages/AgentChatPage.tsx +++ b/app/src/pages/AgentChatPage.tsx @@ -591,6 +591,7 @@ function AgentChat({ specId, library, language, token, onLibraryChange }: AgentC onRequestTheme={(version, theme) => void session.requestTheme(version, theme)} onFetchArtifact={session.fetchArtifact} onTrack={trackEvent} + onOutcome={session.reportOutcome} onSendFeedback={session.sendFeedback} /> diff --git a/app/src/pages/DebugPage.test.tsx b/app/src/pages/DebugPage.test.tsx index e4e92a039bc..ee5e7f3da4d 100644 --- a/app/src/pages/DebugPage.test.tsx +++ b/app/src/pages/DebugPage.test.tsx @@ -2,7 +2,7 @@ import { fireEvent } from '@testing-library/react'; import { beforeEach, describe, expect, it, vi } from 'vitest'; import { DebugPage } from 'src/pages/DebugPage'; -import { render, screen, waitFor } from 'src/test-utils'; +import { render, screen, waitFor, within } from 'src/test-utils'; const mockDebugData = { total_specs: 100, @@ -426,3 +426,136 @@ describe('DebugPage', () => { expect(copied).toContain('Open a pull request'); }); }); + +describe('DebugPage agent usage', () => { + const USAGE = { + status: 'ok', + days: 7, + sessions: 12, + plots_shipped: 30, + kept_rate: 0.42, + repair_rate: null, + top_specs: [ + { key: 'scatter-basic', count: 5 }, + { key: 'bar-grouped', count: 2 }, + ], + top_formats: [{ key: 'csv', count: 9 }], + top_refusals: [], + generated_at: '2026-10-16T12:00:00Z', + }; + + /** The page's own calls answered as in the tests above; the usage call by `usage`. */ + function stubFetch(usage: (days: string) => Promise) { + const fetchMock = vi.fn((url: string) => { + if (url.includes('/debug/agent-usage')) { + return usage(new URL(url).searchParams.get('days') ?? ''); + } + if (url.includes('/debug/agent/cases')) { + return Promise.resolve({ + ok: true, + json: () => Promise.resolve({ items: [], has_more: false }), + }); + } + if (url.includes('/debug/feedback/')) { + return Promise.resolve({ ok: true, json: () => Promise.resolve([]) }); + } + return Promise.resolve({ ok: true, json: () => Promise.resolve(mockDebugData) }); + }); + vi.stubGlobal('fetch', fetchMock); + return fetchMock; + } + + const ok = (body: unknown) => Promise.resolve({ ok: true, json: () => Promise.resolve(body) }); + + it('shows the tiles and the ranked lists of an ok answer', async () => { + const fetchMock = stubFetch(() => ok(USAGE)); + render(); + + const section = await screen.findByRole('region', { name: 'Agent usage section' }); + expect(await within(section).findByRole('group', { name: 'sessions' })).toHaveTextContent('12'); + expect(within(section).getByRole('group', { name: 'plots shipped' })).toHaveTextContent('30'); + expect(within(section).getByRole('group', { name: 'kept rate' })).toHaveTextContent('42%'); + // No runs in the window: the rate has no base. + expect(within(section).getByRole('group', { name: 'repair rate' })).toHaveTextContent('—'); + const specs = within(section).getByRole('group', { name: 'top specs' }); + expect( + within(specs) + .getAllByRole('listitem') + .map(item => item.textContent) + ).toEqual(['5scatter-basic', '2bar-grouped']); + expect(within(section).getByRole('group', { name: 'top formats' })).toHaveTextContent('9csv'); + expect(within(section).getByRole('group', { name: 'top refusal codes' })).toHaveTextContent( + 'none in this window' + ); + const urls = fetchMock.mock.calls.map(call => call[0]); + expect(urls.filter(url => url.includes('/debug/agent-usage'))).toEqual([ + expect.stringMatching(/\/debug\/agent-usage\?days=7$/), + ]); + }); + + it('says that BigQuery is not configured yet, without tiles', async () => { + stubFetch(() => + ok({ ...USAGE, status: 'not_configured', sessions: 0, plots_shipped: 0, kept_rate: null }) + ); + render(); + + expect(await screen.findByTestId('agent-usage-empty')).toHaveTextContent( + 'BigQuery telemetry is not configured yet' + ); + expect(screen.queryByRole('group', { name: 'kept rate' })).not.toBeInTheDocument(); + }); + + it('says so for an empty window, a failed query and a failed request', async () => { + let answer: () => Promise = () => + ok({ ...USAGE, sessions: 0, plots_shipped: 0, top_specs: [], top_formats: [] }); + stubFetch(() => answer()); + render(); + expect(await screen.findByTestId('agent-usage-empty')).toHaveTextContent( + 'no sessions in this window' + ); + + answer = () => ok({ ...USAGE, status: 'unavailable' }); + fireEvent.click(screen.getByRole('button', { name: '30d' })); + await waitFor(() => + expect(screen.getByTestId('agent-usage-empty')).toHaveTextContent( + 'usage data unavailable right now' + ) + ); + + answer = () => Promise.reject(new TypeError('Failed to fetch')); + fireEvent.click(screen.getByRole('button', { name: '7d' })); + await waitFor(() => + expect(screen.getByTestId('agent-usage-empty')).toHaveTextContent( + 'usage data unavailable right now' + ) + ); + // The rest of the page stays. + expect(screen.getAllByText('scatter-basic').length).toBeGreaterThan(0); + }); + + it('fetches the 30-day window on the toggle', async () => { + const fetchMock = stubFetch(days => + ok({ ...USAGE, days: Number(days), sessions: days === '30' ? 77 : 12 }) + ); + render(); + + expect(await screen.findByRole('group', { name: 'sessions' })).toHaveTextContent('12'); + expect(screen.getByRole('button', { name: '7d' })).toHaveAttribute('aria-pressed', 'true'); + + fireEvent.click(screen.getByRole('button', { name: '30d' })); + await waitFor(() => + expect(screen.getByRole('group', { name: 'sessions' })).toHaveTextContent('77') + ); + expect(screen.getByRole('button', { name: '30d' })).toHaveAttribute('aria-pressed', 'true'); + expect(screen.getByRole('region', { name: 'Agent usage section' })).toHaveTextContent( + 'content-free telemetry · last 30 days' + ); + const usageUrls = fetchMock.mock.calls + .map(call => call[0]) + .filter(url => url.includes('/debug/agent-usage')); + expect(usageUrls).toEqual([ + expect.stringMatching(/\?days=7$/), + expect.stringMatching(/\?days=30$/), + ]); + }); +}); diff --git a/app/src/pages/DebugPage.tsx b/app/src/pages/DebugPage.tsx index 579cdcf9bac..8addb3d5ccc 100644 --- a/app/src/pages/DebugPage.tsx +++ b/app/src/pages/DebugPage.tsx @@ -16,6 +16,7 @@ import { useAnalytics, useCopyCode } from 'src/hooks'; import { fetchWithAuth } from 'src/lib/api'; import { specPath } from 'src/routes/paths'; import { AgentCasesSection } from 'src/sections/debug/AgentCasesSection'; +import { AgentUsageSection } from 'src/sections/debug/AgentUsageSection'; import { timeAgo } from 'src/sections/debug/format'; import { debugControlSx } from 'src/sections/debug/styles'; import { colors, fontSize, semanticColors, typography } from 'src/theme'; @@ -1292,6 +1293,9 @@ export function DebugPage() { )} + {/* Agent usage: the agent chat's content-free telemetry, 7 or 30 days */} + + {/* Agent cases: the triage of the agent chat's quick feedback */} diff --git a/app/src/sections/agent-chat/ChatThread.tsx b/app/src/sections/agent-chat/ChatThread.tsx index 2373fe452d3..28f7fe60955 100644 --- a/app/src/sections/agent-chat/ChatThread.tsx +++ b/app/src/sections/agent-chat/ChatThread.tsx @@ -22,7 +22,7 @@ import type { ArtifactName, Theme } from 'src/lib/agent'; import { Ellipsis } from 'src/sections/agent-chat/Ellipsis'; import { ERROR_TEXT, PLOT_FAILED_TEXT } from 'src/sections/agent-chat/messages'; import { ReloadHint } from 'src/sections/agent-chat/ReloadHint'; -import { ResultCard } from 'src/sections/agent-chat/ResultCard'; +import { type CardOutcome, ResultCard } from 'src/sections/agent-chat/ResultCard'; import { type PendingReply, pendingReply } from 'src/sections/agent-chat/steps'; import { bodyTextSx, labelSx } from 'src/sections/agent-chat/styles'; import { colors, typography } from 'src/theme'; @@ -41,6 +41,8 @@ interface ChatThreadProps { onRequestTheme: (version: number, theme: Theme) => void; onFetchArtifact: (version: number, name: ArtifactName) => Promise; onTrack: (event: string, props?: Record) => void; + /** The result cards' usage beacon (`ResultCard`'s `onOutcome`). */ + onOutcome?: (action: CardOutcome, version: number) => void; /** The result cards' quick feedback; without it they show no control. */ onSendFeedback?: (version: number, input: FeedbackInput) => Promise; } @@ -156,6 +158,7 @@ function Item({ onRequestTheme={props.onRequestTheme} onFetchArtifact={props.onFetchArtifact} onTrack={props.onTrack} + onOutcome={props.onOutcome} onSendFeedback={props.onSendFeedback} /> ); diff --git a/app/src/sections/agent-chat/ResultCard.test.tsx b/app/src/sections/agent-chat/ResultCard.test.tsx index 1ca93df6cf4..175a00e45e1 100644 --- a/app/src/sections/agent-chat/ResultCard.test.tsx +++ b/app/src/sections/agent-chat/ResultCard.test.tsx @@ -29,6 +29,7 @@ function renderCard(props: Partial = {}) { onRequestTheme: vi.fn(), onFetchArtifact: vi.fn(async () => new Blob(['a,b\n1,2\n'], { type: 'text/csv' })), onTrack: vi.fn(), + onOutcome: vi.fn(), }; render( { value: { write, writeText: vi.fn() }, configurable: true, }); - renderCard(); + const { onOutcome } = renderCard(); await userEvent.click(screen.getByRole('button', { name: 'Copy image' })); expect(write).toHaveBeenCalledTimes(1); const [items] = write.mock.calls[0] as unknown as [{ items: Record }[]]; expect(items[0].items['image/png']).toBe(pngBlob); expect(await screen.findByText('>>> .copied')).toBeInTheDocument(); expect(clicked).toEqual([]); + expect(onOutcome.mock.calls).toEqual([['copied_image', 2]]); }); it('downloads the PNG when the browser cannot copy images', async () => { vi.stubGlobal('ClipboardItem', undefined); - renderCard(); + const { onOutcome } = renderCard(); await userEvent.click(screen.getByRole('button', { name: 'Copy image' })); expect(clicked).toEqual([ { href: 'blob:download', download: 'scatter-basic-matplotlib-v2-light.png' }, ]); expect(await screen.findByText('>>> .downloaded')).toBeInTheDocument(); + // The user got a download, not a copy, and the beacon says so. + expect(onOutcome.mock.calls).toEqual([['downloaded_png', 2]]); }); it('downloads the PNG and opens it full size', async () => { const open = vi.spyOn(window, 'open').mockImplementation(() => null); - renderCard(); + const { onOutcome } = renderCard(); await userEvent.click(screen.getByRole('button', { name: 'Download PNG' })); expect(clicked.map(entry => entry.download)).toEqual(['scatter-basic-matplotlib-v2-light.png']); await userEvent.click(screen.getByRole('button', { name: 'Open full size' })); expect(open).toHaveBeenCalledWith('blob:light', '_blank', 'noopener,noreferrer'); + // Opening full size is no outcome of its own. + expect(onOutcome.mock.calls).toEqual([['downloaded_png', 2]]); }); it('copies the code and records copy_code for the agent chat', async () => { @@ -136,7 +142,7 @@ describe('ResultCard', () => { value: { writeText }, configurable: true, }); - const { onTrack } = renderCard(); + const { onTrack, onOutcome } = renderCard(); await userEvent.click(screen.getByRole('button', { name: 'Copy code' })); expect(writeText).toHaveBeenCalledWith('import pandas as pd\ndf = pd.read_csv("data.csv")\n'); expect(onTrack).toHaveBeenCalledWith('copy_code', { @@ -145,10 +151,22 @@ describe('ResultCard', () => { method: 'agent', page: 'agent_chat', }); + expect(onOutcome.mock.calls).toEqual([['copied_code', 2]]); + }); + + it('reports no copied code when the clipboard refuses it', async () => { + vi.spyOn(console, 'error').mockImplementation(() => undefined); + Object.defineProperty(navigator, 'clipboard', { + value: { writeText: vi.fn(async () => Promise.reject(new Error('denied'))) }, + configurable: true, + }); + const { onOutcome } = renderCard(); + await userEvent.click(screen.getByRole('button', { name: 'Copy code' })); + expect(onOutcome).not.toHaveBeenCalled(); }); it('downloads plot.py and data.csv under the names the code expects', async () => { - const { onFetchArtifact } = renderCard(); + const { onFetchArtifact, onOutcome } = renderCard(); await userEvent.click(screen.getByRole('button', { name: 'Download plot.py' })); await userEvent.click(screen.getByRole('button', { name: 'Download data.csv' })); await waitFor(() => @@ -157,6 +175,19 @@ describe('ResultCard', () => { // plot.py comes from the code on screen; data.csv from the artifact route. expect(onFetchArtifact).toHaveBeenCalledTimes(1); expect(onFetchArtifact).toHaveBeenCalledWith(2, 'data.csv'); + expect(onOutcome.mock.calls).toEqual([ + ['downloaded_code', 2], + ['downloaded_data', 2], + ]); + }); + + it('reports no download when the file could not be fetched', async () => { + const { onOutcome } = renderCard({ + onFetchArtifact: vi.fn(async () => Promise.reject(new Error('gone'))), + }); + await userEvent.click(screen.getByRole('button', { name: 'Download data.csv' })); + expect(await screen.findByRole('alert')).toHaveTextContent('could not download data.csv'); + expect(onOutcome).not.toHaveBeenCalled(); }); it('asks for the other theme and shows that it renders', async () => { diff --git a/app/src/sections/agent-chat/ResultCard.tsx b/app/src/sections/agent-chat/ResultCard.tsx index 8a79f7094e7..5df04e9f038 100644 --- a/app/src/sections/agent-chat/ResultCard.tsx +++ b/app/src/sections/agent-chat/ResultCard.tsx @@ -16,6 +16,8 @@ * theme on screen. * - The code is the exported `plot.py`, byte-identical to what rendered; it * runs unchanged next to `data.csv`, so both download under those names. + * - A copy or a download that succeeded calls `onOutcome` with the action and + * the version, the content-free usage beacon; nothing else travels. * - Below the code, the quick-feedback control (`ResultFeedback`) stores the * version as a feedback case; it shows when the page passes `onSendFeedback`. * @@ -37,7 +39,7 @@ import Tooltip from '@mui/material/Tooltip'; import type { FeedbackInput, PlotVersion } from 'src/hooks/useAgentSession'; import { useCopyCode } from 'src/hooks/useCopyCode'; import { useTheme } from 'src/hooks/useLayoutContext'; -import { type ArtifactName, type Theme } from 'src/lib/agent'; +import { type ArtifactName, type OutcomeAction, type Theme } from 'src/lib/agent'; import { copyImage, downloadBlob } from 'src/sections/agent-chat/files'; import { IMAGE_FAILURE } from 'src/sections/agent-chat/messages'; import { ResultFeedback } from 'src/sections/agent-chat/ResultFeedback'; @@ -56,6 +58,9 @@ const TOAST_MS = 1500; */ const PLACEHOLDER_ASPECT = '3200/1864'; +/** The beacon actions a card reports; the save and the feedback report themselves. */ +export type CardOutcome = Exclude; + const STATUS_LABEL: Record = { ok: 'ok', needs_attention: 'needs attention', @@ -76,6 +81,11 @@ export interface ResultCardProps { onRequestTheme: (version: number, theme: Theme) => void; onFetchArtifact: (version: number, name: ArtifactName) => Promise; onTrack: (event: string, props?: Record) => void; + /** + * The usage beacon, called once a copy or a download succeeded, with the + * action and this card's version only; fire and forget. + */ + onOutcome?: (action: CardOutcome, version: number) => void; /** * Store the version as a feedback case (`.send()` in the quick-feedback * control); without it the card shows no control. Never held back by `busy`. @@ -92,6 +102,7 @@ export function ResultCard({ onRequestTheme, onFetchArtifact, onTrack, + onOutcome, onSendFeedback, }: ResultCardProps) { const { isDark } = useTheme(); @@ -128,8 +139,10 @@ export function ResultCard({ const pngName = `${specId}-${library}-v${number}-${shownTheme}.png`; const { copied, copyToClipboard } = useCopyCode({ - onCopy: () => - onTrack('copy_code', { spec: specId, library, method: 'agent', page: 'agent_chat' }), + onCopy: () => { + onTrack('copy_code', { spec: specId, library, method: 'agent', page: 'agent_chat' }); + onOutcome?.('copied_code', number); + }, }); /** Whether showing `theme` needs a render on the server first. */ @@ -150,12 +163,15 @@ export function ResultCard({ if (!readyImage) return; const outcome = await copyImage(readyImage.blob, pngName); showToast(outcome === 'copied' ? '>>> .copied' : '>>> .downloaded'); + // Where the browser cannot copy an image, the user got the PNG as a download. + onOutcome?.(outcome === 'copied' ? 'copied_image' : 'downloaded_png', number); }; const handleDownloadPng = () => { if (!readyImage) return; downloadBlob(readyImage.blob, pngName); showToast('>>> .downloaded'); + onOutcome?.('downloaded_png', number); }; const handleOpen = () => { @@ -178,6 +194,7 @@ export function ResultCard({ // The exported code reads `data.csv` from its own directory, so the pair // keeps exactly these names. downloadBlob(blob, name); + onOutcome?.(name === 'plot.py' ? 'downloaded_code' : 'downloaded_data', number); } catch { setFileError(`could not download ${name}`); } diff --git a/app/src/sections/debug/AgentUsageSection.tsx b/app/src/sections/debug/AgentUsageSection.tsx new file mode 100644 index 00000000000..5d381fd591e --- /dev/null +++ b/app/src/sections/debug/AgentUsageSection.tsx @@ -0,0 +1,314 @@ +/** + * The debug page's "agent usage" section: what the agent chat's content-free + * telemetry adds up to over the last 7 or 30 days (`GET /debug/agent-usage`). + * + * - Four tiles: sessions, plots shipped, the kept rate (sessions with a shipped + * plot whose user copied, downloaded or saved it) and the repair rate (runs + * that needed more than one round); a rate without a base reads `—`. + * - Three short ranked lists: the specs, the data formats and the refusal + * codes seen most often. + * - BigQuery not set up yet (`not_configured`), a failed query (`unavailable`) + * and a window without sessions are one muted line each. A failed request or + * an answer the page cannot read is the `unavailable` line too: the section + * never breaks the rest of the page, and never reloads it for Cloudflare + * Access (the page's own `/debug/status` call owns that). + */ + +import { useEffect, useState } from 'react'; + +import Box from '@mui/material/Box'; +import Typography from '@mui/material/Typography'; + +import { SectionHeader } from 'src/components/SectionHeader'; +import { DEBUG_API_URL } from 'src/constants'; +import { fetchWithAuth } from 'src/lib/api'; +import { timeAgo } from 'src/sections/debug/format'; +import { debugControlSx, listMutedSx, listTextSx } from 'src/sections/debug/styles'; +import { colors, fontSize, semanticColors, typography } from 'src/theme'; + +/** The windows the endpoint takes, in days. */ +const USAGE_WINDOWS = [7, 30] as const; +type UsageWindow = (typeof USAGE_WINDOWS)[number]; + +export interface UsageCount { + key: string; + count: number; +} + +/** The answer of `GET /debug/agent-usage?days=N`. */ +export interface AgentUsage { + status: 'ok' | 'not_configured' | 'unavailable'; + days: number; + sessions: number; + plots_shipped: number; + kept_rate: number | null; + repair_rate: number | null; + top_specs: UsageCount[]; + top_formats: UsageCount[]; + top_refusals: UsageCount[]; + generated_at: string | null; +} + +/** The longest list the endpoint sends; anything beyond is dropped. */ +const MAX_TOP = 5; + +const STATUSES = new Set(['ok', 'not_configured', 'unavailable']); + +const count = (value: unknown): number => + typeof value === 'number' && Number.isFinite(value) && value >= 0 ? Math.floor(value) : 0; + +const rate = (value: unknown): number | null => + typeof value === 'number' && Number.isFinite(value) && value >= 0 && value <= 1 ? value : null; + +const topList = (value: unknown): UsageCount[] => + Array.isArray(value) + ? value + .filter( + (item): item is UsageCount => + !!item && + typeof item === 'object' && + typeof (item as UsageCount).key === 'string' && + typeof (item as UsageCount).count === 'number' + ) + .slice(0, MAX_TOP) + .map(item => ({ key: item.key, count: count(item.count) })) + : []; + +/** The answer as the section reads it; `null` for one without a known status. */ +function readAgentUsage(body: unknown): AgentUsage | null { + if (!body || typeof body !== 'object') return null; + const record = body as Record; + if (typeof record.status !== 'string' || !STATUSES.has(record.status)) return null; + return { + status: record.status as AgentUsage['status'], + days: count(record.days), + sessions: count(record.sessions), + plots_shipped: count(record.plots_shipped), + kept_rate: rate(record.kept_rate), + repair_rate: rate(record.repair_rate), + top_specs: topList(record.top_specs), + top_formats: topList(record.top_formats), + top_refusals: topList(record.top_refusals), + generated_at: typeof record.generated_at === 'string' ? record.generated_at : null, + }; +} + +const percent = (value: number | null): string => + value === null ? '—' : `${Math.round(value * 100)}%`; + +const EMPTY_TEXT: Record<'not_configured' | 'unavailable' | 'empty', string> = { + not_configured: 'BigQuery telemetry is not configured yet', + unavailable: 'usage data unavailable right now', + empty: 'no sessions in this window', +}; + +export interface AgentUsageSectionProps { + /** The admin token of the page (`X-Admin-Token`), or `''` behind Cloudflare Access. */ + token: string; + /** The page's reload count: a change fetches the numbers again. */ + reloadCounter: number; +} + +interface UsageState { + /** The token, reload count and window the answer belongs to. */ + key: string; + /** `null` when the request failed or the answer was unreadable. */ + usage: AgentUsage | null; +} + +export function AgentUsageSection({ token, reloadCounter }: AgentUsageSectionProps) { + const [days, setDays] = useState(7); + const key = `${token}\n${reloadCounter}\n${days}`; + const [answer, setAnswer] = useState(null); + + useEffect(() => { + let cancelled = false; + fetchWithAuth(`${DEBUG_API_URL}/debug/agent-usage?days=${days}`, token) + .then(async response => (response.ok ? readAgentUsage(await response.json()) : null)) + .catch(() => null) + .then(usage => { + if (!cancelled) setAnswer({ key, usage }); + }); + return () => { + cancelled = true; + }; + }, [token, days, key]); + + const current = answer && answer.key === key ? answer : null; + const usage = current?.usage ?? null; + const empty: keyof typeof EMPTY_TEXT | null = !current + ? null + : !usage || usage.status === 'unavailable' + ? 'unavailable' + : usage.status === 'not_configured' + ? 'not_configured' + : usage.sessions === 0 && usage.plots_shipped === 0 + ? 'empty' + : null; + + return ( + + + agent usage} /> + + {USAGE_WINDOWS.map(span => { + const active = span === days; + return ( + setDays(span)} + sx={{ + ...debugControlSx, + cursor: 'pointer', + color: active ? 'var(--ink)' : 'var(--ink-soft)', + fontWeight: active ? 600 : 400, + borderColor: active ? colors.primary : 'var(--rule)', + }} + > + {span}d + + ); + })} + + + + content-free telemetry · last {days} days + {usage?.generated_at ? ` · generated ${timeAgo(usage.generated_at)}` : ''} + + + {!current ? ( + loading usage… + ) : empty || !usage ? ( + + {EMPTY_TEXT[empty ?? 'unavailable']} + + ) : ( + <> + + + + + + + + + + + + + )} + + ); +} + +/** One KPI tile, as the page's own KPI grid draws them. */ +function UsageTile({ label, value }: { label: string; value: string }) { + return ( + + + {value} + + {label} + + ); +} + +/** A short ranked list: the count, then the key. */ +function UsageTopList({ title, items }: { title: string; items: UsageCount[] }) { + return ( + + + {title} + + {items.length === 0 ? ( + + none in this window + + ) : ( + + {items.map(item => ( + + + {item.count} + + + {item.key} + + + ))} + + )} + + ); +} From 4a85209d71339bcb8048d09d9fa9c21a94e393dc Mon Sep 17 00:00:00 2001 From: Markus Neusinger <2921697+MarkusNeusinger@users.noreply.github.com> Date: Fri, 16 Oct 2026 15:18:07 +0200 Subject: [PATCH 3/6] docs(changelog): reference #12205 Co-Authored-By: Claude Opus 5.5 Claude-Session: https://claude.ai/code/session_01GdMSqLR5ww4ji74EUSmk9M --- changelog.d/agent-usage-telemetry.md | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/changelog.d/agent-usage-telemetry.md b/changelog.d/agent-usage-telemetry.md index e5ef4e27e98..62dd5e0f99c 100644 --- a/changelog.d/agent-usage-telemetry.md +++ b/changelog.d/agent-usage-telemetry.md @@ -11,7 +11,7 @@ (`POST /debug/agent/sessions/{sid}/outcome`), and the eval harnesses tag their lines `harness`. Committed BigQuery views (`agents/telemetry/views/`) deduplicate by idempotency key and add sessions with a 30-minute inactivity rule, usage, failures and cost; `docs/reference/agent-telemetry.md` - lists the one-time setup. + lists the one-time setup. (#12205) - **Agent usage card on `/debug`.** Sessions, plots shipped, kept and repair rates, and the top specs, input formats and refusal codes of the last 7 or 30 days, read through the new admin-only - `GET /debug/agent-usage` (aggregates only; an empty state while `AGENT_TELEMETRY_DATASET` is unset). + `GET /debug/agent-usage` (aggregates only; an empty state while `AGENT_TELEMETRY_DATASET` is unset). (#12205) From d880e93670a3cfd817e0821a62dde9c6ee1116d7 Mon Sep 17 00:00:00 2001 From: Markus Neusinger <2921697+MarkusNeusinger@users.noreply.github.com> Date: Fri, 16 Oct 2026 15:30:48 +0200 Subject: [PATCH 4/6] fix(agents): telemetry review findings (attempts, harness scope, cancelled turns, usage views) - pipeline_run.attempts counts every adapter call (run.attempts), rounds stay the page's - the eval harnesses tag lines harness inside telemetry.harness(), restored on every exit - a turn whose client went writes its chat_turn once (error cancelled) after the producer stopped; the normal line is written before the error event is yielded - usage groups by library per line; input formats move to their own view Co-Authored-By: Claude Opus 5.5 Claude-Session: https://claude.ai/code/session_01GdMSqLR5ww4ji74EUSmk9M --- agents/anyplot/pipeline.py | 2 +- agents/anyplot/telemetry.py | 18 ++++- agents/evals/matrix.py | 22 +++++- agents/evals/prep.py | 17 +++- agents/main.py | 10 ++- agents/telemetry/views/04_usage_failures.sql | 81 ++++++++++---------- docs/reference/agent-telemetry.md | 2 +- tests/unit/agents/runtime/test_telemetry.py | 15 +++- 8 files changed, 109 insertions(+), 58 deletions(-) diff --git a/agents/anyplot/pipeline.py b/agents/anyplot/pipeline.py index f0cdb048509..73357f58233 100644 --- a/agents/anyplot/pipeline.py +++ b/agents/anyplot/pipeline.py @@ -1400,7 +1400,7 @@ def _telemetry_run( status=result.status, reason=result.reason, stage=run.stage, - attempts=result.attempts, + attempts=run.attempts, # every adapter call, retries and proofreads included; `rounds` is what the page shows rounds=len(run.rounds), reviews=run.reviews, score_round1=telemetry.score_bucket(run.round_scores.get(1)), diff --git a/agents/anyplot/telemetry.py b/agents/anyplot/telemetry.py index 7b0cdfb9861..4dea2da3f63 100644 --- a/agents/anyplot/telemetry.py +++ b/agents/anyplot/telemetry.py @@ -49,7 +49,8 @@ import re import time from collections import OrderedDict -from collections.abc import Iterable +from collections.abc import Iterable, Iterator +from contextlib import contextmanager from dataclasses import dataclass, field from typing import TYPE_CHECKING, Annotated, Any, Literal, cast, get_args @@ -90,6 +91,18 @@ def set_source(source: Source) -> None: _source = source +@contextmanager +def harness() -> Iterator[None]: + """Tag every line written inside the block `harness`, and restore the previous source on every exit, an + exception included, so ordinary requests after an eval run in the same process count as `live` again.""" + previous = _source + set_source("harness") + try: + yield + finally: + set_source(previous) + + def session_key(session_id: str) -> str: """The salted hash of a session id: 16 hex characters, stable for this process only.""" return salted(session_id, "telemetry", 16) @@ -242,7 +255,8 @@ def _library(value: str) -> str: "daily_limit", "other", ] -ErrorCode = Literal["guard_unavailable", "capacity", "deadline", "internal", "other"] +ErrorCode = Literal["guard_unavailable", "capacity", "deadline", "internal", "cancelled", "other"] +"""`cancelled`: the client closed the stream before the turn ended (the page's cancel, a reload, a lost connection).""" LanguageSource = Literal["default", "pasted", "detected", "chosen"] Step = Literal["root", "adapter", "reviewer", "judge", "other"] diff --git a/agents/evals/matrix.py b/agents/evals/matrix.py index 9abc81058a3..c94c2dcfc79 100644 --- a/agents/evals/matrix.py +++ b/agents/evals/matrix.py @@ -1299,13 +1299,29 @@ async def run_matrix( runtime: "Runtime | None" = None, echo: Callable[[str], None] = _echo, ) -> MatrixResult: - """Run every case `config.repeats` times, write the report, the summary and the galleries, compare and decide.""" - from agents import main as service + """Run every case `config.repeats` times, write the report, the summary and the galleries, compare and decide. + + Every usage-telemetry line of the run is tagged `harness` (the views count live sessions only, + docs/reference/agent-telemetry.md), and the process's source is restored on every exit. + """ from agents.anyplot import telemetry + + with telemetry.harness(): + return await _run_matrix(cases, config, services=services, runtime=runtime, echo=echo) + + +async def _run_matrix( + cases: list["EvalCase"], + config: MatrixConfig, + *, + services: "Services | None", + runtime: "Runtime | None", + echo: Callable[[str], None], +) -> MatrixResult: + from agents import main as service from agents.anyplot.services import Services, set_services from agents.anyplot.settings import get_settings - telemetry.set_source("harness") # the usage views count live sessions only (docs/reference/agent-telemetry.md) settings = get_settings() services = services or Services() recorder: AttemptRecorder | None = None diff --git a/agents/evals/prep.py b/agents/evals/prep.py index 925ba1f705f..893a0fabf3a 100644 --- a/agents/evals/prep.py +++ b/agents/evals/prep.py @@ -586,13 +586,24 @@ async def _read_calls(self, result: CaseResult, user: str, sid: str, decimal: st async def run_cases( cases: Sequence[PrepCase], *, runtime: Runtime, services: Services, pacer: Pacer | None = None, echo: Any = None ) -> list[CaseResult]: - """Every case through the in-process app, one at a time and paced by `pacer`, in the case order.""" - from agents import main as service + """Every case through the in-process app, one at a time and paced by `pacer`, in the case order. + + Every usage-telemetry line is tagged `harness` (docs/reference/agent-telemetry.md); the process's source is + restored on every exit. + """ from agents.anyplot import telemetry + + with telemetry.harness(): + return await _run_cases(cases, runtime=runtime, services=services, pacer=pacer, echo=echo) + + +async def _run_cases( + cases: Sequence[PrepCase], *, runtime: Runtime, services: Services, pacer: Pacer | None, echo: Any +) -> list[CaseResult]: + from agents import main as service from agents.anyplot.services import set_services from agents.anyplot.settings import get_settings - telemetry.set_source("harness") # the usage views count live sessions only (docs/reference/agent-telemetry.md) settings = get_settings() set_services(services) saved_overrides = dict(service.app.dependency_overrides) diff --git a/agents/main.py b/agents/main.py index 04db307a5c9..ee954d3fd0d 100644 --- a/agents/main.py +++ b/agents/main.py @@ -1845,6 +1845,7 @@ async def stream() -> AsyncIterator[str]: ledger.run_slot = TurnSlot(runtime, run, sid, ledger, deadline, outbox).take producer: asyncio.Task[None] | None = None failure: str | None = None + closed_early = False try: yield translator.ready() deadline.arm(runtime.now()) @@ -1863,6 +1864,7 @@ async def stream() -> AsyncIterator[str]: logger.warning("run failed (ref %s): %s", rid, type(exc).__name__) failure = _error_code(exc) except BaseException: # the generator is closed or cancelled (the client went): release the run now + closed_early = True runtime.finish(sid, run) raise finally: @@ -1872,16 +1874,20 @@ async def stream() -> AsyncIterator[str]: with contextlib.suppress(BaseException): await producer CURRENT_LEDGER.reset(token) + if closed_early: + # The client went: the turn's line still books what it spent, once, after the producer stopped. + _telemetry_turn(ledger, facts, "cancelled") try: if ledger.error is not None: failure = ledger.error elif run.deadline_hit and not translator.plot_sent: failure = "deadline" + attribution("run", ledger, model_versions=sorted(ledger.model_versions), kind=ledger.kind) + # Written before anything else is yielded: a client that goes during the error event loses no line. + _telemetry_turn(ledger, facts, failure) if failure is not None: for chunk in translator.error(failure): yield chunk - attribution("run", ledger, model_versions=sorted(ledger.model_versions), kind=ledger.kind) - _telemetry_turn(ledger, facts, failure) # Read while the run is still active: no undo or paste can slip in between the run's end # and the `done` that reports its dataset, so the version and the change lines agree. sync = await _session_sync(runtime, user, sid, dataset_before=dataset_before, ledger=ledger) diff --git a/agents/telemetry/views/04_usage_failures.sql b/agents/telemetry/views/04_usage_failures.sql index 27188b7906e..e149fb04100 100644 --- a/agents/telemetry/views/04_usage_failures.sql +++ b/agents/telemetry/views/04_usage_failures.sql @@ -1,49 +1,46 @@ --- Usage by spec, library and input format, and the failures by kind and code. Run after 03_sessions.sql. +-- Usage by spec and library, the inputs by format, and the failures by kind and code. Run after 03_sessions.sql. --- One row per day, spec, library and input format: what was pasted, what ran, what shipped, what was kept. --- `format` is the format of the session's stored data inputs on that day (`(none)` for sessions without one, --- such as data computed in the chat from a formula or code). +-- One row per day, spec and library: what was pasted, what ran, what shipped, what was kept. Every line carries +-- the library it was written under, so a session that switched from matplotlib to seaborn counts each library's +-- runs under that library (and is one session under each). CREATE OR REPLACE VIEW `anyplot.agent_telemetry.usage` AS -WITH formats AS ( - SELECT DISTINCT DATE(ts) AS day, session, source, format - FROM `anyplot.agent_telemetry.data_inputs` - WHERE outcome = 'ok' -), -daily AS ( - SELECT - DATE(ts) AS day, +SELECT + DATE(ts) AS day, + source, + spec_id, + library, + COUNT(DISTINCT session) AS sessions, + COUNTIF(event = 'data_input') AS data_inputs, + COUNTIF(event = 'chat_turn') AS turns, + COUNTIF(event = 'pipeline_run') AS runs, + COUNTIF(event = 'pipeline_run' AND JSON_VALUE(payload.status) IN ('ok', 'needs_attention')) AS plots_shipped, + COUNT(DISTINCT IF( + event = 'outcome' + AND JSON_VALUE(payload.action) IN ( + 'copied_code', 'copied_image', 'downloaded_png', 'downloaded_code', 'downloaded_data', 'saved_session' + ), session, - source, - ANY_VALUE(spec_id) AS spec_id, - ANY_VALUE(library) AS library, - COUNTIF(event = 'data_input') AS data_inputs, - COUNTIF(event = 'chat_turn') AS turns, - COUNTIF(event = 'pipeline_run') AS runs, - COUNTIF(event = 'pipeline_run' AND JSON_VALUE(payload.status) IN ('ok', 'needs_attention')) AS plots_shipped, - COUNTIF( - event = 'outcome' - AND JSON_VALUE(payload.action) IN ( - 'copied_code', 'copied_image', 'downloaded_png', 'downloaded_code', 'downloaded_data', 'saved_session' - ) - ) AS keeps - FROM `anyplot.agent_telemetry.events` - GROUP BY day, session, source -) + NULL + )) AS sessions_kept +FROM `anyplot.agent_telemetry.events` +GROUP BY day, source, spec_id, library; + +-- One row per day, spec, library and input format: the data inputs of that format and how they ended. Each +-- input counts under its own format only; the session totals live in `usage`, never copied onto every format. +CREATE OR REPLACE VIEW `anyplot.agent_telemetry.input_formats` AS SELECT - daily.day, - daily.source, - daily.spec_id, - daily.library, - IFNULL(formats.format, '(none)') AS format, - COUNT(DISTINCT daily.session) AS sessions, - SUM(daily.data_inputs) AS data_inputs, - SUM(daily.turns) AS turns, - SUM(daily.runs) AS runs, - SUM(daily.plots_shipped) AS plots_shipped, - COUNT(DISTINCT IF(daily.keeps > 0, daily.session, NULL)) AS sessions_kept -FROM daily -LEFT JOIN formats USING (day, session, source) -GROUP BY day, source, spec_id, library, format; + DATE(ts) AS day, + source, + spec_id, + library, + format, + entry, + COUNT(*) AS inputs, + COUNTIF(outcome = 'ok') AS stored, + COUNTIF(outcome NOT IN ('ok', 'transcribed')) AS refused, + COUNT(DISTINCT session) AS sessions +FROM `anyplot.agent_telemetry.data_inputs` +GROUP BY day, source, spec_id, library, format, entry; -- One row per failure: a refused or failed data input, a refused or failed turn, a failed pipeline run, and -- every sandbox failure class and failed gate of a run. `kind` says which, `code` the closed code. diff --git a/docs/reference/agent-telemetry.md b/docs/reference/agent-telemetry.md index 0f439671f99..3a235b03c93 100644 --- a/docs/reference/agent-telemetry.md +++ b/docs/reference/agent-telemetry.md @@ -106,7 +106,7 @@ and dataset names written out. | `01_events.sql` | `events` | One row per line, deduplicated by `idempotency_key` (`QUALIFY ROW_NUMBER()`), with the envelope as columns and the line as `payload` | | `02_event_views.sql` | `data_inputs`, `chat_turns`, `pipeline_runs`, `outcomes` | One typed column per field | | `03_sessions.sql` | `sessions` | One row per session: data inputs, turns, runs, shipped plots, `turns_to_first_plot`, keeps, `funnel_last_step`, and `state` under a 30-minute inactivity rule (`active`, `kept`, `refined`, `abandoned_after_plot`, `abandoned_before_plot`) | -| `04_usage_failures.sql` | `usage`, `failures` | Usage by day, spec, library and input format; one row per failure with its kind and closed code | +| `04_usage_failures.sql` | `usage`, `input_formats`, `failures` | Usage by day, spec and library (each line under the library it was written under, so a library switch splits a session's runs correctly); data inputs by format and entry, each input under its own format; one row per failure with its kind and closed code | | `05_prices_cost.sql` | `model_prices`, `cost` | The `eu` list prices of `agents/evals/pricing.py` (a test keeps both equal) and the cost of every turn step and data-input judge call | The cost view prices the `chat_turn` and `data_input` lines only: `pipeline_run.steps` repeats the adapter's diff --git a/tests/unit/agents/runtime/test_telemetry.py b/tests/unit/agents/runtime/test_telemetry.py index 8d0b2dbc400..fe222a77cd2 100644 --- a/tests/unit/agents/runtime/test_telemetry.py +++ b/tests/unit/agents/runtime/test_telemetry.py @@ -358,9 +358,16 @@ def test_spec_fit() -> None: def test_the_harness_tags_its_lines(caplog: pytest.LogCaptureFixture) -> None: caplog.set_level(logging.INFO) ledger = RequestLedger(session_id="s1", spec_id="scatter-basic", library="matplotlib") - try: - telemetry.set_source("harness") + with telemetry.harness(): record = telemetry.write(telemetry.Outcome, ledger, action="saved_session") - finally: - telemetry.set_source("live") + after = telemetry.write(telemetry.Outcome, ledger, action="saved_session") assert record is not None and record["source"] == "harness" + assert after is not None and after["source"] == "live" + + +def test_the_harness_source_is_restored_after_a_failure() -> None: + ledger = RequestLedger(session_id="s1", spec_id="scatter-basic", library="matplotlib") + with pytest.raises(RuntimeError), telemetry.harness(): + raise RuntimeError("the eval run failed") + record = telemetry.write(telemetry.Outcome, ledger, action="saved_session") + assert record is not None and record["source"] == "live" From ce8b40db52a7d694408bbd3c552471941fa9797d Mon Sep 17 00:00:00 2001 From: Markus Neusinger <2921697+MarkusNeusinger@users.noreply.github.com> Date: Fri, 16 Oct 2026 15:42:32 +0200 Subject: [PATCH 5/6] fix(agents): telemetry review round 2 (not_ready runs, outcome library and keys, API setting via the pipeline) - a plot asked for before the data is ready writes a not_ready pipeline_run with its reason - an outcome on a version carries that version's library (404 for an unknown version); every click is its own row (request id in the key) - AGENT_TELEMETRY_DATASET comes from the deploy-api trigger's substitution, like AGENT_SERVICE_URL, so the smoke-tested pipeline activates it Co-Authored-By: Claude Opus 5.5 Claude-Session: https://claude.ai/code/session_01GdMSqLR5ww4ji74EUSmk9M --- agents/anyplot/pipeline.py | 11 +++++++ agents/anyplot/telemetry.py | 4 ++- agents/main.py | 13 ++++++-- api/cloudbuild.yaml | 7 ++++- docs/reference/agent-telemetry.md | 16 +++++++--- docs/reference/api.md | 7 +++-- tests/unit/agents/runtime/test_telemetry.py | 34 +++++++++++++++++++-- 7 files changed, 77 insertions(+), 15 deletions(-) diff --git a/agents/anyplot/pipeline.py b/agents/anyplot/pipeline.py index 73357f58233..69d12823e03 100644 --- a/agents/anyplot/pipeline.py +++ b/agents/anyplot/pipeline.py @@ -1284,6 +1284,17 @@ async def run_pipeline(ctx: Context, node_input: PipelineArgs) -> AsyncGenerator view, dataset, not_ready = load_inputs(ctx, services) if not_ready is not None or view is None or dataset is None: keep_pending_request(ctx.state, node_input, not_ready) + # An attempted plot the data was not ready for is a run of its own in the telemetry: the funnel sees it. + index = telemetry.run_index(ctx.session.id, ctx.invocation_id) + telemetry.write( + telemetry.PipelineRun, + ledger, + attempt=f"run{index}", + run_index=index, + base=node_input.base, + status="not_ready", + reason=not_ready or "no_dataset", + ) yield _result_event(PlotResult.not_ready(not_ready or "no_dataset")) return ledger.pipeline_active = True diff --git a/agents/anyplot/telemetry.py b/agents/anyplot/telemetry.py index 4dea2da3f63..dbbc1fd93b7 100644 --- a/agents/anyplot/telemetry.py +++ b/agents/anyplot/telemetry.py @@ -262,6 +262,8 @@ def _library(value: str) -> str: PlotStatus = Literal["ok", "needs_attention", "failed", "not_ready"] FailureReason = Literal["validation", "render", "deadline", "budget", "error"] +NotReadyReason = Literal["no_dataset", "incomplete_bindings", "data_unconfirmed"] +"""`schemas.NotReadyReason`: why a `not_ready` run did not start (pinned equal by a test).""" ScoreBucket = Literal["lt50", "50_69", "70_79", "80_89", "90_plus"] SandboxFailure = Literal["syntax", "name", "key", "value", "type", "timeout", "memory", "other"] PipelineStage = Literal[ @@ -372,7 +374,7 @@ class PipelineRun(_Event): run_index: Annotated[int, Field(ge=1, le=10_000)] base: Literal["catalogue", "previous"] status: PlotStatus - reason: FailureReason | None = None + reason: FailureReason | NotReadyReason | None = None stage: PipelineStage | None = None attempts: Count = 0 rounds: Count = 0 diff --git a/agents/main.py b/agents/main.py index ee954d3fd0d..029d912bbb4 100644 --- a/agents/main.py +++ b/agents/main.py @@ -2127,20 +2127,29 @@ async def post_outcome(sid: SessionId, body: OutcomeBody, user: User, runtime: R The page sends it after a copy, a download, a session save or a feedback send; the BFF forwards it, so every telemetry line comes from this one writer with the same salted session hash. Content-free by its schema (closed `action` and `reaction`, a version number). `429 rate_limited` / `daily_limit` past - `OUTCOMES_PER_MINUTE` / `_PER_DAY` per user, which the page ignores; `404 session_expired` as everywhere. + `OUTCOMES_PER_MINUTE` / `_PER_DAY` per user, which the page ignores; `404 session_expired` as everywhere, and + `404 not_found` for a version the session does not have. A version's line carries that version's library, + so a card made before a library switch counts for its own library; version 0 carries the session's. No model call, no state write: the session's events are untouched. """ session = await runtime.session(user, sid) services = get_services() + version = services.versions.get(sid, body.version) if body.version else None + if body.version and version is None: + raise AgentsError(404, "not_found") limit = services.usage.outcome_limit(user) if limit is not None: raise AgentsError(429, limit) services.usage.add_outcome(user) ledger = _stamp(RequestLedger(request_id=rid, user_id=user, session_id=sid), read_session(session.state), session) + if version is not None: + # A result card of an earlier library stays usable after a switch: its outcome counts for its own library. + ledger.library = version.library telemetry.write( telemetry.Outcome, ledger, - attempt=f"{body.action}:{body.version}:{body.reaction or ''}", + # Every click is its own row (the request id, hashed into the key only); a line delivered twice still repeats it. + attempt=f"{body.action}:{body.version}:{rid}", action=body.action, version=body.version, reaction=body.reaction, diff --git a/api/cloudbuild.yaml b/api/cloudbuild.yaml index 5980633127d..2d496486e8c 100644 --- a/api/cloudbuild.yaml +++ b/api/cloudbuild.yaml @@ -42,6 +42,11 @@ substitutions: # secret which does not exist, so the binding followed the secret). _AGENT_ENABLED: "false" _AGENT_SERVICE_URL: "" + # `.` of the agent usage telemetry's BigQuery views, which the + # admin card on /debug reads (docs/reference/agent-telemetry.md). Empty: the card + # says "not configured". Set it on the deploy-api trigger once the views exist, + # for the same reason as the two above: this pipeline writes it on every deploy. + _AGENT_TELEMETRY_DATASET: "" steps: # Build the container image @@ -109,7 +114,7 @@ steps: # both flags means one deploy cannot undo an out-of-band change; the cost # is that a variable dropped from this line is no longer removed by the # next deploy, which is the trade already accepted for the secrets. - - "--update-env-vars=^|^ENVIRONMENT=production|GOOGLE_CLOUD_PROJECT=$PROJECT_ID|GCS_BUCKET=anyplot-images|CF_ACCESS_TEAM_DOMAIN=${_CF_ACCESS_TEAM_DOMAIN}|CF_ACCESS_AUD=${_CF_ACCESS_AUD}|ADMIN_ALLOWED_EMAILS=${_ADMIN_ALLOWED_EMAILS}|AGENT_ENABLED=${_AGENT_ENABLED}|AGENT_SERVICE_URL=${_AGENT_SERVICE_URL}" + - "--update-env-vars=^|^ENVIRONMENT=production|GOOGLE_CLOUD_PROJECT=$PROJECT_ID|GCS_BUCKET=anyplot-images|CF_ACCESS_TEAM_DOMAIN=${_CF_ACCESS_TEAM_DOMAIN}|CF_ACCESS_AUD=${_CF_ACCESS_AUD}|ADMIN_ALLOWED_EMAILS=${_ADMIN_ALLOWED_EMAILS}|AGENT_ENABLED=${_AGENT_ENABLED}|AGENT_SERVICE_URL=${_AGENT_SERVICE_URL}|AGENT_TELEMETRY_DATASET=${_AGENT_TELEMETRY_DATASET}" - "--cpu-throttling" # 40, not 15: with _MAX_INSTANCES=1 this limit is the only capacity the # service has, and 15 was a default nobody had measured against. On this diff --git a/docs/reference/agent-telemetry.md b/docs/reference/agent-telemetry.md index 3a235b03c93..3d4e3537995 100644 --- a/docs/reference/agent-telemetry.md +++ b/docs/reference/agent-telemetry.md @@ -34,7 +34,7 @@ The agents service writes one wide JSON line per event to stdout on its own logg |---|---|---| | `data_input` | A paste or a dropped file reaches the dataset route, or the file conversion refuses a file | `format`, `entry` (`paste` or `file`), `outcome`, `cause`, `size_bucket`, `rows_bucket`, `cols_bucket`, column counts by type, `max_categories_bucket`, `missing_bucket`, `warnings` (codes), `tables`, `sheets`, `notes` and `file_name` (booleans), `spec_fit`, judge tokens, `latency_ms` | | `chat_turn` | A `/messages` turn ends, refused ones included; `manual_binding_changed` counts only a `PUT /bindings` whose bindings differ from the ones the session held, because the page echoes the parse's defaults after every paste | `kind` (`plot`, `prep`, `formula`, `code`, `talk`, `refused`, `error`), `action`, `asked_question`, `tools`, `prep_ops`, `undo`, `to_long`, `manual_binding_changed`, `binding_warnings`, `refusal`, `error`, `reply_reasked`, `language_source`, `steps` (calls, tokens and model per agent), `latency_ms` | -| `pipeline_run` | A `plot_pipeline` run ends | `run_index` (1 is the session's first plot), `base`, `status`, `reason`, `stage`, `attempts`, `rounds`, `score_round1`, `score_round2` (buckets), `shipped_attempt`, `forced_repair`, `gates_failed`, `reviewer_defects`, `residual_defects`, `sandbox_failures` (`syntax`, `name`, `key`, `value`, `type`, `timeout`, `memory`, `other`), time per stage, `steps` | +| `pipeline_run` | A `plot_pipeline` run ends, or the root asked for one before the data was ready (`status: not_ready` with its reason) | `run_index` (1 is the session's first plot), `base`, `status`, `reason`, `stage`, `attempts`, `rounds`, `score_round1`, `score_round2` (buckets), `shipped_attempt`, `forced_repair`, `gates_failed`, `reviewer_defects`, `residual_defects`, `sandbox_failures` (`syntax`, `name`, `key`, `value`, `type`, `timeout`, `memory`, `other`), time per stage, `steps` | | `outcome` | The page's beacon after a copy, a download, a session save, or a feedback send | `action` (`copied_code`, `copied_image`, `downloaded_png`, `downloaded_code`, `downloaded_data`, `saved_session`, `feedback`), `version`, `reaction` | Every line carries the same envelope: @@ -194,14 +194,20 @@ Run these steps once, as a project owner. They write nothing but the resources n --role=roles/bigquery.dataViewer anyplot:agent_telemetry_logs ``` -10. Point the API at the views. `--update-env-vars` adds the variable, and the next deploy keeps it - (`api/cloudbuild.yaml` updates its variables additively): +10. Point the API at the views through the deploy pipeline, like `_AGENT_SERVICE_URL`: set the + `_AGENT_TELEMETRY_DATASET` substitution on the `deploy-api` trigger. The pipeline writes the variable on + every deploy, smoke-tests the new revision and moves the pinned traffic to it; a + `gcloud run services update` would create a revision that serves no traffic and that the next deploy + overwrites. ```bash - gcloud run services update anyplot-api --region=europe-west4 \ - --update-env-vars=AGENT_TELEMETRY_DATASET=anyplot.agent_telemetry + gcloud builds triggers update github deploy-api --region=europe-west4 \ + --update-substitutions=_AGENT_TELEMETRY_DATASET=anyplot.agent_telemetry ``` + The setting takes effect with the next API deploy; run the trigger by hand if you want it at once + (`gcloud builds triggers run deploy-api --region=europe-west4 --branch=main`). + 11. Connect Looker Studio to the `anyplot.agent_telemetry` views with your own account, which needs the same two roles. diff --git a/docs/reference/api.md b/docs/reference/api.md index 74eaafcd4e2..4b89c44b2ce 100644 --- a/docs/reference/api.md +++ b/docs/reference/api.md @@ -1019,9 +1019,10 @@ could not be read; `PATCH /cases/{id}` answers `422 no_data` and | `AGENT_TELEMETRY_DATASET` | Unset | `.` of the usage telemetry's BigQuery views (`anyplot.agent_telemetry`), which `GET /debug/agent-usage` reads; unset, that route answers `status: not_configured` ([Agent usage telemetry](agent-telemetry.md)) | | `AGENT_CASES_BUCKET` | `anyplot-agent-cases` | The private EU bucket the feedback cases go to and the case routes read from (see [Feedback case](#feedback-case)); never the public image bucket. The default is the production bucket, so a local API with Application Default Credentials stores its cases there too; set a bucket of your own for local runs | -In production, `AGENT_ENABLED` and `AGENT_SERVICE_URL` come from the -`_AGENT_ENABLED` and `_AGENT_SERVICE_URL` substitutions in `api/cloudbuild.yaml`. -Change them there: the deploy rewrites both variables on every build. +In production, `AGENT_ENABLED`, `AGENT_SERVICE_URL` and `AGENT_TELEMETRY_DATASET` +come from the `_AGENT_ENABLED`, `_AGENT_SERVICE_URL` and `_AGENT_TELEMETRY_DATASET` +substitutions in `api/cloudbuild.yaml`. Change them there: the deploy rewrites all +three variables on every build. --- diff --git a/tests/unit/agents/runtime/test_telemetry.py b/tests/unit/agents/runtime/test_telemetry.py index fe222a77cd2..75bcace6aa0 100644 --- a/tests/unit/agents/runtime/test_telemetry.py +++ b/tests/unit/agents/runtime/test_telemetry.py @@ -96,9 +96,15 @@ async def test_a_whole_session_writes_the_four_events_and_none_of_its_content( f"/v1/sessions/{sid}/outcome", headers=HEADERS, json={"action": "copied_code", "version": 1} ) assert beacon.status_code == 204 + again = await client.post( # a second click on the same button is a second row, not a duplicate + f"/v1/sessions/{sid}/outcome", + headers={**HEADERS, "X-Request-Id": "req-telemetry-2"}, + json={"action": "copied_code", "version": 1}, + ) + assert again.status_code == 204 lines = telemetry_lines(caplog) - assert [line["event"] for line in lines] == ["data_input", "pipeline_run", "chat_turn", "outcome"] + assert [line["event"] for line in lines] == ["data_input", "pipeline_run", "chat_turn", "outcome", "outcome"] raw = json.dumps(lines) for canary in (COLUMN_CANARY, CELL_CANARY, NOTE_CANARY, FILE_CANARY, MESSAGE_CANARY, USER, sid, "Exam Score"): assert canary not in raw, canary @@ -108,7 +114,7 @@ async def test_a_whole_session_writes_the_four_events_and_none_of_its_content( assert line["spec_id"] == "scatter-basic" and line["library"] == "matplotlib" assert len(line["session"]) == 16 and line["session"] == lines[0]["session"] - data_input, run, chat_turn, outcome = lines + data_input, run, chat_turn, outcome, _ = lines assert data_input["outcome"] == "ok" and data_input["format"] == "csv" and data_input["entry"] == "file" assert data_input["file_name"] is True and data_input["notes"] is True and data_input["text_cols"] == 1 assert data_input["rows_bucket"] == "11_100" and data_input["cols_bucket"] == "3_5" @@ -119,7 +125,7 @@ async def test_a_whole_session_writes_the_four_events_and_none_of_its_content( assert chat_turn["tools"] == ["plot_pipeline"] and chat_turn["manual_binding_changed"] is True assert {step["step"] for step in chat_turn["steps"]} >= {"root", "adapter", "reviewer"} assert outcome["action"] == "copied_code" and outcome["version"] == 1 and outcome["turn"] == 1 - assert len({line["idempotency_key"] for line in lines}) == 4 + assert len({line["idempotency_key"] for line in lines}) == 5 async def test_the_attribution_lines_still_carry_the_user_and_the_telemetry_lines_never( @@ -168,6 +174,8 @@ async def test_the_beacon_takes_closed_values_only(client: httpx.AsyncClient) -> assert (await client.post(url, headers=HEADERS, json=body)).status_code == 422, body gone = await client.post("/v1/sessions/nope/outcome", headers=HEADERS, json={"action": "copied_code"}) assert gone.status_code == 404 + unknown = await client.post(url, headers=HEADERS, json={"action": "copied_code", "version": 7}) + assert (unknown.status_code, unknown.json()["detail"]) == (404, "not_found") async def test_the_beacon_is_rate_limited_per_user(client: httpx.AsyncClient) -> None: @@ -227,6 +235,26 @@ def test_the_reactions_are_the_feedback_reactions() -> None: assert get_args(telemetry.Reaction) == FEEDBACK_REACTIONS +def test_the_run_reasons_are_the_pipeline_reasons() -> None: + from agents.anyplot.schemas import FailureReason, NotReadyReason + + assert get_args(telemetry.FailureReason) == get_args(FailureReason) + assert get_args(telemetry.NotReadyReason) == get_args(NotReadyReason) + + +async def test_a_plot_before_the_data_is_ready_is_a_not_ready_run( + client: httpx.AsyncClient, swap_models, caplog: pytest.LogCaptureFixture +) -> None: + caplog.set_level(logging.INFO) + swap_models("gemini", default_script()) + sid = await open_session(client) + await client.post(f"/v1/sessions/{sid}/dataset", headers=HEADERS, json={"text": CANARY_DATA}) + # No `.confirm()`: the pipeline answers not_ready, and the telemetry still counts the attempt. + await client.post(f"/v1/sessions/{sid}/messages", headers=HEADERS, json={"action": "create_plot"}) + runs = [line for line in telemetry_lines(caplog) if line["event"] == "pipeline_run"] + assert [(run["status"], run["reason"], run["run_index"]) for run in runs] == [("not_ready", "data_unconfirmed", 1)] + + def test_the_prep_ops_are_the_operations() -> None: from agents.anyplot.data.prepare import Operation From d6403a69c7d8ab181c730f6c948af21b38eff08a Mon Sep 17 00:00:00 2001 From: Markus Neusinger <2921697+MarkusNeusinger@users.noreply.github.com> Date: Fri, 16 Oct 2026 16:04:15 +0200 Subject: [PATCH 6/6] fix(agents): Fable + Gemini review fixes for the usage telemetry - /debug query reads the linked log view with the window filter before the dedupe (partition pruning), a 10-minute floored @since parameter (cacheable) and a 10-minute in-process cache of ok answers; client start failures are never cached - cost: model ids normalised to the price key (version suffixes, models/ prefix), one step entry per (step, model), the judges carry the judge model - telemetry source is a ContextVar: a harness run cannot tag live requests - sessions/usage: not_ready asks are no runs, new funnel steps (outcome_only, transcription_offered, plot_not_ready), a transcription counts once - every parser warning in the data test corpus must have a code (conftest) - a converted file's kind survives a paste refused at the limits - test: two outcome actions never share a key Co-Authored-By: Claude Opus 5.5 Claude-Session: https://claude.ai/code/session_01GdMSqLR5ww4ji74EUSmk9M --- agents/anyplot/telemetry.py | 72 +++++---- agents/main.py | 1 + agents/telemetry/views/03_sessions.sql | 39 +++-- agents/telemetry/views/04_usage_failures.sql | 9 +- agents/telemetry/views/05_prices_cost.sql | 4 +- api/agent_usage.py | 138 ++++++++++++++---- core/config.py | 4 +- docs/reference/agent-telemetry.md | 14 +- tests/unit/agents/data/conftest.py | 29 ++++ tests/unit/agents/data/test_parse.py | 17 ++- .../agents/evals/test_telemetry_prices.py | 35 +++++ tests/unit/agents/runtime/test_telemetry.py | 46 ++++++ tests/unit/api/test_agent_usage.py | 99 +++++++++++-- 13 files changed, 410 insertions(+), 97 deletions(-) create mode 100644 tests/unit/agents/data/conftest.py diff --git a/agents/anyplot/telemetry.py b/agents/anyplot/telemetry.py index dbbc1fd93b7..f86d1d4526b 100644 --- a/agents/anyplot/telemetry.py +++ b/agents/anyplot/telemetry.py @@ -26,7 +26,7 @@ Envelope (every line): `log_type`, `schema`, `event`, `service` and `revision` (Cloud Run's `K_SERVICE` and `K_REVISION`), `source` (`live`, or `harness` for the eval -harnesses, `set_source`), `session` (a salted hash), `spec_id`, `library`, `turn` (the +harnesses, `harness()`), `session` (a salted hash), `spec_id`, `library`, `turn` (the session's user turns so far, from the request ledger), and `idempotency_key`, a hash of session, turn, attempt and event: the BigQuery base view keeps one row per key, so a line Cloud Logging delivers twice or a pipeline node ADK re-runs counts once. @@ -51,6 +51,7 @@ from collections import OrderedDict from collections.abc import Iterable, Iterator from contextlib import contextmanager +from contextvars import ContextVar from dataclasses import dataclass, field from typing import TYPE_CHECKING, Annotated, Any, Literal, cast, get_args @@ -81,26 +82,21 @@ """The line format's version: a field renamed or removed is a new schema; a field added is not.""" Source = Literal["live", "harness"] -_source: Source = "live" - - -def set_source(source: Source) -> None: - """Tag every later line with `source`: the eval harnesses (`evals/matrix.py`, `evals/prep.py`) call it with - `harness` before their first request, so the views can leave their runs out.""" - global _source - _source = source +SOURCE: ContextVar[Source] = ContextVar("anyplot_telemetry_source", default="live") +"""The source of the lines written in this context. A context variable, not a module global: the eval harnesses +run the app in process (`evals/matrix.py`, `evals/prep.py`), and only the requests they send, which run in the +harness's own context, may be tagged `harness`; a live request served by the same process keeps `live`.""" @contextmanager def harness() -> Iterator[None]: - """Tag every line written inside the block `harness`, and restore the previous source on every exit, an - exception included, so ordinary requests after an eval run in the same process count as `live` again.""" - previous = _source - set_source("harness") + """Tag every line written inside the block, in this context and the tasks it starts, `harness`; the previous + source is back on every exit, an exception included.""" + token = SOURCE.set("harness") try: yield finally: - set_source(previous) + SOURCE.reset(token) def session_key(session_id: str) -> str: @@ -124,8 +120,9 @@ def _library(value: str) -> str: Library = Annotated[str, AfterValidator(_library)] """One of `core.constants.SUPPORTED_LIBRARIES`.""" SpecId = Annotated[str, Field(pattern=r"^[a-z0-9-]{1,100}$")] -ModelId = Annotated[str, Field(pattern=r"^[a-z0-9][a-z0-9.@_-]{0,79}$")] -"""A model id as the provider names it (`claude-haiku-5-5`, `gemini-3.8-flash`, `...@20261001`).""" +ModelId = Annotated[str, Field(pattern=r"^[a-z0-9][a-z0-9._-]{0,79}$")] +"""A model id as the price table keys it (`claude-haiku-5-5`, `gemini-3.8-flash`): `model_id` drops version +suffixes before a line is written.""" DefectId = Annotated[str, Field(pattern=r"^[A-Z]{2}-[0-9]{2}$")] GateId = Annotated[str, Field(pattern=r"^(?:R[0-9](?:-[a-z_]{1,24})?|G[0-9]{1,2})$")] @@ -443,7 +440,7 @@ def emit(event: AnyEvent, ledger: RequestLedger, *, attempt: str = "0") -> dict[ envelope = _Envelope( service=service, revision=revision, - source=_source, + source=SOURCE.get(), session=session, spec_id=ledger.spec_id or None, library=ledger.library or None, @@ -668,33 +665,42 @@ def step_of(agent: str) -> Step: def step_tokens( usage: dict[str, StepUsage], *, steps: Iterable[Step], judge_model: str | None = None ) -> list[dict[str, Any]]: - """The ledger's per-agent usage folded by step, for the steps asked for, in a fixed order.""" + """The ledger's per-agent usage folded by step and model, for the steps asked for, in a fixed step order. + + One entry per (step, model), so two models in one step (an experiment's adapter model beside the default) + are each priced at their own rate by the cost view. The judges book no model of their own (`book_judge`): + their entry carries `judge_model`, the settings' judge model, so the cost view can price it. + """ wanted = list(steps) - folded: dict[Step, dict[str, Any]] = {} + folded: dict[tuple[Step, str | None], dict[str, Any]] = {} for agent, spent in usage.items(): step: Step = "judge" if agent == "judge" else step_of(agent) if step not in wanted: continue + model = model_id(spent.model or (judge_model if step == "judge" else None)) item = folded.setdefault( - step, {"step": step, "model": None, "calls": 0, "input": 0, "output": 0, "cached": 0, "cache_write": 0} + (step, model), + {"step": step, "model": model, "calls": 0, "input": 0, "output": 0, "cached": 0, "cache_write": 0}, ) item["calls"] += spent.calls item["input"] += spent.input item["output"] += spent.output item["cached"] += spent.cached item["cache_write"] += spent.cache_write - item["model"] = item["model"] or _model_id(spent.model or (judge_model if step == "judge" else None)) - return [folded[step] for step in wanted if step in folded] + return [folded[key] for key in sorted(folded, key=lambda key: (wanted.index(key[0]), key[1] or ""))] -_MODEL_ID = re.compile(r"^[a-z0-9][a-z0-9.@_-]{0,79}$") +_MODEL_ID = re.compile(r"^[a-z0-9][a-z0-9._-]{0,79}$") +_MODEL_SUFFIX = re.compile(r"(?:@.*|-[0-9]{8})$") -def _model_id(value: str | None) -> str | None: - """A model id that fits `ModelId`, else None: a provider's odd version string never fails the whole line.""" +def model_id(value: str | None) -> str | None: + """A model id as the price table keys it, else None: lower case, without a `models/` prefix and without a + version suffix (`claude-haiku-5-5@20261001` and `claude-haiku-5-5-20261001` are `claude-haiku-5-5`). A + provider's odd version string becomes None and never fails the whole line.""" if not value: return None - text = value.strip().lower().removeprefix("models/") + text = _MODEL_SUFFIX.sub("", value.strip().lower().removeprefix("models/")) return text if _MODEL_ID.fullmatch(text) else None @@ -852,10 +858,12 @@ class InputFacts: @classmethod def for_paste(cls, session_id: str, *, size: int, source: bool, confirmed: bool) -> InputFacts: - """A paste's facts; a paste that names a file after a conversion takes the converted file's kind, once.""" + """A paste's facts; a paste that names a file after a conversion takes the converted file's kind. + + The kind is only read here: the route clears it (`clear_converted`) once the paste passed its limits, so a + paste refused at the rate limit and sent again still knows the file it came from.""" mark = marks(session_id) kind, sheets = mark.convert_kind, mark.convert_sheets - mark.convert_kind, mark.convert_sheets = None, 0 fmt: str | None = None if source and kind is not None: fmt = kind @@ -899,7 +907,7 @@ def write(self, ledger: RequestLedger, clock: Clock, judge_model: str | None = N sheets=self.sheets, judge_input=judge.input if judge else 0, judge_output=judge.output if judge else 0, - judge_model=_model_id(judge_model) if judge else None, + judge_model=model_id(judge_model) if judge else None, latency_ms=clock.ms(), **self.fields, ) @@ -922,6 +930,12 @@ def file_refusal( return facts.write(ledger, clock) +def clear_converted(session_id: str) -> None: + """The paste that followed a conversion reached the parser: the file's kind is spent.""" + mark = marks(session_id) + mark.convert_kind, mark.convert_sheets = None, 0 + + def converted(session_id: str, *, kind: str, sheets: int) -> None: """A conversion succeeded: the paste that follows it with the file's name is that kind of file.""" mark = marks(session_id) diff --git a/agents/main.py b/agents/main.py index 79640022e98..c9f4f9111a8 100644 --- a/agents/main.py +++ b/agents/main.py @@ -923,6 +923,7 @@ async def upload_dataset( if limit is not None: attribution("paste_limit", ledger, verdict=limit) raise AgentsError(429, limit) + telemetry.clear_converted(sid) # past the limits: this paste is the converted file's one runtime.pasting.add(user) # no await since the check above, so no second paste slipped in between try: return await _ingest_paste(services, runtime, body, user, sid, ledger, view, facts) diff --git a/agents/telemetry/views/03_sessions.sql b/agents/telemetry/views/03_sessions.sql index ba97dac423f..e5ad60afd4e 100644 --- a/agents/telemetry/views/03_sessions.sql +++ b/agents/telemetry/views/03_sessions.sql @@ -2,13 +2,18 @@ -- Run after 02_event_views.sql. -- -- `funnel_last_step` is the furthest step the session reached: --- data_refused every data input was refused or failed --- data_ok a dataset was stored --- chatted a chat turn ran --- plot_run a pipeline run ran --- plot_shipped a pipeline run shipped a plot (status ok or needs_attention) --- kept the user copied or downloaded a result or saved the session --- A session that never sent a data input or a turn writes no line, so the funnel starts at the first input. +-- outcome_only only outcome beacons (feedback on a chat, a save) and no input, turn or run +-- data_refused every data input was refused or failed +-- transcription_offered free text was read as a table, and the user never confirmed it +-- data_ok a dataset was stored +-- chatted a chat turn ran +-- plot_not_ready a plot was asked for before the data was confirmed, and none started +-- plot_run a pipeline run started (`not_ready` ones are not runs here) +-- plot_shipped a pipeline run shipped a plot (status ok or needs_attention) +-- kept the user copied or downloaded a result or saved the session +-- A session that never sent a data input, a turn or a beacon writes no line, so the funnel starts there. +-- A transcription writes two data_input rows (the offer, outcome `transcribed`, and the confirmed paste); +-- `data_inputs` counts the input once, without the offer. -- -- `state` applies a 30-minute inactivity rule: a session whose last line is younger than 30 minutes is -- `active`; any other session ended, as @@ -33,7 +38,13 @@ spans AS ( GROUP BY session, source ), inputs AS ( - SELECT session, source, COUNT(*) AS data_inputs, COUNTIF(outcome = 'ok') AS data_ok + SELECT + session, + source, + COUNTIF(outcome != 'transcribed') AS data_inputs, + COUNTIF(outcome = 'ok') AS data_ok, + COUNTIF(outcome = 'transcribed') AS transcriptions_offered, + COUNTIF(outcome NOT IN ('ok', 'transcribed')) AS data_refused FROM `anyplot.agent_telemetry.data_inputs` GROUP BY session, source ), @@ -46,10 +57,11 @@ runs AS ( SELECT session, source, - COUNT(*) AS runs, + COUNTIF(status != 'not_ready') AS runs, + COUNTIF(status = 'not_ready') AS not_ready_runs, COUNTIF(shipped) AS plots_shipped, MIN(IF(shipped, turn, NULL)) AS first_plot_turn, - COUNTIF(rounds > 1) AS repaired_runs + COUNTIF(status != 'not_ready' AND rounds > 1) AS repaired_runs FROM `anyplot.agent_telemetry.pipeline_runs` GROUP BY session, source ), @@ -69,8 +81,10 @@ SELECT TIMESTAMP_DIFF(spans.last_at, spans.first_at, SECOND) AS duration_s, IFNULL(inputs.data_inputs, 0) AS data_inputs, IFNULL(inputs.data_ok, 0) AS data_ok, + IFNULL(inputs.transcriptions_offered, 0) AS transcriptions_offered, IFNULL(turns.turns, 0) AS turns, IFNULL(runs.runs, 0) AS runs, + IFNULL(runs.not_ready_runs, 0) AS not_ready_runs, IFNULL(runs.plots_shipped, 0) AS plots_shipped, IFNULL(runs.repaired_runs, 0) AS repaired_runs, runs.first_plot_turn AS turns_to_first_plot, @@ -80,9 +94,12 @@ SELECT WHEN IFNULL(kept.keeps, 0) > 0 THEN 'kept' WHEN IFNULL(runs.plots_shipped, 0) > 0 THEN 'plot_shipped' WHEN IFNULL(runs.runs, 0) > 0 THEN 'plot_run' + WHEN IFNULL(runs.not_ready_runs, 0) > 0 THEN 'plot_not_ready' WHEN IFNULL(turns.turns, 0) > 0 THEN 'chatted' WHEN IFNULL(inputs.data_ok, 0) > 0 THEN 'data_ok' - ELSE 'data_refused' + WHEN IFNULL(inputs.transcriptions_offered, 0) > 0 THEN 'transcription_offered' + WHEN IFNULL(inputs.data_refused, 0) > 0 THEN 'data_refused' + ELSE 'outcome_only' END AS funnel_last_step, CASE WHEN spans.last_at > TIMESTAMP_SUB(CURRENT_TIMESTAMP(), INTERVAL 30 MINUTE) THEN 'active' diff --git a/agents/telemetry/views/04_usage_failures.sql b/agents/telemetry/views/04_usage_failures.sql index e149fb04100..5f21929b186 100644 --- a/agents/telemetry/views/04_usage_failures.sql +++ b/agents/telemetry/views/04_usage_failures.sql @@ -10,9 +10,9 @@ SELECT spec_id, library, COUNT(DISTINCT session) AS sessions, - COUNTIF(event = 'data_input') AS data_inputs, + COUNTIF(event = 'data_input' AND JSON_VALUE(payload.outcome) != 'transcribed') AS data_inputs, COUNTIF(event = 'chat_turn') AS turns, - COUNTIF(event = 'pipeline_run') AS runs, + COUNTIF(event = 'pipeline_run' AND JSON_VALUE(payload.status) != 'not_ready') AS runs, COUNTIF(event = 'pipeline_run' AND JSON_VALUE(payload.status) IN ('ok', 'needs_attention')) AS plots_shipped, COUNT(DISTINCT IF( event = 'outcome' @@ -27,6 +27,8 @@ GROUP BY day, source, spec_id, library; -- One row per day, spec, library and input format: the data inputs of that format and how they ended. Each -- input counts under its own format only; the session totals live in `usage`, never copied onto every format. +-- A transcription writes two rows (the offer, outcome `transcribed`, then the confirmed paste): `inputs` +-- counts it once, and `transcriptions_offered` counts the offers. CREATE OR REPLACE VIEW `anyplot.agent_telemetry.input_formats` AS SELECT DATE(ts) AS day, @@ -35,7 +37,8 @@ SELECT library, format, entry, - COUNT(*) AS inputs, + COUNTIF(outcome != 'transcribed') AS inputs, + COUNTIF(outcome = 'transcribed') AS transcriptions_offered, COUNTIF(outcome = 'ok') AS stored, COUNTIF(outcome NOT IN ('ok', 'transcribed')) AS refused, COUNT(DISTINCT session) AS sessions diff --git a/agents/telemetry/views/05_prices_cost.sql b/agents/telemetry/views/05_prices_cost.sql index 4b9f8eae312..4c15060954d 100644 --- a/agents/telemetry/views/05_prices_cost.sql +++ b/agents/telemetry/views/05_prices_cost.sql @@ -24,7 +24,7 @@ WITH steps AS ( SELECT turns.ts, turns.source, turns.session, turns.spec_id, turns.library, 'chat_turn' AS origin, JSON_VALUE(step.step) AS step, - REGEXP_EXTRACT(JSON_VALUE(step.model), r'^[^@]+') AS model, + REGEXP_REPLACE(JSON_VALUE(step.model), r'(@.*|-[0-9]{8})$', '') AS model, IFNULL(LAX_INT64(step.calls), 0) AS calls, IFNULL(LAX_INT64(step.input), 0) AS input_tokens, IFNULL(LAX_INT64(step.output), 0) AS output_tokens, @@ -34,7 +34,7 @@ WITH steps AS ( UNION ALL SELECT ts, source, session, spec_id, library, 'data_input', 'judge', - REGEXP_EXTRACT(judge_model, r'^[^@]+'), + REGEXP_REPLACE(judge_model, r'(@.*|-[0-9]{8})$', ''), IF(judge_input + judge_output > 0, 1, 0), judge_input, judge_output, 0, 0 FROM `anyplot.agent_telemetry.data_inputs` diff --git a/api/agent_usage.py b/api/agent_usage.py index ee0469b27ab..6d8dccc11a3 100644 --- a/api/agent_usage.py +++ b/api/agent_usage.py @@ -1,24 +1,35 @@ """Aggregates of the agent usage telemetry for the admin debug page (`GET /debug/agent-usage`). The agents service writes content-free telemetry lines (`agents/anyplot/telemetry.py`); a -Cloud Logging sink routes them into a log bucket that BigQuery reads through a linked dataset, -and the views in `agents/telemetry/views/*.sql` shape them (`docs/reference/agent-telemetry.md`). -This module runs one query against those views and answers aggregates only: counts, two rates -and the five most frequent specs, input formats and refusal codes, every key an id or a code -from a closed set. Live sessions only (`source = 'live'`): the eval harnesses tag theirs -`harness`. +Cloud Logging sink routes them into a log bucket that BigQuery reads through a linked dataset +(`docs/reference/agent-telemetry.md`). This module runs one query and answers aggregates only: +counts, two rates and the five most frequent specs, input formats and refusal codes, every key +an id or a code from a closed set. Live sessions only (`source = 'live'`): the eval harnesses +tag theirs `harness`. + +Cost. The query reads the linked log dataset `_logs` (`agent_telemetry_logs` for +`AGENT_TELEMETRY_DATASET=anyplot.agent_telemetry`) directly, not the `events` view: the view +deduplicates over the whole bucket with a window function, which no time filter can prune. The +query filters `timestamp` (the log view's partition column) to the window first and +deduplicates only those rows, so a load scans the window, never the whole 180 days. The window +starts at `since`, the current time floored to `WINDOW_STEP_S` minus the days, passed as a +query parameter: within one step the query text and its parameters stay byte-identical, so +BigQuery may answer from its result cache, and on top of that this module keeps each answer for +`CACHE_TTL_S` in the process (`_CACHE`), so reloading `/debug` runs no query at all. `AGENT_TELEMETRY_DATASET` unset answers `not_configured`; a query that fails (no table yet, no permission, a timeout) answers `unavailable`, logged with the exception class only. Both carry -zeros, so the page shows its empty state and never an error. +zeros, so the page shows its empty state and never an error. Neither is cached. """ from __future__ import annotations import logging import re -from datetime import UTC, datetime -from functools import lru_cache +import threading +import time +from collections.abc import Callable +from datetime import UTC, datetime, timedelta from typing import Any, Literal, Protocol from pydantic import BaseModel, Field @@ -30,8 +41,15 @@ USAGE_WINDOWS = (7, 30) TOP_ITEMS = 5 QUERY_TIMEOUT_S = 20.0 -DATASET_PATTERN = re.compile(r"^[a-z][a-z0-9-]{4,28}[a-z0-9]\.[A-Za-z_][A-Za-z0-9_]{0,1023}$") -"""`.`, checked before it goes into the SQL text: a dataset name cannot be a query parameter.""" +WINDOW_STEP_S = 600 +"""The window's start moves in steps of ten minutes, so repeated loads send the same query.""" +CACHE_TTL_S = 600.0 +"""How long an `ok` answer is kept in the process.""" +DATASET_PATTERN = re.compile(r"^[a-z][a-z0-9-]{4,28}[a-z0-9]\.[A-Za-z_][A-Za-z0-9_]{0,1018}$") +"""`.`, checked before it goes into the SQL text: a dataset name cannot be a query parameter. +The dataset leaves room for the `_logs` suffix of the linked log dataset (1,024 characters at most).""" +LOGS_SUFFIX = "_logs" +"""The linked log dataset is the views dataset's name plus this suffix (`docs/reference/agent-telemetry.md`).""" KEEP_ACTIONS = ("copied_code", "copied_image", "downloaded_png", "downloaded_code", "downloaded_data", "saved_session") @@ -49,7 +67,7 @@ class AgentUsage(BaseModel): kept_rate: float | None = None """Sessions whose user kept a result (copied, downloaded, saved) among the sessions with a shipped plot.""" repair_rate: float | None = None - """Pipeline runs that needed more than one round among all runs.""" + """Pipeline runs that needed more than one round among the runs that started (not the `not_ready` ones).""" top_specs: list[UsageItem] = Field(default_factory=list) top_formats: list[UsageItem] = Field(default_factory=list) top_refusals: list[UsageItem] = Field(default_factory=list) @@ -57,9 +75,9 @@ class AgentUsage(BaseModel): class UsageQuery(Protocol): - """Runs one standard-SQL query with an INT64 `@days` parameter and returns its rows as dicts.""" + """Runs one standard-SQL query with a TIMESTAMP `@since` parameter and returns its rows as dicts.""" - def rows(self, sql: str, days: int) -> list[dict[str, Any]]: ... + def rows(self, sql: str, since: datetime) -> list[dict[str, Any]]: ... BIGQUERY_SCOPE = "https://www.googleapis.com/auth/bigquery" @@ -87,7 +105,7 @@ def __init__(self, project: str) -> None: self._session = AuthorizedSession(credentials) self._url = QUERY_URL.format(project=project) - def rows(self, sql: str, days: int) -> list[dict[str, Any]]: + def rows(self, sql: str, since: datetime) -> list[dict[str, Any]]: body = { "query": sql, "useLegacySql": False, @@ -95,7 +113,11 @@ def rows(self, sql: str, days: int) -> list[dict[str, Any]]: "timeoutMs": int(QUERY_TIMEOUT_S * 1000), "parameterMode": "NAMED", "queryParameters": [ - {"name": "days", "parameterType": {"type": "INT64"}, "parameterValue": {"value": str(days)}} + { + "name": "since", + "parameterType": {"type": "TIMESTAMP"}, + "parameterValue": {"value": since.strftime("%Y-%m-%d %H:%M:%S+00:00")}, + } ], } response = self._session.post(self._url, json=body, timeout=QUERY_TIMEOUT_S + 10) @@ -137,17 +159,39 @@ def _scalar(field: dict[str, Any], value: Any) -> Any: return value +def window_start(days: int, now: datetime | None = None) -> datetime: + """The window's start: now, floored to `WINDOW_STEP_S`, minus `days`.""" + current = now or datetime.now(UTC) + floored = datetime.fromtimestamp(int(current.timestamp()) // WINDOW_STEP_S * WINDOW_STEP_S, UTC) + return floored - timedelta(days=days) + + def usage_sql(dataset: str) -> str: - """The one query behind the card, over the views of `dataset` (`.`).""" + """The one query behind the card: the window's lines of the linked log dataset, deduplicated, aggregated. + + The `timestamp` filter comes before the deduplication, so BigQuery prunes the log view's + partitions to the window. Transcription offers (`outcome: transcribed`) are left out of the + formats: the confirmed paste that follows carries the same format, so each input counts once. + `not_ready` runs (a plot asked for before the data was confirmed) count neither as shipped + nor in the repair rate's base. + """ keep = ", ".join(f"'{action}'" for action in KEEP_ACTIONS) return f""" WITH ev AS ( - SELECT * FROM `{dataset}.events` - WHERE source = 'live' AND ts >= TIMESTAMP_SUB(CURRENT_TIMESTAMP(), INTERVAL @days DAY) + SELECT + JSON_VALUE(json_payload.event) AS event, + JSON_VALUE(json_payload.session) AS session, + JSON_VALUE(json_payload.spec_id) AS spec_id, + json_payload AS payload + FROM `{dataset}{LOGS_SUFFIX}._AllLogs` + WHERE timestamp >= @since + AND JSON_VALUE(json_payload.log_type) = 'agent_telemetry' + AND JSON_VALUE(json_payload.source) = 'live' + QUALIFY ROW_NUMBER() OVER (PARTITION BY JSON_VALUE(json_payload.idempotency_key) ORDER BY timestamp) = 1 ), runs AS ( SELECT JSON_VALUE(payload.status) AS status, LAX_INT64(payload.rounds) AS rounds, session - FROM ev WHERE event = 'pipeline_run' + FROM ev WHERE event = 'pipeline_run' AND JSON_VALUE(payload.status) != 'not_ready' ), shipped_sessions AS ( SELECT DISTINCT session FROM runs WHERE status IN ('ok', 'needs_attention') @@ -173,7 +217,8 @@ def usage_sql(dataset: str) -> str: ) AS top_specs, ARRAY( SELECT AS STRUCT JSON_VALUE(payload.format) AS key, COUNT(*) AS count FROM ev - WHERE event = 'data_input' GROUP BY key ORDER BY count DESC, key LIMIT {TOP_ITEMS} + WHERE event = 'data_input' AND JSON_VALUE(payload.outcome) != 'transcribed' + GROUP BY key ORDER BY count DESC, key LIMIT {TOP_ITEMS} ) AS top_formats, ARRAY( SELECT AS STRUCT key, COUNT(*) AS count FROM refusals @@ -201,19 +246,39 @@ def _rate(value: Any) -> float | None: return round(float(value), 4) if isinstance(value, int | float) else None -def read_usage(query: UsageQuery | None, dataset: str | None, days: int) -> AgentUsage: - """The card's aggregates for the last `days` days; `not_configured` or `unavailable` with zeros otherwise.""" +_CACHE: dict[tuple[str, int], tuple[float, AgentUsage]] = {} +_CACHE_LOCK = threading.Lock() + + +def clear_usage_cache() -> None: + """Forget every kept answer (tests, and a dataset change at runtime).""" + with _CACHE_LOCK: + _CACHE.clear() + + +def read_usage( + query: UsageQuery | None, dataset: str | None, days: int, *, clock: Callable[[], float] = time.monotonic +) -> AgentUsage: + """The card's aggregates for the last `days` days; `not_configured` or `unavailable` with zeros otherwise. + + An `ok` answer is kept for `CACHE_TTL_S` per dataset and window; a failure is never kept, so the card + recovers on the next load once BigQuery answers. + """ if not dataset or not DATASET_PATTERN.fullmatch(dataset): return AgentUsage(status="not_configured", days=days, generated_at=_now()) + with _CACHE_LOCK: + kept = _CACHE.get((dataset, days)) + if kept is not None and clock() < kept[0]: + return kept[1] if query is None: # configured, but the client could not start (no credentials) return AgentUsage(status="unavailable", days=days, generated_at=_now()) try: - rows = query.rows(usage_sql(dataset), days) + rows = query.rows(usage_sql(dataset), window_start(days)) except Exception as exc: # no table yet, no permission, a timeout: the page shows its empty state logger.warning("agent usage query failed: %s", type(exc).__name__) return AgentUsage(status="unavailable", days=days, generated_at=_now()) row = rows[0] if rows else {} - return AgentUsage( + usage = AgentUsage( status="ok", days=days, sessions=int(row.get("sessions") or 0), @@ -225,16 +290,31 @@ def read_usage(query: UsageQuery | None, dataset: str | None, days: int) -> Agen top_refusals=_items(row.get("top_refusals")), generated_at=_now(), ) + with _CACHE_LOCK: + _CACHE[(dataset, days)] = (clock() + CACHE_TTL_S, usage) + return usage + + +_CLIENTS: dict[str, UsageQuery] = {} +_CLIENTS_LOCK = threading.Lock() -@lru_cache(maxsize=4) def usage_query(dataset: str | None) -> UsageQuery | None: - """The BigQuery client for `dataset`'s project, one per process, or None when the dataset is unset or the - client cannot start.""" + """The BigQuery client for `dataset`'s project, one per process once it started, or None when the dataset is + unset or the client cannot start. A failure is never kept: the next request tries again (credentials that + appear later, a transient metadata-server error).""" if not dataset or not DATASET_PATTERN.fullmatch(dataset): return None + project = dataset.split(".", 1)[0] + with _CLIENTS_LOCK: + client = _CLIENTS.get(project) + if client is not None: + return client try: - return BigQueryUsage(project=dataset.split(".", 1)[0]) + client = BigQueryUsage(project=project) except Exception as exc: # no credentials (a local run): the card's `unavailable` state logger.warning("BigQuery client unavailable: %s", type(exc).__name__) return None + with _CLIENTS_LOCK: + _CLIENTS.setdefault(project, client) + return client diff --git a/core/config.py b/core/config.py index 435d4af72f7..347c19e9ab9 100644 --- a/core/config.py +++ b/core/config.py @@ -308,7 +308,9 @@ def _parse_admin_allowed_emails(cls, value: Any) -> Any: agent_telemetry_dataset: str | None = None """`.` of the agent usage telemetry's BigQuery views (`agents/telemetry/views/*.sql`, `docs/reference/agent-telemetry.md`), such as - `anyplot.agent_telemetry`. Unset, `GET /debug/agent-usage` answers + `anyplot.agent_telemetry`; the card's query reads the linked log dataset of + the same name plus `_logs` (`anyplot.agent_telemetry_logs`). Unset or empty, + `GET /debug/agent-usage` answers `status: not_configured` with empty numbers. The API's runtime identity needs `roles/bigquery.dataViewer` on that dataset and on the linked log dataset the views read, and `roles/bigquery.jobUser` on the project.""" diff --git a/docs/reference/agent-telemetry.md b/docs/reference/agent-telemetry.md index 3d4e3537995..82b8d55f713 100644 --- a/docs/reference/agent-telemetry.md +++ b/docs/reference/agent-telemetry.md @@ -105,7 +105,7 @@ and dataset names written out. |---|---|---| | `01_events.sql` | `events` | One row per line, deduplicated by `idempotency_key` (`QUALIFY ROW_NUMBER()`), with the envelope as columns and the line as `payload` | | `02_event_views.sql` | `data_inputs`, `chat_turns`, `pipeline_runs`, `outcomes` | One typed column per field | -| `03_sessions.sql` | `sessions` | One row per session: data inputs, turns, runs, shipped plots, `turns_to_first_plot`, keeps, `funnel_last_step`, and `state` under a 30-minute inactivity rule (`active`, `kept`, `refined`, `abandoned_after_plot`, `abandoned_before_plot`) | +| `03_sessions.sql` | `sessions` | One row per session: data inputs, transcription offers, turns, runs, `not_ready` asks, shipped plots, `turns_to_first_plot`, keeps, `funnel_last_step` (from `outcome_only` and `data_refused` through `transcription_offered`, `data_ok`, `chatted`, `plot_not_ready`, `plot_run` and `plot_shipped` to `kept`), and `state` under a 30-minute inactivity rule (`active`, `kept`, `refined`, `abandoned_after_plot`, `abandoned_before_plot`). A transcription writes two `data_input` rows, the offer (outcome `transcribed`) and the confirmed paste; every view counts the input once and the offers apart. `not_ready` runs are never counted as runs or in the repair rate | | `04_usage_failures.sql` | `usage`, `input_formats`, `failures` | Usage by day, spec and library (each line under the library it was written under, so a library switch splits a session's runs correctly); data inputs by format and entry, each input under its own format; one row per failure with its kind and closed code | | `05_prices_cost.sql` | `model_prices`, `cost` | The `eu` list prices of `agents/evals/pricing.py` (a test keeps both equal) and the cost of every turn step and data-input judge call | @@ -216,10 +216,16 @@ Run these steps once, as a project owner. They write nothing but the resources n The admin page `/debug` shows an "agent usage" card for the last 7 or 30 days: sessions, plots shipped, the kept rate (sessions whose user kept a result, among the sessions with a shipped plot), the repair rate (runs that needed more than one round), and the five most frequent specs, input formats and refusal codes. -It reads `GET /debug/agent-usage?days=7|30` (`api/agent_usage.py`), one query over the `events` view of the -dataset that `AGENT_TELEMETRY_DATASET` names. The answer holds aggregates only. `status` is +It reads `GET /debug/agent-usage?days=7|30` (`api/agent_usage.py`). The API needs the linked log dataset to +be named like the views dataset plus `_logs` (`anyplot.agent_telemetry_logs` for +`AGENT_TELEMETRY_DATASET=anyplot.agent_telemetry`), because it queries the log view `_AllLogs` directly: it +filters `timestamp` to the window before it deduplicates, so a load scans the window's partitions only and +never the whole 180 days, which the `events` view, deduplicated over the whole bucket, cannot avoid. The +window starts at the current time floored to 10 minutes, passed as a query parameter, so repeated loads send +the same query and BigQuery may answer from its result cache; on top of that the API keeps each answer for 10 +minutes, so reloading `/debug` runs no query at all. The answer holds aggregates only. `status` is `not_configured` while the variable is unset, and `unavailable` when the query fails (no table yet, no -permission, a timeout); both come with zeros, and the card shows its empty state. +permission, a timeout); both come with zeros, the card shows its empty state, and a failure is never kept. ## Before the chat opens to the public diff --git a/tests/unit/agents/data/conftest.py b/tests/unit/agents/data/conftest.py new file mode 100644 index 00000000000..85ba4b05273 --- /dev/null +++ b/tests/unit/agents/data/conftest.py @@ -0,0 +1,29 @@ +"""Every warning the data tests make the parser write has a closed telemetry code. + +The usage telemetry logs warning codes, never sentences (`agents/anyplot/data/warning_codes.py`); +a reworded or new warning without its rule would silently become `other`. This fixture wraps +`ParsedDataset.__init__` for every test in this package, so each parse the parser test corpus +runs is checked, and fails the test that produced a sentence no rule knows. +""" + +from collections.abc import Iterator +from typing import Any + +import pytest + +from agents.anyplot.data.parse import ParsedDataset +from agents.anyplot.data.warning_codes import warning_code + + +@pytest.fixture(autouse=True) +def warnings_have_codes(monkeypatch: pytest.MonkeyPatch) -> Iterator[None]: + unknown: list[str] = [] + original = ParsedDataset.__init__ + + def checked(self: ParsedDataset, *args: Any, **kwargs: Any) -> None: + original(self, *args, **kwargs) + unknown.extend(line for line in self.warnings if warning_code(line) == "other") + + monkeypatch.setattr(ParsedDataset, "__init__", checked) + yield + assert unknown == [], f"warning sentences without a telemetry code (add a rule in warning_codes.py): {unknown}" diff --git a/tests/unit/agents/data/test_parse.py b/tests/unit/agents/data/test_parse.py index 86eabe337c7..0f85253defa 100644 --- a/tests/unit/agents/data/test_parse.py +++ b/tests/unit/agents/data/test_parse.py @@ -35,6 +35,7 @@ ) from agents.anyplot.data.prepare import Operation, apply_operations from agents.anyplot.data.roles import DataRole +from agents.anyplot.opening import NOTES_INSTRUCTION_WARNING, NOTES_REFUSED_WARNING, NOTES_UNCHECKED_WARNING from agents.anyplot.schemas import ( MAX_CELL_CHARS, MAX_COLUMN_NAME_CHARS, @@ -725,21 +726,23 @@ def test_the_notes_are_cut_at_the_chat_message_cap_with_a_warning(self) -> None: def test_without_context_drops_the_notes_and_puts_its_warning_first(self) -> None: parsed = parse_dataset(f"Title\n{OPEN_METEO}") - dropped = without_context(parsed, "the notes were left out") + dropped = without_context(parsed, NOTES_REFUSED_WARNING) assert dropped.context_lines == [] and dropped.csv == parsed.csv - assert dropped.warnings[0] == dropped.profile.warnings[0] == "the notes were left out" + assert dropped.warnings[0] == dropped.profile.warnings[0] == NOTES_REFUSED_WARNING assert dropped.warnings[1:] == parsed.warnings def test_without_context_keeps_the_warning_cap_and_the_omitted_count(self) -> None: parsed = parse_dataset("x,y\n1,2\n3,4\n5,6\n") - full = [f"w{index}" for index in range(MAX_WARNINGS - 1)] + ["7 more warnings omitted"] + # Real sentences (the data tests check that every warning has a telemetry code, conftest.py). + full = [f"column 'c{index}': spaces around the text removed in 1 cell" for index in range(MAX_WARNINGS - 1)] + full += ["7 more warnings omitted"] capped = replace(parsed, warnings=full, profile=parsed.profile.model_copy(update={"warnings": full})) - dropped = without_context(capped, "first") + dropped = without_context(capped, NOTES_INSTRUCTION_WARNING) assert len(dropped.warnings) == MAX_WARNINGS - assert dropped.warnings[0] == "first" and dropped.warnings[-1] == "8 more warnings omitted" + assert dropped.warnings[0] == NOTES_INSTRUCTION_WARNING and dropped.warnings[-1] == "8 more warnings omitted" class TestPythonLiterals: @@ -1683,10 +1686,10 @@ def test_only_complete_text_columns_of_two_rows_count(self, text: str) -> None: def test_refused_notes_bring_the_columns_back(self) -> None: """The notes judge refuses the notes: the values they carried come back as columns, never lost.""" - parsed = without_context(parse_dataset(OFFICE_EXPORT), "notes dropped") + parsed = without_context(parse_dataset(OFFICE_EXPORT), NOTES_UNCHECKED_WARNING) assert [column.name for column in parsed.profile.columns] == ["Year", "Region", "Origin", "Sex", "Population"] - assert parsed.context_lines == [] and parsed.warnings[0] == "notes dropped" + assert parsed.context_lines == [] and parsed.warnings[0] == NOTES_UNCHECKED_WARNING assert not any("moved to the notes" in warning for warning in parsed.warnings) assert read_back(parsed)["Region"].tolist() == ["Country"] * 4 diff --git a/tests/unit/agents/evals/test_telemetry_prices.py b/tests/unit/agents/evals/test_telemetry_prices.py index f36eb2a3f2e..37b733e7b8f 100644 --- a/tests/unit/agents/evals/test_telemetry_prices.py +++ b/tests/unit/agents/evals/test_telemetry_prices.py @@ -36,6 +36,41 @@ def test_the_cost_view_uses_the_cache_factors() -> None: assert f"input_usd_per_m * {CACHE_WRITE_FACTOR}" in text +@pytest.mark.parametrize("model", sorted(PRICES)) +@pytest.mark.parametrize("spelling", ["{}", "{}@20261001", "{}-20261001", "models/{}", " {} "]) +def test_every_spelling_of_a_priced_model_finds_its_row(model: str, spelling: str) -> None: + """The ids a provider reports (version suffixes, a `models/` prefix, case) reach the price row's key.""" + from agents.anyplot.telemetry import model_id + + assert model_id(spelling.format(model)) == model + assert model_id(spelling.format(model).upper()) == model + assert model in price_rows() + + +def test_a_turn_prices_each_step_at_its_model() -> None: + """Every step of a turn carries a priced model: the judges' too (they book none of their own), and two models + in one step stay two entries, each at its own price.""" + from agents.anyplot.plugins.ledger import StepUsage + from agents.anyplot.telemetry import step_tokens + + usage = { + "anyplot": StepUsage(calls=2, input=1000, output=100, model="claude-haiku-5-5@20261001"), + "adapter_matplotlib": StepUsage(calls=1, input=500, output=50, model="claude-haiku-5-5"), + "adapter_seaborn": StepUsage(calls=1, input=500, output=50, model="claude-sonnet-5-5"), + "reviewer": StepUsage(calls=1, input=300, output=30, model="gemini-3.8-flash"), + "judge": StepUsage(calls=3, input=90, output=9), + } + steps = step_tokens(usage, steps=("root", "adapter", "reviewer", "judge"), judge_model="gemini-3.5-flash-lite") + assert [(step["step"], step["model"]) for step in steps] == [ + ("root", "claude-haiku-5-5"), + ("adapter", "claude-haiku-5-5"), + ("adapter", "claude-sonnet-5-5"), + ("reviewer", "gemini-3.8-flash"), + ("judge", "gemini-3.5-flash-lite"), + ] + assert all(step["model"] in price_rows() for step in steps) + + def test_the_settings_models_are_priced() -> None: from agents.anyplot.settings import AgentSettings diff --git a/tests/unit/agents/runtime/test_telemetry.py b/tests/unit/agents/runtime/test_telemetry.py index 75bcace6aa0..21f8e2c7720 100644 --- a/tests/unit/agents/runtime/test_telemetry.py +++ b/tests/unit/agents/runtime/test_telemetry.py @@ -178,6 +178,18 @@ async def test_the_beacon_takes_closed_values_only(client: httpx.AsyncClient) -> assert (unknown.status_code, unknown.json()["detail"]) == (404, "not_found") +async def test_two_actions_never_share_a_key(client: httpx.AsyncClient, caplog: pytest.LogCaptureFixture) -> None: + """The key holds the action and the version besides the request id: even under one request id (a proxy that + reuses it) two different actions on one version stay two rows.""" + caplog.set_level(logging.INFO) + sid = await open_session(client) + for action in ("saved_session", "feedback"): + response = await client.post(f"/v1/sessions/{sid}/outcome", headers=HEADERS, json={"action": action}) + assert response.status_code == 204 + keys = [line["idempotency_key"] for line in telemetry_lines(caplog) if line["event"] == "outcome"] + assert len(keys) == 2 and keys[0] != keys[1] + + async def test_the_beacon_is_rate_limited_per_user(client: httpx.AsyncClient) -> None: sid = await open_session(client) url = f"/v1/sessions/{sid}/outcome" @@ -306,6 +318,17 @@ def test_a_rerun_node_keeps_its_run_index() -> None: telemetry.forget("sess-x") +def test_a_converted_file_kind_survives_a_refused_paste() -> None: + telemetry.forget("sess-z") + telemetry.converted("sess-z", kind="xlsx", sheets=3) + first = telemetry.InputFacts.for_paste("sess-z", size=10, source=True, confirmed=False) + retry = telemetry.InputFacts.for_paste("sess-z", size=10, source=True, confirmed=False) # after a 429 + assert (first.format, first.sheets) == (retry.format, retry.sheets) == ("xlsx", 3) + telemetry.clear_converted("sess-z") # the paste passed its limits + assert telemetry.InputFacts.for_paste("sess-z", size=10, source=True, confirmed=False).format is None + telemetry.forget("sess-z") + + def test_the_turn_marks_are_taken_once() -> None: telemetry.forget("sess-y") telemetry.marks("sess-y").undo = True @@ -393,6 +416,29 @@ def test_the_harness_tags_its_lines(caplog: pytest.LogCaptureFixture) -> None: assert after is not None and after["source"] == "live" +async def test_a_live_request_beside_a_harness_run_stays_live() -> None: + """The source is a context variable: a harness run in another task of the same loop tags only its own lines.""" + import asyncio + + ledger = RequestLedger(session_id="s1", spec_id="scatter-basic", library="matplotlib") + inside, release = asyncio.Event(), asyncio.Event() + records: list[dict[str, Any] | None] = [] + + async def harness_run() -> None: + with telemetry.harness(): + records.append(telemetry.write(telemetry.Outcome, ledger, action="copied_code")) + inside.set() + await release.wait() + + task = asyncio.create_task(harness_run()) + await inside.wait() + live = telemetry.write(telemetry.Outcome, ledger, action="copied_image") + release.set() + await task + assert records[0] is not None and records[0]["source"] == "harness" + assert live is not None and live["source"] == "live" + + def test_the_harness_source_is_restored_after_a_failure() -> None: ledger = RequestLedger(session_id="s1", spec_id="scatter-basic", library="matplotlib") with pytest.raises(RuntimeError), telemetry.harness(): diff --git a/tests/unit/api/test_agent_usage.py b/tests/unit/api/test_agent_usage.py index 276797ea8b8..254b72cc8d7 100644 --- a/tests/unit/api/test_agent_usage.py +++ b/tests/unit/api/test_agent_usage.py @@ -4,13 +4,25 @@ No test reaches BigQuery: `get_usage_query` is overridden with a fake that records the SQL and answers rows. """ +from collections.abc import Iterator +from datetime import UTC, datetime, timedelta from typing import Any from unittest.mock import patch import pytest from fastapi.testclient import TestClient -from api.agent_usage import DATASET_PATTERN, decode_rows, read_usage, usage_sql +from api import agent_usage +from api.agent_usage import ( + CACHE_TTL_S, + DATASET_PATTERN, + clear_usage_cache, + decode_rows, + read_usage, + usage_query, + usage_sql, + window_start, +) from api.main import app, fastapi_app from api.routers.debug import get_usage_query, require_admin from core.config import settings @@ -30,17 +42,24 @@ class FakeQuery: def __init__(self, rows: list[dict[str, Any]] | None = None, error: Exception | None = None) -> None: - self.calls: list[tuple[str, int]] = [] + self.calls: list[tuple[str, datetime]] = [] self._rows = rows if rows is not None else [ROW] self._error = error - def rows(self, sql: str, days: int) -> list[dict[str, Any]]: - self.calls.append((sql, days)) + def rows(self, sql: str, since: datetime) -> list[dict[str, Any]]: + self.calls.append((sql, since)) if self._error is not None: raise self._error return self._rows +@pytest.fixture(autouse=True) +def fresh_cache() -> Iterator[None]: + clear_usage_cache() + yield + clear_usage_cache() + + @pytest.fixture def fake() -> FakeQuery: return FakeQuery() @@ -66,11 +85,19 @@ def test_answers_the_aggregates(self, client, fake) -> None: assert body["top_specs"] == [{"key": "scatter-basic", "count": 5}, {"key": "bar-basic", "count": 3}] assert body["top_refusals"] == [{"key": "unparseable", "count": 3}] # a null key is dropped assert body["generated_at"].endswith("Z") - assert fake.calls[0][1] == 30 and f"`{DATASET}.events`" in fake.calls[0][0] + sql, since = fake.calls[0] + assert f"`{DATASET}_logs._AllLogs`" in sql + assert timedelta(days=30) <= datetime.now(UTC) - since < timedelta(days=30, minutes=11) def test_defaults_to_seven_days(self, client, fake) -> None: assert client.get("/debug/agent-usage").json()["days"] == 7 - assert fake.calls[0][1] == 7 + assert datetime.now(UTC) - fake.calls[0][1] < timedelta(days=7, minutes=11) + + def test_a_second_load_runs_no_query(self, client, fake) -> None: + client.get("/debug/agent-usage") + client.get("/debug/agent-usage") + client.get("/debug/agent-usage?days=30") + assert len(fake.calls) == 2 # one per window @pytest.mark.parametrize("days", ["8", "0", "365", "seven"]) def test_other_windows_are_refused(self, client, fake, days) -> None: @@ -102,6 +129,50 @@ def test_needs_the_admin_gate(self) -> None: assert response.status_code == 401 +class TestCache: + def test_an_answer_is_kept_for_the_ttl(self) -> None: + fake, now = FakeQuery(), [100.0] + read_usage(fake, DATASET, 7, clock=lambda: now[0]) + now[0] += CACHE_TTL_S - 1 + read_usage(fake, DATASET, 7, clock=lambda: now[0]) + assert len(fake.calls) == 1 + now[0] += 2 + read_usage(fake, DATASET, 7, clock=lambda: now[0]) + assert len(fake.calls) == 2 + + def test_a_failure_is_never_kept(self) -> None: + failing = FakeQuery(error=RuntimeError("timeout")) + assert read_usage(failing, DATASET, 7).status == "unavailable" + working = FakeQuery() + assert read_usage(working, DATASET, 7).status == "ok" and len(working.calls) == 1 + + def test_the_window_start_moves_in_ten_minute_steps(self) -> None: + a = window_start(7, datetime(2026, 10, 16, 12, 1, 5, tzinfo=UTC)) + b = window_start(7, datetime(2026, 10, 16, 12, 9, 59, tzinfo=UTC)) + c = window_start(7, datetime(2026, 10, 16, 12, 10, 0, tzinfo=UTC)) + assert a == b == datetime(2026, 10, 9, 12, 0, tzinfo=UTC) + assert c == datetime(2026, 10, 9, 12, 10, tzinfo=UTC) + + def test_a_client_that_failed_to_start_is_tried_again(self, monkeypatch: pytest.MonkeyPatch) -> None: + attempts: list[str] = [] + + class Flaky: + def __init__(self, project: str) -> None: + attempts.append(project) + if len(attempts) == 1: + raise RuntimeError("no credentials yet") + + def rows(self, sql: str, since: datetime) -> list[dict[str, Any]]: + return [] + + monkeypatch.setattr(agent_usage, "BigQueryUsage", Flaky) + monkeypatch.setattr(agent_usage, "_CLIENTS", {}) + assert usage_query(DATASET) is None + first = usage_query(DATASET) + assert first is not None and usage_query(DATASET) is first + assert attempts == ["anyplot", "anyplot"] + + class TestRestAnswer: """`decode_rows` reads BigQuery's `jobs.query` REST answer, where every scalar is a string.""" @@ -147,7 +218,7 @@ def test_types_the_cells_by_the_schema(self) -> None: def test_feeds_the_card(self) -> None: class Rest: - def rows(self, sql: str, days: int) -> list[dict[str, Any]]: + def rows(self, sql: str, since: datetime) -> list[dict[str, Any]]: return decode_rows(TestRestAnswer.ANSWER) usage = read_usage(Rest(), DATASET, 7) @@ -156,7 +227,7 @@ def rows(self, sql: str, days: int) -> list[dict[str, Any]]: def test_an_incomplete_job_is_unavailable(self) -> None: class Slow: - def rows(self, sql: str, days: int) -> list[dict[str, Any]]: + def rows(self, sql: str, since: datetime) -> list[dict[str, Any]]: return decode_rows({"jobComplete": False}) assert read_usage(Slow(), DATASET, 7).status == "unavailable" @@ -166,10 +237,16 @@ def test_no_rows_is_empty(self) -> None: class TestQuery: - def test_counts_live_sessions_only_and_takes_days_as_a_parameter(self) -> None: + def test_prunes_to_the_window_before_it_deduplicates(self) -> None: + sql = usage_sql(DATASET) + assert sql.index("timestamp >= @since") < sql.index("QUALIFY ROW_NUMBER()") + assert "CURRENT_TIMESTAMP" not in sql # the text stays byte-identical, so the result cache can answer + assert "'live'" in sql and "LIMIT 5" in sql + + def test_counts_an_input_once_and_leaves_not_ready_runs_out(self) -> None: sql = usage_sql(DATASET) - assert "source = 'live'" in sql and "INTERVAL @days DAY" in sql - assert "LIMIT 5" in sql + assert "JSON_VALUE(payload.outcome) != 'transcribed'" in sql + assert "JSON_VALUE(payload.status) != 'not_ready'" in sql @pytest.mark.parametrize("dataset", ["anyplot.agent_telemetry", "my-project-1.usage_views"]) def test_accepts_a_dataset_name(self, dataset) -> None: