From c61cee9bfc0659fefdcf8a8e468f836a3c12ecb4 Mon Sep 17 00:00:00 2001 From: shfunc Date: Wed, 19 Aug 2026 19:41:33 +0200 Subject: [PATCH 1/5] feat(cli): emit anonymous usage events --- docs/docs.json | 2 +- docs/v6/reference/telemetry.mdx | 53 +++++++++ hud/cli/__init__.py | 49 +++++---- hud/cli/tests/test_usage.py | 136 +++++++++++++++++++++++ hud/cli/utils/usage.py | 187 ++++++++++++++++++++++++++++++++ 5 files changed, 403 insertions(+), 24 deletions(-) create mode 100644 docs/v6/reference/telemetry.mdx create mode 100644 hud/cli/tests/test_usage.py create mode 100644 hud/cli/utils/usage.py diff --git a/docs/docs.json b/docs/docs.json index 2b990a3c6..7e63e5b40 100644 --- a/docs/docs.json +++ b/docs/docs.json @@ -64,7 +64,7 @@ "groups": [ { "group": "Start here", "pages": ["v6/start/index", "v6/start/quickstart", "v6/start/overview"] }, { "group": "Guides", "pages": ["v6/guides/creating-an-environment", "v6/guides/running-an-eval", "v6/guides/training-agents"] }, - { "group": "Reference", "pages": ["v6/reference/environment", "v6/reference/tasks", "v6/reference/capabilities", "v6/reference/agents", "v6/reference/runtime", "v6/reference/graders", "v6/reference/advice", "v6/reference/training", "v6/reference/types", "v6/reference/cli"] }, + { "group": "Reference", "pages": ["v6/reference/environment", "v6/reference/tasks", "v6/reference/capabilities", "v6/reference/agents", "v6/reference/runtime", "v6/reference/graders", "v6/reference/advice", "v6/reference/training", "v6/reference/types", "v6/reference/cli", "v6/reference/telemetry"] }, { "group": "Advanced", "pages": [ { "group": "Advanced", "expanded": false, "pages": ["v6/advanced/extending", "v6/advanced/robots"] } ] }, diff --git a/docs/v6/reference/telemetry.mdx b/docs/v6/reference/telemetry.mdx new file mode 100644 index 000000000..85e283233 --- /dev/null +++ b/docs/v6/reference/telemetry.mdx @@ -0,0 +1,53 @@ +--- +title: "Telemetry" +description: "What the HUD SDK sends, when, and how to turn it off." +icon: "satellite-dish" +--- + +The SDK sends two kinds of telemetry. Both are disabled by the same switch: + +```bash +hud set HUD_TELEMETRY_ENABLED=0 +``` + +or set `HUD_TELEMETRY_ENABLED=0` in your environment. + +## Eval trace telemetry + +When you run evaluations with an API key configured, the SDK uploads trace +spans (steps, tool calls, timings) to your team's traces on the HUD platform. +This is the product feature that powers the trace viewer — it is scoped to +your team and requires your API key. + +- **No API key (the default):** spans are not uploaded anywhere. +- **`HUD_TELEMETRY_LOCAL_DIR=./spans`:** spans are also written to local + files. Works without an API key, and keeps working with telemetry disabled. +- **Fully local:** `HUD_TELEMETRY_ENABLED=0` plus `HUD_TELEMETRY_LOCAL_DIR` + keeps every span on your machine and sends nothing. + +## Anonymous CLI usage events + +Each CLI invocation sends one small event recording which command ran and +how it exited, so HUD can improve the commands people actually use and fix +the ones that fail. The payload is a fixed allowlist: + +| Field | Example | +| --- | --- | +| command, subcommand | `eval`, `models list` | +| exit code and error class | `1`, `HudException` | +| duration | `1500` (ms) | +| SDK, Python version, OS | `0.6.13`, `3.12.9`, `darwin` | +| CI flag | `true` in CI environments | +| install id | random UUID, stored in `~/.hud/.env` | + +Never sent: command arguments, file paths or contents, error messages, +environment variables, or API keys. A token from the command line is recorded +only when it names a command registered on the CLI itself — anything else, a +task file, a trace id, a typo, is recorded as `other` or omitted, so positional +input is never transmitted. The install id is a random identifier for one +machine — it is not tied to your account. + +Events are sent in the background and abandoned after about a second, so a +slow or failed send can never break a command and adds at most that bound to +its exit. A one-time notice is printed the first time the install id is +created. diff --git a/hud/cli/__init__.py b/hud/cli/__init__.py index a228ae4d9..ae4089f72 100644 --- a/hud/cli/__init__.py +++ b/hud/cli/__init__.py @@ -139,33 +139,36 @@ def main() -> None: console.print(f"HUD CLI version: [cyan]{__version__}[/cyan]") return - try: - if len(sys.argv) == 1 or (len(sys.argv) == 2 and sys.argv[1] in ["--help", "-h"]): - console.print( - Panel.fit( - "[bold cyan]HUD CLI[/bold cyan]\nBuild, test, and deploy environments", - border_style="cyan", - ) - ) - console.print("\n[yellow]Quick Start:[/yellow]") - console.print(" Run evaluations: [cyan]hud eval tasks.py claude[/cyan]\n") + from .utils.usage import recorded_invocation - app() - except typer.Exit as e: + with recorded_invocation(sys.argv): try: - exit_code = getattr(e, "exit_code", 0) - except Exception: - exit_code = 1 - if exit_code != 0: + if len(sys.argv) == 1 or (len(sys.argv) == 2 and sys.argv[1] in ["--help", "-h"]): + console.print( + Panel.fit( + "[bold cyan]HUD CLI[/bold cyan]\nBuild, test, and deploy environments", + border_style="cyan", + ) + ) + console.print("\n[yellow]Quick Start:[/yellow]") + console.print(" Run evaluations: [cyan]hud eval tasks.py claude[/cyan]\n") + + app() + except typer.Exit as e: + try: + exit_code = getattr(e, "exit_code", 0) + except Exception: + exit_code = 1 + if exit_code != 0: + from hud.utils.hud_console import hud_console + + hud_console.info(SUPPORT_HINT) + raise + except HudException as e: from hud.utils.hud_console import hud_console - hud_console.info(SUPPORT_HINT) - raise - except HudException as e: - from hud.utils.hud_console import hud_console - - hud_console.render_exception(e) - raise typer.Exit(1) from e + hud_console.render_exception(e) + raise typer.Exit(1) from e if __name__ == "__main__": diff --git a/hud/cli/tests/test_usage.py b/hud/cli/tests/test_usage.py new file mode 100644 index 000000000..87bf89cb3 --- /dev/null +++ b/hud/cli/tests/test_usage.py @@ -0,0 +1,136 @@ +"""Tests for anonymous CLI usage events.""" + +from __future__ import annotations + +import uuid +from typing import TYPE_CHECKING, Any + +from hud.cli.utils import usage + +if TYPE_CHECKING: + from pathlib import Path + + import pytest + + +class TestCommandTokens: + def test_arguments_are_never_captured(self) -> None: + """The second token of a plain command is user input and must not be sent.""" + assert usage._command_tokens(["hud", "eval", "tasks.py", "claude"]) == ("eval", None) + + def test_registered_subcommands_are_captured(self) -> None: + assert usage._command_tokens(["hud", "models", "list"]) == ("models", "list") + + def test_callback_group_positionals_are_never_captured(self) -> None: + """``hud trace `` and ``hud jobs `` take user data, not subcommands.""" + assert usage._command_tokens(["hud", "trace", "8b1f2c3d4e5f"]) == ("trace", None) + assert usage._command_tokens(["hud", "jobs", "0f9e8d7c"]) == ("jobs", None) + + def test_unregistered_command_is_other(self) -> None: + """A token that is not a registered command is never sent verbatim.""" + assert usage._command_tokens(["hud", "secret-name"]) == ("other", None) + + def test_bare_invocation_is_help(self) -> None: + assert usage._command_tokens(["hud"]) == ("help", None) + + def test_flags_are_skipped(self) -> None: + assert usage._command_tokens(["hud", "--verbose", "serve"]) == ("serve", None) + + +class TestClassify: + def test_typer_exit_from_hud_exception_names_the_cause(self) -> None: + """The CLI converts HudException via ``raise typer.Exit(1) from e``.""" + import typer + + from hud.utils.exceptions import HudException + + try: + try: + raise HudException("boom") + except HudException as e: + raise typer.Exit(1) from e + except typer.Exit as converted: + assert usage._classify(converted) == (1, "HudException") + + def test_plain_exit_has_no_error_class(self) -> None: + import typer + + assert usage._classify(typer.Exit(2)) == (2, None) + + def test_keyboard_interrupt(self) -> None: + assert usage._classify(KeyboardInterrupt()) == (130, "KeyboardInterrupt") + + def test_unexpected_exception(self) -> None: + assert usage._classify(ValueError("x")) == (1, "ValueError") + + +class TestInstallId: + def test_created_once_and_persisted( + self, tmp_path: Path, monkeypatch: pytest.MonkeyPatch, capsys: pytest.CaptureFixture[str] + ) -> None: + """First call creates an id and prints the notice; later calls reuse it silently.""" + monkeypatch.setattr("pathlib.Path.home", lambda: tmp_path) + + first = usage._install_id() + second = usage._install_id() + + assert first == second + assert uuid.UUID(first) + captured = capsys.readouterr() + assert captured.err.count("anonymous usage data") == 1 + + +class TestRecordInvocation: + def test_disabled_sends_nothing(self, monkeypatch: pytest.MonkeyPatch) -> None: + from hud.settings import settings + + monkeypatch.setattr(settings, "telemetry_enabled", False) + + assert ( + usage.record_invocation( + ["hud", "eval"], exit_code=0, error_class=None, duration_ms=10 + ) + is None + ) + + def test_payload_is_the_allowlist( + self, tmp_path: Path, monkeypatch: pytest.MonkeyPatch + ) -> None: + """The sent payload holds command facts only — no argv, paths, or messages.""" + monkeypatch.setattr("pathlib.Path.home", lambda: tmp_path) + from hud.settings import settings + + monkeypatch.setattr(settings, "telemetry_enabled", True) + monkeypatch.setattr(settings, "hud_api_url", "https://api.example.test") + sent: list[tuple[str, dict[str, Any]]] = [] + monkeypatch.setattr(usage, "_post", lambda url, payload: sent.append((url, payload))) + + thread = usage.record_invocation( + ["hud", "serve", "my_env.py"], + exit_code=1, + error_class="HudException", + duration_ms=42, + ) + + assert thread is not None + thread.join(timeout=5) + (url, payload) = sent[0] + assert url == "https://api.example.test/v2/sdk-events/cli" + (event,) = payload["events"] + assert event["command"] == "serve" + assert event["subcommand"] is None # my_env.py must not appear + assert event["exit_code"] == 1 + assert event["error_class"] == "HudException" + assert "my_env.py" not in str(payload) + assert set(event) == { + "command", + "subcommand", + "exit_code", + "error_class", + "duration_ms", + "cli_version", + "python_version", + "os", + "is_ci", + "install_id", + } diff --git a/hud/cli/utils/usage.py b/hud/cli/utils/usage.py new file mode 100644 index 000000000..398445e16 --- /dev/null +++ b/hud/cli/utils/usage.py @@ -0,0 +1,187 @@ +"""Anonymous CLI usage events: command name, outcome, and environment facts. + +The payload is a strict allowlist — never arguments, paths, error messages, or +env values, which can carry secrets. A token from argv is recorded only when +it names a command registered on the CLI itself, so positional user input +(task files, trace ids) can never be transmitted. ``HUD_TELEMETRY_ENABLED=0`` +disables sending. +""" + +from __future__ import annotations + +import contextlib +import functools +import os +import sys +import threading +import time +import uuid +from typing import TYPE_CHECKING + +from .config import load_env_file, set_env_values + +if TYPE_CHECKING: + from collections.abc import Iterator + +_INSTALL_ID_KEY = "HUD_INSTALL_ID" +_JOIN_TIMEOUT_S = 1.5 + +_CI_ENV_VARS = ( + "CI", + "GITHUB_ACTIONS", + "GITLAB_CI", + "BUILDKITE", + "CIRCLECI", + "JENKINS_URL", + "TF_BUILD", +) + +_FIRST_RUN_NOTICE = ( + "hud collects anonymous usage data (command names and outcomes; never " + "arguments, file contents, or keys). Disable with: hud set HUD_TELEMETRY_ENABLED=0\n" + "Details: https://docs.hud.ai/v6/reference/telemetry" +) + + +def _is_ci() -> bool: + return any(os.environ.get(var) for var in _CI_ENV_VARS) + + +def _install_id() -> str: + """The persistent anonymous install id, created (with a notice) on first use.""" + env = load_env_file() + existing = env.get(_INSTALL_ID_KEY, "") + try: + return str(uuid.UUID(existing)) + except ValueError: + pass + created = str(uuid.uuid4()) + set_env_values({_INSTALL_ID_KEY: created}) + sys.stderr.write(_FIRST_RUN_NOTICE + "\n") + return created + + +@functools.cache +def _registered_commands() -> dict[str, frozenset[str]]: + """Registered top-level command names mapped to their subcommand names. + + Introspected from the CLI itself so the allowlist cannot drift from the + real command tree; groups invoked via callback (``hud trace ``) have no + subcommands, so their positional arguments never match. + """ + try: + import click + import typer + + from hud.cli import app + + group = typer.main.get_group(app) + return { + name: frozenset(command.commands) if isinstance(command, click.Group) else frozenset() + for name, command in group.commands.items() + } + except Exception: # conservative: unknown registry records nothing specific + return {} + + +def _command_tokens(argv: list[str]) -> tuple[str, str | None]: + words = [arg for arg in argv[1:] if not arg.startswith("-")] + if not words: + return "help", None + registry = _registered_commands() + if words[0] not in registry: + return "other", None + command = words[0] + subcommand = words[1] if len(words) > 1 and words[1] in registry[command] else None + return command, subcommand + + +def _classify(error: BaseException) -> tuple[int, str | None]: + """Map a propagating error to (exit_code, error_class). + + ``typer.Exit`` carries ``exit_code`` and ``SystemExit`` carries ``code``; + when either was raised ``from`` an original error (the CLI converts + ``HudException`` this way), the cause names the real error class. + """ + if isinstance(error, KeyboardInterrupt): + return 130, "KeyboardInterrupt" + exit_code = getattr(error, "exit_code", None) + if exit_code is None and isinstance(error, SystemExit): + exit_code = error.code if isinstance(error.code, int) else 1 + if isinstance(exit_code, int): + cause = error.__cause__ + return exit_code, type(cause).__name__ if cause is not None else None + return 1, type(error).__name__ + + +def _post(url: str, payload: dict[str, object]) -> None: + import httpx # lazy: keeps CLI startup light + + # Telemetry must never surface a failure. + with contextlib.suppress(Exception): + httpx.post(url, json=payload, timeout=httpx.Timeout(1.0, connect=0.5)) + + +def record_invocation( + argv: list[str], + *, + exit_code: int, + error_class: str | None, + duration_ms: int, +) -> threading.Thread | None: + """Send one usage event in a background thread; returns it for a bounded join. + + Returns ``None`` (and sends nothing) when telemetry is disabled. + """ + try: + from hud.settings import settings + + if not settings.telemetry_enabled: + return None + from hud import __version__ + + command, subcommand = _command_tokens(argv) + payload = { + "events": [ + { + "command": command, + "subcommand": subcommand, + "exit_code": exit_code, + "error_class": error_class, + "duration_ms": duration_ms, + "cli_version": __version__, + "python_version": ".".join(map(str, sys.version_info[:3])), + "os": sys.platform if sys.platform in ("linux", "darwin", "win32") else "other", + "is_ci": _is_ci(), + "install_id": _install_id(), + } + ] + } + url = f"{settings.hud_api_url.rstrip('/')}/v2/sdk-events/cli" + thread = threading.Thread(target=_post, args=(url, payload), daemon=True) + thread.start() + except Exception: # telemetry must never break a command + return None + return thread + + +@contextlib.contextmanager +def recorded_invocation(argv: list[str]) -> Iterator[None]: + """Record one CLI invocation around the wrapped block, then re-raise as-is.""" + started = time.monotonic() + exit_code = 0 + error_class: str | None = None + try: + yield + except BaseException as error: + exit_code, error_class = _classify(error) + raise + finally: + sender = record_invocation( + argv, + exit_code=exit_code, + error_class=error_class, + duration_ms=int((time.monotonic() - started) * 1000), + ) + if sender is not None: + sender.join(timeout=_JOIN_TIMEOUT_S) From 09c4b35eecdda67715fd006a253d63c681c61667 Mon Sep 17 00:00:00 2001 From: shfunc Date: Wed, 19 Aug 2026 20:19:23 +0200 Subject: [PATCH 2/5] fix(cli): re-read settings at send time so opt-out applies immediately --- hud/cli/tests/test_usage.py | 14 ++++++++------ hud/cli/utils/usage.py | 3 ++- 2 files changed, 10 insertions(+), 7 deletions(-) diff --git a/hud/cli/tests/test_usage.py b/hud/cli/tests/test_usage.py index 87bf89cb3..8e4b129d5 100644 --- a/hud/cli/tests/test_usage.py +++ b/hud/cli/tests/test_usage.py @@ -81,10 +81,14 @@ def test_created_once_and_persisted( class TestRecordInvocation: - def test_disabled_sends_nothing(self, monkeypatch: pytest.MonkeyPatch) -> None: + def test_opt_out_applies_immediately(self, monkeypatch: pytest.MonkeyPatch) -> None: + """The opt-out is re-read at send time, so it applies to the very + invocation that set it — even though the import-time settings + singleton still says enabled.""" from hud.settings import settings - monkeypatch.setattr(settings, "telemetry_enabled", False) + monkeypatch.setattr(settings, "telemetry_enabled", True) + monkeypatch.setenv("HUD_TELEMETRY_ENABLED", "0") assert ( usage.record_invocation( @@ -98,10 +102,8 @@ def test_payload_is_the_allowlist( ) -> None: """The sent payload holds command facts only — no argv, paths, or messages.""" monkeypatch.setattr("pathlib.Path.home", lambda: tmp_path) - from hud.settings import settings - - monkeypatch.setattr(settings, "telemetry_enabled", True) - monkeypatch.setattr(settings, "hud_api_url", "https://api.example.test") + monkeypatch.setenv("HUD_TELEMETRY_ENABLED", "1") + monkeypatch.setenv("HUD_API_URL", "https://api.example.test") sent: list[tuple[str, dict[str, Any]]] = [] monkeypatch.setattr(usage, "_post", lambda url, payload: sent.append((url, payload))) diff --git a/hud/cli/utils/usage.py b/hud/cli/utils/usage.py index 398445e16..895750991 100644 --- a/hud/cli/utils/usage.py +++ b/hud/cli/utils/usage.py @@ -134,8 +134,9 @@ def record_invocation( Returns ``None`` (and sends nothing) when telemetry is disabled. """ try: - from hud.settings import settings + from hud.settings import Settings + settings = Settings() if not settings.telemetry_enabled: return None from hud import __version__ From 1d1993cc84d60f5818a889298442ec2d4a6b7182 Mon Sep 17 00:00:00 2001 From: shfunc Date: Wed, 19 Aug 2026 20:49:21 +0200 Subject: [PATCH 3/5] fix(cli): harden usage-event tests and command introspection --- hud/cli/tests/test_usage.py | 13 ++++++++----- hud/cli/utils/usage.py | 5 +++-- hud/conftest.py | 1 + 3 files changed, 12 insertions(+), 7 deletions(-) diff --git a/hud/cli/tests/test_usage.py b/hud/cli/tests/test_usage.py index 8e4b129d5..d12d93ff8 100644 --- a/hud/cli/tests/test_usage.py +++ b/hud/cli/tests/test_usage.py @@ -5,15 +5,20 @@ import uuid from typing import TYPE_CHECKING, Any +import pytest + from hud.cli.utils import usage if TYPE_CHECKING: from pathlib import Path - import pytest - class TestCommandTokens: + @pytest.fixture(autouse=True) + def _fresh_registry(self) -> None: + """Rebuild the cached registry so earlier tests cannot poison it.""" + usage._registered_commands.cache_clear() + def test_arguments_are_never_captured(self) -> None: """The second token of a plain command is user input and must not be sent.""" assert usage._command_tokens(["hud", "eval", "tasks.py", "claude"]) == ("eval", None) @@ -91,9 +96,7 @@ def test_opt_out_applies_immediately(self, monkeypatch: pytest.MonkeyPatch) -> N monkeypatch.setenv("HUD_TELEMETRY_ENABLED", "0") assert ( - usage.record_invocation( - ["hud", "eval"], exit_code=0, error_class=None, duration_ms=10 - ) + usage.record_invocation(["hud", "eval"], exit_code=0, error_class=None, duration_ms=10) is None ) diff --git a/hud/cli/utils/usage.py b/hud/cli/utils/usage.py index 895750991..46951b1ad 100644 --- a/hud/cli/utils/usage.py +++ b/hud/cli/utils/usage.py @@ -70,14 +70,15 @@ def _registered_commands() -> dict[str, frozenset[str]]: subcommands, so their positional arguments never match. """ try: - import click import typer from hud.cli import app group = typer.main.get_group(app) + # Duck-typed on ``commands``: isinstance(click.Group) breaks when + # tests mock or reload click. return { - name: frozenset(command.commands) if isinstance(command, click.Group) else frozenset() + name: frozenset(getattr(command, "commands", {})) for name, command in group.commands.items() } except Exception: # conservative: unknown registry records nothing specific diff --git a/hud/conftest.py b/hud/conftest.py index f1fd5eb37..c26fce731 100644 --- a/hud/conftest.py +++ b/hud/conftest.py @@ -27,5 +27,6 @@ def _isolate_hud_settings(request: pytest.FixtureRequest) -> None: mp = pytest.MonkeyPatch() request.addfinalizer(mp.undo) mp.setattr(settings, "telemetry_enabled", False) + mp.setenv("HUD_TELEMETRY_ENABLED", "0") mp.setattr(settings, "api_key", None) mp.setattr(settings, "telemetry_local_dir", None) From 90c14106862ad818573ed7a7b7056dfbd54cf944 Mon Sep 17 00:00:00 2001 From: shfunc Date: Wed, 19 Aug 2026 21:25:09 +0200 Subject: [PATCH 4/5] refactor(cli): send usage events to the telemetry service --- hud/cli/tests/test_usage.py | 4 ++-- hud/cli/utils/usage.py | 2 +- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/hud/cli/tests/test_usage.py b/hud/cli/tests/test_usage.py index d12d93ff8..2aa09bb5a 100644 --- a/hud/cli/tests/test_usage.py +++ b/hud/cli/tests/test_usage.py @@ -106,7 +106,7 @@ def test_payload_is_the_allowlist( """The sent payload holds command facts only — no argv, paths, or messages.""" monkeypatch.setattr("pathlib.Path.home", lambda: tmp_path) monkeypatch.setenv("HUD_TELEMETRY_ENABLED", "1") - monkeypatch.setenv("HUD_API_URL", "https://api.example.test") + monkeypatch.setenv("HUD_TELEMETRY_URL", "https://t.example.test/v3/api") sent: list[tuple[str, dict[str, Any]]] = [] monkeypatch.setattr(usage, "_post", lambda url, payload: sent.append((url, payload))) @@ -120,7 +120,7 @@ def test_payload_is_the_allowlist( assert thread is not None thread.join(timeout=5) (url, payload) = sent[0] - assert url == "https://api.example.test/v2/sdk-events/cli" + assert url == "https://t.example.test/v3/api/v2/sdk-events/cli" (event,) = payload["events"] assert event["command"] == "serve" assert event["subcommand"] is None # my_env.py must not appear diff --git a/hud/cli/utils/usage.py b/hud/cli/utils/usage.py index 46951b1ad..777b4f9fb 100644 --- a/hud/cli/utils/usage.py +++ b/hud/cli/utils/usage.py @@ -159,7 +159,7 @@ def record_invocation( } ] } - url = f"{settings.hud_api_url.rstrip('/')}/v2/sdk-events/cli" + url = f"{settings.hud_telemetry_url.rstrip('/')}/v2/sdk-events/cli" thread = threading.Thread(target=_post, args=(url, payload), daemon=True) thread.start() except Exception: # telemetry must never break a command From c6b252f5a12e05396ab8076d9db14e775f04ed2b Mon Sep 17 00:00:00 2001 From: Jaideep <67646710+jdchawla29@users.noreply.github.com> Date: Wed, 19 Aug 2026 14:15:35 -0700 Subject: [PATCH 5/5] refactor(cli): separate usage analytics controls --- docs/v6/reference/telemetry.mdx | 21 ++++++++++---- hud/cli/tests/test_usage.py | 25 ++++++----------- hud/cli/utils/usage.py | 50 +++++++++++++-------------------- hud/conftest.py | 2 +- hud/settings.py | 6 ++++ hud/tests/test_settings.py | 9 ++++++ 6 files changed, 61 insertions(+), 52 deletions(-) diff --git a/docs/v6/reference/telemetry.mdx b/docs/v6/reference/telemetry.mdx index 85e283233..fb44eb938 100644 --- a/docs/v6/reference/telemetry.mdx +++ b/docs/v6/reference/telemetry.mdx @@ -1,10 +1,15 @@ --- -title: "Telemetry" -description: "What the HUD SDK sends, when, and how to turn it off." +title: "Telemetry and analytics" +description: "What the HUD SDK sends and how to turn it off." icon: "satellite-dish" --- -The SDK sends two kinds of telemetry. Both are disabled by the same switch: +The SDK sends eval trace telemetry and anonymous CLI usage analytics. They are +independent and have separate controls. + +## Eval trace telemetry + +Disable trace uploads with: ```bash hud set HUD_TELEMETRY_ENABLED=0 @@ -12,8 +17,6 @@ hud set HUD_TELEMETRY_ENABLED=0 or set `HUD_TELEMETRY_ENABLED=0` in your environment. -## Eval trace telemetry - When you run evaluations with an API key configured, the SDK uploads trace spans (steps, tool calls, timings) to your team's traces on the HUD platform. This is the product feature that powers the trace viewer — it is scoped to @@ -27,6 +30,14 @@ your team and requires your API key. ## Anonymous CLI usage events +Disable CLI usage events with: + +```bash +hud set HUD_CLI_ANALYTICS_ENABLED=0 +``` + +or set `HUD_CLI_ANALYTICS_ENABLED=0` in your environment. + Each CLI invocation sends one small event recording which command ran and how it exited, so HUD can improve the commands people actually use and fix the ones that fail. The payload is a fixed allowlist: diff --git a/hud/cli/tests/test_usage.py b/hud/cli/tests/test_usage.py index 2aa09bb5a..f44fbeed2 100644 --- a/hud/cli/tests/test_usage.py +++ b/hud/cli/tests/test_usage.py @@ -87,40 +87,33 @@ def test_created_once_and_persisted( class TestRecordInvocation: def test_opt_out_applies_immediately(self, monkeypatch: pytest.MonkeyPatch) -> None: - """The opt-out is re-read at send time, so it applies to the very - invocation that set it — even though the import-time settings - singleton still says enabled.""" - from hud.settings import settings + monkeypatch.setenv("HUD_CLI_ANALYTICS_ENABLED", "0") + sent: list[dict[str, object]] = [] + monkeypatch.setattr(usage, "_post", lambda _url, payload: sent.append(payload)) - monkeypatch.setattr(settings, "telemetry_enabled", True) - monkeypatch.setenv("HUD_TELEMETRY_ENABLED", "0") + usage.record_invocation(["hud", "eval"], exit_code=0, error_class=None, duration_ms=10) - assert ( - usage.record_invocation(["hud", "eval"], exit_code=0, error_class=None, duration_ms=10) - is None - ) + assert sent == [] def test_payload_is_the_allowlist( self, tmp_path: Path, monkeypatch: pytest.MonkeyPatch ) -> None: """The sent payload holds command facts only — no argv, paths, or messages.""" monkeypatch.setattr("pathlib.Path.home", lambda: tmp_path) - monkeypatch.setenv("HUD_TELEMETRY_ENABLED", "1") - monkeypatch.setenv("HUD_TELEMETRY_URL", "https://t.example.test/v3/api") + monkeypatch.setenv("HUD_CLI_ANALYTICS_ENABLED", "1") + monkeypatch.setenv("HUD_TELEMETRY_URL", "https://telemetry.example.test/v3/api") sent: list[tuple[str, dict[str, Any]]] = [] monkeypatch.setattr(usage, "_post", lambda url, payload: sent.append((url, payload))) - thread = usage.record_invocation( + usage.record_invocation( ["hud", "serve", "my_env.py"], exit_code=1, error_class="HudException", duration_ms=42, ) - assert thread is not None - thread.join(timeout=5) (url, payload) = sent[0] - assert url == "https://t.example.test/v3/api/v2/sdk-events/cli" + assert url == "https://telemetry.example.test/v3/api/sdk-events/cli" (event,) = payload["events"] assert event["command"] == "serve" assert event["subcommand"] is None # my_env.py must not appear diff --git a/hud/cli/utils/usage.py b/hud/cli/utils/usage.py index 777b4f9fb..c3067d239 100644 --- a/hud/cli/utils/usage.py +++ b/hud/cli/utils/usage.py @@ -3,8 +3,7 @@ The payload is a strict allowlist — never arguments, paths, error messages, or env values, which can carry secrets. A token from argv is recorded only when it names a command registered on the CLI itself, so positional user input -(task files, trace ids) can never be transmitted. ``HUD_TELEMETRY_ENABLED=0`` -disables sending. +(task files, trace ids) can never be transmitted. """ from __future__ import annotations @@ -38,7 +37,8 @@ _FIRST_RUN_NOTICE = ( "hud collects anonymous usage data (command names and outcomes; never " - "arguments, file contents, or keys). Disable with: hud set HUD_TELEMETRY_ENABLED=0\n" + "arguments, file contents, or keys). Disable with: " + "hud set HUD_CLI_ANALYTICS_ENABLED=0\n" "Details: https://docs.hud.ai/v6/reference/telemetry" ) @@ -69,20 +69,15 @@ def _registered_commands() -> dict[str, frozenset[str]]: real command tree; groups invoked via callback (``hud trace ``) have no subcommands, so their positional arguments never match. """ - try: - import typer + import typer - from hud.cli import app + from hud.cli import app - group = typer.main.get_group(app) - # Duck-typed on ``commands``: isinstance(click.Group) breaks when - # tests mock or reload click. - return { - name: frozenset(getattr(command, "commands", {})) - for name, command in group.commands.items() - } - except Exception: # conservative: unknown registry records nothing specific - return {} + group = typer.main.get_group(app) + return { + name: frozenset(getattr(command, "commands", {})) + for name, command in group.commands.items() + } def _command_tokens(argv: list[str]) -> tuple[str, str | None]: @@ -118,7 +113,7 @@ def _classify(error: BaseException) -> tuple[int, str | None]: def _post(url: str, payload: dict[str, object]) -> None: import httpx # lazy: keeps CLI startup light - # Telemetry must never surface a failure. + # Analytics must never surface a failure. with contextlib.suppress(Exception): httpx.post(url, json=payload, timeout=httpx.Timeout(1.0, connect=0.5)) @@ -129,17 +124,14 @@ def record_invocation( exit_code: int, error_class: str | None, duration_ms: int, -) -> threading.Thread | None: - """Send one usage event in a background thread; returns it for a bounded join. - - Returns ``None`` (and sends nothing) when telemetry is disabled. - """ +) -> None: + """Send one usage event without delaying CLI exit beyond the join timeout.""" try: from hud.settings import Settings settings = Settings() - if not settings.telemetry_enabled: - return None + if not settings.cli_analytics_enabled: + return from hud import __version__ command, subcommand = _command_tokens(argv) @@ -159,12 +151,12 @@ def record_invocation( } ] } - url = f"{settings.hud_telemetry_url.rstrip('/')}/v2/sdk-events/cli" + url = f"{settings.hud_telemetry_url.rstrip('/')}/sdk-events/cli" thread = threading.Thread(target=_post, args=(url, payload), daemon=True) thread.start() - except Exception: # telemetry must never break a command - return None - return thread + thread.join(timeout=_JOIN_TIMEOUT_S) + except Exception: + return @contextlib.contextmanager @@ -179,11 +171,9 @@ def recorded_invocation(argv: list[str]) -> Iterator[None]: exit_code, error_class = _classify(error) raise finally: - sender = record_invocation( + record_invocation( argv, exit_code=exit_code, error_class=error_class, duration_ms=int((time.monotonic() - started) * 1000), ) - if sender is not None: - sender.join(timeout=_JOIN_TIMEOUT_S) diff --git a/hud/conftest.py b/hud/conftest.py index c26fce731..e58852e02 100644 --- a/hud/conftest.py +++ b/hud/conftest.py @@ -27,6 +27,6 @@ def _isolate_hud_settings(request: pytest.FixtureRequest) -> None: mp = pytest.MonkeyPatch() request.addfinalizer(mp.undo) mp.setattr(settings, "telemetry_enabled", False) - mp.setenv("HUD_TELEMETRY_ENABLED", "0") + mp.setenv("HUD_CLI_ANALYTICS_ENABLED", "0") mp.setattr(settings, "api_key", None) mp.setattr(settings, "telemetry_local_dir", None) diff --git a/hud/settings.py b/hud/settings.py index 5eb64b6eb..27d896b32 100644 --- a/hud/settings.py +++ b/hud/settings.py @@ -134,6 +134,12 @@ def settings_customise_sources( validation_alias="HUD_TELEMETRY_ENABLED", ) + cli_analytics_enabled: bool = Field( + default=True, + description="Send anonymous HUD CLI usage analytics", + validation_alias="HUD_CLI_ANALYTICS_ENABLED", + ) + telemetry_local_dir: str | None = Field( default=None, description="If set, also write each telemetry span to /.jsonl " diff --git a/hud/tests/test_settings.py b/hud/tests/test_settings.py index 41e16f6b1..25db843d0 100644 --- a/hud/tests/test_settings.py +++ b/hud/tests/test_settings.py @@ -36,6 +36,15 @@ def test_file_tracking_can_be_disabled_by_env(monkeypatch): assert Settings().file_tracking_enabled is False +def test_cli_analytics_is_independent_of_trace_telemetry(monkeypatch): + monkeypatch.setenv("HUD_TELEMETRY_ENABLED", "true") + monkeypatch.setenv("HUD_CLI_ANALYTICS_ENABLED", "false") + configured = Settings() + + assert configured.telemetry_enabled is True + assert configured.cli_analytics_enabled is False + + def test_settings_singleton(): """Test that settings is a singleton.""" s1 = get_settings()