diff --git a/docs/docs.json b/docs/docs.json index 2b990a3c6..7e63e5b40 100644 --- a/docs/docs.json +++ b/docs/docs.json @@ -64,7 +64,7 @@ "groups": [ { "group": "Start here", "pages": ["v6/start/index", "v6/start/quickstart", "v6/start/overview"] }, { "group": "Guides", "pages": ["v6/guides/creating-an-environment", "v6/guides/running-an-eval", "v6/guides/training-agents"] }, - { "group": "Reference", "pages": ["v6/reference/environment", "v6/reference/tasks", "v6/reference/capabilities", "v6/reference/agents", "v6/reference/runtime", "v6/reference/graders", "v6/reference/advice", "v6/reference/training", "v6/reference/types", "v6/reference/cli"] }, + { "group": "Reference", "pages": ["v6/reference/environment", "v6/reference/tasks", "v6/reference/capabilities", "v6/reference/agents", "v6/reference/runtime", "v6/reference/graders", "v6/reference/advice", "v6/reference/training", "v6/reference/types", "v6/reference/cli", "v6/reference/telemetry"] }, { "group": "Advanced", "pages": [ { "group": "Advanced", "expanded": false, "pages": ["v6/advanced/extending", "v6/advanced/robots"] } ] }, diff --git a/docs/v6/reference/telemetry.mdx b/docs/v6/reference/telemetry.mdx new file mode 100644 index 000000000..fb44eb938 --- /dev/null +++ b/docs/v6/reference/telemetry.mdx @@ -0,0 +1,64 @@ +--- +title: "Telemetry and analytics" +description: "What the HUD SDK sends and how to turn it off." +icon: "satellite-dish" +--- + +The SDK sends eval trace telemetry and anonymous CLI usage analytics. They are +independent and have separate controls. + +## Eval trace telemetry + +Disable trace uploads with: + +```bash +hud set HUD_TELEMETRY_ENABLED=0 +``` + +or set `HUD_TELEMETRY_ENABLED=0` in your environment. + +When you run evaluations with an API key configured, the SDK uploads trace +spans (steps, tool calls, timings) to your team's traces on the HUD platform. +This is the product feature that powers the trace viewer — it is scoped to +your team and requires your API key. + +- **No API key (the default):** spans are not uploaded anywhere. +- **`HUD_TELEMETRY_LOCAL_DIR=./spans`:** spans are also written to local + files. Works without an API key, and keeps working with telemetry disabled. +- **Fully local:** `HUD_TELEMETRY_ENABLED=0` plus `HUD_TELEMETRY_LOCAL_DIR` + keeps every span on your machine and sends nothing. + +## Anonymous CLI usage events + +Disable CLI usage events with: + +```bash +hud set HUD_CLI_ANALYTICS_ENABLED=0 +``` + +or set `HUD_CLI_ANALYTICS_ENABLED=0` in your environment. + +Each CLI invocation sends one small event recording which command ran and +how it exited, so HUD can improve the commands people actually use and fix +the ones that fail. The payload is a fixed allowlist: + +| Field | Example | +| --- | --- | +| command, subcommand | `eval`, `models list` | +| exit code and error class | `1`, `HudException` | +| duration | `1500` (ms) | +| SDK, Python version, OS | `0.6.13`, `3.12.9`, `darwin` | +| CI flag | `true` in CI environments | +| install id | random UUID, stored in `~/.hud/.env` | + +Never sent: command arguments, file paths or contents, error messages, +environment variables, or API keys. A token from the command line is recorded +only when it names a command registered on the CLI itself — anything else, a +task file, a trace id, a typo, is recorded as `other` or omitted, so positional +input is never transmitted. The install id is a random identifier for one +machine — it is not tied to your account. + +Events are sent in the background and abandoned after about a second, so a +slow or failed send can never break a command and adds at most that bound to +its exit. A one-time notice is printed the first time the install id is +created. diff --git a/hud/cli/__init__.py b/hud/cli/__init__.py index a228ae4d9..ae4089f72 100644 --- a/hud/cli/__init__.py +++ b/hud/cli/__init__.py @@ -139,33 +139,36 @@ def main() -> None: console.print(f"HUD CLI version: [cyan]{__version__}[/cyan]") return - try: - if len(sys.argv) == 1 or (len(sys.argv) == 2 and sys.argv[1] in ["--help", "-h"]): - console.print( - Panel.fit( - "[bold cyan]HUD CLI[/bold cyan]\nBuild, test, and deploy environments", - border_style="cyan", - ) - ) - console.print("\n[yellow]Quick Start:[/yellow]") - console.print(" Run evaluations: [cyan]hud eval tasks.py claude[/cyan]\n") + from .utils.usage import recorded_invocation - app() - except typer.Exit as e: + with recorded_invocation(sys.argv): try: - exit_code = getattr(e, "exit_code", 0) - except Exception: - exit_code = 1 - if exit_code != 0: + if len(sys.argv) == 1 or (len(sys.argv) == 2 and sys.argv[1] in ["--help", "-h"]): + console.print( + Panel.fit( + "[bold cyan]HUD CLI[/bold cyan]\nBuild, test, and deploy environments", + border_style="cyan", + ) + ) + console.print("\n[yellow]Quick Start:[/yellow]") + console.print(" Run evaluations: [cyan]hud eval tasks.py claude[/cyan]\n") + + app() + except typer.Exit as e: + try: + exit_code = getattr(e, "exit_code", 0) + except Exception: + exit_code = 1 + if exit_code != 0: + from hud.utils.hud_console import hud_console + + hud_console.info(SUPPORT_HINT) + raise + except HudException as e: from hud.utils.hud_console import hud_console - hud_console.info(SUPPORT_HINT) - raise - except HudException as e: - from hud.utils.hud_console import hud_console - - hud_console.render_exception(e) - raise typer.Exit(1) from e + hud_console.render_exception(e) + raise typer.Exit(1) from e if __name__ == "__main__": diff --git a/hud/cli/tests/test_usage.py b/hud/cli/tests/test_usage.py new file mode 100644 index 000000000..f44fbeed2 --- /dev/null +++ b/hud/cli/tests/test_usage.py @@ -0,0 +1,134 @@ +"""Tests for anonymous CLI usage events.""" + +from __future__ import annotations + +import uuid +from typing import TYPE_CHECKING, Any + +import pytest + +from hud.cli.utils import usage + +if TYPE_CHECKING: + from pathlib import Path + + +class TestCommandTokens: + @pytest.fixture(autouse=True) + def _fresh_registry(self) -> None: + """Rebuild the cached registry so earlier tests cannot poison it.""" + usage._registered_commands.cache_clear() + + def test_arguments_are_never_captured(self) -> None: + """The second token of a plain command is user input and must not be sent.""" + assert usage._command_tokens(["hud", "eval", "tasks.py", "claude"]) == ("eval", None) + + def test_registered_subcommands_are_captured(self) -> None: + assert usage._command_tokens(["hud", "models", "list"]) == ("models", "list") + + def test_callback_group_positionals_are_never_captured(self) -> None: + """``hud trace `` and ``hud jobs `` take user data, not subcommands.""" + assert usage._command_tokens(["hud", "trace", "8b1f2c3d4e5f"]) == ("trace", None) + assert usage._command_tokens(["hud", "jobs", "0f9e8d7c"]) == ("jobs", None) + + def test_unregistered_command_is_other(self) -> None: + """A token that is not a registered command is never sent verbatim.""" + assert usage._command_tokens(["hud", "secret-name"]) == ("other", None) + + def test_bare_invocation_is_help(self) -> None: + assert usage._command_tokens(["hud"]) == ("help", None) + + def test_flags_are_skipped(self) -> None: + assert usage._command_tokens(["hud", "--verbose", "serve"]) == ("serve", None) + + +class TestClassify: + def test_typer_exit_from_hud_exception_names_the_cause(self) -> None: + """The CLI converts HudException via ``raise typer.Exit(1) from e``.""" + import typer + + from hud.utils.exceptions import HudException + + try: + try: + raise HudException("boom") + except HudException as e: + raise typer.Exit(1) from e + except typer.Exit as converted: + assert usage._classify(converted) == (1, "HudException") + + def test_plain_exit_has_no_error_class(self) -> None: + import typer + + assert usage._classify(typer.Exit(2)) == (2, None) + + def test_keyboard_interrupt(self) -> None: + assert usage._classify(KeyboardInterrupt()) == (130, "KeyboardInterrupt") + + def test_unexpected_exception(self) -> None: + assert usage._classify(ValueError("x")) == (1, "ValueError") + + +class TestInstallId: + def test_created_once_and_persisted( + self, tmp_path: Path, monkeypatch: pytest.MonkeyPatch, capsys: pytest.CaptureFixture[str] + ) -> None: + """First call creates an id and prints the notice; later calls reuse it silently.""" + monkeypatch.setattr("pathlib.Path.home", lambda: tmp_path) + + first = usage._install_id() + second = usage._install_id() + + assert first == second + assert uuid.UUID(first) + captured = capsys.readouterr() + assert captured.err.count("anonymous usage data") == 1 + + +class TestRecordInvocation: + def test_opt_out_applies_immediately(self, monkeypatch: pytest.MonkeyPatch) -> None: + monkeypatch.setenv("HUD_CLI_ANALYTICS_ENABLED", "0") + sent: list[dict[str, object]] = [] + monkeypatch.setattr(usage, "_post", lambda _url, payload: sent.append(payload)) + + usage.record_invocation(["hud", "eval"], exit_code=0, error_class=None, duration_ms=10) + + assert sent == [] + + def test_payload_is_the_allowlist( + self, tmp_path: Path, monkeypatch: pytest.MonkeyPatch + ) -> None: + """The sent payload holds command facts only — no argv, paths, or messages.""" + monkeypatch.setattr("pathlib.Path.home", lambda: tmp_path) + monkeypatch.setenv("HUD_CLI_ANALYTICS_ENABLED", "1") + monkeypatch.setenv("HUD_TELEMETRY_URL", "https://telemetry.example.test/v3/api") + sent: list[tuple[str, dict[str, Any]]] = [] + monkeypatch.setattr(usage, "_post", lambda url, payload: sent.append((url, payload))) + + usage.record_invocation( + ["hud", "serve", "my_env.py"], + exit_code=1, + error_class="HudException", + duration_ms=42, + ) + + (url, payload) = sent[0] + assert url == "https://telemetry.example.test/v3/api/sdk-events/cli" + (event,) = payload["events"] + assert event["command"] == "serve" + assert event["subcommand"] is None # my_env.py must not appear + assert event["exit_code"] == 1 + assert event["error_class"] == "HudException" + assert "my_env.py" not in str(payload) + assert set(event) == { + "command", + "subcommand", + "exit_code", + "error_class", + "duration_ms", + "cli_version", + "python_version", + "os", + "is_ci", + "install_id", + } diff --git a/hud/cli/utils/usage.py b/hud/cli/utils/usage.py new file mode 100644 index 000000000..c3067d239 --- /dev/null +++ b/hud/cli/utils/usage.py @@ -0,0 +1,179 @@ +"""Anonymous CLI usage events: command name, outcome, and environment facts. + +The payload is a strict allowlist — never arguments, paths, error messages, or +env values, which can carry secrets. A token from argv is recorded only when +it names a command registered on the CLI itself, so positional user input +(task files, trace ids) can never be transmitted. +""" + +from __future__ import annotations + +import contextlib +import functools +import os +import sys +import threading +import time +import uuid +from typing import TYPE_CHECKING + +from .config import load_env_file, set_env_values + +if TYPE_CHECKING: + from collections.abc import Iterator + +_INSTALL_ID_KEY = "HUD_INSTALL_ID" +_JOIN_TIMEOUT_S = 1.5 + +_CI_ENV_VARS = ( + "CI", + "GITHUB_ACTIONS", + "GITLAB_CI", + "BUILDKITE", + "CIRCLECI", + "JENKINS_URL", + "TF_BUILD", +) + +_FIRST_RUN_NOTICE = ( + "hud collects anonymous usage data (command names and outcomes; never " + "arguments, file contents, or keys). Disable with: " + "hud set HUD_CLI_ANALYTICS_ENABLED=0\n" + "Details: https://docs.hud.ai/v6/reference/telemetry" +) + + +def _is_ci() -> bool: + return any(os.environ.get(var) for var in _CI_ENV_VARS) + + +def _install_id() -> str: + """The persistent anonymous install id, created (with a notice) on first use.""" + env = load_env_file() + existing = env.get(_INSTALL_ID_KEY, "") + try: + return str(uuid.UUID(existing)) + except ValueError: + pass + created = str(uuid.uuid4()) + set_env_values({_INSTALL_ID_KEY: created}) + sys.stderr.write(_FIRST_RUN_NOTICE + "\n") + return created + + +@functools.cache +def _registered_commands() -> dict[str, frozenset[str]]: + """Registered top-level command names mapped to their subcommand names. + + Introspected from the CLI itself so the allowlist cannot drift from the + real command tree; groups invoked via callback (``hud trace ``) have no + subcommands, so their positional arguments never match. + """ + import typer + + from hud.cli import app + + group = typer.main.get_group(app) + return { + name: frozenset(getattr(command, "commands", {})) + for name, command in group.commands.items() + } + + +def _command_tokens(argv: list[str]) -> tuple[str, str | None]: + words = [arg for arg in argv[1:] if not arg.startswith("-")] + if not words: + return "help", None + registry = _registered_commands() + if words[0] not in registry: + return "other", None + command = words[0] + subcommand = words[1] if len(words) > 1 and words[1] in registry[command] else None + return command, subcommand + + +def _classify(error: BaseException) -> tuple[int, str | None]: + """Map a propagating error to (exit_code, error_class). + + ``typer.Exit`` carries ``exit_code`` and ``SystemExit`` carries ``code``; + when either was raised ``from`` an original error (the CLI converts + ``HudException`` this way), the cause names the real error class. + """ + if isinstance(error, KeyboardInterrupt): + return 130, "KeyboardInterrupt" + exit_code = getattr(error, "exit_code", None) + if exit_code is None and isinstance(error, SystemExit): + exit_code = error.code if isinstance(error.code, int) else 1 + if isinstance(exit_code, int): + cause = error.__cause__ + return exit_code, type(cause).__name__ if cause is not None else None + return 1, type(error).__name__ + + +def _post(url: str, payload: dict[str, object]) -> None: + import httpx # lazy: keeps CLI startup light + + # Analytics must never surface a failure. + with contextlib.suppress(Exception): + httpx.post(url, json=payload, timeout=httpx.Timeout(1.0, connect=0.5)) + + +def record_invocation( + argv: list[str], + *, + exit_code: int, + error_class: str | None, + duration_ms: int, +) -> None: + """Send one usage event without delaying CLI exit beyond the join timeout.""" + try: + from hud.settings import Settings + + settings = Settings() + if not settings.cli_analytics_enabled: + return + from hud import __version__ + + command, subcommand = _command_tokens(argv) + payload = { + "events": [ + { + "command": command, + "subcommand": subcommand, + "exit_code": exit_code, + "error_class": error_class, + "duration_ms": duration_ms, + "cli_version": __version__, + "python_version": ".".join(map(str, sys.version_info[:3])), + "os": sys.platform if sys.platform in ("linux", "darwin", "win32") else "other", + "is_ci": _is_ci(), + "install_id": _install_id(), + } + ] + } + url = f"{settings.hud_telemetry_url.rstrip('/')}/sdk-events/cli" + thread = threading.Thread(target=_post, args=(url, payload), daemon=True) + thread.start() + thread.join(timeout=_JOIN_TIMEOUT_S) + except Exception: + return + + +@contextlib.contextmanager +def recorded_invocation(argv: list[str]) -> Iterator[None]: + """Record one CLI invocation around the wrapped block, then re-raise as-is.""" + started = time.monotonic() + exit_code = 0 + error_class: str | None = None + try: + yield + except BaseException as error: + exit_code, error_class = _classify(error) + raise + finally: + record_invocation( + argv, + exit_code=exit_code, + error_class=error_class, + duration_ms=int((time.monotonic() - started) * 1000), + ) diff --git a/hud/conftest.py b/hud/conftest.py index f1fd5eb37..e58852e02 100644 --- a/hud/conftest.py +++ b/hud/conftest.py @@ -27,5 +27,6 @@ def _isolate_hud_settings(request: pytest.FixtureRequest) -> None: mp = pytest.MonkeyPatch() request.addfinalizer(mp.undo) mp.setattr(settings, "telemetry_enabled", False) + mp.setenv("HUD_CLI_ANALYTICS_ENABLED", "0") mp.setattr(settings, "api_key", None) mp.setattr(settings, "telemetry_local_dir", None) diff --git a/hud/settings.py b/hud/settings.py index 5eb64b6eb..27d896b32 100644 --- a/hud/settings.py +++ b/hud/settings.py @@ -134,6 +134,12 @@ def settings_customise_sources( validation_alias="HUD_TELEMETRY_ENABLED", ) + cli_analytics_enabled: bool = Field( + default=True, + description="Send anonymous HUD CLI usage analytics", + validation_alias="HUD_CLI_ANALYTICS_ENABLED", + ) + telemetry_local_dir: str | None = Field( default=None, description="If set, also write each telemetry span to /.jsonl " diff --git a/hud/tests/test_settings.py b/hud/tests/test_settings.py index 41e16f6b1..25db843d0 100644 --- a/hud/tests/test_settings.py +++ b/hud/tests/test_settings.py @@ -36,6 +36,15 @@ def test_file_tracking_can_be_disabled_by_env(monkeypatch): assert Settings().file_tracking_enabled is False +def test_cli_analytics_is_independent_of_trace_telemetry(monkeypatch): + monkeypatch.setenv("HUD_TELEMETRY_ENABLED", "true") + monkeypatch.setenv("HUD_CLI_ANALYTICS_ENABLED", "false") + configured = Settings() + + assert configured.telemetry_enabled is True + assert configured.cli_analytics_enabled is False + + def test_settings_singleton(): """Test that settings is a singleton.""" s1 = get_settings()