feat(cli): emit anonymous usage events

docs/docs.json

@@ -64,7 +64,7 @@
         "groups": [
           { "group": "Start here", "pages": ["v6/start/index", "v6/start/quickstart", "v6/start/overview"] },
           { "group": "Guides", "pages": ["v6/guides/creating-an-environment", "v6/guides/running-an-eval", "v6/guides/training-agents"] },
-          { "group": "Reference", "pages": ["v6/reference/environment", "v6/reference/tasks", "v6/reference/capabilities", "v6/reference/agents", "v6/reference/runtime", "v6/reference/graders", "v6/reference/advice", "v6/reference/training", "v6/reference/types", "v6/reference/cli"] },
+          { "group": "Reference", "pages": ["v6/reference/environment", "v6/reference/tasks", "v6/reference/capabilities", "v6/reference/agents", "v6/reference/runtime", "v6/reference/graders", "v6/reference/advice", "v6/reference/training", "v6/reference/types", "v6/reference/cli", "v6/reference/telemetry"] },
           { "group": "Advanced", "pages": [
             { "group": "Advanced", "expanded": false, "pages": ["v6/advanced/extending", "v6/advanced/robots"] }
           ] },

docs/v6/reference/telemetry.mdx

@@ -0,0 +1,64 @@
+---
+title: "Telemetry and analytics"
+description: "What the HUD SDK sends and how to turn it off."
+icon: "satellite-dish"
+---
+
+The SDK sends eval trace telemetry and anonymous CLI usage analytics. They are
+independent and have separate controls.
+
+## Eval trace telemetry
+
+Disable trace uploads with:
+
+```bash
+hud set HUD_TELEMETRY_ENABLED=0
+```
+
+or set `HUD_TELEMETRY_ENABLED=0` in your environment.
+
+When you run evaluations with an API key configured, the SDK uploads trace
+spans (steps, tool calls, timings) to your team's traces on the HUD platform.
+This is the product feature that powers the trace viewer — it is scoped to
+your team and requires your API key.
+
+- **No API key (the default):** spans are not uploaded anywhere.
+- **`HUD_TELEMETRY_LOCAL_DIR=./spans`:** spans are also written to local
+  files. Works without an API key, and keeps working with telemetry disabled.
+- **Fully local:** `HUD_TELEMETRY_ENABLED=0` plus `HUD_TELEMETRY_LOCAL_DIR`
+  keeps every span on your machine and sends nothing.
+
+## Anonymous CLI usage events
+
+Disable CLI usage events with:
+
+```bash
+hud set HUD_CLI_ANALYTICS_ENABLED=0
+```
+
+or set `HUD_CLI_ANALYTICS_ENABLED=0` in your environment.
+
+Each CLI invocation sends one small event recording which command ran and
+how it exited, so HUD can improve the commands people actually use and fix
+the ones that fail. The payload is a fixed allowlist:
+
+| Field | Example |
+| --- | --- |
+| command, subcommand | `eval`, `models list` |
+| exit code and error class | `1`, `HudException` |
+| duration | `1500` (ms) |
+| SDK, Python version, OS | `0.6.13`, `3.12.9`, `darwin` |
+| CI flag | `true` in CI environments |
+| install id | random UUID, stored in `~/.hud/.env` |
+
+Never sent: command arguments, file paths or contents, error messages,
+environment variables, or API keys. A token from the command line is recorded
+only when it names a command registered on the CLI itself — anything else, a
+task file, a trace id, a typo, is recorded as `other` or omitted, so positional
+input is never transmitted. The install id is a random identifier for one
+machine — it is not tied to your account.
+
+Events are sent in the background and abandoned after about a second, so a
+slow or failed send can never break a command and adds at most that bound to
+its exit. A one-time notice is printed the first time the install id is
+created.

hud/cli/__init__.py

@@ -139,33 +139,36 @@ def main() -> None:
         console.print(f"HUD CLI version: [cyan]{__version__}[/cyan]")
         return
 
-    try:
-        if len(sys.argv) == 1 or (len(sys.argv) == 2 and sys.argv[1] in ["--help", "-h"]):
-            console.print(
-                Panel.fit(
-                    "[bold cyan]HUD CLI[/bold cyan]\nBuild, test, and deploy environments",
-                    border_style="cyan",
-                )
-            )
-            console.print("\n[yellow]Quick Start:[/yellow]")
-            console.print("  Run evaluations: [cyan]hud eval tasks.py claude[/cyan]\n")
+    from .utils.usage import recorded_invocation
 
-        app()
-    except typer.Exit as e:
+    with recorded_invocation(sys.argv):
         try:
-            exit_code = getattr(e, "exit_code", 0)
-        except Exception:
-            exit_code = 1
-        if exit_code != 0:
+            if len(sys.argv) == 1 or (len(sys.argv) == 2 and sys.argv[1] in ["--help", "-h"]):
+                console.print(
+                    Panel.fit(
+                        "[bold cyan]HUD CLI[/bold cyan]\nBuild, test, and deploy environments",
+                        border_style="cyan",
+                    )
+                )
+                console.print("\n[yellow]Quick Start:[/yellow]")
+                console.print("  Run evaluations: [cyan]hud eval tasks.py claude[/cyan]\n")
+
+            app()
+        except typer.Exit as e:
+            try:
+                exit_code = getattr(e, "exit_code", 0)
+            except Exception:
+                exit_code = 1
+            if exit_code != 0:
+                from hud.utils.hud_console import hud_console
+
+                hud_console.info(SUPPORT_HINT)
+            raise
+        except HudException as e:
             from hud.utils.hud_console import hud_console
 
-            hud_console.info(SUPPORT_HINT)
-        raise
-    except HudException as e:
-        from hud.utils.hud_console import hud_console
-
-        hud_console.render_exception(e)
-        raise typer.Exit(1) from e
+            hud_console.render_exception(e)
+            raise typer.Exit(1) from e
 
 
 if __name__ == "__main__":

hud/cli/tests/test_usage.py

@@ -0,0 +1,134 @@
+"""Tests for anonymous CLI usage events."""
+
+from __future__ import annotations
+
+import uuid
+from typing import TYPE_CHECKING, Any
+
+import pytest
+
+from hud.cli.utils import usage
+
+if TYPE_CHECKING:
+    from pathlib import Path
+
+
+class TestCommandTokens:
+    @pytest.fixture(autouse=True)
+    def _fresh_registry(self) -> None:
+        """Rebuild the cached registry so earlier tests cannot poison it."""
+        usage._registered_commands.cache_clear()
+
+    def test_arguments_are_never_captured(self) -> None:
+        """The second token of a plain command is user input and must not be sent."""
+        assert usage._command_tokens(["hud", "eval", "tasks.py", "claude"]) == ("eval", None)
+
+    def test_registered_subcommands_are_captured(self) -> None:
+        assert usage._command_tokens(["hud", "models", "list"]) == ("models", "list")
+
+    def test_callback_group_positionals_are_never_captured(self) -> None:
+        """``hud trace <id>`` and ``hud jobs <id>`` take user data, not subcommands."""
+        assert usage._command_tokens(["hud", "trace", "8b1f2c3d4e5f"]) == ("trace", None)
+        assert usage._command_tokens(["hud", "jobs", "0f9e8d7c"]) == ("jobs", None)
+
+    def test_unregistered_command_is_other(self) -> None:
+        """A token that is not a registered command is never sent verbatim."""
+        assert usage._command_tokens(["hud", "secret-name"]) == ("other", None)
+
+    def test_bare_invocation_is_help(self) -> None:
+        assert usage._command_tokens(["hud"]) == ("help", None)
+
+    def test_flags_are_skipped(self) -> None:
+        assert usage._command_tokens(["hud", "--verbose", "serve"]) == ("serve", None)
+
+
+class TestClassify:
+    def test_typer_exit_from_hud_exception_names_the_cause(self) -> None:
+        """The CLI converts HudException via ``raise typer.Exit(1) from e``."""
+        import typer
+
+        from hud.utils.exceptions import HudException
+
+        try:
+            try:
+                raise HudException("boom")
+            except HudException as e:
+                raise typer.Exit(1) from e
+        except typer.Exit as converted:
+            assert usage._classify(converted) == (1, "HudException")
+
+    def test_plain_exit_has_no_error_class(self) -> None:
+        import typer
+
+        assert usage._classify(typer.Exit(2)) == (2, None)
+
+    def test_keyboard_interrupt(self) -> None:
+        assert usage._classify(KeyboardInterrupt()) == (130, "KeyboardInterrupt")
+
+    def test_unexpected_exception(self) -> None:
+        assert usage._classify(ValueError("x")) == (1, "ValueError")
+
+
+class TestInstallId:
+    def test_created_once_and_persisted(
+        self, tmp_path: Path, monkeypatch: pytest.MonkeyPatch, capsys: pytest.CaptureFixture[str]
+    ) -> None:
+        """First call creates an id and prints the notice; later calls reuse it silently."""
+        monkeypatch.setattr("pathlib.Path.home", lambda: tmp_path)
+
+        first = usage._install_id()
+        second = usage._install_id()
+
+        assert first == second
+        assert uuid.UUID(first)
+        captured = capsys.readouterr()
+        assert captured.err.count("anonymous usage data") == 1
+
+
+class TestRecordInvocation:
+    def test_opt_out_applies_immediately(self, monkeypatch: pytest.MonkeyPatch) -> None:
+        monkeypatch.setenv("HUD_CLI_ANALYTICS_ENABLED", "0")
+        sent: list[dict[str, object]] = []
+        monkeypatch.setattr(usage, "_post", lambda _url, payload: sent.append(payload))
+
+        usage.record_invocation(["hud", "eval"], exit_code=0, error_class=None, duration_ms=10)
+
+        assert sent == []
+
+    def test_payload_is_the_allowlist(
+        self, tmp_path: Path, monkeypatch: pytest.MonkeyPatch
+    ) -> None:
+        """The sent payload holds command facts only — no argv, paths, or messages."""
+        monkeypatch.setattr("pathlib.Path.home", lambda: tmp_path)
+        monkeypatch.setenv("HUD_CLI_ANALYTICS_ENABLED", "1")
+        monkeypatch.setenv("HUD_TELEMETRY_URL", "https://telemetry.example.test/v3/api")
+        sent: list[tuple[str, dict[str, Any]]] = []
+        monkeypatch.setattr(usage, "_post", lambda url, payload: sent.append((url, payload)))
+
+        usage.record_invocation(
+            ["hud", "serve", "my_env.py"],
+            exit_code=1,
+            error_class="HudException",
+            duration_ms=42,
+        )
+
+        (url, payload) = sent[0]
+        assert url == "https://telemetry.example.test/v3/api/sdk-events/cli"
+        (event,) = payload["events"]
+        assert event["command"] == "serve"
+        assert event["subcommand"] is None  # my_env.py must not appear
+        assert event["exit_code"] == 1
+        assert event["error_class"] == "HudException"
+        assert "my_env.py" not in str(payload)
+        assert set(event) == {
+            "command",
+            "subcommand",
+            "exit_code",
+            "error_class",
+            "duration_ms",
+            "cli_version",
+            "python_version",
+            "os",
+            "is_ci",
+            "install_id",
+        }