From 1c3b0634f6c378f3741f697358bf2401bc29704a Mon Sep 17 00:00:00 2001
From: FlopBut
Date: Sun, 15 Nov 2026 14:22:47 +0100
Subject: [PATCH 1/3] feat(auth): ingest keys authorize, actor tokens identify
(schema v12)
Paperclip v2026.1106 strips every agent's OTEL_* env before each run and
applies one company-wide OTLP destination and header set, so a per-agent
Authorization header can no longer reach cotel; agent telemetry has been
dark since 2026-11-07. Authorization and identity are now separate
(ADR-0028):
- Ingest keys (cotel_ik_<64 hex>): shared credentials created and revoked
in Setup -> Settings, stored as sha256 with a 12-char display prefix,
shown once. They authorize /v1/traces and /v1/logs only - never export,
import or the dashboard, since agents can read them from their env.
- Actor tokens are unchanged and not reissued. With an ingest key they
travel in X-Cotel-Actor-Token; a valid key without (or with an unknown)
actor token is stored as Anonymous even when allow_anonymous is off,
because the request is authorized. Authorization: Bearer
keeps working as before.
- No DB write per request: the key check is a read.
- /v1/logs accepts and discards OTLP logs so Paperclip's log exporter does
not 404.
- scripts/otel-headers-helper.py, for Claude Code's otelHeadersHelper,
emits the full header set: the Authorization Paperclip set plus
X-Cotel-Actor-Token from COTEL_ACTOR_TOKEN; it always prints valid JSON.
- scripts/agent-set-cotel-token.sh now writes COTEL_ACTOR_TOKEN instead of
the per-agent OTEL_EXPORTER_OTLP_HEADERS Paperclip strips.
- Actor page says "Actor token" and shows both ways to send it.
Co-Authored-By: Claude Opus 5.5
---
.github/workflows/health-probe.yml | 4 +
CHANGELOG.md | 1 +
README.md | 2 +
cmd/cotel/main.go | 5 +-
docs/.vitepress/config.js | 1 +
.../0028-ingest-keys-and-actor-tokens.md | 122 ++++++++
docs/decisions/index.md | 1 +
docs/operations/actors-and-auth.md | 130 +++++++--
docs/operations/api-reference.md | 65 ++++-
frontend/src/api/index.ts | 52 ++++
frontend/src/pages/ActorDetail.module.css | 29 +-
frontend/src/pages/ActorDetail.tsx | 16 +-
frontend/src/pages/Setup.module.css | 203 ++++++++++++++
frontend/src/pages/Setup.tsx | 260 +++++++++++++++++-
internal/api/auth/ingest_middleware_test.go | 199 ++++++++++++++
internal/api/auth/middleware.go | 67 ++++-
internal/api/handler.go | 5 +
internal/api/ingest_keys.go | 130 +++++++++
internal/api/ingest_keys_test.go | 167 +++++++++++
internal/ingest/logs.go | 30 ++
internal/ingest/logs_test.go | 45 +++
internal/storage/actors_migration_test.go | 4 +-
internal/storage/ingest_keys.go | 117 ++++++++
internal/storage/ingest_keys_test.go | 255 +++++++++++++++++
internal/storage/schema.sql | 16 +-
internal/storage/testdata/schema_v11.sql | 165 +++++++++++
scripts/agent-set-cotel-token.sh | 45 +--
scripts/otel-headers-helper.py | 122 ++++++++
scripts/otel-headers-helper_test.sh | 137 +++++++++
29 files changed, 2322 insertions(+), 73 deletions(-)
create mode 100644 docs/decisions/0028-ingest-keys-and-actor-tokens.md
create mode 100644 internal/api/auth/ingest_middleware_test.go
create mode 100644 internal/api/ingest_keys.go
create mode 100644 internal/api/ingest_keys_test.go
create mode 100644 internal/ingest/logs.go
create mode 100644 internal/ingest/logs_test.go
create mode 100644 internal/storage/ingest_keys.go
create mode 100644 internal/storage/ingest_keys_test.go
create mode 100644 internal/storage/testdata/schema_v11.sql
create mode 100755 scripts/otel-headers-helper.py
create mode 100755 scripts/otel-headers-helper_test.sh
diff --git a/.github/workflows/health-probe.yml b/.github/workflows/health-probe.yml
index 43ce62d..67f8b9b 100644
--- a/.github/workflows/health-probe.yml
+++ b/.github/workflows/health-probe.yml
@@ -57,6 +57,8 @@ on:
- "scripts/probe-edge-ingest_test.sh"
- "scripts/page-cotel-health.sh"
- "scripts/page-cotel-health_test.sh"
+ - "scripts/otel-headers-helper.py"
+ - "scripts/otel-headers-helper_test.sh"
- ".github/workflows/health-probe.yml"
permissions:
@@ -75,6 +77,8 @@ jobs:
run: bash scripts/probe-edge-ingest_test.sh
- name: Run pager tests
run: bash scripts/page-cotel-health_test.sh
+ - name: Run OTEL headers helper tests
+ run: bash scripts/otel-headers-helper_test.sh
probe-ingest:
name: Probe the public OTLP ingest
diff --git a/CHANGELOG.md b/CHANGELOG.md
index 765dbeb..47e58c5 100644
--- a/CHANGELOG.md
+++ b/CHANGELOG.md
@@ -8,6 +8,7 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0
## [Unreleased]
### Added
+- **Ingest keys: one shared key authorizes, each sender's actor token identifies** ([ADR-0028](docs/decisions/0028-ingest-keys-and-actor-tokens.md)). Paperclip now strips every `OTEL_*` and `CLAUDE_CODE_*TELEMETRY*` variable from an agent's env and applies one company-wide set of OTLP headers, so a per-agent `Authorization: Bearer ` can no longer be configured, and with that setting empty it turns telemetry off, which is why agent telemetry stopped on 2026-11-07. An ingest key (`cotel_ik_<64 hex>`, created and revoked under Setup → Settings → Ingest keys, stored as a hash with a 12-character display prefix, shown in full once) authorizes OTLP ingest and nothing else, never export/import or the dashboard; the existing actor token, unchanged and not reissued, rides next to it in `X-Cotel-Actor-Token`. A valid key with a missing or unknown actor token is accepted as Anonymous even when allow-anonymous is off, because the request was authorized; `X-Cotel-Actor-Token` without a bearer never identifies anyone, and a bearer actor token works exactly as before. Schema v12 adds `ingest_keys`; the dashboard API gains `GET`/`POST /api/v1/ingest-keys` and `DELETE /api/v1/ingest-keys/{id}`. `POST /v1/logs` accepts and discards OTLP logs (`200 {}`), so Paperclip's log exporter stops getting 404. `scripts/otel-headers-helper.py` is a Claude Code `otelHeadersHelper` that prints the complete header set — `OTEL_EXPORTER_OTLP_HEADERS` overlaid with `OTEL_EXPORTER_OTLP_TRACES_HEADERS`, plus `X-Cotel-Actor-Token` from `COTEL_ACTOR_TOKEN` — and always exits 0 with one JSON object, since a failing helper stops Claude Code's exports. Docs → [Actors and Authentication](docs/operations/actors-and-auth.md#ingest-keys-one-key-for-many-senders)
- `cotel --backfill-cost` (dry run) and `--backfill-cost-apply` recompute `cost_usd` for every span of a priced model from its stored token counts at today's rates, revived from the closed PR #67 as reviewed there (FLO-551/552). Spans with an unknown or empty model are counted and left alone; `daily_usage` is not touched, because a day rolled up after the backfill derives its cost from the corrected spans. The apply needs the database to itself, so it runs while the server is stopped; the dry run restores the newest snapshot into a throwaway file instead of copying a live one. README → Maintenance commands has both procedures for the current compose deploy, with `--entrypoint` set so a one-off does not start a second Cloudflare tunnel connector
- A synthetic reproducer for the ART-index corruption that took production down in September 2026: `TestARTIndexSurvivesAddedColumnAndBulkReplace` in `internal/storage/art_repro_test.go`, built from the schema with no production bytes — every value is derived from the row number, so it carries no user id, session id or span attribute that belonged to anyone. It rebuilds the trigger rather than a statement from it: `spans` as it stood at schema version 2 — without `user_id`, and *with* the mid-table `duration_ms` generated column, so that the one ALTER production ran on a long-lived filled table (v9 → v10, six weeks before the abort) runs here too instead of passing over a column the fixture never had, and so the indexes are built over the logical/physical column skew they were built over in production — with that era's three ART indexes, filled across the 30-day retention boundary; `storage.Open` then runs the real `schema.sql`, which adds `user_id VARCHAR` to the filled indexed table and puts `idx_spans_user_id` on that added column; a bulk `UPDATE` writes the column, cotel's own retention roll-up supplies the large replace transaction (the `INSERT INTO daily_usage` and the `DELETE FROM spans` are one transaction, so it is the real mutation and not an imitation); then checkpoint, close, reopen, checkpoint, ingest more, repeat. The repetition is the point — the damage accumulates and production ran for weeks after its migration before an open aborted — and so is the restart: this class of fault is only ever reported by the checkpoint that folds the index or the open that reads it back. Each pass asserts that `duration_ms` was there before the migration and is gone after it (a `DROP COLUMN IF EXISTS` that no-ops must not be able to come back unnoticed), that the reopen succeeds, that both checkpoints fold — a present-but-empty `.wal` counts as folded, so an engine that truncates rather than unlinks is not reported as this defect — that all four indexes survive, that a bare `user_id = ` folded into the scan agrees with the same predicate wrapped in an expression, which is not, and that re-inserting a stored `span_id` is still rejected while an unused one is accepted. The last two are the silent form of the damage: a wrong answer rather than an error. They are phrased that way because no `SELECT` over `spans` is planned as an ART index scan on this engine — measured, including on the primary key and on a textbook unique match with the thresholds raised — so an “indexed probe agrees with an unindexed probe” assertion would have been two sequential scans agreeing; the pushdown pair is what [ADR-0013](docs/decisions/0013-spans-has-no-derived-columns.md)'s wrong answers actually came out of, and unique-constraint enforcement is the one path that provably traverses an ART index. It runs in the ordinary test set rather than behind a build tag, because the moment it most needs to run is a driver or engine bump and that is exactly when an optional test is forgotten; `COTEL_ART_REPRO_SPANS` and `COTEL_ART_REPRO_CYCLES` scale it past the defaults (20 000 spans, 2 cycles, 4.1 s). Against the engine the tree links today — `duckdb-go/v2 v2.10506.0`, `SELECT version()` → `v1.5.6` — it did **not** reproduce, at the defaults or at 200 000 spans over 6 cycles. That measures the half of the forensic volume's release condition that was previously unevaluable, and measures it negative rather than exonerating: upstream reports the same class of fault on 1.5.5 against a ~19 GB database, the file that was damaged here took weeks of ingest to get there, and two clean runs at our scale are not absence ([docs/operations/duckdb-recovery.md](docs/operations/duckdb-recovery.md))
diff --git a/README.md b/README.md
index eda1973..04ad324 100644
--- a/README.md
+++ b/README.md
@@ -86,6 +86,8 @@ Open the dashboard → **Actors** (people icon in the sidebar) → **Add actor**
By default, cotel accepts spans without an `Authorization` header (allow-anonymous mode). To enforce strict auth: open **Setup** → **Settings** tab → disable **Allow anonymous OTLP**. Requests without a valid token then receive `401 Unauthorized`.
+For a fleet that shares one set of OTLP headers, such as agents run by Paperclip, create an **ingest key** under **Setup** → **Settings** → **Ingest keys** and send it as the bearer instead. The key only authorizes ingest; each sender still names its actor by sending its own actor token in the `X-Cotel-Actor-Token` header, which `scripts/otel-headers-helper.py` adds from `COTEL_ACTOR_TOKEN` as a Claude Code `otelHeadersHelper`. See [Ingest keys](docs/operations/actors-and-auth.md#ingest-keys-one-key-for-many-senders).
+
See **[docs/operations/actors-and-auth.md](docs/operations/actors-and-auth.md)** for the full guide: one token per sender, token rotation, and security considerations.
## Publishing with Cloudflare
diff --git a/cmd/cotel/main.go b/cmd/cotel/main.go
index 9d86835..5fea1bc 100644
--- a/cmd/cotel/main.go
+++ b/cmd/cotel/main.go
@@ -127,10 +127,11 @@ func main() {
go db.RunSnapshotWorker(snapshotCfg, envDuration("COTEL_SNAPSHOT_INTERVAL", storage.DefaultSnapshotInterval))
ingestMux := http.NewServeMux()
- ingestMux.Handle("/v1/traces", auth.Middleware(db, ingest.New(db)))
+ ingestMux.Handle("/v1/traces", auth.IngestMiddleware(db, ingest.New(db)))
+ ingestMux.Handle("/v1/logs", auth.IngestMiddleware(db, ingest.NewLogsSink()))
ro := db.ReadOnly()
- apiHandler := api.New(ro).SetPublicIngestURL(parsePublicIngestURL(os.Getenv("COTEL_PUBLIC_INGEST_URL"))).SetActorStore(db)
+ apiHandler := api.New(ro).SetPublicIngestURL(parsePublicIngestURL(os.Getenv("COTEL_PUBLIC_INGEST_URL"))).SetActorStore(db).SetIngestKeyStore(db)
dashMux := http.NewServeMux()
dashMux.Handle("/api/v1/export", auth.Middleware(db, export.NewHandler(db)))
dashMux.Handle("/api/v1/import", auth.Middleware(db, importpkg.NewHandler(db)))
diff --git a/docs/.vitepress/config.js b/docs/.vitepress/config.js
index 4627518..2240585 100644
--- a/docs/.vitepress/config.js
+++ b/docs/.vitepress/config.js
@@ -68,6 +68,7 @@ export default defineConfig({
{ text: 'ADR-0025 - Best-Effort Ingest Watcher in Actions', link: '/decisions/0025-best-effort-ingest-watcher-in-actions' },
{ text: 'ADR-0026 - The Deploy Host Asks /healthz on a Schedule', link: '/decisions/0026-scheduled-loopback-probe-with-a-pager' },
{ text: 'ADR-0027 - Users Become Actors', link: '/decisions/0027-users-become-actors' },
+ { text: 'ADR-0028 - Ingest Keys Authorize, Actor Tokens Identify', link: '/decisions/0028-ingest-keys-and-actor-tokens' },
],
},
],
diff --git a/docs/decisions/0028-ingest-keys-and-actor-tokens.md b/docs/decisions/0028-ingest-keys-and-actor-tokens.md
new file mode 100644
index 0000000..6147e11
--- /dev/null
+++ b/docs/decisions/0028-ingest-keys-and-actor-tokens.md
@@ -0,0 +1,122 @@
+# ADR 0028 — Ingest keys authorize, actor tokens identify
+
+**Date:** 2026-11-15
+**Status:** Accepted
+**Deciders:** Flop (board)
+
+---
+
+## Context
+
+Since ADR-0008 every sender proves two things with one header:
+`Authorization: Bearer ` both lets the request in and names the
+actor its spans belong to. Each Paperclip agent carried its own token in its own
+adapter env, set by `scripts/agent-set-cotel-token.sh`.
+
+Paperclip now manages Claude Code's telemetry env itself. Before every agent run
+it removes every `OTEL_*` and `CLAUDE_CODE_*TELEMETRY*` variable from the
+agent's env and applies **one** company-wide OTLP destination with **one** set
+of headers (its "Agent observability" setting): http/protobuf, traces to
+`{base}/v1/traces`, logs to `{base}/v1/logs`, the headers in
+`OTEL_EXPORTER_OTLP_TRACES_HEADERS` and `OTEL_EXPORTER_OTLP_LOGS_HEADERS` with
+percent-encoded values, and the resource attributes `paperclip.agent.id` and
+`paperclip.company.id`. With that setting empty it sets
+`CLAUDE_CODE_ENABLE_TELEMETRY=0`.
+
+That is why agent telemetry stopped on 2026-11-07: the per-agent headers were
+stripped and telemetry was switched off. Configuring the setting brings
+telemetry back, but only with headers that are identical for every agent. A
+per-agent `Authorization` header cannot be expressed through Paperclip, so with
+one header doing both jobs the choice is between no agent telemetry and every
+agent merged into one actor.
+
+## Decision
+
+Split authorization from identification.
+
+- An **ingest key** is a new shared credential, `cotel_ik_<64 hex>`. It is
+ created and revoked in the dashboard (**Setup → Settings → Ingest keys**) and
+ travels as `Authorization: Bearer `. Many senders can share one.
+- The **actor token** is the existing per-actor `cotel_<64 hex>` token,
+ unchanged and not reissued. Alongside an ingest key it travels in
+ `X-Cotel-Actor-Token`. Agents keep it in the env var `COTEL_ACTOR_TOKEN`,
+ which Paperclip's stripping does not touch.
+
+### How ingest decides
+
+| Request | Result |
+|---|---|
+| `Authorization: Bearer ` + `X-Cotel-Actor-Token: ` | Accepted, attributed to that actor |
+| Valid ingest key, actor token missing, empty or unknown | Accepted as **Anonymous**, even with allow-anonymous off |
+| Invalid `cotel_ik_` key | `401` |
+| `Authorization: Bearer ` (legacy) | Accepted, attributed to that actor; `X-Cotel-Actor-Token` is ignored |
+| Any other unknown `cotel_` bearer | `401` |
+| No bearer (a bearer not starting with `cotel_` counts as none) | Allow-anonymous decides, as before; `X-Cotel-Actor-Token` alone never identifies |
+
+A request with a valid ingest key and no usable actor token is stored as
+Anonymous even when allow-anonymous is off. The toggle exists to keep out
+requests nobody authorized; this one was authorized, and refusing it would drop
+real telemetry over a missing label. The actor token alone is never enough:
+it is an identity, not a credential, so `X-Cotel-Actor-Token` without a bearer
+falls through to allow-anonymous like any request without one.
+
+### Storage and scope
+
+- Schema v12 adds `ingest_keys (id, name, token_hash UNIQUE, prefix,
+ created_at)`. The key is stored **hashed**; only a 12-character display prefix
+ (e.g. `cotel_ik_ab1`) is kept in clear. The full value is shown once, when the
+ key is created, and cannot be retrieved afterwards. Revoking a key deletes its
+ row.
+- An ingest key authorizes **OTLP ingest only** (`/v1/traces`, `/v1/logs`). It
+ never opens export/import or the dashboard API.
+- Management is in the dashboard API: `GET`, `POST /api/v1/ingest-keys` and
+ `DELETE /api/v1/ingest-keys/{id}`.
+- `POST /v1/logs` exists only so Paperclip's log exporter does not get `404`. It
+ accepts the body and discards it (`200 {}`). cotel stores spans, not logs.
+
+### Getting the actor header out of Claude Code
+
+Claude Code does not turn arbitrary env vars into headers. It does run an
+`otelHeadersHelper` from `settings.json`: a command that prints a JSON object of
+HTTP headers, run at startup and every 29 minutes
+(`CLAUDE_CODE_OTEL_HEADERS_HELPER_DEBOUNCE_MS`), for http/protobuf and
+http/json only. If the helper fails or prints invalid JSON, exports fail until it
+works again.
+
+`scripts/otel-headers-helper.py` is that helper. It rebuilds the headers from
+`OTEL_EXPORTER_OTLP_HEADERS` overlaid with `OTEL_EXPORTER_OTLP_TRACES_HEADERS`,
+adds `X-Cotel-Actor-Token` from `COTEL_ACTOR_TOKEN`, and always prints one JSON
+object and exits 0. It emits the **complete** header set, not just the actor
+header, because how Claude Code merges the helper's output with the static
+`OTEL_EXPORTER_OTLP_*HEADERS` vars is not documented: if the helper's output
+replaced them, a helper that printed only `X-Cotel-Actor-Token` would silently
+drop the ingest key and every export would get `401`.
+
+## Alternatives considered
+
+- **Patch our Paperclip fork to stop stripping the env.** Rejected: it fights
+ upstream behaviour on purpose, and the patch would have to survive the weekly
+ rebase indefinitely.
+- **Map the `paperclip.agent.id` resource attribute to actors.** Viable, but
+ Paperclip-only and a second identity scheme to keep in sync with the actors
+ table. It may be added later as a fallback for requests without an actor
+ token.
+- **One shared actor for all agents.** Works today with no code change, and
+ loses per-agent attribution, which is what ADR-0008 exists for.
+
+## Consequences
+
+- **The ingest key is readable by agent code.** It sits in the agent's env, so
+ anything the agent runs can read it. That is why its scope is ingest-only: the
+ worst a leaked key does is write spans.
+- **Revoking a key cuts every sender using it** at once. Rotating a shared key
+ means creating the new one, updating the shared configuration, then revoking
+ the old one.
+- **Unknown actor tokens land in Anonymous silently.** A typo in
+ `COTEL_ACTOR_TOKEN`, a rotated token or a deleted actor gives no error to the
+ sender; its spans appear under Anonymous. Check the Actors page after wiring a
+ new sender.
+- **No last-used timestamp.** Ingest does not write to the database per request
+ to record key use, so there is no way to see which keys are still in use other
+ than revoking one and watching what stops.
+- Senders using a bearer actor token directly keep working unchanged.
diff --git a/docs/decisions/index.md b/docs/decisions/index.md
index 6a394df..624fb3d 100644
--- a/docs/decisions/index.md
+++ b/docs/decisions/index.md
@@ -35,3 +35,4 @@ New ADRs go in this directory as `NNNN-short-title.md`, numbered sequentially.
| [ADR-0025](./0025-best-effort-ingest-watcher-in-actions) | cotel is watched by a best-effort ingest probe in GitHub Actions | Accepted |
| [ADR-0026](./0026-scheduled-loopback-probe-with-a-pager) | The deploy host asks `/healthz` on a schedule, and it pages | Accepted |
| [ADR-0027](./0027-users-become-actors) | Users become actors, down to the schema | Accepted |
+| [ADR-0028](./0028-ingest-keys-and-actor-tokens) | Ingest keys authorize, actor tokens identify | Accepted |
diff --git a/docs/operations/actors-and-auth.md b/docs/operations/actors-and-auth.md
index 42b5079..cb74b19 100644
--- a/docs/operations/actors-and-auth.md
+++ b/docs/operations/actors-and-auth.md
@@ -1,16 +1,17 @@
# Actors and Authentication
-cotel ties OTLP ingest to named **actors**: whoever sends telemetry, whether a person, an AI agent or a host. Each actor has a bearer token; spans ingested with that token are attributed to that actor across the dashboard. This guide covers creating and managing actors, configuring Claude Code to send an authenticated token, the allow-anonymous toggle, and what happens when strict auth is enforced.
+cotel ties OTLP ingest to named **actors**: whoever sends telemetry, whether a person, an AI agent or a host. Each actor has an **actor token**; spans ingested with that token are attributed to that actor across the dashboard. Senders that must all share one set of headers authorize with a shared **ingest key** instead and name their actor in a separate header. This guide covers creating and managing actors, configuring Claude Code to send its actor token, ingest keys, the allow-anonymous toggle, and what happens when strict auth is enforced.
## Concepts
| Term | Meaning |
|---|---|
| **Actor** | A named sender of telemetry — a developer, an AI agent or a machine. Called a *user* before schema v11 ([ADR-0027](../decisions/0027-users-become-actors.md)) |
-| **Token** | A `cotel_<64 hex chars>` bearer token bound to one actor |
-| **Allow-anonymous** | When on (default), spans without a token are accepted and shown without an actor label |
+| **Actor token** | A `cotel_<64 hex chars>` token bound to one actor. Sent as the bearer token, or in `X-Cotel-Actor-Token` next to an ingest key |
+| **Ingest key** | A shared `cotel_ik_<64 hex chars>` bearer credential that authorizes OTLP ingest only and identifies nobody ([ADR-0028](../decisions/0028-ingest-keys-and-actor-tokens.md)) |
+| **Allow-anonymous** | When on (default), spans sent without a bearer token are accepted and shown without an actor label |
-Tokens are plain bearer strings stored in the database. They are always retrievable from the dashboard — rotating a token replaces it with a new one and immediately revokes the old one.
+Actor tokens are plain strings stored in the database. They are always retrievable from the dashboard — rotating an actor token replaces it with a new one and immediately revokes the old one. Ingest keys work differently; see [Ingest keys](#ingest-keys-one-key-for-many-senders).
## Dashboard walkthrough
@@ -26,7 +27,7 @@ Click the **people icon** in the left sidebar. The Actors page is a single list
| **Created** | When the actor was created (all-time) |
| **Last seen** | Most recent span (all-time) |
-There is no token column and no per-row buttons — a row is a single click through to that actor's own page.
+There is no actor-token column and no per-row buttons — a row is a single click through to that actor's own page.
**Range switcher.** A segmented control above the list scopes **Cost** and **Sessions** to a rolling window — All / Year / Month / Week / Day, defaulting to **Month** (last 30 days). The two columns it governs show the active window in their headers (e.g. `Cost (30d)`). Created and Last seen are always all-time. Your choice is remembered between visits via the `cotel_actors_range` cookie.
@@ -38,20 +39,20 @@ There is no token column and no per-row buttons — a row is a single click thro
2. Enter a display name — use something that identifies the sender, such as a machine hostname, agent name, or developer alias (e.g. `alice`, `prod-box`, `ci-runner`).
3. Click **Create**.
-A banner appears showing the new actor's token. Copy it now — you will paste it into Claude Code settings in the next step. You can retrieve the token again any time from the actor's page.
+A banner appears showing the new actor's actor token. Copy it now — you will paste it into Claude Code settings in the next step. You can retrieve it again any time from the actor's page.
### The per-actor page
Click any row to open `/actors/` (the old `/users/` links redirect there). This page shows the actor's name, their range-scoped **Cost** and **Sessions** (with the same range switcher as the list), all-time **Created** and **Last seen**, and the actions:
-- **Token** with a copy button — the full `cotel_…` string, retrievable any time.
-- **Rotate token** — generates a new token and immediately revokes the old one. Any Claude Code instance using the old token will start receiving `401 Unauthorized` until updated. Copy the new token and update `OTEL_EXPORTER_OTLP_HEADERS` in every `settings.json` that used the old one.
-- **Delete actor** — opens a modal with two choices: *Delete actor only* (revokes the token; history stays attributed to the name and is reversible by re-creating the actor) or *Delete actor + history* (permanently removes the actor and all their telemetry; type the name to confirm). After deletion you return to the list.
+- The **actor token** with a copy button — the full `cotel_…` string, retrievable any time.
+- **Rotate token** — generates a new actor token and immediately revokes the old one. Any Claude Code instance sending the old token as its bearer will start receiving `401 Unauthorized` until updated; one sending it in `X-Cotel-Actor-Token` next to an ingest key keeps being accepted, but its spans land under **Anonymous**. Copy the new token and update `OTEL_EXPORTER_OTLP_HEADERS` (or `COTEL_ACTOR_TOKEN`) everywhere the old one was used.
+- **Delete actor** — opens a modal with two choices: *Delete actor only* (revokes the actor token; history stays attributed to the name and is reversible by re-creating the actor) or *Delete actor + history* (permanently removes the actor and all their telemetry; type the name to confirm). After deletion you return to the list.
- **View activity** and **View sessions** — jump to the dashboard and Sessions views filtered to this actor.
### The Anonymous actor
-Unattributed spans (ingested with allow-anonymous on) roll up under a synthetic **Anonymous** row that sorts and pages inline with real actors. Its per-actor page has no token and no rotate — only **Delete anonymous data** (permanently removes all unattributed spans) alongside the activity and sessions links.
+Unattributed spans (ingested with allow-anonymous on) roll up under a synthetic **Anonymous** row that sorts and pages inline with real actors. Its per-actor page has no actor token and no rotate — only **Delete anonymous data** (permanently removes all unattributed spans) alongside the activity and sessions links.
## Configuring Claude Code
@@ -65,16 +66,16 @@ Add `OTEL_EXPORTER_OTLP_HEADERS` to `~/.claude/settings.json`:
"OTEL_TRACES_EXPORTER": "otlp",
"OTEL_EXPORTER_OTLP_PROTOCOL": "http/json",
"OTEL_EXPORTER_OTLP_TRACES_ENDPOINT": "http://localhost:4318/v1/traces",
- "OTEL_EXPORTER_OTLP_HEADERS": "Authorization=Bearer cotel_"
+ "OTEL_EXPORTER_OTLP_HEADERS": "Authorization=Bearer cotel_"
}
}
```
-Replace `cotel_` with the token you copied from the dashboard. Restart Claude Code to apply the change.
+Replace `cotel_` with the actor token you copied from the dashboard. Restart Claude Code to apply the change.
### Multiple machines or agents
-Create a separate actor for each sender — one per developer or machine. Each instance gets its own `settings.json` entry with its own `OTEL_EXPORTER_OTLP_HEADERS` value. Spans appear with the corresponding actor label in the dashboard.
+Create a separate actor for each sender — one per developer or machine. Each instance gets its own `settings.json` entry with its own `OTEL_EXPORTER_OTLP_HEADERS` value. Spans appear with the corresponding actor label in the dashboard. When the senders cannot have headers of their own — every agent run by Paperclip gets the same ones — use an [ingest key](#ingest-keys-one-key-for-many-senders) instead.
### Remote ingest (Cloudflare Tunnel)
@@ -82,19 +83,101 @@ When using a public ingest URL, the endpoint changes but the header format is id
```json
"OTEL_EXPORTER_OTLP_TRACES_ENDPOINT": "https://ingest.yourdomain.com/v1/traces",
-"OTEL_EXPORTER_OTLP_HEADERS": "Authorization=Bearer cotel_"
+"OTEL_EXPORTER_OTLP_HEADERS": "Authorization=Bearer cotel_"
```
See the [Cloudflare Tunnel guide](./cloudflare-tunnel-remote.md) for end-to-end setup.
+## Ingest keys (one key for many senders)
+
+An **ingest key** is a shared credential for OTLP ingest. It answers *may this request write spans?* and nothing else; which actor the spans belong to comes from the sender's actor token, sent in a second header, `X-Cotel-Actor-Token` ([ADR-0028](../decisions/0028-ingest-keys-and-actor-tokens.md)).
+
+Use one when many senders must share one set of headers. Paperclip is the case it was built for: before every agent run it removes all `OTEL_*` and `CLAUDE_CODE_*TELEMETRY*` variables from the agent's env and applies one company-wide OTLP destination with one set of headers, so a per-agent `Authorization` header cannot be set through it. Senders that can each have their own headers need no ingest key; sending the actor token as the bearer keeps working.
+
+| | Actor token | Ingest key |
+|---|---|---|
+| Format | `cotel_<64 hex>` | `cotel_ik_<64 hex>` |
+| Identifies | One actor | Nobody |
+| Authorizes | OTLP ingest | OTLP ingest only (`/v1/traces`, `/v1/logs`) — never export/import or the dashboard |
+| Sent as | `Authorization: Bearer …`, or `X-Cotel-Actor-Token: …` next to an ingest key | `Authorization: Bearer …` |
+| Stored | Plaintext, retrievable any time | Hashed; only a 12-character prefix such as `cotel_ik_ab1` is kept |
+
+### Creating and revoking an ingest key
+
+Open **Setup** → **Settings** tab → **Ingest keys**.
+
+- **Create:** give the key a name that says who uses it (e.g. `paperclip`) and create it. The full key is shown **once**, right after creation. Copy it then; cotel stores only its hash and cannot show it again. The list afterwards shows each key's name, prefix and creation time.
+- **Revoke:** deletes the key. Every sender using it gets `401 Unauthorized` on its next request. To replace a key without a gap, create the new one, update the shared headers, then revoke the old one.
+
+There is no last-used time: ingest does not write to the database to record which key a request used.
+
+### How ingest decides
+
+| Request | Result |
+|---|---|
+| `Authorization: Bearer ` + `X-Cotel-Actor-Token: ` | Accepted, attributed to that actor |
+| Valid ingest key, actor token missing, empty or unknown | Accepted as **Anonymous** — even with allow-anonymous off, because the key authorized the request |
+| Unknown `cotel_ik_` key | `401` |
+| `Authorization: Bearer ` | Accepted, attributed to that actor; `X-Cotel-Actor-Token` is ignored |
+| Any other unknown `cotel_` bearer | `401` |
+| No bearer (a bearer not starting with `cotel_` counts as none) | [Allow-anonymous](#allow-anonymous-mode) decides; `X-Cotel-Actor-Token` alone never identifies a request |
+
+A wrong actor token is not an error. A typo, a rotated token or a deleted actor sends the spans to **Anonymous** without telling the sender, so check the [Actors page](#the-actors-list) after wiring up a new sender.
+
+`POST /v1/logs` accepts the same credentials and discards the body (`200 {}`). It exists so that a sender exporting logs as well as traces, as Paperclip does, does not get `404`; cotel stores spans only.
+
+### Configuring a sender
+
+Each sender needs three things:
+
+1. **The ingest key** in its OTLP headers: `Authorization=Bearer cotel_ik_` in `OTEL_EXPORTER_OTLP_HEADERS` or `OTEL_EXPORTER_OTLP_TRACES_HEADERS`. With Paperclip this is the company-wide header set; Paperclip puts it in the per-signal header vars.
+2. **Its own actor token** in the env var `COTEL_ACTOR_TOKEN`, in that sender's own process env (for a Paperclip agent, its adapter env). Do not put it in the `env` block of a shared `settings.json`: that block applies to every launch and overrides the inherited env, so every sender would claim the same actor.
+3. **The headers helper.** Claude Code does not send env vars as headers, so it cannot send `COTEL_ACTOR_TOKEN` by itself. It does run an `otelHeadersHelper` command and sends the headers it prints. `scripts/otel-headers-helper.py` (Python 3.9+, stdlib only) prints the complete header set: `OTEL_EXPORTER_OTLP_HEADERS`, overlaid with `OTEL_EXPORTER_OTLP_TRACES_HEADERS`, plus `X-Cotel-Actor-Token` from `COTEL_ACTOR_TOKEN`. Point the `settings.json` that the sender's Claude Code reads at it:
+
+```json
+{
+ "otelHeadersHelper": "/path/to/cotel/scripts/otel-headers-helper.py"
+}
+```
+
+Things to know about the helper:
+
+- Claude Code runs it at startup and again every 29 minutes (`CLAUDE_CODE_OTEL_HEADERS_HELPER_DEBOUNCE_MS`), and only for the `http/protobuf` and `http/json` protocols. It reads the env Claude Code was started with, so a changed `COTEL_ACTOR_TOKEN` takes effect from the next Claude Code run.
+- If a helper fails or prints anything but a JSON object, Claude Code's exports fail until it works again. This one always prints one JSON object and exits 0; on an unexpected error it prints what it had built, or `{}`.
+- It repeats the headers from the env rather than printing only `X-Cotel-Actor-Token`, because how Claude Code merges a helper's output with the static `OTEL_EXPORTER_OTLP_*HEADERS` vars is not documented. If the output replaces them, a helper that printed only the actor header would drop the ingest key.
+- The file must be executable and `python3` must be on the sender's `PATH`. Run it by hand with the sender's env to see what it will send; its tests are `scripts/otel-headers-helper_test.sh`.
+
+### Verifying an ingest key
+
+```sh
+# Should return {} (OK): the key is accepted
+curl -s -w "\n%{http_code}" \
+ -H "Authorization: Bearer cotel_ik_" \
+ -H "X-Cotel-Actor-Token: cotel_" \
+ -H "Content-Type: application/json" \
+ -d '{"resourceSpans":[]}' \
+ http://localhost:4318/v1/traces
+
+# Should return 401: the key is not one cotel issued
+curl -s -w "\n%{http_code}" \
+ -H "Authorization: Bearer cotel_ik_0000000000000000000000000000000000000000000000000000000000000000" \
+ -H "Content-Type: application/json" \
+ -d '{"resourceSpans":[]}' \
+ http://localhost:4318/v1/traces
+```
+
+An empty `resourceSpans` stores nothing, so the first command proves the key is accepted but does not show attribution; that shows up on the Actors page once the sender's real spans arrive.
+
## Allow-anonymous mode
The **Allow anonymous OTLP** toggle lives in **Setup** → **Settings** tab.
| State | Behaviour |
|---|---|
-| **On** (default) | OTLP requests without a valid `Authorization` header are accepted; spans are stored with no actor attribution |
-| **Off** | Every OTLP request must present `Authorization: Bearer cotel_`; requests without a valid token receive `401 Unauthorized` |
+| **On** (default) | OTLP requests without an `Authorization` header are accepted; spans are stored with no actor attribution |
+| **Off** | Every OTLP request must present `Authorization: Bearer` with an actor token or an ingest key; requests without one receive `401 Unauthorized` |
+
+In both states an unknown `cotel_…` bearer gets `401`, and a request with a valid ingest key is accepted (as Anonymous if it names no known actor).
### When to keep allow-anonymous on
@@ -110,15 +193,15 @@ The **Allow anonymous OTLP** toggle lives in **Setup** → **Settings** tab.
### Disabling allow-anonymous
-1. Create at least one actor (see above) so at least one valid token exists.
-2. Configure all senders with their token before flipping the toggle.
+1. Create at least one actor (see above) so at least one valid actor token exists.
+2. Configure all senders with their actor token (or an ingest key) before flipping the toggle.
3. Open **Setup** → **Settings** tab → disable **Allow anonymous OTLP**.
4. Verify with `curl`:
```sh
# Should return {} (OK)
curl -s -w "\n%{http_code}" \
- -H "Authorization: Bearer cotel_" \
+ -H "Authorization: Bearer cotel_" \
-H "Content-Type: application/json" \
-d '{"resourceSpans":[]}' \
http://localhost:4318/v1/traces
@@ -147,7 +230,7 @@ Content-Type: application/json
Claude Code's OTel SDK logs a warning and drops the trace batch. No spans are stored.
-**Recovery:** copy the correct token from the Actors page and update `OTEL_EXPORTER_OTLP_HEADERS` in `settings.json`, then restart Claude Code.
+**Recovery:** copy the correct actor token from the Actors page and update `OTEL_EXPORTER_OTLP_HEADERS` in `settings.json`, then restart Claude Code. A sender using an ingest key gets `401` only when the key itself is wrong or revoked: create a new key and update the shared headers.
## Token security
@@ -157,16 +240,19 @@ Actor tokens are 32-byte cryptographically random values encoded as lowercase he
### Protecting the database
-Because tokens are stored in plaintext, anyone with filesystem access to the DuckDB file (`/data/cotel.duckdb` inside the container) can read all tokens. Protect with:
+Because actor tokens are stored in plaintext, anyone with filesystem access to the DuckDB file (`/data/cotel.duckdb` inside the container) can read all actor tokens. Protect with:
- Standard Docker volume permissions (owned by the container user, not world-readable on the host).
- Cloudflare Access or equivalent to restrict dashboard access to trusted operators.
-- Rotate any token that may have been exposed.
+- Rotate any actor token that may have been exposed.
+
+Ingest keys are the exception: only a hash and a 12-character display prefix are stored, so the database file does not reveal them.
## References
- [ADR-0004 — Multi-User Separation](../decisions/0004-multi-user-separation.md)
- [ADR-0027 — Users Become Actors](../decisions/0027-users-become-actors.md)
+- [ADR-0028 — Ingest Keys Authorize, Actor Tokens Identify](../decisions/0028-ingest-keys-and-actor-tokens.md)
- [ADR-0006 — Cloudflare Tunnel + Token Auth](../decisions/0006-cloudflare-tunnel-and-token-auth.md)
- [Cloudflare Tunnel — Token Mode](./cloudflare-tunnel-remote.md)
- [Cloudflare Tunnel — Local Config](./cloudflare-tunnel-local.md)
diff --git a/docs/operations/api-reference.md b/docs/operations/api-reference.md
index 743493e..1269cee 100644
--- a/docs/operations/api-reference.md
+++ b/docs/operations/api-reference.md
@@ -2,7 +2,9 @@
The dashboard is a client of the same public JSON API you can call yourself.
Everything below is mounted under `/api/v1/` on the dashboard port (`:8080` by
-default) and returns `application/json`.
+default) and returns `application/json`. The OTLP ingest endpoints, `/v1/traces`
+and `/v1/logs`, are the exception: they live on the ingest port (`:4318` by
+default) and are described [at the end](#otlp-ingest-v1-traces-and-v1-logs).
## The list-query contract
@@ -249,6 +251,66 @@ returns one actor in the same shape and accepts `range`. See
[Actors and Authentication](./actors-and-auth.md) for the management endpoints
(create, rotate, delete).
+## Ingest keys
+
+Shared credentials that authorize OTLP ingest without identifying an actor
+([ADR-0028](../decisions/0028-ingest-keys-and-actor-tokens.md)). The key itself
+is stored hashed, so it appears in exactly one response: the one that creates
+it. An ingest key authorizes `/v1/*` ingest only; it cannot call these
+endpoints or any other `/api/v1/` endpoint. See
+[Ingest keys](./actors-and-auth.md#ingest-keys-one-key-for-many-senders) for
+how senders use them.
+
+### `GET /ingest-keys`
+
+Every key, oldest first. Not paginated, and it takes no parameters.
+
+```json
+{
+ "ingest_keys": [
+ { "id": "…", "name": "paperclip", "prefix": "cotel_ik_ab1", "created_at": "2026-11-15T10:02:11Z" }
+ ]
+}
+```
+
+`prefix` is the first 12 characters of the key, kept so a key can be told apart
+in the list. It is not enough to authenticate with.
+
+### `POST /ingest-keys`
+
+Creates a key. Body: `{"name": "paperclip"}`. The name is trimmed, must not be
+empty and is at most 100 characters; otherwise `400`.
+
+Answers `201` with the row and the full key in `token`:
+
+```json
+{ "id": "…", "name": "paperclip", "prefix": "cotel_ik_ab1", "created_at": "2026-11-15T10:02:11Z", "token": "cotel_ik_ab1…" }
+```
+
+`token` is never returned again. A lost key cannot be recovered; create a new
+one and revoke the old.
+
+### `DELETE /ingest-keys/{id}`
+
+Revokes a key by deleting it. Answers `204` with no body, or `404` when no key
+has that id. Every sender using the key gets `401` from its next request on.
+
+## OTLP ingest: `/v1/traces` and `/v1/logs`
+
+These two are not under `/api/v1/` and not on the dashboard port: they are
+served on the **ingest port** (`:4318` by default), and they are what Claude
+Code's OTLP exporter talks to. Both take the same credentials — an actor token,
+or an ingest key with `X-Cotel-Actor-Token` — and answer `401
+{"error":"unauthorized"}` when those are wrong. The full decision table is in
+[How ingest decides](./actors-and-auth.md#how-ingest-decides).
+
+- **`POST /v1/traces`** stores the spans in an OTLP/HTTP trace export
+ (`application/json` or `application/x-protobuf`) and answers `200 {}`.
+- **`POST /v1/logs`** accepts an OTLP/HTTP log export and **discards** it,
+ answering `200 {}`. cotel stores spans only; the endpoint exists so a sender
+ that exports logs as well as traces (Paperclip configures both) gets a success
+ instead of `404`.
+
## `GET /health`
Instance health. Takes no parameters and is never range-scoped.
@@ -288,3 +350,4 @@ README's *Health check* section for how the container `HEALTHCHECK` uses it.
- [ADR-0012 — Tools list: ranged stats and server-side sort](../decisions/0012-tools-list-ranged-stats-and-server-side-sort.md)
- [ADR-0014 — Overview: one range selector every panel obeys](../decisions/0014-overview-single-range-selector.md)
- [ADR-0009 — daily_usage unknown sentinel](../decisions/0009-daily-usage-unknown-sentinel.md)
+- [ADR-0028 — Ingest keys authorize, actor tokens identify](../decisions/0028-ingest-keys-and-actor-tokens.md)
diff --git a/frontend/src/api/index.ts b/frontend/src/api/index.ts
index 0a890af..1c2467e 100644
--- a/frontend/src/api/index.ts
+++ b/frontend/src/api/index.ts
@@ -231,6 +231,58 @@ export function useSettings() {
return useSWR('/api/v1/settings', fetcher)
}
+// Ingest keys authorize OTLP ingest for many senders at once; each sender names
+// itself with its actor token in X-Cotel-Actor-Token. The list carries only the
+// key's prefix: the full key is returned once, by createIngestKey.
+export interface IngestKey {
+ id: string
+ name: string
+ prefix: string
+ created_at: string
+}
+
+export interface IngestKeysResponse {
+ ingest_keys: IngestKey[]
+}
+
+export interface CreatedIngestKey extends IngestKey {
+ token: string
+}
+
+export const INGEST_KEYS_URL = '/api/v1/ingest-keys'
+
+// responseError turns a failed response into an Error, preferring the
+// server's {"error": "..."} message over the raw body.
+async function responseError(res: Response): Promise {
+ const text = await res.text().catch(() => '')
+ try {
+ const body = JSON.parse(text) as { error?: unknown }
+ if (typeof body.error === 'string' && body.error) return new Error(body.error)
+ } catch {
+ // not JSON: fall through to the raw text
+ }
+ return new Error(text || `${res.status} ${res.statusText}`)
+}
+
+export function useIngestKeys() {
+ return useSWR(INGEST_KEYS_URL, fetcher)
+}
+
+export async function createIngestKey(name: string): Promise {
+ const res = await fetch(INGEST_KEYS_URL, {
+ method: 'POST',
+ headers: { 'Content-Type': 'application/json' },
+ body: JSON.stringify({ name }),
+ })
+ if (!res.ok) throw await responseError(res)
+ return res.json()
+}
+
+export async function revokeIngestKey(id: string): Promise {
+ const res = await fetch(`${INGEST_KEYS_URL}/${encodeURIComponent(id)}`, { method: 'DELETE' })
+ if (!res.ok) throw await responseError(res)
+}
+
export async function createActor(name: string): Promise {
const res = await fetch('/api/v1/actors', {
method: 'POST',
diff --git a/frontend/src/pages/ActorDetail.module.css b/frontend/src/pages/ActorDetail.module.css
index e21d805..de59832 100644
--- a/frontend/src/pages/ActorDetail.module.css
+++ b/frontend/src/pages/ActorDetail.module.css
@@ -87,13 +87,6 @@
color: var(--color-accent);
}
-.hint {
- margin: var(--space-3) 0 0;
- font-size: var(--text-xs);
- color: var(--color-text-2);
- word-break: break-all;
-}
-
.code {
font-family: var(--font-mono);
font-size: var(--text-mono-sm);
@@ -159,3 +152,25 @@
color: var(--color-text-2);
margin: 0 0 var(--space-4);
}
+
+.hintList {
+ margin: var(--space-3) 0 0;
+ padding-left: var(--space-5);
+ font-size: var(--text-xs);
+ color: var(--color-text-2);
+ line-height: 1.7;
+ overflow-wrap: anywhere;
+}
+
+.hintList .code {
+ word-break: break-all;
+}
+
+.hintLink {
+ color: var(--color-accent);
+ text-decoration: none;
+}
+
+.hintLink:hover {
+ text-decoration: underline;
+}
diff --git a/frontend/src/pages/ActorDetail.tsx b/frontend/src/pages/ActorDetail.tsx
index edfcb7a..c71b9e5 100644
--- a/frontend/src/pages/ActorDetail.tsx
+++ b/frontend/src/pages/ActorDetail.tsx
@@ -130,7 +130,7 @@ export default function ActorDetail() {
{!isAnon && (
-
+
{data.token}
{notice &&
{notice}
}
-
- Set OTEL_EXPORTER_OTLP_HEADERS=Authorization=Bearer {data.token} in your Claude Code settings.
-
+
+
+ Send it directly: set{' '}
+ OTEL_EXPORTER_OTLP_HEADERS=Authorization=Bearer {data.token}.
+
+
+ With an ingest key: set{' '}
+ COTEL_ACTOR_TOKEN={data.token}, sent as{' '}
+ X-Cotel-Actor-Token.
+
When enabled, spans without an Authorization header are accepted and attributed to no actor.
- Disable to require every OTLP sender to present a valid actor token.
+ Disable to require every OTLP sender to present a valid actor token or ingest key.