From cd6f444f3ee896ac3fb97ffd22095774f166dc8e Mon Sep 17 00:00:00 2001 From: FlopBut Date: Sun, 15 Nov 2026 20:36:27 +0100 Subject: [PATCH 1/2] feat(ingest): drop Codex spans without a session (ADR-0030) Codex exports every internal tracing span: two short Paperclip runs sent 31,373 spans, 78% of the day's traffic, and only the 207 that carry a session feed any page. Ingest now drops a Codex span the adapter finds no session for, and logs the dropped count at most once a minute. --purge-dropped (dry run) and --purge-dropped-apply apply the same rule, through ingest.WouldDrop, to spans stored before it. Only spans without a session are candidates. Co-Authored-By: Claude Opus 5.5 --- CHANGELOG.md | 1 + README.md | 22 ++++ cmd/cotel/main.go | 39 ++++++ ...dex-spans-without-a-session-are-dropped.md | 72 +++++++++++ docs/decisions/index.md | 1 + internal/ingest/handler.go | 34 ++++- internal/ingest/producers.go | 28 +++- internal/ingest/producers_test.go | 72 ++++++++--- internal/storage/purge.go | 122 ++++++++++++++++++ internal/storage/purge_test.go | 86 ++++++++++++ 10 files changed, 458 insertions(+), 19 deletions(-) create mode 100644 docs/decisions/0030-codex-spans-without-a-session-are-dropped.md create mode 100644 internal/storage/purge.go create mode 100644 internal/storage/purge_test.go diff --git a/CHANGELOG.md b/CHANGELOG.md index cdb20b0..309a925 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -14,6 +14,7 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 - A synthetic reproducer for the ART-index corruption that took production down in September 2026: `TestARTIndexSurvivesAddedColumnAndBulkReplace` in `internal/storage/art_repro_test.go`, built from the schema with no production bytes — every value is derived from the row number, so it carries no user id, session id or span attribute that belonged to anyone. It rebuilds the trigger rather than a statement from it: `spans` as it stood at schema version 2 — without `user_id`, and *with* the mid-table `duration_ms` generated column, so that the one ALTER production ran on a long-lived filled table (v9 → v10, six weeks before the abort) runs here too instead of passing over a column the fixture never had, and so the indexes are built over the logical/physical column skew they were built over in production — with that era's three ART indexes, filled across the 30-day retention boundary; `storage.Open` then runs the real `schema.sql`, which adds `user_id VARCHAR` to the filled indexed table and puts `idx_spans_user_id` on that added column; a bulk `UPDATE` writes the column, cotel's own retention roll-up supplies the large replace transaction (the `INSERT INTO daily_usage` and the `DELETE FROM spans` are one transaction, so it is the real mutation and not an imitation); then checkpoint, close, reopen, checkpoint, ingest more, repeat. The repetition is the point — the damage accumulates and production ran for weeks after its migration before an open aborted — and so is the restart: this class of fault is only ever reported by the checkpoint that folds the index or the open that reads it back. Each pass asserts that `duration_ms` was there before the migration and is gone after it (a `DROP COLUMN IF EXISTS` that no-ops must not be able to come back unnoticed), that the reopen succeeds, that both checkpoints fold — a present-but-empty `.wal` counts as folded, so an engine that truncates rather than unlinks is not reported as this defect — that all four indexes survive, that a bare `user_id = ` folded into the scan agrees with the same predicate wrapped in an expression, which is not, and that re-inserting a stored `span_id` is still rejected while an unused one is accepted. The last two are the silent form of the damage: a wrong answer rather than an error. They are phrased that way because no `SELECT` over `spans` is planned as an ART index scan on this engine — measured, including on the primary key and on a textbook unique match with the thresholds raised — so an “indexed probe agrees with an unindexed probe” assertion would have been two sequential scans agreeing; the pushdown pair is what [ADR-0013](docs/decisions/0013-spans-has-no-derived-columns.md)'s wrong answers actually came out of, and unique-constraint enforcement is the one path that provably traverses an ART index. It runs in the ordinary test set rather than behind a build tag, because the moment it most needs to run is a driver or engine bump and that is exactly when an optional test is forgotten; `COTEL_ART_REPRO_SPANS` and `COTEL_ART_REPRO_CYCLES` scale it past the defaults (20 000 spans, 2 cycles, 4.1 s). Against the engine the tree links today — `duckdb-go/v2 v2.10506.0`, `SELECT version()` → `v1.5.6` — it did **not** reproduce, at the defaults or at 200 000 spans over 6 cycles. That measures the half of the forensic volume's release condition that was previously unevaluable, and measures it negative rather than exonerating: upstream reports the same class of fault on 1.5.5 against a ~19 GB database, the file that was damaged here took weeks of ingest to get there, and two clean runs at our scale are not absence ([docs/operations/duckdb-recovery.md](docs/operations/duckdb-recovery.md)) ### Changed +- **Codex spans without a session are dropped at ingest** ([ADR-0030](docs/decisions/0030-codex-spans-without-a-session-are-dropped.md)). Codex exports every internal tracing span. Two short runs sent 31,373 spans, of which the 207 that carry a session were the only ones any page showed. Sessions, tokens, cost and tool calls are unchanged. The ingest log reports the dropped count at most once a minute. `cotel --purge-dropped` (dry run) and `--purge-dropped-apply` delete such spans that were stored before the rule. - Database snapshots are taken once a day and fourteen are kept (`COTEL_SNAPSHOT_INTERVAL=24h`, `COTEL_SNAPSHOT_KEEP=14`), down from every six hours with 56 kept. Reach stays fourteen days; a restore now loses up to a day of telemetry instead of six hours, for about a quarter of the disk. The probe's `SNAPSHOT_STALE_AFTER_SECONDS` default moves from 12 h to 48 h with it, so a healthy daily worker does not read as stale ([ADR-0023 amendment](docs/decisions/0023-production-database-snapshots.md)) - **Users are now actors, down to the schema** ([ADR-0027](docs/decisions/0027-users-become-actors.md)). What a token belongs to was never only a person: production holds one person, one host and fifteen Paperclip agents. Schema v11 renames the `users` table to `actors` and `spans.user_id` / `daily_usage.user_id` to `actor_id`; the API moves to `/api/v1/actors`, `?actor_id=` and `actors_count`; the dashboard's Users page becomes Actors at `/actors`; export archives become `format_version` 2. The rename runs in Go before `schema.sql`, because a rename cannot be idempotent in SQL and `CREATE TABLE IF NOT EXISTS actors` would otherwise create an empty twin beside the populated table; it checks the catalog per step, drops and lets `schema.sql` recreate the `spans` indexes (DuckDB will not ALTER an indexed table), commits as one transaction, and refuses to start if both tables exist. Requests using the old names keep working - `/api/v1/users`, `?user_id=`, `mode=user_only|user_and_history`, the `/users` dashboard URLs, and v1 export archives - but **response bodies use the new field names only**, and **an older binary cannot open a migrated file without losing its tokens**: rolling back across v11 means restoring a pre-v11 snapshot - `docker compose build` tags its result `cotel-local:latest` instead of `ghcr.io/flopsstuff/cotel:latest`. Compose treats `image:` next to `build:` as the name to tag the local build with, so the published name was being reused for an artifact that never came from the registry: on the production host — where the deploy is `docker compose build --pull` followed by `up -d`, and `--pull` refreshes the Dockerfile's base images rather than the app image — `Image=ghcr.io/flopsstuff/cotel:latest` named a locally built binary, and built-here and pulled-from-registry were the same string. Three things follow from splitting them. You can read which artifact a host is running off the name. The thing the release pipeline publishes, and which `release.yml` moves only on an explicit dispatch, is no longer shadowed by every merge to `main`. And `docker pull ghcr.io/flopsstuff/cotel:latest` on the production host stops being able to swap the running service's binary at its next restart for content no deploy step validated — it now lands on a name nothing there runs. Production still *builds* its image, deliberately: a pull-only production would mean a merge to `main` no longer changes production, which is a separate decision from this cleanup and would need its own ADR. The operations pages name both images where the distinction matters — the recovery procedure's "match the CLI to the version linked into cotel" step asks `cotel-local:latest` on a host that builds its own image — and they also record that `ghcr.io/flopsstuff/cotel:latest` is now a multi-arch index whose `linux/arm64` entry runs natively on a Pi or Apple Silicon, so the documented `docker run --entrypoint /usr/local/bin/cotel … --db-query` and `--db-import` commands are honest on arm64 without a local build ([docs/operations/duckdb-recovery.md](docs/operations/duckdb-recovery.md), [docs/operations/duckdb-snapshots.md](docs/operations/duckdb-snapshots.md)) diff --git a/README.md b/README.md index 9c6190d..0d9a6c9 100644 --- a/README.md +++ b/README.md @@ -613,6 +613,28 @@ stored token counts, not scaled, so a second run changes nothing. > output ~156:1, so a recompute from the stored counters would understate the cost. Run the backfill > before the affected days leave the raw window (`COTEL_RETENTION_RAW_DAYS`, 30 days by default). +### Purge spans ingest now drops + +Codex spans without a session are dropped at ingest ([ADR-0030](docs/decisions/0030-codex-spans-without-a-session-are-dropped.md)). +`--purge-dropped` lists the stored spans the same rule would drop, by service and span name, and +`--purge-dropped-apply` deletes them. Only spans without a session are candidates. The same +exclusive-lock rule as for the cost backfill applies, so the steps are the same: + +```bash +# Dry run against the newest snapshot (no writes, no downtime) +docker compose run --rm --no-deps -e COTEL_DB_PATH=/tmp/dry.duckdb --entrypoint sh cotel -c \ + 'cotel --db-import "$(ls -d /snapshots/20* | tail -1)" && cotel --purge-dropped' + +# Apply +docker compose stop cotel +docker compose run --rm --no-deps --entrypoint /usr/local/bin/cotel cotel --purge-dropped-apply +docker compose start cotel +``` + +It is idempotent: a second run finds nothing. A `DELETE` does not shrink the DuckDB file; the freed +space is reused for new spans, and the file only gets smaller when it is rebuilt from a snapshot +([Database snapshots](docs/operations/duckdb-snapshots.md)). + ## Architecture ``` diff --git a/cmd/cotel/main.go b/cmd/cotel/main.go index 5fea1bc..365d18d 100644 --- a/cmd/cotel/main.go +++ b/cmd/cotel/main.go @@ -33,6 +33,8 @@ func main() { healthcheck := flag.Bool("healthcheck", false, "probe the local dashboard /healthz and exit 0 (ready) or 1; used by the container HEALTHCHECK") backfillCost := flag.Bool("backfill-cost", false, "dry-run: show what recalculating cost_usd from stored token counts would change (no writes), then exit") backfillCostApply := flag.Bool("backfill-cost-apply", false, "recalculate cost_usd for all spans of priced models from their stored token counts, then exit") + purgeDropped := flag.Bool("purge-dropped", false, "dry-run: show which stored spans today's ingest would drop (Codex spans without a session, ADR-0030), then exit") + purgeDroppedApply := flag.Bool("purge-dropped-apply", false, "delete the stored spans today's ingest would drop (Codex spans without a session, ADR-0030), then exit") flag.Parse() dbPath := env("COTEL_DB_PATH", "/data/cotel.duckdb") @@ -74,6 +76,9 @@ func main() { if *backfillCost || *backfillCostApply { os.Exit(runBackfill(dbPath, *backfillCostApply)) } + if *purgeDropped || *purgeDroppedApply { + os.Exit(runPurgeDropped(dbPath, *purgeDroppedApply)) + } // Bind BEFORE storage.Open: WAL replay + schema migration can block for // minutes on a large DB, and a bound port answering a retryable 503 keeps @@ -186,6 +191,40 @@ func runBackfill(dbPath string, apply bool) int { return 0 } +// runPurgeDropped deletes the stored spans that ingest would drop today, for +// data ingested before the rule (ADR-0030). apply=false is a dry run. +func runPurgeDropped(dbPath string, apply bool) int { + db, err := storage.Open(dbPath) + if err != nil { + log.Printf("open storage: %v", err) + return 1 + } + defer db.Close() + + if apply { + fmt.Println("=== DELETING spans ingest would drop ===") + } else { + fmt.Println("=== DRY RUN (no writes) - pass --purge-dropped-apply to delete ===") + } + rep, err := db.PurgeSpans(ingest.WouldDrop, apply) + if err != nil { + log.Printf("purge: %v", err) + return 1 + } + fmt.Printf("\n%-24s %-45s %8s\n", "service", "span name", "spans") + fmt.Println(strings.Repeat("-", 81)) + for _, row := range rep.Rows { + fmt.Printf("%-24s %-45s %8d\n", row.Service, row.Name, row.Spans) + } + fmt.Println(strings.Repeat("-", 81)) + verb := "would delete" + if apply { + verb = "deleted" + } + fmt.Printf("%s: %d spans\n\n", verb, rep.Total) + return 0 +} + // printBackfillReport prints a BackfillReport to stdout. func printBackfillReport(rep storage.BackfillReport, applied bool) { verb := "would change" diff --git a/docs/decisions/0030-codex-spans-without-a-session-are-dropped.md b/docs/decisions/0030-codex-spans-without-a-session-are-dropped.md new file mode 100644 index 0000000..621fd73 --- /dev/null +++ b/docs/decisions/0030-codex-spans-without-a-session-are-dropped.md @@ -0,0 +1,72 @@ +# ADR 0030 — Codex spans without a session are dropped at ingest + +**Date:** 2026-11-15 +**Status:** Accepted +**Deciders:** Flop (board) + +--- + +## Context + +Codex (`service.name` `codex-app-server`) exports every span its Rust tracing +layer opens, not a curated set like Claude Code's. On 2026-11-15 two short +Paperclip ping runs of one Codex agent sent 31,373 spans, 78% of that day's +traffic. All of production then held about 102,000 raw spans, covering 30 days. + +| Spans | Count | What they are | +|---|---|---| +| `receiving`, `handle_responses` | 18,946 | A pair per streamed response event, about 225 events per model request | +| `append_items`, `persist_rollout_items` | 4,788 | Writes to Codex's own rollout file | +| `fs.get_metadata`, `fs.read_file` | 2,677 | Filesystem calls | +| `realtime_conversation.running_state` | 1,294 | A state poll | +| tool routing, MCP, plugins, `auth`, config | the rest | Set-up repeated before each step | + +Only 207 of them (0.7%) feed anything cotel shows. These are the turn span +(`session_task.turn`, the model and token totals) and the tool-call spans. They +are also exactly the spans that carry the conversation id, which the Codex +adapter reads as the session. Every page that shows a span (Sessions, Tools, +Costs, Models, the session timeline) shows only spans with a session. The +other 99.3% were stored and never shown. They stayed for the 30-day raw +retention and went into every daily snapshot. Volume grows with the number of +model requests, about 750 spans each, so one real task would add more spans +than all other senders send in a month. + +Codex itself cannot be told to send less. Its binary has no span filter by +name or level. `OTEL_TRACES_SAMPLER` drops whole traces, turn spans included, +and Paperclip strips `OTEL_*` from agents' environment anyway (ADR-0028). + +## Decision + +**cotel drops at ingest every Codex span that has no session** after the Codex +adapter has run. The adapter takes the session from `conversation.id`, or from +`thread.id` when that is a UUID string. Dropped spans are not stored. The +request still succeeds, because the drop is a choice and not an error. The +ingest log reports how many spans were dropped, at most once a minute. + +The rule keys on the session, not on a list of span names. Codex adds internal +spans in new versions, and a list of names would let each new one through +until someone noticed. A span that the dashboard could show always has a +session, so the rule cannot drop one of those. + +Only Codex is filtered. Claude Code and Grok CLI spans are all stored. Grok +sends about one fiftieth of Codex's volume. + +`cotel --purge-dropped` (dry run) and `--purge-dropped-apply` apply the same +rule to spans already stored, using the ingest code itself (`ingest.WouldDrop`). +Only spans without a session are candidates. + +## Consequences + +- Codex volume falls by about 99%. Sessions, tokens, cost and tool calls are + unchanged. +- An error status on a dropped span is lost. Such errors never affected a + session's status, because that status reads only spans in the session, so + nothing visible changes. +- The raw export has fewer Codex spans than were sent. Anyone who needs the + full Codex trace must send it to a tracing backend, not to cotel. +- A Codex version that stops sending the conversation id would have every span + dropped. The ingest log would show it as a jump in dropped spans, and Codex + sessions would stop appearing. +- `DELETE` does not shrink a DuckDB file. Space freed by the purge is reused + for new spans. The file itself gets smaller only when it is rebuilt from a + snapshot ([Database snapshots](../operations/duckdb-snapshots.md)). diff --git a/docs/decisions/index.md b/docs/decisions/index.md index f5c6706..414f07d 100644 --- a/docs/decisions/index.md +++ b/docs/decisions/index.md @@ -37,3 +37,4 @@ New ADRs go in this directory as `NNNN-short-title.md`, numbered sequentially. | [ADR-0027](./0027-users-become-actors) | Users become actors, down to the schema | Accepted | | [ADR-0028](./0028-ingest-keys-and-actor-tokens) | Ingest keys authorize, actor tokens identify | Accepted | | [ADR-0029](./0029-paperclip-agents-by-agent-id) | Paperclip agents are identified by `paperclip.agent.id` | Accepted | +| [ADR-0030](./0030-codex-spans-without-a-session-are-dropped) | Codex spans without a session are dropped at ingest | Accepted | diff --git a/internal/ingest/handler.go b/internal/ingest/handler.go index 8bce592..ec8376d 100644 --- a/internal/ingest/handler.go +++ b/internal/ingest/handler.go @@ -8,6 +8,7 @@ import ( "log" "net/http" "strings" + "sync" "time" "google.golang.org/protobuf/encoding/protojson" @@ -39,6 +40,11 @@ type Handler struct { store Storer queue chan storage.Span done chan struct{} + + // Spans dropped by dropSpan since the last log line; see noteDropped. + dropMu sync.Mutex + dropped int64 + dropLogged time.Time } func New(store Storer) *Handler { @@ -66,6 +72,25 @@ func (h *Handler) Flush() { <-h.done } +// dropLogEvery bounds how often noteDropped logs: Codex sends a batch every few +// seconds while it runs, and a line per batch would drown the log. +const dropLogEvery = time.Minute + +// noteDropped counts spans dropped by dropSpan and logs the running total at +// most once per dropLogEvery, so the volume being discarded stays visible. +func (h *Handler) noteDropped(n int64) { + if n == 0 { + return + } + h.dropMu.Lock() + defer h.dropMu.Unlock() + h.dropped += n + if now := time.Now(); now.Sub(h.dropLogged) >= dropLogEvery { + log.Printf("ingest: dropped %d Codex spans without a session (ADR-0030)", h.dropped) + h.dropped, h.dropLogged = 0, now + } +} + // ServeHTTP handles POST /v1/traces (OTLP/HTTP protobuf or JSON). func (h *Handler) ServeHTTP(w http.ResponseWriter, r *http.Request) { if r.Method != http.MethodPost { @@ -103,6 +128,7 @@ func (h *Handler) ServeHTTP(w http.ResponseWriter, r *http.Request) { byAgent := actorName == "" && IngestKeyAuthFromContext(r.Context()) resolve := h.agentResolver(byAgent) + var dropped int64 for _, rs := range req.ResourceSpans { svcName := attrStr(rs.GetResource().GetAttributes(), "service.name") resAttrs := attrsToMap(rs.GetResource().GetAttributes()) @@ -119,10 +145,16 @@ func (h *Handler) ServeHTTP(w http.ResponseWriter, r *http.Request) { // Some exporters put the attribute on the span, not the resource. spanActor = resolve(attrStr(sp.GetAttributes(), PaperclipAgentIDAttr)) } - h.queue <- decodeSpan(sp, svcName, resAttrs, string(resJSON), spanActor) + s := decodeSpan(sp, svcName, resAttrs, string(resJSON), spanActor) + if dropSpan(svcName, &s) { + dropped++ + continue + } + h.queue <- s } } } + h.noteDropped(dropped) writeOTLPSuccess(w, r) } diff --git a/internal/ingest/producers.go b/internal/ingest/producers.go index 1cb8e21..2ab50a3 100644 --- a/internal/ingest/producers.go +++ b/internal/ingest/producers.go @@ -20,15 +20,37 @@ import ( // normalizeProducer applies the adapter for svcName, if there is one. func normalizeProducer(svcName, spanName string, attrs map[string]any, s *storage.Span) { - svc := strings.ToLower(svcName) switch { - case strings.HasPrefix(svc, "codex"): + case isCodex(svcName): normalizeCodex(attrs, s) - case strings.HasPrefix(svc, "grok"): + case strings.HasPrefix(strings.ToLower(svcName), "grok"): normalizeGrok(spanName, attrs, s) } } +func isCodex(svcName string) bool { + return strings.HasPrefix(strings.ToLower(svcName), "codex") +} + +// dropSpan reports whether a decoded span is kept out of storage. Codex exports +// every internal tracing span - one pair per streamed response event, rollout +// writes, filesystem calls, state polls - and only the ones that carry a +// session feed anything cotel shows, so a Codex span without one is dropped +// (ADR-0030). Every other producer's spans are all stored. +func dropSpan(svcName string, s *storage.Span) bool { + return isCodex(svcName) && s.SessionID == "" +} + +// WouldDrop reports whether ingest would drop a span with this service, name +// and attributes today. It is meant for spans already stored without a +// session, to purge what was ingested before the rule existed: attrs is the +// span's stored attribute map, as decoded from JSON. +func WouldDrop(svcName, spanName string, attrs map[string]any) bool { + var s storage.Span + normalizeProducer(svcName, spanName, attrs, &s) + return dropSpan(svcName, &s) +} + // Codex (service.name "codex-app-server", Rust tracing over OTLP). // // - Session: conversation.id on exec spans; on the turn span and the tool diff --git a/internal/ingest/producers_test.go b/internal/ingest/producers_test.go index cc2408d..c8ebb83 100644 --- a/internal/ingest/producers_test.go +++ b/internal/ingest/producers_test.go @@ -31,6 +31,16 @@ func i(v string) string { return `{"intValue":"` + v + `"}` } // ingestOne posts one span from service svc and returns what was stored. func ingestOne(t *testing.T, svc, spanName string, attrs ...kv) storage.Span { + t.Helper() + stored := ingestSpans(t, svc, spanName, attrs...) + if len(stored) != 1 { + t.Fatalf("expected 1 span, got %d", len(stored)) + } + return stored[0] +} + +// ingestSpans posts one span from service svc and returns every span stored. +func ingestSpans(t *testing.T, svc, spanName string, attrs ...kv) []storage.Span { t.Helper() parts := make([]string, 0, len(attrs)) for _, a := range attrs { @@ -52,10 +62,7 @@ func ingestOne(t *testing.T, svc, spanName string, attrs ...kv) storage.Span { req.Header.Set("Content-Type", "application/json") h.ServeHTTP(httptest.NewRecorder(), req) h.Flush() - if len(store.spans) != 1 { - t.Fatalf("expected 1 span, got %d", len(store.spans)) - } - return store.spans[0] + return store.spans } func wantTokens(t *testing.T, name string, got *int64, want int64) { @@ -119,28 +126,63 @@ func TestCodexTurnSpanDerivesNonCachedInput(t *testing.T) { wantTokens(t, "cache_read", sp.CacheReadTokens, 600) } -// thread.id on most Codex spans is the tracing layer's OS thread number. +// thread.id on most Codex spans is the tracing layer's OS thread number, not a +// session, and a Codex span without a session is not stored (ADR-0030). func TestCodexIntegerThreadIDIsNotASession(t *testing.T) { - sp := ingestOne(t, "codex-app-server", "fs.get_metadata", kv{"thread.id", i("3")}) - if sp.SessionID != "" { - t.Errorf("session_id: got %q, want empty", sp.SessionID) + if got := ingestSpans(t, "codex-app-server", "fs.get_metadata", kv{"thread.id", i("3")}); len(got) != 0 { + t.Errorf("stored %d spans, want 0 (session_id %q)", len(got), got[0].SessionID) } } // handle_responses repeats the turn's usage per request as gen_ai.usage.*; the -// turn span already carries the sum, so these must not add to it. +// turn span already carries the sum. It has no session, so it is not stored and +// cannot add to the turn's tokens. func TestCodexPerRequestUsageIsNotCounted(t *testing.T) { - sp := ingestOne(t, "codex-app-server", "handle_responses", + got := ingestSpans(t, "codex-app-server", "handle_responses", + kv{"thread.id", i("5")}, kv{"gen_ai.usage.input_tokens", i("24325")}, kv{"gen_ai.usage.output_tokens", i("196")}, kv{"gen_ai.usage.cache_read.input_tokens", i("11264")}, ) - if sp.InputTokens != nil || sp.OutputTokens != nil || sp.CacheReadTokens != nil { - t.Errorf("handle_responses must carry no tokens, got in=%v out=%v cr=%v", - sp.InputTokens, sp.OutputTokens, sp.CacheReadTokens) + if len(got) != 0 { + t.Errorf("handle_responses: stored %d spans, want 0", len(got)) + } +} + +// The Codex spans that make up nearly all of its volume carry no session and +// are dropped; every other producer's sessionless spans are stored (ADR-0030). +func TestOnlyCodexSessionlessSpansAreDropped(t *testing.T) { + for _, name := range []string{"receiving", "append_items", "persist_rollout_items", "realtime_conversation.running_state"} { + if got := ingestSpans(t, "codex-app-server", name, kv{"thread.id", i("7")}); len(got) != 0 { + t.Errorf("codex %s: stored %d spans, want 0", name, len(got)) + } + } + for _, svc := range []string{"grok-cli", "claude-code", ""} { + if got := ingestSpans(t, svc, "anything"); len(got) != 1 { + t.Errorf("%q sessionless span: stored %d, want 1", svc, len(got)) + } + } +} + +// WouldDrop applies the same rule to a stored span's JSON-decoded attributes, +// where integers arrive as float64. +func TestWouldDrop(t *testing.T) { + cases := []struct { + svc, name string + attrs map[string]any + want bool + }{ + {"codex-app-server", "receiving", map[string]any{"thread.id": float64(5)}, true}, + {"codex-app-server", "fs.read_file", map[string]any{}, true}, + {"codex-app-server", "session_task.turn", map[string]any{"thread.id": codexConv}, false}, + {"codex-app-server", "exec", map[string]any{"conversation.id": codexConv}, false}, + {"grok-cli", "sampling.stream_decode", map[string]any{}, false}, + {"claude-code", "claude_code.tool", map[string]any{}, false}, } - if sp.CostUSD != nil { - t.Errorf("handle_responses must carry no cost, got %v", *sp.CostUSD) + for _, c := range cases { + if got := ingest.WouldDrop(c.svc, c.name, c.attrs); got != c.want { + t.Errorf("WouldDrop(%s, %s) = %v, want %v", c.svc, c.name, got, c.want) + } } } diff --git a/internal/storage/purge.go b/internal/storage/purge.go new file mode 100644 index 0000000..fc597e8 --- /dev/null +++ b/internal/storage/purge.go @@ -0,0 +1,122 @@ +package storage + +import ( + "encoding/json" + "fmt" + "log" + "sort" +) + +// PurgeReport summarises PurgeSpans: how many stored spans matched, by service +// and span name. +type PurgeReport struct { + Rows []PurgeRow + Total int64 +} + +// PurgeRow is one service and span name with the number of spans matched. +type PurgeRow struct { + Service string + Name string + Spans int64 +} + +// PurgeSpans deletes the stored spans that drop matches, among spans with no +// session; with apply=false it only reports what it would delete. A span with a +// session is never a candidate, whatever drop says. drop receives the span's +// service name, span name and attributes decoded from their stored JSON; a span +// whose attributes do not decode is kept. +// +// The caller passes ingest's rule (ingest.WouldDrop), so the purge removes +// exactly what ingest would no longer store: storage cannot import ingest. +func (db *DB) PurgeSpans(drop func(svc, name string, attrs map[string]any) bool, apply bool) (PurgeReport, error) { + rows, err := db.rw.Query(` + SELECT span_id, COALESCE(service_name, ''), name, COALESCE(CAST(attributes AS VARCHAR), '') + FROM spans + WHERE COALESCE(session_id, '') = ''`) + if err != nil { + return PurgeReport{}, fmt.Errorf("purge: read spans: %w", err) + } + type key struct{ svc, name string } + counts := map[key]int64{} + var ids []string + for rows.Next() { + var id, svc, name, attrsJSON string + if err := rows.Scan(&id, &svc, &name, &attrsJSON); err != nil { + rows.Close() + return PurgeReport{}, fmt.Errorf("purge: scan span: %w", err) + } + attrs := map[string]any{} + if attrsJSON != "" { + if err := json.Unmarshal([]byte(attrsJSON), &attrs); err != nil { + continue + } + } + if drop(svc, name, attrs) { + counts[key{svc, name}]++ + ids = append(ids, id) + } + } + err = rows.Err() + rows.Close() + if err != nil { + return PurgeReport{}, fmt.Errorf("purge: read spans: %w", err) + } + + rep := PurgeReport{Total: int64(len(ids))} + for k, n := range counts { + rep.Rows = append(rep.Rows, PurgeRow{Service: k.svc, Name: k.name, Spans: n}) + } + sort.Slice(rep.Rows, func(i, j int) bool { + if rep.Rows[i].Spans != rep.Rows[j].Spans { + return rep.Rows[i].Spans > rep.Rows[j].Spans + } + return rep.Rows[i].Service+rep.Rows[i].Name < rep.Rows[j].Service+rep.Rows[j].Name + }) + if !apply || len(ids) == 0 { + return rep, nil + } + if err := db.deleteSpanIDs(ids); err != nil { + return rep, err + } + log.Printf("purge: deleted %d spans", len(ids)) + return rep, nil +} + +// deleteSpanIDs deletes the spans with the given ids in one transaction. The ids +// go through a temporary table so the delete is one statement, not one per span. +func (db *DB) deleteSpanIDs(ids []string) (err error) { + tx, err := db.rw.Begin() + if err != nil { + return fmt.Errorf("purge: begin tx: %w", err) + } + defer func() { + if err != nil { + _ = tx.Rollback() + } + }() + if _, err = tx.Exec(`CREATE TEMP TABLE purge_ids (span_id VARCHAR)`); err != nil { + return fmt.Errorf("purge: create temp table: %w", err) + } + stmt, err := tx.Prepare(`INSERT INTO purge_ids VALUES (?)`) + if err != nil { + return fmt.Errorf("purge: prepare insert: %w", err) + } + for _, id := range ids { + if _, err = stmt.Exec(id); err != nil { + stmt.Close() + return fmt.Errorf("purge: stage span %s: %w", id, err) + } + } + stmt.Close() + if _, err = tx.Exec(`DELETE FROM spans WHERE span_id IN (SELECT span_id FROM purge_ids)`); err != nil { + return fmt.Errorf("purge: delete spans: %w", err) + } + if _, err = tx.Exec(`DROP TABLE purge_ids`); err != nil { + return fmt.Errorf("purge: drop temp table: %w", err) + } + if err = tx.Commit(); err != nil { + return fmt.Errorf("purge: commit: %w", err) + } + return nil +} diff --git a/internal/storage/purge_test.go b/internal/storage/purge_test.go new file mode 100644 index 0000000..f5ba21f --- /dev/null +++ b/internal/storage/purge_test.go @@ -0,0 +1,86 @@ +package storage + +import ( + "testing" + "time" +) + +func insertPurgeSpan(t *testing.T, db *DB, spanID, svc, name, session, attrs string) { + t.Helper() + now := time.Now() + _, err := db.rw.Exec(` + INSERT INTO spans (trace_id, span_id, name, start_time, end_time, service_name, session_id, attributes) + VALUES ('t1', ?, ?, ?, ?, ?, ?, ?)`, + spanID, name, now, now.Add(time.Millisecond), svc, session, attrs) + if err != nil { + t.Fatalf("insert %s: %v", spanID, err) + } +} + +// dropNoise stands in for ingest.WouldDrop, which storage cannot import: it +// drops service "noisy" spans unless their attributes name a conversation. +func dropNoise(svc, _ string, attrs map[string]any) bool { + return svc == "noisy" && attrs["conversation.id"] == nil +} + +func TestPurgeSpans(t *testing.T) { + db, err := Open(":memory:") + if err != nil { + t.Fatal(err) + } + defer db.Close() + + insertPurgeSpan(t, db, "a", "noisy", "receiving", "", `{"thread.id":5}`) + insertPurgeSpan(t, db, "b", "noisy", "receiving", "", `{}`) + insertPurgeSpan(t, db, "c", "noisy", "fs.read_file", "", `{}`) + insertPurgeSpan(t, db, "d", "noisy", "exec", "", `{"conversation.id":"x"}`) // drop says keep + insertPurgeSpan(t, db, "e", "noisy", "receiving", "sess-1", `{}`) // has a session: never a candidate + insertPurgeSpan(t, db, "f", "quiet", "anything", "", `{}`) + + count := func() int64 { + var n int64 + if err := db.rw.QueryRow(`SELECT count(*) FROM spans`).Scan(&n); err != nil { + t.Fatal(err) + } + return n + } + + rep, err := db.PurgeSpans(dropNoise, false) + if err != nil { + t.Fatal(err) + } + if rep.Total != 3 || count() != 6 { + t.Fatalf("dry run: total %d, stored %d; want 3 matched and nothing deleted", rep.Total, count()) + } + if len(rep.Rows) != 2 || rep.Rows[0] != (PurgeRow{"noisy", "receiving", 2}) { + t.Errorf("dry run rows: %+v", rep.Rows) + } + + if rep, err = db.PurgeSpans(dropNoise, true); err != nil { + t.Fatal(err) + } + if rep.Total != 3 || count() != 3 { + t.Fatalf("apply: total %d, stored %d; want 3 deleted, 3 left", rep.Total, count()) + } + rows, err := db.rw.Query(`SELECT span_id FROM spans ORDER BY span_id`) + if err != nil { + t.Fatal(err) + } + defer rows.Close() + var left string + for rows.Next() { + var id string + if err := rows.Scan(&id); err != nil { + t.Fatal(err) + } + left += id + } + if left != "def" { + t.Errorf("spans left: %q, want \"def\"", left) + } + + // A second apply finds nothing and does not fail on the temp table. + if rep, err = db.PurgeSpans(dropNoise, true); err != nil || rep.Total != 0 { + t.Errorf("second apply: total %d, err %v", rep.Total, err) + } +} From eef4014592b9ff03e73c2aa7b0331d4d7aadc256 Mon Sep 17 00:00:00 2001 From: FlopBut Date: Sun, 15 Nov 2026 20:42:46 +0100 Subject: [PATCH 2/2] test(ingest): producer tests assert the request succeeded Co-Authored-By: Claude Opus 5.5 --- internal/ingest/producers_test.go | 8 +++++++- 1 file changed, 7 insertions(+), 1 deletion(-) diff --git a/internal/ingest/producers_test.go b/internal/ingest/producers_test.go index c8ebb83..d85eda9 100644 --- a/internal/ingest/producers_test.go +++ b/internal/ingest/producers_test.go @@ -60,8 +60,14 @@ func ingestSpans(t *testing.T, svc, spanName string, attrs ...kv) []storage.Span h := ingest.New(store) req := httptest.NewRequest(http.MethodPost, "/v1/traces", bytes.NewBufferString(payload)) req.Header.Set("Content-Type", "application/json") - h.ServeHTTP(httptest.NewRecorder(), req) + w := httptest.NewRecorder() + h.ServeHTTP(w, req) h.Flush() + // A dropped span is still a successful request; without this a rejected + // request would pass every "nothing stored" assertion. + if w.Code != http.StatusOK { + t.Fatalf("status %d: %s", w.Code, w.Body.String()) + } return store.spans }